*! supershape v2.0.1 Jeremy Nighohossian 19jun2014
*! this command is an enhanced reshape command that enables retaining labels
*!   and filling in zeros

*Note: As this is a complicated program, it's pretty messy, and I haven't yet
*      cleaned it up.

*v2.0.0 fixed some issues with going wide when j is the same as labels
*		also, fixed variable naming to try to avoid errors
*v2.0.01 removed hyphen at end of variable labels
*v2.0.2 added check for the labels variable being a numeric variable w/ value label
*v2.1 modify to split up variable names and labels for ease and to correct character problems
*v2.1.01 fix mis-aligned labels issue
*v2.2 revamped labels and variables for labels option
*v2.2.1 changed temp file to circumvent access restrictions and gave ability to work through
*		missing values
*v2.2.5 revamped labels and variables for keep option
*v2.2.6 27feb2018 addressed issues with supershape long and labels stemming from vartype of _j 
*v2.2.7 30mar2020 - added some error handling for when values are not unique when rshaping wide
*		and when creating varnames that will not be unique even though values are

program supershape
syntax anything, i(varlist) [j(namelist) KEEPlabels LABels(varname) Fill Zeros String]

local debug=0
tempname wl
tempvar labs jvar labelvar
tempname fdir labelfile

local maxstub=0
local badchars ", - @ & . / '"

if `debug' n display _n "Debugging supershape command"

*In case of error, revert to original dataset
tempfile orig
qui save `orig'

n capture {

*Getting arguments
local wl=substr("`anything'",1,4) //reshape wide or long
if !inlist("`wl'","wide","long") {
	n display as error "Must designate wide/long"
	exit
}
local stub_text=substr("`anything'",6,.) // stubs
qui describe `stub_text'*, varlist
local stubs = r(varlist)
local stubcount: list sizeof stubs	// number of stubs
local maxstub=0
foreach stub of local stubs {
	local temp=length("`stub'")
	if `temp'>`maxstub' local maxstub=`temp'
}

if `maxstub'>10 & "`wl'"=="wide" {
//	display "`maxstub'"
//	n display as error "Please use a shorter stub"
//	exit
}
if "`zeros'"=="zeros" & "`fill'"=="" {
	local fill="zeros"
}
	
n display as result	_n "`stubcount' stub(s): `stubs'" ///
			_n	"Longest stub is `maxstub' characters"

*Preliminary Error Checking
local error=0
if "`wl'"=="wide" & "`j'"=="" local error=1
if "`j'"!="" & "`wl'"=="wide" {
	local j_count: list sizeof j
	if `j_count'>1 local error=2
	local labels_count: list sizeof labels
	if `labels_count'>1 local error=3

	local j_str=0
	local j_type: type `j'
	local j_str=substr("`j_type'",1,3)=="str"
	if `j_str' count if `j'==""
	if !`j_str' count if `j'==.
	if r(N)>0 local error=4 
	
	*Error checking for unique-ness
	if (r(N)>r(unique_value)) local error=5
}

	if `error'==1 n display as error "j option must be populated when reshaping wide"
	if `error'==2 n display as error "supershape cannot handle more than 1 variable in j"
	if `error'==3 n display as error "supershape cannot handle more than 1 variable in labels option"
	if `error'==4 n display as error "`j' contains missing values; please remove or replace"
	if `error'==5 n display as error "Values in `i' `j' not unique, please re-specify or modify data"

if `error' exit
*Pre-processing

**Reshape Wide
***Data types for j and labels
if "`wl'"=="wide" {
	if `debug' n display "Preparing variables for reshape wide"
	tempvar j_temp labels_temp
	local j_labels:	value label `j'
	*If j is str, all is good
	if `j_str' g `j_temp'=`j'
	*If j is numeric w/o value labels, string it
	if !`j_str' qui tostring(`j'), g(`j_temp')
	*If j is numeric w/ value labels, decode it
	foreach char of local badchars {
		qui replace `j_temp'=subinstr(`j_temp',"`char'","",.)
	}
	qui replace `j_temp'=subinstr(`j_temp'," ","_",.)
	qui replace `j_temp'=substr(`j_temp',1,32-`maxstub')

	*Check for duplicate values now that names are abbreviated
	qui duplicates report `i' `j_temp'
	if r(N)>r(unique_value) local error=5
	if `error'==5 {
		tempvar jind_full j_uniq jind jind_max
		bysort `j': g `j_uniq'=_n==1
		g `jind'=_n
		bysort `j_temp' (`jind'): g `jind_full'=sum(`j_uniq')
		bysort `j_temp': egen `jind_max'=max(`jind_full')
		replace `j_temp'=substr(`j_temp',1,31-`maxstub')+string(`jind_full') if inrange(`jind_max',2,9)
	}

	if `labels_count' {
	if `debug' n display "Preparing labels for reshape wide"
		local labels_str=0
		local labels_type: type `labels'
		local labels_str=substr("`labels_type'",1,3)=="str"
		local labels_labels:	value label `labels'
		*If j is str, all is good
		if `labels_str' g `labels_temp'=`labels'
		*If j is numeric w/o value labels, string it
		if !`labels_str' & "`labels_labels'"=="" qui tostring(`labels'), g(`labels_temp')
		*If j is numeric w/ value labels, decode it
		if !`labels_str' & "`labels_labels'"!="" decode `labels', g(`labels_temp')
	}
}

tokenize "`stubs'"

*For reshape long: code to retain labels
if "`keeplabels'"!="" {
	if `debug' n display "saving labels"
	local `fdir'=c(tmpdir)
	if `debug' n display "File saved ``fdir''/`labelfile'.txt"
	n display "Stubcount is `stubcount'"
	file open lblfl using ``fdir''/`labelfile'.txt, write replace
	forv s=1/`stubcount' {
		if `debug' n display "Working on Stub ``s''"
		local stub="``s''"
		local stub=subinstr("``s''","@","*",1)
		foreach var of varlist `stub'* {
			local `var': var label `var'
			display "`stub'"
			local var2=regexr("`var'","^`stub_text'","")
			n display "`var'|``var''" _n
			file write lblfl "`var2'|``var''" _n
		}
	}
	file close lblfl
	preserve
		import delimited using "``fdir''/`labelfile'.txt", delimiter("|") clear
		rename (v1 v2) (_j lbl)
		local jtype: type _j
		if substr("`jtype'",1,3)=="str" drop if _j==""
		else drop if _j==.
		duplicates drop
		save lblfl, replace
	restore
	n display "`varlabels'"
}

*Reshape Wide: Prepare variable names & Save labels for later application
if "`wl'"=="wide" {
if `debug' n display "Saving labels file"
preserve
	if `labels_count' keep `j_temp' `labels_temp'
		else keep `j_temp'
	qui duplicates drop
	local `fdir'=c(tmpdir)
	qui export delimited ``fdir''/`labelfile'.txt, delimiter("|") replace
restore
}

if `debug' n display "List of variables: `varsmacro'"

if `debug' n display "Reshaping..."
keep `stubs' `i' `j_temp'
if substr("`jtype'",1,3)=="str" local string="string"
if "`j'"=="" {
	n display _n "reshape `wl' `stub_text', i(`i') `string'"  _n
	n reshape `wl' `stub_text', i(`i') "`string'"
}
else {
	n display _n "reshape `wl' `stub_text', i(`i') j(`j_temp') `string'" _n
	capture reshape `wl' `stub_text', i(`i') j(`j_temp') string
	if `debug' n display "Error Handling"
	*Error Handling
	if (_rc==9) {
		n display as error "Error when values converted to variable names. Try encoding `j' and setting original to label."
	}
	else {
		if (_rc>0) n display as error "Error with Reshape. Review Setup and try again."
	}
	qui compress
}

if `debug' n display "Reshaping Complete"

if "`keeplabels'"!="" {
	n display "Looping through `stubcount' stubs"
	forval s=1/`stubcount' {
		n display "On Stub `s' cleaning up ``s''"
	}
	merge m:1 _j using lblfl, nogen
	erase ``fdir''/`labelfile'.txt
}

if "`wl'"=="long" & "`j'"!="" rename _j `j'

if "`labels'"!="" {
	file open labels_file using ``fdir''/`labelfile'.txt, read text
	file read labels_file line_red
	while r(eof)==0 {
		file read labels_file line_red
		local temp_var=substr("`line_red'",1,strpos("`line_red'","|")-1)
		local temp_label=substr("`line_red'",strpos("`line_red'","|")+1,.)
		//if `debug' n display "label var `temp_var' `temp_label'"
		capture label var `stub_text'`temp_var' `"`temp_label'"'
	}
	file close labels_file
	erase ``fdir''/`labelfile'.txt
	if `debug' n display "Labels Applied"
}

*fill missing numerical variables with zeros
if "`wl'"=="wide" & "`fill'"!="" {
	forv s=1/`stubcount' {
		foreach var of varlist `stub_text'* {
			local vtype: type `var'
			if substr("`vtype'",1,3)!="str" qui replace `var'=0 if `var'==.
		}
	}
}
}

if !`debug' & _rc use `orig', replace
end