* CHANGE THE DIRECTORY TO THE ONE WHERE THIS CODE IS LOCATED
local topfolder C:/Users/hspamann/Documents/Projects/In_progress/crimlaw_legorig/drafts/CURRENT/paper/ALERsubmission/code/data_generation

cd `topfolder'/datasources
clear
set more off

use Population_USCensusBureau/Census_IDB_for_export.dta
merge 1:1 ISO3code year using arms/SmallArmsSurvey.dta, nogenerate
	replace arms=arms/100
	label variable arms `"Firearms per capita (2004; source: Small Arms Survey)"'
merge m:1 ISO3code using DBlabor_2008.dta, nogenerate
merge 1:1 ISO3code year using drugs/WDR_forexport.dta, nogenerate
	drop WDRcountry
merge 1:1 ISO3code year using Federalism/fedccp_forexport.dta, keepusing(fedccp) nogenerate
merge 1:1 ISO3code year using inequality/utip_forexport.dta, keepusing(UTIP*) nogenerate
merge 1:1 ISO3code year using Political_vars/FH/fh.dta, keepusing(free) nogenerate
merge 1:1 ISO3code year using Political_vars/PolityIV/polity_forexport.dta, keepusing(polity) nogenerate
merge 1:1 ISO3code year using Political_vars/Propvote/propvote_forexport.dta, keepusing(liec propvote democracy propdem)
	drop if ISO3code=="YMD" & _merge==2 // this is the People's Democratic Republic of Yemen, which has no data elsewhere
	drop _merge
merge 1:1 ISO3code year using ICPS/ICPS_forexport.dta, nogenerate
	gen prisrate=100000*prisoners/population
	label variable prisrate "prisoners per 100,000 (ICPS)"
	gen ln_prisrate=ln(prisrate)
	label variable ln_prisrate "ln(prisrate)"
merge 1:1 ISO3code year using UNCTS/UNCTS_forexport.dta, nogenerate
	gen ln_UNCTSprisrate=ln(UNCTSprisrate)
	label variable ln_UNCTSprisrate "ln(UNCTSprisrate)"
	gen ln_UNCTShomrate=ln(UNCTShomrate)
	label variable ln_UNCTShomrate "ln(UNCTShomrate)"
merge 1:1 ISO3code year using UNCTS/UNCTS_ms.dta, nogenerate
merge 1:1 ISO3code year using PWT/PWT_forexport.dta, nogenerate
	replace rgdpl2=rgdpl2/1000
	label variable rgdpl2 `"GDP pc PPP (PWT 7.1) (in 1,000s)"'
	gen ln_rgdpl2=ln( rgdpl2)
	label variable ln_rgdpl2 `"ln(rgdpl2)"'
merge 1:1 ISO3code year using Religion/ARDA_worldreligion_natl_download27Feb2014_forexport.dta, keepusing(chprtpct chcatpct isgenpct chgenpct) nogenerate
merge 1:1 ISO3code year using UN_DYB_historic/UN_DYB.dta, nogenerate
merge 1:1  ISO3code year using WDI/WDI.dta, keepusing(lngdp gdp nlngdp ngdp gini inc* migrants urban city unempl_* military literacy pop_total) nogenerate
	rename gini gini_WDI
merge 1:1  ISO3code year using WHO_mortality_database_WHOSIS/WHOSIS_homicides.dta, nogenerate
	gen homrate = 100000*homicides/population
	label variable homrate "homicides per 100,000 population (WHO-SIS, Census IDB)"
	gen ln_homrate = ln(homrate) // some countries have zero homicides in some years so they drop out here. But they are either very small, so this is noise anyway, or the numbers are not credible.
	label variable ln_homrate "ln(homrate)"
	encode ICDversion, gen(numICDversion)
merge 1:1 ISO3code year using IHME_GBD_2010/IHME_GBD_2010_complete.dta, keepusing(IHME* firearm*) nogenerate
	gen ln_IHMEviolentdeaths=ln(IHMEviolentdeaths)
	label variable ln_IHMEviolent "ln(IHMEviolentdeaths)"
	gen ln_IHMEdrugdeaths=ln(IHMEdrugdeaths)
	label var ln_IHMEdrugdeaths "ln(IHMEdrugdeaths)"
merge 1:1 ISO3code year using WHO_GlobalHealthObsDataRepository/mentalhealthbeds_2011.dta, keepusing(mental) nogenerate
merge 1:m ISO3code year using ICVS_EUICS/ICVS_forexport.dta, keepusing(victim9 C01A100 C06A100 C10A100 C09A100 C12A100 I002A I003) nogenerate
	label variable I002A "ICVS: sweep"
	label variable I003 "ICVS: type of survey (nat'l or capital)"
merge 1:1 ISO3code year using inequality/OECD/OECDgini.dta, nogenerate
	sum year if mi(ISO3code)==1
	list country year if mi(ISO3code)==1
	drop if mi(year)==1
merge m:1 ISO3code using fractionalization/fractionalization_forexport.dta, nogenerate keep(1 3)
merge m:1 ISO3code using Klermanetal.dta, keep(1 3) nogenerate
merge m:1 ISO3code using LegalOrigin/LO.dta, keep(1 3) nogenerate
merge m:1 ISO3code using slavery/slavery_forexport.dta, keep(1 3) nogenerate
	drop wbcode wbname
	foreach var of varlist slave_total slave_total1m slave_total500k slave_total250k {
		replace `var'=`var'*(1-africa)
	}
merge m:1 ISO3code using OECD.dta, nogenerate keep(1 3)
	recode oecd (.=0) 
merge 1:1 ISO3code year using BJS/BJShomicide.dta, nogenerate


tab country if mi(ISO3code)==1
drop country
merge m:1  ISO3code using ISO3.dta, keepusing(country) keep(1 3) nogenerate update replace
drop if mi(ISO3code)==1


label variable year ""
encode ISO3code, gen(numISO3code)
gen gini=gini_oecd_post
replace gini=gini_WDI if mi(gini)
label var gini "Gini: preferably OECD, otherwise WDI (both post transf n taxes)"
order country  ISO3code numISO3code year  prisoners prisrate ln_prisrate UNCTS*source UNCTSprisrate ln_UNCTSprisrate UNCTShomrate ln_UNCTShomrate ///
	homicides  homrate ln_homrate ICDversion numICDversion ln_IHMEviolentdeaths IHMEviolentdeaths ///
	victim9 C??A100 I002A I003 cocaine ecstasy opiates ln_IHMEdrugdeaths IHMEdrugdeaths rgdpl2 ln_rgdpl2 gini gini_* UTIPgini UTIPtheil
xtset numISO year

gen L25teenshare=L25.teenshare
label var L25teenshare "teen births %, t-25"
gen L15teenshare=L15.teenshare
label var L15teenshare "teen births %, t-15"


* interpolating
sort numISO year
foreach var of varlist ln_prisrate UTIP* ch*pct is*pct free gini incsh_low* migrants urban city unempl_*_natl teenshare L*5teenshare literacy {
	by numISO: ipolate `var' year, generate(i_`var')
	label variable i_`var' "same as `var', but missing values linearly interpolated"
}



***************************
*** some clean-up**********


** common knowledge US values
recode free fedccp dem (.=1) if ISO3=="USA" // US is always 1 on "free" where not missing
recode propdem (.=0) if ISO3=="USA"

** MS1519, combine UNCTS and Census IDB data for longer time series, interpolate, and use 1975 values for 1970-1974
tempvar combo filler
gen `combo'=ms1519
replace `combo'=UNCTSms if mi(`combo')
by numISO: ipolate `combo' year, gen(combo_ms1519)
by numISO: egen `filler'=mean(UNCTSms) if year<=1975 // the "mean" is the observation from 1975 -- no observations prior
replace combo=`filler' if inrange(year,1970,1974) & mi(combo)
label var combo "males 15-19 as share of population, combined UNCTS and Census-IDB, interpolated"


** FREE, use Czechoslovakian value for the two constituent republics
replace free=F.free if year==1992 & inlist(country,"Czech Republic","Slovakia")

** FEDCCP
* subunits evaluated for themselves (i.e., not automatically federal) (YUG was non-federal in fedccp 2003-2005 anyway, so a fortiori constituents SRB and MNE)
replace fedccp=0 if country=="Serbia" | country=="Hong Kong" | ISO=="MAC" | ISO=="MNE" | ISO=="PRI" 
* Myanmar suspended constitution in 1988 but did not have federalism before and after: set to 0
replace fedccp=0 if ISO3=="MMR" & year>1987 & year<2008
* Nigeria suspended constitution in 1998, but had federalism before and after
replace fedccp=1 if year==1998 & country=="Nigeria"
* same or coding error (missing) for Kyrgyzstan 2006
replace fedccp=0 if ISO3=="KGZ" & year==2006
* add from Wikipedia
replace fedccp=0 if ISO3=="BMU" & year>1966


** DEM & PROPDEM.
* additions from Wikipedia -- see separate Word file in DPI folder -- years chosen (hopefully) conservatively
replace democracy = 1 if (ISO=="AND" & year>1993) | (ISO=="ATG" & year>1951) | (ISO=="BMU" & year>1966) | (ISO=="FSM" & year>1999) | (ISO=="HKG" & year>1996) | (ISO=="KIR" & year>1999) | (ISO=="KNA" & year>1999) | (ISO=="MHL" & year>1979) | (ISO=="MNE" & year>1999) | ISO=="PRI" | (ISO=="SRB" & (year>1999 | year==1996)) | (ISO=="YUG" & year>1999) | (ISO=="STP" & year>1989) | (ISO=="SYC" & (year>2002 & year<2012)) | (ISO=="VCT" & year>1999)
replace democracy=0 if (ISO=="DMA" & year>1999) | (ISO=="MAC" & year>1999) | (ISO=="SYC" & (year<=2002 | year>=2012)) | (ISO=="TON" & year<2008)
replace propdem = 1 if (ISO=="HKG" & year>1996) | (ISO=="MNE" & year>1999) | (ISO=="SRB" & (year>1999 | year==1996)) | (ISO=="YUG" & year>1999) | (ISO=="STP" & year>1989)
replace propdem=2/3 if (ISO=="AND" & year>1993) | (ISO=="SYC" & (year>2002 & year<2012)) 
replace propdem=1/3 if ISO=="PRI"
replace propdem=0 if (ISO=="ATG" & year>1951) | (ISO=="BMU" & year>1966) | (ISO=="MAC" & year>1999) | (ISO=="MHL" & year>1979) | (ISO=="DMA" & year>1999) | (ISO=="FSM" & year>1999) | (ISO=="KIR" & year>1999) | (ISO=="KNA" & year>1999) | (ISO=="SYC" & (year<=2002 | year>=2012)) | (ISO=="TON" & year<2008) | (ISO=="VCT" & year>1999)
* from Wikipedia etc via RA: already integrated into propvote_forexport.dta


** RELIGION
* Germany from 1987 census (results at http://fowid.de/fileadmin/datenarchiv/Religionszugehoerigkeit/Religionszugehoerigkeit_Bevoelkerung_1970_2011.pdf, visited 2/27/2014). These are West German data but all available outcome data are from West Germany before 1990
recode i_chprtpct (.=0.416) if country=="Germany" & inrange(year,1985,1990)
recode i_chcatpct (.=0.429) if country=="Germany" & inrange(year,1985,1990)
recode i_chgenpct (.=0.845) if country=="Germany" & inrange(year,1985,1990)
recode i_isgenpct (.=0.027) if country=="Germany" & inrange(year,1985,1990)

* Hong Kong 2005-2010 from Wikipedia data for 2010 (http://en.wikipedia.org/wiki/Religion_in_Hong_Kong visited 2/27/2014)
recode i_chprtpct (.=0.067) if ISO=="HKG" & inrange(year,2005,2010)
recode i_chcatpct (.=0.049) if ISO=="HKG" & inrange(year,2005,2010)
recode i_chgenpct (.=0.116) if ISO=="HKG" & inrange(year,2005,2010)
recode i_isgenpct (.=0.031) if ISO=="HKG" & inrange(year,2005,2010)

* Macao 2000-2010 based on 2006 data from http://www.thearda.com/internationalData/countries/Country_49_2.asp visited 2/27/2014
recode i_chprtpct (.=0.04) if ISO=="MAC" & inrange(year,2000,2010)
recode i_chcatpct (.=0.011) if ISO=="MAC" & inrange(year,2000,2010)
recode i_chgenpct (.=0.072) if ISO=="MAC" & inrange(year,2000,2010)
recode i_isgenpct (.=0.002) if ISO=="MAC" & inrange(year,2000,2010)

* Puerto Rico in 2000-2010, based on 2006 data (without source) from Wikipedia and undated data from Enc. Britannica Online, using 2/3 catholic from the latter and assuming remaining 30% Christian from the former are Protestant
recode i_chprtpct (.=0.30) if ISO=="PRI" & inrange(year,2000,2010)
recode i_chcatpct (.=0.67) if ISO=="PRI" & inrange(year,2000,2010)
recode i_chgenpct (.=0.97) if ISO=="PRI" & inrange(year,2000,2010)
recode i_isgenpct (.=0.013) if ISO=="PRI" & inrange(year,2000,2010)

* Bermuda from 2010 census available at http://www.govsubportal.com/images/Cabinet_Office/Dept_of_Statistics/Docs/Census/Census2010/Census%20Report%202010-Final%20%282%29.pdf, visited 2/27/2014
recode chprtpct (.=0.39) if ISO=="BMU" & year==2010
recode chcatpct (.=0.15) if ISO=="BMU" & year==2010
* recode changpct (.=0.16) if ISO=="BMU" & year==2010 // currently not used
recode chgenpct (.=0.70) if ISO=="BMU" & year==2010
recode isgenpct (.=0) if ISO=="BMU" & year==2010

recode chprtpct (.=0.40) if ISO=="BMU" & year==2000
recode chcatpct (.=0.15) if ISO=="BMU" & year==2000
* recode changpct (.=0.23) if ISO=="BMU" & year==2000 // currently not used
recode chgenpct (.=0.78) if ISO=="BMU" & year==2000
recode isgenpct (.=0) if ISO=="BMU" & year==2000

foreach var of varlist chprtpct chcatpct chgenpct isgenpct {
	ipolate `var' year if ISO=="BMU", generate(BMU_`var')
	replace i_`var'=BMU_`var' if inrange(year,2000,2010) & ISO=="BMU"
	drop BMU_`var'
}


* fill for SRB, MNE, and YUG from each other in 2000s (data sets differ in their convention; in 2000s, seems safe to assume stability)
foreach religion of varlist i_ch*pct i_is*pct {
	replace `religion'=L.`religion' if ISO3=="YUG" & year>2000 & year<2011
}

tempvar inversetime
gen `inversetime'=2010-year
xtset numISO `inversetime'

foreach religion of varlist i_ch*pct i_is*pct {
	replace `religion'=L.`religion' if (ISO3=="SRB" | ISO3=="MNE") & `inversetime'>0 & `inversetime'<11
}

* for Serbia in 1996, fill from Wikipedia for use with ICVS city survey Belgrade: Wiki shows that religion was fairly stable Serbia excluding Kosovo in 1991-2002, so seems fair to use mean for Belgrade
recode i_chprtpct (.=0.011) if ISO=="SRB" & year==1996
recode i_chcatpct (.=0.06) if ISO=="SRB" & year==1996
recode i_chgenpct (.=0.904) if ISO=="SRB" & year==1996
recode i_isgenpct (.=0.03) if ISO=="SRB" & year==1996

** GINI: filling in observations (this uses tempvar inversetime from religion step)

* information from UTIP -- first by country, then from other countries
gen p_gini=i_gini

foreach predictor in "i_UTIP*" i_UTIPg i_UTIPt {
	reg i_gini `predictor' year i.numISO
	bys numISO: egen countryobs=max(e(sample))
	predict g if countryobs==1
	replace p_gini=g if mi(p_gini)
	drop countryobs g
}

foreach predictor in "i_UTIP*" i_UTIPg i_UTIPt {
	reg i_gini `predictor' year
	predict g
	replace p_gini=g if mi(p_gini)
	drop g
}

label var p_gini "i_gini, with mi replaced by predicted value from reg on UTIP where possible"

* extrapolation
gen f_gini=i_gini
replace f_gini=L.f_gini if f_gini>=.
gen fp_gini=p_gini
replace fp_gini=L.fp_gini if fp_gini>=.
xtset numISO year
replace f_gini=L.f_gini if f_gini>=.
replace fp_gini=L.fp_gini if fp_gini>=.
label variable f_gini "same as i_gini, but missing early/late obs replaced by first/last known value"
label variable fp_gini "same as p_gini, but missing early/late obs replaced by first/last known value"


******************************
******************************

gen USA=ISO3=="USA"
label var USA "dummy for US"

drop __*
compress
cd `topfolder'
save paneldata.dta, replace
