diff --git a/scripts/world_bank/wdi/README.md b/scripts/world_bank/wdi/README.md index ef1f0f5dc6..239d0f7277 100644 --- a/scripts/world_bank/wdi/README.md +++ b/scripts/world_bank/wdi/README.md @@ -146,5 +146,24 @@ If you want to perform "only download", run the below command: python3 worldbank.py --mode=download ``` +### Added golden files and increased the threshold with golden checks in validation_config.json. + +The `GOLDENS_CHECK` validator confirms that the import includes a specific set of expected records. This is useful for verifying that critical StatVars, Places, or specific metadata combinations are consistently present in the output. + +The validator compares the input data (usually from the stats data source) against one or more "golden" files (MCF or CSV). + +If any combination of values in a golden file row is missing from the input, the validation fails. The missing golden rows are then listed in the validation report JSON. + +If you want to get goldens, run the below command: +```bash +#goldens from output csv +python3 validator_goldens.py --validate_goldens_input=../../scripts/world_bank/wdi/output/WorldBank.csv --generate_goldens=golden_data/golden_observations.csv --goldens_must_include="ISO3166Alpha3:gs://unresolved_mcf/import_validation/top_100k_places.csv" --generate_goldens_property_sets="ISO3166Alpha3" +``` + +#goldens from summary reports +```bash +python3 validator_goldens.py --validate_goldens_input="summary_report.csv" --generate_goldens=golden_data/golden_summary_report.csv --generate_goldens_property_sets="StatVar|Units|MinDate|MeasurementMethods|observationPeriod" +``` + We highly recommend the use of the import validation tool for this import which you can find in https://github.com/datacommonsorg/tools/tree/master/import-validation-helper. diff --git a/scripts/world_bank/wdi/golden_data/golden_observations.csv b/scripts/world_bank/wdi/golden_data/golden_observations.csv new file mode 100644 index 0000000000..336749595a --- /dev/null +++ b/scripts/world_bank/wdi/golden_data/golden_observations.csv @@ -0,0 +1,217 @@ +"ISO3166Alpha3" +"Earth" +"country/AGO" +"country/ALB" +"country/ARE" +"country/ARG" +"country/ARM" +"country/AUS" +"country/AUT" +"country/AZE" +"country/BEL" +"country/BEN" +"country/BFA" +"country/BGD" +"country/BGR" +"country/BHR" +"country/BIH" +"country/BLR" +"country/BOL" +"country/BRA" +"country/BRN" +"country/BWA" +"country/CAN" +"country/CHE" +"country/CHL" +"country/CHN" +"country/CIV" +"country/CMR" +"country/COD" +"country/COG" +"country/COL" +"country/CRI" +"country/CUB" +"country/CUW" +"country/CYP" +"country/CZE" +"country/DEU" +"country/DNK" +"country/DOM" +"country/DZA" +"country/ECU" +"country/EGY" +"country/ERI" +"country/ESP" +"country/EST" +"country/ETH" +"country/FIN" +"country/FRA" +"country/GAB" +"country/GBR" +"country/GEO" +"country/GHA" +"country/GIB" +"country/GNQ" +"country/GRC" +"country/GTM" +"country/HKG" +"country/HND" +"country/HRV" +"country/HTI" +"country/HUN" +"country/IDN" +"country/IND" +"country/IRL" +"country/IRN" +"country/IRQ" +"country/ISL" +"country/ISR" +"country/ITA" +"country/JAM" +"country/JOR" +"country/JPN" +"country/KAZ" +"country/KEN" +"country/KGZ" +"country/KHM" +"country/KOR" +"country/KWT" +"country/LAO" +"country/LBN" +"country/LBY" +"country/LKA" +"country/LTU" +"country/LUX" +"country/LVA" +"country/MAR" +"country/MDA" +"country/MDG" +"country/MEX" +"country/MKD" +"country/MLT" +"country/MMR" +"country/MNE" +"country/MNG" +"country/MOZ" +"country/MUS" +"country/MYS" +"country/NAM" +"country/NER" +"country/NGA" +"country/NIC" +"country/NLD" +"country/NOR" +"country/NPL" +"country/NZL" +"country/OMN" +"country/PAK" +"country/PAN" +"country/PER" +"country/PHL" +"country/POL" +"country/PRK" +"country/PRT" +"country/PRY" +"country/QAT" +"country/ROU" +"country/RUS" +"country/RWA" +"country/SAU" +"country/SDN" +"country/SEN" +"country/SGP" +"country/SLV" +"country/SRB" +"country/SSD" +"country/SUR" +"country/SVK" +"country/SVN" +"country/SWE" +"country/SWZ" +"country/SYR" +"country/TCD" +"country/TGO" +"country/THA" +"country/TJK" +"country/TKM" +"country/TTO" +"country/TUN" +"country/TUR" +"country/TZA" +"country/UGA" +"country/UKR" +"country/URY" +"country/USA" +"country/UZB" +"country/VEN" +"country/VNM" +"country/XKS" +"country/YEM" +"country/ZAF" +"country/ZMB" +"country/ZWE" +"country/ATG" +"country/BHS" +"country/BLZ" +"country/BRB" +"country/BTN" +"country/COM" +"country/CPV" +"country/DJI" +"country/DMA" +"country/FJI" +"country/GMB" +"country/GNB" +"country/GRD" +"country/GUY" +"country/KIR" +"country/KNA" +"country/LCA" +"country/LSO" +"country/MDV" +"country/MHL" +"country/PLW" +"country/SLB" +"country/STP" +"country/SYC" +"country/TLS" +"country/TON" +"country/VCT" +"country/VUT" +"country/WSM" +"ChannelIslands" +"country/ABW" +"country/AFG" +"country/AND" +"country/ASM" +"country/BDI" +"country/BMU" +"country/CAF" +"country/CYM" +"country/FRO" +"country/FSM" +"country/GIN" +"country/GRL" +"country/GUM" +"country/IMN" +"country/LBR" +"country/LIE" +"country/MAC" +"country/MAF" +"country/MCO" +"country/MLI" +"country/MNP" +"country/MRT" +"country/MWI" +"country/NCL" +"country/PNG" +"country/PRI" +"country/PSE" +"country/PYF" +"country/SLE" +"country/SMR" +"country/SOM" +"country/SXM" +"country/TCA" +"country/VIR" +"country/VGB" diff --git a/scripts/world_bank/wdi/golden_data/golden_summary_report.csv b/scripts/world_bank/wdi/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..92453f1249 --- /dev/null +++ b/scripts/world_bank/wdi/golden_data/golden_summary_report.csv @@ -0,0 +1,80 @@ +"NumPlaces","StatVar","ScalingFactors","Units","MinDate","MeasurementMethods","observationPeriods" +"186","Count_Death_IntentionalSelfHarm_Male_AsFractionOf_Count_Person_Male","[]","[Per100000Males]","2000","[]","[P1Y]" +"203","Amount_EconomicActivity_GrossNationalIncome_PurchasingPowerParity","[]","[InternationalDollar]","1990","[]","[P1Y]" +"165","Count_Person_Upto4Years_Wasting_AsFractionOf_Count_Person_Upto4Years","[100]","[Percent]","1983","[JointChildMalnutritionEstimate]","[P1Y]" +"144","Count_Person_25OrMoreYears_DoctorateDegree_AsFractionOf_Count_Person_25OrMoreYears","[]","[]","1994","[]","[P1Y]" +"204","Amount_Emissions_CarbonDioxide_PerCapita","[]","[MetricTon]","1970","[]","[P1Y]" +"185","Count_Person_25OrMoreYears_Male_TertiaryEducation_AsFractionOf_Count_Person_25OrMoreYears_Male","[]","[]","1970","[]","[P1Y]" +"218","LifeExpectancy_Person_Female","[]","[Year]","1960","[]","[P1Y]" +"139","Count_Person_25OrMoreYears_Male_DoctorateDegree_AsFractionOf_Count_Person_25OrMoreYears_Male","[]","[]","1994","[]","[P1Y]" +"197","Count_Death_0Years_Female_AsFractionOf_Count_BirthEvent_LiveBirth_Female","[]","[Per1000FemaleLiveBirths]","1960","[UnitedNationsIGMEEstimate]","[P1Y]" +"197","Count_CriminalActivities_MurderAndNonNegligentManslaughter_AsFractionOf_Count_Person","[]","[Per100000Persons]","1990","[]","[P1Y]" +"194","Amount_EconomicActivity_ExpenditureActivity_HealthcareExpenditure_AsFractionOf_Count_Person","[]","[InternationalDollar, USDollar]","2000","[]","[P1Y]" +"202","Amount_EconomicActivity_ExpenditureActivity_EducationExpenditure_Government_AsFractionOf_Amount_EconomicActivity_ExpenditureActivity_Government","[100]","[Percent]","1980","[]","[P1Y]" +"188","Count_Person_25OrMoreYears_Male_BachelorsDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Male","[]","[]","1970","[]","[P1Y]" +"218","FertilityRate_Person_Female","[]","[]","1960","[]","[]" +"218","Count_Person_Rural","[]","[]","1960","[WorldBankEstimate]","[P1Y]" +"184","Count_Person_25OrMoreYears_Female_TertiaryEducation_AsFractionOf_Count_Person_25OrMoreYears_Female","[]","[]","1970","[]","[P1Y]" +"218","Count_Person_Urban","[]","[]","1960","[WorldBankEstimate]","[P1Y]" +"165","Count_Person_Upto4Years_Overweight_AsFractionOf_Count_Person_Upto4Years","[]","[]","1983","[]","[P1Y]" +"218","LifeExpectancy_Person_Male","[]","[Year]","1960","[]","[P1Y]" +"218","Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","[]","[Per1000Persons]","1960","[]","[P1Y]" +"197","MortalityRate_Person_Upto4Years_AsFractionOf_Count_BirthEvent_LiveBirth","[]","[Per1000LiveBirths]","1960","[]","[P1Y]" +"218","Count_Person","[]","[]","1960","[]","[P1Y]" +"100","Count_Person_7To14Years_Male_Employed_AsFractionOf_Count_Person_7To14Years_Male","[100]","[Percent]","1994","[]","[P1Y]" +"160","Count_Person_Upto4Years_Male_Wasting_AsFractionOf_Count_Person_Upto4Years_Male","[100]","[Percent]","1986","[JointChildMalnutritionEstimate]","[P1Y]" +"204","Amount_EconomicActivity_ExpenditureActivity_EducationExpenditure_Government_AsFractionOf_Amount_EconomicActivity_GrossDomesticProduction_Nominal","[100]","[Percent]","1970","[]","[P1Y]" +"188","Count_Person_25OrMoreYears_BachelorsDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears","[]","[]","1970","[]","[P1Y]" +"171","Count_Person_15OrMoreYears_Female_Smoking_AsFractionOf_Count_Person_15OrMoreYears_Female","[]","[]","2000","[AgeAdjustedPrevalence]","[P1Y]" +"171","Count_Person_15OrMoreYears_Smoking_AsFractionOf_Count_Person_15OrMoreYears","[]","[]","2000","[AgeAdjustedPrevalence]","[P1Y]" +"203","Amount_EconomicActivity_GrossNationalIncome_PurchasingPowerParity_PerCapita","[]","[InternationalDollar]","1990","[]","[P1Y]" +"160","Count_Person_Upto4Years_Male_Overweight_AsFractionOf_Count_Person_Upto4Years_Male","[]","[]","1986","[]","[P1Y]" +"197","Count_Death_0Years","[]","[]","1960","[UnitedNationsIGMEEstimate]","[P1Y]" +"195","Amount_EconomicActivity_ExpenditureActivity_TertiaryEducationExpenditure_Government_AsFractionOf_Amount_EconomicActivity_ExpenditureActivity_EducationExpenditure_Government","[]","[]","1970","[]","[P1Y]" +"159","Count_Person_Upto4Years_Male_SevereWasting_AsFractionOf_Count_Person_Upto4Years_Male","[100]","[Percent]","1986","[JointChildMalnutritionEstimate]","[P1Y]" +"151","Amount_Consumption_Electricity_PerCapita","[]","[KilowattHour]","1990","[]","[P1Y]" +"197","Count_Death_0Years_Male_AsFractionOf_Count_BirthEvent_LiveBirth_Male","[]","[Per1000MaleLiveBirths]","1960","[UnitedNationsIGMEEstimate]","[P1Y]" +"180","Amount_Consumption_Energy_PerCapita","[]","[KilogramOfOilEquivalent]","1990","[]","[P1Y]" +"186","Count_Death_IntentionalSelfHarm_Female_AsFractionOf_Count_Person_Female","[]","[Per100000Females]","2000","[]","[P1Y]" +"171","Count_Person_15OrMoreYears_Male_Smoking_AsFractionOf_Count_Person_15OrMoreYears_Male","[]","[]","2000","[AgeAdjustedPrevalence]","[P1Y]" +"149","Count_CriminalActivities_MurderAndNonNegligentManslaughter_Male_AsFractionOf_Count_Person_Male","[]","[Per100000Males]","1990","[]","[P1Y]" +"200","Amount_Remittance_InwardRemittance_AsFractionOf_Amount_EconomicActivity_GrossDomesticProduction_Nominal","[100]","[Percent]","1970","[WorldBankEstimate]","[P1Y]" +"188","Count_Person_15To64Years_InLaborForce_AsFractionOf_Count_Person_15To64Years","[]","[]","1990","[]","[P1Y]" +"100","Count_Person_7To14Years_Employed_AsFractionOf_Count_Person_7To14Years","[100]","[Percent]","1994","[]","[P1Y]" +"171","GiniIndex_EconomicActivity","[]","[]","1963","[WorldBankEstimate]","[P1Y]" +"162","Count_Person_25OrMoreYears_Female_MastersDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Female","[]","[]","1990","[]","[P1Y]" +"170","Count_Person_25OrMoreYears_MastersDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears","[]","[]","1990","[]","[P1Y]" +"152","Count_CriminalActivities_MurderAndNonNegligentManslaughter_Female_AsFractionOf_Count_Person_Female","[]","[Per100000Females]","1990","[]","[P1Y]" +"188","Count_Person_15To64Years_Female_InLaborForce_AsFractionOf_Count_Person_15To64Years_Female","[]","[]","1990","[]","[P1Y]" +"104","Amount_Stock_AsFractionOf_Amount_EconomicActivity_GrossDomesticProduction_Nominal","[100]","[Percent]","1975","[]","[P1Y]" +"131","Count_Person_25OrMoreYears_Female_DoctorateDegree_AsFractionOf_Count_Person_25OrMoreYears_Female","[]","[]","1994","[]","[P1Y]" +"215","GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","[]","[]","1961","[]","[P1Y]" +"218","Count_Death_AsAFractionOfCount_Person","[]","[Per1000Persons]","1960","[WorldBankWeightedAverage]","[P1Y]" +"215","Amount_EconomicActivity_GrossDomesticProduction_Nominal","[]","[USDollar]","1960","[]","[P1Y]" +"188","Count_Person_15To64Years_Male_InLaborForce_AsFractionOf_Count_Person_15To64Years_Male","[]","[]","1990","[]","[P1Y]" +"200","Amount_Remittance_InwardRemittance","[]","[USDollar]","1970","[WorldBankEstimate]","[P1Y]" +"161","Count_Person_Upto4Years_SevereWasting_AsFractionOf_Count_Person_Upto4Years","[100]","[Percent]","1983","[JointChildMalnutritionEstimate]","[P1Y]" +"188","Count_Person_25OrMoreYears_Female_BachelorsDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Female","[]","[]","1970","[]","[P1Y]" +"100","Count_Person_7To14Years_Female_Employed_AsFractionOf_Count_Person_7To14Years_Female","[100]","[Percent]","1994","[]","[P1Y]" +"167","Count_Person_25OrMoreYears_Male_MastersDegreeOrHigher_AsFractionOf_Count_Person_25OrMoreYears_Male","[]","[]","1990","[]","[P1Y]" +"215","Amount_EconomicActivity_GrossDomesticProduction_Nominal_PerCapita","[]","[USDollar]","1960","[]","[P1Y]" +"188","Amount_Consumption_Alcohol_15OrMoreYears_AsFractionOf_Count_Person_15OrMoreYears","[]","[Liter]","2000","[WorldHealthOrganizationEstimates]","[P1Y]" +"188","Count_Person_15OrMoreYears_InLaborForce_Female_AsFractionOf_Count_Person_InLaborForce","[]","[]","1990","[]","[P1Y]" +"216","Count_Person_ResidingLessThan5MetersAboveSeaLevel_AsFractionOf_Count_Person","[]","[]","1990","[]","[P1Y]" +"215","Count_Product_MobileCellularSubscription_AsFractionOf_Count_Person","[]","[]","1960","[]","[P1Y]" +"188","Count_Person_InLaborForce","[]","[]","1990","[InternationalLaborOrganization]","[P1Y]" +"186","Count_Death_IntentionalSelfHarm_AsFractionOf_Count_Person","[]","[Per100000Persons]","2000","[]","[P1Y]" +"197","Count_Death_0Years_AsFractionOf_Count_BirthEvent_LiveBirth","[]","[Per1000LiveBirths]","1960","[UnitedNationsIGMEEstimate]","[P1Y]" +"160","Count_Person_Upto4Years_Female_Wasting_AsFractionOf_Count_Person_Upto4Years_Female","[100]","[Percent]","1986","[JointChildMalnutritionEstimate]","[P1Y]" +"201","Amount_Remittance_OutwardRemittance","[]","[USDollar]","1966","[WorldBankEstimate]","[P1Y]" +"160","Count_Person_Upto4Years_Female_Overweight_AsFractionOf_Count_Person_Upto4Years_Female","[]","[]","1986","[]","[P1Y]" +"214","Count_Person_IsInternetUser_PerCapita","[100]","[]","1990","[]","[P1Y]" +"210","Amount_Production_ElectricityFromNuclearSources_AsFractionOf_Amount_Production_Energy","[]","[]","1990","[]","[P1Y]" +"159","Count_Person_Upto4Years_Female_SevereWasting_AsFractionOf_Count_Person_Upto4Years_Female","[100]","[Percent]","1986","[JointChildMalnutritionEstimate]","[P1Y]" +"185","Count_Person_25OrMoreYears_TertiaryEducation_AsFractionOf_Count_Person_25OrMoreYears","[]","[]","1970","[]","[P1Y]" +"210","Amount_Production_ElectricityFromOilGasOrCoalSources_AsFractionOf_Amount_Production_Energy","[]","[]","1990","[]","[P1Y]" +"218","GrowthRate_Count_Person","[]","[]","1961","[]","[P1Y]" +"213","Amount_Consumption_RenewableEnergy_AsFractionOf_Amount_Consumption_Energy","[]","[]","1990","[]","[P1Y]" +"104","Amount_Stock","[]","[USDollar]","1975","[]","[P1Y]" +"218","LifeExpectancy_Person","[]","[Year]","1960","[]","[]" +"210","Count_Person_20To79Years_Diabetes_AsFractionOf_Count_Person_20To79Years","[]","[]","2000","[]","[P1Y]" diff --git a/scripts/world_bank/wdi/manifest.json b/scripts/world_bank/wdi/manifest.json index bc3927141e..eb427c0472 100644 --- a/scripts/world_bank/wdi/manifest.json +++ b/scripts/world_bank/wdi/manifest.json @@ -20,7 +20,8 @@ "WorldBankCountries.csv", "schema_csvs/WorldBankIndicators_prod.csv" ], - "cron_schedule": "0 11 * * 2" + "cron_schedule": "0 11 * * 2", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/scripts/world_bank/wdi/validation_config.json b/scripts/world_bank/wdi/validation_config.json new file mode 100644 index 0000000000..252f615d20 --- /dev/null +++ b/scripts/world_bank/wdi/validation_config.json @@ -0,0 +1,29 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv" + } + }, + { + "rule_id": "Check_goldens_output_csv", + "description": "Verifies the generated output CSV data matches established critical golden records", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_observations.csv", + "input_files": "../../../../output/WorldBank.csv" + } + } + ] +} diff --git a/scripts/world_bank/wdi/worldbank.py b/scripts/world_bank/wdi/worldbank.py index d864bfeaa2..4e6cdda2d3 100644 --- a/scripts/world_bank/wdi/worldbank.py +++ b/scripts/world_bank/wdi/worldbank.py @@ -39,6 +39,10 @@ "indicatorSchemaFile", os.path.join(_MODULE_DIR, "schema_csvs/WorldBankIndicators_prod.csv"), "") flags.DEFINE_string('mode', '', 'Options: download or process') +flags.DEFINE_string( + 'historical_gcs_path', + 'gs://unresolved_mcf/world_bank/wdi/deleted_rows_07_2026.csv', + 'GCS path to the deleted historical data CSV file') # Remaps the columns provided by World Bank API. WORLDBANK_COL_REMAP = { @@ -504,9 +508,40 @@ def output_csv_and_tmcf_by_grouping(worldbank_dataframe, df = df.replace({'StatisticalVariable': RESOLUTION_TO_EXISTING_DCID}) if saveOutput: logging.info("Writing output csv") - df.drop('IndicatorCode', axis=1).to_csv('output/WorldBank.csv', + output_file_path = 'output/WorldBank.csv' + df.drop('IndicatorCode', axis=1).to_csv(output_file_path, float_format='%.10f', index=False) + + # Read and append historical deleted data from GCS. + try: + logging.info( + f"Reading historical deleted data from GCS: {_FLAGS.historical_gcs_path}" + ) + final_df = pd.read_csv(output_file_path) + deleted_df = pd.read_csv(_FLAGS.historical_gcs_path) + + # Combine dataframes. final_df is placed first so its versions are preferred. + final_df = pd.concat([final_df, deleted_df], ignore_index=True) + + # Deduplicate based on composite keys, keeping the first occurrence (from final_df) + composite_keys = [ + 'StatisticalVariable', 'ISO3166Alpha3', 'Year', + 'observationPeriod', 'unit', 'measurementMethod', + 'scalingFactor' + ] + final_df = final_df.drop_duplicates(subset=composite_keys, + keep='first') + final_df.to_csv(output_file_path, + float_format='%.10f', + index=False) + logging.info( + "Successfully merged and de-duplicated deleted historical data." + ) + except Exception as e: + logging.warning( + f"Could not read historical deleted data from GCS: {e}. Proceeding with fresh data only." + ) else: return df except Exception as e: