From 0b7cb73bc438e2b721e773ca4d36b33174dce41e Mon Sep 17 00:00:00 2001 From: GregorSchroeder Date: Tue, 30 Jun 2026 14:40:30 -0700 Subject: [PATCH 01/14] #247 - hh by workers tract controls --- .../get_tract_controls_hh_by_workers.sql | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 sql/hh_characteristics/get_tract_controls_hh_by_workers.sql diff --git a/sql/hh_characteristics/get_tract_controls_hh_by_workers.sql b/sql/hh_characteristics/get_tract_controls_hh_by_workers.sql new file mode 100644 index 0000000..2b31941 --- /dev/null +++ b/sql/hh_characteristics/get_tract_controls_hh_by_workers.sql @@ -0,0 +1,143 @@ +/* +Get census tract controls for households by number of workers. Census tracts without a +distribution default to using the regional distribution. + +Controls are generated by the 5-year ACS Detailed Tables + B08202 - Household Size by Number of Workers in Household + +Two input parameters are used + run_id - the run identifier is used to get the list of census tracts in tandem with + the year parameter that determines their vintage + year - the year parameter is used to identify the 5-year ACS Detailed Tables release + to use (ex. 2023 maps to 2019-2023) and the census tract geography vintage + (2010-2019 uses 2010, 2020-2029 uses 2020) +*/ + +SET NOCOUNT ON; + +-- Initialize parameters --------------------------------------------------------------- +DECLARE @run_id integer = :run_id; +DECLARE @year integer = :year; +DECLARE @series INTEGER = (SELECT [series] FROM [metadata].[run] WHERE [run_id] = @run_id); + +-- Send error message if no data exists ------------------------------------------------ +DECLARE @msg nvarchar(45) = 'ACS 5-Year Table does not exist'; +IF NOT EXISTS ( + SELECT TOP (1) * + FROM [acs].[detailed].[tables] + WHERE + [name] = 'B08202' + AND [year] = @year + AND [product] = '5Y' +) +SELECT @msg AS [msg] +ELSE +BEGIN + + -- Build the expected return table Tract x Workers + DROP TABLE IF EXISTS [#tt_shell]; + SELECT + [tract], + [workers] + INTO [#tt_shell] + FROM ( + SELECT DISTINCT [tract] + FROM [demographic_warehouse].[dim].[mgra] + INNER JOIN [demographic_warehouse].[dim].[mgra_xref] + ON [mgra].[mgra_id] = [mgra_xref].[mgra_id] + AND [mgra_xref].[xref_year] = @year + WHERE [mgra].[series] = @series + ) AS [tracts] + CROSS JOIN ( + SELECT [workers] FROM ( + VALUES + (0), + (1), + (2), + (3) + ) AS [tt] ([workers]) + ) AS [workers]; + + + -- Prepare intermediary results from ACS datasets ---------------------------------- + -- 5-year ACS Detailed Table B08202 - Household Size by Number of Workers in Household + DROP TABLE IF EXISTS [#hh_by_workers]; + SELECT + [tract], + [workers], + SUM([value]) AS [hh] + INTO [#hh_by_workers] + FROM ( + SELECT + [tract], + -- The '%' wildcard matches both 'Family households' and 'Nonfamily + -- households' + CASE + WHEN REPLACE([label], ':', '') = 'Estimate!!Total!!No workers' THEN 0 + WHEN REPLACE([label], ':', '') = 'Estimate!!Total!!1 worker' THEN 1 + WHEN REPLACE([label], ':', '') = 'Estimate!!Total!!2 workers' THEN 2 + WHEN REPLACE([label], ':', '') = 'Estimate!!Total!!3 or more workers' THEN 3 + ELSE NULL -- NULL values for Margin of Error fields removed in subsequent WHERE clause + END AS [workers], + [value] + FROM [acs].[detailed].[values] + LEFT JOIN [acs].[detailed].[geography] + ON [values].[geography_id] = [geography].[geography_id] + LEFT JOIN [acs].[detailed].[variables] + ON [values].[variable] = [variables].[variable] + AND [values].[table_id] = [variables].[table_id] + LEFT JOIN [acs].[detailed].[tables] + ON [values].[table_id] = [tables].[table_id] + WHERE + [tables].[name] = 'B08202' + AND [tables].[year] = @year + AND [tables].[product] = '5Y' + ) AS [b08202] + WHERE [workers] IS NOT NULL + GROUP BY [tract], [workers] + + -- Create hh workers distribution -------------------------------------------------- + -- Calculate regional hh by workers distribution + DECLARE @total_hh INTEGER = (SELECT SUM([hh]) FROM [#hh_by_workers]); + WITH [region_workers_distribution] AS ( + SELECT + [workers], + 1.0 * SUM([hh]) / @total_hh AS [hh_dist] + FROM [#hh_by_workers] + GROUP BY [workers] + ), + -- Calculate census tract hh by workers distribution + [tract_distribution] AS ( + SELECT + [#hh_by_workers].[tract], + [workers], + CASE + WHEN [tract_total_hh].[hh] = 0 THEN NULL + ELSE [#hh_by_workers].[hh] / [tract_total_hh].[hh] + END AS [hh_dist] + FROM [#hh_by_workers] + LEFT JOIN ( + SELECT + [tract], + SUM([hh]) AS [hh] + FROM [#hh_by_workers] + GROUP BY [tract] + ) AS [tract_total_hh] + ON [#hh_by_workers].[tract] = [tract_total_hh].[tract] + ) + SELECT + @run_id AS [run_id], + @year AS [year], + [#tt_shell].[tract], + [#tt_shell].[workers], + ISNULL([tract_distribution].[hh_dist], [region_workers_distribution].[hh_dist]) AS [value] + FROM [#tt_shell] + LEFT JOIN [region_workers_distribution] + ON [#tt_shell].[workers] = [region_workers_distribution].[workers] + LEFT JOIN [tract_distribution] + ON [#tt_shell].[tract] = [tract_distribution].[tract] + AND [#tt_shell].[workers] = [tract_distribution].[workers] + ORDER BY + [#tt_shell].[tract], + [#tt_shell].[workers]; +END \ No newline at end of file From 86a604ebc1298a3a96c7d9f620d3c70f34dca699 Mon Sep 17 00:00:00 2001 From: GregorSchroeder Date: Thu, 2 Jul 2026 09:27:19 -0700 Subject: [PATCH 02/14] #247 - hh by workers mgra controls --- .../get_mgra_controls_hh_by_size.sql | 7 +--- ...get_mgra_controls_hh_by_workers_18plus.sql | 35 ++++++++++++++++ .../get_mgra_controls_hh_by_workers_size.sql | 41 +++++++++++++++++++ 3 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 sql/hh_characteristics/get_mgra_controls_hh_by_workers_18plus.sql create mode 100644 sql/hh_characteristics/get_mgra_controls_hh_by_workers_size.sql diff --git a/sql/hh_characteristics/get_mgra_controls_hh_by_size.sql b/sql/hh_characteristics/get_mgra_controls_hh_by_size.sql index e967bf5..1292710 100644 --- a/sql/hh_characteristics/get_mgra_controls_hh_by_size.sql +++ b/sql/hh_characteristics/get_mgra_controls_hh_by_size.sql @@ -3,11 +3,8 @@ Get MGRA controls for household population used in households by household size. This data is pulled from previously run modules. Two input parameters are used - run_id - the run identifier is used to get the list of census tracts in tandem with - the year parameter that determines their vintage - year - the year parameter is used to identify the 5-year ACS Detailed Tables release - to use (ex. 2023 maps to 2019-2023) and the census tract geography vintage - (2010-2019 uses 2010, 2020-2029 uses 2020) + run_id - the run identifier for this run + year - the year parameter grabs the year of data to use */ -- Initialize parameters diff --git a/sql/hh_characteristics/get_mgra_controls_hh_by_workers_18plus.sql b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_18plus.sql new file mode 100644 index 0000000..b74b2d3 --- /dev/null +++ b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_18plus.sql @@ -0,0 +1,35 @@ +/* +Get MGRA controls for persons aged 18+ used in households by number of workers. +This data is pulled from previously run modules. + +It would be preferable to use 16+ as that is the age of eligibility for labor +force participation in the ACS but the Estimates Program uses 15 to 17 as an +age category so there is not an easy way to select only 16+. + +Two input parameters are used + run_id - the run identifier for this run + year - the year parameter grabs the year of data to use +*/ + +-- Initialize parameters +DECLARE @run_id integer = :run_id; +DECLARE @year integer = :year; + +-- Get population aged 18+ +SELECT + @run_id AS [run_id], + @year AS [year], + [mgra], + SUM([value]) AS [persons_18plus] +FROM [outputs].[ase] +WHERE + [run_id] = @run_id + AND [year] = @year + AND [age_group] NOT IN ( + 'Under 5', + '5 to 9', + '10 to 14', + '15 to 17' + ) +GROUP BY [mgra] +ORDER BY [mgra] \ No newline at end of file diff --git a/sql/hh_characteristics/get_mgra_controls_hh_by_workers_size.sql b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_size.sql new file mode 100644 index 0000000..dfb2b3f --- /dev/null +++ b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_size.sql @@ -0,0 +1,41 @@ +/* +Get MGRA controls for households by size used in households by number of workers. +This data is pulled from previously run portions of this module. + +Since households by workers only contains (0,1,2,3+) categories, the household +size categories are collapsed to (1,2,3+) to match the households by workers +categories. + +Two input parameters are used + run_id - the run identifier for this run + year - the year parameter grabs the year of data to use +*/ + +-- Initialize parameters +DECLARE @run_id integer = :run_id; +DECLARE @year integer = :year; + +-- Get households by size +SELECT + @run_id AS [run_id], + @year AS [year], + [mgra], + CASE + WHEN [metric] IN ('Household Size - 1', 'Household Size - 2') THEN [metric] + ELSE 'Household Size - 3+' + END AS [metric], + SUM([value]) AS [hh] +FROM [outputs].[hh_characteristics] +WHERE + [run_id] = @run_id + AND [year] = @year + AND [metric] LIKE 'Household Size%' +GROUP BY + [mgra], + CASE + WHEN [metric] IN ('Household Size - 1', 'Household Size - 2') THEN [metric] + ELSE 'Household Size - 3+' + END +ORDER BY + [mgra], + [metric] \ No newline at end of file From d3e8ec2029751444c9293d6b9c4163dab475c391 Mon Sep 17 00:00:00 2001 From: GregorSchroeder Date: Thu, 2 Jul 2026 10:17:47 -0700 Subject: [PATCH 03/14] #247 - hh by workers input and validation --- python/hh_characteristics.py | 161 +++++++++++++++--- python/tests.py | 8 +- python/utils.py | 5 +- ...get_mgra_controls_hh_by_workers_18plus.sql | 37 ++-- ...> get_mgra_controls_hh_by_workers_hhs.sql} | 4 +- .../get_tract_controls_hh_by_workers.sql | 30 ++-- 6 files changed, 184 insertions(+), 61 deletions(-) rename sql/hh_characteristics/{get_mgra_controls_hh_by_workers_size.sql => get_mgra_controls_hh_by_workers_hhs.sql} (94%) diff --git a/python/hh_characteristics.py b/python/hh_characteristics.py index 2b9b11d..884be48 100644 --- a/python/hh_characteristics.py +++ b/python/hh_characteristics.py @@ -15,31 +15,43 @@ def run_hh_characteristics(year: int, debug: bool) -> None: """Orchestrator function to calculate and insert household characteristics. - The exact household characteristics created are: - 1. Households split by household income category - 2. Households split by number of people in each household + The household characteristics created are: + 1. Households by household income category + 2. Households by number of people in each household (size) + 3. Households by number of workers in each household (workers) - Both characteristics are generated by applying ACS data to MGRA level households, - which are created by the HS/HH module. + All characteristics are generated by applying ACS data to MGRA level households, + which are created by the Housing and Households module. Functionality is segmented + into functions for code encapsulation. - Functionality is segmented into functions for code encapsulation. The following are - used for households split by income category: + The following functions are used for households by income category: _get_hh_income_inputs - Get MGRA households and ACS tract distributions for - income - _validate_hh_income_inputs - Validate the hh income input data - _create_hh_income - Calculate the hh income, control to MGRA households - _validate_hh_income_outputs - Validate the hh income output data - _insert_hh_income - Insert hh income and tract level income distributions to - database - - The following functions are used for households split by size + household income categories + _validate_hh_income_inputs - Validate the household income category input data + _create_hh_income - Calculate the household income, control to MGRA households + _validate_hh_income_outputs - Validate the household income output data + _insert_hh_income - Insert household income and tract level income + distributions to the production database + + The following functions are used for households by size: _get_hh_size_inputs - Get MGRA households, MGRA household population, and ACS - tract distributions for size - _validate_hh_size_inputs - Validate the hh size input data - _create_hh_size - Calculate the hh size, control to MGRA households and MGRA - household population - _validate_hh_size_outputs - Validate the hh size output data - _insert_hh_size - Insert hh size and tract level size distributions to database + tract distributions for household size + _validate_hh_size_inputs - Validate the household size input data + _create_hh_size - Calculate the household size, control to MGRA households and + MGRA household population + _validate_hh_size_outputs - Validate the household size output data + _insert_hh_size - Insert household size and tract level size distributions to + the production database + + The following functions are used for households by workers: + _get_hh_workers_inputs - Get MGRA households, MGRA population aged 18+, MGRA + households by size, and ACS tract distributions for households workers + _validate_hh_workers_inputs - Validate the household workers input data + _create_hh_workers - Calculate the household workers, control to MGRA + households, MGRA population aged 18+, and MGRA households by size + _validate_hh_workers_outputs - Validate the household workers output data + _insert_hh_workers - Insert household workers and tract level workers + distributions to the production database Args: year (int): estimates year @@ -62,6 +74,10 @@ def run_hh_characteristics(year: int, debug: bool) -> None: _insert_hh_size(hh_size_inputs, hh_size_outputs, debug) + # And finally do households by workers + hh_workers_inputs = _get_hh_workers_inputs(year) + _validate_hh_workers_inputs(year, hh_workers_inputs) + def _get_hh_income_inputs(year: int) -> dict[str, pd.DataFrame]: """Get households and various tract level datas""" @@ -100,12 +116,12 @@ def _get_hh_size_inputs(year: int) -> dict[str, pd.DataFrame]: """Get households and various tract level datas.""" with utils.ESTIMATES_ENGINE.connect() as con: # Store results here - hh_char_inputs = {} + hh_size_inputs = {} # Get MGRA level households. Note we re-use the SQL script from the 'Population # by Type' module with open(utils.SQL_FOLDER / "pop_type" / "get_mgra_hh.sql") as file: - hh_char_inputs["hh"] = pd.read_sql_query( + hh_size_inputs["hh"] = pd.read_sql_query( sql=sql.text(file.read()), con=con, params={ @@ -120,7 +136,7 @@ def _get_hh_size_inputs(year: int) -> dict[str, pd.DataFrame]: / "hh_characteristics" / "get_tract_controls_hh_by_size.sql" ) as file: - hh_char_inputs["hhs_tract_controls"] = utils.read_sql_query_fallback( + hh_size_inputs["hhs_tract_controls"] = utils.read_sql_query_fallback( sql=sql.text(file.read()), # type: ignore con=con, # type: ignore params={"run_id": utils.RUN_ID, "year": year}, @@ -130,13 +146,70 @@ def _get_hh_size_inputs(year: int) -> dict[str, pd.DataFrame]: with open( utils.SQL_FOLDER / "hh_characteristics" / "get_mgra_controls_hh_by_size.sql" ) as file: - hh_char_inputs["hhs_mgra_controls"] = pd.read_sql_query( + hh_size_inputs["hhs_mgra_controls"] = pd.read_sql_query( + sql=sql.text(file.read()), + con=con, + params={"run_id": utils.RUN_ID, "year": year}, # type: ignore + ) + + return hh_size_inputs + + +def _get_hh_workers_inputs(year: int) -> dict[str, pd.DataFrame]: + """Get MGRA and tract level controls for households by workers.""" + with utils.ESTIMATES_ENGINE.connect() as con: + # Store results here + hh_workers_inputs = {} + + # Get MGRA level households. Note we re-use the SQL script from the 'Population + # by Type' module + with open(utils.SQL_FOLDER / "pop_type" / "get_mgra_hh.sql") as file: + hh_workers_inputs["hh"] = pd.read_sql_query( + sql=sql.text(file.read()), + con=con, + params={ + "run_id": utils.RUN_ID, + "year": year, + }, # type: ignore + ).drop(columns=["jurisdiction"]) + + # Get MGRA level population aged 18+ + with open( + utils.SQL_FOLDER + / "hh_characteristics" + / "get_mgra_controls_hh_by_workers_18plus.sql" + ) as file: + hh_workers_inputs["mgra_18plus"] = pd.read_sql_query( + sql=sql.text(file.read()), + con=con, + params={"run_id": utils.RUN_ID, "year": year}, # type: ignore + ) + + # Get MGRA level households by size + with open( + utils.SQL_FOLDER + / "hh_characteristics" + / "get_mgra_controls_hh_by_workers_hhs.sql" + ) as file: + hh_workers_inputs["mgra_hhs"] = pd.read_sql_query( sql=sql.text(file.read()), con=con, params={"run_id": utils.RUN_ID, "year": year}, # type: ignore ) - return hh_char_inputs + # Get tract level households by workers distributions + with open( + utils.SQL_FOLDER + / "hh_characteristics" + / "get_tract_controls_hh_by_workers.sql" + ) as file: + hh_workers_inputs["tract_workers"] = pd.read_sql_query( + sql=sql.text(file.read()), + con=con, + params={"run_id": utils.RUN_ID, "year": year}, # type: ignore + ) + + return hh_workers_inputs def _validate_hh_income_inputs( @@ -186,6 +259,42 @@ def _validate_hh_size_inputs( ) +def _validate_hh_workers_inputs( + year: int, hh_workers_inputs: dict[str, pd.DataFrame] +) -> None: + """Validate the household characteristics input data""" + tests.validate_data( + "MGRA households", + hh_workers_inputs["hh"], + row_count={"key_columns": {"mgra"}}, + negative={}, + null={}, + ) + tests.validate_data( + "MGRA persons aged 18+ controls", + hh_workers_inputs["mgra_18plus"], + row_count={"key_columns": {"mgra"}}, + negative={}, + null={}, + ) + tests.validate_data( + "MGRA households by household size controls", + hh_workers_inputs["mgra_hhs"], + row_count={ + "key_columns": {"mgra", "household_size_3plus"} + }, # Household size categories collapsed to (1, 2, 3+) + negative={}, + null={}, + ) + tests.validate_data( + "Tract households by household workers distribution", + hh_workers_inputs["tract_workers"], + row_count={"key_columns": {"tract", "household_workers"}, "year": year}, + negative={}, + null={}, + ) + + def _create_hh_income( hh_income_inputs: dict[str, pd.DataFrame], ) -> dict[str, pd.DataFrame]: diff --git a/python/tests.py b/python/tests.py index e033c83..5f747a8 100644 --- a/python/tests.py +++ b/python/tests.py @@ -31,6 +31,8 @@ "structure_type": 4, "income_category": 10, "household_size": 7, + "household_size_3plus": 3, + "household_workers": 4, "tract": { 2010: 627, 2020: 736, @@ -39,7 +41,7 @@ # The industry_code is a variable to group employment data into. Almost all # codes are 2-digit naics codes. The 2-digit naics code 72 was split into 721 # and 722 3-digit naics code. Self employment and military active duty data do - # not natively have a naics code so it is therefore assigned to 'SE' and 'MIL' + # not natively have a naics code so it is therefore assigned to 'SE' and 'MIL' # respectively, as the naics codes are being treated as strings. "industry_code": 23, }, @@ -52,7 +54,9 @@ # For ease of access, combine the constants part of the dictionary with the current # MGRA series -_DISTINCT_COUNTS = _DISTINCT_COUNTS["constants"] | _DISTINCT_COUNTS["series"][utils.SERIES] +_DISTINCT_COUNTS = ( + _DISTINCT_COUNTS["constants"] | _DISTINCT_COUNTS["series"][utils.SERIES] +) ######### diff --git a/python/utils.py b/python/utils.py index 5eeecf2..c8ab73b 100644 --- a/python/utils.py +++ b/python/utils.py @@ -9,7 +9,6 @@ import python.parsers as parsers - ######### # PATHS # ######### @@ -72,8 +71,8 @@ + _secrets["sql"]["gis"]["server"] + "/" + _secrets["sql"]["gis"]["database"] - + "?driver=ODBC Driver 18 for SQL Server" + - "&TrustServerCertificate=yes", + + "?driver=ODBC Driver 18 for SQL Server" + + "&TrustServerCertificate=yes", fast_executemany=True, ) diff --git a/sql/hh_characteristics/get_mgra_controls_hh_by_workers_18plus.sql b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_18plus.sql index b74b2d3..b9ef05b 100644 --- a/sql/hh_characteristics/get_mgra_controls_hh_by_workers_18plus.sql +++ b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_18plus.sql @@ -16,20 +16,31 @@ DECLARE @run_id integer = :run_id; DECLARE @year integer = :year; -- Get population aged 18+ +WITH [ase_data] AS ( + SELECT + [mgra], + SUM([value]) AS [value] + FROM [outputs].[ase] + WHERE + [run_id] = @run_id + AND [year] = @year + AND [age_group] NOT IN ( + 'Under 5', + '5 to 9', + '10 to 14', + '15 to 17' + ) + GROUP BY [mgra] +) +-- Ensure all MGRAs are included SELECT @run_id AS [run_id], @year AS [year], - [mgra], - SUM([value]) AS [persons_18plus] -FROM [outputs].[ase] + [mgra].[mgra], + ISNULL([value], 0) AS [persons_18plus] +FROM [inputs].[mgra] +LEFT OUTER JOIN [ase_data] + ON [mgra].[mgra] = [ase_data].[mgra] WHERE - [run_id] = @run_id - AND [year] = @year - AND [age_group] NOT IN ( - 'Under 5', - '5 to 9', - '10 to 14', - '15 to 17' - ) -GROUP BY [mgra] -ORDER BY [mgra] \ No newline at end of file + [mgra].[run_id] = @run_id +ORDER BY [mgra].[mgra] \ No newline at end of file diff --git a/sql/hh_characteristics/get_mgra_controls_hh_by_workers_size.sql b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_hhs.sql similarity index 94% rename from sql/hh_characteristics/get_mgra_controls_hh_by_workers_size.sql rename to sql/hh_characteristics/get_mgra_controls_hh_by_workers_hhs.sql index dfb2b3f..ad37fb1 100644 --- a/sql/hh_characteristics/get_mgra_controls_hh_by_workers_size.sql +++ b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_hhs.sql @@ -23,7 +23,7 @@ SELECT CASE WHEN [metric] IN ('Household Size - 1', 'Household Size - 2') THEN [metric] ELSE 'Household Size - 3+' - END AS [metric], + END AS [household_size_3plus], SUM([value]) AS [hh] FROM [outputs].[hh_characteristics] WHERE @@ -38,4 +38,4 @@ GROUP BY END ORDER BY [mgra], - [metric] \ No newline at end of file + [household_size_3plus] \ No newline at end of file diff --git a/sql/hh_characteristics/get_tract_controls_hh_by_workers.sql b/sql/hh_characteristics/get_tract_controls_hh_by_workers.sql index 2b31941..5eaa39a 100644 --- a/sql/hh_characteristics/get_tract_controls_hh_by_workers.sql +++ b/sql/hh_characteristics/get_tract_controls_hh_by_workers.sql @@ -38,7 +38,7 @@ BEGIN DROP TABLE IF EXISTS [#tt_shell]; SELECT [tract], - [workers] + [household_workers] INTO [#tt_shell] FROM ( SELECT DISTINCT [tract] @@ -49,14 +49,14 @@ BEGIN WHERE [mgra].[series] = @series ) AS [tracts] CROSS JOIN ( - SELECT [workers] FROM ( + SELECT [household_workers] FROM ( VALUES (0), (1), (2), (3) - ) AS [tt] ([workers]) - ) AS [workers]; + ) AS [tt] ([household_workers]) + ) AS [household_workers]; -- Prepare intermediary results from ACS datasets ---------------------------------- @@ -64,7 +64,7 @@ BEGIN DROP TABLE IF EXISTS [#hh_by_workers]; SELECT [tract], - [workers], + [household_workers], SUM([value]) AS [hh] INTO [#hh_by_workers] FROM ( @@ -78,7 +78,7 @@ BEGIN WHEN REPLACE([label], ':', '') = 'Estimate!!Total!!2 workers' THEN 2 WHEN REPLACE([label], ':', '') = 'Estimate!!Total!!3 or more workers' THEN 3 ELSE NULL -- NULL values for Margin of Error fields removed in subsequent WHERE clause - END AS [workers], + END AS [household_workers], [value] FROM [acs].[detailed].[values] LEFT JOIN [acs].[detailed].[geography] @@ -93,24 +93,24 @@ BEGIN AND [tables].[year] = @year AND [tables].[product] = '5Y' ) AS [b08202] - WHERE [workers] IS NOT NULL - GROUP BY [tract], [workers] + WHERE [household_workers] IS NOT NULL + GROUP BY [tract], [household_workers] -- Create hh workers distribution -------------------------------------------------- -- Calculate regional hh by workers distribution DECLARE @total_hh INTEGER = (SELECT SUM([hh]) FROM [#hh_by_workers]); WITH [region_workers_distribution] AS ( SELECT - [workers], + [household_workers], 1.0 * SUM([hh]) / @total_hh AS [hh_dist] FROM [#hh_by_workers] - GROUP BY [workers] + GROUP BY [household_workers] ), -- Calculate census tract hh by workers distribution [tract_distribution] AS ( SELECT [#hh_by_workers].[tract], - [workers], + [household_workers], CASE WHEN [tract_total_hh].[hh] = 0 THEN NULL ELSE [#hh_by_workers].[hh] / [tract_total_hh].[hh] @@ -129,15 +129,15 @@ BEGIN @run_id AS [run_id], @year AS [year], [#tt_shell].[tract], - [#tt_shell].[workers], + [#tt_shell].[household_workers], ISNULL([tract_distribution].[hh_dist], [region_workers_distribution].[hh_dist]) AS [value] FROM [#tt_shell] LEFT JOIN [region_workers_distribution] - ON [#tt_shell].[workers] = [region_workers_distribution].[workers] + ON [#tt_shell].[household_workers] = [region_workers_distribution].[household_workers] LEFT JOIN [tract_distribution] ON [#tt_shell].[tract] = [tract_distribution].[tract] - AND [#tt_shell].[workers] = [tract_distribution].[workers] + AND [#tt_shell].[household_workers] = [tract_distribution].[household_workers] ORDER BY [#tt_shell].[tract], - [#tt_shell].[workers]; + [#tt_shell].[household_workers]; END \ No newline at end of file From a1faaac37a3af31f37981fa58cc018c33dc50f71 Mon Sep 17 00:00:00 2001 From: GregorSchroeder Date: Fri, 10 Jul 2026 09:21:12 -0700 Subject: [PATCH 04/14] #247 - hh by workers create, validate, and insert --- python/hh_characteristics.py | 281 +++++++++++++++++- python/tests.py | 1 - python/utils.py | 2 + .../get_mgra_controls_hh_by_workers_hhs.sql | 29 +- 4 files changed, 296 insertions(+), 17 deletions(-) diff --git a/python/hh_characteristics.py b/python/hh_characteristics.py index 884be48..786559f 100644 --- a/python/hh_characteristics.py +++ b/python/hh_characteristics.py @@ -78,6 +78,11 @@ def run_hh_characteristics(year: int, debug: bool) -> None: hh_workers_inputs = _get_hh_workers_inputs(year) _validate_hh_workers_inputs(year, hh_workers_inputs) + hh_workers_outputs = _create_hh_workers(hh_workers_inputs) + _validate_hh_workers_outputs(hh_workers_outputs) + + _insert_hh_workers(hh_workers_inputs, hh_workers_outputs, debug) + def _get_hh_income_inputs(year: int) -> dict[str, pd.DataFrame]: """Get households and various tract level datas""" @@ -280,9 +285,7 @@ def _validate_hh_workers_inputs( tests.validate_data( "MGRA households by household size controls", hh_workers_inputs["mgra_hhs"], - row_count={ - "key_columns": {"mgra", "household_size_3plus"} - }, # Household size categories collapsed to (1, 2, 3+) + row_count={"key_columns": {"mgra"}}, negative={}, null={}, ) @@ -566,6 +569,210 @@ def adjust_mgra(mgra_data: pd.Series) -> pd.Series: } +def _create_hh_workers( + hh_workers_inputs: dict[str, pd.DataFrame], +) -> dict[str, pd.DataFrame]: + """Code to compute MGRA level households by workers + + Similar to how income and size work, this function takes MGRA households, applies + tract level rates, then integerizes the data. But additionally, we control to the + MGRA population aged 18+ and MGRA households by size. This is because the + distribution of households by household workers implies some minimum number of + workers in the MGRA, which cannot exceed the actual number of workers in the MGRA. + Because the actual number of workers in the MGRA is not known, we use the + population aged 18+ as a proxy. Additionally, the distribution of households by + household workers implies a minimum number of households by size in the (1,2,3+) + categories, which cannot exceed the actual number of households by size within each + category in the MGRA. + """ + hh = hh_workers_inputs["hh"] + tract_workers_dist = hh_workers_inputs["tract_workers"] + mgra_18plus = hh_workers_inputs["mgra_18plus"] + mgra_hhs = hh_workers_inputs["mgra_hhs"] + + # Combine the total households in each MGRA with the distribution of households by + # number of workers + hh_workers = ( + hh.merge(tract_workers_dist, on=["run_id", "year", "tract"], how="left") + .assign(hh=lambda df: df["hh"] * df["value"]) + .drop(columns=["value"]) + .pivot( + index=["run_id", "year", "mgra", "tract"], + columns="household_workers", + values="hh", + ) + .reset_index(drop=False) + .sort_values(by="mgra") + .reset_index(drop=True) + ) + + # To ensure that we pretty much exactly match ACS distributions, we will do two- + # dimensional controlling on the MGRA level data. After splitting the data into + # separate tracts, row controls will be total households in each MGRA and column + # controls will be tract level households by size + controlled_groups = [] + for tract, group in hh_workers.groupby("tract"): + seed_data = group[utils.HOUSEHOLD_WORKERS].to_numpy() + + row_controls = ( + group[utils.HOUSEHOLD_WORKERS].sum(axis=1).to_numpy().round(0).astype(int) + ) + + col_controls = utils.integerize_1d( + group[utils.HOUSEHOLD_WORKERS].sum(axis=0), generator=generator + ) + + controlled_data = utils.integerize_2d( + data=seed_data, + row_ctrls=row_controls, + col_ctrls=col_controls, + condition="exact", + suppress_warnings=True, + generator=generator, + ) + + # Assign the controlled data back to the tract + group[utils.HOUSEHOLD_WORKERS] = controlled_data + controlled_groups.append(group) + + # Recombine all the data in preparation for the next controlling step + hh_workers = pd.concat(controlled_groups) + + # For every MGRA, compute the minimum implied worker population + # And difference between implied worker and 18+ population + hh_workers = ( + hh_workers.merge(mgra_18plus, on=["run_id", "year", "mgra"], how="left") + .astype({workers: int for workers in utils.HOUSEHOLD_WORKERS}) + .assign( + min_implied_workers=lambda df: df[1] + (2 * df[2]) + (3 * df[3]), + # Compute the difference between the 18+ population and the implied minimum + decrease_min=lambda df: np.where( + df["min_implied_workers"] > df["persons_18plus"], + df["min_implied_workers"] - df["persons_18plus"], + 0, + ), + ) + ) + + # The methodology to adjust each individual MGRA for minimum implied workers + # TODO: Consider ways to parallelize + def adjust_mgra_18plus(mgra_data: pd.Series) -> pd.Series: + if mgra_data["decrease_min"] == 0: + return mgra_data + while True: + # Choose a random household workers category to decrease + # weighted by the number of households in each category + workers_to_decrease = generator.choice( + utils.HOUSEHOLD_WORKERS, + p=mgra_data[utils.HOUSEHOLD_WORKERS] + / mgra_data[utils.HOUSEHOLD_WORKERS].sum(), + ) + + # Look below the chosen category for a category to increase + if mgra_data["decrease_min"] > 0 and workers_to_decrease != 0: + # Avoid overshooting the decrease + workers_to_increase = generator.choice( + range( + np.max([0, workers_to_decrease - mgra_data["decrease_min"]]), + workers_to_decrease, + ) + ) + # Continue if we need to decrease implied workers but the randomly chosen + # category was already 0 workers, restart the loop to choose a new category + else: + continue + + # Execute the change and recompute the remaining change needed + mgra_data[workers_to_decrease] -= 1 + mgra_data[workers_to_increase] += 1 + mgra_data["decrease_min"] += workers_to_increase - workers_to_decrease + + # Check if we are done with this MGRA + if mgra_data["decrease_min"] == 0: + return mgra_data + + # Apply the MGRA adjustments for implied workers and 18+ population + # Note we apply minimum implied workers first then household size categories + # This is because the household size categories are a hard constraint whereas + # the minimum implied workers is a soft constraint due to data limitations + hh_workers = hh_workers.apply(adjust_mgra_18plus, axis=1).drop( + columns=["tract", "persons_18plus", "min_implied_workers", "decrease_min"] + ) + + # For every MGRA, compute the difference between households by number of workers + # and the households by size categories to determine necessary adjustments + # Note that the household size categories are collapsed to (1, 2, 3+) + hh_workers = hh_workers.merge( + mgra_hhs, on=["run_id", "year", "mgra"], how="left" + ).assign( + diff1=lambda df: df["1"] - df[1], + diff2=lambda df: df["2"] - df[2], + diff3=lambda df: df["3"] - df[3], + ) + + # The methodology to adjust each individual MGRA to not violate household size + # constraints. Note that the household size categories are collapsed to (1, 2, 3+) + # and that the 0 worker category can always be added to + def adjust_mgra_hhs(mgra_data: pd.Series) -> pd.Series: + if ( + mgra_data["diff1"] >= 0 + and mgra_data["diff2"] >= 0 + and mgra_data["diff3"] >= 0 + ): + return mgra_data + while True: + # Identify households by worker categories that need adjustment + # As well as categories that can accomodate additional households + workers_to_decrease = [ + category for category in [1, 2, 3] if mgra_data[f"diff{category}"] < 0 + ] + + workers_to_increase = [ + category for category in [1, 2, 3] if mgra_data[f"diff{category}"] > 0 + ] + + # Take the first worker category to decrease and select a worker category to increase + # Use a weighted random methodology to select the category to increase + # Note the 0-worker category is always eligible to increase + workers_to_decrease = workers_to_decrease[0] + + # If all categories to increase are 0 just choose one at a random + if mgra_data[[0] + workers_to_increase].sum() == 0: + workers_to_increase = generator.choice([0] + workers_to_increase) + else: + workers_to_increase = generator.choice( + [0] + workers_to_increase, + p=mgra_data[[0] + workers_to_increase] + / mgra_data[[0] + workers_to_increase].sum(), + ) + + # Execute the change and recompute the remaining change needed + mgra_data[workers_to_decrease] -= 1 + mgra_data[f"diff{workers_to_decrease}"] += 1 + mgra_data[workers_to_increase] += 1 + + # Check if we are done with this MGRA + if ( + mgra_data["diff1"] >= 0 + and mgra_data["diff2"] >= 0 + and mgra_data["diff3"] >= 0 + ): + return mgra_data + + # Apply the MGRA adjustments for household size categories + hh_workers = hh_workers.apply(adjust_mgra_hhs, axis=1).drop( + columns=["1", "2", "3", "diff1", "diff2", "diff3"] + ) + + return { + "hh_workers": hh_workers.melt( + id_vars=["run_id", "year", "mgra"], + var_name="household_workers", + value_name="hh", + ) + } + + def _validate_hh_income_outputs(hh_income_outputs: dict[str, pd.DataFrame]) -> None: """Validate the household income output data""" tests.validate_data( @@ -588,6 +795,17 @@ def _validate_hh_size_outputs(hh_size_outputs: dict[str, pd.DataFrame]) -> None: ) +def _validate_hh_workers_outputs(hh_workers_outputs: dict[str, pd.DataFrame]) -> None: + """Validate the household workers output data""" + tests.validate_data( + "MGRA Households by Workers", + hh_workers_outputs["hh_workers"], + row_count={"key_columns": {"mgra", "household_workers"}}, + negative={}, + null={}, + ) + + def _insert_hh_income( hh_income_inputs: dict[str, pd.DataFrame], hh_income_outputs: dict[str, pd.DataFrame], @@ -693,3 +911,60 @@ def _insert_hh_size( con=con, index=False, ) + + +def _insert_hh_workers( + hh_workers_inputs: dict[str, pd.DataFrame], + hh_workers_outputs: dict[str, pd.DataFrame], + debug: bool, +) -> None: + """Insert hh characteristics and tract level controls to database""" + + inputs_controls_tract = ( + hh_workers_inputs["tract_workers"] + .rename(columns={"household_workers": "metric"}) + .assign( + metric=lambda df: "Household Workers - " + + df["metric"].astype(str).replace("3", "3+") + ) + ) + outputs_hh_characteristics = ( + hh_workers_outputs["hh_workers"][ + ["run_id", "year", "mgra", "household_workers", "hh"] + ] + .rename(columns={"household_workers": "metric", "hh": "value"}) + .assign( + metric=lambda df: "Household Workers - " + + df["metric"].astype(str).replace("3", "3+") + ) + ) + + # Save locally if in debug mode + if debug: + inputs_controls_tract.to_csv( + utils.DEBUG_OUTPUT_FOLDER / "inputs_controls_tract_hh_workers.csv", + index=False, + ) + outputs_hh_characteristics.to_csv( + utils.DEBUG_OUTPUT_FOLDER / "outputs_hh_characteristics_hh_workers.csv", + index=False, + ) + + # Otherwise, load to database + else: + with utils.ESTIMATES_ENGINE.connect() as con: + inputs_controls_tract.to_sql( + schema="inputs", + name="controls_tract", + if_exists="append", + con=con, + index=False, + ) + + outputs_hh_characteristics.to_sql( + schema="outputs", + name="hh_characteristics", + if_exists="append", + con=con, + index=False, + ) diff --git a/python/tests.py b/python/tests.py index 5f747a8..d1ac869 100644 --- a/python/tests.py +++ b/python/tests.py @@ -31,7 +31,6 @@ "structure_type": 4, "income_category": 10, "household_size": 7, - "household_size_3plus": 3, "household_workers": 4, "tract": { 2010: 627, diff --git a/python/utils.py b/python/utils.py index c8ab73b..fae1c2c 100644 --- a/python/utils.py +++ b/python/utils.py @@ -126,6 +126,8 @@ HOUSEHOLD_SIZES = list(range(1, 8)) +HOUSEHOLD_WORKERS = list(range(0, 4)) + ASE = ["age_group", "sex", "ethnicity"] INCOME_CATEGORIES = [ diff --git a/sql/hh_characteristics/get_mgra_controls_hh_by_workers_hhs.sql b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_hhs.sql index ad37fb1..79350bf 100644 --- a/sql/hh_characteristics/get_mgra_controls_hh_by_workers_hhs.sql +++ b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_hhs.sql @@ -19,23 +19,26 @@ DECLARE @year integer = :year; SELECT @run_id AS [run_id], @year AS [year], + [mgra], + [1], + [2], + [3] +FROM ( +SELECT [mgra], CASE - WHEN [metric] IN ('Household Size - 1', 'Household Size - 2') THEN [metric] - ELSE 'Household Size - 3+' - END AS [household_size_3plus], - SUM([value]) AS [hh] + WHEN [metric] = 'Household Size - 1' THEN '1' + WHEN [metric] = 'Household Size - 2' THEN '2' + ELSE '3' + END AS [metric], + [value] FROM [outputs].[hh_characteristics] WHERE [run_id] = @run_id AND [year] = @year AND [metric] LIKE 'Household Size%' -GROUP BY - [mgra], - CASE - WHEN [metric] IN ('Household Size - 1', 'Household Size - 2') THEN [metric] - ELSE 'Household Size - 3+' - END -ORDER BY - [mgra], - [household_size_3plus] \ No newline at end of file +) AS [to_pvt] +PIVOT ( + SUM([value]) FOR [metric] IN ([1], [2], [3]) +) AS [pvt] +ORDER BY [mgra] \ No newline at end of file From 015a545c3d46857d59af5c7e35696232022c99e8 Mon Sep 17 00:00:00 2001 From: GregorSchroeder Date: Fri, 10 Jul 2026 10:52:37 -0700 Subject: [PATCH 05/14] #278 - CoPilot PR Feedback --- python/hh_characteristics.py | 6 +++--- sql/hh_characteristics/get_tract_controls_hh_by_workers.sql | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/python/hh_characteristics.py b/python/hh_characteristics.py index 786559f..0fdce40 100644 --- a/python/hh_characteristics.py +++ b/python/hh_characteristics.py @@ -208,7 +208,7 @@ def _get_hh_workers_inputs(year: int) -> dict[str, pd.DataFrame]: / "hh_characteristics" / "get_tract_controls_hh_by_workers.sql" ) as file: - hh_workers_inputs["tract_workers"] = pd.read_sql_query( + hh_workers_inputs["tract_workers"] = utils.read_sql_query_fallback( sql=sql.text(file.read()), con=con, params={"run_id": utils.RUN_ID, "year": year}, # type: ignore @@ -609,7 +609,7 @@ def _create_hh_workers( # To ensure that we pretty much exactly match ACS distributions, we will do two- # dimensional controlling on the MGRA level data. After splitting the data into # separate tracts, row controls will be total households in each MGRA and column - # controls will be tract level households by size + # controls will be tract level households by workers controlled_groups = [] for tract, group in hh_workers.groupby("tract"): seed_data = group[utils.HOUSEHOLD_WORKERS].to_numpy() @@ -722,7 +722,7 @@ def adjust_mgra_hhs(mgra_data: pd.Series) -> pd.Series: return mgra_data while True: # Identify households by worker categories that need adjustment - # As well as categories that can accomodate additional households + # As well as categories that can accommodate additional households workers_to_decrease = [ category for category in [1, 2, 3] if mgra_data[f"diff{category}"] < 0 ] diff --git a/sql/hh_characteristics/get_tract_controls_hh_by_workers.sql b/sql/hh_characteristics/get_tract_controls_hh_by_workers.sql index 5eaa39a..00095b8 100644 --- a/sql/hh_characteristics/get_tract_controls_hh_by_workers.sql +++ b/sql/hh_characteristics/get_tract_controls_hh_by_workers.sql @@ -70,8 +70,6 @@ BEGIN FROM ( SELECT [tract], - -- The '%' wildcard matches both 'Family households' and 'Nonfamily - -- households' CASE WHEN REPLACE([label], ':', '') = 'Estimate!!Total!!No workers' THEN 0 WHEN REPLACE([label], ':', '') = 'Estimate!!Total!!1 worker' THEN 1 From 9ae9e95edcdc3de8e08822664896bf3bb7909e91 Mon Sep 17 00:00:00 2001 From: GregorSchroeder Date: Fri, 10 Jul 2026 13:52:54 -0700 Subject: [PATCH 06/14] #278 - correct implied hh sizes --- python/hh_characteristics.py | 39 ++++++++----------- .../get_mgra_controls_hh_by_workers_hhs.sql | 38 +++++++++--------- 2 files changed, 36 insertions(+), 41 deletions(-) diff --git a/python/hh_characteristics.py b/python/hh_characteristics.py index 0fdce40..afb304d 100644 --- a/python/hh_characteristics.py +++ b/python/hh_characteristics.py @@ -581,7 +581,7 @@ def _create_hh_workers( workers in the MGRA, which cannot exceed the actual number of workers in the MGRA. Because the actual number of workers in the MGRA is not known, we use the population aged 18+ as a proxy. Additionally, the distribution of households by - household workers implies a minimum number of households by size in the (1,2,3+) + household workers implies a minimum number of households by size in the 2+ and 3+ categories, which cannot exceed the actual number of households by size within each category in the MGRA. """ @@ -701,67 +701,60 @@ def adjust_mgra_18plus(mgra_data: pd.Series) -> pd.Series: # For every MGRA, compute the difference between households by number of workers # and the households by size categories to determine necessary adjustments - # Note that the household size categories are collapsed to (1, 2, 3+) + # Note that the household size categories are collapsed to 2+ and 3+ hh_workers = hh_workers.merge( mgra_hhs, on=["run_id", "year", "mgra"], how="left" ).assign( - diff1=lambda df: df["1"] - df[1], diff2=lambda df: df["2"] - df[2], diff3=lambda df: df["3"] - df[3], ) # The methodology to adjust each individual MGRA to not violate household size - # constraints. Note that the household size categories are collapsed to (1, 2, 3+) - # and that the 0 worker category can always be added to + # constraints. Note that the household size categories are collapsed to 2+ and 3+ + # and that the 0 and 1 worker categories can always be added to as they imply + # a minimum household size of 1+ which is satisfied by every category def adjust_mgra_hhs(mgra_data: pd.Series) -> pd.Series: - if ( - mgra_data["diff1"] >= 0 - and mgra_data["diff2"] >= 0 - and mgra_data["diff3"] >= 0 - ): + if mgra_data["diff2"] >= 0 and mgra_data["diff3"] >= 0: return mgra_data while True: # Identify households by worker categories that need adjustment # As well as categories that can accommodate additional households workers_to_decrease = [ - category for category in [1, 2, 3] if mgra_data[f"diff{category}"] < 0 + category for category in [2, 3] if mgra_data[f"diff{category}"] < 0 ] workers_to_increase = [ - category for category in [1, 2, 3] if mgra_data[f"diff{category}"] > 0 + category for category in [2, 3] if mgra_data[f"diff{category}"] > 0 ] # Take the first worker category to decrease and select a worker category to increase # Use a weighted random methodology to select the category to increase - # Note the 0-worker category is always eligible to increase + # Note the 0 and 1 worker categories are always eligible to increase workers_to_decrease = workers_to_decrease[0] # If all categories to increase are 0 just choose one at a random - if mgra_data[[0] + workers_to_increase].sum() == 0: + if mgra_data[[0, 1] + workers_to_increase].sum() == 0: workers_to_increase = generator.choice([0] + workers_to_increase) else: workers_to_increase = generator.choice( - [0] + workers_to_increase, - p=mgra_data[[0] + workers_to_increase] - / mgra_data[[0] + workers_to_increase].sum(), + [0, 1] + workers_to_increase, + p=mgra_data[[0, 1] + workers_to_increase] + / mgra_data[[0, 1] + workers_to_increase].sum(), ) # Execute the change and recompute the remaining change needed mgra_data[workers_to_decrease] -= 1 mgra_data[f"diff{workers_to_decrease}"] += 1 mgra_data[workers_to_increase] += 1 + mgra_data[f"diff{workers_to_increase}"] -= 1 # Check if we are done with this MGRA - if ( - mgra_data["diff1"] >= 0 - and mgra_data["diff2"] >= 0 - and mgra_data["diff3"] >= 0 - ): + if mgra_data["diff2"] >= 0 and mgra_data["diff3"] >= 0: return mgra_data # Apply the MGRA adjustments for household size categories hh_workers = hh_workers.apply(adjust_mgra_hhs, axis=1).drop( - columns=["1", "2", "3", "diff1", "diff2", "diff3"] + columns=["2", "3", "diff2", "diff3"] ) return { diff --git a/sql/hh_characteristics/get_mgra_controls_hh_by_workers_hhs.sql b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_hhs.sql index 79350bf..020720d 100644 --- a/sql/hh_characteristics/get_mgra_controls_hh_by_workers_hhs.sql +++ b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_hhs.sql @@ -3,8 +3,10 @@ Get MGRA controls for households by size used in households by number of workers This data is pulled from previously run portions of this module. Since households by workers only contains (0,1,2,3+) categories, the household -size categories are collapsed to (1,2,3+) to match the households by workers -categories. +size categories are collapsed to (2+,3+) to match the minimum implied household +sizes from the households by workers categories (e.g. 0 and 1 worker households +can be any size 1+, a 2 worker household must be at least size 2, and a three +worker household must be at least size 3). Two input parameters are used run_id - the run identifier for this run @@ -20,25 +22,25 @@ SELECT @run_id AS [run_id], @year AS [year], [mgra], - [1], - [2], - [3] -FROM ( -SELECT - [mgra], - CASE - WHEN [metric] = 'Household Size - 1' THEN '1' - WHEN [metric] = 'Household Size - 2' THEN '2' - ELSE '3' - END AS [metric], - [value] + SUM( + CASE + WHEN [metric] != 'Household Size - 1' THEN [value] + ELSE 0 + END + ) AS [2], -- 2+ household size + SUM( + CASE + WHEN [metric] NOT IN ( + 'Household Size - 1', + 'Household Size - 2' + ) THEN [value] + ELSE 0 + END + ) AS [3] -- 3+ household size FROM [outputs].[hh_characteristics] WHERE [run_id] = @run_id AND [year] = @year AND [metric] LIKE 'Household Size%' -) AS [to_pvt] -PIVOT ( - SUM([value]) FOR [metric] IN ([1], [2], [3]) -) AS [pvt] +GROUP BY [mgra] ORDER BY [mgra] \ No newline at end of file From 9f8000a12438c0f7ef8b93029d0185679b7a0135 Mon Sep 17 00:00:00 2001 From: GregorSchroeder Date: Fri, 10 Jul 2026 17:37:40 -0700 Subject: [PATCH 07/14] #278 - clean up hh workers hh size adjustment method --- python/hh_characteristics.py | 58 +++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/python/hh_characteristics.py b/python/hh_characteristics.py index afb304d..0616c96 100644 --- a/python/hh_characteristics.py +++ b/python/hh_characteristics.py @@ -705,7 +705,10 @@ def adjust_mgra_18plus(mgra_data: pd.Series) -> pd.Series: hh_workers = hh_workers.merge( mgra_hhs, on=["run_id", "year", "mgra"], how="left" ).assign( + # Expect to see differences >= 0 for Household Sizes - Household Workers + # Household Worker category 2 should be <= Household Size 2+ diff2=lambda df: df["2"] - df[2], + # Household Worker category 3+ should be <= Household Size 3+ diff3=lambda df: df["3"] - df[3], ) @@ -718,35 +721,54 @@ def adjust_mgra_hhs(mgra_data: pd.Series) -> pd.Series: return mgra_data while True: # Identify households by worker categories that need adjustment - # As well as categories that can accommodate additional households - workers_to_decrease = [ - category for category in [2, 3] if mgra_data[f"diff{category}"] < 0 - ] - - workers_to_increase = [ - category for category in [2, 3] if mgra_data[f"diff{category}"] > 0 - ] + # and take the first worker category identified to decrease + if mgra_data["diff2"] < 0: + workers_to_decrease = 2 + elif mgra_data["diff3"] < 0: + workers_to_decrease = 3 + else: + break - # Take the first worker category to decrease and select a worker category to increase - # Use a weighted random methodology to select the category to increase + # Identify categories that can accommodate additional households + # that are smaller than the worker category we are decreasing # Note the 0 and 1 worker categories are always eligible to increase - workers_to_decrease = workers_to_decrease[0] + if workers_to_decrease == 3: + if mgra_data["diff2"] > 0: + workers_to_increase = [0, 1, 2] + else: + workers_to_increase = [0, 1] + elif workers_to_decrease == 2: + workers_to_increase = [0, 1] + else: + break + # Use a weighted random methodology to select the category to increase # If all categories to increase are 0 just choose one at a random - if mgra_data[[0, 1] + workers_to_increase].sum() == 0: - workers_to_increase = generator.choice([0] + workers_to_increase) + if mgra_data[workers_to_increase].sum() == 0: + workers_to_increase = generator.choice(workers_to_increase) else: workers_to_increase = generator.choice( - [0, 1] + workers_to_increase, - p=mgra_data[[0, 1] + workers_to_increase] - / mgra_data[[0, 1] + workers_to_increase].sum(), + workers_to_increase, + p=mgra_data[workers_to_increase] + / mgra_data[workers_to_increase].sum(), ) # Execute the change and recompute the remaining change needed + + # Both 0 and 1 worker categories are not eligible to be decreased mgra_data[workers_to_decrease] -= 1 - mgra_data[f"diff{workers_to_decrease}"] += 1 + if workers_to_decrease == 2: + mgra_data["diff2"] += 1 + elif workers_to_decrease == 3: + mgra_data["diff3"] += 1 + + # Note we only track changes for the 2 and 3+ worker categories + # Although both 0 and 1 worker categories are elgibile to be increased mgra_data[workers_to_increase] += 1 - mgra_data[f"diff{workers_to_increase}"] -= 1 + if workers_to_increase == 2: + mgra_data["diff2"] -= 1 + elif workers_to_increase == 3: + mgra_data["diff3"] -= 1 # Check if we are done with this MGRA if mgra_data["diff2"] >= 0 and mgra_data["diff3"] >= 0: From ba21fd6700a0876f23c3588c35bc1b9544230fb7 Mon Sep 17 00:00:00 2001 From: GregorSchroeder <148280701+GregorSchroeder@users.noreply.github.com> Date: Fri, 10 Jul 2026 17:40:50 -0700 Subject: [PATCH 08/14] Potential fix for pull request finding 'Unreachable code' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- python/hh_characteristics.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/hh_characteristics.py b/python/hh_characteristics.py index 0616c96..66daafc 100644 --- a/python/hh_characteristics.py +++ b/python/hh_characteristics.py @@ -739,8 +739,6 @@ def adjust_mgra_hhs(mgra_data: pd.Series) -> pd.Series: workers_to_increase = [0, 1] elif workers_to_decrease == 2: workers_to_increase = [0, 1] - else: - break # Use a weighted random methodology to select the category to increase # If all categories to increase are 0 just choose one at a random From 6a6cfd0b3c9750829b3a824a32679fdc959173f5 Mon Sep 17 00:00:00 2001 From: GregorSchroeder Date: Fri, 10 Jul 2026 17:42:21 -0700 Subject: [PATCH 09/14] #278 - Code quality fix --- python/hh_characteristics.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/hh_characteristics.py b/python/hh_characteristics.py index 66daafc..0227648 100644 --- a/python/hh_characteristics.py +++ b/python/hh_characteristics.py @@ -726,8 +726,6 @@ def adjust_mgra_hhs(mgra_data: pd.Series) -> pd.Series: workers_to_decrease = 2 elif mgra_data["diff3"] < 0: workers_to_decrease = 3 - else: - break # Identify categories that can accommodate additional households # that are smaller than the worker category we are decreasing From 772413bd9559047e3bdacdf692e31d01acdd944a Mon Sep 17 00:00:00 2001 From: GregorSchroeder Date: Tue, 14 Jul 2026 16:57:33 -0700 Subject: [PATCH 10/14] #278 - pr feedback restrict 18+ to household population --- .../get_mgra_controls_hh_by_workers_18plus.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/hh_characteristics/get_mgra_controls_hh_by_workers_18plus.sql b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_18plus.sql index b9ef05b..72528ff 100644 --- a/sql/hh_characteristics/get_mgra_controls_hh_by_workers_18plus.sql +++ b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_18plus.sql @@ -24,6 +24,7 @@ WITH [ase_data] AS ( WHERE [run_id] = @run_id AND [year] = @year + AND [pop_type] = 'Household Population' AND [age_group] NOT IN ( 'Under 5', '5 to 9', From 7f380832d5080d17dc73cb0e0a61b6408849431f Mon Sep 17 00:00:00 2001 From: GregorSchroeder Date: Tue, 14 Jul 2026 17:05:17 -0700 Subject: [PATCH 11/14] #278 - wiki update --- wiki/Employment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki/Employment.md b/wiki/Employment.md index b7943ed..8e788eb 100644 --- a/wiki/Employment.md +++ b/wiki/Employment.md @@ -30,7 +30,7 @@ The initial siting and share‑assignment effort was led by SANDAG's transportat Confidential point geometry employment by ownership and industry is provided to SANDAG by the California Employment Development Department (CA EDD). This dataset is only used to prepare and allocate census block employment by ownership and industry to MGRAs and is **NOT used to create employment/jobs counts**. See private SANDAG repository [EMPCORE](https://github.com/SANDAG/EMPCORE). ## Census block employment by ownership and industry -The [United States Census Bureau Longitudinal Employer-Household Dynamics (LEHD) Origin-Destination Employment Statistics (LODES)](https://lehd.ces.census.gov/data/) dataset provides census block employment by ownership and industry. This dataset requires a split of its block level two-digit NAICS 72 sector, Accommodation and Food Services, into sectors 721 (Accommodation) and 722 (Food Services) using the point geometry employment dataset and is allocated to MGRAs using a combination of the point geometry employment dataset and a simple land area intersection. This dataset is then scaled to match the regional employment controls by ownership and industry creating employment/jobs by MGRA. See private SANDAG repository [Census-LEHD](https://github.com/SANDAG/Census-LEHD). +The [United States Census Bureau Longitudinal Employer-Household Dynamics (LEHD) Origin-Destination Employment Statistics (LODES)](https://lehd.ces.census.gov/data/) dataset provides census block employment by ownership and industry. This dataset requires a split of its block level two-digit NAICS 72 sector, Accommodation and Food Services, into sectors 721 (Accommodation) and 722 (Food Services) using the point geometry employment dataset and is allocated to MGRAs using a combination of the point geometry employment dataset and a simple land area intersection. The allocation process first attempts to allocate from census block to MGRAs using the point geometry employment dataset at the industry level, then using the point geometry employment dataset without taking industry into account, and finally defaulting to the simple land area intersection between census block and MGRAs. This dataset is then scaled to match the regional employment controls by ownership and industry creating employment/jobs by MGRA. See private SANDAG repository [Census-LEHD](https://github.com/SANDAG/Census-LEHD). ## Subregional self-employment counts Census block group (2013+) or tract (2010-2012) counts of self-employed individuals are gotten from the American Community Survey (ACS) table [B24080 | SEX BY CLASS OF WORKER FOR THE CIVILIAN EMPLOYED POPULATION 16 YEARS AND OVER](https://data.census.gov/table/ACSDT5Y2020.B24080?q=B24080) using the total count of *Self-employed in own not incorporated business workers*. The counts are allocated to MGRAs using a hierarchy of cross references. From 0de17c247a9757dbeb7d7b919bfc01048255802d Mon Sep 17 00:00:00 2001 From: GregorSchroeder Date: Tue, 14 Jul 2026 17:06:35 -0700 Subject: [PATCH 12/14] Revert "#278 - wiki update" This reverts commit 7f380832d5080d17dc73cb0e0a61b6408849431f. --- wiki/Employment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki/Employment.md b/wiki/Employment.md index 8e788eb..b7943ed 100644 --- a/wiki/Employment.md +++ b/wiki/Employment.md @@ -30,7 +30,7 @@ The initial siting and share‑assignment effort was led by SANDAG's transportat Confidential point geometry employment by ownership and industry is provided to SANDAG by the California Employment Development Department (CA EDD). This dataset is only used to prepare and allocate census block employment by ownership and industry to MGRAs and is **NOT used to create employment/jobs counts**. See private SANDAG repository [EMPCORE](https://github.com/SANDAG/EMPCORE). ## Census block employment by ownership and industry -The [United States Census Bureau Longitudinal Employer-Household Dynamics (LEHD) Origin-Destination Employment Statistics (LODES)](https://lehd.ces.census.gov/data/) dataset provides census block employment by ownership and industry. This dataset requires a split of its block level two-digit NAICS 72 sector, Accommodation and Food Services, into sectors 721 (Accommodation) and 722 (Food Services) using the point geometry employment dataset and is allocated to MGRAs using a combination of the point geometry employment dataset and a simple land area intersection. The allocation process first attempts to allocate from census block to MGRAs using the point geometry employment dataset at the industry level, then using the point geometry employment dataset without taking industry into account, and finally defaulting to the simple land area intersection between census block and MGRAs. This dataset is then scaled to match the regional employment controls by ownership and industry creating employment/jobs by MGRA. See private SANDAG repository [Census-LEHD](https://github.com/SANDAG/Census-LEHD). +The [United States Census Bureau Longitudinal Employer-Household Dynamics (LEHD) Origin-Destination Employment Statistics (LODES)](https://lehd.ces.census.gov/data/) dataset provides census block employment by ownership and industry. This dataset requires a split of its block level two-digit NAICS 72 sector, Accommodation and Food Services, into sectors 721 (Accommodation) and 722 (Food Services) using the point geometry employment dataset and is allocated to MGRAs using a combination of the point geometry employment dataset and a simple land area intersection. This dataset is then scaled to match the regional employment controls by ownership and industry creating employment/jobs by MGRA. See private SANDAG repository [Census-LEHD](https://github.com/SANDAG/Census-LEHD). ## Subregional self-employment counts Census block group (2013+) or tract (2010-2012) counts of self-employed individuals are gotten from the American Community Survey (ACS) table [B24080 | SEX BY CLASS OF WORKER FOR THE CIVILIAN EMPLOYED POPULATION 16 YEARS AND OVER](https://data.census.gov/table/ACSDT5Y2020.B24080?q=B24080) using the total count of *Self-employed in own not incorporated business workers*. The counts are allocated to MGRAs using a hierarchy of cross references. From e959239d94b65b1d6e4199e7fd91547099a2fb0b Mon Sep 17 00:00:00 2001 From: GregorSchroeder Date: Tue, 14 Jul 2026 17:35:05 -0700 Subject: [PATCH 13/14] #278 - pr feedback wiki update --- wiki/Household-Characteristics.md | 44 ++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/wiki/Household-Characteristics.md b/wiki/Household-Characteristics.md index d94f947..8365ea0 100644 --- a/wiki/Household-Characteristics.md +++ b/wiki/Household-Characteristics.md @@ -1,13 +1,15 @@ # Inputs -| Input | Module Source | Usage | -|-------------------------------------------------------|------------------------|---------------------------------------------------------------| -| MGRA Geography (`[inputs].[mgra]`) | Startup | Used to merge MGRA-level values with census tract rates | -| MGRA Cross References | Demographic Warehouse | Provides cross reference from MGRAs to census tracts | -| Households in each MGRA (`[outputs].[hh]`) | Housing and Households | Used to generate household characteristics | -| Household population in each MGRA (`[outputs].[hhp]`) | Housing and Households | Used to balance household size implied household population | -| Census tract household income distribution | External (ACS) | Apply to households to create households by income categories | -| Census tract households by size distribution | External (ACS) | Apply to households to create households by size categories | +| Input | Module Source | Usage | +|-----------------------------------------------------------|---------------------------------|----------------------------------------------------------------------| +| MGRA Geography (`[inputs].[mgra]`) | Startup | Used to merge MGRA-level values with census tract rates | +| MGRA Cross References | Demographic Warehouse | Provides cross reference from MGRAs to census tracts | +| Households in each MGRA (`[outputs].[hh]`) | Housing and Households | Used to generate household characteristics | +| Household population in each MGRA (`[outputs].[hhp]`) | Housing and Households | Used to balance household size implied household population | +| Household population 18+ in each MGRA (`[outputs].[ase]`) | Population by Age Sex Ethnicity | Used to balance household workers implied worker population | +| Census tract household income distribution | External (ACS) | Apply to households to create households by income categories | +| Census tract households by size distribution | External (ACS) | Apply to households to create households by size categories | +| Census tract households by workers distribution | External (ACS) | Apply to households to create households by workers categories | ## MGRA Geography (`[inputs].[mgra]`) See [Startup](https://github.com/SANDAG/Estimates-Program/wiki/Startup). @@ -21,6 +23,9 @@ See [Housing and Households](https://github.com/SANDAG/Estimates-Program/wiki/Ho ## Household population in each MGRA (`[outputs].[hhp]`) See [Housing and Households](https://github.com/SANDAG/Estimates-Program/wiki/Housing-and-Households). +## Household population 18+ in each MGRA (`[outputs].[ase]`) +See [Population by Age Sex Ethnicity](https://github.com/SANDAG/Estimates-Program/wiki/Population-by-Age-Sex-Ethnicity). + ## Census tract household income distribution The household income distribution is derived from American Community Survey (ACS) table [B19001 | HOUSEHOLD INCOME IN THE PAST 12 MONTHS](https://data.census.gov/table/ACSDT5Y2020.B19001?q=B19001). The table provides households by household income. Dividing each category by total households provides census tract household income distributions. The year of the ACS table provides the inflation-adjusted dollars year. @@ -49,6 +54,20 @@ To avoid division by zero errors, the households by size distribution is set to \forall hhs \in \text{Household Size Category}; \text{Regional Household Size Distribution}_{hhs} = \frac{\sum \text{Households in Size Category}_{hhs}}{\sum \text{Households}} ``` +## Census tract households by workers distribution + +The households by workers distribution is derived from ACS table [B08202 | HOUSEHOLD SIZE BY NUMBER OF WORKERS IN HOUSEHOLD](https://data.census.gov/table/ACSDT5Y2020.B08202?q=B08202). The table provides households by number of workers category. Dividing each category by total households provides census tract households by workers distributions. + +```math +\forall t \in \text{San Diego Tracts}, \forall hhs \in \text{Household Worker Category}; \text{Distribution}_{t,hhs} = \frac{\text{Households in Worker Category}_{t,hhs}}{\text{Households}_t} +``` + +To avoid division by zero errors, the households by workers distribution is set to `NULL` if the households are zero within a census tract. This can lead to conflict between ACS data and SANDAG's LUDU. Within a census tract, the ACS may have no housing units, no households, and a `NULL` households by workers distribution while SANDAG's LUDU contains housing units. In this situation, the regional households by workers distribution is used. + +```math +\forall hhs \in \text{Household Worker Category}; \text{Regional Household Worker Distribution}_{hhs} = \frac{\sum \text{Households in Worker Category}_{hhs}}{\sum \text{Households}} +``` + # Outputs (`[outputs].[hh_characteristics]`) Each row of this table contains the following information: @@ -67,4 +86,11 @@ MGRA households by income category. Calculated by applying census tract distribu ## Households by size category in each MGRA MGRA households by size category. Calculated by applying census tract distributions to households. Adjusts size categories within MGRAs such that the implied household population range (min-max) contains the actual MGRA household population value. -For example, within a MGRA with 10 households of size one, 10 households of size two, ..., 10 households of size 7+, the minimum amount of household population would be 1x10 + 2x20 + ... + 7x10 = 280. The maximum amount of household population, assuming the 7+ category all average 11 people (see [issue #112](https://github.com/SANDAG/Estimates-Program/issues/112)), would be 1x10 + 2x20 + ... + 11x10 = 320. The actual amount of household population in this MGRA must be between these two values. If it is not, the households in size categories are adjusted until the condition is satisfied. \ No newline at end of file +For example, within a MGRA with 10 households of size one, 10 households of size two, ..., 10 households of size 7+, the minimum amount of household population would be 1x10 + 2x20 + ... + 7x10 = 280. The maximum amount of household population, assuming the 7+ category all average 11 people (see [issue #112](https://github.com/SANDAG/Estimates-Program/issues/112)), would be 1x10 + 2x20 + ... + 11x10 = 320. The actual amount of household population in this MGRA must be between these two values. If it is not, the households in size categories are adjusted until the condition is satisfied. + +## Households by workers category in each MGRA +MGRA households by workers category. Calculated by applying census tract distributions to households. Adjusts workers categories within MGRAs such that the implied minimum worker population does not exceed the MGRA household population of persons aged 18+. Then adjusts households by workers categories such that they respect the households by size categories in each MGRA. + +For example, within a MGRA with 10 households with 0 workers, 10 households with 1 workers, 10 households with 2 workers, 10 households with 3+ workers, the minimum implied household population of persons aged 18+ is 0x10 + 1x10 + 2x10 + 3x10 = 60. If the MGRA household population of persons aged 18+ is less than 60 then the households by workers categories are adjusted until the condition is satisfied. This is a soft constraint as we do not estimate the true number of workers in each MGRA. + +The process then moves onto adjusting households by workers categories such that they respect the households by size categories in each MGRA. Again, within a MGRA with 10 households with 0 workers, 10 households with 1 workers, 10 households with 2 workers, 10 households with 3+ workers, this implies there are at least 10 households of size 2+ and 10 household of size 3+. The 0 and 1 worker households can be of any size 1+ and do not violate any households by size category conditions as the total households within each MGRA is respected. If the MGRA households by size categories aggregated to 2+ and 3+ are less than 10 then the households by workers categories are adjusted until the condition is satisfied. \ No newline at end of file From a7596e3a13dd292330124a1f7f7e096ea36fb1a0 Mon Sep 17 00:00:00 2001 From: GregorSchroeder Date: Thu, 16 Jul 2026 08:51:58 -0700 Subject: [PATCH 14/14] #278 - add qc checks --- reporting/check_hhworkers_hhsize.sql | 39 +++++++++++++++++++++++ reporting/check_implied_workers.sql | 47 ++++++++++++++++++++++++++++ reporting/reporting.py | 4 ++- 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 reporting/check_hhworkers_hhsize.sql create mode 100644 reporting/check_implied_workers.sql diff --git a/reporting/check_hhworkers_hhsize.sql b/reporting/check_hhworkers_hhsize.sql new file mode 100644 index 0000000..ddb6efc --- /dev/null +++ b/reporting/check_hhworkers_hhsize.sql @@ -0,0 +1,39 @@ +-- SQL script to check that households by workers do not exceed households by size +DECLARE @run_id INTEGER = :run_id; + + +-- Check hard constraint of household size is not violated +WITH [Household Size] AS ( + SELECT + [year], + [mgra], + SUM(CASE WHEN [metric] != 'Household Size - 1' THEN [value] ELSE 0 END) AS [2+ size], -- 2+ household size + SUM(CASE WHEN [metric] NOT IN ('Household Size - 1','Household Size - 2') THEN [value] ELSE 0 END) AS [3+ size] -- 3+ household size + FROM [outputs].[hh_characteristics] + WHERE [run_id] = @run_id AND [metric] LIKE 'Household Size%' + GROUP BY [year], [mgra] +), +[Household Workers] AS ( + SELECT + [year], + [mgra], + SUM(CASE WHEN [metric] = 'Household Workers - 2' THEN [value] ELSE 0 END) AS [2 workers], + SUM(CASE WHEN [metric] = 'Household Workers - 3+' THEN [value] ELSE 0 END) AS [3+ workers] + FROM [outputs].[hh_characteristics] + WHERE [run_id] = @run_id AND [metric] LIKE 'Household Workers%' + GROUP BY [year], [mgra] +) +SELECT + [Household Size].[year], + [Household Size].[mgra], + [Household Size].[2+ size], + [Household Size].[3+ size], + [Household Workers].[2 workers], + [Household Workers].[3+ workers] +FROM [Household Size] +INNER JOIN [Household Workers] + ON [Household Size].[year] = [Household Workers].[year] + AND [Household Size].[mgra] = [Household Workers].[mgra] +WHERE + [Household Size].[2+ size] < [Household Workers].[2 workers] + OR [Household Size].[3+ size] < [Household Workers].[3+ workers] \ No newline at end of file diff --git a/reporting/check_implied_workers.sql b/reporting/check_implied_workers.sql new file mode 100644 index 0000000..2a1b480 --- /dev/null +++ b/reporting/check_implied_workers.sql @@ -0,0 +1,47 @@ +-- SQL script to check that implied workers do not exceed persons aged 18+ +DECLARE @run_id INTEGER = :run_id; + + +-- Check soft constraint of persons aged 18+ +WITH [Household Size] AS ( + SELECT + [year], + [mgra], + SUM( + CASE + WHEN [metric] = 'Household Workers - 1' THEN [value] + WHEN [metric] = 'Household Workers - 2' THEN [value]*2 + WHEN [metric] = 'Household Workers - 3+' THEN [value]*3 + ELSE 0 + END + ) AS [min_workers] + FROM [outputs].[hh_characteristics] + WHERE [run_id] = @run_id AND [metric] LIKE 'Household Workers%' + GROUP BY [year], [mgra] +), +[Population Aged 18+] AS ( + SELECT + [year], + [mgra], + SUM([value]) AS [persons_18plus] + FROM [outputs].[ase] + WHERE + [run_id] = @run_id + AND [age_group] NOT IN ( + 'Under 5', + '5 to 9', + '10 to 14', + '15 to 17' + ) + GROUP BY [year], [mgra] +) +SELECT + [Household Size].[year], + [Household Size].[mgra], + [Household Size].[min_workers], + ISNULL([Population Aged 18+].[persons_18plus], 0) AS [persons_18plus] +FROM [Household Size] +LEFT OUTER JOIN [Population Aged 18+] + ON [Household Size].[year] = [Population Aged 18+].[year] + AND [Household Size].[mgra] = [Population Aged 18+].[mgra] +WHERE [min_workers] > ISNULL([Population Aged 18+].[persons_18plus], 0); \ No newline at end of file diff --git a/reporting/reporting.py b/reporting/reporting.py index b7f935f..3edf284 100644 --- a/reporting/reporting.py +++ b/reporting/reporting.py @@ -4,7 +4,7 @@ # https://github.com/SANDAG/Series-15-Urban-Development-Model/blob/main/Other/Significant%20Change.xlsx # Main configuration, what run_id to operate on -RUN_ID = 245 +RUN_ID = 229 # We cannot import python.utils, as just importing will cause a new [run_id]` value and # new log file to be created. Instead, copy what we need for now :( @@ -46,6 +46,8 @@ "Check every HHP has HH": "check_every_hhp_has_hh.sql", "Check implied HHP vs actual HHP": "check_implied_hhp_vs_actual_hhp.sql", "Check householders vs households": "check_householders_vs_households.sql", + "Check implied workers vs persons 18+": "check_implied_workers.sql", + "Check households by workers vs households by size": "check_hhworkers_hhsize.sql", } # Run each script, printing out status messages if necessary: