diff --git a/python/hh_characteristics.py b/python/hh_characteristics.py index 2b9b11d..0227648 100644 --- a/python/hh_characteristics.py +++ b/python/hh_characteristics.py @@ -15,31 +15,43 @@ def run_hh_characteristics(year: int, debug: bool) -> None: """Orchestrator function to calculate and insert household characteristics. - The exact household characteristics created are: - 1. Households split by household income category - 2. Households split by number of people in each household + The household characteristics created are: + 1. Households by household income category + 2. Households by number of people in each household (size) + 3. Households by number of workers in each household (workers) - Both characteristics are generated by applying ACS data to MGRA level households, - which are created by the HS/HH module. + All characteristics are generated by applying ACS data to MGRA level households, + which are created by the Housing and Households module. Functionality is segmented + into functions for code encapsulation. - Functionality is segmented into functions for code encapsulation. The following are - used for households split by income category: + The following functions are used for households by income category: _get_hh_income_inputs - Get MGRA households and ACS tract distributions for - income - _validate_hh_income_inputs - Validate the hh income input data - _create_hh_income - Calculate the hh income, control to MGRA households - _validate_hh_income_outputs - Validate the hh income output data - _insert_hh_income - Insert hh income and tract level income distributions to - database - - The following functions are used for households split by size + household income categories + _validate_hh_income_inputs - Validate the household income category input data + _create_hh_income - Calculate the household income, control to MGRA households + _validate_hh_income_outputs - Validate the household income output data + _insert_hh_income - Insert household income and tract level income + distributions to the production database + + The following functions are used for households by size: _get_hh_size_inputs - Get MGRA households, MGRA household population, and ACS - tract distributions for size - _validate_hh_size_inputs - Validate the hh size input data - _create_hh_size - Calculate the hh size, control to MGRA households and MGRA - household population - _validate_hh_size_outputs - Validate the hh size output data - _insert_hh_size - Insert hh size and tract level size distributions to database + tract distributions for household size + _validate_hh_size_inputs - Validate the household size input data + _create_hh_size - Calculate the household size, control to MGRA households and + MGRA household population + _validate_hh_size_outputs - Validate the household size output data + _insert_hh_size - Insert household size and tract level size distributions to + the production database + + The following functions are used for households by workers: + _get_hh_workers_inputs - Get MGRA households, MGRA population aged 18+, MGRA + households by size, and ACS tract distributions for households workers + _validate_hh_workers_inputs - Validate the household workers input data + _create_hh_workers - Calculate the household workers, control to MGRA + households, MGRA population aged 18+, and MGRA households by size + _validate_hh_workers_outputs - Validate the household workers output data + _insert_hh_workers - Insert household workers and tract level workers + distributions to the production database Args: year (int): estimates year @@ -62,6 +74,15 @@ def run_hh_characteristics(year: int, debug: bool) -> None: _insert_hh_size(hh_size_inputs, hh_size_outputs, debug) + # And finally do households by workers + hh_workers_inputs = _get_hh_workers_inputs(year) + _validate_hh_workers_inputs(year, hh_workers_inputs) + + hh_workers_outputs = _create_hh_workers(hh_workers_inputs) + _validate_hh_workers_outputs(hh_workers_outputs) + + _insert_hh_workers(hh_workers_inputs, hh_workers_outputs, debug) + def _get_hh_income_inputs(year: int) -> dict[str, pd.DataFrame]: """Get households and various tract level datas""" @@ -100,12 +121,12 @@ def _get_hh_size_inputs(year: int) -> dict[str, pd.DataFrame]: """Get households and various tract level datas.""" with utils.ESTIMATES_ENGINE.connect() as con: # Store results here - hh_char_inputs = {} + hh_size_inputs = {} # Get MGRA level households. Note we re-use the SQL script from the 'Population # by Type' module with open(utils.SQL_FOLDER / "pop_type" / "get_mgra_hh.sql") as file: - hh_char_inputs["hh"] = pd.read_sql_query( + hh_size_inputs["hh"] = pd.read_sql_query( sql=sql.text(file.read()), con=con, params={ @@ -120,7 +141,7 @@ def _get_hh_size_inputs(year: int) -> dict[str, pd.DataFrame]: / "hh_characteristics" / "get_tract_controls_hh_by_size.sql" ) as file: - hh_char_inputs["hhs_tract_controls"] = utils.read_sql_query_fallback( + hh_size_inputs["hhs_tract_controls"] = utils.read_sql_query_fallback( sql=sql.text(file.read()), # type: ignore con=con, # type: ignore params={"run_id": utils.RUN_ID, "year": year}, @@ -130,13 +151,70 @@ def _get_hh_size_inputs(year: int) -> dict[str, pd.DataFrame]: with open( utils.SQL_FOLDER / "hh_characteristics" / "get_mgra_controls_hh_by_size.sql" ) as file: - hh_char_inputs["hhs_mgra_controls"] = pd.read_sql_query( + hh_size_inputs["hhs_mgra_controls"] = pd.read_sql_query( sql=sql.text(file.read()), con=con, params={"run_id": utils.RUN_ID, "year": year}, # type: ignore ) - return hh_char_inputs + return hh_size_inputs + + +def _get_hh_workers_inputs(year: int) -> dict[str, pd.DataFrame]: + """Get MGRA and tract level controls for households by workers.""" + with utils.ESTIMATES_ENGINE.connect() as con: + # Store results here + hh_workers_inputs = {} + + # Get MGRA level households. Note we re-use the SQL script from the 'Population + # by Type' module + with open(utils.SQL_FOLDER / "pop_type" / "get_mgra_hh.sql") as file: + hh_workers_inputs["hh"] = pd.read_sql_query( + sql=sql.text(file.read()), + con=con, + params={ + "run_id": utils.RUN_ID, + "year": year, + }, # type: ignore + ).drop(columns=["jurisdiction"]) + + # Get MGRA level population aged 18+ + with open( + utils.SQL_FOLDER + / "hh_characteristics" + / "get_mgra_controls_hh_by_workers_18plus.sql" + ) as file: + hh_workers_inputs["mgra_18plus"] = pd.read_sql_query( + sql=sql.text(file.read()), + con=con, + params={"run_id": utils.RUN_ID, "year": year}, # type: ignore + ) + + # Get MGRA level households by size + with open( + utils.SQL_FOLDER + / "hh_characteristics" + / "get_mgra_controls_hh_by_workers_hhs.sql" + ) as file: + hh_workers_inputs["mgra_hhs"] = pd.read_sql_query( + sql=sql.text(file.read()), + con=con, + params={"run_id": utils.RUN_ID, "year": year}, # type: ignore + ) + + # Get tract level households by workers distributions + with open( + utils.SQL_FOLDER + / "hh_characteristics" + / "get_tract_controls_hh_by_workers.sql" + ) as file: + hh_workers_inputs["tract_workers"] = utils.read_sql_query_fallback( + sql=sql.text(file.read()), + con=con, + params={"run_id": utils.RUN_ID, "year": year}, # type: ignore + ) + + return hh_workers_inputs def _validate_hh_income_inputs( @@ -186,6 +264,40 @@ def _validate_hh_size_inputs( ) +def _validate_hh_workers_inputs( + year: int, hh_workers_inputs: dict[str, pd.DataFrame] +) -> None: + """Validate the household characteristics input data""" + tests.validate_data( + "MGRA households", + hh_workers_inputs["hh"], + row_count={"key_columns": {"mgra"}}, + negative={}, + null={}, + ) + tests.validate_data( + "MGRA persons aged 18+ controls", + hh_workers_inputs["mgra_18plus"], + row_count={"key_columns": {"mgra"}}, + negative={}, + null={}, + ) + tests.validate_data( + "MGRA households by household size controls", + hh_workers_inputs["mgra_hhs"], + row_count={"key_columns": {"mgra"}}, + negative={}, + null={}, + ) + tests.validate_data( + "Tract households by household workers distribution", + hh_workers_inputs["tract_workers"], + row_count={"key_columns": {"tract", "household_workers"}, "year": year}, + negative={}, + null={}, + ) + + def _create_hh_income( hh_income_inputs: dict[str, pd.DataFrame], ) -> dict[str, pd.DataFrame]: @@ -457,6 +569,221 @@ def adjust_mgra(mgra_data: pd.Series) -> pd.Series: } +def _create_hh_workers( + hh_workers_inputs: dict[str, pd.DataFrame], +) -> dict[str, pd.DataFrame]: + """Code to compute MGRA level households by workers + + Similar to how income and size work, this function takes MGRA households, applies + tract level rates, then integerizes the data. But additionally, we control to the + MGRA population aged 18+ and MGRA households by size. This is because the + distribution of households by household workers implies some minimum number of + workers in the MGRA, which cannot exceed the actual number of workers in the MGRA. + Because the actual number of workers in the MGRA is not known, we use the + population aged 18+ as a proxy. Additionally, the distribution of households by + household workers implies a minimum number of households by size in the 2+ and 3+ + categories, which cannot exceed the actual number of households by size within each + category in the MGRA. + """ + hh = hh_workers_inputs["hh"] + tract_workers_dist = hh_workers_inputs["tract_workers"] + mgra_18plus = hh_workers_inputs["mgra_18plus"] + mgra_hhs = hh_workers_inputs["mgra_hhs"] + + # Combine the total households in each MGRA with the distribution of households by + # number of workers + hh_workers = ( + hh.merge(tract_workers_dist, on=["run_id", "year", "tract"], how="left") + .assign(hh=lambda df: df["hh"] * df["value"]) + .drop(columns=["value"]) + .pivot( + index=["run_id", "year", "mgra", "tract"], + columns="household_workers", + values="hh", + ) + .reset_index(drop=False) + .sort_values(by="mgra") + .reset_index(drop=True) + ) + + # To ensure that we pretty much exactly match ACS distributions, we will do two- + # dimensional controlling on the MGRA level data. After splitting the data into + # separate tracts, row controls will be total households in each MGRA and column + # controls will be tract level households by workers + controlled_groups = [] + for tract, group in hh_workers.groupby("tract"): + seed_data = group[utils.HOUSEHOLD_WORKERS].to_numpy() + + row_controls = ( + group[utils.HOUSEHOLD_WORKERS].sum(axis=1).to_numpy().round(0).astype(int) + ) + + col_controls = utils.integerize_1d( + group[utils.HOUSEHOLD_WORKERS].sum(axis=0), generator=generator + ) + + controlled_data = utils.integerize_2d( + data=seed_data, + row_ctrls=row_controls, + col_ctrls=col_controls, + condition="exact", + suppress_warnings=True, + generator=generator, + ) + + # Assign the controlled data back to the tract + group[utils.HOUSEHOLD_WORKERS] = controlled_data + controlled_groups.append(group) + + # Recombine all the data in preparation for the next controlling step + hh_workers = pd.concat(controlled_groups) + + # For every MGRA, compute the minimum implied worker population + # And difference between implied worker and 18+ population + hh_workers = ( + hh_workers.merge(mgra_18plus, on=["run_id", "year", "mgra"], how="left") + .astype({workers: int for workers in utils.HOUSEHOLD_WORKERS}) + .assign( + min_implied_workers=lambda df: df[1] + (2 * df[2]) + (3 * df[3]), + # Compute the difference between the 18+ population and the implied minimum + decrease_min=lambda df: np.where( + df["min_implied_workers"] > df["persons_18plus"], + df["min_implied_workers"] - df["persons_18plus"], + 0, + ), + ) + ) + + # The methodology to adjust each individual MGRA for minimum implied workers + # TODO: Consider ways to parallelize + def adjust_mgra_18plus(mgra_data: pd.Series) -> pd.Series: + if mgra_data["decrease_min"] == 0: + return mgra_data + while True: + # Choose a random household workers category to decrease + # weighted by the number of households in each category + workers_to_decrease = generator.choice( + utils.HOUSEHOLD_WORKERS, + p=mgra_data[utils.HOUSEHOLD_WORKERS] + / mgra_data[utils.HOUSEHOLD_WORKERS].sum(), + ) + + # Look below the chosen category for a category to increase + if mgra_data["decrease_min"] > 0 and workers_to_decrease != 0: + # Avoid overshooting the decrease + workers_to_increase = generator.choice( + range( + np.max([0, workers_to_decrease - mgra_data["decrease_min"]]), + workers_to_decrease, + ) + ) + # Continue if we need to decrease implied workers but the randomly chosen + # category was already 0 workers, restart the loop to choose a new category + else: + continue + + # Execute the change and recompute the remaining change needed + mgra_data[workers_to_decrease] -= 1 + mgra_data[workers_to_increase] += 1 + mgra_data["decrease_min"] += workers_to_increase - workers_to_decrease + + # Check if we are done with this MGRA + if mgra_data["decrease_min"] == 0: + return mgra_data + + # Apply the MGRA adjustments for implied workers and 18+ population + # Note we apply minimum implied workers first then household size categories + # This is because the household size categories are a hard constraint whereas + # the minimum implied workers is a soft constraint due to data limitations + hh_workers = hh_workers.apply(adjust_mgra_18plus, axis=1).drop( + columns=["tract", "persons_18plus", "min_implied_workers", "decrease_min"] + ) + + # For every MGRA, compute the difference between households by number of workers + # and the households by size categories to determine necessary adjustments + # Note that the household size categories are collapsed to 2+ and 3+ + hh_workers = hh_workers.merge( + mgra_hhs, on=["run_id", "year", "mgra"], how="left" + ).assign( + # Expect to see differences >= 0 for Household Sizes - Household Workers + # Household Worker category 2 should be <= Household Size 2+ + diff2=lambda df: df["2"] - df[2], + # Household Worker category 3+ should be <= Household Size 3+ + diff3=lambda df: df["3"] - df[3], + ) + + # The methodology to adjust each individual MGRA to not violate household size + # constraints. Note that the household size categories are collapsed to 2+ and 3+ + # and that the 0 and 1 worker categories can always be added to as they imply + # a minimum household size of 1+ which is satisfied by every category + def adjust_mgra_hhs(mgra_data: pd.Series) -> pd.Series: + if mgra_data["diff2"] >= 0 and mgra_data["diff3"] >= 0: + return mgra_data + while True: + # Identify households by worker categories that need adjustment + # and take the first worker category identified to decrease + if mgra_data["diff2"] < 0: + workers_to_decrease = 2 + elif mgra_data["diff3"] < 0: + workers_to_decrease = 3 + + # Identify categories that can accommodate additional households + # that are smaller than the worker category we are decreasing + # Note the 0 and 1 worker categories are always eligible to increase + if workers_to_decrease == 3: + if mgra_data["diff2"] > 0: + workers_to_increase = [0, 1, 2] + else: + workers_to_increase = [0, 1] + elif workers_to_decrease == 2: + workers_to_increase = [0, 1] + + # Use a weighted random methodology to select the category to increase + # If all categories to increase are 0 just choose one at a random + if mgra_data[workers_to_increase].sum() == 0: + workers_to_increase = generator.choice(workers_to_increase) + else: + workers_to_increase = generator.choice( + workers_to_increase, + p=mgra_data[workers_to_increase] + / mgra_data[workers_to_increase].sum(), + ) + + # Execute the change and recompute the remaining change needed + + # Both 0 and 1 worker categories are not eligible to be decreased + mgra_data[workers_to_decrease] -= 1 + if workers_to_decrease == 2: + mgra_data["diff2"] += 1 + elif workers_to_decrease == 3: + mgra_data["diff3"] += 1 + + # Note we only track changes for the 2 and 3+ worker categories + # Although both 0 and 1 worker categories are elgibile to be increased + mgra_data[workers_to_increase] += 1 + if workers_to_increase == 2: + mgra_data["diff2"] -= 1 + elif workers_to_increase == 3: + mgra_data["diff3"] -= 1 + + # Check if we are done with this MGRA + if mgra_data["diff2"] >= 0 and mgra_data["diff3"] >= 0: + return mgra_data + + # Apply the MGRA adjustments for household size categories + hh_workers = hh_workers.apply(adjust_mgra_hhs, axis=1).drop( + columns=["2", "3", "diff2", "diff3"] + ) + + return { + "hh_workers": hh_workers.melt( + id_vars=["run_id", "year", "mgra"], + var_name="household_workers", + value_name="hh", + ) + } + + def _validate_hh_income_outputs(hh_income_outputs: dict[str, pd.DataFrame]) -> None: """Validate the household income output data""" tests.validate_data( @@ -479,6 +806,17 @@ def _validate_hh_size_outputs(hh_size_outputs: dict[str, pd.DataFrame]) -> None: ) +def _validate_hh_workers_outputs(hh_workers_outputs: dict[str, pd.DataFrame]) -> None: + """Validate the household workers output data""" + tests.validate_data( + "MGRA Households by Workers", + hh_workers_outputs["hh_workers"], + row_count={"key_columns": {"mgra", "household_workers"}}, + negative={}, + null={}, + ) + + def _insert_hh_income( hh_income_inputs: dict[str, pd.DataFrame], hh_income_outputs: dict[str, pd.DataFrame], @@ -584,3 +922,60 @@ def _insert_hh_size( con=con, index=False, ) + + +def _insert_hh_workers( + hh_workers_inputs: dict[str, pd.DataFrame], + hh_workers_outputs: dict[str, pd.DataFrame], + debug: bool, +) -> None: + """Insert hh characteristics and tract level controls to database""" + + inputs_controls_tract = ( + hh_workers_inputs["tract_workers"] + .rename(columns={"household_workers": "metric"}) + .assign( + metric=lambda df: "Household Workers - " + + df["metric"].astype(str).replace("3", "3+") + ) + ) + outputs_hh_characteristics = ( + hh_workers_outputs["hh_workers"][ + ["run_id", "year", "mgra", "household_workers", "hh"] + ] + .rename(columns={"household_workers": "metric", "hh": "value"}) + .assign( + metric=lambda df: "Household Workers - " + + df["metric"].astype(str).replace("3", "3+") + ) + ) + + # Save locally if in debug mode + if debug: + inputs_controls_tract.to_csv( + utils.DEBUG_OUTPUT_FOLDER / "inputs_controls_tract_hh_workers.csv", + index=False, + ) + outputs_hh_characteristics.to_csv( + utils.DEBUG_OUTPUT_FOLDER / "outputs_hh_characteristics_hh_workers.csv", + index=False, + ) + + # Otherwise, load to database + else: + with utils.ESTIMATES_ENGINE.connect() as con: + inputs_controls_tract.to_sql( + schema="inputs", + name="controls_tract", + if_exists="append", + con=con, + index=False, + ) + + outputs_hh_characteristics.to_sql( + schema="outputs", + name="hh_characteristics", + if_exists="append", + con=con, + index=False, + ) diff --git a/python/tests.py b/python/tests.py index e033c83..d1ac869 100644 --- a/python/tests.py +++ b/python/tests.py @@ -31,6 +31,7 @@ "structure_type": 4, "income_category": 10, "household_size": 7, + "household_workers": 4, "tract": { 2010: 627, 2020: 736, @@ -39,7 +40,7 @@ # The industry_code is a variable to group employment data into. Almost all # codes are 2-digit naics codes. The 2-digit naics code 72 was split into 721 # and 722 3-digit naics code. Self employment and military active duty data do - # not natively have a naics code so it is therefore assigned to 'SE' and 'MIL' + # not natively have a naics code so it is therefore assigned to 'SE' and 'MIL' # respectively, as the naics codes are being treated as strings. "industry_code": 23, }, @@ -52,7 +53,9 @@ # For ease of access, combine the constants part of the dictionary with the current # MGRA series -_DISTINCT_COUNTS = _DISTINCT_COUNTS["constants"] | _DISTINCT_COUNTS["series"][utils.SERIES] +_DISTINCT_COUNTS = ( + _DISTINCT_COUNTS["constants"] | _DISTINCT_COUNTS["series"][utils.SERIES] +) ######### diff --git a/python/utils.py b/python/utils.py index 5eeecf2..fae1c2c 100644 --- a/python/utils.py +++ b/python/utils.py @@ -9,7 +9,6 @@ import python.parsers as parsers - ######### # PATHS # ######### @@ -72,8 +71,8 @@ + _secrets["sql"]["gis"]["server"] + "/" + _secrets["sql"]["gis"]["database"] - + "?driver=ODBC Driver 18 for SQL Server" + - "&TrustServerCertificate=yes", + + "?driver=ODBC Driver 18 for SQL Server" + + "&TrustServerCertificate=yes", fast_executemany=True, ) @@ -127,6 +126,8 @@ HOUSEHOLD_SIZES = list(range(1, 8)) +HOUSEHOLD_WORKERS = list(range(0, 4)) + ASE = ["age_group", "sex", "ethnicity"] INCOME_CATEGORIES = [ diff --git a/reporting/check_hhworkers_hhsize.sql b/reporting/check_hhworkers_hhsize.sql new file mode 100644 index 0000000..ddb6efc --- /dev/null +++ b/reporting/check_hhworkers_hhsize.sql @@ -0,0 +1,39 @@ +-- SQL script to check that households by workers do not exceed households by size +DECLARE @run_id INTEGER = :run_id; + + +-- Check hard constraint of household size is not violated +WITH [Household Size] AS ( + SELECT + [year], + [mgra], + SUM(CASE WHEN [metric] != 'Household Size - 1' THEN [value] ELSE 0 END) AS [2+ size], -- 2+ household size + SUM(CASE WHEN [metric] NOT IN ('Household Size - 1','Household Size - 2') THEN [value] ELSE 0 END) AS [3+ size] -- 3+ household size + FROM [outputs].[hh_characteristics] + WHERE [run_id] = @run_id AND [metric] LIKE 'Household Size%' + GROUP BY [year], [mgra] +), +[Household Workers] AS ( + SELECT + [year], + [mgra], + SUM(CASE WHEN [metric] = 'Household Workers - 2' THEN [value] ELSE 0 END) AS [2 workers], + SUM(CASE WHEN [metric] = 'Household Workers - 3+' THEN [value] ELSE 0 END) AS [3+ workers] + FROM [outputs].[hh_characteristics] + WHERE [run_id] = @run_id AND [metric] LIKE 'Household Workers%' + GROUP BY [year], [mgra] +) +SELECT + [Household Size].[year], + [Household Size].[mgra], + [Household Size].[2+ size], + [Household Size].[3+ size], + [Household Workers].[2 workers], + [Household Workers].[3+ workers] +FROM [Household Size] +INNER JOIN [Household Workers] + ON [Household Size].[year] = [Household Workers].[year] + AND [Household Size].[mgra] = [Household Workers].[mgra] +WHERE + [Household Size].[2+ size] < [Household Workers].[2 workers] + OR [Household Size].[3+ size] < [Household Workers].[3+ workers] \ No newline at end of file diff --git a/reporting/check_implied_workers.sql b/reporting/check_implied_workers.sql new file mode 100644 index 0000000..2a1b480 --- /dev/null +++ b/reporting/check_implied_workers.sql @@ -0,0 +1,47 @@ +-- SQL script to check that implied workers do not exceed persons aged 18+ +DECLARE @run_id INTEGER = :run_id; + + +-- Check soft constraint of persons aged 18+ +WITH [Household Size] AS ( + SELECT + [year], + [mgra], + SUM( + CASE + WHEN [metric] = 'Household Workers - 1' THEN [value] + WHEN [metric] = 'Household Workers - 2' THEN [value]*2 + WHEN [metric] = 'Household Workers - 3+' THEN [value]*3 + ELSE 0 + END + ) AS [min_workers] + FROM [outputs].[hh_characteristics] + WHERE [run_id] = @run_id AND [metric] LIKE 'Household Workers%' + GROUP BY [year], [mgra] +), +[Population Aged 18+] AS ( + SELECT + [year], + [mgra], + SUM([value]) AS [persons_18plus] + FROM [outputs].[ase] + WHERE + [run_id] = @run_id + AND [age_group] NOT IN ( + 'Under 5', + '5 to 9', + '10 to 14', + '15 to 17' + ) + GROUP BY [year], [mgra] +) +SELECT + [Household Size].[year], + [Household Size].[mgra], + [Household Size].[min_workers], + ISNULL([Population Aged 18+].[persons_18plus], 0) AS [persons_18plus] +FROM [Household Size] +LEFT OUTER JOIN [Population Aged 18+] + ON [Household Size].[year] = [Population Aged 18+].[year] + AND [Household Size].[mgra] = [Population Aged 18+].[mgra] +WHERE [min_workers] > ISNULL([Population Aged 18+].[persons_18plus], 0); \ No newline at end of file diff --git a/reporting/reporting.py b/reporting/reporting.py index b7f935f..3edf284 100644 --- a/reporting/reporting.py +++ b/reporting/reporting.py @@ -4,7 +4,7 @@ # https://github.com/SANDAG/Series-15-Urban-Development-Model/blob/main/Other/Significant%20Change.xlsx # Main configuration, what run_id to operate on -RUN_ID = 245 +RUN_ID = 229 # We cannot import python.utils, as just importing will cause a new [run_id]` value and # new log file to be created. Instead, copy what we need for now :( @@ -46,6 +46,8 @@ "Check every HHP has HH": "check_every_hhp_has_hh.sql", "Check implied HHP vs actual HHP": "check_implied_hhp_vs_actual_hhp.sql", "Check householders vs households": "check_householders_vs_households.sql", + "Check implied workers vs persons 18+": "check_implied_workers.sql", + "Check households by workers vs households by size": "check_hhworkers_hhsize.sql", } # Run each script, printing out status messages if necessary: diff --git a/sql/hh_characteristics/get_mgra_controls_hh_by_size.sql b/sql/hh_characteristics/get_mgra_controls_hh_by_size.sql index e967bf5..1292710 100644 --- a/sql/hh_characteristics/get_mgra_controls_hh_by_size.sql +++ b/sql/hh_characteristics/get_mgra_controls_hh_by_size.sql @@ -3,11 +3,8 @@ Get MGRA controls for household population used in households by household size. This data is pulled from previously run modules. Two input parameters are used - run_id - the run identifier is used to get the list of census tracts in tandem with - the year parameter that determines their vintage - year - the year parameter is used to identify the 5-year ACS Detailed Tables release - to use (ex. 2023 maps to 2019-2023) and the census tract geography vintage - (2010-2019 uses 2010, 2020-2029 uses 2020) + run_id - the run identifier for this run + year - the year parameter grabs the year of data to use */ -- Initialize parameters diff --git a/sql/hh_characteristics/get_mgra_controls_hh_by_workers_18plus.sql b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_18plus.sql new file mode 100644 index 0000000..72528ff --- /dev/null +++ b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_18plus.sql @@ -0,0 +1,47 @@ +/* +Get MGRA controls for persons aged 18+ used in households by number of workers. +This data is pulled from previously run modules. + +It would be preferable to use 16+ as that is the age of eligibility for labor +force participation in the ACS but the Estimates Program uses 15 to 17 as an +age category so there is not an easy way to select only 16+. + +Two input parameters are used + run_id - the run identifier for this run + year - the year parameter grabs the year of data to use +*/ + +-- Initialize parameters +DECLARE @run_id integer = :run_id; +DECLARE @year integer = :year; + +-- Get population aged 18+ +WITH [ase_data] AS ( + SELECT + [mgra], + SUM([value]) AS [value] + FROM [outputs].[ase] + WHERE + [run_id] = @run_id + AND [year] = @year + AND [pop_type] = 'Household Population' + AND [age_group] NOT IN ( + 'Under 5', + '5 to 9', + '10 to 14', + '15 to 17' + ) + GROUP BY [mgra] +) +-- Ensure all MGRAs are included +SELECT + @run_id AS [run_id], + @year AS [year], + [mgra].[mgra], + ISNULL([value], 0) AS [persons_18plus] +FROM [inputs].[mgra] +LEFT OUTER JOIN [ase_data] + ON [mgra].[mgra] = [ase_data].[mgra] +WHERE + [mgra].[run_id] = @run_id +ORDER BY [mgra].[mgra] \ No newline at end of file diff --git a/sql/hh_characteristics/get_mgra_controls_hh_by_workers_hhs.sql b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_hhs.sql new file mode 100644 index 0000000..020720d --- /dev/null +++ b/sql/hh_characteristics/get_mgra_controls_hh_by_workers_hhs.sql @@ -0,0 +1,46 @@ +/* +Get MGRA controls for households by size used in households by number of workers. +This data is pulled from previously run portions of this module. + +Since households by workers only contains (0,1,2,3+) categories, the household +size categories are collapsed to (2+,3+) to match the minimum implied household +sizes from the households by workers categories (e.g. 0 and 1 worker households +can be any size 1+, a 2 worker household must be at least size 2, and a three +worker household must be at least size 3). + +Two input parameters are used + run_id - the run identifier for this run + year - the year parameter grabs the year of data to use +*/ + +-- Initialize parameters +DECLARE @run_id integer = :run_id; +DECLARE @year integer = :year; + +-- Get households by size +SELECT + @run_id AS [run_id], + @year AS [year], + [mgra], + SUM( + CASE + WHEN [metric] != 'Household Size - 1' THEN [value] + ELSE 0 + END + ) AS [2], -- 2+ household size + SUM( + CASE + WHEN [metric] NOT IN ( + 'Household Size - 1', + 'Household Size - 2' + ) THEN [value] + ELSE 0 + END + ) AS [3] -- 3+ household size +FROM [outputs].[hh_characteristics] +WHERE + [run_id] = @run_id + AND [year] = @year + AND [metric] LIKE 'Household Size%' +GROUP BY [mgra] +ORDER BY [mgra] \ No newline at end of file diff --git a/sql/hh_characteristics/get_tract_controls_hh_by_workers.sql b/sql/hh_characteristics/get_tract_controls_hh_by_workers.sql new file mode 100644 index 0000000..00095b8 --- /dev/null +++ b/sql/hh_characteristics/get_tract_controls_hh_by_workers.sql @@ -0,0 +1,141 @@ +/* +Get census tract controls for households by number of workers. Census tracts without a +distribution default to using the regional distribution. + +Controls are generated by the 5-year ACS Detailed Tables + B08202 - Household Size by Number of Workers in Household + +Two input parameters are used + run_id - the run identifier is used to get the list of census tracts in tandem with + the year parameter that determines their vintage + year - the year parameter is used to identify the 5-year ACS Detailed Tables release + to use (ex. 2023 maps to 2019-2023) and the census tract geography vintage + (2010-2019 uses 2010, 2020-2029 uses 2020) +*/ + +SET NOCOUNT ON; + +-- Initialize parameters --------------------------------------------------------------- +DECLARE @run_id integer = :run_id; +DECLARE @year integer = :year; +DECLARE @series INTEGER = (SELECT [series] FROM [metadata].[run] WHERE [run_id] = @run_id); + +-- Send error message if no data exists ------------------------------------------------ +DECLARE @msg nvarchar(45) = 'ACS 5-Year Table does not exist'; +IF NOT EXISTS ( + SELECT TOP (1) * + FROM [acs].[detailed].[tables] + WHERE + [name] = 'B08202' + AND [year] = @year + AND [product] = '5Y' +) +SELECT @msg AS [msg] +ELSE +BEGIN + + -- Build the expected return table Tract x Workers + DROP TABLE IF EXISTS [#tt_shell]; + SELECT + [tract], + [household_workers] + INTO [#tt_shell] + FROM ( + SELECT DISTINCT [tract] + FROM [demographic_warehouse].[dim].[mgra] + INNER JOIN [demographic_warehouse].[dim].[mgra_xref] + ON [mgra].[mgra_id] = [mgra_xref].[mgra_id] + AND [mgra_xref].[xref_year] = @year + WHERE [mgra].[series] = @series + ) AS [tracts] + CROSS JOIN ( + SELECT [household_workers] FROM ( + VALUES + (0), + (1), + (2), + (3) + ) AS [tt] ([household_workers]) + ) AS [household_workers]; + + + -- Prepare intermediary results from ACS datasets ---------------------------------- + -- 5-year ACS Detailed Table B08202 - Household Size by Number of Workers in Household + DROP TABLE IF EXISTS [#hh_by_workers]; + SELECT + [tract], + [household_workers], + SUM([value]) AS [hh] + INTO [#hh_by_workers] + FROM ( + SELECT + [tract], + CASE + WHEN REPLACE([label], ':', '') = 'Estimate!!Total!!No workers' THEN 0 + WHEN REPLACE([label], ':', '') = 'Estimate!!Total!!1 worker' THEN 1 + WHEN REPLACE([label], ':', '') = 'Estimate!!Total!!2 workers' THEN 2 + WHEN REPLACE([label], ':', '') = 'Estimate!!Total!!3 or more workers' THEN 3 + ELSE NULL -- NULL values for Margin of Error fields removed in subsequent WHERE clause + END AS [household_workers], + [value] + FROM [acs].[detailed].[values] + LEFT JOIN [acs].[detailed].[geography] + ON [values].[geography_id] = [geography].[geography_id] + LEFT JOIN [acs].[detailed].[variables] + ON [values].[variable] = [variables].[variable] + AND [values].[table_id] = [variables].[table_id] + LEFT JOIN [acs].[detailed].[tables] + ON [values].[table_id] = [tables].[table_id] + WHERE + [tables].[name] = 'B08202' + AND [tables].[year] = @year + AND [tables].[product] = '5Y' + ) AS [b08202] + WHERE [household_workers] IS NOT NULL + GROUP BY [tract], [household_workers] + + -- Create hh workers distribution -------------------------------------------------- + -- Calculate regional hh by workers distribution + DECLARE @total_hh INTEGER = (SELECT SUM([hh]) FROM [#hh_by_workers]); + WITH [region_workers_distribution] AS ( + SELECT + [household_workers], + 1.0 * SUM([hh]) / @total_hh AS [hh_dist] + FROM [#hh_by_workers] + GROUP BY [household_workers] + ), + -- Calculate census tract hh by workers distribution + [tract_distribution] AS ( + SELECT + [#hh_by_workers].[tract], + [household_workers], + CASE + WHEN [tract_total_hh].[hh] = 0 THEN NULL + ELSE [#hh_by_workers].[hh] / [tract_total_hh].[hh] + END AS [hh_dist] + FROM [#hh_by_workers] + LEFT JOIN ( + SELECT + [tract], + SUM([hh]) AS [hh] + FROM [#hh_by_workers] + GROUP BY [tract] + ) AS [tract_total_hh] + ON [#hh_by_workers].[tract] = [tract_total_hh].[tract] + ) + SELECT + @run_id AS [run_id], + @year AS [year], + [#tt_shell].[tract], + [#tt_shell].[household_workers], + ISNULL([tract_distribution].[hh_dist], [region_workers_distribution].[hh_dist]) AS [value] + FROM [#tt_shell] + LEFT JOIN [region_workers_distribution] + ON [#tt_shell].[household_workers] = [region_workers_distribution].[household_workers] + LEFT JOIN [tract_distribution] + ON [#tt_shell].[tract] = [tract_distribution].[tract] + AND [#tt_shell].[household_workers] = [tract_distribution].[household_workers] + ORDER BY + [#tt_shell].[tract], + [#tt_shell].[household_workers]; +END \ No newline at end of file diff --git a/wiki/Household-Characteristics.md b/wiki/Household-Characteristics.md index d94f947..8365ea0 100644 --- a/wiki/Household-Characteristics.md +++ b/wiki/Household-Characteristics.md @@ -1,13 +1,15 @@ # Inputs -| Input | Module Source | Usage | -|-------------------------------------------------------|------------------------|---------------------------------------------------------------| -| MGRA Geography (`[inputs].[mgra]`) | Startup | Used to merge MGRA-level values with census tract rates | -| MGRA Cross References | Demographic Warehouse | Provides cross reference from MGRAs to census tracts | -| Households in each MGRA (`[outputs].[hh]`) | Housing and Households | Used to generate household characteristics | -| Household population in each MGRA (`[outputs].[hhp]`) | Housing and Households | Used to balance household size implied household population | -| Census tract household income distribution | External (ACS) | Apply to households to create households by income categories | -| Census tract households by size distribution | External (ACS) | Apply to households to create households by size categories | +| Input | Module Source | Usage | +|-----------------------------------------------------------|---------------------------------|----------------------------------------------------------------------| +| MGRA Geography (`[inputs].[mgra]`) | Startup | Used to merge MGRA-level values with census tract rates | +| MGRA Cross References | Demographic Warehouse | Provides cross reference from MGRAs to census tracts | +| Households in each MGRA (`[outputs].[hh]`) | Housing and Households | Used to generate household characteristics | +| Household population in each MGRA (`[outputs].[hhp]`) | Housing and Households | Used to balance household size implied household population | +| Household population 18+ in each MGRA (`[outputs].[ase]`) | Population by Age Sex Ethnicity | Used to balance household workers implied worker population | +| Census tract household income distribution | External (ACS) | Apply to households to create households by income categories | +| Census tract households by size distribution | External (ACS) | Apply to households to create households by size categories | +| Census tract households by workers distribution | External (ACS) | Apply to households to create households by workers categories | ## MGRA Geography (`[inputs].[mgra]`) See [Startup](https://github.com/SANDAG/Estimates-Program/wiki/Startup). @@ -21,6 +23,9 @@ See [Housing and Households](https://github.com/SANDAG/Estimates-Program/wiki/Ho ## Household population in each MGRA (`[outputs].[hhp]`) See [Housing and Households](https://github.com/SANDAG/Estimates-Program/wiki/Housing-and-Households). +## Household population 18+ in each MGRA (`[outputs].[ase]`) +See [Population by Age Sex Ethnicity](https://github.com/SANDAG/Estimates-Program/wiki/Population-by-Age-Sex-Ethnicity). + ## Census tract household income distribution The household income distribution is derived from American Community Survey (ACS) table [B19001 | HOUSEHOLD INCOME IN THE PAST 12 MONTHS](https://data.census.gov/table/ACSDT5Y2020.B19001?q=B19001). The table provides households by household income. Dividing each category by total households provides census tract household income distributions. The year of the ACS table provides the inflation-adjusted dollars year. @@ -49,6 +54,20 @@ To avoid division by zero errors, the households by size distribution is set to \forall hhs \in \text{Household Size Category}; \text{Regional Household Size Distribution}_{hhs} = \frac{\sum \text{Households in Size Category}_{hhs}}{\sum \text{Households}} ``` +## Census tract households by workers distribution + +The households by workers distribution is derived from ACS table [B08202 | HOUSEHOLD SIZE BY NUMBER OF WORKERS IN HOUSEHOLD](https://data.census.gov/table/ACSDT5Y2020.B08202?q=B08202). The table provides households by number of workers category. Dividing each category by total households provides census tract households by workers distributions. + +```math +\forall t \in \text{San Diego Tracts}, \forall hhs \in \text{Household Worker Category}; \text{Distribution}_{t,hhs} = \frac{\text{Households in Worker Category}_{t,hhs}}{\text{Households}_t} +``` + +To avoid division by zero errors, the households by workers distribution is set to `NULL` if the households are zero within a census tract. This can lead to conflict between ACS data and SANDAG's LUDU. Within a census tract, the ACS may have no housing units, no households, and a `NULL` households by workers distribution while SANDAG's LUDU contains housing units. In this situation, the regional households by workers distribution is used. + +```math +\forall hhs \in \text{Household Worker Category}; \text{Regional Household Worker Distribution}_{hhs} = \frac{\sum \text{Households in Worker Category}_{hhs}}{\sum \text{Households}} +``` + # Outputs (`[outputs].[hh_characteristics]`) Each row of this table contains the following information: @@ -67,4 +86,11 @@ MGRA households by income category. Calculated by applying census tract distribu ## Households by size category in each MGRA MGRA households by size category. Calculated by applying census tract distributions to households. Adjusts size categories within MGRAs such that the implied household population range (min-max) contains the actual MGRA household population value. -For example, within a MGRA with 10 households of size one, 10 households of size two, ..., 10 households of size 7+, the minimum amount of household population would be 1x10 + 2x20 + ... + 7x10 = 280. The maximum amount of household population, assuming the 7+ category all average 11 people (see [issue #112](https://github.com/SANDAG/Estimates-Program/issues/112)), would be 1x10 + 2x20 + ... + 11x10 = 320. The actual amount of household population in this MGRA must be between these two values. If it is not, the households in size categories are adjusted until the condition is satisfied. \ No newline at end of file +For example, within a MGRA with 10 households of size one, 10 households of size two, ..., 10 households of size 7+, the minimum amount of household population would be 1x10 + 2x20 + ... + 7x10 = 280. The maximum amount of household population, assuming the 7+ category all average 11 people (see [issue #112](https://github.com/SANDAG/Estimates-Program/issues/112)), would be 1x10 + 2x20 + ... + 11x10 = 320. The actual amount of household population in this MGRA must be between these two values. If it is not, the households in size categories are adjusted until the condition is satisfied. + +## Households by workers category in each MGRA +MGRA households by workers category. Calculated by applying census tract distributions to households. Adjusts workers categories within MGRAs such that the implied minimum worker population does not exceed the MGRA household population of persons aged 18+. Then adjusts households by workers categories such that they respect the households by size categories in each MGRA. + +For example, within a MGRA with 10 households with 0 workers, 10 households with 1 workers, 10 households with 2 workers, 10 households with 3+ workers, the minimum implied household population of persons aged 18+ is 0x10 + 1x10 + 2x10 + 3x10 = 60. If the MGRA household population of persons aged 18+ is less than 60 then the households by workers categories are adjusted until the condition is satisfied. This is a soft constraint as we do not estimate the true number of workers in each MGRA. + +The process then moves onto adjusting households by workers categories such that they respect the households by size categories in each MGRA. Again, within a MGRA with 10 households with 0 workers, 10 households with 1 workers, 10 households with 2 workers, 10 households with 3+ workers, this implies there are at least 10 households of size 2+ and 10 household of size 3+. The 0 and 1 worker households can be of any size 1+ and do not violate any households by size category conditions as the total households within each MGRA is respected. If the MGRA households by size categories aggregated to 2+ and 3+ are less than 10 then the households by workers categories are adjusted until the condition is satisfied. \ No newline at end of file