diff --git a/README.md b/README.md index 8c73906..9fd7db2 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ The default version of the runtime configuration file is copied here, with comme # Whether to use the 'run' section. Mutually exclusive with 'debug' mode enabled = true -# The MGRA series to use for this run. Currently only '15' is valid +# The MGRA series to use for this run. Currently only '15' and '16' are valid series = 15 # The first year inclusive to start running from diff --git a/python/employment.py b/python/employment.py index c669226..e083731 100644 --- a/python/employment.py +++ b/python/employment.py @@ -75,7 +75,8 @@ def _get_controls_inputs(year: int) -> dict[str, pd.DataFrame]: "params": { "run_id": utils.RUN_ID, "year": year, - "series": utils.SERIES, + "estimates_server": utils.ESTIMATES_SERVER, + "estimates_database": utils.ESTIMATES_DATABASE, }, }, "region_edd": { @@ -113,7 +114,8 @@ def _get_controls_inputs(year: int) -> dict[str, pd.DataFrame]: "params": { "run_id": utils.RUN_ID, "year": year, - "series": utils.SERIES, + "estimates_server": utils.ESTIMATES_SERVER, + "estimates_database": utils.ESTIMATES_DATABASE, }, }, "region_qcew": { @@ -669,8 +671,10 @@ def _get_jobs_inputs(year: int) -> dict[str, pd.DataFrame]: sql=sql.text(file.read()), con=con, params={ - "series": utils.SERIES, + "run_id": utils.RUN_ID, "year": year, + "estimates_server": utils.ESTIMATES_SERVER, + "estimates_database": utils.ESTIMATES_DATABASE, }, ) @@ -682,7 +686,8 @@ def _get_jobs_inputs(year: int) -> dict[str, pd.DataFrame]: params={ "run_id": utils.RUN_ID, "year": year, - "series": utils.SERIES, + "estimates_server": utils.ESTIMATES_SERVER, + "estimates_database": utils.ESTIMATES_DATABASE, }, ) diff --git a/python/parsers.py b/python/parsers.py index 33e2bad..067a52d 100644 --- a/python/parsers.py +++ b/python/parsers.py @@ -142,7 +142,7 @@ def _validate_config(self) -> None: "type": "dict", "schema": { "enabled": {"type": "boolean"}, - "series": {"type": "integer", "allowed": [15]}, + "series": {"type": "integer", "allowed": [15, 16]}, "start_year": {"type": "integer", "min": min_max_years[0]}, "end_year": {"type": "integer", "max": min_max_years[1]}, "version": {"type": "string", "allowed": versions}, diff --git a/python/tests.py b/python/tests.py index 6d72b8b..490fe79 100644 --- a/python/tests.py +++ b/python/tests.py @@ -52,9 +52,8 @@ ("naics3", "naics_sector"): 22, }, "series": { - 15: { - "mgra": 24321, - }, + 15: {"mgra": 24321}, + 16: {"mgra": 24209}, }, } diff --git a/python/utils.py b/python/utils.py index 2d66012..63877ed 100644 --- a/python/utils.py +++ b/python/utils.py @@ -78,6 +78,7 @@ # Other SQL configuration(s) ESTIMATES_SERVER = _secrets["sql"]["estimates"]["server"] +ESTIMATES_DATABASE = _secrets["sql"]["estimates"]["database"] GIS_SERVER = _secrets["sql"]["gis"]["server"] # Temporary file staging location for SQL BULK Inserts diff --git a/sql/create_objects.sql b/sql/create_objects.sql index 61a81f9..c1cc142 100644 --- a/sql/create_objects.sql +++ b/sql/create_objects.sql @@ -128,6 +128,25 @@ INSERT INTO [inputs].[special_mgras] ( The determination was made that juveniles would be allowed in this MGRA but women would not due to the facility being majority male and all other facilities being male only.'), (15, 18741, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 18, NULL, 'Vista Detention Facility (VDF) operates as both a male and + female facility intake facility but the majority of housed inmates are male as women are + transferred to the Las Colinas Detention Facility.'), + (16, 18232, 2017, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 18, NULL, 'CAI Boston Avenue'), + (16, 16362, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', NULL, 18, NULL, 'Metropolitan Correctional Center, San Diego (MCC San Diego)'), + (16, 16289, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', NULL, 18, NULL, 'Western Region Detention Facility'), + (16, 16290, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 18, NULL, 'San Diego Central Jail'), + (16, 14489, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Female', 18, NULL, 'Las Colinas Detention Facility'), + (16, 23402, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 18, NULL, 'Richard J. Donovan Correctional Facility (RJD)'), + (16, 12166, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', NULL, 10, 18, 'Kearney Mesa Juvenile Detention Facility'), + (16, 20974, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 18, NULL, 'South Bay Detention Facility'), + (16, 23417, 2018, 2025, 'Group Quarters - Institutional Correctional Facilities', NULL, 18, NULL, 'Otay Mesa Detention Center'), + (16, 23409, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 18, NULL, 'Richard J. Donovan Correctional Facility (RJD)'), + (16, 23399, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 18, NULL, 'Richard J. Donovan Correctional Facility (RJD)'), + (16, 23400, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 10, NULL, 'Includes the Rock Mountain Detention Facility (RMDF), + George Bailey Detention Facility (GBDF), East Mesa Reentry Facility (EMRF), and the East Mesa + Juvenile Detention Facility (EMJDF). Note that the EMJDF is a juvenile facility that allows women. + The determination was made that juveniles would be allowed in this MGRA but women would not due + to the facility being majority male and all other facilities being male only.'), + (16, 2438, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 18, NULL, 'Vista Detention Facility (VDF) operates as both a male and female facility intake facility but the majority of housed inmates are male as women are transferred to the Las Colinas Detention Facility.') GO diff --git a/sql/employment/get_military_employment.sql b/sql/employment/get_military_employment.sql index f036370..ffa3061 100644 --- a/sql/employment/get_military_employment.sql +++ b/sql/employment/get_military_employment.sql @@ -1,22 +1,38 @@ /* -This query grabs the Military Active Duty (Job) Data and assigns counts to MGRA15. +This query grabs the Military Active Duty (Job) Data and assigns counts to MGRAs. This will assign 0s to MGRAs where there are no military jobs. -Notes: - 1) This is assuming a connection to the GIS server - 2) currently only works using MGRA15 +Note: This is assuming a connection to the GIS server */ +SET NOCOUNT ON; -- Initialize parameters ----------------------------------------------------- DECLARE @run_id INTEGER = :run_id; DECLARE @year INTEGER = :year; -DECLARE @series INTEGER = :series; +DECLARE @estimates_server nvarchar(20) = :estimates_server; +DECLARE @estimates_database nvarchar(20) = :estimates_database; --- Check for MGRA series and stop execution if not Series 15 -IF @series != 15 -BEGIN - THROW 50000, 'Spatial join with military locations only valid for Series 15 MGRAs',1 -END + +-- Get MGRA geography and insert to temporary table +-- Build the OPENQUERY to the Estimates database to get the MGRA geography +-- Note the statement stores results in a temporary table for later use +DROP TABLE IF EXISTS [#mgra]; +CREATE TABLE [#mgra] ( + [mgra] INTEGER NOT NULL, + [shape] GEOMETRY NOT NULL, + CONSTRAINT [pk_tt_mgra] PRIMARY KEY ([mgra]) +); + +DECLARE @qry NVARCHAR(max) = ' + INSERT INTO [#mgra] + SELECT [mgra], [shape] + FROM OPENQUERY([' + @estimates_server + '], '' + SELECT [mgra], [shape] + FROM ' + @estimates_database + '.[inputs].[mgra] + WHERE [run_id] = ' + CONVERT(NVARCHAR, @run_id) + ' + '') +' +EXEC sp_executesql @qry; -- Send error message if no data exists -------------------------------------- @@ -39,13 +55,15 @@ BEGIN 'Federal Government' AS [ownership_title], 'MIL' AS [industry_code], COALESCE(SUM([site_active_duty]), 0) AS [value] - FROM [GeoDepot].[sde].[MGRA15] + FROM [#mgra] LEFT OUTER JOIN ( SELECT [site_active_duty], [shape] FROM [EMPCORE].[dbo].[mil_active_duty] WHERE [yr] = @year ) AS [mil_active_duty] - ON [mil_active_duty].[shape].STWithin([MGRA15].[shape]) = 1 + ON [mil_active_duty].[shape].STWithin([#mgra].[shape]) = 1 GROUP BY [mgra] - ORDER BY [mgra] + ORDER BY [mgra]; + + DROP TABLE IF EXISTS [#mgra]; END \ No newline at end of file diff --git a/sql/employment/xref_block_to_mgra.sql b/sql/employment/xref_block_to_mgra.sql index afe4942..2adee02 100644 --- a/sql/employment/xref_block_to_mgra.sql +++ b/sql/employment/xref_block_to_mgra.sql @@ -1,5 +1,5 @@ /* -This query provides a many-to-many cross reference mapping 2020 Census Blocks to Series 15 MGRAs +This query provides a many-to-many cross reference mapping 2020 Census Blocks to MGRAs There are two cross references for separate use cases 1) Cross reference based on EDD point-level jobs data within SANDAG employment categories 2) Cross reference based on EDD point-level jobs data without considering SANDAG employment categories @@ -19,16 +19,34 @@ Notes: SET NOCOUNT ON; -- Initialize parameters and return table ------------------------------------ +DECLARE @run_id INTEGER = :run_id; DECLARE @year INTEGER = :year; -DECLARE @series INTEGER = :series; +DECLARE @estimates_server nvarchar(20) = :estimates_server; +DECLARE @estimates_database nvarchar(20) = :estimates_database; DECLARE @msg nvarchar(45) = 'EDD point-level data does not exist'; --- Check for MGRA series and stop execution if not Series 15 -IF @series != 15 -BEGIN - THROW 50000, 'EDD xref only valid for Series 15 MGRAs',1; -END +-- Get MGRA geography and insert to temporary table +-- Build the OPENQUERY to the Estimates database to get the MGRA geography +-- Note the statement stores results in a temporary table for later use +DROP TABLE IF EXISTS [#mgra]; +CREATE TABLE [#mgra] ( + [mgra] INTEGER NOT NULL, + [shape] GEOMETRY NOT NULL, + CONSTRAINT [pk_tt_mgra] PRIMARY KEY ([mgra]) +) + +DECLARE @qry NVARCHAR(max) = ' + INSERT INTO [#mgra] + SELECT [mgra], [shape] + FROM OPENQUERY([' + @estimates_server + '], '' + SELECT [mgra], [shape] + FROM ' + @estimates_database + '.[inputs].[mgra] + WHERE [run_id] = ' + CONVERT(NVARCHAR, @run_id) + ' + '') +' +EXEC sp_executesql @qry; + -- Create shell table of 2020 Census Block x Ownership Title x Industry Code DROP TABLE IF EXISTS [#tt_block_category]; @@ -306,33 +324,33 @@ BEGIN [CENSUSBLOCKS].[GEOID20] AS [block], [ownership_title], [industry_code], - [MGRA15].[MGRA] AS [mgra], + [#mgra].[mgra], SUM([jobs]) / SUM(SUM([jobs])) OVER (PARTITION BY [CENSUSBLOCKS].[GEOID20], [ownership_title], [industry_code]) AS [pct_edd_category] FROM [#edd] INNER JOIN [GeoDepot].[sde].[CENSUSBLOCKS] ON [#edd].[Shape].STIntersects([CENSUSBLOCKS].[Shape]) = 1 - INNER JOIN [GeoDepot].[sde].[MGRA15] - ON [#edd].[Shape].STIntersects([MGRA15].[Shape]) = 1 + INNER JOIN [#mgra] + ON [#edd].[Shape].STIntersects([#mgra].[shape]) = 1 GROUP BY - [CENSUSBLOCKS].[GEOID20], [ownership_title], [industry_code], [MGRA15].[MGRA] + [CENSUSBLOCKS].[GEOID20], [ownership_title], [industry_code], [#mgra].[mgra] ), -- Calculate % allocation of Census 2020 Block jobs to MGRAs [xref_edd] AS ( SELECT [CENSUSBLOCKS].[GEOID20] AS [block], - [MGRA15].[MGRA] AS [mgra], - SUM(SUM([jobs])) OVER (PARTITION BY [CENSUSBLOCKS].[GEOID20], [MGRA15].[MGRA]) + [#mgra].[mgra], + SUM(SUM([jobs])) OVER (PARTITION BY [CENSUSBLOCKS].[GEOID20], [#mgra].[mgra]) / SUM(SUM([jobs])) OVER (PARTITION BY [CENSUSBLOCKS].[GEOID20]) AS [pct_edd] FROM [#edd] INNER JOIN [GeoDepot].[sde].[CENSUSBLOCKS] ON [#edd].[Shape].STIntersects([CENSUSBLOCKS].[Shape]) = 1 - INNER JOIN [GeoDepot].[sde].[MGRA15] - ON [#edd].[Shape].STIntersects([MGRA15].[Shape]) = 1 + INNER JOIN [#mgra] + ON [#edd].[Shape].STIntersects([#mgra].[shape]) = 1 GROUP BY - [CENSUSBLOCKS].[GEOID20], [MGRA15].[MGRA] + [CENSUSBLOCKS].[GEOID20], [#mgra].[mgra] ), -- Get % area overlap of Census 2020 Block area and MGRAs [xref_area] AS ( @@ -344,14 +362,14 @@ BEGIN FROM ( SELECT [CENSUSBLOCKS].[GEOID20] AS [block], - [MGRA15].[MGRA] AS [mgra], - ([CENSUSBLOCKS].[Shape].STIntersection([MGRA15].[Shape]).STArea() + [#mgra].[mgra], + ([CENSUSBLOCKS].[Shape].STIntersection([#mgra].[shape]).STArea() / [CENSUSBLOCKS].[Shape].STArea()) AS [pct_area] FROM [GeoDepot].[sde].[CENSUSBLOCKS] - LEFT OUTER JOIN [GeoDepot].[sde].[MGRA15] - ON [CENSUSBLOCKS].[Shape].STIntersects([MGRA15].[Shape]) = 1 - WHERE ([CENSUSBLOCKS].[Shape].STIntersection([MGRA15].[Shape]).STArea() + LEFT OUTER JOIN [#mgra] + ON [CENSUSBLOCKS].[Shape].STIntersects([#mgra].[shape]) = 1 + WHERE ([CENSUSBLOCKS].[Shape].STIntersection([#mgra].[shape]).STArea() / [CENSUSBLOCKS].[Shape].STArea()) > 0.01 ) AS [raw_xref_area] ) @@ -386,5 +404,6 @@ BEGIN [#tt_block_category].[industry_code] END +DROP TABLE IF EXISTS [#mgra]; DROP TABLE IF EXISTS [#tt_block_category]; DROP TABLE IF EXISTS [#edd]; \ No newline at end of file