[PULL REQUEST] Update Estimates to allow for Series 16 MGRAs

README.md

@@ -44,7 +44,7 @@ The default version of the runtime configuration file is copied here, with comme
 # Whether to use the 'run' section. Mutually exclusive with 'debug' mode
 enabled = true
 
-# The MGRA series to use for this run. Currently only '15' is valid
+# The MGRA series to use for this run. Currently only '15' and '16' are valid
 series = 15
 
 # The first year inclusive to start running from

python/employment.py

@@ -75,7 +75,8 @@ def _get_controls_inputs(year: int) -> dict[str, pd.DataFrame]:
                 "params": {
                     "run_id": utils.RUN_ID,
                     "year": year,
-                    "series": utils.SERIES,
+                    "estimates_server": utils.ESTIMATES_SERVER,
+                    "estimates_database": utils.ESTIMATES_DATABASE,
                 },
             },
             "region_edd": {
@@ -113,7 +114,8 @@ def _get_controls_inputs(year: int) -> dict[str, pd.DataFrame]:
                 "params": {
                     "run_id": utils.RUN_ID,
                     "year": year,
-                    "series": utils.SERIES,
+                    "estimates_server": utils.ESTIMATES_SERVER,
+                    "estimates_database": utils.ESTIMATES_DATABASE,
                 },
             },
             "region_qcew": {
@@ -669,8 +671,10 @@ def _get_jobs_inputs(year: int) -> dict[str, pd.DataFrame]:
                 sql=sql.text(file.read()),
                 con=con,
                 params={
-                    "series": utils.SERIES,
+                    "run_id": utils.RUN_ID,
                     "year": year,
+                    "estimates_server": utils.ESTIMATES_SERVER,
+                    "estimates_database": utils.ESTIMATES_DATABASE,
                 },
             )
 
@@ -682,7 +686,8 @@ def _get_jobs_inputs(year: int) -> dict[str, pd.DataFrame]:
                 params={
                     "run_id": utils.RUN_ID,
                     "year": year,
-                    "series": utils.SERIES,
+                    "estimates_server": utils.ESTIMATES_SERVER,
+                    "estimates_database": utils.ESTIMATES_DATABASE,
                 },
             )
 

python/parsers.py

@@ -142,7 +142,7 @@ def _validate_config(self) -> None:
                 "type": "dict",
                 "schema": {
                     "enabled": {"type": "boolean"},
-                    "series": {"type": "integer", "allowed": [15]},
+                    "series": {"type": "integer", "allowed": [15, 16]},
                     "start_year": {"type": "integer", "min": min_max_years[0]},
                     "end_year": {"type": "integer", "max": min_max_years[1]},
                     "version": {"type": "string", "allowed": versions},

python/tests.py

@@ -52,9 +52,8 @@
         ("naics3", "naics_sector"): 22,
     },
     "series": {
-        15: {
-            "mgra": 24321,
-        },
+        15: {"mgra": 24321},
+        16: {"mgra": 24209},
     },
 }
 

python/utils.py

@@ -78,6 +78,7 @@
 
 # Other SQL configuration(s)
 ESTIMATES_SERVER = _secrets["sql"]["estimates"]["server"]
+ESTIMATES_DATABASE = _secrets["sql"]["estimates"]["database"]
 GIS_SERVER = _secrets["sql"]["gis"]["server"]
 
 # Temporary file staging location for SQL BULK Inserts

sql/create_objects.sql

@@ -128,6 +128,25 @@ INSERT INTO [inputs].[special_mgras] (
         The determination was made that juveniles would be allowed in this MGRA but women would not due
         to the facility being majority male and all other facilities being male only.'),
     (15, 18741, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 18, NULL, 'Vista Detention Facility (VDF) operates as both a male and
+        female facility intake facility but the majority of housed inmates are male as women are
+        transferred to the Las Colinas Detention Facility.'),
+    (16, 18232, 2017, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 18, NULL, 'CAI Boston Avenue'),
+    (16, 16362, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', NULL, 18, NULL, 'Metropolitan Correctional Center, San Diego (MCC San Diego)'),
+    (16, 16289, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', NULL, 18, NULL, 'Western Region Detention Facility'),
+    (16, 16290, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 18, NULL, 'San Diego Central Jail'),
+    (16, 14489, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Female', 18, NULL, 'Las Colinas Detention Facility'),
+    (16, 23402, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 18, NULL, 'Richard J. Donovan Correctional Facility (RJD)'),
+    (16, 12166, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', NULL, 10, 18, 'Kearney Mesa Juvenile Detention Facility'),
+    (16, 20974, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 18, NULL, 'South Bay Detention Facility'),
+    (16, 23417, 2018, 2025, 'Group Quarters - Institutional Correctional Facilities', NULL, 18, NULL, 'Otay Mesa Detention Center'),
+    (16, 23409, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 18, NULL, 'Richard J. Donovan Correctional Facility (RJD)'),
+    (16, 23399, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 18, NULL, 'Richard J. Donovan Correctional Facility (RJD)'),
+    (16, 23400, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 10, NULL, 'Includes the Rock Mountain Detention Facility (RMDF), 
+        George Bailey Detention Facility (GBDF), East Mesa Reentry Facility (EMRF), and the East Mesa
+        Juvenile Detention Facility (EMJDF). Note that the EMJDF is a juvenile facility that allows women.
+        The determination was made that juveniles would be allowed in this MGRA but women would not due
+        to the facility being majority male and all other facilities being male only.'),
+    (16, 2438, 2010, 2025, 'Group Quarters - Institutional Correctional Facilities', 'Male', 18, NULL, 'Vista Detention Facility (VDF) operates as both a male and
         female facility intake facility but the majority of housed inmates are male as women are
         transferred to the Las Colinas Detention Facility.')
 GO

sql/employment/get_military_employment.sql

@@ -1,22 +1,38 @@
 /* 
-This query grabs the Military Active Duty (Job) Data and assigns counts to MGRA15.
+This query grabs the Military Active Duty (Job) Data and assigns counts to MGRAs.
 This will assign 0s to MGRAs where there are no military jobs.
 
-Notes:
-    1) This is assuming a connection to the GIS server
-    2) currently only works using MGRA15
+Note: This is assuming a connection to the GIS server
 */
 
+SET NOCOUNT ON;
 -- Initialize parameters -----------------------------------------------------
 DECLARE @run_id INTEGER = :run_id;
 DECLARE @year INTEGER = :year;
-DECLARE @series INTEGER = :series;
+DECLARE @estimates_server nvarchar(20) = :estimates_server;
+DECLARE @estimates_database nvarchar(20) = :estimates_database;
 
--- Check for MGRA series and stop execution if not Series 15
-IF @series != 15
-BEGIN
-    THROW 50000, 'Spatial join with military locations only valid for Series 15 MGRAs',1
-END
+
+-- Get MGRA geography and insert to temporary table
+-- Build the OPENQUERY to the Estimates database to get the MGRA geography
+-- Note the statement stores results in a temporary table for later use
+DROP TABLE IF EXISTS [#mgra];
+CREATE TABLE [#mgra] (
+    [mgra] INTEGER NOT NULL,
+    [shape] GEOMETRY NOT NULL,
+    CONSTRAINT [pk_tt_mgra] PRIMARY KEY ([mgra])
+);
+
+DECLARE @qry NVARCHAR(max) = '
+    INSERT INTO [#mgra]
+    SELECT [mgra], [shape]
+    FROM OPENQUERY([' + @estimates_server + '], ''
+        SELECT [mgra], [shape]
+        FROM ' + @estimates_database + '.[inputs].[mgra]
+        WHERE [run_id] = ' + CONVERT(NVARCHAR, @run_id) + '
+    '')
+'
+EXEC sp_executesql @qry;
 
 
 -- Send error message if no data exists --------------------------------------
@@ -39,13 +55,15 @@ BEGIN
         'Federal Government' AS [ownership_title],
         'MIL' AS [industry_code],
         COALESCE(SUM([site_active_duty]), 0) AS [value]
-    FROM [GeoDepot].[sde].[MGRA15]
+    FROM [#mgra]
     LEFT OUTER JOIN (
         SELECT [site_active_duty], [shape] 
         FROM [EMPCORE].[dbo].[mil_active_duty]
         WHERE [yr] = @year
         ) AS [mil_active_duty]
-        ON [mil_active_duty].[shape].STWithin([MGRA15].[shape]) = 1
+        ON [mil_active_duty].[shape].STWithin([#mgra].[shape]) = 1
     GROUP BY [mgra]
-    ORDER BY [mgra]
+    ORDER BY [mgra];
+
+    DROP TABLE IF EXISTS [#mgra];
 END

sql/employment/xref_block_to_mgra.sql

@@ -1,5 +1,5 @@
 /*
-This query provides a many-to-many cross reference mapping 2020 Census Blocks to Series 15 MGRAs
+This query provides a many-to-many cross reference mapping 2020 Census Blocks to MGRAs
 There are two cross references for separate use cases
   1) Cross reference based on EDD point-level jobs data within SANDAG employment categories
   2) Cross reference based on EDD point-level jobs data without considering SANDAG employment categories
@@ -19,16 +19,34 @@ Notes:
 
 SET NOCOUNT ON;
 -- Initialize parameters and return table ------------------------------------
+DECLARE @run_id INTEGER = :run_id;
 DECLARE @year INTEGER = :year;
-DECLARE @series INTEGER = :series;
+DECLARE @estimates_server nvarchar(20) = :estimates_server;
+DECLARE @estimates_database nvarchar(20) = :estimates_database;
 DECLARE @msg nvarchar(45) = 'EDD point-level data does not exist';
 
 
--- Check for MGRA series and stop execution if not Series 15
-IF @series != 15
-BEGIN
-    THROW 50000, 'EDD xref only valid for Series 15 MGRAs',1;
-END
+-- Get MGRA geography and insert to temporary table
+-- Build the OPENQUERY to the Estimates database to get the MGRA geography
+-- Note the statement stores results in a temporary table for later use
+DROP TABLE IF EXISTS [#mgra];
+CREATE TABLE [#mgra] (
+    [mgra] INTEGER NOT NULL,
+    [shape] GEOMETRY NOT NULL,
+    CONSTRAINT [pk_tt_mgra] PRIMARY KEY ([mgra])
+)
+
+DECLARE @qry NVARCHAR(max) = '
+    INSERT INTO [#mgra]
+    SELECT [mgra], [shape]
+    FROM OPENQUERY([' + @estimates_server + '], ''
+        SELECT [mgra], [shape]
+        FROM ' + @estimates_database + '.[inputs].[mgra]
+        WHERE [run_id] = ' + CONVERT(NVARCHAR, @run_id) + '
+    '')
+'
+EXEC sp_executesql @qry;
+
 
 -- Create shell table of 2020 Census Block x Ownership Title x Industry Code
 DROP TABLE IF EXISTS [#tt_block_category];
@@ -306,33 +324,33 @@ BEGIN
             [CENSUSBLOCKS].[GEOID20] AS [block],
             [ownership_title],
             [industry_code],
-            [MGRA15].[MGRA] AS [mgra],
+            [#mgra].[mgra],
             SUM([jobs]) 
                 / SUM(SUM([jobs])) OVER (PARTITION BY [CENSUSBLOCKS].[GEOID20], [ownership_title], [industry_code])
             AS [pct_edd_category]
         FROM [#edd]
         INNER JOIN [GeoDepot].[sde].[CENSUSBLOCKS]
             ON [#edd].[Shape].STIntersects([CENSUSBLOCKS].[Shape]) = 1
-        INNER JOIN [GeoDepot].[sde].[MGRA15]
-            ON [#edd].[Shape].STIntersects([MGRA15].[Shape]) = 1
+        INNER JOIN [#mgra]
+            ON [#edd].[Shape].STIntersects([#mgra].[shape]) = 1
         GROUP BY 
-            [CENSUSBLOCKS].[GEOID20], [ownership_title], [industry_code], [MGRA15].[MGRA]
+            [CENSUSBLOCKS].[GEOID20], [ownership_title], [industry_code], [#mgra].[mgra]
     ),
     -- Calculate % allocation of Census 2020 Block jobs to MGRAs
     [xref_edd] AS (
         SELECT
             [CENSUSBLOCKS].[GEOID20] AS [block],
-            [MGRA15].[MGRA] AS [mgra],
-            SUM(SUM([jobs])) OVER (PARTITION BY [CENSUSBLOCKS].[GEOID20], [MGRA15].[MGRA]) 
+            [#mgra].[mgra],
+            SUM(SUM([jobs])) OVER (PARTITION BY [CENSUSBLOCKS].[GEOID20], [#mgra].[mgra]) 
                 / SUM(SUM([jobs])) OVER (PARTITION BY [CENSUSBLOCKS].[GEOID20])
             AS [pct_edd]
         FROM [#edd]
         INNER JOIN [GeoDepot].[sde].[CENSUSBLOCKS]
             ON [#edd].[Shape].STIntersects([CENSUSBLOCKS].[Shape]) = 1
-        INNER JOIN [GeoDepot].[sde].[MGRA15]
-            ON [#edd].[Shape].STIntersects([MGRA15].[Shape]) = 1
+        INNER JOIN [#mgra]
+            ON [#edd].[Shape].STIntersects([#mgra].[shape]) = 1
         GROUP BY 
-            [CENSUSBLOCKS].[GEOID20], [MGRA15].[MGRA]
+            [CENSUSBLOCKS].[GEOID20], [#mgra].[mgra]
     ),
     -- Get % area overlap of Census 2020 Block area and MGRAs
     [xref_area] AS (
@@ -344,14 +362,14 @@ BEGIN
         FROM (
             SELECT
                 [CENSUSBLOCKS].[GEOID20] AS [block],
-                [MGRA15].[MGRA] AS [mgra],
-                ([CENSUSBLOCKS].[Shape].STIntersection([MGRA15].[Shape]).STArea() 
+                [#mgra].[mgra],
+                ([CENSUSBLOCKS].[Shape].STIntersection([#mgra].[shape]).STArea() 
                     / [CENSUSBLOCKS].[Shape].STArea())
                 AS [pct_area]
             FROM [GeoDepot].[sde].[CENSUSBLOCKS]
-            LEFT OUTER JOIN [GeoDepot].[sde].[MGRA15] 
-                ON [CENSUSBLOCKS].[Shape].STIntersects([MGRA15].[Shape]) = 1
-            WHERE ([CENSUSBLOCKS].[Shape].STIntersection([MGRA15].[Shape]).STArea() 
+            LEFT OUTER JOIN [#mgra]
+                ON [CENSUSBLOCKS].[Shape].STIntersects([#mgra].[shape]) = 1
+            WHERE ([CENSUSBLOCKS].[Shape].STIntersection([#mgra].[shape]).STArea() 
                 / [CENSUSBLOCKS].[Shape].STArea()) > 0.01
         ) AS [raw_xref_area]
     )
@@ -386,5 +404,6 @@ BEGIN
         [#tt_block_category].[industry_code]
 END
 
+DROP TABLE IF EXISTS [#mgra];
 DROP TABLE IF EXISTS [#tt_block_category];
 DROP TABLE IF EXISTS [#edd];