feat!: read raw-free medallion names and fail loudly when absent

README.md

@@ -79,9 +79,11 @@ structurally impossible.
 
 **Only the dbt template's `ci` target is prefixed.** `g3dt config dbt-env`
 emits, alongside the real names, the CI-isolation variants the template's
-`ci` target consumes: `G3DT_DB_RAW_SILVER_CI` / `G3DT_DB_RAW_GOLD_CI`
+`ci` target consumes: `G3DT_DB_SILVER_CI` / `G3DT_DB_GOLD_CI`
 (`ci_` + the real database name) and `G3DT_S3_SILVER_DATA_DIR_CI` /
-`G3DT_S3_GOLD_DATA_DIR_CI` (`dbt_ci/` under the same buckets). Commit-
+`G3DT_S3_GOLD_DATA_DIR_CI` (`dbt_ci/` under the same buckets). Toolkit
+releases >= 3 read the raw-free medallion SSM keys and therefore require a
+pipeline deployment >= v2.0.0, which publishes them. Commit-
 triggered CI builds land there; every other target (default, local) and the
 release build keep the real, unprefixed names — so CI can never advance the
 warehouse's Iceberg snapshots that releases pin. The library enforces the

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gen3-dataops-toolkit"
-version = "2.3.0"
+version = "3.0.0"
 description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"

src/g3dt/cli/config_cmds.py

@@ -25,6 +25,27 @@
 )
 
 
+def _req_key(rc, key: str) -> str:
+    """Return the SSM leaf ``key`` from ``rc``, failing loudly if absent.
+
+    The medallion names (``buckets/silver|gold``, ``glue/db/bronze|silver|gold``)
+    are published under these raw-free keys by pipeline deployments >= v2.0.0.
+    Before this guard, a missing key propagated as ``None``: dbt-env silently
+    dropped the G3DT_DB_* vars and emitted ``s3://None/dbt/`` data dirs, and
+    the release search silently fell back to an account-wide catalog walk.
+    """
+    value = rc.get(key)
+    if value is None:
+        raise config.ConfigError(
+            f"SSM parameter /{rc.project}/{rc.env}/{key} is missing. "
+            f"gen3-dataops-toolkit >= 3 reads the raw-free medallion keys "
+            f"published by gen3-aws-data-pipeline >= v2.0.0; a pipeline "
+            f"deployment older than v2.0.0 still publishes raw-prefixed keys. "
+            f"Upgrade the pipeline deployment (or pin gen3-dataops-toolkit<3)."
+        )
+    return value
+
+
 @app.command()
 def envs() -> None:
     """List the environments with a deployed SSM tree for this project."""
@@ -208,28 +229,31 @@ def dbt_env(
     profile = None if env.endswith("_ec2") else config.aws_profile_for(base, marker)
     try:
         rc = resolver.resolve(project, base, profile=profile)
+        silver_db = _req_key(rc, "glue/db/silver")
+        gold_db = _req_key(rc, "glue/db/gold")
+        silver_bucket = _req_key(rc, "buckets/silver")
+        gold_bucket = _req_key(rc, "buckets/gold")
+        bronze_db = _req_key(rc, "glue/db/bronze")
     except config.ConfigError as exc:
         typer.secho(str(exc), fg=typer.colors.RED, err=True)
         raise typer.Exit(1)
 
-    raw_silver_db = rc.get("glue/db/rawSilver")
-    raw_gold_db = rc.get("glue/db/rawGold")
     values = {
         "G3DT_REGION": rc.region,
         "G3DT_ATHENA_WORKGROUP": rc.athena_workgroup,
         "G3DT_ATHENA_OUTPUT": rc.athena_output_location,
-        "G3DT_DB_RAW_BRONZE": rc.get("glue/db/rawBronze"),
-        "G3DT_DB_RAW_SILVER": raw_silver_db,
-        "G3DT_DB_RAW_GOLD": raw_gold_db,
-        "G3DT_S3_SILVER_DATA_DIR": f"s3://{rc.get('buckets/rawSilver')}/dbt/",
-        "G3DT_S3_GOLD_DATA_DIR": f"s3://{rc.get('buckets/rawGold')}/dbt/",
+        "G3DT_DB_BRONZE": bronze_db,
+        "G3DT_DB_SILVER": silver_db,
+        "G3DT_DB_GOLD": gold_db,
+        "G3DT_S3_SILVER_DATA_DIR": f"s3://{silver_bucket}/dbt/",
+        "G3DT_S3_GOLD_DATA_DIR": f"s3://{gold_bucket}/dbt/",
         # CI isolation: the dbt template's `ci` target builds into these
         # instead — same grammar as the CDK's ci_ databases, same buckets
         # under a dbt_ci/ prefix. Real names above are never prefixed.
-        "G3DT_DB_RAW_SILVER_CI": f"ci_{raw_silver_db}" if raw_silver_db else None,
-        "G3DT_DB_RAW_GOLD_CI": f"ci_{raw_gold_db}" if raw_gold_db else None,
-        "G3DT_S3_SILVER_DATA_DIR_CI": f"s3://{rc.get('buckets/rawSilver')}/dbt_ci/",
-        "G3DT_S3_GOLD_DATA_DIR_CI": f"s3://{rc.get('buckets/rawGold')}/dbt_ci/",
+        "G3DT_DB_SILVER_CI": f"ci_{silver_db}",
+        "G3DT_DB_GOLD_CI": f"ci_{gold_db}",
+        "G3DT_S3_SILVER_DATA_DIR_CI": f"s3://{silver_bucket}/dbt_ci/",
+        "G3DT_S3_GOLD_DATA_DIR_CI": f"s3://{gold_bucket}/dbt_ci/",
     }
     if profile:
         # A named profile means a laptop run: select the dbt target that

src/g3dt/cli/release_cmds.py

@@ -57,10 +57,14 @@ def write(
         )
         # The env's own silver/gold DBs (from SSM) scope the model->DB search:
         # deterministic in shared accounts, no account-wide Glue perms needed.
+        # _req_key fails loudly if the raw-free keys are absent (pre-v2.0.0
+        # pipeline) instead of silently widening to an account-wide walk.
+        from g3dt.cli.config_cmds import _req_key
+
         search_databases = [
-            db for db in (rc.get("glue/db/rawSilver"), rc.get("glue/db/rawGold"))
-            if db
-        ] or None
+            _req_key(rc, "glue/db/silver"),
+            _req_key(rc, "glue/db/gold"),
+        ]
         release_writer.run(
             dbt_schema_path=dbt_schema_path,
             release_db=rc.release_db,

tests/test_athena_utils.py

@@ -1675,7 +1675,7 @@ def test_find_db_for_model_skips_ci_databases(mock_boto_session, mock_wr, scoped
         paths: the scoped databases= list and the account-wide catalog walk.
 
     Steps:
-        1. Present ['ci_proj_test_raw_gold_db', 'proj_test_raw_gold_db'],
+        1. Present ['ci_proj_test_gold_db', 'proj_test_gold_db'],
            both containing 'gold_model', with the ci_ database FIRST.
         2. Call find_db_for_model, scoped and unscoped.
 
@@ -1685,18 +1685,18 @@ def test_find_db_for_model_skips_ci_databases(mock_boto_session, mock_wr, scoped
     """
     mock_session_instance = MagicMock()
     mock_boto_session.return_value = mock_session_instance
-    dbs = ['ci_proj_test_raw_gold_db', 'proj_test_raw_gold_db']
+    dbs = ['ci_proj_test_gold_db', 'proj_test_gold_db']
     mock_wr.catalog.databases.return_value = {'Database': dbs}
     mock_wr.catalog.get_tables.return_value = [{'Name': 'gold_model'}]
 
     result = athena_query_instance.find_db_for_model(
         'gold_model', databases=dbs if scoped else None
     )
 
-    assert result == 'proj_test_raw_gold_db'
+    assert result == 'proj_test_gold_db'
     assert mock_wr.catalog.get_tables.call_count == 1
     mock_wr.catalog.get_tables.assert_called_once_with(
-        database='proj_test_raw_gold_db', boto3_session=mock_session_instance
+        database='proj_test_gold_db', boto3_session=mock_session_instance
     )
 
 

tests/test_release_cmds.py

@@ -50,11 +50,11 @@ def _seed(project="etl", env="test"):
     leaves = {
         "meta/region": REGION,
         "buckets/metadata": f"{project}-{env}-metadata-{ACCOUNT}-{REGION}",
-        "buckets/rawSilver": f"{project}-{env}-raw-silver-{ACCOUNT}-{REGION}",
-        "buckets/rawGold": f"{project}-{env}-raw-gold-{ACCOUNT}-{REGION}",
-        "glue/db/rawBronze": f"{project}_{env}_raw_bronze_db",
-        "glue/db/rawSilver": f"{project}_{env}_raw_silver_db",
-        "glue/db/rawGold": f"{project}_{env}_raw_gold_db",
+        "buckets/silver": f"{project}-{env}-silver-{ACCOUNT}-{REGION}",
+        "buckets/gold": f"{project}-{env}-gold-{ACCOUNT}-{REGION}",
+        "glue/db/bronze": f"{project}_{env}_bronze_db",
+        "glue/db/silver": f"{project}_{env}_silver_db",
+        "glue/db/gold": f"{project}_{env}_gold_db",
         "release/db": f"{project}_{env}_dataops_metadata_db",
         "release/table": "releases",
         "athena/workgroup": f"{project}-{env}",
@@ -96,7 +96,7 @@ def test_release_write_resolves_names_from_ssm(mock_run):
     # env's own DBs so shared accounts can't cross-match
     assert kwargs["workgroup"] == "etl-test"
     assert kwargs["search_databases"] == [
-        "etl_test_raw_silver_db", "etl_test_raw_gold_db"
+        "etl_test_silver_db", "etl_test_gold_db"
     ]
     # the resolved target is echoed so a build log always shows where rows go
     assert "etl_test_dataops_metadata_db.releases" in result.output
@@ -144,7 +144,7 @@ def test_release_writer_dry_run_writes_nothing():
         insert_release_row(
             athena_config=athena_config,
             model_name="silver_x",
-            db_name="etl_test_raw_silver_db",
+            db_name="etl_test_silver_db",
             snapshot_id=1,
             committed_at="2026-01-01 00:00:00",
             release_db="etl_test_dataops_metadata_db",
@@ -171,11 +171,11 @@ def test_config_dbt_env_emits_every_dbt_setting():
     assert "export G3DT_ATHENA_WORKGROUP=etl-test" in out
     assert f"export G3DT_ATHENA_OUTPUT=s3://etl-test-athena-results-{ACCOUNT}-{REGION}/" in out
     assert f"export G3DT_REGION={REGION}" in out
-    assert "export G3DT_DB_RAW_BRONZE=etl_test_raw_bronze_db" in out
-    assert "export G3DT_DB_RAW_SILVER=etl_test_raw_silver_db" in out
-    assert "export G3DT_DB_RAW_GOLD=etl_test_raw_gold_db" in out
-    assert f"export G3DT_S3_SILVER_DATA_DIR=s3://etl-test-raw-silver-{ACCOUNT}-{REGION}/dbt/" in out
-    assert f"export G3DT_S3_GOLD_DATA_DIR=s3://etl-test-raw-gold-{ACCOUNT}-{REGION}/dbt/" in out
+    assert "export G3DT_DB_BRONZE=etl_test_bronze_db" in out
+    assert "export G3DT_DB_SILVER=etl_test_silver_db" in out
+    assert "export G3DT_DB_GOLD=etl_test_gold_db" in out
+    assert f"export G3DT_S3_SILVER_DATA_DIR=s3://etl-test-silver-{ACCOUNT}-{REGION}/dbt/" in out
+    assert f"export G3DT_S3_GOLD_DATA_DIR=s3://etl-test-gold-{ACCOUNT}-{REGION}/dbt/" in out
     # no profile configured -> ambient credentials and the default dbt target
     assert "G3DT_AWS_PROFILE" not in out
     assert "G3DT_DBT_TARGET" not in out
@@ -228,13 +228,63 @@ def test_config_dbt_env_emits_ci_isolation_vars():
     result = runner.invoke(app, ["config", "dbt-env", "--env", "test"])
     assert result.exit_code == 0, result.output
     out = result.output
-    assert "export G3DT_DB_RAW_SILVER_CI=ci_etl_test_raw_silver_db" in out
-    assert "export G3DT_DB_RAW_GOLD_CI=ci_etl_test_raw_gold_db" in out
-    assert f"export G3DT_S3_SILVER_DATA_DIR_CI=s3://etl-test-raw-silver-{ACCOUNT}-{REGION}/dbt_ci/" in out
-    assert f"export G3DT_S3_GOLD_DATA_DIR_CI=s3://etl-test-raw-gold-{ACCOUNT}-{REGION}/dbt_ci/" in out
+    assert "export G3DT_DB_SILVER_CI=ci_etl_test_silver_db" in out
+    assert "export G3DT_DB_GOLD_CI=ci_etl_test_gold_db" in out
+    assert f"export G3DT_S3_SILVER_DATA_DIR_CI=s3://etl-test-silver-{ACCOUNT}-{REGION}/dbt_ci/" in out
+    assert f"export G3DT_S3_GOLD_DATA_DIR_CI=s3://etl-test-gold-{ACCOUNT}-{REGION}/dbt_ci/" in out
     # the real names remain unprefixed
-    assert "export G3DT_DB_RAW_SILVER=etl_test_raw_silver_db" in out
-    assert "export G3DT_DB_RAW_GOLD=etl_test_raw_gold_db" in out
+    assert "export G3DT_DB_SILVER=etl_test_silver_db" in out
+    assert "export G3DT_DB_GOLD=etl_test_gold_db" in out
+
+
+@mock_aws
+@patch("g3dt.utils.release_writer.run")
+def test_missing_medallion_keys_fail_loudly(mock_run):
+    """
+    Background:
+        gen3-aws-data-pipeline < v2.0.0 published the medallion names under
+        SSM keys carrying a legacy raw prefix in the leaf name. Toolkit
+        versions < 3 read them with rc.get(), which returns None for
+        an absent key — so a key-name mismatch FAILED SILENTLY: dbt-env
+        dropped the G3DT_DB_* export lines entirely and emitted
+        `s3://None/dbt/` data dirs, and `release write` quietly fell back to
+        an account-wide Glue catalog walk. Toolkit >= 3 reads only the
+        raw-free keys and must fail loudly when they are missing.
+
+    Inputs:  an SSM tree seeded WITHOUT the medallion keys — exactly what a
+             pipeline deployment older than v2.0.0 presents to toolkit >= 3.
+    Expected: `config dbt-env` and `release write` both exit 1 with a
+              ConfigError naming the missing SSM key and pointing at the
+              pipeline upgrade (>= v2.0.0); no partial exports, no s3://None,
+              and the release writer is never invoked.
+    """
+    ssm = boto3.client("ssm", region_name=REGION)
+    leaves = {  # everything _seed publishes EXCEPT the medallion keys
+        "meta/region": REGION,
+        "buckets/metadata": f"etl-test-metadata-{ACCOUNT}-{REGION}",
+        "release/db": "etl_test_dataops_metadata_db",
+        "release/table": "releases",
+        "athena/workgroup": "etl-test",
+        "athena/outputLocation": f"s3://etl-test-athena-results-{ACCOUNT}-{REGION}/",
+    }
+    for rel, value in leaves.items():
+        ssm.put_parameter(Name=f"/etl/test/{rel}", Value=value, Type="String")
+
+    result = runner.invoke(app, ["config", "dbt-env", "--env", "test"])
+    assert result.exit_code == 1
+    assert "/etl/test/glue/db/silver is missing" in result.output
+    assert "v2.0.0" in result.output
+    assert "export" not in result.output  # all-or-nothing: no partial env
+    assert "s3://None" not in result.output
+
+    result = runner.invoke(
+        app,
+        ["release", "write", "--env", "test", "--data-release-version", "1.0.0"],
+    )
+    assert result.exit_code == 1
+    assert "/etl/test/glue/db/silver is missing" in result.output
+    assert "v2.0.0" in result.output
+    mock_run.assert_not_called()
 
 
 def test_release_writer_run_aggregates_model_failures():
@@ -267,7 +317,7 @@ def fake_snapshot(self, return_commit_datetime=False):
     with patch.object(rw, "get_model_names", return_value=["silver_a", "silver_bad", "silver_b"]), \
          patch.object(rw, "insert_release_row", side_effect=fake_insert), \
          patch.object(rw.AthenaQuery, "create_release_table", MagicMock()), \
-         patch.object(rw.AthenaQuery, "find_db_for_model", lambda self, m, databases=None: "etl_test_raw_silver_db"), \
+         patch.object(rw.AthenaQuery, "find_db_for_model", lambda self, m, databases=None: "etl_test_silver_db"), \
          patch.object(rw.AthenaValidationWriter, "_get_latest_snapshot_id", fake_snapshot):
         with pytest.raises(RuntimeError, match="silver_bad"):
             rw.run(