diff --git a/README.md b/README.md index f07932f..d4dce1a 100644 --- a/README.md +++ b/README.md @@ -79,13 +79,13 @@ structurally impossible. **Only the dbt template's `ci` target is prefixed.** `g3dt config dbt-env` emits, alongside the real names, the CI-isolation variants the template's -`ci` target consumes: `G3DT_DB_BRONZE_CI` / `G3DT_DB_SILVER_CI` / -`G3DT_DB_GOLD_CI` (`ci_` + the real database name) and -`G3DT_S3_BRONZE_DATA_DIR_CI` / `G3DT_S3_SILVER_DATA_DIR_CI` / -`G3DT_S3_GOLD_DATA_DIR_CI` (`dbt_ci/` under the same buckets). Bronze can be -dbt-managed from gen3-dbt-template's synthetic-data revision onwards; a -pipeline deployment >= v2.1.0 provides the matching `ci_..._bronze_db` Glue -database. Toolkit +`ci` target consumes: `G3DT_DB_SILVER_CI` / `G3DT_DB_GOLD_CI` +(`ci_` + the real database name) and `G3DT_S3_SILVER_DATA_DIR_CI` / +`G3DT_S3_GOLD_DATA_DIR_CI` (`dbt_ci/` under the same buckets). Bronze is +ingest-only — dbt never writes it, and the template's synthetic demo data +generates at silver — so bronze gets only `G3DT_DB_BRONZE` (real-ingest +deployments resolve their sources.yml schema from it); pairs with an +aws-gen3-pipeline deployment >= v2.2.0. Toolkit releases >= 3 read the raw-free medallion SSM keys and therefore require a pipeline deployment >= v2.0.0, which publishes them. Commit- triggered CI builds land there; every other target (default, local) and the diff --git a/pyproject.toml b/pyproject.toml index 3f2d92b..8433dc5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gen3-dataops-toolkit" -version = "3.1.0" +version = "3.2.0" description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments" authors = ["JoshuaHarris391 "] readme = "README.md" diff --git a/src/g3dt/cli/config_cmds.py b/src/g3dt/cli/config_cmds.py index 49808c9..6bfa703 100644 --- a/src/g3dt/cli/config_cmds.py +++ b/src/g3dt/cli/config_cmds.py @@ -233,7 +233,6 @@ def dbt_env( gold_db = _req_key(rc, "glue/db/gold") silver_bucket = _req_key(rc, "buckets/silver") gold_bucket = _req_key(rc, "buckets/gold") - bronze_bucket = _req_key(rc, "buckets/bronze") bronze_db = _req_key(rc, "glue/db/bronze") except config.ConfigError as exc: typer.secho(str(exc), fg=typer.colors.RED, err=True) @@ -246,16 +245,15 @@ def dbt_env( "G3DT_DB_BRONZE": bronze_db, "G3DT_DB_SILVER": silver_db, "G3DT_DB_GOLD": gold_db, - "G3DT_S3_BRONZE_DATA_DIR": f"s3://{bronze_bucket}/dbt/", "G3DT_S3_SILVER_DATA_DIR": f"s3://{silver_bucket}/dbt/", "G3DT_S3_GOLD_DATA_DIR": f"s3://{gold_bucket}/dbt/", # CI isolation: the dbt template's `ci` target builds into these # instead — same grammar as the CDK's ci_ databases, same buckets # under a dbt_ci/ prefix. Real names above are never prefixed. - "G3DT_DB_BRONZE_CI": f"ci_{bronze_db}", + # Bronze has no CI variants: bronze is ingest-only (never written + # by dbt), so only its real database name is exported. "G3DT_DB_SILVER_CI": f"ci_{silver_db}", "G3DT_DB_GOLD_CI": f"ci_{gold_db}", - "G3DT_S3_BRONZE_DATA_DIR_CI": f"s3://{bronze_bucket}/dbt_ci/", "G3DT_S3_SILVER_DATA_DIR_CI": f"s3://{silver_bucket}/dbt_ci/", "G3DT_S3_GOLD_DATA_DIR_CI": f"s3://{gold_bucket}/dbt_ci/", } diff --git a/tests/test_release_cmds.py b/tests/test_release_cmds.py index 648112e..0f469d7 100644 --- a/tests/test_release_cmds.py +++ b/tests/test_release_cmds.py @@ -52,7 +52,6 @@ def _seed(project="etl", env="test"): "buckets/metadata": f"{project}-{env}-metadata-{ACCOUNT}-{REGION}", "buckets/silver": f"{project}-{env}-silver-{ACCOUNT}-{REGION}", "buckets/gold": f"{project}-{env}-gold-{ACCOUNT}-{REGION}", - "buckets/bronze": f"{project}-{env}-bronze-{ACCOUNT}-{REGION}", "glue/db/bronze": f"{project}_{env}_bronze_db", "glue/db/silver": f"{project}_{env}_silver_db", "glue/db/gold": f"{project}_{env}_gold_db", @@ -162,8 +161,10 @@ def test_config_dbt_env_emits_every_dbt_setting(): """ Inputs: g3dt config dbt-env --env test Expected Output: shell-evaluable `export` lines carrying the workgroup, - Athena output, region, bronze/silver/gold DBs and the bronze/silver/gold + Athena output, region, bronze/silver/gold DBs and the silver/gold s3_data_dir values — the full env_var() contract of the dbt template. + Bronze is ingest-only, so it gets a database name (for sources.yml + schema resolution) but no dbt write-path data dir. """ _seed() result = runner.invoke(app, ["config", "dbt-env", "--env", "test"]) @@ -175,9 +176,10 @@ def test_config_dbt_env_emits_every_dbt_setting(): assert "export G3DT_DB_BRONZE=etl_test_bronze_db" in out assert "export G3DT_DB_SILVER=etl_test_silver_db" in out assert "export G3DT_DB_GOLD=etl_test_gold_db" in out - assert f"export G3DT_S3_BRONZE_DATA_DIR=s3://etl-test-bronze-{ACCOUNT}-{REGION}/dbt/" in out assert f"export G3DT_S3_SILVER_DATA_DIR=s3://etl-test-silver-{ACCOUNT}-{REGION}/dbt/" in out assert f"export G3DT_S3_GOLD_DATA_DIR=s3://etl-test-gold-{ACCOUNT}-{REGION}/dbt/" in out + # bronze is ingest-only: no dbt write path for it + assert "G3DT_S3_BRONZE_DATA_DIR" not in out # no profile configured -> ambient credentials and the default dbt target assert "G3DT_AWS_PROFILE" not in out assert "G3DT_DBT_TARGET" not in out @@ -230,12 +232,13 @@ def test_config_dbt_env_emits_ci_isolation_vars(): result = runner.invoke(app, ["config", "dbt-env", "--env", "test"]) assert result.exit_code == 0, result.output out = result.output - assert "export G3DT_DB_BRONZE_CI=ci_etl_test_bronze_db" in out assert "export G3DT_DB_SILVER_CI=ci_etl_test_silver_db" in out assert "export G3DT_DB_GOLD_CI=ci_etl_test_gold_db" in out - assert f"export G3DT_S3_BRONZE_DATA_DIR_CI=s3://etl-test-bronze-{ACCOUNT}-{REGION}/dbt_ci/" in out assert f"export G3DT_S3_SILVER_DATA_DIR_CI=s3://etl-test-silver-{ACCOUNT}-{REGION}/dbt_ci/" in out assert f"export G3DT_S3_GOLD_DATA_DIR_CI=s3://etl-test-gold-{ACCOUNT}-{REGION}/dbt_ci/" in out + # bronze is ingest-only: no CI variants, only the real database name + assert "G3DT_DB_BRONZE_CI" not in out + assert "G3DT_S3_BRONZE_DATA_DIR_CI" not in out # the real names remain unprefixed assert "export G3DT_DB_BRONZE=etl_test_bronze_db" in out assert "export G3DT_DB_SILVER=etl_test_silver_db" in out