From 70ec5390dbcee2e0e8843152219074e93cad8d2e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 20 Aug 2026 11:10:01 +0200 Subject: [PATCH] direct: fail bind when the deployment state cannot be read Bind treated any failure to open the state as "resource not bound": it skipped the already-managed check and then bound over state whose ownership was never verified, overwriting it. A crashed deploy that left a WAL behind was enough to trigger this, and bind still reported success. A state file that does not exist opens as an empty one, so absence of a binding is already covered; return the error instead. Co-authored-by: Isaac --- .nextchanges/bundles/bind-unreadable-state.md | 1 + .../bind/job/unreadable-state/databricks.yml | 7 +++++ .../bind/job/unreadable-state/out.test.toml | 2 ++ .../bind/job/unreadable-state/output.txt | 21 +++++++++++++++ .../job/unreadable-state/resources.json.wal | 2 ++ .../bind/job/unreadable-state/script | 14 ++++++++++ .../bind/job/unreadable-state/test.toml | 5 ++++ bundle/direct/bind.go | 27 +++++++++++-------- 8 files changed, 68 insertions(+), 11 deletions(-) create mode 100644 .nextchanges/bundles/bind-unreadable-state.md create mode 100644 acceptance/bundle/deployment/bind/job/unreadable-state/databricks.yml create mode 100644 acceptance/bundle/deployment/bind/job/unreadable-state/out.test.toml create mode 100644 acceptance/bundle/deployment/bind/job/unreadable-state/output.txt create mode 100644 acceptance/bundle/deployment/bind/job/unreadable-state/resources.json.wal create mode 100644 acceptance/bundle/deployment/bind/job/unreadable-state/script create mode 100644 acceptance/bundle/deployment/bind/job/unreadable-state/test.toml diff --git a/.nextchanges/bundles/bind-unreadable-state.md b/.nextchanges/bundles/bind-unreadable-state.md new file mode 100644 index 00000000000..d9cc7cf05d0 --- /dev/null +++ b/.nextchanges/bundles/bind-unreadable-state.md @@ -0,0 +1 @@ +Fix `bundle deployment bind` silently binding over a deployment state it could not read, which could take over a resource that was already managed by the bundle. diff --git a/acceptance/bundle/deployment/bind/job/unreadable-state/databricks.yml b/acceptance/bundle/deployment/bind/job/unreadable-state/databricks.yml new file mode 100644 index 00000000000..4077df9f8b3 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/unreadable-state/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + jobs: + job_1: + name: Job 1 diff --git a/acceptance/bundle/deployment/bind/job/unreadable-state/out.test.toml b/acceptance/bundle/deployment/bind/job/unreadable-state/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/unreadable-state/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/job/unreadable-state/output.txt b/acceptance/bundle/deployment/bind/job/unreadable-state/output.txt new file mode 100644 index 00000000000..bc3424895b0 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/unreadable-state/output.txt @@ -0,0 +1,21 @@ + +=== Deploy job_1 +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.job_1 +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Leave behind a WAL from a different lineage, as a crashed deploy would +=== Bind must refuse: the state cannot be read, so we cannot tell if job_1 is already managed + +>>> errcode [CLI] bundle deployment bind job_1 [EXTERNAL_JOB_ID] --auto-approve +Error: cannot check whether resources.jobs.job_1 is already bound: reading state from [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: WAL recovery failed: WAL lineage ("wal-lineage-bbb") does not match state lineage ("[UUID]") + + +Exit code: 1 + +=== Nothing was bound: the WAL is still there and no bind state was written +>>> assert_exists.py .databricks/bundle/default/resources.json.wal + +>>> assert_not_exists.py .databricks/bundle/default/resources.json.temp-bind diff --git a/acceptance/bundle/deployment/bind/job/unreadable-state/resources.json.wal b/acceptance/bundle/deployment/bind/job/unreadable-state/resources.json.wal new file mode 100644 index 00000000000..5ecc88124f0 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/unreadable-state/resources.json.wal @@ -0,0 +1,2 @@ +{"lineage":"wal-lineage-bbb","serial":2} +{"k":"resources.jobs.job_1","v":{"__id__":"1001","state":{"name":"Job 1"}}} diff --git a/acceptance/bundle/deployment/bind/job/unreadable-state/script b/acceptance/bundle/deployment/bind/job/unreadable-state/script new file mode 100644 index 00000000000..221470a9cc4 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/unreadable-state/script @@ -0,0 +1,14 @@ +title "Deploy job_1" +trace $CLI bundle deploy + +title "Leave behind a WAL from a different lineage, as a crashed deploy would" +cp resources.json.wal .databricks/bundle/default/ + +title "Bind must refuse: the state cannot be read, so we cannot tell if job_1 is already managed\n" +job_id=$($CLI jobs create --json '{"name": "External Job"}' | jq -r '.job_id') +add_repl "$job_id" EXTERNAL_JOB_ID +trace errcode $CLI bundle deployment bind job_1 "$job_id" --auto-approve + +title "Nothing was bound: the WAL is still there and no bind state was written" +trace assert_exists.py .databricks/bundle/default/resources.json.wal +trace assert_not_exists.py .databricks/bundle/default/resources.json.temp-bind diff --git a/acceptance/bundle/deployment/bind/job/unreadable-state/test.toml b/acceptance/bundle/deployment/bind/job/unreadable-state/test.toml new file mode 100644 index 00000000000..fc6e9d7bbf7 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/unreadable-state/test.toml @@ -0,0 +1,5 @@ +Cloud = false + +Ignore = ["resources.json.wal"] + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 9760ce95666..b487db4b000 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -62,17 +62,22 @@ type BindResult struct { func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*BindResult, error) { // Check if the resource is already managed (bound to a different ID) var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false)); err == nil { - existingID := checkStateDB.GetResourceID(resourceKey) - if _, err := checkStateDB.Finalize(ctx); err != nil { - log.Warnf(ctx, "failed to finalize state: %v", err) - } - if existingID != "" { - return nil, ErrResourceAlreadyBound{ - ResourceKey: resourceKey, - ExistingID: existingID, - NewID: resourceID, - } + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + // State that cannot be read is not the same as state without a binding: + // the resource may well be managed already, and binding on top of it + // would take over a resource whose ownership was never checked. A state + // file that does not exist yet opens successfully as an empty one. + return nil, fmt.Errorf("cannot check whether %s is already bound: %w", resourceKey, err) + } + existingID := checkStateDB.GetResourceID(resourceKey) + if _, err := checkStateDB.Finalize(ctx); err != nil { + log.Warnf(ctx, "failed to finalize state: %v", err) + } + if existingID != "" { + return nil, ErrResourceAlreadyBound{ + ResourceKey: resourceKey, + ExistingID: existingID, + NewID: resourceID, } }