Skip to content

feat: per-study delete versions and duplicate-upload pre-flight #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
JoshuaHarris391 wants to merge 2 commits into from
Closed

feat: per-study delete versions and duplicate-upload pre-flight #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b25d54d
feat(cli): per-study name:version specs for delete metadata
JoshuaHarris391 Aug 4, 2026
6c05037
feat(upload): refuse duplicate metadata uploads; add --force-reupload
JoshuaHarris391 Aug 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
156 changes: 116 additions & 40 deletions src/g3dt/cli/delete_cmds.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,16 +56,82 @@ def _normalise_version(raw: str, where: str) -> str:
return match.group(1)


def _parse_study_specs(studies: str, fallback, env: str):
"""Turn ``--studies`` into ``[(resolved_study_key, version), ...]``.

Each comma-separated entry is ``name`` or ``name:version``. A bare name
takes *fallback* (the ``--version`` default); *fallback* is ``None`` when
``--version`` was not given, which makes a bare name a usage error.

Every entry is validated before anything is dispatched, so a typo in the
last study cannot leave the earlier ones already deleted.
"""
specs = []
for entry in studies.split(","):
entry = entry.strip()
if not entry:
continue

# partition() rather than split(), so a trailing colon ("ausdiab:") is
# distinguishable from a bare name and can be rejected instead of
# silently taking the fallback.
name, sep, raw_version = entry.partition(":")
name = name.strip()

if not name:
typer.secho(
f"Invalid --studies entry '{entry}': missing study name.",
fg=typer.colors.RED,
err=True,
)
raise typer.Exit(2)

if sep and not raw_version.strip():
typer.secho(
f"Invalid --studies entry '{entry}': ':' with no version. "
f"Use '{name}:0.9.8', '{name}:all', or a bare '{name}' to take "
"the --version default.",
fg=typer.colors.RED,
err=True,
)
raise typer.Exit(2)

if sep:
version = _normalise_version(raw_version, f"for study '{name}'")
elif fallback is not None:
version = fallback
else:
typer.secho(
f"No version for study '{name}': add ':<version>' to it "
f"(e.g. '{name}:0.9.8'), or pass --version as the default for "
"every study. Use 'all' to delete every version.",
fg=typer.colors.RED,
err=True,
)
raise typer.Exit(2)

specs.append((study_of(name, env).key, version))

if not specs:
typer.secho("--studies is empty.", fg=typer.colors.RED, err=True)
raise typer.Exit(2)
return specs


@app.command()
def metadata(
studies: str = typer.Option(
..., "--studies", help="Comma-separated studies, e.g. ausdiab,caughtcad."
...,
"--studies",
help="Comma-separated studies, each optionally 'name:version', "
"e.g. ausdiab:0.7.5,cdah:0.8.1,edcad.",
),
env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
version: str = typer.Option(
None,
"--version",
help="Metadata version to delete, e.g. 0.9.8, or 'all' for every version.",
help="Default version for studies written without their own "
"':version', e.g. 0.9.8, or 'all' for every version.",
),
node: str = typer.Option(None, "--node", help="Delete only this node type."),
yes: bool = typer.Option(
Expand All @@ -77,55 +143,65 @@ def metadata(

Studies are processed one at a time. A study that exists but has no data at
the requested version is skipped, and the job continues to the next study.
"""
if version is None:
typer.secho(
"--version is required: specify a version (e.g. 0.9.8) or 'all' "
"to delete every version.",
fg=typer.colors.RED,
err=True,
)
raise typer.Exit(2)

version = _normalise_version(version, "for --version")
Each study may carry its own version as ``name:version``; ``--version``
supplies the default for any study written bare. Examples:

names = [s.strip() for s in studies.split(",") if s.strip()]
keys = [study_of(name, env).key for name in names]
target = ",".join(keys)
all_versions = version == "all"

if all_versions:
# Deleting every version is the most destructive path: always prompt
# (pass assume_yes=False so --yes can't bypass it; prod still types the
# target).
safety.confirm_destructive("deletion of ALL VERSIONS", target, env, False)
g3dt delete metadata --studies "ausdiab:0.7.5,cdah:0.8.1" --env staging
g3dt delete metadata --studies "ausdiab:all,cdah" --version 0.9.8 --env staging
"""
fallback = (
_normalise_version(version, "for --version") if version is not None else None
)
specs = _parse_study_specs(studies, fallback, env)
versions = [v for _, v in specs]

# The typed production confirmation stays the study keys alone: short
# enough to retype accurately, while the per-study versions are spelled
# out in the action line printed directly above the prompt.
target = ",".join(key for key, _ in specs)
uniform = len(set(versions)) == 1
any_all = "all" in versions

if uniform and versions[0] == "all":
action = "deletion of ALL VERSIONS"
elif uniform:
action = f"deletion of v{versions[0]}"
else:
safety.confirm_destructive(f"deletion of v{version}", target, env, yes)
plan = ", ".join(f"{key}:{v}" for key, v in specs)
action = f"deletion of per-study versions [{plan}]"

# Deleting every version is the most destructive path: always prompt (pass
# assume_yes=False so --yes can't bypass it; prod still types the target).
# One 'all' anywhere in the list is enough to force the prompt, so an 'all'
# buried mid-list cannot ride along on a batch marked unattended.
safety.confirm_destructive(action, target, env, False if any_all else yes)

def build_args(env_name):
a = [
"--studies",
target,
"--env",
env_name,
"--version",
"all" if all_versions else version,
]
if uniform:
# Canonical (and historical) shape: one --version for every study.
# Emitting it keeps a newer CLI compatible with an older installed
# service script on the box, which can lag a pip upgrade.
a = ["--studies", target, "--env", env_name, "--version", versions[0]]
else:
a = [
"--studies",
",".join(f"{key}:{v}" for key, v in specs),
"--env",
env_name,
]
if node:
a += ["--node", node]
return a

def remote_cli(env_name):
# --yes: confirmation already happened locally; the remote job must
# not prompt (SSM has no TTY). The remote re-check is version-specific
# only, and 'all' was already confirmed above.
a = [
"delete", "metadata",
"--studies", studies,
"--env", env_name,
"--version", "all" if all_versions else version,
"--yes",
]
# not prompt (SSM has no TTY). The raw --studies string is forwarded
# verbatim — the remote re-entry re-parses and re-validates it.
a = ["delete", "metadata", "--studies", studies, "--env", env_name]
if version is not None:
a += ["--version", version]
a.append("--yes")
if node:
a += ["--node", node]
return a
Expand Down
22 changes: 22 additions & 0 deletions src/g3dt/cli/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,19 @@ def upload(
study: str = typer.Option(..., "--study", "-s", help="Study, e.g. ausdiab."),
env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
node: str = typer.Option(None, "--node", help="Submit only this node type."),
force_reupload: bool = typer.Option(
False, "--force-reupload",
help="Proceed even if this project+version was already uploaded to "
"this commons (uploads are additive: re-running duplicates records).",
),
on: Target = typer.Option(Target.local, "--on", help="Run local or on ec2."),
) -> None:
"""Upload a study's release metadata to Gen3 sheepdog.

The worker refuses (exit 2) when the audit table already records an
upload of the same project + version + endpoint — re-running would
duplicate every record. ``--force-reupload`` overrides.

Examples:
g3dt metadata upload --study ausdiab --env staging
g3dt metadata upload --study ausdiab --env staging --on ec2
Expand All @@ -36,12 +45,16 @@ def build_args(env_name):
a = ["--study", s.key, "--env", env_name]
if node:
a += ["--specific-node", node]
if force_reupload:
a.append("--force-reupload")
return a

def remote_cli(env_name):
a = ["metadata", "upload", "--study", study, "--env", env_name]
if node:
a += ["--node", node]
if force_reupload:
a.append("--force-reupload")
return a

dispatch.run_or_dispatch(
Expand All @@ -64,6 +77,11 @@ def upload_all(
help="Internal: set by the remote re-entry after the typed "
"confirmation already happened locally. Never pass by hand.",
),
force_reupload: bool = typer.Option(
False, "--force-reupload",
help="Proceed even for project+versions the audit table says were "
"already uploaded to this commons.",
),
on: Target = typer.Option(Target.local, "--on", help="Run local or on ec2."),
) -> None:
"""Upload several studies sequentially (wraps upload_all_studies.sh).
Expand Down Expand Up @@ -95,6 +113,8 @@ def build_args(env_name):
a = ["--studies", ",".join(keys), "--env", env_name]
if allow_prod:
a.append("--allow-prod")
if force_reupload:
a.append("--force-reupload")
return a

def remote_cli(env_name):
Expand All @@ -103,6 +123,8 @@ def remote_cli(env_name):
# The typed confirmation already happened locally above; the box
# has no TTY, so the re-entry must not prompt again.
a += ["--allow-prod", "--prod-confirmed"]
if force_reupload:
a.append("--force-reupload")
return a

dispatch.run_or_dispatch(
Expand Down
70 changes: 52 additions & 18 deletions src/g3dt/services/delete/delete_metadata.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,21 @@ SKIP_EXIT_CODE=3

usage() {
cat <<EOF
Usage: $(basename "$0") --studies <comma-separated-studies> --env <environment> --version <version|all> [--node <node>]
Usage: $(basename "$0") --studies <name[:version|all],...> --env <environment> [--version <version|all>] [--node <node>]

Delete metadata for each study sequentially, in a single job.

Arguments:
--studies Comma-separated list of study config keys (e.g. ausdiab_staging,caughtcad_staging)
--studies Comma-separated study config keys, each optionally qualified with
its own version (e.g. ausdiab_staging:0.7.5,cdah_staging:0.8.1,
or bare ausdiab_staging to take the --version default)
--env Environment string passed to the Python worker (e.g. staging_ec2)
--version Metadata version to delete (e.g. 0.9.8), or 'all' for every version
--version Default version for bare --studies entries (e.g. 0.9.8), or 'all'
--node (optional) Restrict deletion to a single node type

Behaviour:
* --version all -> delete_all_metadata_for_project.py (deletes whole nodes)
* --version <x.y.z> -> delete_metadata_by_guid.py (Athena GUID lookup for that version)
* version 'all' -> delete_all_metadata_for_project.py (deletes whole nodes)
* version <x.y.z> -> delete_metadata_by_guid.py (Athena GUID lookup for that version)

A study that exists but has no data at the requested version is skipped and the
loop continues. Only genuine errors (Gen3/AWS failures) count as failures.
Expand Down Expand Up @@ -69,13 +71,39 @@ while [[ $# -gt 0 ]]; do
esac
done

if [[ -z "$STUDIES" || -z "$ENV" || -z "$VERSION" ]]; then
echo "ERROR: --studies, --env and --version are required."
if [[ -z "$STUDIES" || -z "$ENV" ]]; then
echo "ERROR: --studies and --env are required."
usage
fi

# Lower-case the version so 'ALL'/'All' are treated as 'all'.
VERSION_LC="$(echo "$VERSION" | tr '[:upper:]' '[:lower:]')"
# Expand '--studies name[:version],...' into two parallel arrays. An entry with
# no ':version' takes the --version default. Validating the whole list up front
# means a typo in the last entry cannot leave the earlier studies already
# deleted.
IFS=',' read -ra STUDY_ENTRIES <<< "$STUDIES"
STUDY_NAMES=()
STUDY_VERSIONS=()
for entry in "${STUDY_ENTRIES[@]}"; do
name="${entry%%:*}"
# Test for the ':' explicitly: for a bare 'name', "${entry#*:}" expands to
# 'name' rather than to the empty string, which would silently become the
# version.
if [[ "$entry" == *:* ]]; then
entry_version="${entry#*:}"
else
entry_version="$VERSION"
fi
if [[ -z "$name" ]]; then
echo "ERROR: empty study name in --studies entry '${entry}'."
usage
fi
if [[ -z "$entry_version" ]]; then
echo "ERROR: study '${name}' has no version: use '${name}:<version|all>' in --studies, or pass --version as the default."
usage
fi
STUDY_NAMES+=("$name")
STUDY_VERSIONS+=("$entry_version")
done

# ---------- Setup ----------
# Logs go outside the installed package.
Expand All @@ -84,7 +112,6 @@ TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
FAILED_LOG="${LOG_DIR}/${TIMESTAMP}_delete_failed.log"
mkdir -p "${LOG_DIR}"

IFS=',' read -ra STUDY_LIST <<< "$STUDIES"
DELETED_COUNT=0
SKIPPED_COUNT=0
FAIL_COUNT=0
Expand All @@ -93,25 +120,30 @@ echo "============================================"
echo "Metadata delete started at $(date)"
echo "Environment : ${ENV}"
echo "Studies : ${STUDIES}"
echo "Version : ${VERSION}"
echo "Version : ${VERSION:-(per study, from --studies)}"
[[ -n "$NODE" ]] && echo "Node : ${NODE}"
echo "Failure log : ${FAILED_LOG}"
echo "============================================"
echo ""

# ---------- Sequential execution ----------
for study in "${STUDY_LIST[@]}"; do
for i in "${!STUDY_NAMES[@]}"; do
study="${STUDY_NAMES[$i]}"
study_version="${STUDY_VERSIONS[$i]}"
# Lower-cased for the 'all' comparison only; the worker gets the original.
study_version_lc="$(echo "$study_version" | tr '[:upper:]' '[:lower:]')"

echo "--------------------------------------------"
echo "[$(date +%Y-%m-%d\ %H:%M:%S)] Starting deletion for study: ${study} (version: ${VERSION})"
echo "[$(date +%Y-%m-%d\ %H:%M:%S)] Starting deletion for study: ${study} (version: ${study_version})"
echo "--------------------------------------------"

if [[ "$VERSION_LC" == "all" ]]; then
if [[ "$study_version_lc" == "all" ]]; then
CMD=(python3 "${SCRIPT_DIR}/delete_all_metadata_for_project.py"
--study "$study" --env "$ENV")
[[ -n "$NODE" ]] && CMD+=(--node "$NODE")
else
CMD=(python3 "${SCRIPT_DIR}/delete_metadata_by_guid.py"
--study "$study" --env "$ENV" --version "$VERSION" --skip-if-empty)
--study "$study" --env "$ENV" --version "$study_version" --skip-if-empty)
[[ -n "$NODE" ]] && CMD+=(--node "$NODE")
fi

Expand All @@ -125,12 +157,14 @@ for study in "${STUDY_LIST[@]}"; do
echo "[$(date +%Y-%m-%d\ %H:%M:%S)] Completed: ${study}"
DELETED_COUNT=$((DELETED_COUNT + 1))
elif [[ $EXIT_CODE -eq $SKIP_EXIT_CODE ]]; then
echo "[$(date +%Y-%m-%d\ %H:%M:%S)] Skipped (no data at version ${VERSION}): ${study}"
echo "[$(date +%Y-%m-%d\ %H:%M:%S)] Skipped (no data at version ${study_version}): ${study}"
SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
else
FAIL_COUNT=$((FAIL_COUNT + 1))
echo "[$(date +%Y-%m-%d\ %H:%M:%S)] FAILED: ${study} (exit code ${EXIT_CODE})"
echo "[$(date +%Y-%m-%d\ %H:%M:%S)] ${study} exit_code=${EXIT_CODE}" >> "$FAILED_LOG"
# The version is recorded because one job can now delete two versions
# of the same study.
echo "[$(date +%Y-%m-%d\ %H:%M:%S)] ${study} version=${study_version} exit_code=${EXIT_CODE}" >> "$FAILED_LOG"
fi

echo ""
Expand All @@ -139,7 +173,7 @@ done
# ---------- Summary ----------
echo "============================================"
echo "Metadata delete finished at $(date)"
echo "Total studies : ${#STUDY_LIST[@]}"
echo "Total studies : ${#STUDY_NAMES[@]}"
echo "Deleted : ${DELETED_COUNT}"
echo "Skipped : ${SKIPPED_COUNT}"
echo "Failures : ${FAIL_COUNT}"
Expand Down
Loading
Loading