diff --git a/scripts/create_resources/combine/process_datasets_allen_nebius.sh b/scripts/create_resources/combine/process_datasets_allen_nebius.sh new file mode 100644 index 000000000..ec29fdb2b --- /dev/null +++ b/scripts/create_resources/combine/process_datasets_allen_nebius.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +# Process ONLY the Allen Brain Cell Atlas (ABCA) MERFISH datasets (combine step) — FOUR +# whole-brain sections, ALL paired with the SAME mouse-brain SC reference (like the LTX/MPII +# combine scripts, and unlike Kuppe which uses condition-specific references): +# mouse1_coronal, mouse2_coronal, mouse3_sagittal, mouse4_sagittal +# <-> allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2 (ABCA 2023 Yao 10xv2 atlas) +# +# Reads each spatial input (process_allen_brain_cell_atlas_merfish loader output) and the SC +# reference from the local /scratch raw folder, and writes the combined datasets to the +# /scratch datasets folder (same layout the other *_nebius.sh combine scripts use). +# +# Prerequisites (both publish to the same /scratch raw folder used below): +# 1. scripts/create_resources/spatial/process_allen_brain_cell_atlas_merfish_nebius.sh +# -> /scratch/.../raw/allen_brain_cell_atlas_merfish/mouse{1,2,3,4}_*/rep1/dataset.zarr +# 2. scripts/create_resources/sc/process_allen_brain_cell_atlas_brain_nebius.sh (log_cp -> hvg +# -> pca -> knn, i.e. it must carry a 'normalized' layer) +# -> /scratch/.../raw/allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2/dataset.h5ad + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +# process_allen_brain_cell_atlas_merfish_nebius.sh + process_allen_brain_cell_atlas_brain_nebius.sh +# both publish here. +raw_dir='/scratch/task_ist_preprocessing/raw' +publish_dir='/scratch/task_ist_preprocessing/datasets' + +# shared SC reference for all four sections (ABCA 2023 Yao whole-brain 10xv2 atlas) +sc_ref="$raw_dir/allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2/dataset.h5ad" + +launch_batch() { + local params_file="$1" + local label="$2" + tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/process_datasets/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file "$params_file" \ + --config src/base/labels_nebius.config \ + --labels "task_ist_preprocessing,process_datasets,$label" +} + +cat > /tmp/params_allen.yaml << HERE +param_list: + + - id: "allen_brain_cell_atlas_merfish_combined/mouse1_coronal/rep1" + input_sp: "$raw_dir/allen_brain_cell_atlas_merfish/mouse1_coronal/rep1/dataset.zarr" + input_sc: "$sc_ref" + dataset_id: "allen_brain_cell_atlas_merfish_combined/mouse1_coronal/rep1" + dataset_name: "Mouse brain combined ABCA MERFISH mouse1 coronal + 2023 Yao scRNAseq" + dataset_url: "https://download.brainimagelibrary.org/29/3c/293cc39ceea87f6d/" + dataset_reference: "10.1038/s41586-023-06812-z" + dataset_summary: "Allen Brain Cell Atlas whole-brain MERFISH mouse 1 (coronal, ~1100-gene panel) + 2023 Yao mouse-brain scRNAseq" + dataset_description: "Brain-wide MERFISH spatial transcriptomics (Zhuang lab, Allen Brain Cell Atlas), mouse 1 coronal section, paired with the ABCA 2023 Yao whole-mouse-brain 10xv2 scRNAseq reference." + dataset_organism: "mus_musculus" + + - id: "allen_brain_cell_atlas_merfish_combined/mouse2_coronal/rep1" + input_sp: "$raw_dir/allen_brain_cell_atlas_merfish/mouse2_coronal/rep1/dataset.zarr" + input_sc: "$sc_ref" + dataset_id: "allen_brain_cell_atlas_merfish_combined/mouse2_coronal/rep1" + dataset_name: "Mouse brain combined ABCA MERFISH mouse2 coronal + 2023 Yao scRNAseq" + dataset_url: "https://download.brainimagelibrary.org/29/3c/293cc39ceea87f6d/" + dataset_reference: "10.1038/s41586-023-06812-z" + dataset_summary: "Allen Brain Cell Atlas whole-brain MERFISH mouse 2 (coronal, ~1100-gene panel) + 2023 Yao mouse-brain scRNAseq" + dataset_description: "Brain-wide MERFISH spatial transcriptomics (Zhuang lab, Allen Brain Cell Atlas), mouse 2 coronal section, paired with the ABCA 2023 Yao whole-mouse-brain 10xv2 scRNAseq reference." + dataset_organism: "mus_musculus" + + - id: "allen_brain_cell_atlas_merfish_combined/mouse3_sagittal/rep1" + input_sp: "$raw_dir/allen_brain_cell_atlas_merfish/mouse3_sagittal/rep1/dataset.zarr" + input_sc: "$sc_ref" + dataset_id: "allen_brain_cell_atlas_merfish_combined/mouse3_sagittal/rep1" + dataset_name: "Mouse brain combined ABCA MERFISH mouse3 sagittal + 2023 Yao scRNAseq" + dataset_url: "https://download.brainimagelibrary.org/29/3c/293cc39ceea87f6d/" + dataset_reference: "10.1038/s41586-023-06812-z" + dataset_summary: "Allen Brain Cell Atlas whole-brain MERFISH mouse 3 (sagittal, ~1100-gene panel) + 2023 Yao mouse-brain scRNAseq" + dataset_description: "Brain-wide MERFISH spatial transcriptomics (Zhuang lab, Allen Brain Cell Atlas), mouse 3 sagittal section, paired with the ABCA 2023 Yao whole-mouse-brain 10xv2 scRNAseq reference." + dataset_organism: "mus_musculus" + + - id: "allen_brain_cell_atlas_merfish_combined/mouse4_sagittal/rep1" + input_sp: "$raw_dir/allen_brain_cell_atlas_merfish/mouse4_sagittal/rep1/dataset.zarr" + input_sc: "$sc_ref" + dataset_id: "allen_brain_cell_atlas_merfish_combined/mouse4_sagittal/rep1" + dataset_name: "Mouse brain combined ABCA MERFISH mouse4 sagittal + 2023 Yao scRNAseq" + dataset_url: "https://download.brainimagelibrary.org/29/3c/293cc39ceea87f6d/" + dataset_reference: "10.1038/s41586-023-06812-z" + dataset_summary: "Allen Brain Cell Atlas whole-brain MERFISH mouse 4 (sagittal, ~1100-gene panel) + 2023 Yao mouse-brain scRNAseq" + dataset_description: "Brain-wide MERFISH spatial transcriptomics (Zhuang lab, Allen Brain Cell Atlas), mouse 4 sagittal section, paired with the ABCA 2023 Yao whole-mouse-brain 10xv2 scRNAseq reference." + dataset_organism: "mus_musculus" + +# ABCA whole-brain images are enormous (~83k x 102k px) and the tissue sits off-centre, +# so an image-centred crop can miss it (mouse1: 0 of 42M transcripts). Centre the crop +# on the transcript density instead (opt-in flag; default is image-centred). +tissue_centered_crop: true + +output_sc: "\$id/output_sc.h5ad" +output_sp: "\$id/output_sp.zarr" +output_state: "\$id/state.yaml" +publish_dir: "$publish_dir" +HERE + +launch_batch /tmp/params_allen.yaml "allen_brain_cell_atlas_merfish" diff --git a/scripts/run_benchmark/param_sweep/run_test_baysor_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_baysor_nebius.sh new file mode 100644 index 000000000..dd1d42b4e --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_baysor_nebius.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# Nebius test run: all default methods + baysor at the transcript-assignment stage, +# with a parameter sweep over baysor's tuning knobs. +# See src/methods_transcript_assignment/baysor/NOTES.md ("Optimization / tuning"). +# +# The stage default (basic_transcript_assignment) stays enabled alongside baysor so the run +# also produces the baseline the sweep is scored against — the workflow allows at most ONE +# non-default variant at a time, and baysor is that one non-default method here; every OTHER +# stage stays on its single default. +# +# PARAMS-FILE CAVEAT: `method_parameters_yaml` is opened by the WORKFLOW at runtime on the cloud +# (readYaml -> Nextflow file()), so a local /tmp path won't exist there and /scratch is read-only +# from the launch host. file() DOES stage http(s):// and this repo is public, so the sweep lives +# in a COMMITTED file read from GitHub via its raw URL => commit AND PUSH +# scripts/run_benchmark/param_sweep/baysor_params.yaml to $params_branch before launching. +# This is independent of --revision (which selects the pipeline CODE). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" +# Results publish to /scratch — created and written by the cloud compute env (read-only here). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_baysor" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch defaults to +# the branch you are on; the file must be pushed there. (Independent of --revision below.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/baysor_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment + - baysor +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/baysor_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,baysor diff --git a/scripts/run_benchmark/param_sweep/run_test_binning_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_binning_nebius.sh index 527e0d3d6..3cab53120 100644 --- a/scripts/run_benchmark/param_sweep/run_test_binning_nebius.sh +++ b/scripts/run_benchmark/param_sweep/run_test_binning_nebius.sh @@ -31,7 +31,7 @@ cd "$REPO_ROOT" set -e -resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" # Results publish to /scratch — created and written by the cloud compute env, so # the launcher does NOT create it here (it is read-only from the launch host). publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_binning" diff --git a/scripts/run_benchmark/param_sweep/run_test_cellpose_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_cellpose_nebius.sh index e3458e9a6..05d49e0bb 100644 --- a/scripts/run_benchmark/param_sweep/run_test_cellpose_nebius.sh +++ b/scripts/run_benchmark/param_sweep/run_test_cellpose_nebius.sh @@ -29,7 +29,7 @@ cd "$REPO_ROOT" set -e -resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" # Results publish to /scratch — created and written by the cloud compute env, so # the launcher does NOT create it here (it is read-only from the launch host). publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_cellpose" diff --git a/scripts/run_benchmark/param_sweep/run_test_cellposev4_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_cellposev4_nebius.sh index 5c092195e..b2877beef 100644 --- a/scripts/run_benchmark/param_sweep/run_test_cellposev4_nebius.sh +++ b/scripts/run_benchmark/param_sweep/run_test_cellposev4_nebius.sh @@ -15,7 +15,7 @@ # publish) is READ-ONLY from the launch host — which is why the binning # method_params block is commented out in run_test_nebius.sh. file() does stage # http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED -# file (scripts/run_benchmark/cellposev4_params.yaml) and read it from GitHub via +# file (scripts/run_benchmark/param_sweep/cellposev4_params.yaml) and read it from GitHub via # its raw URL. => the params file must be committed AND PUSHED to $params_branch # before launching (edit the file there, not here, to change the sweep). @@ -27,7 +27,7 @@ cd "$REPO_ROOT" set -e -resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" # Results publish to /scratch — created and written by the cloud compute env, so # the launcher does NOT create it here (it is read-only from the launch host). publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_cellposev4" @@ -37,7 +37,7 @@ publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_cellposev4" # is independent of --revision below, which selects the pipeline CODE to run.) params_repo="openproblems-bio/task_ist_preprocessing" params_branch="$(git rev-parse --abbrev-ref HEAD)" -params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/cellposev4_params.yaml" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/cellposev4_params.yaml" cat > /tmp/params_settings.yaml << HERE default_methods: @@ -89,7 +89,7 @@ HERE if ! curl -fsSL -o /dev/null "$params_url"; then echo "ERROR: params file not reachable at:" >&2 echo " $params_url" >&2 - echo "Commit and push scripts/run_benchmark/cellposev4_params.yaml to '$params_branch' first." >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/cellposev4_params.yaml to '$params_branch' first." >&2 exit 1 fi diff --git a/scripts/run_benchmark/param_sweep/run_test_clustermap_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_clustermap_nebius.sh new file mode 100644 index 000000000..4183a0b4a --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_clustermap_nebius.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# Nebius test run: all default methods + clustermap at the transcript-assignment stage, +# with a parameter sweep over clustermap's tuning knobs. +# See src/methods_transcript_assignment/clustermap/NOTES.md ("Optimization / tuning"). +# +# The stage default (basic_transcript_assignment) stays enabled alongside clustermap so the run +# also produces the baseline the sweep is scored against — the workflow allows at most ONE +# non-default variant at a time, and clustermap is that one non-default method here; every OTHER +# stage stays on its single default. +# +# PARAMS-FILE CAVEAT: `method_parameters_yaml` is opened by the WORKFLOW at runtime on the cloud +# (readYaml -> Nextflow file()), so a local /tmp path won't exist there and /scratch is read-only +# from the launch host. file() DOES stage http(s):// and this repo is public, so the sweep lives +# in a COMMITTED file read from GitHub via its raw URL => commit AND PUSH +# scripts/run_benchmark/param_sweep/clustermap_params.yaml to $params_branch before launching. +# This is independent of --revision (which selects the pipeline CODE). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" +# Results publish to /scratch — created and written by the cloud compute env (read-only here). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_clustermap" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch defaults to +# the branch you are on; the file must be pushed there. (Independent of --revision below.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/clustermap_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment + - clustermap +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/clustermap_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,clustermap diff --git a/scripts/run_benchmark/param_sweep/run_test_comseg_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_comseg_nebius.sh new file mode 100644 index 000000000..d31e7d018 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_comseg_nebius.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# Nebius test run: all default methods + comseg at the transcript-assignment stage, +# with a parameter sweep over comseg's tuning knobs. +# See src/methods_transcript_assignment/comseg/NOTES.md ("Optimization / tuning"). +# +# The stage default (basic_transcript_assignment) stays enabled alongside comseg so the run +# also produces the baseline the sweep is scored against — the workflow allows at most ONE +# non-default variant at a time, and comseg is that one non-default method here; every OTHER +# stage stays on its single default. +# +# PARAMS-FILE CAVEAT: `method_parameters_yaml` is opened by the WORKFLOW at runtime on the cloud +# (readYaml -> Nextflow file()), so a local /tmp path won't exist there and /scratch is read-only +# from the launch host. file() DOES stage http(s):// and this repo is public, so the sweep lives +# in a COMMITTED file read from GitHub via its raw URL => commit AND PUSH +# scripts/run_benchmark/param_sweep/comseg_params.yaml to $params_branch before launching. +# This is independent of --revision (which selects the pipeline CODE). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" +# Results publish to /scratch — created and written by the cloud compute env (read-only here). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_comseg" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch defaults to +# the branch you are on; the file must be pushed there. (Independent of --revision below.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/comseg_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment + - comseg +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/comseg_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,comseg diff --git a/scripts/run_benchmark/param_sweep/run_test_fastreseg_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_fastreseg_nebius.sh new file mode 100644 index 000000000..acaebb3f7 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_fastreseg_nebius.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# Nebius test run: all default methods + fastreseg at the transcript-assignment stage, +# with a parameter sweep over fastreseg's tuning knobs. +# See src/methods_transcript_assignment/fastreseg/NOTES.md ("Optimization / tuning"). +# +# The stage default (basic_transcript_assignment) stays enabled alongside fastreseg so the run +# also produces the baseline the sweep is scored against — the workflow allows at most ONE +# non-default variant at a time, and fastreseg is that one non-default method here; every OTHER +# stage stays on its single default. +# +# PARAMS-FILE CAVEAT: `method_parameters_yaml` is opened by the WORKFLOW at runtime on the cloud +# (readYaml -> Nextflow file()), so a local /tmp path won't exist there and /scratch is read-only +# from the launch host. file() DOES stage http(s):// and this repo is public, so the sweep lives +# in a COMMITTED file read from GitHub via its raw URL => commit AND PUSH +# scripts/run_benchmark/param_sweep/fastreseg_params.yaml to $params_branch before launching. +# This is independent of --revision (which selects the pipeline CODE). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" +# Results publish to /scratch — created and written by the cloud compute env (read-only here). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_fastreseg" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch defaults to +# the branch you are on; the file must be pushed there. (Independent of --revision below.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/fastreseg_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment + - fastreseg +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/fastreseg_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,fastreseg diff --git a/scripts/run_benchmark/param_sweep/run_test_moscot_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_moscot_nebius.sh index 7319cfb74..27d1a5c6e 100644 --- a/scripts/run_benchmark/param_sweep/run_test_moscot_nebius.sh +++ b/scripts/run_benchmark/param_sweep/run_test_moscot_nebius.sh @@ -36,7 +36,7 @@ cd "$REPO_ROOT" set -e -resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" # Results publish to /scratch — created and written by the cloud compute env, so # the launcher does NOT create it here (it is read-only from the launch host). publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_moscot" diff --git a/scripts/run_benchmark/param_sweep/run_test_pciseq_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_pciseq_nebius.sh new file mode 100644 index 000000000..be965b33c --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_pciseq_nebius.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# Nebius test run: all default methods + pciseq at the transcript-assignment stage, +# with a parameter sweep over pciseq's tuning knobs. +# See src/methods_transcript_assignment/pciseq/NOTES.md ("Optimization / tuning"). +# +# The stage default (basic_transcript_assignment) stays enabled alongside pciseq so the run +# also produces the baseline the sweep is scored against — the workflow allows at most ONE +# non-default variant at a time, and pciseq is that one non-default method here; every OTHER +# stage stays on its single default. +# +# PARAMS-FILE CAVEAT: `method_parameters_yaml` is opened by the WORKFLOW at runtime on the cloud +# (readYaml -> Nextflow file()), so a local /tmp path won't exist there and /scratch is read-only +# from the launch host. file() DOES stage http(s):// and this repo is public, so the sweep lives +# in a COMMITTED file read from GitHub via its raw URL => commit AND PUSH +# scripts/run_benchmark/param_sweep/pciseq_params.yaml to $params_branch before launching. +# This is independent of --revision (which selects the pipeline CODE). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" +# Results publish to /scratch — created and written by the cloud compute env (read-only here). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_pciseq" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch defaults to +# the branch you are on; the file must be pushed there. (Independent of --revision below.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/pciseq_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment + - pciseq +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/pciseq_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,pciseq diff --git a/scripts/run_benchmark/param_sweep/run_test_proseg_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_proseg_nebius.sh new file mode 100644 index 000000000..bbcff5c34 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_proseg_nebius.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# Nebius test run: all default methods + proseg at the transcript-assignment stage, +# with a parameter sweep over proseg's tuning knobs. +# See src/methods_transcript_assignment/proseg/NOTES.md ("Optimization / tuning"). +# +# The stage default (basic_transcript_assignment) stays enabled alongside proseg so the run +# also produces the baseline the sweep is scored against — the workflow allows at most ONE +# non-default variant at a time, and proseg is that one non-default method here; every OTHER +# stage stays on its single default. +# +# PARAMS-FILE CAVEAT: `method_parameters_yaml` is opened by the WORKFLOW at runtime on the cloud +# (readYaml -> Nextflow file()), so a local /tmp path won't exist there and /scratch is read-only +# from the launch host. file() DOES stage http(s):// and this repo is public, so the sweep lives +# in a COMMITTED file read from GitHub via its raw URL => commit AND PUSH +# scripts/run_benchmark/param_sweep/proseg_params.yaml to $params_branch before launching. +# This is independent of --revision (which selects the pipeline CODE). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" +# Results publish to /scratch — created and written by the cloud compute env (read-only here). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_proseg" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch defaults to +# the branch you are on; the file must be pushed there. (Independent of --revision below.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/proseg_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment + - proseg +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/proseg_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,proseg diff --git a/scripts/run_benchmark/param_sweep/run_test_rctd_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_rctd_nebius.sh index 438c3a14e..5144b1d94 100644 --- a/scripts/run_benchmark/param_sweep/run_test_rctd_nebius.sh +++ b/scripts/run_benchmark/param_sweep/run_test_rctd_nebius.sh @@ -34,7 +34,7 @@ cd "$REPO_ROOT" set -e -resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" # Results publish to /scratch — created and written by the cloud compute env, so # the launcher does NOT create it here (it is read-only from the launch host). publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_rctd" diff --git a/scripts/run_benchmark/param_sweep/run_test_resolvi_correction_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_resolvi_correction_nebius.sh index a96ab96f0..f09969f85 100644 --- a/scripts/run_benchmark/param_sweep/run_test_resolvi_correction_nebius.sh +++ b/scripts/run_benchmark/param_sweep/run_test_resolvi_correction_nebius.sh @@ -29,7 +29,7 @@ cd "$REPO_ROOT" set -e -resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" # Results publish to /scratch — created and written by the cloud compute env, so the launcher # does NOT create it here (it is read-only from the launch host). publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_resolvi_correction" diff --git a/scripts/run_benchmark/param_sweep/run_test_segger_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_segger_nebius.sh new file mode 100644 index 000000000..6763f1df3 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_segger_nebius.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +# Nebius test run: all default methods + segger at the transcript-assignment stage, +# with a parameter sweep over segger's tuning knobs. +# See src/methods_transcript_assignment/segger/NOTES.md ("Optimization / tuning"). +# segger is GPU-only: the `gpuh100` label in its config + src/base/labels_nebius.config +# (runAsUser:0 + /dev/shm volume) pin it to the GPU node group — no GPU-specific change is +# needed here (same as the existing run_test_segger_nebius.sh). +# +# The stage default (basic_transcript_assignment) stays enabled alongside segger so the run +# also produces the baseline the sweep is scored against — the workflow allows at most ONE +# non-default variant at a time, and segger is that one non-default method here; every OTHER +# stage stays on its single default. +# +# PARAMS-FILE CAVEAT: `method_parameters_yaml` is opened by the WORKFLOW at runtime on the cloud +# (readYaml -> Nextflow file()), so a local /tmp path won't exist there and /scratch is read-only +# from the launch host. file() DOES stage http(s):// and this repo is public, so the sweep lives +# in a COMMITTED file read from GitHub via its raw URL => commit AND PUSH +# scripts/run_benchmark/param_sweep/segger_params.yaml to $params_branch before launching. +# This is independent of --revision (which selects the pipeline CODE). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" +# Results publish to /scratch — created and written by the cloud compute env (read-only here). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_segger" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch defaults to +# the branch you are on; the file must be pushed there. (Independent of --revision below.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/segger_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment + - segger +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/segger_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,segger diff --git a/scripts/run_benchmark/param_sweep/run_test_singler_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_singler_nebius.sh new file mode 100644 index 000000000..db0323fe7 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_singler_nebius.sh @@ -0,0 +1,113 @@ +#!/bin/bash + +# Nebius test run: all default methods + SingleR (singler) cell-type annotation, +# with a parameter sweep over the one exposed optimization lever for SingleR. +# See src/methods_cell_type_annotation/singler/NOTES.md ("Optimization / tuning"). +# +# SingleR (singler-py) is a CPU-only reference-correlation labeller — NO GPU, so +# there is no `gpu` label here (the non-GPU parts mirror run_test_nebius.sh). The +# compute env is the same one all these test runs use. +# +# The annotation stage keeps its default method `tacco` alongside `singler` so the +# run has a baseline to compare the singler variants against. Every other stage is +# left on its single default (the swept method must be the only non-default thing). +# +# !!! SUBMITTABLE-BUT-NO-OP CAVEAT !!! +# The sweep varies `celltype_key`, which ALREADY EXISTS in build/main's config, so +# this launches WITHOUT a rebuild. BUT build/main's baked-in script.py hardcodes the +# reference label column (ref_labels = ...column("cell_type")) and never reads +# par['celltype_key'] -- so on the deployed build/main container all four variants +# produce IDENTICAL annotations. The sweep only measures anything after a one-line +# wiring fix (...column(par["celltype_key"])) AND a container rebuild. See NOTES.md. +# +# PARAMS-FILE CAVEAT (why this reads from GitHub): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host. file() does stage http(s):// though, +# and this repo is public, so we keep the sweep in a COMMITTED file +# (scripts/run_benchmark/param_sweep/singler_params.yaml) and read it from GitHub +# via its raw URL. => the params file must be committed AND PUSHED to +# $params_branch before launching (edit the file there, not here, to change the +# sweep). This is independent of --revision below, which selects the pipeline CODE. + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_singler" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/singler_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco + - singler +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/singler_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,singler diff --git a/scripts/run_benchmark/param_sweep/run_test_split_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_split_nebius.sh index 0a20da769..bb63ff97b 100644 --- a/scripts/run_benchmark/param_sweep/run_test_split_nebius.sh +++ b/scripts/run_benchmark/param_sweep/run_test_split_nebius.sh @@ -37,7 +37,7 @@ cd "$REPO_ROOT" set -e -resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" # Results publish to /scratch — created and written by the cloud compute env, so # the launcher does NOT create it here (it is read-only from the launch host). publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_split" diff --git a/scripts/run_benchmark/param_sweep/run_test_ssam_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_ssam_nebius.sh index 7ac988081..6d0517717 100644 --- a/scripts/run_benchmark/param_sweep/run_test_ssam_nebius.sh +++ b/scripts/run_benchmark/param_sweep/run_test_ssam_nebius.sh @@ -30,7 +30,7 @@ cd "$REPO_ROOT" set -e -resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" # Results publish to /scratch -- created and written by the cloud compute env, so # the launcher does NOT create it here (it is read-only from the launch host). publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_ssam" diff --git a/scripts/run_benchmark/param_sweep/run_test_stardist_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_stardist_nebius.sh index 21d22c6bc..31d0d6765 100644 --- a/scripts/run_benchmark/param_sweep/run_test_stardist_nebius.sh +++ b/scripts/run_benchmark/param_sweep/run_test_stardist_nebius.sh @@ -27,7 +27,7 @@ cd "$REPO_ROOT" set -e -resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" # Results publish to /scratch — created and written by the cloud compute env, so # the launcher does NOT create it here (it is read-only from the launch host). publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_stardist" diff --git a/scripts/run_benchmark/param_sweep/run_test_tangram_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_tangram_nebius.sh index be7c40618..a9ddf4d95 100644 --- a/scripts/run_benchmark/param_sweep/run_test_tangram_nebius.sh +++ b/scripts/run_benchmark/param_sweep/run_test_tangram_nebius.sh @@ -30,7 +30,7 @@ cd "$REPO_ROOT" set -e -resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" # Results publish to /scratch — created and written by the cloud compute env, so # the launcher does NOT create it here (it is read-only from the launch host). publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_tangram" diff --git a/scripts/run_benchmark/param_sweep/run_test_watershed_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_watershed_nebius.sh index 657635223..4118a4e0c 100644 --- a/scripts/run_benchmark/param_sweep/run_test_watershed_nebius.sh +++ b/scripts/run_benchmark/param_sweep/run_test_watershed_nebius.sh @@ -27,7 +27,7 @@ cd "$REPO_ROOT" set -e -resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/" # Results publish to /scratch — created and written by the cloud compute env, so # the launcher does NOT create it here (it is read-only from the launch host). publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_watershed" diff --git a/src/api/comp_data_preprocessor.yaml b/src/api/comp_data_preprocessor.yaml index ddc535078..25b26c512 100644 --- a/src/api/comp_data_preprocessor.yaml +++ b/src/api/comp_data_preprocessor.yaml @@ -61,3 +61,14 @@ argument_groups: type: string description: The organism of the sample in the dataset. required: false + - name: Processing options + arguments: + - name: "--tissue_centered_crop" + type: boolean + default: false + description: | + When cropping an oversized image, centre the crop window on the transcript + density (median global x/y) instead of the image centre. Needed for huge + whole-section canvases where the tissue sits well off-centre (e.g. ABCA + whole-brain MERFISH), for which an image-centred window can miss the tissue + entirely. Default false keeps the historical image-centred crop. diff --git a/src/data_processors/process_dataset/script.py b/src/data_processors/process_dataset/script.py index 31b3ce1f8..b311e70db 100644 --- a/src/data_processors/process_dataset/script.py +++ b/src/data_processors/process_dataset/script.py @@ -31,7 +31,8 @@ "dataset_reference": "https://www.10xgenomics.com/datasets/fresh-frozen-mouse-brain-replicates-1-standard;10.1038/s41586-023-06812-z", "dataset_summary": "Demonstration of gene expression profiling for fresh frozen mouse brain on the Xenium platform using the pre-designed Mouse Brain Gene Expression Panel (v1);A high-resolution scRNAseq atlas of cell types in the whole mouse brain", "dataset_description": "Demonstration of gene expression profiling for fresh frozen mouse brain on the Xenium platform using the pre-designed Mouse Brain Gene Expression Panel (v1). Replicate results demonstrate the high reproducibility of data generated by the platform. 10x Genomics obtained tissue from a C57BL/6 mouse from Charles River Laboratories. Three adjacent 10µm sections were placed on the same slide. Tissues were prepared following the demonstrated protocols Xenium In Situ for Fresh Frozen Tissues - Tissue Preparation Guide (CG000579) and Xenium In Situ for Fresh Frozen Tissues - Fixation & Permeabilization (CG000581).;See dataset_reference for more information. Note that we only took the 10xv2 data from the dataset.", - "dataset_organism": "mus_musculus" + "dataset_organism": "mus_musculus", + "tissue_centered_crop": False } ### VIASH END @@ -50,7 +51,28 @@ def _full_res_image(img): return img["scale0"].image -def get_crop_coords(sdata, max_n_pixels=20000*20000): #50000*50000): +def _transcript_center_global(sdata): + """Median (x, y) of the transcripts in the GLOBAL coordinate system. + + Used to centre the crop window on the tissue rather than on the image centre. + Global coords are derived from the element's affine transform (same math as + crop_points_by_global_xy), so it is correct for Scale/Affine/identity alike, and + the median is robust to sparse background points scattered at the canvas edges. + """ + import dask + from spatialdata.transformations import get_transformation + pts = sdata['transcripts'] + axes = [a for a in ("x", "y", "z") if a in pts.columns] + trans = get_transformation(pts, get_all=True) + M = trans["global"].to_affine_matrix(input_axes=tuple(axes), output_axes=tuple(axes)) + ix, iy = axes.index("x"), axes.index("y") + gx = sum(M[ix, i] * pts[a] for i, a in enumerate(axes)) + M[ix, len(axes)] + gy = sum(M[iy, i] * pts[a] for i, a in enumerate(axes)) + M[iy, len(axes)] + cx, cy = dask.compute(gx.quantile(0.5), gy.quantile(0.5)) + return float(cx), float(cy) + + +def get_crop_coords(sdata, max_n_pixels=20000*20000, tissue_centered=False): #50000*50000): """Get the crop coordinates to subset the sdata to max_n_pixels Arguments @@ -84,10 +106,24 @@ def get_crop_coords(sdata, max_n_pixels=20000*20000): #50000*50000): w_crop = w h_crop = int(max_n_pixels / w_crop) - # Center the crop - h_offset = (h - h_crop) // 2 - w_offset = (w - w_crop) // 2 - + # Position the crop window. By default it is centred on the IMAGE (historical + # behaviour). With tissue_centered=True it is centred on the TRANSCRIPT DENSITY + # instead — needed for huge whole-section canvases where the tissue sits well + # off-centre (e.g. ABCA whole-brain MERFISH, 83361x102170 px: an image-centred + # 20000px window caught 0 of 42M transcripts). The window is clamped inside the + # image; a degenerate/unavailable transcript centre falls back to the image centre. + if tissue_centered: + try: + cx, cy = _transcript_center_global(sdata) + if not (np.isfinite(cx) and np.isfinite(cy)): + cx, cy = w / 2, h / 2 + except Exception: + cx, cy = w / 2, h / 2 + else: + cx, cy = w / 2, h / 2 + w_offset = int(min(max(cx - w_crop / 2, 0), max(w - w_crop, 0))) + h_offset = int(min(max(cy - h_crop / 2, 0), max(h - h_crop, 0))) + crop = [[h_offset, h_offset + h_crop], [w_offset, w_offset + w_crop]] return crop @@ -319,7 +355,7 @@ def subsample_adata_group_balanced(adata, group_key, n_samples, seed=0): _tbl.obs.index.name = None # Crop datasets that are too large -crop_coords = get_crop_coords(sdata) +crop_coords = get_crop_coords(sdata, tissue_centered=par.get("tissue_centered_crop", False)) if crop_coords is not None: (y0, y1), (x0, x1) = crop_coords # global box (raw image is at identity -> px == global) diff --git a/src/methods_segmentation/stardist/NOTES.md b/src/methods_segmentation/stardist/NOTES.md index 0121111b1..d41b06d23 100644 --- a/src/methods_segmentation/stardist/NOTES.md +++ b/src/methods_segmentation/stardist/NOTES.md @@ -46,22 +46,33 @@ detector**: the script feeds it `image[0]` only (see Tier 0 below). 3. **Load model** (`:59`) — `StarDist2D.from_pretrained(par['model'])`. Default `2D_versatile_fluo`. This also loads the model's optimized `thresholds.json` (prob=0.479071, nms=0.3 for that model) as the fallback thresholds. -4. **Percentile normalizer** (`:64-79`) — a csbdeep `Normalizer` subclass that min-max +4. **Percentile normalizer** (`:64-77`) — a csbdeep `Normalizer` subclass that min-max scales the image to its **1st / 99.8th percentiles** (`normalize_mi_ma`), the recommended StarDist preprocessing but with fixed percentile bounds. `block_size` - and `offset` (block overlap/context) are derived from the image width so a large - panel is processed in tiles. + and `context` are derived from the image width so a large panel is processed in + tiles; **`min_overlap` is derived from object size, not block size** (see step 6 and + the min_overlap gotcha below). 5. **Build eval-params** (`:85-89`) — collects the newly exposed tunables (`prob_thresh, nms_thresh, scale`) from `par`, **dropping any that are `None`**. A dropped key ⇒ `predict_instances` uses the model's own optimized value (so the no-args call is byte-for-byte the pre-tuning behaviour). Mirrors the cellposev4 eval-params pattern. -6. **Segment** (`:92-95`) — `model.predict_instances_big(image[0], axes='YX', +6. **Segment** (`:92-131`) — `model.predict_instances_big(image[0], axes='YX', block_size=…, min_overlap=…, context=…, normalizer=…, **eval_params)`. `predict_instances_big` splits the image into `block_size` blocks, calls `predict_instances` on each (forwarding `**eval_params` unchanged — it only overrides `axes/overlap_label/return_labels/return_predict`), and reassembles the labels into global coordinates. `image[0]` = first channel → a single 2D plane. + **The stitching invariant is that every predicted object is smaller than + `min_overlap`** (an object bigger than the overlap can span a block seam and can't be + uniquely assigned → `RuntimeError: ...violates the assumption of being smaller than + 'min_overlap'`). So `min_overlap` is set from an **object-size** bound + (`max_object_diameter`, default **192 px**), *not* from `block_size` (the old + `block_size // 5.5` shrank it to 64 px on small panels while real blobs reached + ~110 px → crash). `block_size` is then grown if needed to satisfy + `min_overlap + 2*context < block_size`, and the call is wrapped in a **retry that + doubles `min_overlap` on that specific error** so a rare oversized blob self-heals + instead of failing the run. 7. **Post-process** (`:100-104`) — `convert_to_lower_dtype` downcasts the label array to the smallest uint that holds `max label`; wrap as an `xarray.DataArray`, `Labels2DModel.parse` with the copied transform, store as @@ -109,6 +120,7 @@ Base image `openproblems/base_tensorflow_nvidia:1` (GPU-capable TF). The pip pin | `--prob_thresh` | double | *(unset → model 0.479071)* | `predict_instances(prob_thresh=)` | | `--nms_thresh` | double | *(unset → model 0.3)* | `predict_instances(nms_thresh=)` | | `--scale` | double | *(unset → no rescale)* | `predict_instances(scale=)` | +| `--max_object_diameter` | integer | *(unset → 192 px)* | `predict_instances_big(min_overlap=)` | `prob_thresh`, `nms_thresh`, `scale` were **added during tuning** (see below). They are **optional with no static default on purpose**: leaving them unset makes StarDist use diff --git a/src/methods_segmentation/stardist/config.vsh.yaml b/src/methods_segmentation/stardist/config.vsh.yaml index cba9c2272..13e153995 100644 --- a/src/methods_segmentation/stardist/config.vsh.yaml +++ b/src/methods_segmentation/stardist/config.vsh.yaml @@ -47,6 +47,17 @@ arguments: If unset, no rescaling. Use it to match nucleus size to the model's training size: >1 upscales (small nuclei appear larger), <1 downscales. StarDist's analogue of Cellpose's diameter. + - name: --max_object_diameter + type: integer + required: false + description: >- + Upper bound (in original image pixels) on the largest object StarDist may produce. + It sets predict_instances_big's `min_overlap` (the guaranteed block-stitching + overlap), which MUST exceed the biggest nucleus/blob — otherwise stitching fails + with "violates the assumption of being smaller than 'min_overlap'". Defaults to + 192 px if unset, and the script auto-doubles it on demand, so this is a + robustness/geometry knob, NOT a quality knob: it does not change which pixels get + segmented. Raise it if you hit that error or your nuclei are unusually large. resources: - type: python_script diff --git a/src/methods_segmentation/stardist/script.py b/src/methods_segmentation/stardist/script.py index 4bf502c61..4014f2772 100644 --- a/src/methods_segmentation/stardist/script.py +++ b/src/methods_segmentation/stardist/script.py @@ -43,6 +43,7 @@ def convert_to_lower_dtype(arr): "prob_thresh": None, "nms_thresh": None, "scale": None, + "max_object_diameter": None, } ## VIASH END @@ -75,8 +76,6 @@ def do_after(self): mi, ma = np.percentile(image, [1,99.8]) normalizer = MyNormalizer(mi, ma) -block_size = min(image.shape[1] // 3, 4096) -offset = min(block_size // 5.5, 128) # Tunable knobs forwarded through predict_instances_big -> predict_instances. # A value left as None (i.e. omitted from par) means "use the model's own optimized @@ -89,10 +88,44 @@ def do_after(self): } print(f"predict_instances_big overrides: {eval_params}", flush=True) -labels, _ = model.predict_instances_big( - image[0,:,:], axes='YX', block_size=block_size, min_overlap=offset, - context=offset, normalizer=normalizer, **eval_params # n_tiles left to block_size -) +# predict_instances_big tiles the image and stitches the per-block predictions. Its +# stitching invariant is that EVERY predicted object is smaller than `min_overlap`; +# an object spanning a block seam that is larger than the overlap can't be assigned to +# a single block, which raises "Found object of shape (...), which violates the +# assumption of being smaller than 'min_overlap'". So `min_overlap` must be +# OBJECT-SIZE-based, not block-size-based — the old `block_size // 5.5` shrank the +# overlap below real nuclei/blobs on small panels (min_overlap fell to 64 px while +# objects reached ~110 px). Objects are measured in ORIGINAL image pixels +# (predict_instances undoes `scale` internally), so this bound is in original px and is +# independent of `scale`. `context` is only the receptive-field margin discarded around +# each block, so deriving it from the image size is fine. +block_size = min(image.shape[1] // 3, 4096) +context = int(min(block_size // 5.5, 128)) +min_overlap = int(par.get("max_object_diameter") or 192) # px; must exceed largest object +# predict_instances_big asserts: min_overlap + 2*context < block_size. +block_size = max(block_size, min_overlap + 2 * context + 1) + +# Self-heal: if a rare oversized blob (merged nuclei / debris) still exceeds +# `min_overlap`, double it (and grow block_size to keep the geometry constraint) and +# retry, rather than failing the whole segmentation. +while True: + try: + labels, _ = model.predict_instances_big( + image[0, :, :], axes='YX', block_size=block_size, + min_overlap=min_overlap, context=context, + normalizer=normalizer, **eval_params, # n_tiles left to block_size + ) + break + except RuntimeError as e: + if "min_overlap" not in str(e) or min_overlap >= 2048: + raise + min_overlap *= 2 + block_size = max(block_size, min_overlap + 2 * context + 1) + print( + "predict_instances_big: an object exceeded min_overlap; retrying with " + f"min_overlap={min_overlap}, block_size={block_size}", + flush=True, + ) diff --git a/src/methods_transcript_assignment/segger/NOTES.md b/src/methods_transcript_assignment/segger/NOTES.md index 0ce302627..f3a1e138e 100644 --- a/src/methods_transcript_assignment/segger/NOTES.md +++ b/src/methods_transcript_assignment/segger/NOTES.md @@ -218,6 +218,24 @@ Nextflow runner labels: `[hightime, midcpu, highmem, gpuh100]`. ## Risk points / gotchas +- **Never start a continuation line in `script.py` with `|` — viash's Nextflow codegen + deletes it (Groovy `stripMargin`).** viash embeds `script.py` into the Nextflow module + (`target/nextflow/.../segger/main.nf`) as a Groovy string and runs it through + `.stripMargin()`, whose **default margin delimiter is `|`**: any line matching + `^\s*\|` has the leading whitespace **and the `|`** stripped. A wrapped boolean/bitwise + expression like + `n_oob = int(np.count_nonzero(\n (y_coords < 0) | (y_coords >= H)\n | (x_coords < 0) | (x_coords >= W)\n))` + becomes `... (y_coords >= H)\n (x_coords < 0) ...` in the generated module — the two + parenthesised groups then **juxtapose into a call**, so at run time Python raises + `TypeError: 'numpy.ndarray' object is not callable` (the caret points at the *first* + group, the "callable"). **This corrupts ONLY the `=nextflow=>` target**, so `viash test` / + `viash run` (the `=executable=>` target, no `stripMargin`) pass clean and the bug is + invisible until a Nextflow/Tower benchmark run. It also survives across builds: the + `build/main` executable can be correct while the `build/main` *nextflow module* is + corrupt. **Fix = put the operator at the END of the line** (`... (y_coords >= H) |` / + next line `(x_coords < 0) ...`) so no line begins with `|`; or keep it on one line. + Grep guard before committing: `grep -nE '^[[:space:]]*\|' script.py` must be empty. + (Diagnosed 2026-08-06: reproduced byte-for-byte, incl. the exact caret placement.) - **numba must use the NVIDIA (cuda-python) driver binding, or cudf/numba see 0 GPUs.** The load-bearing GPU fix: `run_segger` sets **`NUMBA_CUDA_USE_NVIDIA_BINDING=1`** for the subprocess. RAPIDS (cudf/cuml/cugraph) builds its CUDA context through the cuda-python diff --git a/src/methods_transcript_assignment/segger/script.py b/src/methods_transcript_assignment/segger/script.py index 71e1eb29d..69c0de530 100644 --- a/src/methods_transcript_assignment/segger/script.py +++ b/src/methods_transcript_assignment/segger/script.py @@ -173,9 +173,16 @@ # rounding is all that remains, and it's handled below. NOTE: this does NOT touch the separate # empty-`bd`-batch crash (a cell-free INTERIOR tile), which is intra-field, not OOB-driven, and # still needs the segger-encoder guard -- see NOTES.md. +# NB: keep the `|` operators at the END of each line -- a continuation line must NOT start +# with `|`. viash embeds this script into the Nextflow module as a Groovy string and runs it +# through `.stripMargin()`, which treats a leading `|` as the margin delimiter and DELETES it. +# A leading-`|` line like `| (x_coords < 0) ...` becomes ` (x_coords < 0) ...`, so the previous +# line's `(...)` and this line's `(...)` juxtapose into a call -> the Nextflow run dies with +# `TypeError: 'numpy.ndarray' object is not callable` while `viash test`/`viash run` (the +# executable target, no stripMargin) pass. Trailing operators sidestep it entirely. n_oob = int(np.count_nonzero( - (y_coords < 0) | (y_coords >= label_image.shape[0]) - | (x_coords < 0) | (x_coords >= label_image.shape[1]) + (y_coords < 0) | (y_coords >= label_image.shape[0]) | + (x_coords < 0) | (x_coords >= label_image.shape[1]) )) y_coords = np.clip(y_coords, 0, label_image.shape[0] - 1) x_coords = np.clip(x_coords, 0, label_image.shape[1] - 1) diff --git a/src/workflows/process_datasets/main.nf b/src/workflows/process_datasets/main.nf index 2bfc4c3f9..70948ecf8 100644 --- a/src/workflows/process_datasets/main.nf +++ b/src/workflows/process_datasets/main.nf @@ -22,7 +22,8 @@ workflow run_wf { "dataset_reference", "dataset_summary", "dataset_description", - "dataset_organism" + "dataset_organism", + "tissue_centered_crop" ], toState: [ "output_sc",