Fixes

scripts/create_resources/combine/process_datasets_allen_nebius.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+
+# Process ONLY the Allen Brain Cell Atlas (ABCA) MERFISH datasets (combine step) — FOUR
+# whole-brain sections, ALL paired with the SAME mouse-brain SC reference (like the LTX/MPII
+# combine scripts, and unlike Kuppe which uses condition-specific references):
+#   mouse1_coronal, mouse2_coronal, mouse3_sagittal, mouse4_sagittal
+#     <-> allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2 (ABCA 2023 Yao 10xv2 atlas)
+#
+# Reads each spatial input (process_allen_brain_cell_atlas_merfish loader output) and the SC
+# reference from the local /scratch raw folder, and writes the combined datasets to the
+# /scratch datasets folder (same layout the other *_nebius.sh combine scripts use).
+#
+# Prerequisites (both publish to the same /scratch raw folder used below):
+#   1. scripts/create_resources/spatial/process_allen_brain_cell_atlas_merfish_nebius.sh
+#        -> /scratch/.../raw/allen_brain_cell_atlas_merfish/mouse{1,2,3,4}_*/rep1/dataset.zarr
+#   2. scripts/create_resources/sc/process_allen_brain_cell_atlas_brain_nebius.sh  (log_cp -> hvg
+#      -> pca -> knn, i.e. it must carry a 'normalized' layer)
+#        -> /scratch/.../raw/allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2/dataset.h5ad
+
+# get the root of the directory
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+# ensure that the command below is run from the root of the repository
+cd "$REPO_ROOT"
+
+set -e
+
+# process_allen_brain_cell_atlas_merfish_nebius.sh + process_allen_brain_cell_atlas_brain_nebius.sh
+# both publish here.
+raw_dir='/scratch/task_ist_preprocessing/raw'
+publish_dir='/scratch/task_ist_preprocessing/datasets'
+
+# shared SC reference for all four sections (ABCA 2023 Yao whole-brain 10xv2 atlas)
+sc_ref="$raw_dir/allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2/dataset.h5ad"
+
+launch_batch() {
+  local params_file="$1"
+  local label="$2"
+  tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
+    --revision build/main \
+    --pull-latest \
+    --main-script target/nextflow/workflows/process_datasets/main.nf \
+    --workspace 167877437119966 \
+    --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \
+    --params-file "$params_file" \
+    --config src/base/labels_nebius.config \
+    --labels "task_ist_preprocessing,process_datasets,$label"
+}
+
+cat > /tmp/params_allen.yaml << HERE
+param_list:
+
+  - id: "allen_brain_cell_atlas_merfish_combined/mouse1_coronal/rep1"
+    input_sp: "$raw_dir/allen_brain_cell_atlas_merfish/mouse1_coronal/rep1/dataset.zarr"
+    input_sc: "$sc_ref"
+    dataset_id: "allen_brain_cell_atlas_merfish_combined/mouse1_coronal/rep1"
+    dataset_name: "Mouse brain combined ABCA MERFISH mouse1 coronal + 2023 Yao scRNAseq"
+    dataset_url: "https://download.brainimagelibrary.org/29/3c/293cc39ceea87f6d/"
+    dataset_reference: "10.1038/s41586-023-06812-z"
+    dataset_summary: "Allen Brain Cell Atlas whole-brain MERFISH mouse 1 (coronal, ~1100-gene panel) + 2023 Yao mouse-brain scRNAseq"
+    dataset_description: "Brain-wide MERFISH spatial transcriptomics (Zhuang lab, Allen Brain Cell Atlas), mouse 1 coronal section, paired with the ABCA 2023 Yao whole-mouse-brain 10xv2 scRNAseq reference."
+    dataset_organism: "mus_musculus"
+
+  - id: "allen_brain_cell_atlas_merfish_combined/mouse2_coronal/rep1"
+    input_sp: "$raw_dir/allen_brain_cell_atlas_merfish/mouse2_coronal/rep1/dataset.zarr"
+    input_sc: "$sc_ref"
+    dataset_id: "allen_brain_cell_atlas_merfish_combined/mouse2_coronal/rep1"
+    dataset_name: "Mouse brain combined ABCA MERFISH mouse2 coronal + 2023 Yao scRNAseq"
+    dataset_url: "https://download.brainimagelibrary.org/29/3c/293cc39ceea87f6d/"
+    dataset_reference: "10.1038/s41586-023-06812-z"
+    dataset_summary: "Allen Brain Cell Atlas whole-brain MERFISH mouse 2 (coronal, ~1100-gene panel) + 2023 Yao mouse-brain scRNAseq"
+    dataset_description: "Brain-wide MERFISH spatial transcriptomics (Zhuang lab, Allen Brain Cell Atlas), mouse 2 coronal section, paired with the ABCA 2023 Yao whole-mouse-brain 10xv2 scRNAseq reference."
+    dataset_organism: "mus_musculus"
+
+  - id: "allen_brain_cell_atlas_merfish_combined/mouse3_sagittal/rep1"
+    input_sp: "$raw_dir/allen_brain_cell_atlas_merfish/mouse3_sagittal/rep1/dataset.zarr"
+    input_sc: "$sc_ref"
+    dataset_id: "allen_brain_cell_atlas_merfish_combined/mouse3_sagittal/rep1"
+    dataset_name: "Mouse brain combined ABCA MERFISH mouse3 sagittal + 2023 Yao scRNAseq"
+    dataset_url: "https://download.brainimagelibrary.org/29/3c/293cc39ceea87f6d/"
+    dataset_reference: "10.1038/s41586-023-06812-z"
+    dataset_summary: "Allen Brain Cell Atlas whole-brain MERFISH mouse 3 (sagittal, ~1100-gene panel) + 2023 Yao mouse-brain scRNAseq"
+    dataset_description: "Brain-wide MERFISH spatial transcriptomics (Zhuang lab, Allen Brain Cell Atlas), mouse 3 sagittal section, paired with the ABCA 2023 Yao whole-mouse-brain 10xv2 scRNAseq reference."
+    dataset_organism: "mus_musculus"
+
+  - id: "allen_brain_cell_atlas_merfish_combined/mouse4_sagittal/rep1"
+    input_sp: "$raw_dir/allen_brain_cell_atlas_merfish/mouse4_sagittal/rep1/dataset.zarr"
+    input_sc: "$sc_ref"
+    dataset_id: "allen_brain_cell_atlas_merfish_combined/mouse4_sagittal/rep1"
+    dataset_name: "Mouse brain combined ABCA MERFISH mouse4 sagittal + 2023 Yao scRNAseq"
+    dataset_url: "https://download.brainimagelibrary.org/29/3c/293cc39ceea87f6d/"
+    dataset_reference: "10.1038/s41586-023-06812-z"
+    dataset_summary: "Allen Brain Cell Atlas whole-brain MERFISH mouse 4 (sagittal, ~1100-gene panel) + 2023 Yao mouse-brain scRNAseq"
+    dataset_description: "Brain-wide MERFISH spatial transcriptomics (Zhuang lab, Allen Brain Cell Atlas), mouse 4 sagittal section, paired with the ABCA 2023 Yao whole-mouse-brain 10xv2 scRNAseq reference."
+    dataset_organism: "mus_musculus"
+
+# ABCA whole-brain images are enormous (~83k x 102k px) and the tissue sits off-centre,
+# so an image-centred crop can miss it (mouse1: 0 of 42M transcripts). Centre the crop
+# on the transcript density instead (opt-in flag; default is image-centred).
+tissue_centered_crop: true
+
+output_sc: "\$id/output_sc.h5ad"
+output_sp: "\$id/output_sp.zarr"
+output_state: "\$id/state.yaml"
+publish_dir: "$publish_dir"
+HERE
+
+launch_batch /tmp/params_allen.yaml "allen_brain_cell_atlas_merfish"

scripts/run_benchmark/param_sweep/run_test_baysor_nebius.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+
+# Nebius test run: all default methods + baysor at the transcript-assignment stage,
+# with a parameter sweep over baysor's tuning knobs.
+# See src/methods_transcript_assignment/baysor/NOTES.md ("Optimization / tuning").
+#
+# The stage default (basic_transcript_assignment) stays enabled alongside baysor so the run
+# also produces the baseline the sweep is scored against — the workflow allows at most ONE
+# non-default variant at a time, and baysor is that one non-default method here; every OTHER
+# stage stays on its single default.
+#
+# PARAMS-FILE CAVEAT: `method_parameters_yaml` is opened by the WORKFLOW at runtime on the cloud
+# (readYaml -> Nextflow file()), so a local /tmp path won't exist there and /scratch is read-only
+# from the launch host. file() DOES stage http(s):// and this repo is public, so the sweep lives
+# in a COMMITTED file read from GitHub via its raw URL => commit AND PUSH
+# scripts/run_benchmark/param_sweep/baysor_params.yaml to $params_branch before launching.
+# This is independent of --revision (which selects the pipeline CODE).
+
+# get the root of the directory
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+# ensure that the command below is run from the root of the repository
+cd "$REPO_ROOT"
+
+set -e
+
+resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/"
+# Results publish to /scratch — created and written by the cloud compute env (read-only here).
+publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_baysor"
+
+# The sweep lives in a committed file, read from GitHub at runtime. $params_branch defaults to
+# the branch you are on; the file must be pushed there. (Independent of --revision below.)
+params_repo="openproblems-bio/task_ist_preprocessing"
+params_branch="$(git rev-parse --abbrev-ref HEAD)"
+params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/baysor_params.yaml"
+
+cat > /tmp/params_settings.yaml << HERE
+default_methods:
+  - custom_segmentation
+  - basic_transcript_assignment
+  - basic_count_aggregation
+  - basic_qc_filter
+  - alpha_shapes
+  - normalize_by_volume
+  - tacco
+  - no_correction
+segmentation_methods:
+  - custom_segmentation
+transcript_assignment_methods:
+  - basic_transcript_assignment
+  - baysor
+count_aggregation_methods:
+  - basic_count_aggregation
+qc_filtering_methods:
+  - basic_qc_filter
+volume_calculation_methods:
+  - alpha_shapes
+normalization_methods:
+  - normalize_by_volume
+celltype_annotation_methods:
+  - tacco
+expression_correction_methods:
+  - no_correction
+gene_efficiency_correction_methods:
+  - no_correction
+method_parameters_yaml: $params_url
+HERE
+
+# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this)
+cat > /tmp/params.yaml << HERE
+input_states: $resources_test_s3/**/state.yaml
+rename_keys: 'input_sc:output_sc;input_sp:output_sp'
+save_spatial_data: false
+settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)'
+output_state: "state.yaml"
+publish_dir: "$publish_dir"
+HERE
+
+# Fail early with a clear message if the params file isn't reachable on GitHub yet.
+if ! curl -fsSL -o /dev/null "$params_url"; then
+  echo "ERROR: params file not reachable at:" >&2
+  echo "  $params_url" >&2
+  echo "Commit and push scripts/run_benchmark/param_sweep/baysor_params.yaml to '$params_branch' first." >&2
+  exit 1
+fi
+
+tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
+  --revision build/main \
+  --pull-latest \
+  --main-script target/nextflow/workflows/run_benchmark/main.nf \
+  --workspace 167877437119966 \
+  --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \
+  --params-file /tmp/params.yaml \
+  --entry-name auto \
+  --config src/base/labels_nebius.config \
+  --labels task_ist_preprocessing,test,baysor

scripts/run_benchmark/param_sweep/run_test_binning_nebius.sh

@@ -31,7 +31,7 @@ cd "$REPO_ROOT"
 
 set -e
 
-resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing
+resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/"
 # Results publish to /scratch — created and written by the cloud compute env, so
 # the launcher does NOT create it here (it is read-only from the launch host).
 publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_binning"

scripts/run_benchmark/param_sweep/run_test_cellpose_nebius.sh

@@ -29,7 +29,7 @@ cd "$REPO_ROOT"
 
 set -e
 
-resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing
+resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/"
 # Results publish to /scratch — created and written by the cloud compute env, so
 # the launcher does NOT create it here (it is read-only from the launch host).
 publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_cellpose"

scripts/run_benchmark/param_sweep/run_test_cellposev4_nebius.sh

@@ -15,7 +15,7 @@
 #   publish) is READ-ONLY from the launch host — which is why the binning
 #   method_params block is commented out in run_test_nebius.sh. file() does stage
 #   http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED
-#   file (scripts/run_benchmark/cellposev4_params.yaml) and read it from GitHub via
+#   file (scripts/run_benchmark/param_sweep/cellposev4_params.yaml) and read it from GitHub via
 #   its raw URL. => the params file must be committed AND PUSHED to $params_branch
 #   before launching (edit the file there, not here, to change the sweep).
 
@@ -27,7 +27,7 @@ cd "$REPO_ROOT"
 
 set -e
 
-resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing
+resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/"
 # Results publish to /scratch — created and written by the cloud compute env, so
 # the launcher does NOT create it here (it is read-only from the launch host).
 publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_cellposev4"
@@ -37,7 +37,7 @@ publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_cellposev4"
 # is independent of --revision below, which selects the pipeline CODE to run.)
 params_repo="openproblems-bio/task_ist_preprocessing"
 params_branch="$(git rev-parse --abbrev-ref HEAD)"
-params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/cellposev4_params.yaml"
+params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/cellposev4_params.yaml"
 
 cat > /tmp/params_settings.yaml << HERE
 default_methods:
@@ -89,7 +89,7 @@ HERE
 if ! curl -fsSL -o /dev/null "$params_url"; then
   echo "ERROR: params file not reachable at:" >&2
   echo "  $params_url" >&2
-  echo "Commit and push scripts/run_benchmark/cellposev4_params.yaml to '$params_branch' first." >&2
+  echo "Commit and push scripts/run_benchmark/param_sweep/cellposev4_params.yaml to '$params_branch' first." >&2
   exit 1
 fi
 

scripts/run_benchmark/param_sweep/run_test_clustermap_nebius.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+
+# Nebius test run: all default methods + clustermap at the transcript-assignment stage,
+# with a parameter sweep over clustermap's tuning knobs.
+# See src/methods_transcript_assignment/clustermap/NOTES.md ("Optimization / tuning").
+#
+# The stage default (basic_transcript_assignment) stays enabled alongside clustermap so the run
+# also produces the baseline the sweep is scored against — the workflow allows at most ONE
+# non-default variant at a time, and clustermap is that one non-default method here; every OTHER
+# stage stays on its single default.
+#
+# PARAMS-FILE CAVEAT: `method_parameters_yaml` is opened by the WORKFLOW at runtime on the cloud
+# (readYaml -> Nextflow file()), so a local /tmp path won't exist there and /scratch is read-only
+# from the launch host. file() DOES stage http(s):// and this repo is public, so the sweep lives
+# in a COMMITTED file read from GitHub via its raw URL => commit AND PUSH
+# scripts/run_benchmark/param_sweep/clustermap_params.yaml to $params_branch before launching.
+# This is independent of --revision (which selects the pipeline CODE).
+
+# get the root of the directory
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+# ensure that the command below is run from the root of the repository
+cd "$REPO_ROOT"
+
+set -e
+
+resources_test_s3="/scratch/task_ist_preprocessing/resources_test/task_ist_preprocessing/"
+# Results publish to /scratch — created and written by the cloud compute env (read-only here).
+publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_clustermap"
+
+# The sweep lives in a committed file, read from GitHub at runtime. $params_branch defaults to
+# the branch you are on; the file must be pushed there. (Independent of --revision below.)
+params_repo="openproblems-bio/task_ist_preprocessing"
+params_branch="$(git rev-parse --abbrev-ref HEAD)"
+params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/clustermap_params.yaml"
+
+cat > /tmp/params_settings.yaml << HERE
+default_methods:
+  - custom_segmentation
+  - basic_transcript_assignment
+  - basic_count_aggregation
+  - basic_qc_filter
+  - alpha_shapes
+  - normalize_by_volume
+  - tacco
+  - no_correction
+segmentation_methods:
+  - custom_segmentation
+transcript_assignment_methods:
+  - basic_transcript_assignment
+  - clustermap
+count_aggregation_methods:
+  - basic_count_aggregation
+qc_filtering_methods:
+  - basic_qc_filter
+volume_calculation_methods:
+  - alpha_shapes
+normalization_methods:
+  - normalize_by_volume
+celltype_annotation_methods:
+  - tacco
+expression_correction_methods:
+  - no_correction
+gene_efficiency_correction_methods:
+  - no_correction
+method_parameters_yaml: $params_url
+HERE
+
+# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this)
+cat > /tmp/params.yaml << HERE
+input_states: $resources_test_s3/**/state.yaml
+rename_keys: 'input_sc:output_sc;input_sp:output_sp'
+save_spatial_data: false
+settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)'
+output_state: "state.yaml"
+publish_dir: "$publish_dir"
+HERE
+
+# Fail early with a clear message if the params file isn't reachable on GitHub yet.
+if ! curl -fsSL -o /dev/null "$params_url"; then
+  echo "ERROR: params file not reachable at:" >&2
+  echo "  $params_url" >&2
+  echo "Commit and push scripts/run_benchmark/param_sweep/clustermap_params.yaml to '$params_branch' first." >&2
+  exit 1
+fi
+
+tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
+  --revision build/main \
+  --pull-latest \
+  --main-script target/nextflow/workflows/run_benchmark/main.nf \
+  --workspace 167877437119966 \
+  --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \
+  --params-file /tmp/params.yaml \
+  --entry-name auto \
+  --config src/base/labels_nebius.config \
+  --labels task_ist_preprocessing,test,clustermap