From 60f69c6832073662d3517ec817c9dc13e36093a5 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Fri, 24 Jul 2026 15:09:48 +0000 Subject: [PATCH 01/15] feat(example): adds an additional example with default image, updates copier --- Dockerfile | 4 +- .../templates/example_in_image.txt.jinja | 196 ++++++++++++++++++ ...ple_template_within_default_image.py.jinja | 172 +++++++++++++++ .../templates/example_in_image.yaml | 196 ++++++++++++++++++ ...e_example_template_within_default_image.py | 172 +++++++++++++++ 5 files changed, 738 insertions(+), 2 deletions(-) create mode 100644 src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja create mode 100644 src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja create mode 100644 src/python_interface_to_workflows/templates/example_in_image.yaml create mode 100644 src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py diff --git a/Dockerfile b/Dockerfile index 0736493..bfc7efa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ ENV UV_PYTHON_INSTALL_DIR=/python # Sync the project without its dev dependencies RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --locked --no-editable --no-dev --managed-python + uv sync --locked --no-dev --no-editable --managed-python # The runtime stage copies the built venv into a runtime container FROM ubuntu:resolute AS runtime @@ -43,7 +43,7 @@ FROM ubuntu:resolute AS runtime # Copy the python installation from the build stage COPY --from=build /python /python -# Copy the environment, but not the source code +# Copy the environment, and the source code COPY --from=build /app/.venv /app/.venv ENV PATH=/app/.venv/bin:$PATH diff --git a/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja new file mode 100644 index 0000000..6614a15 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja @@ -0,0 +1,196 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: hera-example-in-image- + annotations: + workflows.argoproj.io/description: |- + Replicates the functionality of + example.yaml + workflows.argoproj.io/title: example remade via hera + workflows.diamond.ac.uk/repository: https://github.com/{{github_org}}/{{repo_name}} + labels: + workflows.diamond.ac.uk/science-group-examples: 'true' +spec: + entrypoint: workflowentry + templates: + - name: workflowentry + dag: + tasks: + - name: params + template: generate-parameters + arguments: + parameters: + - name: png + value: 'True' + - name: jpg + value: 'True' + - name: jpeg + value: 'True' + - name: tif + value: 'True' + - name: tiff + value: 'True' + - name: create-image + depends: params + template: create-image + withParam: '{{tasks.params.outputs.parameters.out-parameters}}' + arguments: + parameters: + - name: width + value: '{{item.width}}' + - name: height + value: '{{item.height}}' + - name: weights + value: '{{item.weights}}' + - name: extension + value: '{{item.extension}}' + - name: to-hdf5 + depends: create-image + template: to-hdf5 + arguments: + parameters: + - name: paths + value: '{{tasks.create-image.outputs.parameters.out-paths}}' + - name: generate-parameters + inputs: + parameters: + - name: png + - name: jpg + - name: jpeg + - name: tif + - name: tiff + outputs: + parameters: + - name: out-parameters + valueFrom: + path: /tmp/parameters.json + script: + image: ghcr.io/matt-carre/{{repo_name}}-default-image + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: jpeg = json.loads(r'''{{inputs.parameters.jpeg}}''') + except: jpeg = r'''{{inputs.parameters.jpeg}}''' + try: jpg = json.loads(r'''{{inputs.parameters.jpg}}''') + except: jpg = r'''{{inputs.parameters.jpg}}''' + try: png = json.loads(r'''{{inputs.parameters.png}}''') + except: png = r'''{{inputs.parameters.png}}''' + try: tif = json.loads(r'''{{inputs.parameters.tif}}''') + except: tif = r'''{{inputs.parameters.tif}}''' + try: tiff = json.loads(r'''{{inputs.parameters.tiff}}''') + except: tiff = r'''{{inputs.parameters.tiff}}''' + + import json + params: list[dict[str, int | list[int] | str] | None] = [{'width': 500, 'height': 500, 'weights': [255, 1, 100], 'extension': 'png'} if png.lower() == 'true' else None, {'width': 600, 'height': 200, 'weights': [100, 150, 100], 'extension': 'jpg'} if jpg.lower() == 'true' else None, {'width': 300, 'height': 400, 'weights': [100, 150, 100], 'extension': 'jpeg'} if jpeg.lower() == 'true' else None, {'width': 300, 'height': 200, 'weights': [230, 100, 1], 'extension': 'tif'} if tif.lower() == 'true' else None, {'width': 200, 'height': 300, 'weights': [230, 100, 1], 'extension': 'tiff'} if tiff.lower() == 'true' else None] + params_to_write: list[dict[str, int | list[int] | str]] = [image_params for image_params in params if image_params is not None] + with open('/tmp/parameters.json', 'w') as f: + json.dump(params_to_write, f) + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + - name: create-image + inputs: + parameters: + - name: width + - name: height + - name: weights + - name: extension + outputs: + artifacts: + - name: '{{inputs.parameters.extension}}-image' + path: /tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}} + archive: + none: {} + parameters: + - name: out-paths + valueFrom: + path: /tmp/{{inputs.parameters.extension}}-path.json + script: + image: ghcr.io/matt-carre/{{repo_name}}-default-image + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: extension = json.loads(r'''{{inputs.parameters.extension}}''') + except: extension = r'''{{inputs.parameters.extension}}''' + try: height = json.loads(r'''{{inputs.parameters.height}}''') + except: height = r'''{{inputs.parameters.height}}''' + try: weights = json.loads(r'''{{inputs.parameters.weights}}''') + except: weights = r'''{{inputs.parameters.weights}}''' + try: width = json.loads(r'''{{inputs.parameters.width}}''') + except: width = r'''{{inputs.parameters.width}}''' + + import json + from PIL import Image + + def create_pattern(width: int, height: int, weights: tuple[int, int, int]) -> Image.Image: + print(f'width: {width}') + print(f'height: {height}') + print(f'RBG weights: {weights}') + image = Image.new('RGB', (width, height)) + pixels = image.load() + for i in range(width): + for j in range(height): + pixels[i, j] = ((i + j * 50) % weights[0], weights[1], (i * 300 + j) % weights[2]) + return image + image = create_pattern(width, height, weights) + path = f'/tmp/{extension}-image.{extension}' + image.save(path) + with open(f'/tmp/{extension}-path.json', 'w') as f: + json.dump(path, f) + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + - name: to-hdf5 + inputs: + parameters: + - name: paths + outputs: + artifacts: + - name: hdf5output + path: /tmp/images.hdf5 + archive: + none: {} + script: + image: ghcr.io/matt-carre/{{repo_name}}-default-image + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: paths = json.loads(r'''{{inputs.parameters.paths}}''') + except: paths = r'''{{inputs.parameters.paths}}''' + + import h5py + import numpy as np + from PIL import Image + print('creating hdf5 file') + with h5py.File('/tmp/images.hdf5', 'w') as f: + for i, path in enumerate(paths): + path = path.strip('"') + print(f'Got {path}') + with Image.open(path) as image: + arr = np.array(image) + f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) + print('done') + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + volumeClaimTemplates: + - metadata: + name: tmpdir + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja new file mode 100644 index 0000000..eaaf2d4 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja @@ -0,0 +1,172 @@ +import os + +from hera.shared import global_config +from hera.workflows import ( + DAG, + Artifact, + Parameter, + Script, + Volume, + Workflow, + script, # pyright: ignore[reportUnknownVariableType] +) +from hera.workflows import models as m +from hera.workflows.archive import NoneArchiveStrategy + +global_config.set_class_defaults( # pyright: ignore + Script, image=str(os.environ.get("DEFAULT_IMAGE")) +) + + +@script( + command=["python"], + outputs=Parameter( + name="out-parameters", value_from=m.ValueFrom(path="/tmp/parameters.json") + ), + volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], +) +def generate_parameters( + png: str, + jpg: str, + jpeg: str, + tif: str, + tiff: str, +): + import json + + params: list[dict[str, int | list[int] | str] | None] = [ + {"width": 500, "height": 500, "weights": [255, 1, 100], "extension": "png"} + if png.lower() == "true" + else None, + {"width": 600, "height": 200, "weights": [100, 150, 100], "extension": "jpg"} + if jpg.lower() == "true" + else None, + {"width": 300, "height": 400, "weights": [100, 150, 100], "extension": "jpeg"} + if jpeg.lower() == "true" + else None, + {"width": 300, "height": 200, "weights": [230, 100, 1], "extension": "tif"} + if tif.lower() == "true" + else None, + {"width": 200, "height": 300, "weights": [230, 100, 1], "extension": "tiff"} + if tiff.lower() == "true" + else None, + ] + params_to_write: list[dict[str, int | list[int] | str]] = [ + image_params for image_params in params if image_params is not None + ] + with open("/tmp/parameters.json", "w") as f: + json.dump(params_to_write, f) + + +@script( + command=["python"], + volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], + outputs=[ + Parameter( + name="out-paths", + value_from=m.ValueFrom( + path="/tmp/{{inputs.parameters.extension}}-path.json" + ), + ), + Artifact( + name="{{inputs.parameters.extension}}-image", + path="/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}", + archive=NoneArchiveStrategy(), + ), + ], +) +def create_image( + width: int, height: int, weights: tuple[int, int, int], extension: str +): + import json + + from PIL import Image + + def create_pattern( + width: int, + height: int, + weights: tuple[int, int, int], + ) -> Image.Image: + print(f"width: {width}") + print(f"height: {height}") + print(f"RBG weights: {weights}") + image = Image.new("RGB", (width, height)) + pixels = image.load() + for i in range(width): + for j in range(height): + pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript] + (i + j * 50) % weights[0], + weights[1], + (i * 300 + j) % weights[2], + ) + return image + + image = create_pattern(width, height, weights) + path = f"/tmp/{extension}-image.{extension}" + image.save(path) + with open(f"/tmp/{extension}-path.json", "w") as f: + json.dump(path, f) + + +@script( + command=["python"], + volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], + outputs=Artifact( + name="hdf5output", + path="/tmp/images.hdf5", + archive=NoneArchiveStrategy(), + ), +) +def to_hdf5(paths: str): + + import h5py # pyright: ignore[reportMissingTypeStubs] + import numpy as np + from PIL import Image + + print("creating hdf5 file") + with h5py.File("/tmp/images.hdf5", "w") as f: + for i, path in enumerate(paths): + path = path.strip('"') + print(f"Got {path}") + with Image.open(path) as image: + arr = np.array(image) + f.create_dataset( # pyright: ignore[reportUnknownMemberType] + f"image_{i}", data=arr, dtype=arr.dtype + ) + print("done") + + +with Workflow( + generate_name="hera-example-in-image-", # name on graphql + entrypoint="workflowentry", + api_version="argoproj.io/v1alpha1", + kind="Workflow", # ClusterWorkflowTemplate", when on graphql + labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, + annotations={ + "workflows.argoproj.io/title": "example remade via hera", + "workflows.argoproj.io/description": """Replicates the functionality of +example.yaml""", + "workflows.diamond.ac.uk/repository": "https://github.com/{{github_org}}/{{repo_name}}", + }, + volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), +) as w: + with DAG(name="workflowentry"): + params = generate_parameters( + name="params", + arguments={ + "png": "True", + "jpg": "True", + "jpeg": "True", + "tif": "True", + "tiff": "True", + }, + ) + makeimages = create_image(with_param=params.get_parameter("out-parameters")) + makehdf5 = to_hdf5( + arguments={ + "paths": makeimages.get_parameter("out-paths"), + } + ) + params >> makeimages >> makehdf5 # pyright: ignore +with open("example_in_image.txt", "w") as div: + div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] diff --git a/src/python_interface_to_workflows/templates/example_in_image.yaml b/src/python_interface_to_workflows/templates/example_in_image.yaml new file mode 100644 index 0000000..64f9565 --- /dev/null +++ b/src/python_interface_to_workflows/templates/example_in_image.yaml @@ -0,0 +1,196 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: hera-example-in-image- + annotations: + workflows.argoproj.io/description: |- + Replicates the functionality of + example.yaml + workflows.argoproj.io/title: example remade via hera + workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows + labels: + workflows.diamond.ac.uk/science-group-examples: 'true' +spec: + entrypoint: workflowentry + templates: + - name: workflowentry + dag: + tasks: + - name: params + template: generate-parameters + arguments: + parameters: + - name: png + value: 'True' + - name: jpg + value: 'True' + - name: jpeg + value: 'True' + - name: tif + value: 'True' + - name: tiff + value: 'True' + - name: create-image + depends: params + template: create-image + withParam: '{{tasks.params.outputs.parameters.out-parameters}}' + arguments: + parameters: + - name: width + value: '{{item.width}}' + - name: height + value: '{{item.height}}' + - name: weights + value: '{{item.weights}}' + - name: extension + value: '{{item.extension}}' + - name: to-hdf5 + depends: create-image + template: to-hdf5 + arguments: + parameters: + - name: paths + value: '{{tasks.create-image.outputs.parameters.out-paths}}' + - name: generate-parameters + inputs: + parameters: + - name: png + - name: jpg + - name: jpeg + - name: tif + - name: tiff + outputs: + parameters: + - name: out-parameters + valueFrom: + path: /tmp/parameters.json + script: + image: ghcr.io/matt-carre/python-interface-to-workflows-default-image + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: jpeg = json.loads(r'''{{inputs.parameters.jpeg}}''') + except: jpeg = r'''{{inputs.parameters.jpeg}}''' + try: jpg = json.loads(r'''{{inputs.parameters.jpg}}''') + except: jpg = r'''{{inputs.parameters.jpg}}''' + try: png = json.loads(r'''{{inputs.parameters.png}}''') + except: png = r'''{{inputs.parameters.png}}''' + try: tif = json.loads(r'''{{inputs.parameters.tif}}''') + except: tif = r'''{{inputs.parameters.tif}}''' + try: tiff = json.loads(r'''{{inputs.parameters.tiff}}''') + except: tiff = r'''{{inputs.parameters.tiff}}''' + + import json + params: list[dict[str, int | list[int] | str] | None] = [{'width': 500, 'height': 500, 'weights': [255, 1, 100], 'extension': 'png'} if png.lower() == 'true' else None, {'width': 600, 'height': 200, 'weights': [100, 150, 100], 'extension': 'jpg'} if jpg.lower() == 'true' else None, {'width': 300, 'height': 400, 'weights': [100, 150, 100], 'extension': 'jpeg'} if jpeg.lower() == 'true' else None, {'width': 300, 'height': 200, 'weights': [230, 100, 1], 'extension': 'tif'} if tif.lower() == 'true' else None, {'width': 200, 'height': 300, 'weights': [230, 100, 1], 'extension': 'tiff'} if tiff.lower() == 'true' else None] + params_to_write: list[dict[str, int | list[int] | str]] = [image_params for image_params in params if image_params is not None] + with open('/tmp/parameters.json', 'w') as f: + json.dump(params_to_write, f) + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + - name: create-image + inputs: + parameters: + - name: width + - name: height + - name: weights + - name: extension + outputs: + artifacts: + - name: '{{inputs.parameters.extension}}-image' + path: /tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}} + archive: + none: {} + parameters: + - name: out-paths + valueFrom: + path: /tmp/{{inputs.parameters.extension}}-path.json + script: + image: ghcr.io/matt-carre/python-interface-to-workflows-default-image + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: extension = json.loads(r'''{{inputs.parameters.extension}}''') + except: extension = r'''{{inputs.parameters.extension}}''' + try: height = json.loads(r'''{{inputs.parameters.height}}''') + except: height = r'''{{inputs.parameters.height}}''' + try: weights = json.loads(r'''{{inputs.parameters.weights}}''') + except: weights = r'''{{inputs.parameters.weights}}''' + try: width = json.loads(r'''{{inputs.parameters.width}}''') + except: width = r'''{{inputs.parameters.width}}''' + + import json + from PIL import Image + + def create_pattern(width: int, height: int, weights: tuple[int, int, int]) -> Image.Image: + print(f'width: {width}') + print(f'height: {height}') + print(f'RBG weights: {weights}') + image = Image.new('RGB', (width, height)) + pixels = image.load() + for i in range(width): + for j in range(height): + pixels[i, j] = ((i + j * 50) % weights[0], weights[1], (i * 300 + j) % weights[2]) + return image + image = create_pattern(width, height, weights) + path = f'/tmp/{extension}-image.{extension}' + image.save(path) + with open(f'/tmp/{extension}-path.json', 'w') as f: + json.dump(path, f) + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + - name: to-hdf5 + inputs: + parameters: + - name: paths + outputs: + artifacts: + - name: hdf5output + path: /tmp/images.hdf5 + archive: + none: {} + script: + image: ghcr.io/matt-carre/python-interface-to-workflows-default-image + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: paths = json.loads(r'''{{inputs.parameters.paths}}''') + except: paths = r'''{{inputs.parameters.paths}}''' + + import h5py + import numpy as np + from PIL import Image + print('creating hdf5 file') + with h5py.File('/tmp/images.hdf5', 'w') as f: + for i, path in enumerate(paths): + path = path.strip('"') + print(f'Got {path}') + with Image.open(path) as image: + arr = np.array(image) + f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) + print('done') + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + volumeClaimTemplates: + - metadata: + name: tmpdir + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py b/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py new file mode 100644 index 0000000..88740ca --- /dev/null +++ b/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py @@ -0,0 +1,172 @@ +import os + +from hera.shared import global_config +from hera.workflows import ( + DAG, + Artifact, + Parameter, + Script, + Volume, + Workflow, + script, # pyright: ignore[reportUnknownVariableType] +) +from hera.workflows import models as m +from hera.workflows.archive import NoneArchiveStrategy + +global_config.set_class_defaults( # pyright: ignore + Script, image=str(os.environ.get("DEFAULT_IMAGE")) +) + + +@script( + command=["python"], + outputs=Parameter( + name="out-parameters", value_from=m.ValueFrom(path="/tmp/parameters.json") + ), + volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], +) +def generate_parameters( + png: str, + jpg: str, + jpeg: str, + tif: str, + tiff: str, +): + import json + + params: list[dict[str, int | list[int] | str] | None] = [ + {"width": 500, "height": 500, "weights": [255, 1, 100], "extension": "png"} + if png.lower() == "true" + else None, + {"width": 600, "height": 200, "weights": [100, 150, 100], "extension": "jpg"} + if jpg.lower() == "true" + else None, + {"width": 300, "height": 400, "weights": [100, 150, 100], "extension": "jpeg"} + if jpeg.lower() == "true" + else None, + {"width": 300, "height": 200, "weights": [230, 100, 1], "extension": "tif"} + if tif.lower() == "true" + else None, + {"width": 200, "height": 300, "weights": [230, 100, 1], "extension": "tiff"} + if tiff.lower() == "true" + else None, + ] + params_to_write: list[dict[str, int | list[int] | str]] = [ + image_params for image_params in params if image_params is not None + ] + with open("/tmp/parameters.json", "w") as f: + json.dump(params_to_write, f) + + +@script( + command=["python"], + volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], + outputs=[ + Parameter( + name="out-paths", + value_from=m.ValueFrom( + path="/tmp/{{inputs.parameters.extension}}-path.json" + ), + ), + Artifact( + name="{{inputs.parameters.extension}}-image", + path="/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}", + archive=NoneArchiveStrategy(), + ), + ], +) +def create_image( + width: int, height: int, weights: tuple[int, int, int], extension: str +): + import json + + from PIL import Image + + def create_pattern( + width: int, + height: int, + weights: tuple[int, int, int], + ) -> Image.Image: + print(f"width: {width}") + print(f"height: {height}") + print(f"RBG weights: {weights}") + image = Image.new("RGB", (width, height)) + pixels = image.load() + for i in range(width): + for j in range(height): + pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript] + (i + j * 50) % weights[0], + weights[1], + (i * 300 + j) % weights[2], + ) + return image + + image = create_pattern(width, height, weights) + path = f"/tmp/{extension}-image.{extension}" + image.save(path) + with open(f"/tmp/{extension}-path.json", "w") as f: + json.dump(path, f) + + +@script( + command=["python"], + volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], + outputs=Artifact( + name="hdf5output", + path="/tmp/images.hdf5", + archive=NoneArchiveStrategy(), + ), +) +def to_hdf5(paths: str): + + import h5py # pyright: ignore[reportMissingTypeStubs] + import numpy as np + from PIL import Image + + print("creating hdf5 file") + with h5py.File("/tmp/images.hdf5", "w") as f: + for i, path in enumerate(paths): + path = path.strip('"') + print(f"Got {path}") + with Image.open(path) as image: + arr = np.array(image) + f.create_dataset( # pyright: ignore[reportUnknownMemberType] + f"image_{i}", data=arr, dtype=arr.dtype + ) + print("done") + + +with Workflow( + generate_name="hera-example-in-image-", # name on graphql + entrypoint="workflowentry", + api_version="argoproj.io/v1alpha1", + kind="Workflow", # ClusterWorkflowTemplate", when on graphql + labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, + annotations={ + "workflows.argoproj.io/title": "example remade via hera", + "workflows.argoproj.io/description": """Replicates the functionality of +example.yaml""", + "workflows.diamond.ac.uk/repository": "https://github.com/DiamondLightSource/python-interface-to-workflows", + }, + volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), +) as w: + with DAG(name="workflowentry"): + params = generate_parameters( + name="params", + arguments={ + "png": "True", + "jpg": "True", + "jpeg": "True", + "tif": "True", + "tiff": "True", + }, + ) + makeimages = create_image(with_param=params.get_parameter("out-parameters")) + makehdf5 = to_hdf5( + arguments={ + "paths": makeimages.get_parameter("out-paths"), + } + ) + params >> makeimages >> makehdf5 # pyright: ignore +with open("example_in_image.txt", "w") as div: + div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] From e0d962072c83b69d73e22894f98f168281370f34 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Mon, 27 Jul 2026 09:12:39 +0000 Subject: [PATCH 02/15] docs(copier): adds instructions on how to create and set images in copier readme --- Dockerfile | 2 +- src/copier_template/src/README.md.jinja | 17 +++++++++++++++++ .../templates/example_in_image.txt.jinja | 10 ++++++---- ...ample_template_within_default_image.py.jinja | 4 +++- ...ample_in_image.yaml => example_in_image.txt} | 0 5 files changed, 27 insertions(+), 6 deletions(-) rename src/python_interface_to_workflows/templates/{example_in_image.yaml => example_in_image.txt} (100%) diff --git a/Dockerfile b/Dockerfile index bfc7efa..2a69ffb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ FROM ubuntu:resolute AS runtime # Copy the python installation from the build stage COPY --from=build /python /python -# Copy the environment, and the source code +# Copy the environment, but not the source code COPY --from=build /app/.venv /app/.venv ENV PATH=/app/.venv/bin:$PATH diff --git a/src/copier_template/src/README.md.jinja b/src/copier_template/src/README.md.jinja index c09fef6..828529c 100644 --- a/src/copier_template/src/README.md.jinja +++ b/src/copier_template/src/README.md.jinja @@ -25,3 +25,20 @@ submit_workflow(w) NOTE: Be sure to remove this line upon commiting changes, as all workflow definition files are by default, ran on pre-commit, to ensure that any yaml files they create are up to date. + +# Building a custom image +While in src, the same folder as a Dockerfile: + +podman build -t ghcr.io/Your-Github-Name/image-name . +podman login ghcr.io +podman push ghcr.io/Your-Github-Name/image-name + +Then go to your github profile, packages, and set image-name's visibility to public +After this, you may add 'image' in the script decorator, to run specific scripts within that image +Alternatively, you can set the default image at the top of the file by adding: + +```python +global_config.set_class_defaults( # pyright: ignore + Script, image=str(os.environ.get("DEFAULT_IMAGE")) +) +``` diff --git a/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja index 6614a15..78b1316 100644 --- a/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja @@ -1,3 +1,4 @@ +{% raw %} apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: @@ -7,7 +8,7 @@ metadata: Replicates the functionality of example.yaml workflows.argoproj.io/title: example remade via hera - workflows.diamond.ac.uk/repository: https://github.com/{{github_org}}/{{repo_name}} + workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %} labels: workflows.diamond.ac.uk/science-group-examples: 'true' spec: @@ -65,7 +66,7 @@ spec: valueFrom: path: /tmp/parameters.json script: - image: ghcr.io/matt-carre/{{repo_name}}-default-image + image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image source: |- import os import sys @@ -110,7 +111,7 @@ spec: valueFrom: path: /tmp/{{inputs.parameters.extension}}-path.json script: - image: ghcr.io/matt-carre/{{repo_name}}-default-image + image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image source: |- import os import sys @@ -159,7 +160,7 @@ spec: archive: none: {} script: - image: ghcr.io/matt-carre/{{repo_name}}-default-image + image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image source: |- import os import sys @@ -194,3 +195,4 @@ spec: resources: requests: storage: 1Gi +{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja index eaaf2d4..80e5e29 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja @@ -1,3 +1,4 @@ +{% raw %} import os from hera.shared import global_config @@ -146,7 +147,7 @@ with Workflow( "workflows.argoproj.io/title": "example remade via hera", "workflows.argoproj.io/description": """Replicates the functionality of example.yaml""", - "workflows.diamond.ac.uk/repository": "https://github.com/{{github_org}}/{{repo_name}}", + "workflows.diamond.ac.uk/repository": "https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}", }, volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), ) as w: @@ -170,3 +171,4 @@ example.yaml""", params >> makeimages >> makehdf5 # pyright: ignore with open("example_in_image.txt", "w") as div: div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] +{% endraw %} diff --git a/src/python_interface_to_workflows/templates/example_in_image.yaml b/src/python_interface_to_workflows/templates/example_in_image.txt similarity index 100% rename from src/python_interface_to_workflows/templates/example_in_image.yaml rename to src/python_interface_to_workflows/templates/example_in_image.txt From 185ba731bfaccf4263c915597f3bf24df5775e44 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 29 Jul 2026 11:09:50 +0000 Subject: [PATCH 03/15] feat(example): updates templates and copier files to new standards --- .../{{ project_name }}/templates/example_in_image.txt.jinja | 4 ++-- .../create_example_template_within_default_image.py.jinja | 4 ++-- .../notebooks/notebook_example.ipynb.jinja | 4 ++-- .../templates/example_in_image.txt | 4 ++-- .../create_example_template_within_default_image.py | 4 ++-- .../workflow_definitions/notebooks/notebook_division.ipynb | 6 +++--- .../workflow_definitions/notebooks/notebook_example.ipynb | 4 ++-- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja index 78b1316..2606a5f 100644 --- a/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja @@ -1,8 +1,8 @@ {% raw %} apiVersion: argoproj.io/v1alpha1 -kind: Workflow +kind: WorkflowTemplate metadata: - generateName: hera-example-in-image- + name: hera-example-in-image annotations: workflows.argoproj.io/description: |- Replicates the functionality of diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja index 80e5e29..3e604a5 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja @@ -138,10 +138,10 @@ def to_hdf5(paths: str): with Workflow( - generate_name="hera-example-in-image-", # name on graphql + name="hera-example-in-image", entrypoint="workflowentry", api_version="argoproj.io/v1alpha1", - kind="Workflow", # ClusterWorkflowTemplate", when on graphql + kind="WorkflowTemplate", labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, annotations={ "workflows.argoproj.io/title": "example remade via hera", diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja index 0f829f6..41e2409 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja @@ -166,10 +166,10 @@ "\n", "\n", "with Workflow(\n", - " generate_name=\"hera-example-\", # when running on graphql this should be name\n", + " name=\"hera-example-\",\n", " entrypoint=\"workflowentry\",\n", " api_version=\"argoproj.io/v1alpha1\",\n", - " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", + " kind=\"WorkflowTemplate\",\n", " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", " annotations={\n", " \"workflows.argoproj.io/title\": \"example remade via hera\",\n", diff --git a/src/python_interface_to_workflows/templates/example_in_image.txt b/src/python_interface_to_workflows/templates/example_in_image.txt index 64f9565..ac6be98 100644 --- a/src/python_interface_to_workflows/templates/example_in_image.txt +++ b/src/python_interface_to_workflows/templates/example_in_image.txt @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 -kind: Workflow +kind: WorkflowTemplate metadata: - generateName: hera-example-in-image- + name: hera-example-in-image annotations: workflows.argoproj.io/description: |- Replicates the functionality of diff --git a/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py b/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py index 88740ca..779921c 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py @@ -137,10 +137,10 @@ def to_hdf5(paths: str): with Workflow( - generate_name="hera-example-in-image-", # name on graphql + name="hera-example-in-image", entrypoint="workflowentry", api_version="argoproj.io/v1alpha1", - kind="Workflow", # ClusterWorkflowTemplate", when on graphql + kind="WorkflowTemplate", labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, annotations={ "workflows.argoproj.io/title": "example remade via hera", diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb index 9113672..107824e 100644 --- a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb +++ b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "494174ef", "metadata": {}, "outputs": [], @@ -50,10 +50,10 @@ "\n", "\n", "with Workflow(\n", - " generate_name=\"hera-division-\", # when running on graphql this should be name\n", + " name=\"hera-division\",\n", " entrypoint=\"divide\",\n", " api_version=\"argoproj.io/v1alpha1\",\n", - " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", + " kind=\"WorkflowTemplate\",\n", " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", " annotations={\n", " \"workflows.argoproj.io/title\": \"Division via hera test\",\n", diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb index c1018ea..4d3b47f 100644 --- a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb +++ b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb @@ -166,10 +166,10 @@ "\n", "\n", "with Workflow(\n", - " generate_name=\"hera-example-\", # when running on graphql this should be name\n", + " name=\"hera-example\",\n", " entrypoint=\"workflowentry\",\n", " api_version=\"argoproj.io/v1alpha1\",\n", - " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", + " kind=\"WorkflowTemplate\",\n", " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", " annotations={\n", " \"workflows.argoproj.io/title\": \"example remade via hera\",\n", From 9e17abc63b7b63bf715109c067c73dbdec012206 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 29 Jul 2026 11:39:19 +0000 Subject: [PATCH 04/15] feat(graphql): makes submit_workflow async --- .vscode/settings.json | 5 ++++- Dockerfile | 2 +- pyproject.toml | 2 ++ src/copier_template/pyproject.toml.jinja | 2 ++ src/copier_template/src/README.md.jinja | 2 +- .../submit_workflow.py.jinja | 4 ++-- .../notebooks/notebook_example.ipynb.jinja | 2 +- .../tests/test_submit_to_graphql.py.jinja | 17 ++++++++++------- .../submit_workflow.py | 4 ++-- .../notebooks/notebook_division.ipynb | 2 +- .../notebooks/notebook_example.ipynb | 6 +++--- tests/test_submit_to_graphql.py | 19 ++++++++++++------- uv.lock | 17 +++++++++++++++++ 13 files changed, 58 insertions(+), 26 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 7ac45f8..fb1f0e8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,5 +9,8 @@ "[python]": { "editor.defaultFormatter": "charliermarsh.ruff", }, - "python.envFile": "${workspaceFolder}/workspaces/python-interface-to-workflows/src/.env" + "python.envFile": "${workspaceFolder}/workspaces/python-interface-to-workflows/src/.env", + "python.testing.pytestArgs": [ + "tests" + ] } diff --git a/Dockerfile b/Dockerfile index 2a69ffb..0736493 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ ENV UV_PYTHON_INSTALL_DIR=/python # Sync the project without its dev dependencies RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --locked --no-dev --no-editable --managed-python + uv sync --locked --no-editable --no-dev --managed-python # The runtime stage copies the built venv into a runtime container FROM ubuntu:resolute AS runtime diff --git a/pyproject.toml b/pyproject.toml index 3552c32..50f1105 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ "h5py", "dotenv", "python-keycloak", + "pytest-asyncio", ] # Add project dependencies here, e.g. ["click", "numpy"] dynamic = ["version"] license.file = "LICENSE" @@ -46,6 +47,7 @@ dev = [ "h5py", "dotenv", "python-keycloak", + "pytest-asyncio", ] [project.scripts] diff --git a/src/copier_template/pyproject.toml.jinja b/src/copier_template/pyproject.toml.jinja index 9eec731..3d5c5bb 100644 --- a/src/copier_template/pyproject.toml.jinja +++ b/src/copier_template/pyproject.toml.jinja @@ -24,6 +24,7 @@ dependencies = [ "h5py", "dotenv", "python-keycloak", + "pytest-asyncio", ] # Add project dependencies here, e.g. ["click", "numpy"] dynamic = ["version"] license.file = "LICENSE" @@ -46,6 +47,7 @@ dev = [ "h5py", "dotenv", "python-keycloak", + "pytest-asyncio", ] diff --git a/src/copier_template/src/README.md.jinja b/src/copier_template/src/README.md.jinja index 828529c..23da404 100644 --- a/src/copier_template/src/README.md.jinja +++ b/src/copier_template/src/README.md.jinja @@ -2,7 +2,7 @@ 1. run "uv lock" to generate the uv.lock file 2. Create .env in this folder (with the path src/.env) containing the following variables: -HOST=https://argo-workflows.workflows.diamond.ac.uk/ (to submit to the production cluster) +HOST=https://workflows.diamond.ac.uk/graphql (to submit to the production cluster) DEFAULT_IMAGE= (usually python 3.10) VISIT= (the Visit you wish to run the template on) TOKEN= diff --git a/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja b/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja index a2dfed9..e2d353d 100644 --- a/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja +++ b/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja @@ -9,7 +9,7 @@ from hera.workflows import Workflow from {% endraw %}{{project_name}}{% raw %}.auth.keycloak_checker import set_token_env_variable -def submit_workflow(w: Workflow): +async def submit_workflow(w: Workflow): yamlstr = w.to_yaml() # pyright:ignore dotenv.load_dotenv(dotenv_path="src/.env", override=True) token: str = set_token_env_variable() @@ -34,7 +34,7 @@ mutation Submit($visit: VisitInput!, $manifest: String!) { } } """) - result = client.execute( + result = await client.execute_async( mutation, variable_values={ "visit": { diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja index 41e2409..f5c3c68 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja @@ -222,7 +222,7 @@ "source": [ "from {% endraw %}{{project_name}}{% raw %}.submit_workflow import submit_workflow\n", "\n", - "submit_workflow(w)" + "await submit_workflow(w)" ] } ], diff --git a/src/copier_template/tests/test_submit_to_graphql.py.jinja b/src/copier_template/tests/test_submit_to_graphql.py.jinja index 265d189..2427bd4 100644 --- a/src/copier_template/tests/test_submit_to_graphql.py.jinja +++ b/src/copier_template/tests/test_submit_to_graphql.py.jinja @@ -1,27 +1,30 @@ -from unittest.mock import MagicMock, call, patch +from unittest.mock import AsyncMock, MagicMock, call, patch from {{project_name}}.submit_workflow import submit_workflow +@pytest.mark.asyncio @patch("{{project_name}}.submit_workflow.os.environ.get") @patch("{{project_name}}.submit_workflow.dotenv.load_dotenv") @patch("{{project_name}}.submit_workflow.Workflow") @patch("{{project_name}}.submit_workflow.set_token_env_variable") @patch("{{project_name}}.submit_workflow.Client") -def test_submit_workflow_to_graphql( - mock_client: MagicMock, +async def test_submit_workflow_to_graphql( + mock_client: AsyncMock, mock_key: MagicMock, mock_workflow: MagicMock, mock_load_env: MagicMock, mock_os_get: MagicMock, ): - mock_instance = MagicMock() + mock_instance = AsyncMock() mock_key.return_value = "token" mock_client.return_value = mock_instance - mock_instance.execute.return_value = {"submitWorkflow": {"name": "workflow123"}} - submit_workflow(mock_workflow) + mock_instance.execute_async = AsyncMock( + return_value={"submitWorkflow": {"name": "workflow123"}} + ) + await submit_workflow(mock_workflow) mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) - mock_instance.execute.assert_called_once() + mock_instance.execute_async.assert_called_once() mock_workflow.to_yaml.assert_called_once() mock_os_get.assert_has_calls([call("VISIT"), call("HOST")], any_order=True) diff --git a/src/python_interface_to_workflows/submit_workflow.py b/src/python_interface_to_workflows/submit_workflow.py index da3422a..6a668c8 100644 --- a/src/python_interface_to_workflows/submit_workflow.py +++ b/src/python_interface_to_workflows/submit_workflow.py @@ -8,7 +8,7 @@ from python_interface_to_workflows.auth.keycloak_checker import set_token_env_variable -def submit_workflow(w: Workflow): +async def submit_workflow(w: Workflow): yamlstr = w.to_yaml() # pyright:ignore dotenv.load_dotenv(dotenv_path="src/.env", override=True) token: str = set_token_env_variable(True) @@ -33,7 +33,7 @@ def submit_workflow(w: Workflow): } } """) - result = client.execute( + result = await client.execute_async( mutation, variable_values={ "visit": { diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb index 107824e..4d0e147 100644 --- a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb +++ b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb @@ -88,7 +88,7 @@ "source": [ "from python_interface_to_workflows.submit_workflow import submit_workflow\n", "\n", - "submit_workflow(w)" + "await submit_workflow(w)" ] } ], diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb index 4d3b47f..1d7bfe4 100644 --- a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb +++ b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb @@ -223,13 +223,13 @@ "source": [ "from python_interface_to_workflows.submit_workflow import submit_workflow\n", "\n", - "submit_workflow(w)" + "await submit_workflow(w)" ] } ], "metadata": { "kernelspec": { - "display_name": "python-interface-to-workflows (3.11.x)", + "display_name": "python-interface-to-workflows (broken)", "language": "python", "name": "python3" }, @@ -243,7 +243,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.15" + "version": "3.11.13" } }, "nbformat": 4, diff --git a/tests/test_submit_to_graphql.py b/tests/test_submit_to_graphql.py index 1eb8020..3a84665 100644 --- a/tests/test_submit_to_graphql.py +++ b/tests/test_submit_to_graphql.py @@ -1,27 +1,32 @@ -from unittest.mock import MagicMock, call, patch +from unittest.mock import AsyncMock, MagicMock, call, patch + +import pytest from python_interface_to_workflows.submit_workflow import submit_workflow +@pytest.mark.asyncio @patch("python_interface_to_workflows.submit_workflow.os.environ.get") @patch("python_interface_to_workflows.submit_workflow.dotenv.load_dotenv") @patch("python_interface_to_workflows.submit_workflow.Workflow") @patch("python_interface_to_workflows.submit_workflow.set_token_env_variable") @patch("python_interface_to_workflows.submit_workflow.Client") -def test_submit_workflow_to_graphql( - mock_client: MagicMock, +async def test_submit_workflow_to_graphql( + mock_client: AsyncMock, mock_key: MagicMock, mock_workflow: MagicMock, mock_load_env: MagicMock, mock_os_get: MagicMock, ): - mock_instance = MagicMock() + mock_instance = AsyncMock() mock_key.return_value = "token" mock_client.return_value = mock_instance - mock_instance.execute.return_value = {"submitWorkflow": {"name": "workflow123"}} - submit_workflow(mock_workflow) + mock_instance.execute_async = AsyncMock( + return_value={"submitWorkflow": {"name": "workflow123"}} + ) + await submit_workflow(mock_workflow) mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) - mock_instance.execute.assert_called_once() + mock_instance.execute_async.assert_called_once() mock_workflow.to_yaml.assert_called_once() mock_os_get.assert_has_calls([call("VISIT"), call("HOST")], any_order=True) diff --git a/uv.lock b/uv.lock index a65926b..0391cc9 100644 --- a/uv.lock +++ b/uv.lock @@ -1767,6 +1767,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] +[[package]] +name = "pytest-asyncio" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, +] + [[package]] name = "pytest-cov" version = "7.1.0" @@ -1814,6 +1827,7 @@ dependencies = [ { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "pillow" }, + { name = "pytest-asyncio" }, { name = "python-keycloak" }, { name = "pyyaml" }, { name = "requests" }, @@ -1830,6 +1844,7 @@ dev = [ { name = "pre-commit" }, { name = "pyright" }, { name = "pytest" }, + { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "python-keycloak" }, { name = "pyyaml" }, @@ -1847,6 +1862,7 @@ requires-dist = [ { name = "hera" }, { name = "numpy" }, { name = "pillow" }, + { name = "pytest-asyncio" }, { name = "python-keycloak" }, { name = "pyyaml" }, { name = "requests" }, @@ -1862,6 +1878,7 @@ dev = [ { name = "pre-commit" }, { name = "pyright" }, { name = "pytest" }, + { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "python-keycloak" }, { name = "pyyaml" }, From 50d99a2da6f769eddfbf07b05d8c94d4a0a5459c Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Thu, 30 Jul 2026 09:56:17 +0000 Subject: [PATCH 05/15] feat(CI): adds auto-updating image --- .github/workflows/_update_image.yml | 51 +++++++++++++++++++++++++++++ .github/workflows/ci.yml | 3 ++ 2 files changed, 54 insertions(+) create mode 100644 .github/workflows/_update_image.yml diff --git a/.github/workflows/_update_image.yml b/.github/workflows/_update_image.yml new file mode 100644 index 0000000..022f244 --- /dev/null +++ b/.github/workflows/_update_image.yml @@ -0,0 +1,51 @@ +name: Update Docker Image + +on: + workflow_call: + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout Code + uses: actions/checkout@v6 + + - name: Generate Image Name + run: echo IMAGE_REPOSITORY=ghcr.io/$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]' | tr '[_]' '[\-]')-image >> $GITHUB_ENV + + - name: Log in to GitHub Docker Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v4.1.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker Metadata + id: meta + uses: docker/metadata-action@v6.1.0 + with: + images: ${{ env.IMAGE_REPOSITORY }} + tags: | + type=ref,event=branch + type=raw,value=latest,enable={{is_default_branch}} + type=match,pattern=python-interface-to-workflows@v?(.+),group=1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4.0.0 + with: + driver-opts: network=host + + - name: Build Image + uses: docker/build-push-action@v6.18.0 + with: + context: . + push: ${{ github.event_name == 'push' }} + load: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f576ab..13ffd5e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,9 @@ jobs: commit-lint: uses: ./.github/workflows/_commit_msg.yml + updateimage: + uses: ./.github/workflows/_update_image.yml + test: strategy: matrix: From 89c6b340fa34efe212ce56432382120c85d85dbe Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Thu, 30 Jul 2026 10:59:33 +0000 Subject: [PATCH 06/15] docs(copier): updates copier template and improves example --- scripts/makecopiercorrect.sh | 7 +- .../templates/example.txt.jinja | 41 ++-- .../templates/example_in_image.txt.jinja | 198 ------------------ .../create_example_template.py.jinja | 60 ++++-- ...ple_template_within_default_image.py.jinja | 174 --------------- ...nja => notebook_image_example.ipynb.jinja} | 109 ++++++---- .../auth/keycloak_checker.py | 2 +- .../templates/example.txt | 41 ++-- .../templates/example_in_image.txt | 196 ----------------- .../create_example_template.py | 60 ++++-- ...e_example_template_within_default_image.py | 172 --------------- .../notebooks/notebook_division.ipynb | 4 +- ...ple.ipynb => notebook_image_example.ipynb} | 97 ++++++--- tests/test_keycloak_checker.py | 5 +- 14 files changed, 258 insertions(+), 908 deletions(-) delete mode 100644 src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja delete mode 100644 src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja rename src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/{notebook_example.ipynb.jinja => notebook_image_example.ipynb.jinja} (78%) delete mode 100644 src/python_interface_to_workflows/templates/example_in_image.txt delete mode 100644 src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py rename src/python_interface_to_workflows/workflow_definitions/notebooks/{notebook_example.ipynb => notebook_image_example.ipynb} (80%) diff --git a/scripts/makecopiercorrect.sh b/scripts/makecopiercorrect.sh index 55b5d63..fd9765e 100644 --- a/scripts/makecopiercorrect.sh +++ b/scripts/makecopiercorrect.sh @@ -11,6 +11,7 @@ do sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file" sed -i 's/DiamondLightSource/{{github_org}}/g' "$file" + sed -i 's/python_interface_to_workflows/{{project_name}}/g' "$file" sed -i '1i{% raw %}' "$file" echo '{% endraw %}' >> "$file" @@ -18,6 +19,7 @@ do sed -i \ -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \ -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \ + -e 's/{{project_name}}/{% endraw %}{{project_name}}{% raw %}/g' \ "$file" mv "$file" "$file.jinja" @@ -30,12 +32,14 @@ do sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file" sed -i 's/DiamondLightSource/{{github_org}}/g' "$file" + sed -i 's/python_interface_to_workflows/{{project_name}}/g' "$file" sed -i '1i{% raw %}' "$file" echo '{% endraw %}' >> "$file" sed -i \ -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \ -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \ + -e 's/{{project_name}}/{% endraw %}{{project_name}}{% raw %}/g' \ "$file" @@ -48,13 +52,14 @@ do sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file" sed -i 's/DiamondLightSource/{{github_org}}/g' "$file" - + sed -i 's/python_interface_to_workflows/{{project_name}}/g' "$file" sed -i '1i{% raw %}' "$file" echo '{% endraw %}' >> "$file" sed -i \ -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \ -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \ + -e 's/{{project_name}}/{% endraw %}{{project_name}}{% raw %}/g' \ "$file" mv "$file" "$file.jinja" diff --git a/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja index 78cb885..b017cf1 100644 --- a/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja @@ -13,12 +13,12 @@ metadata: workflows.diamond.ac.uk/science-group-examples: 'true' spec: entrypoint: workflowentry + podSpecPatch: '{"containers": [{"name": "main", "resources": {"limits": {"cpu": + "1", "memory": "1Gi"}, "requests": {"cpu": "1", "memory": "1Gi"}}}]}' templates: - name: workflowentry dag: tasks: - - name: install - template: install-dependencies - name: params template: generate-parameters arguments: @@ -34,7 +34,7 @@ spec: - name: tiff value: 'True' - name: create-image - depends: install && params + depends: params template: create-image withParam: '{{tasks.params.outputs.parameters.out-parameters}}' arguments: @@ -54,22 +54,6 @@ spec: parameters: - name: paths value: '{{tasks.create-image.outputs.parameters.out-paths}}' - - name: install-dependencies - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import subprocess - print('creating venv') - subprocess.check_call(['python', '-m', 'venv', '/tmp/venv']) - subprocess.check_call(['/tmp/venv/bin/pip', 'install', 'pillow', 'h5py', 'numpy', 'hera']) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - name: generate-parameters inputs: parameters: @@ -84,7 +68,7 @@ spec: valueFrom: path: /tmp/parameters.json script: - image: python:3.10 + image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image source: |- import os import sys @@ -129,7 +113,7 @@ spec: valueFrom: path: /tmp/{{inputs.parameters.extension}}-path.json script: - image: python:3.10 + image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image source: |- import os import sys @@ -163,7 +147,7 @@ spec: with open(f'/tmp/{extension}-path.json', 'w') as f: json.dump(path, f) command: - - /tmp/venv/bin/python + - python volumeMounts: - name: tmpdir mountPath: /tmp @@ -178,7 +162,7 @@ spec: archive: none: {} script: - image: python:3.10 + image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image source: |- import os import sys @@ -200,10 +184,19 @@ spec: f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) print('done') command: - - /tmp/venv/bin/python + - python volumeMounts: - name: tmpdir mountPath: /tmp + tolerations: + - effect: NoSchedule + key: nodetype + operator: Equal + value: gpu + - effect: NoSchedule + key: nodegroup + operator: Equal + value: workflows volumeClaimTemplates: - metadata: name: tmpdir diff --git a/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja deleted file mode 100644 index 2606a5f..0000000 --- a/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja +++ /dev/null @@ -1,198 +0,0 @@ -{% raw %} -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: hera-example-in-image - annotations: - workflows.argoproj.io/description: |- - Replicates the functionality of - example.yaml - workflows.argoproj.io/title: example remade via hera - workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %} - labels: - workflows.diamond.ac.uk/science-group-examples: 'true' -spec: - entrypoint: workflowentry - templates: - - name: workflowentry - dag: - tasks: - - name: params - template: generate-parameters - arguments: - parameters: - - name: png - value: 'True' - - name: jpg - value: 'True' - - name: jpeg - value: 'True' - - name: tif - value: 'True' - - name: tiff - value: 'True' - - name: create-image - depends: params - template: create-image - withParam: '{{tasks.params.outputs.parameters.out-parameters}}' - arguments: - parameters: - - name: width - value: '{{item.width}}' - - name: height - value: '{{item.height}}' - - name: weights - value: '{{item.weights}}' - - name: extension - value: '{{item.extension}}' - - name: to-hdf5 - depends: create-image - template: to-hdf5 - arguments: - parameters: - - name: paths - value: '{{tasks.create-image.outputs.parameters.out-paths}}' - - name: generate-parameters - inputs: - parameters: - - name: png - - name: jpg - - name: jpeg - - name: tif - - name: tiff - outputs: - parameters: - - name: out-parameters - valueFrom: - path: /tmp/parameters.json - script: - image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: jpeg = json.loads(r'''{{inputs.parameters.jpeg}}''') - except: jpeg = r'''{{inputs.parameters.jpeg}}''' - try: jpg = json.loads(r'''{{inputs.parameters.jpg}}''') - except: jpg = r'''{{inputs.parameters.jpg}}''' - try: png = json.loads(r'''{{inputs.parameters.png}}''') - except: png = r'''{{inputs.parameters.png}}''' - try: tif = json.loads(r'''{{inputs.parameters.tif}}''') - except: tif = r'''{{inputs.parameters.tif}}''' - try: tiff = json.loads(r'''{{inputs.parameters.tiff}}''') - except: tiff = r'''{{inputs.parameters.tiff}}''' - - import json - params: list[dict[str, int | list[int] | str] | None] = [{'width': 500, 'height': 500, 'weights': [255, 1, 100], 'extension': 'png'} if png.lower() == 'true' else None, {'width': 600, 'height': 200, 'weights': [100, 150, 100], 'extension': 'jpg'} if jpg.lower() == 'true' else None, {'width': 300, 'height': 400, 'weights': [100, 150, 100], 'extension': 'jpeg'} if jpeg.lower() == 'true' else None, {'width': 300, 'height': 200, 'weights': [230, 100, 1], 'extension': 'tif'} if tif.lower() == 'true' else None, {'width': 200, 'height': 300, 'weights': [230, 100, 1], 'extension': 'tiff'} if tiff.lower() == 'true' else None] - params_to_write: list[dict[str, int | list[int] | str]] = [image_params for image_params in params if image_params is not None] - with open('/tmp/parameters.json', 'w') as f: - json.dump(params_to_write, f) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: create-image - inputs: - parameters: - - name: width - - name: height - - name: weights - - name: extension - outputs: - artifacts: - - name: '{{inputs.parameters.extension}}-image' - path: /tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}} - archive: - none: {} - parameters: - - name: out-paths - valueFrom: - path: /tmp/{{inputs.parameters.extension}}-path.json - script: - image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: extension = json.loads(r'''{{inputs.parameters.extension}}''') - except: extension = r'''{{inputs.parameters.extension}}''' - try: height = json.loads(r'''{{inputs.parameters.height}}''') - except: height = r'''{{inputs.parameters.height}}''' - try: weights = json.loads(r'''{{inputs.parameters.weights}}''') - except: weights = r'''{{inputs.parameters.weights}}''' - try: width = json.loads(r'''{{inputs.parameters.width}}''') - except: width = r'''{{inputs.parameters.width}}''' - - import json - from PIL import Image - - def create_pattern(width: int, height: int, weights: tuple[int, int, int]) -> Image.Image: - print(f'width: {width}') - print(f'height: {height}') - print(f'RBG weights: {weights}') - image = Image.new('RGB', (width, height)) - pixels = image.load() - for i in range(width): - for j in range(height): - pixels[i, j] = ((i + j * 50) % weights[0], weights[1], (i * 300 + j) % weights[2]) - return image - image = create_pattern(width, height, weights) - path = f'/tmp/{extension}-image.{extension}' - image.save(path) - with open(f'/tmp/{extension}-path.json', 'w') as f: - json.dump(path, f) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: to-hdf5 - inputs: - parameters: - - name: paths - outputs: - artifacts: - - name: hdf5output - path: /tmp/images.hdf5 - archive: - none: {} - script: - image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: paths = json.loads(r'''{{inputs.parameters.paths}}''') - except: paths = r'''{{inputs.parameters.paths}}''' - - import h5py - import numpy as np - from PIL import Image - print('creating hdf5 file') - with h5py.File('/tmp/images.hdf5', 'w') as f: - for i, path in enumerate(paths): - path = path.strip('"') - print(f'Got {path}') - with Image.open(path) as image: - arr = np.array(image) - f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) - print('done') - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - volumeClaimTemplates: - - metadata: - name: tmpdir - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi -{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja index f438048..d3b03e3 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja @@ -1,8 +1,13 @@ {% raw %} +import json +import os + +from hera.shared import global_config from hera.workflows import ( DAG, Artifact, Parameter, + Script, Volume, Workflow, script, # pyright: ignore[reportUnknownVariableType] @@ -10,20 +15,9 @@ from hera.workflows import ( from hera.workflows import models as m from hera.workflows.archive import NoneArchiveStrategy - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], +global_config.set_class_defaults( # pyright: ignore + Script, image=str(os.environ.get("DEFAULT_IMAGE")) ) -def install_dependencies(): - import subprocess - - print("creating venv") - - subprocess.check_call(["python", "-m", "venv", "/tmp/venv"]) - subprocess.check_call( - ["/tmp/venv/bin/pip", "install", "pillow", "h5py", "numpy", "hera"] - ) @script( @@ -67,7 +61,7 @@ def generate_parameters( @script( - command=["/tmp/venv/bin/python"], + command=["python"], volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], outputs=[ Parameter( @@ -117,7 +111,7 @@ def create_image( @script( - command=["/tmp/venv/bin/python"], + command=["python"], volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], outputs=Artifact( name="hdf5output", @@ -145,10 +139,37 @@ def to_hdf5(paths: str): with Workflow( - name="hera-example", # when running on argo this should be generate_name: ...- + pod_spec_patch=json.dumps( + { + "containers": [ + { + "name": "main", + "resources": { + "limits": { + "cpu": "1", + "memory": "1Gi", + }, + "requests": { + "cpu": "1", + "memory": "1Gi", + }, + }, + } + ] + } + ), + tolerations=[ + m.Toleration( + key="nodetype", operator="Equal", value="gpu", effect="NoSchedule" + ), + m.Toleration( + key="nodegroup", operator="Equal", value="workflows", effect="NoSchedule" + ), + ], + name="hera-example", entrypoint="workflowentry", api_version="argoproj.io/v1alpha1", - kind="WorkflowTemplate", # ClusterWorkflowTemplate", when on graphql + kind="WorkflowTemplate", labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, annotations={ "workflows.argoproj.io/title": "example remade via hera", @@ -159,7 +180,6 @@ example.yaml""", volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), ) as w: with DAG(name="workflowentry"): - install = install_dependencies(name="install") params = generate_parameters( name="params", arguments={ @@ -176,9 +196,9 @@ example.yaml""", "paths": makeimages.get_parameter("out-paths"), } ) - [install, params] >> makeimages >> makehdf5 # pyright: ignore + params >> makeimages >> makehdf5 # pyright: ignore -with open("example.txt", "w") as div: +with open("src/{% endraw %}{{project_name}}{% raw %}/templates/example.txt", "w") as div: div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] {% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja deleted file mode 100644 index 3e604a5..0000000 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja +++ /dev/null @@ -1,174 +0,0 @@ -{% raw %} -import os - -from hera.shared import global_config -from hera.workflows import ( - DAG, - Artifact, - Parameter, - Script, - Volume, - Workflow, - script, # pyright: ignore[reportUnknownVariableType] -) -from hera.workflows import models as m -from hera.workflows.archive import NoneArchiveStrategy - -global_config.set_class_defaults( # pyright: ignore - Script, image=str(os.environ.get("DEFAULT_IMAGE")) -) - - -@script( - command=["python"], - outputs=Parameter( - name="out-parameters", value_from=m.ValueFrom(path="/tmp/parameters.json") - ), - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], -) -def generate_parameters( - png: str, - jpg: str, - jpeg: str, - tif: str, - tiff: str, -): - import json - - params: list[dict[str, int | list[int] | str] | None] = [ - {"width": 500, "height": 500, "weights": [255, 1, 100], "extension": "png"} - if png.lower() == "true" - else None, - {"width": 600, "height": 200, "weights": [100, 150, 100], "extension": "jpg"} - if jpg.lower() == "true" - else None, - {"width": 300, "height": 400, "weights": [100, 150, 100], "extension": "jpeg"} - if jpeg.lower() == "true" - else None, - {"width": 300, "height": 200, "weights": [230, 100, 1], "extension": "tif"} - if tif.lower() == "true" - else None, - {"width": 200, "height": 300, "weights": [230, 100, 1], "extension": "tiff"} - if tiff.lower() == "true" - else None, - ] - params_to_write: list[dict[str, int | list[int] | str]] = [ - image_params for image_params in params if image_params is not None - ] - with open("/tmp/parameters.json", "w") as f: - json.dump(params_to_write, f) - - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], - outputs=[ - Parameter( - name="out-paths", - value_from=m.ValueFrom( - path="/tmp/{{inputs.parameters.extension}}-path.json" - ), - ), - Artifact( - name="{{inputs.parameters.extension}}-image", - path="/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}", - archive=NoneArchiveStrategy(), - ), - ], -) -def create_image( - width: int, height: int, weights: tuple[int, int, int], extension: str -): - import json - - from PIL import Image - - def create_pattern( - width: int, - height: int, - weights: tuple[int, int, int], - ) -> Image.Image: - print(f"width: {width}") - print(f"height: {height}") - print(f"RBG weights: {weights}") - image = Image.new("RGB", (width, height)) - pixels = image.load() - for i in range(width): - for j in range(height): - pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript] - (i + j * 50) % weights[0], - weights[1], - (i * 300 + j) % weights[2], - ) - return image - - image = create_pattern(width, height, weights) - path = f"/tmp/{extension}-image.{extension}" - image.save(path) - with open(f"/tmp/{extension}-path.json", "w") as f: - json.dump(path, f) - - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], - outputs=Artifact( - name="hdf5output", - path="/tmp/images.hdf5", - archive=NoneArchiveStrategy(), - ), -) -def to_hdf5(paths: str): - - import h5py # pyright: ignore[reportMissingTypeStubs] - import numpy as np - from PIL import Image - - print("creating hdf5 file") - with h5py.File("/tmp/images.hdf5", "w") as f: - for i, path in enumerate(paths): - path = path.strip('"') - print(f"Got {path}") - with Image.open(path) as image: - arr = np.array(image) - f.create_dataset( # pyright: ignore[reportUnknownMemberType] - f"image_{i}", data=arr, dtype=arr.dtype - ) - print("done") - - -with Workflow( - name="hera-example-in-image", - entrypoint="workflowentry", - api_version="argoproj.io/v1alpha1", - kind="WorkflowTemplate", - labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, - annotations={ - "workflows.argoproj.io/title": "example remade via hera", - "workflows.argoproj.io/description": """Replicates the functionality of -example.yaml""", - "workflows.diamond.ac.uk/repository": "https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}", - }, - volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), -) as w: - with DAG(name="workflowentry"): - params = generate_parameters( - name="params", - arguments={ - "png": "True", - "jpg": "True", - "jpeg": "True", - "tif": "True", - "tiff": "True", - }, - ) - makeimages = create_image(with_param=params.get_parameter("out-parameters")) - makehdf5 = to_hdf5( - arguments={ - "paths": makeimages.get_parameter("out-paths"), - } - ) - params >> makeimages >> makehdf5 # pyright: ignore -with open("example_in_image.txt", "w") as div: - div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] -{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_image_example.ipynb.jinja similarity index 78% rename from src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja rename to src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_image_example.ipynb.jinja index f5c3c68..8eebe05 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_image_example.ipynb.jinja @@ -1,4 +1,5 @@ -{% raw %}{ +{% raw %} +{ "cells": [ { "cell_type": "markdown", @@ -16,14 +17,18 @@ { "cell_type": "code", "execution_count": null, - "id": "494174ef", + "id": "c1d5a92b", "metadata": {}, "outputs": [], "source": [ + "import json\n", + "\n", + "from hera.shared import global_config\n", "from hera.workflows import (\n", " DAG,\n", " Artifact,\n", " Parameter,\n", + " Script,\n", " Volume,\n", " Workflow,\n", " script, # pyright: ignore[reportUnknownVariableType]\n", @@ -31,22 +36,18 @@ "from hera.workflows import models as m\n", "from hera.workflows.archive import NoneArchiveStrategy\n", "\n", - "\n", - "@script(\n", - " command=[\"python\"],\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - ")\n", - "def install_dependencies():\n", - " import subprocess\n", - "\n", - " print(\"creating venv\")\n", - "\n", - " subprocess.check_call([\"python\", \"-m\", \"venv\", \"/tmp/venv\"])\n", - " subprocess.check_call(\n", - " [\"/tmp/venv/bin/pip\", \"install\", \"pillow\", \"h5py\", \"numpy\", \"hera\"]\n", - " )\n", - "\n", - "\n", + "global_config.set_class_defaults( # pyright: ignore\n", + " Script, image=\"ghcr.io/diamondlightsource/python-interface-to-workflows-image\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "494174ef", + "metadata": {}, + "outputs": [], + "source": [ "@script(\n", " command=[\"python\"],\n", " outputs=Parameter(\n", @@ -84,11 +85,18 @@ " image_params for image_params in params if image_params is not None\n", " ]\n", " with open(\"/tmp/parameters.json\", \"w\") as f:\n", - " json.dump(params_to_write, f)\n", - "\n", - "\n", + " json.dump(params_to_write, f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ec312fb", + "metadata": {}, + "outputs": [], + "source": [ "@script(\n", - " command=[\"/tmp/venv/bin/python\"],\n", + " command=[\"python\"],\n", " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", " outputs=[\n", " Parameter(\n", @@ -134,11 +142,18 @@ " path = f\"/tmp/{extension}-image.{extension}\"\n", " image.save(path)\n", " with open(f\"/tmp/{extension}-path.json\", \"w\") as f:\n", - " json.dump(path, f)\n", - "\n", - "\n", + " json.dump(path, f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff25236c", + "metadata": {}, + "outputs": [], + "source": [ "@script(\n", - " command=[\"/tmp/venv/bin/python\"],\n", + " command=[\"python\"],\n", " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", " outputs=Artifact(\n", " name=\"hdf5output\",\n", @@ -162,11 +177,29 @@ " f.create_dataset( # pyright: ignore[reportUnknownMemberType]\n", " f\"image_{i}\", data=arr, dtype=arr.dtype\n", " )\n", - " print(\"done\")\n", - "\n", - "\n", - "with Workflow(\n", - " name=\"hera-example-\",\n", + " print(\"done\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c93a15d1", + "metadata": {}, + "outputs": [], + "source": [ + "with Workflow(pod_spec_patch=json.dumps({\"containers\":\n", + " [{\"name\":\"main\",\n", + " \"resources\":\n", + " {\"limits\":{\"cpu\":\"1\",\n", + " \"memory\":\"1Gi\",\n", + " },\n", + " \"requests\":{\"cpu\":\"1\",\n", + " \"memory\":\"1Gi\",\n", + " }}}]}),\n", + " tolerations=[\n", + " m.Toleration(key=\"nodetype\",operator=\"Equal\",value=\"gpu\",effect=\"NoSchedule\"),\n", + " m.Toleration(key=\"nodegroup\",operator=\"Equal\",value=\"workflows\",effect=\"NoSchedule\")],\n", + " name=\"hera-example\",\n", " entrypoint=\"workflowentry\",\n", " api_version=\"argoproj.io/v1alpha1\",\n", " kind=\"WorkflowTemplate\",\n", @@ -180,7 +213,6 @@ " volumes=Volume(name=\"tmpdir\", mount_path=\"/tmp/\", size=\"1Gi\"),\n", ") as w:\n", " with DAG(name=\"workflowentry\"):\n", - " install = install_dependencies(name=\"install\")\n", " params = generate_parameters(\n", " name=\"params\",\n", " arguments={\n", @@ -197,9 +229,7 @@ " \"paths\": makeimages.get_parameter(\"out-paths\"),\n", " }\n", " )\n", - " [install, params] >> makeimages >> makehdf5 # pyright: ignore\n", - "\n", - "\n" + " params >> makeimages >> makehdf5 # pyright: ignore" ] }, { @@ -210,7 +240,7 @@ "outputs": [], "source": [ "\n", - "with open(\"example.txt\", \"w\") as div:\n", + "with open(\"src/{% endraw %}{{project_name}}{% raw %}/templates/example.txt\", \"w\") as div:\n", " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" ] }, @@ -219,7 +249,8 @@ "execution_count": null, "id": "7e9f88cb", "metadata": {}, - "source": [ + "outputs": [], + "source": [ "from {% endraw %}{{project_name}}{% raw %}.submit_workflow import submit_workflow\n", "\n", "await submit_workflow(w)" @@ -228,7 +259,7 @@ ], "metadata": { "kernelspec": { - "display_name": "{% endraw %}{{repo_name}}{% raw %} (3.11.x)", + "display_name": "{% endraw %}{{repo_name}}{% raw %} (broken)", "language": "python", "name": "python3" }, @@ -242,7 +273,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.15" + "version": "3.11.13" } }, "nbformat": 4, diff --git a/src/python_interface_to_workflows/auth/keycloak_checker.py b/src/python_interface_to_workflows/auth/keycloak_checker.py index 6f6ff11..e024fa0 100644 --- a/src/python_interface_to_workflows/auth/keycloak_checker.py +++ b/src/python_interface_to_workflows/auth/keycloak_checker.py @@ -70,7 +70,7 @@ def set_token_env_variable(staging: bool) -> str: ), ) try: - expire_time = int(token_info["exp"]) + 1800 + expire_time = int(token_info["exp"]) + 1500 dotenv.set_key("src/.env", "EXPIRY", str(expire_time)) dotenv.set_key("src/.env", "TOKEN", token["access_token"].strip("'")) dotenv.set_key("src/.env", "REFRESHTOKEN", token["refresh_token"].strip("'")) diff --git a/src/python_interface_to_workflows/templates/example.txt b/src/python_interface_to_workflows/templates/example.txt index 1ad8e0e..9dedc96 100644 --- a/src/python_interface_to_workflows/templates/example.txt +++ b/src/python_interface_to_workflows/templates/example.txt @@ -12,12 +12,12 @@ metadata: workflows.diamond.ac.uk/science-group-examples: 'true' spec: entrypoint: workflowentry + podSpecPatch: '{"containers": [{"name": "main", "resources": {"limits": {"cpu": + "1", "memory": "1Gi"}, "requests": {"cpu": "1", "memory": "1Gi"}}}]}' templates: - name: workflowentry dag: tasks: - - name: install - template: install-dependencies - name: params template: generate-parameters arguments: @@ -33,7 +33,7 @@ spec: - name: tiff value: 'True' - name: create-image - depends: install && params + depends: params template: create-image withParam: '{{tasks.params.outputs.parameters.out-parameters}}' arguments: @@ -53,22 +53,6 @@ spec: parameters: - name: paths value: '{{tasks.create-image.outputs.parameters.out-paths}}' - - name: install-dependencies - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import subprocess - print('creating venv') - subprocess.check_call(['python', '-m', 'venv', '/tmp/venv']) - subprocess.check_call(['/tmp/venv/bin/pip', 'install', 'pillow', 'h5py', 'numpy', 'hera']) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - name: generate-parameters inputs: parameters: @@ -83,7 +67,7 @@ spec: valueFrom: path: /tmp/parameters.json script: - image: python:3.10 + image: ghcr.io/matt-carre/python-interface-to-workflows-default-image source: |- import os import sys @@ -128,7 +112,7 @@ spec: valueFrom: path: /tmp/{{inputs.parameters.extension}}-path.json script: - image: python:3.10 + image: ghcr.io/matt-carre/python-interface-to-workflows-default-image source: |- import os import sys @@ -162,7 +146,7 @@ spec: with open(f'/tmp/{extension}-path.json', 'w') as f: json.dump(path, f) command: - - /tmp/venv/bin/python + - python volumeMounts: - name: tmpdir mountPath: /tmp @@ -177,7 +161,7 @@ spec: archive: none: {} script: - image: python:3.10 + image: ghcr.io/matt-carre/python-interface-to-workflows-default-image source: |- import os import sys @@ -199,10 +183,19 @@ spec: f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) print('done') command: - - /tmp/venv/bin/python + - python volumeMounts: - name: tmpdir mountPath: /tmp + tolerations: + - effect: NoSchedule + key: nodetype + operator: Equal + value: gpu + - effect: NoSchedule + key: nodegroup + operator: Equal + value: workflows volumeClaimTemplates: - metadata: name: tmpdir diff --git a/src/python_interface_to_workflows/templates/example_in_image.txt b/src/python_interface_to_workflows/templates/example_in_image.txt deleted file mode 100644 index ac6be98..0000000 --- a/src/python_interface_to_workflows/templates/example_in_image.txt +++ /dev/null @@ -1,196 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: hera-example-in-image - annotations: - workflows.argoproj.io/description: |- - Replicates the functionality of - example.yaml - workflows.argoproj.io/title: example remade via hera - workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows - labels: - workflows.diamond.ac.uk/science-group-examples: 'true' -spec: - entrypoint: workflowentry - templates: - - name: workflowentry - dag: - tasks: - - name: params - template: generate-parameters - arguments: - parameters: - - name: png - value: 'True' - - name: jpg - value: 'True' - - name: jpeg - value: 'True' - - name: tif - value: 'True' - - name: tiff - value: 'True' - - name: create-image - depends: params - template: create-image - withParam: '{{tasks.params.outputs.parameters.out-parameters}}' - arguments: - parameters: - - name: width - value: '{{item.width}}' - - name: height - value: '{{item.height}}' - - name: weights - value: '{{item.weights}}' - - name: extension - value: '{{item.extension}}' - - name: to-hdf5 - depends: create-image - template: to-hdf5 - arguments: - parameters: - - name: paths - value: '{{tasks.create-image.outputs.parameters.out-paths}}' - - name: generate-parameters - inputs: - parameters: - - name: png - - name: jpg - - name: jpeg - - name: tif - - name: tiff - outputs: - parameters: - - name: out-parameters - valueFrom: - path: /tmp/parameters.json - script: - image: ghcr.io/matt-carre/python-interface-to-workflows-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: jpeg = json.loads(r'''{{inputs.parameters.jpeg}}''') - except: jpeg = r'''{{inputs.parameters.jpeg}}''' - try: jpg = json.loads(r'''{{inputs.parameters.jpg}}''') - except: jpg = r'''{{inputs.parameters.jpg}}''' - try: png = json.loads(r'''{{inputs.parameters.png}}''') - except: png = r'''{{inputs.parameters.png}}''' - try: tif = json.loads(r'''{{inputs.parameters.tif}}''') - except: tif = r'''{{inputs.parameters.tif}}''' - try: tiff = json.loads(r'''{{inputs.parameters.tiff}}''') - except: tiff = r'''{{inputs.parameters.tiff}}''' - - import json - params: list[dict[str, int | list[int] | str] | None] = [{'width': 500, 'height': 500, 'weights': [255, 1, 100], 'extension': 'png'} if png.lower() == 'true' else None, {'width': 600, 'height': 200, 'weights': [100, 150, 100], 'extension': 'jpg'} if jpg.lower() == 'true' else None, {'width': 300, 'height': 400, 'weights': [100, 150, 100], 'extension': 'jpeg'} if jpeg.lower() == 'true' else None, {'width': 300, 'height': 200, 'weights': [230, 100, 1], 'extension': 'tif'} if tif.lower() == 'true' else None, {'width': 200, 'height': 300, 'weights': [230, 100, 1], 'extension': 'tiff'} if tiff.lower() == 'true' else None] - params_to_write: list[dict[str, int | list[int] | str]] = [image_params for image_params in params if image_params is not None] - with open('/tmp/parameters.json', 'w') as f: - json.dump(params_to_write, f) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: create-image - inputs: - parameters: - - name: width - - name: height - - name: weights - - name: extension - outputs: - artifacts: - - name: '{{inputs.parameters.extension}}-image' - path: /tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}} - archive: - none: {} - parameters: - - name: out-paths - valueFrom: - path: /tmp/{{inputs.parameters.extension}}-path.json - script: - image: ghcr.io/matt-carre/python-interface-to-workflows-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: extension = json.loads(r'''{{inputs.parameters.extension}}''') - except: extension = r'''{{inputs.parameters.extension}}''' - try: height = json.loads(r'''{{inputs.parameters.height}}''') - except: height = r'''{{inputs.parameters.height}}''' - try: weights = json.loads(r'''{{inputs.parameters.weights}}''') - except: weights = r'''{{inputs.parameters.weights}}''' - try: width = json.loads(r'''{{inputs.parameters.width}}''') - except: width = r'''{{inputs.parameters.width}}''' - - import json - from PIL import Image - - def create_pattern(width: int, height: int, weights: tuple[int, int, int]) -> Image.Image: - print(f'width: {width}') - print(f'height: {height}') - print(f'RBG weights: {weights}') - image = Image.new('RGB', (width, height)) - pixels = image.load() - for i in range(width): - for j in range(height): - pixels[i, j] = ((i + j * 50) % weights[0], weights[1], (i * 300 + j) % weights[2]) - return image - image = create_pattern(width, height, weights) - path = f'/tmp/{extension}-image.{extension}' - image.save(path) - with open(f'/tmp/{extension}-path.json', 'w') as f: - json.dump(path, f) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: to-hdf5 - inputs: - parameters: - - name: paths - outputs: - artifacts: - - name: hdf5output - path: /tmp/images.hdf5 - archive: - none: {} - script: - image: ghcr.io/matt-carre/python-interface-to-workflows-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: paths = json.loads(r'''{{inputs.parameters.paths}}''') - except: paths = r'''{{inputs.parameters.paths}}''' - - import h5py - import numpy as np - from PIL import Image - print('creating hdf5 file') - with h5py.File('/tmp/images.hdf5', 'w') as f: - for i, path in enumerate(paths): - path = path.strip('"') - print(f'Got {path}') - with Image.open(path) as image: - arr = np.array(image) - f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) - print('done') - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - volumeClaimTemplates: - - metadata: - name: tmpdir - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi diff --git a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py index 013387a..804351a 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py @@ -1,7 +1,12 @@ +import json +import os + +from hera.shared import global_config from hera.workflows import ( DAG, Artifact, Parameter, + Script, Volume, Workflow, script, # pyright: ignore[reportUnknownVariableType] @@ -9,20 +14,9 @@ from hera.workflows import models as m from hera.workflows.archive import NoneArchiveStrategy - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], +global_config.set_class_defaults( # pyright: ignore + Script, image=str(os.environ.get("DEFAULT_IMAGE")) ) -def install_dependencies(): - import subprocess - - print("creating venv") - - subprocess.check_call(["python", "-m", "venv", "/tmp/venv"]) - subprocess.check_call( - ["/tmp/venv/bin/pip", "install", "pillow", "h5py", "numpy", "hera"] - ) @script( @@ -66,7 +60,7 @@ def generate_parameters( @script( - command=["/tmp/venv/bin/python"], + command=["python"], volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], outputs=[ Parameter( @@ -116,7 +110,7 @@ def create_pattern( @script( - command=["/tmp/venv/bin/python"], + command=["python"], volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], outputs=Artifact( name="hdf5output", @@ -144,10 +138,37 @@ def to_hdf5(paths: str): with Workflow( - name="hera-example", # when running on argo this should be generate_name: ...- + pod_spec_patch=json.dumps( + { + "containers": [ + { + "name": "main", + "resources": { + "limits": { + "cpu": "1", + "memory": "1Gi", + }, + "requests": { + "cpu": "1", + "memory": "1Gi", + }, + }, + } + ] + } + ), + tolerations=[ + m.Toleration( + key="nodetype", operator="Equal", value="gpu", effect="NoSchedule" + ), + m.Toleration( + key="nodegroup", operator="Equal", value="workflows", effect="NoSchedule" + ), + ], + name="hera-example", entrypoint="workflowentry", api_version="argoproj.io/v1alpha1", - kind="WorkflowTemplate", # ClusterWorkflowTemplate", when on graphql + kind="WorkflowTemplate", labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, annotations={ "workflows.argoproj.io/title": "example remade via hera", @@ -158,7 +179,6 @@ def to_hdf5(paths: str): volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), ) as w: with DAG(name="workflowentry"): - install = install_dependencies(name="install") params = generate_parameters( name="params", arguments={ @@ -175,8 +195,8 @@ def to_hdf5(paths: str): "paths": makeimages.get_parameter("out-paths"), } ) - [install, params] >> makeimages >> makehdf5 # pyright: ignore + params >> makeimages >> makehdf5 # pyright: ignore -with open("example.txt", "w") as div: +with open("src/python_interface_to_workflows/templates/example.txt", "w") as div: div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] diff --git a/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py b/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py deleted file mode 100644 index 779921c..0000000 --- a/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py +++ /dev/null @@ -1,172 +0,0 @@ -import os - -from hera.shared import global_config -from hera.workflows import ( - DAG, - Artifact, - Parameter, - Script, - Volume, - Workflow, - script, # pyright: ignore[reportUnknownVariableType] -) -from hera.workflows import models as m -from hera.workflows.archive import NoneArchiveStrategy - -global_config.set_class_defaults( # pyright: ignore - Script, image=str(os.environ.get("DEFAULT_IMAGE")) -) - - -@script( - command=["python"], - outputs=Parameter( - name="out-parameters", value_from=m.ValueFrom(path="/tmp/parameters.json") - ), - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], -) -def generate_parameters( - png: str, - jpg: str, - jpeg: str, - tif: str, - tiff: str, -): - import json - - params: list[dict[str, int | list[int] | str] | None] = [ - {"width": 500, "height": 500, "weights": [255, 1, 100], "extension": "png"} - if png.lower() == "true" - else None, - {"width": 600, "height": 200, "weights": [100, 150, 100], "extension": "jpg"} - if jpg.lower() == "true" - else None, - {"width": 300, "height": 400, "weights": [100, 150, 100], "extension": "jpeg"} - if jpeg.lower() == "true" - else None, - {"width": 300, "height": 200, "weights": [230, 100, 1], "extension": "tif"} - if tif.lower() == "true" - else None, - {"width": 200, "height": 300, "weights": [230, 100, 1], "extension": "tiff"} - if tiff.lower() == "true" - else None, - ] - params_to_write: list[dict[str, int | list[int] | str]] = [ - image_params for image_params in params if image_params is not None - ] - with open("/tmp/parameters.json", "w") as f: - json.dump(params_to_write, f) - - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], - outputs=[ - Parameter( - name="out-paths", - value_from=m.ValueFrom( - path="/tmp/{{inputs.parameters.extension}}-path.json" - ), - ), - Artifact( - name="{{inputs.parameters.extension}}-image", - path="/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}", - archive=NoneArchiveStrategy(), - ), - ], -) -def create_image( - width: int, height: int, weights: tuple[int, int, int], extension: str -): - import json - - from PIL import Image - - def create_pattern( - width: int, - height: int, - weights: tuple[int, int, int], - ) -> Image.Image: - print(f"width: {width}") - print(f"height: {height}") - print(f"RBG weights: {weights}") - image = Image.new("RGB", (width, height)) - pixels = image.load() - for i in range(width): - for j in range(height): - pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript] - (i + j * 50) % weights[0], - weights[1], - (i * 300 + j) % weights[2], - ) - return image - - image = create_pattern(width, height, weights) - path = f"/tmp/{extension}-image.{extension}" - image.save(path) - with open(f"/tmp/{extension}-path.json", "w") as f: - json.dump(path, f) - - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], - outputs=Artifact( - name="hdf5output", - path="/tmp/images.hdf5", - archive=NoneArchiveStrategy(), - ), -) -def to_hdf5(paths: str): - - import h5py # pyright: ignore[reportMissingTypeStubs] - import numpy as np - from PIL import Image - - print("creating hdf5 file") - with h5py.File("/tmp/images.hdf5", "w") as f: - for i, path in enumerate(paths): - path = path.strip('"') - print(f"Got {path}") - with Image.open(path) as image: - arr = np.array(image) - f.create_dataset( # pyright: ignore[reportUnknownMemberType] - f"image_{i}", data=arr, dtype=arr.dtype - ) - print("done") - - -with Workflow( - name="hera-example-in-image", - entrypoint="workflowentry", - api_version="argoproj.io/v1alpha1", - kind="WorkflowTemplate", - labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, - annotations={ - "workflows.argoproj.io/title": "example remade via hera", - "workflows.argoproj.io/description": """Replicates the functionality of -example.yaml""", - "workflows.diamond.ac.uk/repository": "https://github.com/DiamondLightSource/python-interface-to-workflows", - }, - volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), -) as w: - with DAG(name="workflowentry"): - params = generate_parameters( - name="params", - arguments={ - "png": "True", - "jpg": "True", - "jpeg": "True", - "tif": "True", - "tiff": "True", - }, - ) - makeimages = create_image(with_param=params.get_parameter("out-parameters")) - makehdf5 = to_hdf5( - arguments={ - "paths": makeimages.get_parameter("out-paths"), - } - ) - params >> makeimages >> makehdf5 # pyright: ignore -with open("example_in_image.txt", "w") as div: - div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb index 4d0e147..de4001c 100644 --- a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb +++ b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb @@ -64,8 +64,8 @@ " volumes=EmptyDirVolume(name=\"output-dir\", mount_path=\"/output-dir\"),\n", ") as w:\n", " with Steps(name=\"divide\"):\n", - " do_division(name=\"first\", arguments={\"a\": 2, \"b\": 5})\n", - "\n" + " do_division(name=\"first\", arguments={\"a\":\"{{.Files.get '/notebooks/a.json'}}\",\n", + " \"b\":\"{{.Files.get '/notebooks/b.json'}}\"}) # file.get here\n" ] }, { diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_image_example.ipynb similarity index 80% rename from src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb rename to src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_image_example.ipynb index 1d7bfe4..ef7becc 100644 --- a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb +++ b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_image_example.ipynb @@ -16,14 +16,18 @@ { "cell_type": "code", "execution_count": null, - "id": "494174ef", + "id": "c1d5a92b", "metadata": {}, "outputs": [], "source": [ + "import json\n", + "\n", + "from hera.shared import global_config\n", "from hera.workflows import (\n", " DAG,\n", " Artifact,\n", " Parameter,\n", + " Script,\n", " Volume,\n", " Workflow,\n", " script, # pyright: ignore[reportUnknownVariableType]\n", @@ -31,22 +35,18 @@ "from hera.workflows import models as m\n", "from hera.workflows.archive import NoneArchiveStrategy\n", "\n", - "\n", - "@script(\n", - " command=[\"python\"],\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - ")\n", - "def install_dependencies():\n", - " import subprocess\n", - "\n", - " print(\"creating venv\")\n", - "\n", - " subprocess.check_call([\"python\", \"-m\", \"venv\", \"/tmp/venv\"])\n", - " subprocess.check_call(\n", - " [\"/tmp/venv/bin/pip\", \"install\", \"pillow\", \"h5py\", \"numpy\", \"hera\"]\n", - " )\n", - "\n", - "\n", + "global_config.set_class_defaults( # pyright: ignore\n", + " Script, image=str(os.environ.get(\"DEFAULT_IMAGE\"))\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "494174ef", + "metadata": {}, + "outputs": [], + "source": [ "@script(\n", " command=[\"python\"],\n", " outputs=Parameter(\n", @@ -84,11 +84,18 @@ " image_params for image_params in params if image_params is not None\n", " ]\n", " with open(\"/tmp/parameters.json\", \"w\") as f:\n", - " json.dump(params_to_write, f)\n", - "\n", - "\n", + " json.dump(params_to_write, f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ec312fb", + "metadata": {}, + "outputs": [], + "source": [ "@script(\n", - " command=[\"/tmp/venv/bin/python\"],\n", + " command=[\"python\"],\n", " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", " outputs=[\n", " Parameter(\n", @@ -134,11 +141,18 @@ " path = f\"/tmp/{extension}-image.{extension}\"\n", " image.save(path)\n", " with open(f\"/tmp/{extension}-path.json\", \"w\") as f:\n", - " json.dump(path, f)\n", - "\n", - "\n", + " json.dump(path, f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff25236c", + "metadata": {}, + "outputs": [], + "source": [ "@script(\n", - " command=[\"/tmp/venv/bin/python\"],\n", + " command=[\"python\"],\n", " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", " outputs=Artifact(\n", " name=\"hdf5output\",\n", @@ -162,10 +176,28 @@ " f.create_dataset( # pyright: ignore[reportUnknownMemberType]\n", " f\"image_{i}\", data=arr, dtype=arr.dtype\n", " )\n", - " print(\"done\")\n", - "\n", - "\n", - "with Workflow(\n", + " print(\"done\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c93a15d1", + "metadata": {}, + "outputs": [], + "source": [ + "with Workflow(pod_spec_patch=json.dumps({\"containers\":\n", + " [{\"name\":\"main\",\n", + " \"resources\":\n", + " {\"limits\":{\"cpu\":\"1\",\n", + " \"memory\":\"1Gi\",\n", + " },\n", + " \"requests\":{\"cpu\":\"1\",\n", + " \"memory\":\"1Gi\",\n", + " }}}]}),\n", + " tolerations=[\n", + " m.Toleration(key=\"nodetype\",operator=\"Equal\",value=\"gpu\",effect=\"NoSchedule\"),\n", + " m.Toleration(key=\"nodegroup\",operator=\"Equal\",value=\"workflows\",effect=\"NoSchedule\")],\n", " name=\"hera-example\",\n", " entrypoint=\"workflowentry\",\n", " api_version=\"argoproj.io/v1alpha1\",\n", @@ -180,7 +212,6 @@ " volumes=Volume(name=\"tmpdir\", mount_path=\"/tmp/\", size=\"1Gi\"),\n", ") as w:\n", " with DAG(name=\"workflowentry\"):\n", - " install = install_dependencies(name=\"install\")\n", " params = generate_parameters(\n", " name=\"params\",\n", " arguments={\n", @@ -197,9 +228,7 @@ " \"paths\": makeimages.get_parameter(\"out-paths\"),\n", " }\n", " )\n", - " [install, params] >> makeimages >> makehdf5 # pyright: ignore\n", - "\n", - "\n" + " params >> makeimages >> makehdf5 # pyright: ignore" ] }, { @@ -210,7 +239,7 @@ "outputs": [], "source": [ "\n", - "with open(\"../../templates/example_from_jupyter.txt\", \"w\") as div:\n", + "with open(\"src/python_interface_to_workflows/templates/example.txt\", \"w\") as div:\n", " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" ] }, diff --git a/tests/test_keycloak_checker.py b/tests/test_keycloak_checker.py index c658d0c..095133c 100644 --- a/tests/test_keycloak_checker.py +++ b/tests/test_keycloak_checker.py @@ -64,7 +64,7 @@ def test_set_token_env_variable( keycloak.token.assert_not_called() mock_set_key.assert_has_calls( [ - call("src/.env", "EXPIRY", str(123456789 + 1800)), + call("src/.env", "EXPIRY", str(123456789 + 1500)), call("src/.env", "TOKEN", "fake_token"), call("src/.env", "REFRESHTOKEN", "fake_refresh"), ] @@ -97,8 +97,7 @@ def test_set_token_env_variable_attribute_error( mock_gen_code_challenge.return_value = ("challenge", "S256") mock_token_expired.return_value = True - os.environ["REFRESHTOKEN"] = "refresh" - + os.environ["REFRESHTOKEN"] = "fake_refresh" keycloak = MagicMock() mock_gen_keycloak_id.return_value = keycloak From f53b4407708c611705f17bdc0552d8db64f19b59 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 5 Aug 2026 09:30:29 +0000 Subject: [PATCH 07/15] fix(auth): removes erroring element in auth --- .../src/{{ project_name }}/auth/keycloak_checker.py.jinja | 2 +- src/copier_template/tests/test_keycloak_checker.py.jinja | 2 +- src/python_interface_to_workflows/auth/keycloak_checker.py | 2 +- tests/test_keycloak_checker.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja b/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja index ff9db43..3f6fa48 100644 --- a/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja +++ b/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja @@ -59,7 +59,7 @@ def set_token_env_variable() -> str: ), ) try: - expire_time = int(token_info["exp"]) + 1800 + expire_time = int(token_info["exp"]) dotenv.set_key("src/.env", "EXPIRY", str(expire_time)) dotenv.set_key("src/.env", "TOKEN", token["access_token"].strip("'")) dotenv.set_key("src/.env", "REFRESHTOKEN", token["refresh_token"].strip("'")) diff --git a/src/copier_template/tests/test_keycloak_checker.py.jinja b/src/copier_template/tests/test_keycloak_checker.py.jinja index b960007..d2619a6 100644 --- a/src/copier_template/tests/test_keycloak_checker.py.jinja +++ b/src/copier_template/tests/test_keycloak_checker.py.jinja @@ -61,7 +61,7 @@ def test_set_token_env_variable( keycloak.token.assert_not_called() mock_set_key.assert_has_calls( [ - call("src/.env", "EXPIRY", str(123456789 + 1800)), + call("src/.env", "EXPIRY", str(123456789)), call("src/.env", "TOKEN", "fake_token"), call("src/.env", "REFRESHTOKEN", "fake_refresh"), ] diff --git a/src/python_interface_to_workflows/auth/keycloak_checker.py b/src/python_interface_to_workflows/auth/keycloak_checker.py index e024fa0..8474fa7 100644 --- a/src/python_interface_to_workflows/auth/keycloak_checker.py +++ b/src/python_interface_to_workflows/auth/keycloak_checker.py @@ -70,7 +70,7 @@ def set_token_env_variable(staging: bool) -> str: ), ) try: - expire_time = int(token_info["exp"]) + 1500 + expire_time = int(token_info["exp"]) dotenv.set_key("src/.env", "EXPIRY", str(expire_time)) dotenv.set_key("src/.env", "TOKEN", token["access_token"].strip("'")) dotenv.set_key("src/.env", "REFRESHTOKEN", token["refresh_token"].strip("'")) diff --git a/tests/test_keycloak_checker.py b/tests/test_keycloak_checker.py index 095133c..ace1387 100644 --- a/tests/test_keycloak_checker.py +++ b/tests/test_keycloak_checker.py @@ -64,7 +64,7 @@ def test_set_token_env_variable( keycloak.token.assert_not_called() mock_set_key.assert_has_calls( [ - call("src/.env", "EXPIRY", str(123456789 + 1500)), + call("src/.env", "EXPIRY", str(123456789)), call("src/.env", "TOKEN", "fake_token"), call("src/.env", "REFRESHTOKEN", "fake_refresh"), ] From 8c248c2cee5c3311ada4aee9c6f3d3789867f279 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Tue, 4 Aug 2026 16:33:04 +0000 Subject: [PATCH 08/15] feat(images): adds new image with mounted_files path --- .github/workflows/_update_image.yml | 2 +- Dockerfile | 1 + .../templates/example_import_files.txt.jinja | 65 ++++++++++ .../templates/example_import_files.txt | 63 ++++++++++ .../create_notebook_in_image.py | 94 +++++++++++++++ .../mounted_files/pandas.ipynb | 111 ++++++++++++++++++ .../mounted_files/requirements.txt | 8 ++ 7 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja create mode 100644 src/python_interface_to_workflows/templates/example_import_files.txt create mode 100644 src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py create mode 100644 src/python_interface_to_workflows/workflow_definitions/mounted_files/pandas.ipynb create mode 100644 src/python_interface_to_workflows/workflow_definitions/mounted_files/requirements.txt diff --git a/.github/workflows/_update_image.yml b/.github/workflows/_update_image.yml index 022f244..5f7ee7d 100644 --- a/.github/workflows/_update_image.yml +++ b/.github/workflows/_update_image.yml @@ -14,7 +14,7 @@ jobs: uses: actions/checkout@v6 - name: Generate Image Name - run: echo IMAGE_REPOSITORY=ghcr.io/$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]' | tr '[_]' '[\-]')-image >> $GITHUB_ENV + run: echo IMAGE_REPOSITORY=ghcr.io/$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]' | tr '[_]' '[\-]')-mounted-in-image >> $GITHUB_ENV - name: Log in to GitHub Docker Registry if: github.event_name != 'pull_request' diff --git a/Dockerfile b/Dockerfile index 0736493..33def25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,7 @@ COPY --from=build /python /python # Copy the environment, but not the source code COPY --from=build /app/.venv /app/.venv +COPY --from=build /app/src/python_interface_to_workflows/workflow_definitions/mounted_files /mounted_files/ ENV PATH=/app/.venv/bin:$PATH # change this entrypoint if it is not the same as the repo diff --git a/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja new file mode 100644 index 0000000..480b685 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja @@ -0,0 +1,65 @@ +{% raw %} +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: hera-example-pandas + annotations: + workflows.argoproj.io/description: |- + Replicates the functionality of + notebook.yaml + workflows.argoproj.io/title: notebook.yaml remade via hera + workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %} + labels: + workflows.diamond.ac.uk/science-group-examples: 'true' +spec: + entrypoint: workflowentry + podSpecPatch: '{"containers": [{"name": "main", "resources": {"limits": {"cpu": + "500m", "memory": "2Gi"}, "requests": {"cpu": "500m", "memory": "2Gi"}}}]}' + templates: + - name: workflowentry + dag: + tasks: + - name: mount-files + template: mount-files + - name: mount-files + outputs: + artifacts: + - name: notebook + path: /tmp/notebook.html + archive: + none: {} + script: + image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-mounted-image:latest + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import subprocess + subprocess.call('python -m venv /tmp/venv', shell=True) + subprocess.call('/tmp/venv/bin/pip install -r /mounted_files/requirements.txt', shell=True) + subprocess.call('/tmp/venv/bin/python -m ipykernel install --prefix=/tmp/venv --name=venv', shell=True) + subprocess.call('/tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /mounted_files/notebook.ipynb', shell=True) + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + tolerations: + - effect: NoSchedule + key: nodetype + operator: Equal + value: gpu + - effect: NoSchedule + key: nodegroup + operator: Equal + value: workflows + volumeClaimTemplates: + - metadata: + name: tmpdir + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +{% endraw %} diff --git a/src/python_interface_to_workflows/templates/example_import_files.txt b/src/python_interface_to_workflows/templates/example_import_files.txt new file mode 100644 index 0000000..8d4bc9b --- /dev/null +++ b/src/python_interface_to_workflows/templates/example_import_files.txt @@ -0,0 +1,63 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: hera-example-pandas + annotations: + workflows.argoproj.io/description: |- + Replicates the functionality of + notebook.yaml + workflows.argoproj.io/title: notebook.yaml remade via hera + workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows + labels: + workflows.diamond.ac.uk/science-group-examples: 'true' +spec: + entrypoint: workflowentry + podSpecPatch: '{"containers": [{"name": "main", "resources": {"limits": {"cpu": + "500m", "memory": "2Gi"}, "requests": {"cpu": "500m", "memory": "2Gi"}}}]}' + templates: + - name: workflowentry + dag: + tasks: + - name: mount-files + template: mount-files + - name: mount-files + outputs: + artifacts: + - name: notebook + path: /tmp/notebook.html + archive: + none: {} + script: + image: ghcr.io/matt-carre/python-interface-to-workflows-mounted-image:latest + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import subprocess + subprocess.call('python -m venv /tmp/venv', shell=True) + subprocess.call('/tmp/venv/bin/pip install -r /mounted_files/requirements.txt', shell=True) + subprocess.call('/tmp/venv/bin/python -m ipykernel install --prefix=/tmp/venv --name=venv', shell=True) + subprocess.call('/tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /mounted_files/notebook.ipynb', shell=True) + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + tolerations: + - effect: NoSchedule + key: nodetype + operator: Equal + value: gpu + - effect: NoSchedule + key: nodegroup + operator: Equal + value: workflows + volumeClaimTemplates: + - metadata: + name: tmpdir + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py b/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py new file mode 100644 index 0000000..c7f685a --- /dev/null +++ b/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py @@ -0,0 +1,94 @@ +import json + +from hera.shared import global_config +from hera.workflows import ( + DAG, + Artifact, + Script, + Volume, + Workflow, + script, # pyright: ignore[reportUnknownVariableType] +) +from hera.workflows import models as m +from hera.workflows.archive import NoneArchiveStrategy + +global_config.set_class_defaults( # pyright: ignore + Script, + image="ghcr.io/matt-carre/python-interface-to-workflows-mounted-image:latest", +) + + +@script( + command=["python"], + outputs=Artifact( + name="notebook", path="/tmp/notebook.html", archive=NoneArchiveStrategy() + ), + volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], +) +def mount_files(): + import subprocess + + subprocess.call("python -m venv /tmp/venv", shell=True) + subprocess.call( + "/tmp/venv/bin/pip install -r /mounted_files/requirements.txt", shell=True + ) + subprocess.call( + "/tmp/venv/bin/python -m ipykernel install --prefix=/tmp/venv --name=venv", + shell=True, + ) + subprocess.call( + "/tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /mounted_files/notebook.ipynb", # noqa: E501 + shell=True, + ) + + +with Workflow( + pod_spec_patch=json.dumps( + { + "containers": [ + { + "name": "main", + "resources": { + "limits": { + "cpu": "500m", + "memory": "2Gi", + }, + "requests": { + "cpu": "500m", + "memory": "2Gi", + }, + }, + } + ] + } + ), + tolerations=[ + m.Toleration( + key="nodetype", operator="Equal", value="gpu", effect="NoSchedule" + ), + m.Toleration( + key="nodegroup", operator="Equal", value="workflows", effect="NoSchedule" + ), + ], + name="hera-example-pandas", + entrypoint="workflowentry", + api_version="argoproj.io/v1alpha1", + kind="WorkflowTemplate", + labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, + annotations={ + "workflows.argoproj.io/title": "notebook.yaml remade via hera", + "workflows.argoproj.io/description": """Replicates the functionality of +notebook.yaml""", + "workflows.diamond.ac.uk/repository": "https://github.com/DiamondLightSource/python-interface-to-workflows", + }, + volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), +) as w: + with DAG(name="workflowentry"): + files = mount_files() + files # pyright: ignore # noqa: B018 + + +with open( + "src/python_interface_to_workflows/templates/example_import_files.txt", "w" +) as div: + div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] diff --git a/src/python_interface_to_workflows/workflow_definitions/mounted_files/pandas.ipynb b/src/python_interface_to_workflows/workflow_definitions/mounted_files/pandas.ipynb new file mode 100644 index 0000000..96b333a --- /dev/null +++ b/src/python_interface_to_workflows/workflow_definitions/mounted_files/pandas.ipynb @@ -0,0 +1,111 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Manipulation and Visualization Example\n", + "\n", + "This notebook demonstrates:\n", + "- Creating synthetic test data\n", + "- Performing data manipulation with pandas\n", + "- Visualizing the results with matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Set environment variables\n", + "os.environ[\"MPLCONFIGDIR\"] = \"/tmp/.config/matplotlib\"\n", + "\n", + "# Ensure the directories exist\n", + "os.makedirs(os.environ[\"MPLCONFIGDIR\"], exist_ok=True)\n", + "\n", + "# Import required libraries\n", + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Set a random seed for reproducibility\n", + "np.random.seed(42)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create synthetic test data\n", + "dates = pd.date_range(start='2023-01-01', periods=100)\n", + "categories = ['A', 'B', 'C']\n", + "\n", + "data = pd.DataFrame({\n", + " 'Date': dates,\n", + " 'Category': np.random.choice(categories, size=100),\n", + " 'Value': np.random.normal(loc=50, scale=10, size=100)\n", + "})\n", + "\n", + "# Display first few rows\n", + "data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Data manipulation\n", + "# 1. Add a rolling average column\n", + "data['RollingAvg'] = data['Value'].rolling(window=7).mean()\n", + "\n", + "# 2. Group by Category and calculate mean value\n", + "category_means = data.groupby('Category')['Value'].mean()\n", + "category_means" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Visualization\n", + "plt.figure(figsize=(12, 6))\n", + "\n", + "# Plot the original Value and Rolling Average for each category\n", + "for cat in data['Category'].unique():\n", + " subset = data[data['Category'] == cat]\n", + " plt.plot(subset['Date'], subset['Value'], label=f'{cat} Value', alpha=0.3)\n", + " plt.plot(subset['Date'], subset['RollingAvg'], label=f'{cat} RollingAvg')\n", + "\n", + "plt.xlabel('Date')\n", + "plt.ylabel('Value')\n", + "plt.title('Value and Rolling Average by Category')\n", + "plt.legend()\n", + "plt.grid(True)\n", + "plt.tight_layout()\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/python_interface_to_workflows/workflow_definitions/mounted_files/requirements.txt b/src/python_interface_to_workflows/workflow_definitions/mounted_files/requirements.txt new file mode 100644 index 0000000..58a5482 --- /dev/null +++ b/src/python_interface_to_workflows/workflow_definitions/mounted_files/requirements.txt @@ -0,0 +1,8 @@ +numpy==2.2.5 +matplotlib==3.10.3 +pandas==2.2.3 +scipy==1.15.3 +nbconvert==7.17.1 +ipykernel==6.29.5 +ipython==9.2.0 +papermill==2.6.0 From bfe33132049d113bd2883a32de5d157e5fc1ba88 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 5 Aug 2026 08:21:28 +0000 Subject: [PATCH 09/15] feat(mounts): adds a new example which uses mounted files --- src/copier_template/Dockerfile.jinja | 1 + .../templates/example_import_files.txt.jinja | 2 +- .../create_notebook_in_image.py.jinja | 96 +++++++++++++++ .../mounted_files/pandas.ipynb | 111 ++++++++++++++++++ .../mounted_files/requirements.txt | 8 ++ src/python_interface_to_workflows/__main__.py | 12 ++ .../templates/example_import_files.txt | 2 +- .../create_notebook_in_image.py | 8 +- 8 files changed, 234 insertions(+), 6 deletions(-) create mode 100644 src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja create mode 100644 src/copier_template/src/{{ project_name }}/workflow_definitions/mounted_files/pandas.ipynb create mode 100644 src/copier_template/src/{{ project_name }}/workflow_definitions/mounted_files/requirements.txt diff --git a/src/copier_template/Dockerfile.jinja b/src/copier_template/Dockerfile.jinja index 145b50a..14f5a33 100644 --- a/src/copier_template/Dockerfile.jinja +++ b/src/copier_template/Dockerfile.jinja @@ -45,6 +45,7 @@ COPY --from=build /python /python # Copy the environment, but not the source code COPY --from=build /app/.venv /app/.venv +COPY --from=build /app/src/{{project_name}}/workflow_definitions/mounted_files /mounted_files/ ENV PATH=/app/.venv/bin:$PATH # change this entrypoint if it is not the same as the repo diff --git a/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja index 480b685..18e5936 100644 --- a/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja @@ -38,7 +38,7 @@ spec: subprocess.call('python -m venv /tmp/venv', shell=True) subprocess.call('/tmp/venv/bin/pip install -r /mounted_files/requirements.txt', shell=True) subprocess.call('/tmp/venv/bin/python -m ipykernel install --prefix=/tmp/venv --name=venv', shell=True) - subprocess.call('/tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /mounted_files/notebook.ipynb', shell=True) + subprocess.call('/tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /mounted_files/pandas.ipynb', shell=True) command: - python volumeMounts: diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja new file mode 100644 index 0000000..49da1c3 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja @@ -0,0 +1,96 @@ +{% raw %} +import json + +from hera.shared import global_config +from hera.workflows import ( + DAG, + Artifact, + Script, + Volume, + Workflow, + script, # pyright: ignore[reportUnknownVariableType] +) +from hera.workflows import models as m +from hera.workflows.archive import NoneArchiveStrategy + +global_config.set_class_defaults( # pyright: ignore + Script, + image="ghcr.io/matt-carre/python-interface-to-workflows-mounted-image:latest", +) + + +@script( + command=["python"], + outputs=Artifact( + name="notebook", path="/tmp/notebook.html", archive=NoneArchiveStrategy() + ), + volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], +) +def mount_files(): + import subprocess + + subprocess.call("python -m venv /tmp/venv", shell=True) + subprocess.call( + "/tmp/venv/bin/pip install -r /mounted_files/requirements.txt", shell=True + ) + subprocess.call( + "/tmp/venv/bin/python -m ipykernel install --prefix=/tmp/venv --name=venv", + shell=True, + ) + subprocess.call( + "/tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /mounted_files/pandas.ipynb", # noqa: E501 + shell=True, + ) + + +with Workflow( + pod_spec_patch=json.dumps( + { + "containers": [ + { + "name": "main", + "resources": { + "limits": { + "cpu": "500m", + "memory": "2Gi", + }, + "requests": { + "cpu": "500m", + "memory": "2Gi", + }, + }, + } + ] + } + ), + tolerations=[ + m.Toleration( + key="nodetype", operator="Equal", value="gpu", effect="NoSchedule" + ), + m.Toleration( + key="nodegroup", operator="Equal", value="workflows", effect="NoSchedule" + ), + ], + name="hera-example-pandas", + entrypoint="workflowentry", + api_version="argoproj.io/v1alpha1", + kind="WorkflowTemplate", + labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, + annotations={ + "workflows.argoproj.io/title": "notebook.yaml remade via hera", + "workflows.argoproj.io/description": """Runs pandas.ipynb and converts it to an +html file""", + "workflows.diamond.ac.uk/repository": "https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}", + }, + volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), +) as w: + with DAG(name="workflowentry"): + files = mount_files() + files # pyright: ignore # noqa: B018 + + +with open( + "src/{% endraw %}{{project_name}}{% raw %}/templates/example_import_files.txt", "w" +) as div: + div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] +{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/mounted_files/pandas.ipynb b/src/copier_template/src/{{ project_name }}/workflow_definitions/mounted_files/pandas.ipynb new file mode 100644 index 0000000..96b333a --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/mounted_files/pandas.ipynb @@ -0,0 +1,111 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Manipulation and Visualization Example\n", + "\n", + "This notebook demonstrates:\n", + "- Creating synthetic test data\n", + "- Performing data manipulation with pandas\n", + "- Visualizing the results with matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Set environment variables\n", + "os.environ[\"MPLCONFIGDIR\"] = \"/tmp/.config/matplotlib\"\n", + "\n", + "# Ensure the directories exist\n", + "os.makedirs(os.environ[\"MPLCONFIGDIR\"], exist_ok=True)\n", + "\n", + "# Import required libraries\n", + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Set a random seed for reproducibility\n", + "np.random.seed(42)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create synthetic test data\n", + "dates = pd.date_range(start='2023-01-01', periods=100)\n", + "categories = ['A', 'B', 'C']\n", + "\n", + "data = pd.DataFrame({\n", + " 'Date': dates,\n", + " 'Category': np.random.choice(categories, size=100),\n", + " 'Value': np.random.normal(loc=50, scale=10, size=100)\n", + "})\n", + "\n", + "# Display first few rows\n", + "data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Data manipulation\n", + "# 1. Add a rolling average column\n", + "data['RollingAvg'] = data['Value'].rolling(window=7).mean()\n", + "\n", + "# 2. Group by Category and calculate mean value\n", + "category_means = data.groupby('Category')['Value'].mean()\n", + "category_means" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Visualization\n", + "plt.figure(figsize=(12, 6))\n", + "\n", + "# Plot the original Value and Rolling Average for each category\n", + "for cat in data['Category'].unique():\n", + " subset = data[data['Category'] == cat]\n", + " plt.plot(subset['Date'], subset['Value'], label=f'{cat} Value', alpha=0.3)\n", + " plt.plot(subset['Date'], subset['RollingAvg'], label=f'{cat} RollingAvg')\n", + "\n", + "plt.xlabel('Date')\n", + "plt.ylabel('Value')\n", + "plt.title('Value and Rolling Average by Category')\n", + "plt.legend()\n", + "plt.grid(True)\n", + "plt.tight_layout()\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/mounted_files/requirements.txt b/src/copier_template/src/{{ project_name }}/workflow_definitions/mounted_files/requirements.txt new file mode 100644 index 0000000..58a5482 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/mounted_files/requirements.txt @@ -0,0 +1,8 @@ +numpy==2.2.5 +matplotlib==3.10.3 +pandas==2.2.3 +scipy==1.15.3 +nbconvert==7.17.1 +ipykernel==6.29.5 +ipython==9.2.0 +papermill==2.6.0 diff --git a/src/python_interface_to_workflows/__main__.py b/src/python_interface_to_workflows/__main__.py index ff69253..c44d97c 100644 --- a/src/python_interface_to_workflows/__main__.py +++ b/src/python_interface_to_workflows/__main__.py @@ -17,8 +17,20 @@ def main(args: Sequence[str] | None = None) -> None: action="version", version=__version__, ) + parser.add_argument( + "-sleep", + ) parser.parse_args(args) + if "-sleep" in vars(parser.parse_args(args)): + while True: + import time + + time.sleep(200) if __name__ == "__main__": main() + while True: + import time + + time.sleep(200) diff --git a/src/python_interface_to_workflows/templates/example_import_files.txt b/src/python_interface_to_workflows/templates/example_import_files.txt index 8d4bc9b..c873986 100644 --- a/src/python_interface_to_workflows/templates/example_import_files.txt +++ b/src/python_interface_to_workflows/templates/example_import_files.txt @@ -37,7 +37,7 @@ spec: subprocess.call('python -m venv /tmp/venv', shell=True) subprocess.call('/tmp/venv/bin/pip install -r /mounted_files/requirements.txt', shell=True) subprocess.call('/tmp/venv/bin/python -m ipykernel install --prefix=/tmp/venv --name=venv', shell=True) - subprocess.call('/tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /mounted_files/notebook.ipynb', shell=True) + subprocess.call('/tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /mounted_files/pandas.ipynb', shell=True) command: - python volumeMounts: diff --git a/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py b/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py index c7f685a..9069f5b 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py @@ -14,7 +14,7 @@ global_config.set_class_defaults( # pyright: ignore Script, - image="ghcr.io/matt-carre/python-interface-to-workflows-mounted-image:latest", + image="ghcr.io/diamondlightsource/python-interface-to-workflows-mounted-in-image:latest", ) @@ -37,7 +37,7 @@ def mount_files(): shell=True, ) subprocess.call( - "/tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /mounted_files/notebook.ipynb", # noqa: E501 + "/tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /mounted_files/pandas.ipynb", # noqa: E501 shell=True, ) @@ -77,8 +77,8 @@ def mount_files(): labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, annotations={ "workflows.argoproj.io/title": "notebook.yaml remade via hera", - "workflows.argoproj.io/description": """Replicates the functionality of -notebook.yaml""", + "workflows.argoproj.io/description": """Runs pandas.ipynb and converts it to an +html file""", "workflows.diamond.ac.uk/repository": "https://github.com/DiamondLightSource/python-interface-to-workflows", }, volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), From caec5181c1cd771cb3cf66085c94d6785af80617 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Thu, 6 Aug 2026 10:51:42 +0000 Subject: [PATCH 10/15] feat(cleanup): begins clearing up repo --- pyproject.toml | 2 + .../auth/keycloak_checker.py.jinja | 72 --------- .../{{ project_name }}/auth/open_auth_url.py | 55 ------- .../auth/keycloak_checker.py | 83 ---------- .../auth/open_auth_url.py | 55 ------- .../submit_workflow.py | 48 ------ tests/test_keycloak_checker.py | 123 --------------- tests/test_open_auth_url.py | 142 ------------------ tests/test_submit_to_graphql.py | 32 ---- uv.lock | 34 +++++ 10 files changed, 36 insertions(+), 610 deletions(-) delete mode 100644 src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja delete mode 100644 src/copier_template/src/{{ project_name }}/auth/open_auth_url.py delete mode 100644 src/python_interface_to_workflows/auth/keycloak_checker.py delete mode 100644 src/python_interface_to_workflows/auth/open_auth_url.py delete mode 100644 src/python_interface_to_workflows/submit_workflow.py delete mode 100644 tests/test_keycloak_checker.py delete mode 100644 tests/test_open_auth_url.py delete mode 100644 tests/test_submit_to_graphql.py diff --git a/pyproject.toml b/pyproject.toml index 50f1105..242640e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "dotenv", "python-keycloak", "pytest-asyncio", + "python-workflow-submitter", ] # Add project dependencies here, e.g. ["click", "numpy"] dynamic = ["version"] license.file = "LICENSE" @@ -48,6 +49,7 @@ dev = [ "dotenv", "python-keycloak", "pytest-asyncio", + "python-workflow-submitter", ] [project.scripts] diff --git a/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja b/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja deleted file mode 100644 index 3f6fa48..0000000 --- a/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja +++ /dev/null @@ -1,72 +0,0 @@ -import os -from typing import TypedDict, cast - -import dotenv -from keycloak import KeycloakOpenID -from keycloak.pkce_utils import generate_code_challenge, generate_code_verifier - -from {{project_name}}.auth.open_auth_url import token_expired - - -class TokenResponse(TypedDict): - access_token: str - refresh_token: str - - -class DecodedToken(TypedDict): - exp: int - - -def set_token_env_variable() -> str: - keycloak_openid = KeycloakOpenID( - client_id="workflows-cli", - server_url="https://identity.diamond.ac.uk/", - realm_name="dls", - client_secret_key="", - pool_maxsize=1, - ) - port = 8000 - code_verifier = generate_code_verifier() - code_challenge, code_challenge_method = generate_code_challenge(code_verifier) - auth_url = keycloak_openid.auth_url( - redirect_uri=f"http://localhost:{port}/", - scope="openid posix-uid profile email fedid", - state="", - code_challenge=code_challenge, - code_challenge_method=code_challenge_method, - ) - if token_expired(auth_url, port): - token = cast( - TokenResponse, - keycloak_openid.token( # pyright: ignore[reportUnknownMemberType] - grant_type="authorization_code", - code=str(os.environ.get("AUTH")), - redirect_uri=f"http://localhost:{port}/", - code_verifier=code_verifier, - ), - ) - else: - token = cast( - TokenResponse, - keycloak_openid.refresh_token( # pyright: ignore[reportUnknownMemberType] - str(os.environ.get("REFRESHTOKEN")) - ), - ) - token_info = cast( - DecodedToken, - keycloak_openid.decode_token( # pyright: ignore[reportUnknownMemberType] - token["access_token"] - ), - ) - try: - expire_time = int(token_info["exp"]) - dotenv.set_key("src/.env", "EXPIRY", str(expire_time)) - dotenv.set_key("src/.env", "TOKEN", token["access_token"].strip("'")) - dotenv.set_key("src/.env", "REFRESHTOKEN", token["refresh_token"].strip("'")) - except AttributeError: - print("ERROR:") - exit(1) - finally: - dotenv.load_dotenv(dotenv_path="src/.env", override=True) - - return token["access_token"] diff --git a/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py b/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py deleted file mode 100644 index f8c2581..0000000 --- a/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -import socket -import time -import urllib.parse -import webbrowser -from http.server import BaseHTTPRequestHandler, HTTPServer -from typing import cast - -import dotenv - - -class _ReusingHTTPServer(HTTPServer): - allow_reuse_address = True - auth_code: str - - -class CallbackHandler(BaseHTTPRequestHandler): - def do_GET(self): - query = urllib.parse.urlparse(self.path).query - params = urllib.parse.parse_qs(query) - if "code" in params: - cast(_ReusingHTTPServer, self.server).auth_code = params["code"][0] - self.send_response(200) - self.end_headers() - self.wfile.write(b"Authorization successful. You can close this window.") - else: - self.send_response(400) - self.end_headers() - self.wfile.write(b"Missing authorization code.") - - -def token_expired(auth_url: str, port: int) -> bool: - dotenv.load_dotenv(dotenv_path="src/.env", override=True) - expiry_str: str = str(os.environ.get("EXPIRY")).strip("'") - if (expiry_str == "" or int(expiry_str)) <= float(time.time()): - _open_auth_url(auth_url, port) - return True - else: - return False - - -def _open_auth_url(auth_url: str, port: int) -> None: - httpd = _ReusingHTTPServer(("localhost", port), CallbackHandler) - webbrowser.open(auth_url) - try: - httpd.handle_request() - os.environ["AUTH"] = httpd.auth_code - dotenv.set_key("src/.env", "AUTH", httpd.auth_code) - except OSError: - os.environ["AUTH"] = "" - print("ERROR: Port in use. Please restart your terminal.") - exit(1) - finally: - httpd.socket.shutdown(socket.SHUT_RDWR) - httpd.server_close() diff --git a/src/python_interface_to_workflows/auth/keycloak_checker.py b/src/python_interface_to_workflows/auth/keycloak_checker.py deleted file mode 100644 index 8474fa7..0000000 --- a/src/python_interface_to_workflows/auth/keycloak_checker.py +++ /dev/null @@ -1,83 +0,0 @@ -import os -from typing import TypedDict, cast - -import dotenv -from keycloak import KeycloakOpenID -from keycloak.pkce_utils import generate_code_challenge, generate_code_verifier - -from python_interface_to_workflows.auth.open_auth_url import token_expired - - -class TokenResponse(TypedDict): - access_token: str - refresh_token: str - - -class DecodedToken(TypedDict): - exp: int - - -def set_token_env_variable(staging: bool) -> str: - match staging: - case True: - keycloak_openid = KeycloakOpenID( - server_url="https://identity-test.diamond.ac.uk/", - client_id="workflows-ui-dev", - realm_name="dls", - client_secret_key="", - pool_maxsize=1, - ) - port = 5173 - case False: - keycloak_openid = KeycloakOpenID( - client_id="workflows-cli", - server_url="https://identity.diamond.ac.uk/", - realm_name="dls", - client_secret_key="", - pool_maxsize=1, - ) - port = 8000 - code_verifier = generate_code_verifier() - code_challenge, code_challenge_method = generate_code_challenge(code_verifier) - auth_url = keycloak_openid.auth_url( - redirect_uri=f"http://localhost:{port}/", - scope="openid posix-uid profile email fedid", - state="", - code_challenge=code_challenge, - code_challenge_method=code_challenge_method, - ) - if token_expired(auth_url, port): - token = cast( - TokenResponse, - keycloak_openid.token( # pyright: ignore[reportUnknownMemberType] - grant_type="authorization_code", - code=str(os.environ.get("AUTH")), - redirect_uri=f"http://localhost:{port}/", - code_verifier=code_verifier, - ), - ) - else: - token = cast( - TokenResponse, - keycloak_openid.refresh_token( # pyright: ignore[reportUnknownMemberType] - str(os.environ.get("REFRESHTOKEN")) - ), - ) - token_info = cast( - DecodedToken, - keycloak_openid.decode_token( # pyright: ignore[reportUnknownMemberType] - token["access_token"] - ), - ) - try: - expire_time = int(token_info["exp"]) - dotenv.set_key("src/.env", "EXPIRY", str(expire_time)) - dotenv.set_key("src/.env", "TOKEN", token["access_token"].strip("'")) - dotenv.set_key("src/.env", "REFRESHTOKEN", token["refresh_token"].strip("'")) - except AttributeError: - print("ERROR:") - exit(1) - finally: - dotenv.load_dotenv(dotenv_path="src/.env", override=True) - - return token["access_token"] diff --git a/src/python_interface_to_workflows/auth/open_auth_url.py b/src/python_interface_to_workflows/auth/open_auth_url.py deleted file mode 100644 index f8c2581..0000000 --- a/src/python_interface_to_workflows/auth/open_auth_url.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -import socket -import time -import urllib.parse -import webbrowser -from http.server import BaseHTTPRequestHandler, HTTPServer -from typing import cast - -import dotenv - - -class _ReusingHTTPServer(HTTPServer): - allow_reuse_address = True - auth_code: str - - -class CallbackHandler(BaseHTTPRequestHandler): - def do_GET(self): - query = urllib.parse.urlparse(self.path).query - params = urllib.parse.parse_qs(query) - if "code" in params: - cast(_ReusingHTTPServer, self.server).auth_code = params["code"][0] - self.send_response(200) - self.end_headers() - self.wfile.write(b"Authorization successful. You can close this window.") - else: - self.send_response(400) - self.end_headers() - self.wfile.write(b"Missing authorization code.") - - -def token_expired(auth_url: str, port: int) -> bool: - dotenv.load_dotenv(dotenv_path="src/.env", override=True) - expiry_str: str = str(os.environ.get("EXPIRY")).strip("'") - if (expiry_str == "" or int(expiry_str)) <= float(time.time()): - _open_auth_url(auth_url, port) - return True - else: - return False - - -def _open_auth_url(auth_url: str, port: int) -> None: - httpd = _ReusingHTTPServer(("localhost", port), CallbackHandler) - webbrowser.open(auth_url) - try: - httpd.handle_request() - os.environ["AUTH"] = httpd.auth_code - dotenv.set_key("src/.env", "AUTH", httpd.auth_code) - except OSError: - os.environ["AUTH"] = "" - print("ERROR: Port in use. Please restart your terminal.") - exit(1) - finally: - httpd.socket.shutdown(socket.SHUT_RDWR) - httpd.server_close() diff --git a/src/python_interface_to_workflows/submit_workflow.py b/src/python_interface_to_workflows/submit_workflow.py deleted file mode 100644 index 6a668c8..0000000 --- a/src/python_interface_to_workflows/submit_workflow.py +++ /dev/null @@ -1,48 +0,0 @@ -import os - -import dotenv -from gql import Client, gql -from gql.transport.aiohttp import AIOHTTPTransport -from hera.workflows import Workflow - -from python_interface_to_workflows.auth.keycloak_checker import set_token_env_variable - - -async def submit_workflow(w: Workflow): - yamlstr = w.to_yaml() # pyright:ignore - dotenv.load_dotenv(dotenv_path="src/.env", override=True) - token: str = set_token_env_variable(True) - host: str = os.environ.get("HOST") # pyright:ignore - visit: str = os.environ.get("VISIT") # pyright:ignore - - transport = AIOHTTPTransport( - url=host, - headers={"Authorization": f"Bearer {token}"}, - ) - client = Client( - transport=transport, - fetch_schema_from_transport=True, - ) - mutation = gql(""" -mutation Submit($visit: VisitInput!, $manifest: String!) { - submitWorkflow( - visit: $visit - manifest: $manifest - ) { - name - } -} -""") - result = await client.execute_async( - mutation, - variable_values={ - "visit": { - "proposalCode": str(visit[:2]), - "proposalNumber": int(visit[2:7]), - "number": int(visit[-1]), - }, - "manifest": f"""{yamlstr}""", - }, - ) - name = str(result["submitWorkflow"]["name"]) - print(f"Job '{name}' submitted to {visit}") diff --git a/tests/test_keycloak_checker.py b/tests/test_keycloak_checker.py deleted file mode 100644 index ace1387..0000000 --- a/tests/test_keycloak_checker.py +++ /dev/null @@ -1,123 +0,0 @@ -import os -from unittest.mock import MagicMock, call, patch - -from pytest import mark - -from python_interface_to_workflows.auth.keycloak_checker import set_token_env_variable - - -@mark.parametrize( - "staging,port,return_present", - [ - (True, 5173, True), - (False, 8000, False), - (True, 5173, False), - (False, 8000, True), - ], -) -@patch("python_interface_to_workflows.auth.keycloak_checker.dotenv.load_dotenv") -@patch("python_interface_to_workflows.auth.keycloak_checker.dotenv.set_key") -@patch("python_interface_to_workflows.auth.keycloak_checker.KeycloakOpenID") -@patch("python_interface_to_workflows.auth.keycloak_checker.generate_code_verifier") -@patch("python_interface_to_workflows.auth.keycloak_checker.generate_code_challenge") -@patch("python_interface_to_workflows.auth.keycloak_checker.token_expired") -def test_set_token_env_variable( - mock_token_expired: MagicMock, - mock_gen_code_challenge: MagicMock, - mock_gen_code_verifier: MagicMock, - mock_gen_keycloak_id: MagicMock, - mock_set_key: MagicMock, - mock_load_env: MagicMock, - staging: bool, - port: int, - return_present: bool, -): - mock_gen_code_verifier.return_value = "verifier" - mock_gen_code_challenge.return_value = ("challenge", "S256") - os.environ["AUTH"] = "auth_url_code" - os.environ["REFRESHTOKEN"] = "refresh" - - keycloak = MagicMock() - mock_gen_keycloak_id.return_value = keycloak - mock_token_expired.return_value = return_present - keycloak.auth_url.return_value = "https://mock.site" - token = { - "access_token": "fake_token", - "refresh_token": "fake_refresh", - } - keycloak.token.return_value = token - keycloak.refresh_token.return_value = token - keycloak.decode_token.return_value = {"exp": 123456789} - assert set_token_env_variable(staging) == "fake_token" - - mock_token_expired.assert_called_once_with("https://mock.site", port) - if return_present: - keycloak.token.assert_called_once_with( - grant_type="authorization_code", - code="auth_url_code", - redirect_uri=f"http://localhost:{port}/", - code_verifier="verifier", - ) - keycloak.refresh_token.assert_not_called() - else: - keycloak.refresh_token.assert_called_once_with("refresh") - keycloak.token.assert_not_called() - mock_set_key.assert_has_calls( - [ - call("src/.env", "EXPIRY", str(123456789)), - call("src/.env", "TOKEN", "fake_token"), - call("src/.env", "REFRESHTOKEN", "fake_refresh"), - ] - ) - mock_load_env.assert_called_once_with( - dotenv_path="src/.env", - override=True, - ) - - -@patch("python_interface_to_workflows.auth.keycloak_checker.exit") -@patch("python_interface_to_workflows.auth.keycloak_checker.print") -@patch("python_interface_to_workflows.auth.keycloak_checker.dotenv.load_dotenv") -@patch("python_interface_to_workflows.auth.keycloak_checker.dotenv.set_key") -@patch("python_interface_to_workflows.auth.keycloak_checker.KeycloakOpenID") -@patch("python_interface_to_workflows.auth.keycloak_checker.generate_code_verifier") -@patch("python_interface_to_workflows.auth.keycloak_checker.generate_code_challenge") -@patch("python_interface_to_workflows.auth.keycloak_checker.token_expired") -def test_set_token_env_variable_attribute_error( - mock_token_expired: MagicMock, - mock_gen_code_challenge: MagicMock, - mock_gen_code_verifier: MagicMock, - mock_gen_keycloak_id: MagicMock, - mock_set_key: MagicMock, - mock_load_env: MagicMock, - mock_print: MagicMock, - mock_exit: MagicMock, -): - mock_gen_code_verifier.return_value = "verifier" - mock_gen_code_challenge.return_value = ("challenge", "S256") - mock_token_expired.return_value = True - - os.environ["REFRESHTOKEN"] = "fake_refresh" - keycloak = MagicMock() - mock_gen_keycloak_id.return_value = keycloak - - token = { - "access_token": "fake_token", - "refresh_token": "fake_refresh", - } - - keycloak.refresh_token.return_value = token - - decoded = MagicMock() - decoded.__getitem__.side_effect = AttributeError - keycloak.decode_token.return_value = decoded - - set_token_env_variable(True) - - mock_print.assert_called_once_with("ERROR:") - mock_exit.assert_called_once_with(1) - mock_set_key.assert_not_called() - mock_load_env.assert_called_once_with( - dotenv_path="src/.env", - override=True, - ) diff --git a/tests/test_open_auth_url.py b/tests/test_open_auth_url.py deleted file mode 100644 index 0acd322..0000000 --- a/tests/test_open_auth_url.py +++ /dev/null @@ -1,142 +0,0 @@ -import os -from unittest.mock import MagicMock, patch - -from python_interface_to_workflows.auth.open_auth_url import ( - CallbackHandler, - _open_auth_url, # pyright:ignore - token_expired, -) - - -@patch("python_interface_to_workflows.auth.open_auth_url.dotenv.set_key") -@patch("python_interface_to_workflows.auth.open_auth_url.time.time") -@patch("python_interface_to_workflows.auth.open_auth_url.webbrowser.open") -@patch("python_interface_to_workflows.auth.open_auth_url._ReusingHTTPServer") -def test_open_auth_url_normal_function( - mock_http_server: MagicMock, - mock_open_browser: MagicMock, - mock_time: MagicMock, - mock_set_key: MagicMock, -): - mock_time.return_value = 100 - os.environ["EXPIRY"] = "" - - server = mock_http_server.return_value - server.auth_code = "this_is_your_code" - - _open_auth_url("url", 5173) - server.handle_request.assert_called_once() - mock_open_browser.assert_called_once_with("url") - mock_set_key.assert_called_once_with( - "src/.env", - "AUTH", - "this_is_your_code", - ) - server.socket.shutdown.assert_called_once() - server.server_close.assert_called_once() - assert os.environ["AUTH"] == "this_is_your_code" - - -@patch("python_interface_to_workflows.auth.open_auth_url.dotenv.set_key") -@patch("python_interface_to_workflows.auth.open_auth_url.time.time") -@patch("python_interface_to_workflows.auth.open_auth_url.exit") -@patch("python_interface_to_workflows.auth.open_auth_url.webbrowser.open") -@patch("python_interface_to_workflows.auth.open_auth_url._ReusingHTTPServer") -def test_open_auth_url_raises_error( - mock_http_server: MagicMock, - mock_open_browser: MagicMock, - mock_exit: MagicMock, - mock_time: MagicMock, - mock_set_key: MagicMock, -): - mock_time.return_value = 100 - os.environ["EXPIRY"] = "" - - server = mock_http_server.return_value - server.handle_request.side_effect = OSError - - _open_auth_url("url", 5173) - mock_open_browser.assert_called_once_with("url") - assert os.environ["AUTH"] == "" - mock_exit.assert_called_once_with(1) - mock_set_key.assert_not_called() - server.socket.shutdown.assert_called_once() - server.server_close.assert_called_once() - - -@patch("python_interface_to_workflows.auth.open_auth_url._open_auth_url") -@patch("python_interface_to_workflows.auth.open_auth_url.dotenv.load_dotenv") -@patch("python_interface_to_workflows.auth.open_auth_url.time.time") -def test_token_expired( - mock_time: MagicMock, - mock_load_env: MagicMock, - mock_open_auth_url: MagicMock, -): - mock_time.return_value = 100 - os.environ["EXPIRY"] = "50" - - assert token_expired("url", 5173) is True - - mock_load_env.assert_called_once_with( - dotenv_path="src/.env", - override=True, - ) - mock_open_auth_url.assert_called_once_with("url", 5173) - - -@patch("python_interface_to_workflows.auth.open_auth_url._open_auth_url") -@patch("python_interface_to_workflows.auth.open_auth_url.dotenv.load_dotenv") -@patch("python_interface_to_workflows.auth.open_auth_url.time.time") -def test_token_not_expired( - mock_time: MagicMock, - mock_load_env: MagicMock, - mock_open_auth_url: MagicMock, -): - mock_time.return_value = 100 - os.environ["EXPIRY"] = "200" - - assert token_expired("url", 5173) is False - - mock_load_env.assert_called_once_with( - dotenv_path="src/.env", - override=True, - ) - mock_open_auth_url.assert_not_called() - - -def test_handler_normal_function(): - handler = CallbackHandler.__new__(CallbackHandler) - handler.path = "/?code=this_is_your_code" - - handler.server = MagicMock() - - handler.send_response = MagicMock() - handler.end_headers = MagicMock() - handler.wfile = MagicMock() - handler.wfile.write = MagicMock() - - handler.do_GET() - - assert handler.server.auth_code == "this_is_your_code" - handler.send_response.assert_called_once_with(200) - handler.wfile.write.assert_called_once_with( - b"Authorization successful. You can close this window." - ) - - -def test_handler_error_response(): - handler = CallbackHandler.__new__(CallbackHandler) - handler.path = "/" - - handler.server = MagicMock() - - handler.send_response = MagicMock() - handler.end_headers = MagicMock() - handler.wfile = MagicMock() - handler.wfile.write = MagicMock() - - handler.do_GET() - - handler.send_response.assert_called_once_with(400) - handler.end_headers.assert_called_once() - handler.wfile.write.assert_called_once_with(b"Missing authorization code.") diff --git a/tests/test_submit_to_graphql.py b/tests/test_submit_to_graphql.py deleted file mode 100644 index 3a84665..0000000 --- a/tests/test_submit_to_graphql.py +++ /dev/null @@ -1,32 +0,0 @@ -from unittest.mock import AsyncMock, MagicMock, call, patch - -import pytest - -from python_interface_to_workflows.submit_workflow import submit_workflow - - -@pytest.mark.asyncio -@patch("python_interface_to_workflows.submit_workflow.os.environ.get") -@patch("python_interface_to_workflows.submit_workflow.dotenv.load_dotenv") -@patch("python_interface_to_workflows.submit_workflow.Workflow") -@patch("python_interface_to_workflows.submit_workflow.set_token_env_variable") -@patch("python_interface_to_workflows.submit_workflow.Client") -async def test_submit_workflow_to_graphql( - mock_client: AsyncMock, - mock_key: MagicMock, - mock_workflow: MagicMock, - mock_load_env: MagicMock, - mock_os_get: MagicMock, -): - - mock_instance = AsyncMock() - mock_key.return_value = "token" - mock_client.return_value = mock_instance - mock_instance.execute_async = AsyncMock( - return_value={"submitWorkflow": {"name": "workflow123"}} - ) - await submit_workflow(mock_workflow) - mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) - mock_instance.execute_async.assert_called_once() - mock_workflow.to_yaml.assert_called_once() - mock_os_get.assert_has_calls([call("VISIT"), call("HOST")], any_order=True) diff --git a/uv.lock b/uv.lock index 0391cc9..713428c 100644 --- a/uv.lock +++ b/uv.lock @@ -177,6 +177,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, ] +[[package]] +name = "asyncio" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/71/ea/26c489a11f7ca862d5705db67683a7361ce11c23a7b98fc6c2deaeccede2/asyncio-4.0.0.tar.gz", hash = "sha256:570cd9e50db83bc1629152d4d0b7558d6451bb1bfd5dfc2e935d96fc2f40329b", size = 5371, upload-time = "2025-08-05T02:51:46.605Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/64/eff2564783bd650ca25e15938d1c5b459cda997574a510f7de69688cb0b4/asyncio-4.0.0-py3-none-any.whl", hash = "sha256:c1eddb0659231837046809e68103969b2bef8b0400d59cfa6363f6b5ed8cc88b", size = 5555, upload-time = "2025-08-05T02:51:45.767Z" }, +] + [[package]] name = "attrs" version = "26.1.0" @@ -1829,6 +1838,7 @@ dependencies = [ { name = "pillow" }, { name = "pytest-asyncio" }, { name = "python-keycloak" }, + { name = "python-workflow-submitter" }, { name = "pyyaml" }, { name = "requests" }, ] @@ -1847,6 +1857,7 @@ dev = [ { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "python-keycloak" }, + { name = "python-workflow-submitter" }, { name = "pyyaml" }, { name = "ruff" }, { name = "tox-uv" }, @@ -1864,6 +1875,7 @@ requires-dist = [ { name = "pillow" }, { name = "pytest-asyncio" }, { name = "python-keycloak" }, + { name = "python-workflow-submitter" }, { name = "pyyaml" }, { name = "requests" }, ] @@ -1881,6 +1893,7 @@ dev = [ { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "python-keycloak" }, + { name = "python-workflow-submitter" }, { name = "pyyaml" }, { name = "ruff" }, { name = "tox-uv" }, @@ -1904,6 +1917,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/9f/569a8bbdb0859498d33d8b86273bc82849bd0b445ac01416ad34996ca3d8/python_keycloak-7.1.1-py3-none-any.whl", hash = "sha256:d8295bec6c4805ab7335b03bc92753c8c6258d5511b080cac061da20ae77f61c", size = 87607, upload-time = "2026-02-15T08:45:57.054Z" }, ] +[[package]] +name = "python-workflow-submitter" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "asyncio" }, + { name = "dotenv" }, + { name = "gql" }, + { name = "h5py" }, + { name = "hera" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "pillow" }, + { name = "pytest-asyncio" }, + { name = "python-keycloak" }, + { name = "pyyaml" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/00/faf63331a1204fc82532d24967246c452116e4c053ea46fb6eb92d132b45/python_workflow_submitter-0.4.0.tar.gz", hash = "sha256:43060752d80fac674ed354c5cb7599115126ce0da6b142a0621827c5b94a6038", size = 181430, upload-time = "2026-08-06T09:24:22.086Z" } + [[package]] name = "pyyaml" version = "6.0.3" From 2934275203d2d60c3c0a3363c4df57acbbb3c94f Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Thu, 6 Aug 2026 13:13:24 +0000 Subject: [PATCH 11/15] docs(copier): updates copier and runthefiles.sh --- scripts/runthefiles.sh | 4 + .../scripts/runthefiles.sh.jinja | 4 + .../tests/test_keycloak_checker.py.jinja | 120 --------------- .../tests/test_open_auth_url.py.jinja | 142 ------------------ .../tests/test_submit_to_graphql.py.jinja | 30 ---- uv.lock | 4 +- 6 files changed, 10 insertions(+), 294 deletions(-) delete mode 100644 src/copier_template/tests/test_keycloak_checker.py.jinja delete mode 100644 src/copier_template/tests/test_open_auth_url.py.jinja delete mode 100644 src/copier_template/tests/test_submit_to_graphql.py.jinja diff --git a/scripts/runthefiles.sh b/scripts/runthefiles.sh index 662bf8e..245900c 100644 --- a/scripts/runthefiles.sh +++ b/scripts/runthefiles.sh @@ -3,6 +3,10 @@ cd src/python_interface_to_workflows/workflow_definitions for file in * do [[ -d "$file" ]] && continue + if grep -Eq '^\s*(from|import)\s+python_workflow_submitter\b' "$file"; then + echo "ERROR: $file imports submitter" + exit 1 + fi uv run "$file" done mv *.txt ../templates/ diff --git a/src/copier_template/scripts/runthefiles.sh.jinja b/src/copier_template/scripts/runthefiles.sh.jinja index c4d14a7..fa7fc42 100644 --- a/src/copier_template/scripts/runthefiles.sh.jinja +++ b/src/copier_template/scripts/runthefiles.sh.jinja @@ -3,6 +3,10 @@ cd src/{{project_name}}/workflow_definitions for file in * do [[ -d "$file" ]] && continue + if grep -Eq '^\s*(from|import)\s+python_workflow_submitter\b' "$file"; then + echo "ERROR: $file imports submitter" + exit 1 + fi uv run "$file" done mv *.txt src/{{project_name}}/templates/ diff --git a/src/copier_template/tests/test_keycloak_checker.py.jinja b/src/copier_template/tests/test_keycloak_checker.py.jinja deleted file mode 100644 index d2619a6..0000000 --- a/src/copier_template/tests/test_keycloak_checker.py.jinja +++ /dev/null @@ -1,120 +0,0 @@ -import os -from unittest.mock import MagicMock, call, patch - -from pytest import mark - -from {{project_name}}.auth.keycloak_checker import set_token_env_variable - - -@mark.parametrize( - "return_present", - [ - (False), - (True), - ], -) -@patch("{{project_name}}.auth.keycloak_checker.dotenv.load_dotenv") -@patch("{{project_name}}.auth.keycloak_checker.dotenv.set_key") -@patch("{{project_name}}.auth.keycloak_checker.KeycloakOpenID") -@patch("{{project_name}}.auth.keycloak_checker.generate_code_verifier") -@patch("{{project_name}}.auth.keycloak_checker.generate_code_challenge") -@patch("{{project_name}}.auth.keycloak_checker.token_expired") -def test_set_token_env_variable( - mock_token_expired: MagicMock, - mock_gen_code_challenge: MagicMock, - mock_gen_code_verifier: MagicMock, - mock_gen_keycloak_id: MagicMock, - mock_set_key: MagicMock, - mock_load_env: MagicMock, - return_present: bool, -): - port=8000 - mock_gen_code_verifier.return_value = "verifier" - mock_gen_code_challenge.return_value = ("challenge", "S256") - os.environ["AUTH"] = "auth_url_code" - os.environ["REFRESHTOKEN"] = "refresh" - - keycloak = MagicMock() - mock_gen_keycloak_id.return_value = keycloak - mock_token_expired.return_value = return_present - keycloak.auth_url.return_value = "https://mock.site" - token = { - "access_token": "fake_token", - "refresh_token": "fake_refresh", - } - keycloak.token.return_value = token - keycloak.refresh_token.return_value = token - keycloak.decode_token.return_value = {"exp": 123456789} - assert set_token_env_variable() == "fake_token" - - mock_token_expired.assert_called_once_with("https://mock.site", port) - if return_present: - keycloak.token.assert_called_once_with( - grant_type="authorization_code", - code="auth_url_code", - redirect_uri=f"http://localhost:{port}/", - code_verifier="verifier", - ) - keycloak.refresh_token.assert_not_called() - else: - keycloak.refresh_token.assert_called_once_with("refresh") - keycloak.token.assert_not_called() - mock_set_key.assert_has_calls( - [ - call("src/.env", "EXPIRY", str(123456789)), - call("src/.env", "TOKEN", "fake_token"), - call("src/.env", "REFRESHTOKEN", "fake_refresh"), - ] - ) - mock_load_env.assert_called_once_with( - dotenv_path="src/.env", - override=True, - ) - -@patch("{{project_name}}.auth.keycloak_checker.exit") -@patch("{{project_name}}.auth.keycloak_checker.print") -@patch("{{project_name}}.auth.keycloak_checker.dotenv.load_dotenv") -@patch("{{project_name}}.auth.keycloak_checker.dotenv.set_key") -@patch("{{project_name}}.auth.keycloak_checker.KeycloakOpenID") -@patch("{{project_name}}.auth.keycloak_checker.generate_code_verifier") -@patch("{{project_name}}.auth.keycloak_checker.generate_code_challenge") -@patch("{{project_name}}.auth.keycloak_checker.token_expired") -def test_set_token_env_variable_attribute_error( - mock_token_expired: MagicMock, - mock_gen_code_challenge: MagicMock, - mock_gen_code_verifier: MagicMock, - mock_gen_keycloak_id: MagicMock, - mock_set_key: MagicMock, - mock_load_env: MagicMock, - mock_print: MagicMock, - mock_exit: MagicMock, -): - mock_gen_code_verifier.return_value = "verifier" - mock_gen_code_challenge.return_value = ("challenge", "S256") - mock_token_expired.return_value = True - - os.environ["REFRESHTOKEN"] = "refresh" - - keycloak = MagicMock() - mock_gen_keycloak_id.return_value = keycloak - - token = { - "access_token": "fake_token", - "refresh_token": "fake_refresh", - } - - keycloak.refresh_token.return_value = token - - decoded = MagicMock() - decoded.__getitem__.side_effect = AttributeError - keycloak.decode_token.return_value = decoded - - set_token_env_variable() - - mock_print.assert_called_once_with("ERROR:") - mock_exit.assert_called_once_with(1) - mock_set_key.assert_not_called() - mock_load_env.assert_called_once_with( - dotenv_path="src/.env", - override=True, - ) diff --git a/src/copier_template/tests/test_open_auth_url.py.jinja b/src/copier_template/tests/test_open_auth_url.py.jinja deleted file mode 100644 index f35aca1..0000000 --- a/src/copier_template/tests/test_open_auth_url.py.jinja +++ /dev/null @@ -1,142 +0,0 @@ -import os -from unittest.mock import MagicMock, patch - -from {{project_name}}.auth.open_auth_url import ( - CallbackHandler, - _open_auth_url, # pyright:ignore - token_expired, -) - - -@patch("{{project_name}}.auth.open_auth_url.dotenv.set_key") -@patch("{{project_name}}.auth.open_auth_url.time.time") -@patch("{{project_name}}.auth.open_auth_url.webbrowser.open") -@patch("{{project_name}}.auth.open_auth_url._ReusingHTTPServer") -def test_open_auth_url_normal_function( - mock_http_server: MagicMock, - mock_open_browser: MagicMock, - mock_time: MagicMock, - mock_set_key: MagicMock, -): - mock_time.return_value = 100 - os.environ["EXPIRY"] = "" - - server = mock_http_server.return_value - server.auth_code = "this_is_your_code" - - _open_auth_url("url", 5173) - server.handle_request.assert_called_once() - mock_open_browser.assert_called_once_with("url") - mock_set_key.assert_called_once_with( - "src/.env", - "AUTH", - "this_is_your_code", - ) - server.socket.shutdown.assert_called_once() - server.server_close.assert_called_once() - assert os.environ["AUTH"] == "this_is_your_code" - - -@patch("{{project_name}}.auth.open_auth_url.dotenv.set_key") -@patch("{{project_name}}.auth.open_auth_url.time.time") -@patch("{{project_name}}.auth.open_auth_url.exit") -@patch("{{project_name}}.auth.open_auth_url.webbrowser.open") -@patch("{{project_name}}.auth.open_auth_url._ReusingHTTPServer") -def test_open_auth_url_raises_error( - mock_http_server: MagicMock, - mock_open_browser: MagicMock, - mock_exit: MagicMock, - mock_time: MagicMock, - mock_set_key: MagicMock, -): - mock_time.return_value = 100 - os.environ["EXPIRY"] = "" - - server = mock_http_server.return_value - server.handle_request.side_effect = OSError - - _open_auth_url("url", 5173) - mock_open_browser.assert_called_once_with("url") - assert os.environ["AUTH"] == "" - mock_exit.assert_called_once_with(1) - mock_set_key.assert_not_called() - server.socket.shutdown.assert_called_once() - server.server_close.assert_called_once() - - -@patch("{{project_name}}.auth.open_auth_url._open_auth_url") -@patch("{{project_name}}.auth.open_auth_url.dotenv.load_dotenv") -@patch("{{project_name}}.auth.open_auth_url.time.time") -def test_token_expired( - mock_time: MagicMock, - mock_load_env: MagicMock, - mock_open_auth_url: MagicMock, -): - mock_time.return_value = 100 - os.environ["EXPIRY"] = "50" - - assert token_expired("url", 5173) is True - - mock_load_env.assert_called_once_with( - dotenv_path="src/.env", - override=True, - ) - mock_open_auth_url.assert_called_once_with("url", 5173) - - -@patch("{{project_name}}.auth.open_auth_url._open_auth_url") -@patch("{{project_name}}.auth.open_auth_url.dotenv.load_dotenv") -@patch("{{project_name}}.auth.open_auth_url.time.time") -def test_token_not_expired( - mock_time: MagicMock, - mock_load_env: MagicMock, - mock_open_auth_url: MagicMock, -): - mock_time.return_value = 100 - os.environ["EXPIRY"] = "200" - - assert token_expired("url", 5173) is False - - mock_load_env.assert_called_once_with( - dotenv_path="src/.env", - override=True, - ) - mock_open_auth_url.assert_not_called() - - -def test_handler_normal_function(): - handler = CallbackHandler.__new__(CallbackHandler) - handler.path = "/?code=this_is_your_code" - - handler.server = MagicMock() - - handler.send_response = MagicMock() - handler.end_headers = MagicMock() - handler.wfile = MagicMock() - handler.wfile.write = MagicMock() - - handler.do_GET() - - assert handler.server.auth_code == "this_is_your_code" - handler.send_response.assert_called_once_with(200) - handler.wfile.write.assert_called_once_with( - b"Authorization successful. You can close this window." - ) - - -def test_handler_error_response(): - handler = CallbackHandler.__new__(CallbackHandler) - handler.path = "/" - - handler.server = MagicMock() - - handler.send_response = MagicMock() - handler.end_headers = MagicMock() - handler.wfile = MagicMock() - handler.wfile.write = MagicMock() - - handler.do_GET() - - handler.send_response.assert_called_once_with(400) - handler.end_headers.assert_called_once() - handler.wfile.write.assert_called_once_with(b"Missing authorization code.") diff --git a/src/copier_template/tests/test_submit_to_graphql.py.jinja b/src/copier_template/tests/test_submit_to_graphql.py.jinja deleted file mode 100644 index 2427bd4..0000000 --- a/src/copier_template/tests/test_submit_to_graphql.py.jinja +++ /dev/null @@ -1,30 +0,0 @@ -from unittest.mock import AsyncMock, MagicMock, call, patch - -from {{project_name}}.submit_workflow import submit_workflow - - -@pytest.mark.asyncio -@patch("{{project_name}}.submit_workflow.os.environ.get") -@patch("{{project_name}}.submit_workflow.dotenv.load_dotenv") -@patch("{{project_name}}.submit_workflow.Workflow") -@patch("{{project_name}}.submit_workflow.set_token_env_variable") -@patch("{{project_name}}.submit_workflow.Client") -async def test_submit_workflow_to_graphql( - mock_client: AsyncMock, - mock_key: MagicMock, - mock_workflow: MagicMock, - mock_load_env: MagicMock, - mock_os_get: MagicMock, -): - - mock_instance = AsyncMock() - mock_key.return_value = "token" - mock_client.return_value = mock_instance - mock_instance.execute_async = AsyncMock( - return_value={"submitWorkflow": {"name": "workflow123"}} - ) - await submit_workflow(mock_workflow) - mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) - mock_instance.execute_async.assert_called_once() - mock_workflow.to_yaml.assert_called_once() - mock_os_get.assert_has_calls([call("VISIT"), call("HOST")], any_order=True) diff --git a/uv.lock b/uv.lock index 713428c..620fa72 100644 --- a/uv.lock +++ b/uv.lock @@ -1919,7 +1919,7 @@ wheels = [ [[package]] name = "python-workflow-submitter" -version = "0.4.0" +version = "0.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -1936,7 +1936,7 @@ dependencies = [ { name = "pyyaml" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/68/00/faf63331a1204fc82532d24967246c452116e4c053ea46fb6eb92d132b45/python_workflow_submitter-0.4.0.tar.gz", hash = "sha256:43060752d80fac674ed354c5cb7599115126ce0da6b142a0621827c5b94a6038", size = 181430, upload-time = "2026-08-06T09:24:22.086Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/f8/39caad75cb0b5e9205424e91baf9dd1c719096f95dca218ca114293c045a/python_workflow_submitter-0.4.2.tar.gz", hash = "sha256:f5ff6d56c42d6f0c7c4337a59979f7d4e6f733e19a159accc5a309f8c41a4fda", size = 181442, upload-time = "2026-08-06T12:01:11.956Z" } [[package]] name = "pyyaml" From 15199a94ffda7b2d6148f6805cb7cc79c6b4fea6 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Fri, 7 Aug 2026 08:58:22 +0000 Subject: [PATCH 12/15] docs(copier): adds explainers to relevant files --- src/copier_template/src/README.md.jinja | 16 +- .../submit_workflow.py.jinja | 50 ---- .../templates/example.txt.jinja | 209 ------------- .../create_example_template.py.jinja | 204 ------------- .../create_notebook_in_image.py.jinja | 32 +- .../notebook_image_example.ipynb.jinja | 282 ------------------ .../templates/divisionyaml.txt | 60 ---- .../templates/example.txt | 207 ------------- .../create_division_yaml.py | 49 --- .../create_example_template.py | 202 ------------- .../create_notebook_in_image.py | 32 +- .../notebooks/notebook_division.ipynb | 116 ------- .../notebooks/notebook_image_example.ipynb | 280 ----------------- 13 files changed, 57 insertions(+), 1682 deletions(-) delete mode 100644 src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja delete mode 100644 src/copier_template/src/{{ project_name }}/templates/example.txt.jinja delete mode 100644 src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja delete mode 100644 src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_image_example.ipynb.jinja delete mode 100644 src/python_interface_to_workflows/templates/divisionyaml.txt delete mode 100644 src/python_interface_to_workflows/templates/example.txt delete mode 100644 src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py delete mode 100644 src/python_interface_to_workflows/workflow_definitions/create_example_template.py delete mode 100644 src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb delete mode 100644 src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_image_example.ipynb diff --git a/src/copier_template/src/README.md.jinja b/src/copier_template/src/README.md.jinja index 23da404..ef19d16 100644 --- a/src/copier_template/src/README.md.jinja +++ b/src/copier_template/src/README.md.jinja @@ -1,14 +1,9 @@ # Setting up a development environment 1. run "uv lock" to generate the uv.lock file -2. Create .env in this folder (with the path src/.env) containing the following variables: - -HOST=https://workflows.diamond.ac.uk/graphql (to submit to the production cluster) -DEFAULT_IMAGE= (usually python 3.10) -VISIT= (the Visit you wish to run the template on) -TOKEN= -EXPIRY= -AUTH= - +2. Create .env in this folder (with the path src/.env) containing the following variable (everything else will be set automatically): +```bash +VISIT={your visit} +``` 3. Build the dev container # Submitting a workflow @@ -39,6 +34,5 @@ Alternatively, you can set the default image at the top of the file by adding: ```python global_config.set_class_defaults( # pyright: ignore - Script, image=str(os.environ.get("DEFAULT_IMAGE")) -) + Script, image=ghcr.io/Your-Github-Name/image-name) ``` diff --git a/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja b/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja deleted file mode 100644 index e2d353d..0000000 --- a/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja +++ /dev/null @@ -1,50 +0,0 @@ -{% raw %} -import os - -import dotenv -from gql import Client, gql -from gql.transport.aiohttp import AIOHTTPTransport -from hera.workflows import Workflow - -from {% endraw %}{{project_name}}{% raw %}.auth.keycloak_checker import set_token_env_variable - - -async def submit_workflow(w: Workflow): - yamlstr = w.to_yaml() # pyright:ignore - dotenv.load_dotenv(dotenv_path="src/.env", override=True) - token: str = set_token_env_variable() - host: str = os.environ.get("HOST") # pyright:ignore - visit: str = os.environ.get("VISIT") # pyright:ignore - - transport = AIOHTTPTransport( - url=host, - headers={"Authorization": f"Bearer {token}"}, - ) - client = Client( - transport=transport, - fetch_schema_from_transport=True, - ) - mutation = gql(""" -mutation Submit($visit: VisitInput!, $manifest: String!) { - submitWorkflow( - visit: $visit - manifest: $manifest - ) { - name - } -} -""") - result = await client.execute_async( - mutation, - variable_values={ - "visit": { - "proposalCode": str(visit[:2]), - "proposalNumber": int(visit[2:7]), - "number": int(visit[-1]), - }, - "manifest": f"""{yamlstr}""", - }, - ) - name = str(result["submitWorkflow"]["name"]) - print(f"Job '{name}' submitted to {visit}") -{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja deleted file mode 100644 index b017cf1..0000000 --- a/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja +++ /dev/null @@ -1,209 +0,0 @@ -{% raw %} -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: hera-example - annotations: - workflows.argoproj.io/description: |- - Replicates the functionality of - example.yaml - workflows.argoproj.io/title: example remade via hera - workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %} - labels: - workflows.diamond.ac.uk/science-group-examples: 'true' -spec: - entrypoint: workflowentry - podSpecPatch: '{"containers": [{"name": "main", "resources": {"limits": {"cpu": - "1", "memory": "1Gi"}, "requests": {"cpu": "1", "memory": "1Gi"}}}]}' - templates: - - name: workflowentry - dag: - tasks: - - name: params - template: generate-parameters - arguments: - parameters: - - name: png - value: 'True' - - name: jpg - value: 'True' - - name: jpeg - value: 'True' - - name: tif - value: 'True' - - name: tiff - value: 'True' - - name: create-image - depends: params - template: create-image - withParam: '{{tasks.params.outputs.parameters.out-parameters}}' - arguments: - parameters: - - name: width - value: '{{item.width}}' - - name: height - value: '{{item.height}}' - - name: weights - value: '{{item.weights}}' - - name: extension - value: '{{item.extension}}' - - name: to-hdf5 - depends: create-image - template: to-hdf5 - arguments: - parameters: - - name: paths - value: '{{tasks.create-image.outputs.parameters.out-paths}}' - - name: generate-parameters - inputs: - parameters: - - name: png - - name: jpg - - name: jpeg - - name: tif - - name: tiff - outputs: - parameters: - - name: out-parameters - valueFrom: - path: /tmp/parameters.json - script: - image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: jpeg = json.loads(r'''{{inputs.parameters.jpeg}}''') - except: jpeg = r'''{{inputs.parameters.jpeg}}''' - try: jpg = json.loads(r'''{{inputs.parameters.jpg}}''') - except: jpg = r'''{{inputs.parameters.jpg}}''' - try: png = json.loads(r'''{{inputs.parameters.png}}''') - except: png = r'''{{inputs.parameters.png}}''' - try: tif = json.loads(r'''{{inputs.parameters.tif}}''') - except: tif = r'''{{inputs.parameters.tif}}''' - try: tiff = json.loads(r'''{{inputs.parameters.tiff}}''') - except: tiff = r'''{{inputs.parameters.tiff}}''' - - import json - params: list[dict[str, int | list[int] | str] | None] = [{'width': 500, 'height': 500, 'weights': [255, 1, 100], 'extension': 'png'} if png.lower() == 'true' else None, {'width': 600, 'height': 200, 'weights': [100, 150, 100], 'extension': 'jpg'} if jpg.lower() == 'true' else None, {'width': 300, 'height': 400, 'weights': [100, 150, 100], 'extension': 'jpeg'} if jpeg.lower() == 'true' else None, {'width': 300, 'height': 200, 'weights': [230, 100, 1], 'extension': 'tif'} if tif.lower() == 'true' else None, {'width': 200, 'height': 300, 'weights': [230, 100, 1], 'extension': 'tiff'} if tiff.lower() == 'true' else None] - params_to_write: list[dict[str, int | list[int] | str]] = [image_params for image_params in params if image_params is not None] - with open('/tmp/parameters.json', 'w') as f: - json.dump(params_to_write, f) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: create-image - inputs: - parameters: - - name: width - - name: height - - name: weights - - name: extension - outputs: - artifacts: - - name: '{{inputs.parameters.extension}}-image' - path: /tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}} - archive: - none: {} - parameters: - - name: out-paths - valueFrom: - path: /tmp/{{inputs.parameters.extension}}-path.json - script: - image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: extension = json.loads(r'''{{inputs.parameters.extension}}''') - except: extension = r'''{{inputs.parameters.extension}}''' - try: height = json.loads(r'''{{inputs.parameters.height}}''') - except: height = r'''{{inputs.parameters.height}}''' - try: weights = json.loads(r'''{{inputs.parameters.weights}}''') - except: weights = r'''{{inputs.parameters.weights}}''' - try: width = json.loads(r'''{{inputs.parameters.width}}''') - except: width = r'''{{inputs.parameters.width}}''' - - import json - from PIL import Image - - def create_pattern(width: int, height: int, weights: tuple[int, int, int]) -> Image.Image: - print(f'width: {width}') - print(f'height: {height}') - print(f'RBG weights: {weights}') - image = Image.new('RGB', (width, height)) - pixels = image.load() - for i in range(width): - for j in range(height): - pixels[i, j] = ((i + j * 50) % weights[0], weights[1], (i * 300 + j) % weights[2]) - return image - image = create_pattern(width, height, weights) - path = f'/tmp/{extension}-image.{extension}' - image.save(path) - with open(f'/tmp/{extension}-path.json', 'w') as f: - json.dump(path, f) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: to-hdf5 - inputs: - parameters: - - name: paths - outputs: - artifacts: - - name: hdf5output - path: /tmp/images.hdf5 - archive: - none: {} - script: - image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: paths = json.loads(r'''{{inputs.parameters.paths}}''') - except: paths = r'''{{inputs.parameters.paths}}''' - - import h5py - import numpy as np - from PIL import Image - print('creating hdf5 file') - with h5py.File('/tmp/images.hdf5', 'w') as f: - for i, path in enumerate(paths): - path = path.strip('"') - print(f'Got {path}') - with Image.open(path) as image: - arr = np.array(image) - f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) - print('done') - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - tolerations: - - effect: NoSchedule - key: nodetype - operator: Equal - value: gpu - - effect: NoSchedule - key: nodegroup - operator: Equal - value: workflows - volumeClaimTemplates: - - metadata: - name: tmpdir - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi -{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja deleted file mode 100644 index d3b03e3..0000000 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja +++ /dev/null @@ -1,204 +0,0 @@ -{% raw %} -import json -import os - -from hera.shared import global_config -from hera.workflows import ( - DAG, - Artifact, - Parameter, - Script, - Volume, - Workflow, - script, # pyright: ignore[reportUnknownVariableType] -) -from hera.workflows import models as m -from hera.workflows.archive import NoneArchiveStrategy - -global_config.set_class_defaults( # pyright: ignore - Script, image=str(os.environ.get("DEFAULT_IMAGE")) -) - - -@script( - command=["python"], - outputs=Parameter( - name="out-parameters", value_from=m.ValueFrom(path="/tmp/parameters.json") - ), - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], -) -def generate_parameters( - png: str, - jpg: str, - jpeg: str, - tif: str, - tiff: str, -): - import json - - params: list[dict[str, int | list[int] | str] | None] = [ - {"width": 500, "height": 500, "weights": [255, 1, 100], "extension": "png"} - if png.lower() == "true" - else None, - {"width": 600, "height": 200, "weights": [100, 150, 100], "extension": "jpg"} - if jpg.lower() == "true" - else None, - {"width": 300, "height": 400, "weights": [100, 150, 100], "extension": "jpeg"} - if jpeg.lower() == "true" - else None, - {"width": 300, "height": 200, "weights": [230, 100, 1], "extension": "tif"} - if tif.lower() == "true" - else None, - {"width": 200, "height": 300, "weights": [230, 100, 1], "extension": "tiff"} - if tiff.lower() == "true" - else None, - ] - params_to_write: list[dict[str, int | list[int] | str]] = [ - image_params for image_params in params if image_params is not None - ] - with open("/tmp/parameters.json", "w") as f: - json.dump(params_to_write, f) - - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], - outputs=[ - Parameter( - name="out-paths", - value_from=m.ValueFrom( - path="/tmp/{{inputs.parameters.extension}}-path.json" - ), - ), - Artifact( - name="{{inputs.parameters.extension}}-image", - path="/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}", - archive=NoneArchiveStrategy(), - ), - ], -) -def create_image( - width: int, height: int, weights: tuple[int, int, int], extension: str -): - import json - - from PIL import Image - - def create_pattern( - width: int, - height: int, - weights: tuple[int, int, int], - ) -> Image.Image: - print(f"width: {width}") - print(f"height: {height}") - print(f"RBG weights: {weights}") - image = Image.new("RGB", (width, height)) - pixels = image.load() - for i in range(width): - for j in range(height): - pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript] - (i + j * 50) % weights[0], - weights[1], - (i * 300 + j) % weights[2], - ) - return image - - image = create_pattern(width, height, weights) - path = f"/tmp/{extension}-image.{extension}" - image.save(path) - with open(f"/tmp/{extension}-path.json", "w") as f: - json.dump(path, f) - - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], - outputs=Artifact( - name="hdf5output", - path="/tmp/images.hdf5", - archive=NoneArchiveStrategy(), - ), -) -def to_hdf5(paths: str): - - import h5py # pyright: ignore[reportMissingTypeStubs] - import numpy as np - from PIL import Image - - print("creating hdf5 file") - with h5py.File("/tmp/images.hdf5", "w") as f: - for i, path in enumerate(paths): - path = path.strip('"') - print(f"Got {path}") - with Image.open(path) as image: - arr = np.array(image) - f.create_dataset( # pyright: ignore[reportUnknownMemberType] - f"image_{i}", data=arr, dtype=arr.dtype - ) - print("done") - - -with Workflow( - pod_spec_patch=json.dumps( - { - "containers": [ - { - "name": "main", - "resources": { - "limits": { - "cpu": "1", - "memory": "1Gi", - }, - "requests": { - "cpu": "1", - "memory": "1Gi", - }, - }, - } - ] - } - ), - tolerations=[ - m.Toleration( - key="nodetype", operator="Equal", value="gpu", effect="NoSchedule" - ), - m.Toleration( - key="nodegroup", operator="Equal", value="workflows", effect="NoSchedule" - ), - ], - name="hera-example", - entrypoint="workflowentry", - api_version="argoproj.io/v1alpha1", - kind="WorkflowTemplate", - labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, - annotations={ - "workflows.argoproj.io/title": "example remade via hera", - "workflows.argoproj.io/description": """Replicates the functionality of -example.yaml""", - "workflows.diamond.ac.uk/repository": "https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}", - }, - volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), -) as w: - with DAG(name="workflowentry"): - params = generate_parameters( - name="params", - arguments={ - "png": "True", - "jpg": "True", - "jpeg": "True", - "tif": "True", - "tiff": "True", - }, - ) - makeimages = create_image(with_param=params.get_parameter("out-parameters")) - makehdf5 = to_hdf5( - arguments={ - "paths": makeimages.get_parameter("out-paths"), - } - ) - params >> makeimages >> makehdf5 # pyright: ignore - - -with open("src/{% endraw %}{{project_name}}{% raw %}/templates/example.txt", "w") as div: - div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] -{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja index 49da1c3..e1d49b6 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja @@ -3,9 +3,9 @@ import json from hera.shared import global_config from hera.workflows import ( - DAG, Artifact, Script, + Steps, Volume, Workflow, script, # pyright: ignore[reportUnknownVariableType] @@ -13,22 +13,31 @@ from hera.workflows import ( from hera.workflows import models as m from hera.workflows.archive import NoneArchiveStrategy +# Sets the default image, unless specified otherwise, to this. +# This image has access to the /workflow_definitions/mounted_files folder +# and was created with the included dockerfile. global_config.set_class_defaults( # pyright: ignore Script, - image="ghcr.io/matt-carre/python-interface-to-workflows-mounted-image:latest", + image="ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-mounted-image:latest", ) +# The script decorator allows hera to convert python code into yaml @script( - command=["python"], + command=["python"], # optional + # makes output an artifact to allow it to be downloaded outputs=Artifact( name="notebook", path="/tmp/notebook.html", archive=NoneArchiveStrategy() ), + # mounts a volume named "tmpdir" at path "/tmp" volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], ) def mount_files(): + # notably, we install dependencies for scripts *within* the function we run. + # They are not interpreted as functions, but as stand-alone scripts. import subprocess + # install our dependencies subprocess.call("python -m venv /tmp/venv", shell=True) subprocess.call( "/tmp/venv/bin/pip install -r /mounted_files/requirements.txt", shell=True @@ -37,6 +46,7 @@ def mount_files(): "/tmp/venv/bin/python -m ipykernel install --prefix=/tmp/venv --name=venv", shell=True, ) + # convert the notebook file into an html file subprocess.call( "/tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /mounted_files/pandas.ipynb", # noqa: E501 shell=True, @@ -44,6 +54,7 @@ def mount_files(): with Workflow( + # assures that the container has enough resources for our workflow pod_spec_patch=json.dumps( { "containers": [ @@ -63,6 +74,7 @@ with Workflow( ] } ), + # assures we run this in a pod with a gpu and no other scheduled workflow tolerations=[ m.Toleration( key="nodetype", operator="Equal", value="gpu", effect="NoSchedule" @@ -71,7 +83,10 @@ with Workflow( key="nodegroup", operator="Equal", value="workflows", effect="NoSchedule" ), ], + # name of workflow - will append a short identifier automatically. name="hera-example-pandas", + # the following is the same as for writing any yaml workflow, but as variables. + # All of these are required aside from "workflows.argoproj.io/description". entrypoint="workflowentry", api_version="argoproj.io/v1alpha1", kind="WorkflowTemplate", @@ -82,15 +97,20 @@ with Workflow( html file""", "workflows.diamond.ac.uk/repository": "https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}", }, + # We use Volume objects to define volumes. volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), ) as w: - with DAG(name="workflowentry"): + # We establish the order of templates to run here, using '>>' to determine order + # We can use a DAG instead, in which we write out the dag's architecture afterwards: + # grouping tasks with [], i.e., files >> [a, b] will run files, then will + # run a and b simultaneously. + with Steps(name="workflowentry"): files = mount_files() - files # pyright: ignore # noqa: B018 +# produce a yaml file so we can lint and submit it with python_workflow_submitter with open( - "src/{% endraw %}{{project_name}}{% raw %}/templates/example_import_files.txt", "w" + "src/{% endraw %}{{project_name}}{% raw %}/templates/example_import_files.yaml", "w" ) as div: div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] {% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_image_example.ipynb.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_image_example.ipynb.jinja deleted file mode 100644 index 8eebe05..0000000 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_image_example.ipynb.jinja +++ /dev/null @@ -1,282 +0,0 @@ -{% raw %} -{ - "cells": [ - { - "cell_type": "markdown", - "id": "1e3146f1", - "metadata": {}, - "source": [ - "# Running in VSCode:\n", - "\n", - "1. Set kernal to {% endraw %}{{repo_name}}{% raw %} 3.11.x\n", - "2. Hit F1, run Jupyter: Import Notebook to Script\n", - "3. Click notebook_example.ipynb\n", - "4. Run the cells sequentially" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c1d5a92b", - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "\n", - "from hera.shared import global_config\n", - "from hera.workflows import (\n", - " DAG,\n", - " Artifact,\n", - " Parameter,\n", - " Script,\n", - " Volume,\n", - " Workflow,\n", - " script, # pyright: ignore[reportUnknownVariableType]\n", - ")\n", - "from hera.workflows import models as m\n", - "from hera.workflows.archive import NoneArchiveStrategy\n", - "\n", - "global_config.set_class_defaults( # pyright: ignore\n", - " Script, image=\"ghcr.io/diamondlightsource/python-interface-to-workflows-image\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "494174ef", - "metadata": {}, - "outputs": [], - "source": [ - "@script(\n", - " command=[\"python\"],\n", - " outputs=Parameter(\n", - " name=\"out-parameters\", value_from=m.ValueFrom(path=\"/tmp/parameters.json\")\n", - " ),\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - ")\n", - "def generate_parameters(\n", - " png: str,\n", - " jpg: str,\n", - " jpeg: str,\n", - " tif: str,\n", - " tiff: str,\n", - "):\n", - " import json\n", - "\n", - " params: list[dict[str, int | list[int] | str] | None] = [\n", - " {\"width\": 500, \"height\": 500, \"weights\": [255, 1, 100], \"extension\": \"png\"}\n", - " if png.lower() == \"true\"\n", - " else None,\n", - " {\"width\": 600, \"height\": 200, \"weights\": [100, 150, 100], \"extension\": \"jpg\"}\n", - " if jpg.lower() == \"true\"\n", - " else None,\n", - " {\"width\": 300, \"height\": 400, \"weights\": [100, 150, 100], \"extension\": \"jpeg\"}\n", - " if jpeg.lower() == \"true\"\n", - " else None,\n", - " {\"width\": 300, \"height\": 200, \"weights\": [230, 100, 1], \"extension\": \"tif\"}\n", - " if tif.lower() == \"true\"\n", - " else None,\n", - " {\"width\": 200, \"height\": 300, \"weights\": [230, 100, 1], \"extension\": \"tiff\"}\n", - " if tiff.lower() == \"true\"\n", - " else None,\n", - " ]\n", - " params_to_write: list[dict[str, int | list[int] | str]] = [\n", - " image_params for image_params in params if image_params is not None\n", - " ]\n", - " with open(\"/tmp/parameters.json\", \"w\") as f:\n", - " json.dump(params_to_write, f)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8ec312fb", - "metadata": {}, - "outputs": [], - "source": [ - "@script(\n", - " command=[\"python\"],\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - " outputs=[\n", - " Parameter(\n", - " name=\"out-paths\",\n", - " value_from=m.ValueFrom(\n", - " path=\"/tmp/{{inputs.parameters.extension}}-path.json\"\n", - " ),\n", - " ),\n", - " Artifact(\n", - " name=\"{{inputs.parameters.extension}}-image\",\n", - " path=\"/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}\",\n", - " archive=NoneArchiveStrategy(),\n", - " ),\n", - " ],\n", - ")\n", - "def create_image(\n", - " width: int, height: int, weights: tuple[int, int, int], extension: str\n", - "):\n", - " import json\n", - "\n", - " from PIL import Image\n", - "\n", - " def create_pattern(\n", - " width: int,\n", - " height: int,\n", - " weights: tuple[int, int, int],\n", - " ) -> Image.Image:\n", - " print(f\"width: {width}\")\n", - " print(f\"height: {height}\")\n", - " print(f\"RBG weights: {weights}\")\n", - " image = Image.new(\"RGB\", (width, height))\n", - " pixels = image.load()\n", - " for i in range(width):\n", - " for j in range(height):\n", - " pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript]\n", - " (i + j * 50) % weights[0],\n", - " weights[1],\n", - " (i * 300 + j) % weights[2],\n", - " )\n", - " return image\n", - "\n", - " image = create_pattern(width, height, weights)\n", - " path = f\"/tmp/{extension}-image.{extension}\"\n", - " image.save(path)\n", - " with open(f\"/tmp/{extension}-path.json\", \"w\") as f:\n", - " json.dump(path, f)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ff25236c", - "metadata": {}, - "outputs": [], - "source": [ - "@script(\n", - " command=[\"python\"],\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - " outputs=Artifact(\n", - " name=\"hdf5output\",\n", - " path=\"/tmp/images.hdf5\",\n", - " archive=NoneArchiveStrategy(),\n", - " ),\n", - ")\n", - "def to_hdf5(paths: str):\n", - "\n", - " import h5py # pyright: ignore[reportMissingTypeStubs]\n", - " import numpy as np\n", - " from PIL import Image\n", - "\n", - " print(\"creating hdf5 file\")\n", - " with h5py.File(\"/tmp/images.hdf5\", \"w\") as f:\n", - " for i, path in enumerate(paths):\n", - " path = path.strip('\"')\n", - " print(f\"Got {path}\")\n", - " with Image.open(path) as image:\n", - " arr = np.array(image)\n", - " f.create_dataset( # pyright: ignore[reportUnknownMemberType]\n", - " f\"image_{i}\", data=arr, dtype=arr.dtype\n", - " )\n", - " print(\"done\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c93a15d1", - "metadata": {}, - "outputs": [], - "source": [ - "with Workflow(pod_spec_patch=json.dumps({\"containers\":\n", - " [{\"name\":\"main\",\n", - " \"resources\":\n", - " {\"limits\":{\"cpu\":\"1\",\n", - " \"memory\":\"1Gi\",\n", - " },\n", - " \"requests\":{\"cpu\":\"1\",\n", - " \"memory\":\"1Gi\",\n", - " }}}]}),\n", - " tolerations=[\n", - " m.Toleration(key=\"nodetype\",operator=\"Equal\",value=\"gpu\",effect=\"NoSchedule\"),\n", - " m.Toleration(key=\"nodegroup\",operator=\"Equal\",value=\"workflows\",effect=\"NoSchedule\")],\n", - " name=\"hera-example\",\n", - " entrypoint=\"workflowentry\",\n", - " api_version=\"argoproj.io/v1alpha1\",\n", - " kind=\"WorkflowTemplate\",\n", - " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", - " annotations={\n", - " \"workflows.argoproj.io/title\": \"example remade via hera\",\n", - " \"workflows.argoproj.io/description\": \"\"\"Replicates the functionality of\n", - "example.yaml\"\"\",\n", - " \"workflows.diamond.ac.uk/repository\": \"https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}\",\n", - " },\n", - " volumes=Volume(name=\"tmpdir\", mount_path=\"/tmp/\", size=\"1Gi\"),\n", - ") as w:\n", - " with DAG(name=\"workflowentry\"):\n", - " params = generate_parameters(\n", - " name=\"params\",\n", - " arguments={\n", - " \"png\": \"True\",\n", - " \"jpg\": \"True\",\n", - " \"jpeg\": \"True\",\n", - " \"tif\": \"True\",\n", - " \"tiff\": \"True\",\n", - " },\n", - " )\n", - " makeimages = create_image(with_param=params.get_parameter(\"out-parameters\"))\n", - " makehdf5 = to_hdf5(\n", - " arguments={\n", - " \"paths\": makeimages.get_parameter(\"out-paths\"),\n", - " }\n", - " )\n", - " params >> makeimages >> makehdf5 # pyright: ignore" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "04708f46", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "with open(\"src/{% endraw %}{{project_name}}{% raw %}/templates/example.txt\", \"w\") as div:\n", - " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7e9f88cb", - "metadata": {}, - "outputs": [], - "source": [ - "from {% endraw %}{{project_name}}{% raw %}.submit_workflow import submit_workflow\n", - "\n", - "await submit_workflow(w)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "{% endraw %}{{repo_name}}{% raw %} (broken)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} -{% endraw %} diff --git a/src/python_interface_to_workflows/templates/divisionyaml.txt b/src/python_interface_to_workflows/templates/divisionyaml.txt deleted file mode 100644 index c9ca1ea..0000000 --- a/src/python_interface_to_workflows/templates/divisionyaml.txt +++ /dev/null @@ -1,60 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: hera-division - annotations: - workflows.argoproj.io/description: |- - Takes a numerical input and returns - the remainder, output float, and output string to a json file - workflows.argoproj.io/title: Division via hera test - workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows - labels: - workflows.diamond.ac.uk/science-group-examples: 'true' -spec: - entrypoint: divide - templates: - - name: divide - steps: - - - name: first - template: do-division - arguments: - parameters: - - name: a - value: '2' - - name: b - value: '5' - - name: do-division - inputs: - parameters: - - name: a - - name: b - outputs: - artifacts: - - name: json-output - path: /output-dir/output.json - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: a = json.loads(r'''{{inputs.parameters.a}}''') - except: a = r'''{{inputs.parameters.a}}''' - try: b = json.loads(r'''{{inputs.parameters.b}}''') - except: b = r'''{{inputs.parameters.b}}''' - - div = a / b - intdiv = a // b - remain = a % b - dictionary_of_results = {'divide': div, 'quotient': intdiv, 'remainder': remain} - with open('/output-dir/output.json', 'w') as otpt: - json.dump(dictionary_of_results, otpt) - command: - - python - volumeMounts: - - name: output-dir - mountPath: /output-dir/ - volumes: - - name: output-dir - emptyDir: {} diff --git a/src/python_interface_to_workflows/templates/example.txt b/src/python_interface_to_workflows/templates/example.txt deleted file mode 100644 index 9dedc96..0000000 --- a/src/python_interface_to_workflows/templates/example.txt +++ /dev/null @@ -1,207 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: hera-example - annotations: - workflows.argoproj.io/description: |- - Replicates the functionality of - example.yaml - workflows.argoproj.io/title: example remade via hera - workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows - labels: - workflows.diamond.ac.uk/science-group-examples: 'true' -spec: - entrypoint: workflowentry - podSpecPatch: '{"containers": [{"name": "main", "resources": {"limits": {"cpu": - "1", "memory": "1Gi"}, "requests": {"cpu": "1", "memory": "1Gi"}}}]}' - templates: - - name: workflowentry - dag: - tasks: - - name: params - template: generate-parameters - arguments: - parameters: - - name: png - value: 'True' - - name: jpg - value: 'True' - - name: jpeg - value: 'True' - - name: tif - value: 'True' - - name: tiff - value: 'True' - - name: create-image - depends: params - template: create-image - withParam: '{{tasks.params.outputs.parameters.out-parameters}}' - arguments: - parameters: - - name: width - value: '{{item.width}}' - - name: height - value: '{{item.height}}' - - name: weights - value: '{{item.weights}}' - - name: extension - value: '{{item.extension}}' - - name: to-hdf5 - depends: create-image - template: to-hdf5 - arguments: - parameters: - - name: paths - value: '{{tasks.create-image.outputs.parameters.out-paths}}' - - name: generate-parameters - inputs: - parameters: - - name: png - - name: jpg - - name: jpeg - - name: tif - - name: tiff - outputs: - parameters: - - name: out-parameters - valueFrom: - path: /tmp/parameters.json - script: - image: ghcr.io/matt-carre/python-interface-to-workflows-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: jpeg = json.loads(r'''{{inputs.parameters.jpeg}}''') - except: jpeg = r'''{{inputs.parameters.jpeg}}''' - try: jpg = json.loads(r'''{{inputs.parameters.jpg}}''') - except: jpg = r'''{{inputs.parameters.jpg}}''' - try: png = json.loads(r'''{{inputs.parameters.png}}''') - except: png = r'''{{inputs.parameters.png}}''' - try: tif = json.loads(r'''{{inputs.parameters.tif}}''') - except: tif = r'''{{inputs.parameters.tif}}''' - try: tiff = json.loads(r'''{{inputs.parameters.tiff}}''') - except: tiff = r'''{{inputs.parameters.tiff}}''' - - import json - params: list[dict[str, int | list[int] | str] | None] = [{'width': 500, 'height': 500, 'weights': [255, 1, 100], 'extension': 'png'} if png.lower() == 'true' else None, {'width': 600, 'height': 200, 'weights': [100, 150, 100], 'extension': 'jpg'} if jpg.lower() == 'true' else None, {'width': 300, 'height': 400, 'weights': [100, 150, 100], 'extension': 'jpeg'} if jpeg.lower() == 'true' else None, {'width': 300, 'height': 200, 'weights': [230, 100, 1], 'extension': 'tif'} if tif.lower() == 'true' else None, {'width': 200, 'height': 300, 'weights': [230, 100, 1], 'extension': 'tiff'} if tiff.lower() == 'true' else None] - params_to_write: list[dict[str, int | list[int] | str]] = [image_params for image_params in params if image_params is not None] - with open('/tmp/parameters.json', 'w') as f: - json.dump(params_to_write, f) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: create-image - inputs: - parameters: - - name: width - - name: height - - name: weights - - name: extension - outputs: - artifacts: - - name: '{{inputs.parameters.extension}}-image' - path: /tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}} - archive: - none: {} - parameters: - - name: out-paths - valueFrom: - path: /tmp/{{inputs.parameters.extension}}-path.json - script: - image: ghcr.io/matt-carre/python-interface-to-workflows-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: extension = json.loads(r'''{{inputs.parameters.extension}}''') - except: extension = r'''{{inputs.parameters.extension}}''' - try: height = json.loads(r'''{{inputs.parameters.height}}''') - except: height = r'''{{inputs.parameters.height}}''' - try: weights = json.loads(r'''{{inputs.parameters.weights}}''') - except: weights = r'''{{inputs.parameters.weights}}''' - try: width = json.loads(r'''{{inputs.parameters.width}}''') - except: width = r'''{{inputs.parameters.width}}''' - - import json - from PIL import Image - - def create_pattern(width: int, height: int, weights: tuple[int, int, int]) -> Image.Image: - print(f'width: {width}') - print(f'height: {height}') - print(f'RBG weights: {weights}') - image = Image.new('RGB', (width, height)) - pixels = image.load() - for i in range(width): - for j in range(height): - pixels[i, j] = ((i + j * 50) % weights[0], weights[1], (i * 300 + j) % weights[2]) - return image - image = create_pattern(width, height, weights) - path = f'/tmp/{extension}-image.{extension}' - image.save(path) - with open(f'/tmp/{extension}-path.json', 'w') as f: - json.dump(path, f) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: to-hdf5 - inputs: - parameters: - - name: paths - outputs: - artifacts: - - name: hdf5output - path: /tmp/images.hdf5 - archive: - none: {} - script: - image: ghcr.io/matt-carre/python-interface-to-workflows-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: paths = json.loads(r'''{{inputs.parameters.paths}}''') - except: paths = r'''{{inputs.parameters.paths}}''' - - import h5py - import numpy as np - from PIL import Image - print('creating hdf5 file') - with h5py.File('/tmp/images.hdf5', 'w') as f: - for i, path in enumerate(paths): - path = path.strip('"') - print(f'Got {path}') - with Image.open(path) as image: - arr = np.array(image) - f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) - print('done') - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - tolerations: - - effect: NoSchedule - key: nodetype - operator: Equal - value: gpu - - effect: NoSchedule - key: nodegroup - operator: Equal - value: workflows - volumeClaimTemplates: - - metadata: - name: tmpdir - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi diff --git a/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py b/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py deleted file mode 100644 index 327618f..0000000 --- a/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py +++ /dev/null @@ -1,49 +0,0 @@ -import json - -from hera.workflows import ( - Artifact, - EmptyDirVolume, - Steps, - Workflow, - script, # pyright: ignore[reportUnknownVariableType] -) -from hera.workflows import models as m - - -@script( - volume_mounts=[m.VolumeMount(name="output-dir", mount_path="/output-dir/")], - outputs=Artifact(name="json-output", path="/output-dir/output.json"), -) -def do_division(a: int, b: int): - div = a / b - intdiv = a // b - remain = a % b - dictionary_of_results = { - "divide": div, - "quotient": intdiv, - "remainder": remain, - } - with open("/output-dir/output.json", "w") as otpt: - json.dump(dictionary_of_results, otpt) - - -with Workflow( - name="hera-division", # when running on argo this should be generate_name: ...- - entrypoint="divide", - api_version="argoproj.io/v1alpha1", - kind="WorkflowTemplate", # ClusterWorkflowTemplate", when on graphql - labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, - annotations={ - "workflows.argoproj.io/title": "Division via hera test", - "workflows.argoproj.io/description": """Takes a numerical input and returns - the remainder, output float, and output string to a json file""", - "workflows.diamond.ac.uk/repository": "https://github.com/DiamondLightSource/python-interface-to-workflows", - }, - volumes=EmptyDirVolume(name="output-dir", mount_path="/output-dir"), -) as w: - with Steps(name="divide"): - do_division(name="first", arguments={"a": 2, "b": 5}) - - -with open("divisionyaml.txt", "w") as div: - div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] diff --git a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py deleted file mode 100644 index 804351a..0000000 --- a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py +++ /dev/null @@ -1,202 +0,0 @@ -import json -import os - -from hera.shared import global_config -from hera.workflows import ( - DAG, - Artifact, - Parameter, - Script, - Volume, - Workflow, - script, # pyright: ignore[reportUnknownVariableType] -) -from hera.workflows import models as m -from hera.workflows.archive import NoneArchiveStrategy - -global_config.set_class_defaults( # pyright: ignore - Script, image=str(os.environ.get("DEFAULT_IMAGE")) -) - - -@script( - command=["python"], - outputs=Parameter( - name="out-parameters", value_from=m.ValueFrom(path="/tmp/parameters.json") - ), - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], -) -def generate_parameters( - png: str, - jpg: str, - jpeg: str, - tif: str, - tiff: str, -): - import json - - params: list[dict[str, int | list[int] | str] | None] = [ - {"width": 500, "height": 500, "weights": [255, 1, 100], "extension": "png"} - if png.lower() == "true" - else None, - {"width": 600, "height": 200, "weights": [100, 150, 100], "extension": "jpg"} - if jpg.lower() == "true" - else None, - {"width": 300, "height": 400, "weights": [100, 150, 100], "extension": "jpeg"} - if jpeg.lower() == "true" - else None, - {"width": 300, "height": 200, "weights": [230, 100, 1], "extension": "tif"} - if tif.lower() == "true" - else None, - {"width": 200, "height": 300, "weights": [230, 100, 1], "extension": "tiff"} - if tiff.lower() == "true" - else None, - ] - params_to_write: list[dict[str, int | list[int] | str]] = [ - image_params for image_params in params if image_params is not None - ] - with open("/tmp/parameters.json", "w") as f: - json.dump(params_to_write, f) - - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], - outputs=[ - Parameter( - name="out-paths", - value_from=m.ValueFrom( - path="/tmp/{{inputs.parameters.extension}}-path.json" - ), - ), - Artifact( - name="{{inputs.parameters.extension}}-image", - path="/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}", - archive=NoneArchiveStrategy(), - ), - ], -) -def create_image( - width: int, height: int, weights: tuple[int, int, int], extension: str -): - import json - - from PIL import Image - - def create_pattern( - width: int, - height: int, - weights: tuple[int, int, int], - ) -> Image.Image: - print(f"width: {width}") - print(f"height: {height}") - print(f"RBG weights: {weights}") - image = Image.new("RGB", (width, height)) - pixels = image.load() - for i in range(width): - for j in range(height): - pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript] - (i + j * 50) % weights[0], - weights[1], - (i * 300 + j) % weights[2], - ) - return image - - image = create_pattern(width, height, weights) - path = f"/tmp/{extension}-image.{extension}" - image.save(path) - with open(f"/tmp/{extension}-path.json", "w") as f: - json.dump(path, f) - - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], - outputs=Artifact( - name="hdf5output", - path="/tmp/images.hdf5", - archive=NoneArchiveStrategy(), - ), -) -def to_hdf5(paths: str): - - import h5py # pyright: ignore[reportMissingTypeStubs] - import numpy as np - from PIL import Image - - print("creating hdf5 file") - with h5py.File("/tmp/images.hdf5", "w") as f: - for i, path in enumerate(paths): - path = path.strip('"') - print(f"Got {path}") - with Image.open(path) as image: - arr = np.array(image) - f.create_dataset( # pyright: ignore[reportUnknownMemberType] - f"image_{i}", data=arr, dtype=arr.dtype - ) - print("done") - - -with Workflow( - pod_spec_patch=json.dumps( - { - "containers": [ - { - "name": "main", - "resources": { - "limits": { - "cpu": "1", - "memory": "1Gi", - }, - "requests": { - "cpu": "1", - "memory": "1Gi", - }, - }, - } - ] - } - ), - tolerations=[ - m.Toleration( - key="nodetype", operator="Equal", value="gpu", effect="NoSchedule" - ), - m.Toleration( - key="nodegroup", operator="Equal", value="workflows", effect="NoSchedule" - ), - ], - name="hera-example", - entrypoint="workflowentry", - api_version="argoproj.io/v1alpha1", - kind="WorkflowTemplate", - labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, - annotations={ - "workflows.argoproj.io/title": "example remade via hera", - "workflows.argoproj.io/description": """Replicates the functionality of -example.yaml""", - "workflows.diamond.ac.uk/repository": "https://github.com/DiamondLightSource/python-interface-to-workflows", - }, - volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), -) as w: - with DAG(name="workflowentry"): - params = generate_parameters( - name="params", - arguments={ - "png": "True", - "jpg": "True", - "jpeg": "True", - "tif": "True", - "tiff": "True", - }, - ) - makeimages = create_image(with_param=params.get_parameter("out-parameters")) - makehdf5 = to_hdf5( - arguments={ - "paths": makeimages.get_parameter("out-paths"), - } - ) - params >> makeimages >> makehdf5 # pyright: ignore - - -with open("src/python_interface_to_workflows/templates/example.txt", "w") as div: - div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] diff --git a/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py b/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py index 9069f5b..716e65f 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py @@ -2,9 +2,9 @@ from hera.shared import global_config from hera.workflows import ( - DAG, Artifact, Script, + Steps, Volume, Workflow, script, # pyright: ignore[reportUnknownVariableType] @@ -12,22 +12,31 @@ from hera.workflows import models as m from hera.workflows.archive import NoneArchiveStrategy +# Sets the default image, unless specified otherwise, to this. +# This image has access to the /workflow_definitions/mounted_files folder +# and was created with the included dockerfile. global_config.set_class_defaults( # pyright: ignore Script, - image="ghcr.io/diamondlightsource/python-interface-to-workflows-mounted-in-image:latest", + image="ghcr.io/matt-carre/python-interface-to-workflows-mounted-image:latest", ) +# The script decorator allows hera to convert python code into yaml @script( - command=["python"], + command=["python"], # optional + # makes output an artifact to allow it to be downloaded outputs=Artifact( name="notebook", path="/tmp/notebook.html", archive=NoneArchiveStrategy() ), + # mounts a volume named "tmpdir" at path "/tmp" volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], ) def mount_files(): + # notably, we install dependencies for scripts *within* the function we run. + # They are not interpreted as functions, but as stand-alone scripts. import subprocess + # install our dependencies subprocess.call("python -m venv /tmp/venv", shell=True) subprocess.call( "/tmp/venv/bin/pip install -r /mounted_files/requirements.txt", shell=True @@ -36,6 +45,7 @@ def mount_files(): "/tmp/venv/bin/python -m ipykernel install --prefix=/tmp/venv --name=venv", shell=True, ) + # convert the notebook file into an html file subprocess.call( "/tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /mounted_files/pandas.ipynb", # noqa: E501 shell=True, @@ -43,6 +53,7 @@ def mount_files(): with Workflow( + # assures that the container has enough resources for our workflow pod_spec_patch=json.dumps( { "containers": [ @@ -62,6 +73,7 @@ def mount_files(): ] } ), + # assures we run this in a pod with a gpu and no other scheduled workflow tolerations=[ m.Toleration( key="nodetype", operator="Equal", value="gpu", effect="NoSchedule" @@ -70,7 +82,10 @@ def mount_files(): key="nodegroup", operator="Equal", value="workflows", effect="NoSchedule" ), ], + # name of workflow - will append a short identifier automatically. name="hera-example-pandas", + # the following is the same as for writing any yaml workflow, but as variables. + # All of these are required aside from "workflows.argoproj.io/description". entrypoint="workflowentry", api_version="argoproj.io/v1alpha1", kind="WorkflowTemplate", @@ -81,14 +96,19 @@ def mount_files(): html file""", "workflows.diamond.ac.uk/repository": "https://github.com/DiamondLightSource/python-interface-to-workflows", }, + # We use Volume objects to define volumes. volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), ) as w: - with DAG(name="workflowentry"): + # We establish the order of templates to run here, using '>>' to determine order + # We can use a DAG instead, in which we write out the dag's architecture afterwards: + # grouping tasks with [], i.e., files >> [a, b] will run files, then will + # run a and b simultaneously. + with Steps(name="workflowentry"): files = mount_files() - files # pyright: ignore # noqa: B018 +# produce a yaml file so we can lint and submit it with python_workflow_submitter with open( - "src/python_interface_to_workflows/templates/example_import_files.txt", "w" + "src/python_interface_to_workflows/templates/example_import_files.yaml", "w" ) as div: div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb deleted file mode 100644 index de4001c..0000000 --- a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb +++ /dev/null @@ -1,116 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "1e3146f1", - "metadata": {}, - "source": [ - "# Running in VSCode:\n", - "\n", - "1. Set kernal to python-interface-to-workflows 3.11.x\n", - "2. Hit F1, run Jupyter: Import Notebook to Script\n", - "3. Click notebook_division.ipynb\n", - "4. Run the cells sequentially" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "494174ef", - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "\n", - "from hera.workflows import (\n", - " Artifact,\n", - " EmptyDirVolume,\n", - " Steps,\n", - " Workflow,\n", - " script, # pyright: ignore[reportUnknownVariableType]\n", - ")\n", - "from hera.workflows import models as m\n", - "\n", - "\n", - "@script(\n", - " volume_mounts=[m.VolumeMount(name=\"output-dir\", mount_path=\"/output-dir/\")],\n", - " outputs=Artifact(name=\"json-output\", path=\"/output-dir/output.json\"),\n", - ")\n", - "def do_division(a: int, b: int):\n", - " div = a / b\n", - " intdiv = a // b\n", - " remain = a % b\n", - " dictionary_of_results = {\n", - " \"divide\": div,\n", - " \"quotient\": intdiv,\n", - " \"remainder\": remain,\n", - " }\n", - " with open(\"/output-dir/output.json\", \"w\") as otpt:\n", - " json.dump(dictionary_of_results, otpt)\n", - "\n", - "\n", - "with Workflow(\n", - " name=\"hera-division\",\n", - " entrypoint=\"divide\",\n", - " api_version=\"argoproj.io/v1alpha1\",\n", - " kind=\"WorkflowTemplate\",\n", - " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", - " annotations={\n", - " \"workflows.argoproj.io/title\": \"Division via hera test\",\n", - " \"workflows.argoproj.io/description\": \"\"\"Takes a numerical input and returns\n", - " the remainder, output float, and output string to a json file\"\"\",\n", - " \"workflows.diamond.ac.uk/repository\": \"https://github.com/DiamondLightSource/python-interface-to-workflows\",\n", - " },\n", - " volumes=EmptyDirVolume(name=\"output-dir\", mount_path=\"/output-dir\"),\n", - ") as w:\n", - " with Steps(name=\"divide\"):\n", - " do_division(name=\"first\", arguments={\"a\":\"{{.Files.get '/notebooks/a.json'}}\",\n", - " \"b\":\"{{.Files.get '/notebooks/b.json'}}\"}) # file.get here\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "04708f46", - "metadata": {}, - "outputs": [], - "source": [ - "with open(\"../../templates/division_from_jupyter.txt\", \"w\") as div:\n", - " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7e9f88cb", - "metadata": {}, - "outputs": [], - "source": [ - "from python_interface_to_workflows.submit_workflow import submit_workflow\n", - "\n", - "await submit_workflow(w)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "python-interface-to-workflows (3.11.x)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.-1" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_image_example.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_image_example.ipynb deleted file mode 100644 index ef7becc..0000000 --- a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_image_example.ipynb +++ /dev/null @@ -1,280 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "1e3146f1", - "metadata": {}, - "source": [ - "# Running in VSCode:\n", - "\n", - "1. Set kernal to python-interface-to-workflows 3.11.x\n", - "2. Hit F1, run Jupyter: Import Notebook to Script\n", - "3. Click notebook_example.ipynb\n", - "4. Run the cells sequentially" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c1d5a92b", - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "\n", - "from hera.shared import global_config\n", - "from hera.workflows import (\n", - " DAG,\n", - " Artifact,\n", - " Parameter,\n", - " Script,\n", - " Volume,\n", - " Workflow,\n", - " script, # pyright: ignore[reportUnknownVariableType]\n", - ")\n", - "from hera.workflows import models as m\n", - "from hera.workflows.archive import NoneArchiveStrategy\n", - "\n", - "global_config.set_class_defaults( # pyright: ignore\n", - " Script, image=str(os.environ.get(\"DEFAULT_IMAGE\"))\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "494174ef", - "metadata": {}, - "outputs": [], - "source": [ - "@script(\n", - " command=[\"python\"],\n", - " outputs=Parameter(\n", - " name=\"out-parameters\", value_from=m.ValueFrom(path=\"/tmp/parameters.json\")\n", - " ),\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - ")\n", - "def generate_parameters(\n", - " png: str,\n", - " jpg: str,\n", - " jpeg: str,\n", - " tif: str,\n", - " tiff: str,\n", - "):\n", - " import json\n", - "\n", - " params: list[dict[str, int | list[int] | str] | None] = [\n", - " {\"width\": 500, \"height\": 500, \"weights\": [255, 1, 100], \"extension\": \"png\"}\n", - " if png.lower() == \"true\"\n", - " else None,\n", - " {\"width\": 600, \"height\": 200, \"weights\": [100, 150, 100], \"extension\": \"jpg\"}\n", - " if jpg.lower() == \"true\"\n", - " else None,\n", - " {\"width\": 300, \"height\": 400, \"weights\": [100, 150, 100], \"extension\": \"jpeg\"}\n", - " if jpeg.lower() == \"true\"\n", - " else None,\n", - " {\"width\": 300, \"height\": 200, \"weights\": [230, 100, 1], \"extension\": \"tif\"}\n", - " if tif.lower() == \"true\"\n", - " else None,\n", - " {\"width\": 200, \"height\": 300, \"weights\": [230, 100, 1], \"extension\": \"tiff\"}\n", - " if tiff.lower() == \"true\"\n", - " else None,\n", - " ]\n", - " params_to_write: list[dict[str, int | list[int] | str]] = [\n", - " image_params for image_params in params if image_params is not None\n", - " ]\n", - " with open(\"/tmp/parameters.json\", \"w\") as f:\n", - " json.dump(params_to_write, f)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8ec312fb", - "metadata": {}, - "outputs": [], - "source": [ - "@script(\n", - " command=[\"python\"],\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - " outputs=[\n", - " Parameter(\n", - " name=\"out-paths\",\n", - " value_from=m.ValueFrom(\n", - " path=\"/tmp/{{inputs.parameters.extension}}-path.json\"\n", - " ),\n", - " ),\n", - " Artifact(\n", - " name=\"{{inputs.parameters.extension}}-image\",\n", - " path=\"/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}\",\n", - " archive=NoneArchiveStrategy(),\n", - " ),\n", - " ],\n", - ")\n", - "def create_image(\n", - " width: int, height: int, weights: tuple[int, int, int], extension: str\n", - "):\n", - " import json\n", - "\n", - " from PIL import Image\n", - "\n", - " def create_pattern(\n", - " width: int,\n", - " height: int,\n", - " weights: tuple[int, int, int],\n", - " ) -> Image.Image:\n", - " print(f\"width: {width}\")\n", - " print(f\"height: {height}\")\n", - " print(f\"RBG weights: {weights}\")\n", - " image = Image.new(\"RGB\", (width, height))\n", - " pixels = image.load()\n", - " for i in range(width):\n", - " for j in range(height):\n", - " pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript]\n", - " (i + j * 50) % weights[0],\n", - " weights[1],\n", - " (i * 300 + j) % weights[2],\n", - " )\n", - " return image\n", - "\n", - " image = create_pattern(width, height, weights)\n", - " path = f\"/tmp/{extension}-image.{extension}\"\n", - " image.save(path)\n", - " with open(f\"/tmp/{extension}-path.json\", \"w\") as f:\n", - " json.dump(path, f)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ff25236c", - "metadata": {}, - "outputs": [], - "source": [ - "@script(\n", - " command=[\"python\"],\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - " outputs=Artifact(\n", - " name=\"hdf5output\",\n", - " path=\"/tmp/images.hdf5\",\n", - " archive=NoneArchiveStrategy(),\n", - " ),\n", - ")\n", - "def to_hdf5(paths: str):\n", - "\n", - " import h5py # pyright: ignore[reportMissingTypeStubs]\n", - " import numpy as np\n", - " from PIL import Image\n", - "\n", - " print(\"creating hdf5 file\")\n", - " with h5py.File(\"/tmp/images.hdf5\", \"w\") as f:\n", - " for i, path in enumerate(paths):\n", - " path = path.strip('\"')\n", - " print(f\"Got {path}\")\n", - " with Image.open(path) as image:\n", - " arr = np.array(image)\n", - " f.create_dataset( # pyright: ignore[reportUnknownMemberType]\n", - " f\"image_{i}\", data=arr, dtype=arr.dtype\n", - " )\n", - " print(\"done\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c93a15d1", - "metadata": {}, - "outputs": [], - "source": [ - "with Workflow(pod_spec_patch=json.dumps({\"containers\":\n", - " [{\"name\":\"main\",\n", - " \"resources\":\n", - " {\"limits\":{\"cpu\":\"1\",\n", - " \"memory\":\"1Gi\",\n", - " },\n", - " \"requests\":{\"cpu\":\"1\",\n", - " \"memory\":\"1Gi\",\n", - " }}}]}),\n", - " tolerations=[\n", - " m.Toleration(key=\"nodetype\",operator=\"Equal\",value=\"gpu\",effect=\"NoSchedule\"),\n", - " m.Toleration(key=\"nodegroup\",operator=\"Equal\",value=\"workflows\",effect=\"NoSchedule\")],\n", - " name=\"hera-example\",\n", - " entrypoint=\"workflowentry\",\n", - " api_version=\"argoproj.io/v1alpha1\",\n", - " kind=\"WorkflowTemplate\",\n", - " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", - " annotations={\n", - " \"workflows.argoproj.io/title\": \"example remade via hera\",\n", - " \"workflows.argoproj.io/description\": \"\"\"Replicates the functionality of\n", - "example.yaml\"\"\",\n", - " \"workflows.diamond.ac.uk/repository\": \"https://github.com/DiamondLightSource/python-interface-to-workflows\",\n", - " },\n", - " volumes=Volume(name=\"tmpdir\", mount_path=\"/tmp/\", size=\"1Gi\"),\n", - ") as w:\n", - " with DAG(name=\"workflowentry\"):\n", - " params = generate_parameters(\n", - " name=\"params\",\n", - " arguments={\n", - " \"png\": \"True\",\n", - " \"jpg\": \"True\",\n", - " \"jpeg\": \"True\",\n", - " \"tif\": \"True\",\n", - " \"tiff\": \"True\",\n", - " },\n", - " )\n", - " makeimages = create_image(with_param=params.get_parameter(\"out-parameters\"))\n", - " makehdf5 = to_hdf5(\n", - " arguments={\n", - " \"paths\": makeimages.get_parameter(\"out-paths\"),\n", - " }\n", - " )\n", - " params >> makeimages >> makehdf5 # pyright: ignore" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "04708f46", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "with open(\"src/python_interface_to_workflows/templates/example.txt\", \"w\") as div:\n", - " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7e9f88cb", - "metadata": {}, - "outputs": [], - "source": [ - "from python_interface_to_workflows.submit_workflow import submit_workflow\n", - "\n", - "await submit_workflow(w)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "python-interface-to-workflows (broken)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From f191090755261069e7112514140105621d7fe1a0 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Fri, 7 Aug 2026 09:15:38 +0000 Subject: [PATCH 13/15] docs(copier): includes helm template in copier --- .../.github/workflows/_update_image.yml | 51 ++++++++ .../src/{{ project_name }}/helm/Chart.yaml | 5 + .../helm/notebooks/pandas.ipynb | 111 ++++++++++++++++++ .../helm/notebooks/requirements.txt | 8 ++ .../helm/templates/notebook.txt.jinja | 87 ++++++++++++++ .../create_notebook_in_image.py.jinja | 2 +- src/python_interface_to_workflows/__main__.py | 12 -- .../helm/Chart.yaml | 5 + .../helm/notebooks/pandas.ipynb | 111 ++++++++++++++++++ .../helm/notebooks/requirements.txt | 8 ++ .../helm/templates/notebook.yaml | 85 ++++++++++++++ uv.lock | 4 +- 12 files changed, 474 insertions(+), 15 deletions(-) create mode 100644 src/copier_template/.github/workflows/_update_image.yml create mode 100644 src/copier_template/src/{{ project_name }}/helm/Chart.yaml create mode 100644 src/copier_template/src/{{ project_name }}/helm/notebooks/pandas.ipynb create mode 100644 src/copier_template/src/{{ project_name }}/helm/notebooks/requirements.txt create mode 100644 src/copier_template/src/{{ project_name }}/helm/templates/notebook.txt.jinja create mode 100644 src/python_interface_to_workflows/helm/Chart.yaml create mode 100644 src/python_interface_to_workflows/helm/notebooks/pandas.ipynb create mode 100644 src/python_interface_to_workflows/helm/notebooks/requirements.txt create mode 100644 src/python_interface_to_workflows/helm/templates/notebook.yaml diff --git a/src/copier_template/.github/workflows/_update_image.yml b/src/copier_template/.github/workflows/_update_image.yml new file mode 100644 index 0000000..5f7ee7d --- /dev/null +++ b/src/copier_template/.github/workflows/_update_image.yml @@ -0,0 +1,51 @@ +name: Update Docker Image + +on: + workflow_call: + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout Code + uses: actions/checkout@v6 + + - name: Generate Image Name + run: echo IMAGE_REPOSITORY=ghcr.io/$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]' | tr '[_]' '[\-]')-mounted-in-image >> $GITHUB_ENV + + - name: Log in to GitHub Docker Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v4.1.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker Metadata + id: meta + uses: docker/metadata-action@v6.1.0 + with: + images: ${{ env.IMAGE_REPOSITORY }} + tags: | + type=ref,event=branch + type=raw,value=latest,enable={{is_default_branch}} + type=match,pattern=python-interface-to-workflows@v?(.+),group=1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4.0.0 + with: + driver-opts: network=host + + - name: Build Image + uses: docker/build-push-action@v6.18.0 + with: + context: . + push: ${{ github.event_name == 'push' }} + load: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/src/copier_template/src/{{ project_name }}/helm/Chart.yaml b/src/copier_template/src/{{ project_name }}/helm/Chart.yaml new file mode 100644 index 0000000..d04b657 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/helm/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: examples +description: A chart example of WorkflowTemplates +type: application +version: "0.1" diff --git a/src/copier_template/src/{{ project_name }}/helm/notebooks/pandas.ipynb b/src/copier_template/src/{{ project_name }}/helm/notebooks/pandas.ipynb new file mode 100644 index 0000000..96b333a --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/helm/notebooks/pandas.ipynb @@ -0,0 +1,111 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Manipulation and Visualization Example\n", + "\n", + "This notebook demonstrates:\n", + "- Creating synthetic test data\n", + "- Performing data manipulation with pandas\n", + "- Visualizing the results with matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Set environment variables\n", + "os.environ[\"MPLCONFIGDIR\"] = \"/tmp/.config/matplotlib\"\n", + "\n", + "# Ensure the directories exist\n", + "os.makedirs(os.environ[\"MPLCONFIGDIR\"], exist_ok=True)\n", + "\n", + "# Import required libraries\n", + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Set a random seed for reproducibility\n", + "np.random.seed(42)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create synthetic test data\n", + "dates = pd.date_range(start='2023-01-01', periods=100)\n", + "categories = ['A', 'B', 'C']\n", + "\n", + "data = pd.DataFrame({\n", + " 'Date': dates,\n", + " 'Category': np.random.choice(categories, size=100),\n", + " 'Value': np.random.normal(loc=50, scale=10, size=100)\n", + "})\n", + "\n", + "# Display first few rows\n", + "data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Data manipulation\n", + "# 1. Add a rolling average column\n", + "data['RollingAvg'] = data['Value'].rolling(window=7).mean()\n", + "\n", + "# 2. Group by Category and calculate mean value\n", + "category_means = data.groupby('Category')['Value'].mean()\n", + "category_means" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Visualization\n", + "plt.figure(figsize=(12, 6))\n", + "\n", + "# Plot the original Value and Rolling Average for each category\n", + "for cat in data['Category'].unique():\n", + " subset = data[data['Category'] == cat]\n", + " plt.plot(subset['Date'], subset['Value'], label=f'{cat} Value', alpha=0.3)\n", + " plt.plot(subset['Date'], subset['RollingAvg'], label=f'{cat} RollingAvg')\n", + "\n", + "plt.xlabel('Date')\n", + "plt.ylabel('Value')\n", + "plt.title('Value and Rolling Average by Category')\n", + "plt.legend()\n", + "plt.grid(True)\n", + "plt.tight_layout()\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/copier_template/src/{{ project_name }}/helm/notebooks/requirements.txt b/src/copier_template/src/{{ project_name }}/helm/notebooks/requirements.txt new file mode 100644 index 0000000..58a5482 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/helm/notebooks/requirements.txt @@ -0,0 +1,8 @@ +numpy==2.2.5 +matplotlib==3.10.3 +pandas==2.2.3 +scipy==1.15.3 +nbconvert==7.17.1 +ipykernel==6.29.5 +ipython==9.2.0 +papermill==2.6.0 diff --git a/src/copier_template/src/{{ project_name }}/helm/templates/notebook.txt.jinja b/src/copier_template/src/{{ project_name }}/helm/templates/notebook.txt.jinja new file mode 100644 index 0000000..599fa44 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/helm/templates/notebook.txt.jinja @@ -0,0 +1,87 @@ +{% raw %} +# This is from workflows/example https://github.com/DiamondLightSource/workflows/blob/main/examples/helm-based-templates/templates/notebook.yaml +apiVersion: argoproj.io/v1alpha1 +kind: ClusterWorkflowTemplate +metadata: + name: notebook + labels: + workflows.diamond.ac.uk/science-group-examples: "true" + annotations: + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/{% endraw %}{{project_name}}{% raw %}" +spec: + entrypoint: notebook + volumeClaimTemplates: + - metadata: + name: tmp + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + storageClassName: netapp + podResources: + requests: + cpu: 600m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + podSpecPatch: | + containers: + - name: main + resources: + limits: + memory: "1Gi" + templates: + - name: mount-files + script: + image: docker.io/library/python:bookworm + command: [bash] + source: | + echo '{{ .Files.Get "notebooks/pandas.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb + echo '{{ .Files.Get "notebooks/requirements.txt" | b64enc }}' | base64 -d > /tmp/requirements.txt + volumeMounts: + - name: tmp + mountPath: /tmp + - name: convert-notebook + podSpecPatch: '{"containers":[{"name":"main", "resources":{"limits":{"cpu": "600m"}}}]}' + tolerations: + - effect: NoSchedule + key: nvidia.com/gpu + operator: Exists + - effect: NoSchedule + key: nodetype + operator: Equal + value: gpu + - effect: NoSchedule + key: nodegroup + operator: Equal + value: workflows + script: + image: docker.io/library/python:bookworm + command: [bash] + source: | + python -m venv /tmp/venv + /tmp/venv/bin/pip install -r /tmp/requirements.txt + /tmp/venv/bin/python -m ipykernel install --prefix=/tmp/venv --name=venv + /tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /tmp/notebook.ipynb + volumeMounts: + - name: tmp + mountPath: /tmp + outputs: + artifacts: + - name: notebook + path: /tmp/notebook.html + archive: + none: {} + + - name: notebook + dag: + tasks: + - name: files + template: mount-files + - name: convert + template: convert-notebook + dependencies: [files] +{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja index e1d49b6..28a0c2d 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja @@ -18,7 +18,7 @@ from hera.workflows.archive import NoneArchiveStrategy # and was created with the included dockerfile. global_config.set_class_defaults( # pyright: ignore Script, - image="ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-mounted-image:latest", + image="ghcr.io/matt-carre/python-interface-to-workflows-mounted-image:latest", ) diff --git a/src/python_interface_to_workflows/__main__.py b/src/python_interface_to_workflows/__main__.py index c44d97c..ff69253 100644 --- a/src/python_interface_to_workflows/__main__.py +++ b/src/python_interface_to_workflows/__main__.py @@ -17,20 +17,8 @@ def main(args: Sequence[str] | None = None) -> None: action="version", version=__version__, ) - parser.add_argument( - "-sleep", - ) parser.parse_args(args) - if "-sleep" in vars(parser.parse_args(args)): - while True: - import time - - time.sleep(200) if __name__ == "__main__": main() - while True: - import time - - time.sleep(200) diff --git a/src/python_interface_to_workflows/helm/Chart.yaml b/src/python_interface_to_workflows/helm/Chart.yaml new file mode 100644 index 0000000..d04b657 --- /dev/null +++ b/src/python_interface_to_workflows/helm/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: examples +description: A chart example of WorkflowTemplates +type: application +version: "0.1" diff --git a/src/python_interface_to_workflows/helm/notebooks/pandas.ipynb b/src/python_interface_to_workflows/helm/notebooks/pandas.ipynb new file mode 100644 index 0000000..96b333a --- /dev/null +++ b/src/python_interface_to_workflows/helm/notebooks/pandas.ipynb @@ -0,0 +1,111 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Manipulation and Visualization Example\n", + "\n", + "This notebook demonstrates:\n", + "- Creating synthetic test data\n", + "- Performing data manipulation with pandas\n", + "- Visualizing the results with matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Set environment variables\n", + "os.environ[\"MPLCONFIGDIR\"] = \"/tmp/.config/matplotlib\"\n", + "\n", + "# Ensure the directories exist\n", + "os.makedirs(os.environ[\"MPLCONFIGDIR\"], exist_ok=True)\n", + "\n", + "# Import required libraries\n", + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Set a random seed for reproducibility\n", + "np.random.seed(42)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create synthetic test data\n", + "dates = pd.date_range(start='2023-01-01', periods=100)\n", + "categories = ['A', 'B', 'C']\n", + "\n", + "data = pd.DataFrame({\n", + " 'Date': dates,\n", + " 'Category': np.random.choice(categories, size=100),\n", + " 'Value': np.random.normal(loc=50, scale=10, size=100)\n", + "})\n", + "\n", + "# Display first few rows\n", + "data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Data manipulation\n", + "# 1. Add a rolling average column\n", + "data['RollingAvg'] = data['Value'].rolling(window=7).mean()\n", + "\n", + "# 2. Group by Category and calculate mean value\n", + "category_means = data.groupby('Category')['Value'].mean()\n", + "category_means" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Visualization\n", + "plt.figure(figsize=(12, 6))\n", + "\n", + "# Plot the original Value and Rolling Average for each category\n", + "for cat in data['Category'].unique():\n", + " subset = data[data['Category'] == cat]\n", + " plt.plot(subset['Date'], subset['Value'], label=f'{cat} Value', alpha=0.3)\n", + " plt.plot(subset['Date'], subset['RollingAvg'], label=f'{cat} RollingAvg')\n", + "\n", + "plt.xlabel('Date')\n", + "plt.ylabel('Value')\n", + "plt.title('Value and Rolling Average by Category')\n", + "plt.legend()\n", + "plt.grid(True)\n", + "plt.tight_layout()\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/python_interface_to_workflows/helm/notebooks/requirements.txt b/src/python_interface_to_workflows/helm/notebooks/requirements.txt new file mode 100644 index 0000000..58a5482 --- /dev/null +++ b/src/python_interface_to_workflows/helm/notebooks/requirements.txt @@ -0,0 +1,8 @@ +numpy==2.2.5 +matplotlib==3.10.3 +pandas==2.2.3 +scipy==1.15.3 +nbconvert==7.17.1 +ipykernel==6.29.5 +ipython==9.2.0 +papermill==2.6.0 diff --git a/src/python_interface_to_workflows/helm/templates/notebook.yaml b/src/python_interface_to_workflows/helm/templates/notebook.yaml new file mode 100644 index 0000000..c7e3abf --- /dev/null +++ b/src/python_interface_to_workflows/helm/templates/notebook.yaml @@ -0,0 +1,85 @@ +# This is from workflows/example https://github.com/DiamondLightSource/workflows/blob/main/examples/helm-based-templates/templates/notebook.yaml +apiVersion: argoproj.io/v1alpha1 +kind: ClusterWorkflowTemplate +metadata: + name: notebook + labels: + workflows.diamond.ac.uk/science-group-examples: "true" + annotations: + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/python-interface-to-workflows" +spec: + entrypoint: notebook + volumeClaimTemplates: + - metadata: + name: tmp + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + storageClassName: netapp + podResources: + requests: + cpu: 600m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + podSpecPatch: | + containers: + - name: main + resources: + limits: + memory: "1Gi" + templates: + - name: mount-files + script: + image: docker.io/library/python:bookworm + command: [bash] + source: | + echo '{{ .Files.Get "notebooks/pandas.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb + echo '{{ .Files.Get "notebooks/requirements.txt" | b64enc }}' | base64 -d > /tmp/requirements.txt + volumeMounts: + - name: tmp + mountPath: /tmp + - name: convert-notebook + podSpecPatch: '{"containers":[{"name":"main", "resources":{"limits":{"cpu": "600m"}}}]}' + tolerations: + - effect: NoSchedule + key: nvidia.com/gpu + operator: Exists + - effect: NoSchedule + key: nodetype + operator: Equal + value: gpu + - effect: NoSchedule + key: nodegroup + operator: Equal + value: workflows + script: + image: docker.io/library/python:bookworm + command: [bash] + source: | + python -m venv /tmp/venv + /tmp/venv/bin/pip install -r /tmp/requirements.txt + /tmp/venv/bin/python -m ipykernel install --prefix=/tmp/venv --name=venv + /tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /tmp/notebook.ipynb + volumeMounts: + - name: tmp + mountPath: /tmp + outputs: + artifacts: + - name: notebook + path: /tmp/notebook.html + archive: + none: {} + + - name: notebook + dag: + tasks: + - name: files + template: mount-files + - name: convert + template: convert-notebook + dependencies: [files] diff --git a/uv.lock b/uv.lock index 620fa72..9e33579 100644 --- a/uv.lock +++ b/uv.lock @@ -1919,7 +1919,7 @@ wheels = [ [[package]] name = "python-workflow-submitter" -version = "0.4.2" +version = "0.4.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -1936,7 +1936,7 @@ dependencies = [ { name = "pyyaml" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1c/f8/39caad75cb0b5e9205424e91baf9dd1c719096f95dca218ca114293c045a/python_workflow_submitter-0.4.2.tar.gz", hash = "sha256:f5ff6d56c42d6f0c7c4337a59979f7d4e6f733e19a159accc5a309f8c41a4fda", size = 181442, upload-time = "2026-08-06T12:01:11.956Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/0b/aea3418dd83b4eb607571c88188a19cd2ff5fc2d64cd3c4b87314189e342/python_workflow_submitter-0.4.3.tar.gz", hash = "sha256:3cabcbe3be68feab94a5887ef9b31c8ca04a3b203c9dd515282da71331a05279", size = 180804, upload-time = "2026-08-06T12:20:03.847Z" } [[package]] name = "pyyaml" From 663951b1f958add5a0549883d36ec5a17f6b3bbb Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Mon, 10 Aug 2026 11:42:10 +0000 Subject: [PATCH 14/15] fix(uv): rebuilds uv lock --- README.md | 45 +++++++++++-------- scripts/runthefiles.sh | 6 ++- src/copier_template/pyproject.toml.jinja | 2 + .../scripts/checkyamlcompliance.py.jinja | 2 + .../scripts/runthefiles.sh.jinja | 6 ++- .../templates/example_import_files.txt.jinja | 9 ++-- .../create_notebook_in_image.py.jinja | 8 +++- .../templates/example_import_files.txt | 9 ++-- .../create_notebook_in_image.py | 8 +++- uv.lock | 8 ++-- 10 files changed, 64 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 0c5acd2..5d6fc7a 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,7 @@ Python alternative to creating and running argo workflows in the Data Analysis Platform -This is where you should write a short paragraph that describes what your module does, -how it does it, and why people should use it. +This provides a copier template and demonstrating how to use Hera to rewrite workflows using python. What | Where :---: | :---: @@ -16,21 +15,6 @@ Source | -This is where you should put some images or code snippets that illustrate -some relevant examples. If it is a library then you might put some -introductory code here: - -```python -from python_interface_to_workflows import __version__ - -print(f"Hello python_interface_to_workflows {__version__}") -``` - -Or if it is a commandline tool then you might put some example commands here: - -``` -python -m python_interface_to_workflows --version -``` # Using Copier ```bash @@ -49,6 +33,29 @@ copier copy {this_repo's_path} {new_directory_path} ``` or: ```bash -copier copy git@github.com:DiamondLightSource/python-copier-template.git new directory path +copier copy git@github.com:DiamondLightSource/python-copier-template.git . +code . +``` + +Then: +1) create a .env file in src +2) run uv lock +3) rebuild and reopen in container + +To submit your yaml files in a notebook, append the following: + +```python +from python_workflow_submitter.submit_workflow import submit_workflow_yaml + +await submit_workflow_yaml("example.yaml") +``` + +Alternatively: +```python +import asyncio +import os + +from python_workflow_submitter.submit_workflow import submit_workflow_yaml + +asyncio.run(submit_workflow_yaml("example.yaml", visit=os.environ.get("VISIT"))) ``` -rebuild in dev container without cache diff --git a/scripts/runthefiles.sh b/scripts/runthefiles.sh index 245900c..dde5ae1 100644 --- a/scripts/runthefiles.sh +++ b/scripts/runthefiles.sh @@ -9,5 +9,7 @@ do fi uv run "$file" done -mv *.txt ../templates/ -git add -u ../templates/ +if compgen -G "*.yaml" > /dev/null || compgen -G "*.txt" > /dev/null; then + mv -- *.yaml *.txt ../templates/ + git add ../templates/ +fi diff --git a/src/copier_template/pyproject.toml.jinja b/src/copier_template/pyproject.toml.jinja index 3d5c5bb..6266061 100644 --- a/src/copier_template/pyproject.toml.jinja +++ b/src/copier_template/pyproject.toml.jinja @@ -25,6 +25,7 @@ dependencies = [ "dotenv", "python-keycloak", "pytest-asyncio", + "python-workflow-submitter", ] # Add project dependencies here, e.g. ["click", "numpy"] dynamic = ["version"] license.file = "LICENSE" @@ -48,6 +49,7 @@ dev = [ "dotenv", "python-keycloak", "pytest-asyncio", + "python-workflow-submitter", ] diff --git a/src/copier_template/scripts/checkyamlcompliance.py.jinja b/src/copier_template/scripts/checkyamlcompliance.py.jinja index b020cd6..af3573f 100644 --- a/src/copier_template/scripts/checkyamlcompliance.py.jinja +++ b/src/copier_template/scripts/checkyamlcompliance.py.jinja @@ -28,6 +28,8 @@ for file in yamllist: clst_tmpt = True case "Workflow": clst_tmpt = True + case "WorkflowTemplate": + clst_tmpt = True case _: clst_tmpt = False else: diff --git a/src/copier_template/scripts/runthefiles.sh.jinja b/src/copier_template/scripts/runthefiles.sh.jinja index fa7fc42..9c6350d 100644 --- a/src/copier_template/scripts/runthefiles.sh.jinja +++ b/src/copier_template/scripts/runthefiles.sh.jinja @@ -9,5 +9,7 @@ do fi uv run "$file" done -mv *.txt src/{{project_name}}/templates/ -git add -u src/{{project_name}}/templates/ +if compgen -G "*.yaml" > /dev/null || compgen -G "*.txt" > /dev/null; then + mv -- *.yaml *.txt ../templates/ + git add ../templates/ +fi diff --git a/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja index 18e5936..1d3db39 100644 --- a/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja @@ -5,8 +5,8 @@ metadata: name: hera-example-pandas annotations: workflows.argoproj.io/description: |- - Replicates the functionality of - notebook.yaml + Runs pandas.ipynb and converts it to an + html file workflows.argoproj.io/title: notebook.yaml remade via hera workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %} labels: @@ -17,9 +17,8 @@ spec: "500m", "memory": "2Gi"}, "requests": {"cpu": "500m", "memory": "2Gi"}}}]}' templates: - name: workflowentry - dag: - tasks: - - name: mount-files + steps: + - - name: mount-files template: mount-files - name: mount-files outputs: diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja index 28a0c2d..807d427 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja @@ -109,8 +109,14 @@ html file""", # produce a yaml file so we can lint and submit it with python_workflow_submitter +# python_workflow_submitter may only submit yaml files, so rename .txt to .yaml if you +# wish to test this example. +# Furthermore, absolute paths are preferred (such as the one shown here) to place them +# within the correct folder upon commiting (see scripts/runthefiles.sh), however these +# are not required. with open( - "src/{% endraw %}{{project_name}}{% raw %}/templates/example_import_files.yaml", "w" + "/workspaces/{% endraw %}{{project_name}}{% raw %}/src/{% endraw %}{{project_name}}{% raw %}/templates/example_import_files.txt", + "w" ) as div: div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] {% endraw %} diff --git a/src/python_interface_to_workflows/templates/example_import_files.txt b/src/python_interface_to_workflows/templates/example_import_files.txt index c873986..e900759 100644 --- a/src/python_interface_to_workflows/templates/example_import_files.txt +++ b/src/python_interface_to_workflows/templates/example_import_files.txt @@ -4,8 +4,8 @@ metadata: name: hera-example-pandas annotations: workflows.argoproj.io/description: |- - Replicates the functionality of - notebook.yaml + Runs pandas.ipynb and converts it to an + html file workflows.argoproj.io/title: notebook.yaml remade via hera workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows labels: @@ -16,9 +16,8 @@ spec: "500m", "memory": "2Gi"}, "requests": {"cpu": "500m", "memory": "2Gi"}}}]}' templates: - name: workflowentry - dag: - tasks: - - name: mount-files + steps: + - - name: mount-files template: mount-files - name: mount-files outputs: diff --git a/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py b/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py index 716e65f..2078735 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py @@ -108,7 +108,13 @@ def mount_files(): # produce a yaml file so we can lint and submit it with python_workflow_submitter +# python_workflow_submitter may only submit yaml files, so rename .txt to .yaml if you +# wish to test this example. +# Furthermore, absolute paths are preferred (such as the one shown here) to place them +# within the correct folder upon commiting (see scripts/runthefiles.sh), however these +# are not required. with open( - "src/python_interface_to_workflows/templates/example_import_files.yaml", "w" + "/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/templates/example_import_files.txt", + "w", ) as div: div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] diff --git a/uv.lock b/uv.lock index 9e33579..8bdf0b2 100644 --- a/uv.lock +++ b/uv.lock @@ -1929,7 +1929,7 @@ dependencies = [ { name = "h5py" }, { name = "hera" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "pillow" }, { name = "pytest-asyncio" }, { name = "python-keycloak" }, @@ -2186,14 +2186,14 @@ wheels = [ [[package]] name = "typing-inspection" -version = "0.4.2" +version = "0.4.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/bc/4eae18cd40c65798a16267572ba346c11f599d44b01603dbd843342042bc/typing_inspection-0.4.3.tar.gz", hash = "sha256:c5f9ec1530b5c1e2c9bc34a84d9a3466ed1b2f3f2fa9f901368d9c5596210e4d", size = 76711, upload-time = "2026-08-10T09:39:18.063Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, + { url = "https://files.pythonhosted.org/packages/42/f7/7a3935abdebd5cf18705a5f0335dd6a3a18bef3baa7cb9edc3b6b9922cc8/typing_inspection-0.4.3-py3-none-any.whl", hash = "sha256:5f42b23858a91e0b4ef521f5418f03a0da3c9216fd2995ef5e73463100e676cd", size = 14693, upload-time = "2026-08-10T09:39:16.693Z" }, ] [[package]] From ccc9b152b75b7906fabdb75c8d76e84903c0a656 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Mon, 10 Aug 2026 16:31:28 +0000 Subject: [PATCH 15/15] docs(copier): updates copier for final release --- scripts/checkyamlcompliance.py | 20 ++------- src/copier_template/README.md.jinja | 5 +++ src/copier_template/renovate.json.jinja | 41 +++++++++++++++++++ .../scripts/checkyamlcompliance.py.jinja | 4 -- .../templates/example_import_files.txt.jinja | 4 +- .../create_notebook_in_image.py.jinja | 4 +- .../templates/example_import_files.txt | 4 +- .../create_notebook_in_image.py | 4 +- 8 files changed, 58 insertions(+), 28 deletions(-) create mode 100644 src/copier_template/renovate.json.jinja diff --git a/scripts/checkyamlcompliance.py b/scripts/checkyamlcompliance.py index 2f64a09..2efed7b 100644 --- a/scripts/checkyamlcompliance.py +++ b/scripts/checkyamlcompliance.py @@ -24,12 +24,6 @@ api_ver = False if "kind" in yamldata.keys(): match yamldata["kind"]: - case "ClusterWorkflowTemplate": - clst_tmpt = True - case "Workflow": - clst_tmpt = True - case "WorkflowTemplate": - clst_tmpt = True case "ClusterWorkflowTemplate": clst_tmpt = True case _: @@ -38,7 +32,6 @@ clst_tmpt = False if "metadata" in yamldata.keys(): metadata = True - gen_name = "generateName" in yamldata["metadata"].keys() normal_name = "name" in yamldata["metadata"].keys() if "annotations" in yamldata["metadata"].keys(): annotations = True @@ -64,19 +57,14 @@ else: group = labels = False else: - metadata = normal_name = gen_name = False - if all([api_ver, clst_tmpt, title, repo, group, annotations, labels]): - if gen_name != normal_name: - exit(0) - else: - print( - "generated_name and normal_name error, must have one of these not both" - ) - exit(1) + metadata = normal_name = False + if all([api_ver, clst_tmpt, title, repo, group, annotations, labels, normal_name]): + exit(0) else: print(f""" within file: {file}... metadata present?: {metadata} + name present?: {normal_name} annotations present?: {annotations} labels present?: {labels} api_ver present?: {api_ver} diff --git a/src/copier_template/README.md.jinja b/src/copier_template/README.md.jinja index e462aa9..fb65706 100644 --- a/src/copier_template/README.md.jinja +++ b/src/copier_template/README.md.jinja @@ -7,6 +7,11 @@ Python alternative to creating and running argo workflows in the Data Analysis Platform +Update pyproject.toml for the following use cases: + - Pyright linting + - Dependencies + + This is where you should write a short paragraph that describes what your module does, how it does it, and why people should use it. diff --git a/src/copier_template/renovate.json.jinja b/src/copier_template/renovate.json.jinja new file mode 100644 index 0000000..c60c307 --- /dev/null +++ b/src/copier_template/renovate.json.jinja @@ -0,0 +1,41 @@ +{{% raw %}} +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:recommended" + ], + "lockFileMaintenance": { + "description": "Keep uv.lock up to date, merging if tests pass", + "enabled": true, + "automerge": true + }, + "packageRules": [ + { + "description": "Disable python version as that is managed by {{repo_name}}", + "matchManagers": [ + "pyenv" + ], + "enabled": false + }, + { + "description": "Disable github actions that are managed by {{repo_name}}", + "matchPackageNames": [ + "actions/checkout", + "astral-sh/setup-uv", + "actions/upload-artifact", + "actions/download-artifact", + "softprops/action-gh-release", + "codecov/codecov-action", + "docker/setup-buildx-action", + "docker/login-action", + "docker/build-push-action", + "docker/metadata-action" + ], + "matchManagers": [ + "github-actions" + ], + "enabled": false + } + ] +} +{{% endraw %}} diff --git a/src/copier_template/scripts/checkyamlcompliance.py.jinja b/src/copier_template/scripts/checkyamlcompliance.py.jinja index af3573f..5b0467a 100644 --- a/src/copier_template/scripts/checkyamlcompliance.py.jinja +++ b/src/copier_template/scripts/checkyamlcompliance.py.jinja @@ -26,10 +26,6 @@ for file in yamllist: match yamldata["kind"]: case "ClusterWorkflowTemplate": clst_tmpt = True - case "Workflow": - clst_tmpt = True - case "WorkflowTemplate": - clst_tmpt = True case _: clst_tmpt = False else: diff --git a/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja index 1d3db39..4dd7e8a 100644 --- a/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja @@ -1,6 +1,6 @@ {% raw %} apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +kind: ClusterWorkflowTemplate metadata: name: hera-example-pandas annotations: @@ -28,7 +28,7 @@ spec: archive: none: {} script: - image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-mounted-image:latest + image: ghcr.io/diamondlightsource/{% endraw %}{{repo_name}}{% raw %}-mounted-image:latest source: |- import os import sys diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja index 807d427..d305429 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja @@ -18,7 +18,7 @@ from hera.workflows.archive import NoneArchiveStrategy # and was created with the included dockerfile. global_config.set_class_defaults( # pyright: ignore Script, - image="ghcr.io/matt-carre/python-interface-to-workflows-mounted-image:latest", + image="ghcr.io/diamondlightsource/python-interface-to-workflows-mounted-image:latest", ) @@ -89,7 +89,7 @@ with Workflow( # All of these are required aside from "workflows.argoproj.io/description". entrypoint="workflowentry", api_version="argoproj.io/v1alpha1", - kind="WorkflowTemplate", + kind="ClusterWorkflowTemplate", labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, annotations={ "workflows.argoproj.io/title": "notebook.yaml remade via hera", diff --git a/src/python_interface_to_workflows/templates/example_import_files.txt b/src/python_interface_to_workflows/templates/example_import_files.txt index e900759..028c475 100644 --- a/src/python_interface_to_workflows/templates/example_import_files.txt +++ b/src/python_interface_to_workflows/templates/example_import_files.txt @@ -1,5 +1,5 @@ apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +kind: ClusterWorkflowTemplate metadata: name: hera-example-pandas annotations: @@ -27,7 +27,7 @@ spec: archive: none: {} script: - image: ghcr.io/matt-carre/python-interface-to-workflows-mounted-image:latest + image: ghcr.io/diamondlightsource/python-interface-to-workflows-mounted-image:latest source: |- import os import sys diff --git a/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py b/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py index 2078735..5df63fd 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py @@ -17,7 +17,7 @@ # and was created with the included dockerfile. global_config.set_class_defaults( # pyright: ignore Script, - image="ghcr.io/matt-carre/python-interface-to-workflows-mounted-image:latest", + image="ghcr.io/diamondlightsource/python-interface-to-workflows-mounted-image:latest", ) @@ -88,7 +88,7 @@ def mount_files(): # All of these are required aside from "workflows.argoproj.io/description". entrypoint="workflowentry", api_version="argoproj.io/v1alpha1", - kind="WorkflowTemplate", + kind="ClusterWorkflowTemplate", labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, annotations={ "workflows.argoproj.io/title": "notebook.yaml remade via hera",