diff --git a/.github/workflows/_update_image.yml b/.github/workflows/_update_image.yml new file mode 100644 index 0000000..5f7ee7d --- /dev/null +++ b/.github/workflows/_update_image.yml @@ -0,0 +1,51 @@ +name: Update Docker Image + +on: + workflow_call: + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout Code + uses: actions/checkout@v6 + + - name: Generate Image Name + run: echo IMAGE_REPOSITORY=ghcr.io/$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]' | tr '[_]' '[\-]')-mounted-in-image >> $GITHUB_ENV + + - name: Log in to GitHub Docker Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v4.1.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker Metadata + id: meta + uses: docker/metadata-action@v6.1.0 + with: + images: ${{ env.IMAGE_REPOSITORY }} + tags: | + type=ref,event=branch + type=raw,value=latest,enable={{is_default_branch}} + type=match,pattern=python-interface-to-workflows@v?(.+),group=1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4.0.0 + with: + driver-opts: network=host + + - name: Build Image + uses: docker/build-push-action@v6.18.0 + with: + context: . + push: ${{ github.event_name == 'push' }} + load: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f576ab..13ffd5e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,9 @@ jobs: commit-lint: uses: ./.github/workflows/_commit_msg.yml + updateimage: + uses: ./.github/workflows/_update_image.yml + test: strategy: matrix: diff --git a/.vscode/settings.json b/.vscode/settings.json index 7ac45f8..fb1f0e8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,5 +9,8 @@ "[python]": { "editor.defaultFormatter": "charliermarsh.ruff", }, - "python.envFile": "${workspaceFolder}/workspaces/python-interface-to-workflows/src/.env" + "python.envFile": "${workspaceFolder}/workspaces/python-interface-to-workflows/src/.env", + "python.testing.pytestArgs": [ + "tests" + ] } diff --git a/Dockerfile b/Dockerfile index 0736493..33def25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,7 @@ COPY --from=build /python /python # Copy the environment, but not the source code COPY --from=build /app/.venv /app/.venv +COPY --from=build /app/src/python_interface_to_workflows/workflow_definitions/mounted_files /mounted_files/ ENV PATH=/app/.venv/bin:$PATH # change this entrypoint if it is not the same as the repo diff --git a/README.md b/README.md index 0c5acd2..5d6fc7a 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,7 @@ Python alternative to creating and running argo workflows in the Data Analysis Platform -This is where you should write a short paragraph that describes what your module does, -how it does it, and why people should use it. +This provides a copier template and demonstrating how to use Hera to rewrite workflows using python. What | Where :---: | :---: @@ -16,21 +15,6 @@ Source | -This is where you should put some images or code snippets that illustrate -some relevant examples. If it is a library then you might put some -introductory code here: - -```python -from python_interface_to_workflows import __version__ - -print(f"Hello python_interface_to_workflows {__version__}") -``` - -Or if it is a commandline tool then you might put some example commands here: - -``` -python -m python_interface_to_workflows --version -``` # Using Copier ```bash @@ -49,6 +33,29 @@ copier copy {this_repo's_path} {new_directory_path} ``` or: ```bash -copier copy git@github.com:DiamondLightSource/python-copier-template.git new directory path +copier copy git@github.com:DiamondLightSource/python-copier-template.git . +code . +``` + +Then: +1) create a .env file in src +2) run uv lock +3) rebuild and reopen in container + +To submit your yaml files in a notebook, append the following: + +```python +from python_workflow_submitter.submit_workflow import submit_workflow_yaml + +await submit_workflow_yaml("example.yaml") +``` + +Alternatively: +```python +import asyncio +import os + +from python_workflow_submitter.submit_workflow import submit_workflow_yaml + +asyncio.run(submit_workflow_yaml("example.yaml", visit=os.environ.get("VISIT"))) ``` -rebuild in dev container without cache diff --git a/pyproject.toml b/pyproject.toml index 3552c32..242640e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,8 @@ dependencies = [ "h5py", "dotenv", "python-keycloak", + "pytest-asyncio", + "python-workflow-submitter", ] # Add project dependencies here, e.g. ["click", "numpy"] dynamic = ["version"] license.file = "LICENSE" @@ -46,6 +48,8 @@ dev = [ "h5py", "dotenv", "python-keycloak", + "pytest-asyncio", + "python-workflow-submitter", ] [project.scripts] diff --git a/scripts/checkyamlcompliance.py b/scripts/checkyamlcompliance.py index 2f64a09..2efed7b 100644 --- a/scripts/checkyamlcompliance.py +++ b/scripts/checkyamlcompliance.py @@ -24,12 +24,6 @@ api_ver = False if "kind" in yamldata.keys(): match yamldata["kind"]: - case "ClusterWorkflowTemplate": - clst_tmpt = True - case "Workflow": - clst_tmpt = True - case "WorkflowTemplate": - clst_tmpt = True case "ClusterWorkflowTemplate": clst_tmpt = True case _: @@ -38,7 +32,6 @@ clst_tmpt = False if "metadata" in yamldata.keys(): metadata = True - gen_name = "generateName" in yamldata["metadata"].keys() normal_name = "name" in yamldata["metadata"].keys() if "annotations" in yamldata["metadata"].keys(): annotations = True @@ -64,19 +57,14 @@ else: group = labels = False else: - metadata = normal_name = gen_name = False - if all([api_ver, clst_tmpt, title, repo, group, annotations, labels]): - if gen_name != normal_name: - exit(0) - else: - print( - "generated_name and normal_name error, must have one of these not both" - ) - exit(1) + metadata = normal_name = False + if all([api_ver, clst_tmpt, title, repo, group, annotations, labels, normal_name]): + exit(0) else: print(f""" within file: {file}... metadata present?: {metadata} + name present?: {normal_name} annotations present?: {annotations} labels present?: {labels} api_ver present?: {api_ver} diff --git a/scripts/makecopiercorrect.sh b/scripts/makecopiercorrect.sh index 55b5d63..fd9765e 100644 --- a/scripts/makecopiercorrect.sh +++ b/scripts/makecopiercorrect.sh @@ -11,6 +11,7 @@ do sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file" sed -i 's/DiamondLightSource/{{github_org}}/g' "$file" + sed -i 's/python_interface_to_workflows/{{project_name}}/g' "$file" sed -i '1i{% raw %}' "$file" echo '{% endraw %}' >> "$file" @@ -18,6 +19,7 @@ do sed -i \ -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \ -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \ + -e 's/{{project_name}}/{% endraw %}{{project_name}}{% raw %}/g' \ "$file" mv "$file" "$file.jinja" @@ -30,12 +32,14 @@ do sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file" sed -i 's/DiamondLightSource/{{github_org}}/g' "$file" + sed -i 's/python_interface_to_workflows/{{project_name}}/g' "$file" sed -i '1i{% raw %}' "$file" echo '{% endraw %}' >> "$file" sed -i \ -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \ -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \ + -e 's/{{project_name}}/{% endraw %}{{project_name}}{% raw %}/g' \ "$file" @@ -48,13 +52,14 @@ do sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file" sed -i 's/DiamondLightSource/{{github_org}}/g' "$file" - + sed -i 's/python_interface_to_workflows/{{project_name}}/g' "$file" sed -i '1i{% raw %}' "$file" echo '{% endraw %}' >> "$file" sed -i \ -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \ -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \ + -e 's/{{project_name}}/{% endraw %}{{project_name}}{% raw %}/g' \ "$file" mv "$file" "$file.jinja" diff --git a/scripts/runthefiles.sh b/scripts/runthefiles.sh index 662bf8e..dde5ae1 100644 --- a/scripts/runthefiles.sh +++ b/scripts/runthefiles.sh @@ -3,7 +3,13 @@ cd src/python_interface_to_workflows/workflow_definitions for file in * do [[ -d "$file" ]] && continue + if grep -Eq '^\s*(from|import)\s+python_workflow_submitter\b' "$file"; then + echo "ERROR: $file imports submitter" + exit 1 + fi uv run "$file" done -mv *.txt ../templates/ -git add -u ../templates/ +if compgen -G "*.yaml" > /dev/null || compgen -G "*.txt" > /dev/null; then + mv -- *.yaml *.txt ../templates/ + git add ../templates/ +fi diff --git a/src/copier_template/.github/workflows/_update_image.yml b/src/copier_template/.github/workflows/_update_image.yml new file mode 100644 index 0000000..5f7ee7d --- /dev/null +++ b/src/copier_template/.github/workflows/_update_image.yml @@ -0,0 +1,51 @@ +name: Update Docker Image + +on: + workflow_call: + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout Code + uses: actions/checkout@v6 + + - name: Generate Image Name + run: echo IMAGE_REPOSITORY=ghcr.io/$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]' | tr '[_]' '[\-]')-mounted-in-image >> $GITHUB_ENV + + - name: Log in to GitHub Docker Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v4.1.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker Metadata + id: meta + uses: docker/metadata-action@v6.1.0 + with: + images: ${{ env.IMAGE_REPOSITORY }} + tags: | + type=ref,event=branch + type=raw,value=latest,enable={{is_default_branch}} + type=match,pattern=python-interface-to-workflows@v?(.+),group=1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4.0.0 + with: + driver-opts: network=host + + - name: Build Image + uses: docker/build-push-action@v6.18.0 + with: + context: . + push: ${{ github.event_name == 'push' }} + load: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/src/copier_template/Dockerfile.jinja b/src/copier_template/Dockerfile.jinja index 145b50a..14f5a33 100644 --- a/src/copier_template/Dockerfile.jinja +++ b/src/copier_template/Dockerfile.jinja @@ -45,6 +45,7 @@ COPY --from=build /python /python # Copy the environment, but not the source code COPY --from=build /app/.venv /app/.venv +COPY --from=build /app/src/{{project_name}}/workflow_definitions/mounted_files /mounted_files/ ENV PATH=/app/.venv/bin:$PATH # change this entrypoint if it is not the same as the repo diff --git a/src/copier_template/README.md.jinja b/src/copier_template/README.md.jinja index e462aa9..fb65706 100644 --- a/src/copier_template/README.md.jinja +++ b/src/copier_template/README.md.jinja @@ -7,6 +7,11 @@ Python alternative to creating and running argo workflows in the Data Analysis Platform +Update pyproject.toml for the following use cases: + - Pyright linting + - Dependencies + + This is where you should write a short paragraph that describes what your module does, how it does it, and why people should use it. diff --git a/src/copier_template/pyproject.toml.jinja b/src/copier_template/pyproject.toml.jinja index 9eec731..6266061 100644 --- a/src/copier_template/pyproject.toml.jinja +++ b/src/copier_template/pyproject.toml.jinja @@ -24,6 +24,8 @@ dependencies = [ "h5py", "dotenv", "python-keycloak", + "pytest-asyncio", + "python-workflow-submitter", ] # Add project dependencies here, e.g. ["click", "numpy"] dynamic = ["version"] license.file = "LICENSE" @@ -46,6 +48,8 @@ dev = [ "h5py", "dotenv", "python-keycloak", + "pytest-asyncio", + "python-workflow-submitter", ] diff --git a/src/copier_template/renovate.json.jinja b/src/copier_template/renovate.json.jinja new file mode 100644 index 0000000..c60c307 --- /dev/null +++ b/src/copier_template/renovate.json.jinja @@ -0,0 +1,41 @@ +{{% raw %}} +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:recommended" + ], + "lockFileMaintenance": { + "description": "Keep uv.lock up to date, merging if tests pass", + "enabled": true, + "automerge": true + }, + "packageRules": [ + { + "description": "Disable python version as that is managed by {{repo_name}}", + "matchManagers": [ + "pyenv" + ], + "enabled": false + }, + { + "description": "Disable github actions that are managed by {{repo_name}}", + "matchPackageNames": [ + "actions/checkout", + "astral-sh/setup-uv", + "actions/upload-artifact", + "actions/download-artifact", + "softprops/action-gh-release", + "codecov/codecov-action", + "docker/setup-buildx-action", + "docker/login-action", + "docker/build-push-action", + "docker/metadata-action" + ], + "matchManagers": [ + "github-actions" + ], + "enabled": false + } + ] +} +{{% endraw %}} diff --git a/src/copier_template/scripts/checkyamlcompliance.py.jinja b/src/copier_template/scripts/checkyamlcompliance.py.jinja index b020cd6..5b0467a 100644 --- a/src/copier_template/scripts/checkyamlcompliance.py.jinja +++ b/src/copier_template/scripts/checkyamlcompliance.py.jinja @@ -26,8 +26,6 @@ for file in yamllist: match yamldata["kind"]: case "ClusterWorkflowTemplate": clst_tmpt = True - case "Workflow": - clst_tmpt = True case _: clst_tmpt = False else: diff --git a/src/copier_template/scripts/runthefiles.sh.jinja b/src/copier_template/scripts/runthefiles.sh.jinja index c4d14a7..9c6350d 100644 --- a/src/copier_template/scripts/runthefiles.sh.jinja +++ b/src/copier_template/scripts/runthefiles.sh.jinja @@ -3,7 +3,13 @@ cd src/{{project_name}}/workflow_definitions for file in * do [[ -d "$file" ]] && continue + if grep -Eq '^\s*(from|import)\s+python_workflow_submitter\b' "$file"; then + echo "ERROR: $file imports submitter" + exit 1 + fi uv run "$file" done -mv *.txt src/{{project_name}}/templates/ -git add -u src/{{project_name}}/templates/ +if compgen -G "*.yaml" > /dev/null || compgen -G "*.txt" > /dev/null; then + mv -- *.yaml *.txt ../templates/ + git add ../templates/ +fi diff --git a/src/copier_template/src/README.md.jinja b/src/copier_template/src/README.md.jinja index c09fef6..ef19d16 100644 --- a/src/copier_template/src/README.md.jinja +++ b/src/copier_template/src/README.md.jinja @@ -1,14 +1,9 @@ # Setting up a development environment 1. run "uv lock" to generate the uv.lock file -2. Create .env in this folder (with the path src/.env) containing the following variables: - -HOST=https://argo-workflows.workflows.diamond.ac.uk/ (to submit to the production cluster) -DEFAULT_IMAGE= (usually python 3.10) -VISIT= (the Visit you wish to run the template on) -TOKEN= -EXPIRY= -AUTH= - +2. Create .env in this folder (with the path src/.env) containing the following variable (everything else will be set automatically): +```bash +VISIT={your visit} +``` 3. Build the dev container # Submitting a workflow @@ -25,3 +20,19 @@ submit_workflow(w) NOTE: Be sure to remove this line upon commiting changes, as all workflow definition files are by default, ran on pre-commit, to ensure that any yaml files they create are up to date. + +# Building a custom image +While in src, the same folder as a Dockerfile: + +podman build -t ghcr.io/Your-Github-Name/image-name . +podman login ghcr.io +podman push ghcr.io/Your-Github-Name/image-name + +Then go to your github profile, packages, and set image-name's visibility to public +After this, you may add 'image' in the script decorator, to run specific scripts within that image +Alternatively, you can set the default image at the top of the file by adding: + +```python +global_config.set_class_defaults( # pyright: ignore + Script, image=ghcr.io/Your-Github-Name/image-name) +``` diff --git a/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja b/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja deleted file mode 100644 index ff9db43..0000000 --- a/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja +++ /dev/null @@ -1,72 +0,0 @@ -import os -from typing import TypedDict, cast - -import dotenv -from keycloak import KeycloakOpenID -from keycloak.pkce_utils import generate_code_challenge, generate_code_verifier - -from {{project_name}}.auth.open_auth_url import token_expired - - -class TokenResponse(TypedDict): - access_token: str - refresh_token: str - - -class DecodedToken(TypedDict): - exp: int - - -def set_token_env_variable() -> str: - keycloak_openid = KeycloakOpenID( - client_id="workflows-cli", - server_url="https://identity.diamond.ac.uk/", - realm_name="dls", - client_secret_key="", - pool_maxsize=1, - ) - port = 8000 - code_verifier = generate_code_verifier() - code_challenge, code_challenge_method = generate_code_challenge(code_verifier) - auth_url = keycloak_openid.auth_url( - redirect_uri=f"http://localhost:{port}/", - scope="openid posix-uid profile email fedid", - state="", - code_challenge=code_challenge, - code_challenge_method=code_challenge_method, - ) - if token_expired(auth_url, port): - token = cast( - TokenResponse, - keycloak_openid.token( # pyright: ignore[reportUnknownMemberType] - grant_type="authorization_code", - code=str(os.environ.get("AUTH")), - redirect_uri=f"http://localhost:{port}/", - code_verifier=code_verifier, - ), - ) - else: - token = cast( - TokenResponse, - keycloak_openid.refresh_token( # pyright: ignore[reportUnknownMemberType] - str(os.environ.get("REFRESHTOKEN")) - ), - ) - token_info = cast( - DecodedToken, - keycloak_openid.decode_token( # pyright: ignore[reportUnknownMemberType] - token["access_token"] - ), - ) - try: - expire_time = int(token_info["exp"]) + 1800 - dotenv.set_key("src/.env", "EXPIRY", str(expire_time)) - dotenv.set_key("src/.env", "TOKEN", token["access_token"].strip("'")) - dotenv.set_key("src/.env", "REFRESHTOKEN", token["refresh_token"].strip("'")) - except AttributeError: - print("ERROR:") - exit(1) - finally: - dotenv.load_dotenv(dotenv_path="src/.env", override=True) - - return token["access_token"] diff --git a/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py b/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py deleted file mode 100644 index f8c2581..0000000 --- a/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -import socket -import time -import urllib.parse -import webbrowser -from http.server import BaseHTTPRequestHandler, HTTPServer -from typing import cast - -import dotenv - - -class _ReusingHTTPServer(HTTPServer): - allow_reuse_address = True - auth_code: str - - -class CallbackHandler(BaseHTTPRequestHandler): - def do_GET(self): - query = urllib.parse.urlparse(self.path).query - params = urllib.parse.parse_qs(query) - if "code" in params: - cast(_ReusingHTTPServer, self.server).auth_code = params["code"][0] - self.send_response(200) - self.end_headers() - self.wfile.write(b"Authorization successful. You can close this window.") - else: - self.send_response(400) - self.end_headers() - self.wfile.write(b"Missing authorization code.") - - -def token_expired(auth_url: str, port: int) -> bool: - dotenv.load_dotenv(dotenv_path="src/.env", override=True) - expiry_str: str = str(os.environ.get("EXPIRY")).strip("'") - if (expiry_str == "" or int(expiry_str)) <= float(time.time()): - _open_auth_url(auth_url, port) - return True - else: - return False - - -def _open_auth_url(auth_url: str, port: int) -> None: - httpd = _ReusingHTTPServer(("localhost", port), CallbackHandler) - webbrowser.open(auth_url) - try: - httpd.handle_request() - os.environ["AUTH"] = httpd.auth_code - dotenv.set_key("src/.env", "AUTH", httpd.auth_code) - except OSError: - os.environ["AUTH"] = "" - print("ERROR: Port in use. Please restart your terminal.") - exit(1) - finally: - httpd.socket.shutdown(socket.SHUT_RDWR) - httpd.server_close() diff --git a/src/copier_template/src/{{ project_name }}/helm/Chart.yaml b/src/copier_template/src/{{ project_name }}/helm/Chart.yaml new file mode 100644 index 0000000..d04b657 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/helm/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: examples +description: A chart example of WorkflowTemplates +type: application +version: "0.1" diff --git a/src/copier_template/src/{{ project_name }}/helm/notebooks/pandas.ipynb b/src/copier_template/src/{{ project_name }}/helm/notebooks/pandas.ipynb new file mode 100644 index 0000000..96b333a --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/helm/notebooks/pandas.ipynb @@ -0,0 +1,111 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Manipulation and Visualization Example\n", + "\n", + "This notebook demonstrates:\n", + "- Creating synthetic test data\n", + "- Performing data manipulation with pandas\n", + "- Visualizing the results with matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Set environment variables\n", + "os.environ[\"MPLCONFIGDIR\"] = \"/tmp/.config/matplotlib\"\n", + "\n", + "# Ensure the directories exist\n", + "os.makedirs(os.environ[\"MPLCONFIGDIR\"], exist_ok=True)\n", + "\n", + "# Import required libraries\n", + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Set a random seed for reproducibility\n", + "np.random.seed(42)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create synthetic test data\n", + "dates = pd.date_range(start='2023-01-01', periods=100)\n", + "categories = ['A', 'B', 'C']\n", + "\n", + "data = pd.DataFrame({\n", + " 'Date': dates,\n", + " 'Category': np.random.choice(categories, size=100),\n", + " 'Value': np.random.normal(loc=50, scale=10, size=100)\n", + "})\n", + "\n", + "# Display first few rows\n", + "data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Data manipulation\n", + "# 1. Add a rolling average column\n", + "data['RollingAvg'] = data['Value'].rolling(window=7).mean()\n", + "\n", + "# 2. Group by Category and calculate mean value\n", + "category_means = data.groupby('Category')['Value'].mean()\n", + "category_means" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Visualization\n", + "plt.figure(figsize=(12, 6))\n", + "\n", + "# Plot the original Value and Rolling Average for each category\n", + "for cat in data['Category'].unique():\n", + " subset = data[data['Category'] == cat]\n", + " plt.plot(subset['Date'], subset['Value'], label=f'{cat} Value', alpha=0.3)\n", + " plt.plot(subset['Date'], subset['RollingAvg'], label=f'{cat} RollingAvg')\n", + "\n", + "plt.xlabel('Date')\n", + "plt.ylabel('Value')\n", + "plt.title('Value and Rolling Average by Category')\n", + "plt.legend()\n", + "plt.grid(True)\n", + "plt.tight_layout()\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/copier_template/src/{{ project_name }}/helm/notebooks/requirements.txt b/src/copier_template/src/{{ project_name }}/helm/notebooks/requirements.txt new file mode 100644 index 0000000..58a5482 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/helm/notebooks/requirements.txt @@ -0,0 +1,8 @@ +numpy==2.2.5 +matplotlib==3.10.3 +pandas==2.2.3 +scipy==1.15.3 +nbconvert==7.17.1 +ipykernel==6.29.5 +ipython==9.2.0 +papermill==2.6.0 diff --git a/src/copier_template/src/{{ project_name }}/helm/templates/notebook.txt.jinja b/src/copier_template/src/{{ project_name }}/helm/templates/notebook.txt.jinja new file mode 100644 index 0000000..599fa44 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/helm/templates/notebook.txt.jinja @@ -0,0 +1,87 @@ +{% raw %} +# This is from workflows/example https://github.com/DiamondLightSource/workflows/blob/main/examples/helm-based-templates/templates/notebook.yaml +apiVersion: argoproj.io/v1alpha1 +kind: ClusterWorkflowTemplate +metadata: + name: notebook + labels: + workflows.diamond.ac.uk/science-group-examples: "true" + annotations: + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/{% endraw %}{{project_name}}{% raw %}" +spec: + entrypoint: notebook + volumeClaimTemplates: + - metadata: + name: tmp + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + storageClassName: netapp + podResources: + requests: + cpu: 600m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + podSpecPatch: | + containers: + - name: main + resources: + limits: + memory: "1Gi" + templates: + - name: mount-files + script: + image: docker.io/library/python:bookworm + command: [bash] + source: | + echo '{{ .Files.Get "notebooks/pandas.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb + echo '{{ .Files.Get "notebooks/requirements.txt" | b64enc }}' | base64 -d > /tmp/requirements.txt + volumeMounts: + - name: tmp + mountPath: /tmp + - name: convert-notebook + podSpecPatch: '{"containers":[{"name":"main", "resources":{"limits":{"cpu": "600m"}}}]}' + tolerations: + - effect: NoSchedule + key: nvidia.com/gpu + operator: Exists + - effect: NoSchedule + key: nodetype + operator: Equal + value: gpu + - effect: NoSchedule + key: nodegroup + operator: Equal + value: workflows + script: + image: docker.io/library/python:bookworm + command: [bash] + source: | + python -m venv /tmp/venv + /tmp/venv/bin/pip install -r /tmp/requirements.txt + /tmp/venv/bin/python -m ipykernel install --prefix=/tmp/venv --name=venv + /tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /tmp/notebook.ipynb + volumeMounts: + - name: tmp + mountPath: /tmp + outputs: + artifacts: + - name: notebook + path: /tmp/notebook.html + archive: + none: {} + + - name: notebook + dag: + tasks: + - name: files + template: mount-files + - name: convert + template: convert-notebook + dependencies: [files] +{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja b/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja deleted file mode 100644 index a2dfed9..0000000 --- a/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja +++ /dev/null @@ -1,50 +0,0 @@ -{% raw %} -import os - -import dotenv -from gql import Client, gql -from gql.transport.aiohttp import AIOHTTPTransport -from hera.workflows import Workflow - -from {% endraw %}{{project_name}}{% raw %}.auth.keycloak_checker import set_token_env_variable - - -def submit_workflow(w: Workflow): - yamlstr = w.to_yaml() # pyright:ignore - dotenv.load_dotenv(dotenv_path="src/.env", override=True) - token: str = set_token_env_variable() - host: str = os.environ.get("HOST") # pyright:ignore - visit: str = os.environ.get("VISIT") # pyright:ignore - - transport = AIOHTTPTransport( - url=host, - headers={"Authorization": f"Bearer {token}"}, - ) - client = Client( - transport=transport, - fetch_schema_from_transport=True, - ) - mutation = gql(""" -mutation Submit($visit: VisitInput!, $manifest: String!) { - submitWorkflow( - visit: $visit - manifest: $manifest - ) { - name - } -} -""") - result = client.execute( - mutation, - variable_values={ - "visit": { - "proposalCode": str(visit[:2]), - "proposalNumber": int(visit[2:7]), - "number": int(visit[-1]), - }, - "manifest": f"""{yamlstr}""", - }, - ) - name = str(result["submitWorkflow"]["name"]) - print(f"Job '{name}' submitted to {visit}") -{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja deleted file mode 100644 index 78cb885..0000000 --- a/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja +++ /dev/null @@ -1,216 +0,0 @@ -{% raw %} -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: hera-example - annotations: - workflows.argoproj.io/description: |- - Replicates the functionality of - example.yaml - workflows.argoproj.io/title: example remade via hera - workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %} - labels: - workflows.diamond.ac.uk/science-group-examples: 'true' -spec: - entrypoint: workflowentry - templates: - - name: workflowentry - dag: - tasks: - - name: install - template: install-dependencies - - name: params - template: generate-parameters - arguments: - parameters: - - name: png - value: 'True' - - name: jpg - value: 'True' - - name: jpeg - value: 'True' - - name: tif - value: 'True' - - name: tiff - value: 'True' - - name: create-image - depends: install && params - template: create-image - withParam: '{{tasks.params.outputs.parameters.out-parameters}}' - arguments: - parameters: - - name: width - value: '{{item.width}}' - - name: height - value: '{{item.height}}' - - name: weights - value: '{{item.weights}}' - - name: extension - value: '{{item.extension}}' - - name: to-hdf5 - depends: create-image - template: to-hdf5 - arguments: - parameters: - - name: paths - value: '{{tasks.create-image.outputs.parameters.out-paths}}' - - name: install-dependencies - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import subprocess - print('creating venv') - subprocess.check_call(['python', '-m', 'venv', '/tmp/venv']) - subprocess.check_call(['/tmp/venv/bin/pip', 'install', 'pillow', 'h5py', 'numpy', 'hera']) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: generate-parameters - inputs: - parameters: - - name: png - - name: jpg - - name: jpeg - - name: tif - - name: tiff - outputs: - parameters: - - name: out-parameters - valueFrom: - path: /tmp/parameters.json - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: jpeg = json.loads(r'''{{inputs.parameters.jpeg}}''') - except: jpeg = r'''{{inputs.parameters.jpeg}}''' - try: jpg = json.loads(r'''{{inputs.parameters.jpg}}''') - except: jpg = r'''{{inputs.parameters.jpg}}''' - try: png = json.loads(r'''{{inputs.parameters.png}}''') - except: png = r'''{{inputs.parameters.png}}''' - try: tif = json.loads(r'''{{inputs.parameters.tif}}''') - except: tif = r'''{{inputs.parameters.tif}}''' - try: tiff = json.loads(r'''{{inputs.parameters.tiff}}''') - except: tiff = r'''{{inputs.parameters.tiff}}''' - - import json - params: list[dict[str, int | list[int] | str] | None] = [{'width': 500, 'height': 500, 'weights': [255, 1, 100], 'extension': 'png'} if png.lower() == 'true' else None, {'width': 600, 'height': 200, 'weights': [100, 150, 100], 'extension': 'jpg'} if jpg.lower() == 'true' else None, {'width': 300, 'height': 400, 'weights': [100, 150, 100], 'extension': 'jpeg'} if jpeg.lower() == 'true' else None, {'width': 300, 'height': 200, 'weights': [230, 100, 1], 'extension': 'tif'} if tif.lower() == 'true' else None, {'width': 200, 'height': 300, 'weights': [230, 100, 1], 'extension': 'tiff'} if tiff.lower() == 'true' else None] - params_to_write: list[dict[str, int | list[int] | str]] = [image_params for image_params in params if image_params is not None] - with open('/tmp/parameters.json', 'w') as f: - json.dump(params_to_write, f) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: create-image - inputs: - parameters: - - name: width - - name: height - - name: weights - - name: extension - outputs: - artifacts: - - name: '{{inputs.parameters.extension}}-image' - path: /tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}} - archive: - none: {} - parameters: - - name: out-paths - valueFrom: - path: /tmp/{{inputs.parameters.extension}}-path.json - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: extension = json.loads(r'''{{inputs.parameters.extension}}''') - except: extension = r'''{{inputs.parameters.extension}}''' - try: height = json.loads(r'''{{inputs.parameters.height}}''') - except: height = r'''{{inputs.parameters.height}}''' - try: weights = json.loads(r'''{{inputs.parameters.weights}}''') - except: weights = r'''{{inputs.parameters.weights}}''' - try: width = json.loads(r'''{{inputs.parameters.width}}''') - except: width = r'''{{inputs.parameters.width}}''' - - import json - from PIL import Image - - def create_pattern(width: int, height: int, weights: tuple[int, int, int]) -> Image.Image: - print(f'width: {width}') - print(f'height: {height}') - print(f'RBG weights: {weights}') - image = Image.new('RGB', (width, height)) - pixels = image.load() - for i in range(width): - for j in range(height): - pixels[i, j] = ((i + j * 50) % weights[0], weights[1], (i * 300 + j) % weights[2]) - return image - image = create_pattern(width, height, weights) - path = f'/tmp/{extension}-image.{extension}' - image.save(path) - with open(f'/tmp/{extension}-path.json', 'w') as f: - json.dump(path, f) - command: - - /tmp/venv/bin/python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: to-hdf5 - inputs: - parameters: - - name: paths - outputs: - artifacts: - - name: hdf5output - path: /tmp/images.hdf5 - archive: - none: {} - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: paths = json.loads(r'''{{inputs.parameters.paths}}''') - except: paths = r'''{{inputs.parameters.paths}}''' - - import h5py - import numpy as np - from PIL import Image - print('creating hdf5 file') - with h5py.File('/tmp/images.hdf5', 'w') as f: - for i, path in enumerate(paths): - path = path.strip('"') - print(f'Got {path}') - with Image.open(path) as image: - arr = np.array(image) - f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) - print('done') - command: - - /tmp/venv/bin/python - volumeMounts: - - name: tmpdir - mountPath: /tmp - volumeClaimTemplates: - - metadata: - name: tmpdir - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi -{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja new file mode 100644 index 0000000..4dd7e8a --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/templates/example_import_files.txt.jinja @@ -0,0 +1,64 @@ +{% raw %} +apiVersion: argoproj.io/v1alpha1 +kind: ClusterWorkflowTemplate +metadata: + name: hera-example-pandas + annotations: + workflows.argoproj.io/description: |- + Runs pandas.ipynb and converts it to an + html file + workflows.argoproj.io/title: notebook.yaml remade via hera + workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %} + labels: + workflows.diamond.ac.uk/science-group-examples: 'true' +spec: + entrypoint: workflowentry + podSpecPatch: '{"containers": [{"name": "main", "resources": {"limits": {"cpu": + "500m", "memory": "2Gi"}, "requests": {"cpu": "500m", "memory": "2Gi"}}}]}' + templates: + - name: workflowentry + steps: + - - name: mount-files + template: mount-files + - name: mount-files + outputs: + artifacts: + - name: notebook + path: /tmp/notebook.html + archive: + none: {} + script: + image: ghcr.io/diamondlightsource/{% endraw %}{{repo_name}}{% raw %}-mounted-image:latest + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import subprocess + subprocess.call('python -m venv /tmp/venv', shell=True) + subprocess.call('/tmp/venv/bin/pip install -r /mounted_files/requirements.txt', shell=True) + subprocess.call('/tmp/venv/bin/python -m ipykernel install --prefix=/tmp/venv --name=venv', shell=True) + subprocess.call('/tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /mounted_files/pandas.ipynb', shell=True) + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + tolerations: + - effect: NoSchedule + key: nodetype + operator: Equal + value: gpu + - effect: NoSchedule + key: nodegroup + operator: Equal + value: workflows + volumeClaimTemplates: + - metadata: + name: tmpdir + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja deleted file mode 100644 index f438048..0000000 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja +++ /dev/null @@ -1,184 +0,0 @@ -{% raw %} -from hera.workflows import ( - DAG, - Artifact, - Parameter, - Volume, - Workflow, - script, # pyright: ignore[reportUnknownVariableType] -) -from hera.workflows import models as m -from hera.workflows.archive import NoneArchiveStrategy - - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], -) -def install_dependencies(): - import subprocess - - print("creating venv") - - subprocess.check_call(["python", "-m", "venv", "/tmp/venv"]) - subprocess.check_call( - ["/tmp/venv/bin/pip", "install", "pillow", "h5py", "numpy", "hera"] - ) - - -@script( - command=["python"], - outputs=Parameter( - name="out-parameters", value_from=m.ValueFrom(path="/tmp/parameters.json") - ), - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], -) -def generate_parameters( - png: str, - jpg: str, - jpeg: str, - tif: str, - tiff: str, -): - import json - - params: list[dict[str, int | list[int] | str] | None] = [ - {"width": 500, "height": 500, "weights": [255, 1, 100], "extension": "png"} - if png.lower() == "true" - else None, - {"width": 600, "height": 200, "weights": [100, 150, 100], "extension": "jpg"} - if jpg.lower() == "true" - else None, - {"width": 300, "height": 400, "weights": [100, 150, 100], "extension": "jpeg"} - if jpeg.lower() == "true" - else None, - {"width": 300, "height": 200, "weights": [230, 100, 1], "extension": "tif"} - if tif.lower() == "true" - else None, - {"width": 200, "height": 300, "weights": [230, 100, 1], "extension": "tiff"} - if tiff.lower() == "true" - else None, - ] - params_to_write: list[dict[str, int | list[int] | str]] = [ - image_params for image_params in params if image_params is not None - ] - with open("/tmp/parameters.json", "w") as f: - json.dump(params_to_write, f) - - -@script( - command=["/tmp/venv/bin/python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], - outputs=[ - Parameter( - name="out-paths", - value_from=m.ValueFrom( - path="/tmp/{{inputs.parameters.extension}}-path.json" - ), - ), - Artifact( - name="{{inputs.parameters.extension}}-image", - path="/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}", - archive=NoneArchiveStrategy(), - ), - ], -) -def create_image( - width: int, height: int, weights: tuple[int, int, int], extension: str -): - import json - - from PIL import Image - - def create_pattern( - width: int, - height: int, - weights: tuple[int, int, int], - ) -> Image.Image: - print(f"width: {width}") - print(f"height: {height}") - print(f"RBG weights: {weights}") - image = Image.new("RGB", (width, height)) - pixels = image.load() - for i in range(width): - for j in range(height): - pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript] - (i + j * 50) % weights[0], - weights[1], - (i * 300 + j) % weights[2], - ) - return image - - image = create_pattern(width, height, weights) - path = f"/tmp/{extension}-image.{extension}" - image.save(path) - with open(f"/tmp/{extension}-path.json", "w") as f: - json.dump(path, f) - - -@script( - command=["/tmp/venv/bin/python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], - outputs=Artifact( - name="hdf5output", - path="/tmp/images.hdf5", - archive=NoneArchiveStrategy(), - ), -) -def to_hdf5(paths: str): - - import h5py # pyright: ignore[reportMissingTypeStubs] - import numpy as np - from PIL import Image - - print("creating hdf5 file") - with h5py.File("/tmp/images.hdf5", "w") as f: - for i, path in enumerate(paths): - path = path.strip('"') - print(f"Got {path}") - with Image.open(path) as image: - arr = np.array(image) - f.create_dataset( # pyright: ignore[reportUnknownMemberType] - f"image_{i}", data=arr, dtype=arr.dtype - ) - print("done") - - -with Workflow( - name="hera-example", # when running on argo this should be generate_name: ...- - entrypoint="workflowentry", - api_version="argoproj.io/v1alpha1", - kind="WorkflowTemplate", # ClusterWorkflowTemplate", when on graphql - labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, - annotations={ - "workflows.argoproj.io/title": "example remade via hera", - "workflows.argoproj.io/description": """Replicates the functionality of -example.yaml""", - "workflows.diamond.ac.uk/repository": "https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}", - }, - volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), -) as w: - with DAG(name="workflowentry"): - install = install_dependencies(name="install") - params = generate_parameters( - name="params", - arguments={ - "png": "True", - "jpg": "True", - "jpeg": "True", - "tif": "True", - "tiff": "True", - }, - ) - makeimages = create_image(with_param=params.get_parameter("out-parameters")) - makehdf5 = to_hdf5( - arguments={ - "paths": makeimages.get_parameter("out-paths"), - } - ) - [install, params] >> makeimages >> makehdf5 # pyright: ignore - - -with open("example.txt", "w") as div: - div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] -{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja new file mode 100644 index 0000000..d305429 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja @@ -0,0 +1,122 @@ +{% raw %} +import json + +from hera.shared import global_config +from hera.workflows import ( + Artifact, + Script, + Steps, + Volume, + Workflow, + script, # pyright: ignore[reportUnknownVariableType] +) +from hera.workflows import models as m +from hera.workflows.archive import NoneArchiveStrategy + +# Sets the default image, unless specified otherwise, to this. +# This image has access to the /workflow_definitions/mounted_files folder +# and was created with the included dockerfile. +global_config.set_class_defaults( # pyright: ignore + Script, + image="ghcr.io/diamondlightsource/python-interface-to-workflows-mounted-image:latest", +) + + +# The script decorator allows hera to convert python code into yaml +@script( + command=["python"], # optional + # makes output an artifact to allow it to be downloaded + outputs=Artifact( + name="notebook", path="/tmp/notebook.html", archive=NoneArchiveStrategy() + ), + # mounts a volume named "tmpdir" at path "/tmp" + volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], +) +def mount_files(): + # notably, we install dependencies for scripts *within* the function we run. + # They are not interpreted as functions, but as stand-alone scripts. + import subprocess + + # install our dependencies + subprocess.call("python -m venv /tmp/venv", shell=True) + subprocess.call( + "/tmp/venv/bin/pip install -r /mounted_files/requirements.txt", shell=True + ) + subprocess.call( + "/tmp/venv/bin/python -m ipykernel install --prefix=/tmp/venv --name=venv", + shell=True, + ) + # convert the notebook file into an html file + subprocess.call( + "/tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /mounted_files/pandas.ipynb", # noqa: E501 + shell=True, + ) + + +with Workflow( + # assures that the container has enough resources for our workflow + pod_spec_patch=json.dumps( + { + "containers": [ + { + "name": "main", + "resources": { + "limits": { + "cpu": "500m", + "memory": "2Gi", + }, + "requests": { + "cpu": "500m", + "memory": "2Gi", + }, + }, + } + ] + } + ), + # assures we run this in a pod with a gpu and no other scheduled workflow + tolerations=[ + m.Toleration( + key="nodetype", operator="Equal", value="gpu", effect="NoSchedule" + ), + m.Toleration( + key="nodegroup", operator="Equal", value="workflows", effect="NoSchedule" + ), + ], + # name of workflow - will append a short identifier automatically. + name="hera-example-pandas", + # the following is the same as for writing any yaml workflow, but as variables. + # All of these are required aside from "workflows.argoproj.io/description". + entrypoint="workflowentry", + api_version="argoproj.io/v1alpha1", + kind="ClusterWorkflowTemplate", + labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, + annotations={ + "workflows.argoproj.io/title": "notebook.yaml remade via hera", + "workflows.argoproj.io/description": """Runs pandas.ipynb and converts it to an +html file""", + "workflows.diamond.ac.uk/repository": "https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}", + }, + # We use Volume objects to define volumes. + volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), +) as w: + # We establish the order of templates to run here, using '>>' to determine order + # We can use a DAG instead, in which we write out the dag's architecture afterwards: + # grouping tasks with [], i.e., files >> [a, b] will run files, then will + # run a and b simultaneously. + with Steps(name="workflowentry"): + files = mount_files() + + +# produce a yaml file so we can lint and submit it with python_workflow_submitter +# python_workflow_submitter may only submit yaml files, so rename .txt to .yaml if you +# wish to test this example. +# Furthermore, absolute paths are preferred (such as the one shown here) to place them +# within the correct folder upon commiting (see scripts/runthefiles.sh), however these +# are not required. +with open( + "/workspaces/{% endraw %}{{project_name}}{% raw %}/src/{% endraw %}{{project_name}}{% raw %}/templates/example_import_files.txt", + "w" +) as div: + div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] +{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/mounted_files/pandas.ipynb b/src/copier_template/src/{{ project_name }}/workflow_definitions/mounted_files/pandas.ipynb new file mode 100644 index 0000000..96b333a --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/mounted_files/pandas.ipynb @@ -0,0 +1,111 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Manipulation and Visualization Example\n", + "\n", + "This notebook demonstrates:\n", + "- Creating synthetic test data\n", + "- Performing data manipulation with pandas\n", + "- Visualizing the results with matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Set environment variables\n", + "os.environ[\"MPLCONFIGDIR\"] = \"/tmp/.config/matplotlib\"\n", + "\n", + "# Ensure the directories exist\n", + "os.makedirs(os.environ[\"MPLCONFIGDIR\"], exist_ok=True)\n", + "\n", + "# Import required libraries\n", + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Set a random seed for reproducibility\n", + "np.random.seed(42)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create synthetic test data\n", + "dates = pd.date_range(start='2023-01-01', periods=100)\n", + "categories = ['A', 'B', 'C']\n", + "\n", + "data = pd.DataFrame({\n", + " 'Date': dates,\n", + " 'Category': np.random.choice(categories, size=100),\n", + " 'Value': np.random.normal(loc=50, scale=10, size=100)\n", + "})\n", + "\n", + "# Display first few rows\n", + "data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Data manipulation\n", + "# 1. Add a rolling average column\n", + "data['RollingAvg'] = data['Value'].rolling(window=7).mean()\n", + "\n", + "# 2. Group by Category and calculate mean value\n", + "category_means = data.groupby('Category')['Value'].mean()\n", + "category_means" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Visualization\n", + "plt.figure(figsize=(12, 6))\n", + "\n", + "# Plot the original Value and Rolling Average for each category\n", + "for cat in data['Category'].unique():\n", + " subset = data[data['Category'] == cat]\n", + " plt.plot(subset['Date'], subset['Value'], label=f'{cat} Value', alpha=0.3)\n", + " plt.plot(subset['Date'], subset['RollingAvg'], label=f'{cat} RollingAvg')\n", + "\n", + "plt.xlabel('Date')\n", + "plt.ylabel('Value')\n", + "plt.title('Value and Rolling Average by Category')\n", + "plt.legend()\n", + "plt.grid(True)\n", + "plt.tight_layout()\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/mounted_files/requirements.txt b/src/copier_template/src/{{ project_name }}/workflow_definitions/mounted_files/requirements.txt new file mode 100644 index 0000000..58a5482 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/mounted_files/requirements.txt @@ -0,0 +1,8 @@ +numpy==2.2.5 +matplotlib==3.10.3 +pandas==2.2.3 +scipy==1.15.3 +nbconvert==7.17.1 +ipykernel==6.29.5 +ipython==9.2.0 +papermill==2.6.0 diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja deleted file mode 100644 index 0f829f6..0000000 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja +++ /dev/null @@ -1,251 +0,0 @@ -{% raw %}{ - "cells": [ - { - "cell_type": "markdown", - "id": "1e3146f1", - "metadata": {}, - "source": [ - "# Running in VSCode:\n", - "\n", - "1. Set kernal to {% endraw %}{{repo_name}}{% raw %} 3.11.x\n", - "2. Hit F1, run Jupyter: Import Notebook to Script\n", - "3. Click notebook_example.ipynb\n", - "4. Run the cells sequentially" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "494174ef", - "metadata": {}, - "outputs": [], - "source": [ - "from hera.workflows import (\n", - " DAG,\n", - " Artifact,\n", - " Parameter,\n", - " Volume,\n", - " Workflow,\n", - " script, # pyright: ignore[reportUnknownVariableType]\n", - ")\n", - "from hera.workflows import models as m\n", - "from hera.workflows.archive import NoneArchiveStrategy\n", - "\n", - "\n", - "@script(\n", - " command=[\"python\"],\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - ")\n", - "def install_dependencies():\n", - " import subprocess\n", - "\n", - " print(\"creating venv\")\n", - "\n", - " subprocess.check_call([\"python\", \"-m\", \"venv\", \"/tmp/venv\"])\n", - " subprocess.check_call(\n", - " [\"/tmp/venv/bin/pip\", \"install\", \"pillow\", \"h5py\", \"numpy\", \"hera\"]\n", - " )\n", - "\n", - "\n", - "@script(\n", - " command=[\"python\"],\n", - " outputs=Parameter(\n", - " name=\"out-parameters\", value_from=m.ValueFrom(path=\"/tmp/parameters.json\")\n", - " ),\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - ")\n", - "def generate_parameters(\n", - " png: str,\n", - " jpg: str,\n", - " jpeg: str,\n", - " tif: str,\n", - " tiff: str,\n", - "):\n", - " import json\n", - "\n", - " params: list[dict[str, int | list[int] | str] | None] = [\n", - " {\"width\": 500, \"height\": 500, \"weights\": [255, 1, 100], \"extension\": \"png\"}\n", - " if png.lower() == \"true\"\n", - " else None,\n", - " {\"width\": 600, \"height\": 200, \"weights\": [100, 150, 100], \"extension\": \"jpg\"}\n", - " if jpg.lower() == \"true\"\n", - " else None,\n", - " {\"width\": 300, \"height\": 400, \"weights\": [100, 150, 100], \"extension\": \"jpeg\"}\n", - " if jpeg.lower() == \"true\"\n", - " else None,\n", - " {\"width\": 300, \"height\": 200, \"weights\": [230, 100, 1], \"extension\": \"tif\"}\n", - " if tif.lower() == \"true\"\n", - " else None,\n", - " {\"width\": 200, \"height\": 300, \"weights\": [230, 100, 1], \"extension\": \"tiff\"}\n", - " if tiff.lower() == \"true\"\n", - " else None,\n", - " ]\n", - " params_to_write: list[dict[str, int | list[int] | str]] = [\n", - " image_params for image_params in params if image_params is not None\n", - " ]\n", - " with open(\"/tmp/parameters.json\", \"w\") as f:\n", - " json.dump(params_to_write, f)\n", - "\n", - "\n", - "@script(\n", - " command=[\"/tmp/venv/bin/python\"],\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - " outputs=[\n", - " Parameter(\n", - " name=\"out-paths\",\n", - " value_from=m.ValueFrom(\n", - " path=\"/tmp/{{inputs.parameters.extension}}-path.json\"\n", - " ),\n", - " ),\n", - " Artifact(\n", - " name=\"{{inputs.parameters.extension}}-image\",\n", - " path=\"/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}\",\n", - " archive=NoneArchiveStrategy(),\n", - " ),\n", - " ],\n", - ")\n", - "def create_image(\n", - " width: int, height: int, weights: tuple[int, int, int], extension: str\n", - "):\n", - " import json\n", - "\n", - " from PIL import Image\n", - "\n", - " def create_pattern(\n", - " width: int,\n", - " height: int,\n", - " weights: tuple[int, int, int],\n", - " ) -> Image.Image:\n", - " print(f\"width: {width}\")\n", - " print(f\"height: {height}\")\n", - " print(f\"RBG weights: {weights}\")\n", - " image = Image.new(\"RGB\", (width, height))\n", - " pixels = image.load()\n", - " for i in range(width):\n", - " for j in range(height):\n", - " pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript]\n", - " (i + j * 50) % weights[0],\n", - " weights[1],\n", - " (i * 300 + j) % weights[2],\n", - " )\n", - " return image\n", - "\n", - " image = create_pattern(width, height, weights)\n", - " path = f\"/tmp/{extension}-image.{extension}\"\n", - " image.save(path)\n", - " with open(f\"/tmp/{extension}-path.json\", \"w\") as f:\n", - " json.dump(path, f)\n", - "\n", - "\n", - "@script(\n", - " command=[\"/tmp/venv/bin/python\"],\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - " outputs=Artifact(\n", - " name=\"hdf5output\",\n", - " path=\"/tmp/images.hdf5\",\n", - " archive=NoneArchiveStrategy(),\n", - " ),\n", - ")\n", - "def to_hdf5(paths: str):\n", - "\n", - " import h5py # pyright: ignore[reportMissingTypeStubs]\n", - " import numpy as np\n", - " from PIL import Image\n", - "\n", - " print(\"creating hdf5 file\")\n", - " with h5py.File(\"/tmp/images.hdf5\", \"w\") as f:\n", - " for i, path in enumerate(paths):\n", - " path = path.strip('\"')\n", - " print(f\"Got {path}\")\n", - " with Image.open(path) as image:\n", - " arr = np.array(image)\n", - " f.create_dataset( # pyright: ignore[reportUnknownMemberType]\n", - " f\"image_{i}\", data=arr, dtype=arr.dtype\n", - " )\n", - " print(\"done\")\n", - "\n", - "\n", - "with Workflow(\n", - " generate_name=\"hera-example-\", # when running on graphql this should be name\n", - " entrypoint=\"workflowentry\",\n", - " api_version=\"argoproj.io/v1alpha1\",\n", - " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", - " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", - " annotations={\n", - " \"workflows.argoproj.io/title\": \"example remade via hera\",\n", - " \"workflows.argoproj.io/description\": \"\"\"Replicates the functionality of\n", - "example.yaml\"\"\",\n", - " \"workflows.diamond.ac.uk/repository\": \"https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}\",\n", - " },\n", - " volumes=Volume(name=\"tmpdir\", mount_path=\"/tmp/\", size=\"1Gi\"),\n", - ") as w:\n", - " with DAG(name=\"workflowentry\"):\n", - " install = install_dependencies(name=\"install\")\n", - " params = generate_parameters(\n", - " name=\"params\",\n", - " arguments={\n", - " \"png\": \"True\",\n", - " \"jpg\": \"True\",\n", - " \"jpeg\": \"True\",\n", - " \"tif\": \"True\",\n", - " \"tiff\": \"True\",\n", - " },\n", - " )\n", - " makeimages = create_image(with_param=params.get_parameter(\"out-parameters\"))\n", - " makehdf5 = to_hdf5(\n", - " arguments={\n", - " \"paths\": makeimages.get_parameter(\"out-paths\"),\n", - " }\n", - " )\n", - " [install, params] >> makeimages >> makehdf5 # pyright: ignore\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "04708f46", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "with open(\"example.txt\", \"w\") as div:\n", - " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7e9f88cb", - "metadata": {}, - "source": [ - "from {% endraw %}{{project_name}}{% raw %}.submit_workflow import submit_workflow\n", - "\n", - "submit_workflow(w)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "{% endraw %}{{repo_name}}{% raw %} (3.11.x)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.15" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} -{% endraw %} diff --git a/src/copier_template/tests/test_keycloak_checker.py.jinja b/src/copier_template/tests/test_keycloak_checker.py.jinja deleted file mode 100644 index b960007..0000000 --- a/src/copier_template/tests/test_keycloak_checker.py.jinja +++ /dev/null @@ -1,120 +0,0 @@ -import os -from unittest.mock import MagicMock, call, patch - -from pytest import mark - -from {{project_name}}.auth.keycloak_checker import set_token_env_variable - - -@mark.parametrize( - "return_present", - [ - (False), - (True), - ], -) -@patch("{{project_name}}.auth.keycloak_checker.dotenv.load_dotenv") -@patch("{{project_name}}.auth.keycloak_checker.dotenv.set_key") -@patch("{{project_name}}.auth.keycloak_checker.KeycloakOpenID") -@patch("{{project_name}}.auth.keycloak_checker.generate_code_verifier") -@patch("{{project_name}}.auth.keycloak_checker.generate_code_challenge") -@patch("{{project_name}}.auth.keycloak_checker.token_expired") -def test_set_token_env_variable( - mock_token_expired: MagicMock, - mock_gen_code_challenge: MagicMock, - mock_gen_code_verifier: MagicMock, - mock_gen_keycloak_id: MagicMock, - mock_set_key: MagicMock, - mock_load_env: MagicMock, - return_present: bool, -): - port=8000 - mock_gen_code_verifier.return_value = "verifier" - mock_gen_code_challenge.return_value = ("challenge", "S256") - os.environ["AUTH"] = "auth_url_code" - os.environ["REFRESHTOKEN"] = "refresh" - - keycloak = MagicMock() - mock_gen_keycloak_id.return_value = keycloak - mock_token_expired.return_value = return_present - keycloak.auth_url.return_value = "https://mock.site" - token = { - "access_token": "fake_token", - "refresh_token": "fake_refresh", - } - keycloak.token.return_value = token - keycloak.refresh_token.return_value = token - keycloak.decode_token.return_value = {"exp": 123456789} - assert set_token_env_variable() == "fake_token" - - mock_token_expired.assert_called_once_with("https://mock.site", port) - if return_present: - keycloak.token.assert_called_once_with( - grant_type="authorization_code", - code="auth_url_code", - redirect_uri=f"http://localhost:{port}/", - code_verifier="verifier", - ) - keycloak.refresh_token.assert_not_called() - else: - keycloak.refresh_token.assert_called_once_with("refresh") - keycloak.token.assert_not_called() - mock_set_key.assert_has_calls( - [ - call("src/.env", "EXPIRY", str(123456789 + 1800)), - call("src/.env", "TOKEN", "fake_token"), - call("src/.env", "REFRESHTOKEN", "fake_refresh"), - ] - ) - mock_load_env.assert_called_once_with( - dotenv_path="src/.env", - override=True, - ) - -@patch("{{project_name}}.auth.keycloak_checker.exit") -@patch("{{project_name}}.auth.keycloak_checker.print") -@patch("{{project_name}}.auth.keycloak_checker.dotenv.load_dotenv") -@patch("{{project_name}}.auth.keycloak_checker.dotenv.set_key") -@patch("{{project_name}}.auth.keycloak_checker.KeycloakOpenID") -@patch("{{project_name}}.auth.keycloak_checker.generate_code_verifier") -@patch("{{project_name}}.auth.keycloak_checker.generate_code_challenge") -@patch("{{project_name}}.auth.keycloak_checker.token_expired") -def test_set_token_env_variable_attribute_error( - mock_token_expired: MagicMock, - mock_gen_code_challenge: MagicMock, - mock_gen_code_verifier: MagicMock, - mock_gen_keycloak_id: MagicMock, - mock_set_key: MagicMock, - mock_load_env: MagicMock, - mock_print: MagicMock, - mock_exit: MagicMock, -): - mock_gen_code_verifier.return_value = "verifier" - mock_gen_code_challenge.return_value = ("challenge", "S256") - mock_token_expired.return_value = True - - os.environ["REFRESHTOKEN"] = "refresh" - - keycloak = MagicMock() - mock_gen_keycloak_id.return_value = keycloak - - token = { - "access_token": "fake_token", - "refresh_token": "fake_refresh", - } - - keycloak.refresh_token.return_value = token - - decoded = MagicMock() - decoded.__getitem__.side_effect = AttributeError - keycloak.decode_token.return_value = decoded - - set_token_env_variable() - - mock_print.assert_called_once_with("ERROR:") - mock_exit.assert_called_once_with(1) - mock_set_key.assert_not_called() - mock_load_env.assert_called_once_with( - dotenv_path="src/.env", - override=True, - ) diff --git a/src/copier_template/tests/test_open_auth_url.py.jinja b/src/copier_template/tests/test_open_auth_url.py.jinja deleted file mode 100644 index f35aca1..0000000 --- a/src/copier_template/tests/test_open_auth_url.py.jinja +++ /dev/null @@ -1,142 +0,0 @@ -import os -from unittest.mock import MagicMock, patch - -from {{project_name}}.auth.open_auth_url import ( - CallbackHandler, - _open_auth_url, # pyright:ignore - token_expired, -) - - -@patch("{{project_name}}.auth.open_auth_url.dotenv.set_key") -@patch("{{project_name}}.auth.open_auth_url.time.time") -@patch("{{project_name}}.auth.open_auth_url.webbrowser.open") -@patch("{{project_name}}.auth.open_auth_url._ReusingHTTPServer") -def test_open_auth_url_normal_function( - mock_http_server: MagicMock, - mock_open_browser: MagicMock, - mock_time: MagicMock, - mock_set_key: MagicMock, -): - mock_time.return_value = 100 - os.environ["EXPIRY"] = "" - - server = mock_http_server.return_value - server.auth_code = "this_is_your_code" - - _open_auth_url("url", 5173) - server.handle_request.assert_called_once() - mock_open_browser.assert_called_once_with("url") - mock_set_key.assert_called_once_with( - "src/.env", - "AUTH", - "this_is_your_code", - ) - server.socket.shutdown.assert_called_once() - server.server_close.assert_called_once() - assert os.environ["AUTH"] == "this_is_your_code" - - -@patch("{{project_name}}.auth.open_auth_url.dotenv.set_key") -@patch("{{project_name}}.auth.open_auth_url.time.time") -@patch("{{project_name}}.auth.open_auth_url.exit") -@patch("{{project_name}}.auth.open_auth_url.webbrowser.open") -@patch("{{project_name}}.auth.open_auth_url._ReusingHTTPServer") -def test_open_auth_url_raises_error( - mock_http_server: MagicMock, - mock_open_browser: MagicMock, - mock_exit: MagicMock, - mock_time: MagicMock, - mock_set_key: MagicMock, -): - mock_time.return_value = 100 - os.environ["EXPIRY"] = "" - - server = mock_http_server.return_value - server.handle_request.side_effect = OSError - - _open_auth_url("url", 5173) - mock_open_browser.assert_called_once_with("url") - assert os.environ["AUTH"] == "" - mock_exit.assert_called_once_with(1) - mock_set_key.assert_not_called() - server.socket.shutdown.assert_called_once() - server.server_close.assert_called_once() - - -@patch("{{project_name}}.auth.open_auth_url._open_auth_url") -@patch("{{project_name}}.auth.open_auth_url.dotenv.load_dotenv") -@patch("{{project_name}}.auth.open_auth_url.time.time") -def test_token_expired( - mock_time: MagicMock, - mock_load_env: MagicMock, - mock_open_auth_url: MagicMock, -): - mock_time.return_value = 100 - os.environ["EXPIRY"] = "50" - - assert token_expired("url", 5173) is True - - mock_load_env.assert_called_once_with( - dotenv_path="src/.env", - override=True, - ) - mock_open_auth_url.assert_called_once_with("url", 5173) - - -@patch("{{project_name}}.auth.open_auth_url._open_auth_url") -@patch("{{project_name}}.auth.open_auth_url.dotenv.load_dotenv") -@patch("{{project_name}}.auth.open_auth_url.time.time") -def test_token_not_expired( - mock_time: MagicMock, - mock_load_env: MagicMock, - mock_open_auth_url: MagicMock, -): - mock_time.return_value = 100 - os.environ["EXPIRY"] = "200" - - assert token_expired("url", 5173) is False - - mock_load_env.assert_called_once_with( - dotenv_path="src/.env", - override=True, - ) - mock_open_auth_url.assert_not_called() - - -def test_handler_normal_function(): - handler = CallbackHandler.__new__(CallbackHandler) - handler.path = "/?code=this_is_your_code" - - handler.server = MagicMock() - - handler.send_response = MagicMock() - handler.end_headers = MagicMock() - handler.wfile = MagicMock() - handler.wfile.write = MagicMock() - - handler.do_GET() - - assert handler.server.auth_code == "this_is_your_code" - handler.send_response.assert_called_once_with(200) - handler.wfile.write.assert_called_once_with( - b"Authorization successful. You can close this window." - ) - - -def test_handler_error_response(): - handler = CallbackHandler.__new__(CallbackHandler) - handler.path = "/" - - handler.server = MagicMock() - - handler.send_response = MagicMock() - handler.end_headers = MagicMock() - handler.wfile = MagicMock() - handler.wfile.write = MagicMock() - - handler.do_GET() - - handler.send_response.assert_called_once_with(400) - handler.end_headers.assert_called_once() - handler.wfile.write.assert_called_once_with(b"Missing authorization code.") diff --git a/src/copier_template/tests/test_submit_to_graphql.py.jinja b/src/copier_template/tests/test_submit_to_graphql.py.jinja deleted file mode 100644 index 265d189..0000000 --- a/src/copier_template/tests/test_submit_to_graphql.py.jinja +++ /dev/null @@ -1,27 +0,0 @@ -from unittest.mock import MagicMock, call, patch - -from {{project_name}}.submit_workflow import submit_workflow - - -@patch("{{project_name}}.submit_workflow.os.environ.get") -@patch("{{project_name}}.submit_workflow.dotenv.load_dotenv") -@patch("{{project_name}}.submit_workflow.Workflow") -@patch("{{project_name}}.submit_workflow.set_token_env_variable") -@patch("{{project_name}}.submit_workflow.Client") -def test_submit_workflow_to_graphql( - mock_client: MagicMock, - mock_key: MagicMock, - mock_workflow: MagicMock, - mock_load_env: MagicMock, - mock_os_get: MagicMock, -): - - mock_instance = MagicMock() - mock_key.return_value = "token" - mock_client.return_value = mock_instance - mock_instance.execute.return_value = {"submitWorkflow": {"name": "workflow123"}} - submit_workflow(mock_workflow) - mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) - mock_instance.execute.assert_called_once() - mock_workflow.to_yaml.assert_called_once() - mock_os_get.assert_has_calls([call("VISIT"), call("HOST")], any_order=True) diff --git a/src/python_interface_to_workflows/auth/keycloak_checker.py b/src/python_interface_to_workflows/auth/keycloak_checker.py deleted file mode 100644 index 6f6ff11..0000000 --- a/src/python_interface_to_workflows/auth/keycloak_checker.py +++ /dev/null @@ -1,83 +0,0 @@ -import os -from typing import TypedDict, cast - -import dotenv -from keycloak import KeycloakOpenID -from keycloak.pkce_utils import generate_code_challenge, generate_code_verifier - -from python_interface_to_workflows.auth.open_auth_url import token_expired - - -class TokenResponse(TypedDict): - access_token: str - refresh_token: str - - -class DecodedToken(TypedDict): - exp: int - - -def set_token_env_variable(staging: bool) -> str: - match staging: - case True: - keycloak_openid = KeycloakOpenID( - server_url="https://identity-test.diamond.ac.uk/", - client_id="workflows-ui-dev", - realm_name="dls", - client_secret_key="", - pool_maxsize=1, - ) - port = 5173 - case False: - keycloak_openid = KeycloakOpenID( - client_id="workflows-cli", - server_url="https://identity.diamond.ac.uk/", - realm_name="dls", - client_secret_key="", - pool_maxsize=1, - ) - port = 8000 - code_verifier = generate_code_verifier() - code_challenge, code_challenge_method = generate_code_challenge(code_verifier) - auth_url = keycloak_openid.auth_url( - redirect_uri=f"http://localhost:{port}/", - scope="openid posix-uid profile email fedid", - state="", - code_challenge=code_challenge, - code_challenge_method=code_challenge_method, - ) - if token_expired(auth_url, port): - token = cast( - TokenResponse, - keycloak_openid.token( # pyright: ignore[reportUnknownMemberType] - grant_type="authorization_code", - code=str(os.environ.get("AUTH")), - redirect_uri=f"http://localhost:{port}/", - code_verifier=code_verifier, - ), - ) - else: - token = cast( - TokenResponse, - keycloak_openid.refresh_token( # pyright: ignore[reportUnknownMemberType] - str(os.environ.get("REFRESHTOKEN")) - ), - ) - token_info = cast( - DecodedToken, - keycloak_openid.decode_token( # pyright: ignore[reportUnknownMemberType] - token["access_token"] - ), - ) - try: - expire_time = int(token_info["exp"]) + 1800 - dotenv.set_key("src/.env", "EXPIRY", str(expire_time)) - dotenv.set_key("src/.env", "TOKEN", token["access_token"].strip("'")) - dotenv.set_key("src/.env", "REFRESHTOKEN", token["refresh_token"].strip("'")) - except AttributeError: - print("ERROR:") - exit(1) - finally: - dotenv.load_dotenv(dotenv_path="src/.env", override=True) - - return token["access_token"] diff --git a/src/python_interface_to_workflows/auth/open_auth_url.py b/src/python_interface_to_workflows/auth/open_auth_url.py deleted file mode 100644 index f8c2581..0000000 --- a/src/python_interface_to_workflows/auth/open_auth_url.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -import socket -import time -import urllib.parse -import webbrowser -from http.server import BaseHTTPRequestHandler, HTTPServer -from typing import cast - -import dotenv - - -class _ReusingHTTPServer(HTTPServer): - allow_reuse_address = True - auth_code: str - - -class CallbackHandler(BaseHTTPRequestHandler): - def do_GET(self): - query = urllib.parse.urlparse(self.path).query - params = urllib.parse.parse_qs(query) - if "code" in params: - cast(_ReusingHTTPServer, self.server).auth_code = params["code"][0] - self.send_response(200) - self.end_headers() - self.wfile.write(b"Authorization successful. You can close this window.") - else: - self.send_response(400) - self.end_headers() - self.wfile.write(b"Missing authorization code.") - - -def token_expired(auth_url: str, port: int) -> bool: - dotenv.load_dotenv(dotenv_path="src/.env", override=True) - expiry_str: str = str(os.environ.get("EXPIRY")).strip("'") - if (expiry_str == "" or int(expiry_str)) <= float(time.time()): - _open_auth_url(auth_url, port) - return True - else: - return False - - -def _open_auth_url(auth_url: str, port: int) -> None: - httpd = _ReusingHTTPServer(("localhost", port), CallbackHandler) - webbrowser.open(auth_url) - try: - httpd.handle_request() - os.environ["AUTH"] = httpd.auth_code - dotenv.set_key("src/.env", "AUTH", httpd.auth_code) - except OSError: - os.environ["AUTH"] = "" - print("ERROR: Port in use. Please restart your terminal.") - exit(1) - finally: - httpd.socket.shutdown(socket.SHUT_RDWR) - httpd.server_close() diff --git a/src/python_interface_to_workflows/helm/Chart.yaml b/src/python_interface_to_workflows/helm/Chart.yaml new file mode 100644 index 0000000..d04b657 --- /dev/null +++ b/src/python_interface_to_workflows/helm/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: examples +description: A chart example of WorkflowTemplates +type: application +version: "0.1" diff --git a/src/python_interface_to_workflows/helm/notebooks/pandas.ipynb b/src/python_interface_to_workflows/helm/notebooks/pandas.ipynb new file mode 100644 index 0000000..96b333a --- /dev/null +++ b/src/python_interface_to_workflows/helm/notebooks/pandas.ipynb @@ -0,0 +1,111 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Manipulation and Visualization Example\n", + "\n", + "This notebook demonstrates:\n", + "- Creating synthetic test data\n", + "- Performing data manipulation with pandas\n", + "- Visualizing the results with matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Set environment variables\n", + "os.environ[\"MPLCONFIGDIR\"] = \"/tmp/.config/matplotlib\"\n", + "\n", + "# Ensure the directories exist\n", + "os.makedirs(os.environ[\"MPLCONFIGDIR\"], exist_ok=True)\n", + "\n", + "# Import required libraries\n", + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Set a random seed for reproducibility\n", + "np.random.seed(42)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create synthetic test data\n", + "dates = pd.date_range(start='2023-01-01', periods=100)\n", + "categories = ['A', 'B', 'C']\n", + "\n", + "data = pd.DataFrame({\n", + " 'Date': dates,\n", + " 'Category': np.random.choice(categories, size=100),\n", + " 'Value': np.random.normal(loc=50, scale=10, size=100)\n", + "})\n", + "\n", + "# Display first few rows\n", + "data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Data manipulation\n", + "# 1. Add a rolling average column\n", + "data['RollingAvg'] = data['Value'].rolling(window=7).mean()\n", + "\n", + "# 2. Group by Category and calculate mean value\n", + "category_means = data.groupby('Category')['Value'].mean()\n", + "category_means" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Visualization\n", + "plt.figure(figsize=(12, 6))\n", + "\n", + "# Plot the original Value and Rolling Average for each category\n", + "for cat in data['Category'].unique():\n", + " subset = data[data['Category'] == cat]\n", + " plt.plot(subset['Date'], subset['Value'], label=f'{cat} Value', alpha=0.3)\n", + " plt.plot(subset['Date'], subset['RollingAvg'], label=f'{cat} RollingAvg')\n", + "\n", + "plt.xlabel('Date')\n", + "plt.ylabel('Value')\n", + "plt.title('Value and Rolling Average by Category')\n", + "plt.legend()\n", + "plt.grid(True)\n", + "plt.tight_layout()\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/python_interface_to_workflows/helm/notebooks/requirements.txt b/src/python_interface_to_workflows/helm/notebooks/requirements.txt new file mode 100644 index 0000000..58a5482 --- /dev/null +++ b/src/python_interface_to_workflows/helm/notebooks/requirements.txt @@ -0,0 +1,8 @@ +numpy==2.2.5 +matplotlib==3.10.3 +pandas==2.2.3 +scipy==1.15.3 +nbconvert==7.17.1 +ipykernel==6.29.5 +ipython==9.2.0 +papermill==2.6.0 diff --git a/src/python_interface_to_workflows/helm/templates/notebook.yaml b/src/python_interface_to_workflows/helm/templates/notebook.yaml new file mode 100644 index 0000000..c7e3abf --- /dev/null +++ b/src/python_interface_to_workflows/helm/templates/notebook.yaml @@ -0,0 +1,85 @@ +# This is from workflows/example https://github.com/DiamondLightSource/workflows/blob/main/examples/helm-based-templates/templates/notebook.yaml +apiVersion: argoproj.io/v1alpha1 +kind: ClusterWorkflowTemplate +metadata: + name: notebook + labels: + workflows.diamond.ac.uk/science-group-examples: "true" + annotations: + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/python-interface-to-workflows" +spec: + entrypoint: notebook + volumeClaimTemplates: + - metadata: + name: tmp + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + storageClassName: netapp + podResources: + requests: + cpu: 600m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + podSpecPatch: | + containers: + - name: main + resources: + limits: + memory: "1Gi" + templates: + - name: mount-files + script: + image: docker.io/library/python:bookworm + command: [bash] + source: | + echo '{{ .Files.Get "notebooks/pandas.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb + echo '{{ .Files.Get "notebooks/requirements.txt" | b64enc }}' | base64 -d > /tmp/requirements.txt + volumeMounts: + - name: tmp + mountPath: /tmp + - name: convert-notebook + podSpecPatch: '{"containers":[{"name":"main", "resources":{"limits":{"cpu": "600m"}}}]}' + tolerations: + - effect: NoSchedule + key: nvidia.com/gpu + operator: Exists + - effect: NoSchedule + key: nodetype + operator: Equal + value: gpu + - effect: NoSchedule + key: nodegroup + operator: Equal + value: workflows + script: + image: docker.io/library/python:bookworm + command: [bash] + source: | + python -m venv /tmp/venv + /tmp/venv/bin/pip install -r /tmp/requirements.txt + /tmp/venv/bin/python -m ipykernel install --prefix=/tmp/venv --name=venv + /tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /tmp/notebook.ipynb + volumeMounts: + - name: tmp + mountPath: /tmp + outputs: + artifacts: + - name: notebook + path: /tmp/notebook.html + archive: + none: {} + + - name: notebook + dag: + tasks: + - name: files + template: mount-files + - name: convert + template: convert-notebook + dependencies: [files] diff --git a/src/python_interface_to_workflows/submit_workflow.py b/src/python_interface_to_workflows/submit_workflow.py deleted file mode 100644 index da3422a..0000000 --- a/src/python_interface_to_workflows/submit_workflow.py +++ /dev/null @@ -1,48 +0,0 @@ -import os - -import dotenv -from gql import Client, gql -from gql.transport.aiohttp import AIOHTTPTransport -from hera.workflows import Workflow - -from python_interface_to_workflows.auth.keycloak_checker import set_token_env_variable - - -def submit_workflow(w: Workflow): - yamlstr = w.to_yaml() # pyright:ignore - dotenv.load_dotenv(dotenv_path="src/.env", override=True) - token: str = set_token_env_variable(True) - host: str = os.environ.get("HOST") # pyright:ignore - visit: str = os.environ.get("VISIT") # pyright:ignore - - transport = AIOHTTPTransport( - url=host, - headers={"Authorization": f"Bearer {token}"}, - ) - client = Client( - transport=transport, - fetch_schema_from_transport=True, - ) - mutation = gql(""" -mutation Submit($visit: VisitInput!, $manifest: String!) { - submitWorkflow( - visit: $visit - manifest: $manifest - ) { - name - } -} -""") - result = client.execute( - mutation, - variable_values={ - "visit": { - "proposalCode": str(visit[:2]), - "proposalNumber": int(visit[2:7]), - "number": int(visit[-1]), - }, - "manifest": f"""{yamlstr}""", - }, - ) - name = str(result["submitWorkflow"]["name"]) - print(f"Job '{name}' submitted to {visit}") diff --git a/src/python_interface_to_workflows/templates/divisionyaml.txt b/src/python_interface_to_workflows/templates/divisionyaml.txt deleted file mode 100644 index c9ca1ea..0000000 --- a/src/python_interface_to_workflows/templates/divisionyaml.txt +++ /dev/null @@ -1,60 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: hera-division - annotations: - workflows.argoproj.io/description: |- - Takes a numerical input and returns - the remainder, output float, and output string to a json file - workflows.argoproj.io/title: Division via hera test - workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows - labels: - workflows.diamond.ac.uk/science-group-examples: 'true' -spec: - entrypoint: divide - templates: - - name: divide - steps: - - - name: first - template: do-division - arguments: - parameters: - - name: a - value: '2' - - name: b - value: '5' - - name: do-division - inputs: - parameters: - - name: a - - name: b - outputs: - artifacts: - - name: json-output - path: /output-dir/output.json - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: a = json.loads(r'''{{inputs.parameters.a}}''') - except: a = r'''{{inputs.parameters.a}}''' - try: b = json.loads(r'''{{inputs.parameters.b}}''') - except: b = r'''{{inputs.parameters.b}}''' - - div = a / b - intdiv = a // b - remain = a % b - dictionary_of_results = {'divide': div, 'quotient': intdiv, 'remainder': remain} - with open('/output-dir/output.json', 'w') as otpt: - json.dump(dictionary_of_results, otpt) - command: - - python - volumeMounts: - - name: output-dir - mountPath: /output-dir/ - volumes: - - name: output-dir - emptyDir: {} diff --git a/src/python_interface_to_workflows/templates/example.txt b/src/python_interface_to_workflows/templates/example.txt deleted file mode 100644 index 1ad8e0e..0000000 --- a/src/python_interface_to_workflows/templates/example.txt +++ /dev/null @@ -1,214 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: hera-example - annotations: - workflows.argoproj.io/description: |- - Replicates the functionality of - example.yaml - workflows.argoproj.io/title: example remade via hera - workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows - labels: - workflows.diamond.ac.uk/science-group-examples: 'true' -spec: - entrypoint: workflowentry - templates: - - name: workflowentry - dag: - tasks: - - name: install - template: install-dependencies - - name: params - template: generate-parameters - arguments: - parameters: - - name: png - value: 'True' - - name: jpg - value: 'True' - - name: jpeg - value: 'True' - - name: tif - value: 'True' - - name: tiff - value: 'True' - - name: create-image - depends: install && params - template: create-image - withParam: '{{tasks.params.outputs.parameters.out-parameters}}' - arguments: - parameters: - - name: width - value: '{{item.width}}' - - name: height - value: '{{item.height}}' - - name: weights - value: '{{item.weights}}' - - name: extension - value: '{{item.extension}}' - - name: to-hdf5 - depends: create-image - template: to-hdf5 - arguments: - parameters: - - name: paths - value: '{{tasks.create-image.outputs.parameters.out-paths}}' - - name: install-dependencies - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import subprocess - print('creating venv') - subprocess.check_call(['python', '-m', 'venv', '/tmp/venv']) - subprocess.check_call(['/tmp/venv/bin/pip', 'install', 'pillow', 'h5py', 'numpy', 'hera']) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: generate-parameters - inputs: - parameters: - - name: png - - name: jpg - - name: jpeg - - name: tif - - name: tiff - outputs: - parameters: - - name: out-parameters - valueFrom: - path: /tmp/parameters.json - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: jpeg = json.loads(r'''{{inputs.parameters.jpeg}}''') - except: jpeg = r'''{{inputs.parameters.jpeg}}''' - try: jpg = json.loads(r'''{{inputs.parameters.jpg}}''') - except: jpg = r'''{{inputs.parameters.jpg}}''' - try: png = json.loads(r'''{{inputs.parameters.png}}''') - except: png = r'''{{inputs.parameters.png}}''' - try: tif = json.loads(r'''{{inputs.parameters.tif}}''') - except: tif = r'''{{inputs.parameters.tif}}''' - try: tiff = json.loads(r'''{{inputs.parameters.tiff}}''') - except: tiff = r'''{{inputs.parameters.tiff}}''' - - import json - params: list[dict[str, int | list[int] | str] | None] = [{'width': 500, 'height': 500, 'weights': [255, 1, 100], 'extension': 'png'} if png.lower() == 'true' else None, {'width': 600, 'height': 200, 'weights': [100, 150, 100], 'extension': 'jpg'} if jpg.lower() == 'true' else None, {'width': 300, 'height': 400, 'weights': [100, 150, 100], 'extension': 'jpeg'} if jpeg.lower() == 'true' else None, {'width': 300, 'height': 200, 'weights': [230, 100, 1], 'extension': 'tif'} if tif.lower() == 'true' else None, {'width': 200, 'height': 300, 'weights': [230, 100, 1], 'extension': 'tiff'} if tiff.lower() == 'true' else None] - params_to_write: list[dict[str, int | list[int] | str]] = [image_params for image_params in params if image_params is not None] - with open('/tmp/parameters.json', 'w') as f: - json.dump(params_to_write, f) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: create-image - inputs: - parameters: - - name: width - - name: height - - name: weights - - name: extension - outputs: - artifacts: - - name: '{{inputs.parameters.extension}}-image' - path: /tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}} - archive: - none: {} - parameters: - - name: out-paths - valueFrom: - path: /tmp/{{inputs.parameters.extension}}-path.json - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: extension = json.loads(r'''{{inputs.parameters.extension}}''') - except: extension = r'''{{inputs.parameters.extension}}''' - try: height = json.loads(r'''{{inputs.parameters.height}}''') - except: height = r'''{{inputs.parameters.height}}''' - try: weights = json.loads(r'''{{inputs.parameters.weights}}''') - except: weights = r'''{{inputs.parameters.weights}}''' - try: width = json.loads(r'''{{inputs.parameters.width}}''') - except: width = r'''{{inputs.parameters.width}}''' - - import json - from PIL import Image - - def create_pattern(width: int, height: int, weights: tuple[int, int, int]) -> Image.Image: - print(f'width: {width}') - print(f'height: {height}') - print(f'RBG weights: {weights}') - image = Image.new('RGB', (width, height)) - pixels = image.load() - for i in range(width): - for j in range(height): - pixels[i, j] = ((i + j * 50) % weights[0], weights[1], (i * 300 + j) % weights[2]) - return image - image = create_pattern(width, height, weights) - path = f'/tmp/{extension}-image.{extension}' - image.save(path) - with open(f'/tmp/{extension}-path.json', 'w') as f: - json.dump(path, f) - command: - - /tmp/venv/bin/python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: to-hdf5 - inputs: - parameters: - - name: paths - outputs: - artifacts: - - name: hdf5output - path: /tmp/images.hdf5 - archive: - none: {} - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: paths = json.loads(r'''{{inputs.parameters.paths}}''') - except: paths = r'''{{inputs.parameters.paths}}''' - - import h5py - import numpy as np - from PIL import Image - print('creating hdf5 file') - with h5py.File('/tmp/images.hdf5', 'w') as f: - for i, path in enumerate(paths): - path = path.strip('"') - print(f'Got {path}') - with Image.open(path) as image: - arr = np.array(image) - f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) - print('done') - command: - - /tmp/venv/bin/python - volumeMounts: - - name: tmpdir - mountPath: /tmp - volumeClaimTemplates: - - metadata: - name: tmpdir - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi diff --git a/src/python_interface_to_workflows/templates/example_import_files.txt b/src/python_interface_to_workflows/templates/example_import_files.txt new file mode 100644 index 0000000..028c475 --- /dev/null +++ b/src/python_interface_to_workflows/templates/example_import_files.txt @@ -0,0 +1,62 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ClusterWorkflowTemplate +metadata: + name: hera-example-pandas + annotations: + workflows.argoproj.io/description: |- + Runs pandas.ipynb and converts it to an + html file + workflows.argoproj.io/title: notebook.yaml remade via hera + workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows + labels: + workflows.diamond.ac.uk/science-group-examples: 'true' +spec: + entrypoint: workflowentry + podSpecPatch: '{"containers": [{"name": "main", "resources": {"limits": {"cpu": + "500m", "memory": "2Gi"}, "requests": {"cpu": "500m", "memory": "2Gi"}}}]}' + templates: + - name: workflowentry + steps: + - - name: mount-files + template: mount-files + - name: mount-files + outputs: + artifacts: + - name: notebook + path: /tmp/notebook.html + archive: + none: {} + script: + image: ghcr.io/diamondlightsource/python-interface-to-workflows-mounted-image:latest + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import subprocess + subprocess.call('python -m venv /tmp/venv', shell=True) + subprocess.call('/tmp/venv/bin/pip install -r /mounted_files/requirements.txt', shell=True) + subprocess.call('/tmp/venv/bin/python -m ipykernel install --prefix=/tmp/venv --name=venv', shell=True) + subprocess.call('/tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /mounted_files/pandas.ipynb', shell=True) + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + tolerations: + - effect: NoSchedule + key: nodetype + operator: Equal + value: gpu + - effect: NoSchedule + key: nodegroup + operator: Equal + value: workflows + volumeClaimTemplates: + - metadata: + name: tmpdir + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py b/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py deleted file mode 100644 index 327618f..0000000 --- a/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py +++ /dev/null @@ -1,49 +0,0 @@ -import json - -from hera.workflows import ( - Artifact, - EmptyDirVolume, - Steps, - Workflow, - script, # pyright: ignore[reportUnknownVariableType] -) -from hera.workflows import models as m - - -@script( - volume_mounts=[m.VolumeMount(name="output-dir", mount_path="/output-dir/")], - outputs=Artifact(name="json-output", path="/output-dir/output.json"), -) -def do_division(a: int, b: int): - div = a / b - intdiv = a // b - remain = a % b - dictionary_of_results = { - "divide": div, - "quotient": intdiv, - "remainder": remain, - } - with open("/output-dir/output.json", "w") as otpt: - json.dump(dictionary_of_results, otpt) - - -with Workflow( - name="hera-division", # when running on argo this should be generate_name: ...- - entrypoint="divide", - api_version="argoproj.io/v1alpha1", - kind="WorkflowTemplate", # ClusterWorkflowTemplate", when on graphql - labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, - annotations={ - "workflows.argoproj.io/title": "Division via hera test", - "workflows.argoproj.io/description": """Takes a numerical input and returns - the remainder, output float, and output string to a json file""", - "workflows.diamond.ac.uk/repository": "https://github.com/DiamondLightSource/python-interface-to-workflows", - }, - volumes=EmptyDirVolume(name="output-dir", mount_path="/output-dir"), -) as w: - with Steps(name="divide"): - do_division(name="first", arguments={"a": 2, "b": 5}) - - -with open("divisionyaml.txt", "w") as div: - div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] diff --git a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py deleted file mode 100644 index 013387a..0000000 --- a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py +++ /dev/null @@ -1,182 +0,0 @@ -from hera.workflows import ( - DAG, - Artifact, - Parameter, - Volume, - Workflow, - script, # pyright: ignore[reportUnknownVariableType] -) -from hera.workflows import models as m -from hera.workflows.archive import NoneArchiveStrategy - - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], -) -def install_dependencies(): - import subprocess - - print("creating venv") - - subprocess.check_call(["python", "-m", "venv", "/tmp/venv"]) - subprocess.check_call( - ["/tmp/venv/bin/pip", "install", "pillow", "h5py", "numpy", "hera"] - ) - - -@script( - command=["python"], - outputs=Parameter( - name="out-parameters", value_from=m.ValueFrom(path="/tmp/parameters.json") - ), - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], -) -def generate_parameters( - png: str, - jpg: str, - jpeg: str, - tif: str, - tiff: str, -): - import json - - params: list[dict[str, int | list[int] | str] | None] = [ - {"width": 500, "height": 500, "weights": [255, 1, 100], "extension": "png"} - if png.lower() == "true" - else None, - {"width": 600, "height": 200, "weights": [100, 150, 100], "extension": "jpg"} - if jpg.lower() == "true" - else None, - {"width": 300, "height": 400, "weights": [100, 150, 100], "extension": "jpeg"} - if jpeg.lower() == "true" - else None, - {"width": 300, "height": 200, "weights": [230, 100, 1], "extension": "tif"} - if tif.lower() == "true" - else None, - {"width": 200, "height": 300, "weights": [230, 100, 1], "extension": "tiff"} - if tiff.lower() == "true" - else None, - ] - params_to_write: list[dict[str, int | list[int] | str]] = [ - image_params for image_params in params if image_params is not None - ] - with open("/tmp/parameters.json", "w") as f: - json.dump(params_to_write, f) - - -@script( - command=["/tmp/venv/bin/python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], - outputs=[ - Parameter( - name="out-paths", - value_from=m.ValueFrom( - path="/tmp/{{inputs.parameters.extension}}-path.json" - ), - ), - Artifact( - name="{{inputs.parameters.extension}}-image", - path="/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}", - archive=NoneArchiveStrategy(), - ), - ], -) -def create_image( - width: int, height: int, weights: tuple[int, int, int], extension: str -): - import json - - from PIL import Image - - def create_pattern( - width: int, - height: int, - weights: tuple[int, int, int], - ) -> Image.Image: - print(f"width: {width}") - print(f"height: {height}") - print(f"RBG weights: {weights}") - image = Image.new("RGB", (width, height)) - pixels = image.load() - for i in range(width): - for j in range(height): - pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript] - (i + j * 50) % weights[0], - weights[1], - (i * 300 + j) % weights[2], - ) - return image - - image = create_pattern(width, height, weights) - path = f"/tmp/{extension}-image.{extension}" - image.save(path) - with open(f"/tmp/{extension}-path.json", "w") as f: - json.dump(path, f) - - -@script( - command=["/tmp/venv/bin/python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], - outputs=Artifact( - name="hdf5output", - path="/tmp/images.hdf5", - archive=NoneArchiveStrategy(), - ), -) -def to_hdf5(paths: str): - - import h5py # pyright: ignore[reportMissingTypeStubs] - import numpy as np - from PIL import Image - - print("creating hdf5 file") - with h5py.File("/tmp/images.hdf5", "w") as f: - for i, path in enumerate(paths): - path = path.strip('"') - print(f"Got {path}") - with Image.open(path) as image: - arr = np.array(image) - f.create_dataset( # pyright: ignore[reportUnknownMemberType] - f"image_{i}", data=arr, dtype=arr.dtype - ) - print("done") - - -with Workflow( - name="hera-example", # when running on argo this should be generate_name: ...- - entrypoint="workflowentry", - api_version="argoproj.io/v1alpha1", - kind="WorkflowTemplate", # ClusterWorkflowTemplate", when on graphql - labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, - annotations={ - "workflows.argoproj.io/title": "example remade via hera", - "workflows.argoproj.io/description": """Replicates the functionality of -example.yaml""", - "workflows.diamond.ac.uk/repository": "https://github.com/DiamondLightSource/python-interface-to-workflows", - }, - volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), -) as w: - with DAG(name="workflowentry"): - install = install_dependencies(name="install") - params = generate_parameters( - name="params", - arguments={ - "png": "True", - "jpg": "True", - "jpeg": "True", - "tif": "True", - "tiff": "True", - }, - ) - makeimages = create_image(with_param=params.get_parameter("out-parameters")) - makehdf5 = to_hdf5( - arguments={ - "paths": makeimages.get_parameter("out-paths"), - } - ) - [install, params] >> makeimages >> makehdf5 # pyright: ignore - - -with open("example.txt", "w") as div: - div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] diff --git a/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py b/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py new file mode 100644 index 0000000..5df63fd --- /dev/null +++ b/src/python_interface_to_workflows/workflow_definitions/create_notebook_in_image.py @@ -0,0 +1,120 @@ +import json + +from hera.shared import global_config +from hera.workflows import ( + Artifact, + Script, + Steps, + Volume, + Workflow, + script, # pyright: ignore[reportUnknownVariableType] +) +from hera.workflows import models as m +from hera.workflows.archive import NoneArchiveStrategy + +# Sets the default image, unless specified otherwise, to this. +# This image has access to the /workflow_definitions/mounted_files folder +# and was created with the included dockerfile. +global_config.set_class_defaults( # pyright: ignore + Script, + image="ghcr.io/diamondlightsource/python-interface-to-workflows-mounted-image:latest", +) + + +# The script decorator allows hera to convert python code into yaml +@script( + command=["python"], # optional + # makes output an artifact to allow it to be downloaded + outputs=Artifact( + name="notebook", path="/tmp/notebook.html", archive=NoneArchiveStrategy() + ), + # mounts a volume named "tmpdir" at path "/tmp" + volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], +) +def mount_files(): + # notably, we install dependencies for scripts *within* the function we run. + # They are not interpreted as functions, but as stand-alone scripts. + import subprocess + + # install our dependencies + subprocess.call("python -m venv /tmp/venv", shell=True) + subprocess.call( + "/tmp/venv/bin/pip install -r /mounted_files/requirements.txt", shell=True + ) + subprocess.call( + "/tmp/venv/bin/python -m ipykernel install --prefix=/tmp/venv --name=venv", + shell=True, + ) + # convert the notebook file into an html file + subprocess.call( + "/tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /mounted_files/pandas.ipynb", # noqa: E501 + shell=True, + ) + + +with Workflow( + # assures that the container has enough resources for our workflow + pod_spec_patch=json.dumps( + { + "containers": [ + { + "name": "main", + "resources": { + "limits": { + "cpu": "500m", + "memory": "2Gi", + }, + "requests": { + "cpu": "500m", + "memory": "2Gi", + }, + }, + } + ] + } + ), + # assures we run this in a pod with a gpu and no other scheduled workflow + tolerations=[ + m.Toleration( + key="nodetype", operator="Equal", value="gpu", effect="NoSchedule" + ), + m.Toleration( + key="nodegroup", operator="Equal", value="workflows", effect="NoSchedule" + ), + ], + # name of workflow - will append a short identifier automatically. + name="hera-example-pandas", + # the following is the same as for writing any yaml workflow, but as variables. + # All of these are required aside from "workflows.argoproj.io/description". + entrypoint="workflowentry", + api_version="argoproj.io/v1alpha1", + kind="ClusterWorkflowTemplate", + labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, + annotations={ + "workflows.argoproj.io/title": "notebook.yaml remade via hera", + "workflows.argoproj.io/description": """Runs pandas.ipynb and converts it to an +html file""", + "workflows.diamond.ac.uk/repository": "https://github.com/DiamondLightSource/python-interface-to-workflows", + }, + # We use Volume objects to define volumes. + volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), +) as w: + # We establish the order of templates to run here, using '>>' to determine order + # We can use a DAG instead, in which we write out the dag's architecture afterwards: + # grouping tasks with [], i.e., files >> [a, b] will run files, then will + # run a and b simultaneously. + with Steps(name="workflowentry"): + files = mount_files() + + +# produce a yaml file so we can lint and submit it with python_workflow_submitter +# python_workflow_submitter may only submit yaml files, so rename .txt to .yaml if you +# wish to test this example. +# Furthermore, absolute paths are preferred (such as the one shown here) to place them +# within the correct folder upon commiting (see scripts/runthefiles.sh), however these +# are not required. +with open( + "/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/templates/example_import_files.txt", + "w", +) as div: + div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] diff --git a/src/python_interface_to_workflows/workflow_definitions/mounted_files/pandas.ipynb b/src/python_interface_to_workflows/workflow_definitions/mounted_files/pandas.ipynb new file mode 100644 index 0000000..96b333a --- /dev/null +++ b/src/python_interface_to_workflows/workflow_definitions/mounted_files/pandas.ipynb @@ -0,0 +1,111 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Manipulation and Visualization Example\n", + "\n", + "This notebook demonstrates:\n", + "- Creating synthetic test data\n", + "- Performing data manipulation with pandas\n", + "- Visualizing the results with matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Set environment variables\n", + "os.environ[\"MPLCONFIGDIR\"] = \"/tmp/.config/matplotlib\"\n", + "\n", + "# Ensure the directories exist\n", + "os.makedirs(os.environ[\"MPLCONFIGDIR\"], exist_ok=True)\n", + "\n", + "# Import required libraries\n", + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Set a random seed for reproducibility\n", + "np.random.seed(42)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create synthetic test data\n", + "dates = pd.date_range(start='2023-01-01', periods=100)\n", + "categories = ['A', 'B', 'C']\n", + "\n", + "data = pd.DataFrame({\n", + " 'Date': dates,\n", + " 'Category': np.random.choice(categories, size=100),\n", + " 'Value': np.random.normal(loc=50, scale=10, size=100)\n", + "})\n", + "\n", + "# Display first few rows\n", + "data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Data manipulation\n", + "# 1. Add a rolling average column\n", + "data['RollingAvg'] = data['Value'].rolling(window=7).mean()\n", + "\n", + "# 2. Group by Category and calculate mean value\n", + "category_means = data.groupby('Category')['Value'].mean()\n", + "category_means" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Visualization\n", + "plt.figure(figsize=(12, 6))\n", + "\n", + "# Plot the original Value and Rolling Average for each category\n", + "for cat in data['Category'].unique():\n", + " subset = data[data['Category'] == cat]\n", + " plt.plot(subset['Date'], subset['Value'], label=f'{cat} Value', alpha=0.3)\n", + " plt.plot(subset['Date'], subset['RollingAvg'], label=f'{cat} RollingAvg')\n", + "\n", + "plt.xlabel('Date')\n", + "plt.ylabel('Value')\n", + "plt.title('Value and Rolling Average by Category')\n", + "plt.legend()\n", + "plt.grid(True)\n", + "plt.tight_layout()\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/python_interface_to_workflows/workflow_definitions/mounted_files/requirements.txt b/src/python_interface_to_workflows/workflow_definitions/mounted_files/requirements.txt new file mode 100644 index 0000000..58a5482 --- /dev/null +++ b/src/python_interface_to_workflows/workflow_definitions/mounted_files/requirements.txt @@ -0,0 +1,8 @@ +numpy==2.2.5 +matplotlib==3.10.3 +pandas==2.2.3 +scipy==1.15.3 +nbconvert==7.17.1 +ipykernel==6.29.5 +ipython==9.2.0 +papermill==2.6.0 diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb deleted file mode 100644 index 9113672..0000000 --- a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb +++ /dev/null @@ -1,116 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "1e3146f1", - "metadata": {}, - "source": [ - "# Running in VSCode:\n", - "\n", - "1. Set kernal to python-interface-to-workflows 3.11.x\n", - "2. Hit F1, run Jupyter: Import Notebook to Script\n", - "3. Click notebook_division.ipynb\n", - "4. Run the cells sequentially" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "494174ef", - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "\n", - "from hera.workflows import (\n", - " Artifact,\n", - " EmptyDirVolume,\n", - " Steps,\n", - " Workflow,\n", - " script, # pyright: ignore[reportUnknownVariableType]\n", - ")\n", - "from hera.workflows import models as m\n", - "\n", - "\n", - "@script(\n", - " volume_mounts=[m.VolumeMount(name=\"output-dir\", mount_path=\"/output-dir/\")],\n", - " outputs=Artifact(name=\"json-output\", path=\"/output-dir/output.json\"),\n", - ")\n", - "def do_division(a: int, b: int):\n", - " div = a / b\n", - " intdiv = a // b\n", - " remain = a % b\n", - " dictionary_of_results = {\n", - " \"divide\": div,\n", - " \"quotient\": intdiv,\n", - " \"remainder\": remain,\n", - " }\n", - " with open(\"/output-dir/output.json\", \"w\") as otpt:\n", - " json.dump(dictionary_of_results, otpt)\n", - "\n", - "\n", - "with Workflow(\n", - " generate_name=\"hera-division-\", # when running on graphql this should be name\n", - " entrypoint=\"divide\",\n", - " api_version=\"argoproj.io/v1alpha1\",\n", - " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", - " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", - " annotations={\n", - " \"workflows.argoproj.io/title\": \"Division via hera test\",\n", - " \"workflows.argoproj.io/description\": \"\"\"Takes a numerical input and returns\n", - " the remainder, output float, and output string to a json file\"\"\",\n", - " \"workflows.diamond.ac.uk/repository\": \"https://github.com/DiamondLightSource/python-interface-to-workflows\",\n", - " },\n", - " volumes=EmptyDirVolume(name=\"output-dir\", mount_path=\"/output-dir\"),\n", - ") as w:\n", - " with Steps(name=\"divide\"):\n", - " do_division(name=\"first\", arguments={\"a\": 2, \"b\": 5})\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "04708f46", - "metadata": {}, - "outputs": [], - "source": [ - "with open(\"../../templates/division_from_jupyter.txt\", \"w\") as div:\n", - " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7e9f88cb", - "metadata": {}, - "outputs": [], - "source": [ - "from python_interface_to_workflows.submit_workflow import submit_workflow\n", - "\n", - "submit_workflow(w)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "python-interface-to-workflows (3.11.x)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.-1" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb deleted file mode 100644 index c1018ea..0000000 --- a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb +++ /dev/null @@ -1,251 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "1e3146f1", - "metadata": {}, - "source": [ - "# Running in VSCode:\n", - "\n", - "1. Set kernal to python-interface-to-workflows 3.11.x\n", - "2. Hit F1, run Jupyter: Import Notebook to Script\n", - "3. Click notebook_example.ipynb\n", - "4. Run the cells sequentially" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "494174ef", - "metadata": {}, - "outputs": [], - "source": [ - "from hera.workflows import (\n", - " DAG,\n", - " Artifact,\n", - " Parameter,\n", - " Volume,\n", - " Workflow,\n", - " script, # pyright: ignore[reportUnknownVariableType]\n", - ")\n", - "from hera.workflows import models as m\n", - "from hera.workflows.archive import NoneArchiveStrategy\n", - "\n", - "\n", - "@script(\n", - " command=[\"python\"],\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - ")\n", - "def install_dependencies():\n", - " import subprocess\n", - "\n", - " print(\"creating venv\")\n", - "\n", - " subprocess.check_call([\"python\", \"-m\", \"venv\", \"/tmp/venv\"])\n", - " subprocess.check_call(\n", - " [\"/tmp/venv/bin/pip\", \"install\", \"pillow\", \"h5py\", \"numpy\", \"hera\"]\n", - " )\n", - "\n", - "\n", - "@script(\n", - " command=[\"python\"],\n", - " outputs=Parameter(\n", - " name=\"out-parameters\", value_from=m.ValueFrom(path=\"/tmp/parameters.json\")\n", - " ),\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - ")\n", - "def generate_parameters(\n", - " png: str,\n", - " jpg: str,\n", - " jpeg: str,\n", - " tif: str,\n", - " tiff: str,\n", - "):\n", - " import json\n", - "\n", - " params: list[dict[str, int | list[int] | str] | None] = [\n", - " {\"width\": 500, \"height\": 500, \"weights\": [255, 1, 100], \"extension\": \"png\"}\n", - " if png.lower() == \"true\"\n", - " else None,\n", - " {\"width\": 600, \"height\": 200, \"weights\": [100, 150, 100], \"extension\": \"jpg\"}\n", - " if jpg.lower() == \"true\"\n", - " else None,\n", - " {\"width\": 300, \"height\": 400, \"weights\": [100, 150, 100], \"extension\": \"jpeg\"}\n", - " if jpeg.lower() == \"true\"\n", - " else None,\n", - " {\"width\": 300, \"height\": 200, \"weights\": [230, 100, 1], \"extension\": \"tif\"}\n", - " if tif.lower() == \"true\"\n", - " else None,\n", - " {\"width\": 200, \"height\": 300, \"weights\": [230, 100, 1], \"extension\": \"tiff\"}\n", - " if tiff.lower() == \"true\"\n", - " else None,\n", - " ]\n", - " params_to_write: list[dict[str, int | list[int] | str]] = [\n", - " image_params for image_params in params if image_params is not None\n", - " ]\n", - " with open(\"/tmp/parameters.json\", \"w\") as f:\n", - " json.dump(params_to_write, f)\n", - "\n", - "\n", - "@script(\n", - " command=[\"/tmp/venv/bin/python\"],\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - " outputs=[\n", - " Parameter(\n", - " name=\"out-paths\",\n", - " value_from=m.ValueFrom(\n", - " path=\"/tmp/{{inputs.parameters.extension}}-path.json\"\n", - " ),\n", - " ),\n", - " Artifact(\n", - " name=\"{{inputs.parameters.extension}}-image\",\n", - " path=\"/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}\",\n", - " archive=NoneArchiveStrategy(),\n", - " ),\n", - " ],\n", - ")\n", - "def create_image(\n", - " width: int, height: int, weights: tuple[int, int, int], extension: str\n", - "):\n", - " import json\n", - "\n", - " from PIL import Image\n", - "\n", - " def create_pattern(\n", - " width: int,\n", - " height: int,\n", - " weights: tuple[int, int, int],\n", - " ) -> Image.Image:\n", - " print(f\"width: {width}\")\n", - " print(f\"height: {height}\")\n", - " print(f\"RBG weights: {weights}\")\n", - " image = Image.new(\"RGB\", (width, height))\n", - " pixels = image.load()\n", - " for i in range(width):\n", - " for j in range(height):\n", - " pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript]\n", - " (i + j * 50) % weights[0],\n", - " weights[1],\n", - " (i * 300 + j) % weights[2],\n", - " )\n", - " return image\n", - "\n", - " image = create_pattern(width, height, weights)\n", - " path = f\"/tmp/{extension}-image.{extension}\"\n", - " image.save(path)\n", - " with open(f\"/tmp/{extension}-path.json\", \"w\") as f:\n", - " json.dump(path, f)\n", - "\n", - "\n", - "@script(\n", - " command=[\"/tmp/venv/bin/python\"],\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - " outputs=Artifact(\n", - " name=\"hdf5output\",\n", - " path=\"/tmp/images.hdf5\",\n", - " archive=NoneArchiveStrategy(),\n", - " ),\n", - ")\n", - "def to_hdf5(paths: str):\n", - "\n", - " import h5py # pyright: ignore[reportMissingTypeStubs]\n", - " import numpy as np\n", - " from PIL import Image\n", - "\n", - " print(\"creating hdf5 file\")\n", - " with h5py.File(\"/tmp/images.hdf5\", \"w\") as f:\n", - " for i, path in enumerate(paths):\n", - " path = path.strip('\"')\n", - " print(f\"Got {path}\")\n", - " with Image.open(path) as image:\n", - " arr = np.array(image)\n", - " f.create_dataset( # pyright: ignore[reportUnknownMemberType]\n", - " f\"image_{i}\", data=arr, dtype=arr.dtype\n", - " )\n", - " print(\"done\")\n", - "\n", - "\n", - "with Workflow(\n", - " generate_name=\"hera-example-\", # when running on graphql this should be name\n", - " entrypoint=\"workflowentry\",\n", - " api_version=\"argoproj.io/v1alpha1\",\n", - " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", - " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", - " annotations={\n", - " \"workflows.argoproj.io/title\": \"example remade via hera\",\n", - " \"workflows.argoproj.io/description\": \"\"\"Replicates the functionality of\n", - "example.yaml\"\"\",\n", - " \"workflows.diamond.ac.uk/repository\": \"https://github.com/DiamondLightSource/python-interface-to-workflows\",\n", - " },\n", - " volumes=Volume(name=\"tmpdir\", mount_path=\"/tmp/\", size=\"1Gi\"),\n", - ") as w:\n", - " with DAG(name=\"workflowentry\"):\n", - " install = install_dependencies(name=\"install\")\n", - " params = generate_parameters(\n", - " name=\"params\",\n", - " arguments={\n", - " \"png\": \"True\",\n", - " \"jpg\": \"True\",\n", - " \"jpeg\": \"True\",\n", - " \"tif\": \"True\",\n", - " \"tiff\": \"True\",\n", - " },\n", - " )\n", - " makeimages = create_image(with_param=params.get_parameter(\"out-parameters\"))\n", - " makehdf5 = to_hdf5(\n", - " arguments={\n", - " \"paths\": makeimages.get_parameter(\"out-paths\"),\n", - " }\n", - " )\n", - " [install, params] >> makeimages >> makehdf5 # pyright: ignore\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "04708f46", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "with open(\"../../templates/example_from_jupyter.txt\", \"w\") as div:\n", - " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7e9f88cb", - "metadata": {}, - "outputs": [], - "source": [ - "from python_interface_to_workflows.submit_workflow import submit_workflow\n", - "\n", - "submit_workflow(w)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "python-interface-to-workflows (3.11.x)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.15" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/tests/test_keycloak_checker.py b/tests/test_keycloak_checker.py deleted file mode 100644 index c658d0c..0000000 --- a/tests/test_keycloak_checker.py +++ /dev/null @@ -1,124 +0,0 @@ -import os -from unittest.mock import MagicMock, call, patch - -from pytest import mark - -from python_interface_to_workflows.auth.keycloak_checker import set_token_env_variable - - -@mark.parametrize( - "staging,port,return_present", - [ - (True, 5173, True), - (False, 8000, False), - (True, 5173, False), - (False, 8000, True), - ], -) -@patch("python_interface_to_workflows.auth.keycloak_checker.dotenv.load_dotenv") -@patch("python_interface_to_workflows.auth.keycloak_checker.dotenv.set_key") -@patch("python_interface_to_workflows.auth.keycloak_checker.KeycloakOpenID") -@patch("python_interface_to_workflows.auth.keycloak_checker.generate_code_verifier") -@patch("python_interface_to_workflows.auth.keycloak_checker.generate_code_challenge") -@patch("python_interface_to_workflows.auth.keycloak_checker.token_expired") -def test_set_token_env_variable( - mock_token_expired: MagicMock, - mock_gen_code_challenge: MagicMock, - mock_gen_code_verifier: MagicMock, - mock_gen_keycloak_id: MagicMock, - mock_set_key: MagicMock, - mock_load_env: MagicMock, - staging: bool, - port: int, - return_present: bool, -): - mock_gen_code_verifier.return_value = "verifier" - mock_gen_code_challenge.return_value = ("challenge", "S256") - os.environ["AUTH"] = "auth_url_code" - os.environ["REFRESHTOKEN"] = "refresh" - - keycloak = MagicMock() - mock_gen_keycloak_id.return_value = keycloak - mock_token_expired.return_value = return_present - keycloak.auth_url.return_value = "https://mock.site" - token = { - "access_token": "fake_token", - "refresh_token": "fake_refresh", - } - keycloak.token.return_value = token - keycloak.refresh_token.return_value = token - keycloak.decode_token.return_value = {"exp": 123456789} - assert set_token_env_variable(staging) == "fake_token" - - mock_token_expired.assert_called_once_with("https://mock.site", port) - if return_present: - keycloak.token.assert_called_once_with( - grant_type="authorization_code", - code="auth_url_code", - redirect_uri=f"http://localhost:{port}/", - code_verifier="verifier", - ) - keycloak.refresh_token.assert_not_called() - else: - keycloak.refresh_token.assert_called_once_with("refresh") - keycloak.token.assert_not_called() - mock_set_key.assert_has_calls( - [ - call("src/.env", "EXPIRY", str(123456789 + 1800)), - call("src/.env", "TOKEN", "fake_token"), - call("src/.env", "REFRESHTOKEN", "fake_refresh"), - ] - ) - mock_load_env.assert_called_once_with( - dotenv_path="src/.env", - override=True, - ) - - -@patch("python_interface_to_workflows.auth.keycloak_checker.exit") -@patch("python_interface_to_workflows.auth.keycloak_checker.print") -@patch("python_interface_to_workflows.auth.keycloak_checker.dotenv.load_dotenv") -@patch("python_interface_to_workflows.auth.keycloak_checker.dotenv.set_key") -@patch("python_interface_to_workflows.auth.keycloak_checker.KeycloakOpenID") -@patch("python_interface_to_workflows.auth.keycloak_checker.generate_code_verifier") -@patch("python_interface_to_workflows.auth.keycloak_checker.generate_code_challenge") -@patch("python_interface_to_workflows.auth.keycloak_checker.token_expired") -def test_set_token_env_variable_attribute_error( - mock_token_expired: MagicMock, - mock_gen_code_challenge: MagicMock, - mock_gen_code_verifier: MagicMock, - mock_gen_keycloak_id: MagicMock, - mock_set_key: MagicMock, - mock_load_env: MagicMock, - mock_print: MagicMock, - mock_exit: MagicMock, -): - mock_gen_code_verifier.return_value = "verifier" - mock_gen_code_challenge.return_value = ("challenge", "S256") - mock_token_expired.return_value = True - - os.environ["REFRESHTOKEN"] = "refresh" - - keycloak = MagicMock() - mock_gen_keycloak_id.return_value = keycloak - - token = { - "access_token": "fake_token", - "refresh_token": "fake_refresh", - } - - keycloak.refresh_token.return_value = token - - decoded = MagicMock() - decoded.__getitem__.side_effect = AttributeError - keycloak.decode_token.return_value = decoded - - set_token_env_variable(True) - - mock_print.assert_called_once_with("ERROR:") - mock_exit.assert_called_once_with(1) - mock_set_key.assert_not_called() - mock_load_env.assert_called_once_with( - dotenv_path="src/.env", - override=True, - ) diff --git a/tests/test_open_auth_url.py b/tests/test_open_auth_url.py deleted file mode 100644 index 0acd322..0000000 --- a/tests/test_open_auth_url.py +++ /dev/null @@ -1,142 +0,0 @@ -import os -from unittest.mock import MagicMock, patch - -from python_interface_to_workflows.auth.open_auth_url import ( - CallbackHandler, - _open_auth_url, # pyright:ignore - token_expired, -) - - -@patch("python_interface_to_workflows.auth.open_auth_url.dotenv.set_key") -@patch("python_interface_to_workflows.auth.open_auth_url.time.time") -@patch("python_interface_to_workflows.auth.open_auth_url.webbrowser.open") -@patch("python_interface_to_workflows.auth.open_auth_url._ReusingHTTPServer") -def test_open_auth_url_normal_function( - mock_http_server: MagicMock, - mock_open_browser: MagicMock, - mock_time: MagicMock, - mock_set_key: MagicMock, -): - mock_time.return_value = 100 - os.environ["EXPIRY"] = "" - - server = mock_http_server.return_value - server.auth_code = "this_is_your_code" - - _open_auth_url("url", 5173) - server.handle_request.assert_called_once() - mock_open_browser.assert_called_once_with("url") - mock_set_key.assert_called_once_with( - "src/.env", - "AUTH", - "this_is_your_code", - ) - server.socket.shutdown.assert_called_once() - server.server_close.assert_called_once() - assert os.environ["AUTH"] == "this_is_your_code" - - -@patch("python_interface_to_workflows.auth.open_auth_url.dotenv.set_key") -@patch("python_interface_to_workflows.auth.open_auth_url.time.time") -@patch("python_interface_to_workflows.auth.open_auth_url.exit") -@patch("python_interface_to_workflows.auth.open_auth_url.webbrowser.open") -@patch("python_interface_to_workflows.auth.open_auth_url._ReusingHTTPServer") -def test_open_auth_url_raises_error( - mock_http_server: MagicMock, - mock_open_browser: MagicMock, - mock_exit: MagicMock, - mock_time: MagicMock, - mock_set_key: MagicMock, -): - mock_time.return_value = 100 - os.environ["EXPIRY"] = "" - - server = mock_http_server.return_value - server.handle_request.side_effect = OSError - - _open_auth_url("url", 5173) - mock_open_browser.assert_called_once_with("url") - assert os.environ["AUTH"] == "" - mock_exit.assert_called_once_with(1) - mock_set_key.assert_not_called() - server.socket.shutdown.assert_called_once() - server.server_close.assert_called_once() - - -@patch("python_interface_to_workflows.auth.open_auth_url._open_auth_url") -@patch("python_interface_to_workflows.auth.open_auth_url.dotenv.load_dotenv") -@patch("python_interface_to_workflows.auth.open_auth_url.time.time") -def test_token_expired( - mock_time: MagicMock, - mock_load_env: MagicMock, - mock_open_auth_url: MagicMock, -): - mock_time.return_value = 100 - os.environ["EXPIRY"] = "50" - - assert token_expired("url", 5173) is True - - mock_load_env.assert_called_once_with( - dotenv_path="src/.env", - override=True, - ) - mock_open_auth_url.assert_called_once_with("url", 5173) - - -@patch("python_interface_to_workflows.auth.open_auth_url._open_auth_url") -@patch("python_interface_to_workflows.auth.open_auth_url.dotenv.load_dotenv") -@patch("python_interface_to_workflows.auth.open_auth_url.time.time") -def test_token_not_expired( - mock_time: MagicMock, - mock_load_env: MagicMock, - mock_open_auth_url: MagicMock, -): - mock_time.return_value = 100 - os.environ["EXPIRY"] = "200" - - assert token_expired("url", 5173) is False - - mock_load_env.assert_called_once_with( - dotenv_path="src/.env", - override=True, - ) - mock_open_auth_url.assert_not_called() - - -def test_handler_normal_function(): - handler = CallbackHandler.__new__(CallbackHandler) - handler.path = "/?code=this_is_your_code" - - handler.server = MagicMock() - - handler.send_response = MagicMock() - handler.end_headers = MagicMock() - handler.wfile = MagicMock() - handler.wfile.write = MagicMock() - - handler.do_GET() - - assert handler.server.auth_code == "this_is_your_code" - handler.send_response.assert_called_once_with(200) - handler.wfile.write.assert_called_once_with( - b"Authorization successful. You can close this window." - ) - - -def test_handler_error_response(): - handler = CallbackHandler.__new__(CallbackHandler) - handler.path = "/" - - handler.server = MagicMock() - - handler.send_response = MagicMock() - handler.end_headers = MagicMock() - handler.wfile = MagicMock() - handler.wfile.write = MagicMock() - - handler.do_GET() - - handler.send_response.assert_called_once_with(400) - handler.end_headers.assert_called_once() - handler.wfile.write.assert_called_once_with(b"Missing authorization code.") diff --git a/tests/test_submit_to_graphql.py b/tests/test_submit_to_graphql.py deleted file mode 100644 index 1eb8020..0000000 --- a/tests/test_submit_to_graphql.py +++ /dev/null @@ -1,27 +0,0 @@ -from unittest.mock import MagicMock, call, patch - -from python_interface_to_workflows.submit_workflow import submit_workflow - - -@patch("python_interface_to_workflows.submit_workflow.os.environ.get") -@patch("python_interface_to_workflows.submit_workflow.dotenv.load_dotenv") -@patch("python_interface_to_workflows.submit_workflow.Workflow") -@patch("python_interface_to_workflows.submit_workflow.set_token_env_variable") -@patch("python_interface_to_workflows.submit_workflow.Client") -def test_submit_workflow_to_graphql( - mock_client: MagicMock, - mock_key: MagicMock, - mock_workflow: MagicMock, - mock_load_env: MagicMock, - mock_os_get: MagicMock, -): - - mock_instance = MagicMock() - mock_key.return_value = "token" - mock_client.return_value = mock_instance - mock_instance.execute.return_value = {"submitWorkflow": {"name": "workflow123"}} - submit_workflow(mock_workflow) - mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) - mock_instance.execute.assert_called_once() - mock_workflow.to_yaml.assert_called_once() - mock_os_get.assert_has_calls([call("VISIT"), call("HOST")], any_order=True) diff --git a/uv.lock b/uv.lock index a65926b..8bdf0b2 100644 --- a/uv.lock +++ b/uv.lock @@ -177,6 +177,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, ] +[[package]] +name = "asyncio" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/71/ea/26c489a11f7ca862d5705db67683a7361ce11c23a7b98fc6c2deaeccede2/asyncio-4.0.0.tar.gz", hash = "sha256:570cd9e50db83bc1629152d4d0b7558d6451bb1bfd5dfc2e935d96fc2f40329b", size = 5371, upload-time = "2025-08-05T02:51:46.605Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/64/eff2564783bd650ca25e15938d1c5b459cda997574a510f7de69688cb0b4/asyncio-4.0.0-py3-none-any.whl", hash = "sha256:c1eddb0659231837046809e68103969b2bef8b0400d59cfa6363f6b5ed8cc88b", size = 5555, upload-time = "2025-08-05T02:51:45.767Z" }, +] + [[package]] name = "attrs" version = "26.1.0" @@ -1767,6 +1776,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] +[[package]] +name = "pytest-asyncio" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, +] + [[package]] name = "pytest-cov" version = "7.1.0" @@ -1814,7 +1836,9 @@ dependencies = [ { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "pillow" }, + { name = "pytest-asyncio" }, { name = "python-keycloak" }, + { name = "python-workflow-submitter" }, { name = "pyyaml" }, { name = "requests" }, ] @@ -1830,8 +1854,10 @@ dev = [ { name = "pre-commit" }, { name = "pyright" }, { name = "pytest" }, + { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "python-keycloak" }, + { name = "python-workflow-submitter" }, { name = "pyyaml" }, { name = "ruff" }, { name = "tox-uv" }, @@ -1847,7 +1873,9 @@ requires-dist = [ { name = "hera" }, { name = "numpy" }, { name = "pillow" }, + { name = "pytest-asyncio" }, { name = "python-keycloak" }, + { name = "python-workflow-submitter" }, { name = "pyyaml" }, { name = "requests" }, ] @@ -1862,8 +1890,10 @@ dev = [ { name = "pre-commit" }, { name = "pyright" }, { name = "pytest" }, + { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "python-keycloak" }, + { name = "python-workflow-submitter" }, { name = "pyyaml" }, { name = "ruff" }, { name = "tox-uv" }, @@ -1887,6 +1917,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/9f/569a8bbdb0859498d33d8b86273bc82849bd0b445ac01416ad34996ca3d8/python_keycloak-7.1.1-py3-none-any.whl", hash = "sha256:d8295bec6c4805ab7335b03bc92753c8c6258d5511b080cac061da20ae77f61c", size = 87607, upload-time = "2026-02-15T08:45:57.054Z" }, ] +[[package]] +name = "python-workflow-submitter" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "asyncio" }, + { name = "dotenv" }, + { name = "gql" }, + { name = "h5py" }, + { name = "hera" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "pillow" }, + { name = "pytest-asyncio" }, + { name = "python-keycloak" }, + { name = "pyyaml" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/0b/aea3418dd83b4eb607571c88188a19cd2ff5fc2d64cd3c4b87314189e342/python_workflow_submitter-0.4.3.tar.gz", hash = "sha256:3cabcbe3be68feab94a5887ef9b31c8ca04a3b203c9dd515282da71331a05279", size = 180804, upload-time = "2026-08-06T12:20:03.847Z" } + [[package]] name = "pyyaml" version = "6.0.3" @@ -2135,14 +2186,14 @@ wheels = [ [[package]] name = "typing-inspection" -version = "0.4.2" +version = "0.4.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/bc/4eae18cd40c65798a16267572ba346c11f599d44b01603dbd843342042bc/typing_inspection-0.4.3.tar.gz", hash = "sha256:c5f9ec1530b5c1e2c9bc34a84d9a3466ed1b2f3f2fa9f901368d9c5596210e4d", size = 76711, upload-time = "2026-08-10T09:39:18.063Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, + { url = "https://files.pythonhosted.org/packages/42/f7/7a3935abdebd5cf18705a5f0335dd6a3a18bef3baa7cb9edc3b6b9922cc8/typing_inspection-0.4.3-py3-none-any.whl", hash = "sha256:5f42b23858a91e0b4ef521f5418f03a0da3c9216fd2995ef5e73463100e676cd", size = 14693, upload-time = "2026-08-10T09:39:16.693Z" }, ] [[package]]