Cleans up files for final release

.github/workflows/_update_image.yml

@@ -0,0 +1,51 @@
+name: Update Docker Image
+
+on:
+  workflow_call:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v6
+
+      - name: Generate Image Name
+        run: echo IMAGE_REPOSITORY=ghcr.io/$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]' | tr '[_]' '[\-]')-mounted-in-image >> $GITHUB_ENV
+
+      - name: Log in to GitHub Docker Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v4.1.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker Metadata
+        id: meta
+        uses: docker/metadata-action@v6.1.0
+        with:
+          images: ${{ env.IMAGE_REPOSITORY }}
+          tags: |
+            type=ref,event=branch
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=match,pattern=python-interface-to-workflows@v?(.+),group=1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4.0.0
+        with:
+          driver-opts: network=host
+
+      - name: Build Image
+        uses: docker/build-push-action@v6.18.0
+        with:
+          context: .
+          push: ${{ github.event_name == 'push' }}
+          load: false
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/ci.yml

@@ -17,6 +17,9 @@ jobs:
   commit-lint:
     uses: ./.github/workflows/_commit_msg.yml
 
+  updateimage:
+    uses: ./.github/workflows/_update_image.yml
+
   test:
     strategy:
       matrix:

.vscode/settings.json

@@ -9,5 +9,8 @@
     "[python]": {
         "editor.defaultFormatter": "charliermarsh.ruff",
     },
-    "python.envFile": "${workspaceFolder}/workspaces/python-interface-to-workflows/src/.env"
+    "python.envFile": "${workspaceFolder}/workspaces/python-interface-to-workflows/src/.env",
+    "python.testing.pytestArgs": [
+        "tests"
+    ]
 }

Dockerfile

@@ -45,6 +45,7 @@ COPY --from=build /python /python
 
 # Copy the environment, but not the source code
 COPY --from=build /app/.venv /app/.venv
+COPY --from=build /app/src/python_interface_to_workflows/workflow_definitions/mounted_files /mounted_files/
 ENV PATH=/app/.venv/bin:$PATH
 
 # change this entrypoint if it is not the same as the repo

README.md

@@ -7,30 +7,14 @@
 
  Python alternative to creating and running argo workflows in the Data Analysis Platform
 
-This is where you should write a short paragraph that describes what your module does,
-how it does it, and why people should use it.
+This provides a copier template and demonstrating how to use Hera to rewrite workflows using python.
 
 What            | Where
 :---:           | :---:
 Source          | <https://github.com/DiamondLightSource/python-interface-to-workflows>
 Docker          | `docker run ghcr.io/diamondlightsource/python-interface-to-workflows:latest`
 Releases        | <https://github.com/DiamondLightSource/python-interface-to-workflows/releases>
 
-This is where you should put some images or code snippets that illustrate
-some relevant examples. If it is a library then you might put some
-introductory code here:
-
-```python
-from python_interface_to_workflows import __version__
-
-print(f"Hello python_interface_to_workflows {__version__}")
-```
-
-Or if it is a commandline tool then you might put some example commands here:
-
-```
-python -m python_interface_to_workflows --version
-```
 
 # Using Copier
 ```bash
@@ -49,6 +33,29 @@ copier copy {this_repo's_path} {new_directory_path}
 ```
 or:
 ```bash
-copier copy git@github.com:DiamondLightSource/python-copier-template.git new directory path
+copier copy git@github.com:DiamondLightSource/python-copier-template.git .
+code .
+```
+
+Then:
+1) create a .env file in src
+2) run uv lock
+3) rebuild and reopen in container
+
+To submit your yaml files in a notebook, append the following:
+
+```python
+from python_workflow_submitter.submit_workflow import submit_workflow_yaml
+
+await submit_workflow_yaml("example.yaml")
+```
+
+Alternatively:
+```python
+import asyncio
+import os
+
+from python_workflow_submitter.submit_workflow import submit_workflow_yaml
+
+asyncio.run(submit_workflow_yaml("example.yaml", visit=os.environ.get("VISIT")))
 ```
-rebuild in dev container without cache

pyproject.toml

@@ -24,6 +24,8 @@ dependencies = [
     "h5py",
     "dotenv",
     "python-keycloak",
+    "pytest-asyncio",
+    "python-workflow-submitter",
 ] # Add project dependencies here, e.g. ["click", "numpy"]
 dynamic = ["version"]
 license.file = "LICENSE"
@@ -46,6 +48,8 @@ dev = [
     "h5py",
     "dotenv",
     "python-keycloak",
+    "pytest-asyncio",
+    "python-workflow-submitter",
 ]
 
 [project.scripts]

scripts/checkyamlcompliance.py

@@ -24,12 +24,6 @@
         api_ver = False
     if "kind" in yamldata.keys():
         match yamldata["kind"]:
-            case "ClusterWorkflowTemplate":
-                clst_tmpt = True
-            case "Workflow":
-                clst_tmpt = True
-            case "WorkflowTemplate":
-                clst_tmpt = True
             case "ClusterWorkflowTemplate":
                 clst_tmpt = True
             case _:
@@ -38,7 +32,6 @@
         clst_tmpt = False
     if "metadata" in yamldata.keys():
         metadata = True
-        gen_name = "generateName" in yamldata["metadata"].keys()
         normal_name = "name" in yamldata["metadata"].keys()
         if "annotations" in yamldata["metadata"].keys():
             annotations = True
@@ -64,19 +57,14 @@
         else:
             group = labels = False
     else:
-        metadata = normal_name = gen_name = False
-    if all([api_ver, clst_tmpt, title, repo, group, annotations, labels]):
-        if gen_name != normal_name:
-            exit(0)
-        else:
-            print(
-                "generated_name and normal_name error, must have one of these not both"
-            )
-            exit(1)
+        metadata = normal_name = False
+    if all([api_ver, clst_tmpt, title, repo, group, annotations, labels, normal_name]):
+        exit(0)
     else:
         print(f"""
                 within file: {file}...
                 metadata present?: {metadata}
+                name present?: {normal_name}
                 annotations present?: {annotations}
                 labels present?: {labels}
                 api_ver present?: {api_ver}

scripts/makecopiercorrect.sh

@@ -11,13 +11,15 @@ do
 
     sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file"
     sed -i 's/DiamondLightSource/{{github_org}}/g' "$file"
+    sed -i 's/python_interface_to_workflows/{{project_name}}/g' "$file"
 
     sed -i '1i{% raw %}' "$file"
     echo '{% endraw %}' >> "$file"
 
     sed -i \
         -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \
         -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \
+        -e 's/{{project_name}}/{% endraw %}{{project_name}}{% raw %}/g' \
         "$file"
 
     mv "$file" "$file.jinja"
@@ -30,12 +32,14 @@ do
     sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file"
     sed -i 's/DiamondLightSource/{{github_org}}/g' "$file"
 
+    sed -i 's/python_interface_to_workflows/{{project_name}}/g' "$file"
     sed -i '1i{% raw %}' "$file"
     echo '{% endraw %}' >> "$file"
 
     sed -i \
         -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \
         -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \
+        -e 's/{{project_name}}/{% endraw %}{{project_name}}{% raw %}/g' \
         "$file"
 
 
@@ -48,13 +52,14 @@ do
 
     sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file"
     sed -i 's/DiamondLightSource/{{github_org}}/g' "$file"
-
+    sed -i 's/python_interface_to_workflows/{{project_name}}/g' "$file"
     sed -i '1i{% raw %}' "$file"
     echo '{% endraw %}' >> "$file"
 
     sed -i \
         -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \
         -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \
+        -e 's/{{project_name}}/{% endraw %}{{project_name}}{% raw %}/g' \
         "$file"
 
     mv "$file" "$file.jinja"

scripts/runthefiles.sh

@@ -3,7 +3,13 @@ cd src/python_interface_to_workflows/workflow_definitions
 for file in *
 do
 [[ -d "$file" ]] && continue
+    if grep -Eq '^\s*(from|import)\s+python_workflow_submitter\b' "$file"; then
+        echo "ERROR: $file imports submitter"
+        exit 1
+    fi
 uv run "$file"
 done
-mv *.txt ../templates/
-git add -u ../templates/
+if compgen -G "*.yaml" > /dev/null || compgen -G "*.txt" > /dev/null; then
+    mv -- *.yaml *.txt ../templates/
+    git add ../templates/
+fi

src/copier_template/.github/workflows/_update_image.yml

@@ -0,0 +1,51 @@
+name: Update Docker Image
+
+on:
+  workflow_call:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v6
+
+      - name: Generate Image Name
+        run: echo IMAGE_REPOSITORY=ghcr.io/$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]' | tr '[_]' '[\-]')-mounted-in-image >> $GITHUB_ENV
+
+      - name: Log in to GitHub Docker Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v4.1.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker Metadata
+        id: meta
+        uses: docker/metadata-action@v6.1.0
+        with:
+          images: ${{ env.IMAGE_REPOSITORY }}
+          tags: |
+            type=ref,event=branch
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=match,pattern=python-interface-to-workflows@v?(.+),group=1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4.0.0
+        with:
+          driver-opts: network=host
+
+      - name: Build Image
+        uses: docker/build-push-action@v6.18.0
+        with:
+          context: .
+          push: ${{ github.event_name == 'push' }}
+          load: false
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

src/copier_template/Dockerfile.jinja

@@ -45,6 +45,7 @@ COPY --from=build /python /python
 
 # Copy the environment, but not the source code
 COPY --from=build /app/.venv /app/.venv
+COPY --from=build /app/src/{{project_name}}/workflow_definitions/mounted_files /mounted_files/
 ENV PATH=/app/.venv/bin:$PATH
 
 # change this entrypoint if it is not the same as the repo

src/copier_template/README.md.jinja

@@ -7,6 +7,11 @@
 
  Python alternative to creating and running argo workflows in the Data Analysis Platform
 
+Update pyproject.toml for the following use cases:
+ - Pyright linting
+ - Dependencies
+
+
 This is where you should write a short paragraph that describes what your module does,
 how it does it, and why people should use it.
 

src/copier_template/pyproject.toml.jinja

@@ -24,6 +24,8 @@ dependencies = [
     "h5py",
     "dotenv",
     "python-keycloak",
+    "pytest-asyncio",
+    "python-workflow-submitter",
 ] # Add project dependencies here, e.g. ["click", "numpy"]
 dynamic = ["version"]
 license.file = "LICENSE"
@@ -46,6 +48,8 @@ dev = [
     "h5py",
     "dotenv",
     "python-keycloak",
+    "pytest-asyncio",
+    "python-workflow-submitter",
 ]