diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index ef587cf0..113ae8ea 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -99,17 +99,17 @@ jobs: 'scripts/build-docs.sh', 'scripts/publish-agent-markdown.py', 'scripts/render-dev-notes.py', - 'scripts/stage-privacy-guard-docs.py', + 'scripts/stage-egress-gate-docs.py', 'tests/test_agent_markdown.py', 'tests/test_docs_404.py', 'tests/test_render_dev_notes.py', - 'tests/test_stage_privacy_guard_docs.py', + 'tests/test_stage_egress_gate_docs.py', 'zensical.toml', ]); const docsChanged = files.some( ({ filename }) => filename.startsWith('docs/') || filename.startsWith('overrides/') || - filename.startsWith('projects/privacy-guard/docs/') || + filename.startsWith('projects/egress-gate/docs/') || exactInputs.has(filename), ); operation = docsChanged ? 'deploy' : 'remove'; diff --git a/.github/workflows/docs-preview.yml b/.github/workflows/docs-preview.yml index 6f54a221..a90d3f84 100644 --- a/.github/workflows/docs-preview.yml +++ b/.github/workflows/docs-preview.yml @@ -47,17 +47,17 @@ jobs: 'scripts/build-docs.sh', 'scripts/publish-agent-markdown.py', 'scripts/render-dev-notes.py', - 'scripts/stage-privacy-guard-docs.py', + 'scripts/stage-egress-gate-docs.py', 'tests/test_agent_markdown.py', 'tests/test_docs_404.py', 'tests/test_render_dev_notes.py', - 'tests/test_stage_privacy_guard_docs.py', + 'tests/test_stage_egress_gate_docs.py', 'zensical.toml', ]); const docsChanged = files.some( ({ filename }) => filename.startsWith('docs/') || filename.startsWith('overrides/') || - filename.startsWith('projects/privacy-guard/docs/') || + filename.startsWith('projects/egress-gate/docs/') || exactInputs.has(filename), ); core.setOutput('operation', docsChanged ? 'deploy' : 'remove'); diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2f98ced6..047fbea3 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -50,6 +50,9 @@ jobs: - name: Check navigation JavaScript syntax run: node --check docs/javascripts/navigation-drawer.js + - name: Test navigation JavaScript behavior + run: node tests/navigation-drawer.test.js + - name: Build documentation run: scripts/build-docs.sh diff --git a/.github/workflows/privacy-guard.yml b/.github/workflows/egress-gate.yml similarity index 69% rename from .github/workflows/privacy-guard.yml rename to .github/workflows/egress-gate.yml index 07a9c3bb..b16f6c91 100644 --- a/.github/workflows/privacy-guard.yml +++ b/.github/workflows/egress-gate.yml @@ -1,4 +1,4 @@ -name: Privacy Guard +name: Egress Gate "on": pull_request: @@ -11,12 +11,12 @@ permissions: contents: read concurrency: - group: privacy-guard-${{ github.workflow }}-${{ github.ref }} + group: egress-gate-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: check: - name: Check Privacy Guard (Python ${{ matrix.python-version }}) + name: Check Egress Gate (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest strategy: fail-fast: false @@ -26,7 +26,7 @@ jobs: - "3.14" defaults: run: - working-directory: projects/privacy-guard + working-directory: projects/egress-gate steps: - name: Checkout uses: actions/checkout@v7 @@ -41,8 +41,8 @@ jobs: - name: Configure isolated uv paths run: | - echo "UV_CACHE_DIR=$RUNNER_TEMP/privacy-guard-uv-cache" >> "$GITHUB_ENV" - echo "UV_PROJECT_ENVIRONMENT=$RUNNER_TEMP/privacy-guard-venv" >> "$GITHUB_ENV" + echo "UV_CACHE_DIR=$RUNNER_TEMP/egress-gate-uv-cache" >> "$GITHUB_ENV" + echo "UV_PROJECT_ENVIRONMENT=$RUNNER_TEMP/egress-gate-venv" >> "$GITHUB_ENV" - name: Install locked dependencies run: uv sync --frozen diff --git a/.gitignore b/.gitignore index ee6c962b..8e2e0357 100644 --- a/.gitignore +++ b/.gitignore @@ -121,7 +121,7 @@ lib/ # Static site and docs output public/ site/ -docs/documentation/privacy-guard/ +docs/documentation/egress-gate/ .docusaurus/ .vitepress/cache/ .vitepress/dist/ diff --git a/docs/development/index.md b/docs/development/index.md index ceef7e54..c1380883 100644 --- a/docs/development/index.md +++ b/docs/development/index.md @@ -92,8 +92,8 @@ scripts/build-docs.sh ``` `scripts/build-docs.sh` recreates `.venv-docs`, installs the pinned toolchain, -stages canonical Privacy Guard documentation from -`projects/privacy-guard/docs/`, renders Dev Notes metadata, and runs +stages canonical Egress Gate documentation from +`projects/egress-gate/docs/`, renders Dev Notes metadata, and runs `zensical build --clean --strict`. Do not report success unless it completes without issues. diff --git a/docs/documentation/index.md b/docs/documentation/index.md index 8f2d1e54..e5ee2124 100644 --- a/docs/documentation/index.md +++ b/docs/documentation/index.md @@ -9,4 +9,5 @@ agent_markdown: true Technical documentation and references for installing, using, and extending OpenShell Research projects. -- [Privacy Guard](privacy-guard/index.md): middleware for protecting sensitive data in OpenShell. +- [Egress Gate](egress-gate/index.md): extensible middleware for applying gates + to outgoing HTTP requests. diff --git a/docs/javascripts/navigation-drawer.js b/docs/javascripts/navigation-drawer.js index 83ebaea0..1644d407 100644 --- a/docs/javascripts/navigation-drawer.js +++ b/docs/javascripts/navigation-drawer.js @@ -1,4 +1,6 @@ (() => { + const drawerStateKey = "openshell.navigationDrawerOpen"; + const modalDrawerQuery = "(max-width: 63.99rem)"; let cleanup = () => {}; function enhanceNavigationDrawer() { @@ -7,32 +9,19 @@ const toggle = document.querySelector("#__drawer"); const sidebar = document.querySelector(".md-sidebar--primary"); const overlay = document.querySelector('.md-overlay[for="__drawer"]'); - const legacyControl = document.querySelector( - '.md-header__button[for="__drawer"]', - ); - - if (!(toggle instanceof HTMLInputElement) || !(sidebar instanceof HTMLElement)) { - cleanup = () => {}; - return; - } - - let button = document.querySelector(".openshell-drawer-button"); - if (!(button instanceof HTMLButtonElement) && legacyControl instanceof HTMLElement) { - button = document.createElement("button"); - button.type = "button"; - button.className = legacyControl.className; - button.classList.add("openshell-drawer-button"); - button.innerHTML = legacyControl.innerHTML; - legacyControl.replaceWith(button); - } - - if (!(button instanceof HTMLButtonElement)) { + const modalDrawer = window.matchMedia(modalDrawerQuery); + const button = document.querySelector(".openshell-drawer-button"); + + if ( + !(toggle instanceof HTMLInputElement) || + !(sidebar instanceof HTMLElement) || + !(button instanceof HTMLElement) + ) { cleanup = () => {}; return; } sidebar.id = "primary-navigation"; - sidebar.setAttribute("role", "dialog"); sidebar.setAttribute("aria-label", "Primary navigation"); button.setAttribute("aria-controls", sidebar.id); @@ -61,26 +50,42 @@ const focusableElements = () => Array.from( sidebar.querySelectorAll( - 'a[href], button:not([disabled]), input:not([disabled]), [tabindex]:not([tabindex="-1"])', + 'a[href], button:not([disabled]), input:not([disabled]):not(.md-toggle), [tabindex]:not([tabindex="-1"])', ), - ).filter((element) => element instanceof HTMLElement && !element.hidden); + ).filter( + (element) => + element instanceof HTMLElement && + element.tabIndex >= 0 && + element.getClientRects().length > 0 && + window.getComputedStyle(element).visibility === "visible" && + !element.closest("[inert]"), + ); const synchronize = ({ moveFocus = false, restoreFocus = false } = {}) => { const isOpen = toggle.checked; + const isModal = modalDrawer.matches; + document.documentElement.dataset.navigationDrawer = isOpen ? "open" : "closed"; button.setAttribute("aria-expanded", String(isOpen)); button.setAttribute("aria-label", isOpen ? "Close navigation" : "Open navigation"); sidebar.setAttribute("aria-hidden", String(!isOpen)); - if (isOpen) { + if (isModal) { + sidebar.setAttribute("role", "dialog"); + sidebar.setAttribute("aria-label", "Primary navigation"); + } else { + sidebar.removeAttribute("role"); + sidebar.removeAttribute("aria-label"); + } + if (isOpen && isModal) { sidebar.setAttribute("aria-modal", "true"); } else { sidebar.removeAttribute("aria-modal"); } sidebar.inert = !isOpen; backgroundElements.forEach((wasInert, element) => { - element.inert = isOpen || wasInert; + element.inert = (isOpen && isModal) || wasInert; }); - if (isOpen && moveFocus) { + if (isOpen && isModal && moveFocus) { focusableElements()[0]?.focus(); } else if (!isOpen && restoreFocus) { returnFocus.focus(); @@ -92,27 +97,42 @@ returnFocus = button; } toggle.checked = isOpen; + writeDrawerState(isOpen); synchronize(options); }; - const onButtonClick = () => { + const onButtonClick = (event) => { + event.preventDefault(); setOpen(!toggle.checked, { moveFocus: !toggle.checked, restoreFocus: toggle.checked, }); }; - const onToggleChange = () => synchronize(); - const onOverlayClick = (event) => { - event.preventDefault(); - setOpen(false, { restoreFocus: true }); + const onToggleChange = () => { + writeDrawerState(toggle.checked); + synchronize(); }; const onSidebarClick = (event) => { - if (event.target instanceof Element && event.target.closest("a[href]")) { + const link = event.target instanceof Element && event.target.closest("a[href]"); + if (link && modalDrawer.matches) { setOpen(false); } }; + const onOverlayClick = (event) => { + event.preventDefault(); + setOpen(false, { restoreFocus: true }); + }; const onKeyDown = (event) => { + if ( + document.activeElement === button && + (event.key === " " || event.key === "Enter") + ) { + event.preventDefault(); + onButtonClick(event); + return; + } + if (!toggle.checked) return; if (event.key === "Escape") { @@ -121,7 +141,7 @@ return; } - if (event.key !== "Tab") return; + if (event.key !== "Tab" || !modalDrawer.matches) return; const focusable = focusableElements(); if (!focusable.length) { @@ -142,23 +162,54 @@ button.addEventListener("click", onButtonClick); toggle.addEventListener("change", onToggleChange); - overlay?.addEventListener("click", onOverlayClick); sidebar.addEventListener("click", onSidebarClick); + overlay?.addEventListener("click", onOverlayClick); + const onDrawerModeChange = () => { + const shouldMoveFocus = + toggle.checked && modalDrawer.matches && !sidebar.contains(document.activeElement); + synchronize({ moveFocus: shouldMoveFocus }); + }; + modalDrawer.addEventListener("change", onDrawerModeChange); document.addEventListener("keydown", onKeyDown); - synchronize(); + document.documentElement.classList.add("openshell-drawer-restoring"); + toggle.checked = readDrawerState(); + synchronize({ moveFocus: toggle.checked && modalDrawer.matches }); + window.requestAnimationFrame(() => { + window.requestAnimationFrame(() => { + document.documentElement.classList.remove("openshell-drawer-restoring"); + }); + }); cleanup = () => { button.removeEventListener("click", onButtonClick); toggle.removeEventListener("change", onToggleChange); - overlay?.removeEventListener("click", onOverlayClick); sidebar.removeEventListener("click", onSidebarClick); + overlay?.removeEventListener("click", onOverlayClick); + modalDrawer.removeEventListener("change", onDrawerModeChange); document.removeEventListener("keydown", onKeyDown); + document.documentElement.classList.remove("openshell-drawer-restoring"); backgroundElements.forEach((wasInert, element) => { element.inert = wasInert; }); }; } + const readDrawerState = () => { + try { + return window.sessionStorage.getItem(drawerStateKey) === "true"; + } catch { + return false; + } + }; + + const writeDrawerState = (isOpen) => { + try { + window.sessionStorage.setItem(drawerStateKey, String(isOpen)); + } catch { + // Keep the drawer usable when browser storage is unavailable. + } + }; + if (window.document$?.subscribe) { window.document$.subscribe(enhanceNavigationDrawer); } else if (document.readyState === "loading") { diff --git a/docs/stylesheets/dev-notes.css b/docs/stylesheets/dev-notes.css index b2e9e37a..ae1eea31 100644 --- a/docs/stylesheets/dev-notes.css +++ b/docs/stylesheets/dev-notes.css @@ -9,6 +9,10 @@ */ :root { + --openshell-sidebar-width: 15.25rem; + --openshell-header-height: 3.7rem; + --openshell-header-control-size: 2.4rem; + --openshell-header-icon-size: 1.2rem; --openshell-green: #76b900; --openshell-green-soft: #8dc63f; --openshell-accent: #3c626b; @@ -92,7 +96,7 @@ body { } .md-header__inner { - height: 3.7rem; + height: var(--openshell-header-height); } .md-header__inner > [for="__drawer"], @@ -100,15 +104,64 @@ body { display: inline-flex; align-items: center; justify-content: center; + width: var(--openshell-header-control-size); + height: var(--openshell-header-control-size); + margin: 0; + padding: 0.6rem; order: -2; } +.md-header__option .md-header__button, +.md-header__inner > [for="__search"], +.md-header__source .md-source { + box-sizing: border-box; + width: var(--openshell-header-control-size); + height: var(--openshell-header-control-size); + margin: 0; + padding: 0.6rem; +} + +.md-header__option .md-header__button svg, +.md-header__inner > [for="__search"] svg, +.openshell-drawer-button svg, +.md-header__source .md-source__icon, +.md-header__source .md-source__icon svg { + width: var(--openshell-header-icon-size); + height: var(--openshell-header-icon-size); +} + .openshell-drawer-button:focus-visible { border-radius: 0.2rem; outline: 2px solid var(--openshell-accent); outline-offset: 0.15rem; } +.openshell-drawer-button svg { + fill: none; + stroke: currentColor; + stroke-width: 1.7; + stroke-linecap: round; + stroke-linejoin: round; +} + +.openshell-drawer-icon-collapse, +:root[data-navigation-drawer="open"] .openshell-drawer-icon-expand, +#__drawer:checked ~ .md-header .openshell-drawer-icon-expand { + display: none; +} + +:root[data-navigation-drawer="open"] .openshell-drawer-icon-collapse, +#__drawer:checked ~ .md-header .openshell-drawer-icon-collapse { + display: inline; +} + +.openshell-drawer-restoring .openshell-drawer-button, +.openshell-drawer-restoring .md-main, +.openshell-drawer-restoring .md-sidebar--primary, +.openshell-drawer-restoring .md-footer { + transition: none !important; +} + .md-header__inner > .md-logo { order: -1; } @@ -145,6 +198,7 @@ body { .md-header__source .md-source { display: inline-flex; align-items: center; + justify-content: center; color: var(--openshell-ink); } @@ -158,13 +212,9 @@ body { display: inline-flex; align-items: center; justify-content: center; - width: 1.8rem; - height: 1.8rem; } .md-header__source .md-source__icon svg { - width: 1.1rem; - height: 1.1rem; margin: 0; } @@ -183,6 +233,7 @@ body { transition: opacity 180ms ease; } +:root[data-navigation-drawer="open"] .md-overlay, #__drawer:checked ~ .md-overlay { opacity: 1; pointer-events: auto; @@ -195,7 +246,7 @@ body { bottom: auto !important; left: 0.25rem !important; display: block; - width: 15.25rem; + width: var(--openshell-sidebar-width); height: calc(100vh - 1rem) !important; padding: 0; border: 1px solid var(--openshell-rule); @@ -207,12 +258,74 @@ body { transition: transform 200ms ease, visibility 0s linear 200ms !important; } +:root[data-navigation-drawer="open"] .md-sidebar--primary, #__drawer:checked ~ .md-container .md-sidebar--primary { transform: translateX(0) !important; visibility: visible; transition-delay: 0s !important; } +@media (min-width: 64rem) { + .md-header { + z-index: 6; + } + + .md-header__inner > .openshell-drawer-button { + position: fixed; + z-index: 7; + top: calc((var(--openshell-header-height) - var(--openshell-header-control-size)) / 2); + left: 0.85rem; + color: var(--openshell-ink); + background: transparent; + border: 0; + border-radius: 0.35rem; + transition: color 120ms ease, background-color 120ms ease; + } + + .md-header__inner > .openshell-drawer-button:hover { + color: var(--md-default-fg-color); + background: color-mix(in srgb, var(--md-default-fg-color) 8%, transparent); + } + + .md-overlay, + :root[data-navigation-drawer="open"] .md-overlay, + #__drawer:checked ~ .md-overlay { + opacity: 0; + pointer-events: none; + } + + .md-main { + transition: padding-left 200ms ease; + } + + :root[data-navigation-drawer="open"] .md-main { + padding-left: var(--openshell-sidebar-width); + } + + :root[data-navigation-drawer="open"] .md-footer { + padding-left: var(--openshell-sidebar-width); + } + + .md-sidebar--primary { + top: var(--openshell-header-height) !important; + bottom: 0 !important; + left: 0 !important; + height: calc(100vh - var(--openshell-header-height)) !important; + border-width: 0 1px 0 0; + border-radius: 0; + box-shadow: none; + transform: translateX(calc(-100% - 1px)) !important; + visibility: hidden; + transition: transform 200ms ease, visibility 0s linear 200ms !important; + } + + :root[data-navigation-drawer="open"] .md-sidebar--primary { + transform: translateX(0) !important; + visibility: visible; + transition-delay: 0s !important; + } +} + .md-sidebar--primary .md-sidebar__scrollwrap { height: 100%; margin: 0; @@ -277,9 +390,7 @@ body { } .md-main { - background: - linear-gradient(90deg, transparent 0, transparent calc(50% - 36rem), color-mix(in srgb, var(--openshell-rule) 20%, transparent) calc(50% - 36rem), transparent calc(50% - 35.95rem)), - var(--openshell-paper); + background: var(--openshell-paper); } .md-main__inner { @@ -326,8 +437,108 @@ body { } .md-footer { + box-sizing: border-box; border-top: 0; background: var(--openshell-paper); + transition: padding-left 200ms ease; +} + +.md-footer__inner { + gap: 2rem; + width: min(calc(100% - 3rem), 60rem); + max-width: none; + margin: 3.5rem auto 0; + padding: 1.25rem 0 2.25rem; + border-top: 1px solid var(--openshell-rule); +} + +.md-footer__link { + align-items: center; + flex: 1 1 0; + gap: 0.75rem; + min-width: 0; + max-width: calc(50% - 1rem); + margin: 0; + padding: 0.65rem 0; + color: var(--openshell-muted); + opacity: 1; +} + +.md-footer__link:hover, +.md-footer__link:focus-visible { + color: var(--openshell-ink); + opacity: 1; +} + +.md-footer__link--next { + margin-left: auto; + text-align: right; +} + +.md-footer__button { + flex: 0 0 auto; + width: 1.2rem; + height: 1.2rem; + margin: 0; +} + +.md-footer__button svg { + width: 1rem; + height: 1rem; +} + +.md-footer__title { + flex: 1 1 auto; + min-width: 0; + max-width: none; + padding: 0; + font-family: var(--openshell-serif); + font-size: 1rem; + line-height: 1.3; + white-space: normal; +} + +.md-footer__title .md-ellipsis { + display: -webkit-box; + overflow: hidden; + white-space: normal; + text-overflow: clip; + -webkit-box-orient: vertical; + -webkit-line-clamp: 2; +} + +.md-footer__direction { + display: block; + margin-bottom: 0.2rem; + color: var(--openshell-muted); + font-family: var(--openshell-mono); + font-size: 0.58rem; + font-weight: 600; + letter-spacing: 0.1em; + line-height: 1.4; + opacity: 1; + text-transform: uppercase; +} + +.md-footer__link:hover .md-footer__direction, +.md-footer__link:focus-visible .md-footer__direction { + color: var(--openshell-accent); +} + +@media screen and (max-width: 44.984375em) { + .md-footer__inner { + flex-direction: column; + gap: 0.25rem; + } + + .md-footer__link { + width: 100%; + max-width: none; + } + + .md-footer__link--prev .md-footer__title { + display: block; + } } .md-footer-meta { @@ -380,8 +591,12 @@ body { /* Landing page ------------------------------------------------------------ */ -body:has(.dev-notes-page) .md-grid, -body:has(.openshell-home-page) .md-grid { +body:has(.dev-notes-page) .md-main__inner.md-grid, +body:has(.dev-notes-page) .md-footer__inner.md-grid, +body:has(.dev-notes-page) .md-footer-meta__inner.md-grid, +body:has(.openshell-home-page) .md-main__inner.md-grid, +body:has(.openshell-home-page) .md-footer__inner.md-grid, +body:has(.openshell-home-page) .md-footer-meta__inner.md-grid { max-width: 66rem; } @@ -414,7 +629,7 @@ body:has(.openshell-home-page) .md-path { } .research-masthead { - min-height: min(30rem, calc(100vh - 3.7rem)); + min-height: min(30rem, calc(100vh - var(--openshell-header-height))); padding: clamp(3.5rem, 9vw, 7.5rem) 0 clamp(3.2rem, 7vw, 5.8rem); border-bottom: 1px solid var(--openshell-rule-strong); } @@ -1072,6 +1287,11 @@ body[data-md-color-scheme="slate"] .openshell-home-brand__dark { width: calc(100% - 2rem); } + .md-footer__inner { + width: calc(100% - 2rem); + margin-top: 2.75rem; + } + .research-masthead { padding-top: 2.5rem; } @@ -1163,6 +1383,18 @@ body[data-md-color-scheme="slate"] .openshell-home-brand__dark { html { scroll-behavior: auto; } + + .openshell-drawer-button, + .md-overlay, + .md-sidebar--primary, + .md-main, + .md-footer, + .dev-note-card__visual::before, + .dev-note-card__visual::after, + .dev-note-card__visual-image, + .dev-note-card__read::after { + transition: none !important; + } } /* Dev note diagrams (SVG figures) */ diff --git a/docs/stylesheets/documentation.css b/docs/stylesheets/documentation.css index 3348bccf..aa3da39e 100644 --- a/docs/stylesheets/documentation.css +++ b/docs/stylesheets/documentation.css @@ -1,21 +1,58 @@ /* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ /* SPDX-License-Identifier: Apache-2.0 */ -/* - * Keep documentation diagrams subordinate to the surrounding explanation. - * Small screens retain the full available width so labels remain legible. - */ +/* Documentation figures use one quiet visual treatment across light and dark pages. */ +.md-typeset .documentation-figure { + width: 100%; + margin: 2rem auto 2.25rem; + text-align: center; +} + +.md-typeset .documentation-figure img, .md-typeset img[src*="assets/diagrams/"] { display: block; - width: 88%; + width: 100%; max-width: 52rem; height: auto; - margin-right: auto; - margin-left: auto; + margin-inline: auto; + border: 1px solid var(--openshell-rule); + border-radius: 1rem; + background: var(--openshell-paper-raised); + box-shadow: 0 0.8rem 2.2rem rgba(20, 28, 27, 0.1); +} + +.md-typeset .documentation-figure--wide img { + max-width: 60rem; +} + +.md-typeset .documentation-figure--portrait img { + max-width: 40rem; +} + +.md-typeset .documentation-figure figcaption { + max-width: 42rem; + margin: 0.75rem auto 0; + color: var(--openshell-muted); + font-size: 0.78rem; + line-height: 1.5; + text-wrap: balance; +} + +.md-typeset .documentation-figure figcaption code { + font-size: 0.75rem; +} + +body[data-md-color-scheme="slate"] .md-typeset .documentation-figure img { + box-shadow: 0 0.9rem 2.4rem rgba(0, 0, 0, 0.28); } @media screen and (max-width: 48rem) { + .md-typeset .documentation-figure { + margin-block: 1.5rem 1.8rem; + } + + .md-typeset .documentation-figure img, .md-typeset img[src*="assets/diagrams/"] { - width: 100%; + border-radius: 0.7rem; } } diff --git a/overrides/main.html b/overrides/main.html index f922ff93..adeeb5fa 100644 --- a/overrides/main.html +++ b/overrides/main.html @@ -3,6 +3,16 @@ {% block extrahead %} {{ super() }} + {% if page.meta and page.meta.agent_markdown %} {% endif %} diff --git a/overrides/partials/footer.html b/overrides/partials/footer.html new file mode 100644 index 00000000..2d4d6e02 --- /dev/null +++ b/overrides/partials/footer.html @@ -0,0 +1,57 @@ +{% set area_landing = page.url == "dev-notes/" or page.url == "documentation/" %} + diff --git a/overrides/partials/header.html b/overrides/partials/header.html new file mode 100644 index 00000000..7d373ad0 --- /dev/null +++ b/overrides/partials/header.html @@ -0,0 +1,70 @@ +{% set class = "md-header" %} +{% if "navigation.tabs.sticky" in features %} + {% set class = class ~ " md-header--shadow md-header--lifted" %} +{% elif "navigation.tabs" not in features %} + {% set class = class ~ " md-header--shadow" %} +{% endif %} +
+ + {% if "navigation.tabs.sticky" in features %} + {% if "navigation.tabs" in features %} + {% include "partials/tabs.html" %} + {% endif %} + {% endif %} +
diff --git a/plans/egress-gate-refactor.md b/plans/egress-gate-refactor.md new file mode 100644 index 00000000..16c98e32 --- /dev/null +++ b/plans/egress-gate-refactor.md @@ -0,0 +1,1643 @@ +# Egress Gate refactor and reframing plan + +## Status + +Historical implementation plan. The public contract evolved while the refactor +was implemented, so names and configuration examples below may be superseded. +Use the project documentation and source as the canonical references. The +phased sequence records the original construction and acceptance boundaries; +it is not a remaining-work checklist and must not be replayed. + +This plan intentionally makes no provision for backwards compatibility. The +superseded package name, Python imports, CLI, policy schema, public classes, +examples, documentation routes, and tests may all be removed or replaced. Do +not add compatibility aliases, schema translation, deprecation warnings, +legacy command names, or dual implementations. + +## Executive decision + +Build **Egress Gate** as an extensible OpenShell middleware that evaluates and +transforms sandbox HTTP egress during the pre-credentials request phase. + +The product is not a fixed DLP service and is not a standalone forward proxy. +It is an OpenShell supervisor middleware designed directly around OpenShell's +pre-credentials HTTP request contract, with: + +- a bounded immutable OpenShell HTTP request model +- an ordered pipeline of strongly typed, registry-provided gates +- explicit proceed, allow, and deny control flow +- validated request mutations +- audit-safe findings and decision metadata +- application-owned resources for custom integrations +- bounded preparation, atomic policy replacement, execution, and concurrency +- a protobuf-free processing core behind the OpenShell gRPC service boundary + +Body inspection and redaction are a first-party configuration of the built-in +regex gate, not a separate compatibility layer. The generic registry remains +available for organization-specific gates without shipping another concrete +integration. Deterministic destination and request constraints remain in +OpenShell policy, which already owns egress enforcement. + +The refactor does **not** implement an HTTP/HTTPS MITM proxy. OpenShell owns +request interception, routing, egress enforcement, and credential attachment; +Egress Gate owns policy evaluation through the supervisor-middleware +contract. A standalone proxy is outside this project's scope and roadmap. If a +separate proxy project is ever proposed, it must not shape this middleware's +gate API or package structure. + +## Working names + +Use these names throughout the refactor unless product naming is deliberately +revisited before implementation starts: + +| Surface | New name | +| --- | --- | +| Product | Egress Gate | +| Project directory | `projects/egress-gate/` | +| Python distribution | `egress-gate` | +| Python package | `egress_gate` | +| CLI | `egress-gate` | +| Service manifest name | `egress-gate` | +| Documentation route | `documentation/egress-gate/` | +| GitHub workflow | `egress-gate.yml` | + +The regex redaction composition uses the same binary, service, import path, and +configuration schema as every other Egress Gate policy. + +## Goals + +1. Preserve and strengthen customization as a first-class feature. +2. Make regex-based detect, deny, and replace setups concise. +3. Let custom gates reason about the complete bounded request rather than only + one decoded text body. +4. Retain strict typed configuration generated from the exact installed gate + registry. +5. Keep policy behavior in policy configuration and operational dependencies + in application-owned resources. +6. Make gate order, mutation visibility, terminal decisions, defaults, and + failures explicit and mechanically testable. +7. Preserve content-safe findings, errors, and logs by default. +8. Reuse one prepared active policy and make policy changes atomic without + restarting Egress Gate. +9. Keep OpenShell protobuf and gRPC details inside `service/` while modeling + the processing domain directly on OpenShell's request semantics. +10. Provide offline policy evaluation and shadow operation without requiring a + raw production-traffic database. + +## Project non-goals + +- Preserving any superseded API or configuration. +- Supporting runtimes or transports outside OpenShell. +- Implementing a forward proxy or TLS interception. +- Inspecting or transforming HTTP responses; the current OpenShell protocol + exposes only pre-credentials HTTP requests. +- Inspecting files, transcripts, tool calls, or harness persistence. +- Acting as a WAF, network firewall, identity provider, credential broker, or + general authorization server. +- Providing vendor-specific LLM SDKs or implementing semantic/LLM judgment, + including as a runnable example. The only built-in gate is regex body. +- Duplicating deterministic host, port, method, path, query, or process rules + already owned by OpenShell policy. Organization-specific request logic may + still be implemented through the custom-gate API when OpenShell policy is + insufficient. +- Persisting request bodies, headers, query strings, or response content by + default. +- Automatically publishing policies inferred from observed traffic. +- Hot-reloading installed Python code or registry factories. +- Serving genuinely different active policy configurations concurrently from + one Egress Gate service; deploy separate service instances for that case. +- Carrying the names or branding of external comparison tools into Egress Gate + code, configuration, examples, tests, logs, metrics, or product documentation. + +## Design principles + +### Terminology is part of the contract + +Use the nouns consistently in code, configuration, documentation, diagnostics, +and tests: + +- **Egress Gate** is the OpenShell middleware product and service. +- A **gate type** is one registered `Gate` implementation selected by the + literal `config.gate` discriminator. `GateType` is that stable discriminator, + never a Python class name. +- A **configured gate** or **gate instance** is one named pipeline entry with a + gate type and exact configuration. `GateName` is its stable instance identity. +- A **gate** is the general extension noun when the type/instance distinction is + irrelevant. It evaluates the current request and may produce findings, + propose mutations, proceed, allow, or deny. A gate need not make a terminal + decision. +- A **pipeline** is the ordered composition of configured gates plus its + required default decision. +- `GateEvaluation` is the validated output of one gate invocation. +- `EgressResult` is the final middleware-domain result after the pipeline has + terminated or applied its default decision. + +Diagnostics, traces, discovery, and any future metrics use `GateName` and +`GateType`, not implementation class names. Reserve **pipeline** or +**composition** for multi-gate behavior; do not call an entire pipeline a gate. + +Do not use “stage” as an internal or public synonym for gate. OpenShell's +pre-credentials **phase** remains a separate protocol concept and should +still be called a phase. + +### The current implementation is the design reference + +The refactor changes the product scope and public contract, but it should keep +the general taste of the current implementation. Its layout and extension +mechanics were deliberate and are the starting point for the redesign, not +legacy structure to discard casually. + +In particular, preserve these qualities unless the new behavior provides a +concrete reason to change one: + +- a small, readable package map with behavior-owning modules +- strict immutable Pydantic domain models +- public orchestration methods wrapping protected extension hooks +- exact typed configuration for every registered implementation +- optional typed operational resources injected at construction +- no arbitrary custom constructors +- registry finalization before serving +- OpenShell protobuf and gRPC isolation under `service/` +- one shared monotonic timeout across an evaluation +- stable content-safe errors, logs, and findings +- tests that mirror source boundaries +- focused dependencies and an understandable import graph +- public declarations before private implementation details where practical + +Broader scope does not justify framework-shaped abstraction layers, deep +directory nesting, generic dependency injection, event buses, or a +proliferation of interfaces. Extend the current design in the smallest coherent +way: widen the processor input from one text value to one immutable HTTP +request, widen the registry from entity engines to request gates, and retain +the existing construction, validation, execution, and service seams where they +still own the same behavior. + +### The runtime owns policy execution + +The runtime owns ordering, deadlines, gate invocation, gate-evaluation +validation, mutation application, decision termination, finding aggregation, +error translation, and final result construction. A custom gate cannot +redefine those mechanics. + +### Gates own one explicit behavior + +A gate is configured for one request-level responsibility, such as matching +request facts, inspecting or rewriting body text, or integrating a custom +external decision service. Prefer specific gate classes over a generic +callback or interceptor API. + +### The current request is the only gate input + +Each gate sees the request after mutations from all preceding gates. This +preserves ordered replacement behavior and enables a redaction gate to rewrite +a body before a later custom gate sees it. + +The runtime may retain the original OpenShell request privately so it can +produce one final mutation result for OpenShell. Gates must not receive an +implicit escape hatch to the unmodified original request. + +### Control flow is explicit + +Every successful gate returns one of: + +- `proceed`: apply validated mutations and invoke the next gate +- `allow`: require an empty patch, stop the pipeline, and allow the current + request including mutations already applied by earlier proceeding gates +- `deny`: require an empty patch, stop the pipeline, and deny the request + +If every gate proceeds, the policy's required `default_decision` determines +the result. There is no implicit allow and no hidden fallback. + +An early terminal allow intentionally skips later gates. Configuration and +documentation must make this visible because it bypasses every later custom or +built-in gate. + +### Failures are not decisions + +Invalid input, invalid configuration, gate contract violations, gate +execution errors, and unexpected failures remain evaluation failures. The +OpenShell middleware registration's `on_error` setting owns their request +effect. A failure must not silently become `proceed` or `allow`. + +Expected runtime safety-limit exhaustion returns a stable fail-closed deny when +the request envelope and policy were otherwise valid, following the existing +fail-closed approach. Do not introduce a general passthrough fallback. + +Use this normative outcome matrix: + +| Condition | Outcome | Active-policy effect | `on_error` | +| --- | --- | --- | --- | +| Invalid phase, envelope, policy, or incoming request bound | gRPC `INVALID_ARGUMENT` | None | Applies | +| Invalid candidate gate configuration | gRPC `INVALID_ARGUMENT` | Candidate is not published | Applies | +| Deadline expiry during validation, slot wait, replacement-lock wait, preparation, or the final pre-publication check | Deny with source `runtime_limit` and code `egress_gate_limit_exceeded` | Candidate is not published | Does not apply | +| Deadline expiry after candidate publication, including gate execution or result construction | Deny with source `runtime_limit` and code `egress_gate_limit_exceeded` | The atomically published candidate remains active; do not roll it back | Does not apply | +| Runtime input or domain-conversion limit before candidate publication | Deny with source `runtime_limit` and code `egress_gate_limit_exceeded` | Candidate is not published | Does not apply | +| Runtime mutation, finding, or encoded-output limit after candidate publication | Deny with source `runtime_limit` and code `egress_gate_limit_exceeded` | The published candidate remains active; do not roll it back | Does not apply | +| Gate contract violation, gate execution failure, or unexpected internal failure | gRPC `INTERNAL` | Any already published candidate remains active; do not roll it back | Applies | +| Explicit gate or pipeline decision | Return the corresponding `EgressResult` | A complete candidate may already have been published | Does not apply | +| RPC cancellation | Propagate cancellation; synchronous work still owns its slot until exit | Do not publish an unpublished candidate | Gateway cancellation behavior applies | + +Check the deadline after acquiring the replacement lock and again immediately +before candidate publication. An expired or cancelled request cannot prepare or +publish a new active policy. Runnable registrations and examples configure +OpenShell's `on_error` to deny. + +### Data minimization is the default + +Findings and logs exclude request content by default. Any traffic-discovery or +content-capture mode must be separately named, opt-in, bounded, and documented +as expanding the trust boundary. + +### Semantic judgment is deferred + +Do not implement or document a concrete semantic/LLM gate in this refactor, +including under `examples/`. A later proposal must define its own data +minimization, failure, evaluation, and prompt-injection boundaries before any +implementation is added. + +## Target architecture + +```text +OpenShell gateway + | + v +SupervisorMiddleware service boundary + - validates protobuf bounds and phase + - converts the evaluation to immutable domain models + - resolves a prepared policy pipeline + | + v +RequestProcessor + - shared deadline + - ordered gate execution + - request mutation validation + - terminal decision handling + - finding aggregation + | + +--> regex body gate + +--> custom organization gate + | + v +EgressResult + - allow or deny + - body/header mutations + - audit-safe findings + - stable reason code + | + +--> OpenShell HttpRequestResult serialization + +--> content-safe operational logging +``` + +## Core domain model + +Create protobuf-free immutable models in focused top-level modules such as +`request.py` and `result.py`. They should directly represent the bounded fields +and semantics of OpenShell's `HttpRequestEvaluation` and `HttpRequestResult`, +without pretending to be a generic cross-transport HTTP abstraction. Keep the +package flat unless a directory owns a real family of implementations. Only +`service/` may import gRPC or generated protobuf bindings. + +### `HttpRequest` + +The gate-visible request contains: + +```python +class HttpRequest(StrictDomainModel): + context: RequestContext + target: HttpTarget + headers: tuple[HttpHeader, ...] + body: bytes +``` + +`RequestContext` contains the bounded request ID, sandbox ID, and originating +process information already supplied by OpenShell. `HttpTarget` contains +scheme, host, port, method, path, and raw query. `HttpHeader` preserves ordered +repeated fields. + +The request remains byte-oriented. Text gates explicitly perform strict +decoding according to their configuration and content requirements. The core +runtime must not assume every HTTP body is UTF-8. + +### `RequestPatch` + +A gate proposes mutations rather than mutating the request object: + +```python +class ExistingHeaderAction(StrEnum): + APPEND = "append" + OVERWRITE = "overwrite" + SKIP = "skip" + + +class WriteHeaderMutation(StrictDomainModel): + operation: Literal["write"] = "write" + name: HeaderName + value: HeaderValue + on_existing: ExistingHeaderAction + + +class RemoveHeaderMutation(StrictDomainModel): + operation: Literal["remove"] = "remove" + name: HeaderName + + +HeaderMutation = Annotated[ + WriteHeaderMutation | RemoveHeaderMutation, + Field(discriminator="operation"), +] + + +class RequestPatch(StrictDomainModel): + replacement_body: bytes | None = None + header_mutations: tuple[HeaderMutation, ...] = () +``` + +`replacement_body=None` means no body replacement, while `b""` explicitly +replaces the body with an empty value. The OpenShell service adapter derives +the wire-level `has_body` flag from that single domain representation. These +protobuf-free header types live in `request.py` and mirror the OpenShell write, +remove, and existing-header actions exactly. Header names match +case-insensitively. Operations apply in tuple order: `append` adds one value at +the end; `overwrite` removes every matching field and appends the new value; +`skip` appends only when no matching field exists; and `remove` removes every +matching field. Unrelated headers retain their relative order. + +The runtime validates protected-header restrictions and all per-patch and +request-wide mutation limits before making any operation visible to the next +gate. For an allowed result, it serializes the validated operations from +proceeding gates in the same order without collapsing or synthesizing a +different sequence. Thus intermediate and final wire semantics are identical, +including for repeated headers, and operation/encoded-size limits apply both +incrementally and to the final concatenated sequence. An empty patch has +`replacement_body=None` and no header mutations. + +```python +RequestPatch( + header_mutations=( + WriteHeaderMutation( + name="x-openshell-middleware-policy-reviewed", + value="true", + on_existing=ExistingHeaderAction.OVERWRITE, + ), + ), +) +``` + +### `GateControl` + +```python +class GateControl(StrEnum): + PROCEED = "proceed" + ALLOW = "allow" + DENY = "deny" +``` + +### `GateEvaluation` + +```python +class GateEvaluation(StrictDomainModel): + control: GateControl + patch: RequestPatch = RequestPatch() + findings: tuple[Finding, ...] = () + reason_code: str | None = None + + @classmethod + def proceed( + cls, + *, + patch: RequestPatch | None = None, + findings: tuple[Finding, ...] = (), + ) -> Self: ... + + @classmethod + def allow( + cls, + *, + findings: tuple[Finding, ...] = (), + ) -> Self: ... + + @classmethod + def deny( + cls, + reason_code: ReasonCode, + *, + findings: tuple[Finding, ...] = (), + ) -> Self: ... +``` + +`proceed` treats `patch=None` as an empty patch. These are the complete v0 +construction helpers; callers needing no helper may instantiate the strict +model directly. + +Required invariants include: + +- `reason_code` is present if and only if control is `deny`; both `proceed` and + `allow` require it to be `None` +- a deny must carry a stable reason code +- both terminal controls require an empty patch; only `proceed` can propose + mutations +- a denied `EgressResult` contains no mutation, including mutations accumulated + privately from earlier proceeding gates +- findings satisfy shared per-gate and final-result count and encoded-size + limits +- strings crossing the result boundary are audit-safe and bounded +- mutations are validated before they become visible to a later gate +- a failed gate contributes no partial patch or findings + +### General finding model + +`Finding` is the extensible, audit-safe result vocabulary shared by built-in +and custom gates. It must be general enough to describe different classes of +observation without becoming an arbitrary payload channel. + +Use one stable envelope: + +```python +FindingCount = Annotated[int, Field(ge=1, le=(2**32 - 1))] + +class Finding(StrictDomainModel): + type: FindingType + label: AuditSafeFindingLabel + count: FindingCount = 1 + confidence: AuditSafeFindingValue | None = None + severity: AuditSafeFindingValue | None = None +``` + +The fields have distinct roles: + +- `type` is a stable machine-readable category owned by the gate +- `label` is a stable audit-safe identifier or concise display label within + that category +- `count` aggregates equivalent observations +- `confidence` and `severity` are optional gate-defined scalar vocabularies + +These are exactly the fields in the released OpenShell finding contract. Do +not add an internal attributes field that cannot be serialized faithfully. +When a custom gate needs a richer result, it should emit several stable finding +categories or use a separately designed offline/export surface rather than +encoding structured data into a label or result metadata. + +`FindingCount` is an integer from 1 through the canonical OpenShell protobuf +maximum (`2**32 - 1` for the current `uint32` field). Aggregation uses checked +addition; overflow produces the same atomic stable limit result as count or +encoded-size exhaustion. + +Example finding categories include: + +| Gate | `type` | Example `label` | Optional scalar vocabulary | +| --- | --- | --- | --- | +| Regex privacy | `sensitive_entity` | `email` | `confidence=high` | +| Custom JSON validation | `body_schema_violation` | `required_field_missing` | `severity=error` | +| Custom gate | Custom stable type | Custom stable label | Gate-defined confidence/severity | + +Finding values must not contain matched request text, body fragments, header or +query values, raw model output, arbitrary exception text, credentials, or +other request-derived content. A gate may use configured identifiers, +catalog-owned entity names, rule IDs, schema IDs, approved profile names, and +other stable values that satisfy shared bounds. + +A gate needing to report several independent observations emits several +findings. A use case requiring structured, large, or content-bearing output +belongs in an explicitly designed opt-in export or evaluation surface, not in +middleware findings. + +### Finding declarations + +Each gate class declares the finding types it may emit: + +```python +finding_types = ( + FindingTypeDefinition( + type="body_schema_violation", + ), +) +``` + +The declaration is part of gate discovery and contract validation. The public +gate wrapper rejects: + +- undeclared finding types +- invalid identifiers +- values outside shared string bounds +- per-gate or per-request count and encoded-size limits +- finding content that violates structurally enforceable safety rules + +Helper bases declare common finding definitions or provide concise helpers so a +simple custom gate does not need repetitive boilerplate. The CLI reports each +installed gate's possible finding types. Custom-gate contract tests verify +emitted findings against the declarations. + +### Runtime-owned provenance and aggregation + +A gate returns only its own `Finding` values. `RequestProcessor` wraps every +accepted finding with runtime-owned provenance before adding it to the final +result: + +```python +class SourcedFinding(StrictDomainModel): + source_gate: GateName + finding: Finding +``` + +Gates cannot spoof another gate's source. The runtime aggregates only +findings with identical source, type, label, confidence, and severity. +Aggregation adds their counts and rechecks the request-wide bounds with checked +arithmetic. + +Egress Gate is one OpenShell middleware result regardless of how many internal +gates ran. Set a fixed per-gate finding cap no larger than the canonical +OpenShell result cap, and cap the final serialized `EgressResult` at the +canonical one-result limit (currently 32 finding groups, not 32 per internal +gate). Enforce the final cap incrementally while aggregating. If several +individually valid gate evaluations exceed it together, return the stable +fail-closed limit result with no partial findings or mutations. + +Findings remain observations, not hidden decision inputs. A gate's +`GateControl` decides whether processing proceeds, allows, or denies; the +presence or severity of a finding has no implicit runtime disposition. + +### OpenShell finding contract + +The released OpenShell `Finding` protobuf has `type`, `label`, `count`, +`confidence`, and `severity`. Those five fields are the complete v0 public +finding contract for built-in and custom gates. + +`RequestProcessor` may retain `source_gate` around findings in its internal +`EgressResult` so traces, logs, and offline evaluation can attribute which gate +emitted an observation. The OpenShell adapter serializes only the nested +five-field `Finding`; per-gate provenance is not available on the wire. This is +an explicit platform limitation, not a reason to invent a compatibility +encoding. + +Do not encode provenance or structured attributes into `label`, flatten them +into indexed `HttpRequestResult.metadata` keys, or serialize JSON into a string +field. `EgressResult.metadata` is runtime-owned and remains reserved for facts +about the complete pipeline evaluation. Gates cannot emit or overwrite +result-level metadata. + +### `EgressResult` + +The final domain result contains the terminal decision, decision source, +complete final patch relative to the original OpenShell evaluation when +allowed, accumulated internally sourced findings, stable reason code, bounded +runtime-owned metadata, policy fingerprint, and per-gate content-safe trace +data. A denied result always has an empty patch. + +`DecisionSource` has a bounded kind—`gate`, `pipeline_default`, or +`runtime_limit`—plus optional `GateName` and `GateType` fields. Those fields are +required only for kind `gate` and forbidden for the other kinds. A +default-sourced decision has no terminal gate; default deny +uses the runtime-owned `egress_gate_default_deny` reason code, while default +allow has no deny reason code. A runtime-generated fail-closed limit denial has +no terminal gate, uses `runtime_limit`, and carries the appropriate stable +runtime limit code such as `egress_gate_limit_exceeded`. Logs and offline +evaluation use these fixed source values rather than misattributing either +outcome to the last proceeding gate. + +Use this complete v0 deny-code ownership table: + +| Denial source | Reason code | +| --- | --- | +| `regex-body` match | `egress_gate_regex_denied` | +| Custom gate | A validated gate- or policy-owned code | +| Pipeline default | `egress_gate_default_deny` | +| Runtime safety limit | `egress_gate_limit_exceeded` | + +No other runtime-generated deny code is part of v0. Evaluation failures use +the service error mapping rather than inventing deny codes. + +Each `GateTrace` records `GateName`, `GateType`, control result, duration, +finding count, and mutation kinds. `GateType` is the registry discriminator, +never the Python implementation class. The trace does not contain bodies, +header values, query values, matched text, model output, arbitrary exception +messages, or gate-defined free-form metadata. + +## Gate extension system + +The reframed product has three intentionally separate customization levels: + +1. **Policy composition:** operators assemble installed gates without writing + Python. This is how regex inspection and redaction deployments should be + built. +2. **Gate authoring:** developers add one focused request behavior through a + typed config and `Gate` implementation. +3. **Application assembly:** deployers register trusted gates and inject typed + operational resources through one registry factory. + +Additional transports are outside the extension model and outside this +project's scope. Gate authors target the stable protobuf-free representation +of OpenShell's supervisor-middleware request contract. + +### Public gate contract + +Replace `EntityProcessingEngine` as the primary extension contract with a +request-level `Gate` contract: + +```python +GateConfigT = TypeVar("GateConfigT", bound=GateConfig) +GateResourcesT = TypeVar( + "GateResourcesT", + bound=GateResources | None, + default=None, +) + + +class Gate(Generic[GateConfigT, GateResourcesT]): + @final + def __init__( + self, + config: GateConfigT, + resources: GateResourcesT, + *, + timeout: Timeout | None = None, + ) -> None: ... + + @classmethod + def get_config_type(cls) -> type[GateConfig]: ... + + @classmethod + def get_resources_type(cls) -> type[GateResources] | None: ... + + @property + def config(self) -> GateConfigT: ... + + @property + def resources(self) -> GateResourcesT: ... + + def _initialize(self, *, timeout: Timeout | None = None) -> None: ... + + @final + def evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: ... + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: ... +``` + +As today, custom implementations do not define arbitrary constructors. The +base class owns construction, read-only config access, optional resource +validation, the public `evaluate` wrapper, input/result validation, and +content-safe error translation. Implementations provide a protected initialization hook for +derived reusable state and a protected `_evaluate` method. Construction +validates the exact config and resource types, stores them as read-only +properties, and calls `_initialize` exactly once with the preparation timeout. +The resulting instance must be safe for concurrent calls and is shared by +evaluations; the runtime does not claim Python-level deep immutability. +The wrapper rejects terminal evaluations with non-empty patches before they +reach the pipeline. + +The generic arguments are the single source of truth for exact config and +resource types, preserving the former extension pattern. The resources type +defaults to `None`, so `Gate[KeywordDenyConfig]` is the complete declaration +for a resource-free gate. Base-owned read-only class methods infer the runtime +types; implementations do not repeat them as class attributes, and registry +validation rejects missing, unresolved, or invalid generic declarations. +Use `typing_extensions.TypeVar` for the defaulted type parameter on Python +3.11. + +### Extension-author experience + +The clean custom-gate path is a primary product surface. A resource-free gate +should normally require only: + +1. one strict configuration model +2. one gate class +3. one compact capability declaration, or a helper base that supplies it +4. one protected `_evaluate` implementation +5. one registry call + +For example, the documentation should be able to present a complete custom +gate with approximately this shape: + +```python +class KeywordDenyConfig(GateConfig): + gate: Literal["keyword-deny"] + keyword: str + reason_code: ReasonCode + + +class KeywordDenyGate(Gate[KeywordDenyConfig]): + capabilities = GateCapabilities(reads_body=True, may_deny=True) + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + timeout.raise_if_expired() + if self.config.keyword.encode() in request.body: + return GateEvaluation.deny(self.config.reason_code) + return GateEvaluation.proceed() +``` + +An extension author should not need to understand gRPC, protobufs, active-policy +replacement, server lifecycle, mutation serialization, or runtime concurrency. + +Resource-free gates omit a resources generic parameter and resource-backed +gates add one typed `GateResources` model, matching the current extension +pattern. Export the complete supported authoring surface from +`egress_gate.gates` so examples do not import private modules. + +Registration is equally direct: `registry.register(KeywordDenyGate)` for a +resource-free gate, and `registry.register(OrganizationGate, +resources=resources)` for a resource-backed gate. The registry, not policy +configuration, supplies the constructor argument; it passes `None` explicitly +for resource-free gates, while a missing or wrong resource for a +resource-backed gate is a registration error. + +Document the gate contract and test the example custom gate directly. Add a +reusable external contract-test package only after a concrete second consumer +shows that the abstraction reduces duplication. + +### Gate configuration + +Every concrete gate declares an exact strict Pydantic config model with a +literal `gate` discriminator: + +```python +class KeywordDenyConfig(GateConfig): + gate: Literal["keyword-deny"] + keyword: str +``` + +The application registry builds the discriminated union from exactly the +installed gate types. Unknown fields and unknown gates are rejected. + +### Operational resources + +Retain the existing distinction between policy behavior and deployment-owned +resources. Resource bundles contain concurrency-safe clients, approved +endpoints, or credential providers. They contain no per-request state and +cannot override policy configuration. + +Prepared gates borrow these application-owned resources; policy replacement +never closes them. In v0, `_initialize` may create only immutable, ordinarily +garbage-collected derived state, not independently closable resources. During +shutdown the server stops admitting RPCs and waits for synchronous workers to +exit; only then may the assembling application close its resources. Egress +Gate itself never closes a borrowed resource. + +A custom resource-backed gate may select an operator-approved resource profile, +but policy configuration may not provide arbitrary provider URLs, credentials, +Python imports, or client implementations. + +### Capability declarations + +Each gate type declares immutable capabilities used for validation and +discovery: + +- reads target +- reads context +- reads headers +- reads body +- replaces body +- mutates headers +- produces findings +- may terminally allow +- may deny +- uses external resources + +Read capabilities are declarative discovery and linting metadata because every +trusted gate receives the complete immutable `HttpRequest`; the wrapper cannot +prove which fields Python code inspected. Output capabilities—body replacement, +header mutation, finding production, terminal allow, and deny—are mechanically +enforced against each `GateEvaluation` at the public wrapper. + +Capabilities are not a permission system for hostile Python code; registry +factories and custom gate modules remain trusted deployment code. Do not imply +that read declarations create field isolation. + +Keep capability authoring compact. Helper bases should provide correct defaults +for common cases, and capabilities that can be derived unambiguously from a +base class or result type should not require repetitive declarations. + +### Helper bases + +Provide one narrow helper base for the built-in text-inspection pattern: + +- `Utf8BodyGate`: strict UTF-8 decoding, unchanged-body checks, and bounded + body replacement + +The built-in regex implementation should use `Utf8BodyGate`. A custom privacy +gate should remain comparably compact. + +## Registry and application assembly + +Rename and generalize `EngineRegistry` to `GateRegistry`. + +The registry must: + +1. register concrete gate config, implementation, and optional resource types +2. reject duplicate discriminators and incomplete declarations +3. bind application-owned resource profiles +4. generate the exact pipeline configuration schema +5. validate gate capabilities and finding-type declarations +6. prepare reusable gate instances that satisfy the concurrent-call contract +7. expose content-safe gate and finding discovery information for the CLI +8. finalize exactly once before serving requests + +Preserve `module:factory` application assembly through a renamed +`--registry-factory` option. The factory returns one finalized application +registry. Configuration cannot choose or import a factory. + +## Pipeline configuration + +Use one top-level schema: + +```yaml +pipeline: + gates: + - name: stable-diagnostic-name + config: + gate: concrete-gate-discriminator + # exact gate-specific fields + default_decision: allow +``` + +Requirements: + +- one through ten gates initially +- a required, explicit, unique `name` for every gate +- required `default_decision` +- exact strict gate configuration +- complete validation before preparation +- complete candidate preparation before active-policy publication +- no global `on_detection` concept +- no implicit action derived from findings +- no backwards-compatible acceptance of `entity_processing` + +Gate configuration owns the relationship between its observations and its +control result. This permits the regex gate and installed custom gates to +detect, replace, allow, or deny as their explicit contracts define. + +## Built-in gates + +The default registry ships exactly one concrete gate: `regex-body`. +Organization-specific behavior may use the custom-gate registry surface. +Deterministic egress constraints belong in OpenShell policy. Semantic judgment +and other proposed built-ins are deferred; do not add another concrete gate or +example implementation in this refactor. + +### Regex body gate + +Port the hardened catalog validation, bounded compiled cache, timeout-capable +matching, overlap behavior, detections, and replacement implementation into a +request-level `regex-body` gate. + +Proposed configuration: + +```yaml +gate: regex-body +pattern_catalog: patterns.yaml +mode: replace # detect | deny | replace +replacement: + strategy: template + template: "[{entity}]" +``` + +Behavior: + +- `detect`: findings plus `proceed`, no mutation +- `deny`: deny on a match; otherwise proceed +- `replace`: replace the body, emit findings, and proceed + +The gate reads the current body as strict UTF-8. Later gates see the replaced +body. Preserve the current safety bounds and atomic failure behavior, but adopt +new names and APIs without aliases. + +## Reference compositions + +### Regex redaction composition + +Ship a runnable example and complete documentation for: + +```yaml +pipeline: + gates: + - name: identifiers + config: + gate: regex-body + pattern_catalog: + entities: + - name: email + rules: + - pattern: '(? + + + + + + EgressGate comprehensive QA and remediation report — 2026-08-05 + + + + +
+
+

Independent battle test

+

EgressGate comprehensive QA and remediation report

+

Functional, extension, service, resilience, content-safety, packaging, and operator-experience validation followed by verified remediation of every finding.

+
+ 2026-08-05 UTC + commit aa74572 + branch johnny/egress-gate-refactor + EgressGate 0.1.0 + 13 of 13 findings addressed +
+
+
+ +
+ + +
+

Executive summary

+
+
Verified
Remediation assessment
+
217 / 217
Authoritative tests passed on 3.11 and 3.14
+
0
Critical or high findings
+
0 open
5 medium + 8 low findings addressed
+
+

Bottom line: EgressGate's core decision pipeline, regex behavior, custom-gate contract, content safety, limits, concurrency, policy replacement, packaging, and shutdown behavior remain strong. All 13 QA findings have now been addressed through code, tests, or explicit operator guidance, and the integrated project and documentation checks pass.

+

No request-content disclosure was found. Adversarial failures were generally fail-closed, bounded, and recoverable. The one implementation-detail disclosure contains Python/protobuf type information, not user request data.

+ +

Release gate

+
+ + + + + + + + + + +
AreaAssessmentRationale
Core correctnessPassAll 217 repository tests pass on Python 3.11 and 3.14; additional regex and service battle suites passed.
Custom gatesPassTwo novel gates and the bundled example worked through schema, validation, offline evaluation, downstream mutation, and live gRPC.
Security/content safetyPassSentinel request/config values stay secret; malformed protobuf now returns cataloged INVALID_ARGUMENT without implementation details.
ResiliencePassDeadlines, cancellation, worker and RPC saturation, invalid policy replacement, oversize input, and close/restart behavior recovered.
Operator UXPassValidation, preparation, and corpus failures now provide bounded actionable context; command discovery and narrow-terminal help are verified.
PackagingPasssdist and wheel build and install cleanly on Python 3.11; packaged guidance now separates installed and source-checkout workflows and uses durable links.
+
+
+ +
+

Verified remediation status

+

The original QA session found five medium- and eight low-severity issues. Eleven new regression tests were added during remediation. Transport behaviors owned by grpcio were resolved with explicit bounded operational guidance instead of weakening EgressGate's admission controls or hiding HTTP/2 faults.

+
+ + + + + + + + + + + + + + + + + +
FindingResolutionStatus
EG-QA-01A server interceptor now catches protobuf decode failures before dispatch and returns cataloged request_protobuf_invalid with gRPC INVALID_ARGUMENT. Raw-wire recovery is regression-tested.Verified
EG-QA-02Gate preparation failures map to config_preparation_failed with safe built-in regex remediation rather than custom-resource guidance.Verified
EG-QA-03Policy errors now report one trusted schema path and safe category while excluding submitted values, Pydantic inputs, context, and URLs.Verified
EG-QA-04The README separates installed and source-checkout workflows, states that examples/docs are repository assets, and uses durable absolute links. Wheel metadata was inspected after a clean build.Verified
EG-QA-05Automatic logging color honors the presence of NO_COLOR; explicit application-owned ALWAYS remains an override. Empty and non-empty values are tested.Verified
EG-QA-06Execution failures identify the validated case name and render safe results completed before the failure without exposing request content.Verified
EG-QA-07Bare invocation now renders help and exits 0.Verified
EG-QA-08Operations guidance now specifies short bounded 5/10/20 ms backoff within the middleware deadline. EgressGate retains grpcio's 16-RPC transport guard.Documented
EG-QA-09Operations guidance distinguishes expected HTTP/2 GOAWAY/cancellation during zero-grace planned shutdown from actionable out-of-window transport faults.Documented
EG-QA-10Generated schemas rewrite Pydantic generic definition names and references to stable ConfiguredGate and PipelineConfig names.Verified
EG-QA-11egress-gate --version reports the installed distribution version and exits 0.Verified
EG-QA-12Plain help preserves complete option identifiers at 40 columns; concise command summaries remain complete at standard widths.Verified
EG-QA-13Operations documentation now gives policy, transport, and controlled sandbox end-to-end readiness checks and explains the limits of each layer.Verified
+
+
+ +
+

Original QA findings

+ +
+
Medium

EG-QA-01 — Malformed protobuf returns UNKNOWN with implementation details

+
+
Observed
A malformed nested protobuf returned UNKNOWN and named google.protobuf.message.DecodeError plus the generated message type.
+
Expected
A stable, content-safe INVALID_ARGUMENT response consistent with the documented invalid-request contract.
+
Impact
Clients cannot classify all bad input consistently, and the response exposes runtime implementation detail. The server did recover immediately.
+
Likely seam
The failure occurs before the servicer method, so handling probably belongs at the gRPC deserialization or interceptor boundary.
+
+
Reproduction and evidence + 
raw = channel.unary_unary(
+    "/openshell.middleware.v1.SupervisorMiddleware/EvaluateHttpRequest",
+    request_serializer=lambda value: value,
+    response_deserializer=lambda value: value,
+)
+await raw(b"\x12\x02\x0a\xff")
+
+status=UNKNOWN
+details="Unexpected <class 'google.protobuf.message.DecodeError'>:
+Error parsing message with type
+'openshell.middleware.v1.HttpRequestEvaluation'"
+
+
+ +
+
Medium

EG-QA-02 — Built-in regex preparation failure gives custom-gate guidance

+
+
Observed
A structurally valid policy with a forbidden named capture group passes validate, then evaluate reports generic execution_failed and tells the user to inspect custom resources.
+
Expected
A cataloged configuration/preparation error that identifies the regex-policy remediation without echoing pattern content.
+
Impact
The error is safe but sends operators to the wrong subsystem, increasing time to diagnose a built-in configuration issue.
+
+
Reproduction and evidence + 
$ egress-gate validate --policy qa_policy_named_group.yaml
+✓ Policy is valid
+
+$ egress-gate evaluate --policy qa_policy_named_group.yaml --cases qa_cases.yaml
+Evaluation failed [execution_failed]
+An unexpected error stopped the evaluation.
+Next: Check custom gate and application-owned resource setup, then retry.
+[exit 2]
+
+
+ +
+
Medium

EG-QA-03 — Policy validation diagnostics lack a field path

+
+
Observed
A typo such as scna is reduced to a generic schema mismatch; the CLI does not identify the gate, YAML path, unknown key, or missing scan field. Missing and malformed files also share the same invalid_input text.
+
Expected
A bounded structural location and reason, while continuing to suppress submitted values and raw exception text.
+
Impact
Safe but slow troubleshooting, especially in a large multi-gate policy.
+
+
Observed output + 
Policy validation failed [config_invalid]
+The policy does not match the schema for the installed gates.
+Next: Run egress-gate gates schema, then check the pipeline, gate kinds,
+required fields, and pattern catalog.
+[exit 1]
+
+
+ +
+
Medium

EG-QA-04 — Packaged README quickstart depends on files that are not shipped

+
+
Observed
The wheel and sdist include sources, license, and README but omit examples/, docs/, and uv.lock. The embedded README tells users to validate examples/regex-redaction/egress-gate-config.yaml and links to relative documentation files.
+
Expected
An installed-package quickstart that works from a neutral directory, or an explicit “from a source checkout” label with absolute repository/documentation links.
+
Impact
A successful clean installation leads directly to a failing advertised first workflow and broken local documentation links.
+
+
Distribution evidence + 
$ egress-gate validate \
+    --policy examples/regex-redaction/egress-gate-config.yaml
+Policy validation failed [invalid_input]
+[exit 1]
+
+
+ +
+
Medium

EG-QA-05 — NO_COLOR is ignored by interactive service logging

+
+
Observed
In a pseudo-TTY, NO_COLOR=1 egress-gate --debug serve still emitted ANSI sequences for timestamp, level, and logger name.
+
Expected
The standard opt-out should disable styling in logging as well as command output.
+
Impact
Accessibility preferences are not honored and captured terminal logs may contain unwanted escape codes.
+
+
+ +
+
Low

EG-QA-06 — Execution failure omits the failing corpus case

+

An invalid-UTF-8 case aborts with body_encoding_invalid but does not print its bounded case name or already completed results. In a large corpus this forces manual bisection. Include the validated case name without rendering request fields.

+
+ +
+
Low

EG-QA-07 — Bare command prints help but exits 2

+

Running egress-gate with no arguments renders useful top-level help but returns usage-error status 2. This is common CLI-framework behavior, but exit 0 would better match a discovery-oriented first run.

+
+ +
+
Low

EG-QA-08 — Immediate retry can briefly remain saturated

+

After 20 concurrent calls produced 16 allows and four expected RESOURCE_EXHAUSTED results, one immediate retry was also rejected. A retry 5 ms later succeeded. This may be grpcio accounting teardown rather than EgressGate logic; document retry/backoff or smooth the recovery if practical.

+
+ +
+
Low

EG-QA-09 — Successful shutdown can emit confusing GOAWAY noise

+

Normal live-server teardown emitted grpcio core messages including Got goaway and Cancelling all calls. No work was lost. Consider logging guidance or filtering so expected shutdown does not resemble an incident.

+
+ +
+
Low

EG-QA-10 — Generated schema definition names are unwieldy

+

The JSON is valid, but custom-gate definitions can receive long Pydantic-derived names such as ConfiguredGate_Annotated_Union_RegexConfig__PathPrefixDenyConfig.... Stable human-oriented titles or a concise YAML schema summary would make diagnostics and discussion easier.

+
+ +
+
Low

EG-QA-11 — No --version command

+

egress-gate --version returns “No such option” with exit 2. Operators lack a direct way to correlate a running CLI with package and protocol versions.

+
+ +
+
Low

EG-QA-12 — Very narrow help truncates option names

+

At a 40-column pseudo-TTY, required registration options render as --host… and --conf…. Prefer a stacked/plain layout at narrow widths so identifiers remain copyable.

+
+ +
+
Low

EG-QA-13 — Readiness verification is under-documented

+

Operations guidance covers binding, registration, restart, and logs but no explicit health or end-to-end gateway reachability check. Add a concrete readiness verification workflow.

+
+
+ +
+

Custom-gate release-critical track

+

QA did not rely only on the bundled keyword example. Two disposable gates were independently authored in isolated copies using the documented public API.

+
+ + + + + + + + +
GatePurposePath exercisedResult
stamp-or-denyWrite a header unless a body token requires denial.--registry-factory, list, schema, validate, offline evaluate, downstream regex observation.Pass
qa-probeResource-backed keyword deny, controlled delay, counters, and deliberate exception.Real grpc.aio server, Describe, ValidateConfig, EvaluateHttpRequest, concurrency, policy replacement, failure redaction.Pass
Deliberately invalid gateReturn an undeclared terminal deny.Public capability enforcement and content-safe CLI failure.Rejected correctly
Bundled keyword-denyDocumentation smoke test.Discovery, policy validation, two-case offline corpus.Pass
+
+ +

What the authored gates proved

+ + 
$ egress-gate --registry-factory qa_custom_gate:create_registry evaluate \
+    --policy qa_custom_policy.yaml --cases qa_custom_cases.yaml
+2 passed · 0 failed · 2 total
+[exit 0]
+
+$ egress-gate --registry-factory qa_bad_gate:create_registry evaluate \
+    --policy qa_bad_policy.yaml --cases qa_bad_cases.yaml
+Evaluation failed [gate_output_invalid]
+A gate returned an invalid result.
+[exit 2]
+
+ +
+

Battle-test matrix

+
+ + + + + + + + + + + + + + + + + + +
AreaScenariosResult
Repository checksTests, formatting, Ruff, ty, import smoke test, dependency audit on Python 3.11.15 and 3.14.4.Pass
Regex gateBody, path, raw query, selected/repeated/case-insensitive headers, detect, deny, replace, overlap ranking, downstream mutation, ordering.Pass
Policy/corpus strictnessDuplicate keys/cases, YAML aliases, unknown fields, noncanonical base64, unsupported replacements, absolute/traversal/symlink catalogs, missing files.Rejected safely; diagnostics finding
CLIHelp, gate list/schema, validation, evaluation, stable exits 0/1/2, timeout bounds, registry reference errors, non-TTY and 40-column pseudo-TTY output.Pass; UX findings
PackagingLocked sync, sdist/wheel build, clean wheel install on Python 3.11, Python 3.10 rejection, installed console script, embedded README workflow.Runtime pass; packaged quickstart finding
Live gRPCDescribe, ValidateConfig, EvaluateHttpRequest, custom resource gate, active-policy changes, invalid candidate rollback.Pass
Deadlines/queueing12 calls, four workers, 90 ms work, 40 ms service timeout; atomic limit denials; post-drain recovery.Pass
CancellationEight calls with 15 ms client deadline and 100 ms work; slot retention, worker bound, recovery.Pass
Saturation20 calls against 16-RPC bound.16 allowed, 4 expected exhausted; one transient retry finding
Boundaries5,242,881-byte frame, 4 MiB + 1 body, NaN configuration, malformed protobuf.Limits enforced; malformed-wire finding
Content safetySentinels in bodies, policies, paths, templates, malformed configs, and raised exceptions.No request-content leak found
LifecycleClose during in-flight gate, active processor cleanup, repeated start/stop.Pass
Gateway registrationInput rejection, XDG creation, mode 0600, idempotent add/update, removal, operator next steps.Pass
AccessibilityNormal and narrow terminal rendering, piped schema, NO_COLOR service logging.Readable overall; color and narrow-help findings
+
+
+ +
+

Evidence and environments

+

Each specialist copied the project to a unique temporary directory and selected a distinct UV_PROJECT_ENVIRONMENT. Disposable gates, policies, corpora, and battle tests existed only in those copies. The shared worktree was used read-only until this report was added.

+ +

Authoritative validation

+ 
$ UV_PROJECT_ENVIRONMENT=/tmp/egress-gate-remediation-py311 make check-py311
+Using CPython 3.11.15
+217 passed in 1.68s
+41 files already formatted
+All checks passed!  # Ruff
+All checks passed!  # ty
+No known vulnerabilities found
+
+$ UV_PROJECT_ENVIRONMENT=/tmp/egress-gate-remediation-qa make check
+Using CPython 3.14.4
+217 passed in 1.61s
+41 files already formatted
+All checks passed!  # Ruff
+All checks passed!  # ty
+No known vulnerabilities found
+

The audit skipped only local unpublished egress-gate 0.1.0, as expected. Repeated cachecontrol cache-deserialization warnings were environment/tooling noise and did not affect the result.

+ +

Additional suites and measurements

+ + +
Representative commands + 
# Baselines
+UV_PROJECT_ENVIRONMENT=/tmp/egress-gate-root-qa-venv make check
+UV_PROJECT_ENVIRONMENT=/tmp/egress-gate-root-qa-py311 make check-py311
+
+# Built-in workflow
+uv run --frozen egress-gate gates list
+uv run --frozen egress-gate validate \
+  --policy examples/regex-redaction/egress-gate-config.yaml
+uv run --frozen egress-gate evaluate \
+  --policy examples/regex-redaction/egress-gate-config.yaml \
+  --cases examples/regex-redaction/cases.yaml
+
+# Bundled custom gate
+uv run --frozen egress-gate \
+  --registry-factory examples.custom-gate.keyword_gate:create_registry \
+  evaluate --policy examples/custom-gate/egress-gate-config.yaml \
+  --cases examples/custom-gate/cases.yaml
+
+# Packaging in a disposable source copy
+uv build
+uv venv /tmp/egress-gate-package-qa/install-venv --python 3.11
+uv pip install --python /tmp/egress-gate-package-qa/install-venv/bin/python \
+  dist/egress_gate-0.1.0-py3-none-any.whl
+/tmp/egress-gate-package-qa/install-venv/bin/egress-gate gates list
+
+
+ +
+

Implemented actions and guardrails

+
    +
  1. Normalized malformed protobuf errors. Decode failures now return stable, content-safe INVALID_ARGUMENT; a raw-wire regression protects the boundary.
    2. +
  2. Repaired the installed-package journey. Source-checkout commands are labeled, the installed quickstart is standalone, and documentation links are absolute.
    3. +
  3. Translated preparation errors precisely. Built-in GateConfigurationError failures map to a cataloged preparation response with relevant next steps.
    4. +
  4. Added safe structural diagnostics. Bounded locations such as pipeline.gates[2].config.scan and safe categories replace generic schema errors.
    5. +
  5. Honored color preferences. NO_COLOR applies to automatic service logging and has regression coverage.
    6. +
  6. Added case context to evaluator failures. The validated case name and already completed safe projections are retained.
    7. +
  7. Documented overload retry and readiness behavior. Operations guidance defines bounded RESOURCE_EXHAUSTED backoff and layered readiness checks.
    8. +
  8. Polished command discovery. --version, exit-0 bare help, complete narrow-width option identifiers, concise command summaries, and stable schema titles are verified.
    9. +
+

Release stance after remediation: no open QA finding blocks controlled trusted-network deployment. Keep the new raw-wire, content-safety, CLI, schema, logging, and documentation checks in the release gate.

+
+ +
+

Scope and limitations

+ +
+
+ + + + diff --git a/projects/privacy-guard/analysis/render_latency_plot.py b/projects/egress-gate/analysis/render_latency_plot.py similarity index 92% rename from projects/privacy-guard/analysis/render_latency_plot.py rename to projects/egress-gate/analysis/render_latency_plot.py index da4ff2b4..1cec7ffc 100644 --- a/projects/privacy-guard/analysis/render_latency_plot.py +++ b/projects/egress-gate/analysis/render_latency_plot.py @@ -1,4 +1,4 @@ -"""Render the Privacy Guard latency proof-of-concept figure as deterministic SVG.""" +"""Render the Egress Gate latency proof-of-concept figure as deterministic SVG.""" from __future__ import annotations @@ -10,13 +10,13 @@ from pathlib import Path _ANALYSIS_DIR = Path(__file__).resolve().parent -_DEFAULT_DATA = _ANALYSIS_DIR / "privacy-guard-latency.csv" +_DEFAULT_DATA = _ANALYSIS_DIR / "egress-gate-latency.csv" _DEFAULT_OUTPUT = ( _ANALYSIS_DIR.parent / "docs" / "assets" / "analysis" - / "privacy-guard-latency-vs-prompt-size.svg" + / "egress-gate-latency-vs-prompt-size.svg" ) _WIDTH = 1200 @@ -47,7 +47,7 @@ class Measurement: observed_at_utc: str prompt_tokens: int - privacy_guard_latency_ms: float + egress_gate_latency_ms: float entity_count: int phase: str openshell_observed_ms: float | None @@ -97,7 +97,7 @@ def main() -> None: completed_turns = [row for row in measurements if row.turn_elapsed_ms is not None] mean_turn_share = statistics.fmean( - row.privacy_guard_latency_ms / row.turn_elapsed_ms + row.egress_gate_latency_ms / row.turn_elapsed_ms for row in completed_turns if row.turn_elapsed_ms is not None ) @@ -118,7 +118,7 @@ def _load_measurements(path: Path) -> list[Measurement]: Measurement( observed_at_utc=row["observed_at_utc"], prompt_tokens=int(row["prompt_tokens"]), - privacy_guard_latency_ms=float(row["privacy_guard_latency_ms"]), + egress_gate_latency_ms=float(row["egress_gate_latency_ms"]), entity_count=int(row["entity_count"]), phase=row["phase"], openshell_observed_ms=_optional_float(row["openshell_observed_ms"]), @@ -139,7 +139,7 @@ def _optional_float(value: str) -> float | None: def _linear_fit(measurements: list[Measurement]) -> LinearFit: x_values = [row.prompt_tokens / 100_000.0 for row in measurements] - y_values = [row.privacy_guard_latency_ms for row in measurements] + y_values = [row.egress_gate_latency_ms for row in measurements] x_mean = statistics.fmean(x_values) y_mean = statistics.fmean(y_values) x_variance = sum((value - x_mean) ** 2 for value in x_values) @@ -173,7 +173,7 @@ def _render_svg(measurements: list[Measurement], fit: LinearFit) -> str: maximum_entities = max(row.entity_count for row in measurements) completed_turns = [row for row in measurements if row.turn_elapsed_ms is not None] mean_turn_share = statistics.fmean( - row.privacy_guard_latency_ms / row.turn_elapsed_ms + row.egress_gate_latency_ms / row.turn_elapsed_ms for row in completed_turns if row.turn_elapsed_ms is not None ) @@ -184,12 +184,12 @@ def _render_svg(measurements: list[Measurement], fit: LinearFit) -> str: f'viewBox="0 0 {_WIDTH} {_HEIGHT}" role="img" ' f'aria-labelledby="title description">' ), - 'Privacy Guard latency versus prompt size', + 'Egress Gate latency versus prompt size', ( - 'Scatter plot of 96 Privacy Guard service ' + 'Scatter plot of 96 Egress Gate service ' "latency measurements from 18 thousand to 1.141 million prompt " "tokens, with one linear fit and a one-million-token threshold. " - f"Privacy Guard averaged {100.0 * mean_turn_share:.2f} percent of " + f"Egress Gate averaged {100.0 * mean_turn_share:.2f} percent of " "end-to-end time across 12 completed turns." ), "", @@ -223,7 +223,7 @@ def _render_svg(measurements: list[Measurement], fit: LinearFit) -> str: "", ( f'' - "Privacy Guard latency (ms) · log scale" + "Egress Gate latency (ms) · log scale" ), ] @@ -280,10 +280,10 @@ def _render_svg(measurements: list[Measurement], fit: LinearFit) -> str: ) parts.append( f'' f"{row.prompt_tokens:,} tokens; " - f"{row.privacy_guard_latency_ms:.1f} ms; " + f"{row.egress_gate_latency_ms:.1f} ms; " f"{row.entity_count} entities detected" ) @@ -293,7 +293,7 @@ def _render_svg(measurements: list[Measurement], fit: LinearFit) -> str: [ ( f'' - f"Privacy Guard averaged {100.0 * mean_turn_share:.2f}%" + f"Egress Gate averaged {100.0 * mean_turn_share:.2f}%" ), ( f' + Inside Egress Gate, the gRPC service adapter is separate from the protobuf-free pipeline processor and request gates. +
The external OpenShell supervisor talks only to the Egress Gate service adapter. The pipeline processor and gates use local domain models.
+ + +## Component ownership + +| Module | Responsibility | +| --- | --- | +| `request.py` | Immutable request, headers, and `RequestMutations` | +| `result.py` | Gate evaluations, five-field findings, provenance, traces, and result invariants | +| `gates/base.py` | Gate lifecycle, capabilities, output validation, and UTF-8 helper | +| `gates/registry.py` | Trusted registration, exact pipeline schema, resources, discovery, and processor preparation | +| `gates/regex.py` | Typed scan and action selection, bounded matching, overlap handling, caching, and body replacement | +| `config.py` | Strict ordered gates and required default decision | +| `request_processor.py` | Shared deadline, immutable snapshot construction, control flow, aggregation, and provenance | +| `service/` | Protobuf validation/conversion, worker slots, lifecycle, and wire serialization | + +The CLI's offline evaluator parses bounded YAML. It uses +`GateRegistry.prepare_processor()` and the production `RequestProcessor`. It +does not add a second execution path or import the transport adapter. + +Only `service/` imports generated protobuf/gRPC bindings. The pipeline processor +and gates receive domain values and can be tested offline. + +## Pipeline execution + +
+ A request moves through pipeline processor controls and an ordered gate pipeline before Egress Gate returns a result. +
Each gate proposes changes to its current snapshot. The pipeline processor builds the next snapshot, the service adapter maps the final mutations, and the OpenShell supervisor applies them.
+
+ +## Trust and state + +Registry factories and custom gate modules are trusted deployment code. +Capabilities mechanically constrain outputs but do not sandbox Python reads. +Prepared gates can use application-owned resources that are safe for concurrent +use. Egress Gate does not close these resources. + +One validated policy and one prepared pipeline processor (`RequestProcessor`) +are active at a time. +Preparation is serialized and a complete candidate is published only after +the shared deadline checks. A failed candidate leaves the existing policy +unchanged. Gate instances are reused across worker threads, so per-request +state must remain local to `evaluate`. + +See [Request lifecycle](request-lifecycle.md) and [Service boundary](service-boundary.md). diff --git a/projects/egress-gate/docs/architecture/request-lifecycle.md b/projects/egress-gate/docs/architecture/request-lifecycle.md new file mode 100644 index 00000000..47e42d56 --- /dev/null +++ b/projects/egress-gate/docs/architecture/request-lifecycle.md @@ -0,0 +1,64 @@ +--- +title: Request lifecycle +description: How one OpenShell evaluation becomes an EgressResult. +agent_markdown: true +--- + +# Request lifecycle + +
+ The Egress Gate service validates an OpenShell request, prepares the policy, creates immutable snapshots for the gate pipeline, and maps the result back to OpenShell. +
The pipeline processor updates local snapshots. The OpenShell supervisor applies final mutations to the intercepted request.
+
+ +## 1. Validate the transport + +The service checks the pre-credentials phase, exact protobuf configuration, +context, target, header, and body bounds. Domain models then enforce bounded +scalar and aggregate values. Invalid input produces a cataloged gRPC failure. + +## 2. Validate and prepare the policy + +The service converts the protobuf `Struct` to a mapping. The sealed +`GateRegistry` validates it as an exact `EgressGateConfig`. The registry then +prepares each configured gate and creates a `RequestProcessor`. Preparation +uses one replacement lock and the request `Timeout`. The service publishes the +candidate only after a final deadline check. + +## 3. Execute the pipeline + +For each configured gate, the Egress Gate pipeline processor: + +1. Check the shared deadline. +2. Pass the current read-only `HttpRequest` snapshot to the gate. +3. Reconstruct and validate the returned `GateEvaluation`. +4. Add a content-safe `GateTrace` and `SourcedFinding` values owned by the + pipeline processor. +5. On `proceed`, validate the request mutations and construct the next request + snapshot. +6. On terminal `allow` or `deny`, stop without invoking later gates. + +The pipeline processor never changes a request object in place. It keeps the +first snapshot private, constructs a new snapshot after each validated mutation +set, and passes that snapshot to the next gate. The final allowed result +combines these mutations in order. A denied result always has an empty mutation +set. Body replacement `None` and `b""` remain distinct. Header mutation variants +use the required `kind` values `write` and `remove`. + +If every gate proceeds, `default_decision` controls the result. Default deny +uses `egress_gate_default_deny`. Default allow has no reason code. + +## 4. Handle pipeline processor limits + +Deadline expiry, worker-slot exhaustion, mutation bounds, finding limits, and +encoded output limits return an atomic deny with source `runtime_limit` and +`egress_gate_limit_exceeded`. No partial mutations or findings are returned. +Gate contract and execution failures remain gRPC failures. + +## 5. Serialize the result + +The Egress Gate service adapter maps the protobuf-free `EgressResult` to +OpenShell's `HttpRequestResult`. It serializes the final body and header +mutations, exactly five finding fields, and no internal provenance. An explicit +empty replacement sets `has_body=true` with an empty body. After an allow, the +OpenShell supervisor applies these mutations to the intercepted request. diff --git a/projects/egress-gate/docs/architecture/service-boundary.md b/projects/egress-gate/docs/architecture/service-boundary.md new file mode 100644 index 00000000..00f3ed4c --- /dev/null +++ b/projects/egress-gate/docs/architecture/service-boundary.md @@ -0,0 +1,69 @@ +--- +title: Service boundary +description: Protobuf conversion, validation, worker scheduling, and lifecycle. +agent_markdown: true +--- + +# Service boundary + +The `service/` package is the only handwritten package that imports OpenShell +protobuf/gRPC bindings. It owns exact encoded wire limits and transport status +mapping. Domain models own protobuf-free invariants. + +The OpenShell supervisor owns the intercepted request. Egress Gate receives its +request data over gRPC and works with local immutable `HttpRequest` snapshots. +The Egress Gate service adapter returns a decision and final mutations; the +supervisor applies allowed mutations to the intercepted request. + +## RPCs + +| RPC | Behavior | +| --- | --- | +| `Describe` | Advertise Egress Gate and the pre-credentials HTTP binding | +| `ValidateConfig` | Validate a complete registry-backed pipeline without publishing it | +| `EvaluateHttpRequest` | Adapt one request, prepare/reuse policy, execute, and serialize | + +The configuration arrives as `google.protobuf.Struct`. The adapter normalizes +safe integral doubles before strict domain validation and rejects oversized +encoded configuration before registry parsing. + +## Shared deadline and workers + +`EvaluateHttpRequest` creates one monotonic `Timeout`. That same deadline is +used for semaphore acquisition, policy preparation, replacement-lock waits, +gate execution, and final result checks. `RequestProcessor.process` accepts the +caller-owned timeout and never creates or stores one. + +Synchronous work runs in a bounded four-slot executor. The gRPC server permits +sixteen concurrent RPCs. Cancellation does not stop Python code that already +runs in a worker. The worker owns its slot until it exits. + +## Wire findings and mutations + +The current OpenShell `Finding` contains exactly `type`, `label`, `count`, +`confidence`, and `severity`. `SourcedFinding.source_gate`, decision sources, +and traces belong to the pipeline processor and are not serialized. Decision +sources use a strict `kind`-discriminated union. The adapter rechecks protobuf +finding and header sizes before returning a response. + +`RequestMutations` is Egress Gate's internal aggregate. A gate returns it with +`proceed` instead of modifying its input. The pipeline processor validates and +applies it to a new local `HttpRequest` snapshot for the next gate. + +At the service boundary, the adapter maps the accumulated +`RequestMutations.replacement_body` to `HttpRequestResult.body` and `has_body`. +It maps each ordered header operation to +`HttpRequestResult.header_mutations`. `None` means no body replacement, while +empty bytes are emitted with `has_body=true`. The OpenShell supervisor applies +these wire mutations after an allow. + +## Lifecycle and errors + +The active policy contains one validated configuration and one prepared +pipeline processor. An equal configuration reuses the active pipeline +processor. The service prepares a changed candidate before it publishes that +candidate. An invalid candidate does not replace the active policy. + +Invalid input maps to `INVALID_ARGUMENT`. Internal gate or service failures map +to `INTERNAL`. A pipeline processor limit denial is not a gRPC failure. It uses +`egress_gate_limit_exceeded`. diff --git a/projects/privacy-guard/docs/assets/analysis/privacy-guard-latency-vs-prompt-size.svg b/projects/egress-gate/docs/assets/analysis/egress-gate-latency-vs-prompt-size.svg similarity index 97% rename from projects/privacy-guard/docs/assets/analysis/privacy-guard-latency-vs-prompt-size.svg rename to projects/egress-gate/docs/assets/analysis/egress-gate-latency-vs-prompt-size.svg index e6b18663..fb18bc60 100644 --- a/projects/privacy-guard/docs/assets/analysis/privacy-guard-latency-vs-prompt-size.svg +++ b/projects/egress-gate/docs/assets/analysis/egress-gate-latency-vs-prompt-size.svg @@ -1,6 +1,6 @@ -Privacy Guard latency versus prompt size -Scatter plot of 96 Privacy Guard service latency measurements from 18 thousand to 1.141 million prompt tokens, with one linear fit and a one-million-token threshold. Privacy Guard averaged 0.56 percent of end-to-end time across 12 completed turns. +Egress Gate latency versus prompt size +Scatter plot of 96 Egress Gate service latency measurements from 18 thousand to 1.141 million prompt tokens, with one linear fit and a one-million-token threshold. Egress Gate averaged 0.56 percent of end-to-end time across 12 completed turns. @@ -20,7 +20,7 @@ text{font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Ar .threshold{stroke:#20262d;stroke-width:2;stroke-dasharray:8 7} @media(prefers-color-scheme:dark){text{fill:#edf2f7}.axis,.annotation{fill:#aeb8c5}.grid{stroke:#52606d}.threshold{stroke:#d8dee5}.point{stroke:#e5e9ee}} -Privacy Guard latency (ms) · log scale +Egress Gate latency (ms) · log scale 10k @@ -142,7 +142,7 @@ text{font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Ar 428,405 tokens; 103.4 ms; 83 entities detected 628,418 tokens; 196.2 ms; 123 entities detected 801,456 tokens; 176.0 ms; 163 entities detected -Privacy Guard averaged 0.56% +Egress Gate averaged 0.56% of end-to-end turn time across 12 completed turns Entities detected diff --git a/projects/privacy-guard/docs/assets/diagrams/component-architecture.svg b/projects/egress-gate/docs/assets/diagrams/component-architecture.svg similarity index 62% rename from projects/privacy-guard/docs/assets/diagrams/component-architecture.svg rename to projects/egress-gate/docs/assets/diagrams/component-architecture.svg index e1013014..0f5db97c 100644 --- a/projects/privacy-guard/docs/assets/diagrams/component-architecture.svg +++ b/projects/egress-gate/docs/assets/diagrams/component-architecture.svg @@ -1,6 +1,6 @@ - Privacy Guard component architecture - Four architecture layers show transport and startup adapters, typed configuration and registry, request processing, and entity-processing engines. Downward arrows show configuration flowing into the request processor and the processor invoking concrete engines through their shared wrapper. + Egress Gate component architecture + All four layers are inside Egress Gate. The Egress Gate gRPC adapter exchanges HttpRequestEvaluation and HttpRequestResult messages with the external OpenShell supervisor. The protobuf-free pipeline processor invokes request gates and builds immutable local snapshots. - + @@ -39,42 +41,42 @@ CLI and gateway config cli.py · gateway_config.py - OpenShell gRPC adapter - service/ · protobuf validation · result serialization + Egress Gate gRPC adapter + HttpRequestEvaluation · HttpRequestResult - TYPED POLICY AND RUNTIME INVENTORY + TYPED POLICY AND GATE INVENTORY Policy configuration - config.py · ordered stages · final action + config.py · ordered gates · default decision - Engine registry - config union · resources · engine construction + Gate registry + gate union · resources · gate construction - REQUEST PROCESSING + PIPELINE PROCESSOR RequestProcessor - stage order · one timeout · aggregation · allow or deny + gate order · one timeout · aggregation · allow or deny - ENTITY PROCESSING - - Engine wrapper · contract and bounds + REQUEST GATES + + Gate wrapper · contract and bounds - RegexEngine - built-in implementation + regex + typed scan · explicit action - Custom engines + Custom gates trusted integrations - - validate config + + validate config validated policy - construct engines - run stages + construct gates + run gates diff --git a/projects/privacy-guard/docs/assets/diagrams/processing-pipeline.svg b/projects/egress-gate/docs/assets/diagrams/processing-pipeline.svg similarity index 58% rename from projects/privacy-guard/docs/assets/diagrams/processing-pipeline.svg rename to projects/egress-gate/docs/assets/diagrams/processing-pipeline.svg index 05d78185..f1f26e99 100644 --- a/projects/privacy-guard/docs/assets/diagrams/processing-pipeline.svg +++ b/projects/egress-gate/docs/assets/diagrams/processing-pipeline.svg @@ -1,94 +1,96 @@ - Privacy Guard processing pipeline - Validated policy and input text produce one detect or replace strategy and shared timeout. Ordered engine stages run and validate their output. Privacy Guard aggregates detections and applies the final detect, block, or replace action. + Egress Gate processing pipeline + The Egress Gate pipeline processor passes an immutable request snapshot to each gate. A gate can propose RequestMutations. The pipeline processor validates them, builds the next snapshot, and accumulates final mutations. The service adapter maps them to HttpRequestResult, and the OpenShell supervisor applies them after an allow. - + 1 Input - Validated policy + UTF-8 text + OpenShell request + policy 2 - Invocation controls - DETECT or REPLACE + one Timeout + Pipeline processor + One shared timeout + bounds - + 3 - Stage 1 - engine.run(current text) + Evaluate gate + gate.evaluate(current snapshot) 4 - Validate result - Text · spans · limits · timeout + Validate output + Control · mutations · findings - text + validated - + 5 - Next stage - Repeat in configured order + Apply control + Build next snapshot or stop 6 - Aggregate - Stage + entity + confidence + Finalize + Findings · provenance · default 7 - Apply the policy action + Return the explicit result - detect - allow original + Decision + allow or deny - block - deny on detection + Final mutations + for HttpRequestResult - replace - allow final text + Findings + bounded + sourced - - repeat until final stage + + proceed · next gate diff --git a/projects/egress-gate/docs/assets/diagrams/request-lifecycle.svg b/projects/egress-gate/docs/assets/diagrams/request-lifecycle.svg new file mode 100644 index 00000000..59918805 --- /dev/null +++ b/projects/egress-gate/docs/assets/diagrams/request-lifecycle.svg @@ -0,0 +1,112 @@ + + Egress Gate request lifecycle + The OpenShell supervisor sends a request to Egress Gate. The service validates the request and policy. The Egress Gate pipeline processor passes immutable local snapshots through the gates and accumulates final mutations. The service returns an HttpRequestResult, and the supervisor applies allowed mutations to the intercepted request. + + + + + + + + + 1 · TRANSPORT + + Receive request + OpenShell evaluation RPC + pre-credentials phase + + + Validate input + phase · request context + target · headers · body + size · encoding bounds + + + + + 2 · CONFIGURATION + + Validate policy + strict policy schema + gates · default decision + + + Prepare pipeline + reuse unchanged policy + prepare changed policy + activate when complete + + + + + 3 · PIPELINE PROCESSOR + + Create snapshot + from OpenShell request + immutable HttpRequest + + + Run gates + gate reads current snapshot + returns RequestMutations + build next snapshot + + + + + 4 · SERVICE BOUNDARY + + Finalize result + allow or deny + final mutations + bounded findings + + + Map to OpenShell + HttpRequestResult + mutations + findings + + + RPC FAILURE + Return a gRPC error + INVALID_ARGUMENT · invalid input + INTERNAL · gate or service failure + + + + + + SUCCESSFUL MIDDLEWARE RESULT + Return to OpenShell supervisor + Allow + final mutations, or deny + Supervisor applies mutations after allow + + diff --git a/projects/privacy-guard/docs/assets/diagrams/request-path.svg b/projects/egress-gate/docs/assets/diagrams/request-path.svg similarity index 51% rename from projects/privacy-guard/docs/assets/diagrams/request-path.svg rename to projects/egress-gate/docs/assets/diagrams/request-path.svg index d19a5a82..60aa7957 100644 --- a/projects/privacy-guard/docs/assets/diagrams/request-path.svg +++ b/projects/egress-gate/docs/assets/diagrams/request-path.svg @@ -1,36 +1,50 @@ - Privacy Guard request path - A provider-bound request travels from a sandbox application through the OpenShell supervisor to Privacy Guard. Privacy Guard returns allow, replacement, or deny. OpenShell attaches credentials only after an allow result and then sends the request to the provider. + Egress Gate request path + The OpenShell supervisor sends an intercepted request and policy to Egress Gate. Egress Gate runs gates that inspect the request and propose mutations, then returns an allow with final mutations or a deny. The supervisor applies allowed mutations, attaches credentials, and sends the request to the provider. - + - - + + Sandbox application Creates a provider-bound HTTP request @@ -38,37 +52,37 @@ request - - + + OpenShell supervisor Routes the request before credentials are attached - body + policy over gRPC + request + policy over gRPC - + PRE-CREDENTIALS - - - - Privacy Guard - Detects entities and applies the configured action + + + + Egress Gate + Runs gates that inspect requests and propose mutations - allow original, replacement, or deny + allow + final mutations, or deny - - + + OpenShell supervisor - Stops denied requests; attaches credentials after allow + Applies mutations · attaches credentials after allow authorized request - - + + Provider diff --git a/projects/egress-gate/docs/configuration.md b/projects/egress-gate/docs/configuration.md new file mode 100644 index 00000000..b0e48cdc --- /dev/null +++ b/projects/egress-gate/docs/configuration.md @@ -0,0 +1,92 @@ +--- +title: Configure policies +description: Configure Egress Gate pipelines and request-level gates. +agent_markdown: true +--- + +# Configure policies + +OpenShell embeds the Egress Gate policy in a `network_middlewares` entry. The +registry validates the complete strict configuration before preparing a +pipeline processor. + +```yaml title="OpenShell policy" +network_middlewares: + egress_gate: + name: Inspect provider requests + middleware: egress-gate + order: 0 + config: + gates: + - name: identifiers + kind: regex + scan: + kind: body + action: + kind: replace + template: '[{entity}]' + pattern_catalog: patterns.yaml + default_decision: allow + on_error: fail_closed + endpoints: + include: [api.anthropic.com] +``` + +Relative catalog paths resolve from the Egress Gate process working directory, +not from the policy file. Use an inline catalog when the process does not have a +stable working directory. + +The Egress Gate policy has two required fields: + +- `gates` contains one through ten named gate configurations. +- `default_decision` is `allow` or `deny`. + +Each gate entry has a unique, bounded `name`. Its literal `kind` field selects +the exact gate type and its remaining fields. The registry rejects unknown +fields, unknown gate types, missing defaults, and duplicate names. + +## Built-in gates + +The shipped registry contains only `regex`. See +[Regex gate](gates/regex.md) for scans, actions, catalogs, and replacement +templates. `scan.kind` selects the body, path, query, or named headers. +`scan.action.kind` selects `detect` or `deny`; a body scan can also select +`replace`. The schema does not permit `replace` for another scan kind. A +trusted application registry supplies other behavior. + +## Inspect the installed registry + +Run these commands from the +[`projects/egress-gate/`](https://github.com/NVIDIA/OpenShell-Research/tree/main/projects/egress-gate) +directory in a source checkout: + +```bash title="Inspect the default registry" +uv run egress-gate gates list +uv run egress-gate gates schema +uv run egress-gate validate --policy path/to/policy.yaml +``` + +Custom registries use the same module attribute for inspection and serving: + +```bash title="Inspect a custom registry" +uv run egress-gate \ + --registry my_gates:registry gates list +uv run egress-gate \ + --registry my_gates:registry gates schema +``` + +The attribute can contain a `GateRegistry` or a zero-argument factory that +returns one. A factory is useful when a deployment must construct typed +`GateResources` dynamically. Policy configuration cannot import Python, choose +a resource implementation, or provide credentials. + +`validate` checks the policy and registered resources. It also reads and checks +a file-backed pattern catalog. It does not construct gates, prepare a +pipeline processor, or change the active policy. Use `evaluate` to check +artifacts that the gate creates during preparation. The gRPC service checks the +exact encoded size of the OpenShell configuration. + +For repeatable request-level checks, the `evaluate` command accepts a pipeline +policy and a strict version-one corpus. It uses the registry's prepared +pipeline processor path and does not start the gRPC service. See +[Test policies offline](evaluation.md). diff --git a/projects/egress-gate/docs/evaluation.md b/projects/egress-gate/docs/evaluation.md new file mode 100644 index 00000000..a6650a28 --- /dev/null +++ b/projects/egress-gate/docs/evaluation.md @@ -0,0 +1,152 @@ +--- +title: Test policies offline +description: Test policy decisions and findings before deployment. +agent_markdown: true +--- + +# Test policies offline + +A policy can be valid and still do the wrong thing. It might allow a request +that you meant to deny, invoke the wrong gate, or stop reporting a finding +after a rule changes. + +`egress-gate evaluate` lets you catch these problems before the policy handles +live traffic. You give it a policy and a set of request examples. Each example +states the result that you expect. Egress Gate runs every request through the +same prepared `RequestProcessor` that the service uses and reports any +difference. + +This is useful when you want to: + +- check a new policy before rollout +- turn a fixed bug into a permanent regression test +- test a custom gate without starting the gRPC service +- compare the behavior of two policy revisions +- build a repeatable request set for a separate performance benchmark + +The command tests correctness. It does not report latency or throughput. Use a +benchmark harness around the same request set when you need performance data. + +## Try the included example + +The repository includes a regex policy and two request cases. Run them from +`projects/egress-gate/`; `uv` prepares the project environment automatically: + +```bash title="Run the example policy tests" +uv run egress-gate evaluate \ + --policy examples/regex-redaction/egress-gate-config.yaml \ + --cases examples/regex-redaction/cases.yaml \ + --timeout-seconds 1 +``` + +The command prepares the policy once, runs each case with a fresh timeout, and +shows whether each request produced its expected result: + +```text title="Evaluation output" +Policy evaluation +Status Case Details +PASS email-is-detected-and-request-is-allowed All checks matched +PASS ordinary-body-is-allowed All checks matched +2 passed · 0 failed · 2 total +``` + +If a case fails, the Details column shows each field that differed and its +expected and actual values. The summary and exit status make the same result +easy to use in CI. + +No request goes to an upstream service. The command does not start gRPC, +attach credentials, or persist request data. + +## Write one test case + +The CLI calls the cases file a *corpus*. In plain terms, it is a versioned YAML +test suite. `version: 1` selects the current file format. You do not need to +manage multiple versions. + +This example checks that the regex policy reports an email finding and then +allows the request through its default decision: + +```yaml title="cases.yaml" +version: 1 +cases: + - name: email-is-detected + provenance: + kind: synthetic + redacted: false + request: + context: + request_id: test-email + sandbox_id: test-sandbox + target: + scheme: https + host: api.example.com + port: 443 + method: POST + path: /v1/messages + query: "" + headers: [] + body: + encoding: utf8 + value: "send alice@example.com" + expected: + decision: allow + finding_types: [regex_match] +``` + +Each case has three parts: + +- `provenance` records whether the request is synthetic or captured and + whether its content is redacted. +- `request` contains the first read-only HTTP request snapshot that the gates + will evaluate. +- `expected` contains the result fields that must match. + +Only `expected.decision` is required. Add more expected fields when they make +the test more useful: + +| Expected field | What it checks | +| --- | --- | +| `decision_source_kind` | Whether a gate, the pipeline default, or a pipeline processor limit made the decision | +| `gate_name` | Which configured gate made a terminal decision | +| `gate_type` | Which gate implementation made a terminal decision | +| `finding_types` | The ordered finding types returned by the pipeline | + +Gate name and gate type apply only when `decision_source_kind` is `gate`. +Omitted fields are not compared. The current evaluator does not compare the +contents of request mutations. + +## Grow the suite with the policy + +Start with one normal request and one request for each important deny or +finding rule. Add a case whenever you fix a policy bug. Keep captured requests +small, deliberate, and redacted when possible. + +Case names must be unique. Optional tags can group cases for external tooling. +The parser also rejects aliases, duplicate keys, unknown fields, invalid +base64, and values that exceed pipeline processor limits. These checks keep tests +repeatable and ensure that test requests follow the same bounds as service +requests. + +Use `--registry` when the policy contains application-owned custom gates: + +```bash title="Test a custom gate" +uv run egress-gate \ + --registry examples.custom-gate.keyword_gate:registry \ + evaluate \ + --policy examples/custom-gate/egress-gate-config.yaml \ + --cases examples/custom-gate/cases.yaml +``` + +## Use the result in automation + +The command uses stable exit statuses: + +| Status | Meaning | +| ---: | --- | +| `0` | Every case matched | +| `1` | One or more cases did not match | +| `2` | The policy, cases, preparation, or execution failed | + +Failure output contains decision metadata and finding types. It does not print +request bodies or raw exception text. This makes the command suitable for CI +logs while keeping request content out of normal output. diff --git a/projects/egress-gate/docs/gates/custom.md b/projects/egress-gate/docs/gates/custom.md new file mode 100644 index 00000000..4fc9a701 --- /dev/null +++ b/projects/egress-gate/docs/gates/custom.md @@ -0,0 +1,164 @@ +--- +title: Custom gates +description: Author and register trusted request-level gates. +agent_markdown: true +--- + +# Custom gates + +Custom gates are trusted application code. They target the protobuf-free +`egress_gate.request` and `egress_gate.result` models and do not import gRPC, +protobuf, or `RequestProcessor` internals. Use the function helper for a small, +stateless gate. Use the class-based API when a gate needs initialization, +helper-base behavior, or operational resources. + +The repository includes runnable examples for both extension styles: + +- [Function-based custom gate](https://github.com/NVIDIA/OpenShell-Research/tree/main/projects/egress-gate/examples/custom-gate) +- [Class-based custom gate](https://github.com/NVIDIA/OpenShell-Research/tree/main/projects/egress-gate/examples/class-based-gate) + +Each example pairs one implementation with a policy and two offline evaluation +cases. Run the function example from `projects/egress-gate/`: + +```bash title="Run the function-based example" +uv run egress-gate \ + --registry examples.custom-gate.keyword_gate:registry \ + evaluate \ + --policy examples/custom-gate/egress-gate-config.yaml \ + --cases examples/custom-gate/cases.yaml +``` + +The executable resolves the explicit `module:attribute` reference from the +working directory. The attribute can contain a registry or a zero-argument +registry factory. A packaged deployment can resolve the same reference from an +installed custom-gate package. + +```python title="examples/custom-gate/keyword_gate.py" +from typing import Literal + +from egress_gate.gates import ( + GateCapability, + GateConfig, + GateRegistry, +) +from egress_gate.request import HttpRequest +from egress_gate.result import GateEvaluation +from egress_gate.timeout import Timeout + + +class KeywordDenyConfig(GateConfig): + kind: Literal["keyword-deny"] + keyword: str + + +registry = GateRegistry(include_builtin_gates=True) + + +@registry.gate( + config=KeywordDenyConfig, + capabilities=frozenset({GateCapability.READ_BODY, GateCapability.DENY}), +) +def keyword_deny( + request: HttpRequest, + config: KeywordDenyConfig, + *, + timeout: Timeout, +) -> GateEvaluation: + timeout.raise_if_expired() + if config.keyword.encode("utf-8") in request.body: + return GateEvaluation.deny("keyword_denied") + return GateEvaluation.proceed() + + +``` + +`registry.gate` creates an ordinary resource-free `Gate` type and adds it to +the application-owned registry. The existing public wrapper still validates +configuration, capabilities, findings, mutations, timeouts, and errors. The +registry stays open while the module declares gates. The CLI or service seals +it automatically on first use. + +On first use, `GateRegistry` constructs the exact discriminated `gates` schema +from the registered config types. A registry factory remains available when a +deployment must construct typed `GateResources` dynamically. Policy +configuration cannot construct or replace those resources. + +`GateConfig` supplies the common required `name` field. Custom config classes +inherit it and do not redefine or alias it. Each config declares one required +literal `kind` and keeps that serialized field name. Nested unions follow the +same discriminator rule. This gives policy parsers and generated schemas one +consistent way to select an exact configuration shape. + +Declare capabilities and finding types accurately. The public wrapper +rejects undeclared body replacements, header mutations, terminal decisions, +and finding types. Read capabilities are discovery metadata. They do not limit +which request fields trusted Python code can read. Keep request state local so +the gate is safe for concurrent calls. + +Declare capabilities as a `frozenset` of `GateCapability` values. Read access, +body replacement, header mutation, terminal allow, and deny are explicit. +Resource use comes from the gate's `GateResources` type, and finding support +comes from `finding_types`, so a gate does not declare either fact twice. + +A custom gate must not edit its `HttpRequest` input. To propose a change, return +`GateEvaluation.proceed(request_mutations=RequestMutations(...))`. The pipeline +processor validates the mutations and creates the next immutable snapshot. + +## Class-based gates + +The function helper does not replace the class-based extension API. Implement +`Gate[ConfigType, ResourcesType]` directly when a gate needs `_initialize`, a +helper base such as `Utf8BodyGate`, or typed `GateResources`. Resource-free +class-based gates use `registry.register(GateType)`. + +```python title="examples/class-based-gate/keyword_gate.py" +from typing import Literal + +from egress_gate.gates import Gate, GateCapability, GateConfig, GateRegistry +from egress_gate.request import HttpRequest +from egress_gate.result import GateEvaluation +from egress_gate.timeout import Timeout + + +class KeywordDenyConfig(GateConfig): + kind: Literal["keyword-deny"] + keyword: str + + +class KeywordDenyGate(Gate[KeywordDenyConfig, None]): + capabilities = frozenset( + {GateCapability.READ_BODY, GateCapability.DENY} + ) + finding_types = () + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + timeout.raise_if_expired() + if self.config.keyword.encode("utf-8") in request.body: + return GateEvaluation.deny("keyword_denied") + return GateEvaluation.proceed() + + +registry = GateRegistry(include_builtin_gates=True) +registry.register(KeywordDenyGate) +``` + +Run the complete class-based example with: + +```bash title="Run the class-based example" +uv run egress-gate \ + --registry examples.class-based-gate.keyword_gate:registry \ + evaluate \ + --policy examples/class-based-gate/egress-gate-config.yaml \ + --cases examples/class-based-gate/cases.yaml +``` + +For a resource-backed gate, define a typed `GateResources` bundle. Pass the +bundle to `registry.register(..., resources=resources)`. Resources are trusted, +application-owned dependencies that must be safe for concurrent use. Policy +configuration can select behavior. It cannot construct clients, provide +credentials, or replace the registered resource implementation. diff --git a/projects/egress-gate/docs/gates/index.md b/projects/egress-gate/docs/gates/index.md new file mode 100644 index 00000000..2db9fa65 --- /dev/null +++ b/projects/egress-gate/docs/gates/index.md @@ -0,0 +1,25 @@ +--- +title: Gates +description: Built-in request matching and trusted custom gates. +agent_markdown: true +--- + +# Gates + +A gate receives a read-only `HttpRequest` snapshot, one shared `Timeout`, and +its exact typed configuration. It cannot change that request object in place. +To change the request, the gate returns `proceed` with a `RequestMutations` +value. + +The pipeline processor validates the mutations and constructs a new read-only +snapshot for the next gate. It also accumulates the mutations that the Egress +Gate service adapter will map to `HttpRequestResult` if the pipeline allows the +request. The OpenShell supervisor then applies those final mutations to the +intercepted request. A gate can instead return terminal `allow` or terminal +`deny` to stop the pipeline. + +The default registry ships exactly `regex`. Application registries can add +trusted custom gates. Egress Gate does not isolate trusted Python gate code. + +- [Regex gate](regex.md) +- [Custom gates](custom.md) diff --git a/projects/egress-gate/docs/gates/regex.md b/projects/egress-gate/docs/gates/regex.md new file mode 100644 index 00000000..4ae31031 --- /dev/null +++ b/projects/egress-gate/docs/gates/regex.md @@ -0,0 +1,108 @@ +--- +title: Regex gate +description: Scan one configured part of a request with bounded regular expressions. +agent_markdown: true +--- + +# Regex gate + +The `regex` gate matches one configured part of the current request. It can +inspect the body, path, query, or selected header values. It returns audit-safe +findings with type `regex_match`. + +Choose what to scan with `scan.kind`, then choose what to do with +`scan.action.kind`. This example replaces matches in the request body: + +```yaml title="Inline regex catalog" +name: customer-identifiers +kind: regex +scan: + kind: body + action: + kind: replace + template: '[{entity}]' +pattern_catalog: + entities: + - name: customer-id + rules: + - name: customer-id-rule + pattern: '\bCUST-[0-9]{8}\b' + confidence: high +``` + +The body is decoded as strict UTF-8. Path and query scans use the exact text in +the request model. A header scan matches each selected header value on its own; +a match cannot span two values. Header names are case-insensitive: + +```yaml title="Selected request headers" +name: labeled-headers +kind: regex +scan: + kind: header + names: [x-customer-note, x-request-label] + action: + kind: deny +pattern_catalog: patterns.yaml +``` + +The header scan sees the current request snapshot, including validated header +mutations from earlier gates. The regex gate does not return header mutations +itself. +OpenShell permits writes only in the `x-openshell-middleware-` namespace, so a +general regex replacement cannot rewrite arbitrary selected headers. A custom +gate can return supported header writes or removals when it declares the +`GateCapability.MUTATE_HEADERS` capability. + +A catalog can be inline or in a relative `.yaml` or `.yml` file. Relative paths +resolve from the Egress Gate process working directory, not from the policy +file. Use an inline catalog when the process does not have a stable working +directory. The gate rejects absolute paths, path traversal, symlinks, YAML +aliases, duplicate keys, invalid body UTF-8, unsafe patterns, and oversized +catalogs. + +Each entity has a stable, bounded name and one or more rules. Rule confidence +is `low`, `medium`, or `high`. Optional flags are `ignore_case`, `multiline`, +`dot_all`, and `ascii`. Do not use named capture groups or inline flags. +Patterns must produce non-empty matches. Findings include overlapping +detections. Replacement uses deterministic, non-overlapping matches. + +## Actions + +| `scan.action.kind` | Match result | +| --- | --- | +| `detect` | `proceed`, findings, no request mutation | +| `deny` | terminal `deny`, findings, `egress_gate_regex_denied` | +| `replace` | `proceed`, findings, explicit body replacement | + +`detect` and `deny` work with every scan kind. `replace` exists only in the +body scan schema. It cannot be configured for a path, query, or header scan. +This structure keeps unsupported combinations out of generated schemas and +editor suggestions. OpenShell middleware results cannot rewrite a request path +or query. Header replacement is not part of the built-in gate. + +The replace action owns its template. It returns a body replacement even when +there is no match. This preserves the operator's explicit intent to replace the +current body. Invalid body UTF-8 is a stable `body_encoding_invalid` service +failure. + +The regex gate does not edit the body in place. It returns the replacement in +`RequestMutations`. The pipeline processor uses it to build the next immutable +`HttpRequest` snapshot. If the pipeline allows the request, Egress Gate includes +the replacement in the final mutations returned to the OpenShell supervisor. + +Replacement templates contain literal text and the `{entity}` field only. +Output size is projected before rendering and is bounded by the advertised +OpenShell body limit. + +## Scan reference + +| `scan.kind` | Additional fields | Supported `action.kind` values | +| --- | --- | --- | +| `body` | none | `detect`, `deny`, `replace` | +| `path` | none | `detect`, `deny` | +| `query` | none | `detect`, `deny` | +| `header` | non-empty `names` list | `detect`, `deny` | + +Configure another regex gate when different request parts need different +catalogs or actions. Keeping one scan per gate makes matches, findings, and +replacement offsets unambiguous. diff --git a/projects/egress-gate/docs/index.md b/projects/egress-gate/docs/index.md new file mode 100644 index 00000000..1f969d05 --- /dev/null +++ b/projects/egress-gate/docs/index.md @@ -0,0 +1,109 @@ +--- +title: Egress Gate +description: Overview and quickstart for request-level OpenShell middleware. +agent_markdown: true +--- + +# Egress Gate + +Egress Gate is extensible OpenShell middleware for applying an ordered pipeline +of gates to outgoing HTTP requests from sandboxes. A gate can inspect a +request, report findings, rewrite supported content, allow it, or deny it. Use +the built-in regex gate or install trusted custom gates for application-specific +checks. + +Egress Gate provides one typed, configurable place for request-level controls. +You can combine gates, test policies offline, and add new gate types without +changing OpenShell. + +OpenShell still owns interception, routing, network policy, and credential +attachment. Egress Gate is not a forward proxy, TLS interceptor, response +filter, or storage control. + +## Quickstart + +From the +[`projects/egress-gate/`](https://github.com/NVIDIA/OpenShell-Research/tree/main/projects/egress-gate) +directory in a source checkout: + +First, inspect the installed gates and their configuration schema: + +```bash title="Explore available gates" +uv run egress-gate gates list +uv run egress-gate gates schema +``` + +Then validate a policy before you use it: + +```bash title="Validate a policy" +uv run egress-gate validate \ + --policy examples/regex-redaction/egress-gate-config.yaml +``` + +Start Egress Gate in the foreground when the policy is ready: + +```bash title="Start the server" +uv run egress-gate serve --listen 127.0.0.1:50051 +``` + +Use the [regex guide](gates/regex.md) for an OpenShell policy and a +regex-pattern "catalog". + +Use [offline policy tests](evaluation.md) to check saved request examples with +the same prepared `RequestProcessor` used by the service. No request goes to an +upstream provider. + +## How a request moves through the pipeline + +1. The OpenShell supervisor sends the intercepted request to Egress Gate. +2. Each gate reads the current read-only `HttpRequest` and can propose + `RequestMutations`. A gate does not modify its input in place. +3. The Egress Gate pipeline processor validates and applies the requested + mutations by creating a new local `HttpRequest`. +4. The next gate receives the updated request. +5. If the pipeline allows the request, the service adapter maps the accumulated + mutations to OpenShell's `HttpRequestResult`. +6. The OpenShell supervisor applies the returned mutations to the intercepted + request before it attaches credentials. + +A denial returns no request mutations. + +
+
The OpenShell supervisor sends an intercepted request to Egress Gate, receives the final decision and mutations, and applies allowed mutations before it attaches credentials. +
Egress Gate evaluates the request. The OpenShell supervisor owns and updates the intercepted request.
+ + +Within Egress Gate, the +[gRPC service adapter](https://github.com/NVIDIA/OpenShell-Research/tree/main/projects/egress-gate/src/egress_gate/service) +is the only code that imports OpenShell's generated protobuf bindings. Gates +and the pipeline processor do not depend on protobuf or gRPC, so the same +policy pipeline can run in offline tests. + +## Core rules + +- A policy has one through ten named gates and a required `default_decision`. +- Each gate receives the current read-only request. +- Each gate returns one control result. `proceed` continues to the next gate; + `allow` and `deny` stop the pipeline. Only `proceed` can include request + mutations. +- `None` body replacement means no replacement. `b""` is an explicit empty + replacement. +- When the pipeline processor reaches a safety limit, Egress Gate denies the + request. The result uses source `runtime_limit` and code + `egress_gate_limit_exceeded`. +- Pipeline default deny uses source `pipeline_default` and code + `egress_gate_default_deny`. +- The released Finding wire contract has only five fields. Gate source and + decision provenance remain internal to the pipeline processor. + +## Further reading + +- [Configuration](configuration.md) +- [Test policies offline](evaluation.md) +- [Operations](operations.md) +- [Gate authoring](gates/custom.md) +- [Regex gate](gates/regex.md) +- [Architecture](architecture/index.md) +- [Request lifecycle](architecture/request-lifecycle.md) +- [Service boundary](architecture/service-boundary.md) +- [Limits and failures](reference/limits-and-failures.md) diff --git a/projects/egress-gate/docs/operations.md b/projects/egress-gate/docs/operations.md new file mode 100644 index 00000000..4ff39f8c --- /dev/null +++ b/projects/egress-gate/docs/operations.md @@ -0,0 +1,163 @@ +--- +title: Run and operate Egress Gate +description: Start, register, observe, and troubleshoot Egress Gate. +agent_markdown: true +--- + +# Run and operate Egress Gate + +The OpenShell gateway and sandbox supervisors call Egress Gate through gRPC. +Run the service from `projects/egress-gate`. `uv run` prepares the project +environment as needed. + +```bash title="Start Egress Gate" +uv run egress-gate gates list +uv run egress-gate gates schema +uv run egress-gate serve --listen 0.0.0.0:50051 --timeout-seconds 4 +``` + +Use a reachable non-loopback address only when the supervisor is outside the +host network namespace of the service. Plaintext gRPC is for a restricted, +trusted network. Do not expose the port to an untrusted network. + +## OpenShell registration + +Before you change the gateway configuration, stop any running OpenShell +gateways that use it. A running gateway does not reload middleware +registrations. + +```bash title="Register Egress Gate" +uv run egress-gate add-gateway-registration \ + --host-ip YOUR_HOST_IPV4 --name egress-gate --port 50051 +``` + +The command updates `OPENSHELL_GATEWAY_CONFIG`, then +`$XDG_CONFIG_HOME/openshell/gateway.toml`, then +`~/.config/openshell/gateway.toml`. Use `--config PATH` for another file. +Start the gateways again with the same commands or service managers that you +normally use. + +To remove a registration, stop any running gateways that use the configuration +again. List the available names with: + +```bash title="List middleware registrations" +uv run egress-gate list-gateway-registrations +``` + +The gateway config does not identify which service owns a registration. The +command therefore lists all external middleware. Use its exact name to remove +the registration you no longer need: + +```bash title="Remove the registration" +uv run egress-gate remove-gateway-registration --name egress-gate +``` + +Start the gateways again after the command completes. + +The generated OpenShell middleware timeout is five seconds. Keep the Egress +Gate `--timeout-seconds` below it so queueing, preparation, and transport have +headroom. + +If the middleware RPC returns gRPC `RESOURCE_EXHAUSTED`, capacity may remain +accounted for briefly while completed RPCs are torn down. The OpenShell gateway +or supervisor should retry the middleware RPC with short, bounded exponential +backoff, for example 5, 10, then 20 milliseconds, while staying inside its +middleware deadline. Do not turn this into an unbounded application-level +retry or replay an outbound request unless its request semantics permit that. + +## Verify readiness + +Egress Gate does not expose a separate gRPC health service. Verify readiness at +the policy, transport, and end-to-end layers instead. The commands below use an +installed `egress-gate` executable; prefix them with `uv run` in a source +checkout. + +1. Validate and evaluate the exact deployment policy before starting the + service: + + ```bash + egress-gate validate --policy /absolute/path/to/policy.yaml + egress-gate evaluate \ + --policy /absolute/path/to/policy.yaml \ + --cases /absolute/path/to/cases.yaml + ``` + +2. Start Egress Gate and wait for the content-safe + `egress_gate_server_bound` log entry. From the OpenShell gateway host or + network namespace, confirm the registered address accepts a TCP connection: + + ```bash + python3 -c 'import socket; socket.create_connection(("EGRESS_GATE_HOST", 50051), timeout=2).close()' + ``` + + This proves transport reachability only; it does not exercise the gRPC + contract or a policy. + +3. After restarting the OpenShell gateway, send one harmless request from a + sandbox whose policy uses the registration. Choose an endpoint explicitly + allowed by that policy: + + ```bash + openshell sandbox exec --name SANDBOX_NAME --no-tty -- \ + curl --fail --silent --show-error https://ALLOWED_TEST_ENDPOINT/health + openshell logs SANDBOX_NAME -n 100 --source sandbox + ``` + + Readiness requires the request to receive its expected allow or deny result + without a middleware connection, timeout, or configuration error. A TCP + check alone is not sufficient. + +## Logging and decisions + +`--debug` enables content-safe diagnostics. Egress Gate does not log request or +replacement bodies. Set `NO_COLOR` to any value to suppress ANSI styling when +default logging writes to an interactive terminal. Application code can still +request colors explicitly with `LoggingConfig(color_mode=ColorMode.ALWAYS)`. + +Successful policy outcomes are distinct from gRPC failures: + +| Outcome | Wire result | +| --- | --- | +| Gate deny | deny, gate-owned reason code | +| Pipeline default deny | deny, `egress_gate_default_deny` | +| Pipeline processor reaches a safety limit | deny, `egress_gate_limit_exceeded` | +| Invalid request or config | gRPC `INVALID_ARGUMENT` | +| Gate or service failure | gRPC `INTERNAL` | + +Results caused by pipeline processor limits contain no partial mutations or +findings. A failed candidate does not replace the active policy. See +[Limits and failures](reference/limits-and-failures.md). + +## Policy rollout + +Each Egress Gate service keeps one active prepared policy. To change the +policy, first stop requests that use the old configuration. Let all admitted +requests finish. Then, send a request that uses the new configuration. Use +separate service instances when different policies must be active at the same +time. + +## Shutdown + +Use Ctrl-C for an interactive process or send `SIGINT` through the service +manager, then wait for the process to exit before replacing it. Egress Gate +stops the gRPC server with zero transport grace and closes its worker resources; +callers with an active RPC may observe cancellation or unavailability and +should follow the bounded retry guidance above. grpcio messages such as +`Got goaway` or `Cancelling all calls` are expected during a planned shutdown +when the process exits normally. Investigate them when they occur outside a +deployment or shutdown window, accompany lost work, or the process does not +exit. + +## Troubleshooting + +Inspect a finite OpenShell log window: + +```bash title="Inspect recent sandbox logs" +openshell status +openshell logs SANDBOX_NAME -n 100 --source sandbox +``` + +Check the request ID and stable error code in content-safe Egress Gate logs. +Reduce request, header, finding, metadata, or regex catalog size when the +limit reason is returned. Check the exact schema with `gates schema` +when validation fails. diff --git a/projects/egress-gate/docs/reference/limits-and-failures.md b/projects/egress-gate/docs/reference/limits-and-failures.md new file mode 100644 index 00000000..2bec400a --- /dev/null +++ b/projects/egress-gate/docs/reference/limits-and-failures.md @@ -0,0 +1,52 @@ +--- +title: Limits and failure behavior +description: Bounded Egress Gate domain and service behavior. +agent_markdown: true +--- + +# Limits and failure behavior + +Limits are fail-closed and content-safe. The `service/` package checks exact +encoded protobuf sizes. Domain models check scalar, aggregate, and result +limits. + +| Area | Limit | +| --- | ---: | +| Request body | 4 MiB | +| Pipeline gates | 10 | +| Finding groups per gate/result | 32 | +| Estimated finding wire size | 4 KiB | +| Result metadata entries | 64 | +| Result metadata aggregate strings | 32 KiB | +| Gate traces per result | 10 | +| Header mutations per gate evaluation | 64 | +| Processing timeout | 30 seconds maximum | +| Concurrent processing slots | 4 | + +Request context and target aggregates, headers, replacement bodies, regex +catalogs, individual patterns, and diagnostic strings have additional bounded +limits in `constants.py`. Tests cover exact accepted boundaries and the first +rejected value. + +## Outcomes + +| Condition | Outcome | +| --- | --- | +| Invalid phase, envelope, policy, or input encoding | gRPC `INVALID_ARGUMENT` | +| Gate contract or unexpected execution failure | gRPC `INTERNAL` | +| Deadline or pipeline processor limit | deny, source `runtime_limit`, code `egress_gate_limit_exceeded` | +| Gate terminal deny | deny, source `gate`, gate-owned reason code | +| Pipeline default deny | deny, source `pipeline_default`, code `egress_gate_default_deny` | +| Pipeline default allow | allow, source `pipeline_default`, no reason code | + +Pipeline processor limit results contain no partial mutations, findings, or +trace details. Failed policy preparation leaves the active policy unchanged. +Stable error catalogs and reason codes never include request content or +arbitrary exception text. + +## Finding contract + +The released OpenShell wire contract has five fields. The pipeline processor's +`SourcedFinding` and `DecisionSource` preserve provenance for internal tests, +traces, and logging only. Do not encode source or attributes into labels or +metadata while the canonical protocol remains five-field. diff --git a/projects/egress-gate/examples/class-based-gate/README.md b/projects/egress-gate/examples/class-based-gate/README.md new file mode 100644 index 00000000..fecbe9d5 --- /dev/null +++ b/projects/egress-gate/examples/class-based-gate/README.md @@ -0,0 +1,118 @@ +# Class-based custom gate + +This example implements the same `keyword-deny` behavior as the +function-based example, but uses the full `Gate` API. Use this form when a gate +needs initialization, a helper base, or typed operational resources. + +The implementation has three pieces: + +1. `KeywordDenyConfig` defines the policy fields and `kind` discriminator. +2. `KeywordDenyGate._evaluate` implements the request decision. +3. The module creates a registry and registers the gate class. + +Run the example from `projects/egress-gate/`. First inspect the registry: + +```bash +uv run egress-gate \ + --registry examples.class-based-gate.keyword_gate:registry \ + gates list +``` + +Then test the policy against two saved requests: + +```bash +uv run egress-gate \ + --registry examples.class-based-gate.keyword_gate:registry \ + evaluate \ + --policy examples/class-based-gate/egress-gate-config.yaml \ + --cases examples/class-based-gate/cases.yaml +``` + +The first case contains the configured keyword and is denied. The second gate +evaluation proceeds, so `default_decision: allow` determines its result. + +## Run it with OpenShell + +Start Egress Gate with this example registry and content-safe debug diagnostics: + +```bash +uv run egress-gate \ + --debug \ + --registry examples.class-based-gate.keyword_gate:registry \ + serve --listen 0.0.0.0:50051 --timeout-seconds 4 +``` + +Before you change the gateway configuration, stop any running OpenShell +gateways that use it. A running gateway does not reload middleware +registrations. + +In another terminal, register the service in your default gateway +configuration. Replace `YOUR_HOST_IPV4` with a non-loopback address that the +gateway and sandbox supervisors can reach. + +```bash +uv run egress-gate add-gateway-registration \ + --host-ip YOUR_HOST_IPV4 \ + --name egress-class \ + --port 50051 +``` + +Start the OpenShell gateway again with the same command or service manager that +you normally use. Then create a sandbox and launch Claude Code: + +```bash +openshell sandbox create \ + --name egress-class \ + --from base \ + --no-auto-providers \ + --policy examples/class-based-gate/policy.yaml \ + -- env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude +``` + +This command uses the base sandbox image, prevents OpenShell from creating or +attaching a provider, and starts Claude Code with nonessential traffic disabled. +The policy can therefore omit telemetry and error-reporting endpoints. + +On the first run, complete Claude Code's browser sign-in from inside the +sandbox. The session uses your Claude subscription directly; OpenShell does not +attach an Anthropic API-key provider. + +At the Claude prompt, enter a normal request: + +```text +Reply with only the word OK. +``` + +Claude should reply normally, and the Egress Gate terminal should record an +allow decision. Then enter a request that contains the configured keyword: + +```text +Reply with only the word SECRET. +``` + +The request must fail before Claude answers. The Egress Gate terminal must +record `action=deny` and `decision_source_kind=gate`. Together, the normal +response and denied request confirm that the class-based gate is active. + +Exit Claude Code and delete the sandbox. Stop any running OpenShell gateways +before you remove the registration. Then start the gateways again: + +```bash +openshell sandbox delete egress-class +uv run egress-gate remove-gateway-registration --name egress-class +``` + +OpenShell names used by this example have a 19-character limit. The chosen +names stay within that limit. + +The base class owns construction and the public `evaluate` wrapper. A custom +class implements `_evaluate` and reads its validated configuration from +`self.config`. Do not override `__init__` or `evaluate`. Use `_initialize` for +reusable derived state. + +This teaching gate searches the body bytes for the UTF-8 encoding of the +configured keyword. It is not a robust content classifier. A production gate +must define its encoding, normalization, and matching behavior. Add limits only +for work that belongs to the gate. Do not put request content in errors or +findings. Check the shared timeout during expensive work, and keep request state +local. diff --git a/projects/egress-gate/examples/class-based-gate/cases.yaml b/projects/egress-gate/examples/class-based-gate/cases.yaml new file mode 100644 index 00000000..4a911910 --- /dev/null +++ b/projects/egress-gate/examples/class-based-gate/cases.yaml @@ -0,0 +1,53 @@ +version: 1 +cases: + - name: configured-keyword-is-denied + tags: [class-based-gate] + provenance: + kind: synthetic + redacted: true + request: + context: + request_id: class-gate-deny + sandbox_id: sandbox-example + target: + scheme: https + host: api.example.com + port: 443 + method: POST + path: /messages + query: "" + headers: [] + body: + encoding: utf8 + value: "do not send this SECRET" + expected: + decision: deny + decision_source_kind: gate + gate_name: block-secret-keyword + gate_type: keyword-deny + finding_types: [] + + - name: other-bodies-proceed-to-the-default + tags: [class-based-gate] + provenance: + kind: synthetic + redacted: true + request: + context: + request_id: class-gate-allow + sandbox_id: sandbox-example + target: + scheme: https + host: api.example.com + port: 443 + method: POST + path: /messages + query: "" + headers: [] + body: + encoding: utf8 + value: "ordinary text" + expected: + decision: allow + decision_source_kind: pipeline_default + finding_types: [] diff --git a/projects/egress-gate/examples/class-based-gate/egress-gate-config.yaml b/projects/egress-gate/examples/class-based-gate/egress-gate-config.yaml new file mode 100644 index 00000000..05186e04 --- /dev/null +++ b/projects/egress-gate/examples/class-based-gate/egress-gate-config.yaml @@ -0,0 +1,5 @@ +gates: + - name: block-secret-keyword + kind: keyword-deny + keyword: SECRET +default_decision: allow diff --git a/projects/egress-gate/examples/class-based-gate/keyword_gate.py b/projects/egress-gate/examples/class-based-gate/keyword_gate.py new file mode 100644 index 00000000..a3eee4bb --- /dev/null +++ b/projects/egress-gate/examples/class-based-gate/keyword_gate.py @@ -0,0 +1,37 @@ +"""A minimal class-based Egress Gate implementation.""" + +from typing import Literal + +from egress_gate.gates import Gate, GateCapability, GateConfig, GateRegistry +from egress_gate.request import HttpRequest +from egress_gate.result import GateEvaluation +from egress_gate.timeout import Timeout + + +class KeywordDenyConfig(GateConfig): + """Policy fields accepted by the custom gate.""" + + kind: Literal["keyword-deny"] + keyword: str + + +class KeywordDenyGate(Gate[KeywordDenyConfig, None]): + """Deny requests whose body contains the configured UTF-8 keyword.""" + + capabilities = frozenset({GateCapability.READ_BODY, GateCapability.DENY}) + finding_types = () + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + timeout.raise_if_expired() + if self.config.keyword.encode("utf-8") in request.body: + return GateEvaluation.deny("keyword_denied") + return GateEvaluation.proceed() + + +registry = GateRegistry(include_builtin_gates=True) +registry.register(KeywordDenyGate) diff --git a/projects/privacy-guard/examples/custom-engine/policy.yaml b/projects/egress-gate/examples/class-based-gate/policy.yaml similarity index 62% rename from projects/privacy-guard/examples/custom-engine/policy.yaml rename to projects/egress-gate/examples/class-based-gate/policy.yaml index 61369d0e..b82b4064 100644 --- a/projects/privacy-guard/examples/custom-engine/policy.yaml +++ b/projects/egress-gate/examples/class-based-gate/policy.yaml @@ -12,7 +12,7 @@ process: network_policies: claude_code: - name: Claude Code subscription access + name: Claude Code access endpoints: - host: api.anthropic.com port: 443 @@ -26,27 +26,21 @@ network_policies: access: full - host: claude.ai port: 443 - - { host: statsig.anthropic.com, port: 443 } - - { host: sentry.io, port: 443 } binaries: - { path: /usr/local/bin/claude } - { path: /usr/bin/node } network_middlewares: - privacy_guard_detect: - name: Detect confidential project names - middleware: privacy-guard-custom-engine + egress_gate_class: + name: Deny the configured keyword + middleware: egress-class order: 0 config: - entity_processing: - stages: - - name: project-names - config: - engine: keyword-tool - entity: confidential-project - keyword: Project Cobalt - on_detection: - action: detect + gates: + - name: block-secret-keyword + kind: keyword-deny + keyword: SECRET + default_decision: allow on_error: fail_closed endpoints: include: diff --git a/projects/egress-gate/examples/custom-gate/README.md b/projects/egress-gate/examples/custom-gate/README.md new file mode 100644 index 00000000..38238df1 --- /dev/null +++ b/projects/egress-gate/examples/custom-gate/README.md @@ -0,0 +1,127 @@ +# Function-based custom gate + +This example adds a `keyword-deny` gate in one Python file. If the configured +keyword occurs in the request body, the gate denies the request. Otherwise, it +returns `proceed`, and the pipeline continues. + +The implementation has three pieces: + +1. `KeywordDenyConfig` defines the exact policy fields and the stable + `kind: keyword-deny` discriminator. +2. `registry.gate` turns the typed `keyword_deny` function into a standard + resource-free gate type and adds it to the application registry. +3. The CLI loads that module-owned registry directly. + +Run the example from `projects/egress-gate/`. First confirm that the custom +gate is installed in this registry: + +```bash +uv run egress-gate \ + --registry examples.custom-gate.keyword_gate:registry \ + gates list +``` + +Then test the policy against two saved requests: + +```bash +uv run egress-gate \ + --registry examples.custom-gate.keyword_gate:registry \ + evaluate \ + --policy examples/custom-gate/egress-gate-config.yaml \ + --cases examples/custom-gate/cases.yaml +``` + +The executable resolves the explicit `module:attribute` reference from the +working directory. The attribute can contain a registry or a zero-argument +registry factory. An installed custom-gate package works the same way. + +The first case contains the configured keyword and is denied. The second gate +evaluation proceeds, so `default_decision: allow` determines its result. + +## Run it with OpenShell + +Start Egress Gate with this example registry and content-safe debug diagnostics: + +```bash +uv run egress-gate \ + --debug \ + --registry examples.custom-gate.keyword_gate:registry \ + serve --listen 0.0.0.0:50051 --timeout-seconds 4 +``` + +Before you change the gateway configuration, stop any running OpenShell +gateways that use it. A running gateway does not reload middleware +registrations. + +In another terminal, register the service in your default gateway +configuration. Replace `YOUR_HOST_IPV4` with a non-loopback address that the +gateway and sandbox supervisors can reach. + +```bash +uv run egress-gate add-gateway-registration \ + --host-ip YOUR_HOST_IPV4 \ + --name egress-function \ + --port 50051 +``` + +Start the OpenShell gateway again with the same command or service manager that +you normally use. Then create a sandbox and launch Claude Code: + +```bash +openshell sandbox create \ + --name egress-function \ + --from base \ + --no-auto-providers \ + --policy examples/custom-gate/policy.yaml \ + -- env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude +``` + +This command uses the base sandbox image, prevents OpenShell from creating or +attaching a provider, and starts Claude Code with nonessential traffic disabled. +The policy can therefore omit telemetry and error-reporting endpoints. + +On the first run, complete Claude Code's browser sign-in from inside the +sandbox. The session uses your Claude subscription directly; OpenShell does not +attach an Anthropic API-key provider. + +At the Claude prompt, enter a normal request: + +```text +Reply with only the word OK. +``` + +Claude should reply normally, and the Egress Gate terminal should record an +allow decision. Then enter a request that contains the configured keyword: + +```text +Reply with only the word SECRET. +``` + +The request must fail before Claude answers. The Egress Gate terminal must +record `action=deny` and `decision_source_kind=gate`. Together, the normal +response and denied request confirm that the custom gate is active. + +Exit Claude Code and delete the sandbox. Stop any running OpenShell gateways +before you remove the registration. Then start the gateways again: + +```bash +openshell sandbox delete egress-function +uv run egress-gate remove-gateway-registration --name egress-function +``` + +OpenShell names used by this example have a 19-character limit. The chosen +names stay within that limit. + +This teaching gate searches the body bytes for the UTF-8 encoding of the +configured keyword. It is not a robust content classifier. The pipeline +processor already checks the `HttpRequest` limits; the gate does not repeat +those checks. + +The decorator is a helper for small, stateless gates. See the runnable +[`class-based-gate`](../class-based-gate/) example when a gate needs reusable +initialization, a helper base, or typed operational resources. + +A production gate must define its encoding, normalization, and matching +behavior. Add limits only for work that belongs to the gate. Do not put request +content in errors or findings. Check the shared timeout during expensive work, +and keep request state local. diff --git a/projects/egress-gate/examples/custom-gate/cases.yaml b/projects/egress-gate/examples/custom-gate/cases.yaml new file mode 100644 index 00000000..53dc2467 --- /dev/null +++ b/projects/egress-gate/examples/custom-gate/cases.yaml @@ -0,0 +1,53 @@ +version: 1 +cases: + - name: configured-keyword-is-denied + tags: [custom-gate] + provenance: + kind: synthetic + redacted: true + request: + context: + request_id: custom-gate-deny + sandbox_id: sandbox-example + target: + scheme: https + host: api.example.com + port: 443 + method: POST + path: /messages + query: "" + headers: [] + body: + encoding: utf8 + value: "do not send this SECRET" + expected: + decision: deny + decision_source_kind: gate + gate_name: block-secret-keyword + gate_type: keyword-deny + finding_types: [] + + - name: other-bodies-proceed-to-the-default + tags: [custom-gate] + provenance: + kind: synthetic + redacted: true + request: + context: + request_id: custom-gate-allow + sandbox_id: sandbox-example + target: + scheme: https + host: api.example.com + port: 443 + method: POST + path: /messages + query: "" + headers: [] + body: + encoding: utf8 + value: "ordinary text" + expected: + decision: allow + decision_source_kind: pipeline_default + finding_types: [] diff --git a/projects/egress-gate/examples/custom-gate/egress-gate-config.yaml b/projects/egress-gate/examples/custom-gate/egress-gate-config.yaml new file mode 100644 index 00000000..05186e04 --- /dev/null +++ b/projects/egress-gate/examples/custom-gate/egress-gate-config.yaml @@ -0,0 +1,5 @@ +gates: + - name: block-secret-keyword + kind: keyword-deny + keyword: SECRET +default_decision: allow diff --git a/projects/egress-gate/examples/custom-gate/keyword_gate.py b/projects/egress-gate/examples/custom-gate/keyword_gate.py new file mode 100644 index 00000000..6cdad089 --- /dev/null +++ b/projects/egress-gate/examples/custom-gate/keyword_gate.py @@ -0,0 +1,39 @@ +"""A minimal custom Egress Gate implementation.""" + +from typing import Literal + +from egress_gate.gates import ( + GateCapability, + GateConfig, + GateRegistry, +) +from egress_gate.request import HttpRequest +from egress_gate.result import GateEvaluation +from egress_gate.timeout import Timeout + + +class KeywordDenyConfig(GateConfig): + """Policy fields accepted by the custom gate.""" + + kind: Literal["keyword-deny"] + keyword: str + + +registry = GateRegistry(include_builtin_gates=True) + + +@registry.gate( + config=KeywordDenyConfig, + capabilities=frozenset({GateCapability.READ_BODY, GateCapability.DENY}), +) +def keyword_deny( + request: HttpRequest, + config: KeywordDenyConfig, + *, + timeout: Timeout, +) -> GateEvaluation: + """Deny requests whose body contains the configured UTF-8 keyword.""" + timeout.raise_if_expired() + if config.keyword.encode("utf-8") in request.body: + return GateEvaluation.deny("keyword_denied") + return GateEvaluation.proceed() diff --git a/projects/egress-gate/examples/custom-gate/policy.yaml b/projects/egress-gate/examples/custom-gate/policy.yaml new file mode 100644 index 00000000..796ca0c9 --- /dev/null +++ b/projects/egress-gate/examples/custom-gate/policy.yaml @@ -0,0 +1,47 @@ +version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + claude_code: + name: Claude Code access + endpoints: + - host: api.anthropic.com + port: 443 + protocol: rest + enforcement: enforce + access: full + - host: platform.claude.com + port: 443 + protocol: rest + enforcement: enforce + access: full + - host: claude.ai + port: 443 + binaries: + - { path: /usr/local/bin/claude } + - { path: /usr/bin/node } + +network_middlewares: + egress_gate_fn: + name: Deny the configured keyword + middleware: egress-function + order: 0 + config: + gates: + - name: block-secret-keyword + kind: keyword-deny + keyword: SECRET + default_decision: allow + on_error: fail_closed + endpoints: + include: + - api.anthropic.com diff --git a/projects/privacy-guard/examples/custom-engine/.gitignore b/projects/egress-gate/examples/regex-redaction/.gitignore similarity index 100% rename from projects/privacy-guard/examples/custom-engine/.gitignore rename to projects/egress-gate/examples/regex-redaction/.gitignore diff --git a/projects/egress-gate/examples/regex-redaction/README.md b/projects/egress-gate/examples/regex-redaction/README.md new file mode 100644 index 00000000..9190b796 --- /dev/null +++ b/projects/egress-gate/examples/regex-redaction/README.md @@ -0,0 +1,103 @@ +# Regex redaction + +This example replaces email addresses and customer IDs in request bodies. The +OpenShell policy applies the built-in `regex` gate to requests for one provider +endpoint. + +Run these commands from `projects/egress-gate/examples/regex-redaction/`. + +## Test the gate + +Inspect the installed gates, then test the standalone policy against two saved +requests: + +```bash +uv run egress-gate gates list +uv run egress-gate evaluate \ + --policy egress-gate-config.yaml \ + --cases cases.yaml +``` + +## Run it with OpenShell + +Start Egress Gate with content-safe debug diagnostics in one terminal. The +working directory contains the pattern catalog referenced by `policy.yaml`. + +```bash +uv run egress-gate --debug serve \ + --listen 0.0.0.0:50051 \ + --timeout-seconds 4 +``` + +Before you change the gateway configuration, stop any running OpenShell +gateways that use it. A running gateway does not reload middleware +registrations. + +In another terminal, add the registration to your default gateway +configuration. Replace `YOUR_HOST_IPV4` with a non-loopback address that the +gateway and sandbox supervisors can reach. + +```bash +uv run egress-gate add-gateway-registration \ + --host-ip YOUR_HOST_IPV4 \ + --name eg-regex \ + --port 50051 +``` + +Start the OpenShell gateway again with the same command or service manager that +you normally use. Then create a sandbox and launch Claude Code: + +```bash +openshell sandbox create \ + --name eg-regex \ + --from base \ + --no-auto-providers \ + --policy policy.yaml \ + -- env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude +``` + +This command uses the base sandbox image, prevents OpenShell from creating or +attaching a provider, and starts Claude Code with nonessential traffic disabled. +The policy can therefore omit telemetry and error-reporting endpoints. + +On the first run, complete Claude Code's browser sign-in from inside the +sandbox. The session uses your Claude subscription directly; OpenShell does not +attach an Anthropic API-key provider. + +At the Claude prompt, enter: + +```text +Reply with exactly this text: alice@example.com CUST-12345678 +``` + +Claude must not receive the original identifiers. Its response should contain +`[email]` and `[customer-id]` instead. The Egress Gate terminal also records an +allow decision with `finding_count=2`, without logging request content. These +two observations confirm that OpenShell called Egress Gate and applied the +replacement before it sent the request to Claude. + +Exit Claude Code, then delete the sandbox when the test is complete: + +```bash +openshell sandbox delete eg-regex +``` + +To remove the example registration, first stop any running OpenShell gateways +that use the configuration. Run this command, then start the gateways again: + +```bash +uv run egress-gate remove-gateway-registration \ + --name eg-regex +``` + +OpenShell names used by this example have a 19-character limit. The chosen +names stay within that limit. + +## What the policy does + +The gate uses `scan.kind: body` with `action.kind: replace`. It strictly +decodes the body as UTF-8, finds catalog matches, and requests a body +replacement. Egress Gate applies that mutation before the request continues. + +Body scans also support `detect` and `deny`. The same gate can detect or deny +matches in the path, query, or selected header values. diff --git a/projects/egress-gate/examples/regex-redaction/cases.yaml b/projects/egress-gate/examples/regex-redaction/cases.yaml new file mode 100644 index 00000000..e9e568c7 --- /dev/null +++ b/projects/egress-gate/examples/regex-redaction/cases.yaml @@ -0,0 +1,50 @@ +version: 1 +cases: + - name: email-is-detected-and-request-is-allowed + tags: [regex, redaction] + provenance: + kind: synthetic + redacted: false + request: + context: + request_id: corpus-email + sandbox_id: sandbox-example + target: + scheme: https + host: api.example.com + port: 443 + method: POST + path: /v1/messages + query: "" + headers: [] + body: + encoding: utf8 + value: "send alice@example.com" + expected: + decision: allow + decision_source_kind: pipeline_default + finding_types: [regex_match] + - name: ordinary-body-is-allowed + tags: [regex] + provenance: + kind: synthetic + redacted: true + request: + context: + request_id: corpus-ordinary + sandbox_id: sandbox-example + target: + scheme: https + host: api.example.com + port: 443 + method: POST + path: /v1/messages + query: "" + headers: [] + body: + encoding: utf8 + value: "ordinary text" + expected: + decision: allow + decision_source_kind: pipeline_default + finding_types: [] diff --git a/projects/egress-gate/examples/regex-redaction/egress-gate-config.yaml b/projects/egress-gate/examples/regex-redaction/egress-gate-config.yaml new file mode 100644 index 00000000..bec57150 --- /dev/null +++ b/projects/egress-gate/examples/regex-redaction/egress-gate-config.yaml @@ -0,0 +1,21 @@ +gates: + - name: identifiers + kind: regex + scan: + kind: body + action: + kind: replace + template: "[{entity}]" + pattern_catalog: + entities: + - name: email + rules: + - name: conventional-email + pattern: '(?=3.11" license = "Apache-2.0" @@ -15,12 +15,13 @@ dependencies = [ "pydantic>=2.11,<3", "pyyaml>=6,<7", "regex>=2026.7.19,<2027", + "rich>=14,<16", "typer>=0.16,<1", "typing-extensions>=4.12,<5", ] [project.scripts] -privacy-guard = "privacy_guard.cli:app" +egress-gate = "egress_gate.cli:app" [project.urls] Repository = "https://github.com/NVIDIA/OpenShell-Research" @@ -43,7 +44,7 @@ testpaths = ["tests"] [tool.ruff] target-version = "py311" -extend-exclude = ["src/privacy_guard/bindings"] +extend-exclude = ["src/egress_gate/bindings"] [tool.ruff.lint] select = ["E", "F", "I", "UP"] @@ -51,7 +52,7 @@ select = ["E", "F", "I", "UP"] [tool.ty.src] # Generated protobuf/gRPC bindings are checked at their handwritten adapter seam; # their generator-owned implementations and incomplete annotations stay excluded. -exclude = ["src/privacy_guard/bindings"] +exclude = ["src/egress_gate/bindings"] [tool.ty.rules] blanket-ignore-comment = "error" diff --git a/projects/privacy-guard/scripts/check.sh b/projects/egress-gate/scripts/check.sh similarity index 90% rename from projects/privacy-guard/scripts/check.sh rename to projects/egress-gate/scripts/check.sh index c82e637d..fcd362a4 100755 --- a/projects/privacy-guard/scripts/check.sh +++ b/projects/egress-gate/scripts/check.sh @@ -16,7 +16,7 @@ fi "${uv_run[@]}" ruff format --check . "${uv_run[@]}" ruff check . "${uv_run[@]}" ty check -"${uv_run[@]}" python -c "import privacy_guard" +"${uv_run[@]}" python -c "import egress_gate" "${uv_run[@]}" pip-audit \ --progress-spinner off \ --local diff --git a/projects/egress-gate/src/egress_gate/__init__.py b/projects/egress-gate/src/egress_gate/__init__.py new file mode 100644 index 00000000..a3f52198 --- /dev/null +++ b/projects/egress-gate/src/egress_gate/__init__.py @@ -0,0 +1 @@ +"""Egress Gate: an OpenShell supervisor middleware. See the package README.""" diff --git a/projects/privacy-guard/src/privacy_guard/base.py b/projects/egress-gate/src/egress_gate/base.py similarity index 100% rename from projects/privacy-guard/src/privacy_guard/base.py rename to projects/egress-gate/src/egress_gate/base.py diff --git a/projects/privacy-guard/src/privacy_guard/bindings/__init__.py b/projects/egress-gate/src/egress_gate/bindings/__init__.py similarity index 100% rename from projects/privacy-guard/src/privacy_guard/bindings/__init__.py rename to projects/egress-gate/src/egress_gate/bindings/__init__.py diff --git a/projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2.py b/projects/egress-gate/src/egress_gate/bindings/supervisor_middleware_pb2.py similarity index 100% rename from projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2.py rename to projects/egress-gate/src/egress_gate/bindings/supervisor_middleware_pb2.py diff --git a/projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2.pyi b/projects/egress-gate/src/egress_gate/bindings/supervisor_middleware_pb2.pyi similarity index 100% rename from projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2.pyi rename to projects/egress-gate/src/egress_gate/bindings/supervisor_middleware_pb2.pyi diff --git a/projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2_grpc.py b/projects/egress-gate/src/egress_gate/bindings/supervisor_middleware_pb2_grpc.py similarity index 100% rename from projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2_grpc.py rename to projects/egress-gate/src/egress_gate/bindings/supervisor_middleware_pb2_grpc.py diff --git a/projects/egress-gate/src/egress_gate/cli.py b/projects/egress-gate/src/egress_gate/cli.py new file mode 100644 index 00000000..fa8e9d1a --- /dev/null +++ b/projects/egress-gate/src/egress_gate/cli.py @@ -0,0 +1,1237 @@ +"""Egress Gate command-line application.""" + +from __future__ import annotations + +import base64 +import binascii +import importlib +import ipaddress +import json +import sys +from collections.abc import Mapping +from dataclasses import dataclass +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Annotated, Literal, Self + +import typer +import yaml +from pydantic import ValidationError, field_validator, model_validator +from rich.console import Console +from rich.panel import Panel +from rich.syntax import Syntax +from rich.table import Table +from rich.text import Text +from yaml.constructor import ConstructorError +from yaml.events import AliasEvent +from yaml.nodes import MappingNode +from yaml.resolver import BaseResolver + +from egress_gate.base import StrictDomainModel +from egress_gate.constants import ( + DEFAULT_TIMEOUT_SECONDS, + MAX_BODY_BYTES, + MAX_EVALUATION_CASE_NAME_BYTES, + MAX_EVALUATION_CASES, + MAX_EVALUATION_FILE_BYTES, + MAX_EVALUATION_TAGS, + MAX_PROTO_FINDING_GROUPS, + MAX_TIMEOUT_SECONDS, +) +from egress_gate.errors import EgressGateError, GateRegistryError +from egress_gate.gates.base import GateCapability +from egress_gate.gates.registry import ( + GateRegistry, + PolicyValidationError, + create_builtin_registry, +) +from egress_gate.gateway_config import ( + MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES, + GatewayConfigError, + GatewayConfigRemoval, + GatewayConfigUpdate, + GatewayMiddlewareRegistration, + default_gateway_config_path, + list_gateway_registrations, + remove_gateway_config, + update_gateway_config, + validate_middleware_name, +) +from egress_gate.logging import LoggingConfig, configure_logging +from egress_gate.request import HttpHeader, HttpRequest, HttpTarget, RequestContext +from egress_gate.result import EgressResult, GateDecisionSource +from egress_gate.string_validators import BoundedMetadataString +from egress_gate.timeout import Timeout, validate_timeout_seconds + +app = typer.Typer( + name="egress-gate", + help=( + "Run the OpenShell middleware, test policies offline, manage the OpenShell " + "gateway registration, and inspect installed gates." + ), + invoke_without_command=True, + no_args_is_help=False, + add_completion=False, + rich_markup_mode=None, +) +gates_app = typer.Typer( + help="Inspect installed gates and the policy schema they accept.", + no_args_is_help=True, + rich_markup_mode=None, +) +app.add_typer( + gates_app, + name="gates", + short_help="Inspect installed gates and policy schema.", +) + + +@app.callback() +def configure_cli( + context: typer.Context, + version_requested: Annotated[ + bool, + typer.Option( + "--version", + help="Show the installed Egress Gate version and exit.", + is_eager=True, + ), + ] = False, + registry: Annotated[ + str | None, + typer.Option( + "--registry", + help=( + "Load a trusted MODULE:ATTRIBUTE containing a GateRegistry or a " + "zero-argument registry factory. This option applies to every command." + ), + ), + ] = None, + debug: Annotated[ + bool, + typer.Option( + "--debug", + help="Log content-safe startup and request diagnostics.", + ), + ] = False, +) -> None: + """Configure the command application and its gate inventory.""" + if version_requested: + _CONSOLE.print(f"egress-gate {_package_version()}") + raise typer.Exit + configure_logging(LoggingConfig(level="DEBUG" if debug else "INFO")) + context.obj = _CommandOptions(registry=_load_registry(registry)) + if context.invoked_subcommand is None: + _CONSOLE.print(context.get_help()) + raise typer.Exit + + +@app.command("serve", short_help="Start the Egress Gate gRPC service.") +def serve( + context: typer.Context, + listen: Annotated[ + str, + typer.Option( + help=( + "Listen address in HOST:PORT form. Use 0.0.0.0 only when sandbox " + "supervisors must connect across a network namespace." + ), + ), + ] = "127.0.0.1:50051", + timeout_seconds: Annotated[ + float, + typer.Option( + help=( + "Total processing time available to all gates for one request. " + f"The value must be greater than 0 and at most {MAX_TIMEOUT_SECONDS:g}." + ), + ), + ] = DEFAULT_TIMEOUT_SECONDS, +) -> None: + """Start the Egress Gate gRPC service and run until shutdown.""" + options = _command_options(context) + from egress_gate.service.server import EgressGateServer + + try: + validated_timeout_seconds = validate_timeout_seconds(timeout_seconds) + except ValueError as error: + raise typer.BadParameter( + str(error), + param_hint="--timeout-seconds", + ) from None + try: + EgressGateServer( + options.registry, + timeout_seconds=validated_timeout_seconds, + ).serve_sync(listen) + except EgressGateError as error: + _render_egress_error("Egress Gate could not start", error) + raise typer.Exit(code=1) from None + + +@app.command( + "add-gateway-registration", + short_help="Register Egress Gate with OpenShell.", +) +def add_gateway_registration( + host_ip: Annotated[ + str, + typer.Option( + help=( + "Non-loopback IPv4 address that the OpenShell gateway and sandbox " + "supervisors can use to reach this Egress Gate service." + ), + ), + ], + config: Annotated[ + Path | None, + typer.Option( + help=( + "OpenShell gateway TOML file to update. By default, use " + "OPENSHELL_GATEWAY_CONFIG, then the standard per-user file." + ), + ), + ] = None, + name: Annotated[ + str, + typer.Option( + help=( + "Registration name used by an OpenShell policy's middleware field. " + "OpenShell allows " + f"1-{MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES} ASCII bytes." + ), + ), + ] = "egress-gate", + port: Annotated[ + int, + typer.Option( + min=1, + max=65535, + help=( + "Port to advertise to OpenShell. It must match the port in the " + "egress-gate serve --listen address." + ), + ), + ] = 50051, +) -> None: + """Add or update Egress Gate in an OpenShell gateway TOML file.""" + try: + address = ipaddress.IPv4Address(host_ip) + except ipaddress.AddressValueError: + raise typer.BadParameter( + "Pass one IPv4 address, for example --host-ip 192.168.1.20.", + param_hint="--host-ip", + ) from None + if address.is_loopback or address.is_unspecified: + raise typer.BadParameter( + "Pass a non-loopback host IPv4 address reachable by sandbox " + "supervisors; do not use 127.0.0.1 or 0.0.0.0.", + param_hint="--host-ip", + ) + try: + validated_name = validate_middleware_name(name) + except GatewayConfigError as error: + raise typer.BadParameter( + str(error), + param_hint="--name", + ) from None + + config_path = config or default_gateway_config_path() + try: + result = update_gateway_config( + config_path, + middleware_name=validated_name, + host_ip=str(address), + port=port, + ) + except GatewayConfigError as error: + _render_cli_error( + "Gateway registration could not be saved", + code="gateway_config_error", + message=str(error), + ) + raise typer.Exit(code=1) from None + + change = { + GatewayConfigUpdate.CREATED: "Created the gateway configuration file", + GatewayConfigUpdate.ADDED: "Added the registration", + GatewayConfigUpdate.UPDATED: "Updated the registration", + GatewayConfigUpdate.UNCHANGED: "Registration was already current", + }[result] + _render_registration( + title="Gateway registration is ready", + config_path=config_path, + name=validated_name, + endpoint=f"http://{address}:{port}", + change=change, + next_step=( + "Start Egress Gate, then restart the OpenShell gateway to load this " + "registration." + ), + ) + + +@app.command( + "list-gateway-registrations", + short_help="List OpenShell middleware registrations.", +) +def list_gateway_registrations_command( + config: Annotated[ + Path | None, + typer.Option( + help=( + "OpenShell gateway TOML file to inspect. By default, use " + "OPENSHELL_GATEWAY_CONFIG, then the standard per-user file." + ), + ), + ] = None, +) -> None: + """List the names and endpoints of registered OpenShell middleware.""" + config_path = config or default_gateway_config_path() + try: + registrations = list_gateway_registrations(config_path) + except GatewayConfigError as error: + _render_cli_error( + "Gateway registrations could not be listed", + code="gateway_config_error", + message=str(error), + ) + raise typer.Exit(code=1) from None + + _render_gateway_registrations(config_path, registrations) + + +@app.command( + "remove-gateway-registration", + short_help="Remove an OpenShell registration.", +) +def remove_gateway_registration( + name: Annotated[ + str, + typer.Option( + help="Exact registration name to remove from the gateway config.", + ), + ], + config: Annotated[ + Path | None, + typer.Option( + help=( + "OpenShell gateway TOML file to update. By default, use " + "OPENSHELL_GATEWAY_CONFIG, then the standard per-user file." + ), + ), + ] = None, +) -> None: + """Remove a named registration from an OpenShell gateway TOML file.""" + config_path = config or default_gateway_config_path() + try: + result = remove_gateway_config( + config_path, + middleware_name=name, + ) + except GatewayConfigError as error: + _render_cli_error( + "Gateway registration could not be removed", + code="gateway_config_error", + message=str(error), + ) + raise typer.Exit(code=1) from None + + if result is GatewayConfigRemoval.REMOVED: + _render_registration( + title="Gateway registration was removed", + config_path=config_path, + name=name, + next_step=("Restart the OpenShell gateway to unload this registration."), + ) + else: + _render_registration( + title="Gateway registration was not found", + config_path=config_path, + name=name, + status_style="bold yellow", + ) + + +@gates_app.command("list") +def list_gates(context: typer.Context) -> None: + """Show what each installed gate can read, change, decide, and report.""" + _render_gates(_command_options(context).registry) + + +@gates_app.command("schema") +def gate_schema(context: typer.Context) -> None: + """Print the complete policy JSON Schema for the installed gates.""" + schema = json.dumps( + _command_options(context).registry.configuration_json_schema(), + indent=2, + ensure_ascii=False, + sort_keys=True, + ) + _CONSOLE.print( + Syntax( + schema, + "json", + theme="ansi_dark", + word_wrap=False, + ), + soft_wrap=True, + ) + + +@app.command("validate", short_help="Check a policy against installed gates.") +def validate_policy( + context: typer.Context, + policy: Annotated[ + Path, + typer.Option( + "--policy", + help="Path to the YAML policy to check.", + ), + ], +) -> None: + """Check a policy without preparing gates or activating the policy.""" + options = _command_options(context) + try: + values = _load_policy(policy) + options.registry.validate_config(values) + except _EvaluationCorpusError: + _render_cli_error( + "Policy validation failed", + code="invalid_input", + message="The policy file could not be read as a supported YAML policy.", + ) + raise typer.Exit(code=1) from None + except PolicyValidationError as error: + _render_cli_error( + "Policy validation failed", + code=error.code.value, + message=(f"Policy field {error.formatted_path}: {error.category.value}."), + hint="Run egress-gate gates schema and correct that field.", + ) + raise typer.Exit(code=1) from None + except EgressGateError: + _render_cli_error( + "Policy validation failed", + code="config_invalid", + message="The policy does not match the schema for the installed gates.", + hint=( + "Run egress-gate gates schema, then check the gate names, kinds, " + "required fields, and pattern catalog." + ), + ) + raise typer.Exit(code=1) from None + _CONSOLE.print("[bold green]✓[/bold green] Policy is valid") + + +@app.command("evaluate", short_help="Test policy cases without starting the service.") +def evaluate( + context: typer.Context, + policy: Annotated[ + Path, + typer.Option( + "--policy", + help="Path to the YAML policy to test.", + ), + ], + cases: Annotated[ + Path, + typer.Option( + "--cases", + help="Path to the YAML file of saved request cases and expected results.", + ), + ], + timeout_seconds: Annotated[ + float, + typer.Option( + help=( + "Maximum seconds for policy preparation and, separately, each case. " + f"The value must be greater than 0 and at most {MAX_TIMEOUT_SECONDS:g}." + ), + ), + ] = DEFAULT_TIMEOUT_SECONDS, +) -> None: + """Test saved requests against a policy without starting the service.""" + options = _command_options(context) + try: + validated_timeout_seconds = validate_timeout_seconds(timeout_seconds) + except ValueError as error: + raise typer.BadParameter( + str(error), + param_hint="--timeout-seconds", + ) from None + try: + policy_values = _load_policy(policy) + except _EvaluationCorpusError: + _render_cli_error( + "Evaluation could not start", + code="invalid_policy_file", + message="The policy file could not be read as a supported YAML policy.", + ) + raise typer.Exit(code=2) from None + try: + corpus = _load_corpus(cases) + except _EvaluationCorpusError: + _render_cli_error( + "Evaluation could not start", + code="invalid_cases_file", + message=( + "The cases file could not be read as a valid version 1 YAML test suite." + ), + ) + raise typer.Exit(code=2) from None + try: + summary = _run_corpus( + options.registry, + policy_values, + corpus, + timeout_seconds=validated_timeout_seconds, + ) + except _CaseExecutionError as error: + if error.completed: + _render_evaluation( + _EvaluationSummary(cases=error.completed), + title="Completed before failure", + ) + failure_title = f"Evaluation failed for case {error.case_name}" + if isinstance(error.cause, EgressGateError): + _render_egress_error(failure_title, error.cause) + else: + _render_cli_error( + failure_title, + code="execution_failed", + message="An unexpected error stopped the evaluation.", + hint="Check the configured gate and its resources, then retry.", + ) + raise typer.Exit(code=2) from None + except EgressGateError as error: + _render_egress_error("Evaluation failed", error) + raise typer.Exit(code=2) from None + except Exception: + _render_cli_error( + "Evaluation failed", + code="execution_failed", + message="An unexpected error stopped the evaluation.", + hint=( + "Check custom gate and application-owned resource setup, then retry." + ), + ) + raise typer.Exit(code=2) from None + + _render_evaluation(summary) + if summary.failed: + raise typer.Exit(code=1) + + +_CONSOLE = Console() +_ERROR_CONSOLE = Console(stderr=True) + + +@dataclass(frozen=True) +class _CommandOptions: + registry: GateRegistry + + +class _EvaluationCorpusError(Exception): + """A content-safe offline policy or corpus input failure.""" + + +class _CaseExecutionError(Exception): + """One case failure plus safe results completed before it.""" + + def __init__( + self, + *, + case_name: str, + completed: tuple[_CaseEvaluation, ...], + cause: Exception, + ) -> None: + self.case_name = case_name + self.completed = completed + self.cause = cause + super().__init__("corpus case execution failed") + + +class _CorpusProvenance(StrictDomainModel): + """Required origin and redaction declaration for one corpus case.""" + + kind: Literal["synthetic", "captured"] + redacted: bool + + +class _CorpusBody(StrictDomainModel): + """One bounded UTF-8 or standard base64 body representation.""" + + encoding: Literal["utf8", "base64"] + value: str + + @model_validator(mode="after") + def _value_is_bounded_and_decodable(self) -> Self: + if self.encoding == "utf8": + try: + if len(self.value.encode("utf-8", errors="strict")) > MAX_BODY_BYTES: + raise ValueError + except UnicodeEncodeError: + raise ValueError("body value is not valid UTF-8") from None + return self + + try: + encoded = self.value.encode("ascii") + decoded = base64.b64decode(encoded, validate=True) + except (UnicodeEncodeError, binascii.Error, ValueError, OverflowError): + raise ValueError("body value is not valid base64") from None + if len(decoded) > MAX_BODY_BYTES: + raise ValueError("decoded body exceeds the size limit") + if base64.b64encode(decoded) != encoded: + raise ValueError("body base64 is not canonical") + return self + + def decode(self) -> bytes: + """Decode the bounded body without exposing input in an error.""" + if self.encoding == "utf8": + return self.value.encode("utf-8") + return base64.b64decode(self.value.encode("ascii"), validate=True) + + +class _CorpusRequest(StrictDomainModel): + """The exact protobuf-free request fields accepted by a corpus case.""" + + context: RequestContext + target: HttpTarget + headers: tuple[HttpHeader, ...] + body: _CorpusBody + + @field_validator("context", mode="before") + @classmethod + def _normalize_context_sequences(cls, value: object) -> object: + if not isinstance(value, Mapping): + return value + context = dict(value) + process = context.get("originating_process") + if isinstance(process, Mapping): + process_values = dict(process) + ancestors = process_values.get("ancestors") + if isinstance(ancestors, list): + process_values["ancestors"] = tuple(ancestors) + context["originating_process"] = process_values + return context + + @field_validator("headers", mode="before") + @classmethod + def _headers_are_a_tuple(cls, value: object) -> object: + if isinstance(value, list): + return tuple(value) + return value + + def to_http_request(self) -> HttpRequest: + """Build the same immutable request value used by the service.""" + return HttpRequest( + context=self.context, + target=self.target, + headers=self.headers, + body=self.body.decode(), + ) + + +class _CorpusExpected(StrictDomainModel): + """Required decision and optional content-safe result projections.""" + + decision: Literal["allow", "deny"] + decision_source_kind: ( + Literal["gate", "pipeline_default", "runtime_limit"] | None + ) = None + gate_name: BoundedMetadataString | None = None + gate_type: BoundedMetadataString | None = None + finding_types: tuple[BoundedMetadataString, ...] | None = None + + @field_validator("finding_types", mode="before") + @classmethod + def _finding_types_are_a_tuple(cls, value: object) -> object: + if isinstance(value, list): + return tuple(value) + return value + + @model_validator(mode="after") + def _optional_fields_are_omitted_or_present(self) -> Self: + for field_name in ( + "decision_source_kind", + "gate_name", + "gate_type", + "finding_types", + ): + if ( + field_name in self.model_fields_set + and getattr(self, field_name) is None + ): + raise ValueError(f"{field_name} must be omitted when not expected") + if ( + self.finding_types is not None + and len(self.finding_types) > MAX_PROTO_FINDING_GROUPS + ): + raise ValueError("too many expected finding types") + gate_fields_present = any( + field_name in self.model_fields_set + for field_name in ("gate_name", "gate_type") + ) + if gate_fields_present and self.decision_source_kind != "gate": + raise ValueError("gate source fields require a gate decision source") + return self + + +class _EvaluationCase(StrictDomainModel): + """One named request, provenance declaration, and expected projection.""" + + name: BoundedMetadataString + tags: tuple[BoundedMetadataString, ...] = () + provenance: _CorpusProvenance + request: _CorpusRequest + expected: _CorpusExpected + + @field_validator("name") + @classmethod + def _name_is_bounded_for_reporting(cls, value: str) -> str: + if len(value.encode("utf-8")) > MAX_EVALUATION_CASE_NAME_BYTES: + raise ValueError("case name exceeds the size limit") + return value + + @field_validator("tags", mode="before") + @classmethod + def _tags_are_a_tuple(cls, value: object) -> object: + if isinstance(value, list): + return tuple(value) + return value + + @model_validator(mode="after") + def _tags_are_bounded(self) -> Self: + if len(self.tags) > MAX_EVALUATION_TAGS: + raise ValueError("case has too many tags") + return self + + +class _EvaluationCorpus(StrictDomainModel): + """Version-one strict bounded corpus document.""" + + version: Literal[1] + cases: tuple[_EvaluationCase, ...] + + @field_validator("cases", mode="before") + @classmethod + def _cases_are_bounded_tuple(cls, value: object) -> object: + if not isinstance(value, list | tuple) or not value: + raise ValueError("corpus cases must be a non-empty list") + if len(value) > MAX_EVALUATION_CASES: + raise ValueError("corpus has too many cases") + return tuple(value) + + @model_validator(mode="after") + def _case_names_are_unique(self) -> Self: + names = tuple(case.name for case in self.cases) + if len(names) != len(set(names)): + raise ValueError("corpus case names must be unique") + return self + + +@dataclass(frozen=True) +class _FieldDifference: + """One stable expected-versus-actual projection difference.""" + + field: str + expected: object + actual: object + + +@dataclass(frozen=True) +class _CaseEvaluation: + """Content-safe result for one corpus case.""" + + name: str + differences: tuple[_FieldDifference, ...] + + @property + def matched(self) -> bool: + return not self.differences + + +@dataclass(frozen=True) +class _EvaluationSummary: + """Stable aggregate for an offline corpus run.""" + + cases: tuple[_CaseEvaluation, ...] + + @property + def total(self) -> int: + return len(self.cases) + + @property + def passed(self) -> int: + return sum(case.matched for case in self.cases) + + @property + def failed(self) -> int: + return self.total - self.passed + + +class _StrictEvaluationLoader(yaml.SafeLoader): + """Safe YAML loader that rejects aliases and duplicate mapping keys.""" + + def compose_node( + self, + parent: object, + index: object, + ) -> yaml.Node: + if self.check_event(AliasEvent): + raise ConstructorError( + None, + None, + "YAML aliases are not supported", + self.peek_event().start_mark, + ) + return super().compose_node(parent, index) + + +def _construct_unique_mapping( + loader: _StrictEvaluationLoader, + node: MappingNode, + deep: bool = False, +) -> dict[object, object]: + loader.flatten_mapping(node) + mapping: dict[object, object] = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + try: + duplicate = key in mapping + except TypeError: + raise ConstructorError( + "while constructing a mapping", + node.start_mark, + "found an unhashable key", + key_node.start_mark, + ) from None + if duplicate: + raise ConstructorError( + "while constructing a mapping", + node.start_mark, + "found a duplicate key", + key_node.start_mark, + ) + mapping[key] = loader.construct_object(value_node, deep=deep) + return mapping + + +_StrictEvaluationLoader.add_constructor( + BaseResolver.DEFAULT_MAPPING_TAG, + _construct_unique_mapping, +) + + +def _load_policy(path: Path) -> Mapping[str, object]: + """Load one bounded strict YAML policy.""" + values = _load_yaml(path) + if not isinstance(values, Mapping): + raise _EvaluationCorpusError + policy_values: dict[str, object] = {} + for key, value in values.items(): + if not isinstance(key, str): + raise _EvaluationCorpusError + policy_values[key] = value + return policy_values + + +def _load_corpus(path: Path) -> _EvaluationCorpus: + """Load and validate one version-one strict YAML corpus.""" + values = _load_yaml(path) + try: + return _EvaluationCorpus.model_validate(values) + except (TypeError, ValueError, ValidationError): + raise _EvaluationCorpusError from None + + +def _run_corpus( + registry: GateRegistry, + policy_values: Mapping[str, object], + corpus: _EvaluationCorpus, + *, + timeout_seconds: float, +) -> _EvaluationSummary: + """Prepare once, then evaluate every case with a fresh shared timeout.""" + validated_timeout = validate_timeout_seconds(timeout_seconds) + validated_config = registry.validate_config(policy_values) + processor = registry.prepare_processor( + validated_config, + timeout=Timeout.from_seconds(validated_timeout), + ) + evaluations: list[_CaseEvaluation] = [] + for case in corpus.cases: + try: + result = processor.process( + case.request.to_http_request(), + timeout=Timeout.from_seconds(validated_timeout), + ) + except Exception as error: + raise _CaseExecutionError( + case_name=case.name, + completed=tuple(evaluations), + cause=error, + ) from None + evaluations.append( + _CaseEvaluation( + name=case.name, + differences=_compare_result(case.expected, result), + ) + ) + return _EvaluationSummary(cases=tuple(evaluations)) + + +def _render_evaluation( + summary: _EvaluationSummary, + *, + title: str = "Policy evaluation", +) -> None: + """Render content-safe case results and their aggregate.""" + table = Table( + title=title, + box=None, + pad_edge=False, + padding=(0, 2), + header_style="bold cyan", + title_style="bold", + title_justify="left", + ) + table.add_column("Status", no_wrap=True) + table.add_column("Case", ratio=2) + table.add_column("Details", ratio=3) + + for evaluation in summary.cases: + if evaluation.matched: + status = Text("PASS", style="bold green") + details = Text("All checks matched", style="dim") + else: + status = Text("FAIL", style="bold red") + details = Text() + for index, difference in enumerate(evaluation.differences): + if index: + details.append("\n") + details.append(f"{difference.field}: ", style="bold") + details.append("expected ", style="dim") + details.append(_format_value(difference.expected)) + details.append(" · actual ", style="dim") + details.append(_format_value(difference.actual)) + table.add_row(status, Text(evaluation.name), details) + + _CONSOLE.print(table) + _CONSOLE.print( + Text.assemble( + (f"{summary.passed} passed", "bold green"), + " · ", + ( + f"{summary.failed} failed", + "bold red" if summary.failed else "dim", + ), + " · ", + (f"{summary.total} total", "dim"), + ) + ) + + +def _render_gates(registry: GateRegistry) -> None: + """Render the installed gate inventory for a person.""" + _CONSOLE.print("[bold]Installed gates[/bold]") + for description in registry.describe_gates(): + finding_types = ( + ", ".join(item.type for item in description.finding_types) + or "None declared" + ) + request_access = ", ".join( + label + for capability, label in _REQUEST_ACCESS_LABELS.items() + if capability in description.capabilities + ) + possible_result_labels = [ + label + for capability, label in _MUTATION_CAPABILITY_LABELS.items() + if capability in description.capabilities + ] + if description.finding_types: + possible_result_labels.append("findings") + possible_result_labels.extend( + label + for capability, label in _DECISION_CAPABILITY_LABELS.items() + if capability in description.capabilities + ) + possible_results = ", ".join(possible_result_labels) + details = Table.grid(padding=(0, 2)) + details.add_column(style="bold cyan", no_wrap=True) + details.add_column() + details.add_row("Description", Text(description.description)) + details.add_row("Request access", Text(request_access or "None declared")) + details.add_row( + "Possible results", + Text(possible_results or "None declared"), + ) + details.add_row("Finding types", Text(finding_types)) + details.add_row("Python config", Text(description.config_type)) + if description.resource_type is not None: + details.add_row("Python resources", Text(description.resource_type)) + _CONSOLE.print( + Panel( + details, + title=Text(description.gate_type, style="bold green"), + title_align="left", + border_style="bright_blue", + ) + ) + + +def _render_registration( + *, + title: str, + config_path: Path, + name: str, + endpoint: str | None = None, + change: str | None = None, + next_step: str | None = None, + status_style: str = "bold green", +) -> None: + """Render one gateway registration outcome and its relevant values.""" + _CONSOLE.print(Text(title, style=status_style)) + details = Table.grid(padding=(0, 2)) + details.add_column(style="bold cyan", no_wrap=True) + details.add_column(overflow="fold") + details.add_row("Gateway file", Text(str(config_path))) + details.add_row("Registration", Text(name)) + if endpoint is not None: + details.add_row("Endpoint", Text(endpoint)) + if change is not None: + details.add_row("Change", Text(change)) + _CONSOLE.print(details) + if next_step is not None: + _CONSOLE.print(Text.assemble(("Next: ", "bold"), next_step)) + + +def _render_gateway_registrations( + config_path: Path, + registrations: tuple[GatewayMiddlewareRegistration, ...], +) -> None: + """Render the middleware names that can be passed to the remove command.""" + _CONSOLE.print("[bold]OpenShell middleware registrations[/bold]") + _CONSOLE.print(Text.assemble(("Gateway file: ", "bold cyan"), str(config_path))) + if not registrations: + _CONSOLE.print("No middleware registrations found.") + return + + table = Table(box=None, pad_edge=False, padding=(0, 2), header_style="bold cyan") + table.add_column("Name", style="bold", no_wrap=True) + table.add_column("Endpoint", overflow="fold") + for registration in registrations: + table.add_row(registration.name, registration.endpoint or "Not set") + _CONSOLE.print(table) + _CONSOLE.print( + Text.assemble( + ("Remove one: ", "bold"), + "egress-gate remove-gateway-registration --name NAME", + ) + ) + + +def _render_egress_error(title: str, error: EgressGateError) -> None: + """Render one cataloged error without internal component terminology.""" + _render_cli_error( + title, + code=error.code.value, + message=error.summary, + hint=error.hint, + ) + + +def _render_cli_error( + title: str, + *, + code: str, + message: str, + hint: str | None = None, +) -> None: + """Render a concise content-safe CLI failure.""" + heading = Text(title, style="bold red") + heading.append(f" [{code}]", style="dim") + _ERROR_CONSOLE.print(heading) + _ERROR_CONSOLE.print(Text(message)) + if hint is not None: + _ERROR_CONSOLE.print(Text.assemble(("Next: ", "bold"), hint)) + + +_REQUEST_ACCESS_LABELS = { + GateCapability.READ_TARGET: "target", + GateCapability.READ_CONTEXT: "request context", + GateCapability.READ_HEADERS: "headers", + GateCapability.READ_BODY: "body", +} +_MUTATION_CAPABILITY_LABELS = { + GateCapability.REPLACE_BODY: "body replacement", + GateCapability.MUTATE_HEADERS: "header changes", +} +_DECISION_CAPABILITY_LABELS = { + GateCapability.ALLOW: "allow decision", + GateCapability.DENY: "deny decision", +} + + +def _load_yaml(path: Path) -> object: + try: + with path.open("rb") as source: + contents = source.read(MAX_EVALUATION_FILE_BYTES + 1) + if len(contents) > MAX_EVALUATION_FILE_BYTES: + raise ValueError + text = contents.decode("utf-8", errors="strict") + return yaml.load(text, Loader=_StrictEvaluationLoader) + except ( + OSError, + RecursionError, + UnicodeError, + ValueError, + yaml.YAMLError, + ): + raise _EvaluationCorpusError from None + + +def _compare_result( + expected: _CorpusExpected, + result: EgressResult, +) -> tuple[_FieldDifference, ...]: + source = result.decision_source + actual: dict[str, object] = { + "decision": result.decision.value, + "decision_source_kind": source.kind.value, + "gate_name": source.gate_name + if isinstance(source, GateDecisionSource) + else None, + "gate_type": source.gate_type + if isinstance(source, GateDecisionSource) + else None, + "finding_types": tuple(item.finding.type for item in result.findings), + } + expected_values: dict[str, object] = {"decision": expected.decision} + for field_name in ( + "decision_source_kind", + "gate_name", + "gate_type", + "finding_types", + ): + if field_name in expected.model_fields_set: + expected_values[field_name] = getattr(expected, field_name) + + differences: list[_FieldDifference] = [] + for field_name in ( + "decision", + "decision_source_kind", + "gate_name", + "gate_type", + "finding_types", + ): + if field_name not in expected_values: + continue + expected_value = expected_values[field_name] + actual_value = actual[field_name] + if expected_value != actual_value: + differences.append( + _FieldDifference( + field=field_name, + expected=expected_value, + actual=actual_value, + ) + ) + return tuple(differences) + + +def _format_value(value: object) -> str: + if isinstance(value, tuple): + value = list(value) + return json.dumps( + value, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + +def _load_registry(reference: str | None) -> GateRegistry: + if reference is None: + registry = create_builtin_registry() + registry.configuration_json_schema() + return registry + module_name, separator, attribute_name = reference.partition(":") + if not separator or not module_name or not attribute_name: + raise typer.BadParameter( + "Use MODULE:ATTRIBUTE, for example my_gates:registry.", + param_hint="--registry", + ) + working_directory = str(Path.cwd()) + if working_directory not in sys.path: + sys.path.insert(0, working_directory) + try: + module = importlib.import_module(module_name) + except Exception: + raise typer.BadParameter( + "Could not import the registry module. Check MODULE:ATTRIBUTE and the " + "module's dependencies.", + param_hint="--registry", + ) from None + try: + candidate = getattr(module, attribute_name) + except Exception: + raise typer.BadParameter( + "Could not find the registry attribute. Check the attribute name in " + "MODULE:ATTRIBUTE.", + param_hint="--registry", + ) from None + if isinstance(candidate, GateRegistry): + registry = candidate + elif callable(candidate): + try: + registry = candidate() + except Exception: + raise typer.BadParameter( + "The registry factory raised an exception. Run it directly to inspect " + "the startup failure.", + param_hint="--registry", + ) from None + else: + raise typer.BadParameter( + "The registry attribute must be a GateRegistry or a zero-argument factory.", + param_hint="--registry", + ) + if not isinstance(registry, GateRegistry): + raise typer.BadParameter( + "The registry factory must return a GateRegistry.", + param_hint="--registry", + ) + try: + registry.configuration_json_schema() + except GateRegistryError: + raise typer.BadParameter( + "The registry could not prepare its policy schema. Register at least one " + "valid gate before loading it.", + param_hint="--registry", + ) from None + return registry + + +def _package_version() -> str: + try: + return version("egress-gate") + except PackageNotFoundError: + return "unknown" + + +def _command_options(context: typer.Context) -> _CommandOptions: + options = context.obj + if not isinstance(options, _CommandOptions): + raise RuntimeError("Egress Gate command context is unavailable") + return options + + +if __name__ == "__main__": + app() + + +__all__ = ["app"] diff --git a/projects/egress-gate/src/egress_gate/config.py b/projects/egress-gate/src/egress_gate/config.py new file mode 100644 index 00000000..4110deec --- /dev/null +++ b/projects/egress-gate/src/egress_gate/config.py @@ -0,0 +1,68 @@ +"""Strict Egress Gate policy configuration.""" + +from __future__ import annotations + +from enum import StrEnum +from typing import Generic, TypeVar + +from pydantic import Field, field_validator + +from egress_gate.base import StrictDomainModel +from egress_gate.constants import MAX_PIPELINE_GATES +from egress_gate.gates.base import GateConfig +from egress_gate.string_validators import validate_scalar_string + + +class DefaultDecision(StrEnum): + """Pipeline disposition when every gate proceeds.""" + + ALLOW = "allow" + DENY = "deny" + + +_GateConfigT = TypeVar("_GateConfigT", bound=GateConfig) + + +class EgressGateConfig(StrictDomainModel, Generic[_GateConfigT]): + """Flat policy with ordered named gates and a required fallback decision.""" + + gates: tuple[_GateConfigT, ...] = Field( + min_length=1, + max_length=MAX_PIPELINE_GATES, + description="Ordered gate configurations. Each gate name must be unique.", + repr=False, + ) + default_decision: DefaultDecision = Field( + description="Decision used when every configured gate proceeds." + ) + + @field_validator("gates", mode="before") + @classmethod + def _gates_are_bounded_tuple(cls, value: object) -> object: + if not isinstance(value, list | tuple): + raise ValueError("policy gates must be a non-empty list") + return tuple(value) + + @field_validator("gates") + @classmethod + def _gate_names_are_unique( + cls, + value: tuple[_GateConfigT, ...], + ) -> tuple[_GateConfigT, ...]: + names = tuple(gate.name for gate in value) + if len(names) != len(set(names)): + raise ValueError("policy gate names must be unique") + return value + + @field_validator("default_decision", mode="before") + @classmethod + def _parse_default_decision(cls, value: object) -> DefaultDecision: + if isinstance(value, DefaultDecision): + return value + return DefaultDecision(validate_scalar_string(value)) + + +__all__ = [ + "DefaultDecision", + "EgressGateConfig", +] diff --git a/projects/privacy-guard/src/privacy_guard/constants.py b/projects/egress-gate/src/egress_gate/constants.py similarity index 53% rename from projects/privacy-guard/src/privacy_guard/constants.py rename to projects/egress-gate/src/egress_gate/constants.py index 4fbda0c8..06716351 100644 --- a/projects/privacy-guard/src/privacy_guard/constants.py +++ b/projects/egress-gate/src/egress_gate/constants.py @@ -1,7 +1,7 @@ -"""Package-wide Privacy Guard constants and operational limits. +"""Package-wide Egress Gate constants and operational limits. Keep this module dependency-free within the package: it must not import from -``privacy_guard``. +``egress_gate``. """ from __future__ import annotations @@ -14,34 +14,46 @@ MAX_TIMEOUT_SECONDS = 30.0 # Middleware identity and stable response values. -SERVICE_NAME = "privacy-guard" -SERVICE_VERSION = version("privacy-guard") -BLOCK_REASON = "Privacy Guard blocked the request" -BLOCK_REASON_CODE = "privacy_guard_blocked" +SERVICE_NAME = "egress-gate" +SERVICE_VERSION = version("egress-gate") +BLOCK_REASON = "Egress Gate blocked the request" +DEFAULT_DENY_REASON_CODE = "egress_gate_default_deny" LIMIT_REASON = ( - "Privacy Guard exceeded a processing safety limit. Check Privacy Guard logs " + "Egress Gate exceeded a processing safety limit. Check Egress Gate logs " "for the limit kind. Reduce the request or replacement size, simplify the " - "configured stages and rules, or increase the processing timeout with " + "configured gates and rules, or increase the processing timeout with " "--timeout-seconds or " - "PrivacyGuardServer(timeout_seconds=...) to at most " + "EgressGateServer(timeout_seconds=...) to at most " f"{MAX_TIMEOUT_SECONDS:g} seconds. If increasing it, give OpenShell's " "middleware timeout additional headroom for queueing and configuration " "preparation, then retry." ) -LIMIT_REASON_CODE = "privacy_guard_limit_exceeded" +LIMIT_REASON_CODE = "egress_gate_limit_exceeded" # Text input limits. MAX_BODY_BYTES = 4 * 1024 * 1024 +MAX_EVALUATION_FILE_BYTES = 16 * 1024 * 1024 +MAX_EVALUATION_CASES = 256 +MAX_EVALUATION_CASE_NAME_BYTES = 128 +MAX_EVALUATION_TAGS = 16 -# Engine and result limits. -MAX_DETECTIONS_PER_STAGE = 256 -MAX_DETECTIONS_PER_REQUEST = 4096 +# Gate and result limits. +MAX_DETECTIONS_PER_GATE = 256 MAX_DIAGNOSTIC_TEXT_BYTES = 1024 -MAX_FINDING_METADATA_ENTRIES = 32 MAX_PROTO_FINDING_GROUPS = 32 MAX_PROTO_FINDING_BYTES = 4 * 1024 +MAX_FINDING_COUNT = 2**32 - 1 +MAX_RESULT_METADATA_ENTRIES = 64 +MAX_RESULT_METADATA_BYTES = 32 * 1024 +MAX_GATE_TRACES = 10 +MAX_TRACE_MUTATION_KINDS = 2 -# Engine configuration and regex execution limits. -MAX_ENTITY_PROCESSING_STAGES = 10 +# Domain request-mutation limits mirrored from the OpenShell middleware +# contract. Encoded protobuf size remains a service-boundary concern. +MAX_HEADER_MUTATIONS = 64 +MAX_HEADER_MUTATION_DATA_BYTES = 32 * 1024 + +# Pipeline configuration and regex execution limits. +MAX_PIPELINE_GATES = 10 MAX_REGEX_NAME_BYTES = 128 MAX_REGEX_ENTITIES_PER_CATALOG = 2_000 MAX_REGEX_RULES_PER_CATALOG = 10_000 @@ -49,14 +61,10 @@ MAX_REGEX_CATALOG_FILE_BYTES = 16 * 1024 * 1024 MAX_REGEX_CATALOG_PATH_BYTES = 1024 -# Prepared-state cache budget. The entry-count cap remains a secondary guard. -MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES = 32 * 1024 * 1024 -REGEX_COMPILED_RULE_WEIGHT_BYTES = 4 * 1024 - # Service concurrency and transport limits. MAX_CONCURRENT_PROCESSING = 4 MAX_CONCURRENT_RPCS = 16 -# Mirrored from the encoded OpenShell v0.0.90 middleware contract. +# Mirrored from the encoded OpenShell middleware contract. MAX_PROTO_CONTEXT_BYTES = 4 * 1024 MAX_PROTO_CONFIG_BYTES = 64 * 1024 MAX_PROTO_TARGET_BYTES = 32 * 1024 @@ -66,4 +74,4 @@ MAX_RECEIVE_MESSAGE_BYTES = MAX_BODY_BYTES + PROTOBUF_ENVELOPE_ALLOWANCE_BYTES # Protocol validation values. -REASON_CODE_PATTERN = re.compile(r"[a-z][a-z0-9_]{0,63}\Z") +REASON_CODE_PATTERN = re.compile(r"\A[a-z][a-z0-9_]{0,63}\Z") diff --git a/projects/privacy-guard/src/privacy_guard/errors.py b/projects/egress-gate/src/egress_gate/errors.py similarity index 60% rename from projects/privacy-guard/src/privacy_guard/errors.py rename to projects/egress-gate/src/egress_gate/errors.py index 84e58be7..3d99f5cf 100644 --- a/projects/privacy-guard/src/privacy_guard/errors.py +++ b/projects/egress-gate/src/egress_gate/errors.py @@ -1,11 +1,11 @@ -"""Content-safe failures shared across Privacy Guard trust boundaries.""" +"""Content-safe failures shared across Egress Gate trust boundaries.""" from __future__ import annotations from dataclasses import dataclass from enum import StrEnum -from privacy_guard.constants import MAX_PROTO_CONFIG_BYTES, MAX_TIMEOUT_SECONDS +from egress_gate.constants import MAX_PROTO_CONFIG_BYTES, MAX_TIMEOUT_SECONDS class ErrorKind(StrEnum): @@ -16,10 +16,10 @@ class ErrorKind(StrEnum): class ErrorComponent(StrEnum): - """The Privacy Guard component responsible for a failure.""" + """The Egress Gate component responsible for a failure.""" CONFIG = "config" - ENGINE = "engine" + GATE = "gate" PROCESSOR = "processor" SERVICE = "service" SERVER = "server" @@ -29,28 +29,19 @@ class ErrorCode(StrEnum): """Stable identifiers for cataloged production failures.""" CONFIG_INVALID = "config_invalid" + CONFIG_PREPARATION_FAILED = "config_preparation_failed" + REQUEST_PROTOBUF_INVALID = "request_protobuf_invalid" REQUEST_PHASE_INVALID = "request_phase_invalid" REQUEST_ENVELOPE_INVALID = "request_envelope_invalid" REQUEST_BODY_TOO_LARGE = "request_body_too_large" BODY_ENCODING_INVALID = "body_encoding_invalid" - ENGINE_OUTPUT_INVALID = "engine_output_invalid" - ENGINE_EXECUTION_FAILED = "engine_execution_failed" + GATE_OUTPUT_INVALID = "gate_output_invalid" + GATE_EXECUTION_FAILED = "gate_execution_failed" SERVER_BIND_FAILED = "server_bind_failed" UNEXPECTED_SERVICE_FAILURE = "unexpected_service_failure" -@dataclass(frozen=True) -class _ErrorSpec: - """Immutable, developer-authored classification and remediation text.""" - - kind: ErrorKind - component: ErrorComponent - operation: str - summary: str - hint: str - - -class PrivacyGuardError(Exception): +class EgressGateError(Exception): """A catalog-only failure whose public representation is content-safe.""" def __init__(self, code: ErrorCode) -> None: @@ -85,39 +76,54 @@ def __str__(self) -> str: ) -class EntityProcessingError(Exception): - """Base for content-safe entity-processing failures.""" +class GateError(Exception): + """Base for content-safe gate lifecycle failures.""" + +class GateConfigurationError(GateError): + """A gate class or configured instance is invalid.""" -class EngineConfigurationError(EntityProcessingError): - """An engine class or configured instance is invalid.""" +class GateContractError(GateError): + """A gate invocation or returned result violated the public contract.""" -class EngineContractError(EntityProcessingError): - """An engine invocation or returned result violated the public contract.""" +class GateExecutionError(GateError): + """A gate's configured runtime failed to complete one request.""" -class EngineExecutionError(EntityProcessingError): - """An engine's configured runtime failed to complete one text input.""" +class GateLimitExceededError(GateError): + """A gate exceeded a bounded configuration or output limit.""" -class EngineLimitExceededError(EntityProcessingError): - """An engine exceeded a bounded configuration or output limit.""" +class GateInputError(GateError): + """A gate could not interpret a bounded request input.""" -class TimeoutExpiredError(EntityProcessingError): - """The shared entity-processing timeout expired.""" + +class TimeoutExpiredError(Exception): + """The shared request-processing timeout expired.""" def __init__(self) -> None: super().__init__( - "Privacy Guard processing timed out. Reduce the request size or simplify " - "the configured stages and rules, or increase the processing timeout " + "Egress Gate processing timed out. Reduce the request size or simplify " + "the configured gates and rules, or increase the processing timeout " f"to at most {MAX_TIMEOUT_SECONDS:g} seconds, then retry." ) -class EngineRegistryError(Exception): - """A content-safe engine registration or registry lifecycle failure.""" +class GateRegistryError(Exception): + """A content-safe gate registration or registry lifecycle failure.""" + + +@dataclass(frozen=True) +class _ErrorSpec: + """Immutable, developer-authored classification and remediation text.""" + + kind: ErrorKind + component: ErrorComponent + operation: str + summary: str + hint: str _ERROR_SPECS: dict[ErrorCode, _ErrorSpec] = { @@ -128,8 +134,25 @@ class EngineRegistryError(Exception): "Policy configuration is invalid.", "Keep the encoded configuration at or below " f"{MAX_PROTO_CONFIG_BYTES // 1024} KiB, compare it with " - "`privacy-guard configuration-schema`, then check the stages, engine " - "settings, pattern catalogs, replacements, and action.", + "`egress-gate gates schema`, then check the gates, " + "pattern catalogs, replacements, and default decision.", + ), + ErrorCode.CONFIG_PREPARATION_FAILED: _ErrorSpec( + ErrorKind.INVALID_INPUT, + ErrorComponent.CONFIG, + "prepare", + "A configured gate could not be prepared.", + "Check the configured gate's rules and resources. For the built-in regex " + "gate, remove named groups, inline flags, invalid expressions, and patterns " + "that can match empty input, then retry.", + ), + ErrorCode.REQUEST_PROTOBUF_INVALID: _ErrorSpec( + ErrorKind.INVALID_INPUT, + ErrorComponent.SERVICE, + "decode_protobuf", + "Request protobuf encoding is invalid.", + "Encode a complete request with the published OpenShell middleware " + "protobuf contract, then retry.", ), ErrorCode.REQUEST_PHASE_INVALID: _ErrorSpec( ErrorKind.INVALID_INPUT, @@ -160,21 +183,21 @@ class EngineRegistryError(Exception): "Request body encoding is invalid.", "Supply a valid UTF-8 request body.", ), - ErrorCode.ENGINE_OUTPUT_INVALID: _ErrorSpec( + ErrorCode.GATE_OUTPUT_INVALID: _ErrorSpec( ErrorKind.INTERNAL, ErrorComponent.PROCESSOR, - "validate_engine", - "An entity-processing engine returned an invalid result.", - "Custom engine developers should check the run contract, result model, " - "spans, processing strategy, and output limits.", + "validate_gate", + "A gate returned an invalid result.", + "Gate authors should check the evaluate contract, capabilities, findings, " + "mutations, and output limits.", ), - ErrorCode.ENGINE_EXECUTION_FAILED: _ErrorSpec( + ErrorCode.GATE_EXECUTION_FAILED: _ErrorSpec( ErrorKind.INTERNAL, - ErrorComponent.ENGINE, - "run", - "An entity-processing engine failed.", + ErrorComponent.GATE, + "evaluate", + "A configured gate failed.", "Check the request ID and error code in service logs, then run the " - "configured engine's focused configuration and single-text tests.", + "configured gate's focused configuration and request tests.", ), ErrorCode.SERVER_BIND_FAILED: _ErrorSpec( ErrorKind.INTERNAL, diff --git a/projects/egress-gate/src/egress_gate/gates/__init__.py b/projects/egress-gate/src/egress_gate/gates/__init__.py new file mode 100644 index 00000000..95ee1ae8 --- /dev/null +++ b/projects/egress-gate/src/egress_gate/gates/__init__.py @@ -0,0 +1,61 @@ +"""Public Egress Gate authoring and built-in gate surface.""" + +from egress_gate.gates.base import ( + Gate, + GateCapability, + GateConfig, + GateResources, + Utf8BodyGate, +) +from egress_gate.gates.regex import ( + ConfidenceLevel, + RegexBodyAction, + RegexBodyScan, + RegexConfig, + RegexDenyAction, + RegexDetectAction, + RegexEntity, + RegexGate, + RegexHeaderScan, + RegexPathScan, + RegexPatternCatalog, + RegexQueryScan, + RegexReadOnlyAction, + RegexReplaceAction, + RegexRule, + RegexScan, +) +from egress_gate.gates.registry import ( + GateDescription, + GateRegistry, + create_builtin_registry, +) +from egress_gate.result import FindingTypeDefinition + +__all__ = [ + "ConfidenceLevel", + "Gate", + "GateCapability", + "GateConfig", + "GateDescription", + "GateRegistry", + "GateResources", + "FindingTypeDefinition", + "RegexBodyAction", + "RegexBodyScan", + "RegexConfig", + "RegexDenyAction", + "RegexDetectAction", + "RegexEntity", + "RegexGate", + "RegexHeaderScan", + "RegexPatternCatalog", + "RegexPathScan", + "RegexQueryScan", + "RegexReadOnlyAction", + "RegexReplaceAction", + "RegexRule", + "RegexScan", + "Utf8BodyGate", + "create_builtin_registry", +] diff --git a/projects/egress-gate/src/egress_gate/gates/base.py b/projects/egress-gate/src/egress_gate/gates/base.py new file mode 100644 index 00000000..46543ce3 --- /dev/null +++ b/projects/egress-gate/src/egress_gate/gates/base.py @@ -0,0 +1,372 @@ +"""Trusted request-level gate extension contract.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from enum import StrEnum +from types import NoneType +from typing import ClassVar, Generic, TypeGuard, final, get_args, get_origin + +from pydantic import Field, ValidationError +from typing_extensions import TypeVar + +from egress_gate.base import StrictDomainModel +from egress_gate.errors import ( + GateConfigurationError, + GateContractError, + GateError, + GateExecutionError, + GateInputError, + TimeoutExpiredError, +) +from egress_gate.request import HttpRequest +from egress_gate.result import ( + FindingTypeDefinition, + GateEvaluation, + GateName, +) +from egress_gate.timeout import Timeout + + +class GateConfig(StrictDomainModel): + """Base for one named gate with a required literal ``kind``.""" + + name: GateName = Field( + description="Unique diagnostic name for this gate in the policy." + ) + + +class GateResources: + """Operator-owned, concurrency-safe resources borrowed by prepared gates.""" + + __slots__ = () + + +class GateCapability(StrEnum): + """One declared request access or permitted gate result.""" + + READ_TARGET = "read_target" + READ_CONTEXT = "read_context" + READ_HEADERS = "read_headers" + READ_BODY = "read_body" + REPLACE_BODY = "replace_body" + MUTATE_HEADERS = "mutate_headers" + ALLOW = "allow" + DENY = "deny" + + +GateConfigT = TypeVar("GateConfigT", bound=GateConfig) +GateResourcesT = TypeVar( + "GateResourcesT", + bound=GateResources | None, + default=None, +) + + +class Gate(ABC, Generic[GateConfigT, GateResourcesT]): + """Typed request-level gate with a validated public evaluation wrapper.""" + + capabilities: ClassVar[frozenset[GateCapability]] + finding_types: ClassVar[tuple[FindingTypeDefinition, ...]] + + @final + def __init__( + self, + config: GateConfigT, + resources: GateResourcesT, + *, + timeout: Timeout | None = None, + ) -> None: + type(self).validate_config(config, resources) + self.__config = config + self.__resources = resources + if timeout is not None and not isinstance(timeout, Timeout): + raise GateConfigurationError("gate preparation timeout is invalid") + self._initialize(timeout=timeout) + + @classmethod + def validate_config( + cls, + config: GateConfigT, + resources: GateResourcesT, + ) -> None: + """Purely validate one exact config and its registered resources.""" + cls._validate_class_contract() + config_type, resources_type = _declared_gate_types(cls) + try: + if type(config) is not config_type: + raise ValueError + config_type.model_validate(config) + except (ValidationError, ValueError): + raise GateConfigurationError("gate configuration is invalid") from None + if not _is_valid_resources(resources, resources_type): + raise GateConfigurationError("gate resources are invalid") + cls._validate_config(config, resources) + + @classmethod + def get_config_type(cls) -> type[GateConfig]: + """Return the concrete ``GateConfig`` type declared by the gate.""" + config_type, _ = _declared_gate_types(cls) + return config_type + + @classmethod + def get_resources_type(cls) -> type[GateResources] | None: + """Return the concrete operational-resource type, if any.""" + _, resources_type = _declared_gate_types(cls) + return resources_type + + @property + def config(self) -> GateConfigT: + """Return the exact validated gate configuration.""" + return self.__config + + @property + def resources(self) -> GateResourcesT: + """Return the borrowed application-owned resources.""" + return self.__resources + + @final + def evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + """Evaluate one immutable current request and validate the result.""" + if not isinstance(request, HttpRequest) or not isinstance(timeout, Timeout): + raise GateContractError("gate input types are invalid") + timeout.raise_if_expired() + try: + raw_result = self._evaluate(request, timeout=timeout) + if not isinstance(raw_result, GateEvaluation): + raise GateContractError("gate output is invalid") + try: + result = GateEvaluation.model_validate(raw_result.model_dump()) + except ValidationError: + raise GateContractError("gate output is invalid") from None + timeout.raise_if_expired() + _validate_gate_output( + type(self).capabilities, + type(self).finding_types, + result, + ) + return result + except (GateError, TimeoutExpiredError): + raise + except Exception: + raise GateExecutionError("gate evaluation failed") from None + + @classmethod + def _validate_class_contract(cls) -> None: + capabilities = getattr(cls, "capabilities", None) + if not isinstance(capabilities, frozenset) or any( + not isinstance(capability, GateCapability) for capability in capabilities + ): + raise GateConfigurationError("gate capabilities are invalid") + finding_types = getattr(cls, "finding_types", None) + if not isinstance(finding_types, tuple) or any( + not isinstance(item, FindingTypeDefinition) for item in finding_types + ): + raise GateConfigurationError("gate finding declarations are invalid") + names = tuple(item.type for item in finding_types) + if len(names) != len(set(names)): + raise GateConfigurationError("gate finding types must be unique") + config_type, _ = _declared_gate_types(cls) + if config_type is GateConfig: + raise GateConfigurationError("gate config type is not concrete") + + @classmethod + def _validate_config( + cls, + config: GateConfigT, + resources: GateResourcesT, + ) -> None: + """Optionally validate resource-backed config without side effects.""" + + def _initialize(self, *, timeout: Timeout | None = None) -> None: + """Optionally derive reusable state under the preparation deadline.""" + + @abstractmethod + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + """Return one explicit control result for the current request.""" + raise NotImplementedError + + +class Utf8BodyGate( + Gate[GateConfigT, GateResourcesT], Generic[GateConfigT, GateResourcesT] +): + """Gate helper that exposes one strict UTF-8 body to an implementation.""" + + capabilities = frozenset({GateCapability.READ_BODY}) + finding_types: ClassVar[tuple[FindingTypeDefinition, ...]] = () + + @final + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + try: + text = request.body.decode("utf-8", errors="strict") + except UnicodeDecodeError: + raise GateInputError("request body is not valid UTF-8") from None + result = self._evaluate_text(text, timeout=timeout) + if not isinstance(result, GateEvaluation): + raise GateContractError("UTF-8 body gate output is invalid") + if result.request_mutations.replacement_body is not None: + try: + result.request_mutations.replacement_body.decode( + "utf-8", errors="strict" + ) + except UnicodeDecodeError: + raise GateContractError( + "UTF-8 body gate returned a non-UTF-8 replacement" + ) from None + return result + + @abstractmethod + def _evaluate_text( + self, + text: str, + *, + timeout: Timeout, + ) -> GateEvaluation: + """Evaluate the decoded body and preserve explicit replacement intent.""" + raise NotImplementedError + + +def _validate_gate_output( + capabilities: frozenset[GateCapability], + finding_types: tuple[FindingTypeDefinition, ...], + result: GateEvaluation, +) -> None: + if ( + result.request_mutations.replacement_body is not None + and GateCapability.REPLACE_BODY not in capabilities + ): + raise GateContractError("gate returned an undeclared body replacement") + if ( + result.request_mutations.header_mutations + and GateCapability.MUTATE_HEADERS not in capabilities + ): + raise GateContractError("gate returned undeclared header mutations") + if result.control.value == "allow" and GateCapability.ALLOW not in capabilities: + raise GateContractError("gate returned an undeclared terminal allow") + if result.control.value == "deny" and GateCapability.DENY not in capabilities: + raise GateContractError("gate returned an undeclared deny") + declared_types = frozenset(item.type for item in finding_types) + if any(finding.type not in declared_types for finding in result.findings): + raise GateContractError("gate returned an undeclared finding type") + + +def _declared_gate_types( + gate_type: type[object], +) -> tuple[type[GateConfig], type[GateResources] | None]: + decorated_config_type = getattr(gate_type, "_decorated_config_type", None) + if _is_gate_config_type(decorated_config_type): + return decorated_config_type, None + for candidate in gate_type.__mro__: + for base in getattr(candidate, "__orig_bases__", ()): + origin = get_origin(base) + if origin is None or not isinstance(origin, type): + continue + if not issubclass(origin, Gate): + continue + arguments = get_args(base) + parameters = getattr(origin, "__parameters__", ()) + substitutions = dict(zip(parameters, arguments, strict=False)) + if origin is Gate: + resolved = tuple( + substitutions.get(argument, argument) for argument in arguments + ) + if len(resolved) == 1: + resolved = (*resolved, NoneType) + if len(resolved) != 2: + break + config_type, resources_type = resolved + if _is_gate_config_type(config_type): + return config_type, _normalize_resources_type(resources_type) + break + try: + inherited = _resolve_gate_base(origin, substitutions) + except GateConfigurationError: + continue + if inherited is not None: + return inherited + raise GateConfigurationError( + "gate must declare concrete configuration and resource types" + ) + + +def _resolve_gate_base( + candidate: type[object], + substitutions: dict[object, object], +) -> tuple[type[GateConfig], type[GateResources] | None] | None: + for base in getattr(candidate, "__orig_bases__", ()): + origin = get_origin(base) + if ( + origin is None + or not isinstance(origin, type) + or not issubclass(origin, Gate) + ): + continue + arguments = tuple( + substitutions.get(argument, argument) for argument in get_args(base) + ) + parameters = getattr(origin, "__parameters__", ()) + nested = dict(substitutions) + nested.update(zip(parameters, arguments, strict=False)) + if origin is Gate: + if len(arguments) == 1: + arguments = (*arguments, NoneType) + if len(arguments) == 2: + config_type, resources_type = arguments + if _is_gate_config_type(config_type): + return config_type, _normalize_resources_type(resources_type) + continue + resolved = _resolve_gate_base(origin, nested) + if resolved is not None: + return resolved + return None + + +def _is_gate_config_type(value: object) -> TypeGuard[type[GateConfig]]: + return isinstance(value, type) and issubclass(value, GateConfig) + + +def _is_gate_resources_type(value: object) -> TypeGuard[type[GateResources]]: + return isinstance(value, type) and issubclass(value, GateResources) + + +def _normalize_resources_type(value: object) -> type[GateResources] | None: + if value is None or value is NoneType: + return None + if _is_gate_resources_type(value): + return value + raise GateConfigurationError("gate resources type is invalid") + + +def _is_valid_resources( + resources: object, + resources_type: type[GateResources] | None, +) -> bool: + if resources_type is None: + return resources is None + return isinstance(resources, resources_type) + + +__all__ = [ + "Gate", + "GateCapability", + "GateConfig", + "GateConfigT", + "GateResources", + "GateResourcesT", + "Utf8BodyGate", +] diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/egress-gate/src/egress_gate/gates/regex.py similarity index 61% rename from projects/privacy-guard/src/privacy_guard/engines/regex.py rename to projects/egress-gate/src/egress_gate/gates/regex.py index 4b0323bc..0dd10b9e 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/egress-gate/src/egress_gate/gates/regex.py @@ -1,17 +1,16 @@ -"""Bounded regular-expression entity detection and replacement.""" +"""Typed, bounded regular-expression scans and actions for HTTP requests.""" from __future__ import annotations -import json import os from collections import OrderedDict -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from dataclasses import dataclass +from enum import StrEnum from pathlib import Path from stat import S_ISREG from string import Formatter -from threading import RLock -from typing import Literal, Protocol, Self +from typing import Annotated, Literal, Protocol, Self, TypeAlias import regex import yaml @@ -21,36 +20,39 @@ from yaml.nodes import MappingNode from yaml.resolver import BaseResolver -from privacy_guard.base import StrictDomainModel -from privacy_guard.constants import ( +from egress_gate.base import StrictDomainModel +from egress_gate.constants import ( MAX_BODY_BYTES, - MAX_DETECTIONS_PER_STAGE, + MAX_DETECTIONS_PER_GATE, MAX_DIAGNOSTIC_TEXT_BYTES, + MAX_PROTO_FINDING_GROUPS, + MAX_PROTO_HEADERS, MAX_REGEX_CATALOG_FILE_BYTES, MAX_REGEX_CATALOG_PATH_BYTES, - MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES, MAX_REGEX_ENTITIES_PER_CATALOG, MAX_REGEX_NAME_BYTES, MAX_REGEX_PATTERN_BYTES, MAX_REGEX_RULES_PER_CATALOG, - REGEX_COMPILED_RULE_WEIGHT_BYTES, ) -from privacy_guard.engines.base import ( - ConfidenceLevel, - EngineConfig, - EntityDetection, - EntityProcessingEngine, - EntityProcessingStrategy, - TextProcessingResult, +from egress_gate.errors import ( + GateConfigurationError, + GateContractError, + GateInputError, + GateLimitExceededError, ) -from privacy_guard.errors import ( - EngineConfigurationError, - EngineContractError, - EngineLimitExceededError, -) -from privacy_guard.logging import get_logger -from privacy_guard.string_validators import ScalarString, validate_scalar_string -from privacy_guard.timeout import Timeout +from egress_gate.gates.base import Gate, GateCapability, GateConfig +from egress_gate.request import HeaderName, HttpRequest, RequestMutations +from egress_gate.result import Finding, FindingTypeDefinition, GateEvaluation +from egress_gate.string_validators import ScalarString, validate_scalar_string +from egress_gate.timeout import Timeout + + +class ConfidenceLevel(StrEnum): + """Categorical certainty reported by the regex gate.""" + + LOW = "low" + MEDIUM = "medium" + HIGH = "high" class RegexRule(StrictDomainModel): @@ -113,7 +115,7 @@ def _supplied_rule_names_are_unique(self) -> Self: class RegexPatternCatalog(StrictDomainModel): - """The complete ordered entity catalog for one RegexEngine stage.""" + """The complete ordered entity catalog for one regex gate.""" entities: tuple[RegexEntity, ...] = Field(repr=False) @@ -139,10 +141,22 @@ def _catalog_is_bounded_and_unambiguous(self) -> Self: return self -class RegexReplacement(StrictDomainModel): - """A constrained template replacement recipe.""" +class RegexDetectAction(StrictDomainModel): + """Report matches and continue without changing the request.""" + + kind: Literal["detect"] + - strategy: Literal["template"] = "template" +class RegexDenyAction(StrictDomainModel): + """Deny the request when the scan finds a match.""" + + kind: Literal["deny"] + + +class RegexReplaceAction(StrictDomainModel): + """Replace body matches with a constrained template.""" + + kind: Literal["replace"] template: ScalarString = Field(default="[{entity}]", repr=False) @field_validator("template") @@ -161,17 +175,76 @@ def _template_is_safe_and_bounded(cls, value: str) -> str: return value -class RegexEngineConfig(EngineConfig): - """Exact policy configuration owned by ``RegexEngine``.""" +RegexReadOnlyAction: TypeAlias = Annotated[ + RegexDetectAction | RegexDenyAction, + Field(discriminator="kind"), +] +RegexBodyAction: TypeAlias = Annotated[ + RegexDetectAction | RegexDenyAction | RegexReplaceAction, + Field(discriminator="kind"), +] + + +class RegexBodyScan(StrictDomainModel): + """Scan the UTF-8 request body and apply a body-compatible action.""" + + kind: Literal["body"] + action: RegexBodyAction + + +class RegexPathScan(StrictDomainModel): + """Scan the request path and detect or deny matches.""" + + kind: Literal["path"] + action: RegexReadOnlyAction + + +class RegexQueryScan(StrictDomainModel): + """Scan the raw request query and detect or deny matches.""" + + kind: Literal["query"] + action: RegexReadOnlyAction + + +class RegexHeaderScan(StrictDomainModel): + """Scan values from named request headers and detect or deny matches.""" + + kind: Literal["header"] + names: tuple[HeaderName, ...] = Field(min_length=1, max_length=MAX_PROTO_HEADERS) + action: RegexReadOnlyAction + + @field_validator("names", mode="before") + @classmethod + def _names_are_a_tuple(cls, value: object) -> object: + if isinstance(value, list | tuple): + return tuple(value) + return value + + @model_validator(mode="after") + def _names_are_unique(self) -> Self: + normalized = tuple(name.casefold() for name in self.names) + if len(normalized) != len(set(normalized)): + raise ValueError("header scan names must be unique") + return self + + +RegexScan: TypeAlias = Annotated[ + RegexBodyScan | RegexPathScan | RegexQueryScan | RegexHeaderScan, + Field(discriminator="kind"), +] + + +class RegexConfig(GateConfig): + """Exact policy configuration owned by ``RegexGate``.""" - engine: Literal["regex"] = "regex" + kind: Literal["regex"] + scan: RegexScan pattern_catalog: RegexPatternCatalog = Field( repr=False, description=( "Complete structured catalog or relative path to a complete YAML catalog." ), ) - replacement: RegexReplacement | None = None @field_validator( "pattern_catalog", @@ -183,60 +256,112 @@ def _load_pattern_catalog( cls, value: object, ) -> object: - del cls if isinstance(value, str): return _load_pattern_catalog_file(value) return value @model_validator(mode="after") - def _rules_are_valid(self) -> Self: - try: - _compile_pattern_catalog(self.pattern_catalog) - except (RecursionError, ValueError, regex.error): - raise ValueError("regex pattern catalog is invalid") from None + def _patterns_are_valid(self) -> Self: + if any( + _contains_inline_flags(rule.pattern) + for entity in self.pattern_catalog.entities + for rule in entity.rules + ): + raise ValueError("regex pattern catalog is invalid") return self -class RegexEngine(EntityProcessingEngine[RegexEngineConfig]): - """Detect every regex match, including matches that share input characters.""" +class RegexGate(Gate[RegexConfig, None]): + """Scan the request body, path, query, or selected headers with regex rules.""" - supported_strategies = frozenset( + capabilities = frozenset( { - EntityProcessingStrategy.DETECT, - EntityProcessingStrategy.REPLACE, + GateCapability.READ_TARGET, + GateCapability.READ_HEADERS, + GateCapability.READ_BODY, + GateCapability.REPLACE_BODY, + GateCapability.DENY, } ) + finding_types = (FindingTypeDefinition(type="regex_match"),) - @classmethod - def _validate_run_config( - cls, - config: RegexEngineConfig, - resources: None, - *, - strategy: EntityProcessingStrategy, - ) -> None: - del cls, resources - if strategy is EntityProcessingStrategy.REPLACE and config.replacement is None: - raise EngineConfigurationError( - "regex replacement configuration is required" - ) - - def _initialize(self) -> None: + def _initialize(self, *, timeout: Timeout | None = None) -> None: try: - self._rules = _compile_pattern_catalog(self.config.pattern_catalog) + self._rules = _compile_pattern_catalog( + self.config.pattern_catalog, + timeout=timeout, + ) except (RecursionError, ValueError, regex.error): - raise EngineConfigurationError( - "regex engine configuration is invalid" + raise GateConfigurationError( + "regex gate configuration is invalid" ) from None - def _run( + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + scan_texts = self._scan_texts(request) + detections_with_identity: list[tuple[_RegexDetection, str]] = [] + for text in scan_texts: + detections_with_identity.extend(self._match_text(text, timeout=timeout)) + if len(detections_with_identity) > MAX_DETECTIONS_PER_GATE: + raise GateLimitExceededError("regex detection count exceeds the limit") + detections = tuple(item[0] for item in detections_with_identity) + findings = _aggregate_findings(detections) + if len(findings) > MAX_PROTO_FINDING_GROUPS: + raise GateLimitExceededError("regex finding groups exceed the limit") + action = self.config.scan.action + if isinstance(action, RegexDenyAction) and detections: + return GateEvaluation.deny( + "egress_gate_regex_denied", + findings=findings, + ) + if not isinstance(action, RegexReplaceAction): + return GateEvaluation.proceed(findings=findings) + + body_text = scan_texts[0] + output_text = body_text + if detections: + winners = _resolve_overlaps(detections_with_identity) + output_text = _render_bounded_replacement( + body_text, + winners, + action.template, + ) + return GateEvaluation.proceed( + request_mutations=RequestMutations( + replacement_body=output_text.encode("utf-8") + ), + findings=findings, + ) + + def _scan_texts(self, request: HttpRequest) -> tuple[str, ...]: + scan = self.config.scan + if isinstance(scan, RegexBodyScan): + try: + return (request.body.decode("utf-8", errors="strict"),) + except UnicodeDecodeError: + raise GateInputError("regex body scan is not valid UTF-8") from None + if isinstance(scan, RegexPathScan): + return (request.target.path,) + if isinstance(scan, RegexQueryScan): + return (request.target.query,) + selected_names = frozenset(name.casefold() for name in scan.names) + return tuple( + header.value + for header in request.headers + if header.name.casefold() in selected_names + ) + + def _match_text( self, text: str, *, - strategy: EntityProcessingStrategy, timeout: Timeout, - ) -> TextProcessingResult: - detections_with_identity: list[tuple[EntityDetection, str]] = [] + ) -> list[tuple[_RegexDetection, str]]: + detections: list[tuple[_RegexDetection, str]] = [] for rule in self._rules: next_position = 0 while next_position <= len(text): @@ -250,28 +375,30 @@ def _run( break start, end = match.span() if start == end: - raise EngineConfigurationError( - "regex engine configuration is invalid" + raise GateConfigurationError( + "regex configuration matches an empty span" ) if match.span(rule.marker) != (end, end): - raise EngineConfigurationError( - "regex engine configuration is invalid" + raise GateConfigurationError( + "regex configuration marker is invalid" + ) + detections.append( + ( + _RegexDetection( + entity=rule.entity, + start=start, + end=end, + confidence=rule.confidence, + ), + rule.rule_identity, ) - detection = EntityDetection( - entity=rule.entity, - start=start, - end=end, - confidence=rule.confidence, - metadata={_RULE_METADATA_KEY: rule.rule_identity}, ) - detections_with_identity.append((detection, rule.rule_identity)) - if len(detections_with_identity) > MAX_DETECTIONS_PER_STAGE: - raise EngineLimitExceededError( + if len(detections) > MAX_DETECTIONS_PER_GATE: + raise GateLimitExceededError( "regex detection count exceeds the limit" ) next_position = start + 1 - - detections_with_identity.sort( + detections.sort( key=lambda item: ( item[0].start, item[0].end, @@ -279,21 +406,7 @@ def _run( item[1], ) ) - detections = tuple(item[0] for item in detections_with_identity) - output_text = text - if strategy is EntityProcessingStrategy.REPLACE and detections: - replacement = self.config.replacement - if replacement is None: - raise EngineConfigurationError( - "regex replacement configuration is required" - ) - winners = _resolve_overlaps(detections_with_identity) - output_text = _render_bounded_replacement( - text, - winners, - replacement.template, - ) - return TextProcessingResult(text=output_text, detections=detections) + return detections @dataclass(frozen=True) @@ -305,6 +418,32 @@ class _CompiledRule: compiled: _CompiledPattern +@dataclass(frozen=True) +class _RegexDetection: + entity: str + start: int + end: int + confidence: ConfidenceLevel + + +def _aggregate_findings( + detections: tuple[_RegexDetection, ...], +) -> tuple[Finding, ...]: + counts: OrderedDict[tuple[str, ConfidenceLevel], int] = OrderedDict() + for detection in detections: + key = (detection.entity, detection.confidence) + counts[key] = counts.get(key, 0) + 1 + return tuple( + Finding( + type="regex_match", + label=entity, + count=count, + confidence=confidence.value, + ) + for (entity, confidence), count in counts.items() + ) + + class _RegexMatch(Protocol): def span(self, group: int | str = 0) -> tuple[int, int]: """Return the matched span for a numbered or named group.""" @@ -475,85 +614,31 @@ def _validate_name(value: str) -> str: def _compile_pattern_catalog( catalog: RegexPatternCatalog, + *, + timeout: Timeout | None = None, ) -> tuple[_CompiledRule, ...]: - with _COMPILED_PATTERN_CACHE_LOCK: - cached = _COMPILED_PATTERN_CACHE.get(catalog) - if cached is not None: - _COMPILED_PATTERN_CACHE.move_to_end(catalog) - return cached[0] - - rules = tuple( - _compile_rule( - entity, - rule, - catalog_index=global_index, - entity_rule_index=rule_index, - ) - for global_index, (entity, rule_index, rule) in enumerate( - _iter_catalog_rules(catalog) - ) - ) - weight_bytes = _compiled_pattern_weight(catalog, len(rules)) - if weight_bytes > MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES: - _LOGGER.debug( - "privacy_guard_cache_skip cache=regex_compiled " - "weight_bytes=%d budget_bytes=%d", - weight_bytes, - MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES, - ) - return rules - - evicted_entries = 0 - evicted_weight_bytes = 0 - with _COMPILED_PATTERN_CACHE_LOCK: - cached = _COMPILED_PATTERN_CACHE.get(catalog) - if cached is not None: - _COMPILED_PATTERN_CACHE.move_to_end(catalog) - return cached[0] - - global _COMPILED_PATTERN_CACHE_WEIGHT_BYTES - _COMPILED_PATTERN_CACHE[catalog] = (rules, weight_bytes) - _COMPILED_PATTERN_CACHE_WEIGHT_BYTES += weight_bytes - while ( - len(_COMPILED_PATTERN_CACHE) > _MAX_CACHED_COMPILED_CATALOGS - or _COMPILED_PATTERN_CACHE_WEIGHT_BYTES - > MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES - ): - _, (_, evicted_weight) = _COMPILED_PATTERN_CACHE.popitem(last=False) - _COMPILED_PATTERN_CACHE_WEIGHT_BYTES -= evicted_weight - evicted_weight_bytes += evicted_weight - evicted_entries += 1 - if evicted_entries: - _LOGGER.debug( - "privacy_guard_cache_eviction cache=regex_compiled " - "entries=%d weight_bytes=%d", - evicted_entries, - evicted_weight_bytes, + _raise_if_expired(timeout) + rules_list: list[_CompiledRule] = [] + for global_index, (entity, rule_index, rule) in enumerate( + _iter_catalog_rules(catalog) + ): + _raise_if_expired(timeout) + rules_list.append( + _compile_rule( + entity, + rule, + catalog_index=global_index, + entity_rule_index=rule_index, + timeout=timeout, + ) ) - return rules + _raise_if_expired(timeout) + return tuple(rules_list) -def _compiled_pattern_weight( - catalog: RegexPatternCatalog, - rule_count: int, -) -> int: - catalog_bytes = len( - json.dumps( - catalog.model_dump(mode="json"), - allow_nan=False, - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") - ) - return catalog_bytes + rule_count * REGEX_COMPILED_RULE_WEIGHT_BYTES - - -def _clear_compiled_pattern_cache() -> None: - global _COMPILED_PATTERN_CACHE_WEIGHT_BYTES - with _COMPILED_PATTERN_CACHE_LOCK: - _COMPILED_PATTERN_CACHE.clear() - _COMPILED_PATTERN_CACHE_WEIGHT_BYTES = 0 +def _raise_if_expired(timeout: Timeout | None) -> None: + if timeout is not None: + timeout.raise_if_expired() def _compile_rule( @@ -561,7 +646,10 @@ def _compile_rule( rule: RegexRule, catalog_index: int, entity_rule_index: int, + *, + timeout: Timeout | None = None, ) -> _CompiledRule: + _raise_if_expired(timeout) flags = 0 if rule.ignore_case: flags |= regex.IGNORECASE @@ -574,12 +662,19 @@ def _compile_rule( if _contains_inline_flags(rule.pattern): raise ValueError("inline flags are unsupported") unmarked = regex.compile(rule.pattern, flags) + _raise_if_expired(timeout) if unmarked.groupindex: raise ValueError("named groups are reserved") - if unmarked.search("") is not None: + if timeout is None: + empty_match = unmarked.search("") + else: + with timeout.enforce(): + empty_match = unmarked.search("", timeout=timeout.remaining_seconds()) + if empty_match is not None: raise ValueError("pattern must not match empty input") - marker = f"_pg_rule_{catalog_index:06d}" + marker = f"_eg_rule_{catalog_index:06d}" compiled = regex.compile(f"(?:{rule.pattern})(?P<{marker}>)", flags) + _raise_if_expired(timeout) if marker not in compiled.groupindex: raise ValueError("internal marker is missing") rule_identity = rule.name or f"{entity.name}.rules[{entity_rule_index}]" @@ -594,12 +689,10 @@ def _compile_rule( def _iter_catalog_rules( catalog: RegexPatternCatalog, -) -> tuple[tuple[RegexEntity, int, RegexRule], ...]: - return tuple( - (entity, rule_index, rule) - for entity in catalog.entities - for rule_index, rule in enumerate(entity.rules) - ) +) -> Iterator[tuple[RegexEntity, int, RegexRule]]: + for entity in catalog.entities: + for rule_index, rule in enumerate(entity.rules): + yield entity, rule_index, rule def _contains_inline_flags(pattern: str) -> bool: @@ -625,9 +718,9 @@ def _contains_inline_flags(pattern: str) -> bool: def _resolve_overlaps( - detections: list[tuple[EntityDetection, str]], -) -> tuple[EntityDetection, ...]: - winners: list[EntityDetection] = [] + detections: list[tuple[_RegexDetection, str]], +) -> tuple[_RegexDetection, ...]: + winners: list[_RegexDetection] = [] ranked = sorted( detections, key=lambda item: ( @@ -655,13 +748,13 @@ def _resolve_overlaps( def _categorical_confidence_rank(confidence: object) -> int: if not isinstance(confidence, ConfidenceLevel): - raise EngineContractError("regex detection confidence is invalid") + raise GateContractError("regex detection confidence is invalid") return _CONFIDENCE_RANK[confidence] def _render_bounded_replacement( text: str, - detections: tuple[EntityDetection, ...], + detections: tuple[_RegexDetection, ...], template: str, ) -> str: projected_size = 0 @@ -670,11 +763,11 @@ def _render_bounded_replacement( projected_size += len(text[cursor : detection.start].encode("utf-8")) projected_size += _rendered_template_size(template, detection.entity) if projected_size > MAX_BODY_BYTES: - raise EngineLimitExceededError("regex replacement exceeds the size limit") + raise GateLimitExceededError("regex replacement exceeds the size limit") cursor = detection.end projected_size += len(text[cursor:].encode("utf-8")) if projected_size > MAX_BODY_BYTES: - raise EngineLimitExceededError("regex replacement exceeds the size limit") + raise GateLimitExceededError("regex replacement exceeds the size limit") parts: list[str] = [] cursor = 0 @@ -703,24 +796,21 @@ def _rendered_template_size(template: str, entity: str) -> int: ConfidenceLevel.MEDIUM: 1, ConfidenceLevel.HIGH: 2, } -_RULE_METADATA_KEY = "rule" -_MAX_CACHED_COMPILED_CATALOGS = 128 -# Python dict preserves insertion order, but this LRU must move cache hits to the -# newest position and efficiently evict the oldest entry. -_COMPILED_PATTERN_CACHE: OrderedDict[ - RegexPatternCatalog, - tuple[tuple[_CompiledRule, ...], int], -] = OrderedDict() -_COMPILED_PATTERN_CACHE_WEIGHT_BYTES = 0 -_COMPILED_PATTERN_CACHE_LOCK = RLock() -_LOGGER = get_logger(__name__) - - __all__ = [ - "RegexEngine", - "RegexEngineConfig", + "ConfidenceLevel", + "RegexBodyAction", + "RegexBodyScan", + "RegexConfig", + "RegexDenyAction", + "RegexDetectAction", "RegexEntity", + "RegexGate", + "RegexHeaderScan", "RegexPatternCatalog", - "RegexReplacement", + "RegexPathScan", + "RegexQueryScan", + "RegexReadOnlyAction", + "RegexReplaceAction", "RegexRule", + "RegexScan", ] diff --git a/projects/egress-gate/src/egress_gate/gates/registry.py b/projects/egress-gate/src/egress_gate/gates/registry.py new file mode 100644 index 00000000..7208d954 --- /dev/null +++ b/projects/egress-gate/src/egress_gate/gates/registry.py @@ -0,0 +1,517 @@ +"""Gate registration and lazy policy-schema construction.""" + +from __future__ import annotations + +import hashlib +import inspect +import json +import re +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from enum import StrEnum +from functools import reduce +from operator import getitem, or_ +from typing import ( + TYPE_CHECKING, + Annotated, + Literal, + Protocol, + TypeGuard, + get_args, + get_origin, +) + +from pydantic import Field, TypeAdapter, ValidationError +from typing_extensions import TypeVar + +from egress_gate.errors import ( + EgressGateError, + ErrorCode, + GateConfigurationError, + GateRegistryError, +) +from egress_gate.gates.base import ( + Gate, + GateCapability, + GateConfig, + GateConfigT, + GateResources, +) +from egress_gate.gates.regex import RegexGate +from egress_gate.request import HttpRequest +from egress_gate.result import FindingTypeDefinition, GateEvaluation +from egress_gate.timeout import Timeout + +if TYPE_CHECKING: + from egress_gate.config import EgressGateConfig + from egress_gate.request_processor import RequestProcessor + + +@dataclass(frozen=True) +class GateDescription: + """Safe discovery metadata for one registered gate.""" + + gate_type: str + description: str + capabilities: frozenset[GateCapability] + finding_types: tuple[FindingTypeDefinition, ...] + resource_type: str | None + config_type: str + + +class PolicyValidationCategory(StrEnum): + """Content-safe category for one policy schema failure.""" + + REQUIRED_FIELD_MISSING = "required field is missing" + UNKNOWN_FIELD = "unknown field is not allowed" + UNKNOWN_VARIANT = "kind does not identify an installed variant" + INVALID_VALUE = "value has the wrong type, shape, or constraints" + + +class PolicyValidationError(EgressGateError): + """Cataloged policy failure with a trusted structural location.""" + + def __init__( + self, + *, + path: tuple[str | int, ...], + category: PolicyValidationCategory, + ) -> None: + super().__init__(ErrorCode.CONFIG_INVALID) + self.path = path + self.category = category + + @property + def formatted_path(self) -> str: + """Render the trusted field path without submitted values.""" + rendered = "" + for component in self.path: + if isinstance(component, int): + rendered += f"[{component}]" + elif rendered: + rendered += f".{component}" + else: + rendered = component + return rendered or "policy" + + @classmethod + def from_validation_error( + cls, + error: ValidationError, + *, + schema: Mapping[str, object], + ) -> PolicyValidationError: + """Reduce Pydantic diagnostics to one bounded, content-safe issue.""" + known_fields = _schema_property_names(schema) + issues = error.errors( + include_url=False, + include_context=False, + include_input=False, + ) + issue = min(issues, key=lambda item: _validation_error_priority(item["type"])) + path = tuple( + component + for component in issue["loc"] + if isinstance(component, int) + or (isinstance(component, str) and component in known_fields) + ) + return cls( + path=path, + category=_validation_error_category(issue["type"]), + ) + + +class GateRegistry: + """Collect trusted gates and seal their exact policy union on first use.""" + + def __init__(self, *, include_builtin_gates: bool = False) -> None: + self._registrations: dict[str, _Registration] = {} + self._config_adapter: TypeAdapter[object] | None = None + if include_builtin_gates: + self.register(RegexGate) + + def register( + self, + gate_type: type[object], + *, + resources: object = None, + ) -> None: + """Register one gate and its application-owned resources.""" + if self._config_adapter is not None: + raise GateRegistryError("cannot register after the registry is in use") + if not _is_gate_type(gate_type): + raise GateRegistryError("registered gate type is invalid") + if gate_type.__init__ is not Gate.__init__: + raise GateRegistryError( + "gate lifecycle contract requires Gate.__init__; use _initialize()" + ) + if gate_type.evaluate is not Gate.evaluate: + raise GateRegistryError( + "gate lifecycle contract requires Gate.evaluate; implement _evaluate()" + ) + + try: + gate_type._validate_class_contract() + config_type = gate_type.get_config_type() + resources_type = gate_type.get_resources_type() + except (AttributeError, TypeError, GateConfigurationError): + raise GateRegistryError("gate generic declaration is invalid") from None + if not isinstance(config_type, type) or not issubclass(config_type, GateConfig): + raise GateRegistryError("gate config type is invalid") + _validate_common_gate_config_fields(config_type) + gate_kind = _gate_kind(config_type) + if gate_kind in self._registrations: + raise GateRegistryError("gate kind is already registered") + if any( + registration.config_type is config_type + for registration in self._registrations.values() + ): + raise GateRegistryError("gate config type is already registered") + + if resources_type is None: + if resources is not None: + raise GateRegistryError("resource-free gate received resources") + elif resources is None or not isinstance(resources, resources_type): + raise GateRegistryError("gate resources do not match their declared type") + + self._registrations[gate_kind] = _Registration( + gate_type=gate_type, + config_type=config_type, + resources=resources, + ) + + def gate( + self, + *, + config: type[GateConfigT], + capabilities: frozenset[GateCapability], + finding_types: tuple[FindingTypeDefinition, ...] = (), + ) -> Callable[[_GateFunction[GateConfigT]], type[Gate[GateConfig, None]]]: + """Register a typed function as one resource-free gate.""" + config_type = config + declared_capabilities = capabilities + declared_finding_types = finding_types + + def decorate( + evaluate: _GateFunction[GateConfigT], + ) -> type[Gate[GateConfig, None]]: + class FunctionGate(Gate[GateConfig, None]): + _decorated_config_type = config_type + capabilities = declared_capabilities + finding_types = declared_finding_types + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + assert isinstance(self.config, config_type) + return evaluate(request, self.config, timeout=timeout) + + FunctionGate.__doc__ = inspect.getdoc(evaluate) or "" + self.register(FunctionGate) + return FunctionGate + + return decorate + + def validate_config(self, values: object) -> EgressGateConfig[GateConfig]: + """Parse and validate one complete pipeline without preparing gates.""" + if not isinstance(values, Mapping): + raise EgressGateError(ErrorCode.CONFIG_INVALID) + try: + config_value = self._require_config_adapter().validate_python(dict(values)) + except ValidationError as error: + raise PolicyValidationError.from_validation_error( + error, + schema=self.configuration_json_schema(), + ) from None + except (TypeError, ValueError): + raise EgressGateError(ErrorCode.CONFIG_INVALID) from None + if not _is_egress_gate_config(config_value): + raise EgressGateError(ErrorCode.CONFIG_INVALID) + config = config_value + for gate_index, configured_gate in enumerate(config.gates): + registration = self._resolve_registration(configured_gate) + try: + registration.gate_type.validate_config( + configured_gate, + registration.resources, + ) + except GateConfigurationError: + raise PolicyValidationError( + path=("gates", gate_index), + category=PolicyValidationCategory.INVALID_VALUE, + ) from None + return config + + def create_gate( + self, + config: GateConfig, + *, + timeout: Timeout | None = None, + ) -> Gate[GateConfig, GateResources | None]: + """Construct one initialized gate from its exact validated config.""" + registration = self._resolve_registration(config) + if type(config) is not registration.config_type: + raise GateRegistryError("gate config concrete type is invalid") + return registration.gate_type( + config, + registration.resources, + timeout=timeout, + ) + + def prepare_processor( + self, + validated_config: EgressGateConfig[GateConfig], + *, + timeout: Timeout, + ) -> RequestProcessor: + """Prepare one processor from a validated policy configuration. + + This is the production preparation seam shared by the service and + offline evaluation. Registration, policy validation, and resource + ownership remain registry responsibilities; the returned processor + owns only the prepared gates and immutable policy metadata. + """ + from egress_gate.request_processor import RequestProcessor + + self._require_config_adapter() + if not _is_egress_gate_config(validated_config): + raise GateRegistryError("processor configuration is invalid") + if not isinstance(timeout, Timeout): + raise GateRegistryError("processor preparation timeout is invalid") + + prepared: list[tuple[str, str, Gate[GateConfig, GateResources | None]]] = [] + for configured_gate in validated_config.gates: + timeout.raise_if_expired() + gate_type = getattr(configured_gate, "kind", None) + if not isinstance(gate_type, str): + raise GateRegistryError("gate config discriminator is invalid") + try: + gate = self.create_gate(configured_gate, timeout=timeout) + except GateConfigurationError: + raise EgressGateError(ErrorCode.CONFIG_PREPARATION_FAILED) from None + prepared.append((configured_gate.name, gate_type, gate)) + timeout.raise_if_expired() + return RequestProcessor( + validated_config, + tuple(prepared), + policy_fingerprint=self.policy_fingerprint(validated_config), + ) + + def configuration_json_schema(self) -> dict[str, object]: + """Return the complete policy JSON Schema.""" + schema = self._require_config_adapter().json_schema() + schema["title"] = "EgressGateConfig" + schema["description"] = ( + "Flat policy for Egress Gate with ordered gates and a default decision." + ) + return schema + + def describe_gates(self) -> tuple[GateDescription, ...]: + """Return safe gate metadata without constructing gate instances.""" + self._require_config_adapter() + return tuple( + GateDescription( + gate_type=gate_kind, + description=_gate_description(registration.gate_type), + capabilities=registration.gate_type.capabilities, + finding_types=registration.gate_type.finding_types, + resource_type=( + resources_type.__name__ + if (resources_type := registration.gate_type.get_resources_type()) + is not None + else None + ), + config_type=registration.config_type.__name__, + ) + for gate_kind, registration in self._registrations.items() + ) + + @staticmethod + def policy_fingerprint(config: EgressGateConfig[GateConfig]) -> str: + """Return the canonical fingerprint passed into a prepared processor.""" + canonical = json.dumps( + config.model_dump(mode="json"), + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return hashlib.sha256(canonical).hexdigest() + + def _resolve_registration(self, config: GateConfig) -> _Registration: + self._require_config_adapter() + try: + gate_kind = getattr(config, "kind") + if not isinstance(gate_kind, str): + raise AttributeError + return self._registrations[gate_kind] + except (AttributeError, KeyError): + raise GateRegistryError("gate config is not registered") from None + + def _require_config_adapter(self) -> TypeAdapter[object]: + if self._config_adapter is None: + try: + config_type = _build_egress_gate_config_type( + tuple( + registration.config_type + for registration in self._registrations.values() + ) + ) + except (TypeError, ValueError): + raise GateRegistryError( + "gate registry has no registered gates" + ) from None + self._config_adapter = TypeAdapter[object](config_type) + return self._config_adapter + + +def create_builtin_registry() -> GateRegistry: + """Build the registry shipped by the base package.""" + return GateRegistry(include_builtin_gates=True) + + +@dataclass(frozen=True) +class _Registration: + gate_type: type[Gate[GateConfig, GateResources | None]] + config_type: type[GateConfig] + resources: GateResources | None + + +_FunctionConfigT = TypeVar( + "_FunctionConfigT", + bound=GateConfig, + contravariant=True, +) + + +class _GateFunction(Protocol[_FunctionConfigT]): + def __call__( + self, + request: HttpRequest, + config: _FunctionConfigT, + *, + timeout: Timeout, + ) -> GateEvaluation: ... + + +def _build_egress_gate_config_type( + config_types: Sequence[type[GateConfig]], +) -> object: + from egress_gate.config import EgressGateConfig + + if not config_types: + raise ValueError("at least one gate config type must be registered") + registered_union = reduce(or_, config_types) + registered_config = getitem( + Annotated, + (registered_union, Field(discriminator="kind")), + ) + return getattr(EgressGateConfig, "__class_getitem__")(registered_config) + + +def _is_gate_type( + value: object, +) -> TypeGuard[type[Gate[GateConfig, GateResources | None]]]: + return isinstance(value, type) and issubclass(value, Gate) + + +def _is_egress_gate_config( + value: object, +) -> TypeGuard[EgressGateConfig[GateConfig]]: + from egress_gate.config import EgressGateConfig + + return isinstance(value, EgressGateConfig) + + +def _gate_kind(config_type: type[GateConfig]) -> str: + field = config_type.model_fields.get("kind") + if field is None: + raise GateRegistryError("gate config lacks a kind discriminator") + if get_origin(field.annotation) is not Literal: + raise GateRegistryError("gate kind must be one string Literal") + values = get_args(field.annotation) + if len(values) != 1 or not isinstance(values[0], str): + raise GateRegistryError("gate kind must be one string Literal") + gate_kind = values[0] + if _GATE_KIND_PATTERN.fullmatch(gate_kind) is None: + raise GateRegistryError("gate kind is invalid") + if not field.is_required(): + raise GateRegistryError("gate kind must be required") + return gate_kind + + +def _validate_common_gate_config_fields(config_type: type[GateConfig]) -> None: + for ancestor in config_type.__mro__: + if ancestor is GateConfig: + break + if "name" in ancestor.__dict__.get("__annotations__", {}): + raise GateRegistryError( + "gate config must inherit name without redefining it" + ) + else: + raise GateRegistryError("gate config type is invalid") + + for field_name in ("name", "kind"): + field = config_type.model_fields.get(field_name) + if field is None: + continue + aliases = (field.alias, field.validation_alias, field.serialization_alias) + if any(alias not in (None, field_name) for alias in aliases): + raise GateRegistryError( + "gate config name and kind must use their canonical field names" + ) + + +def _gate_description(gate_type: type[object]) -> str: + description = inspect.getdoc(gate_type) or "" + first_line = description.splitlines()[0] if description else "" + if len(first_line.encode("utf-8")) > 1024: + return "" + return first_line + + +def _schema_property_names(value: object) -> frozenset[str]: + names: set[str] = set() + if isinstance(value, Mapping): + properties = value.get("properties") + if isinstance(properties, Mapping): + names.update(key for key in properties if isinstance(key, str)) + for nested in value.values(): + names.update(_schema_property_names(nested)) + elif isinstance(value, list): + for nested in value: + names.update(_schema_property_names(nested)) + return frozenset(names) + + +def _validation_error_priority(error_type: object) -> int: + return { + "missing": 0, + "extra_forbidden": 1, + "union_tag_invalid": 2, + "union_tag_not_found": 2, + }.get(error_type, 3) + + +def _validation_error_category(error_type: object) -> PolicyValidationCategory: + return { + "missing": PolicyValidationCategory.REQUIRED_FIELD_MISSING, + "extra_forbidden": PolicyValidationCategory.UNKNOWN_FIELD, + "union_tag_invalid": PolicyValidationCategory.UNKNOWN_VARIANT, + "union_tag_not_found": PolicyValidationCategory.UNKNOWN_VARIANT, + }.get(error_type, PolicyValidationCategory.INVALID_VALUE) + + +_GATE_KIND_PATTERN = re.compile(r"[a-z][a-z0-9-]{0,127}\Z") + + +__all__ = [ + "GateDescription", + "GateRegistry", + "PolicyValidationCategory", + "PolicyValidationError", + "create_builtin_registry", +] diff --git a/projects/privacy-guard/src/privacy_guard/gateway_config.py b/projects/egress-gate/src/egress_gate/gateway_config.py similarity index 86% rename from projects/privacy-guard/src/privacy_guard/gateway_config.py rename to projects/egress-gate/src/egress_gate/gateway_config.py index 2170789c..c6b0eef8 100644 --- a/projects/privacy-guard/src/privacy_guard/gateway_config.py +++ b/projects/egress-gate/src/egress_gate/gateway_config.py @@ -8,12 +8,13 @@ import stat import tempfile import tomllib +from dataclasses import dataclass from enum import Enum from pathlib import Path class GatewayConfigUpdate(Enum): - """Result of writing one Privacy Guard middleware registration.""" + """Result of writing one Egress Gate middleware registration.""" CREATED = "created" ADDED = "added" @@ -22,7 +23,7 @@ class GatewayConfigUpdate(Enum): class GatewayConfigRemoval(Enum): - """Result of removing one Privacy Guard middleware registration.""" + """Result of removing one Egress Gate middleware registration.""" REMOVED = "removed" UNCHANGED = "unchanged" @@ -32,9 +33,17 @@ class GatewayConfigError(ValueError): """A safe, actionable gateway registration management error.""" +@dataclass(frozen=True) +class GatewayMiddlewareRegistration: + """One external middleware registration in an OpenShell gateway config.""" + + name: str + endpoint: str | None + + # Mirrors OpenShell's stable-identifier byte limit for external middleware # registrations. -MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES = 128 +MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES = 19 def default_gateway_config_path() -> Path: @@ -50,6 +59,41 @@ def default_gateway_config_path() -> Path: return Path.home() / ".config" / "openshell" / "gateway.toml" +def list_gateway_registrations( + path: Path, +) -> tuple[GatewayMiddlewareRegistration, ...]: + """List the external middleware registrations in an OpenShell gateway config.""" + try: + contents = path.read_text(encoding="utf-8") + except FileNotFoundError: + return () + except (OSError, UnicodeError) as error: + raise GatewayConfigError( + f"Could not read {path}. Check that the file is readable UTF-8 TOML." + ) from error + + if not contents.strip(): + return () + + registrations: list[GatewayMiddlewareRegistration] = [] + for entry in _middleware_entries(_load_gateway_config(contents, path), path): + name = entry.get("name") + endpoint = entry.get("grpc_endpoint") + if not isinstance(name, str) or not name: + raise GatewayConfigError( + f"{path} contains a middleware registration without a valid name." + ) + if endpoint is not None and not isinstance(endpoint, str): + raise GatewayConfigError( + f"The middleware registration {name!r} in {path} has an invalid " + "grpc_endpoint." + ) + registrations.append( + GatewayMiddlewareRegistration(name=name, endpoint=endpoint) + ) + return tuple(registrations) + + def update_gateway_config( path: Path, *, @@ -57,7 +101,7 @@ def update_gateway_config( host_ip: str, port: int, ) -> GatewayConfigUpdate: - """Add or update one named Privacy Guard middleware registration.""" + """Add or update one named Egress Gate middleware registration.""" validate_middleware_name(middleware_name) endpoint = f"http://{host_ip}:{port}" try: @@ -130,8 +174,7 @@ def remove_gateway_config( *, middleware_name: str, ) -> GatewayConfigRemoval: - """Remove one named Privacy Guard middleware registration.""" - validate_middleware_name(middleware_name) + """Remove one named Egress Gate middleware registration.""" try: original = path.read_text(encoding="utf-8") except FileNotFoundError: @@ -385,10 +428,12 @@ def _write_atomically(path: Path, contents: str) -> None: __all__ = [ "GatewayConfigError", + "GatewayMiddlewareRegistration", "GatewayConfigRemoval", "GatewayConfigUpdate", "MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES", "default_gateway_config_path", + "list_gateway_registrations", "remove_gateway_config", "update_gateway_config", "validate_middleware_name", diff --git a/projects/privacy-guard/src/privacy_guard/logging.py b/projects/egress-gate/src/egress_gate/logging.py similarity index 73% rename from projects/privacy-guard/src/privacy_guard/logging.py rename to projects/egress-gate/src/egress_gate/logging.py index 3489bebf..6459483f 100644 --- a/projects/privacy-guard/src/privacy_guard/logging.py +++ b/projects/egress-gate/src/egress_gate/logging.py @@ -1,16 +1,17 @@ -"""Native Python logging configuration for Privacy Guard.""" +"""Native Python logging configuration for Egress Gate.""" from __future__ import annotations import copy import logging +import os from dataclasses import dataclass from enum import StrEnum from typing import TextIO class ColorMode(StrEnum): - """When Privacy Guard should add ANSI colors to console logs.""" + """When Egress Gate should add ANSI colors to console logs.""" AUTO = "auto" ALWAYS = "always" @@ -19,7 +20,7 @@ class ColorMode(StrEnum): @dataclass(frozen=True) class LoggingConfig: - """Privacy Guard console logging settings.""" + """Egress Gate console logging settings.""" level: int | str = logging.INFO stream: TextIO | None = None @@ -30,44 +31,44 @@ class LoggingConfig: def get_logger(name: str) -> logging.Logger: - """Return a logger governed by Privacy Guard's shared configuration.""" + """Return a logger governed by Egress Gate's shared configuration.""" return logging.getLogger(name) def configure_logging( config: LoggingConfig = DEFAULT_LOGGING_CONFIG, ) -> None: - """Configure consistent console logging for the Privacy Guard package. + """Configure consistent console logging for the Egress Gate package. Repeated calls replace the handler installed by this function. Handlers installed by the containing application are left unchanged. """ - package_logger = get_logger("privacy_guard") + package_logger = get_logger("egress_gate") package_logger.setLevel(config.level) for handler in package_logger.handlers[:]: - if isinstance(handler, _PrivacyGuardStreamHandler): + if isinstance(handler, _EgressGateStreamHandler): package_logger.removeHandler(handler) handler.close() - handler = _PrivacyGuardStreamHandler(config.stream) - use_colors = ( - handler.stream.isatty() - if config.color_mode is ColorMode.AUTO - else config.color_mode is ColorMode.ALWAYS + handler = _EgressGateStreamHandler(config.stream) + use_colors = config.color_mode is ColorMode.ALWAYS or ( + config.color_mode is ColorMode.AUTO + and "NO_COLOR" not in os.environ + and handler.stream.isatty() ) - handler.setFormatter(_PrivacyGuardFormatter(use_colors=use_colors)) + handler.setFormatter(_EgressGateFormatter(use_colors=use_colors)) package_logger.addHandler(handler) package_logger.propagate = False def reset_logging() -> None: """Remove logging configuration installed by :func:`configure_logging`.""" - package_logger = get_logger("privacy_guard") + package_logger = get_logger("egress_gate") managed_handlers = [ handler for handler in package_logger.handlers - if isinstance(handler, _PrivacyGuardStreamHandler) + if isinstance(handler, _EgressGateStreamHandler) ] if not managed_handlers: return @@ -79,11 +80,11 @@ def reset_logging() -> None: package_logger.propagate = True -class _PrivacyGuardStreamHandler(logging.StreamHandler[TextIO]): - """Stream handler owned by Privacy Guard's logging configuration.""" +class _EgressGateStreamHandler(logging.StreamHandler[TextIO]): + """Stream handler owned by Egress Gate's logging configuration.""" -class _PrivacyGuardFormatter(logging.Formatter): +class _EgressGateFormatter(logging.Formatter): """Readable console formatter with optional level-aware color.""" def __init__(self, *, use_colors: bool) -> None: diff --git a/projects/egress-gate/src/egress_gate/request.py b/projects/egress-gate/src/egress_gate/request.py new file mode 100644 index 00000000..9fe191fe --- /dev/null +++ b/projects/egress-gate/src/egress_gate/request.py @@ -0,0 +1,190 @@ +"""Immutable request and mutation models shared by Egress Gate components.""" + +from __future__ import annotations + +from enum import StrEnum +from typing import Annotated, Literal, TypeAlias + +from pydantic import Field, field_validator, model_validator + +from egress_gate.base import StrictDomainModel +from egress_gate.constants import ( + MAX_BODY_BYTES, + MAX_HEADER_MUTATION_DATA_BYTES, + MAX_HEADER_MUTATIONS, + MAX_PROTO_CONTEXT_BYTES, + MAX_PROTO_HEADERS, + MAX_PROTO_HEADERS_BYTES, + MAX_PROTO_TARGET_BYTES, +) +from egress_gate.string_validators import ScalarString + +HeaderName = Annotated[ScalarString, Field(min_length=1)] +HeaderValue = ScalarString + + +class Process(StrictDomainModel): + """The originating workload process and its executable ancestry.""" + + binary: ScalarString + pid: int = Field(ge=0, le=2**32 - 1) + ancestors: tuple[ScalarString, ...] = () + + +class RequestContext(StrictDomainModel): + """Sandbox and request identity supplied by OpenShell.""" + + request_id: ScalarString + sandbox_id: ScalarString + originating_process: Process | None = None + + @model_validator(mode="after") + def _context_strings_are_bounded(self) -> RequestContext: + string_bytes = len(self.request_id.encode("utf-8")) + len( + self.sandbox_id.encode("utf-8") + ) + if self.originating_process is not None: + string_bytes += len(self.originating_process.binary.encode("utf-8")) + string_bytes += sum( + len(ancestor.encode("utf-8")) + for ancestor in self.originating_process.ancestors + ) + if string_bytes > MAX_PROTO_CONTEXT_BYTES: + raise ValueError("request context strings exceed the size limit") + return self + + +class HttpTarget(StrictDomainModel): + """The bounded destination and request target visible before credentials.""" + + scheme: ScalarString + host: ScalarString + port: int = Field(ge=0, le=2**32 - 1) + method: ScalarString + path: ScalarString + query: ScalarString + + @model_validator(mode="after") + def _target_strings_are_bounded(self) -> HttpTarget: + string_bytes = sum( + len(value.encode("utf-8")) + for value in ( + self.scheme, + self.host, + self.method, + self.path, + self.query, + ) + ) + if string_bytes > MAX_PROTO_TARGET_BYTES: + raise ValueError("request target strings exceed the size limit") + return self + + +class HttpHeader(StrictDomainModel): + """One ordered, repeated request-header field.""" + + name: HeaderName + value: HeaderValue + + +class HttpRequest(StrictDomainModel): + """The immutable OpenShell HTTP request exposed to a gate.""" + + context: RequestContext + target: HttpTarget + headers: tuple[HttpHeader, ...] = Field(max_length=MAX_PROTO_HEADERS) + body: bytes = Field(max_length=MAX_BODY_BYTES, repr=False) + + @field_validator("headers") + @classmethod + def _headers_are_bounded( + cls, value: tuple[HttpHeader, ...] + ) -> tuple[HttpHeader, ...]: + encoded_size = sum( + len(header.name.encode("utf-8")) + len(header.value.encode("utf-8")) + for header in value + ) + if encoded_size > MAX_PROTO_HEADERS_BYTES: + raise ValueError("request headers exceed the size limit") + return value + + +class ExistingHeaderAction(StrEnum): + """How a header write handles existing case-insensitive fields.""" + + APPEND = "append" + OVERWRITE = "overwrite" + SKIP = "skip" + + +class WriteHeaderMutation(StrictDomainModel): + """One ordered write operation proposed by a gate.""" + + kind: Literal["write"] + name: HeaderName + value: HeaderValue + on_existing: ExistingHeaderAction + + +class RemoveHeaderMutation(StrictDomainModel): + """One ordered removal operation proposed by a gate.""" + + kind: Literal["remove"] + name: HeaderName + + +HeaderMutation: TypeAlias = Annotated[ + WriteHeaderMutation | RemoveHeaderMutation, + Field(discriminator="kind"), +] + + +class RequestMutations(StrictDomainModel): + """Validated body and header mutations proposed by one gate.""" + + replacement_body: bytes | None = Field( + default=None, + max_length=MAX_BODY_BYTES, + repr=False, + ) + header_mutations: tuple[HeaderMutation, ...] = Field( + default=(), + max_length=MAX_HEADER_MUTATIONS, + ) + + @model_validator(mode="after") + def _mutations_are_bounded(self) -> RequestMutations: + data_size = sum( + len(mutation.name.encode("utf-8")) + + ( + len(mutation.value.encode("utf-8")) + if isinstance(mutation, WriteHeaderMutation) + else 0 + ) + for mutation in self.header_mutations + ) + if data_size > MAX_HEADER_MUTATION_DATA_BYTES: + raise ValueError("request mutation header data exceeds the size limit") + return self + + @property + def is_empty(self) -> bool: + """Whether this set proposes no request mutation.""" + return self.replacement_body is None and not self.header_mutations + + +__all__ = [ + "ExistingHeaderAction", + "HeaderMutation", + "HeaderName", + "HeaderValue", + "HttpHeader", + "HttpRequest", + "HttpTarget", + "Process", + "RemoveHeaderMutation", + "RequestContext", + "RequestMutations", + "WriteHeaderMutation", +] diff --git a/projects/egress-gate/src/egress_gate/request_processor.py b/projects/egress-gate/src/egress_gate/request_processor.py new file mode 100644 index 00000000..06e39618 --- /dev/null +++ b/projects/egress-gate/src/egress_gate/request_processor.py @@ -0,0 +1,402 @@ +"""Ordered gate execution over one mutable-current immutable request value.""" + +from __future__ import annotations + +from collections.abc import Sequence +from time import monotonic + +from pydantic import ValidationError + +from egress_gate.config import DefaultDecision, EgressGateConfig +from egress_gate.constants import ( + DEFAULT_DENY_REASON_CODE, + LIMIT_REASON_CODE, + MAX_FINDING_COUNT, + MAX_PROTO_FINDING_BYTES, + MAX_PROTO_FINDING_GROUPS, +) +from egress_gate.errors import ( + EgressGateError, + ErrorCode, + GateConfigurationError, + GateContractError, + GateError, + GateExecutionError, + GateInputError, + GateLimitExceededError, + TimeoutExpiredError, +) +from egress_gate.gates.base import Gate, GateConfig, GateResources +from egress_gate.logging import get_logger +from egress_gate.request import ( + ExistingHeaderAction, + HttpHeader, + HttpRequest, + RemoveHeaderMutation, + RequestMutations, + WriteHeaderMutation, +) +from egress_gate.result import ( + DecisionSource, + DecisionSourceKind, + EgressDecision, + EgressResult, + Finding, + GateControl, + GateDecisionSource, + GateTrace, + MutationKind, + PipelineDefaultDecisionSource, + RuntimeLimitDecisionSource, + SourcedFinding, +) +from egress_gate.string_validators import validate_scalar_string +from egress_gate.timeout import Timeout + + +class RequestProcessor: + """Run configured gates in order over the current request revision.""" + + def __init__( + self, + config: EgressGateConfig[GateConfig], + configured_gates: Sequence[ + tuple[str, str, Gate[GateConfig, GateResources | None]] + ], + *, + policy_fingerprint: str | None = None, + ) -> None: + gates = tuple(configured_gates) + configured_names = tuple(name for name, _, _ in gates) + configured_types = tuple(gate_type for _, gate_type, _ in gates) + policy_names = tuple(item.name for item in config.gates) + policy_types = tuple(getattr(item, "kind", None) for item in config.gates) + if configured_names != policy_names or configured_types != policy_types: + raise ValueError("configured gates do not match the policy") + if not gates: + raise ValueError("at least one configured gate is required") + if any(not name for name in configured_names): + raise ValueError("gate names must be non-empty") + if len(configured_names) != len(set(configured_names)): + raise ValueError("gate names must be unique") + if policy_fingerprint is not None: + try: + policy_fingerprint = validate_scalar_string(policy_fingerprint) + except ValueError: + raise ValueError("policy fingerprint must be a string") from None + self._config = config + self._gates = gates + self._policy_fingerprint = policy_fingerprint + + def process(self, request: HttpRequest, *, timeout: Timeout) -> EgressResult: + """Evaluate one request and return an atomic final domain result.""" + if not isinstance(request, HttpRequest): + raise EgressGateError(ErrorCode.REQUEST_ENVELOPE_INVALID) + if not isinstance(timeout, Timeout): + raise EgressGateError(ErrorCode.GATE_OUTPUT_INVALID) + current_request = request + accumulated_mutations = RequestMutations() + sourced_findings: list[SourcedFinding] = [] + traces: list[GateTrace] = [] + + try: + for gate_name, gate_type, gate in self._gates: + timeout.raise_if_expired() + started = monotonic() + evaluation = gate.evaluate(current_request, timeout=timeout) + mutation_kinds = _mutation_kinds(evaluation.request_mutations) + trace_finding_count = sum( + finding.count for finding in evaluation.findings + ) + if trace_finding_count > MAX_FINDING_COUNT: + raise GateLimitExceededError( + "gate trace finding count exceeds the limit" + ) + traces.append( + GateTrace( + gate_name=gate_name, + gate_type=gate_type, + control=evaluation.control, + duration_ms=max(0.0, (monotonic() - started) * 1000), + finding_count=trace_finding_count, + mutation_kinds=mutation_kinds, + ) + ) + _append_findings( + sourced_findings, + gate_name=gate_name, + findings=evaluation.findings, + ) + if len(sourced_findings) > MAX_PROTO_FINDING_GROUPS: + raise GateLimitExceededError( + "result finding groups exceed the limit" + ) + + if evaluation.control is GateControl.DENY: + return _result( + decision=EgressDecision.DENY, + source=GateDecisionSource( + kind=DecisionSourceKind.GATE, + gate_name=gate_name, + gate_type=gate_type, + ), + findings=sourced_findings, + reason_code=evaluation.reason_code, + fingerprint=self._policy_fingerprint, + traces=traces, + ) + if evaluation.control is GateControl.ALLOW: + return _result( + decision=EgressDecision.ALLOW, + source=GateDecisionSource( + kind=DecisionSourceKind.GATE, + gate_name=gate_name, + gate_type=gate_type, + ), + request_mutations=accumulated_mutations, + findings=sourced_findings, + fingerprint=self._policy_fingerprint, + traces=traces, + ) + if not evaluation.request_mutations.is_empty: + current_request = apply_request_mutations( + current_request, + evaluation.request_mutations, + ) + accumulated_mutations = _compose_request_mutations( + accumulated_mutations, + evaluation.request_mutations, + ) + timeout.raise_if_expired() + except TimeoutExpiredError: + _LOGGER.info("egress_gate_processing_limit kind=timeout") + return _runtime_limit_result(self._policy_fingerprint) + except GateLimitExceededError: + _LOGGER.info("egress_gate_processing_limit kind=resource") + return _runtime_limit_result(self._policy_fingerprint) + except GateInputError: + raise EgressGateError(ErrorCode.BODY_ENCODING_INVALID) from None + except GateConfigurationError: + raise EgressGateError(ErrorCode.CONFIG_INVALID) from None + except GateContractError: + raise EgressGateError(ErrorCode.GATE_OUTPUT_INVALID) from None + except GateExecutionError: + raise EgressGateError(ErrorCode.GATE_EXECUTION_FAILED) from None + except EgressGateError: + raise + except (ValidationError, ValueError): + raise EgressGateError(ErrorCode.GATE_OUTPUT_INVALID) from None + except GateError: + raise EgressGateError(ErrorCode.GATE_EXECUTION_FAILED) from None + except Exception: + raise EgressGateError(ErrorCode.GATE_EXECUTION_FAILED) from None + + if self._config.default_decision is DefaultDecision.ALLOW: + result = _result( + decision=EgressDecision.ALLOW, + source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + request_mutations=accumulated_mutations, + findings=sourced_findings, + fingerprint=self._policy_fingerprint, + traces=traces, + ) + else: + result = _result( + decision=EgressDecision.DENY, + source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + findings=sourced_findings, + reason_code=DEFAULT_DENY_REASON_CODE, + fingerprint=self._policy_fingerprint, + traces=traces, + ) + return result + + +def apply_request_mutations( + request: HttpRequest, + request_mutations: RequestMutations, +) -> HttpRequest: + """Apply validated mutations to the current request in operation order.""" + if not isinstance(request, HttpRequest) or not isinstance( + request_mutations, RequestMutations + ): + raise GateContractError("request mutation input is invalid") + body = ( + request.body + if request_mutations.replacement_body is None + else request_mutations.replacement_body + ) + headers = list(request.headers) + for mutation in request_mutations.header_mutations: + if isinstance(mutation, WriteHeaderMutation): + _validate_write_mutation(mutation) + matching = _header_indexes(headers, mutation.name) + if mutation.on_existing is ExistingHeaderAction.OVERWRITE: + headers = [ + header + for index, header in enumerate(headers) + if index not in matching + ] + headers.append(HttpHeader(name=mutation.name, value=mutation.value)) + elif mutation.on_existing is ExistingHeaderAction.SKIP and matching: + continue + else: + headers.append(HttpHeader(name=mutation.name, value=mutation.value)) + elif isinstance(mutation, RemoveHeaderMutation): + _validate_remove_mutation(mutation) + matching = _header_indexes(headers, mutation.name) + headers = [ + header for index, header in enumerate(headers) if index not in matching + ] + else: + raise GateContractError("request mutation is invalid") + try: + return HttpRequest( + context=request.context, + target=request.target, + headers=tuple(headers), + body=body, + ) + except (TypeError, ValueError, ValidationError): + raise GateLimitExceededError( + "request mutation exceeds a domain limit" + ) from None + + +def _append_findings( + output: list[SourcedFinding], + *, + gate_name: str, + findings: tuple[Finding, ...], +) -> None: + for finding in findings: + for index, sourced in enumerate(output): + if sourced.source_gate != gate_name or sourced.finding.type != finding.type: + continue + if ( + sourced.finding.label != finding.label + or sourced.finding.confidence != finding.confidence + or sourced.finding.severity != finding.severity + ): + continue + total = sourced.finding.count + finding.count + if total > MAX_FINDING_COUNT: + raise GateLimitExceededError("finding count exceeds the limit") + combined_finding = sourced.finding.model_copy(update={"count": total}) + if combined_finding.encoded_size_bytes > MAX_PROTO_FINDING_BYTES: + raise GateLimitExceededError( + "aggregated finding exceeds the encoded size limit" + ) + output[index] = SourcedFinding( + source_gate=gate_name, + finding=combined_finding, + ) + break + else: + output.append(SourcedFinding(source_gate=gate_name, finding=finding)) + + +def _compose_request_mutations( + first: RequestMutations, + second: RequestMutations, +) -> RequestMutations: + try: + return RequestMutations( + replacement_body=( + second.replacement_body + if second.replacement_body is not None + else first.replacement_body + ), + header_mutations=first.header_mutations + second.header_mutations, + ) + except (TypeError, ValueError, ValidationError): + raise GateLimitExceededError( + "composed request mutations exceed a pipeline processor limit" + ) from None + + +def _mutation_kinds(request_mutations: RequestMutations) -> tuple[MutationKind, ...]: + kinds: list[MutationKind] = [] + if request_mutations.replacement_body is not None: + kinds.append(MutationKind.BODY) + if request_mutations.header_mutations: + kinds.append(MutationKind.HEADERS) + return tuple(kinds) + + +def _result( + *, + decision: EgressDecision, + source: DecisionSource, + request_mutations: RequestMutations | None = None, + findings: Sequence[SourcedFinding] = (), + reason_code: str | None = None, + fingerprint: str | None, + traces: Sequence[GateTrace] = (), +) -> EgressResult: + return EgressResult( + decision=decision, + decision_source=source, + request_mutations=( + RequestMutations() if request_mutations is None else request_mutations + ), + findings=tuple(findings), + reason_code=reason_code, + policy_fingerprint=fingerprint, + traces=tuple(traces), + ) + + +def _runtime_limit_result(fingerprint: str | None) -> EgressResult: + return _result( + decision=EgressDecision.DENY, + source=RuntimeLimitDecisionSource(kind=DecisionSourceKind.RUNTIME_LIMIT), + reason_code=LIMIT_REASON_CODE, + fingerprint=fingerprint, + ) + + +def _header_indexes(headers: Sequence[HttpHeader], name: str) -> set[int]: + lowered = name.lower() + return { + index for index, header in enumerate(headers) if header.name.lower() == lowered + } + + +def _validate_write_mutation(mutation: WriteHeaderMutation) -> None: + if not mutation.name.lower().startswith("x-openshell-middleware-"): + raise GateContractError("header writes require the middleware namespace") + + +def _validate_remove_mutation(mutation: RemoveHeaderMutation) -> None: + if mutation.name.lower() in _PROTECTED_HEADER_NAMES: + raise GateContractError("protected headers cannot be removed") + + +_PROTECTED_HEADER_NAMES = frozenset( + { + "authorization", + "connection", + "content-length", + "host", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "proxy-connection", + "te", + "trailer", + "transfer-encoding", + "upgrade", + } +) +_LOGGER = get_logger(__name__) + + +__all__ = [ + "RequestProcessor", + "apply_request_mutations", +] diff --git a/projects/egress-gate/src/egress_gate/result.py b/projects/egress-gate/src/egress_gate/result.py new file mode 100644 index 00000000..8266f7af --- /dev/null +++ b/projects/egress-gate/src/egress_gate/result.py @@ -0,0 +1,309 @@ +"""Immutable gate evaluations and Egress Gate result models. + +These models deliberately contain no protobuf or gRPC types. ``SourcedFinding`` +keeps gate provenance inside the pipeline processor; the current OpenShell wire +contract serializes only the five fields on ``Finding``. +""" + +from __future__ import annotations + +from enum import StrEnum +from typing import Annotated, Literal, Self, TypeAlias + +from pydantic import ( + Field, + model_validator, +) + +from egress_gate.base import StrictDomainModel +from egress_gate.constants import ( + DEFAULT_DENY_REASON_CODE, + LIMIT_REASON_CODE, + MAX_FINDING_COUNT, + MAX_GATE_TRACES, + MAX_PROTO_FINDING_BYTES, + MAX_PROTO_FINDING_GROUPS, + MAX_RESULT_METADATA_BYTES, + MAX_RESULT_METADATA_ENTRIES, + MAX_TRACE_MUTATION_KINDS, + REASON_CODE_PATTERN, +) +from egress_gate.request import RequestMutations +from egress_gate.string_validators import BoundedMetadataString + +ReasonCode = Annotated[str, Field(pattern=REASON_CODE_PATTERN)] +FindingType: TypeAlias = BoundedMetadataString +FindingLabel: TypeAlias = BoundedMetadataString +GateName: TypeAlias = BoundedMetadataString +GateType: TypeAlias = BoundedMetadataString + + +class GateControl(StrEnum): + """The complete control result of one gate invocation.""" + + PROCEED = "proceed" + ALLOW = "allow" + DENY = "deny" + + +class EgressDecision(StrEnum): + """The final OpenShell request disposition.""" + + ALLOW = "allow" + DENY = "deny" + + +class DecisionSourceKind(StrEnum): + """The owner of the final decision.""" + + GATE = "gate" + PIPELINE_DEFAULT = "pipeline_default" + RUNTIME_LIMIT = "runtime_limit" + + +class MutationKind(StrEnum): + """The kinds of mutation represented in a gate trace.""" + + BODY = "body" + HEADERS = "headers" + + +class Finding(StrictDomainModel): + """One audit-safe observation matching the current OpenShell wire shape.""" + + type: FindingType + label: FindingLabel + count: int = Field(default=1, ge=1, le=MAX_FINDING_COUNT) + confidence: BoundedMetadataString | None = None + severity: BoundedMetadataString | None = None + + @property + def encoded_size_bytes(self) -> int: + """Return the size of this finding in the OpenShell wire format.""" + encoded_size = ( + _encoded_string_field_size(self.type) + + _encoded_string_field_size(self.label) + + 1 + + _varint_size(self.count) + ) + if self.confidence is not None: + encoded_size += _encoded_string_field_size(self.confidence) + if self.severity is not None: + encoded_size += _encoded_string_field_size(self.severity) + return encoded_size + + @model_validator(mode="after") + def _wire_size_is_bounded(self) -> Self: + if self.encoded_size_bytes > MAX_PROTO_FINDING_BYTES: + raise ValueError("finding exceeds the encoded size limit") + return self + + +class FindingTypeDefinition(StrictDomainModel): + """A processor-owned declaration for one possible finding type.""" + + type: FindingType + + +class SourcedFinding(StrictDomainModel): + """Runtime-internal finding provenance excluded from wire serialization.""" + + source_gate: GateName + finding: Finding + + +class GateDecisionSource(StrictDomainModel): + """A final decision made by one configured gate.""" + + kind: Literal[DecisionSourceKind.GATE] + gate_name: GateName + gate_type: GateType + + +class PipelineDefaultDecisionSource(StrictDomainModel): + """A final decision made by the pipeline default.""" + + kind: Literal[DecisionSourceKind.PIPELINE_DEFAULT] + + +class RuntimeLimitDecisionSource(StrictDomainModel): + """A fail-closed decision caused by a pipeline processor safety limit.""" + + kind: Literal[DecisionSourceKind.RUNTIME_LIMIT] + + +DecisionSource: TypeAlias = Annotated[ + GateDecisionSource | PipelineDefaultDecisionSource | RuntimeLimitDecisionSource, + Field(discriminator="kind"), +] + + +class GateEvaluation(StrictDomainModel): + """Validated output of one gate invocation.""" + + control: GateControl + request_mutations: RequestMutations = Field(default_factory=RequestMutations) + findings: tuple[Finding, ...] = Field( + default=(), + max_length=MAX_PROTO_FINDING_GROUPS, + ) + reason_code: ReasonCode | None = None + + @model_validator(mode="after") + def _control_contract_is_valid(self) -> Self: + if self.control is GateControl.PROCEED: + if self.reason_code is not None: + raise ValueError("proceed evaluations cannot carry a reason code") + return self + if not self.request_mutations.is_empty: + raise ValueError("terminal evaluations cannot carry request mutations") + if self.control is GateControl.ALLOW and self.reason_code is not None: + raise ValueError("allow evaluations cannot carry a reason code") + if self.control is GateControl.DENY and self.reason_code is None: + raise ValueError("deny evaluations require a reason code") + return self + + @classmethod + def proceed( + cls, + *, + request_mutations: RequestMutations | None = None, + findings: tuple[Finding, ...] = (), + ) -> Self: + """Create a non-terminal evaluation.""" + return cls( + control=GateControl.PROCEED, + request_mutations=( + RequestMutations() if request_mutations is None else request_mutations + ), + findings=findings, + ) + + @classmethod + def allow(cls, *, findings: tuple[Finding, ...] = ()) -> Self: + """Create a terminal allow evaluation.""" + return cls(control=GateControl.ALLOW, findings=findings) + + @classmethod + def deny( + cls, + reason_code: ReasonCode, + *, + findings: tuple[Finding, ...] = (), + ) -> Self: + """Create a terminal deny evaluation.""" + return cls( + control=GateControl.DENY, + reason_code=reason_code, + findings=findings, + ) + + +class GateTrace(StrictDomainModel): + """Content-safe processor trace data for one configured gate.""" + + gate_name: GateName + gate_type: GateType + control: GateControl + duration_ms: float = Field(ge=0, allow_inf_nan=False) + finding_count: int = Field(ge=0, le=MAX_FINDING_COUNT) + mutation_kinds: tuple[MutationKind, ...] = Field( + default=(), + max_length=MAX_TRACE_MUTATION_KINDS, + ) + + +class ResultMetadata(StrictDomainModel): + """One bounded processor-owned result metadata entry.""" + + key: BoundedMetadataString + value: BoundedMetadataString + + +class EgressResult(StrictDomainModel): + """Final domain result returned after pipeline execution.""" + + decision: EgressDecision + decision_source: DecisionSource + request_mutations: RequestMutations = Field(default_factory=RequestMutations) + findings: tuple[SourcedFinding, ...] = Field( + default=(), + max_length=MAX_PROTO_FINDING_GROUPS, + ) + reason_code: ReasonCode | None = None + metadata: tuple[ResultMetadata, ...] = Field( + default=(), + max_length=MAX_RESULT_METADATA_ENTRIES, + ) + policy_fingerprint: BoundedMetadataString | None = None + traces: tuple[GateTrace, ...] = Field( + default=(), + max_length=MAX_GATE_TRACES, + ) + + @model_validator(mode="after") + def _result_contract_is_valid(self) -> Self: + metadata_bytes = sum( + len(item.key.encode("utf-8")) + len(item.value.encode("utf-8")) + for item in self.metadata + ) + if metadata_bytes > MAX_RESULT_METADATA_BYTES: + raise ValueError("result metadata exceeds the size limit") + source_kind = self.decision_source.kind + if self.decision is EgressDecision.DENY: + if not self.request_mutations.is_empty: + raise ValueError("denied results cannot carry request mutations") + if self.reason_code is None: + raise ValueError("denied results require a reason code") + elif self.reason_code is not None: + raise ValueError("allowed results cannot carry a reason code") + + if source_kind is DecisionSourceKind.RUNTIME_LIMIT: + if self.decision is not EgressDecision.DENY: + raise ValueError("runtime-limit results must deny") + if self.reason_code != LIMIT_REASON_CODE: + raise ValueError("runtime-limit results require the limit reason") + elif source_kind is DecisionSourceKind.PIPELINE_DEFAULT: + if self.decision is EgressDecision.DENY: + if self.reason_code != DEFAULT_DENY_REASON_CODE: + raise ValueError("default denies require the default reason") + elif self.reason_code is not None: + raise ValueError("default allows cannot carry a reason code") + return self + + +def _varint_size(value: int) -> int: + size = 1 + while value >= 0x80: + value >>= 7 + size += 1 + return size + + +def _encoded_string_field_size(value: str) -> int: + length = len(value.encode("utf-8")) + return 1 + _varint_size(length) + length + + +__all__ = [ + "DecisionSource", + "DecisionSourceKind", + "EgressDecision", + "EgressResult", + "Finding", + "FindingTypeDefinition", + "FindingLabel", + "FindingType", + "GateControl", + "GateDecisionSource", + "GateEvaluation", + "GateName", + "GateTrace", + "GateType", + "MutationKind", + "PipelineDefaultDecisionSource", + "ReasonCode", + "ResultMetadata", + "RuntimeLimitDecisionSource", + "SourcedFinding", +] diff --git a/projects/egress-gate/src/egress_gate/service/__init__.py b/projects/egress-gate/src/egress_gate/service/__init__.py new file mode 100644 index 00000000..dd8fcc02 --- /dev/null +++ b/projects/egress-gate/src/egress_gate/service/__init__.py @@ -0,0 +1,6 @@ +"""gRPC transport and servicer for the Egress Gate middleware.""" + +from egress_gate.service.server import EgressGateServer +from egress_gate.service.servicer import EgressGateMiddleware + +__all__ = ["EgressGateMiddleware", "EgressGateServer"] diff --git a/projects/egress-gate/src/egress_gate/service/server.py b/projects/egress-gate/src/egress_gate/service/server.py new file mode 100644 index 00000000..cd5857a4 --- /dev/null +++ b/projects/egress-gate/src/egress_gate/service/server.py @@ -0,0 +1,183 @@ +"""Programmatic Egress Gate gRPC server lifecycle.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable +from typing import Protocol, runtime_checkable + +import grpc +from google.protobuf.message import DecodeError + +from egress_gate.bindings import supervisor_middleware_pb2_grpc as pb2_grpc +from egress_gate.constants import ( + DEFAULT_TIMEOUT_SECONDS, + MAX_CONCURRENT_RPCS, + MAX_RECEIVE_MESSAGE_BYTES, +) +from egress_gate.errors import EgressGateError, ErrorCode +from egress_gate.gates.registry import GateRegistry +from egress_gate.logging import get_logger +from egress_gate.service.servicer import EgressGateMiddleware + +DEFAULT_LISTEN_ADDRESS = "127.0.0.1:50051" + + +class EgressGateServer: + """One-shot programmatic server for an application gate registry.""" + + def __init__( + self, + registry: GateRegistry, + *, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, + ) -> None: + self._middleware = EgressGateMiddleware( + registry, + timeout_seconds=timeout_seconds, + ) + + def serve_sync(self, listen: str = DEFAULT_LISTEN_ADDRESS) -> None: + """Serve synchronously until termination.""" + try: + asyncio.run(self.serve_async(listen)) + except KeyboardInterrupt: + return + + async def serve_async(self, listen: str = DEFAULT_LISTEN_ADDRESS) -> None: + """Serve asynchronously until termination, then close owned resources.""" + server = _create_grpc_server(self._middleware) + try: + try: + requested_port = _validated_listen_port(listen) + bound_port = server.add_insecure_port(listen) + if bound_port != requested_port: + raise EgressGateError(ErrorCode.SERVER_BIND_FAILED) + _LOGGER.info("egress_gate_server_bound listen=%r", listen) + await server.start() + except RuntimeError: + raise EgressGateError(ErrorCode.SERVER_BIND_FAILED) from None + await server.wait_for_termination() + finally: + try: + await _stop_grpc_server(server) + finally: + await self._middleware.close() + + +_LOGGER = get_logger(__name__) + + +def _create_grpc_server( + middleware: EgressGateMiddleware, +) -> grpc.aio.Server: + server = grpc.aio.server( + interceptors=(_MalformedProtobufInterceptor(),), + maximum_concurrent_rpcs=MAX_CONCURRENT_RPCS, + options=(("grpc.max_receive_message_length", MAX_RECEIVE_MESSAGE_BYTES),), + ) + pb2_grpc.add_SupervisorMiddlewareServicer_to_server(middleware, server) + return server + + +class _MalformedProtobufInterceptor(grpc.aio.ServerInterceptor): + """Map protobuf decoding failures to the public invalid-input contract.""" + + async def intercept_service( + self, + continuation: Callable[ + [grpc.HandlerCallDetails], Awaitable[grpc.RpcMethodHandler] + ], + handler_call_details: grpc.HandlerCallDetails, + ) -> grpc.RpcMethodHandler: + generic_handler = await continuation(handler_call_details) + if not isinstance(generic_handler, _UnaryUnaryRpcMethodHandler): + return generic_handler + handler = generic_handler + if handler.request_deserializer is None or handler.unary_unary is None: + return generic_handler + + deserialize = handler.request_deserializer + unary_unary = handler.unary_unary + + def deserialize_safely(data: bytes) -> object: + try: + return deserialize(data) + except DecodeError: + return _MALFORMED_PROTOBUF + + async def invoke_safely( + request: object, + context: grpc.aio.ServicerContext[object, object], + ) -> object: + if request is _MALFORMED_PROTOBUF: + await context.abort( + grpc.StatusCode.INVALID_ARGUMENT, + str(EgressGateError(ErrorCode.REQUEST_PROTOBUF_INVALID)), + ) + return await unary_unary(request, context) + + return grpc.unary_unary_rpc_method_handler( + invoke_safely, + request_deserializer=deserialize_safely, + response_serializer=handler.response_serializer, + ) + + +async def _stop_grpc_server(server: grpc.aio.Server) -> None: + shutdown = asyncio.create_task(server.stop(grace=0)) + try: + await asyncio.shield(shutdown) + except asyncio.CancelledError: + if not shutdown.done(): + await shutdown + raise + + +_MALFORMED_PROTOBUF = object() + + +@runtime_checkable +class _UnaryUnaryRpcMethodHandler(Protocol): + request_deserializer: Callable[[bytes], object] | None + response_serializer: Callable[[object], bytes] | None + unary_unary: ( + Callable[[object, grpc.aio.ServicerContext[object, object]], Awaitable[object]] + | None + ) + + +def _validated_listen_port(listen: str) -> int: + if not isinstance(listen, str): + raise EgressGateError(ErrorCode.SERVER_BIND_FAILED) + if listen.startswith("["): + closing_bracket = listen.rfind("]") + if ( + closing_bracket < 2 + or listen[closing_bracket + 1 : closing_bracket + 2] != ":" + ): + raise EgressGateError(ErrorCode.SERVER_BIND_FAILED) + host = listen[1:closing_bracket] + port_text = listen[closing_bracket + 2 :] + else: + host, separator, port_text = listen.rpartition(":") + if not separator or not host or ":" in host: + raise EgressGateError(ErrorCode.SERVER_BIND_FAILED) + if ( + not host + or not port_text + or len(port_text) > 5 + or not port_text.isascii() + or not port_text.isdecimal() + ): + raise EgressGateError(ErrorCode.SERVER_BIND_FAILED) + port = int(port_text) + if not 1 <= port <= 65_535: + raise EgressGateError(ErrorCode.SERVER_BIND_FAILED) + return port + + +__all__ = [ + "DEFAULT_LISTEN_ADDRESS", + "EgressGateServer", +] diff --git a/projects/egress-gate/src/egress_gate/service/servicer.py b/projects/egress-gate/src/egress_gate/service/servicer.py new file mode 100644 index 00000000..965365be --- /dev/null +++ b/projects/egress-gate/src/egress_gate/service/servicer.py @@ -0,0 +1,568 @@ +"""gRPC boundary for the protobuf-free Egress Gate pipeline processor.""" + +from __future__ import annotations + +import asyncio +import json +import math +import time +from collections.abc import Callable, Iterable +from concurrent.futures import Future, ThreadPoolExecutor +from threading import Lock +from typing import Never, Protocol, TypedDict, TypeVar + +import grpc +from google.protobuf import json_format +from google.protobuf.message import Message + +from egress_gate.bindings import supervisor_middleware_pb2 as pb2 +from egress_gate.bindings import supervisor_middleware_pb2_grpc as pb2_grpc +from egress_gate.config import EgressGateConfig +from egress_gate.constants import ( + BLOCK_REASON, + DEFAULT_TIMEOUT_SECONDS, + LIMIT_REASON, + LIMIT_REASON_CODE, + MAX_BODY_BYTES, + MAX_CONCURRENT_PROCESSING, + MAX_PROTO_CONFIG_BYTES, + MAX_PROTO_CONTEXT_BYTES, + MAX_PROTO_FINDING_BYTES, + MAX_PROTO_FINDING_GROUPS, + MAX_PROTO_HEADERS, + MAX_PROTO_HEADERS_BYTES, + MAX_PROTO_TARGET_BYTES, + REASON_CODE_PATTERN, + SERVICE_NAME, + SERVICE_VERSION, +) +from egress_gate.errors import ( + EgressGateError, + ErrorCode, + ErrorKind, + GateConfigurationError, + GateRegistryError, + TimeoutExpiredError, +) +from egress_gate.gates.base import GateConfig +from egress_gate.gates.registry import GateRegistry +from egress_gate.logging import get_logger +from egress_gate.request import ( + HeaderMutation, + HttpHeader, + HttpRequest, + HttpTarget, + Process, + RemoveHeaderMutation, + RequestContext, + WriteHeaderMutation, +) +from egress_gate.request_processor import RequestProcessor +from egress_gate.result import ( + DecisionSourceKind, + EgressDecision, + EgressResult, + SourcedFinding, +) +from egress_gate.string_validators import validate_bounded_metadata_string +from egress_gate.timeout import Timeout, validate_timeout_seconds + + +class EgressGateMiddleware(pb2_grpc.SupervisorMiddlewareServicer): + """Validate, prepare, resolve, and run Egress Gate policies.""" + + def __init__( + self, + registry: GateRegistry, + *, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, + ) -> None: + registry.configuration_json_schema() + self._registry = registry + self._timeout_seconds = validate_timeout_seconds(timeout_seconds) + self._policy = _ActivePolicy(registry) + self._processing_slots = asyncio.Semaphore(MAX_CONCURRENT_PROCESSING) + self._processing_executor = ThreadPoolExecutor( + max_workers=MAX_CONCURRENT_PROCESSING, + thread_name_prefix="egress-gate-processing", + ) + + async def close(self) -> None: + """Wait for in-flight synchronous gates during shutdown.""" + self._processing_executor.shutdown(wait=True, cancel_futures=True) + self._policy.clear() + + async def Describe( + self, + request: object, + context: grpc.aio.ServicerContext[object, pb2.MiddlewareManifest], + ) -> pb2.MiddlewareManifest: + """Advertise the binding and its complete policy schema.""" + return pb2.MiddlewareManifest( + name=SERVICE_NAME, + service_version=SERVICE_VERSION, + bindings=[ + pb2.MiddlewareBinding( + operation=pb2.SUPERVISOR_MIDDLEWARE_OPERATION_HTTP_REQUEST, + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, + max_body_bytes=MAX_BODY_BYTES, + ) + ], + ) + + async def ValidateConfig( + self, + request: pb2.ValidateConfigRequest, + context: grpc.aio.ServicerContext[ + pb2.ValidateConfigRequest, + pb2.ValidateConfigResponse, + ], + ) -> pb2.ValidateConfigResponse: + """Validate expanded configuration without preparing processor state.""" + return await self._run_in_worker(lambda: self._validate_config(request)) + + async def EvaluateHttpRequest( + self, + request: pb2.HttpRequestEvaluation, + context: grpc.aio.ServicerContext[ + pb2.HttpRequestEvaluation, + pb2.HttpRequestResult, + ], + ) -> pb2.HttpRequestResult: + """Resolve the prepared pipeline and evaluate one current request.""" + return await self._evaluate_rpc(request, context) + + def _validate_config( + self, + request: pb2.ValidateConfigRequest, + ) -> pb2.ValidateConfigResponse: + try: + if request.config.ByteSize() > MAX_PROTO_CONFIG_BYTES: + raise EgressGateError(ErrorCode.CONFIG_INVALID) + self._registry.validate_config(_mapping_from_proto(request.config)) + except EgressGateError as error: + return pb2.ValidateConfigResponse(valid=False, reason=str(error)) + except Exception: + error = EgressGateError(ErrorCode.UNEXPECTED_SERVICE_FAILURE) + return pb2.ValidateConfigResponse(valid=False, reason=str(error)) + return pb2.ValidateConfigResponse(valid=True) + + async def _evaluate_rpc( + self, + request: pb2.HttpRequestEvaluation, + context: _AbortContext, + ) -> pb2.HttpRequestResult: + started = time.monotonic() + request_id = _request_id_for_logging(request.context.request_id) + failure: EgressGateError | None = None + action = "error" + finding_count = 0 + source_kind = "none" + try: + timeout = Timeout.from_seconds(self._timeout_seconds) + response, source_kind = await self._evaluate_http_request( + request, + timeout, + ) + action = "allow" if response.decision == pb2.DECISION_ALLOW else "deny" + finding_count = sum(finding.count for finding in response.findings) + return response + except TimeoutExpiredError: + response = _limit_deny() + action = "deny" + source_kind = DecisionSourceKind.RUNTIME_LIMIT.value + return response + except EgressGateError as error: + failure = error + except Exception: + failure = EgressGateError(ErrorCode.UNEXPECTED_SERVICE_FAILURE) + finally: + log_extra = _evaluation_log_extra( + request_id=request_id, + started=started, + action=action, + finding_count=finding_count, + source_kind=source_kind, + failure=failure, + ) + _LOGGER.info( + "egress_gate_evaluation request_id=%s duration_ms=%.3f " + "action=%s finding_count=%d decision_source_kind=%s error_code=%s", + _request_id_for_log_message(log_extra["request_id"]), + log_extra["duration_ms"], + log_extra["action"], + log_extra["finding_count"], + log_extra["decision_source_kind"], + log_extra["error_code"] or "none", + extra=log_extra, + ) + status = ( + grpc.StatusCode.INVALID_ARGUMENT + if failure.kind is ErrorKind.INVALID_INPUT + else grpc.StatusCode.INTERNAL + ) + await context.abort(status, str(failure)) + + async def _evaluate_http_request( + self, + request: pb2.HttpRequestEvaluation, + timeout: Timeout, + ) -> tuple[pb2.HttpRequestResult, str]: + if request.phase != pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS: + raise EgressGateError(ErrorCode.REQUEST_PHASE_INVALID) + if len(request.body) > MAX_BODY_BYTES: + raise EgressGateError(ErrorCode.REQUEST_BODY_TOO_LARGE) + _validate_evaluation_envelope(request) + result = await self._run_in_worker( + lambda: self._prepare_and_process( + request, + timeout, + ), + timeout=timeout, + ) + timeout.raise_if_expired() + response, source_kind = _result_to_proto(result) + timeout.raise_if_expired() + return response, source_kind + + def _prepare_and_process( + self, + request: pb2.HttpRequestEvaluation, + timeout: Timeout, + ) -> EgressResult: + domain_request = _request_from_proto(request) + values = _mapping_from_proto(request.config) + processor = self._policy.processor_for( + values, + timeout=timeout, + ) + return processor.process(domain_request, timeout=timeout) + + async def _run_in_worker( + self, + operation: Callable[[], _WorkerResultT], + *, + timeout: Timeout | None = None, + ) -> _WorkerResultT: + """Run one bounded synchronous operation without blocking the event loop.""" + try: + if timeout is None: + await self._processing_slots.acquire() + else: + await asyncio.wait_for( + self._processing_slots.acquire(), + timeout=timeout.remaining_seconds(), + ) + except TimeoutError: + raise TimeoutExpiredError from None + try: + if timeout is not None: + timeout.raise_if_expired() + worker = self._processing_executor.submit(operation) + future = asyncio.create_task(_await_worker(worker)) + except BaseException: + self._processing_slots.release() + raise + future.add_done_callback(self._worker_finished) + return await asyncio.shield(future) + + def _worker_finished(self, future: asyncio.Future[object]) -> None: + self._processing_slots.release() + if not future.cancelled(): + future.exception() + + +class _ActivePolicy: + """Own one active validated policy and its prepared immutable gates.""" + + def __init__(self, registry: GateRegistry) -> None: + self._registry = registry + self._config: EgressGateConfig[GateConfig] | None = None + self._processor: RequestProcessor | None = None + self._lock = Lock() + + def processor_for( + self, + values: object, + *, + timeout: Timeout, + ) -> RequestProcessor: + """Validate and activate a complete candidate under the shared deadline.""" + config = self._registry.validate_config(values) + timeout.raise_if_expired() + if not self._lock.acquire(timeout=timeout.remaining_seconds()): + raise TimeoutExpiredError + try: + timeout.raise_if_expired() + if config == self._config and self._processor is not None: + return self._processor + processor = self._registry.prepare_processor(config, timeout=timeout) + timeout.raise_if_expired() + + self._config = config + self._processor = processor + return processor + except (GateConfigurationError, GateRegistryError): + raise EgressGateError(ErrorCode.CONFIG_INVALID) from None + finally: + self._lock.release() + + def clear(self) -> None: + """Release the active policy.""" + with self._lock: + self._config = None + self._processor = None + + +_WorkerResultT = TypeVar("_WorkerResultT") + + +async def _await_worker(worker: Future[_WorkerResultT]) -> _WorkerResultT: + """Bridge a worker without relying on broken cross-thread loop wakeups.""" + while not worker.done(): + await asyncio.sleep(0.001) + return worker.result() + + +class _AbortContext(Protocol): + async def abort(self, code: grpc.StatusCode, details: str) -> Never: ... + + +class _EvaluationLogExtra(TypedDict): + request_id: str + duration_ms: float + action: str + finding_count: int + decision_source_kind: str + error_code: str | None + + +def _evaluation_log_extra( + *, + request_id: str, + started: float, + action: str, + finding_count: int, + source_kind: str, + failure: EgressGateError | None, +) -> _EvaluationLogExtra: + return { + "request_id": request_id, + "duration_ms": round((time.monotonic() - started) * 1000, 3), + "action": action, + "finding_count": finding_count, + "decision_source_kind": source_kind, + "error_code": failure.code.value if failure is not None else None, + } + + +def _request_id_for_logging(request_id: object) -> str: + try: + return validate_bounded_metadata_string(request_id) + except ValueError: + return _INVALID_REQUEST_ID + + +def _request_id_for_log_message(request_id: str) -> str: + return json.dumps(request_id, ensure_ascii=False).replace(" ", r"\u0020") + + +def _mapping_from_proto(config: Message) -> dict[str, object]: + try: + values: object = json_format.MessageToDict(config) + except Exception: + raise EgressGateError(ErrorCode.CONFIG_INVALID) from None + if not isinstance(values, dict) or any(not isinstance(key, str) for key in values): + raise EgressGateError(ErrorCode.CONFIG_INVALID) + return { + key: _normalize_proto_numbers(item) + for key, item in values.items() + if isinstance(key, str) + } + + +def _normalize_proto_numbers(value: object) -> object: + if isinstance(value, float): + if ( + math.isfinite(value) + and value.is_integer() + and -_MAX_PROTO_SAFE_INTEGER <= value <= _MAX_PROTO_SAFE_INTEGER + ): + return int(value) + return value + if isinstance(value, list): + return [_normalize_proto_numbers(item) for item in value] + if isinstance(value, dict): + return {key: _normalize_proto_numbers(item) for key, item in value.items()} + return value + + +def _request_from_proto(request: pb2.HttpRequestEvaluation) -> HttpRequest: + process = None + if request.context.HasField("originating_process"): + process = Process( + binary=request.context.originating_process.binary, + pid=request.context.originating_process.pid, + ancestors=tuple(request.context.originating_process.ancestors), + ) + try: + return HttpRequest( + context=RequestContext( + request_id=request.context.request_id, + sandbox_id=request.context.sandbox_id, + originating_process=process, + ), + target=HttpTarget( + scheme=request.target.scheme, + host=request.target.host, + port=request.target.port, + method=request.target.method, + path=request.target.path, + query=request.target.query, + ), + headers=tuple( + HttpHeader(name=header.name, value=header.value) + for header in request.headers + ), + body=request.body, + ) + except (TypeError, ValueError): + raise EgressGateError(ErrorCode.REQUEST_ENVELOPE_INVALID) from None + + +def _validate_evaluation_envelope(request: pb2.HttpRequestEvaluation) -> None: + if request.config.ByteSize() > MAX_PROTO_CONFIG_BYTES: + raise EgressGateError(ErrorCode.CONFIG_INVALID) + if ( + request.context.ByteSize() > MAX_PROTO_CONTEXT_BYTES + or request.target.ByteSize() > MAX_PROTO_TARGET_BYTES + or len(request.headers) > MAX_PROTO_HEADERS + or _encoded_headers_size(request.headers) > MAX_PROTO_HEADERS_BYTES + ): + raise EgressGateError(ErrorCode.REQUEST_ENVELOPE_INVALID) + + +def _encoded_headers_size(headers: Iterable[Message]) -> int: + total = 0 + for header in headers: + size = header.ByteSize() + total += 1 + _varint_size(size) + size + return total + + +def _varint_size(value: int) -> int: + size = 1 + while value >= 0x80: + value >>= 7 + size += 1 + return size + + +def _result_to_proto( + result: EgressResult, +) -> tuple[pb2.HttpRequestResult, str]: + response = _serialize_result(result) + if ( + response.reason_code == LIMIT_REASON_CODE + and result.reason_code != LIMIT_REASON_CODE + ): + return response, DecisionSourceKind.RUNTIME_LIMIT.value + return response, result.decision_source.kind.value + + +def _serialize_result(result: EgressResult) -> pb2.HttpRequestResult: + try: + response = pb2.HttpRequestResult( + decision=( + pb2.DECISION_ALLOW + if result.decision is EgressDecision.ALLOW + else pb2.DECISION_DENY + ), + reason_code=result.reason_code or "", + ) + for sourced in result.findings: + finding = _finding_to_proto(sourced) + if finding.ByteSize() > MAX_PROTO_FINDING_BYTES: + return _limit_deny() + response.findings.append(finding) + if len(response.findings) > MAX_PROTO_FINDING_GROUPS: + return _limit_deny() + if result.decision is EgressDecision.ALLOW: + if result.request_mutations.replacement_body is not None: + if len(result.request_mutations.replacement_body) > MAX_BODY_BYTES: + return _limit_deny() + response.body = result.request_mutations.replacement_body + response.has_body = True + for mutation in result.request_mutations.header_mutations: + _append_header_mutation(response, mutation) + response.metadata.update( + {entry.key: entry.value for entry in result.metadata} + ) + if ( + _encoded_headers_size(response.header_mutations) + > MAX_PROTO_HEADERS_BYTES + ): + return _limit_deny() + return response + if ( + result.reason_code is None + or REASON_CODE_PATTERN.fullmatch(result.reason_code) is None + ): + return _limit_deny() + response.reason = ( + LIMIT_REASON if result.reason_code == LIMIT_REASON_CODE else BLOCK_REASON + ) + return response + except (TypeError, ValueError): + return _limit_deny() + + +def _finding_to_proto(sourced: SourcedFinding) -> pb2.Finding: + finding = sourced.finding + return pb2.Finding( + type=finding.type, + label=finding.label, + count=finding.count, + confidence=finding.confidence or "", + severity=finding.severity or "", + ) + + +def _append_header_mutation( + response: pb2.HttpRequestResult, + mutation: HeaderMutation, +) -> None: + if isinstance(mutation, WriteHeaderMutation): + action = { + "append": pb2.EXISTING_HEADER_ACTION_APPEND, + "overwrite": pb2.EXISTING_HEADER_ACTION_OVERWRITE, + "skip": pb2.EXISTING_HEADER_ACTION_SKIP, + }[mutation.on_existing.value] + response.header_mutations.add( + write=pb2.WriteHeader( + name=mutation.name, + value=mutation.value, + on_existing=action, + ) + ) + elif isinstance(mutation, RemoveHeaderMutation): + response.header_mutations.add(remove=pb2.RemoveHeader(name=mutation.name)) + else: + raise ValueError("header mutation is invalid") + + +def _limit_deny() -> pb2.HttpRequestResult: + _LOGGER.info("egress_gate_processing_limit kind=resource") + return pb2.HttpRequestResult( + decision=pb2.DECISION_DENY, + reason=LIMIT_REASON, + reason_code=LIMIT_REASON_CODE, + ) + + +_LOGGER = get_logger(__name__) +_INVALID_REQUEST_ID = "invalid" +_MAX_PROTO_SAFE_INTEGER = (1 << 53) - 1 + + +__all__ = ["EgressGateMiddleware"] diff --git a/projects/privacy-guard/src/privacy_guard/string_validators.py b/projects/egress-gate/src/egress_gate/string_validators.py similarity index 95% rename from projects/privacy-guard/src/privacy_guard/string_validators.py rename to projects/egress-gate/src/egress_gate/string_validators.py index e45c2204..b5db5fe3 100644 --- a/projects/privacy-guard/src/privacy_guard/string_validators.py +++ b/projects/egress-gate/src/egress_gate/string_validators.py @@ -4,7 +4,7 @@ from pydantic import BeforeValidator -from privacy_guard.constants import MAX_DIAGNOSTIC_TEXT_BYTES +from egress_gate.constants import MAX_DIAGNOSTIC_TEXT_BYTES def validate_scalar_string(value: object) -> str: diff --git a/projects/privacy-guard/src/privacy_guard/timeout.py b/projects/egress-gate/src/egress_gate/timeout.py similarity index 85% rename from projects/privacy-guard/src/privacy_guard/timeout.py rename to projects/egress-gate/src/egress_gate/timeout.py index 763e131f..cad58c55 100644 --- a/projects/privacy-guard/src/privacy_guard/timeout.py +++ b/projects/egress-gate/src/egress_gate/timeout.py @@ -1,4 +1,4 @@ -"""A shared monotonic timeout for one entity-processing run.""" +"""A shared monotonic timeout for one request-pipeline run.""" from __future__ import annotations @@ -10,9 +10,9 @@ from pydantic import Field -from privacy_guard.base import StrictDomainModel -from privacy_guard.constants import MAX_TIMEOUT_SECONDS -from privacy_guard.errors import TimeoutExpiredError +from egress_gate.base import StrictDomainModel +from egress_gate.constants import MAX_TIMEOUT_SECONDS +from egress_gate.errors import TimeoutExpiredError def validate_timeout_seconds(seconds: object) -> float: @@ -32,7 +32,7 @@ def validate_timeout_seconds(seconds: object) -> float: class Timeout(StrictDomainModel): - """An immutable monotonic deadline shared across processing stages.""" + """A monotonic deadline shared across gate preparation and execution.""" deadline: float = Field(allow_inf_nan=False) diff --git a/projects/egress-gate/tests/__init__.py b/projects/egress-gate/tests/__init__.py new file mode 100644 index 00000000..a82cff11 --- /dev/null +++ b/projects/egress-gate/tests/__init__.py @@ -0,0 +1 @@ +"""Egress Gate test package.""" diff --git a/projects/egress-gate/tests/gates/__init__.py b/projects/egress-gate/tests/gates/__init__.py new file mode 100644 index 00000000..c4f730b7 --- /dev/null +++ b/projects/egress-gate/tests/gates/__init__.py @@ -0,0 +1 @@ +"""Gate contract and built-in gate tests.""" diff --git a/projects/egress-gate/tests/gates/test_base.py b/projects/egress-gate/tests/gates/test_base.py new file mode 100644 index 00000000..85f0086b --- /dev/null +++ b/projects/egress-gate/tests/gates/test_base.py @@ -0,0 +1,197 @@ +"""Contract tests for trusted request-level gates.""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from threading import Lock +from typing import Literal + +import pytest + +from egress_gate.errors import GateContractError, GateInputError +from egress_gate.gates import ( + Gate, + GateCapability, + GateConfig, + GateResources, + RegexConfig, + RegexGate, +) +from egress_gate.request import HttpRequest, HttpTarget, RequestContext +from egress_gate.result import Finding, GateControl, GateEvaluation +from egress_gate.timeout import Timeout + + +class _RequestConfig(GateConfig): + kind: Literal["test-request"] + + +class _RequestGate(Gate[_RequestConfig, None]): + capabilities = frozenset({GateCapability.READ_TARGET, GateCapability.DENY}) + finding_types = () + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + timeout.raise_if_expired() + if request.target.host == "blocked.example": + return GateEvaluation.deny("egress_gate_test_denied") + return GateEvaluation.proceed() + + +class _CounterResources(GateResources): + __slots__ = ("lock", "calls") + + def __init__(self) -> None: + self.lock = Lock() + self.calls = 0 + + +class _CounterConfig(GateConfig): + kind: Literal["test-counter"] + + +class _CounterGate(Gate[_CounterConfig, _CounterResources]): + capabilities = frozenset({GateCapability.READ_BODY}) + finding_types = () + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + del request + timeout.raise_if_expired() + resources = self.resources + with resources.lock: + resources.calls += 1 + return GateEvaluation.proceed() + + +class _UndeclaredOutputGate(Gate[_RequestConfig, None]): + capabilities = frozenset() + finding_types = () + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + del request, timeout + return GateEvaluation.proceed( + findings=(Finding(type="undeclared", label="test"),) + ) + + +class _CapabilityBypassGate(_UndeclaredOutputGate): + def _validate_output(self, result: GateEvaluation) -> None: + del result + + +class _InvalidEvaluationGate(Gate[_RequestConfig, None]): + capabilities = frozenset({GateCapability.DENY}) + finding_types = () + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + del request, timeout + return GateEvaluation.proceed().model_copy(update={"control": GateControl.DENY}) + + +def _request(*, body: bytes = b"payload", host: str = "example.com") -> HttpRequest: + return HttpRequest( + context=RequestContext(request_id="request-1", sandbox_id="sandbox-1"), + target=HttpTarget( + scheme="https", + host=host, + port=443, + method="POST", + path="/v1/items", + query="", + ), + headers=(), + body=body, + ) + + +def test_gate_uses_exact_config_and_resource_types() -> None: + config = _RequestConfig(name="test", kind="test-request") + gate = _RequestGate(config, None) + + assert gate.config is config + assert gate.resources is None + assert _RequestGate.get_config_type() is _RequestConfig + assert _RequestGate.get_resources_type() is None + assert ( + gate.evaluate( + _request(host="blocked.example"), timeout=Timeout.from_seconds(1) + ).control.value + == "deny" + ) + + +def test_gate_public_wrapper_enforces_declared_output_capabilities() -> None: + with pytest.raises(GateContractError, match="undeclared finding"): + _UndeclaredOutputGate( + _RequestConfig(name="test", kind="test-request"), None + ).evaluate(_request(), timeout=Timeout.from_seconds(1)) + + with pytest.raises(GateContractError, match="undeclared finding"): + _CapabilityBypassGate( + _RequestConfig(name="test", kind="test-request"), + None, + ).evaluate(_request(), timeout=Timeout.from_seconds(1)) + + +def test_gate_public_wrapper_classifies_invalid_models_as_contract_errors() -> None: + with pytest.raises(GateContractError, match="gate output is invalid"): + _InvalidEvaluationGate( + _RequestConfig(name="test", kind="test-request"), + None, + ).evaluate(_request(), timeout=Timeout.from_seconds(1)) + + +def test_gate_rejects_invalid_utf8_as_gate_input() -> None: + config = RegexConfig.model_validate( + { + "name": "regex", + "kind": "regex", + "scan": {"kind": "body", "action": {"kind": "detect"}}, + "pattern_catalog": { + "entities": [ + { + "name": "token", + "rules": [{"pattern": "secret", "confidence": "high"}], + } + ] + }, + } + ) + + with pytest.raises(GateInputError, match="valid UTF-8"): + RegexGate(config, None).evaluate( + _request(body=b"\xff"), timeout=Timeout.from_seconds(1) + ) + + +def test_resource_backed_gate_is_safe_for_concurrent_evaluations() -> None: + resources = _CounterResources() + gate = _CounterGate(_CounterConfig(name="counter", kind="test-counter"), resources) + + def evaluate(_: int) -> GateEvaluation: + return gate.evaluate(_request(), timeout=Timeout.from_seconds(1)) + + with ThreadPoolExecutor(max_workers=8) as executor: + results = tuple(executor.map(evaluate, range(32))) + + assert all(result.control.value == "proceed" for result in results) + assert resources.calls == 32 diff --git a/projects/egress-gate/tests/gates/test_function_gate.py b/projects/egress-gate/tests/gates/test_function_gate.py new file mode 100644 index 00000000..8416c72c --- /dev/null +++ b/projects/egress-gate/tests/gates/test_function_gate.py @@ -0,0 +1,159 @@ +"""Tests for the resource-free function-gate authoring helper.""" + +from __future__ import annotations + +from typing import ClassVar, Literal, Self + +import pytest +from pydantic import ConfigDict, model_validator + +from egress_gate.errors import GateContractError, GateRegistryError +from egress_gate.gates import ( + Gate, + GateCapability, + GateConfig, + GateRegistry, +) +from egress_gate.request import HttpRequest, HttpTarget, RequestContext +from egress_gate.result import GateEvaluation +from egress_gate.timeout import Timeout + + +class _KeywordConfig(GateConfig): + kind: Literal["keyword"] + keyword: str + + +_registry = GateRegistry() + + +@_registry.gate( + config=_KeywordConfig, + capabilities=frozenset({GateCapability.READ_BODY, GateCapability.DENY}), +) +def _keyword_gate( + request: HttpRequest, + config: _KeywordConfig, + *, + timeout: Timeout, +) -> GateEvaluation: + """Deny a request that contains the configured keyword.""" + timeout.raise_if_expired() + if config.keyword.encode("utf-8") in request.body: + return GateEvaluation.deny("keyword_denied") + return GateEvaluation.proceed() + + +_invalid_registry = GateRegistry() + + +@_invalid_registry.gate(config=_KeywordConfig, capabilities=frozenset()) +def _undeclared_deny( + request: HttpRequest, + config: _KeywordConfig, + *, + timeout: Timeout, +) -> GateEvaluation: + del request, config, timeout + return GateEvaluation.deny("keyword_denied") + + +def test_gate_decorator_builds_an_ordinary_resource_free_gate_type() -> None: + assert issubclass(_keyword_gate, Gate) + assert _keyword_gate.get_config_type() is _KeywordConfig + assert _keyword_gate.get_resources_type() is None + + configured = _KeywordConfig(name="keywords", kind="keyword", keyword="SECRET") + instance = _keyword_gate(configured, None) + + assert instance.config is configured + assert ( + instance.evaluate( + _request(body=b"contains SECRET"), + timeout=Timeout.from_seconds(1), + ).control.value + == "deny" + ) + + +def test_decorated_gate_uses_the_standard_output_contract() -> None: + configured = _KeywordConfig(name="keywords", kind="keyword", keyword="SECRET") + + with pytest.raises(GateContractError, match="undeclared deny"): + _undeclared_deny(configured, None).evaluate( + _request(), timeout=Timeout.from_seconds(1) + ) + + +def test_registry_bound_decorator_registers_and_seals_on_first_use() -> None: + config = _registry.validate_config( + { + "gates": [ + { + "name": "keywords", + "kind": "keyword", + "keyword": "SECRET", + } + ], + "default_decision": "allow", + } + ) + + descriptions = _registry.describe_gates() + assert tuple(item.gate_type for item in descriptions) == ("keyword",) + assert descriptions[0].description == ( + "Deny a request that contains the configured keyword." + ) + assert _registry.create_gate(config.gates[0]).get_config_type() is _KeywordConfig + with pytest.raises(GateRegistryError, match="registry is in use"): + _registry.register(_keyword_gate) + + +def test_decorated_gate_does_not_revalidate_config_during_evaluation() -> None: + class CountingConfig(GateConfig): + model_config = ConfigDict(revalidate_instances="always") + + kind: Literal["counting"] + validation_count: ClassVar[int] = 0 + + @model_validator(mode="after") + def count_validation(self) -> Self: + type(self).validation_count += 1 + return self + + registry = GateRegistry() + + @registry.gate(config=CountingConfig, capabilities=frozenset()) + def counting_gate( + request: HttpRequest, + config: CountingConfig, + *, + timeout: Timeout, + ) -> GateEvaluation: + del request, config, timeout + return GateEvaluation.proceed() + + configured = CountingConfig(name="counting", kind="counting") + gate = counting_gate(configured, None) + validation_count = CountingConfig.validation_count + + gate.evaluate(_request(), timeout=Timeout.from_seconds(1)) + gate.evaluate(_request(), timeout=Timeout.from_seconds(1)) + + assert CountingConfig.validation_count == validation_count + + +def _request(*, body: bytes = b"ordinary") -> HttpRequest: + return HttpRequest( + context=RequestContext(request_id="request-1", sandbox_id="sandbox-1"), + target=HttpTarget( + scheme="https", + host="example.com", + port=443, + method="POST", + path="/v1/items", + query="", + ), + headers=(), + body=body, + ) diff --git a/projects/egress-gate/tests/gates/test_regex.py b/projects/egress-gate/tests/gates/test_regex.py new file mode 100644 index 00000000..37f40a89 --- /dev/null +++ b/projects/egress-gate/tests/gates/test_regex.py @@ -0,0 +1,559 @@ +"""Behavior, scans, actions, safety, caching, and concurrency tests for regex.""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import Mock + +import pytest +from pydantic import ValidationError + +import egress_gate.gates.regex as regex_module +from egress_gate.errors import ( + GateConfigurationError, + GateLimitExceededError, + TimeoutExpiredError, +) +from egress_gate.gates import RegexConfig, RegexGate, RegexPatternCatalog +from egress_gate.request import HttpHeader, HttpRequest, HttpTarget, RequestContext +from egress_gate.result import GateControl, GateEvaluation +from egress_gate.timeout import Timeout + + +def _config( + rules: list[dict[str, object]], + *, + action_kind: str = "detect", + template: object | None = None, + scan: dict[str, object] | None = None, +) -> RegexConfig: + scan_values = {"kind": "body"} if scan is None else dict(scan) + action: dict[str, object] = {"kind": action_kind} + if template is not None: + action["template"] = template + scan_values["action"] = action + values: dict[str, object] = { + "name": "regex", + "kind": "regex", + "scan": scan_values, + "pattern_catalog": { + "entities": [{"name": "token", "rules": rules}], + }, + } + return RegexConfig.model_validate(values) + + +def _request( + body: bytes, + *, + path: str = "/", + query: str = "", + headers: tuple[HttpHeader, ...] = (), +) -> HttpRequest: + return HttpRequest( + context=RequestContext(request_id="request-1", sandbox_id="sandbox-1"), + target=HttpTarget( + scheme="https", + host="example.com", + port=443, + method="POST", + path=path, + query=query, + ), + headers=headers, + body=body, + ) + + +def _run(config: RegexConfig, text: str) -> GateEvaluation: + return RegexGate(config, None).evaluate( + _request(text.encode("utf-8")), timeout=Timeout.from_seconds(1) + ) + + +def _catalog(pattern: str) -> RegexPatternCatalog: + return RegexPatternCatalog.model_validate( + { + "entities": [ + { + "name": "token", + "rules": [{"pattern": pattern, "confidence": "high"}], + } + ] + } + ) + + +def test_detect_action_reports_overlaps_without_mutating_the_body() -> None: + evaluation = _run( + _config( + [ + {"name": "pair", "pattern": "aa", "confidence": "high"}, + {"name": "suffix", "pattern": "a$", "confidence": "medium"}, + ] + ), + "aaa", + ) + + assert evaluation.control is GateControl.PROCEED + assert evaluation.request_mutations.replacement_body is None + assert len(evaluation.findings) == 2 + assert sum(finding.count for finding in evaluation.findings) == 3 + assert {finding.label for finding in evaluation.findings} == {"token"} + + +def test_equivalent_detections_are_aggregated_before_evaluation_bounds() -> None: + evaluation = _run( + _config([{"pattern": "x", "confidence": "high"}]), + "x" * 33, + ) + + assert evaluation.findings == ( + regex_module.Finding( + type="regex_match", + label="token", + count=33, + confidence="high", + ), + ) + + +def test_deny_action_is_terminal_and_uses_the_stable_gate_reason() -> None: + evaluation = _run( + _config( + [{"pattern": "secret", "confidence": "high"}], + action_kind="deny", + ), + "contains secret", + ) + + assert evaluation.control is GateControl.DENY + assert evaluation.reason_code == "egress_gate_regex_denied" + assert evaluation.request_mutations.is_empty + assert len(evaluation.findings) == 1 + + +def test_replace_action_preserves_explicit_replacement_intent() -> None: + config = _config( + [{"pattern": "secret", "confidence": "high"}], + action_kind="replace", + template="[{entity}]", + ) + + changed = _run(config, "contains secret") + unchanged = _run(config, "no match") + + assert changed.request_mutations.replacement_body == b"contains [token]" + assert unchanged.request_mutations.replacement_body == b"no match" + assert not unchanged.request_mutations.is_empty + + +@pytest.mark.parametrize( + ("scan", "http_request"), + [ + ({"kind": "path"}, _request(b"", path="/contains-secret")), + ({"kind": "query"}, _request(b"", query="value=secret")), + ( + {"kind": "header", "names": ["x-note"]}, + _request( + b"", + headers=( + HttpHeader(name="X-Note", value="contains secret"), + HttpHeader(name="x-other", value="secret"), + ), + ), + ), + ], +) +def test_detect_action_matches_the_configured_request_scan( + scan: dict[str, object], + http_request: HttpRequest, +) -> None: + config = _config( + [{"pattern": "secret", "confidence": "high"}], + scan=scan, + ) + + evaluation = RegexGate(config, None).evaluate( + http_request, + timeout=Timeout.from_seconds(1), + ) + + assert evaluation.control is GateControl.PROCEED + assert len(evaluation.findings) == 1 + assert evaluation.request_mutations.is_empty + + +def test_header_scan_matches_each_selected_repeated_value() -> None: + config = _config( + [{"pattern": "secret", "confidence": "high"}], + scan={"kind": "header", "names": ["x-note"]}, + ) + request = _request( + b"secret in ignored body", + headers=( + HttpHeader(name="X-Note", value="first secret"), + HttpHeader(name="x-note", value="second secret"), + ), + ) + + evaluation = RegexGate(config, None).evaluate( + request, + timeout=Timeout.from_seconds(1), + ) + + assert evaluation.findings[0].count == 2 + + +def test_non_body_scan_can_make_a_terminal_deny_decision() -> None: + config = _config( + [{"pattern": "admin", "confidence": "high"}], + scan={"kind": "path"}, + action_kind="deny", + ) + + evaluation = RegexGate(config, None).evaluate( + _request(b"", path="/admin/settings"), + timeout=Timeout.from_seconds(1), + ) + + assert evaluation.control is GateControl.DENY + assert evaluation.reason_code == "egress_gate_regex_denied" + + +@pytest.mark.parametrize("kind", ["path", "query", "header"]) +def test_replace_action_is_structurally_unavailable_for_non_body_scans( + kind: str, +) -> None: + scan: dict[str, object] = {"kind": kind} + if kind == "header": + scan["names"] = ["x-note"] + + with pytest.raises(ValidationError): + _config( + [{"pattern": "secret", "confidence": "high"}], + scan=scan, + action_kind="replace", + template="[{entity}]", + ) + + +def test_scan_and_action_kinds_and_unique_header_names_are_required() -> None: + with pytest.raises(ValidationError): + _config([{"pattern": "secret", "confidence": "high"}], scan={}) + with pytest.raises(ValidationError): + _config( + [{"pattern": "secret", "confidence": "high"}], + scan={"type": "body"}, + ) + with pytest.raises(ValidationError, match="unique"): + _config( + [{"pattern": "secret", "confidence": "high"}], + scan={"kind": "header", "names": ["X-Note", "x-note"]}, + ) + + +def test_action_shape_rejects_missing_kinds_and_unrelated_template_fields() -> None: + with pytest.raises(ValidationError): + RegexConfig.model_validate( + { + "name": "regex", + "kind": "regex", + "scan": {"kind": "body", "action": {}}, + "pattern_catalog": _catalog("x"), + } + ) + + with pytest.raises(ValidationError): + _config( + [{"pattern": "x", "confidence": "high"}], + action_kind="detect", + template="[{entity}]", + ) + + +def test_retired_flat_source_and_mode_shape_is_rejected() -> None: + with pytest.raises(ValidationError): + RegexConfig.model_validate( + { + "name": "regex", + "kind": "regex", + "source": {"kind": "body"}, + "pattern_catalog": _catalog("x"), + "mode": "detect", + } + ) + + +def test_regex_features_and_backreferences_keep_their_original_semantics() -> None: + backreference = _run( + _config([{"pattern": r"(a)\1", "confidence": "high"}]), + "aa", + ) + flags = _run( + _config( + [ + { + "pattern": "^x.$", + "confidence": "high", + "ignore_case": True, + "multiline": True, + "dot_all": True, + "ascii": True, + } + ] + ), + "X\n", + ) + + assert len(backreference.findings) == 1 + assert len(flags.findings) == 1 + + +@pytest.mark.parametrize("pattern", ["", "(?i:x)"]) +def test_structurally_invalid_patterns_are_rejected_content_safely( + pattern: str, +) -> None: + with pytest.raises(ValidationError) as exception_info: + _config([{"pattern": pattern, "confidence": "high"}]) + + if pattern: + assert pattern not in str(exception_info.value) + + +@pytest.mark.parametrize("pattern", ["x*", "(?Px)"]) +def test_compile_dependent_pattern_errors_are_rejected_during_preparation( + pattern: str, +) -> None: + config = _config([{"pattern": pattern, "confidence": "high"}]) + + with pytest.raises(GateConfigurationError) as exception_info: + RegexGate(config, None, timeout=Timeout.from_seconds(1)) + + assert pattern not in str(exception_info.value) + + +@pytest.mark.parametrize( + ("pattern", "text"), + [ + ("x|(?=SECRET-zero-width-493)", "SECRET-zero-width-493"), + ("(?=secret)", "secret"), + ("(?<=prefix)", "prefix"), + (r"\b", "secret"), + ("x|(?:y|(?=secret))", "secret"), + ], +) +def test_contextual_zero_width_matches_fail_during_evaluation( + pattern: str, + text: str, +) -> None: + config = _config([{"pattern": pattern, "confidence": "high"}]) + + with pytest.raises( + GateConfigurationError, + match="regex configuration matches an empty span", + ) as exception_info: + _run(config, text) + + assert pattern not in str(exception_info.value) + + +@pytest.mark.parametrize( + ("pattern", "text"), + [ + ("(?<=prefix)secret(?=suffix)", "prefixsecretsuffix"), + (r"\bsecret\b", "a secret value"), + ( + r"(? None: + evaluation = _run( + _config([{"pattern": pattern, "confidence": "high"}]), + text, + ) + assert len(evaluation.findings) == 1 + + +def test_duplicate_rule_names_are_rejected_but_unnamed_rules_are_allowed() -> None: + with pytest.raises(ValidationError): + _config( + [ + {"name": "duplicate", "pattern": "x", "confidence": "high"}, + {"name": "duplicate", "pattern": "y", "confidence": "high"}, + ] + ) + + config = _config( + [ + {"pattern": "x", "confidence": "high"}, + {"pattern": "y", "confidence": "high"}, + ] + ) + assert len(config.pattern_catalog.entities[0].rules) == 2 + + +def test_replacement_selects_ranked_non_overlapping_winners() -> None: + evaluation = _run( + _config( + [ + {"name": "long-low", "pattern": "abc", "confidence": "low"}, + {"name": "short-high", "pattern": "bc", "confidence": "high"}, + ], + action_kind="replace", + template="<{entity}>", + ), + "abc", + ) + + assert evaluation.request_mutations.replacement_body == b"a" + assert len(evaluation.findings) == 2 + + +@pytest.mark.parametrize( + "template", + [ + "{unknown}", + "{entity.attr}", + "{entity!r}", + "{entity:>10}", + "{", + ], +) +def test_replacement_template_language_is_constrained( + template: str, +) -> None: + with pytest.raises(ValidationError): + _config( + [{"pattern": "x", "confidence": "high"}], + action_kind="replace", + template=template, + ) + + +def test_replacement_size_is_projected_before_rendering( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(regex_module, "MAX_BODY_BYTES", 4) + config = _config( + [{"pattern": "x", "confidence": "high"}], + action_kind="replace", + template="[{entity}]", + ) + + with pytest.raises(GateLimitExceededError): + _run(config, "x") + + +def test_pattern_search_has_an_enforceable_timeout() -> None: + config = _config([{"pattern": "(a+)+$", "confidence": "high"}]) + + with pytest.raises(TimeoutExpiredError): + RegexGate(config, None).evaluate( + _request((b"a" * 100_000) + b"!"), + timeout=Timeout.from_seconds(0.001), + ) + + +def test_patterns_compile_during_preparation_not_validation_or_each_run( + monkeypatch: pytest.MonkeyPatch, +) -> None: + original_compile = regex_module.regex.compile + recording_compile = Mock(wraps=original_compile) + monkeypatch.setattr(regex_module.regex, "compile", recording_compile) + config = _config([{"pattern": "x", "confidence": "high"}]) + assert recording_compile.call_count == 0 + gate = RegexGate(config, None, timeout=Timeout.from_seconds(1)) + prepared_count = recording_compile.call_count + + gate.evaluate(_request(b"x"), timeout=Timeout.from_seconds(1)) + gate.evaluate(_request(b"x"), timeout=Timeout.from_seconds(1)) + + assert prepared_count > 0 + assert recording_compile.call_count == prepared_count + + +def test_gate_preparation_honors_an_expired_timeout() -> None: + config = _config([{"pattern": "x", "confidence": "high"}]) + + with pytest.raises(TimeoutExpiredError): + RegexGate(config, None, timeout=Timeout(deadline=0)) + + +def test_regex_gate_is_safe_for_concurrent_runs() -> None: + gate = RegexGate( + _config([{"pattern": "x", "confidence": "high"}]), + None, + ) + + def run(text: str) -> int: + return len( + gate.evaluate( + _request(text.encode()), timeout=Timeout.from_seconds(1) + ).findings + ) + + with ThreadPoolExecutor(max_workers=4) as executor: + counts = tuple(executor.map(run, ("x",) * 16)) + + assert counts == (1,) * 16 + + +def test_relative_yaml_catalog_loading_rejects_aliases_and_traversal( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + directory = tmp_path + monkeypatch.chdir(directory) + (directory / "patterns.yaml").write_text( + "entities:\n" + " - name: token\n" + " rules:\n" + " - pattern: secret\n" + " confidence: high\n" + ) + + config = RegexConfig.model_validate( + { + "name": "regex", + "kind": "regex", + "scan": {"kind": "body", "action": {"kind": "detect"}}, + "pattern_catalog": "patterns.yaml", + } + ) + assert len(_run(config, "secret").findings) == 1 + + with pytest.raises(ValidationError): + RegexConfig.model_validate( + { + "name": "regex", + "kind": "regex", + "scan": {"kind": "body", "action": {"kind": "detect"}}, + "pattern_catalog": "../patterns.yaml", + } + ) + (directory / "aliases.yaml").write_text( + "shared: &shared\n" + " name: token\n" + " rules:\n" + " - pattern: secret\n" + " confidence: high\n" + "entities:\n" + " - *shared\n" + ) + with pytest.raises(ValidationError): + RegexConfig.model_validate( + { + "name": "regex", + "kind": "regex", + "scan": {"kind": "body", "action": {"kind": "detect"}}, + "pattern_catalog": "aliases.yaml", + } + ) diff --git a/projects/egress-gate/tests/gates/test_registry.py b/projects/egress-gate/tests/gates/test_registry.py new file mode 100644 index 00000000..736219c4 --- /dev/null +++ b/projects/egress-gate/tests/gates/test_registry.py @@ -0,0 +1,395 @@ +"""Registry sealing and exact pipeline-schema tests.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Literal + +import pytest +from pydantic import ConfigDict, Field + +from egress_gate.constants import MAX_PIPELINE_GATES +from egress_gate.errors import EgressGateError, GateRegistryError +from egress_gate.gates import ( + Gate, + GateCapability, + GateConfig, + GateRegistry, + GateResources, + create_builtin_registry, +) +from egress_gate.request import HttpRequest +from egress_gate.request_processor import RequestProcessor +from egress_gate.result import GateEvaluation +from egress_gate.timeout import Timeout + + +class _RegistryConfig(GateConfig): + kind: Literal["registry-test"] + answer: int + + +class _RegistryGate(Gate[_RegistryConfig, None]): + """A small resource-free gate used to exercise registry assembly.""" + + capabilities = frozenset({GateCapability.READ_CONTEXT}) + finding_types = () + + def _initialize(self, *, timeout: Timeout | None = None) -> None: + self.preparation_timeout = timeout + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + del request + timeout.raise_if_expired() + return GateEvaluation.proceed() + + +class _ResourceConfig(GateConfig): + kind: Literal["resource-test"] + + +class _ResourceBundle(GateResources): + __slots__ = ("name",) + + def __init__(self, name: str) -> None: + self.name = name + + +class _ResourceGate(Gate[_ResourceConfig, _ResourceBundle]): + capabilities = frozenset() + finding_types = () + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + del request + timeout.raise_if_expired() + return GateEvaluation.proceed() + + +def _pipeline(config: dict[str, object]) -> dict[str, object]: + return { + "gates": [{"name": "one", **config}], + "default_decision": "allow", + } + + +def test_builtin_registry_seals_on_first_use_and_contains_only_regex() -> None: + registry = create_builtin_registry() + + assert tuple(item.gate_type for item in registry.describe_gates()) == ("regex",) + schema = registry.configuration_json_schema() + assert _discriminator_names(schema) == {"kind"} + properties = _object_dict(schema.get("properties")) + assert set(properties) == {"gates", "default_decision"} + gates_schema = _object_dict(properties["gates"]) + assert gates_schema["minItems"] == 1 + assert gates_schema["maxItems"] == MAX_PIPELINE_GATES + assert "Ordered gate configurations" in str(gates_schema["description"]) + assert "Flat policy" in str(schema["description"]) + default_schema = _object_dict(properties["default_decision"]) + assert "every configured gate proceeds" in str(default_schema["description"]) + definitions = schema["$defs"] + assert isinstance(definitions, dict) + assert all(isinstance(key, str) for key in definitions) + regex_schema = next( + value for key, value in definitions.items() if key == "RegexConfig" + ) + regex_schema = _object_dict(regex_schema) + required = next(value for key, value in regex_schema.items() if key == "required") + assert isinstance(required, list) + assert "name" in required + assert "kind" in required + assert "scan" in required + regex_properties = _object_dict(regex_schema["properties"]) + name_schema = _object_dict(regex_properties["name"]) + assert "Unique diagnostic name" in str(name_schema["description"]) + body_scan_schema = next( + value for key, value in definitions.items() if key == "RegexBodyScan" + ) + header_scan_schema = next( + value for key, value in definitions.items() if key == "RegexHeaderScan" + ) + assert "RegexReplaceAction" in str(body_scan_schema) + assert "RegexReplaceAction" not in str(header_scan_schema) + + with pytest.raises(EgressGateError): + registry.validate_config( + _pipeline( + { + "pattern_catalog": { + "entities": [ + { + "name": "token", + "rules": [{"pattern": "secret", "confidence": "high"}], + } + ] + }, + "scan": {"kind": "body", "action": {"kind": "detect"}}, + } + ) + ) + with pytest.raises(GateRegistryError, match="registry is in use"): + registry.register(_RegistryGate) + + +def test_registry_validates_exact_pipeline_and_gate_config() -> None: + registry = GateRegistry() + registry.register(_RegistryGate) + + config = registry.validate_config( + _pipeline({"kind": "registry-test", "answer": 42}) + ) + + assert config.default_decision.value == "allow" + assert type(config.gates[0]) is _RegistryConfig + gate = registry.create_gate(config.gates[0]) + assert type(gate) is _RegistryGate + assert gate.config.answer == 42 + + +def test_registry_requires_an_explicit_gate_discriminator() -> None: + class DefaultedConfig(GateConfig): + kind: Literal["defaulted"] = "defaulted" + + class DefaultedGate(Gate[DefaultedConfig, None]): + capabilities = frozenset() + finding_types = () + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + del request, timeout + return GateEvaluation.proceed() + + with pytest.raises(GateRegistryError, match="gate kind must be required"): + GateRegistry().register(DefaultedGate) + + class FactoryDefaultedConfig(GateConfig): + kind: Literal["factory-defaulted"] = Field( + default_factory=lambda: "factory-defaulted" + ) + + class FactoryDefaultedGate(Gate[FactoryDefaultedConfig, None]): + capabilities = frozenset() + finding_types = () + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + del request, timeout + return GateEvaluation.proceed() + + with pytest.raises(GateRegistryError, match="gate kind must be required"): + GateRegistry().register(FactoryDefaultedGate) + + +def test_registry_requires_gate_configs_to_inherit_the_common_name() -> None: + class DefaultNameConfig(GateConfig): + name: str = "implicit" + kind: Literal["default-name"] + + class DefaultNameGate(Gate[DefaultNameConfig, None]): + capabilities = frozenset() + finding_types = () + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + del request, timeout + return GateEvaluation.proceed() + + class IntegerNameConfig(GateConfig): + name: int + kind: Literal["integer-name"] + + class IntegerNameGate(Gate[IntegerNameConfig, None]): + capabilities = frozenset() + finding_types = () + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + del request, timeout + return GateEvaluation.proceed() + + class UnboundedNameConfig(GateConfig): + name: str + kind: Literal["unbounded-name"] + + class UnboundedNameGate(Gate[UnboundedNameConfig, None]): + capabilities = frozenset() + finding_types = () + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + del request, timeout + return GateEvaluation.proceed() + + for gate_type in (DefaultNameGate, IntegerNameGate, UnboundedNameGate): + with pytest.raises(GateRegistryError, match="inherit name"): + GateRegistry().register(gate_type) + + +def test_registry_requires_canonical_common_field_names() -> None: + class AliasedConfig(GateConfig): + model_config = ConfigDict(alias_generator=str.upper) + + kind: Literal["aliased"] + value: str + + class AliasedGate(Gate[AliasedConfig, None]): + capabilities = frozenset() + finding_types = () + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + del request, timeout + return GateEvaluation.proceed() + + with pytest.raises(GateRegistryError, match="canonical field names"): + GateRegistry().register(AliasedGate) + + +def test_registry_forwards_the_shared_preparation_timeout() -> None: + registry = GateRegistry() + registry.register(_RegistryGate) + config = registry.validate_config( + _pipeline({"kind": "registry-test", "answer": 42}) + ) + timeout = Timeout.from_seconds(1) + + gate = registry.create_gate(config.gates[0], timeout=timeout) + + assert isinstance(gate, _RegistryGate) + assert gate.preparation_timeout is timeout + + +def test_registry_prepares_the_production_processor_from_validated_config() -> None: + registry = GateRegistry() + registry.register(_RegistryGate) + config = registry.validate_config( + _pipeline({"kind": "registry-test", "answer": 42}) + ) + + processor = registry.prepare_processor( + config, + timeout=Timeout.from_seconds(1), + ) + + assert isinstance(processor, RequestProcessor) + + +def test_registry_injects_typed_application_resources() -> None: + resources = _ResourceBundle("shared-client") + registry = GateRegistry() + registry.register(_ResourceGate, resources=resources) + + config = registry.validate_config(_pipeline({"kind": "resource-test"})) + gate = registry.create_gate(config.gates[0]) + + assert gate.resources is resources + + with pytest.raises(GateRegistryError): + GateRegistry().register(_ResourceGate) + with pytest.raises(GateRegistryError): + GateRegistry().register(_ResourceGate, resources=object()) + + +def test_registry_rejects_unknown_policy_shapes() -> None: + registry = GateRegistry() + registry.register(_RegistryGate) + + for values in ( + {"unexpected": {}}, + _pipeline({"gate": "registry-test", "answer": 1}), + _pipeline({"kind": "missing", "answer": 1}), + _pipeline({"kind": "registry-test", "answer": 1, "extra": True}), + { + "gates": [{"name": "one", "kind": "registry-test", "answer": 1}], + }, + ): + with pytest.raises(EgressGateError): + registry.validate_config(values) + + +def test_registry_lifecycle_and_fingerprint_are_deterministic() -> None: + registry = GateRegistry() + with pytest.raises(GateRegistryError, match="no registered gates"): + registry.configuration_json_schema() + + registry.register(_RegistryGate) + first = registry.validate_config(_pipeline({"kind": "registry-test", "answer": 1})) + second = registry.validate_config(_pipeline({"kind": "registry-test", "answer": 2})) + with pytest.raises(GateRegistryError, match="registry is in use"): + registry.register(_ResourceGate) + + assert registry.policy_fingerprint(first) != registry.policy_fingerprint(second) + assert registry.policy_fingerprint(first) == registry.policy_fingerprint(first) + + +def test_registry_rejects_duplicate_gate_names_before_preparation() -> None: + registry = GateRegistry() + registry.register(_RegistryGate) + values = { + "gates": [ + {"name": "same", "kind": "registry-test", "answer": 1}, + {"name": "same", "kind": "registry-test", "answer": 2}, + ], + "default_decision": "allow", + } + + with pytest.raises(EgressGateError): + registry.validate_config(values) + + +def _discriminator_names(value: object) -> set[object]: + if isinstance(value, Mapping): + names = { + discriminator.get("propertyName") + for key, discriminator in value.items() + if key == "discriminator" and isinstance(discriminator, Mapping) + } + for nested in value.values(): + names.update(_discriminator_names(nested)) + return names + if isinstance(value, list | tuple): + names: set[object] = set() + for nested in value: + names.update(_discriminator_names(nested)) + return names + return set() + + +def _object_dict(value: object) -> dict[str, object]: + assert isinstance(value, dict) + assert all(isinstance(key, str) for key in value) + return {key: nested for key, nested in value.items() if isinstance(key, str)} diff --git a/projects/egress-gate/tests/service/__init__.py b/projects/egress-gate/tests/service/__init__.py new file mode 100644 index 00000000..a2b97e36 --- /dev/null +++ b/projects/egress-gate/tests/service/__init__.py @@ -0,0 +1 @@ +"""Egress Gate service tests.""" diff --git a/projects/egress-gate/tests/service/test_grpc_integration.py b/projects/egress-gate/tests/service/test_grpc_integration.py new file mode 100644 index 00000000..bad72f9a --- /dev/null +++ b/projects/egress-gate/tests/service/test_grpc_integration.py @@ -0,0 +1,173 @@ +"""Loopback coverage for the generated OpenShell gRPC service.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +import grpc +import pytest +from google.protobuf import empty_pb2, json_format, message_factory +from google.protobuf.message import Message + +from egress_gate.bindings import supervisor_middleware_pb2 as pb2 +from egress_gate.bindings import supervisor_middleware_pb2_grpc as pb2_grpc +from egress_gate.errors import EgressGateError, ErrorCode +from egress_gate.gates import create_builtin_registry +from egress_gate.service import server as server_module +from egress_gate.service.servicer import EgressGateMiddleware + + +def _config(*, action_kind: str = "replace") -> Message: + action: dict[str, object] = {"kind": action_kind} + if action_kind == "replace": + action["template"] = "[{entity}]" + values: dict[str, object] = { + "gates": [ + { + "name": "identifiers", + "kind": "regex", + "scan": {"kind": "body", "action": action}, + "pattern_catalog": { + "entities": [ + { + "name": "email", + "rules": [ + { + "pattern": r"[a-z]+@[a-z]+\.[a-z]+", + "confidence": "high", + } + ], + } + ] + }, + } + ], + "default_decision": "allow", + } + request = pb2.ValidateConfigRequest() + json_format.ParseDict(values, request.config) + return request.config + + +def _evaluation( + body: bytes, + *, + action_kind: str = "replace", +) -> pb2.HttpRequestEvaluation: + return pb2.HttpRequestEvaluation( + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, + context=pb2.RequestContext(request_id="grpc-integration", sandbox_id="sandbox"), + config=_config(action_kind=action_kind), + target=pb2.HttpRequestTarget( + scheme="https", + host="example.com", + port=443, + method="POST", + path="/", + query="", + ), + body=body, + ) + + +@asynccontextmanager +async def _running_stub( + middleware: EgressGateMiddleware, +) -> AsyncIterator[tuple[pb2_grpc.SupervisorMiddlewareStub, grpc.aio.Channel]]: + server = server_module._create_grpc_server(middleware) + port = server.add_insecure_port("127.0.0.1:0") + assert port > 0 + await server.start() + channel = grpc.aio.insecure_channel(f"127.0.0.1:{port}") + try: + yield pb2_grpc.SupervisorMiddlewareStub(channel), channel + finally: + await channel.close() + await server.stop(grace=0) + await middleware.close() + + +@pytest.mark.asyncio +async def test_generated_stub_round_trip_covers_manifest_and_gate_actions() -> None: + middleware = EgressGateMiddleware(create_builtin_registry()) + async with _running_stub(middleware) as (stub, _): + empty_message_type = message_factory.GetMessageClass( + empty_pb2.DESCRIPTOR.message_types_by_name["Empty"] + ) + empty_message: Message = empty_message_type() + manifest = await stub.Describe(empty_message) + replaced = await stub.EvaluateHttpRequest(_evaluation(b"contact a@b.com")) + detected = await stub.EvaluateHttpRequest( + _evaluation(b"contact a@b.com", action_kind="detect") + ) + denied_config = _config(action_kind="deny") + denied_request = _evaluation(b"contact a@b.com", action_kind="deny") + denied_request.config.CopyFrom(denied_config) + denied = await stub.EvaluateHttpRequest(denied_request) + + assert manifest.name == "egress-gate" + assert len(manifest.bindings) == 1 + assert replaced.decision == pb2.DECISION_ALLOW + assert replaced.has_body is True + assert replaced.body == b"contact [email]" + assert detected.decision == pb2.DECISION_ALLOW + assert detected.has_body is False + assert len(detected.findings) == 1 + assert denied.decision == pb2.DECISION_DENY + assert denied.reason_code == "egress_gate_regex_denied" + + +@pytest.mark.asyncio +async def test_generated_stub_maps_invalid_phase_to_invalid_argument() -> None: + middleware = EgressGateMiddleware(create_builtin_registry()) + async with _running_stub(middleware) as (stub, _): + request = _evaluation(b"body") + request.phase = pb2.SUPERVISOR_MIDDLEWARE_PHASE_UNSPECIFIED + + with pytest.raises(grpc.aio.AioRpcError) as error: + await stub.EvaluateHttpRequest(request) + + assert error.value.code() is grpc.StatusCode.INVALID_ARGUMENT + assert "request_phase_invalid" in (error.value.details() or "") + + +@pytest.mark.asyncio +async def test_malformed_protobuf_maps_to_content_safe_invalid_argument() -> None: + middleware = EgressGateMiddleware(create_builtin_registry()) + async with _running_stub(middleware) as (stub, channel): + raw_evaluate = channel.unary_unary( + "/openshell.middleware.v1.SupervisorMiddleware/EvaluateHttpRequest", + request_serializer=lambda value: value, + response_deserializer=lambda value: value, + ) + with pytest.raises(grpc.aio.AioRpcError) as error: + await raw_evaluate(b"\x12\x02\x0a\xff") + + recovered = await stub.EvaluateHttpRequest(_evaluation(b"body")) + + details = error.value.details() or "" + assert error.value.code() is grpc.StatusCode.INVALID_ARGUMENT + assert details == str(EgressGateError(ErrorCode.REQUEST_PROTOBUF_INVALID)) + assert "DecodeError" not in details + assert "HttpRequestEvaluation" not in details + assert recovered.decision == pb2.DECISION_ALLOW + + +@pytest.mark.asyncio +async def test_generated_stub_maps_gate_failure_to_internal( + monkeypatch: pytest.MonkeyPatch, +) -> None: + middleware = EgressGateMiddleware(create_builtin_registry()) + + def fail_processing(*args: object, **kwargs: object) -> object: + del args, kwargs + raise EgressGateError(ErrorCode.GATE_EXECUTION_FAILED) + + monkeypatch.setattr(middleware, "_prepare_and_process", fail_processing) + async with _running_stub(middleware) as (stub, _): + with pytest.raises(grpc.aio.AioRpcError) as error: + await stub.EvaluateHttpRequest(_evaluation(b"body")) + + assert error.value.code() is grpc.StatusCode.INTERNAL + assert "gate_execution_failed" in (error.value.details() or "") diff --git a/projects/egress-gate/tests/service/test_server.py b/projects/egress-gate/tests/service/test_server.py new file mode 100644 index 00000000..736ce38f --- /dev/null +++ b/projects/egress-gate/tests/service/test_server.py @@ -0,0 +1,188 @@ +"""Programmatic Egress Gate server lifecycle and transport-isolation tests.""" + +from __future__ import annotations + +import asyncio +import subprocess +import sys + +import grpc +import pytest + +from egress_gate.constants import MAX_CONCURRENT_RPCS, MAX_RECEIVE_MESSAGE_BYTES +from egress_gate.errors import EgressGateError, ErrorCode, GateRegistryError +from egress_gate.gates import GateRegistry, create_builtin_registry +from egress_gate.service import server as server_module +from egress_gate.service.server import EgressGateServer +from egress_gate.service.servicer import EgressGateMiddleware + + +class _FakeServer: + def __init__(self, *, bound_port: int = 50051) -> None: + self.bound_port = bound_port + self.addresses: list[str] = [] + self.started = False + self.waited = False + self.stop_graces: list[float | None] = [] + + def add_insecure_port(self, address: str) -> int: + self.addresses.append(address) + return self.bound_port + + async def start(self) -> None: + self.started = True + + async def wait_for_termination(self) -> bool: + self.waited = True + return True + + async def stop(self, grace: float | None) -> None: + self.stop_graces.append(grace) + + +def test_server_rejects_a_registry_without_gates() -> None: + with pytest.raises(GateRegistryError, match="no registered gates"): + EgressGateServer(GateRegistry()) + + +@pytest.mark.parametrize("seconds", [True, 0, 31, float("inf")]) +def test_server_validates_the_service_timeout(seconds: bool | int | float) -> None: + with pytest.raises(ValueError, match="finite number greater than 0 and at most 30"): + EgressGateServer(create_builtin_registry(), timeout_seconds=seconds) + + +def test_server_keeps_timeout_ownership_at_the_service_boundary() -> None: + server = EgressGateServer(create_builtin_registry(), timeout_seconds=4.5) + try: + assert server._middleware._timeout_seconds == 4.5 + assert not hasattr(server._middleware._policy, "_timeout_seconds") + finally: + asyncio.run(server._middleware.close()) + + +def test_server_seals_the_registry_during_initialization() -> None: + registry = create_builtin_registry() + server = EgressGateServer(registry) + try: + with pytest.raises(GateRegistryError, match="registry is in use"): + registry.register(object) + finally: + asyncio.run(server._middleware.close()) + + +def test_server_sets_transport_limits_and_registers_middleware( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_server = object() + transport_options: list[tuple[int, tuple[tuple[str, int], ...]]] = [] + registrations: list[tuple[EgressGateMiddleware, object]] = [] + + def fake_factory( + *, + interceptors: tuple[grpc.aio.ServerInterceptor, ...], + maximum_concurrent_rpcs: int, + options: tuple[tuple[str, int], ...], + ) -> object: + assert len(interceptors) == 1 + assert isinstance(interceptors[0], server_module._MalformedProtobufInterceptor) + transport_options.append((maximum_concurrent_rpcs, options)) + return fake_server + + def record_registration(middleware: EgressGateMiddleware, server: object) -> None: + registrations.append((middleware, server)) + + middleware = EgressGateMiddleware(create_builtin_registry()) + monkeypatch.setattr(grpc.aio, "server", fake_factory) + monkeypatch.setattr( + server_module.pb2_grpc, + "add_SupervisorMiddlewareServicer_to_server", + record_registration, + ) + try: + result = server_module._create_grpc_server(middleware) + finally: + asyncio.run(middleware.close()) + + assert result is fake_server + assert transport_options == [ + ( + MAX_CONCURRENT_RPCS, + (("grpc.max_receive_message_length", MAX_RECEIVE_MESSAGE_BYTES),), + ) + ] + assert registrations == [(middleware, fake_server)] + + +@pytest.mark.parametrize( + ("listen", "port"), + [("127.0.0.1:1", 1), ("middleware.local:65535", 65_535), ("[::1]:50051", 50_051)], +) +def test_listen_address_accepts_supported_tcp_forms(listen: str, port: int) -> None: + assert server_module._validated_listen_port(listen) == port + + +@pytest.mark.parametrize( + "listen", + ["127.0.0.1:0", "127.0.0.1:65536", "127.0.0.1:-1", "[::1]", "::1:50051"], +) +def test_listen_address_rejects_invalid_forms(listen: str) -> None: + with pytest.raises(EgressGateError) as error: + server_module._validated_listen_port(listen) + assert error.value.code is ErrorCode.SERVER_BIND_FAILED + + +@pytest.mark.asyncio +async def test_serve_async_starts_waits_and_closes_resources( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_server = _FakeServer(bound_port=50053) + closed: list[EgressGateMiddleware] = [] + + async def record_close(middleware: EgressGateMiddleware) -> None: + closed.append(middleware) + + server = EgressGateServer(create_builtin_registry()) + monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) + monkeypatch.setattr(EgressGateMiddleware, "close", record_close) + + await server.serve_async("127.0.0.1:50053") + + assert fake_server.addresses == ["127.0.0.1:50053"] + assert fake_server.started is True + assert fake_server.waited is True + assert fake_server.stop_graces == [0] + assert closed == [server._middleware] + + +@pytest.mark.asyncio +async def test_serve_async_sanitizes_bind_failures_and_closes_resources( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_server = _FakeServer(bound_port=0) + closed: list[EgressGateMiddleware] = [] + + async def record_close(middleware: EgressGateMiddleware) -> None: + closed.append(middleware) + + server = EgressGateServer(create_builtin_registry()) + monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) + monkeypatch.setattr(EgressGateMiddleware, "close", record_close) + + with pytest.raises(EgressGateError) as error: + await server.serve_async("invalid-sensitive-listen:50051") + + assert error.value.code is ErrorCode.SERVER_BIND_FAILED + assert "invalid-sensitive-listen" not in str(error.value) + assert fake_server.stop_graces == [0] + assert closed == [server._middleware] + + +def test_programmatic_server_import_does_not_load_cli_or_gate_transport() -> None: + probe = ( + "import sys; " + "from egress_gate.service import EgressGateServer; " + "assert EgressGateServer.__name__ == 'EgressGateServer'; " + "assert 'egress_gate.cli' not in sys.modules; " + "assert 'typer' not in sys.modules" + ) + subprocess.run([sys.executable, "-c", probe], check=True) diff --git a/projects/egress-gate/tests/service/test_servicer.py b/projects/egress-gate/tests/service/test_servicer.py new file mode 100644 index 00000000..018c95df --- /dev/null +++ b/projects/egress-gate/tests/service/test_servicer.py @@ -0,0 +1,638 @@ +"""Transport-boundary tests for the canonical OpenShell protobuf adapter.""" + +from __future__ import annotations + +import asyncio +import logging +from concurrent.futures import ThreadPoolExecutor +from threading import Barrier, Event +from typing import Never +from unittest.mock import Mock + +import grpc +import pytest +from google.protobuf import json_format +from google.protobuf.message import Message + +from egress_gate.bindings import supervisor_middleware_pb2 as pb2 +from egress_gate.config import EgressGateConfig +from egress_gate.constants import ( + BLOCK_REASON, + DEFAULT_DENY_REASON_CODE, + LIMIT_REASON, + LIMIT_REASON_CODE, + MAX_BODY_BYTES, + MAX_PROTO_CONFIG_BYTES, + MAX_PROTO_CONTEXT_BYTES, + MAX_PROTO_FINDING_BYTES, + MAX_PROTO_HEADERS, + MAX_PROTO_HEADERS_BYTES, + MAX_PROTO_TARGET_BYTES, +) +from egress_gate.errors import EgressGateError, ErrorCode, GateRegistryError +from egress_gate.gates import ( + GateConfig, + RegexConfig, + RegexReplaceAction, + create_builtin_registry, +) +from egress_gate.request import ( + ExistingHeaderAction, + RequestMutations, + WriteHeaderMutation, +) +from egress_gate.result import ( + DecisionSourceKind, + EgressDecision, + EgressResult, + Finding, + PipelineDefaultDecisionSource, + RuntimeLimitDecisionSource, + SourcedFinding, +) +from egress_gate.service import servicer as servicer_module +from egress_gate.service.servicer import EgressGateMiddleware +from egress_gate.timeout import Timeout + + +def _values( + *, action_kind: str = "detect", default_decision: str = "allow" +) -> dict[str, object]: + action: dict[str, object] = {"kind": action_kind} + if action_kind == "replace": + action["template"] = "[{entity}]" + config: dict[str, object] = { + "kind": "regex", + "scan": {"kind": "body", "action": action}, + "pattern_catalog": { + "entities": [ + { + "name": "token", + "rules": [{"pattern": "secret", "confidence": "high"}], + } + ] + }, + } + return { + "gates": [{"name": "body", **config}], + "default_decision": default_decision, + } + + +def _proto_config(values: dict[str, object]) -> Message: + config = pb2.ValidateConfigRequest().config + json_format.ParseDict(values, config) + return config + + +def _request(body: bytes = b"secret") -> pb2.HttpRequestEvaluation: + return pb2.HttpRequestEvaluation( + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, + context=pb2.RequestContext(request_id="request-1", sandbox_id="sandbox-1"), + config=_proto_config(_values()), + target=pb2.HttpRequestTarget( + scheme="https", + host="example.com", + port=443, + method="POST", + path="/", + query="", + ), + headers=[pb2.HttpHeader(name="x-test", value="one")], + body=body, + ) + + +class _SuccessfulEvaluationContext: + async def abort(self, code: grpc.StatusCode, details: str) -> Never: + del code, details + raise AssertionError("successful evaluation unexpectedly aborted") + + +def test_copied_proto_remains_the_current_five_field_finding_contract() -> None: + evaluation = pb2.HttpRequestEvaluation() + finding = pb2.Finding() + + assert isinstance(evaluation.config, Message) + assert not hasattr(evaluation, "config_fingerprint") + assert not hasattr(finding, "source") + assert not hasattr(finding, "attributes") + + +def test_validate_config_is_pure_and_reports_invalid_config() -> None: + middleware = EgressGateMiddleware(create_builtin_registry()) + try: + valid = middleware._validate_config( + pb2.ValidateConfigRequest(config=_proto_config(_values())) + ) + invalid = middleware._validate_config( + pb2.ValidateConfigRequest(config=_proto_config({"unexpected": {}})) + ) + finally: + asyncio.run(middleware.close()) + + assert valid.valid is True + assert invalid.valid is False + assert "config_invalid" in invalid.reason + + +def test_validate_config_rejects_oversized_wire_config_before_parsing() -> None: + exact = pb2.ValidateConfigRequest() + json_format.ParseDict({"padding": "x" * 65_515}, exact.config) + oversized = pb2.ValidateConfigRequest() + json_format.ParseDict({"padding": "x" * 65_516}, oversized.config) + assert exact.config.ByteSize() == MAX_PROTO_CONFIG_BYTES + assert oversized.config.ByteSize() == MAX_PROTO_CONFIG_BYTES + 1 + + middleware = EgressGateMiddleware(create_builtin_registry()) + try: + exact_response = middleware._validate_config(exact) + oversized_response = middleware._validate_config(oversized) + finally: + asyncio.run(middleware.close()) + + assert exact_response.valid is False + assert oversized_response.valid is False + assert "config_invalid" in oversized_response.reason + + +def test_evaluation_enforces_exact_encoded_transport_boundaries() -> None: + request = _request(body=b"") + request.context.sandbox_id = "" + request.context.request_id = "x" * 4_093 + assert request.context.ByteSize() == MAX_PROTO_CONTEXT_BYTES + servicer_module._validate_evaluation_envelope(request) + request.context.request_id += "x" + assert request.context.ByteSize() == MAX_PROTO_CONTEXT_BYTES + 1 + with pytest.raises(EgressGateError) as context_error: + servicer_module._validate_evaluation_envelope(request) + assert context_error.value.code is ErrorCode.REQUEST_ENVELOPE_INVALID + + request = _request(body=b"") + request.target.Clear() + request.target.host = "x" * 32_764 + assert request.target.ByteSize() == MAX_PROTO_TARGET_BYTES + servicer_module._validate_evaluation_envelope(request) + request.target.host += "x" + with pytest.raises(EgressGateError) as target_error: + servicer_module._validate_evaluation_envelope(request) + assert target_error.value.code is ErrorCode.REQUEST_ENVELOPE_INVALID + + request = _request(body=b"") + request.headers.clear() + request.headers.add(name="x", value="x" * 65_525) + assert servicer_module._encoded_headers_size(request.headers) == ( + MAX_PROTO_HEADERS_BYTES + ) + servicer_module._validate_evaluation_envelope(request) + request.headers[0].value += "x" + with pytest.raises(EgressGateError) as header_error: + servicer_module._validate_evaluation_envelope(request) + assert header_error.value.code is ErrorCode.REQUEST_ENVELOPE_INVALID + + request = _request(body=b"") + request.headers.clear() + for _ in range(MAX_PROTO_HEADERS): + request.headers.add() + servicer_module._validate_evaluation_envelope(request) + request.headers.add() + with pytest.raises(EgressGateError): + servicer_module._validate_evaluation_envelope(request) + + +def test_request_adapter_builds_the_full_domain_request() -> None: + domain = servicer_module._request_from_proto(_request(b"bytes")) + + assert domain.body == b"bytes" + assert domain.context.request_id == "request-1" + assert domain.target.host == "example.com" + assert domain.headers[0].name == "x-test" + + +def test_result_adapter_serializes_only_five_finding_fields_and_empty_body_intent() -> ( + None +): + finding = Finding( + type="t" * 1024, + label="l" * 1024, + confidence="c" * 1024, + severity="s" * 1010, + ) + result = EgressResult( + decision=EgressDecision.ALLOW, + decision_source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + request_mutations=RequestMutations(replacement_body=b""), + findings=(SourcedFinding(source_gate="body", finding=finding),), + ) + + response, _ = servicer_module._result_to_proto(result) + + assert response.decision == pb2.DECISION_ALLOW + assert response.has_body is True + assert response.body == b"" + assert response.findings[0].ByteSize() == MAX_PROTO_FINDING_BYTES + assert {field.name for field, _ in response.findings[0].ListFields()} == { + "type", + "label", + "count", + "confidence", + "severity", + } + + +def test_result_adapter_preserves_ordered_header_mutations_and_deny_reason() -> None: + allowed = EgressResult( + decision=EgressDecision.ALLOW, + decision_source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + request_mutations=RequestMutations( + header_mutations=( + WriteHeaderMutation( + kind="write", + name="x-openshell-middleware-reviewed", + value="true", + on_existing=ExistingHeaderAction.OVERWRITE, + ), + ) + ), + ) + denied = EgressResult( + decision=EgressDecision.DENY, + decision_source=RuntimeLimitDecisionSource( + kind=DecisionSourceKind.RUNTIME_LIMIT + ), + reason_code=LIMIT_REASON_CODE, + ) + + allowed_response, _ = servicer_module._result_to_proto(allowed) + denied_response, _ = servicer_module._result_to_proto(denied) + + assert allowed_response.header_mutations[0].write.name == ( + "x-openshell-middleware-reviewed" + ) + assert denied_response.reason == LIMIT_REASON + assert denied_response.reason_code == LIMIT_REASON_CODE + + +def test_processor_preparation_reuses_only_the_current_validated_policy() -> None: + middleware = EgressGateMiddleware(create_builtin_registry()) + try: + first = middleware._policy.processor_for( + _values(), timeout=Timeout.from_seconds(1) + ) + same = middleware._policy.processor_for( + _values(), timeout=Timeout.from_seconds(1) + ) + changed = middleware._policy.processor_for( + _values(action_kind="replace"), timeout=Timeout.from_seconds(1) + ) + finally: + asyncio.run(middleware.close()) + + assert same is first + assert changed is not first + + +def test_concurrent_same_candidate_is_prepared_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + middleware = EgressGateMiddleware(create_builtin_registry()) + original_create_gate = middleware._registry.create_gate + workers_ready = Barrier(2) + build_started = Event() + release_build = Event() + + def blocked_create_gate( + config: GateConfig, + *, + timeout: Timeout | None = None, + ) -> object: + build_started.set() + assert release_build.wait(2) + return original_create_gate(config, timeout=timeout) + + create_gate = Mock(side_effect=blocked_create_gate) + monkeypatch.setattr(middleware._registry, "create_gate", create_gate) + + def resolve_candidate() -> object: + workers_ready.wait(timeout=2) + return middleware._policy.processor_for( + _values(), + timeout=Timeout.from_seconds(2), + ) + + try: + with ThreadPoolExecutor(max_workers=2) as executor: + first_future = executor.submit(resolve_candidate) + second_future = executor.submit(resolve_candidate) + assert build_started.wait(1) + assert not first_future.done() + assert not second_future.done() + release_build.set() + first = first_future.result() + second = second_future.result() + finally: + release_build.set() + asyncio.run(middleware.close()) + + assert second is first + assert create_gate.call_count == 1 + + +def test_failed_candidate_leaves_the_old_policy_active( + monkeypatch: pytest.MonkeyPatch, +) -> None: + middleware = EgressGateMiddleware(create_builtin_registry()) + old = middleware._policy.processor_for(_values(), timeout=Timeout.from_seconds(1)) + original_create_gate = middleware._registry.create_gate + + def fail_changed_candidate( + config: GateConfig, + *, + timeout: Timeout | None = None, + ) -> object: + if isinstance(config, RegexConfig) and isinstance( + config.scan.action, RegexReplaceAction + ): + raise GateRegistryError("candidate preparation failed") + return original_create_gate(config, timeout=timeout) + + monkeypatch.setattr( + middleware._registry, + "create_gate", + fail_changed_candidate, + ) + try: + with pytest.raises(EgressGateError) as error: + middleware._policy.processor_for( + _values(action_kind="replace"), + timeout=Timeout.from_seconds(1), + ) + active = middleware._policy.processor_for( + _values(), timeout=Timeout.from_seconds(1) + ) + finally: + asyncio.run(middleware.close()) + + assert error.value.code is ErrorCode.CONFIG_INVALID + assert active is old + + +def test_invalid_request_cannot_publish_a_changed_policy() -> None: + middleware = EgressGateMiddleware(create_builtin_registry()) + old = middleware._policy.processor_for(_values(), timeout=Timeout.from_seconds(1)) + invalid = _request() + invalid.config.CopyFrom(_proto_config(_values(action_kind="replace"))) + invalid.headers[0].name = "" + + try: + with pytest.raises(EgressGateError) as error: + middleware._prepare_and_process(invalid, Timeout.from_seconds(1)) + assert middleware._policy._processor is old + finally: + asyncio.run(middleware.close()) + + assert error.value.code is ErrorCode.REQUEST_ENVELOPE_INVALID + + +def test_in_flight_processor_reference_survives_policy_replacement() -> None: + middleware = EgressGateMiddleware(create_builtin_registry()) + try: + old = middleware._policy.processor_for( + _values(), timeout=Timeout.from_seconds(1) + ) + replacement = middleware._policy.processor_for( + _values(action_kind="replace"), timeout=Timeout.from_seconds(1) + ) + domain_request = servicer_module._request_from_proto(_request()) + + old_result = old.process(domain_request, timeout=Timeout.from_seconds(1)) + replacement_result = replacement.process( + domain_request, + timeout=Timeout.from_seconds(1), + ) + finally: + asyncio.run(middleware.close()) + + assert old_result.request_mutations.replacement_body is None + assert replacement_result.request_mutations.replacement_body == b"[token]" + + +@pytest.mark.asyncio +async def test_cancelled_candidate_keeps_its_slot_until_worker_exits( + monkeypatch: pytest.MonkeyPatch, +) -> None: + middleware = EgressGateMiddleware( + create_builtin_registry(), + timeout_seconds=5, + ) + started = Event() + release = Event() + original_build = middleware._registry.prepare_processor + + def blocked_build( + config: EgressGateConfig[GateConfig], + *, + timeout: Timeout, + ) -> object: + started.set() + assert release.wait(2) + return original_build(config, timeout=timeout) + + monkeypatch.setattr(middleware._registry, "prepare_processor", blocked_build) + changed_request = _request() + changed_request.config.CopyFrom(_proto_config(_values(action_kind="replace"))) + task = asyncio.create_task( + middleware._evaluate_http_request( + changed_request, + Timeout.from_seconds(5), + ) + ) + try: + assert await asyncio.to_thread(started.wait, 1) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert middleware._processing_slots._value == 3 + + release.set() + for _ in range(100): + if middleware._processing_slots._value == 4: + break + await asyncio.sleep(0.01) + + assert middleware._processing_slots._value == 4 + finally: + release.set() + await middleware.close() + + +@pytest.mark.asyncio +async def test_result_serialization_is_bracketed_by_the_shared_timeout( + monkeypatch: pytest.MonkeyPatch, +) -> None: + middleware = EgressGateMiddleware(create_builtin_registry()) + result = EgressResult( + decision=EgressDecision.ALLOW, + decision_source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + ) + events: list[str] = [] + original_serialize = servicer_module._result_to_proto + + async def return_result(*args: object, **kwargs: object) -> EgressResult: + del args, kwargs + return result + + def record_deadline_check(self: Timeout) -> None: + del self + events.append("deadline") + + def record_serialization( + value: EgressResult, + ) -> tuple[pb2.HttpRequestResult, str]: + events.append("serialize") + return original_serialize(value) + + monkeypatch.setattr(middleware, "_run_in_worker", return_result) + monkeypatch.setattr(Timeout, "raise_if_expired", record_deadline_check) + monkeypatch.setattr( + servicer_module, + "_result_to_proto", + record_serialization, + ) + try: + await middleware._evaluate_http_request( + _request(), + Timeout.from_seconds(1), + ) + finally: + await middleware.close() + + assert events == ["deadline", "serialize", "deadline"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("action_kind", "expected_source"), + (("detect", "pipeline_default"), ("deny", "gate")), +) +async def test_evaluation_log_records_decision_source( + action_kind: str, + expected_source: str, + caplog: pytest.LogCaptureFixture, +) -> None: + middleware = EgressGateMiddleware(create_builtin_registry()) + request = _request() + request.config.CopyFrom(_proto_config(_values(action_kind=action_kind))) + try: + with caplog.at_level(logging.INFO, logger=servicer_module.__name__): + await middleware._evaluate_rpc(request, _SuccessfulEvaluationContext()) + finally: + await middleware.close() + + record = next( + item + for item in caplog.records + if item.message.startswith("egress_gate_evaluation") + ) + assert getattr(record, "decision_source_kind", None) == expected_source + + +@pytest.mark.asyncio +async def test_evaluation_log_records_runtime_limit_source( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + middleware = EgressGateMiddleware(create_builtin_registry()) + + async def return_limit( + *args: object, + **kwargs: object, + ) -> tuple[pb2.HttpRequestResult, str]: + del args, kwargs + return servicer_module._limit_deny(), DecisionSourceKind.RUNTIME_LIMIT.value + + monkeypatch.setattr( + middleware, + "_evaluate_http_request", + return_limit, + ) + try: + with caplog.at_level(logging.INFO, logger=servicer_module.__name__): + await middleware._evaluate_rpc(_request(), _SuccessfulEvaluationContext()) + finally: + await middleware.close() + + record = next( + item + for item in caplog.records + if item.message.startswith("egress_gate_evaluation") + ) + assert getattr(record, "decision_source_kind", None) == "runtime_limit" + + +def test_serialized_limit_result_reports_runtime_limit_source() -> None: + result = EgressResult( + decision=EgressDecision.DENY, + decision_source=RuntimeLimitDecisionSource( + kind=DecisionSourceKind.RUNTIME_LIMIT + ), + reason_code=LIMIT_REASON_CODE, + ) + + response, source_kind = servicer_module._result_to_proto(result) + + assert response.reason_code == LIMIT_REASON_CODE + assert source_kind == DecisionSourceKind.RUNTIME_LIMIT.value + + +def test_invalid_utf8_is_an_input_failure_before_wire_evaluation() -> None: + middleware = EgressGateMiddleware(create_builtin_registry()) + request = _request(body=b"\xff") + try: + with pytest.raises(EgressGateError) as error: + asyncio.run( + middleware._evaluate_http_request( + request, + Timeout.from_seconds(1), + ) + ) + finally: + asyncio.run(middleware.close()) + + assert error.value.code is ErrorCode.BODY_ENCODING_INVALID + + +def test_service_request_body_limit_is_checked_before_worker_execution() -> None: + request = _request(body=b"x" * (MAX_BODY_BYTES + 1)) + middleware = EgressGateMiddleware(create_builtin_registry()) + try: + with pytest.raises(EgressGateError) as error: + asyncio.run( + middleware._evaluate_http_request( + request, + Timeout.from_seconds(1), + ) + ) + finally: + asyncio.run(middleware.close()) + + assert error.value.code is ErrorCode.REQUEST_BODY_TOO_LARGE + + +def test_default_deny_reason_is_wire_safe() -> None: + result = EgressResult( + decision=EgressDecision.DENY, + decision_source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + reason_code=DEFAULT_DENY_REASON_CODE, + ) + response, _ = servicer_module._result_to_proto(result) + assert response.reason == BLOCK_REASON + assert response.reason_code == DEFAULT_DENY_REASON_CODE diff --git a/projects/egress-gate/tests/test_cli.py b/projects/egress-gate/tests/test_cli.py new file mode 100644 index 00000000..e25a19e5 --- /dev/null +++ b/projects/egress-gate/tests/test_cli.py @@ -0,0 +1,534 @@ +"""Command-line tests for Egress Gate discovery and policy schema surfaces.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from types import ModuleType + +import pytest +import yaml +from rich.text import Text +from typer.testing import CliRunner + +from egress_gate.cli import _load_registry, app +from egress_gate.errors import GateRegistryError +from egress_gate.gates import GateRegistry, create_builtin_registry +from egress_gate.gateway_config import MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES + + +def test_cli_does_not_offer_request_content_logging() -> None: + result = CliRunner().invoke(app, ["--help"], color=True) + + assert result.exit_code == 0 + help_output = Text.from_ansi(result.stdout).plain + assert "--debug" in help_output + assert "--debug-log-content" not in help_output + + +def test_cli_bare_command_is_successful_help() -> None: + result = CliRunner().invoke(app, []) + + assert result.exit_code == 0 + assert "Usage: egress-gate" in result.stdout + assert "Register Egress Gate with OpenShell." in result.stdout + assert "Inspect installed gates and policy schema." in result.stdout + + +def test_cli_reports_the_installed_version() -> None: + result = CliRunner().invoke(app, ["--version"]) + + assert result.exit_code == 0 + assert result.stdout == "egress-gate 0.1.0\n" + + +def test_cli_narrow_help_preserves_complete_option_names() -> None: + result = CliRunner().invoke( + app, + ["add-gateway-registration", "--help"], + env={"COLUMNS": "40"}, + ) + + assert result.exit_code == 0 + assert "--host-ip" in result.stdout + assert "--config" in result.stdout + assert "--host…" not in result.stdout + assert "--conf…" not in result.stdout + + +def test_cli_gates_describes_the_request_level_builtin() -> None: + result = CliRunner().invoke(app, ["gates", "list"]) + + assert result.exit_code == 0 + assert "Installed gates" in result.stdout + assert "regex" in result.stdout + assert "regex_match" in result.stdout + assert "Request access" in result.stdout + assert "target, headers, body" in result.stdout + assert "Possible results" in result.stdout + assert "body replacement, findings, deny decision" in result.stdout + assert "RegexConfig" in result.stdout + assert "Python resources" not in result.stdout + + +def test_cli_configuration_schema_exposes_flat_policy() -> None: + result = CliRunner().invoke(app, ["gates", "schema"]) + + assert result.exit_code == 0 + schema = json.loads(result.stdout) + assert schema["title"] == "EgressGateConfig" + assert set(schema["properties"]) == {"gates", "default_decision"} + assert schema["properties"]["gates"]["minItems"] == 1 + assert schema["properties"]["gates"]["maxItems"] == 10 + + +def test_registry_loader_accepts_a_singleton_or_factory_and_seals_it( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = ModuleType("test_registry_source") + singleton = create_builtin_registry() + module.__dict__["registry"] = singleton + module.__dict__["create_registry"] = create_builtin_registry + module.__dict__["empty"] = GateRegistry() + monkeypatch.setitem(sys.modules, module.__name__, module) + + assert _load_registry("test_registry_source:registry") is singleton + factory_registry = _load_registry("test_registry_source:create_registry") + with pytest.raises(GateRegistryError, match="registry is in use"): + singleton.register(object) + with pytest.raises(GateRegistryError, match="registry is in use"): + factory_registry.register(object) + with pytest.raises(Exception, match="at least one valid gate"): + _load_registry("test_registry_source:empty") + + +@pytest.mark.parametrize( + "reference", + ["missing-separator", "test_registry_source:missing"], +) +def test_registry_loader_rejects_invalid_references(reference: str) -> None: + with pytest.raises(Exception): + _load_registry(reference) + + +def test_cli_evaluate_runs_the_builtin_policy_corpus() -> None: + project_dir = Path(__file__).parents[1] + result = CliRunner().invoke( + app, + [ + "evaluate", + "--policy", + str(project_dir / "examples/regex-redaction/egress-gate-config.yaml"), + "--cases", + str(project_dir / "examples/regex-redaction/cases.yaml"), + ], + ) + + assert result.exit_code == 0, result.output + assert "Policy evaluation" in result.stdout + assert "PASS" in result.stdout + assert "email-is-detected-and-request-is-allowed" in result.stdout + assert "2 passed · 0 failed · 2 total" in result.stdout + + +@pytest.mark.parametrize( + ("registry_reference", "example_directory"), + [ + ("examples.custom-gate.keyword_gate:registry", "custom-gate"), + ("examples.class-based-gate.keyword_gate:registry", "class-based-gate"), + ], +) +def test_cli_evaluate_runs_the_custom_gate_examples( + registry_reference: str, + example_directory: str, +) -> None: + project_dir = Path(__file__).parents[1] + result = CliRunner().invoke( + app, + [ + "--registry", + registry_reference, + "evaluate", + "--policy", + str(project_dir / f"examples/{example_directory}/egress-gate-config.yaml"), + "--cases", + str(project_dir / f"examples/{example_directory}/cases.yaml"), + ], + ) + + assert result.exit_code == 0, result.output + assert "configured-keyword-is-denied" in result.stdout + assert "other-bodies-proceed-to-the-default" in result.stdout + assert "2 passed · 0 failed · 2 total" in result.stdout + + +@pytest.mark.parametrize( + ("registry_reference", "example_directory", "registration_name"), + [ + (None, "regex-redaction", "eg-regex"), + ( + "examples.custom-gate.keyword_gate:registry", + "custom-gate", + "egress-function", + ), + ( + "examples.class-based-gate.keyword_gate:registry", + "class-based-gate", + "egress-class", + ), + ], +) +def test_openshell_example_policies_use_valid_gate_configuration( + monkeypatch: pytest.MonkeyPatch, + registry_reference: str | None, + example_directory: str, + registration_name: str, +) -> None: + project_dir = Path(__file__).parents[1] + policy_path = project_dir / f"examples/{example_directory}/policy.yaml" + policy = yaml.safe_load(policy_path.read_text()) + middleware = next(iter(policy["network_middlewares"].values())) + standalone_config = yaml.safe_load( + (policy_path.parent / "egress-gate-config.yaml").read_text() + ) + assert middleware["middleware"] == registration_name + assert len(registration_name) <= MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES + if example_directory == "regex-redaction": + monkeypatch.chdir(policy_path.parent) + _load_registry(registry_reference).validate_config(middleware["config"]) + + embedded_config = middleware["config"] + for gate in embedded_config["gates"]: + pattern_catalog = gate.get("pattern_catalog") + if isinstance(pattern_catalog, str): + gate["pattern_catalog"] = yaml.safe_load( + (policy_path.parent / pattern_catalog).read_text() + ) + + assert embedded_config == standalone_config + + +@pytest.mark.parametrize( + ("example_directory", "name"), + [ + ("regex-redaction", "eg-regex"), + ("custom-gate", "egress-function"), + ("class-based-gate", "egress-class"), + ], +) +def test_example_workflows_use_one_registration_and_sandbox_name( + example_directory: str, + name: str, +) -> None: + project_dir = Path(__file__).parents[1] + readme = (project_dir / f"examples/{example_directory}/README.md").read_text() + normalized_readme = " ".join(readme.replace("\\\n", " ").split()) + + assert f"--host-ip YOUR_HOST_IPV4 --name {name} --port 50051" in normalized_readme + assert f"openshell sandbox create --name {name}" in normalized_readme + assert f"openshell sandbox delete {name}" in readme + assert f"remove-gateway-registration --name {name}" in normalized_readme + assert "stop any running OpenShell gateways" in normalized_readme + assert ( + "A running gateway does not reload middleware registrations" + in normalized_readme + ) + + +def test_installed_executable_loads_a_registry_from_the_working_directory() -> None: + project_dir = Path(__file__).parents[1] + executable = Path(sys.executable).with_name("egress-gate") + + result = subprocess.run( + [ + executable, + "--registry", + "examples.custom-gate.keyword_gate:registry", + "gates", + "list", + ], + cwd=project_dir, + capture_output=True, + check=False, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert "Installed gates" in result.stdout + assert "keyword-deny" in result.stdout + + +def test_cli_validate_checks_policy_without_preparing_gates( + monkeypatch: pytest.MonkeyPatch, +) -> None: + project_dir = Path(__file__).parents[1] + + def unexpected_preparation(*args: object, **kwargs: object) -> object: + del args, kwargs + raise AssertionError("validation prepared a gate") + + monkeypatch.setattr(GateRegistry, "create_gate", unexpected_preparation) + result = CliRunner().invoke( + app, + [ + "validate", + "--policy", + str(project_dir / "examples/regex-redaction/egress-gate-config.yaml"), + ], + ) + + assert result.exit_code == 0, result.output + assert result.stdout == "✓ Policy is valid\n" + + +def test_cli_validate_rejects_invalid_policy(tmp_path: Path) -> None: + policy = tmp_path / "invalid.yaml" + policy.write_text("unexpected: true\n") + + result = CliRunner().invoke( + app, + ["validate", "--policy", str(policy)], + ) + + assert result.exit_code == 1 + assert "Policy validation failed [config_invalid]" in result.stderr + assert "Policy field gates: required field is missing" in result.stderr + assert "egress-gate gates schema" in result.stderr + + +def test_cli_validate_reports_a_safe_structural_path(tmp_path: Path) -> None: + sentinel = "scna-sensitive-sentinel" + policy = tmp_path / "invalid.yaml" + policy.write_text( + """gates: + - name: one + kind: regex + scna-sensitive-sentinel: {} + pattern_catalog: {} +default_decision: allow +""" + ) + + result = CliRunner().invoke(app, ["validate", "--policy", str(policy)]) + + assert result.exit_code == 1 + assert "Policy field gates[0].scan: required field is missing" in result.stderr + assert sentinel not in result.output + + +def test_cli_evaluate_catalogs_regex_preparation_failures(tmp_path: Path) -> None: + project_dir = Path(__file__).parents[1] + policy = tmp_path / "named-group.yaml" + policy.write_text( + """gates: + - name: identifiers + kind: regex + scan: + kind: body + action: {kind: detect} + pattern_catalog: + entities: + - name: token + rules: + - pattern: '(?Psecret)' + confidence: high +default_decision: allow +""" + ) + + result = CliRunner().invoke( + app, + [ + "evaluate", + "--policy", + str(policy), + "--cases", + str(project_dir / "examples/regex-redaction/cases.yaml"), + ], + ) + + assert result.exit_code == 2 + assert "Evaluation failed [config_preparation_failed]" in result.stderr + assert "remove named groups" in result.stderr + assert "sensitive_name" not in result.output + assert "custom gate and application-owned resource setup" not in result.output + + +def test_cli_evaluate_names_a_failing_case_and_keeps_completed_results( + tmp_path: Path, +) -> None: + project_dir = Path(__file__).parents[1] + original = (project_dir / "examples/regex-redaction/cases.yaml").read_text() + cases = tmp_path / "invalid-utf8.yaml" + cases.write_text( + original.replace( + 'encoding: utf8\n value: "ordinary text"', + 'encoding: base64\n value: "/w=="', + ) + ) + + result = CliRunner().invoke( + app, + [ + "evaluate", + "--policy", + str(project_dir / "examples/regex-redaction/egress-gate-config.yaml"), + "--cases", + str(cases), + ], + ) + + assert result.exit_code == 2 + assert "Completed before failure" in result.stdout + assert "email-is-detected-and-request-is-allowed" in result.stdout + assert "Evaluation failed for case ordinary-body-is-allowed" in result.stderr + assert "[body_encoding_invalid]" in result.stderr + assert '"/w=="' not in result.output + + +def test_cli_add_gateway_registration_reports_the_result(tmp_path: Path) -> None: + config = tmp_path / "gateway.toml" + result = CliRunner().invoke( + app, + [ + "add-gateway-registration", + "--host-ip", + "192.0.2.10", + "--config", + str(config), + ], + ) + + assert result.exit_code == 0, result.output + assert "Gateway registration is ready" in result.stdout + assert "Gateway file" in result.stdout + assert str(config) in "".join(result.stdout.split()) + assert "Registration egress-gate" in result.stdout + assert "Endpoint http://192.0.2.10:50051" in result.stdout + assert "Created the gateway configuration file" in result.stdout + assert "Next: Start Egress Gate" in result.stdout + + +def test_cli_lists_gateway_registration_names_for_removal(tmp_path: Path) -> None: + config = tmp_path / "gateway.toml" + config.write_text( + "[openshell]\n" + "version = 1\n\n" + "[[openshell.supervisor.middleware]]\n" + 'name = "eg-regex"\n' + 'grpc_endpoint = "http://192.0.2.10:50051"\n\n' + "[[openshell.supervisor.middleware]]\n" + 'name = "other-service"\n' + 'grpc_endpoint = "http://192.0.2.20:9000"\n' + ) + + result = CliRunner().invoke( + app, + ["list-gateway-registrations", "--config", str(config)], + ) + + assert result.exit_code == 0, result.output + assert "OpenShell middleware registrations" in result.stdout + assert "eg-regex" in result.stdout + assert "http://192.0.2.10:50051" in result.stdout + assert "other-service" in result.stdout + assert "remove-gateway-registration --name NAME" in result.stdout + + +def test_cli_lists_no_registrations_when_gateway_config_is_missing( + tmp_path: Path, +) -> None: + result = CliRunner().invoke( + app, + [ + "list-gateway-registrations", + "--config", + str(tmp_path / "missing.toml"), + ], + ) + + assert result.exit_code == 0, result.output + assert "No middleware registrations found." in result.stdout + + +def test_cli_evaluate_reports_content_safe_mismatch_status(tmp_path: Path) -> None: + project_dir = Path(__file__).parents[1] + cases = tmp_path / "cases.yaml" + original = (project_dir / "examples/regex-redaction/cases.yaml").read_text() + cases.write_text(original.replace("decision: allow", "decision: deny", 1)) + + result = CliRunner().invoke( + app, + [ + "evaluate", + "--policy", + str(project_dir / "examples/regex-redaction/egress-gate-config.yaml"), + "--cases", + str(cases), + ], + ) + + assert result.exit_code == 1 + assert "FAIL" in result.stdout + assert "email-is-detected" in result.stdout + assert "decision:" in result.stdout + assert '"deny"' in result.stdout + assert '"allow"' in result.stdout + assert "1 passed · 1 failed · 2 total" in result.stdout + assert "{}" not in result.stdout + + +@pytest.mark.parametrize( + "invalid_yaml", + [ + "version: 1\ncases: &cases []\n", + "version: 1\ncases:\n - name: one\n name: two\n", + ], +) +def test_cli_evaluate_rejects_non_strict_corpus_yaml( + tmp_path: Path, + invalid_yaml: str, +) -> None: + project_dir = Path(__file__).parents[1] + cases = tmp_path / "cases.yaml" + cases.write_text(invalid_yaml) + + result = CliRunner().invoke( + app, + [ + "evaluate", + "--policy", + str(project_dir / "examples/regex-redaction/egress-gate-config.yaml"), + "--cases", + str(cases), + ], + ) + + assert result.exit_code == 2 + assert "Evaluation could not start [invalid_cases_file]" in result.stderr + assert "valid version 1 YAML test suite" in result.stderr + assert "YAML aliases" not in result.output + + +def test_cli_evaluate_explains_an_invalid_timeout() -> None: + project_dir = Path(__file__).parents[1] + result = CliRunner().invoke( + app, + [ + "evaluate", + "--policy", + str(project_dir / "examples/regex-redaction/egress-gate-config.yaml"), + "--cases", + str(project_dir / "examples/regex-redaction/cases.yaml"), + "--timeout-seconds", + "0", + ], + color=True, + ) + + assert result.exit_code == 2 + error_output = Text.from_ansi(result.stderr).plain + assert "Invalid value for --timeout-seconds" in error_output + assert "greater than 0" in error_output diff --git a/projects/egress-gate/tests/test_config.py b/projects/egress-gate/tests/test_config.py new file mode 100644 index 00000000..3a91fc7f --- /dev/null +++ b/projects/egress-gate/tests/test_config.py @@ -0,0 +1,127 @@ +"""Strict pipeline configuration tests.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from egress_gate.config import DefaultDecision, EgressGateConfig +from egress_gate.constants import MAX_PIPELINE_GATES +from egress_gate.gates import RegexConfig + + +def _regex_config() -> dict[str, object]: + return { + "kind": "regex", + "scan": {"kind": "body", "action": {"kind": "detect"}}, + "pattern_catalog": { + "entities": [ + { + "name": "token", + "rules": [{"pattern": "secret", "confidence": "high"}], + } + ] + }, + } + + +def _values(*, default_decision: str = "allow") -> dict[str, object]: + return { + "gates": [{"name": "body", **_regex_config()}], + "default_decision": default_decision, + } + + +def test_pipeline_uses_required_default_and_exact_gate_entries() -> None: + config = EgressGateConfig[RegexConfig].model_validate(_values()) + + assert config.default_decision is DefaultDecision.ALLOW + assert config.gates[0].name == "body" + assert type(config.gates[0]) is RegexConfig + + +def test_pipeline_default_deny_is_explicit() -> None: + config = EgressGateConfig[RegexConfig].model_validate( + _values(default_decision="deny") + ) + assert config.default_decision is DefaultDecision.DENY + + missing_default = { + "gates": [{"name": "body", **_regex_config()}], + } + with pytest.raises(ValidationError): + EgressGateConfig[RegexConfig].model_validate(missing_default) + + +def test_pipeline_rejects_unknown_fields_and_duplicate_names() -> None: + unknown = { + "gates": [{"name": "body", **_regex_config()}], + "default_decision": "allow", + "unexpected": True, + } + with pytest.raises(ValidationError): + EgressGateConfig[RegexConfig].model_validate(unknown) + + duplicate = { + "gates": [ + {"name": "body", **_regex_config()}, + {"name": "body", **_regex_config()}, + ], + "default_decision": "allow", + } + with pytest.raises(ValidationError) as duplicate_error: + EgressGateConfig[RegexConfig].model_validate(duplicate) + assert duplicate_error.value.errors()[0]["loc"] == ("gates",) + + +def test_removed_policy_wrappers_are_rejected() -> None: + nested_policy = {"pipeline": _values()} + with pytest.raises(ValidationError): + EgressGateConfig[RegexConfig].model_validate(nested_policy) + + nested_gate = { + "gates": [{"name": "body", "config": _regex_config()}], + "default_decision": "allow", + } + with pytest.raises(ValidationError): + EgressGateConfig[RegexConfig].model_validate(nested_gate) + + +def test_pipeline_gate_count_has_an_exact_boundary() -> None: + exact_gates = [ + {"name": f"body-{index}", **_regex_config()} + for index in range(MAX_PIPELINE_GATES) + ] + exact = { + "gates": exact_gates, + "default_decision": "allow", + } + config = EgressGateConfig[RegexConfig].model_validate(exact) + assert len(config.gates) == MAX_PIPELINE_GATES + + too_many_gates = [ + *exact_gates, + {"name": "body-over", **_regex_config()}, + ] + too_many = { + "gates": too_many_gates, + "default_decision": "allow", + } + with pytest.raises(ValidationError): + EgressGateConfig[RegexConfig].model_validate(too_many) + + +def test_regex_scan_structurally_restricts_header_actions() -> None: + invalid = _regex_config() + invalid["scan"] = { + "kind": "header", + "names": ["x-note"], + "action": {"kind": "replace", "template": "[{entity}]"}, + } + with pytest.raises(ValidationError): + EgressGateConfig[RegexConfig].model_validate( + { + "gates": [{"name": "header", **invalid}], + "default_decision": "allow", + } + ) diff --git a/projects/egress-gate/tests/test_errors.py b/projects/egress-gate/tests/test_errors.py new file mode 100644 index 00000000..ee44af3a --- /dev/null +++ b/projects/egress-gate/tests/test_errors.py @@ -0,0 +1,57 @@ +import inspect + +from egress_gate.errors import ( + EgressGateError, + ErrorCode, + ErrorComponent, + ErrorKind, +) + + +def test_every_error_code_has_one_safe_complete_specification() -> None: + sentinel = "sensitive-request-value-8472" + + assert len({code.value for code in ErrorCode}) == len(ErrorCode) + for code in ErrorCode: + error = EgressGateError(code) + message = str(error) + + assert f"[{code.value}]" in message + assert error.component.value in message + assert error.operation in message + assert error.summary in message + assert error.hint in message + assert sentinel not in message + assert repr(error) == f"EgressGateError({message!r})" + + +def test_error_kinds_distinguish_invalid_input_from_internal_failures() -> None: + assert EgressGateError(ErrorCode.CONFIG_INVALID).kind is ErrorKind.INVALID_INPUT + assert EgressGateError(ErrorCode.GATE_EXECUTION_FAILED).kind is ErrorKind.INTERNAL + assert EgressGateError(ErrorCode.CONFIG_INVALID).component is ErrorComponent.CONFIG + + +def test_config_error_explains_the_transport_size_limit() -> None: + error = EgressGateError(ErrorCode.CONFIG_INVALID) + + assert "encoded configuration at or below 64 KiB" in error.hint + + +def test_config_preparation_error_has_builtin_regex_guidance() -> None: + error = EgressGateError(ErrorCode.CONFIG_PREPARATION_FAILED) + + assert error.kind is ErrorKind.INVALID_INPUT + assert "remove named groups" in error.hint + + +def test_malformed_protobuf_error_gives_safe_wire_contract_guidance() -> None: + error = EgressGateError(ErrorCode.REQUEST_PROTOBUF_INVALID) + + assert error.kind is ErrorKind.INVALID_INPUT + assert error.component is ErrorComponent.SERVICE + assert error.operation == "decode_protobuf" + assert "published OpenShell middleware protobuf contract" in error.hint + + +def test_egress_gate_error_exposes_only_a_catalog_code_parameter() -> None: + assert list(inspect.signature(EgressGateError).parameters) == ["code"] diff --git a/projects/privacy-guard/tests/test_gateway_config.py b/projects/egress-gate/tests/test_gateway_config.py similarity index 78% rename from projects/privacy-guard/tests/test_gateway_config.py rename to projects/egress-gate/tests/test_gateway_config.py index 36333013..a064fee0 100644 --- a/projects/privacy-guard/tests/test_gateway_config.py +++ b/projects/egress-gate/tests/test_gateway_config.py @@ -7,12 +7,14 @@ import pytest -from privacy_guard.gateway_config import ( +from egress_gate.gateway_config import ( MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES, GatewayConfigError, GatewayConfigRemoval, GatewayConfigUpdate, + GatewayMiddlewareRegistration, default_gateway_config_path, + list_gateway_registrations, remove_gateway_config, update_gateway_config, validate_middleware_name, @@ -55,6 +57,7 @@ def test_default_gateway_config_path_honors_openshell_override( def test_middleware_name_validation_matches_openshell_constraints() -> None: + assert MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES == 19 longest_name = "a" * MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES assert validate_middleware_name(longest_name) == longest_name @@ -62,9 +65,9 @@ def test_middleware_name_validation_matches_openshell_constraints() -> None: for invalid_name in ( "", "a" * (MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES + 1), - "privacy guard", + "egress gate", "priväcy-guard", - "openshell/privacy-guard", + "openshell/egress-gate", ): with pytest.raises(GatewayConfigError): validate_middleware_name(invalid_name) @@ -77,7 +80,7 @@ def test_update_gateway_config_creates_minimal_default_config( result = update_gateway_config( path, - middleware_name="privacy-guard", + middleware_name="egress-gate", host_ip="192.168.1.20", port=50051, ) @@ -90,7 +93,7 @@ def test_update_gateway_config_creates_minimal_default_config( "supervisor": { "middleware": [ { - "name": "privacy-guard", + "name": "egress-gate", "grpc_endpoint": "http://192.168.1.20:50051", "max_body_bytes": 4_194_304, "timeout": "5s", @@ -101,6 +104,35 @@ def test_update_gateway_config_creates_minimal_default_config( } +def test_list_gateway_registrations_returns_names_and_endpoints( + tmp_path: Path, +) -> None: + path = tmp_path / "gateway.toml" + path.write_text( + "[openshell]\n" + "version = 1\n\n" + "[[openshell.supervisor.middleware]]\n" + 'name = "eg-regex"\n' + 'grpc_endpoint = "http://10.0.0.3:50051"\n\n' + "[[openshell.supervisor.middleware]]\n" + 'name = "other-service"\n' + ) + + assert list_gateway_registrations(path) == ( + GatewayMiddlewareRegistration( + name="eg-regex", + endpoint="http://10.0.0.3:50051", + ), + GatewayMiddlewareRegistration(name="other-service", endpoint=None), + ) + + +def test_list_gateway_registrations_returns_empty_for_missing_file( + tmp_path: Path, +) -> None: + assert list_gateway_registrations(tmp_path / "missing.toml") == () + + def test_update_gateway_config_appends_without_rewriting_existing_settings( tmp_path: Path, ) -> None: @@ -116,7 +148,7 @@ def test_update_gateway_config_appends_without_rewriting_existing_settings( result = update_gateway_config( path, - middleware_name="privacy-guard", + middleware_name="egress-gate", host_ip="10.0.0.12", port=50052, ) @@ -138,7 +170,7 @@ def test_update_gateway_config_updates_only_the_named_registration( "max_body_bytes = 1000\n" 'timeout = "1s"\n\n' "[[openshell.supervisor.middleware]]\n" - 'name = "privacy-guard"\n' + 'name = "egress-gate"\n' "# Keep this registration comment.\n" 'grpc_endpoint = "http://10.0.0.3:50051"\n' "max_body_bytes = 2048\n" @@ -147,7 +179,7 @@ def test_update_gateway_config_updates_only_the_named_registration( result = update_gateway_config( path, - middleware_name="privacy-guard", + middleware_name="egress-gate", host_ip="10.0.0.4", port=50053, ) @@ -162,7 +194,7 @@ def test_update_gateway_config_updates_only_the_named_registration( repeated = update_gateway_config( path, - middleware_name="privacy-guard", + middleware_name="egress-gate", host_ip="10.0.0.4", port=50053, ) @@ -188,7 +220,7 @@ def test_update_gateway_config_rejects_invalid_existing_config( with pytest.raises(GatewayConfigError): update_gateway_config( path, - middleware_name="privacy-guard", + middleware_name="egress-gate", host_ip="192.168.1.20", port=50051, ) @@ -205,7 +237,7 @@ def test_remove_gateway_config_removes_only_the_named_registration( "[openshell]\n" "version = 1\n\n" "[[openshell.supervisor.middleware]]\n" - 'name = "privacy-guard-regex"\n' + 'name = "egress-gate-regex"\n' 'grpc_endpoint = "http://10.0.0.3:50051"\n' "max_body_bytes = 4194304\n" 'timeout = "5s"\n\n' @@ -219,12 +251,12 @@ def test_remove_gateway_config_removes_only_the_named_registration( result = remove_gateway_config( path, - middleware_name="privacy-guard-regex", + middleware_name="egress-gate-regex", ) assert result is GatewayConfigRemoval.REMOVED contents = path.read_text() - assert "privacy-guard-regex" not in contents + assert "egress-gate-regex" not in contents assert "# Keep this operator comment." in contents assert "# Keep this other-service comment." in contents assert 'name = "other-service"' in contents @@ -238,6 +270,25 @@ def test_remove_gateway_config_removes_only_the_named_registration( ] +def test_remove_gateway_config_can_remove_a_legacy_long_name(tmp_path: Path) -> None: + path = tmp_path / "gateway.toml" + path.write_text( + "[openshell]\n" + "version = 1\n\n" + "[[openshell.supervisor.middleware]]\n" + 'name = "legacy-registration-name"\n' + 'grpc_endpoint = "http://10.0.0.3:50051"\n' + ) + + result = remove_gateway_config( + path, + middleware_name="legacy-registration-name", + ) + + assert result is GatewayConfigRemoval.REMOVED + assert "legacy-registration-name" not in path.read_text() + + @pytest.mark.parametrize("create_file", [False, True]) def test_remove_gateway_config_is_unchanged_when_registration_is_absent( tmp_path: Path, @@ -249,7 +300,7 @@ def test_remove_gateway_config_is_unchanged_when_registration_is_absent( result = remove_gateway_config( path, - middleware_name="privacy-guard-regex", + middleware_name="egress-gate-regex", ) assert result is GatewayConfigRemoval.UNCHANGED @@ -267,16 +318,16 @@ def test_remove_gateway_config_rejects_duplicate_named_registrations( "[openshell]\n" "version = 1\n\n" "[[openshell.supervisor.middleware]]\n" - 'name = "privacy-guard-regex"\n\n' + 'name = "egress-gate-regex"\n\n' "[[openshell.supervisor.middleware]]\n" - 'name = "privacy-guard-regex"\n' + 'name = "egress-gate-regex"\n' ) path.write_text(contents) with pytest.raises(GatewayConfigError, match="multiple middleware registrations"): remove_gateway_config( path, - middleware_name="privacy-guard-regex", + middleware_name="egress-gate-regex", ) assert path.read_text() == contents @@ -290,7 +341,7 @@ def test_remove_gateway_config_rejects_registration_child_tables( "[openshell]\n" "version = 1\n\n" "[[openshell.supervisor.middleware]]\n" - 'name = "privacy-guard-regex"\n' + 'name = "egress-gate-regex"\n' 'grpc_endpoint = "http://10.0.0.3:50051"\n\n' "[openshell.supervisor.middleware.metadata]\n" 'owner = "privacy-team"\n' @@ -300,7 +351,7 @@ def test_remove_gateway_config_rejects_registration_child_tables( with pytest.raises(GatewayConfigError, match="Could not safely remove"): remove_gateway_config( path, - middleware_name="privacy-guard-regex", + middleware_name="egress-gate-regex", ) assert path.read_text() == contents @@ -321,7 +372,7 @@ def test_remove_gateway_config_rejects_table_headers_inside_multiline_strings( '"""\n\n' "[openshell.supervisor]\n" "middleware = [\n" - ' { name = "privacy-guard-regex", ' + ' { name = "egress-gate-regex", ' 'grpc_endpoint = "http://10.0.0.3:50051" },\n' "]\n" ) @@ -330,7 +381,7 @@ def test_remove_gateway_config_rejects_table_headers_inside_multiline_strings( with pytest.raises(GatewayConfigError, match="Could not safely remove"): remove_gateway_config( path, - middleware_name="privacy-guard-regex", + middleware_name="egress-gate-regex", ) assert path.read_text() == contents @@ -344,7 +395,7 @@ def test_remove_gateway_config_reports_unsafe_multiline_registration_layout( "[openshell]\n" "version = 1\n\n" "[[openshell.supervisor.middleware]]\n" - 'name = "privacy-guard-regex"\n' + 'name = "egress-gate-regex"\n' 'description = """\n' "[looks.like.a.table]\n" "still string\n" @@ -356,7 +407,7 @@ def test_remove_gateway_config_reports_unsafe_multiline_registration_layout( with pytest.raises(GatewayConfigError, match="Could not safely remove"): remove_gateway_config( path, - middleware_name="privacy-guard-regex", + middleware_name="egress-gate-regex", ) assert path.read_text() == contents diff --git a/projects/privacy-guard/tests/test_logging.py b/projects/egress-gate/tests/test_logging.py similarity index 68% rename from projects/privacy-guard/tests/test_logging.py rename to projects/egress-gate/tests/test_logging.py index 6521c4de..35c578a7 100644 --- a/projects/privacy-guard/tests/test_logging.py +++ b/projects/egress-gate/tests/test_logging.py @@ -1,4 +1,4 @@ -"""Privacy Guard logging configuration tests.""" +"""Egress Gate logging configuration tests.""" from __future__ import annotations @@ -11,7 +11,7 @@ import pytest -from privacy_guard.logging import ( +from egress_gate.logging import ( DEFAULT_LOGGING_CONFIG, ColorMode, LoggingConfig, @@ -32,13 +32,13 @@ def test_configure_logging_emits_consistent_package_logs() -> None: stream = StringIO() configure_logging(LoggingConfig(stream=stream)) - logging.getLogger("privacy_guard.service").info("server_started") - logging.getLogger("privacy_guard.service").debug("hidden_detail") + logging.getLogger("egress_gate.service").info("server_started") + logging.getLogger("egress_gate.service").debug("hidden_detail") output = stream.getvalue() assert re.fullmatch( r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}" - r" \| INFO \| privacy_guard\.service \| server_started\n", + r" \| INFO \| egress_gate\.service \| server_started\n", output, ) assert "hidden_detail" not in output @@ -53,50 +53,70 @@ def test_default_logging_config_uses_info_and_terminal_aware_colors() -> None: ) -def test_configure_logging_colors_interactive_output() -> None: +def test_configure_logging_colors_interactive_output( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("NO_COLOR", raising=False) stream = _TerminalStream() configure_logging(LoggingConfig(stream=stream)) - logging.getLogger("privacy_guard.service").warning("resource_pressure") + logging.getLogger("egress_gate.service").warning("resource_pressure") output = stream.getvalue() assert "\033[33mWARNING \033[0m" in output - assert "\033[36mprivacy_guard.service\033[0m" in output + assert "\033[36megress_gate.service\033[0m" in output assert output.endswith(" | resource_pressure\n") +@pytest.mark.parametrize("no_color", ["", "1"]) +def test_configure_logging_honors_no_color_for_interactive_output( + monkeypatch: pytest.MonkeyPatch, + no_color: str, +) -> None: + monkeypatch.setenv("NO_COLOR", no_color) + stream = _TerminalStream() + configure_logging(LoggingConfig(stream=stream)) + + logging.getLogger("egress_gate.service").warning("resource_pressure") + + assert "\033[" not in stream.getvalue() + + def test_configure_logging_can_disable_terminal_colors() -> None: stream = _TerminalStream() configure_logging(LoggingConfig(stream=stream, color_mode=ColorMode.NEVER)) - logging.getLogger("privacy_guard.service").error("startup_failed") + logging.getLogger("egress_gate.service").error("startup_failed") assert "\033[" not in stream.getvalue() -def test_configure_logging_can_force_colors_for_redirected_output() -> None: +def test_configure_logging_can_force_colors_for_redirected_output( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("NO_COLOR", "1") stream = StringIO() configure_logging(LoggingConfig(stream=stream, color_mode=ColorMode.ALWAYS)) - logging.getLogger("privacy_guard.service").info("server_started") + logging.getLogger("egress_gate.service").info("server_started") assert "\033[32mINFO \033[0m" in stream.getvalue() def test_get_logger_returns_named_package_logger() -> None: - logger = get_logger("privacy_guard.custom_engine") + logger = get_logger("egress_gate.custom_gate") - assert logger is logging.getLogger("privacy_guard.custom_engine") + assert logger is logging.getLogger("egress_gate.custom_gate") def test_configure_logging_accepts_native_log_levels() -> None: stream = StringIO() configure_logging(LoggingConfig(level=logging.DEBUG, stream=stream)) - logging.getLogger("privacy_guard.request_processor").debug("processing_diagnostic") + logging.getLogger("egress_gate.request_processor").debug("processing_diagnostic") assert ( - "DEBUG | privacy_guard.request_processor | processing_diagnostic" + "DEBUG | egress_gate.request_processor | processing_diagnostic" in stream.getvalue() ) @@ -107,14 +127,14 @@ def test_configure_logging_replaces_its_previous_handler() -> None: configure_logging(LoggingConfig(stream=first_stream)) configure_logging(LoggingConfig(stream=second_stream)) - logging.getLogger("privacy_guard").info("configured_once") + logging.getLogger("egress_gate").info("configured_once") assert "configured_once" not in first_stream.getvalue() assert second_stream.getvalue().count("configured_once") == 1 def test_reset_logging_restores_application_logging() -> None: - package_logger = logging.getLogger("privacy_guard") + package_logger = logging.getLogger("egress_gate") application_handler = logging.NullHandler() package_logger.addHandler(application_handler) configure_logging(LoggingConfig(stream=StringIO())) @@ -148,7 +168,7 @@ def test_source_modules_use_the_shared_logging_module() -> None: assert direct_imports == [] -_SOURCE_ROOT = Path(__file__).parents[1] / "src" / "privacy_guard" +_SOURCE_ROOT = Path(__file__).parents[1] / "src" / "egress_gate" _LOGGING_MODULE = _SOURCE_ROOT / "logging.py" diff --git a/projects/egress-gate/tests/test_request.py b/projects/egress-gate/tests/test_request.py new file mode 100644 index 00000000..e4ccd575 --- /dev/null +++ b/projects/egress-gate/tests/test_request.py @@ -0,0 +1,236 @@ +"""Focused boundary tests for the protobuf-free request domain models.""" + +from __future__ import annotations + +import pytest +from pydantic import TypeAdapter, ValidationError + +from egress_gate.constants import ( + MAX_BODY_BYTES, + MAX_HEADER_MUTATION_DATA_BYTES, + MAX_HEADER_MUTATIONS, + MAX_PROTO_CONTEXT_BYTES, + MAX_PROTO_HEADERS, + MAX_PROTO_HEADERS_BYTES, + MAX_PROTO_TARGET_BYTES, +) +from egress_gate.request import ( + ExistingHeaderAction, + HeaderMutation, + HttpHeader, + HttpRequest, + HttpTarget, + Process, + RemoveHeaderMutation, + RequestContext, + RequestMutations, + WriteHeaderMutation, +) + + +def _request( + *, body: bytes = b"payload", headers: tuple[HttpHeader, ...] = () +) -> HttpRequest: + return HttpRequest( + context=RequestContext( + request_id="request-1", + sandbox_id="sandbox-1", + originating_process=Process( + binary="/usr/bin/python", + pid=42, + ancestors=("/usr/bin/init",), + ), + ), + target=HttpTarget( + scheme="https", + host="example.com", + port=443, + method="POST", + path="/v1/items", + query="page=1", + ), + headers=headers, + body=body, + ) + + +def test_request_is_immutable_and_preserves_ordered_headers() -> None: + headers = ( + HttpHeader(name="x-test", value="first"), + HttpHeader(name="x-test", value="second"), + ) + request = _request(headers=headers) + + assert request.body == b"payload" + assert request.headers == headers + assert request.context.originating_process is not None + assert request.context.originating_process.ancestors == ("/usr/bin/init",) + + with pytest.raises(ValidationError): + setattr(request, "body", b"changed") + + +@pytest.mark.parametrize( + ("body", "valid"), + [(b"x" * MAX_BODY_BYTES, True), (b"x" * (MAX_BODY_BYTES + 1), False)], +) +def test_request_body_boundary(body: bytes, valid: bool) -> None: + if valid: + assert _request(body=body).body == body + else: + with pytest.raises(ValidationError): + _request(body=body) + + +def test_header_count_and_data_boundaries() -> None: + headers = tuple( + HttpHeader(name=f"x-{index}", value="v") for index in range(MAX_PROTO_HEADERS) + ) + assert len(_request(headers=headers).headers) == MAX_PROTO_HEADERS + + with pytest.raises(ValidationError): + _request(headers=headers + (HttpHeader(name="x-over", value="v"),)) + + exact_data = (HttpHeader(name="x", value="x" * (MAX_PROTO_HEADERS_BYTES - 1)),) + assert _request(headers=exact_data).headers == exact_data + with pytest.raises(ValidationError): + _request(headers=(HttpHeader(name="x", value="x" * MAX_PROTO_HEADERS_BYTES),)) + + +def test_request_mutations_distinguish_no_replacement_from_empty_body() -> None: + no_replacement = RequestMutations() + empty_replacement = RequestMutations(replacement_body=b"") + + assert no_replacement.is_empty + assert not empty_replacement.is_empty + + +def test_request_mutations_preserve_ordered_discriminated_header_mutations() -> None: + adapter = TypeAdapter(HeaderMutation) + discriminator = adapter.json_schema().get("discriminator") + assert isinstance(discriminator, dict) + assert discriminator.get("propertyName") == "kind" + + request_mutations = RequestMutations( + header_mutations=( + WriteHeaderMutation( + kind="write", + name="x-test", + value="one", + on_existing=ExistingHeaderAction.APPEND, + ), + RemoveHeaderMutation(kind="remove", name="x-old"), + ) + ) + + assert request_mutations.header_mutations[0].kind == "write" + assert request_mutations.header_mutations[1].kind == "remove" + + with pytest.raises(ValidationError): + adapter.validate_python({"name": "x-test"}) + with pytest.raises(ValidationError): + adapter.validate_python({"operation": "remove", "name": "x-test"}) + + +def test_request_mutations_reject_invalid_bounds() -> None: + mutation = RemoveHeaderMutation(kind="remove", name="x-test") + with pytest.raises(ValidationError): + RequestMutations( + header_mutations=tuple(mutation for _ in range(MAX_HEADER_MUTATIONS + 1)) + ) + + with pytest.raises(ValidationError): + RequestMutations( + header_mutations=( + WriteHeaderMutation( + kind="write", + name="x", + value="x" * MAX_HEADER_MUTATION_DATA_BYTES, + on_existing=ExistingHeaderAction.OVERWRITE, + ), + ) + ) + + +def test_request_models_reject_non_tuple_sequences_and_extra_fields() -> None: + values: dict[str, object] = { + "context": RequestContext(request_id="id", sandbox_id="sandbox"), + "target": HttpTarget( + scheme="https", + host="example.com", + port=443, + method="GET", + path="/", + query="", + ), + "headers": [HttpHeader(name="x", value="y")], + "body": b"payload", + } + with pytest.raises(ValidationError): + HttpRequest.model_validate(values) + + with pytest.raises(ValidationError): + HttpHeader(name="", value="value") + + with pytest.raises(ValidationError): + HttpTarget.model_validate( + { + "scheme": "https", + "host": "example.com", + "port": 443, + "method": "GET", + "path": "/", + "query": "", + "extra": "forbidden", + } + ) + + +def test_request_context_string_aggregate_has_an_exact_boundary() -> None: + exact = RequestContext( + request_id="r" * (MAX_PROTO_CONTEXT_BYTES - 1), + sandbox_id="s", + ) + assert len(exact.request_id.encode()) + len(exact.sandbox_id.encode()) == ( + MAX_PROTO_CONTEXT_BYTES + ) + + with pytest.raises(ValidationError): + RequestContext( + request_id="r" * MAX_PROTO_CONTEXT_BYTES, + sandbox_id="s", + ) + + +def test_http_target_string_aggregate_has_an_exact_boundary() -> None: + exact = HttpTarget( + scheme="s" * (MAX_PROTO_TARGET_BYTES - 4), + host="h", + port=443, + method="m", + path="p", + query="q", + ) + assert ( + sum( + len(value.encode()) + for value in ( + exact.scheme, + exact.host, + exact.method, + exact.path, + exact.query, + ) + ) + == MAX_PROTO_TARGET_BYTES + ) + + with pytest.raises(ValidationError): + HttpTarget( + scheme="s" * (MAX_PROTO_TARGET_BYTES - 3), + host="h", + port=443, + method="m", + path="p", + query="q", + ) diff --git a/projects/egress-gate/tests/test_request_processor.py b/projects/egress-gate/tests/test_request_processor.py new file mode 100644 index 00000000..05dcd094 --- /dev/null +++ b/projects/egress-gate/tests/test_request_processor.py @@ -0,0 +1,587 @@ +"""Ordered current-request execution and decision-provenance tests.""" + +from __future__ import annotations + +import inspect +from time import monotonic +from typing import Literal + +import pytest + +from egress_gate.config import DefaultDecision +from egress_gate.constants import ( + DEFAULT_DENY_REASON_CODE, + LIMIT_REASON_CODE, + MAX_FINDING_COUNT, + MAX_HEADER_MUTATIONS, + MAX_PROTO_FINDING_GROUPS, +) +from egress_gate.errors import ( + EgressGateError, + ErrorCode, + GateContractError, +) +from egress_gate.gates import ( + Gate, + GateCapability, + GateConfig, + GateRegistry, +) +from egress_gate.request import ( + ExistingHeaderAction, + HttpHeader, + HttpRequest, + HttpTarget, + RemoveHeaderMutation, + RequestContext, + RequestMutations, + WriteHeaderMutation, +) +from egress_gate.request_processor import RequestProcessor, apply_request_mutations +from egress_gate.result import ( + DecisionSourceKind, + EgressDecision, + Finding, + FindingTypeDefinition, + GateDecisionSource, + GateEvaluation, +) +from egress_gate.timeout import Timeout + +_BOUNDARY_FINDING_TYPE = "t" * 1024 + + +class _ControlConfig(GateConfig): + kind: Literal["test-control"] + control: Literal["proceed", "allow", "deny"] = "proceed" + replacement: str | None = None + expected_body: str | None = None + header_value: str | None = None + header_count: int = 0 + finding_label: str | None = None + finding_count: int = 1 + emit_twice: bool = False + boundary_finding: bool = False + reason_code: str | None = None + + +class _ControlGate(Gate[_ControlConfig, None]): + capabilities = frozenset( + { + GateCapability.READ_BODY, + GateCapability.REPLACE_BODY, + GateCapability.MUTATE_HEADERS, + GateCapability.ALLOW, + GateCapability.DENY, + } + ) + finding_types = ( + FindingTypeDefinition(type="test_observation"), + FindingTypeDefinition(type=_BOUNDARY_FINDING_TYPE), + ) + + def _evaluate( + self, + request: HttpRequest, + *, + timeout: Timeout, + ) -> GateEvaluation: + timeout.raise_if_expired() + if ( + self.config.expected_body is not None + and request.body.decode("utf-8") != self.config.expected_body + ): + raise AssertionError("later gate did not see the current request") + findings: tuple[Finding, ...] = () + if self.config.boundary_finding: + finding = Finding( + type=_BOUNDARY_FINDING_TYPE, + label="x" * 1024, + count=64, + confidence="c" * 1024, + severity="s" * 1010, + ) + findings = (finding, finding) + elif self.config.finding_label is not None: + finding = Finding( + type="test_observation", + label=self.config.finding_label, + count=self.config.finding_count, + ) + findings = (finding,) + if self.config.emit_twice: + findings += (finding,) + if self.config.control == "deny": + return GateEvaluation.deny( + self.config.reason_code or "egress_gate_test_denied", + findings=findings, + ) + if self.config.control == "allow": + return GateEvaluation.allow(findings=findings) + mutations: tuple[WriteHeaderMutation, ...] = tuple( + WriteHeaderMutation( + kind="write", + name=f"x-openshell-middleware-test-{index}", + value=self.config.header_value or "true", + on_existing=ExistingHeaderAction.OVERWRITE, + ) + for index in range(self.config.header_count) + ) + if self.config.header_value is not None and not mutations: + mutations = ( + WriteHeaderMutation( + kind="write", + name="x-openshell-middleware-test", + value=self.config.header_value, + on_existing=ExistingHeaderAction.OVERWRITE, + ), + ) + return GateEvaluation.proceed( + request_mutations=RequestMutations( + replacement_body=( + None + if self.config.replacement is None + else self.config.replacement.encode("utf-8") + ), + header_mutations=mutations, + ), + findings=findings, + ) + + +def _request( + *, body: bytes = b"original", headers: tuple[HttpHeader, ...] = () +) -> HttpRequest: + return HttpRequest( + context=RequestContext(request_id="request-1", sandbox_id="sandbox-1"), + target=HttpTarget( + scheme="https", + host="example.com", + port=443, + method="POST", + path="/", + query="", + ), + headers=headers, + body=body, + ) + + +def _regex_config( + action_kind: str = "detect", + *, + scan: dict[str, object] | None = None, +) -> dict[str, object]: + scan_values = {"kind": "body"} if scan is None else dict(scan) + action: dict[str, object] = {"kind": action_kind} + if action_kind == "replace": + action["template"] = "[{entity}]" + scan_values["action"] = action + return { + "kind": "regex", + "scan": scan_values, + "pattern_catalog": { + "entities": [ + { + "name": "token", + "rules": [{"pattern": "secret", "confidence": "high"}], + } + ] + }, + } + + +def _processor( + gate_values: tuple[tuple[str, dict[str, object]], ...], + *, + default_decision: DefaultDecision = DefaultDecision.ALLOW, + include_regex: bool = False, +) -> RequestProcessor: + registry = GateRegistry(include_builtin_gates=include_regex) + registry.register(_ControlGate) + values = { + "gates": [{"name": name, **config} for name, config in gate_values], + "default_decision": default_decision.value, + } + config = registry.validate_config(values) + prepared_items = [] + for entry in config.gates: + gate_type = getattr(entry, "kind", None) + if not isinstance(gate_type, str): + raise AssertionError("test gate config has no discriminator") + prepared_items.append((entry.name, gate_type, registry.create_gate(entry))) + prepared = tuple(prepared_items) + return RequestProcessor( + config, + prepared, + policy_fingerprint="policy-fingerprint", + ) + + +def test_processor_process_requires_the_service_created_timeout() -> None: + process_signature = inspect.signature(RequestProcessor.process) + assert "timeout_seconds" not in inspect.signature(RequestProcessor).parameters + assert ( + process_signature.parameters["timeout"].kind is inspect.Parameter.KEYWORD_ONLY + ) + + processor = _processor((("one", {"kind": "test-control", "control": "proceed"}),)) + result = processor.process(_request(), timeout=Timeout.from_seconds(1)) + assert result.decision is EgressDecision.ALLOW + + +def test_processor_applies_mutations_to_the_current_request_and_preserves_intent() -> ( + None +): + processor = _processor( + ( + ( + "redact", + { + "kind": "test-control", + "replacement": "redacted", + "finding_label": "secret", + }, + ), + ( + "observe", + { + "kind": "test-control", + "expected_body": "redacted", + "header_value": "true", + "finding_label": "observed", + }, + ), + ) + ) + + result = processor.process(_request(), timeout=Timeout.from_seconds(1)) + + assert result.decision is EgressDecision.ALLOW + assert result.decision_source.kind is DecisionSourceKind.PIPELINE_DEFAULT + assert result.request_mutations.replacement_body == b"redacted" + mutation = result.request_mutations.header_mutations[0] + assert isinstance(mutation, WriteHeaderMutation) + assert mutation.value == "true" + assert [(item.source_gate, item.finding.label) for item in result.findings] == [ + ("redact", "secret"), + ("observe", "observed"), + ] + assert [trace.gate_type for trace in result.traces] == [ + "test-control", + "test-control", + ] + assert result.policy_fingerprint == "policy-fingerprint" + + +def test_regex_gate_sees_header_mutations_from_an_earlier_gate() -> None: + processor = _processor( + ( + ( + "add-header", + { + "kind": "test-control", + "header_value": "contains secret", + }, + ), + ( + "inspect-header", + _regex_config( + "deny", + scan={ + "kind": "header", + "names": ["x-openshell-middleware-test"], + }, + ), + ), + ), + include_regex=True, + ) + + result = processor.process(_request(), timeout=Timeout.from_seconds(1)) + + assert result.decision is EgressDecision.DENY + assert isinstance(result.decision_source, GateDecisionSource) + assert result.decision_source.gate_name == "inspect-header" + assert result.request_mutations.is_empty + + +def test_processor_aggregates_equivalent_findings_by_gate_provenance() -> None: + processor = _processor( + ( + ( + "one", + {"kind": "test-control", "finding_label": "same", "emit_twice": True}, + ), + ) + ) + result = processor.process(_request(), timeout=Timeout.from_seconds(1)) + + assert len(result.findings) == 1 + assert result.findings[0].source_gate == "one" + assert result.findings[0].finding.count == 2 + + +def test_aggregated_finding_size_exhaustion_returns_a_runtime_limit() -> None: + processor = _processor( + ( + ( + "one", + { + "kind": "test-control", + "boundary_finding": True, + }, + ), + ) + ) + + result = processor.process(_request(), timeout=Timeout.from_seconds(1)) + + assert result.decision is EgressDecision.DENY + assert result.decision_source.kind is DecisionSourceKind.RUNTIME_LIMIT + assert result.reason_code == LIMIT_REASON_CODE + assert result.findings == () + + +def test_terminal_decisions_skip_later_gates() -> None: + deny = _processor( + ( + ( + "deny", + { + "kind": "test-control", + "control": "deny", + "reason_code": "policy_denied", + }, + ), + ( + "never", + { + "kind": "test-control", + "expected_body": "this gate must not run", + }, + ), + ) + ) + allow = _processor( + ( + ("allow", {"kind": "test-control", "control": "allow"}), + ( + "never", + { + "kind": "test-control", + "expected_body": "this gate must not run", + }, + ), + ) + ) + + denied = deny.process(_request(), timeout=Timeout.from_seconds(1)) + allowed = allow.process(_request(), timeout=Timeout.from_seconds(1)) + + assert denied.decision is EgressDecision.DENY + assert denied.decision_source.kind is DecisionSourceKind.GATE + assert isinstance(denied.decision_source, GateDecisionSource) + assert denied.decision_source.gate_name == "deny" + assert denied.reason_code == "policy_denied" + assert allowed.decision is EgressDecision.ALLOW + assert isinstance(allowed.decision_source, GateDecisionSource) + assert allowed.decision_source.gate_name == "allow" + + +def test_default_deny_owns_its_reason_and_discards_accumulated_mutations() -> None: + processor = _processor( + (("redact", {"kind": "test-control", "replacement": "redacted"}),), + default_decision=DefaultDecision.DENY, + ) + + result = processor.process(_request(), timeout=Timeout.from_seconds(1)) + + assert result.decision is EgressDecision.DENY + assert result.decision_source.kind is DecisionSourceKind.PIPELINE_DEFAULT + assert result.reason_code == DEFAULT_DENY_REASON_CODE + assert result.request_mutations.is_empty + + +def test_expired_shared_timeout_returns_atomic_runtime_limit_result() -> None: + processor = _processor((("one", {"kind": "test-control", "control": "proceed"}),)) + + result = processor.process( + _request(), + timeout=Timeout(deadline=monotonic() - 1), + ) + + assert result.decision is EgressDecision.DENY + assert result.decision_source.kind is DecisionSourceKind.RUNTIME_LIMIT + assert result.reason_code == LIMIT_REASON_CODE + assert result.request_mutations.is_empty + + +def test_regex_finding_group_overflow_is_an_atomic_runtime_limit() -> None: + processor = _processor( + ( + ( + "regex", + { + "kind": "regex", + "scan": {"kind": "body", "action": {"kind": "detect"}}, + "pattern_catalog": { + "entities": [ + { + "name": f"entity-{index}", + "rules": [{"pattern": "x", "confidence": "high"}], + } + for index in range(MAX_PROTO_FINDING_GROUPS + 1) + ] + }, + }, + ), + ), + include_regex=True, + ) + + result = processor.process( + _request(body=b"x"), + timeout=Timeout.from_seconds(1), + ) + + assert result.decision is EgressDecision.DENY + assert result.decision_source.kind is DecisionSourceKind.RUNTIME_LIMIT + assert result.reason_code == LIMIT_REASON_CODE + assert not result.findings + assert result.request_mutations.is_empty + + +def test_composed_header_mutation_overflow_is_an_atomic_runtime_limit() -> None: + processor = _processor( + ( + ( + "first", + {"kind": "test-control", "header_count": MAX_HEADER_MUTATIONS // 2}, + ), + ( + "second", + { + "kind": "test-control", + "header_count": MAX_HEADER_MUTATIONS // 2 + 1, + }, + ), + ) + ) + + result = processor.process(_request(), timeout=Timeout.from_seconds(1)) + + assert result.decision is EgressDecision.DENY + assert result.decision_source.kind is DecisionSourceKind.RUNTIME_LIMIT + assert result.reason_code == LIMIT_REASON_CODE + assert result.request_mutations.is_empty + assert result.findings == () + assert result.traces == () + + +def test_trace_finding_count_overflow_is_an_atomic_runtime_limit() -> None: + processor = _processor( + ( + ( + "observations", + { + "kind": "test-control", + "finding_label": "same", + "finding_count": MAX_FINDING_COUNT, + "emit_twice": True, + }, + ), + ) + ) + + result = processor.process(_request(), timeout=Timeout.from_seconds(1)) + + assert result.decision is EgressDecision.DENY + assert result.decision_source.kind is DecisionSourceKind.RUNTIME_LIMIT + assert result.reason_code == LIMIT_REASON_CODE + assert result.request_mutations.is_empty + assert result.findings == () + assert result.traces == () + + +def test_invalid_utf8_is_translated_to_the_stable_input_error() -> None: + processor = _processor( + (("regex", _regex_config()),), + include_regex=True, + ) + + with pytest.raises(EgressGateError) as error: + processor.process(_request(body=b"\xff"), timeout=Timeout.from_seconds(1)) + + assert error.value.code is ErrorCode.BODY_ENCODING_INVALID + + +def test_prepared_gate_type_is_part_of_the_processor_contract() -> None: + registry = GateRegistry() + registry.register(_ControlGate) + config = registry.validate_config( + { + "gates": [{"name": "one", "kind": "test-control", "control": "proceed"}], + "default_decision": "allow", + } + ) + gate = registry.create_gate(config.gates[0]) + + with pytest.raises(ValueError): + RequestProcessor( + config, + (("one", "wrong-type", gate),), + ) + + +def test_header_mutations_are_ordered_and_protected() -> None: + original = _request( + headers=( + HttpHeader(name="x-openshell-middleware-test", value="old"), + HttpHeader(name="x-other", value="keep"), + ) + ) + request_mutations = RequestMutations( + header_mutations=( + WriteHeaderMutation( + kind="write", + name="x-openshell-middleware-test", + value="new", + on_existing=ExistingHeaderAction.OVERWRITE, + ), + WriteHeaderMutation( + kind="write", + name="x-openshell-middleware-added", + value="one", + on_existing=ExistingHeaderAction.APPEND, + ), + WriteHeaderMutation( + kind="write", + name="x-openshell-middleware-added", + value="two", + on_existing=ExistingHeaderAction.SKIP, + ), + RemoveHeaderMutation(kind="remove", name="x-other"), + ) + ) + updated = apply_request_mutations(original, request_mutations) + + assert updated.headers == ( + HttpHeader(name="x-openshell-middleware-test", value="new"), + HttpHeader(name="x-openshell-middleware-added", value="one"), + ) + + with pytest.raises(GateContractError): + apply_request_mutations( + original, + RequestMutations( + header_mutations=( + WriteHeaderMutation( + kind="write", + name="authorization", + value="secret", + on_existing=ExistingHeaderAction.APPEND, + ), + ) + ), + ) diff --git a/projects/egress-gate/tests/test_result.py b/projects/egress-gate/tests/test_result.py new file mode 100644 index 00000000..3dbd31ad --- /dev/null +++ b/projects/egress-gate/tests/test_result.py @@ -0,0 +1,387 @@ +"""Focused invariant and boundary tests for Egress Gate result models.""" + +from __future__ import annotations + +import math + +import pytest +from pydantic import TypeAdapter, ValidationError + +from egress_gate.constants import ( + DEFAULT_DENY_REASON_CODE, + LIMIT_REASON_CODE, + MAX_FINDING_COUNT, + MAX_GATE_TRACES, + MAX_PROTO_FINDING_BYTES, + MAX_PROTO_FINDING_GROUPS, + MAX_RESULT_METADATA_BYTES, + MAX_RESULT_METADATA_ENTRIES, + MAX_TRACE_MUTATION_KINDS, +) +from egress_gate.request import RequestMutations +from egress_gate.result import ( + DecisionSource, + DecisionSourceKind, + EgressDecision, + EgressResult, + Finding, + GateControl, + GateDecisionSource, + GateEvaluation, + GateTrace, + MutationKind, + PipelineDefaultDecisionSource, + ResultMetadata, + RuntimeLimitDecisionSource, + SourcedFinding, +) + + +def _finding(**values: object) -> Finding: + defaults: dict[str, object] = {"type": "sensitive_entity", "label": "email"} + defaults.update(values) + return Finding.model_validate(defaults) + + +def test_finding_matches_the_current_five_field_wire_contract() -> None: + finding = _finding(count=MAX_FINDING_COUNT, confidence="high", severity="medium") + + assert set(Finding.model_fields) == { + "type", + "label", + "count", + "confidence", + "severity", + } + assert finding.count == MAX_FINDING_COUNT + + with pytest.raises(ValidationError): + Finding.model_validate( + {"type": "sensitive_entity", "label": "email", "source_gate": "hidden"} + ) + + +@pytest.mark.parametrize("count", [0, MAX_FINDING_COUNT + 1]) +def test_finding_count_is_bounded(count: int) -> None: + with pytest.raises(ValidationError): + _finding(count=count) + + +def test_finding_encoded_size_has_an_exact_four_kibibyte_boundary() -> None: + exact = Finding( + type="t" * 1024, + label="l" * 1024, + confidence="c" * 1024, + severity="s" * 1010, + ) + assert exact.encoded_size_bytes == MAX_PROTO_FINDING_BYTES + + with pytest.raises(ValidationError): + Finding( + type="t" * 1024, + label="l" * 1024, + confidence="c" * 1024, + severity="s" * 1011, + ) + assert MAX_PROTO_FINDING_BYTES == 4 * 1024 + + +def test_gate_evaluation_helpers_and_control_invariants() -> None: + finding = _finding() + assert GateEvaluation.proceed(findings=(finding,)).control is GateControl.PROCEED + assert GateEvaluation.allow().request_mutations.is_empty + assert GateEvaluation.deny("egress_gate_blocked").reason_code == ( + "egress_gate_blocked" + ) + + with pytest.raises(ValidationError): + GateEvaluation( + control=GateControl.ALLOW, + request_mutations=RequestMutations(replacement_body=b"x"), + ) + with pytest.raises(ValidationError): + GateEvaluation(control=GateControl.DENY) + with pytest.raises(ValidationError): + GateEvaluation(control=GateControl.PROCEED, reason_code="invalid_control") + + +@pytest.mark.parametrize("reason_code", ["", "UPPERCASE", "has-hyphen", "x" * 65]) +def test_reason_codes_use_stable_identifier_format(reason_code: str) -> None: + with pytest.raises(ValidationError): + GateEvaluation.deny(reason_code) + + +def test_decision_source_keeps_gate_provenance_outside_finding() -> None: + adapter = TypeAdapter(DecisionSource) + discriminator = adapter.json_schema().get("discriminator") + assert isinstance(discriminator, dict) + assert discriminator.get("propertyName") == "kind" + source = adapter.validate_python( + { + "kind": "gate", + "gate_name": "identifiers", + "gate_type": "regex", + } + ) + sourced = SourcedFinding(source_gate="identifiers", finding=_finding()) + + assert isinstance(source, GateDecisionSource) + assert source.kind is DecisionSourceKind.GATE + assert sourced.finding.model_dump() == _finding().model_dump() + assert "source_gate" not in sourced.finding.model_dump() + + with pytest.raises(ValidationError): + adapter.validate_python({"kind": "gate", "gate_name": "identifiers"}) + with pytest.raises(ValidationError): + adapter.validate_python({"kind": "runtime_limit", "gate_name": "identifiers"}) + + +def test_egress_result_suppresses_mutations_on_deny_by_rejecting_them() -> None: + finding = SourcedFinding(source_gate="identifiers", finding=_finding()) + allowed = EgressResult( + decision=EgressDecision.ALLOW, + decision_source=GateDecisionSource( + kind=DecisionSourceKind.GATE, gate_name="identifiers", gate_type="regex" + ), + request_mutations=RequestMutations(replacement_body=b"redacted"), + findings=(finding,), + ) + assert allowed.request_mutations.replacement_body == b"redacted" + + with pytest.raises(ValidationError): + EgressResult( + decision=EgressDecision.DENY, + decision_source=RuntimeLimitDecisionSource( + kind=DecisionSourceKind.RUNTIME_LIMIT + ), + request_mutations=RequestMutations(replacement_body=b"must-not-leak"), + reason_code="egress_gate_limit_exceeded", + ) + with pytest.raises(ValidationError): + EgressResult( + decision=EgressDecision.DENY, + decision_source=RuntimeLimitDecisionSource( + kind=DecisionSourceKind.RUNTIME_LIMIT + ), + ) + with pytest.raises(ValidationError): + EgressResult( + decision=EgressDecision.ALLOW, + decision_source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + reason_code="not-allowed-on-allow", + ) + + +def test_egress_result_limits_finding_groups_and_trace_values() -> None: + finding = SourcedFinding(source_gate="identifiers", finding=_finding()) + findings = tuple( + SourcedFinding( + source_gate=f"gate-{index}", finding=_finding(label=f"label-{index}") + ) + for index in range(MAX_PROTO_FINDING_GROUPS) + ) + result = EgressResult( + decision=EgressDecision.ALLOW, + decision_source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + findings=findings, + ) + assert len(result.findings) == MAX_PROTO_FINDING_GROUPS + assert finding.finding.type == "sensitive_entity" + + with pytest.raises(ValidationError): + EgressResult( + decision=EgressDecision.ALLOW, + decision_source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + findings=findings + (finding,), + ) + + trace = GateTrace( + gate_name="identifiers", + gate_type="regex", + control=GateControl.PROCEED, + duration_ms=0, + finding_count=0, + mutation_kinds=(MutationKind.BODY,), + ) + assert trace.duration_ms == 0 + with pytest.raises(ValidationError): + GateTrace( + gate_name="identifiers", + gate_type="regex", + control=GateControl.PROCEED, + duration_ms=math.inf, + finding_count=0, + ) + + +def test_gate_evaluation_and_result_group_limits_have_exact_boundaries() -> None: + findings = tuple( + _finding(label=f"label-{index}") for index in range(MAX_PROTO_FINDING_GROUPS) + ) + assert len(GateEvaluation.proceed(findings=findings).findings) == ( + MAX_PROTO_FINDING_GROUPS + ) + with pytest.raises(ValidationError): + GateEvaluation.proceed(findings=findings + (_finding(label="over"),)) + + sourced = tuple( + SourcedFinding(source_gate=f"gate-{index}", finding=finding) + for index, finding in enumerate(findings) + ) + result = EgressResult( + decision=EgressDecision.ALLOW, + decision_source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + findings=sourced, + ) + assert len(result.findings) == MAX_PROTO_FINDING_GROUPS + with pytest.raises(ValidationError): + EgressResult( + decision=EgressDecision.ALLOW, + decision_source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + findings=sourced + + (SourcedFinding(source_gate="over", finding=_finding(label="over")),), + ) + + +def test_metadata_count_and_aggregate_byte_limits_have_exact_boundaries() -> None: + entries = tuple( + ResultMetadata( + key=f"k{index}", + value="v" * (510 if index < 10 else 509), + ) + for index in range(MAX_RESULT_METADATA_ENTRIES) + ) + result = EgressResult( + decision=EgressDecision.ALLOW, + decision_source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + metadata=entries, + ) + assert len(result.metadata) == MAX_RESULT_METADATA_ENTRIES + assert ( + sum(len(entry.key.encode()) + len(entry.value.encode()) for entry in entries) + == MAX_RESULT_METADATA_BYTES + ) + + with pytest.raises(ValidationError): + EgressResult( + decision=EgressDecision.ALLOW, + decision_source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + metadata=entries[:-1] + + ( + ResultMetadata( + key="k63", + value="v" * 510, + ), + ), + ) + with pytest.raises(ValidationError): + EgressResult( + decision=EgressDecision.ALLOW, + decision_source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + metadata=entries + (ResultMetadata(key="over", value="v"),), + ) + + +def test_trace_count_and_mutation_kind_limits_have_exact_boundaries() -> None: + trace = GateTrace( + gate_name="gate", + gate_type="test", + control=GateControl.PROCEED, + duration_ms=0, + finding_count=0, + mutation_kinds=(MutationKind.BODY, MutationKind.HEADERS), + ) + assert len(trace.mutation_kinds) == MAX_TRACE_MUTATION_KINDS + with pytest.raises(ValidationError): + GateTrace( + gate_name="gate", + gate_type="test", + control=GateControl.PROCEED, + duration_ms=0, + finding_count=0, + mutation_kinds=( + MutationKind.BODY, + MutationKind.HEADERS, + MutationKind.BODY, + ), + ) + + traces = tuple( + trace.model_copy(update={"gate_name": f"gate-{index}"}) + for index in range(MAX_GATE_TRACES) + ) + result = EgressResult( + decision=EgressDecision.ALLOW, + decision_source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + traces=traces, + ) + assert len(result.traces) == MAX_GATE_TRACES + with pytest.raises(ValidationError): + EgressResult( + decision=EgressDecision.ALLOW, + decision_source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + traces=traces + (trace,), + ) + + +def test_decision_source_reason_code_ownership_is_strict() -> None: + default_deny = EgressResult( + decision=EgressDecision.DENY, + decision_source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + reason_code=DEFAULT_DENY_REASON_CODE, + ) + runtime_limit = EgressResult( + decision=EgressDecision.DENY, + decision_source=RuntimeLimitDecisionSource( + kind=DecisionSourceKind.RUNTIME_LIMIT + ), + reason_code=LIMIT_REASON_CODE, + ) + assert default_deny.reason_code == DEFAULT_DENY_REASON_CODE + assert runtime_limit.reason_code == LIMIT_REASON_CODE + + with pytest.raises(ValidationError): + EgressResult( + decision=EgressDecision.ALLOW, + decision_source=RuntimeLimitDecisionSource( + kind=DecisionSourceKind.RUNTIME_LIMIT + ), + ) + with pytest.raises(ValidationError): + EgressResult( + decision=EgressDecision.DENY, + decision_source=RuntimeLimitDecisionSource( + kind=DecisionSourceKind.RUNTIME_LIMIT + ), + reason_code=DEFAULT_DENY_REASON_CODE, + ) + with pytest.raises(ValidationError): + EgressResult( + decision=EgressDecision.DENY, + decision_source=PipelineDefaultDecisionSource( + kind=DecisionSourceKind.PIPELINE_DEFAULT + ), + reason_code=LIMIT_REASON_CODE, + ) diff --git a/projects/privacy-guard/tests/test_timeout.py b/projects/egress-gate/tests/test_timeout.py similarity index 77% rename from projects/privacy-guard/tests/test_timeout.py rename to projects/egress-gate/tests/test_timeout.py index 778e97b3..b6c7957f 100644 --- a/projects/privacy-guard/tests/test_timeout.py +++ b/projects/egress-gate/tests/test_timeout.py @@ -1,11 +1,11 @@ -"""Tests for the shared entity-processing timeout.""" +"""Tests for the shared gate-pipeline timeout.""" from time import monotonic import pytest -from privacy_guard.errors import TimeoutExpiredError -from privacy_guard.timeout import Timeout +from egress_gate.errors import TimeoutExpiredError +from egress_gate.timeout import Timeout @pytest.mark.parametrize("seconds", [True, 0, -1, float("inf"), 31]) @@ -26,8 +26,8 @@ def test_expired_timeout_raises_typed_signal() -> None: timeout.raise_if_expired() assert str(captured.value) == ( - "Privacy Guard processing timed out. Reduce the request size or simplify " - "the configured stages and rules, or increase the processing timeout " + "Egress Gate processing timed out. Reduce the request size or simplify " + "the configured gates and rules, or increase the processing timeout " "to at most 30 seconds, then retry." ) diff --git a/projects/privacy-guard/tests/test_typing_policy.py b/projects/egress-gate/tests/test_typing_policy.py similarity index 99% rename from projects/privacy-guard/tests/test_typing_policy.py rename to projects/egress-gate/tests/test_typing_policy.py index b46991d0..a1df07be 100644 --- a/projects/privacy-guard/tests/test_typing_policy.py +++ b/projects/egress-gate/tests/test_typing_policy.py @@ -9,7 +9,7 @@ from typing_extensions import override _HANDWRITTEN_ROOTS = ("src", "tests", "examples") -_GENERATED_BINDINGS = Path("src/privacy_guard/bindings") +_GENERATED_BINDINGS = Path("src/egress_gate/bindings") _TYPING_MODULES = frozenset({"typing", "typing_extensions"}) @@ -574,7 +574,7 @@ def test_typing_policy_excludes_only_generated_bindings(tmp_path: Path) -> None: "from typing import Any, cast\nvalue: Any = cast(Any, None)\n", encoding="utf-8", ) - handwritten = tmp_path / "src/privacy_guard/generated_elsewhere.py" + handwritten = tmp_path / "src/egress_gate/generated_elsewhere.py" handwritten.write_text("from typing import Any\nvalue: Any\n", encoding="utf-8") violations = _typing_policy_violations(tmp_path) diff --git a/projects/privacy-guard/uv.lock b/projects/egress-gate/uv.lock similarity index 99% rename from projects/privacy-guard/uv.lock rename to projects/egress-gate/uv.lock index 9899c414..f2ecd66e 100644 --- a/projects/privacy-guard/uv.lock +++ b/projects/egress-gate/uv.lock @@ -164,6 +164,51 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, ] +[[package]] +name = "egress-gate" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "rich" }, + { name = "typer" }, + { name = "typing-extensions" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pip-audit" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "ruff" }, + { name = "ty" }, +] + +[package.metadata] +requires-dist = [ + { name = "grpcio", specifier = ">=1.81.1,<2" }, + { name = "protobuf", specifier = ">=6.33.5,<7" }, + { name = "pydantic", specifier = ">=2.11,<3" }, + { name = "pyyaml", specifier = ">=6,<7" }, + { name = "regex", specifier = ">=2026.7.19,<2027" }, + { name = "rich", specifier = ">=14,<16" }, + { name = "typer", specifier = ">=0.16,<1" }, + { name = "typing-extensions", specifier = ">=4.12,<5" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pip-audit", specifier = "==2.10.1" }, + { name = "pytest", specifier = ">=9.0.3,<10" }, + { name = "pytest-asyncio", specifier = ">=0.25,<2" }, + { name = "ruff", specifier = ">=0.12,<0.13" }, + { name = "ty", specifier = ">=0.0.1a16,<0.1" }, +] + [[package]] name = "filelock" version = "3.32.0" @@ -429,49 +474,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] -[[package]] -name = "privacy-guard" -version = "0.1.0" -source = { editable = "." } -dependencies = [ - { name = "grpcio" }, - { name = "protobuf" }, - { name = "pydantic" }, - { name = "pyyaml" }, - { name = "regex" }, - { name = "typer" }, - { name = "typing-extensions" }, -] - -[package.dev-dependencies] -dev = [ - { name = "pip-audit" }, - { name = "pytest" }, - { name = "pytest-asyncio" }, - { name = "ruff" }, - { name = "ty" }, -] - -[package.metadata] -requires-dist = [ - { name = "grpcio", specifier = ">=1.81.1,<2" }, - { name = "protobuf", specifier = ">=6.33.5,<7" }, - { name = "pydantic", specifier = ">=2.11,<3" }, - { name = "pyyaml", specifier = ">=6,<7" }, - { name = "regex", specifier = ">=2026.7.19,<2027" }, - { name = "typer", specifier = ">=0.16,<1" }, - { name = "typing-extensions", specifier = ">=4.12,<5" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "pip-audit", specifier = "==2.10.1" }, - { name = "pytest", specifier = ">=9.0.3,<10" }, - { name = "pytest-asyncio", specifier = ">=0.25,<2" }, - { name = "ruff", specifier = ">=0.12,<0.13" }, - { name = "ty", specifier = ">=0.0.1a16,<0.1" }, -] - [[package]] name = "protobuf" version = "6.33.6" diff --git a/projects/privacy-guard/AGENTS.md b/projects/privacy-guard/AGENTS.md deleted file mode 100644 index 5d7c5f20..00000000 --- a/projects/privacy-guard/AGENTS.md +++ /dev/null @@ -1,130 +0,0 @@ -# Privacy Guard - -Privacy Guard is OpenShell middleware that runs an ordered pipeline of -entity-processing engines over one UTF-8 request body and applies a user-facing -detect, block, or replace action. - -## Development commands - -Run commands from `projects/privacy-guard/`. - -- List targets: `make help` -- Run all checks: `make check` -- Check Python 3.11: `make check-py311` -- Run focused tests: `make test PYTEST_ARGS=tests/test_request_processor.py` - -Run focused tests while working and `make check` before handoff. - -## Engineering approach - -- Backwards compatibility is explicitly not a concern for the v0 redesign. Do - not restore legacy behavior, schemas, imports, names, tests, or examples. -- Add defensive handling only for a concrete failure mode at the layer that - owns it. Avoid speculative guards, duplicate validation, broad catches, - retries, and fallbacks. -- Prefer explicit, domain-specific names. Avoid generic intermediate - abstractions that do not own behavior. -- Keep public declarations before private helper types, functions, methods, and - constants when dependency ordering permits. Put private implementation - details at the bottom of their module or class. - -## Project map - -- `src/privacy_guard/engines/`: engine contract, registry, and built-in implementations -- `src/privacy_guard/config.py`: policy action and ordered stage configuration -- `src/privacy_guard/request_processor.py`: stage execution and policy disposition -- `src/privacy_guard/cli.py`: command parsing, discovery, gateway registration - management, configuration-schema output, and server adapter -- `src/privacy_guard/gateway_config.py`: safe OpenShell gateway TOML - registration management -- `src/privacy_guard/logging.py`: package-scoped standard-library logging configuration -- `src/privacy_guard/base.py`: package-wide strict immutable domain-model base -- `src/privacy_guard/string_validators.py`: shared string validators and field types -- `src/privacy_guard/service/`: gRPC lifecycle and protobuf adapter -- `src/privacy_guard/bindings/`: generated protobuf files; never hand-edit -- `docs/`: canonical user and architecture documentation; the repository docs - build stages this tree at the public Privacy Guard documentation route -- `tests/`: tests that mirror source boundaries -- `examples/`: copyable policy-authoring examples - -Before changing `request_processor.py`, `engines/`, or `service/`, read the -architecture overview and matching topic page. Architecture changes follow -[`docs/development/index.md`](../../docs/development/index.md) and require its -checks. - -## Design boundaries - -- One processor call receives one text string. Do not reintroduce request-body - codecs, format handlers, document regions, or JSON traversal. -- An `EntityProcessingEngine` receives engine configuration, a processing - strategy, and a shared `Timeout`. It never receives or infers the policy - action. -- `RequestProcessor` runs configured stages in order and owns detect, block, or - replace disposition. -- Engine configuration lives inside the OpenShell policy as the exact Pydantic - discriminated-union member registered for that engine. -- Deployment startup owns only installed engine implementations and operational - resources such as clients, endpoints, models, and credentials. -- Engine instances and injected resources serve concurrent requests. Do not - retain request content or mutable per-request state. -- Outside generated `bindings/`, only `service/` may import gRPC or generated - bindings. -- The copied OpenShell `.proto` and generated bindings must be updated only - through `openshell-middleware-manager`; never edit them manually. - -## Extension pattern - -Define a concrete `EngineConfig` and implement `_run`. Custom engines do not -define `__init__`; use optional `_initialize` for derived immutable state. -`@override` is not required. - -Resource-free engines omit the second generic argument. Resource-backed engines -declare an `EngineResources` subclass as that argument; the bundle contains -only operator-owned, concurrency-safe runtime dependencies and no policy -behavior or per-request state. - -```python -from typing import Literal - -from privacy_guard.engines import ( - EngineConfig, - EntityProcessingEngine, - EntityProcessingStrategy, - TextProcessingResult, -) -from privacy_guard.timeout import Timeout - - -class KeywordConfig(EngineConfig): - engine: Literal["keyword"] = "keyword" - keyword: str - - -class KeywordEngine(EntityProcessingEngine[KeywordConfig]): - supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - return TextProcessingResult(text=text, detections=()) -``` - -The public `run` method validates input, strategy, timeout, and extension -output. Custom code checks the timeout itself only for delegated calls or -unique long-running loops. Register every engine before finalizing the registry -so policy serialization retains its exact config type. Custom deployments -expose a `module:factory` callable that returns the application-scoped finalized -registry and pass it to the CLI with `--registry-factory`. - -## Change limits - -- Add or update tests at the layer that owns the behavior. -- Ask before adding dependencies or changing the OpenShell protobuf contract, - stable error codes, protocol limits, or fail-closed defaults. -- Do not remove or weaken relevant tests merely to pass checks. -- Do not add casts, explicit `Any`, blanket ignores, or broad type suppressions - to handwritten code. `tests/test_typing_policy.py` enforces this. diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md deleted file mode 100644 index 21af0c8f..00000000 --- a/projects/privacy-guard/README.md +++ /dev/null @@ -1,194 +0,0 @@ -# Privacy Guard - -Privacy Guard is an OpenShell supervisor middleware that detects, blocks, or -replaces configured entities in provider-bound HTTP request bodies before -OpenShell attaches provider credentials. - -It processes the complete request body as UTF-8 text through an ordered -pipeline of entity-processing engines. - -> **Experimental:** Privacy Guard is a proof of concept. It reduces exposure on -> provider-bound network requests that OpenShell routes through the middleware; -> it does not guarantee that sensitive data cannot leak. - -Privacy Guard does not intercept data before a harness writes it to disk. -Prompts, tool output, transcripts, and session histories may therefore retain -raw sensitive values even when the provider-bound request is later replaced or -blocked. Use harness persistence controls and appropriate storage isolation, -retention, and cleanup in addition to Privacy Guard. - -## What it does - -| Policy action | Behavior | -| --- | --- | -| `detect` | Allow the original body and report bounded findings | -| `block` | Deny requests containing configured entities | -| `replace` | Allow the final body returned by replacement-capable engines | - -Findings contain entity, stage, confidence, and count. Framework-controlled -fields and the built-in `RegexEngine` do not add matched text, surrounding -text, offsets, patterns, headers, or credentials. Custom engines must use -stable entity identifiers that are not derived from request text. - -## Developer start - -Requirements: - -- Python 3.11 or newer -- `uv` 0.11 or newer - -From this directory: - -```bash -uv sync --locked -uv run privacy-guard engines -uv run privacy-guard configuration-schema -``` - -Start the built-in `RegexEngine` service locally: - -```bash -uv run privacy-guard serve \ - --listen 127.0.0.1:50051 -``` - -Use `0.0.0.0` when OpenShell sandbox supervisors outside the host network -namespace must reach the service. The development server uses plaintext gRPC; -restrict the port to trusted host and sandbox networks. - -## Policy configuration - -Privacy behavior comes from the OpenShell policy: - -```yaml -entity_processing: - stages: - - name: identifiers - config: - engine: regex - pattern_catalog: - entities: - - name: email - rules: - - pattern: '(? - Privacy Guard request lifecycle - The request lifecycle has four phases: validate the OpenShell transport, validate and activate configuration, decode and process text, and serialize the result. Empty bodies bypass engine processing. Invalid input fails the RPC, while policy and limit decisions return successful allow or deny results. - - - - - - - - - 1 · TRANSPORT - - Receive - HttpRequestEvaluation - pre-credentials phase - - - Validate bounds - context · config - target · headers · body - phase - - - - - 2 · CONFIGURATION - - Validate policy - typed union · engines - resources · action - - - Resolve processor - reuse equal config - or prepare + activate - - - - - 3 · PROCESSING - - Decode text - strict UTF-8 - empty body bypass - - - Run pipeline - ordered stages - one shared timeout - aggregate findings - - - - - 4 · RESULT - - Apply - detect - block - replace - - - Serialize - allow or deny - body + findings - - - RPC FAILURE - Invalid input or internal failure - INVALID_ARGUMENT or INTERNAL - OpenShell applies middleware on_error - - - - - SUCCESSFUL MIDDLEWARE RESULT - Allow, policy deny, or limit deny - privacy_guard_blocked - privacy_guard_limit_exceeded - - diff --git a/projects/privacy-guard/docs/configuration.md b/projects/privacy-guard/docs/configuration.md deleted file mode 100644 index 357769ae..00000000 --- a/projects/privacy-guard/docs/configuration.md +++ /dev/null @@ -1,281 +0,0 @@ ---- -title: Configure policies -description: Configure Privacy Guard stages, actions, catalogs, and OpenShell middleware routing. -agent_markdown: true ---- - -# Configure policies - -Privacy Guard configuration is embedded in an OpenShell -`network_middlewares` entry. The policy determines: - -- which provider endpoints use Privacy Guard -- the order of entity-processing stages -- each engine's exact configuration -- whether detections are reported, blocked, or replaced -- OpenShell's behavior when the middleware RPC fails - -Privacy Guard validates the complete configuration before processing a request. - -## Complete middleware entry - -```yaml -network_middlewares: - privacy_guard_replace: - name: Replace email addresses and customer IDs - middleware: privacy-guard - order: 0 - config: - entity_processing: - stages: - - name: identifiers - config: - engine: regex - pattern_catalog: - entities: - - name: email - rules: - - name: conventional-email - pattern: '(? stage 1 -> stage 2 -> final replacement text -``` - -Detection offsets belong to the input revision seen by the stage that produced -them. Findings aggregate by stage, entity, and confidence. - -## Detection actions - -Set `on_detection.action` to one of: - -| Action | Engine strategy | No detections | Detections | -| --- | --- | --- | --- | -| `detect` | `DETECT` | Allow original body | Allow original body and report findings | -| `block` | `DETECT` | Allow original body | Deny with `privacy_guard_blocked` | -| `replace` | `REPLACE` | Allow final stage output | Allow final stage output and report findings | - -`replace` requires every configured stage to support replacement and to satisfy -its engine-specific replacement requirements. A replacement recipe may remain -configured when the action is `detect` or `block`; it is not used in those -modes. - -## Common policy recipes - -### Detect without changing the request - -```yaml -entity_processing: - stages: - - name: identifiers - config: - engine: regex - pattern_catalog: patterns.yaml -on_detection: - action: detect -``` - -Use this to observe findings while leaving the provider-bound body unchanged. - -### Block requests containing configured entities - -```yaml -entity_processing: - stages: - - name: restricted-values - config: - engine: regex - pattern_catalog: patterns.yaml -on_detection: - action: block -``` - -The request is denied only when at least one configured entity is detected. - -### Replace entities - -```yaml -entity_processing: - stages: - - name: identifiers - config: - engine: regex - pattern_catalog: patterns.yaml - replacement: - strategy: template - template: "[{entity}]" -on_detection: - action: replace -``` - -`{entity}` is replaced with the catalog entity name. For example, -`user@example.com` becomes `[email]`. - -### Run multiple stages - -```yaml -entity_processing: - stages: - - name: structured-identifiers - config: - engine: regex - pattern_catalog: identifiers.yaml - replacement: - strategy: template - template: "[{entity}]" - - name: organization-model - config: - engine: acme-pii - model_profile: organization-default - replacement: - strategy: native -on_detection: - action: replace -``` - -The `acme-pii` engine and its configuration are examples of a custom -installation. The running registry must contain every engine named by the -policy. - -## Regex catalogs - -`RegexEngine` accepts an inline catalog or a relative YAML path. - -Inline: - -```yaml -pattern_catalog: - entities: - - name: customer-id - rules: - - name: prefixed-eight-digit-id - pattern: '\bCUST-[0-9]{8}\b' - confidence: high -``` - -File-backed: - -```yaml -pattern_catalog: patterns.yaml -``` - -Relative paths resolve beneath Privacy Guard's working directory. The path must -end in `.yaml` or `.yml`. Absolute paths, `..` traversal, and symlinks are -rejected. Start Privacy Guard from the directory that contains the referenced -catalog, or use a path relative to that directory. - -See [RegexEngine](engines/regex.md) for the complete catalog schema. - -## Inspect and validate configuration - -List the engines installed in the selected registry: - -```bash -uv run privacy-guard engines -``` - -Print the exact JSON Schema accepted by that registry: - -```bash -uv run privacy-guard configuration-schema -``` - -For a custom registry, pass the same factory to inspection and serving: - -```bash -uv run privacy-guard \ - --registry-factory my_engines:create_registry \ - engines - -uv run privacy-guard \ - --registry-factory my_engines:create_registry \ - configuration-schema - -uv run privacy-guard \ - --registry-factory my_engines:create_registry \ - serve -``` - -Sandbox creation calls `ValidateConfig`. A successful creation proves that the -middleware registration is reachable and that the supplied config matches the -running registry. - -## Policy and deployment ownership - -Keep privacy behavior in policy: - -- stage order -- entity definitions -- detection settings -- engine-specific replacement recipes -- final action - -Keep operational resources in the Privacy Guard deployment: - -- installed engine implementations -- model clients and SDK adapters -- endpoints and credentials -- approved model profiles -- processing timeout - -A policy cannot select a registry factory or import Python code. - -## Configuration activation - -OpenShell sends the complete configuration on each evaluation. Privacy Guard -validates it and compares the normalized immutable result with the active -configuration: - -- equal configuration reuses the active processor -- changed valid configuration is fully prepared, then atomically activated -- failed validation or preparation leaves the active processor unchanged and - fails the triggering evaluation - -Send one consistent configuration stream to each Privacy Guard process. -Interleaving configurations causes the active processor to switch between them. - -The transport configuration is limited to 64 KiB. File-backed Regex catalogs -carry only their relative path through the transport and are loaded by the -Privacy Guard process. - -## Next steps - -- [RegexEngine](engines/regex.md) -- [Add a custom engine](engines/custom.md) -- [Run and operate Privacy Guard](operations.md) -- [Limits and failure behavior](reference/limits-and-failures.md) diff --git a/projects/privacy-guard/docs/engines/custom.md b/projects/privacy-guard/docs/engines/custom.md deleted file mode 100644 index 990d1221..00000000 --- a/projects/privacy-guard/docs/engines/custom.md +++ /dev/null @@ -1,325 +0,0 @@ ---- -title: Add a custom engine -description: Implement, register, run, and test a typed Privacy Guard entity-processing engine. -agent_markdown: true ---- - -# Add a custom engine - -A custom engine integrates another detector or replacement tool with Privacy -Guard. It receives one text string, an invocation strategy, a shared timeout, -and validated engine-specific configuration. It returns processed text and -bounded detections. - -Custom engine code runs inside the Privacy Guard process and can access request -text. Install only reviewed, trusted implementations. - -## Engine contract - -A custom engine defines: - -1. a concrete `EngineConfig` -2. optional typed `EngineResources` -3. supported invocation strategies -4. optional immutable initialization in `_initialize()` -5. request processing in `_run()` - -Do not override `__init__()` or the public `run()` method. The framework-owned -wrapper validates strategy support, timeouts, detection spans, detection -cardinality, output size, and mutation behavior. - -## Minimal detection engine - -```python -import re -from typing import Literal - -from pydantic import Field - -from privacy_guard.engines import ( - EngineConfig, - EntityDetection, - EntityProcessingEngine, - EntityProcessingStrategy, - TextProcessingResult, -) -from privacy_guard.timeout import Timeout - - -class KeywordEngineConfig(EngineConfig): - engine: Literal["keyword"] = "keyword" - keyword: str = Field(min_length=1, max_length=256) - - -class KeywordEngine(EntityProcessingEngine[KeywordEngineConfig]): - supported_strategies = frozenset( - {EntityProcessingStrategy.DETECT} - ) - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - matches = re.finditer(re.escape(self.config.keyword), text) - return TextProcessingResult.from_detections( - text=text, - detections=( - EntityDetection( - entity="keyword", - start=match.start(), - end=match.end(), - ) - for match in matches - ), - ) -``` - -`TextProcessingResult.from_detections()` stops consuming a lazy detection -stream when the per-stage limit is exceeded. The public engine wrapper remains -the enforcement boundary. - -## Configuration - -Each config class must: - -- subclass `EngineConfig` -- declare one literal `engine` discriminator -- use strict typed fields for all policy-owned behavior -- reject unknown fields through the shared base model -- keep sensitive values out of normal representations when applicable - -```python -class AcmeEngineConfig(EngineConfig): - engine: Literal["acme-pii"] = "acme-pii" - model_profile: str - replacement: AcmeReplacement | None = None -``` - -Privacy Guard adds the exact config type to the registry-built Pydantic -discriminated union. The policy object is passed unchanged to the engine. - -OpenShell transports numbers through protobuf `Struct`. Integer settings must -fit the safe range `-(2^53 - 1)` through `2^53 - 1`. - -## Operational resources - -Use `EngineResources` for deployment-owned clients, adapters, endpoints, -credential providers, or preloaded models: - -```python -from dataclasses import dataclass - -from privacy_guard.engines import EngineResources - - -@dataclass(frozen=True) -class AcmeResources(EngineResources): - client: AcmeClient - - -class AcmeEngine( - EntityProcessingEngine[AcmeEngineConfig, AcmeResources] -): - ... -``` - -Resources must: - -- contain operational dependencies, not policy behavior -- retain no request text or per-request state -- be safe for concurrent use -- be created before request processing - -A resource-free engine omits the second generic argument. - -## Supported strategies - -Declare the exact operations exposed by the engine: - -```python -supported_strategies = frozenset( - { - EntityProcessingStrategy.DETECT, - EntityProcessingStrategy.REPLACE, - } -) -``` - -`block` is not an engine strategy. The processor invokes `DETECT` and applies -the block decision after successful engine execution. - -Override `_validate_run_config()` when a strategy requires additional -configuration. For example, a replacement engine can require a replacement -recipe only when invoked with `REPLACE`. - -## Result requirements - -Return `TextProcessingResult` with: - -| Field | Requirement | -| --- | --- | -| `text` | Complete authoritative stage output | -| `detections` | Every bounded occurrence produced by the stage | - -Each `EntityDetection` provides: - -| Field | Requirement | -| --- | --- | -| `entity` | Stable declared identifier, never a value derived from request text | -| `start` | Inclusive Unicode code-point offset in stage input | -| `end` | Exclusive non-empty offset in stage input | -| `confidence` | Optional `low`, `medium`, or `high` | -| `metadata` | Optional bounded internal attribution | - -For `DETECT`, returned text must exactly equal input text. For `REPLACE`, text -may change only when the result contains at least one detection. Do not return -partial text or detections after a collaborator failure. - -## Timeouts - -One `Timeout` is shared across the complete stage pipeline. Pass its remaining -duration to APIs that accept a timeout: - -```python -result = client.process( - text, - timeout=timeout.remaining_seconds(), -) -``` - -Translate Python `TimeoutError` with the shared context manager: - -```python -with timeout.enforce(): - result = client.process( - text, - timeout=timeout.remaining_seconds(), - ) -``` - -Long-running local loops may call `timeout.raise_if_expired()`. Document and -bound operations that cannot be interrupted. - -## Concurrency - -One configured engine instance may process requests concurrently. Keep request -text, detections, counters, and temporary objects local to `_run()`. Treat -configuration and derived initialization state as immutable. Ensure injected -clients and resources support concurrent calls. - -## Errors and logging - -Translate expected collaborator failures into Privacy Guard's content-safe -engine exceptions. Do not include: - -- input or replacement text -- matched values or surrounding text -- credentials or endpoints -- raw exception messages -- model or SDK response bodies - -Stable engine and entity identifiers may appear in findings and diagnostic -logs when they satisfy shared validation. - -Use a static, content-safe message when translating an operational failure: - -```python -from privacy_guard.engines import EngineExecutionError - -try: - result = self.resources.client.process( - text, - timeout=timeout.remaining_seconds(), - ) -except AcmeClientError: - raise EngineExecutionError("Acme processing failed") from None -``` - -| Exception | Use in a custom engine | -| --- | --- | -| `EngineConfigurationError` | Strategy-specific configuration is unusable | -| `EngineExecutionError` | A collaborator or runtime operation failed | -| `EngineLimitExceededError` | Engine-owned bounded work or output exceeded its limit | - -The framework raises `EngineContractError` when returned text or detections -violate the engine contract; custom engines should not use it for collaborator -failures. - -## Register the engine - -Create one application-scoped registry factory: - -```python -from privacy_guard.engines.registry import EngineRegistry - - -def create_registry() -> EngineRegistry: - registry = EngineRegistry(include_builtin_engines=True) - registry.register(KeywordEngine) - return registry.finalize() -``` - -Pass resources during registration when required: - -```python -registry.register( - AcmeEngine, - resources=AcmeResources(client=client), -) -``` - -Use `include_builtin_engines=True` to add the built-in `RegexEngine`. Omit it -when the registry should contain only explicitly registered custom engines. - -## Inspect and run the registry - -```bash -uv run privacy-guard \ - --registry-factory my_engines:create_registry \ - engines - -uv run privacy-guard \ - --registry-factory my_engines:create_registry \ - configuration-schema - -uv run privacy-guard \ - --registry-factory my_engines:create_registry \ - serve \ - --listen 0.0.0.0:50051 -``` - -The module must be installed or available on `PYTHONPATH`. The factory is -trusted deployment code and executes for each CLI invocation. - -The complete runnable example is in -[`projects/privacy-guard/examples/custom-engine`](https://github.com/NVIDIA/OpenShell-Research/tree/main/projects/privacy-guard/examples/custom-engine). - -## Verify the integration - -Before deploying a custom engine: - -1. Run its unit tests directly against `run()` for every supported strategy. - Assert the exact returned text, entity identifiers, spans, and confidence. - Include Unicode input when offsets come from another library. -2. Run `engines` and `configuration-schema` with the registry factory. Confirm - that the engine and its policy fields appear. -3. Send a representative request through a running Privacy Guard service with - the deployment policy. Confirm the OpenShell decision, replacement body, and - findings. -4. Force collaborator timeouts and failures. Confirm that the request fails - without partial output and that responses and logs contain no request text, - credentials, or raw collaborator errors. - -Test engine-specific behavior and integrations. Privacy Guard's own suite -covers the shared wrapper contract, processor ordering, request-wide limits, -and gRPC result mapping. - -## Related pages - -- [Configure policies](../configuration.md) -- [Run and operate Privacy Guard](../operations.md) -- [System architecture](../architecture/index.md) -- [Limits and failure behavior](../reference/limits-and-failures.md) diff --git a/projects/privacy-guard/docs/engines/index.md b/projects/privacy-guard/docs/engines/index.md deleted file mode 100644 index a20d970b..00000000 --- a/projects/privacy-guard/docs/engines/index.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -title: Engines -description: Understand how Privacy Guard engines detect and replace sensitive entities. -agent_markdown: true ---- - -# Engines - -Engines are pluggable processors that inspect request text for configured -entities. Each policy stage selects an engine, supplies its configuration, and -receives detections plus replacement text when replacement is enabled. - -Engines do not decide whether Privacy Guard allows or denies a request. The -request processor runs the configured stages in order, enforces shared safety -bounds, and applies the policy's final action to their combined results. - -Privacy Guard includes two integration paths: - -- [RegexEngine](regex.md) provides deterministic detection and replacement - using a deployment-defined pattern catalog. -- [Custom engines](custom.md) integrate another detector, model, SDK, or - service through Privacy Guard's engine contract and registry. - -Use the regex engine when the sensitive values have stable, testable formats. -Add a custom engine when detection requires semantics or an external system -that regular expressions cannot provide reliably. diff --git a/projects/privacy-guard/docs/engines/regex.md b/projects/privacy-guard/docs/engines/regex.md deleted file mode 100644 index 2c40a3a1..00000000 --- a/projects/privacy-guard/docs/engines/regex.md +++ /dev/null @@ -1,215 +0,0 @@ ---- -title: RegexEngine -description: Configure RegexEngine catalogs, matching flags, findings, and deterministic replacement. -agent_markdown: true ---- - -# RegexEngine - -`RegexEngine` is the built-in Privacy Guard engine. It detects every configured -regular-expression match and can replace a deterministic non-overlapping subset -with a constrained template. - -Privacy Guard provides the catalog schema and execution bounds. It does not -provide an authoritative pattern catalog. Define and test patterns for the data -your deployment handles. - -## Engine configuration - -```yaml -engine: regex -pattern_catalog: patterns.yaml -replacement: - strategy: template - template: "[{entity}]" -``` - -| Field | Required | Purpose | -| --- | --- | --- | -| `engine` | Yes | Must be `regex` | -| `pattern_catalog` | Yes | Inline catalog or relative YAML path | -| `replacement` | For `replace` actions | Template replacement configuration | - -## Catalog structure - -```yaml -entities: - - name: email - rules: - - name: conventional-email - pattern: '(? TextProcessingResult: - matches = re.finditer(re.escape(self.config.keyword), text) - return TextProcessingResult.from_detections( - text=text, - detections=( - EntityDetection( - entity=self.config.entity, - start=match.start(), - end=match.end(), - confidence=ConfidenceLevel.HIGH, - ) - for match in matches - ), - ) - - -def create_registry() -> EngineRegistry: - """Create a registry containing the built-in and custom engines.""" - registry = EngineRegistry(include_builtin_engines=True) - registry.register(KeywordEngine) - return registry.finalize() diff --git a/projects/privacy-guard/examples/custom-engine/privacy-guard-config.yaml b/projects/privacy-guard/examples/custom-engine/privacy-guard-config.yaml deleted file mode 100644 index 1ee1d76c..00000000 --- a/projects/privacy-guard/examples/custom-engine/privacy-guard-config.yaml +++ /dev/null @@ -1,9 +0,0 @@ -entity_processing: - stages: - - name: project-names - config: - engine: keyword-tool - entity: confidential-project - keyword: Project Cobalt -on_detection: - action: detect diff --git a/projects/privacy-guard/examples/regex-engine/.gitignore b/projects/privacy-guard/examples/regex-engine/.gitignore deleted file mode 100644 index b223234c..00000000 --- a/projects/privacy-guard/examples/regex-engine/.gitignore +++ /dev/null @@ -1 +0,0 @@ -gateway.local.toml diff --git a/projects/privacy-guard/examples/regex-engine/README.md b/projects/privacy-guard/examples/regex-engine/README.md deleted file mode 100644 index 27ca6375..00000000 --- a/projects/privacy-guard/examples/regex-engine/README.md +++ /dev/null @@ -1,177 +0,0 @@ -# RegexEngine end-to-end example - -This example runs Privacy Guard's built-in `RegexEngine` through OpenShell. The -final check sends a Claude Code request containing an email address and customer -ID, then verifies that OpenShell forwards `[email]` and `[customer-id]`. - -Privacy Guard does not ship authoritative regex presets. Copy and adapt -`patterns.yaml` for the data you actually need to identify, and test every -pattern against representative matches, non-matches, and worst-case inputs -before deployment. - -## Prerequisites - -This walkthrough was validated with OpenShell `v0.0.90`, the version recorded -in Privacy Guard's `.openshell-middleware-manifest.json`. A later OpenShell -release can also work if it supports the same supervisor middleware contract -and policy schema. - -Before you start, install: - -- Python 3.11 or newer and `uv` 0.11 or newer -- [OpenShell](https://github.com/NVIDIA/OpenShell) `v0.0.90` or a later - compatible version - -The gateway lifecycle commands below cover macOS Homebrew and Linux Debian/RPM -installations. For another deployment, use its equivalent gateway commands. - -## Stop the local gateway - -First, check the local gateway: - -```bash -openshell status -``` - -If the gateway is running, stop it before you change its configuration. Use the -command for your system: - -```bash -# macOS with Homebrew -brew services stop openshell - -# Linux with a Debian or RPM package -systemctl --user stop openshell-gateway -``` - -## Start Privacy Guard - -In terminal 1, from this example directory: - -```bash -cd projects/privacy-guard/examples/regex-engine -uv run --locked privacy-guard serve --listen 0.0.0.0:50051 -``` - -Leave this terminal running. The development server is unauthenticated -plaintext gRPC and receives potentially sensitive request bodies. Binding to -`0.0.0.0` is necessary for the sandbox supervisor to reach it, but port 50051 -must remain restricted to the host and trusted sandbox network. - -## Configure and start the gateway - -Choose a non-loopback host IPv4 address that both the gateway and sandbox -supervisor can reach. - -In terminal 2, return to the example directory. Replace `YOUR_HOST_IPV4` with -the address you selected, then update the default gateway configuration: - -```bash -cd projects/privacy-guard/examples/regex-engine -uv run privacy-guard add-gateway-registration \ - --host-ip YOUR_HOST_IPV4 \ - --name privacy-guard-regex -``` - -Do not use `127.0.0.1`, a VPN address, or `host.openshell.internal`. The gateway -and sandbox supervisor must both be able to reach the configured endpoint. - -Next, use the command for your system to start the gateway in the background: - -```bash -# macOS with Homebrew -brew services start openshell - -# Linux with a Debian or RPM package -systemctl --user start openshell-gateway -``` - -## Verify OpenShell and create the sandbox - -In terminal 3, from this example directory: - -```bash -openshell status -``` - -Do not continue until status reports that the gateway is connected. - -This walkthrough starts Claude Code in the sandbox. To use a different harness, -replace everything after `--` with its command. Then create the sandbox: - -```bash -openshell sandbox create \ - --name privacy-guard-regex \ - --from base \ - --no-auto-providers \ - --policy "$PWD/policy.yaml" \ - -- env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude -``` - -Sandbox creation validates the external middleware registration and the exact -`RegexEngineConfig` embedded in the policy. - -After authenticating Claude Code, enter: - -```text -Draft a short greeting for user@example.com about customer CUST-12345678. -``` - -Privacy Guard should send `[email]` and `[customer-id]` instead of the original -identifiers to the provider. - -## Verify the middleware result - -From another host terminal: - -```bash -openshell logs privacy-guard-regex -n 100 --source sandbox -``` - -Look for the `api.anthropic.com/v1/messages` request with `transformed:true`, -plus `email (identifiers)` and `customer-id (identifiers)` findings. Findings -must not contain the matched email address or customer ID. - -## Cleanup - -Exit Claude and delete the sandbox: - -```bash -openshell sandbox delete privacy-guard-regex -``` - -Stop Privacy Guard with `Ctrl-C`, then stop the gateway before removing the -example registration: - -```bash -# macOS with Homebrew -brew services stop openshell - -# Linux with a Debian or RPM package -systemctl --user stop openshell-gateway - -uv run privacy-guard remove-gateway-registration \ - --name privacy-guard-regex -``` - -Restart the gateway with the command for your system, then verify its -connection: - -```bash -# macOS with Homebrew -brew services start openshell - -# Linux with a Debian or RPM package -systemctl --user start openshell-gateway - -openshell status -``` - -## Troubleshooting - -- Sandbox creation reports unavailable middleware: confirm terminal 1 is still - running, check the IP in the default gateway configuration, and allow trusted - sandbox traffic to host port 50051. -- Policy or middleware registration fields are rejected: confirm that - `openshell` and `openshell-gateway` use compatible versions. If the error - remains, use the tested `v0.0.90` release. diff --git a/projects/privacy-guard/examples/regex-engine/privacy-guard-config.yaml b/projects/privacy-guard/examples/regex-engine/privacy-guard-config.yaml deleted file mode 100644 index 219c5fb6..00000000 --- a/projects/privacy-guard/examples/regex-engine/privacy-guard-config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -entity_processing: - stages: - - name: identifiers - config: - engine: regex - pattern_catalog: patterns.yaml - replacement: - strategy: template - template: "[{entity}]" -on_detection: - action: replace diff --git a/projects/privacy-guard/src/privacy_guard/__init__.py b/projects/privacy-guard/src/privacy_guard/__init__.py deleted file mode 100644 index da81cd3f..00000000 --- a/projects/privacy-guard/src/privacy_guard/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Privacy Guard: an OpenShell supervisor middleware. See the package README.""" diff --git a/projects/privacy-guard/src/privacy_guard/cli.py b/projects/privacy-guard/src/privacy_guard/cli.py deleted file mode 100644 index 773b898c..00000000 --- a/projects/privacy-guard/src/privacy_guard/cli.py +++ /dev/null @@ -1,381 +0,0 @@ -"""Privacy Guard command-line application.""" - -from __future__ import annotations - -import importlib -import ipaddress -import json -from dataclasses import dataclass -from pathlib import Path -from typing import Annotated - -import typer - -from privacy_guard.constants import DEFAULT_TIMEOUT_SECONDS, MAX_TIMEOUT_SECONDS -from privacy_guard.engines import EntityProcessingStrategy -from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry -from privacy_guard.errors import PrivacyGuardError -from privacy_guard.gateway_config import ( - MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES, - GatewayConfigError, - GatewayConfigRemoval, - GatewayConfigUpdate, - default_gateway_config_path, - remove_gateway_config, - update_gateway_config, - validate_middleware_name, -) -from privacy_guard.logging import LoggingConfig, configure_logging, get_logger -from privacy_guard.service.server import DEFAULT_LISTEN_ADDRESS, PrivacyGuardServer -from privacy_guard.timeout import validate_timeout_seconds - -app = typer.Typer( - name="privacy-guard", - help=( - "Run Privacy Guard, manage local OpenShell gateway registrations, and " - "inspect installed entity-processing engines." - ), - no_args_is_help=True, - add_completion=False, -) - - -@app.callback() -def configure_cli( - context: typer.Context, - registry_factory: Annotated[ - str | None, - typer.Option( - help=( - "Load engines from a trusted Python callable, formatted as " - "module:factory. The callable must return a finalized EngineRegistry." - ), - ), - ] = None, - debug: Annotated[ - bool, - typer.Option( - "--debug", - help=( - "Log content-safe diagnostic details for startup and request handling." - ), - ), - ] = False, - debug_log_content: Annotated[ - bool, - typer.Option( - "--debug-log-content", - help=( - "DANGEROUS: log complete request and processed text, which may " - "contain secrets or personal data." - ), - ), - ] = False, -) -> None: - """Configure the command application and its engine inventory.""" - configure_logging( - LoggingConfig(level="DEBUG" if debug or debug_log_content else "INFO") - ) - context.obj = _CommandOptions( - registry=_load_registry(registry_factory), - log_request_content=debug_log_content, - ) - if debug_log_content: - _LOGGER.warning( - "privacy_guard_request_content_logging_enabled " - "complete_request_text_may_contain_secrets" - ) - - -@app.command("serve") -def serve( - context: typer.Context, - listen: Annotated[ - str, - typer.Option( - help=( - "Host and port on which Privacy Guard listens, formatted as " - "host:port. Use 0.0.0.0 when sandbox supervisors must reach it." - ), - ), - ] = DEFAULT_LISTEN_ADDRESS, - timeout_seconds: Annotated[ - float, - typer.Option( - help=( - "Maximum seconds shared by all processing stages in one request; " - f"must be greater than 0 and at most {MAX_TIMEOUT_SECONDS:g}." - ), - ), - ] = DEFAULT_TIMEOUT_SECONDS, -) -> None: - """Run Privacy Guard until the process receives a shutdown signal.""" - options = _command_options(context) - try: - validated_timeout_seconds = validate_timeout_seconds(timeout_seconds) - except ValueError as error: - raise typer.BadParameter( - str(error), - param_hint="--timeout-seconds", - ) from None - try: - PrivacyGuardServer( - options.registry, - timeout_seconds=validated_timeout_seconds, - log_request_content=options.log_request_content, - ).serve_sync(listen) - except PrivacyGuardError as error: - typer.echo(str(error), err=True) - raise typer.Exit(code=1) from None - - -@app.command("add-gateway-registration") -def add_gateway_registration( - host_ip: Annotated[ - str, - typer.Option( - help=( - "Non-loopback IPv4 address of this host that both the OpenShell " - "gateway and sandbox supervisors can reach." - ), - ), - ], - config: Annotated[ - Path | None, - typer.Option( - help=( - "Gateway TOML to update. Defaults to " - "`$OPENSHELL_GATEWAY_CONFIG` when set, otherwise `gateway.toml` " - "under `$XDG_CONFIG_HOME/openshell`." - ), - ), - ] = None, - name: Annotated[ - str, - typer.Option( - help=( - "Gateway registration name referenced by the policy's middleware " - "field. OpenShell allows " - f"1-{MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES} ASCII bytes." - ), - ), - ] = "privacy-guard", - port: Annotated[ - int, - typer.Option( - min=1, - max=65535, - help=( - "Privacy Guard port. Use the same port in `privacy-guard serve " - "--listen`." - ), - ), - ] = 50051, -) -> None: - """Add or update Privacy Guard in an OpenShell gateway TOML file.""" - try: - address = ipaddress.IPv4Address(host_ip) - except ipaddress.AddressValueError: - raise typer.BadParameter( - "Pass one IPv4 address, for example --host-ip 192.168.1.20.", - param_hint="--host-ip", - ) from None - if address.is_loopback or address.is_unspecified: - raise typer.BadParameter( - "Pass a non-loopback host IPv4 address reachable by sandbox " - "supervisors; do not use 127.0.0.1 or 0.0.0.0.", - param_hint="--host-ip", - ) - try: - validated_name = validate_middleware_name(name) - except GatewayConfigError as error: - raise typer.BadParameter( - str(error), - param_hint="--name", - ) from None - - config_path = config or default_gateway_config_path() - try: - result = update_gateway_config( - config_path, - middleware_name=validated_name, - host_ip=str(address), - port=port, - ) - except GatewayConfigError as error: - typer.echo( - f"Could not add or update the OpenShell gateway registration: {error}", - err=True, - ) - raise typer.Exit(code=1) from None - - action = { - GatewayConfigUpdate.CREATED: "Created", - GatewayConfigUpdate.ADDED: "Added the registration to", - GatewayConfigUpdate.UPDATED: "Updated", - GatewayConfigUpdate.UNCHANGED: "No changes needed in", - }[result] - typer.echo(f"{action} {config_path}") - typer.echo(f"Registered {validated_name} at http://{address}:{port}") - typer.echo( - "Next: start Privacy Guard, then restart the OpenShell gateway so it " - "loads this registration." - ) - - -@app.command("remove-gateway-registration") -def remove_gateway_registration( - name: Annotated[ - str, - typer.Option( - help=( - "Gateway registration name to remove. OpenShell allows " - f"1-{MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES} ASCII bytes." - ), - ), - ], - config: Annotated[ - Path | None, - typer.Option( - help=( - "Gateway TOML to update. Defaults to " - "`$OPENSHELL_GATEWAY_CONFIG` when set, otherwise `gateway.toml` " - "under `$XDG_CONFIG_HOME/openshell`." - ), - ), - ] = None, -) -> None: - """Remove a named registration from an OpenShell gateway TOML file.""" - try: - validated_name = validate_middleware_name(name) - except GatewayConfigError as error: - raise typer.BadParameter( - str(error), - param_hint="--name", - ) from None - - config_path = config or default_gateway_config_path() - try: - result = remove_gateway_config( - config_path, - middleware_name=validated_name, - ) - except GatewayConfigError as error: - typer.echo( - f"Could not remove the OpenShell gateway registration: {error}", - err=True, - ) - raise typer.Exit(code=1) from None - - if result is GatewayConfigRemoval.REMOVED: - typer.echo(f"Removed {validated_name} from {config_path}") - typer.echo( - "Next: restart the OpenShell gateway so it unloads this registration." - ) - else: - typer.echo(f"No registration named {validated_name} found in {config_path}") - - -@app.command("configuration-schema") -def configuration_schema(context: typer.Context) -> None: - """Print the policy configuration JSON Schema for the installed engines.""" - typer.echo( - json.dumps( - _command_options(context).registry.configuration_json_schema(), - indent=2, - ensure_ascii=False, - sort_keys=True, - ) - ) - - -@app.command("engines") -def engines(context: typer.Context) -> None: - """List installed engines, supported strategies, and their behavior.""" - for description in _command_options(context).registry.describe_engines(): - strategies = ",".join( - strategy.value - for strategy in EntityProcessingStrategy - if strategy in description.supported_strategies - ) - typer.echo( - f"{description.engine_name}\t{strategies}\t{description.description}" - ) - - -_LOGGER = get_logger(__name__) - - -@dataclass(frozen=True) -class _CommandOptions: - registry: EngineRegistry - log_request_content: bool - - -def _load_registry(factory_reference: str | None) -> EngineRegistry: - if factory_reference is None: - return create_builtin_registry() - module_name, separator, factory_name = factory_reference.partition(":") - if not separator or not module_name or not factory_name: - raise typer.BadParameter( - "Use module:factory, for example my_engines:create_registry.", - param_hint="--registry-factory", - ) - try: - module = importlib.import_module(module_name) - except Exception: - raise typer.BadParameter( - "Registry module could not be imported. Verify the module:factory " - "reference, then import the module directly with content-safe " - "diagnostics to find missing dependencies or startup failures.", - param_hint="--registry-factory", - ) from None - try: - factory = getattr(module, factory_name) - except Exception: - raise typer.BadParameter( - "Registry factory could not be resolved. Verify the module:factory " - "reference and exported callable, then access it directly with " - "content-safe diagnostics.", - param_hint="--registry-factory", - ) from None - if not callable(factory): - raise typer.BadParameter( - "Registry factory is not callable. Export a callable that returns a " - "finalized EngineRegistry.", - param_hint="--registry-factory", - ) - try: - registry = factory() - except Exception: - raise typer.BadParameter( - "Registry factory failed. Run the factory directly with content-safe " - "diagnostics and fix its startup error.", - param_hint="--registry-factory", - ) from None - if not isinstance(registry, EngineRegistry): - raise typer.BadParameter( - "Registry factory returned an invalid object. Return an EngineRegistry.", - param_hint="--registry-factory", - ) - if not registry.is_finalized: - raise typer.BadParameter( - "Registry factory returned an unfinalized registry. Call finalize() " - "before returning it.", - param_hint="--registry-factory", - ) - return registry - - -def _command_options(context: typer.Context) -> _CommandOptions: - options = context.obj - if not isinstance(options, _CommandOptions): - raise RuntimeError("Privacy Guard command context is unavailable") - return options - - -if __name__ == "__main__": - app() - - -__all__ = ["app"] diff --git a/projects/privacy-guard/src/privacy_guard/config.py b/projects/privacy-guard/src/privacy_guard/config.py deleted file mode 100644 index 0862880f..00000000 --- a/projects/privacy-guard/src/privacy_guard/config.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Strict entity-processing policy configuration. - -The concrete model accepted at the policy boundary is finalized by -``EngineRegistry``. Its stage ``config`` field is a Pydantic discriminated -union containing the exact config model registered by every engine. -""" - -from __future__ import annotations - -from enum import StrEnum -from typing import Generic, Self, TypeVar - -from pydantic import ( - Field, - field_validator, - model_validator, -) - -from privacy_guard.base import StrictDomainModel -from privacy_guard.constants import MAX_ENTITY_PROCESSING_STAGES -from privacy_guard.engines import EngineConfig -from privacy_guard.string_validators import ( - BoundedMetadataString, - validate_scalar_string, -) - - -class PolicyAction(StrEnum): - """User-facing disposition applied after all configured stages run.""" - - DETECT = "detect" - BLOCK = "block" - REPLACE = "replace" - - -class OnDetection(StrictDomainModel): - """Required policy disposition for detected entities.""" - - action: PolicyAction - - @field_validator("action", mode="before") - @classmethod - def _parse_action(cls, value: object) -> PolicyAction: - if isinstance(value, PolicyAction): - return value - return PolicyAction(validate_scalar_string(value)) - - -_EngineConfigT = TypeVar( - "_EngineConfigT", - bound=EngineConfig, -) - - -class EntityProcessingStage( - StrictDomainModel, - Generic[_EngineConfigT], -): - """One ordered invocation of an engine with an optional diagnostic name.""" - - name: BoundedMetadataString | None = None - config: _EngineConfigT = Field(repr=False) - - def diagnostic_name(self, stage_number: int) -> str: - """Return the explicit name or a deterministic one-based source label.""" - if self.name is not None: - return self.name - if isinstance(stage_number, bool) or stage_number < 1: - raise ValueError("stage number must be a positive integer") - engine = getattr(self.config, "engine", None) - if not isinstance(engine, str): - raise ValueError("stage config has no engine discriminator") - return f"{engine}[{stage_number}]" - - -class EntityProcessingStages( - StrictDomainModel, - Generic[_EngineConfigT], -): - """The ordered entity-processing stages for one policy.""" - - stages: tuple[EntityProcessingStage[_EngineConfigT], ...] = Field(repr=False) - - @field_validator("stages", mode="before") - @classmethod - def _parse_stages(cls, value: object) -> object: - if not isinstance(value, list | tuple) or not value: - raise ValueError("stages must be a non-empty list") - if len(value) > MAX_ENTITY_PROCESSING_STAGES: - raise ValueError("policy has too many entity-processing stages") - return tuple(value) - - @model_validator(mode="after") - def _diagnostic_names_are_unique(self) -> Self: - names = [ - stage.diagnostic_name(index) - for index, stage in enumerate(self.stages, start=1) - ] - if len(names) != len(set(names)): - raise ValueError("stage diagnostic names must be unique") - return self - - -class PrivacyGuardConfig( - StrictDomainModel, - Generic[_EngineConfigT], -): - """Complete validated Privacy Guard behavior for one OpenShell policy.""" - - entity_processing: EntityProcessingStages[_EngineConfigT] = Field(repr=False) - on_detection: OnDetection = Field(repr=False) - - -__all__ = [ - "EntityProcessingStage", - "EntityProcessingStages", - "OnDetection", - "PolicyAction", - "PrivacyGuardConfig", -] diff --git a/projects/privacy-guard/src/privacy_guard/engines/__init__.py b/projects/privacy-guard/src/privacy_guard/engines/__init__.py deleted file mode 100644 index 59a843d6..00000000 --- a/projects/privacy-guard/src/privacy_guard/engines/__init__.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Supported entity-processing extension and built-in regex engine surface.""" - -from __future__ import annotations - -from privacy_guard.engines.base import ( - BoundedMetadata, - ConfidenceLevel, - EngineConfig, - EngineResources, - EntityDetection, - EntityName, - EntityProcessingEngine, - EntityProcessingStrategy, - TextProcessingResult, -) -from privacy_guard.engines.regex import ( - RegexEngine, - RegexEngineConfig, - RegexEntity, - RegexPatternCatalog, - RegexReplacement, - RegexRule, -) -from privacy_guard.errors import ( - EngineConfigurationError, - EngineContractError, - EngineExecutionError, - EngineLimitExceededError, - EntityProcessingError, -) - -__all__ = [ - "BoundedMetadata", - "ConfidenceLevel", - "EngineConfig", - "EngineConfigurationError", - "EngineContractError", - "EngineExecutionError", - "EngineLimitExceededError", - "EngineResources", - "EntityDetection", - "EntityName", - "EntityProcessingEngine", - "EntityProcessingError", - "EntityProcessingStrategy", - "RegexEngine", - "RegexEngineConfig", - "RegexEntity", - "RegexPatternCatalog", - "RegexReplacement", - "RegexRule", - "TextProcessingResult", -] diff --git a/projects/privacy-guard/src/privacy_guard/engines/base.py b/projects/privacy-guard/src/privacy_guard/engines/base.py deleted file mode 100644 index c6d60bfe..00000000 --- a/projects/privacy-guard/src/privacy_guard/engines/base.py +++ /dev/null @@ -1,367 +0,0 @@ -"""Core entity-processing engine extension contract.""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from collections.abc import Iterable, Mapping -from enum import StrEnum -from itertools import islice -from types import MappingProxyType -from typing import ( - Annotated, - ClassVar, - Generic, - Self, - TypeAlias, - final, - get_args, - get_origin, -) - -from pydantic import ( - BeforeValidator, - Field, - ValidationError, - field_validator, - model_validator, -) -from typing_extensions import TypeVar - -from privacy_guard.base import StrictDomainModel -from privacy_guard.constants import ( - MAX_BODY_BYTES, - MAX_DETECTIONS_PER_STAGE, - MAX_FINDING_METADATA_ENTRIES, -) -from privacy_guard.errors import ( - EngineConfigurationError, - EngineContractError, - EngineLimitExceededError, -) -from privacy_guard.string_validators import ( - ScalarString, - validate_bounded_metadata_string, - validate_scalar_string, -) -from privacy_guard.timeout import Timeout - - -class EntityProcessingStrategy(StrEnum): - """Select whether one engine invocation detects or replaces entities.""" - - DETECT = "detect" - REPLACE = "replace" - - -class ConfidenceLevel(StrEnum): - """Categorical certainty reported by an entity-processing engine.""" - - LOW = "low" - MEDIUM = "medium" - HIGH = "high" - - -EntityName = Annotated[str, BeforeValidator(validate_bounded_metadata_string)] -MetadataString = Annotated[str, BeforeValidator(validate_bounded_metadata_string)] -BoundedMetadata: TypeAlias = Mapping[MetadataString, MetadataString] - - -class EntityDetection(StrictDomainModel): - """One sensitive entity occurrence in the engine's input text.""" - - entity: EntityName - start: int = Field(ge=0) - end: int - confidence: ConfidenceLevel | None = None - metadata: BoundedMetadata = Field(default_factory=dict, repr=False) - - @field_validator("confidence", mode="before") - @classmethod - def _parse_confidence(cls, value: object) -> object: - if isinstance(value, str): - return ConfidenceLevel(validate_scalar_string(value)) - return value - - @field_validator("metadata") - @classmethod - def _copy_bounded_metadata(cls, value: Mapping[str, str]) -> Mapping[str, str]: - if len(value) > MAX_FINDING_METADATA_ENTRIES: - raise ValueError("detection metadata has too many entries") - copied: dict[str, str] = {} - for key, item in value.items(): - copied[validate_bounded_metadata_string(key)] = ( - validate_bounded_metadata_string(item) - ) - return MappingProxyType(copied) - - @model_validator(mode="after") - def _span_is_non_empty(self) -> EntityDetection: - if self.end <= self.start: - raise ValueError("detection span must be non-empty") - return self - - -class TextProcessingResult(StrictDomainModel): - """The authoritative text and detections returned by one engine run.""" - - text: ScalarString = Field(repr=False) - detections: tuple[EntityDetection, ...] - - @field_validator("detections", mode="before") - @classmethod - def _detections_are_a_tuple(cls, value: object) -> object: - if not isinstance(value, tuple): - raise ValueError("detections must be a tuple") - return value - - @classmethod - def from_detections( - cls, - *, - text: str, - detections: Iterable[EntityDetection], - ) -> Self: - """Safely materialize a lazy stream; ``run()`` still validates the result.""" - bounded = tuple(islice(detections, MAX_DETECTIONS_PER_STAGE + 1)) - if len(bounded) > MAX_DETECTIONS_PER_STAGE: - raise EngineLimitExceededError("engine returned too many detections") - return cls(text=text, detections=bounded) - - -class EngineConfig(StrictDomainModel): - """Nominal base for an engine's exact policy configuration.""" - - -class EngineResources: - """Optional operator-owned runtime dependencies shared by engine instances. - - Resource objects contain initialized operational dependencies such as model - clients, SDK adapters, endpoints, or credential providers. They must not - contain policy behavior or mutable per-request state, and everything they - expose to an engine must be safe for concurrent use. - """ - - __slots__ = () - - -_ConfigT = TypeVar("_ConfigT", bound=EngineConfig) -_ResourcesT = TypeVar( - "_ResourcesT", - bound=EngineResources | None, - default=None, -) - - -class EntityProcessingEngine(ABC, Generic[_ConfigT, _ResourcesT]): - """Nominal, typed extension point for processing one text string.""" - - supported_strategies: ClassVar[frozenset[EntityProcessingStrategy]] - - @final - def __init__( - self, - config: _ConfigT, - resources: _ResourcesT, - ) -> None: - """Validate typed configuration/resources and initialize reusable state.""" - self.validate_config(config, resources) - self.__config = config - self.__resources = resources - self._initialize() - - @classmethod - def validate_config( - cls, - config: _ConfigT, - resources: _ResourcesT, - ) -> None: - """Purely validate one exact config and its registered resources.""" - cls._validate_class_contract() - config_type = cls.get_config_type() - try: - if not isinstance(config, config_type): - raise ValueError - config_type.model_validate(config) - except (ValidationError, ValueError): - raise EngineConfigurationError("engine configuration is invalid") from None - resources_type = cls.get_resources_type() - if not _is_valid_resources(resources, resources_type): - raise EngineConfigurationError("engine resources are invalid") - cls._validate_config(config, resources) - - @classmethod - def validate_run_config( - cls, - config: _ConfigT, - resources: _ResourcesT, - *, - strategy: EntityProcessingStrategy, - ) -> None: - """Validate that one config can execute the requested strategy.""" - if not isinstance(strategy, EntityProcessingStrategy): - raise EngineConfigurationError("engine processing strategy is invalid") - cls.validate_config(config, resources) - if strategy not in cls.supported_strategies: - raise EngineConfigurationError( - "engine does not support the requested strategy" - ) - cls._validate_run_config(config, resources, strategy=strategy) - - @classmethod - def get_config_type(cls) -> type[EngineConfig]: - """Return the concrete ``EngineConfig`` type argument.""" - config_type, _ = _declared_engine_types(cls) - return config_type - - @classmethod - def get_resources_type(cls) -> object: - """Return the concrete runtime-resources generic argument.""" - _, resources_type = _declared_engine_types(cls) - return resources_type - - @property - def config(self) -> _ConfigT: - """Return the immutable, concrete engine configuration.""" - return self.__config - - @property - def resources(self) -> _ResourcesT: - """Return the validated, injected runtime resources.""" - return self.__resources - - @final - def run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - """Process one text value and validate the complete collaborator result.""" - try: - validated_text = validate_scalar_string(text) - except ValueError: - raise EngineContractError("engine input text is invalid") from None - if not isinstance(strategy, EntityProcessingStrategy): - raise EngineContractError("engine processing strategy is invalid") - if not isinstance(timeout, Timeout): - raise EngineContractError("engine timeout is invalid") - if strategy not in self.supported_strategies: - raise EngineContractError("engine does not support the requested strategy") - timeout.raise_if_expired() - result: object = self._run( - validated_text, - strategy=strategy, - timeout=timeout, - ) - timeout.raise_if_expired() - return _validate_result(validated_text, result, strategy=strategy) - - @classmethod - def _validate_config( - cls, - config: _ConfigT, - resources: _ResourcesT, - ) -> None: - """Optionally validate resource-backed config without side effects.""" - - @classmethod - def _validate_run_config( - cls, - config: _ConfigT, - resources: _ResourcesT, - *, - strategy: EntityProcessingStrategy, - ) -> None: - """Optionally validate requirements specific to one run strategy.""" - - @classmethod - def _validate_class_contract(cls) -> None: - supported_strategies = getattr(cls, "supported_strategies", None) - if ( - not isinstance(supported_strategies, frozenset) - or not supported_strategies - or any( - not isinstance(strategy, EntityProcessingStrategy) - for strategy in supported_strategies - ) - ): - raise EngineConfigurationError("engine supported strategies are invalid") - cls.get_config_type() - cls.get_resources_type() - - def _initialize(self) -> None: - """Optionally initialize reusable state from config and resources.""" - - @abstractmethod - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - """Return processed text and every detected entity occurrence.""" - raise NotImplementedError - - -def _declared_engine_types( - engine_type: type[object], -) -> tuple[type[EngineConfig], object]: - for candidate in engine_type.__mro__: - for base in getattr(candidate, "__orig_bases__", ()): - if get_origin(base) is not EntityProcessingEngine: - continue - arguments = get_args(base) - if len(arguments) != 2: - break - config_type, resources_type = arguments - if isinstance(config_type, type) and issubclass(config_type, EngineConfig): - return config_type, resources_type - raise EngineConfigurationError( - "engine must declare concrete configuration and resource types" - ) - - -def _is_valid_resources(resources: object, resources_type: object) -> bool: - if resources_type in (None, type(None)): - return resources is None - origin = get_origin(resources_type) - if origin is not None: - resources_type = origin - return isinstance(resources_type, type) and isinstance(resources, resources_type) - - -def _validate_result( - input_text: str, - result: object, - *, - strategy: EntityProcessingStrategy, -) -> TextProcessingResult: - if not isinstance(result, TextProcessingResult): - raise EngineContractError("engine output is invalid") - if len(result.detections) > MAX_DETECTIONS_PER_STAGE: - raise EngineLimitExceededError("engine returned too many detections") - if len(result.text.encode("utf-8")) > MAX_BODY_BYTES: - raise EngineLimitExceededError("engine output text exceeds the size limit") - for detection in result.detections: - if detection.end > len(input_text): - raise EngineContractError("engine detection span is invalid") - if strategy is EntityProcessingStrategy.DETECT and result.text != input_text: - raise EngineContractError("detection-only engine output changed text") - if result.text != input_text and not result.detections: - raise EngineContractError("engine changed text without a detection") - return result - - -__all__ = [ - "BoundedMetadata", - "ConfidenceLevel", - "EngineConfig", - "EngineResources", - "EntityDetection", - "EntityName", - "EntityProcessingEngine", - "EntityProcessingStrategy", - "TextProcessingResult", -] diff --git a/projects/privacy-guard/src/privacy_guard/engines/registry.py b/projects/privacy-guard/src/privacy_guard/engines/registry.py deleted file mode 100644 index bee9deba..00000000 --- a/projects/privacy-guard/src/privacy_guard/engines/registry.py +++ /dev/null @@ -1,318 +0,0 @@ -"""Engine registration and finalized policy-schema construction.""" - -from __future__ import annotations - -import inspect -import re -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from functools import reduce -from operator import or_ -from types import NoneType -from typing import Annotated, Literal, Self, get_args, get_origin - -from pydantic import Field, TypeAdapter, ValidationError -from pydantic_core import PydanticUndefined - -from privacy_guard.config import ( - PolicyAction, - PrivacyGuardConfig, -) -from privacy_guard.engines.base import ( - EngineConfig, - EngineResources, - EntityProcessingEngine, - EntityProcessingStrategy, -) -from privacy_guard.engines.regex import ( - RegexEngine, -) -from privacy_guard.errors import ( - EngineConfigurationError, - EngineRegistryError, - ErrorCode, - PrivacyGuardError, -) - - -@dataclass(frozen=True) -class EngineDescription: - """Safe discovery metadata for one registered engine.""" - - engine_name: str - description: str - supported_strategies: frozenset[EntityProcessingStrategy] - - -class EngineRegistry: - """Register engine implementations and finalize their exact policy union.""" - - def __init__(self, *, include_builtin_engines: bool = False) -> None: - self._registrations: dict[str, _Registration] = {} - self._config_adapter: TypeAdapter[PrivacyGuardConfig[EngineConfig]] | None = ( - None - ) - if include_builtin_engines: - self.register(RegexEngine) - - @property - def is_finalized(self) -> bool: - return self._config_adapter is not None - - def register( - self, - engine_type: type[object], - *, - resources: object = None, - ) -> None: - """Register one engine implementation and its operator-owned resources.""" - if self.is_finalized: - raise EngineRegistryError("cannot register after finalization") - if not isinstance(engine_type, type) or not issubclass( - engine_type, EntityProcessingEngine - ): - raise EngineRegistryError("registered engine type is invalid") - if engine_type.__init__ is not EntityProcessingEngine.__init__: - raise EngineRegistryError( - "engine lifecycle contract requires EntityProcessingEngine.__init__; " - "use _initialize() instead" - ) - if engine_type.run is not EntityProcessingEngine.run: - raise EngineRegistryError( - "engine lifecycle contract requires EntityProcessingEngine.run; " - "implement _run() instead" - ) - - try: - config_type = engine_type.get_config_type() - resources_type = engine_type.get_resources_type() - except (AttributeError, TypeError): - raise EngineRegistryError("engine generic declaration is invalid") from None - if not isinstance(config_type, type) or not issubclass( - config_type, EngineConfig - ): - raise EngineRegistryError("engine config type is invalid") - resources_runtime_type = ( - NoneType - if resources_type is None - else get_origin(resources_type) or resources_type - ) - if not isinstance(resources_runtime_type, type): - raise EngineRegistryError("engine resources type is invalid") - if resources_runtime_type is not NoneType and not issubclass( - resources_runtime_type, - EngineResources, - ): - raise EngineRegistryError( - "engine resources type must extend EngineResources" - ) - - engine_name = _engine_discriminator(config_type) - if engine_name in self._registrations: - raise EngineRegistryError("engine discriminator is already registered") - if any( - registration.config_type is config_type - for registration in self._registrations.values() - ): - raise EngineRegistryError("engine config type is already registered") - - _supported_strategies(engine_type) - if resources_runtime_type is NoneType: - if resources is not None: - raise EngineRegistryError("resource-free engine received resources") - else: - if resources is not None and not isinstance(resources, EngineResources): - raise EngineRegistryError( - "engine resources must extend EngineResources" - ) - if resources is None or not isinstance(resources, resources_runtime_type): - raise EngineRegistryError( - "engine resources do not match their declared type" - ) - - self._registrations[engine_name] = _Registration( - engine_type=engine_type, - config_type=config_type, - resources=resources, - ) - - def finalize(self) -> Self: - """Freeze registrations, build the policy union, and return this registry.""" - if self.is_finalized: - return self - try: - config_type = _build_privacy_guard_config_type( - tuple( - registration.config_type - for registration in self._registrations.values() - ) - ) - except ValueError: - raise EngineRegistryError( - "cannot finalize an empty engine registry" - ) from None - self._config_adapter = TypeAdapter(config_type) - return self - - def validate_config(self, values: object) -> PrivacyGuardConfig[EngineConfig]: - """Purely parse and validate an expanded Privacy Guard configuration.""" - if not isinstance(values, Mapping): - raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) - try: - config = self._require_config_adapter().validate_python(dict(values)) - except (TypeError, ValueError, ValidationError): - raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None - required_strategy = ( - EntityProcessingStrategy.REPLACE - if config.on_detection.action is PolicyAction.REPLACE - else EntityProcessingStrategy.DETECT - ) - for stage in config.entity_processing.stages: - registration = self._resolve_registration(stage.config) - engine_type = registration.engine_type - if not issubclass(engine_type, EntityProcessingEngine): - raise EngineRegistryError("registered engine type is invalid") - try: - validate_run_config = getattr(engine_type, "validate_run_config") - validate_run_config( - stage.config, - registration.resources, - strategy=required_strategy, - ) - except EngineConfigurationError: - raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None - return config - - def create_engine( - self, - config: EngineConfig, - ) -> EntityProcessingEngine[EngineConfig, EngineResources | None]: - """Construct an initialized engine from its exact validated config.""" - registration = self._resolve_registration(config) - if type(config) is not registration.config_type: - raise EngineRegistryError("engine config concrete type is invalid") - return registration.engine_type(config, registration.resources) - - def configuration_json_schema(self) -> dict[str, object]: - """Return the finalized complete policy JSON Schema.""" - return self._require_config_adapter().json_schema() - - def describe_engines(self) -> tuple[EngineDescription, ...]: - """Return safe engine metadata without constructing runtime engines.""" - return tuple( - EngineDescription( - engine_name=engine, - description=_engine_description(registration.engine_type), - supported_strategies=_supported_strategies(registration.engine_type), - ) - for engine, registration in self._registrations.items() - ) - - def _resolve_registration( - self, - config: EngineConfig, - ) -> _Registration: - if not self.is_finalized: - raise EngineRegistryError("engine registry is not finalized") - try: - engine_name = getattr(config, "engine") - if not isinstance(engine_name, str): - raise AttributeError - registration = self._registrations[engine_name] - except (AttributeError, KeyError): - raise EngineRegistryError("engine config is not registered") from None - return registration - - def _require_config_adapter( - self, - ) -> TypeAdapter[PrivacyGuardConfig[EngineConfig]]: - if self._config_adapter is None: - raise EngineRegistryError("engine registry is not finalized") - return self._config_adapter - - -def create_builtin_registry() -> EngineRegistry: - """Build the finalized registry shipped by the base package.""" - return EngineRegistry(include_builtin_engines=True).finalize() - - -@dataclass(frozen=True) -class _Registration: - engine_type: type[object] - config_type: type[EngineConfig] - resources: EngineResources | None - - -def _build_privacy_guard_config_type( - config_types: Sequence[type[EngineConfig]], -) -> type[PrivacyGuardConfig[EngineConfig]]: - if not config_types: - raise ValueError("at least one engine config type must be registered") - registered_union = reduce(or_, config_types) - registered_config = Annotated[ - registered_union, # ty: ignore[invalid-type-form] - Field(discriminator="engine"), - ] - config_type = PrivacyGuardConfig.__class_getitem__( - registered_config # ty: ignore[invalid-argument-type] - ) - if not isinstance(config_type, type) or not issubclass( - config_type, PrivacyGuardConfig - ): - raise TypeError("Pydantic did not construct a policy config type") - return config_type # ty: ignore[invalid-return-type] - - -def _supported_strategies( - engine_type: type[object], -) -> frozenset[EntityProcessingStrategy]: - supported_strategies = getattr(engine_type, "supported_strategies", None) - if ( - not isinstance(supported_strategies, frozenset) - or not supported_strategies - or any( - not isinstance(strategy, EntityProcessingStrategy) - for strategy in supported_strategies - ) - ): - raise EngineRegistryError("engine supported strategies are invalid") - return supported_strategies - - -def _engine_discriminator( - config_type: type[EngineConfig], -) -> str: - field = config_type.model_fields.get("engine") - if field is None: - raise EngineRegistryError("engine config lacks an engine discriminator") - if get_origin(field.annotation) is not Literal: - raise EngineRegistryError("engine discriminator must be one string Literal") - values = get_args(field.annotation) - if len(values) != 1 or not isinstance(values[0], str): - raise EngineRegistryError("engine discriminator must be one string Literal") - engine = values[0] - if _ENGINE_NAME.fullmatch(engine) is None or len(engine.encode("ascii")) > 128: - raise EngineRegistryError("engine discriminator is invalid") - if field.default is not PydanticUndefined and field.default != engine: - raise EngineRegistryError("engine discriminator default is inconsistent") - return engine - - -def _engine_description( - engine_type: type[object], -) -> str: - description = inspect.getdoc(engine_type) or "" - first_line = description.splitlines()[0] if description else "" - if len(first_line.encode("utf-8")) > 1024: - return "" - return first_line - - -_ENGINE_NAME = re.compile(r"[a-z][a-z0-9-]{0,127}\Z") - - -__all__ = [ - "EngineDescription", - "EngineRegistry", - "create_builtin_registry", -] diff --git a/projects/privacy-guard/src/privacy_guard/request_processor.py b/projects/privacy-guard/src/privacy_guard/request_processor.py deleted file mode 100644 index 459f6fb8..00000000 --- a/projects/privacy-guard/src/privacy_guard/request_processor.py +++ /dev/null @@ -1,212 +0,0 @@ -"""Sequential entity-processing orchestration for one text input.""" - -from __future__ import annotations - -from collections.abc import Sequence -from enum import StrEnum - -from pydantic import Field - -from privacy_guard.base import StrictDomainModel -from privacy_guard.config import PolicyAction, PrivacyGuardConfig -from privacy_guard.constants import ( - BLOCK_REASON_CODE, - DEFAULT_TIMEOUT_SECONDS, - LIMIT_REASON_CODE, - MAX_BODY_BYTES, - MAX_DETECTIONS_PER_REQUEST, -) -from privacy_guard.engines import ( - ConfidenceLevel, - EngineConfig, - EngineResources, - EntityProcessingEngine, - EntityProcessingStrategy, - TextProcessingResult, -) -from privacy_guard.errors import ( - EngineConfigurationError, - EngineContractError, - EngineLimitExceededError, - EntityProcessingError, - ErrorCode, - PrivacyGuardError, - TimeoutExpiredError, -) -from privacy_guard.logging import get_logger -from privacy_guard.string_validators import validate_scalar_string -from privacy_guard.timeout import Timeout, validate_timeout_seconds - - -class RequestDecision(StrEnum): - """Whether OpenShell should continue or stop the request.""" - - ALLOW = "allow" - DENY = "deny" - - -class EntityDetectionSummary(StrictDomainModel): - """One bounded aggregate suitable for user-facing audit output.""" - - entity: str - source_stage: str - confidence: ConfidenceLevel | None = None - count: int = Field(ge=1) - - -class RequestProcessingResult(StrictDomainModel): - """The processor's decision, summaries, and optional replacement text.""" - - decision: RequestDecision - replacement_text: str | None = Field(default=None, repr=False) - detection_summaries: tuple[EntityDetectionSummary, ...] = () - reason_code: str | None = None - - -class RequestProcessor: - """Run configured entity-processing stages once, in policy order.""" - - def __init__( - self, - config: PrivacyGuardConfig[EngineConfig], - configured_engines: Sequence[ - tuple[ - str, - EntityProcessingEngine[EngineConfig, EngineResources | None], - ] - ], - *, - timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, - log_request_content: bool = False, - ) -> None: - engines = tuple(configured_engines) - if len(engines) != len(config.entity_processing.stages): - raise ValueError("configured engines do not match the policy") - if not engines: - raise ValueError("at least one configured engine is required") - sources = tuple(source for source, _ in engines) - if any(not source for source in sources) or len(sources) != len(set(sources)): - raise ValueError("engine sources must be non-empty and unique") - self._config = config - self._engines = engines - self._timeout_seconds = validate_timeout_seconds(timeout_seconds) - self._log_request_content = log_request_content - - def process(self, text: str) -> RequestProcessingResult: - """Process one complete request text and apply the user-facing action.""" - try: - input_text = validate_scalar_string(text) - except ValueError: - raise PrivacyGuardError(ErrorCode.BODY_ENCODING_INVALID) from None - if len(input_text.encode("utf-8")) > MAX_BODY_BYTES: - raise PrivacyGuardError(ErrorCode.REQUEST_BODY_TOO_LARGE) - if self._log_request_content: - _LOGGER.debug("privacy_guard_text_input text=%r", input_text) - - action = self._config.on_detection.action - strategy = ( - EntityProcessingStrategy.REPLACE - if action is PolicyAction.REPLACE - else EntityProcessingStrategy.DETECT - ) - timeout = Timeout.from_seconds(self._timeout_seconds) - current_text = input_text - stage_results: list[tuple[str, TextProcessingResult]] = [] - try: - for source, engine in self._engines: - _LOGGER.debug( - "privacy_guard_stage_run source=%s strategy=%s", - source, - strategy.value, - ) - result = engine.run( - current_text, - strategy=strategy, - timeout=timeout, - ) - if len(result.text.encode("utf-8")) > MAX_BODY_BYTES: - raise EngineLimitExceededError( - "intermediate text exceeds the limit" - ) - if ( - sum(len(item.detections) for _, item in stage_results) - + len(result.detections) - > MAX_DETECTIONS_PER_REQUEST - ): - raise EngineLimitExceededError( - "request detections exceed the limit" - ) - stage_results.append((source, result)) - current_text = result.text - timeout.raise_if_expired() - except TimeoutExpiredError: - _LOGGER.info("privacy_guard_processing_limit kind=timeout") - return RequestProcessingResult( - decision=RequestDecision.DENY, - reason_code=LIMIT_REASON_CODE, - ) - except EngineLimitExceededError: - _LOGGER.info("privacy_guard_processing_limit kind=resource") - return RequestProcessingResult( - decision=RequestDecision.DENY, - reason_code=LIMIT_REASON_CODE, - ) - except EngineConfigurationError: - raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None - except EngineContractError: - raise PrivacyGuardError(ErrorCode.ENGINE_OUTPUT_INVALID) from None - except EntityProcessingError: - raise PrivacyGuardError(ErrorCode.ENGINE_EXECUTION_FAILED) from None - except PrivacyGuardError: - raise - except Exception: - raise PrivacyGuardError(ErrorCode.ENGINE_EXECUTION_FAILED) from None - - detections = _aggregate_detections(stage_results) - if action is PolicyAction.BLOCK and detections: - return RequestProcessingResult( - decision=RequestDecision.DENY, - detection_summaries=detections, - reason_code=BLOCK_REASON_CODE, - ) - replacement_text = current_text if action is PolicyAction.REPLACE else None - if self._log_request_content: - _LOGGER.debug("privacy_guard_text_output text=%r", current_text) - return RequestProcessingResult( - decision=RequestDecision.ALLOW, - replacement_text=replacement_text, - detection_summaries=detections, - ) - - -def _aggregate_detections( - stage_results: Sequence[tuple[str, TextProcessingResult]], -) -> tuple[EntityDetectionSummary, ...]: - groups: dict[ - tuple[str, str, ConfidenceLevel | None], - int, - ] = {} - for source, result in stage_results: - for detection in result.detections: - key = (source, detection.entity, detection.confidence) - groups[key] = groups.get(key, 0) + 1 - return tuple( - EntityDetectionSummary( - source_stage=source, - entity=entity, - confidence=confidence, - count=count, - ) - for (source, entity, confidence), count in groups.items() - ) - - -_LOGGER = get_logger(__name__) - - -__all__ = [ - "EntityDetectionSummary", - "RequestDecision", - "RequestProcessingResult", - "RequestProcessor", -] diff --git a/projects/privacy-guard/src/privacy_guard/service/__init__.py b/projects/privacy-guard/src/privacy_guard/service/__init__.py deleted file mode 100644 index d4923885..00000000 --- a/projects/privacy-guard/src/privacy_guard/service/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""gRPC transport and servicer for the Privacy Guard middleware.""" - -from privacy_guard.service.server import PrivacyGuardServer -from privacy_guard.service.servicer import PrivacyGuardMiddleware - -__all__ = ["PrivacyGuardMiddleware", "PrivacyGuardServer"] diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py deleted file mode 100644 index 4465c046..00000000 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Programmatic Privacy Guard gRPC server lifecycle.""" - -from __future__ import annotations - -import asyncio - -import grpc - -from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc -from privacy_guard.constants import ( - DEFAULT_TIMEOUT_SECONDS, - MAX_CONCURRENT_RPCS, - MAX_RECEIVE_MESSAGE_BYTES, -) -from privacy_guard.engines.registry import EngineRegistry -from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.logging import get_logger -from privacy_guard.service.servicer import PrivacyGuardMiddleware - -DEFAULT_LISTEN_ADDRESS = "127.0.0.1:50051" - - -class PrivacyGuardServer: - """One-shot programmatic server for a finalized engine registry.""" - - def __init__( - self, - registry: EngineRegistry, - *, - timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, - log_request_content: bool = False, - ) -> None: - self._middleware = PrivacyGuardMiddleware( - registry, - timeout_seconds=timeout_seconds, - log_request_content=log_request_content, - ) - - def serve_sync(self, listen: str = DEFAULT_LISTEN_ADDRESS) -> None: - """Serve synchronously until termination.""" - try: - asyncio.run(self.serve_async(listen)) - except KeyboardInterrupt: - return - - async def serve_async(self, listen: str = DEFAULT_LISTEN_ADDRESS) -> None: - """Serve asynchronously until termination, then close owned resources.""" - server = _create_grpc_server(self._middleware) - try: - try: - requested_port = _validated_listen_port(listen) - bound_port = server.add_insecure_port(listen) - if bound_port != requested_port: - raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) - _LOGGER.info("privacy_guard_server_bound listen=%r", listen) - await server.start() - except RuntimeError: - raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) from None - await server.wait_for_termination() - finally: - try: - await _stop_grpc_server(server) - finally: - await self._middleware.close() - - -_LOGGER = get_logger(__name__) - - -def _create_grpc_server( - middleware: PrivacyGuardMiddleware, -) -> grpc.aio.Server: - server = grpc.aio.server( - maximum_concurrent_rpcs=MAX_CONCURRENT_RPCS, - options=(("grpc.max_receive_message_length", MAX_RECEIVE_MESSAGE_BYTES),), - ) - pb2_grpc.add_SupervisorMiddlewareServicer_to_server(middleware, server) - return server - - -async def _stop_grpc_server(server: grpc.aio.Server) -> None: - shutdown = asyncio.create_task(server.stop(grace=0)) - try: - await asyncio.shield(shutdown) - except asyncio.CancelledError: - if not shutdown.done(): - await shutdown - raise - - -def _validated_listen_port(listen: str) -> int: - if not isinstance(listen, str): - raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) - if listen.startswith("["): - closing_bracket = listen.rfind("]") - if ( - closing_bracket < 2 - or listen[closing_bracket + 1 : closing_bracket + 2] != ":" - ): - raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) - host = listen[1:closing_bracket] - port_text = listen[closing_bracket + 2 :] - else: - host, separator, port_text = listen.rpartition(":") - if not separator or not host or ":" in host: - raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) - if ( - not host - or not port_text - or len(port_text) > 5 - or not port_text.isascii() - or not port_text.isdecimal() - ): - raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) - port = int(port_text) - if not 1 <= port <= 65_535: - raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) - return port - - -__all__ = [ - "DEFAULT_LISTEN_ADDRESS", - "PrivacyGuardServer", -] diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py deleted file mode 100644 index 68b30585..00000000 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ /dev/null @@ -1,457 +0,0 @@ -"""gRPC boundary for active entity-processing policy evaluation.""" - -from __future__ import annotations - -import asyncio -import json -import math -import time -from collections.abc import Callable, Iterable -from concurrent.futures import Future, ThreadPoolExecutor -from threading import Lock -from typing import Never, Protocol, TypedDict, TypeVar - -import grpc -from google.protobuf import json_format -from google.protobuf.message import Message - -from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 -from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc -from privacy_guard.config import PrivacyGuardConfig -from privacy_guard.constants import ( - BLOCK_REASON, - BLOCK_REASON_CODE, - DEFAULT_TIMEOUT_SECONDS, - LIMIT_REASON, - LIMIT_REASON_CODE, - MAX_BODY_BYTES, - MAX_CONCURRENT_PROCESSING, - MAX_PROTO_CONFIG_BYTES, - MAX_PROTO_CONTEXT_BYTES, - MAX_PROTO_FINDING_BYTES, - MAX_PROTO_FINDING_GROUPS, - MAX_PROTO_HEADERS, - MAX_PROTO_HEADERS_BYTES, - MAX_PROTO_TARGET_BYTES, - REASON_CODE_PATTERN, - SERVICE_NAME, - SERVICE_VERSION, -) -from privacy_guard.engines import EngineConfig -from privacy_guard.engines.registry import EngineRegistry -from privacy_guard.errors import ( - EngineRegistryError, - ErrorCode, - ErrorKind, - PrivacyGuardError, -) -from privacy_guard.logging import get_logger -from privacy_guard.request_processor import ( - EntityDetectionSummary, - RequestDecision, - RequestProcessingResult, - RequestProcessor, -) -from privacy_guard.string_validators import validate_bounded_metadata_string -from privacy_guard.timeout import validate_timeout_seconds - - -class PrivacyGuardMiddleware(pb2_grpc.SupervisorMiddlewareServicer): - """Validate, prepare, resolve, and run Privacy Guard policies.""" - - def __init__( - self, - registry: EngineRegistry, - *, - timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, - log_request_content: bool = False, - ) -> None: - if not registry.is_finalized: - raise EngineRegistryError("middleware requires a finalized engine registry") - self._registry = registry - self._policy = _ActivePolicy( - registry, - timeout_seconds=validate_timeout_seconds(timeout_seconds), - log_request_content=log_request_content, - ) - self._processing_slots = asyncio.Semaphore(MAX_CONCURRENT_PROCESSING) - self._processing_executor = ThreadPoolExecutor( - max_workers=MAX_CONCURRENT_PROCESSING, - thread_name_prefix="privacy-guard-processing", - ) - - async def close(self) -> None: - """Wait for in-flight synchronous engines during shutdown.""" - self._processing_executor.shutdown(wait=True, cancel_futures=True) - self._policy.clear() - - async def Describe( - self, - request: object, - context: grpc.aio.ServicerContext[object, pb2.MiddlewareManifest], - ) -> pb2.MiddlewareManifest: - """Advertise the binding and its finalized policy schema.""" - return pb2.MiddlewareManifest( - name=SERVICE_NAME, - service_version=SERVICE_VERSION, - bindings=[ - pb2.MiddlewareBinding( - operation=pb2.SUPERVISOR_MIDDLEWARE_OPERATION_HTTP_REQUEST, - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, - max_body_bytes=MAX_BODY_BYTES, - ) - ], - ) - - async def ValidateConfig( - self, - request: pb2.ValidateConfigRequest, - context: grpc.aio.ServicerContext[ - pb2.ValidateConfigRequest, - pb2.ValidateConfigResponse, - ], - ) -> pb2.ValidateConfigResponse: - """Validate expanded configuration without preparing runtime state.""" - return await self._run_in_worker(lambda: self._validate_config(request)) - - async def EvaluateHttpRequest( - self, - request: pb2.HttpRequestEvaluation, - context: grpc.aio.ServicerContext[ - pb2.HttpRequestEvaluation, - pb2.HttpRequestResult, - ], - ) -> pb2.HttpRequestResult: - """Resolve the prepared config, decode one text, and process it.""" - return await self._evaluate_rpc(request, context) - - def _validate_config( - self, - request: pb2.ValidateConfigRequest, - ) -> pb2.ValidateConfigResponse: - try: - if request.config.ByteSize() > MAX_PROTO_CONFIG_BYTES: - raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) - self._registry.validate_config(_mapping_from_proto(request.config)) - except PrivacyGuardError as error: - return pb2.ValidateConfigResponse(valid=False, reason=str(error)) - except Exception: - error = PrivacyGuardError(ErrorCode.UNEXPECTED_SERVICE_FAILURE) - return pb2.ValidateConfigResponse(valid=False, reason=str(error)) - return pb2.ValidateConfigResponse(valid=True) - - async def _evaluate_rpc( - self, - request: pb2.HttpRequestEvaluation, - context: _AbortContext, - ) -> pb2.HttpRequestResult: - started = time.monotonic() - request_id = _request_id_for_logging(request.context.request_id) - failure: PrivacyGuardError | None = None - action = "error" - finding_count = 0 - try: - response = await self._evaluate_http_request(request) - action = "allow" if response.decision == pb2.DECISION_ALLOW else "deny" - finding_count = sum(finding.count for finding in response.findings) - return response - except PrivacyGuardError as error: - failure = error - except Exception: - failure = PrivacyGuardError(ErrorCode.UNEXPECTED_SERVICE_FAILURE) - finally: - log_extra = _evaluation_log_extra( - request_id=request_id, - started=started, - action=action, - finding_count=finding_count, - failure=failure, - ) - _LOGGER.info( - "privacy_guard_evaluation request_id=%s duration_ms=%.3f " - "action=%s finding_count=%d error_code=%s", - _request_id_for_log_message(log_extra["request_id"]), - log_extra["duration_ms"], - log_extra["action"], - log_extra["finding_count"], - log_extra["error_code"] or "none", - extra=log_extra, - ) - status = ( - grpc.StatusCode.INVALID_ARGUMENT - if failure.kind is ErrorKind.INVALID_INPUT - else grpc.StatusCode.INTERNAL - ) - await context.abort(status, str(failure)) - - async def _evaluate_http_request( - self, - request: pb2.HttpRequestEvaluation, - ) -> pb2.HttpRequestResult: - if request.phase != pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS: - raise PrivacyGuardError(ErrorCode.REQUEST_PHASE_INVALID) - if len(request.body) > MAX_BODY_BYTES: - raise PrivacyGuardError(ErrorCode.REQUEST_BODY_TOO_LARGE) - _validate_evaluation_envelope(request) - result = await self._run_in_worker( - lambda: self._prepare_and_process(request.config, request.body) - ) - return _result_to_proto(result) - - def _prepare_and_process( - self, - config: Message, - body: bytes, - ) -> RequestProcessingResult: - values = _mapping_from_proto(config) - processor = self._policy.processor_for(values) - if not body: - return RequestProcessingResult(decision=RequestDecision.ALLOW) - try: - text = body.decode("utf-8", errors="strict") - except UnicodeDecodeError: - raise PrivacyGuardError(ErrorCode.BODY_ENCODING_INVALID) from None - return processor.process(text) - - async def _run_in_worker( - self, - operation: Callable[[], _WorkerResultT], - ) -> _WorkerResultT: - """Run one bounded synchronous operation without blocking the event loop.""" - await self._processing_slots.acquire() - try: - worker = self._processing_executor.submit(operation) - future = asyncio.create_task(_await_worker(worker)) - except BaseException: - self._processing_slots.release() - raise - future.add_done_callback(lambda _: self._processing_slots.release()) - return await asyncio.shield(future) - - -class _ActivePolicy: - """Own the process's active policy and its prepared processor.""" - - def __init__( - self, - registry: EngineRegistry, - *, - timeout_seconds: float, - log_request_content: bool, - ) -> None: - self._registry = registry - self._timeout_seconds = timeout_seconds - self._log_request_content = log_request_content - self._config: PrivacyGuardConfig[EngineConfig] | None = None - self._processor: RequestProcessor | None = None - self._lock = Lock() - - def processor_for(self, values: object) -> RequestProcessor: - """Return the processor for the requested policy, activating it if needed.""" - config = self._registry.validate_config(values) - with self._lock: - if config == self._config and self._processor is not None: - return self._processor - processor = self._build_processor(config) - self._config = config - self._processor = processor - return processor - - def _build_processor( - self, - config: PrivacyGuardConfig[EngineConfig], - ) -> RequestProcessor: - stages = tuple( - ( - stage.diagnostic_name(index), - self._registry.create_engine(stage.config), - ) - for index, stage in enumerate( - config.entity_processing.stages, - start=1, - ) - ) - return RequestProcessor( - config, - stages, - timeout_seconds=self._timeout_seconds, - log_request_content=self._log_request_content, - ) - - def clear(self) -> None: - """Release the active policy.""" - with self._lock: - self._config = None - self._processor = None - - -_WorkerResultT = TypeVar("_WorkerResultT") - - -async def _await_worker(worker: Future[_WorkerResultT]) -> _WorkerResultT: - """Bridge a worker without relying on broken cross-thread loop wakeups.""" - while not worker.done(): - await asyncio.sleep(0.001) - return worker.result() - - -class _AbortContext(Protocol): - async def abort(self, code: grpc.StatusCode, details: str) -> Never: ... - - -class _EvaluationLogExtra(TypedDict): - request_id: str - duration_ms: float - action: str - finding_count: int - error_code: str | None - - -def _evaluation_log_extra( - *, - request_id: str, - started: float, - action: str, - finding_count: int, - failure: PrivacyGuardError | None, -) -> _EvaluationLogExtra: - return { - "request_id": request_id, - "duration_ms": round((time.monotonic() - started) * 1000, 3), - "action": action, - "finding_count": finding_count, - "error_code": failure.code.value if failure is not None else None, - } - - -def _request_id_for_logging(request_id: object) -> str: - try: - return validate_bounded_metadata_string(request_id) - except ValueError: - return _INVALID_REQUEST_ID - - -def _request_id_for_log_message(request_id: str) -> str: - return json.dumps(request_id, ensure_ascii=False).replace(" ", r"\u0020") - - -def _mapping_from_proto(config: Message) -> dict[str, object]: - try: - values: object = json_format.MessageToDict(config) - except Exception: - raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None - if not isinstance(values, dict) or any(not isinstance(key, str) for key in values): - raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) - return { - key: _normalize_proto_numbers(item) - for key, item in values.items() - if isinstance(key, str) - } - - -def _normalize_proto_numbers(value: object) -> object: - if isinstance(value, float): - if ( - math.isfinite(value) - and value.is_integer() - and -_MAX_PROTO_SAFE_INTEGER <= value <= _MAX_PROTO_SAFE_INTEGER - ): - return int(value) - return value - if isinstance(value, list): - return [_normalize_proto_numbers(item) for item in value] - if isinstance(value, dict): - return {key: _normalize_proto_numbers(item) for key, item in value.items()} - return value - - -def _validate_evaluation_envelope(request: pb2.HttpRequestEvaluation) -> None: - if request.config.ByteSize() > MAX_PROTO_CONFIG_BYTES: - raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) - if ( - request.context.ByteSize() > MAX_PROTO_CONTEXT_BYTES - or request.target.ByteSize() > MAX_PROTO_TARGET_BYTES - or len(request.headers) > MAX_PROTO_HEADERS - or _encoded_headers_size(request.headers) > MAX_PROTO_HEADERS_BYTES - ): - raise PrivacyGuardError(ErrorCode.REQUEST_ENVELOPE_INVALID) - - -def _encoded_headers_size(headers: Iterable[Message]) -> int: - total = 0 - for header in headers: - size = header.ByteSize() - total += 1 + _varint_size(size) + size - return total - - -def _varint_size(value: int) -> int: - size = 1 - while value >= 0x80: - value >>= 7 - size += 1 - return size - - -def _result_to_proto(result: RequestProcessingResult) -> pb2.HttpRequestResult: - findings: list[pb2.Finding] = [] - for detection in result.detection_summaries: - finding = _detection_to_proto(detection) - if finding.ByteSize() > MAX_PROTO_FINDING_BYTES: - return _limit_deny() - findings.append(finding) - if len(findings) > MAX_PROTO_FINDING_GROUPS: - return _limit_deny() - if result.decision is RequestDecision.ALLOW: - replacement = result.replacement_text - replacement_body = ( - replacement.encode("utf-8") if replacement is not None else b"" - ) - if len(replacement_body) > MAX_BODY_BYTES: - return _limit_deny() - return pb2.HttpRequestResult( - decision=pb2.DECISION_ALLOW, - body=replacement_body, - has_body=replacement is not None, - findings=findings, - ) - if result.decision is RequestDecision.DENY: - reason_code = result.reason_code or BLOCK_REASON_CODE - if REASON_CODE_PATTERN.fullmatch(reason_code) is None: - return _limit_deny() - return pb2.HttpRequestResult( - decision=pb2.DECISION_DENY, - reason=LIMIT_REASON if reason_code == LIMIT_REASON_CODE else BLOCK_REASON, - reason_code=reason_code, - findings=findings, - ) - raise PrivacyGuardError(ErrorCode.UNEXPECTED_SERVICE_FAILURE) - - -def _detection_to_proto(detection: EntityDetectionSummary) -> pb2.Finding: - confidence = detection.confidence - confidence_text = confidence.value if confidence is not None else "" - result = pb2.Finding( - type="detected_entity", - label=f"{detection.entity} ({detection.source_stage})", - confidence=confidence_text, - count=detection.count, - ) - return result - - -def _limit_deny() -> pb2.HttpRequestResult: - _LOGGER.info("privacy_guard_processing_limit kind=resource") - return pb2.HttpRequestResult( - decision=pb2.DECISION_DENY, - reason=LIMIT_REASON, - reason_code=LIMIT_REASON_CODE, - ) - - -_LOGGER = get_logger(__name__) -_INVALID_REQUEST_ID = "invalid" -_MAX_PROTO_SAFE_INTEGER = (1 << 53) - 1 - - -__all__ = ["PrivacyGuardMiddleware"] diff --git a/projects/privacy-guard/tests/__init__.py b/projects/privacy-guard/tests/__init__.py deleted file mode 100644 index 2017165c..00000000 --- a/projects/privacy-guard/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Privacy Guard test package.""" diff --git a/projects/privacy-guard/tests/engines/test_base.py b/projects/privacy-guard/tests/engines/test_base.py deleted file mode 100644 index 2dcae3d4..00000000 --- a/projects/privacy-guard/tests/engines/test_base.py +++ /dev/null @@ -1,323 +0,0 @@ -from __future__ import annotations - -from collections.abc import Iterator -from dataclasses import dataclass -from typing import Literal - -import pytest -from pydantic import ValidationError - -from privacy_guard.base import StrictDomainModel -from privacy_guard.constants import MAX_DETECTIONS_PER_STAGE -from privacy_guard.engines import ( - ConfidenceLevel, - EngineConfig, - EngineContractError, - EngineLimitExceededError, - EngineResources, - EntityDetection, - EntityProcessingEngine, - EntityProcessingStrategy, - TextProcessingResult, -) -from privacy_guard.timeout import Timeout - - -class _Replacement(StrictDomainModel): - strategy: Literal["token"] = "token" - - -class _Config(EngineConfig): - engine: Literal["test"] = "test" - replacement: _Replacement | None = None - - -@dataclass(frozen=True) -class _Resources(EngineResources): - prefix: str - - -class _CustomEngine(EntityProcessingEngine[_Config, _Resources]): - supported_strategies = frozenset( - { - EntityProcessingStrategy.DETECT, - EntityProcessingStrategy.REPLACE, - } - ) - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - detection = EntityDetection( - entity="token", - start=0, - end=len(text), - confidence=ConfidenceLevel.HIGH, - metadata={"provider": "custom"}, - ) - output = ( - f"{self.resources.prefix}token" - if strategy is EntityProcessingStrategy.REPLACE - else text - ) - return TextProcessingResult(text=output, detections=(detection,)) - - -def test_custom_engine_infers_types_and_needs_no_custom_init() -> None: - config = _Config(replacement=_Replacement()) - resources = _Resources(prefix="[") - - engine = _CustomEngine(config, resources) - - assert _CustomEngine.get_config_type() is _Config - assert _CustomEngine.get_resources_type() is _Resources - assert engine.config is config - assert engine.resources is resources - assert ( - engine.run( - "secret", - strategy=EntityProcessingStrategy.DETECT, - timeout=Timeout.from_seconds(1), - ).text - == "secret" - ) - assert ( - engine.run( - "secret", - strategy=EntityProcessingStrategy.REPLACE, - timeout=Timeout.from_seconds(1), - ).text - == "[token" - ) - - -def test_detection_confidence_and_metadata_are_strict_bounded_values() -> None: - categorical = EntityDetection.model_validate( - { - "entity": "email", - "start": 0, - "end": 1, - "confidence": "high", - "metadata": {"rule": "email.rules[0]"}, - } - ) - assert categorical.confidence is ConfidenceLevel.HIGH - assert type(categorical.metadata).__name__ == "mappingproxy" - with pytest.raises(ValidationError): - EntityDetection.model_validate( - { - "entity": "email", - "start": 0, - "end": 1, - "confidence": 0.25, - } - ) - - -@pytest.mark.parametrize( - "unsafe_value", - [ - "line\nbreak", - "ansi\x1b[31m", - "nul\x00byte", - "right-to-left\u202eoverride", - ], -) -def test_detection_rejects_non_printable_identifiers_and_metadata( - unsafe_value: str, -) -> None: - with pytest.raises(ValidationError): - EntityDetection( - entity=unsafe_value, - start=0, - end=1, - ) - with pytest.raises(ValidationError): - EntityDetection( - entity="token", - start=0, - end=1, - metadata={unsafe_value: "value"}, - ) - with pytest.raises(ValidationError): - EntityDetection( - entity="token", - start=0, - end=1, - metadata={"key": unsafe_value}, - ) - - -def test_detection_accepts_printable_unicode_identifiers_and_metadata() -> None: - detection = EntityDetection( - entity="客户资料", - start=0, - end=1, - metadata={"提供者": "自定义 🛡️"}, - ) - - assert detection.entity == "客户资料" - assert detection.metadata == {"提供者": "自定义 🛡️"} - - -def test_processing_result_bounds_a_lazy_detection_stream() -> None: - produced = 0 - - def detections() -> Iterator[EntityDetection]: - nonlocal produced - for index in range(1_000): - produced += 1 - yield EntityDetection(entity="token", start=index, end=index + 1) - - with pytest.raises(EngineLimitExceededError): - TextProcessingResult.from_detections( - text="x" * 1_000, - detections=detections(), - ) - - assert produced == 257 - - -class _OversizedResultEngine(EntityProcessingEngine[_Config]): - supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - del strategy, timeout - return TextProcessingResult( - text=text, - detections=tuple( - EntityDetection(entity="token", start=0, end=1) - for _ in range(MAX_DETECTIONS_PER_STAGE + 1) - ), - ) - - -def test_engine_boundary_bounds_results_built_without_lazy_helper() -> None: - engine = _OversizedResultEngine(_Config(), None) - - with pytest.raises(EngineLimitExceededError): - engine.run( - "text", - strategy=EntityProcessingStrategy.DETECT, - timeout=Timeout.from_seconds(1), - ) - - -class _DetectOnlyEngine(EntityProcessingEngine[_Config]): - supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - return TextProcessingResult(text=text, detections=()) - - -def test_detect_only_engine_rejects_replacement_before_running() -> None: - engine = _DetectOnlyEngine(_Config(), None) - - assert _DetectOnlyEngine.get_resources_type() is None - assert engine.resources is None - with pytest.raises(EngineContractError): - engine.run( - "text", - strategy=EntityProcessingStrategy.REPLACE, - timeout=Timeout.from_seconds(1), - ) - - -class _ReplaceOnlyEngine(EntityProcessingEngine[_Config]): - supported_strategies = frozenset({EntityProcessingStrategy.REPLACE}) - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - del strategy, timeout - return TextProcessingResult(text=text, detections=()) - - -def test_replace_only_engine_rejects_detection_before_running() -> None: - engine = _ReplaceOnlyEngine(_Config(replacement=_Replacement()), None) - - with pytest.raises(EngineContractError): - engine.run( - "text", - strategy=EntityProcessingStrategy.DETECT, - timeout=Timeout.from_seconds(1), - ) - - -class _MutatingDetectEngine(EntityProcessingEngine[_Config]): - supported_strategies = frozenset( - { - EntityProcessingStrategy.DETECT, - EntityProcessingStrategy.REPLACE, - } - ) - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - return TextProcessingResult( - text="changed", - detections=(EntityDetection(entity="token", start=0, end=len(text)),), - ) - - -def test_detection_strategy_rejects_mutated_text() -> None: - engine = _MutatingDetectEngine(_Config(), None) - - with pytest.raises(EngineContractError): - engine.run( - "text", - strategy=EntityProcessingStrategy.DETECT, - timeout=Timeout.from_seconds(1), - ) - - -class _InvalidSpanEngine(EntityProcessingEngine[_Config]): - supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - return TextProcessingResult( - text=text, - detections=(EntityDetection(entity="token", start=0, end=len(text) + 1),), - ) - - -def test_engine_boundary_rejects_spans_outside_stage_input() -> None: - engine = _InvalidSpanEngine(_Config(), None) - - with pytest.raises(EngineContractError): - engine.run( - "text", - strategy=EntityProcessingStrategy.DETECT, - timeout=Timeout.from_seconds(1), - ) diff --git a/projects/privacy-guard/tests/engines/test_regex.py b/projects/privacy-guard/tests/engines/test_regex.py deleted file mode 100644 index 54ddd3f6..00000000 --- a/projects/privacy-guard/tests/engines/test_regex.py +++ /dev/null @@ -1,476 +0,0 @@ -from __future__ import annotations - -import logging -from concurrent.futures import ThreadPoolExecutor -from threading import Barrier - -import pytest -from pydantic import ValidationError - -import privacy_guard.engines.regex as regex_module -from privacy_guard.engines import ( - EngineConfigurationError, - EngineLimitExceededError, - EntityProcessingStrategy, - RegexEngine, - RegexEngineConfig, - RegexPatternCatalog, -) -from privacy_guard.errors import TimeoutExpiredError -from privacy_guard.timeout import Timeout - - -def _config( - rules: list[dict[str, object]], - *, - replacement: dict[str, object] | None = None, -) -> RegexEngineConfig: - values: dict[str, object] = { - "engine": "regex", - "pattern_catalog": { - "entities": [ - { - "name": "token", - "rules": rules, - } - ] - }, - } - if replacement is not None: - values["replacement"] = replacement - return RegexEngineConfig.model_validate(values) - - -def _run( - config: RegexEngineConfig, - text: str, - strategy: EntityProcessingStrategy = EntityProcessingStrategy.DETECT, -) -> tuple[str, list[tuple[str, int, int, str]]]: - result = RegexEngine(config, None).run( - text, - strategy=strategy, - timeout=Timeout.from_seconds(1), - ) - return result.text, [ - ( - detection.entity, - detection.start, - detection.end, - detection.metadata["rule"], - ) - for detection in result.detections - ] - - -def _catalog(pattern: str) -> RegexPatternCatalog: - return RegexPatternCatalog.model_validate( - { - "entities": [ - { - "name": "token", - "rules": [ - { - "pattern": pattern, - "confidence": "high", - } - ], - } - ] - } - ) - - -def test_detects_overlaps_and_orders_matches_deterministically() -> None: - config = _config( - [ - {"name": "pair", "pattern": "aa", "confidence": "high"}, - {"name": "suffix", "pattern": "a$", "confidence": "medium"}, - ] - ) - - output, detections = _run(config, "aaa") - - assert output == "aaa" - assert detections == [ - ("token", 0, 2, "pair"), - ("token", 1, 3, "pair"), - ("token", 2, 3, "suffix"), - ] - - -def test_optional_names_derive_identity_without_affecting_internal_marker() -> None: - config = _config( - [ - {"name": "same-name", "pattern": "x", "confidence": "high"}, - {"name": "same_name", "pattern": "y", "confidence": "high"}, - {"pattern": "z", "confidence": "high"}, - ] - ) - - _, detections = _run(config, "xyz") - - assert [item[3] for item in detections] == [ - "same-name", - "same_name", - "token.rules[2]", - ] - - -def test_numeric_backreferences_keep_original_group_numbers() -> None: - config = _config([{"pattern": r"(a)\1", "confidence": "high"}]) - - _, detections = _run(config, "aa") - - assert [(item[1], item[2]) for item in detections] == [(0, 2)] - - -def test_explicit_flags_are_supported() -> None: - config = _config( - [ - { - "pattern": "^x.$", - "confidence": "high", - "ignore_case": True, - "multiline": True, - "dot_all": True, - "ascii": True, - } - ] - ) - - _, detections = _run(config, "X\n") - - assert [(item[1], item[2]) for item in detections] == [(0, 2)] - - -@pytest.mark.parametrize( - "pattern", - [ - "", - "x*", - "(?Px)", - "(?i:x)", - ], -) -def test_invalid_patterns_are_rejected_content_safely(pattern: str) -> None: - with pytest.raises(ValidationError) as exception_info: - _config([{"pattern": pattern, "confidence": "high"}]) - - if pattern: - assert pattern not in str(exception_info.value) - - -@pytest.mark.parametrize( - ("pattern", "text"), - [ - ("x|(?=SECRET-zero-width-493)", "SECRET-zero-width-493"), - ("(?=secret)", "secret"), - ("(?<=prefix)", "prefix"), - (r"\b", "secret"), - ("x|(?:y|(?=secret))", "secret"), - ], -) -def test_contextual_zero_width_match_is_invalid_configuration_at_runtime( - pattern: str, - text: str, -) -> None: - config = _config([{"pattern": pattern, "confidence": "high"}]) - engine = RegexEngine(config, None) - - with pytest.raises( - EngineConfigurationError, - match="regex engine configuration is invalid", - ) as exception_info: - engine.run( - text, - strategy=EntityProcessingStrategy.DETECT, - timeout=Timeout.from_seconds(1), - ) - - assert pattern not in str(exception_info.value) - - -@pytest.mark.parametrize( - ("pattern", "text", "expected_span"), - [ - ("(?<=prefix)secret(?=suffix)", "prefixsecretsuffix", (6, 12)), - (r"\bsecret\b", "a secret value", (2, 8)), - ( - r"(? None: - config = _config([{"pattern": pattern, "confidence": "high"}]) - - _, detections = _run(config, text) - - assert [(item[1], item[2]) for item in detections] == [expected_span] - - -def test_duplicate_supplied_names_are_rejected_but_unnamed_rules_are_not() -> None: - with pytest.raises(ValidationError): - _config( - [ - {"name": "duplicate", "pattern": "x", "confidence": "high"}, - {"name": "duplicate", "pattern": "y", "confidence": "high"}, - ] - ) - - config = _config( - [ - {"pattern": "x", "confidence": "high"}, - {"pattern": "y", "confidence": "high"}, - ] - ) - assert len(config.pattern_catalog.entities[0].rules) == 2 - - -def test_replacement_selects_ranked_non_overlapping_winners() -> None: - config = _config( - [ - {"name": "long-low", "pattern": "abc", "confidence": "low"}, - {"name": "short-high", "pattern": "bc", "confidence": "high"}, - ], - replacement={"strategy": "template", "template": "<{entity}>"}, - ) - - output, detections = _run( - config, - "abc", - EntityProcessingStrategy.REPLACE, - ) - - assert output == "a" - assert len(detections) == 2 - - -def test_replacement_requires_an_engine_specific_recipe() -> None: - config = _config([{"pattern": "x", "confidence": "high"}]) - - with pytest.raises(EngineConfigurationError): - _run(config, "x", EntityProcessingStrategy.REPLACE) - - -@pytest.mark.parametrize( - "replacement", - [ - {"strategy": "template", "template": "{unknown}"}, - {"strategy": "template", "template": "{entity.attr}"}, - {"strategy": "template", "template": "{entity!r}"}, - {"strategy": "template", "template": "{entity:>10}"}, - {"strategy": "template", "template": "{"}, - ], -) -def test_template_language_allows_only_literal_text_and_entity( - replacement: dict[str, object], -) -> None: - with pytest.raises(ValidationError): - _config( - [{"pattern": "x", "confidence": "high"}], - replacement=replacement, - ) - - -def test_replacement_size_is_projected_before_rendering( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(regex_module, "MAX_BODY_BYTES", 4) - config = _config( - [{"pattern": "x", "confidence": "high"}], - replacement={"strategy": "template", "template": "[{entity}]"}, - ) - - with pytest.raises(EngineLimitExceededError): - _run(config, "x", EntityProcessingStrategy.REPLACE) - - -def test_pattern_search_has_an_enforceable_timeout() -> None: - config = _config([{"pattern": "(a+)+$", "confidence": "high"}]) - engine = RegexEngine(config, None) - - with pytest.raises(TimeoutExpiredError): - engine.run( - "a" * 100_000 + "!", - strategy=EntityProcessingStrategy.DETECT, - timeout=Timeout.from_seconds(0.001), - ) - - -def test_patterns_compile_during_validation_and_preparation_not_run( - monkeypatch: pytest.MonkeyPatch, -) -> None: - regex_module._clear_compiled_pattern_cache() - compile_count = 0 - original_compile = regex_module.regex.compile - - def recording_compile(pattern: str, flags: int = 0) -> object: - nonlocal compile_count - compile_count += 1 - return original_compile(pattern, flags) - - monkeypatch.setattr(regex_module.regex, "compile", recording_compile) - config = _config([{"pattern": "x", "confidence": "high"}]) - engine = RegexEngine(config, None) - prepared_count = compile_count - - engine.run( - "x", - strategy=EntityProcessingStrategy.DETECT, - timeout=Timeout.from_seconds(1), - ) - - assert prepared_count > 0 - assert compile_count == prepared_count - - -def test_compiled_catalog_cache_evicts_least_recently_used_entry( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - regex_module._clear_compiled_pattern_cache() - catalogs = tuple(_catalog(f"sensitive-pattern-{suffix}") for suffix in "abc") - - try: - first_rules = regex_module._compile_pattern_catalog(catalogs[0]) - entry_weight = regex_module._COMPILED_PATTERN_CACHE[catalogs[0]][1] - monkeypatch.setattr( - regex_module, - "MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES", - entry_weight * 2, - ) - with caplog.at_level(logging.DEBUG, logger="privacy_guard.engines.regex"): - regex_module._compile_pattern_catalog(catalogs[1]) - assert regex_module._compile_pattern_catalog(catalogs[0]) is first_rules - regex_module._compile_pattern_catalog(catalogs[2]) - - assert tuple(regex_module._COMPILED_PATTERN_CACHE) == ( - catalogs[0], - catalogs[2], - ) - assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES == sum( - entry[1] for entry in regex_module._COMPILED_PATTERN_CACHE.values() - ) - assert ( - "privacy_guard_cache_eviction cache=regex_compiled entries=1" in caplog.text - ) - assert "sensitive-pattern" not in caplog.text - finally: - regex_module._clear_compiled_pattern_cache() - - -def test_compiled_catalog_cache_skips_oversized_valid_entry( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - regex_module._clear_compiled_pattern_cache() - monkeypatch.setattr(regex_module, "MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES", 1) - catalog = _catalog("sensitive-oversized-pattern") - - try: - with caplog.at_level(logging.DEBUG, logger="privacy_guard.engines.regex"): - first = regex_module._compile_pattern_catalog(catalog) - second = regex_module._compile_pattern_catalog(catalog) - - assert first is not second - assert regex_module._COMPILED_PATTERN_CACHE == {} - assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES == 0 - assert caplog.text.count("privacy_guard_cache_skip cache=regex_compiled") == 2 - assert "sensitive-oversized-pattern" not in caplog.text - finally: - regex_module._clear_compiled_pattern_cache() - - -def test_compiled_catalog_failure_preserves_existing_weight( - monkeypatch: pytest.MonkeyPatch, -) -> None: - regex_module._clear_compiled_pattern_cache() - retained_catalog = _catalog("retained") - regex_module._compile_pattern_catalog(retained_catalog) - retained_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES - retained_entries = tuple(regex_module._COMPILED_PATTERN_CACHE) - - def fail_compile(*args: object, **kwargs: object) -> object: - del args, kwargs - raise ValueError("expected test failure") - - monkeypatch.setattr(regex_module, "_compile_rule", fail_compile) - try: - with pytest.raises(ValueError, match="expected test failure"): - regex_module._compile_pattern_catalog(_catalog("failing")) - - assert tuple(regex_module._COMPILED_PATTERN_CACHE) == retained_entries - assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES == retained_weight - finally: - regex_module._clear_compiled_pattern_cache() - - -def test_compiled_catalog_same_key_race_accounts_once( - monkeypatch: pytest.MonkeyPatch, -) -> None: - regex_module._clear_compiled_pattern_cache() - worker_count = 4 - workers_ready = Barrier(worker_count) - catalog = _catalog("same-key") - original_compile_rule = regex_module._compile_rule - - def synchronized_compile( - entity: regex_module.RegexEntity, - rule: regex_module.RegexRule, - catalog_index: int, - entity_rule_index: int, - ) -> regex_module._CompiledRule: - workers_ready.wait(timeout=5) - return original_compile_rule( - entity, - rule, - catalog_index, - entity_rule_index, - ) - - monkeypatch.setattr(regex_module, "_compile_rule", synchronized_compile) - try: - with ThreadPoolExecutor(max_workers=worker_count) as executor: - results = tuple( - executor.map( - lambda _: regex_module._compile_pattern_catalog(catalog), - range(worker_count), - ) - ) - - assert all(result is results[0] for result in results) - assert len(regex_module._COMPILED_PATTERN_CACHE) == 1 - assert ( - regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES - == (next(iter(regex_module._COMPILED_PATTERN_CACHE.values()))[1]) - ) - finally: - regex_module._clear_compiled_pattern_cache() - - -def test_regex_engine_is_safe_for_concurrent_runs() -> None: - engine = RegexEngine( - _config([{"pattern": "x", "confidence": "high"}]), - None, - ) - - def run(text: str) -> int: - return len( - engine.run( - text, - strategy=EntityProcessingStrategy.DETECT, - timeout=Timeout.from_seconds(1), - ).detections - ) - - with ThreadPoolExecutor(max_workers=4) as executor: - counts = tuple(executor.map(run, ("x",) * 16)) - - assert counts == (1,) * 16 diff --git a/projects/privacy-guard/tests/engines/test_registry.py b/projects/privacy-guard/tests/engines/test_registry.py deleted file mode 100644 index fc600471..00000000 --- a/projects/privacy-guard/tests/engines/test_registry.py +++ /dev/null @@ -1,398 +0,0 @@ -"""Tests for entity-processing engine registration and schema finalization.""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Literal - -import pytest -from pydantic import field_validator - -from privacy_guard.base import StrictDomainModel -from privacy_guard.engines import ( - EngineConfig, - EngineConfigurationError, - EngineResources, - EntityProcessingEngine, - EntityProcessingStrategy, - RegexEngine, - TextProcessingResult, -) -from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry -from privacy_guard.errors import EngineRegistryError, PrivacyGuardError -from privacy_guard.timeout import Timeout - - -class AcmeReplacement(StrictDomainModel): - strategy: Literal["token"] = "token" - - -class AcmeConfig(EngineConfig): - engine: Literal["acme-pii"] = "acme-pii" - entities: tuple[str, ...] - replacement: AcmeReplacement | None = None - - @field_validator("entities", mode="before") - @classmethod - def _entities_are_a_tuple(cls, value: object) -> object: - if not isinstance(value, list | tuple): - raise ValueError("entities must be a list") - return tuple(value) - - -@dataclass(frozen=True) -class AcmeResources(EngineResources): - prefix: str - - -class AcmeEngine(EntityProcessingEngine[AcmeConfig, AcmeResources]): - supported_strategies = frozenset( - { - EntityProcessingStrategy.DETECT, - EntityProcessingStrategy.REPLACE, - } - ) - - @classmethod - def _validate_run_config( - cls, - config: AcmeConfig, - resources: AcmeResources, - *, - strategy: EntityProcessingStrategy, - ) -> None: - del cls, resources - if strategy is EntityProcessingStrategy.REPLACE and config.replacement is None: - raise EngineConfigurationError("acme replacement configuration is required") - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - del strategy, timeout - return TextProcessingResult(text=text, detections=()) - - -class DetectConfig(EngineConfig): - engine: Literal["detect-only"] = "detect-only" - - -class DetectEngine(EntityProcessingEngine[DetectConfig]): - supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - del strategy, timeout - return TextProcessingResult(text=text, detections=()) - - -def _acme_values(*, action: str = "detect") -> dict[str, object]: - return { - "entity_processing": { - "stages": [ - { - "config": { - "engine": "acme-pii", - "entities": ["account"], - "replacement": {"strategy": "token"}, - } - } - ] - }, - "on_detection": {"action": action}, - } - - -def test_builtin_registry_contains_the_builtin_regex_engine() -> None: - registry = create_builtin_registry() - - assert registry.is_finalized is True - descriptions = registry.describe_engines() - assert tuple(item.engine_name for item in descriptions) == ("regex",) - description = descriptions[0] - assert description.engine_name == "regex" - assert description.supported_strategies == frozenset( - { - EntityProcessingStrategy.DETECT, - EntityProcessingStrategy.REPLACE, - } - ) - - -def test_registry_can_include_builtin_engines_before_custom_registration() -> None: - registry = EngineRegistry(include_builtin_engines=True) - registry.register(AcmeEngine, resources=AcmeResources(prefix="token")) - registry.finalize() - - assert tuple(item.engine_name for item in registry.describe_engines()) == ( - "regex", - "acme-pii", - ) - - -def test_custom_engine_config_joins_the_exact_discriminated_union() -> None: - resources = AcmeResources(prefix="token") - registry = EngineRegistry(include_builtin_engines=True) - registry.register(AcmeEngine, resources=resources) - registry.finalize() - - config = registry.validate_config(_acme_values(action="replace")) - engine = registry.create_engine(config.entity_processing.stages[0].config) - - assert type(config.entity_processing.stages[0].config) is AcmeConfig - assert type(engine) is AcmeEngine - assert engine.config is config.entity_processing.stages[0].config - assert engine.resources is resources - assert tuple(item.engine_name for item in registry.describe_engines()) == ( - "regex", - "acme-pii", - ) - - -def test_detection_only_engine_is_rejected_for_replace_action() -> None: - registry = EngineRegistry() - registry.register(DetectEngine) - registry.finalize() - values = { - "entity_processing": {"stages": [{"config": {"engine": "detect-only"}}]}, - "on_detection": {"action": "replace"}, - } - - with pytest.raises(PrivacyGuardError): - registry.validate_config(values) - - -def test_engine_owns_strategy_specific_configuration_requirements() -> None: - registry = EngineRegistry() - registry.register(AcmeEngine, resources=AcmeResources(prefix="token")) - registry.finalize() - values = { - "entity_processing": { - "stages": [ - { - "config": { - "engine": "acme-pii", - "entities": ["account"], - } - } - ] - }, - "on_detection": {"action": "replace"}, - } - - with pytest.raises(PrivacyGuardError): - registry.validate_config(values) - - -class ReplaceOnlyConfig(EngineConfig): - engine: Literal["replace-only"] = "replace-only" - - -class ReplaceOnlyEngine(EntityProcessingEngine[ReplaceOnlyConfig]): - supported_strategies = frozenset({EntityProcessingStrategy.REPLACE}) - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - del strategy, timeout - return TextProcessingResult(text=text, detections=()) - - -def test_replacement_only_engine_is_rejected_for_detect_action() -> None: - registry = EngineRegistry() - registry.register(ReplaceOnlyEngine) - registry.finalize() - values = { - "entity_processing": { - "stages": [ - { - "config": { - "engine": "replace-only", - } - } - ] - }, - "on_detection": {"action": "detect"}, - } - - with pytest.raises(PrivacyGuardError): - registry.validate_config(values) - - values["on_detection"] = {"action": "replace"} - config = registry.validate_config(values) - - config_type = type(config.entity_processing.stages[0].config) - assert "replacement" not in config_type.model_fields - - -def test_registry_is_frozen_after_finalize_and_finalize_is_idempotent() -> None: - registry = EngineRegistry() - registry.register(RegexEngine) - - assert registry.finalize() is registry - assert registry.finalize() is registry - with pytest.raises(EngineRegistryError): - registry.register(DetectEngine) - - -def test_registry_rejects_duplicate_discriminators_and_resource_mismatch() -> None: - registry = EngineRegistry() - registry.register(AcmeEngine, resources=AcmeResources(prefix="token")) - - with pytest.raises(EngineRegistryError): - registry.register(AcmeEngine, resources=AcmeResources(prefix="other")) - with pytest.raises(EngineRegistryError): - EngineRegistry().register(AcmeEngine) - with pytest.raises(EngineRegistryError, match="must extend EngineResources"): - EngineRegistry().register(AcmeEngine, resources=object()) - with pytest.raises(EngineRegistryError): - EngineRegistry().register(DetectEngine, resources=object()) - - -def _run_without_the_engine_wrapper( - self: DetectEngine, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, -) -> TextProcessingResult: - del self, strategy, timeout - return TextProcessingResult(text=text, detections=()) - - -def _initialize_without_the_engine_constructor( - self: DetectEngine, - config: DetectConfig, - resources: None, -) -> None: - del self, config, resources - - -@pytest.mark.parametrize( - ("method_name", "method", "expected_error"), - [ - ( - "run", - _run_without_the_engine_wrapper, - "engine lifecycle contract requires EntityProcessingEngine.run; " - "implement _run() instead", - ), - ( - "__init__", - _initialize_without_the_engine_constructor, - "engine lifecycle contract requires EntityProcessingEngine.__init__; " - "use _initialize() instead", - ), - ], -) -def test_registry_rejects_direct_and_inherited_lifecycle_overrides( - method_name: str, - method: object, - expected_error: str, -) -> None: - direct_override = type( - "LifecycleOverrideEngine", - (DetectEngine,), - {method_name: method}, - ) - inherited_override = type( - "InheritedOverrideEngine", - (direct_override,), - {}, - ) - - for engine_type in (direct_override, inherited_override): - with pytest.raises(EngineRegistryError) as error: - EngineRegistry().register(engine_type) - - assert str(error.value) == expected_error - - -def test_base_lifecycle_methods_are_final_for_static_feedback() -> None: - assert getattr(EntityProcessingEngine.__init__, "__final__", False) is True - assert getattr(EntityProcessingEngine.run, "__final__", False) is True - - -def test_registry_accepts_base_lifecycle_inherited_through_custom_base() -> None: - intermediate_base = type( - "ValidIntermediateEngineBase", - (DetectEngine,), - {}, - ) - inherited_lifecycle_engine = type( - "InheritedLifecycleEngine", - (intermediate_base,), - {}, - ) - - registry = EngineRegistry() - registry.register(inherited_lifecycle_engine) - - assert inherited_lifecycle_engine.__init__ is EntityProcessingEngine.__init__ - assert inherited_lifecycle_engine.run is EntityProcessingEngine.run - - -@pytest.mark.parametrize( - ("engine_type", "resources"), - [ - (DetectEngine, None), - (AcmeEngine, AcmeResources(prefix="token")), - ], -) -def test_registry_accepts_engines_using_the_base_lifecycle( - engine_type: type[object], - resources: object, -) -> None: - registry = EngineRegistry() - - registry.register(engine_type, resources=resources) - - assert registry.finalize().is_finalized is True - - -def test_describe_does_not_construct_an_engine() -> None: - class CountingEngine(EntityProcessingEngine[DetectConfig]): - supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) - initialized = 0 - - def _initialize(self) -> None: - type(self).initialized += 1 - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - del strategy, timeout - return TextProcessingResult(text=text, detections=()) - - registry = EngineRegistry() - registry.register(CountingEngine) - registry.finalize() - - descriptions = registry.describe_engines() - - assert CountingEngine.initialized == 0 - assert descriptions[0].engine_name == "detect-only" - assert descriptions[0].supported_strategies == frozenset( - {EntityProcessingStrategy.DETECT} - ) - - -def test_registry_requires_at_least_one_engine() -> None: - with pytest.raises(EngineRegistryError): - EngineRegistry().finalize() diff --git a/projects/privacy-guard/tests/examples/test_custom_engine.py b/projects/privacy-guard/tests/examples/test_custom_engine.py deleted file mode 100644 index 0391b373..00000000 --- a/projects/privacy-guard/tests/examples/test_custom_engine.py +++ /dev/null @@ -1,147 +0,0 @@ -"""End-to-end checks for the custom engine application example.""" - -from __future__ import annotations - -import json -import os -import subprocess -import sys -from pathlib import Path - -import yaml - -EXAMPLE_DIRECTORY = Path(__file__).parents[2] / "examples" / "custom-engine" - - -def test_custom_engine_runs_through_the_middleware_boundary() -> None: - probe = r""" -import asyncio -from pathlib import Path - -from google.protobuf import json_format -import yaml - -from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 -from privacy_guard.service.servicer import PrivacyGuardMiddleware -from custom_engine import create_registry - -values = yaml.safe_load(Path("privacy-guard-config.yaml").read_text()) -assert isinstance(values, dict) -config = pb2.HttpRequestEvaluation().config -json_format.ParseDict(values, config) - - -async def evaluate() -> None: - middleware = PrivacyGuardMiddleware(create_registry()) - try: - result = await middleware._evaluate_http_request( - pb2.HttpRequestEvaluation( - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, - config=config, - body=b"Discuss Project Cobalt safely.", - ) - ) - finally: - await middleware.close() - - assert result.decision == pb2.DECISION_ALLOW - assert result.has_body is False - assert result.body == b"" - assert len(result.findings) == 1 - assert result.findings[0].label == ( - "confidential-project (project-names)" - ) - - -asyncio.run(evaluate()) -""" - - subprocess.run( - [sys.executable, "-c", probe], - cwd=EXAMPLE_DIRECTORY, - check=True, - ) - - -def test_custom_registry_drives_cli_discovery_and_schema() -> None: - environment = os.environ.copy() - python_path = str(EXAMPLE_DIRECTORY) - existing_python_path = environment.get("PYTHONPATH") - if existing_python_path: - python_path = os.pathsep.join((python_path, existing_python_path)) - environment["PYTHONPATH"] = python_path - command = [ - str(Path(sys.executable).with_name("privacy-guard")), - "--registry-factory", - "custom_engine:create_registry", - ] - - engines = subprocess.run( - [*command, "engines"], - cwd=EXAMPLE_DIRECTORY, - check=True, - capture_output=True, - text=True, - env=environment, - ) - schema = subprocess.run( - [*command, "configuration-schema"], - cwd=EXAMPLE_DIRECTORY, - check=True, - capture_output=True, - text=True, - env=environment, - ) - - assert engines.stdout.startswith("regex\tdetect,replace\t") - assert "keyword-tool\tdetect\t" in engines.stdout - serialized_schema = json.loads(schema.stdout) - assert "RegexEngineConfig" in serialized_schema["$defs"] - assert "KeywordEngineConfig" in serialized_schema["$defs"] - keyword_properties = serialized_schema["$defs"]["KeywordEngineConfig"]["properties"] - assert set(keyword_properties) == { - "engine", - "entity", - "keyword", - } - - -def test_openshell_walkthrough_uses_the_custom_registry_and_current_policy() -> None: - policy = yaml.safe_load((EXAMPLE_DIRECTORY / "policy.yaml").read_text()) - config = yaml.safe_load( - (EXAMPLE_DIRECTORY / "privacy-guard-config.yaml").read_text() - ) - readme = (EXAMPLE_DIRECTORY / "README.md").read_text() - implementation = (EXAMPLE_DIRECTORY / "custom_engine.py").read_text() - - assert isinstance(policy, dict) - assert isinstance(config, dict) - assert not (EXAMPLE_DIRECTORY / "privacy_guard_app.py").exists() - assert "EngineRegistry(include_builtin_engines=True)" in implementation - assert "def create_registry() -> EngineRegistry:" in implementation - middleware_config = policy["network_middlewares"]["privacy_guard_detect"] - assert middleware_config["middleware"] == "privacy-guard-custom-engine" - assert middleware_config["config"] == config - stage_config = config["entity_processing"]["stages"][0]["config"] - assert stage_config["engine"] == "keyword-tool" - assert config["on_detection"]["action"] == "detect" - assert "--registry-factory custom_engine:create_registry" in readme - assert "cd projects/privacy-guard/examples/custom-engine" in readme - assert "uv sync --locked" not in readme - assert "uv run --locked privacy-guard" in readme - assert 'export PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}"' in readme - assert "uv run privacy-guard add-gateway-registration" in readme - assert "uv run privacy-guard remove-gateway-registration" in readme - assert "--host-ip YOUR_HOST_IPV4" in readme - assert "--name privacy-guard-custom-engine" in readme - assert "--config" not in readme - assert "brew services stop openshell" in readme - assert "brew services start openshell" in readme - assert "systemctl --user stop openshell-gateway" in readme - assert "systemctl --user start openshell-gateway" in readme - assert "openshell-gateway --config" not in readme - assert 'sed "s/REPLACE_WITH_HOST_IP/' not in readme - assert not (EXAMPLE_DIRECTORY / "gateway.toml").exists() - assert "openshell gateway add" not in readme - assert "OpenShell `v0.0.90`" in readme - assert "transformed:false" in readme diff --git a/projects/privacy-guard/tests/examples/test_regex_engine.py b/projects/privacy-guard/tests/examples/test_regex_engine.py deleted file mode 100644 index a235ea4f..00000000 --- a/projects/privacy-guard/tests/examples/test_regex_engine.py +++ /dev/null @@ -1,118 +0,0 @@ -"""End-to-end checks for the built-in RegexEngine example.""" - -from __future__ import annotations - -import asyncio -import json -import subprocess -import sys -from pathlib import Path - -import pytest -import yaml -from google.protobuf import json_format - -from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 -from privacy_guard.engines import RegexPatternCatalog -from privacy_guard.engines.registry import create_builtin_registry -from privacy_guard.service.servicer import PrivacyGuardMiddleware - -EXAMPLE_DIRECTORY = Path(__file__).parents[2] / "examples" / "regex-engine" - - -def test_regex_example_runs_through_the_middleware_boundary( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.chdir(EXAMPLE_DIRECTORY) - values = yaml.safe_load( - (EXAMPLE_DIRECTORY / "privacy-guard-config.yaml").read_text() - ) - assert isinstance(values, dict) - config = pb2.HttpRequestEvaluation().config - json_format.ParseDict(values, config) - - async def evaluate() -> None: - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - try: - result = await middleware._evaluate_http_request( - pb2.HttpRequestEvaluation( - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, - config=config, - body=(b"Contact user@example.com about customer CUST-12345678."), - ) - ) - finally: - await middleware.close() - - assert result.decision == pb2.DECISION_ALLOW - assert result.has_body is True - assert result.body == (b"Contact [email] about customer [customer-id].") - assert {finding.label for finding in result.findings} == { - "email (identifiers)", - "customer-id (identifiers)", - } - - asyncio.run(evaluate()) - - -def test_builtin_registry_drives_documented_cli_discovery_and_schema() -> None: - command = str(Path(sys.executable).with_name("privacy-guard")) - engines = subprocess.run( - [command, "engines"], - cwd=EXAMPLE_DIRECTORY, - check=True, - capture_output=True, - text=True, - ) - schema = subprocess.run( - [command, "configuration-schema"], - cwd=EXAMPLE_DIRECTORY, - check=True, - capture_output=True, - text=True, - ) - - assert engines.stdout.startswith("regex\tdetect,replace\t") - serialized_schema = json.loads(schema.stdout) - assert "RegexEngineConfig" in serialized_schema["$defs"] - assert "RegexPatternCatalog" in serialized_schema["$defs"] - assert "RegexRule" in serialized_schema["$defs"] - assert "RegexReplacement" in serialized_schema["$defs"] - - -def test_regex_walkthrough_uses_current_policy_and_gateway_schema() -> None: - policy = yaml.safe_load((EXAMPLE_DIRECTORY / "policy.yaml").read_text()) - config = yaml.safe_load( - (EXAMPLE_DIRECTORY / "privacy-guard-config.yaml").read_text() - ) - catalog = yaml.safe_load((EXAMPLE_DIRECTORY / "patterns.yaml").read_text()) - readme = (EXAMPLE_DIRECTORY / "README.md").read_text() - - assert isinstance(policy, dict) - assert isinstance(config, dict) - assert isinstance(catalog, dict) - middleware_config = policy["network_middlewares"]["privacy_guard_replace"] - assert middleware_config["middleware"] == "privacy-guard-regex" - assert middleware_config["config"] == config - assert config["on_detection"]["action"] == "replace" - stage_config = config["entity_processing"]["stages"][0]["config"] - assert stage_config["engine"] == "regex" - assert stage_config["pattern_catalog"] == "patterns.yaml" - RegexPatternCatalog.model_validate(catalog) - assert "uv sync --locked" not in readme - assert "uv run --locked privacy-guard serve --listen 0.0.0.0:50051" in readme - assert "uv run privacy-guard add-gateway-registration" in readme - assert "uv run privacy-guard remove-gateway-registration" in readme - assert "--host-ip YOUR_HOST_IPV4" in readme - assert "--name privacy-guard-regex" in readme - assert "--config" not in readme - assert "brew services stop openshell" in readme - assert "brew services start openshell" in readme - assert "systemctl --user stop openshell-gateway" in readme - assert "systemctl --user start openshell-gateway" in readme - assert "openshell-gateway --config" not in readme - assert 'sed "s/REPLACE_WITH_HOST_IP/' not in readme - assert not (EXAMPLE_DIRECTORY / "gateway.toml").exists() - assert "openshell gateway add" not in readme - assert "OpenShell `v0.0.90`" in readme - assert "transformed:true" in readme diff --git a/projects/privacy-guard/tests/service/__init__.py b/projects/privacy-guard/tests/service/__init__.py deleted file mode 100644 index c8634e6b..00000000 --- a/projects/privacy-guard/tests/service/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Privacy Guard service tests.""" diff --git a/projects/privacy-guard/tests/service/test_grpc_integration.py b/projects/privacy-guard/tests/service/test_grpc_integration.py deleted file mode 100644 index 4113c1ea..00000000 --- a/projects/privacy-guard/tests/service/test_grpc_integration.py +++ /dev/null @@ -1,328 +0,0 @@ -"""Real loopback coverage for the generated OpenShell gRPC service.""" - -from __future__ import annotations - -from collections.abc import AsyncIterator -from contextlib import asynccontextmanager -from typing import Literal - -import grpc -import pytest -from google.protobuf import empty_pb2, json_format, message_factory -from google.protobuf.message import Message -from pydantic import field_validator - -from privacy_guard.base import StrictDomainModel -from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 -from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc -from privacy_guard.engines import ( - EngineConfig, - EntityProcessingEngine, - EntityProcessingStrategy, - TextProcessingResult, -) -from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry -from privacy_guard.errors import PrivacyGuardError -from privacy_guard.service.servicer import PrivacyGuardMiddleware -from privacy_guard.timeout import Timeout - - -def _config( - *, - action: str = "replace", - pattern: str = r"[a-z]+@[a-z]+\.[a-z]+", -) -> pb2.ValidateConfigRequest: - request = pb2.ValidateConfigRequest() - json_format.ParseDict( - { - "entity_processing": { - "stages": [ - { - "name": "identifiers", - "config": { - "engine": "regex", - "pattern_catalog": { - "entities": [ - { - "name": "email", - "rules": [ - { - "pattern": pattern, - "confidence": "high", - } - ], - } - ] - }, - "replacement": { - "strategy": "template", - "template": "[{entity}]", - }, - }, - } - ] - }, - "on_detection": {"action": action}, - }, - request.config, - ) - return request - - -def _config_with_stages(stage_count: int) -> pb2.ValidateConfigRequest: - values = json_format.MessageToDict(_config(action="detect").config) - stage = values["entity_processing"]["stages"][0] - stage.pop("name") - values["entity_processing"]["stages"] = [stage] * stage_count - request = pb2.ValidateConfigRequest() - json_format.ParseDict(values, request.config) - return request - - -def _evaluation( - body: bytes, - *, - action: str = "replace", - phase: pb2.SupervisorMiddlewarePhase = ( - pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS - ), -) -> pb2.HttpRequestEvaluation: - return pb2.HttpRequestEvaluation( - phase=phase, - context=pb2.RequestContext(request_id="grpc-integration"), - config=_config(action=action).config, - body=body, - ) - - -@asynccontextmanager -async def _running_stub( - middleware: PrivacyGuardMiddleware, -) -> AsyncIterator[pb2_grpc.SupervisorMiddlewareStub]: - server = grpc.aio.server() - pb2_grpc.add_SupervisorMiddlewareServicer_to_server(middleware, server) - port = server.add_insecure_port("127.0.0.1:0") - assert port > 0 - await server.start() - channel = grpc.aio.insecure_channel(f"127.0.0.1:{port}") - try: - yield pb2_grpc.SupervisorMiddlewareStub(channel) - finally: - await channel.close() - await server.stop(grace=0) - await middleware.close() - - -@pytest.mark.asyncio -async def test_generated_stub_round_trip_covers_manifest_config_and_actions() -> None: - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - async with _running_stub(middleware) as stub: - empty_message_type = message_factory.GetMessageClass( - empty_pb2.DESCRIPTOR.message_types_by_name["Empty"] - ) - empty_message: Message = empty_message_type() - manifest = await stub.Describe(empty_message) - valid = await stub.ValidateConfig(_config()) - invalid = await stub.ValidateConfig(pb2.ValidateConfigRequest()) - detected = await stub.EvaluateHttpRequest( - _evaluation(b"contact a@b.com", action="detect") - ) - replaced = await stub.EvaluateHttpRequest(_evaluation(b"contact a@b.com")) - blocked = await stub.EvaluateHttpRequest( - _evaluation(b"contact a@b.com", action="block") - ) - clean = await stub.EvaluateHttpRequest(_evaluation(b"no match", action="block")) - - assert manifest.name == "privacy-guard" - assert len(manifest.bindings) == 1 - assert valid.valid is True - assert invalid.valid is False - assert "config_invalid" in invalid.reason - assert detected.decision == pb2.DECISION_ALLOW - assert detected.has_body is False - assert len(detected.findings) == 1 - assert replaced.decision == pb2.DECISION_ALLOW - assert replaced.has_body is True - assert replaced.body == b"contact [email]" - assert blocked.decision == pb2.DECISION_DENY - assert blocked.reason_code == "privacy_guard_blocked" - assert clean.decision == pb2.DECISION_ALLOW - - -@pytest.mark.asyncio -async def test_generated_stub_maps_invalid_and_internal_failures( - monkeypatch: pytest.MonkeyPatch, -) -> None: - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - async with _running_stub(middleware) as stub: - with pytest.raises(grpc.aio.AioRpcError) as invalid: - await stub.EvaluateHttpRequest( - _evaluation( - b"body", - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_UNSPECIFIED, - ) - ) - assert invalid.value.code() is grpc.StatusCode.INVALID_ARGUMENT - assert "request_phase_invalid" in (invalid.value.details() or "") - - def fail_unexpectedly(values: object, body: bytes) -> None: - del values, body - raise RuntimeError - - monkeypatch.setattr(middleware, "_prepare_and_process", fail_unexpectedly) - with pytest.raises(grpc.aio.AioRpcError) as internal: - await stub.EvaluateHttpRequest(_evaluation(b"body")) - assert internal.value.code() is grpc.StatusCode.INTERNAL - assert "unexpected_service_failure" in (internal.value.details() or "") - - -@pytest.mark.asyncio -async def test_generated_stub_enforces_ten_stage_limit() -> None: - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - async with _running_stub(middleware) as stub: - exact_config = _config_with_stages(10) - oversized_config = _config_with_stages(11) - exact_validation = await stub.ValidateConfig(exact_config) - oversized_validation = await stub.ValidateConfig(oversized_config) - exact_evaluation = _evaluation(b"no match", action="detect") - exact_evaluation.config.CopyFrom(exact_config.config) - exact_result = await stub.EvaluateHttpRequest(exact_evaluation) - oversized_evaluation = _evaluation(b"no match", action="detect") - oversized_evaluation.config.CopyFrom(oversized_config.config) - with pytest.raises(grpc.aio.AioRpcError) as oversized_result: - await stub.EvaluateHttpRequest(oversized_evaluation) - - assert exact_validation.valid is True - assert oversized_validation.valid is False - assert exact_result.decision == pb2.DECISION_ALLOW - assert oversized_result.value.code() is grpc.StatusCode.INVALID_ARGUMENT - assert "config_invalid" in (oversized_result.value.details() or "") - - -@pytest.mark.asyncio -async def test_generated_stub_maps_contextual_zero_width_to_invalid_config() -> None: - report_pattern = "x|(?=SECRET-zero-width-493)" - config = _config(action="detect", pattern=report_pattern) - evaluation = _evaluation(b"SECRET-zero-width-493", action="detect") - evaluation.config.CopyFrom(config.config) - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - - async with _running_stub(middleware) as stub: - before = await stub.EvaluateHttpRequest( - _evaluation(b"contact a@b.com", action="detect") - ) - validation = await stub.ValidateConfig(config) - with pytest.raises(grpc.aio.AioRpcError) as evaluation_error: - await stub.EvaluateHttpRequest(evaluation) - after = await stub.EvaluateHttpRequest( - _evaluation(b"contact a@b.com", action="detect") - ) - - details = evaluation_error.value.details() or "" - assert len(before.findings) == 1 - assert validation.valid is True - assert evaluation_error.value.code() is grpc.StatusCode.INVALID_ARGUMENT - assert "config_invalid" in details - assert "engine_execution_failed" not in details - assert report_pattern not in details - assert len(after.findings) == 1 - - -class _NumericNestedConfig(StrictDomainModel): - count: int - - -class _NumericEngineConfig(EngineConfig): - engine: Literal["numeric"] = "numeric" - threshold: int - ratio: float - nested: _NumericNestedConfig - values: tuple[int, ...] - - @field_validator("values", mode="before") - @classmethod - def _values_are_a_tuple(cls, value: object) -> object: - if not isinstance(value, list | tuple): - raise ValueError("values must be a list") - return tuple(value) - - -class _NumericEngine(EntityProcessingEngine[_NumericEngineConfig]): - supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - del strategy, timeout - return TextProcessingResult(text=text, detections=()) - - -def _numeric_values( - threshold: int | float, - *, - ratio: float = 3.0, -) -> dict[str, object]: - return { - "entity_processing": { - "stages": [ - { - "config": { - "engine": "numeric", - "threshold": threshold, - "ratio": ratio, - "nested": {"count": 4}, - "values": [5, 6], - } - } - ] - }, - "on_detection": {"action": "detect"}, - } - - -def _numeric_request( - threshold: int | float, - *, - ratio: float = 3.0, -) -> pb2.ValidateConfigRequest: - request = pb2.ValidateConfigRequest() - json_format.ParseDict(_numeric_values(threshold, ratio=ratio), request.config) - return request - - -def _numeric_registry() -> EngineRegistry: - registry = EngineRegistry() - registry.register(_NumericEngine) - return registry.finalize() - - -@pytest.mark.asyncio -async def test_generated_stub_normalizes_transport_safe_integral_numbers() -> None: - registry = _numeric_registry() - with pytest.raises(PrivacyGuardError): - registry.validate_config(_numeric_values(3.0)) - - middleware = PrivacyGuardMiddleware(registry) - async with _running_stub(middleware) as stub: - ordinary = await stub.ValidateConfig(_numeric_request(3, ratio=3.5)) - safe_max = await stub.ValidateConfig(_numeric_request((1 << 53) - 1)) - safe_min = await stub.ValidateConfig(_numeric_request(-((1 << 53) - 1))) - non_integral = await stub.ValidateConfig(_numeric_request(3.5)) - beyond_safe = await stub.ValidateConfig(_numeric_request(1 << 53)) - evaluation = pb2.HttpRequestEvaluation( - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, - config=_numeric_request(3).config, - body=b"body", - ) - result = await stub.EvaluateHttpRequest(evaluation) - - assert ordinary.valid is True - assert safe_max.valid is True - assert safe_min.valid is True - assert non_integral.valid is False - assert beyond_safe.valid is False - assert result.decision == pb2.DECISION_ALLOW diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py deleted file mode 100644 index 3147cfec..00000000 --- a/projects/privacy-guard/tests/service/test_server.py +++ /dev/null @@ -1,406 +0,0 @@ -"""Programmatic Privacy Guard server lifecycle tests.""" - -from __future__ import annotations - -import asyncio -import logging -import subprocess -import sys - -import grpc -import pytest - -from privacy_guard.constants import MAX_CONCURRENT_RPCS, MAX_RECEIVE_MESSAGE_BYTES -from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry -from privacy_guard.errors import EngineRegistryError, ErrorCode, PrivacyGuardError -from privacy_guard.service import server as server_module -from privacy_guard.service.server import PrivacyGuardServer -from privacy_guard.service.servicer import PrivacyGuardMiddleware - - -class _LifecycleServerFake: - """Minimal async-server fake for lifecycle-only tests.""" - - def __init__( - self, - *, - bound_port: int = 50051, - bind_error: RuntimeError | None = None, - start_error: RuntimeError | None = None, - wait_error: BaseException | None = None, - block_stop: bool = False, - ) -> None: - self.bound_port = bound_port - self.bind_error = bind_error - self.start_error = start_error - self.wait_error = wait_error - self.addresses: list[str] = [] - self.started = False - self.waited = False - self.stop_graces: list[float | None] = [] - self.stop_started = asyncio.Event() - self.stop_release = asyncio.Event() - if not block_stop: - self.stop_release.set() - - def add_insecure_port(self, address: str) -> int: - self.addresses.append(address) - if self.bind_error is not None: - raise self.bind_error - return self.bound_port - - async def start(self) -> None: - if self.start_error is not None: - raise self.start_error - self.started = True - - async def wait_for_termination(self, timeout: float | None = None) -> bool: - del timeout - if self.wait_error is not None: - raise self.wait_error - self.waited = True - return True - - async def stop(self, grace: float | None) -> None: - self.stop_graces.append(grace) - self.stop_started.set() - await self.stop_release.wait() - - -def test_programmatic_server_runs_with_injected_registry_and_default_address( - monkeypatch: pytest.MonkeyPatch, -) -> None: - registry = create_builtin_registry() - served: list[tuple[PrivacyGuardServer, str]] = [] - - async def record_serve(self: PrivacyGuardServer, listen: str) -> None: - served.append((self, listen)) - await self._middleware.close() - - monkeypatch.setattr(PrivacyGuardServer, "serve_async", record_serve) - - server = PrivacyGuardServer( - registry=registry, - timeout_seconds=4.5, - log_request_content=True, - ) - server.serve_sync() - - assert served == [(server, "127.0.0.1:50051")] - assert server._middleware._registry is registry - assert server._middleware._policy._timeout_seconds == 4.5 - assert server._middleware._policy._log_request_content is True - - -def test_programmatic_server_requires_an_explicit_finalized_registry() -> None: - with pytest.raises(EngineRegistryError, match="finalized"): - PrivacyGuardServer(EngineRegistry()) - - -@pytest.mark.parametrize("timeout_seconds", [True, 0, 31, float("inf")]) -def test_programmatic_server_rejects_invalid_processing_timeout( - timeout_seconds: bool | int | float, -) -> None: - with pytest.raises( - ValueError, - match="finite number greater than 0 and at most 30", - ): - PrivacyGuardServer( - create_builtin_registry(), - timeout_seconds=timeout_seconds, - ) - - -def test_synchronous_server_exits_cleanly_after_keyboard_interrupt( - monkeypatch: pytest.MonkeyPatch, -) -> None: - server = PrivacyGuardServer(create_builtin_registry()) - - async def interrupt(self: PrivacyGuardServer, listen: str) -> None: - del self, listen - raise KeyboardInterrupt - - monkeypatch.setattr(PrivacyGuardServer, "serve_async", interrupt) - - server.serve_sync() - asyncio.run(server._middleware.close()) - - -def test_programmatic_server_import_does_not_load_the_cli_framework() -> None: - probe = ( - "import sys; " - "from privacy_guard.service import PrivacyGuardServer; " - "assert PrivacyGuardServer.__name__ == 'PrivacyGuardServer'; " - "assert 'privacy_guard.cli' not in sys.modules; " - "assert 'typer' not in sys.modules" - ) - - subprocess.run([sys.executable, "-c", probe], check=True) - - -def test_engine_import_does_not_load_the_server_transport() -> None: - probe = ( - "import sys; " - "import privacy_guard.engines; " - "assert 'privacy_guard.service' not in sys.modules; " - "assert 'grpc' not in sys.modules" - ) - - subprocess.run([sys.executable, "-c", probe], check=True) - - -def test_server_sets_transport_limits_and_registers_middleware( - monkeypatch: pytest.MonkeyPatch, -) -> None: - fake_server = object() - server_options: list[tuple[int, tuple[tuple[str, int], ...]]] = [] - registrations: list[tuple[PrivacyGuardMiddleware, object]] = [] - - def fake_server_factory( - *, - maximum_concurrent_rpcs: int, - options: tuple[tuple[str, int], ...], - ) -> object: - server_options.append((maximum_concurrent_rpcs, options)) - return fake_server - - def record_registration( - middleware: PrivacyGuardMiddleware, - server: object, - ) -> None: - registrations.append((middleware, server)) - - middleware = _middleware() - monkeypatch.setattr(grpc.aio, "server", fake_server_factory) - monkeypatch.setattr( - server_module.pb2_grpc, - "add_SupervisorMiddlewareServicer_to_server", - record_registration, - ) - try: - result = server_module._create_grpc_server(middleware) - finally: - asyncio.run(middleware.close()) - - assert result is fake_server - assert server_options == [ - ( - MAX_CONCURRENT_RPCS, - (("grpc.max_receive_message_length", MAX_RECEIVE_MESSAGE_BYTES),), - ) - ] - assert registrations == [(middleware, fake_server)] - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("fake_server", "sensitive_address"), - [ - (_LifecycleServerFake(bound_port=0), "invalid-sensitive-listen-8472"), - ( - _LifecycleServerFake( - bind_error=RuntimeError("invalid-sensitive-listen-9472") - ), - "invalid-sensitive-listen-9472", - ), - ], -) -async def test_serve_async_sanitizes_bind_failures_and_closes_resources( - monkeypatch: pytest.MonkeyPatch, - fake_server: _LifecycleServerFake, - sensitive_address: str, -) -> None: - closed: list[PrivacyGuardMiddleware] = [] - - async def record_close(middleware: PrivacyGuardMiddleware) -> None: - closed.append(middleware) - - server = PrivacyGuardServer(create_builtin_registry()) - monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) - monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) - - with pytest.raises(PrivacyGuardError) as captured: - await server.serve_async(sensitive_address) - - assert captured.value.code is ErrorCode.SERVER_BIND_FAILED - assert captured.value.__cause__ is None - assert sensitive_address not in str(captured.value) - assert fake_server.started is False - assert fake_server.waited is False - assert fake_server.stop_graces == [0] - assert closed == [server._middleware] - - -@pytest.mark.asyncio -async def test_serve_async_starts_waits_and_closes_on_normal_termination( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - fake_server = _LifecycleServerFake(bound_port=50053) - closed: list[PrivacyGuardMiddleware] = [] - - async def record_close(middleware: PrivacyGuardMiddleware) -> None: - closed.append(middleware) - - server = PrivacyGuardServer(create_builtin_registry()) - monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) - monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) - - with caplog.at_level(logging.INFO, logger="privacy_guard.service.server"): - await server.serve_async("127.0.0.1:50053") - - assert fake_server.addresses == ["127.0.0.1:50053"] - assert fake_server.started is True - assert fake_server.waited is True - assert fake_server.stop_graces == [0] - assert closed == [server._middleware] - assert "privacy_guard_server_bound listen='127.0.0.1:50053'" in caplog.text - - -@pytest.mark.asyncio -async def test_serve_async_propagates_cancellation_after_closing_resources( - monkeypatch: pytest.MonkeyPatch, -) -> None: - fake_server = _LifecycleServerFake( - bound_port=50054, - wait_error=asyncio.CancelledError(), - ) - closed: list[PrivacyGuardMiddleware] = [] - - async def record_close(middleware: PrivacyGuardMiddleware) -> None: - closed.append(middleware) - - server = PrivacyGuardServer(create_builtin_registry()) - monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) - monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) - - with pytest.raises(asyncio.CancelledError): - await server.serve_async("127.0.0.1:50054") - - assert fake_server.started is True - assert fake_server.stop_graces == [0] - assert closed == [server._middleware] - - -@pytest.mark.asyncio -async def test_serve_async_preserves_cancellation_during_server_shutdown( - monkeypatch: pytest.MonkeyPatch, -) -> None: - fake_server = _LifecycleServerFake(bound_port=50055, block_stop=True) - closed: list[PrivacyGuardMiddleware] = [] - - async def record_close(middleware: PrivacyGuardMiddleware) -> None: - closed.append(middleware) - - server = PrivacyGuardServer(create_builtin_registry()) - monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) - monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) - - serving = asyncio.create_task(server.serve_async("127.0.0.1:50055")) - await fake_server.stop_started.wait() - serving.cancel() - await asyncio.sleep(0) - - assert serving.done() is False - - fake_server.stop_release.set() - with pytest.raises(asyncio.CancelledError): - await serving - - assert fake_server.stop_graces == [0] - assert closed == [server._middleware] - - -@pytest.mark.asyncio -async def test_serve_async_sanitizes_startup_failures_and_closes_resources( - monkeypatch: pytest.MonkeyPatch, -) -> None: - fake_server = _LifecycleServerFake( - bound_port=50056, - start_error=RuntimeError("startup failed"), - ) - closed: list[PrivacyGuardMiddleware] = [] - - async def record_close(middleware: PrivacyGuardMiddleware) -> None: - closed.append(middleware) - - server = PrivacyGuardServer(create_builtin_registry()) - monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) - monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) - - with pytest.raises(PrivacyGuardError) as captured: - await server.serve_async("127.0.0.1:50056") - - assert captured.value.code is ErrorCode.SERVER_BIND_FAILED - assert captured.value.__cause__ is None - assert "server.start" in str(captured.value) - assert "startup failed" not in str(captured.value) - assert fake_server.waited is False - assert fake_server.stop_graces == [0] - assert closed == [server._middleware] - - -@pytest.mark.parametrize( - ("listen", "port"), - [ - ("127.0.0.1:1", 1), - ("middleware.local:65535", 65_535), - ("[::1]:50051", 50_051), - ], -) -def test_listen_address_accepts_supported_tcp_forms(listen: str, port: int) -> None: - assert server_module._validated_listen_port(listen) == port - - -@pytest.mark.parametrize( - "listen", - [ - "127.0.0.1:0", - "127.0.0.1:65536", - "127.0.0.1:99999", - "127.0.0.1:-1", - "[::1]", - "::1:50051", - ], -) -def test_listen_address_rejects_invalid_numeric_ports_and_forms( - listen: str, -) -> None: - with pytest.raises(PrivacyGuardError) as captured: - server_module._validated_listen_port(listen) - - assert captured.value.code is ErrorCode.SERVER_BIND_FAILED - - -def test_listen_address_rejects_arbitrarily_long_decimal_port() -> None: - with pytest.raises(PrivacyGuardError) as captured: - server_module._validated_listen_port(f"127.0.0.1:{'9' * 5_000}") - - assert captured.value.code is ErrorCode.SERVER_BIND_FAILED - - -@pytest.mark.asyncio -async def test_serve_async_rejects_mismatched_bound_port( - monkeypatch: pytest.MonkeyPatch, -) -> None: - fake_server = _LifecycleServerFake(bound_port=34_463) - closed: list[PrivacyGuardMiddleware] = [] - - async def record_close(middleware: PrivacyGuardMiddleware) -> None: - closed.append(middleware) - - server = PrivacyGuardServer(create_builtin_registry()) - monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) - monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) - - with pytest.raises(PrivacyGuardError) as captured: - await server.serve_async("127.0.0.1:9999") - - assert captured.value.code is ErrorCode.SERVER_BIND_FAILED - assert fake_server.started is False - assert fake_server.stop_graces == [0] - assert closed == [server._middleware] - - -def _middleware() -> PrivacyGuardMiddleware: - return PrivacyGuardMiddleware(create_builtin_registry()) diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py deleted file mode 100644 index 205c7aa3..00000000 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ /dev/null @@ -1,813 +0,0 @@ -"""Service boundary tests over the canonical OpenShell-owned protobuf.""" - -from __future__ import annotations - -import asyncio -import logging -from concurrent.futures import ThreadPoolExecutor -from copy import deepcopy -from threading import Barrier, Event, Lock, get_ident -from typing import Never - -import grpc -import pytest -from google.protobuf import json_format -from google.protobuf.message import Message - -from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 -from privacy_guard.config import PrivacyGuardConfig -from privacy_guard.constants import ( - LIMIT_REASON, - LIMIT_REASON_CODE, - MAX_DIAGNOSTIC_TEXT_BYTES, - MAX_PROTO_CONFIG_BYTES, - MAX_PROTO_CONTEXT_BYTES, - MAX_PROTO_FINDING_BYTES, - MAX_PROTO_HEADERS, - MAX_PROTO_HEADERS_BYTES, - MAX_PROTO_TARGET_BYTES, -) -from privacy_guard.engines import ( - EngineConfig, -) -from privacy_guard.engines import regex as regex_module -from privacy_guard.engines.registry import create_builtin_registry -from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.request_processor import ( - EntityDetectionSummary, - RequestDecision, - RequestProcessingResult, - RequestProcessor, -) -from privacy_guard.service import servicer as servicer_module -from privacy_guard.service.servicer import PrivacyGuardMiddleware - - -def _values( - action: str = "replace", - *, - rules: list[dict[str, object]] | None = None, - stage_count: int = 1, - stage_name: str | None = None, -) -> dict[str, object]: - if rules is None: - rules = [ - { - "pattern": r"[a-z]+@[a-z]+\.[a-z]+", - "confidence": "high", - } - ] - stage: dict[str, object] = { - "config": { - "engine": "regex", - "pattern_catalog": { - "entities": [ - { - "name": "email", - "rules": rules, - } - ] - }, - "replacement": { - "strategy": "template", - "template": "[{entity}]", - }, - } - } - if stage_name is not None: - stage["name"] = stage_name - return { - "entity_processing": {"stages": [deepcopy(stage) for _ in range(stage_count)]}, - "on_detection": {"action": action}, - } - - -def _proto_config(values: dict[str, object]) -> Message: - result = pb2.ValidateConfigRequest().config - json_format.ParseDict(values, result) - return result - - -def _request(body: bytes, *, action: str = "replace") -> pb2.HttpRequestEvaluation: - return pb2.HttpRequestEvaluation( - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, - config=_proto_config(_values(action)), - body=body, - ) - - -class _SuccessfulEvaluationContext: - async def abort(self, code: grpc.StatusCode, details: str) -> Never: - del code, details - raise AssertionError("successful evaluation unexpectedly aborted") - - -def test_copied_proto_remains_the_current_openshell_contract() -> None: - evaluation = pb2.HttpRequestEvaluation() - finding = pb2.Finding() - - assert isinstance(evaluation.config, Message) - assert not hasattr(evaluation, "config_fingerprint") - assert not hasattr(finding, "source") - - -def test_validate_config_is_pure_and_reports_invalid_config( - monkeypatch: pytest.MonkeyPatch, -) -> None: - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - active = middleware._policy.processor_for(_values(action="replace")) - processor_build_count = 0 - original_build = servicer_module._ActivePolicy._build_processor - - def record_processor_build( - policy: servicer_module._ActivePolicy, - config: PrivacyGuardConfig[EngineConfig], - ) -> RequestProcessor: - nonlocal processor_build_count - processor_build_count += 1 - return original_build(policy, config) - - monkeypatch.setattr( - servicer_module._ActivePolicy, - "_build_processor", - record_processor_build, - ) - try: - valid = middleware._validate_config( - pb2.ValidateConfigRequest(config=_proto_config(_values("detect"))) - ) - invalid = middleware._validate_config( - pb2.ValidateConfigRequest(config=_proto_config({"on_detection": {}})) - ) - still_active = middleware._policy.processor_for(_values(action="replace")) - finally: - asyncio.run(middleware.close()) - - assert valid.valid is True - assert invalid.valid is False - assert "config_invalid" in invalid.reason - assert still_active is active - assert processor_build_count == 0 - - -def test_validate_config_rejects_oversized_proto_before_registry_validation( - monkeypatch: pytest.MonkeyPatch, -) -> None: - validation_count = 0 - original_validate = servicer_module.EngineRegistry.validate_config - - def record_validation( - registry: servicer_module.EngineRegistry, - values: object, - ) -> PrivacyGuardConfig[EngineConfig]: - nonlocal validation_count - validation_count += 1 - return original_validate(registry, values) - - monkeypatch.setattr( - servicer_module.EngineRegistry, - "validate_config", - record_validation, - ) - exact_config = pb2.ValidateConfigRequest() - json_format.ParseDict({"padding": "x" * 65_515}, exact_config.config) - oversized_config = pb2.ValidateConfigRequest() - json_format.ParseDict({"padding": "x" * 65_516}, oversized_config.config) - assert exact_config.config.ByteSize() == MAX_PROTO_CONFIG_BYTES - assert oversized_config.config.ByteSize() == MAX_PROTO_CONFIG_BYTES + 1 - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - try: - exact = middleware._validate_config(exact_config) - oversized = middleware._validate_config(oversized_config) - finally: - asyncio.run(middleware.close()) - - assert exact.valid is False - assert oversized.valid is False - assert validation_count == 1 - - -@pytest.mark.parametrize( - "unsafe_value", - [ - "line\nbreak", - "ansi\x1b[31m", - "nul\x00byte", - "right-to-left\u202eoverride", - ], -) -def test_validate_config_rejects_non_printable_stage_names( - unsafe_value: str, -) -> None: - registry = create_builtin_registry() - - with pytest.raises(PrivacyGuardError) as captured: - registry.validate_config(_values(stage_name=unsafe_value)) - - assert captured.value.code is ErrorCode.CONFIG_INVALID - - -def test_validate_config_accepts_printable_unicode_stage_names() -> None: - config = create_builtin_registry().validate_config(_values(stage_name="身份检查 🛡️")) - - assert config.entity_processing.stages[0].name == "身份检查 🛡️" - - -def test_evaluation_enforces_exact_encoded_transport_boundaries() -> None: - request = _request(b"") - request.context.request_id = "x" * 4_093 - assert request.context.ByteSize() == MAX_PROTO_CONTEXT_BYTES - servicer_module._validate_evaluation_envelope(request) - request.context.request_id += "x" - assert request.context.ByteSize() == MAX_PROTO_CONTEXT_BYTES + 1 - with pytest.raises(PrivacyGuardError) as context_error: - servicer_module._validate_evaluation_envelope(request) - assert context_error.value.code is ErrorCode.REQUEST_ENVELOPE_INVALID - - request = _request(b"") - request.target.host = "x" * 32_764 - assert request.target.ByteSize() == MAX_PROTO_TARGET_BYTES - servicer_module._validate_evaluation_envelope(request) - request.target.host += "x" - assert request.target.ByteSize() == MAX_PROTO_TARGET_BYTES + 1 - with pytest.raises(PrivacyGuardError) as target_error: - servicer_module._validate_evaluation_envelope(request) - assert target_error.value.code is ErrorCode.REQUEST_ENVELOPE_INVALID - - request = _request(b"") - request.headers.add(name="x", value="x" * 65_525) - assert servicer_module._encoded_headers_size(request.headers) == ( - MAX_PROTO_HEADERS_BYTES - ) - servicer_module._validate_evaluation_envelope(request) - request.headers[0].value += "x" - assert servicer_module._encoded_headers_size(request.headers) == ( - MAX_PROTO_HEADERS_BYTES + 1 - ) - with pytest.raises(PrivacyGuardError) as header_size_error: - servicer_module._validate_evaluation_envelope(request) - assert header_size_error.value.code is ErrorCode.REQUEST_ENVELOPE_INVALID - - request = _request(b"") - for _ in range(MAX_PROTO_HEADERS): - request.headers.add() - servicer_module._validate_evaluation_envelope(request) - request.headers.add() - with pytest.raises(PrivacyGuardError) as header_count_error: - servicer_module._validate_evaluation_envelope(request) - assert header_count_error.value.code is ErrorCode.REQUEST_ENVELOPE_INVALID - - -def test_evaluation_enforces_exact_encoded_config_boundary() -> None: - request = _request(b"") - request.config.Clear() - json_format.ParseDict({"padding": "x" * 65_515}, request.config) - assert request.config.ByteSize() == MAX_PROTO_CONFIG_BYTES - servicer_module._validate_evaluation_envelope(request) - request.config.Clear() - json_format.ParseDict({"padding": "x" * 65_516}, request.config) - assert request.config.ByteSize() == MAX_PROTO_CONFIG_BYTES + 1 - - with pytest.raises(PrivacyGuardError) as captured: - servicer_module._validate_evaluation_envelope(request) - - assert captured.value.code is ErrorCode.CONFIG_INVALID - assert "encoded configuration at or below 64 KiB" in str(captured.value) - - -def test_limit_deny_explains_recovery_options() -> None: - result = servicer_module._result_to_proto( - RequestProcessingResult( - decision=RequestDecision.DENY, - reason_code=LIMIT_REASON_CODE, - ) - ) - - assert result.reason == LIMIT_REASON - assert "Check Privacy Guard logs for the limit kind" in result.reason - assert "Reduce the request or replacement size" in result.reason - assert "simplify the configured stages and rules" in result.reason - assert "--timeout-seconds or PrivacyGuardServer(timeout_seconds=...)" in ( - result.reason - ) - assert "additional headroom for queueing and configuration preparation" in ( - result.reason - ) - - -def test_service_limit_deny_logs_a_content_safe_resource_kind( - caplog: pytest.LogCaptureFixture, -) -> None: - sentinel = "sensitive-finding-value" - with caplog.at_level(logging.INFO, logger="privacy_guard.service.servicer"): - result = servicer_module._result_to_proto( - RequestProcessingResult( - decision=RequestDecision.ALLOW, - detection_summaries=( - EntityDetectionSummary( - entity=sentinel + ("x" * MAX_PROTO_FINDING_BYTES), - source_stage="stage", - count=1, - ), - ), - ) - ) - - assert result.reason_code == LIMIT_REASON_CODE - assert "privacy_guard_processing_limit kind=resource" in caplog.text - assert sentinel not in caplog.text - - -@pytest.mark.parametrize( - "invalid_request_id", - [ - "line\nbreak", - "ansi\x1b[31m", - "nul\x00byte", - "right-to-left\u202eoverride", - "x" * (MAX_DIAGNOSTIC_TEXT_BYTES + 1), - ], -) -def test_evaluation_logs_placeholder_for_invalid_request_id( - caplog: pytest.LogCaptureFixture, - invalid_request_id: str, -) -> None: - async def evaluate() -> pb2.HttpRequestResult: - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - request = _request(b"no match", action="detect") - request.context.request_id = invalid_request_id - try: - return await middleware._evaluate_rpc( - request, - _SuccessfulEvaluationContext(), - ) - finally: - await middleware.close() - - with caplog.at_level(logging.INFO, logger="privacy_guard.service.servicer"): - result = asyncio.run(evaluate()) - - records = [ - record - for record in caplog.records - if record.name == "privacy_guard.service.servicer" - and record.getMessage().startswith("privacy_guard_evaluation ") - ] - assert result.decision == pb2.DECISION_ALLOW - assert len(records) == 1 - assert 'request_id="invalid" ' in records[0].getMessage() - assert records[0].getMessage().isprintable() - assert len(caplog.text.splitlines()) == 1 - - -def test_evaluation_logs_printable_unicode_request_id( - caplog: pytest.LogCaptureFixture, -) -> None: - async def evaluate() -> None: - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - request = _request(b"no match", action="detect") - request.context.request_id = "请求-42 🛡️" - try: - await middleware._evaluate_rpc( - request, - _SuccessfulEvaluationContext(), - ) - finally: - await middleware.close() - - with caplog.at_level(logging.INFO, logger="privacy_guard.service.servicer"): - asyncio.run(evaluate()) - - assert r'request_id="请求-42\u0020🛡️"' in caplog.text - assert len(caplog.text.splitlines()) == 1 - - -def test_evaluation_quotes_request_id_delimiters_in_message_log( - caplog: pytest.LogCaptureFixture, -) -> None: - request_id = 'trusted action=allow error_code="none"' - - async def evaluate() -> None: - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - request = _request(b"no match", action="detect") - request.context.request_id = request_id - try: - await middleware._evaluate_rpc( - request, - _SuccessfulEvaluationContext(), - ) - finally: - await middleware.close() - - with caplog.at_level(logging.INFO, logger="privacy_guard.service.servicer"): - asyncio.run(evaluate()) - - records = [ - record - for record in caplog.records - if record.name == "privacy_guard.service.servicer" - and record.getMessage().startswith("privacy_guard_evaluation ") - ] - assert len(records) == 1 - assert getattr(records[0], "request_id") == request_id - assert ( - r'request_id="trusted\u0020action=allow\u0020error_code=\"none\""' - in records[0].getMessage() - ) - assert records[0].getMessage().count(" action=") == 1 - - -def test_middleware_applies_configured_timeout_to_active_processor() -> None: - middleware = PrivacyGuardMiddleware( - create_builtin_registry(), - timeout_seconds=4.5, - ) - try: - processor = middleware._policy.processor_for(_values()) - finally: - asyncio.run(middleware.close()) - - assert processor._timeout_seconds == 4.5 - - -def test_evaluation_decodes_one_utf8_text_and_encodes_replacement() -> None: - async def evaluate() -> pb2.HttpRequestResult: - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - try: - return await middleware._evaluate_http_request(_request(b"email a@b.com")) - finally: - await middleware.close() - - result = asyncio.run(evaluate()) - - assert result.decision == pb2.DECISION_ALLOW - assert result.has_body is True - assert result.body == b"email [email]" - assert len(result.findings) == 1 - assert result.findings[0].type == "detected_entity" - assert result.findings[0].label == "email (regex[1])" - - -def test_evaluation_prepares_configuration_off_the_event_loop( - monkeypatch: pytest.MonkeyPatch, -) -> None: - event_loop_thread = get_ident() - preparation_threads: list[int] = [] - original_processor_for = servicer_module._ActivePolicy.processor_for - - def record_preparation( - policy: servicer_module._ActivePolicy, - values: object, - ) -> RequestProcessor: - preparation_threads.append(get_ident()) - return original_processor_for(policy, values) - - monkeypatch.setattr( - servicer_module._ActivePolicy, - "processor_for", - record_preparation, - ) - - async def evaluate() -> None: - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - try: - await middleware._evaluate_http_request(_request(b"email a@b.com")) - finally: - await middleware.close() - - asyncio.run(evaluate()) - - assert len(preparation_threads) == 1 - assert preparation_threads[0] != event_loop_thread - - -def test_evaluation_revalidates_configuration_before_reusing_active_processor( - monkeypatch: pytest.MonkeyPatch, -) -> None: - validation_count = 0 - original_validate = servicer_module.EngineRegistry.validate_config - - def record_validation( - registry: servicer_module.EngineRegistry, - values: object, - ) -> PrivacyGuardConfig[EngineConfig]: - nonlocal validation_count - validation_count += 1 - return original_validate(registry, values) - - monkeypatch.setattr( - servicer_module.EngineRegistry, - "validate_config", - record_validation, - ) - - async def evaluate_twice() -> None: - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - try: - request = _request(b"email a@b.com") - await middleware._evaluate_http_request(request) - await middleware._evaluate_http_request(request) - finally: - await middleware.close() - - asyncio.run(evaluate_twice()) - - assert validation_count == 2 - - -def test_active_policy_reuses_only_the_current_configuration() -> None: - policy = servicer_module._ActivePolicy( - create_builtin_registry(), - timeout_seconds=1, - log_request_content=False, - ) - first_values = _values(action="detect") - second_values = _values(action="block") - - first = policy.processor_for(first_values) - same = policy.processor_for(deepcopy(first_values)) - second = policy.processor_for(second_values) - rebuilt_first = policy.processor_for(first_values) - - assert same is first - assert second is not first - assert rebuilt_first is not first - assert rebuilt_first is not second - - -@pytest.mark.parametrize("initial_action", [None, "detect"]) -def test_concurrent_requests_for_the_same_policy_build_once( - monkeypatch: pytest.MonkeyPatch, - initial_action: str | None, -) -> None: - worker_count = 4 - workers_ready = Barrier(worker_count) - build_started = Event() - release_build = Event() - build_count = 0 - build_count_lock = Lock() - original_build = servicer_module._ActivePolicy._build_processor - policy = servicer_module._ActivePolicy( - create_builtin_registry(), - timeout_seconds=1, - log_request_content=False, - ) - initial = ( - policy.processor_for(_values(action=initial_action)) - if initial_action is not None - else None - ) - requested_values = _values( - action="block" if initial_action is not None else "detect" - ) - - def pause_build( - active_policy: servicer_module._ActivePolicy, - config: PrivacyGuardConfig[EngineConfig], - ) -> RequestProcessor: - nonlocal build_count - with build_count_lock: - build_count += 1 - build_started.set() - assert release_build.wait(timeout=5) - return original_build(active_policy, config) - - monkeypatch.setattr( - servicer_module._ActivePolicy, - "_build_processor", - pause_build, - ) - - def resolve_policy() -> RequestProcessor: - workers_ready.wait(timeout=5) - return policy.processor_for(requested_values) - - with ThreadPoolExecutor(max_workers=worker_count) as executor: - futures = tuple(executor.submit(resolve_policy) for _ in range(worker_count)) - assert build_started.wait(timeout=5) - assert all(not future.done() for future in futures) - release_build.set() - processors = tuple(future.result(timeout=5) for future in futures) - - assert build_count == 1 - assert all(processor is processors[0] for processor in processors) - assert processors[0] is not initial - assert policy.processor_for(requested_values) is processors[0] - - -def test_different_policy_updates_are_serialized( - monkeypatch: pytest.MonkeyPatch, -) -> None: - first_build_started = Event() - release_first_build = Event() - second_build_started = Event() - build_actions: list[str] = [] - active_builds = 0 - maximum_active_builds = 0 - build_count_lock = Lock() - original_build = servicer_module._ActivePolicy._build_processor - policy = servicer_module._ActivePolicy( - create_builtin_registry(), - timeout_seconds=1, - log_request_content=False, - ) - initial = policy.processor_for(_values(action="detect")) - - def control_build( - active_policy: servicer_module._ActivePolicy, - config: PrivacyGuardConfig[EngineConfig], - ) -> RequestProcessor: - nonlocal active_builds, maximum_active_builds - action = config.on_detection.action.value - with build_count_lock: - active_builds += 1 - maximum_active_builds = max(maximum_active_builds, active_builds) - build_actions.append(action) - try: - if action == "block": - first_build_started.set() - assert release_first_build.wait(timeout=5) - elif action == "replace": - second_build_started.set() - return original_build(active_policy, config) - finally: - with build_count_lock: - active_builds -= 1 - - monkeypatch.setattr( - servicer_module._ActivePolicy, - "_build_processor", - control_build, - ) - - with ThreadPoolExecutor(max_workers=2) as executor: - first_update = executor.submit(policy.processor_for, _values(action="block")) - assert first_build_started.wait(timeout=5) - second_update = executor.submit( - policy.processor_for, - _values(action="replace"), - ) - assert not second_build_started.wait(timeout=0.1) - release_first_build.set() - first_processor = first_update.result(timeout=5) - second_processor = second_update.result(timeout=5) - - assert second_build_started.is_set() - assert build_actions == ["block", "replace"] - assert maximum_active_builds == 1 - assert first_processor is not initial - assert second_processor is not first_processor - assert policy.processor_for(_values(action="replace")) is second_processor - - -def test_failed_policy_update_preserves_the_active_processor( - monkeypatch: pytest.MonkeyPatch, -) -> None: - failure = PrivacyGuardError(ErrorCode.UNEXPECTED_SERVICE_FAILURE) - original_build = servicer_module._ActivePolicy._build_processor - policy = servicer_module._ActivePolicy( - create_builtin_registry(), - timeout_seconds=1, - log_request_content=False, - ) - active_values = _values(action="detect") - update_values = _values(action="block") - active = policy.processor_for(active_values) - - def fail_update( - active_policy: servicer_module._ActivePolicy, - config: PrivacyGuardConfig[EngineConfig], - ) -> RequestProcessor: - if config.on_detection.action.value == "block": - raise failure - return original_build(active_policy, config) - - monkeypatch.setattr( - servicer_module._ActivePolicy, - "_build_processor", - fail_update, - ) - - with pytest.raises(PrivacyGuardError) as captured: - policy.processor_for(update_values) - - assert captured.value is failure - assert policy.processor_for(active_values) is active - - monkeypatch.setattr( - servicer_module._ActivePolicy, - "_build_processor", - original_build, - ) - updated = policy.processor_for(update_values) - - assert updated is not active - assert policy.processor_for(update_values) is updated - - -def test_compiled_cache_eviction_does_not_invalidate_active_processor( - monkeypatch: pytest.MonkeyPatch, -) -> None: - regex_module._clear_compiled_pattern_cache() - registry = create_builtin_registry() - policy = servicer_module._ActivePolicy( - registry, - timeout_seconds=1, - log_request_content=False, - ) - processor_values = _values( - "detect", - rules=[{"pattern": "aaa", "confidence": "high"}], - ) - validation_values = _values( - "detect", - rules=[{"pattern": "bbb", "confidence": "high"}], - ) - - try: - processor = policy.processor_for(processor_values) - entry_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES - monkeypatch.setattr( - regex_module, - "MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES", - entry_weight, - ) - - registry.validate_config(validation_values) - - result = processor.process("aaa") - assert len(result.detection_summaries) == 1 - assert policy.processor_for(processor_values) is processor - assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES <= entry_weight - finally: - policy.clear() - regex_module._clear_compiled_pattern_cache() - - -def test_middleware_shutdown_clears_active_policy() -> None: - regex_module._clear_compiled_pattern_cache() - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - try: - middleware._policy.processor_for(_values("detect")) - - asyncio.run(middleware.close()) - - assert middleware._policy._config is None - assert middleware._policy._processor is None - finally: - middleware._policy.clear() - regex_module._clear_compiled_pattern_cache() - - -def test_oversized_stage_list_fails_before_engine_construction( - monkeypatch: pytest.MonkeyPatch, -) -> None: - values = _values(action="detect", stage_count=10_000) - - def unexpected_call(*args: object, **kwargs: object) -> object: - del args, kwargs - raise AssertionError("oversized stage list reached preparation") - - monkeypatch.setattr( - servicer_module.EngineRegistry, - "create_engine", - unexpected_call, - ) - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - try: - with pytest.raises(PrivacyGuardError) as captured: - middleware._policy.processor_for(values) - finally: - asyncio.run(middleware.close()) - - assert captured.value.code is ErrorCode.CONFIG_INVALID - - -def test_invalid_utf8_fails_before_invoking_an_engine() -> None: - async def evaluate() -> None: - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - try: - with pytest.raises(PrivacyGuardError) as captured: - await middleware._evaluate_http_request(_request(b"\xff")) - assert captured.value.code is ErrorCode.BODY_ENCODING_INVALID - finally: - await middleware.close() - - asyncio.run(evaluate()) - - -def test_detect_returns_no_body_mutation() -> None: - async def evaluate() -> pb2.HttpRequestResult: - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - try: - return await middleware._evaluate_http_request( - _request(b"a@b.com", action="detect") - ) - finally: - await middleware.close() - - result = asyncio.run(evaluate()) - - assert result.decision == pb2.DECISION_ALLOW - assert result.has_body is False - assert result.body == b"" diff --git a/projects/privacy-guard/tests/test_cli.py b/projects/privacy-guard/tests/test_cli.py deleted file mode 100644 index b48ba022..00000000 --- a/projects/privacy-guard/tests/test_cli.py +++ /dev/null @@ -1,429 +0,0 @@ -"""Privacy Guard command-line application tests.""" - -from __future__ import annotations - -import json -import re -from collections.abc import Iterator -from importlib.metadata import entry_points -from pathlib import Path -from types import SimpleNamespace - -import pytest -from typer.testing import CliRunner, Result - -from privacy_guard import cli as cli_module -from privacy_guard.cli import app -from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry -from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.logging import reset_logging -from privacy_guard.service.server import PrivacyGuardServer - - -@pytest.fixture(autouse=True) -def _reset_cli_logging() -> Iterator[None]: - yield - reset_logging() - - -def test_cli_help_exposes_server_and_discovery_commands() -> None: - result = CliRunner().invoke(app, ["--help"]) - - assert result.exit_code == 0 - output = _plain_output(result) - assert "serve" in output - assert "configuration-schema" in output - assert "add-gateway-registration" in output - assert "remove-gateway-registration" in output - assert "engines" in output - assert "--debug" in output - assert "--debug-log-content" in output - assert "--registry-factory" in output - assert "--config" not in output - assert "--profile" not in output - assert "--scanner-name" not in output - - -def test_cli_add_gateway_registration_help_requires_an_explicit_host_ip() -> None: - result = CliRunner().invoke( - app, - ["add-gateway-registration", "--help"], - terminal_width=240, - ) - - assert result.exit_code == 0 - output = _normalized_output(result) - assert "--host-ip" in output - assert "required" in output.lower() - assert "Non-loopback IPv4" in output - assert "$OPENSHELL_GATEWAY_CONFIG" in output - assert "$XDG_CONFIG_HOME/openshell" in output - assert "1-128 ASCII bytes" in output - assert "restart the OpenShell gateway" not in output - - -def test_cli_add_gateway_registration_updates_the_default_xdg_config( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) - - result = CliRunner().invoke( - app, - ["add-gateway-registration", "--host-ip", "192.168.1.20"], - ) - - assert result.exit_code == 0 - config_path = tmp_path / "openshell" / "gateway.toml" - assert config_path.exists() - output = _plain_output(result) - assert f"Created {config_path}" in output - assert "Registered privacy-guard at http://192.168.1.20:50051" in output - assert "start Privacy Guard, then restart the OpenShell gateway" in output - - -@pytest.mark.parametrize("host_ip", ["host.openshell.internal", "127.0.0.1", "0.0.0.0"]) -def test_cli_add_gateway_registration_rejects_unusable_host_ip(host_ip: str) -> None: - result = CliRunner().invoke( - app, - ["add-gateway-registration", "--host-ip", host_ip], - terminal_width=240, - ) - - assert result.exit_code == 2 - output = _normalized_output(result) - assert "--host-ip" in output - assert "IPv4 address" in output - - -@pytest.mark.parametrize( - "name", - [ - "a" * 129, - "privacy guard", - "openshell/privacy-guard", - ], -) -def test_cli_add_gateway_registration_rejects_invalid_registration_name( - name: str, -) -> None: - result = CliRunner().invoke( - app, - [ - "add-gateway-registration", - "--host-ip", - "192.168.1.20", - "--name", - name, - ], - terminal_width=240, - ) - - assert result.exit_code == 2 - assert "--name" in _normalized_output(result) - - -def test_cli_add_gateway_registration_reports_invalid_existing_config( - tmp_path: Path, -) -> None: - path = tmp_path / "gateway.toml" - path.write_text("not valid TOML") - - result = CliRunner().invoke( - app, - [ - "add-gateway-registration", - "--host-ip", - "192.168.1.20", - "--config", - str(path), - ], - ) - - assert result.exit_code == 1 - output = _plain_output(result) - assert "Could not add or update the OpenShell gateway registration" in output - assert "not valid TOML" in output - assert path.read_text() == "not valid TOML" - - -def test_cli_remove_gateway_registration_removes_the_named_registration( - tmp_path: Path, -) -> None: - path = tmp_path / "gateway.toml" - path.write_text( - "[openshell]\n" - "version = 1\n\n" - "[[openshell.supervisor.middleware]]\n" - 'name = "privacy-guard-regex"\n' - 'grpc_endpoint = "http://192.168.1.20:50051"\n' - ) - - result = CliRunner().invoke( - app, - [ - "remove-gateway-registration", - "--name", - "privacy-guard-regex", - "--config", - str(path), - ], - ) - - assert result.exit_code == 0 - output = _plain_output(result) - assert f"Removed privacy-guard-regex from {path}" in output - assert "restart the OpenShell gateway" in output - assert "privacy-guard-regex" not in path.read_text() - - -def test_cli_remove_gateway_registration_requires_a_name() -> None: - result = CliRunner().invoke( - app, - ["remove-gateway-registration"], - terminal_width=240, - ) - - assert result.exit_code == 2 - output = _normalized_output(result) - assert "--name" in output - assert "missing option" in output.lower() - - -def test_cli_remove_gateway_registration_reports_absent_name( - tmp_path: Path, -) -> None: - path = tmp_path / "gateway.toml" - path.write_text("[openshell]\nversion = 1\n") - - result = CliRunner().invoke( - app, - [ - "remove-gateway-registration", - "--name", - "privacy-guard-regex", - "--config", - str(path), - ], - ) - - assert result.exit_code == 0 - assert ( - f"No registration named privacy-guard-regex found in {path}" - in _plain_output(result) - ) - - -def test_console_script_targets_the_cli_module() -> None: - console_script = next( - entry_point - for entry_point in entry_points(group="console_scripts") - if entry_point.name == "privacy-guard" - ) - - assert console_script.value == "privacy_guard.cli:app" - - -def test_cli_serve_help_explains_the_processing_timeout() -> None: - result = CliRunner().invoke(app, ["serve", "--help"]) - - assert result.exit_code == 0 - output = _normalized_output(result) - assert "--timeout-seconds" in output - assert "shared by all processing stages" in output - assert "at most 30" in output - - -def test_cli_engines_describes_the_installed_engine() -> None: - result = CliRunner().invoke(app, ["engines"]) - - assert result.exit_code == 0 - assert result.output.startswith("regex\tdetect,replace\t") - description = ( - "Detect every regex match, including matches that share input characters" - ) - assert description in result.output - - -def test_cli_configuration_schema_prints_finalized_policy_schema() -> None: - result = CliRunner().invoke(app, ["configuration-schema"]) - - assert result.exit_code == 0 - schema = json.loads(result.output) - serialized = json.dumps(schema, sort_keys=True) - assert '"propertyName": "engine"' in serialized - assert '"regex"' in serialized - assert '"on_detection"' in serialized - - -def test_cli_loads_one_finalized_operator_registry( - monkeypatch: pytest.MonkeyPatch, -) -> None: - registry = create_builtin_registry() - factory_calls = 0 - - def create_registry() -> EngineRegistry: - nonlocal factory_calls - factory_calls += 1 - return registry - - monkeypatch.setattr( - cli_module.importlib, - "import_module", - lambda module_name: ( - SimpleNamespace(create_registry=create_registry) - if module_name == "operator_engines" - else None - ), - ) - - result = CliRunner().invoke( - app, - ["--registry-factory", "operator_engines:create_registry", "engines"], - ) - - assert result.exit_code == 0 - assert factory_calls == 1 - assert result.output.startswith("regex\tdetect,replace\t") - - -@pytest.mark.parametrize( - ("factory_reference", "reason"), - [ - ("missing-separator", "my_engines:create_registry"), - ("operator_engines:missing", "Verify the module:factory reference"), - ("operator_engines:not_callable", "Export a callable"), - ("operator_engines:failed", "Run the factory directly"), - ("operator_engines:wrong_type", "Return an EngineRegistry"), - ("operator_engines:unfinished", "Call finalize()"), - ], -) -def test_cli_rejects_invalid_registry_factories( - monkeypatch: pytest.MonkeyPatch, - factory_reference: str, - reason: str, -) -> None: - def fail() -> EngineRegistry: - raise RuntimeError("sensitive factory failure") - - module = SimpleNamespace( - not_callable=object(), - failed=fail, - wrong_type=lambda: object(), - unfinished=lambda: EngineRegistry(), - ) - monkeypatch.setattr( - cli_module.importlib, - "import_module", - lambda _: module, - ) - - result = CliRunner().invoke( - app, - ["--registry-factory", factory_reference, "engines"], - terminal_width=240, - ) - - assert result.exit_code == 2 - assert reason in _normalized_output(result) - assert "sensitive factory failure" not in _plain_output(result) - - -def test_cli_explains_registry_module_import_failures_without_leaking_details( - monkeypatch: pytest.MonkeyPatch, -) -> None: - def fail_import(_: str) -> object: - raise RuntimeError("sensitive import failure") - - monkeypatch.setattr(cli_module.importlib, "import_module", fail_import) - - result = CliRunner().invoke( - app, - ["--registry-factory", "operator_engines:create_registry", "engines"], - terminal_width=240, - ) - - assert result.exit_code == 2 - output = _normalized_output(result) - assert "Registry module could not be imported" in output - assert "import the module directly with content-safe diagnostics" in output - assert "sensitive import failure" not in _plain_output(result) - - -def test_cli_serve_adapts_operational_options_to_the_programmatic_server( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls: list[tuple[str, float, bool]] = [] - - def record_serve_sync(self: PrivacyGuardServer, listen: str) -> None: - calls.append( - ( - listen, - self._middleware._policy._timeout_seconds, - self._middleware._policy._log_request_content, - ) - ) - - monkeypatch.setattr(PrivacyGuardServer, "serve_sync", record_serve_sync) - - result = CliRunner().invoke( - app, - [ - "--debug-log-content", - "serve", - "--listen", - "127.0.0.1:50052", - "--timeout-seconds", - "4.5", - ], - ) - - assert result.exit_code == 0 - assert calls == [("127.0.0.1:50052", 4.5, True)] - assert "privacy_guard_request_content_logging_enabled" in _plain_output(result) - - -def test_cli_serve_prints_cataloged_startup_errors_without_a_traceback( - monkeypatch: pytest.MonkeyPatch, -) -> None: - def fail_safely(_: PrivacyGuardServer, listen: str) -> None: - del listen - raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) - - monkeypatch.setattr(PrivacyGuardServer, "serve_sync", fail_safely) - - result = CliRunner().invoke( - app, - ["serve", "--listen", "sensitive-listen-address"], - ) - - assert result.exit_code == 1 - assert "[server_bind_failed]" in result.output - assert "Choose an available listen address and port, then retry" in result.output - assert "sensitive-listen-address" not in result.output - assert "Traceback" not in result.output - - -@pytest.mark.parametrize("timeout_seconds", ["0", "31", "nan"]) -def test_cli_rejects_invalid_processing_timeout(timeout_seconds: str) -> None: - result = CliRunner().invoke( - app, - ["serve", "--timeout-seconds", timeout_seconds], - terminal_width=240, - ) - - assert result.exit_code == 2 - output = _normalized_output(result) - assert "--timeout-seconds" in output - assert "greater than 0 and at most 30" in output - - -def _normalized_output(result: Result) -> str: - return " ".join(_plain_output(result).replace("│", " ").split()) - - -def _plain_output(result: Result) -> str: - return _ANSI_STYLE_PATTERN.sub("", result.output) - - -_ANSI_STYLE_PATTERN = re.compile(r"\x1b\[[0-9;]*m") diff --git a/projects/privacy-guard/tests/test_config.py b/projects/privacy-guard/tests/test_config.py deleted file mode 100644 index 58b936da..00000000 --- a/projects/privacy-guard/tests/test_config.py +++ /dev/null @@ -1,558 +0,0 @@ -from __future__ import annotations - -import os -import subprocess -import sys -from collections.abc import Callable -from copy import deepcopy -from pathlib import Path - -import pytest -import yaml -from pydantic import ValidationError - -import privacy_guard.engines.regex as regex_module -from privacy_guard.config import ( - PolicyAction, -) -from privacy_guard.engines import ( - RegexEngine, - RegexEngineConfig, - RegexPatternCatalog, -) -from privacy_guard.engines.registry import EngineRegistry -from privacy_guard.errors import ErrorCode, PrivacyGuardError - - -def _registry() -> EngineRegistry: - registry = EngineRegistry() - registry.register(RegexEngine) - registry.finalize() - return registry - - -def _config( - *, - action: str = "detect", - replacement: dict[str, object] | None = None, - stage_name: str | None = None, -): - engine_config = { - "engine": "regex", - "pattern_catalog": { - "entities": [ - { - "name": "email", - "rules": [ - { - "pattern": r"\buser@example\.com\b", - "confidence": "high", - } - ], - } - ] - }, - } - if replacement is not None: - engine_config["replacement"] = replacement - stage = {"config": engine_config} - if stage_name is not None: - stage["name"] = stage_name - return { - "entity_processing": {"stages": [stage]}, - "on_detection": {"action": action}, - } - - -@pytest.mark.parametrize("action", list(PolicyAction)) -def test_policy_action_uses_detect_block_replace(action: PolicyAction) -> None: - replacement: dict[str, object] | None = ( - {"strategy": "template", "template": "[{entity}]"} - if action is PolicyAction.REPLACE - else None - ) - config = _registry().validate_config( - _config(action=action.value, replacement=replacement) - ) - - assert config.on_detection.action is action - assert [item.value for item in PolicyAction] == ["detect", "block", "replace"] - - -def test_known_discriminator_constructs_the_exact_engine_config() -> None: - config = _registry().validate_config(_config()) - stage = config.entity_processing.stages[0] - - assert type(stage.config) is RegexEngineConfig - assert type(stage.config.pattern_catalog) is RegexPatternCatalog - assert stage.config.pattern_catalog.entities[0].rules[0].name is None - assert stage.diagnostic_name(1) == "regex[1]" - - -def test_policy_accepts_ten_stages_and_rejects_eleven() -> None: - registry = _registry() - exact = _config() - stage = deepcopy(exact["entity_processing"]["stages"][0]) - exact["entity_processing"]["stages"] = [deepcopy(stage) for _ in range(10)] - oversized = deepcopy(exact) - oversized["entity_processing"]["stages"].append(deepcopy(stage)) - - parsed = registry.validate_config(exact) - - assert len(parsed.entity_processing.stages) == 10 - with pytest.raises(PrivacyGuardError) as captured: - registry.validate_config(oversized) - assert captured.value.code is ErrorCode.CONFIG_INVALID - - -def test_explicit_stage_name_is_the_diagnostic_source() -> None: - config = _registry().validate_config(_config(stage_name="credentials")) - - assert config.entity_processing.stages[0].diagnostic_name(1) == "credentials" - - -def test_discriminated_union_round_trip_preserves_concrete_fields() -> None: - registry = _registry() - parsed = registry.validate_config( - _config( - action="replace", - replacement={"strategy": "template", "template": "[{entity}]"}, - ) - ) - serialized = parsed.model_dump(mode="json") - reparsed = registry.validate_config(serialized) - - assert type(reparsed.entity_processing.stages[0].config) is RegexEngineConfig - assert reparsed == parsed - assert serialized["entity_processing"]["stages"][0]["config"]["engine"] == "regex" - assert ( - serialized["entity_processing"]["stages"][0]["config"]["replacement"][ - "strategy" - ] - == "template" - ) - - -def test_generated_schema_declares_the_engine_discriminator() -> None: - schema = _registry().configuration_json_schema() - definitions = _required_dict(schema, "$defs") - stage_definition = next( - definition - for name, definition in definitions.items() - if isinstance(name, str) and name.startswith("EntityProcessingStage") - ) - properties = _required_dict(stage_definition, "properties") - config_schema = _required_dict(properties, "config") - - assert config_schema["discriminator"] == { - "mapping": {"regex": "#/$defs/RegexEngineConfig"}, - "propertyName": "engine", - } - - -def _required_dict(mapping: object, key: str): - assert isinstance(mapping, dict) - value = mapping.get(key) - assert isinstance(value, dict) - return value - - -def test_catalog_file_and_inline_catalog_produce_the_same_config( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - registry = _registry() - inline_values = _config() - file_values = deepcopy(inline_values) - inline_catalog = file_values["entity_processing"]["stages"][0]["config"][ - "pattern_catalog" - ] - (tmp_path / "patterns.yaml").write_text( - yaml.safe_dump(inline_catalog), - encoding="utf-8", - ) - file_values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = ( - "patterns.yaml" - ) - monkeypatch.chdir(tmp_path) - - inline_config = registry.validate_config(inline_values) - file_config = registry.validate_config(file_values) - - assert file_config == inline_config - serialized_catalog = file_config.model_dump(mode="json")["entity_processing"][ - "stages" - ][0]["config"]["pattern_catalog"] - inline_serialized_catalog = inline_config.model_dump(mode="json")[ - "entity_processing" - ]["stages"][0]["config"]["pattern_catalog"] - assert serialized_catalog == inline_serialized_catalog - assert isinstance(serialized_catalog, dict) - - -def test_catalog_file_change_produces_a_different_validated_config( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - catalog_path = tmp_path / "patterns.yaml" - values = _config() - catalog = values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] - catalog_path.write_text(yaml.safe_dump(catalog), encoding="utf-8") - values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = ( - "patterns.yaml" - ) - monkeypatch.chdir(tmp_path) - registry = _registry() - first = registry.validate_config(values) - - catalog["entities"][0]["rules"][0]["confidence"] = "low" - catalog_path.write_text(yaml.safe_dump(catalog), encoding="utf-8") - second = registry.validate_config(values) - - assert first != second - - -def test_equivalent_catalogs_reuse_compiled_regex_rules( - monkeypatch: pytest.MonkeyPatch, -) -> None: - regex_module._clear_compiled_pattern_cache() - original_compile_rule = regex_module._compile_rule - compile_calls = 0 - - def record_compile_rule( - entity: regex_module.RegexEntity, - rule: regex_module.RegexRule, - catalog_index: int, - entity_rule_index: int, - ) -> regex_module._CompiledRule: - nonlocal compile_calls - compile_calls += 1 - return original_compile_rule( - entity, - rule, - catalog_index, - entity_rule_index, - ) - - monkeypatch.setattr(regex_module, "_compile_rule", record_compile_rule) - registry = _registry() - first = registry.validate_config(_config()) - registry.create_engine(first.entity_processing.stages[0].config) - second = registry.validate_config(deepcopy(_config())) - registry.create_engine(second.entity_processing.stages[0].config) - changed_values = deepcopy(_config()) - changed_values["entity_processing"]["stages"][0]["config"]["pattern_catalog"][ - "entities" - ][0]["rules"][0]["pattern"] = "changed" - changed = registry.validate_config(changed_values) - registry.create_engine(changed.entity_processing.stages[0].config) - - assert compile_calls == 2 - regex_module._clear_compiled_pattern_cache() - - -@pytest.mark.parametrize( - "catalog_path", - [ - "missing.yaml", - "../patterns.yaml", - "patterns.json", - ], -) -def test_catalog_file_rejects_invalid_paths( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - catalog_path: str, -) -> None: - values = _config() - values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = catalog_path - monkeypatch.chdir(tmp_path) - - with pytest.raises(PrivacyGuardError) as exception_info: - _registry().validate_config(values) - - assert exception_info.value.code is ErrorCode.CONFIG_INVALID - - -def test_catalog_file_rejects_absolute_paths( - tmp_path: Path, -) -> None: - values = _config() - values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = str( - tmp_path / "patterns.yaml" - ) - - with pytest.raises(PrivacyGuardError) as exception_info: - _registry().validate_config(values) - - assert exception_info.value.code is ErrorCode.CONFIG_INVALID - - -def test_catalog_file_rejects_symlinks( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - target = tmp_path / "target.yaml" - target.write_text("entities: []\n", encoding="utf-8") - (tmp_path / "patterns.yaml").symlink_to(target) - values = _config() - values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = ( - "patterns.yaml" - ) - monkeypatch.chdir(tmp_path) - - with pytest.raises(PrivacyGuardError) as exception_info: - _registry().validate_config(values) - - assert exception_info.value.code is ErrorCode.CONFIG_INVALID - - -def test_catalog_file_rejects_a_symlink_swap_during_open( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - catalog_root = tmp_path / "catalog-root" - catalog_root.mkdir() - catalog_path = catalog_root / "patterns.yaml" - values = _config() - catalog = values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] - catalog_path.write_text(yaml.safe_dump(catalog), encoding="utf-8") - outside_path = tmp_path / "outside.yaml" - outside_path.write_text(yaml.safe_dump(catalog), encoding="utf-8") - values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = ( - "patterns.yaml" - ) - monkeypatch.chdir(catalog_root) - - original_open = os.open - swapped = False - - def swap_before_final_open( - path: str | bytes | Path, - flags: int, - mode: int = 0o777, - *, - dir_fd: int | None = None, - ) -> int: - nonlocal swapped - if path == "patterns.yaml" and dir_fd is not None and not swapped: - swapped = True - catalog_path.unlink() - catalog_path.symlink_to(outside_path) - return original_open(path, flags, mode, dir_fd=dir_fd) - - monkeypatch.setattr(regex_module.os, "open", swap_before_final_open) - - with pytest.raises(PrivacyGuardError) as exception_info: - _registry().validate_config(values) - - assert swapped is True - assert exception_info.value.code is ErrorCode.CONFIG_INVALID - - -def test_catalog_file_rejects_a_fifo_without_blocking(tmp_path: Path) -> None: - os.mkfifo(tmp_path / "patterns.yaml") - probe = """ -from privacy_guard.engines.regex import _load_pattern_catalog_file - -try: - _load_pattern_catalog_file("patterns.yaml") -except ValueError: - pass -else: - raise AssertionError("FIFO catalog was accepted") -""" - - subprocess.run( - [sys.executable, "-c", probe], - cwd=tmp_path, - check=True, - timeout=5, - ) - - -@pytest.mark.parametrize( - "contents", - [ - "entities:\n - name: first\n name: duplicate\n rules: []\n", - ( - "entities:\n" - " - &shared\n" - " name: first\n" - " rules:\n" - " - pattern: x\n" - " confidence: high\n" - " - *shared\n" - ), - "entities: !!python/object/apply:builtins.list []\n", - ], -) -def test_catalog_file_rejects_unsafe_yaml( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - contents: str, -) -> None: - (tmp_path / "patterns.yaml").write_text(contents, encoding="utf-8") - values = _config() - values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = ( - "patterns.yaml" - ) - monkeypatch.chdir(tmp_path) - - with pytest.raises(PrivacyGuardError) as exception_info: - _registry().validate_config(values) - - assert exception_info.value.code is ErrorCode.CONFIG_INVALID - - -def test_catalog_file_rejects_invalid_utf8( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - (tmp_path / "patterns.yaml").write_bytes(b"\xff") - values = _config() - values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = ( - "patterns.yaml" - ) - monkeypatch.chdir(tmp_path) - - with pytest.raises(PrivacyGuardError) as exception_info: - _registry().validate_config(values) - - assert exception_info.value.code is ErrorCode.CONFIG_INVALID - - -def test_catalog_file_rejects_oversized_content( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(regex_module, "MAX_REGEX_CATALOG_FILE_BYTES", 1) - (tmp_path / "patterns.yaml").write_text("entities: []\n", encoding="utf-8") - values = _config() - values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = ( - "patterns.yaml" - ) - monkeypatch.chdir(tmp_path) - - with pytest.raises(PrivacyGuardError) as exception_info: - _registry().validate_config(values) - - assert exception_info.value.code is ErrorCode.CONFIG_INVALID - - -def test_replace_requires_a_replacement_recipe_on_every_stage() -> None: - with pytest.raises(PrivacyGuardError) as exception_info: - _registry().validate_config(_config(action="replace")) - - assert exception_info.value.code is ErrorCode.CONFIG_INVALID - - -@pytest.mark.parametrize("action", ["detect", "block"]) -def test_dormant_replacement_recipe_is_valid_for_detection_only_actions( - action: str, -) -> None: - config = _registry().validate_config( - _config( - action=action, - replacement={"strategy": "template", "template": "[redacted]"}, - ) - ) - engine_config = config.entity_processing.stages[0].config - - assert isinstance(engine_config, RegexEngineConfig) - assert engine_config.replacement is not None - - -@pytest.mark.parametrize( - "mutation", - [ - lambda values: values.update({"body_format": "json"}), - lambda values: values.update({"on_finding": {"action": "observe"}}), - lambda values: values["on_detection"].update({"action": "observe"}), - lambda values: values["on_detection"].update({"action": "redact"}), - lambda values: values["entity_processing"]["stages"][0]["config"].update( - {"kind": "regex"} - ), - lambda values: values["entity_processing"]["stages"][0]["config"].update( - {"preset": "pii"} - ), - ], -) -def test_removed_or_unknown_policy_fields_are_rejected( - mutation: Callable[[dict[str, object]], None], -) -> None: - values = _config() - mutation(values) - - with pytest.raises(PrivacyGuardError): - _registry().validate_config(values) - - -def test_stage_list_is_non_empty_and_explicit_names_are_unique() -> None: - empty = _config() - empty["entity_processing"]["stages"] = [] - duplicate = _config(stage_name="same") - duplicate["entity_processing"]["stages"].append( - deepcopy(duplicate["entity_processing"]["stages"][0]) - ) - - with pytest.raises(PrivacyGuardError): - _registry().validate_config(empty) - with pytest.raises(PrivacyGuardError): - _registry().validate_config(duplicate) - - -def test_explicit_stage_name_cannot_collide_with_a_derived_name() -> None: - values = _config(stage_name="regex[2]") - values["entity_processing"]["stages"].append( - deepcopy(_config()["entity_processing"]["stages"][0]) - ) - - with pytest.raises(PrivacyGuardError): - _registry().validate_config(values) - - -def test_regex_rule_names_are_optional_but_supplied_names_are_unique() -> None: - values = _config() - rules = values["entity_processing"]["stages"][0]["config"]["pattern_catalog"][ - "entities" - ][0]["rules"] - rules.extend( - [ - {"pattern": "second", "confidence": "low"}, - {"name": "named", "pattern": "third", "confidence": "medium"}, - ] - ) - config = _registry().validate_config(values) - - regex_config = config.entity_processing.stages[0].config - assert isinstance(regex_config, RegexEngineConfig) - parsed_rules = regex_config.pattern_catalog.entities[0].rules - assert [rule.name for rule in parsed_rules] == [None, None, "named"] - - rules.append({"name": "named", "pattern": "duplicate", "confidence": "high"}) - with pytest.raises(PrivacyGuardError): - _registry().validate_config(values) - - -def test_validated_config_equality_covers_concrete_expanded_config() -> None: - registry = _registry() - first = registry.validate_config(_config()) - equivalent = registry.validate_config(deepcopy(_config())) - changed_values = _config() - changed_values["entity_processing"]["stages"][0]["config"]["pattern_catalog"][ - "entities" - ][0]["rules"][0]["confidence"] = "low" - changed = registry.validate_config(changed_values) - - assert first == equivalent - assert first != changed - - -def test_models_are_frozen_and_hide_engine_configuration_from_repr() -> None: - config = _registry().validate_config(_config()) - pattern = "sensitive-pattern-value" - - with pytest.raises(ValidationError): - setattr(config.on_detection, "action", PolicyAction.BLOCK) - assert pattern not in repr(config) diff --git a/projects/privacy-guard/tests/test_errors.py b/projects/privacy-guard/tests/test_errors.py deleted file mode 100644 index e38e0eda..00000000 --- a/projects/privacy-guard/tests/test_errors.py +++ /dev/null @@ -1,45 +0,0 @@ -import inspect - -from privacy_guard.errors import ( - ErrorCode, - ErrorComponent, - ErrorKind, - PrivacyGuardError, -) - - -def test_every_error_code_has_one_safe_complete_specification() -> None: - sentinel = "sensitive-request-value-8472" - - assert len({code.value for code in ErrorCode}) == len(ErrorCode) - for code in ErrorCode: - error = PrivacyGuardError(code) - message = str(error) - - assert f"[{code.value}]" in message - assert error.component.value in message - assert error.operation in message - assert error.summary in message - assert error.hint in message - assert sentinel not in message - assert repr(error) == f"PrivacyGuardError({message!r})" - - -def test_error_kinds_distinguish_invalid_input_from_internal_failures() -> None: - assert PrivacyGuardError(ErrorCode.CONFIG_INVALID).kind is ErrorKind.INVALID_INPUT - assert ( - PrivacyGuardError(ErrorCode.ENGINE_EXECUTION_FAILED).kind is ErrorKind.INTERNAL - ) - assert ( - PrivacyGuardError(ErrorCode.CONFIG_INVALID).component is ErrorComponent.CONFIG - ) - - -def test_config_error_explains_the_transport_size_limit() -> None: - error = PrivacyGuardError(ErrorCode.CONFIG_INVALID) - - assert "encoded configuration at or below 64 KiB" in error.hint - - -def test_privacy_guard_error_exposes_only_a_catalog_code_parameter() -> None: - assert list(inspect.signature(PrivacyGuardError).parameters) == ["code"] diff --git a/projects/privacy-guard/tests/test_request_processor.py b/projects/privacy-guard/tests/test_request_processor.py deleted file mode 100644 index be5a22cf..00000000 --- a/projects/privacy-guard/tests/test_request_processor.py +++ /dev/null @@ -1,221 +0,0 @@ -"""RequestProcessor tests for the one-text, ordered-stage contract.""" - -from __future__ import annotations - -import logging -from concurrent.futures import ThreadPoolExecutor -from time import monotonic - -import pytest - -from privacy_guard.config import PolicyAction -from privacy_guard.constants import MAX_BODY_BYTES -from privacy_guard.engines import RegexEngine -from privacy_guard.engines.registry import EngineRegistry -from privacy_guard.errors import ( - EngineConfigurationError, - EngineLimitExceededError, - ErrorCode, - PrivacyGuardError, -) -from privacy_guard.request_processor import RequestDecision, RequestProcessor -from privacy_guard.string_validators import validate_scalar_string -from privacy_guard.timeout import Timeout - - -def _values( - action: PolicyAction, - *, - include_second_stage: bool = True, -) -> dict[str, object]: - stages: list[dict[str, object]] = [ - { - "name": "people", - "config": { - "engine": "regex", - "pattern_catalog": { - "entities": [ - { - "name": "person", - "rules": [ - { - "pattern": "Alice", - "confidence": "high", - } - ], - } - ] - }, - "replacement": { - "strategy": "template", - "template": "[{entity}]", - }, - }, - } - ] - if include_second_stage: - stages.append( - { - "config": { - "engine": "regex", - "pattern_catalog": { - "entities": [ - { - "name": "marker", - "rules": [ - { - "pattern": "person", - "confidence": "medium", - } - ], - } - ] - }, - "replacement": { - "strategy": "template", - "template": "<{entity}>", - }, - }, - } - ) - return { - "entity_processing": {"stages": stages}, - "on_detection": {"action": action.value}, - } - - -def _processor( - action: PolicyAction, - *, - include_second_stage: bool = True, -) -> RequestProcessor: - registry = EngineRegistry() - registry.register(RegexEngine) - registry.finalize() - config = registry.validate_config( - _values(action, include_second_stage=include_second_stage) - ) - stages = tuple( - ( - stage.diagnostic_name(index), - registry.create_engine(stage.config), - ) - for index, stage in enumerate(config.entity_processing.stages, start=1) - ) - return RequestProcessor(config, stages) - - -def test_replace_runs_stages_sequentially_over_the_current_text() -> None: - result = _processor(PolicyAction.REPLACE).process("Hello Alice") - - assert result.decision is RequestDecision.ALLOW - assert result.replacement_text == "Hello []" - assert tuple( - (item.entity, item.source_stage, item.count) - for item in result.detection_summaries - ) == ( - ("person", "people", 1), - ("marker", "regex[2]", 1), - ) - - -def test_detect_reports_without_returning_replacement_text() -> None: - result = _processor(PolicyAction.DETECT).process("Hello Alice") - - assert result.decision is RequestDecision.ALLOW - assert result.replacement_text is None - assert tuple(item.entity for item in result.detection_summaries) == ("person",) - - -def test_block_is_a_processor_disposition_not_an_engine_strategy() -> None: - result = _processor(PolicyAction.BLOCK).process("Hello Alice") - - assert result.decision is RequestDecision.DENY - assert result.replacement_text is None - assert result.reason_code == "privacy_guard_blocked" - assert tuple(item.entity for item in result.detection_summaries) == ("person",) - - -def test_scalar_validation_rejects_lone_surrogates() -> None: - with pytest.raises(ValueError, match="Unicode scalar"): - validate_scalar_string("\ud800") - - -def test_processor_accepts_exact_body_limit_and_rejects_one_byte_more() -> None: - processor = _processor(PolicyAction.DETECT, include_second_stage=False) - - exact = processor.process("x" * MAX_BODY_BYTES) - - assert exact.decision is RequestDecision.ALLOW - with pytest.raises(PrivacyGuardError) as captured: - processor.process("x" * (MAX_BODY_BYTES + 1)) - assert captured.value.code is ErrorCode.REQUEST_BODY_TOO_LARGE - - -def test_exact_limit_requests_complete_concurrently() -> None: - processor = _processor(PolicyAction.DETECT, include_second_stage=False) - text = "x" * MAX_BODY_BYTES - - with ThreadPoolExecutor(max_workers=4) as executor: - results = tuple(executor.map(processor.process, (text,) * 4)) - - assert all(result.decision is RequestDecision.ALLOW for result in results) - - -def test_exact_limit_request_completes_with_multiple_stages() -> None: - result = _processor(PolicyAction.DETECT).process("x" * MAX_BODY_BYTES) - - assert result.decision is RequestDecision.ALLOW - - -def test_timeout_returns_the_bounded_limit_deny( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - monkeypatch.setattr( - Timeout, - "from_seconds", - classmethod(lambda cls, seconds: cls(deadline=monotonic() - 1)), - ) - - with caplog.at_level(logging.INFO, logger="privacy_guard.request_processor"): - result = _processor(PolicyAction.DETECT).process("Hello Alice") - - assert result.decision is RequestDecision.DENY - assert result.reason_code == "privacy_guard_limit_exceeded" - assert "privacy_guard_processing_limit kind=timeout" in caplog.text - assert "Alice" not in caplog.text - - -def test_engine_resource_limit_returns_the_bounded_limit_deny( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - def exceed_limit(*_: object, **__: object) -> object: - raise EngineLimitExceededError("sensitive resource detail") - - monkeypatch.setattr(RegexEngine, "_run", exceed_limit) - - with caplog.at_level(logging.INFO, logger="privacy_guard.request_processor"): - result = _processor(PolicyAction.DETECT).process("Hello Alice") - - assert result.decision is RequestDecision.DENY - assert result.reason_code == "privacy_guard_limit_exceeded" - assert "privacy_guard_processing_limit kind=resource" in caplog.text - assert "Alice" not in caplog.text - assert "sensitive resource detail" not in caplog.text - - -def test_engine_configuration_failure_maps_to_invalid_config( - monkeypatch: pytest.MonkeyPatch, -) -> None: - def reject_config(*_: object, **__: object) -> object: - raise EngineConfigurationError("sensitive configuration detail") - - monkeypatch.setattr(RegexEngine, "_run", reject_config) - - with pytest.raises(PrivacyGuardError) as captured: - _processor(PolicyAction.DETECT).process("Hello Alice") - - assert captured.value.code is ErrorCode.CONFIG_INVALID - assert "sensitive configuration detail" not in str(captured.value) diff --git a/scripts/build-docs.sh b/scripts/build-docs.sh index 70457c29..18c0662a 100755 --- a/scripts/build-docs.sh +++ b/scripts/build-docs.sh @@ -30,9 +30,11 @@ fi python -m pip install --upgrade pip python -m pip install -r requirements-docs.txt -python scripts/stage-privacy-guard-docs.py +python scripts/stage-egress-gate-docs.py python scripts/render-dev-notes.py zensical build --clean --strict python scripts/publish-agent-markdown.py REQUIRE_RENDERED_AGENT_MARKDOWN=1 python tests/test_agent_markdown.py REQUIRE_RENDERED_404=1 python tests/test_docs_404.py +REQUIRE_RENDERED_NAVIGATION=1 python tests/test_navigation_drawer.py +REQUIRE_RENDERED_PAGE_NAVIGATION=1 python tests/test_page_navigation.py diff --git a/scripts/stage-privacy-guard-docs.py b/scripts/stage-egress-gate-docs.py similarity index 78% rename from scripts/stage-privacy-guard-docs.py rename to scripts/stage-egress-gate-docs.py index bf277bae..ba73bd7f 100644 --- a/scripts/stage-privacy-guard-docs.py +++ b/scripts/stage-egress-gate-docs.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Stage canonical Privacy Guard documentation in the site source tree.""" +"""Stage canonical Egress Gate documentation in the site source tree.""" from __future__ import annotations @@ -12,11 +12,11 @@ ROOT = Path(__file__).resolve().parents[1] -DEFAULT_SOURCE = ROOT / "projects" / "privacy-guard" / "docs" -DEFAULT_DESTINATION = ROOT / "docs" / "documentation" / "privacy-guard" +DEFAULT_SOURCE = ROOT / "projects" / "egress-gate" / "docs" +DEFAULT_DESTINATION = ROOT / "docs" / "documentation" / "egress-gate" -def stage_privacy_guard_docs(source: Path, destination: Path) -> None: +def stage_egress_gate_docs(source: Path, destination: Path) -> None: """Replace the generated site mirror with one canonical project-docs tree.""" source = source.resolve() @@ -46,8 +46,8 @@ def stage_privacy_guard_docs(source: Path, destination: Path) -> None: def main() -> int: - stage_privacy_guard_docs(DEFAULT_SOURCE, DEFAULT_DESTINATION) - print(f"Staged Privacy Guard documentation from {DEFAULT_SOURCE}.") + stage_egress_gate_docs(DEFAULT_SOURCE, DEFAULT_DESTINATION) + print(f"Staged Egress Gate documentation from {DEFAULT_SOURCE}.") return 0 diff --git a/tests/navigation-drawer.test.js b/tests/navigation-drawer.test.js new file mode 100644 index 00000000..d018b976 --- /dev/null +++ b/tests/navigation-drawer.test.js @@ -0,0 +1,287 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); +const test = require("node:test"); +const vm = require("node:vm"); + +const script = fs.readFileSync( + path.join(__dirname, "..", "docs", "javascripts", "navigation-drawer.js"), + "utf8", +); + +class TestEvent { + constructor(type, options = {}) { + this.type = type; + Object.assign(this, options); + this.defaultPrevented = false; + } + + preventDefault() { + this.defaultPrevented = true; + } +} + +class TestElement { + constructor(tagName, document) { + this.tagName = tagName.toUpperCase(); + this.ownerDocument = document; + this.attributes = new Map(); + this.children = []; + this.listeners = new Map(); + this.focusables = []; + this.parentElement = null; + this.hidden = false; + this.inert = false; + this.tabIndex = 0; + this.visible = true; + } + + addEventListener(type, listener) { + const listeners = this.listeners.get(type) ?? []; + listeners.push(listener); + this.listeners.set(type, listeners); + } + + removeEventListener(type, listener) { + this.listeners.set( + type, + (this.listeners.get(type) ?? []).filter((candidate) => candidate !== listener), + ); + } + + dispatchEvent(event) { + event.target ??= this; + for (const listener of this.listeners.get(event.type) ?? []) listener(event); + } + + append(...children) { + for (const child of children) { + child.parentElement = this; + this.children.push(child); + } + } + + setAttribute(name, value) { + this.attributes.set(name, String(value)); + } + + getAttribute(name) { + return this.attributes.get(name) ?? null; + } + + removeAttribute(name) { + this.attributes.delete(name); + } + + querySelectorAll() { + return this.focusables; + } + + contains(element) { + for (let current = element; current; current = current.parentElement) { + if (current === this) return true; + } + return false; + } + + closest(selector) { + for (let current = this; current; current = current.parentElement) { + if (selector === "[inert]" && current.inert) return current; + if ( + selector === "a[href]" && + current.tagName === "A" && + current.attributes.has("href") + ) { + return current; + } + } + return null; + } + + getClientRects() { + return this.visible ? [{}] : []; + } + + focus() { + this.ownerDocument.activeElement = this; + } +} + +class TestInput extends TestElement { + constructor(document) { + super("input", document); + this.checked = false; + } +} + +class TestDocument extends TestElement { + constructor() { + super("document", null); + this.ownerDocument = this; + this.activeElement = null; + this.readyState = "complete"; + this.elements = new Map(); + this.documentElement = new TestElement("html", this); + this.documentElement.dataset = {}; + this.documentElement.classList = { + add() {}, + remove() {}, + }; + } + + querySelector(selector) { + return this.elements.get(selector) ?? null; + } + + querySelectorAll() { + return []; + } +} + +class TestMediaQuery extends TestElement { + constructor(document, matches) { + super("media-query", document); + this.matches = matches; + } +} + +function createFixture({ modal = false, storedOpen = false } = {}) { + const document = new TestDocument(); + const toggle = new TestInput(document); + const sidebar = new TestElement("aside", document); + const overlay = new TestElement("label", document); + const button = new TestElement("label", document); + const container = new TestElement("div", document); + const header = new TestElement("header", document); + const main = new TestElement("main", document); + const firstLink = new TestElement("a", document); + const hiddenLink = new TestElement("a", document); + const lastLink = new TestElement("a", document); + const outside = new TestElement("a", document); + const media = new TestMediaQuery(document, modal); + const storage = new Map([ + ["openshell.navigationDrawerOpen", String(storedOpen)], + ]); + + firstLink.setAttribute("href", "/first/"); + hiddenLink.setAttribute("href", "/hidden/"); + hiddenLink.visible = false; + lastLink.setAttribute("href", "/last/"); + sidebar.append(firstLink, hiddenLink, lastLink); + sidebar.focusables = [firstLink, hiddenLink, lastLink]; + header.append(button); + main.append(sidebar); + + document.elements.set("#__drawer", toggle); + document.elements.set(".md-sidebar--primary", sidebar); + document.elements.set('.md-overlay[for="__drawer"]', overlay); + document.elements.set(".openshell-drawer-button", button); + document.elements.set(".md-container", container); + + const window = { + document$: undefined, + getComputedStyle(element) { + return { visibility: element.visible ? "visible" : "hidden" }; + }, + matchMedia() { + return media; + }, + requestAnimationFrame(callback) { + callback(); + return 1; + }, + sessionStorage: { + getItem(key) { + return storage.get(key) ?? null; + }, + setItem(key, value) { + storage.set(key, value); + }, + }, + }; + + vm.runInNewContext(script, { + document, + Element: TestElement, + HTMLElement: TestElement, + HTMLInputElement: TestInput, + window, + }); + + return { + button, + document, + firstLink, + hiddenLink, + lastLink, + media, + outside, + sidebar, + storage, + toggle, + }; +} + +test("desktop restores state without adding a duplicate navigation landmark", () => { + const fixture = createFixture({ storedOpen: true }); + + assert.equal(fixture.toggle.checked, true); + assert.equal(fixture.document.documentElement.dataset.navigationDrawer, "open"); + assert.equal(fixture.sidebar.getAttribute("role"), null); + assert.equal(fixture.button.getAttribute("aria-expanded"), "true"); +}); + +test("mobile navigation closes the modal and clears saved state", () => { + const fixture = createFixture({ modal: true, storedOpen: true }); + + assert.equal(fixture.document.activeElement, fixture.firstLink); + assert.equal(fixture.sidebar.getAttribute("role"), "dialog"); + assert.equal(fixture.sidebar.getAttribute("aria-modal"), "true"); + + fixture.sidebar.dispatchEvent( + new TestEvent("click", { target: fixture.lastLink }), + ); + + assert.equal(fixture.toggle.checked, false); + assert.equal(fixture.storage.get("openshell.navigationDrawerOpen"), "false"); + assert.equal(fixture.sidebar.inert, true); +}); + +test("keyboard control, Escape, and visible focus endpoints work", () => { + const fixture = createFixture({ modal: true }); + fixture.button.focus(); + + fixture.document.dispatchEvent(new TestEvent("keydown", { key: "Enter" })); + assert.equal(fixture.toggle.checked, true); + + fixture.button.focus(); + fixture.document.dispatchEvent(new TestEvent("keydown", { key: "Enter" })); + assert.equal(fixture.toggle.checked, false); + + fixture.document.dispatchEvent(new TestEvent("keydown", { key: " " })); + assert.equal(fixture.toggle.checked, true); + + fixture.firstLink.focus(); + fixture.document.dispatchEvent( + new TestEvent("keydown", { key: "Tab", shiftKey: true }), + ); + assert.equal(fixture.document.activeElement, fixture.lastLink); + + fixture.document.dispatchEvent(new TestEvent("keydown", { key: "Escape" })); + assert.equal(fixture.toggle.checked, false); + assert.equal(fixture.document.activeElement, fixture.button); +}); + +test("entering modal mode repairs focus", () => { + const fixture = createFixture({ storedOpen: true }); + fixture.outside.focus(); + fixture.media.matches = true; + + fixture.media.dispatchEvent(new TestEvent("change")); + + assert.equal(fixture.sidebar.getAttribute("role"), "dialog"); + assert.equal(fixture.document.activeElement, fixture.firstLink); +}); diff --git a/tests/test_navigation_drawer.py b/tests/test_navigation_drawer.py new file mode 100644 index 00000000..f60a9549 --- /dev/null +++ b/tests/test_navigation_drawer.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +HEADER_TEMPLATE = ROOT / "overrides" / "partials" / "header.html" +MAIN_TEMPLATE = ROOT / "overrides" / "main.html" +DRAWER_SCRIPT = ROOT / "docs" / "javascripts" / "navigation-drawer.js" +DRAWER_STYLES = ROOT / "docs" / "stylesheets" / "dev-notes.css" +RENDERED_PAGE = ROOT / "site" / "documentation" / "index.html" + + +class NavigationDrawerTests(unittest.TestCase): + def test_header_renders_the_final_control(self) -> None: + header = HEADER_TEMPLATE.read_text(encoding="utf-8") + script = DRAWER_SCRIPT.read_text(encoding="utf-8") + + self.assertEqual(header.count("openshell-drawer-button"), 1) + self.assertIn("openshell-drawer-icon-expand", header) + self.assertIn("openshell-drawer-icon-collapse", header) + self.assertNotIn("material/menu", header) + self.assertNotIn(".innerHTML", script) + self.assertNotIn("replaceWith", script) + + def test_saved_state_is_available_before_first_render(self) -> None: + main = MAIN_TEMPLATE.read_text(encoding="utf-8") + styles = DRAWER_STYLES.read_text(encoding="utf-8") + + self.assertIn("document.documentElement.dataset.navigationDrawer", main) + self.assertIn(':root[data-navigation-drawer="open"] .md-main', styles) + self.assertIn( + ':root[data-navigation-drawer="open"] .openshell-drawer-icon-collapse', + styles, + ) + self.assertNotIn("calc(50% - 36rem)", styles) + + def test_rendered_page_contains_one_stable_control(self) -> None: + if os.environ.get("REQUIRE_RENDERED_NAVIGATION") != "1": + self.skipTest("rendered output is checked after the documentation build") + if not RENDERED_PAGE.exists(): + self.fail("the rendered documentation page does not exist") + + html = RENDERED_PAGE.read_text(encoding="utf-8") + head = html[: html.index("")] + + self.assertEqual(html.count("openshell-drawer-button"), 1) + self.assertIn("openshell-drawer-icon-expand", html) + self.assertIn("openshell-drawer-icon-collapse", html) + self.assertIn("document.documentElement.dataset.navigationDrawer", head) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_page_navigation.py b/tests/test_page_navigation.py new file mode 100644 index 00000000..2649c026 --- /dev/null +++ b/tests/test_page_navigation.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +from pathlib import Path +import re +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +CONFIG = ROOT / "zensical.toml" +STYLES = ROOT / "docs" / "stylesheets" / "dev-notes.css" +DOCUMENTATION_LANDING = ROOT / "site" / "documentation" / "index.html" +EGRESS_GATE_LANDING = ROOT / "site" / "documentation" / "egress-gate" / "index.html" +CONFIGURATION_GUIDE = ( + ROOT / "site" / "documentation" / "egress-gate" / "configuration" / "index.html" +) + + +class PageNavigationTests(unittest.TestCase): + def test_landing_page_width_does_not_change_the_shared_header(self) -> None: + styles = STYLES.read_text(encoding="utf-8") + + self.assertNotIn("body:has(.dev-notes-page) .md-grid", styles) + self.assertNotIn("body:has(.openshell-home-page) .md-grid", styles) + self.assertIn( + "body:has(.openshell-home-page) .md-main__inner.md-grid", + styles, + ) + + def test_footer_navigation_is_enabled(self) -> None: + config = CONFIG.read_text(encoding="utf-8") + styles = STYLES.read_text(encoding="utf-8") + + self.assertIn('"navigation.footer"', config) + self.assertRegex( + styles, + re.compile( + r':root\[data-navigation-drawer="open"\] \.md-footer\s*\{' + r"[^}]*padding-left: var\(--openshell-sidebar-width\)", + re.DOTALL, + ), + ) + + def test_rendered_links_follow_the_reading_path(self) -> None: + if os.environ.get("REQUIRE_RENDERED_PAGE_NAVIGATION") != "1": + self.skipTest("rendered output is checked after the documentation build") + + documentation = DOCUMENTATION_LANDING.read_text(encoding="utf-8") + egress_gate = EGRESS_GATE_LANDING.read_text(encoding="utf-8") + configuration = CONFIGURATION_GUIDE.read_text(encoding="utf-8") + + self.assertIn("Back to OpenShell Research", documentation) + self.assertIn("Next: Egress Gate", documentation) + self.assertNotIn("Previous: Bringing Privacy", documentation) + self.assertIn("Previous: Documentation", egress_gate) + self.assertIn("Next: Configure policies", egress_gate) + self.assertIn("Previous: Egress Gate", configuration) + self.assertIn("Next: Test policies offline", configuration) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_stage_privacy_guard_docs.py b/tests/test_stage_egress_gate_docs.py similarity index 84% rename from tests/test_stage_privacy_guard_docs.py rename to tests/test_stage_egress_gate_docs.py index 0230e9cb..418e0ce6 100644 --- a/tests/test_stage_privacy_guard_docs.py +++ b/tests/test_stage_egress_gate_docs.py @@ -10,16 +10,16 @@ ROOT = Path(__file__).resolve().parents[1] -SCRIPT = ROOT / "scripts" / "stage-privacy-guard-docs.py" +SCRIPT = ROOT / "scripts" / "stage-egress-gate-docs.py" -SPEC = importlib.util.spec_from_file_location("stage_privacy_guard_docs", SCRIPT) +SPEC = importlib.util.spec_from_file_location("stage_egress_gate_docs", SCRIPT) if SPEC is None or SPEC.loader is None: raise RuntimeError(f"could not load {SCRIPT}") STAGER = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(STAGER) -class StagePrivacyGuardDocsTests(unittest.TestCase): +class StageEgressGateDocsTests(unittest.TestCase): def test_stage_replaces_destination_with_source_tree(self) -> None: with tempfile.TemporaryDirectory() as temporary_directory: root = Path(temporary_directory) @@ -34,7 +34,7 @@ def test_stage_replaces_destination_with_source_tree(self) -> None: destination.mkdir() (destination / "stale.md").write_text("# Stale\n", encoding="utf-8") - STAGER.stage_privacy_guard_docs(source, destination) + STAGER.stage_egress_gate_docs(source, destination) self.assertEqual( (destination / "index.md").read_text(encoding="utf-8"), @@ -53,7 +53,7 @@ def test_stage_rejects_symlinks_in_source(self) -> None: (source / "linked.md").symlink_to(target) with self.assertRaisesRegex(ValueError, "must not contain symlinks"): - STAGER.stage_privacy_guard_docs(source, root / "site-docs") + STAGER.stage_egress_gate_docs(source, root / "site-docs") def test_stage_rejects_destination_inside_source(self) -> None: with tempfile.TemporaryDirectory() as temporary_directory: @@ -61,7 +61,7 @@ def test_stage_rejects_destination_inside_source(self) -> None: source.mkdir() with self.assertRaisesRegex(ValueError, "must not overlap"): - STAGER.stage_privacy_guard_docs(source, source / "published") + STAGER.stage_egress_gate_docs(source, source / "published") def test_stage_rejects_source_inside_destination(self) -> None: with tempfile.TemporaryDirectory() as temporary_directory: @@ -70,7 +70,7 @@ def test_stage_rejects_source_inside_destination(self) -> None: source.mkdir(parents=True) with self.assertRaisesRegex(ValueError, "must not overlap"): - STAGER.stage_privacy_guard_docs(source, destination) + STAGER.stage_egress_gate_docs(source, destination) if __name__ == "__main__": diff --git a/zensical.toml b/zensical.toml index 41e1b6b0..d9c7e499 100644 --- a/zensical.toml +++ b/zensical.toml @@ -25,24 +25,25 @@ nav = [ ]}, {"Documentation" = [ "documentation/index.md", - {"Privacy Guard" = [ - "documentation/privacy-guard/index.md", + {"Egress Gate" = [ + "documentation/egress-gate/index.md", {"Guides" = [ - {"Configure policies" = "documentation/privacy-guard/configuration.md"}, - {"Run and operate Privacy Guard" = "documentation/privacy-guard/operations.md"} + {"Configure policies" = "documentation/egress-gate/configuration.md"}, + {"Test policies offline" = "documentation/egress-gate/evaluation.md"}, + {"Run and operate Egress Gate" = "documentation/egress-gate/operations.md"} ]}, - {"Engines" = [ - "documentation/privacy-guard/engines/index.md", - {"RegexEngine" = "documentation/privacy-guard/engines/regex.md"}, - {"Add a custom engine" = "documentation/privacy-guard/engines/custom.md"} + {"Gates" = [ + "documentation/egress-gate/gates/index.md", + {"Regex gate" = "documentation/egress-gate/gates/regex.md"}, + {"Add a custom gate" = "documentation/egress-gate/gates/custom.md"} ]}, {"Architecture" = [ - "documentation/privacy-guard/architecture/index.md", - {"Request lifecycle" = "documentation/privacy-guard/architecture/request-lifecycle.md"}, - {"Service boundary" = "documentation/privacy-guard/architecture/service-boundary.md"} + "documentation/egress-gate/architecture/index.md", + {"Request lifecycle" = "documentation/egress-gate/architecture/request-lifecycle.md"}, + {"Service boundary" = "documentation/egress-gate/architecture/service-boundary.md"} ]}, {"Reference" = [ - {"Limits and failure behavior" = "documentation/privacy-guard/reference/limits-and-failures.md"} + {"Limits and failure behavior" = "documentation/egress-gate/reference/limits-and-failures.md"} ]} ]} ]} @@ -54,15 +55,26 @@ generator = false [project.markdown_extensions.admonition] +[project.markdown_extensions."pymdownx.highlight"] +anchor_linenums = true +line_spans = "__span" +pygments_lang_class = true + +[project.markdown_extensions."pymdownx.inlinehilite"] + +[project.markdown_extensions."pymdownx.superfences"] + [project.theme] custom_dir = "overrides" favicon = "assets/brand/favicon.svg" logo = "assets/brand/openshell-mark.svg" icon.repo = "fontawesome/brands/github" features = [ + "content.code.copy", "navigation.sections", "navigation.indexes", "navigation.path", + "navigation.footer", "navigation.top", "search.highlight", "toc.follow"