pipewright-mlops 1.0.0__tar.gz

pipewright_mlops-1.0.0/.dockerignore

@@ -0,0 +1,5 @@
+# The launcher image copies only the built wheel (images/launcher.Dockerfile: COPY dist/). Send nothing else to the
+# build context, so local files (credentials, .env, .git, virtual environments) can never reach an image.
+*
+!dist/
+dist/*.tar.gz

pipewright_mlops-1.0.0/.gitattributes

@@ -0,0 +1,6 @@
+# Store text files with LF line endings for every contributor; shell scripts must stay LF to run in CI.
+* text=auto eol=lf
+*.sh text eol=lf
+*.csv text eol=lf
+*.png binary
+*.whl binary

pipewright_mlops-1.0.0/.github/workflows/ci.yml

@@ -0,0 +1,64 @@
+# Pipewright product CI: every pull request and push to main, and the first stage of release.yml.
+# Lint and tests, the example's tests, the wheel (contents checked, installed fresh, smoke tested), the launcher
+# image built from that wheel (not pushed). The wheel is kept as the `dist` artifact for release.yml.
+name: ci
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  workflow_call:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.11"
+      - run: uv sync --all-extras
+      - run: uv run ruff check .
+      - run: uv run pytest -q
+      - name: Example use case tests
+        run: uv run --no-project --with-requirements examples/churn/tests/requirements.txt python -m pytest examples/churn/tests -q
+
+  package:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.11"
+      - name: Build wheel and sdist
+        run: uv build --out-dir dist
+      - name: Check wheel contents
+        run: uv run --no-project python scripts/check_package.py dist
+      - name: Install the wheel in a fresh environment and smoke test it
+        run: |
+          python3 -m venv "$RUNNER_TEMP/smoke"
+          "$RUNNER_TEMP/smoke/bin/pip" install "$(ls dist/pipewright_mlops-*-py3-none-any.whl)[gcp]"
+          bash scripts/smoke_test.sh "$RUNNER_TEMP/smoke/bin/pipewright"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+          if-no-files-found: error
+
+  launcher-image:
+    needs: package
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Build the launcher image from the wheel (not pushed)
+        run: |
+          rm -f dist/*.tar.gz
+          docker build -f images/launcher.Dockerfile -t pipewright-launcher:ci .

pipewright_mlops-1.0.0/.github/workflows/release.yml

@@ -0,0 +1,132 @@
+# Pipewright release: push a tag v<version> (the version in pyproject.toml) to publish that version on GitHub.
+#
+#   ci                the whole of ci.yml: tests, wheel, smoke test, launcher image build
+#   version           the tag must be v<pyproject version>, or v<pyproject version>-rehearsal.<n> for a rehearsal
+#   publish-launcher  the launcher image, built from the released wheel -> GitHub Container Registry
+#                     (ghcr.io/<owner>/pipewright-launcher:<version>), linked to this repository so it inherits the
+#                     repository's access; platform teams copy it by digest into their environment's own registry
+#   github-release    the GitHub release: the wheel and sdist as files, and the launcher image digest
+#   publish-pypi      the same wheel and sdist to PyPI (pipewright-mlops), real versions only, after the GitHub
+#                     release; PyPI Trusted Publishing, so no API token is stored anywhere
+#
+# Keyless: GitHub's built-in token publishes the image and the release, and PyPI trusts this workflow through
+# OpenID Connect (a trusted publisher for repository fluxorsoft/pipewright, workflow release.yml, environment pypi). A rehearsal
+# (v<version>-rehearsal.<n>) publishes a pre-release and an image tagged <version>-rehearsal.<n>, to be deleted
+# afterwards. A real version is published once: do not delete and re-create its release.
+name: release
+
+on:
+  push:
+    tags: ["v*"]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: release
+  cancel-in-progress: false
+
+jobs:
+  ci:
+    uses: ./.github/workflows/ci.yml
+
+  version:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.v.outputs.version }}
+      image_tag: ${{ steps.v.outputs.image_tag }}
+      prerelease: ${{ steps.v.outputs.prerelease }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: v
+        env:
+          TAG: ${{ github.ref_name }}
+        run: |
+          version="$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
+          case "$TAG" in
+            "v$version") image_tag="$version"; prerelease=false ;;
+            "v$version"-rehearsal.[0-9]*) image_tag="${TAG#v}"; prerelease=true ;;
+            *) echo "::error::tag $TAG is neither v$version nor v$version-rehearsal.<n> (pyproject.toml version $version)"
+               exit 1 ;;
+          esac
+          case "$version" in *dev*|*a*|*b*|*rc*) prerelease=true ;; esac
+          { echo "version=$version"; echo "image_tag=$image_tag"; echo "prerelease=$prerelease"; } >> "$GITHUB_OUTPUT"
+
+  publish-launcher:
+    needs: [ci, version]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    outputs:
+      image: ${{ steps.push.outputs.image }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Build from the released wheel and push to GitHub Container Registry
+        id: push
+        env:
+          VERSION: ${{ needs.version.outputs.version }}
+          TAG: ${{ needs.version.outputs.image_tag }}
+          TOKEN: ${{ github.token }}
+        run: |
+          rm -f dist/*.tar.gz
+          image="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/pipewright-launcher"
+          echo "$TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin
+          docker build -f images/launcher.Dockerfile --build-arg VERSION="$VERSION" \
+            --label "org.opencontainers.image.source=https://github.com/$GITHUB_REPOSITORY" -t "$image:$TAG" .
+          docker push "$image:$TAG"
+          ref="$(docker inspect --format='{{index .RepoDigests 0}}' "$image:$TAG")"
+          echo "image=$ref" >> "$GITHUB_OUTPUT"
+          echo "Launcher image: \`$ref\`" >> "$GITHUB_STEP_SUMMARY"
+
+  github-release:
+    needs: [version, publish-launcher]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Create the GitHub release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          VERSION: ${{ needs.version.outputs.version }}
+          IMAGE: ${{ needs.publish-launcher.outputs.image }}
+          PRERELEASE: ${{ needs.version.outputs.prerelease }}
+        run: |
+          cat > notes.md <<EOF
+          Pipewright $VERSION
+
+          - **Python package:** download the wheel below (\`gh release download $GITHUB_REF_NAME -R $GITHUB_REPOSITORY\`)
+            and install it, or pin \`pipewright-mlops[gcp]==$VERSION\` from PyPI or your organisation's package index.
+          - **Launcher image:** copy it into your environment's own registry by digest, then pin that digest in the
+            platform's Terraform:
+            \`$IMAGE\`
+
+          What changed: CHANGELOG.md. What stays stable: STABILITY.md.
+          EOF
+          flag=""; [ "$PRERELEASE" = true ] && flag="--prerelease"
+          gh release create "$GITHUB_REF_NAME" dist/* --repo "$GITHUB_REPOSITORY" --title "Pipewright $GITHUB_REF_NAME" \
+            --notes-file notes.md $flag
+
+  publish-pypi:
+    needs: [version, github-release]
+    if: needs.version.outputs.prerelease == 'false'     # rehearsals never reach PyPI: a version there is final
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/pipewright-mlops/${{ needs.version.outputs.version }}/
+    permissions:
+      id-token: write                                   # PyPI Trusted Publishing
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@v1.14.2   # twine 7: accepts metadata 2.5 (hatchling)

pipewright_mlops-1.0.0/.gitignore

@@ -0,0 +1,36 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.venv/
+venv/
+
+# Build output. Anchored: src/pipewright/build/ is a package and must stay tracked.
+/dist/
+/build/
+
+# Tool caches
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.coverage
+htmlcov/
+
+# Pipewright's per-use-case build output (compiled templates, image contexts)
+.pipewright/
+
+# Local settings and secrets: never commit credentials, tokens or CA bundles
+.env
+.env.*
+*.pem
+*.key
+credentials*.json
+*-key.json
+service-account*.json
+
+# Editors and operating systems
+.idea/
+.vscode/
+*.swp
+.DS_Store
+Thumbs.db

pipewright_mlops-1.0.0/CHANGELOG.md

@@ -0,0 +1,128 @@
+# Changelog
+
+All notable changes to Pipewright. The format follows [Keep a Changelog](https://keepachangelog.com/), and versions
+follow [semantic versioning](https://semver.org/); [STABILITY.md](STABILITY.md) says what it covers.
+
+## [1.0.0] - 2026-10-18
+
+First release, on GitHub and on PyPI (`pipewright-mlops`). First published on GitHub on 2026-10-17 and re-cut to
+include synthetic data and, for PyPI, a README whose links work outside the repository.
+
+### Added
+
+- **Help with the YAML.** `pipewright explain <key>` (description, type, allowed values, default, whether it may vary
+  per environment); editor schemas in `.pipewright-schemas/` and a `yaml-language-server` line in every generated
+  YAML file, for completion, hover help and underlined mistakes (`pipewright schemas` for existing repos); every use
+  case key described in `config/schemas/reference.yaml`.
+- **Minimal example** (`new-usecase --from minimal`): the churn training code with only the required configuration.
+- **Data contract from BigQuery** (`new-usecase --from-bq <project.dataset.table> --label <column>`): drafts
+  `config/data.yaml` and the raw table's schema from a table or view.
+- **Evaluation gates checked by Pipewright.** `model approve` and `deploy-model` recompute the metric gates, slice
+  gates and champion margin from the version's evaluation record (`evaluation.json`) and refuse a version that fails;
+  `model list` shows each version's result.
+- **Write-once release contracts** and component image tags derived from the release, so re-releases are
+  idempotent and immutable tags work. Contract metadata (owner, objective, description, contacts, model) for tools
+  without a repository checkout.
+- **Per-tenant pipeline roots.** Runs write their artifacts under `pipeline-root/<tenant>/<usecase>/` of the
+  artifacts bucket, so each tenant's runner can be granted its own prefix only.
+- **`pipewright promote`.** A tested release, and optionally its approved model, from one environment to the next:
+  destination policies first, images by digest, templates checked equal (or an accepted, recorded difference), model
+  files verified, the contract written once with `promoted_from`, the pointer moved last; idempotent; a
+  `release.promoted` event. Models carry the release that trained them (a `release` label).
+- **Retrain decisions.** Drift or a performance drop opens a decision case instead of retraining: modes `off`,
+  `propose` and `candidate`; a decision pack (what drifted, the trend, data checks, the champion, what retraining
+  would use); approver groups with a quorum; snoozes and declines that expire; re-opening when things get worse;
+  escalation and a weekly digest; decision events and an opt-in per-tenant feed; `pipewright retrain list | show |
+  approve | decline | snooze`; launcher routes `/retraining/propose`, `/retraining/decide`, `/retraining/tick`.
+- **Features from feature tables.** `data.features_from` names BigQuery feature tables and their features; the
+  run's source is then the entity table and `read_source` joins each feature as it was at the row's cut-off
+  (the newest value at or before it), in one BigQuery query with the environment's key. Scoring without the
+  cut-off column uses the latest values. A `data.features.resolved` event, `validate` and `doctor` checks, and
+  `new-usecase --from features`.
+- **Feature groups as a feature source.** A `data.features_from` source may name a Feature Store feature group; the
+  Feature Registry gives its table and columns at each run. The manifest can list a tenant's feature datasets and
+  require feature tables (`features.required`); `validate`, `doctor` and `read_source` enforce both.
+- **Ray on Agent Platform steps.** `steps.<step>.ray` gives a step a Ray cluster for one run, within the platform's
+  limits (manifest `ray`, per environment and tenant), created by the launcher as the runner identity with the
+  environment's key and Private Service Connect network attachment, and always deleted (by the step, or by the
+  launcher's tick). `pipewright release` builds the use case's Ray image; `ray_cluster()` and `ray_job()` in the
+  component runtime; a Ray client version check; `new-usecase --from ray`; a `ray.cluster.created` event records
+  what each cluster used.
+- **Private-environment support.** The manifest `security` block (CMEK key, BigQuery key, network, reserved IP
+  ranges) is applied to every pipeline job, model, endpoint and BigQuery query; run requests over Pub/Sub
+  (`launcher.run_requests_topic`, `/pubsub/run`); events published through the regional Pub/Sub endpoint.
+- **Manifest forward compatibility.** Manifest keys this version does not know are ignored with a warning
+  (`validate`, `doctor`, the launcher's log) instead of failing; an optional `requires` (for example `">=1.1"`)
+  makes an older version refuse the manifest with a clear message.
+- **`predict` in the generated operate workflow** (`pipewright ci init`): runs `pipewright model predict` against the
+  use case's endpoint or InferenceService with the `instances` input, inline JSON or a JSON file in the repository
+  (for example `usecases/<name>/tests/predict.json`); the predictions go to the job summary.
+- **CLI plugins** through the `pipewright.commands` entry point group.
+
+### Added: private GKE
+
+- **Orchestrator `kfp`: Kubeflow Pipelines on private GKE.** The same use case YAML, compiled templates and release
+  contracts run on Kubeflow Pipelines inside the cluster, submitted through the Kubernetes API (no exposed endpoint)
+  as each tenant's Kubernetes service account, bound to its runner identity. Tasks inside `dsl.OneOf` branches run
+  uncached on Kubeflow (it cannot resolve a cached branch).
+- **Ray on KubeRay.** On GKE a step's cluster is a RayCluster per run on the platform's Ray node pool, reached
+  inside the cluster; or a tenant's **standing** cluster (`ray.mode: standing`), long-lived and autoscaling, to
+  which steps submit with their release's code. The release records the mode it was built for.
+- **Online serving on KServe** (`model.serving.target: kserve`): the registered container and its Agent Platform
+  environment as an InferenceService in the tenant's serving namespace, so requests and responses match Agent
+  Platform endpoints; gates, approvals, rolling updates, undeploy; exposure in the cluster or an internal load
+  balancer. `pipewright model predict` for either platform.
+- **Schedules as Kubernetes CronJobs** where the manifest says `schedules.backend: kubernetes`.
+- **The launcher as a pull worker** (`launcher.runtime: gke`): the same routes fed from the same subscriptions,
+  for a launcher inside the cluster; `launcher.audiences` so releases keep working across a switch.
+- **`pipewright pipelines-ui`**: the Kubeflow Pipelines UI on localhost, read only, through the cluster's API.
+- A small Kubernetes client (`pipewright.kube`); credentials ask for `userinfo.email` (GKE names callers by email).
+
+### Added: synthetic data
+
+- **Off until a team switches it on.** A `synthetic` section in the use case turns it on, per environment; without
+  it nothing is built, compiled or released for it, and templates and digests are unchanged.
+- **Learned inside the environment.** A built-in `synthetic` pipeline (`pipewright run --pipeline synthetic`, or a
+  schedule with `pipeline: synthetic`): the source is read as training reads it, a real holdout is kept back,
+  candidates are generated in parallel (by default the MOSTLY AI Synthetic Data SDK in local mode, which makes no
+  network calls, at sampling temperatures 1.0 and 1.3; SDV as an engine a team installs itself), and each is scored
+  on fidelity, utility (a model trained on synthetic data, tested on the real holdout) and privacy (exact copies,
+  distance to the closest record, singling-out, linkability and inference attacks). The best candidate that passes
+  every gate is published; otherwise nothing is.
+- **Gates per purpose** (`test-data`, `sharing`), recomputed by Pipewright; `pipewright synthetic gates` lists them;
+  a use case may make privacy gates stricter, never looser. The platform's `policies.synthetic` in the manifest sets
+  the allowed purposes and engines, purposes that need differential privacy, gate floors and required approvers.
+- **Never learned:** keys and personal-data columns get new values from column rules.
+- **Relational and sequential** (`mode: relational`): the entity table with the feature tables of
+  `data.features_from` and the event tables in `synthetic.sequences`, learned together. Keys stay consistent across
+  tables, each history keeps its timing against the cut-off, and the gates are measured on the point-in-time join of
+  the synthetic tables; each table goes to its own output table.
+- **Without real data:** `pipewright synthetic generate` writes rows from the data contract and the column rules,
+  for development, CI and demos.
+- **Released like any pipeline:** `pipewright release` builds two images (the engine, and the evaluation), recorded
+  in the contract by digest, verified, and copied by `promote`. `doctor` checks the section against the live
+  environment. Events `synthetic.evaluated`, `synthetic.rejected` and `synthetic.generated`.
+
+### Changed
+
+Compared with the pre-release builds early users ran:
+
+- `retraining.enabled: true` now means `retraining.mode: propose`: drift no longer requests training directly.
+- Decision emails show the summary as a code block, so line breaks and commands are kept.
+- `.yml` configuration files are refused with a message to rename them to `.yaml`, instead of being ignored.
+- Licensed under the Apache License 2.0.
+- Distribution name `pipewright-mlops` (PyPI); the import and the command stay `pipewright`.
+
+### Fixed
+
+- **`alerts sync` keeps its own notification channels.** Another email channel for the same address (a platform's
+  own alerts) made every sync try to repoint the use case's policies; the channel Pipewright made now wins.
+- **Steps with only `retries` or `cache`** no longer get an empty `resources` block in the compiled template: Agent
+  Platform ran them on its sized-step backend, where the metadata server issues no ID tokens.
+- **A release reuses a contract only when its images and compiled templates are unchanged**, so a Pipewright
+  upgrade that changes how templates compile is released (as `<commit>-<hash>`) instead of silently reusing the old
+  template.
+- **Generated CI workflows fail when a command fails.** Steps run as `bash -eo pipefail`, so a `pipewright ...`
+  piped to `tee` no longer passes when the command fails.
+- A pipeline job that cannot join its Agent Platform experiment (for example in a new project) is reported as
+  submitted with a warning, instead of failing the trigger and being retried as a duplicate.