pipewright-mlops 1.0.0__py3-none-any.whl

pipewright/__init__.py

@@ -0,0 +1,10 @@
+"""Pipewright: configurable, governed MLOps on Google Cloud. Distribution name: pipewright-mlops."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("pipewright-mlops")
+except PackageNotFoundError:  # running from a source checkout without install
+    __version__ = "0.0.0"
+
+API_VERSION = "pipewright/v1"

pipewright/adapters/__init__.py

@@ -0,0 +1,23 @@
+"""Orchestrator adapters: `vertex` (Agent Platform Pipelines) and `kfp` (Kubeflow Pipelines, private-gke)."""
+
+from __future__ import annotations
+
+from pipewright.release.contract import PipelineTarget, ReleaseContract
+
+
+def submit(contract: ReleaseContract, target: PipelineTarget, parameters: dict, job_id: str, labels: dict,
+           security: dict | None = None, manifest: dict | None = None, gcp=None) -> str:
+    """`security` is the manifest's security block (CMEK key, network): it is read at submit time, so the platform
+    can change it without app teams re-releasing. `manifest` and `gcp` are needed by kfp (the cluster and the
+    tenant's Kubernetes service account)."""
+    if contract.orchestrator == "vertex":
+        from pipewright.adapters.vertex import submit as vertex_submit
+
+        return vertex_submit(contract, target, parameters, job_id, labels, security)
+    if contract.orchestrator == "kfp":
+        from pipewright.adapters.kfp import submit as kfp_submit
+
+        if manifest is None or gcp is None:
+            raise ValueError("orchestrator kfp needs the environment manifest and a Google session")
+        return kfp_submit(contract, target, parameters, job_id, labels, manifest, gcp)
+    raise NotImplementedError(f"orchestrator {contract.orchestrator!r} is not supported yet")

pipewright/adapters/kfp.py

@@ -0,0 +1,149 @@
+"""Kubeflow Pipelines adapter (orchestrator kfp, the private-gke blueprint).
+
+A run is created through the Kubeflow Pipelines v2beta1 API, reached through the cluster's Kubernetes API (a service
+proxy): the caller authenticates with its own Google identity and RBAC lets it call only that API, so the Kubeflow
+Pipelines endpoint is never exposed. The run uses the same compiled template, parameters and pipeline root as on
+Agent Platform, and runs as the tenant's Kubernetes service account, bound to its runner (Workload Identity).
+
+Kubeflow Pipelines has no client-chosen run ID, so the deterministic job ID is the run's display name, and a run
+with that name already present is a duplicate delivery (AlreadySubmitted).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+
+import yaml
+
+from pipewright.adapters.vertex import AlreadySubmitted
+from pipewright.kube import KubeClient, message
+from pipewright.release.contract import PipelineTarget, ReleaseContract
+
+log = logging.getLogger("pipewright.launcher")
+
+
+DESCRIPTION_MAX = 255     # Kubeflow Pipelines stores a run's description in a VARCHAR(255) column
+# The labels a run keeps (Kubeflow Pipelines v2 runs have no labels of their own), most important first: what does not
+# fit in the description is left out.
+_DESCRIBED = ("pipewright-usecase", "run-id", "pipewright-pipeline", "pipewright-release", "pipewright-tenant",
+              "trigger")
+
+
+def run_description(labels: dict) -> str:
+    """The run's labels as compact JSON within DESCRIPTION_MAX: {"pw": {"usecase": ..., "run-id": ...}}."""
+    kept: dict = {}
+    for key in _DESCRIBED:
+        if labels.get(key) is None:
+            continue
+        trial = {**kept, key.removeprefix("pipewright-"): labels[key]}
+        if len(json.dumps({"pw": trial}, separators=(",", ":"))) > DESCRIPTION_MAX:
+            break
+        kept = trial
+    return json.dumps({"pw": kept}, separators=(",", ":"))
+
+
+def run_labels(description: str | None) -> dict:
+    """The labels run_description kept, under their label names (pipewright-usecase, run-id, ...); {} otherwise."""
+    try:
+        doc = json.loads(description or "{}") or {}
+        if isinstance(doc.get("labels"), dict):      # runs submitted before the compact form
+            return doc["labels"]
+        kept = doc.get("pw") or {}
+    except (ValueError, AttributeError):
+        return {}
+    names = {k.removeprefix("pipewright-"): k for k in _DESCRIBED}
+    return {names.get(k, k): v for k, v in kept.items()} if isinstance(kept, dict) else {}
+
+
+def api_path(namespace: str) -> str:
+    return f"/api/v1/namespaces/{namespace}/services/ml-pipeline:8888/proxy/apis/v2beta1"
+
+
+def template(gcp, template_uri: str) -> dict:
+    """The compiled pipeline from the Artifact Registry KFP repository: the pipeline spec, and its platform spec
+    when the template has one (a second YAML document)."""
+    r = gcp.session.get(template_uri, timeout=60)
+    if r.status_code != 200:
+        raise RuntimeError(f"reading {template_uri} failed: {r.status_code} {r.text[:300]}")
+    docs = [d for d in yaml.safe_load_all(r.text) if d]
+    if not docs:
+        raise RuntimeError(f"{template_uri} is empty")
+    if len(docs) == 1:
+        return docs[0]
+    return {"pipeline_spec": docs[0], "platform_spec": docs[1]}
+
+
+def uncache_branches(spec: dict) -> list[str]:
+    """Turns caching off for the tasks inside dsl.OneOf branches (components `comp-condition-branches-*` and
+    everything under them), in place; returns their names. Kubeflow Pipelines resolves a OneOf output only from a
+    branch task that ran: when the chosen branch's task was a cache hit, the next step fails ("No successful task
+    found for output key", found 2026-10-17 on 2.17.2). Agent Platform resolves cached branches, so templates keep
+    their caching and only runs on Kubeflow change."""
+    pipeline = spec.get("pipeline_spec", spec)
+    components = pipeline.get("components") or {}
+    changed: list[str] = []
+
+    def walk(component: str, seen: set) -> None:
+        if component in seen:
+            return
+        seen.add(component)
+        for name, task in ((components.get(component) or {}).get("dag") or {}).get("tasks", {}).items():
+            if (task.get("cachingOptions") or {}).get("enableCache"):
+                task["cachingOptions"] = {}
+                changed.append(name)
+            child = (task.get("componentRef") or {}).get("name")
+            if child:
+                walk(child, seen)
+
+    seen: set = set()
+    for component in components:
+        if component.startswith("comp-condition-branches-"):
+            walk(component, seen)
+    return changed
+
+
+def find(kube: KubeClient, namespace: str, display_name: str) -> dict | None:
+    flt = json.dumps({"predicates": [{"key": "display_name", "operation": "EQUALS", "string_value": display_name}]})
+    r = kube.get(f"{api_path(namespace)}/runs", filter=flt, page_size=5)
+    if not r.ok:
+        raise RuntimeError(f"listing Kubeflow runs failed: {r.status} {message(r)}")
+    runs = (r.body or {}).get("runs") or []
+    return runs[0] if runs else None
+
+
+def submit(contract: ReleaseContract, target: PipelineTarget, parameters: dict, job_id: str, labels: dict,
+           manifest: dict, gcp) -> str:
+    k8s = manifest.get("kubernetes") or {}
+    namespace = k8s.get("namespace") or "kubeflow"
+    tenant = (manifest.get("tenants") or {}).get(contract.tenant) or {}
+    service_account = tenant.get("kubernetes_service_account")
+    if not service_account:
+        raise RuntimeError(f"tenant {contract.tenant} has no kubernetes_service_account in the environment manifest")
+    kube = KubeClient.from_manifest(manifest, gcp)
+    if find(kube, namespace, job_id):
+        raise AlreadySubmitted(job_id)
+    spec = template(gcp, target.template_uri)
+    uncached = uncache_branches(spec)
+    if uncached:
+        log.info("kubeflow run %s: no caching for OneOf branch tasks %s", job_id, ", ".join(sorted(uncached)))
+    body = {
+        "display_name": job_id,
+        "description": run_description(labels),
+        "pipeline_spec": spec,
+        "runtime_config": {"parameters": parameters, "pipeline_root": contract.pipeline_root},
+        "service_account": service_account,
+    }
+    r = kube.post(f"{api_path(namespace)}/runs", body)
+    if not r.ok:
+        raise RuntimeError(f"creating Kubeflow run {job_id} failed: {r.status} {message(r)}")
+    run_id = (r.body or {}).get("run_id")
+    log.info("kubeflow run %s (%s) as %s", run_id, job_id, service_account)
+    return f"kubeflow:{namespace}/runs/{run_id}"
+
+
+def run_state(manifest: dict, gcp, job_id: str) -> str | None:
+    """The run's state (PENDING, RUNNING, SUCCEEDED, FAILED, ...) by its job ID, or None when there is no such run."""
+    k8s = manifest.get("kubernetes") or {}
+    run = find(KubeClient.from_manifest(manifest, gcp), k8s.get("namespace") or "kubeflow", job_id)
+    return run.get("state") if run else None

pipewright/adapters/vertex.py

@@ -0,0 +1,65 @@
+"""Agent Platform Pipelines adapter."""
+
+from __future__ import annotations
+
+import logging
+
+from pipewright.release.contract import PipelineTarget, ReleaseContract
+
+log = logging.getLogger("pipewright.launcher")
+
+
+class AlreadySubmitted(Exception):
+    """The deterministic job ID already exists: a duplicate delivery of the same trigger."""
+
+
+def _experiment(contract: ReleaseContract, target: PipelineTarget) -> str | None:
+    """Training runs are grouped in an Agent Platform experiment. A failure here never blocks the run."""
+    if not target.experiment:
+        return None
+    from google.cloud import aiplatform
+
+    try:
+        aiplatform.Experiment.get_or_create(target.experiment, project=contract.project, location=contract.region)
+        return target.experiment
+    except Exception as e:  # noqa: BLE001
+        log.warning("experiment %s unavailable, submitting without it: %s", target.experiment, e)
+        return None
+
+
+def submit(contract: ReleaseContract, target: PipelineTarget, parameters: dict, job_id: str, labels: dict,
+           security: dict | None = None) -> str:
+    from google.api_core import exceptions as gexc
+    from google.cloud import aiplatform
+
+    security = security or {}
+    job = aiplatform.PipelineJob(
+        display_name=job_id,
+        template_path=target.template_uri,
+        job_id=job_id,
+        pipeline_root=contract.pipeline_root,
+        parameter_values=parameters,
+        enable_caching=None,          # per-step caching comes from the template (spec.steps.<step>.cache)
+        labels=labels,
+        encryption_spec_key_name=security.get("kms_key"),
+        project=contract.project,
+        location=contract.region,
+    )
+    try:
+        # Private profiles: the job's workers peer with the environment's VPC (Private Services Access).
+        job.submit(service_account=contract.runner_service_account, network=security.get("network"),
+                   reserved_ip_ranges=security.get("reserved_ip_ranges") or None,
+                   experiment=_experiment(contract, target))
+    except gexc.AlreadyExists as e:
+        raise AlreadySubmitted(job_id) from e
+    except RuntimeError as e:
+        # The SDK creates the job, then joins it to the experiment; joining can fail (for example in a new project
+        # whose experiment is not ready yet). The run is already going: report it, never fail it (retrying would only
+        # meet AlreadyExists).
+        try:
+            name = job.resource_name
+        except Exception:  # noqa: BLE001 - the job was not created: the original error stands
+            raise e from None
+        log.warning("%s submitted, but not joined to experiment %s: %s", name, target.experiment, e)
+        return name
+    return job.resource_name

pipewright/alerts/sync.py

@@ -0,0 +1,246 @@
+"""pipewright alerts sync: email alerts for a use case's Pipewright events (monitoring.alerts in usecase.yaml).
+
+    monitoring:
+      alerts:
+        notify: [team-ml@example.com, ml-oncall@example.com]      # any addresses, including distribution lists
+        events: [monitoring.alert, training.rejected]       # default: monitoring.alert
+        severity: WARNING
+
+Retrain decisions (retraining.decisions.notify) get their own policies: each decision event emails its summary and
+case ID (retrain.escalated also emails escalate_notify; retrain.digest only with decisions.digest). Email never
+approves; the email says how to decide.
+
+For each event: one Cloud Monitoring log-based alert policy that matches the event's structured log entry (the
+labels Pipewright puts on every entry) and emails every address. One email notification channel
+per address, shared by all use cases that notify it. Owned objects carry the label pipewright-usecase; a policy for
+an event no longer configured is deleted. Idempotent, like `schedules sync`.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from pipewright.config import Environment, ResolvedUseCase
+from pipewright.gcp import Gcp, error_text
+
+MON = "https://monitoring.googleapis.com/v3"
+
+# event -> (log filter on the event entry, title, details pulled into the email)
+EVENTS = {
+    "monitoring.alert": ('jsonPayload.pipewright_event.event_type="monitoring.alert"', "drift or performance alert",
+                         ["model_version_id", "drifted_features", "max_feature_psi", "prediction_psi",
+                          "performance_value", "retrain_status"]),
+    "retrain.requested": ('jsonPayload.pipewright_event.event_type="retrain.requested"', "retrain requested",
+                          ["triggers", "job_id", "source_uri"]),
+    "training.rejected": ('jsonPayload.pipewright_event.event_type="training.evaluated" AND '
+                          'jsonPayload.pipewright_event.data.decision="reject"', "model rejected by evaluation",
+                          ["primary_metric", "primary_value", "champion_version", "champion_value"]),
+    "training.evaluated": ('jsonPayload.pipewright_event.event_type="training.evaluated"', "training evaluated",
+                           ["decision", "primary_metric", "primary_value", "champion_version"]),
+    "model.registered": ('jsonPayload.pipewright_event.event_type="model.registered"', "model registered",
+                         ["model", "version", "default", "primary_value"]),
+}
+
+# Retrain decision events (pipewright.retraining): emailed to retraining.decisions.notify.
+DECISION_EVENTS = {
+    "retrain.proposed": "retrain decision needed",
+    "retrain.reopened": "retrain case re-opened",
+    "retrain.escalated": "retrain decision overdue",
+    "retrain.approval": "retrain approval recorded",
+    "retrain.approved": "retrain approved",
+    "retrain.declined": "retrain declined",
+    "retrain.snoozed": "retrain snoozed",
+    "retrain.digest": "weekly retrain digest",
+}
+for _event, _title in DECISION_EVENTS.items():
+    EVENTS[_event] = (f'jsonPayload.pipewright_event.event_type="{_event}"', _title, ["case_id", "summary"])
+
+
+def _label(v: str) -> str:
+    return re.sub(r"[^a-z0-9_-]", "-", v.lower())[:63]
+
+
+def policy_name(uc: ResolvedUseCase, event: str) -> str:
+    return f"pipewright {uc.name} {uc.environment}: {EVENTS[event][1]}"
+
+
+def policy_body(uc: ResolvedUseCase, event: str, channels: list[str], severity: str) -> dict:
+    match, title, details = EVENTS[event]
+    log_filter = (f'labels."pipewright-usecase"="{uc.name}" AND labels."pipewright-environment"="{uc.environment}" '
+                  f"AND {match}")
+    extractors = {d: f"EXTRACT(jsonPayload.pipewright_event.data.{d})" for d in details}
+    extractors["run_id"] = 'EXTRACT(labels."run-id")'
+    extractors["release"] = 'EXTRACT(labels."pipewright-release")'
+    subject = f"[{uc.environment}] {uc.name}: {title}"
+    if event in DECISION_EVENTS:
+        subject += " (${log.extracted_label.case_id})" if event != "retrain.digest" else ""
+        # A code block keeps the summary's line breaks and stops markdown turning "--env" into a dash, so the
+        # commands in it can be copied.
+        lines = [f"**{uc.name}** ({uc.environment}): {title}.", "", "```", "${log.extracted_label.summary}", "```", "",
+                 "This email never approves anything. Decide with your own identity: `pipewright retrain show | "
+                 f"approve | decline | snooze --env {uc.environment} --usecase {uc.name} --case <case>`.",
+                 f"Owner: {uc.owner}."]
+    else:
+        lines = [f"**{uc.name}** ({uc.environment}): {title}.", "",
+                 *[f"- {d}: ${{log.extracted_label.{d}}}" for d in details],
+                 "- run: ${log.extracted_label.run_id}, release ${log.extracted_label.release}", "",
+                 f"Owner: {uc.owner}. Details: `pipewright monitor status --env {uc.environment} --usecase "
+                 f"{uc.name}`, or the run in Agent Platform Pipelines."]
+    return {
+        "displayName": policy_name(uc, event),
+        "combiner": "OR",
+        "conditions": [{"displayName": f"{event} event", "conditionMatchedLog": {
+            "filter": log_filter, "labelExtractors": extractors}}],
+        "alertStrategy": {"notificationRateLimit": {"period": "300s"}, "autoClose": "1800s"},
+        "notificationChannels": channels,
+        "documentation": {"content": "\n".join(lines), "mimeType": "text/markdown",
+                          "subject": subject},
+        "severity": severity,
+        "userLabels": {"pipewright-usecase": _label(uc.name), "pipewright-environment": _label(uc.environment),
+                       "pipewright-event": _label(event.replace(".", "-")), "pipewright-managed": "true"},
+        "enabled": True,
+    }
+
+
+@dataclass
+class AlertsPlan:
+    channels_create: list[str] = field(default_factory=list)       # emails without a channel yet
+    channels: dict = field(default_factory=dict)                    # email -> channel name (existing)
+    create: list[str] = field(default_factory=list)                 # events
+    update: list[str] = field(default_factory=list)
+    delete: list[str] = field(default_factory=list)                 # policy names (resource)
+    unchanged: list[str] = field(default_factory=list)
+    errors: list[str] = field(default_factory=list)
+
+
+def _list(gcp: Gcp, url: str, key: str, **params) -> list[dict]:
+    out, token = [], None
+    while True:
+        r = gcp.get(url, **params, **({"pageToken": token} if token else {}))
+        if not r.ok:
+            raise RuntimeError(error_text(r))
+        out += r.body.get(key, [])
+        token = r.body.get("nextPageToken")
+        if not token:
+            return out
+
+
+def _same(live: dict, want: dict) -> bool:
+    keys = ("displayName", "conditions", "alertStrategy", "documentation", "severity", "enabled")
+    norm = lambda d: {k: d.get(k) for k in keys} | {"channels": sorted(d.get("notificationChannels", []))}  # noqa: E731
+    live_c = [{"displayName": c.get("displayName"), "conditionMatchedLog": c.get("conditionMatchedLog")}
+              for c in live.get("conditions", [])]
+    return norm({**live, "conditions": live_c}) == norm(want)
+
+
+def sync(uc: ResolvedUseCase, env: Environment, gcp: Gcp | None = None, dry_run: bool = False) -> AlertsPlan:
+    gcp = gcp or Gcp()
+    alerts = uc.ml["monitoring"]["alerts"]
+    severity = alerts["severity"]
+    wanted_emails = wanted_policies(uc)
+    emails = sorted({e for addresses in wanted_emails.values() for e in addresses})
+    base = f"{MON}/projects/{env.project}"
+    plan = AlertsPlan()
+    try:
+        live_channels = _list(gcp, f"{base}/notificationChannels", "notificationChannels", filter='type="email"')
+        live_policies = _list(gcp, f"{base}/alertPolicies", "alertPolicies",
+                              filter=f'user_labels."pipewright-usecase"="{_label(uc.name)}" AND '
+                                     f'user_labels."pipewright-environment"="{_label(uc.environment)}"')
+    except RuntimeError as e:
+        plan.errors.append(str(e))
+        return plan
+    # A project may have other email channels for the same address (a platform's own alerts): the one Pipewright
+    # made wins, so a new channel elsewhere never repoints the use case's policies.
+    mine = lambda c: (c.get("userLabels") or {}).get("pipewright-managed") == "true"  # noqa: E731
+    by_email = {c["labels"].get("email_address"): c["name"]
+                for c in sorted(live_channels, key=lambda c: (mine(c), c["name"]))}
+    for email in emails:
+        if email in by_email:
+            plan.channels[email] = by_email[email]
+        else:
+            plan.channels_create.append(email)
+
+    if not dry_run:
+        for email in plan.channels_create:
+            r = gcp.post(f"{base}/notificationChannels", {
+                "type": "email", "displayName": f"pipewright: {email}", "labels": {"email_address": email},
+                "userLabels": {"pipewright-managed": "true"}})
+            if r.ok:
+                plan.channels[email] = r.body["name"]
+            else:
+                plan.errors.append(f"channel {email}: {error_text(r)}")
+
+    live_by_name = {p["displayName"]: p for p in live_policies}
+    wanted = {policy_name(uc, e): e for e in wanted_emails}
+    for name, live in live_by_name.items():
+        if name not in wanted:
+            plan.delete.append(live["name"])
+    for name, event in wanted.items():
+        event_channels = sorted(plan.channels[e] for e in wanted_emails[event] if e in plan.channels)
+        body = policy_body(uc, event, event_channels, severity)
+        live = live_by_name.get(name)
+        if live is None:
+            plan.create.append(event)
+            if not dry_run:
+                r = gcp.post(f"{base}/alertPolicies", body)
+                if not r.ok:
+                    plan.errors.append(f"{event}: {error_text(r)}")
+        elif _same(live, body) and not plan.channels_create:
+            plan.unchanged.append(event)
+        else:
+            plan.update.append(event)
+            if not dry_run:
+                r = gcp.patch(f"{MON}/{live['name']}", body)
+                if not r.ok:
+                    plan.errors.append(f"{event}: {error_text(r)}")
+    if not dry_run:
+        for name in plan.delete:
+            r = gcp.delete(f"{MON}/{name}")
+            if not r.ok and r.status != 404:
+                plan.errors.append(f"delete {name}: {error_text(r)}")
+    return plan
+
+
+def wanted_policies(uc: ResolvedUseCase) -> dict[str, list[str]]:
+    """event -> email addresses: monitoring.alerts for its events, retraining.decisions for decision events."""
+    alerts = uc.ml["monitoring"]["alerts"]
+    out = {e: sorted(set(alerts["notify"])) for e in dict.fromkeys(alerts["events"])} if alerts["notify"] else {}
+    retraining = uc.ml.get("retraining") or {}
+    decisions = retraining.get("decisions") or {}
+    notify = sorted(set(decisions.get("notify") or []))
+    if retraining.get("mode", "off") != "off" and notify:
+        for event in DECISION_EVENTS:
+            if event == "retrain.digest" and not decisions.get("digest", True):
+                continue
+            extra = (decisions.get("escalate_notify") or []) if event == "retrain.escalated" else []
+            out[event] = sorted(set(notify) | set(extra))
+    return out
+
+
+def render(uc: ResolvedUseCase, plan: AlertsPlan, dry_run: bool) -> str:
+    alerts = uc.ml["monitoring"]["alerts"]
+    head = f"Alerts for {uc.name} ({uc.environment})" + ("  [dry run]" if dry_run else "")
+    if not alerts["notify"]:
+        lines = [head, "  INFO  monitoring.alerts.notify is empty: no alert emails"]
+    else:
+        lines = [head, f"  notify: {', '.join(sorted(set(alerts['notify'])))}"]
+    decisions = (uc.ml.get("retraining") or {}).get("decisions") or {}
+    if (uc.ml.get("retraining") or {}).get("mode", "off") != "off":
+        lines.append(f"  retrain decisions: {', '.join(sorted(set(decisions.get('notify') or []))) or 'nobody'}"
+                     + ("" if decisions.get("notify") else "  (set retraining.decisions.notify)"))
+    verb = (lambda w: f"would {w}") if dry_run else (lambda w: f"{w}d" if w.endswith("e") else f"{w}ed")
+    for e in plan.channels_create:
+        lines.append(f"  {'PLAN' if dry_run else 'DONE'}  email channel {e}: {verb('create')}")
+    for e in plan.create:
+        lines.append(f"  {'PLAN' if dry_run else 'DONE'}  {e}: {verb('create')}")
+    for e in plan.update:
+        lines.append(f"  {'PLAN' if dry_run else 'DONE'}  {e}: {verb('update')}")
+    for e in plan.unchanged:
+        lines.append(f"  OK    {e}: up to date")
+    for n in plan.delete:
+        status = "PLAN" if dry_run else "DONE"
+        lines.append(f"  {status}  {n.rsplit('/', 1)[-1]}: {verb('delete')} (no longer configured)")
+    for err in plan.errors:
+        lines.append(f"  FAIL  {err}")
+    return "\n".join(lines)

pipewright/build/__init__.py

@@ -0,0 +1 @@
+"""Turning a use case into artifacts: compiled templates and component images."""