cairn-operator-dataflow-metaflow 0.1.3__tar.gz

cairn_operator_dataflow_metaflow-0.1.3/.gitignore

@@ -0,0 +1,100 @@
+# Secrets
+.env
+prod/.env.prod
+.env.local
+.env.*.local
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+*.egg-info/
+.installed.cfg
+*.egg
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+.coverage.*
+htmlcov/
+
+# Virtualenvs
+.venv/
+venv/
+env/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Data caches
+data/chroma/
+*.log
+
+# Eval reports
+evals/last_report.json
+evals/*.report.json
+
+# Build
+build/
+dist/
+# The platform image's build context (#2514): the public wheels the ship step copies beside
+
+# Observability sinks (src/obs)
+events.jsonl
+replays/
+
+# CLI run artifacts: trace dirs (cairn run) + the hash-chained audit log.
+.cairn/
+audit.jsonl
+
+# DuckDB local analytics (scripts/dashboard.py)
+obs.duckdb
+*.duckdb.wal
+*.duckdb.tmp/
+
+# Optional external MCP backend — operators drop this in to enable CAIRN_BACKEND=otel.
+# Without it, the selector falls back to the mock backend with a warning.
+src/mcp/tools/_backends/otel.py
+tests/test_otel_backend.py
+
+# Refactor scratch — incident agent package split kept on disk for reference
+# but not part of the shipped tree. __init__.py files were removed so Python
+# imports src/agents/incident.py (the working single-file module) instead.
+src/agents/incident/
+
+
+.cache/
+.chainlit/
+.files/
+chainlit_app.py
+chainlit.md
+.deepeval/
+.langgraph_api/
+prod/
+
+# stale pre-rename data dir (runtime traces); data now lives in .cairn/
+.obs-agent/
+
+# Never commit a secrets file under any name (an untracked .env backup once held
+# the prod Postgres password, JWT secret and admin hash), nor local work areas.
+.env.*
+!.env.example
+!deploy/light/.env.infisical.example
+# the feature-proof profile's generated keys (scripts/oss/proof_seed.py), never committed
+deploy/light/.proof/
+.proof-out/
+*.bak
+/private/
+/site/
+/data/
+# the generated hello-cairn proof report (scripts/oss/feature_proof.sh)
+/FEATURE_PROOF.md

cairn_operator_dataflow_metaflow-0.1.3/DESIGN.md

@@ -0,0 +1,113 @@
+# cairn-operator-dataflow-metaflow — design
+
+- **Distribution:** `cairn-operator-dataflow-metaflow` (`cairn/operators/optional/dataflow_metaflow/`),
+  offered by `cairndev[metaflow]`
+- **Capabilities provided:** none as operators — it registers the `metaflow` dataflow engine
+  (`cairn.dataflow_engines`), which `dataflow.run` and `ml.run` submit to when selected
+- **Status:** proven at unit level (the conformance suite, cross-box, secrets, gang scheduling);
+  not in the hello-cairn feature proof
+- **Decisions:** ADR-0083 / ADR-0089 (Metaflow as substrate), ADR-0088 (checkpoints + resume),
+  ADR-0134 D2 (an optional wheel, #2678)
+
+## 1. Purpose
+
+Run a cairn flow — a stage list, either kind — on Metaflow's substrate: Kubernetes or AWS Batch
+compute, a shared S3 datastore, a metadata service that makes a run readable and resumable
+from another box, `@retry` and `resume`. It is the scale rung above the builtin engine
+(in-process or one SSH box). It does NOT add operators, govern anything itself or register
+artifacts: the `dataflow/run` / `ml/run` operators (cairn-operator-dataflow) gate, budget and
+register, and core decides which engine an environment selects. Until ADR-0134 it lived inside
+cairn-operator-dataflow; it moved out so a default `cairndev` install carries only what the
+feature proof covers.
+
+## 2. Architecture
+
+```
+dataflow/run | ml/run  (cairn-operator-dataflow; governance, budget, artifacts)
+        │ resolve_engine(kind)  — core: cairn.core.dataflow.engine_registry
+        ▼
+metaflow_engine(kind, target)        factory.py   reads the metaflow_* backend keys
+        ▼
+MetaflowEngine.submit/poll/result    engines_metaflow.py
+        │ spawns  python -m operator_dataflow_metaflow.metaflow_bridge run …
+        │         (metaflow_train_bridge for a gang, resources.workers > 1)
+        ▼
+CairnFlow (Metaflow FlowSpec)        metaflow_bridge.py — ONE static flow interpreting any
+        │                            spec; the step calls cairn.core.dataflow.runner.run_dataflow
+        ▼
+Metaflow local | k8s | Batch compute, S3 datastore, metadata service
+```
+
+- **One adapter, both kinds.** The engine is kind-blind; the environment maps `ml_engine` and
+  `dataflow_engine` to it independently (ADR-0082 D7).
+- **One static flow.** The bridge is never code-generated (ADR-0082 D1): it reads the spec from
+  `CAIRN_DATAFLOW_SPEC` and runs the core kernel runner, so stage contracts, budgets and
+  checkpoints are core's. `cairn flow export --to metaflow` is the separate, one-way projection.
+- **Shared helpers are core's** (`cairn.core.dataflow.resources`, ADR-0134 D3b): the resource
+  envelope (`aggregate_resources`), the gang size (`gang_workers`), and the environment
+  allowlist this engine extends with its own `METAFLOW_` / `CAIRN_METAFLOW_` / `OBS_METAFLOW_`
+  prefixes.
+- **Re-attach and cancel** recognise the bridge process by its cmdline module path. Jobs
+  launched before ADR-0134 run `operator_dataflow.metaflow_bridge`; both paths are recognised
+  for one release (`_BRIDGE_MODULES`).
+- **Imports:** `cairn.core.*` only (the operator contract, `cairn.core.dataflow`); never another
+  operator pack. `metaflow` is imported lazily, so the wheel registers without it.
+
+## 3. Operators
+
+None. The pack registers one `cairn.dataflow_engines` entry, `metaflow` →
+`operator_dataflow_metaflow.factory:metaflow_engine`. The operators that reach it are
+`dataflow/run` and `ml/run` in cairn-operator-dataflow.
+
+## 4. Configuration
+
+| Key | Kind | Scope | Default | Meaning |
+|---|---|---|---|---|
+| `ml_engine` / `dataflow_engine` = `metaflow` | backend field | deployment | `builtin` | select this engine for a flow kind |
+| `metaflow_metadata_root` (`CAIRN_METAFLOW_ROOT`) | backend field | deployment | the kind's target | where Metaflow's local metadata lives |
+| `metaflow_datastore`, `metaflow_datastore_root` | backend fields | deployment | local | the datastore and its root (S3 for cross-box) |
+| `metaflow_compute`, `metaflow_compute_image`, `metaflow_compute_queue` | backend fields | deployment | `local` | the compute substrate (local / kubernetes / batch), its image and queue |
+| `metaflow_retry_times` | backend field | deployment | 0 | `@retry` count; a non-integer is refused at `validate()` |
+| `metaflow_secrets` | backend field | deployment | none | the connections a run may receive (`CAIRN_CONN_*`), and nothing else |
+| `metaflow_env_isolation` | backend field | deployment | `auto` | how much of the control plane's environment a task inherits |
+| `metaflow_metadata_service` | backend field | deployment | none | the metadata service URL that removes box affinity (`cairn doctor` checks it is reachable) |
+
+The backend fields are read through `cairn.core.operator.selected_backend` (profile → env →
+default); a worker box receives them as the env vars named in `factory._SELECTION_ENV`.
+
+## 5. Governance
+
+The engine is reached only through `dataflow/run` / `ml/run`, so every flow leg passes
+`enforce_flow_gate` on the flow's aggregate stage risk and the flow-engine door
+(`cairn.core.dataflow.engine_door`, ADR-0134 D3c) before any record exists: an environment
+that selects `metaflow` without this wheel installed is refused there, naming
+`cairndev[metaflow]`. Budgets are the kernel runner's (`BudgetExceeded`). The environment
+allowlist withholds every credential a run did not declare.
+
+## 6. Failure modes
+
+| Failure | Cause | Effect | Recovered? |
+|---|---|---|---|
+| Engine not installed | `ml_engine = metaflow` without `cairndev[metaflow]` | refused at the launch door | n/a — install the extra |
+| `metaflow` library absent | the wheel installed without its `[metaflow]` extra | `validate()`/submit names the extra | n/a |
+| Run invisible to a later process (#1364) | metadata under a per-job temp dir | fixed: pathspec on the handle, metadata root | yes |
+| Bridge died, no terminal marker | process killed | re-attach reports failed, retriable | yes — `resume` |
+| Pre-upgrade job after ADR-0134 | the old module path in its cmdline | still recognised for one release | yes |
+
+## 7. Testing
+
+`tests/` in this pack: the conformance suite (`test_metaflow_conformance.py`, needs `metaflow`;
+skipped without it), substrate/config (`test_metaflow_substrate.py`), secrets
+(`test_metaflow_secrets.py`), cross-box (`test_metaflow_cross_box.py`), client polling,
+remote results, gang scheduling (`test_metaflow_train_bridge.py`), and the process-identity
+gates — cancel of a detached bridge and re-attach liveness — each run for the current AND the
+pre-upgrade bridge module path. Run with the worktree on `PYTHONPATH` or the pack installed.
+Remote compute (Kubernetes/Batch) is not exercised in CI.
+
+## 8. History
+
+| Date | Change | Issue |
+|---|---|---|
+| 2026-07-27 | Metaflow adapter — one adapter, both flow kinds (ADR-0083), in cairn-operator-dataflow | #1078 → #1114 |
+| 2026-08-08 | Metaflow as substrate: secrets, cross-box, gang scheduling (ADR-0089) | #1345, #1346, #1366 |
+| 2026-10-16 | Its own optional wheel behind `cairndev[metaflow]` (ADR-0134 D2) | #2678 |

cairn_operator_dataflow_metaflow-0.1.3/LICENSE

@@ -0,0 +1,202 @@
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 The cairn Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

cairn_operator_dataflow_metaflow-0.1.3/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.5
+Name: cairn-operator-dataflow-metaflow
+Version: 0.1.3
+Summary: The Metaflow dataflow engine (ADR-0089) for cairn's ml and data flows — offered by cairndev[metaflow] (ADR-0134 D2).
+Project-URL: Homepage, https://cairndev.sh
+Project-URL: Documentation, https://docs.cairndev.sh
+Project-URL: Source, https://github.com/cairndev-sh/cairn
+Project-URL: Issues, https://github.com/cairndev-sh/cairn/issues
+Author: cairn maintainers
+License-Expression: Apache-2.0
+License-File: LICENSE
+Requires-Python: >=3.12
+Provides-Extra: metaflow
+Requires-Dist: metaflow>=2.12; extra == 'metaflow'
+Description-Content-Type: text/markdown
+
+# cairn-operator-dataflow-metaflow — see DESIGN.md + ADR-0089, ADR-0134 D2.
+
+The Metaflow dataflow engine for cairn: `pip install 'cairndev[metaflow]'`, then set
+`[profile.backends] ml_engine = "metaflow"` (or `dataflow_engine`). The `dataflow/run` and
+`ml/run` operators of `cairn-operator-dataflow` submit to it; this wheel registers the engine
+only.

cairn_operator_dataflow_metaflow-0.1.3/README.md

@@ -0,0 +1,6 @@
+# cairn-operator-dataflow-metaflow — see DESIGN.md + ADR-0089, ADR-0134 D2.
+
+The Metaflow dataflow engine for cairn: `pip install 'cairndev[metaflow]'`, then set
+`[profile.backends] ml_engine = "metaflow"` (or `dataflow_engine`). The `dataflow/run` and
+`ml/run` operators of `cairn-operator-dataflow` submit to it; this wheel registers the engine
+only.

cairn_operator_dataflow_metaflow-0.1.3/pack.toml

@@ -0,0 +1,15 @@
+# cairn-operator-dataflow-metaflow — the Metaflow dataflow engine (ADR-0089), moved out of
+# cairn-operator-dataflow into its own optional wheel (ADR-0134 D2, #2678). It registers no
+# operator: the `dataflow/run` and `ml/run` operators (cairn-operator-dataflow) submit to it
+# when an environment selects `metaflow` as its dataflow_engine / ml_engine.
+[pack]
+name        = "obs-operator-dataflow-metaflow"
+version     = "0.1.0"
+type        = "operator"
+description = "The Metaflow dataflow engine behind dataflow/run and ml/run (cairndev[metaflow])."
+kind        = "service_provider"
+publisher   = "cairn"
+trust_level = "official"
+
+[compat]
+cairn_engine = ">=0.1.0"

cairn_operator_dataflow_metaflow-0.1.3/pyproject.toml

@@ -0,0 +1,38 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "cairn-operator-dataflow-metaflow"
+version = "0.1.3"
+description = "The Metaflow dataflow engine (ADR-0089) for cairn's ml and data flows — offered by cairndev[metaflow] (ADR-0134 D2)."
+readme = "README.md"
+license = "Apache-2.0"
+authors = [{name = "cairn maintainers"}]
+requires-python = ">=3.12"
+# No hard dependency on `metaflow`: the engine and the bridges import it lazily, so this wheel
+# imports and registers without it (its tests run that way). `cairndev[metaflow]` installs
+# both; installing this wheel alone, add the extra below.
+dependencies = []
+
+[project.optional-dependencies]
+# The ADR-0083 scale rung, serving BOTH flow kinds through one adapter. Needs NO Argo/Step
+# Functions: local run, `resume` and `@retry` verified standalone on 2026-07-27. Production adds
+# a shared S3 datastore + metadata service.
+metaflow = ["metaflow>=2.12"]
+
+[project.urls]
+Homepage = "https://cairndev.sh"
+Documentation = "https://docs.cairndev.sh"
+Source = "https://github.com/cairndev-sh/cairn"
+Issues = "https://github.com/cairndev-sh/cairn/issues"
+
+# #1512 — core selects an engine by NAME through this group and never imports this pack.
+[project.entry-points."cairn.dataflow_engines"]
+"metaflow" = "operator_dataflow_metaflow.factory:metaflow_engine"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/operator_dataflow_metaflow"]
+
+[tool.hatch.build.targets.wheel.force-include]
+"pack.toml" = "operator_dataflow_metaflow/pack.toml"

cairn_operator_dataflow_metaflow-0.1.3/src/operator_dataflow_metaflow/__init__.py

@@ -0,0 +1,3 @@
+"""cairn-operator-dataflow-metaflow — the Metaflow dataflow engine (ADR-0089), an optional
+wheel offered by ``cairndev[metaflow]`` (ADR-0134 D2). Registered as ``metaflow`` in the
+``cairn.dataflow_engines`` group; core selects it by name and never imports this package."""