clusop 0.0.1__tar.gz

clusop-0.0.1/.github/workflows/release-pypi.yml

@@ -0,0 +1,118 @@
+name: Publish clusop to PyPI
+
+# One-click release — auto-versioned (next = max(PyPI, tags) + 1 patch). Builds the
+# Scala listener JAR (sbt), bakes it into the wheel, lints/tests, publishes via
+# Trusted Publishing (OIDC, no stored token), then commits the bump + tag. Mirrors
+# the proven self-healing-kit workflow, with an added JVM build step.
+on:
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      contents: write   # commit the version bump + tag (after publish)
+      id-token: write   # OIDC for PyPI Trusted Publishing
+    steps:
+      - uses: actions/checkout@v6
+        with: { ref: main, fetch-depth: 0, fetch-tags: true }
+
+      - uses: actions/setup-python@v6
+        with: { python-version: "3.11" }
+
+      - name: Set up JDK (for the listener JAR)
+        uses: actions/setup-java@v4
+        with: { distribution: temurin, java-version: "17", cache: sbt }
+
+      - name: Install sbt (not preinstalled on ubuntu-latest)
+        uses: sbt/setup-sbt@v1
+
+      - name: Compute next version (latest published + 1 patch)
+        id: ver
+        run: |
+          python -m pip install --quiet packaging
+          python - <<'PY' >> "$GITHUB_OUTPUT"
+          import json, subprocess, sys, urllib.request
+          from packaging.version import Version, InvalidVersion
+          PKG = "clusop"; cand = []
+          try:
+              with urllib.request.urlopen(f"https://pypi.org/pypi/{PKG}/json", timeout=20) as r:
+                  cand += list(json.load(r).get("releases", {}).keys())
+          except Exception as e:
+              # stdout is redirected into $GITHUB_OUTPUT — diagnostics MUST go to stderr,
+              # else a 404 (first publish, package not yet on PyPI) breaks output parsing.
+              print(f"pypi read failed ({e}); tags only", file=sys.stderr)
+          tags = subprocess.run(["git","tag","--list","v*"], capture_output=True, text=True).stdout.split()
+          cand += [t[1:] for t in tags if t.startswith("v")]
+          parsed = []
+          for c in cand:
+              try: parsed.append(Version(c))
+              except InvalidVersion: pass
+          base = max(parsed) if parsed else Version("0.0.0")
+          nxt = f"{base.major}.{base.minor}.{base.micro + 1}"
+          print(f"version={nxt}"); print(f"previous={base}")
+          PY
+
+      - name: Show version
+        run: echo "▶ releasing clusop ${{ steps.ver.outputs.version }} (prev ${{ steps.ver.outputs.previous }})"
+
+      - name: Bump pyproject + __init__ (working tree only)
+        run: |
+          V="${{ steps.ver.outputs.version }}"
+          python - "$V" <<'PY'
+          import re, sys, pathlib
+          v = sys.argv[1]
+          for path, key in (("pyproject.toml", "version"), ("src/clusop/__init__.py", "__version__")):
+              p = pathlib.Path(path); t = p.read_text()
+              pat = rf'(?m)^{re.escape(key)} = ".*"'
+              if not re.search(pat, t): raise SystemExit(f"no {key} in {path}")
+              p.write_text(re.sub(pat, f'{key} = "{v}"', t, count=1))
+          print("bumped to", v)
+          PY
+
+      - name: Build the listener JAR (sbt) and bake it into the package
+        run: |
+          cd scala && sbt -batch package && cd ..
+          mkdir -p src/clusop/jars
+          JAR=$(ls scala/target/*.jar scala/target/scala-*/*.jar 2>/dev/null | head -1)
+          cp "$JAR" src/clusop/jars/photon_listener.jar
+          echo "bundled JAR: $(ls -la src/clusop/jars/*.jar)"
+
+      - name: Lint + unit tests
+        run: |
+          pip install -e ".[dev]"
+          ruff check src/ tests/ --ignore E501
+          pytest tests/ -q
+
+      - name: Build sdist + wheel
+        run: pip install build && python -m build && ls -l dist/
+
+      - name: Gate — .pth at wheel root + JAR bundled
+        run: |
+          WHEEL=$(ls dist/*.whl | head -1)
+          python -m zipfile -l "$WHEEL" | awk '{print $1}' | grep -qxE 'clusop_autoload\.pth' || { echo "FAIL: .pth not at wheel root"; exit 1; }
+          python -m zipfile -l "$WHEEL" | grep -q 'clusop/jars/.*\.jar' || { echo "FAIL: listener JAR not bundled"; exit 1; }
+          echo "OK: .pth at root + JAR bundled"
+
+      - name: Publish to PyPI (Trusted Publishing / OIDC)
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with: { skip-existing: true }
+
+      - name: Commit bump + tag (after a successful publish)
+        run: |
+          V="${{ steps.ver.outputs.version }}"
+          git config user.name "clusop-release-bot"
+          git config user.email "clusop-release-bot@users.noreply.github.com"
+          git commit -am "release v$V" || echo "nothing to commit"
+          git tag "v$V"; git push origin HEAD:main; git push origin "v$V"
+
+      - name: Summary
+        if: always()
+        run: |
+          echo "## 📦 clusop publish" >> "$GITHUB_STEP_SUMMARY"
+          echo "Published \`${{ steps.ver.outputs.version }}\` (prev \`${{ steps.ver.outputs.previous }}\`)" >> "$GITHUB_STEP_SUMMARY"
+          echo "https://pypi.org/project/clusop/${{ steps.ver.outputs.version }}/" >> "$GITHUB_STEP_SUMMARY"

clusop-0.0.1/.gitignore

@@ -0,0 +1,11 @@
+__pycache__/
+*.pyc
+.pytest_cache/
+.ruff_cache/
+dist/
+build/
+*.egg-info/
+.venv/
+scala/target/
+scala/project/target/
+src/clso/jars/*.jar

clusop-0.0.1/PKG-INFO

@@ -0,0 +1,107 @@
+Metadata-Version: 2.4
+Name: clusop
+Version: 0.0.1
+Summary: Photon Fallback Analyzer — zero-touch Databricks cost forensics (.pth self-arm + bundled JVM listener)
+Author: yogasathyandrun
+License: Proprietary
+Keywords: cost,databricks,finops,photon,spark
+Requires-Python: >=3.10
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == 'dev'
+Requires-Dist: ruff>=0.5; extra == 'dev'
+Provides-Extra: service
+Requires-Dist: databricks-sdk>=0.30; extra == 'service'
+Requires-Dist: requests>=2.31; extra == 'service'
+Description-Content-Type: text/markdown
+
+# clusop — Photon Fallback Analyzer
+
+`pip install clusop`
+
+clusop finds the money Databricks Photon quietly burns. When Photon hits an operator it
+can't run (a Python UDF, a struct-IN filter, an unsupported Delta feature), it silently
+**falls back** to the JVM — you keep paying the 2–2.9× Photon DBU premium while getting
+JVM speed. clusop detects those fallbacks from the executed plan, estimates the wasted
+spend, and proposes a fix. It never touches your job.
+
+## One install, two halves, zero config
+
+A single `pip install clusop` ships both halves and arms itself:
+
+- a **Python `.pth`** (`clusop_autoload.pth`) that Python's `site` machinery runs at
+  interpreter startup → imports `clusop.runtime.bootstrap` → arms automatically. You do
+  not `import clusop` anywhere in your job.
+- a bundled **Scala JAR** (`clusop/jars/photon_listener.jar`) — the actual
+  `QueryExecutionListener`. The bootstrap `addJar`s it and registers it over Py4J.
+
+Install it as a **cluster/job library** (production path) so every interpreter that
+starts already has it. `%pip install clusop` + `dbutils.library.restartPython()` works for
+dev.
+
+Everything is **auto-detected** at runtime — cloud (from the instance type), DBR (from
+`DATABRICKS_RUNTIME_VERSION`), cluster shape, and whether the JVM is reachable. No
+per-user setup; a new customer pip-installs and it works.
+
+## What it costs you: nothing if it can't run safely
+
+- **Fail-open everywhere.** Any error in arming, listening, parsing, or dispatch is
+  swallowed. clusop never breaks, slows, or blocks a customer query.
+- **Needs a SINGLE_USER / dedicated cluster** for the JVM listener. On USER_ISOLATION
+  (Shared) clusters the JVM is sealed behind Spark Connect — clusop detects this and stays
+  dormant rather than failing.
+- **Propose-never-apply.** clusop emits a signal and a Teams card. A human decides.
+
+## How a signal is born
+
+1. The JVM listener sees a finished query and reads its **executed plan**.
+2. The **structural parser** ([src/clusop/analysis/parser.py](src/clusop/analysis/parser.py),
+   mirrored in Scala) decides if this is a *real* fallback. The hard part: a clean Photon
+   query always ends in one **terminal** `ColumnarToRow` (the normal result boundary) —
+   that is **not** a fallback. Real fallback is **mid-plan** `ColumnarToRow`,
+   `RowToColumnar` round-trips, or `BatchEvalPython`/`ArrowEvalPython`. Counting raw
+   occurrences false-positives; clusop doesn't.
+3. The **waste model** ([waste.py](src/clusop/analysis/waste.py)) sizes the loss:
+   `runtime × Σ(node DBU/hr) × $rate × (photon_premium−1) × fallback_weight`.
+4. **Confidence** ([confidence.py](src/clusop/analysis/confidence.py)) is decomposed into
+   four legs — parse / diagnosis / cost / recommendation. A *fix* (rewrite the UDF) needs
+   only parse+diagnosis; a *disable-Photon* recommendation is a dollar decision and is
+   capped by the cost leg.
+5. The signal flows to a **driver batcher** (dedup by signature) → **central aggregator**
+   (idempotent upsert, suppression, prioritization) → Teams card.
+
+## Cost: modeled now, billed if granted
+
+clusop always works with a **modeled** cost (public price table → MEDIUM cost confidence).
+If the workspace can read `system.billing.usage`, the
+[CostTierResolver](src/clusop/service/resolver.py) reconciles the estimate to **billed**
+dollars (HIGH confidence). Detection never depends on system tables — they only upgrade
+the cost layer. Prices are reference data, not something clusop invents per-row.
+
+## Certifying it's real
+
+[harness/certify.py](harness/certify.py) runs on a dedicated cluster, captures real
+executed plans for known clean / known-fallback fixtures, and reports catch-rate plus a
+**DBR-stamped Delta feature-support matrix** — derived, never hand-maintained.
+
+## Layout
+
+```
+src/clusop/
+  runtime/      bootstrap (arming), detect (cloud/DBR/shape/jvm)
+  analysis/     parser · waste · confidence · signal
+  pricing/      price_table.json · provider
+  service/      aggregator · resolver · suppression · teams · onboard
+  jars/         photon_listener.jar  (baked in by the release workflow)
+scala/          the QueryExecutionListener (sbt; mirror of the Python parser)
+harness/        certify.py  (catch-rate + Delta matrix, run on a dedicated cluster)
+tests/          parser / waste / confidence
+```
+
+See [SPEC.md](SPEC.md) for the full design and invariants.
+
+## Release
+
+Push-button: the [Publish workflow](.github/workflows/release-pypi.yml) auto-versions
+(`max(PyPI, tags)+1`), builds the JAR with sbt, bakes it into the wheel, lints+tests,
+gates on *(.pth at wheel root AND JAR bundled)*, then publishes via PyPI Trusted
+Publishing (OIDC — no stored token) and tags the bump.

clusop-0.0.1/README.md

@@ -0,0 +1,91 @@
+# clusop — Photon Fallback Analyzer
+
+`pip install clusop`
+
+clusop finds the money Databricks Photon quietly burns. When Photon hits an operator it
+can't run (a Python UDF, a struct-IN filter, an unsupported Delta feature), it silently
+**falls back** to the JVM — you keep paying the 2–2.9× Photon DBU premium while getting
+JVM speed. clusop detects those fallbacks from the executed plan, estimates the wasted
+spend, and proposes a fix. It never touches your job.
+
+## One install, two halves, zero config
+
+A single `pip install clusop` ships both halves and arms itself:
+
+- a **Python `.pth`** (`clusop_autoload.pth`) that Python's `site` machinery runs at
+  interpreter startup → imports `clusop.runtime.bootstrap` → arms automatically. You do
+  not `import clusop` anywhere in your job.
+- a bundled **Scala JAR** (`clusop/jars/photon_listener.jar`) — the actual
+  `QueryExecutionListener`. The bootstrap `addJar`s it and registers it over Py4J.
+
+Install it as a **cluster/job library** (production path) so every interpreter that
+starts already has it. `%pip install clusop` + `dbutils.library.restartPython()` works for
+dev.
+
+Everything is **auto-detected** at runtime — cloud (from the instance type), DBR (from
+`DATABRICKS_RUNTIME_VERSION`), cluster shape, and whether the JVM is reachable. No
+per-user setup; a new customer pip-installs and it works.
+
+## What it costs you: nothing if it can't run safely
+
+- **Fail-open everywhere.** Any error in arming, listening, parsing, or dispatch is
+  swallowed. clusop never breaks, slows, or blocks a customer query.
+- **Needs a SINGLE_USER / dedicated cluster** for the JVM listener. On USER_ISOLATION
+  (Shared) clusters the JVM is sealed behind Spark Connect — clusop detects this and stays
+  dormant rather than failing.
+- **Propose-never-apply.** clusop emits a signal and a Teams card. A human decides.
+
+## How a signal is born
+
+1. The JVM listener sees a finished query and reads its **executed plan**.
+2. The **structural parser** ([src/clusop/analysis/parser.py](src/clusop/analysis/parser.py),
+   mirrored in Scala) decides if this is a *real* fallback. The hard part: a clean Photon
+   query always ends in one **terminal** `ColumnarToRow` (the normal result boundary) —
+   that is **not** a fallback. Real fallback is **mid-plan** `ColumnarToRow`,
+   `RowToColumnar` round-trips, or `BatchEvalPython`/`ArrowEvalPython`. Counting raw
+   occurrences false-positives; clusop doesn't.
+3. The **waste model** ([waste.py](src/clusop/analysis/waste.py)) sizes the loss:
+   `runtime × Σ(node DBU/hr) × $rate × (photon_premium−1) × fallback_weight`.
+4. **Confidence** ([confidence.py](src/clusop/analysis/confidence.py)) is decomposed into
+   four legs — parse / diagnosis / cost / recommendation. A *fix* (rewrite the UDF) needs
+   only parse+diagnosis; a *disable-Photon* recommendation is a dollar decision and is
+   capped by the cost leg.
+5. The signal flows to a **driver batcher** (dedup by signature) → **central aggregator**
+   (idempotent upsert, suppression, prioritization) → Teams card.
+
+## Cost: modeled now, billed if granted
+
+clusop always works with a **modeled** cost (public price table → MEDIUM cost confidence).
+If the workspace can read `system.billing.usage`, the
+[CostTierResolver](src/clusop/service/resolver.py) reconciles the estimate to **billed**
+dollars (HIGH confidence). Detection never depends on system tables — they only upgrade
+the cost layer. Prices are reference data, not something clusop invents per-row.
+
+## Certifying it's real
+
+[harness/certify.py](harness/certify.py) runs on a dedicated cluster, captures real
+executed plans for known clean / known-fallback fixtures, and reports catch-rate plus a
+**DBR-stamped Delta feature-support matrix** — derived, never hand-maintained.
+
+## Layout
+
+```
+src/clusop/
+  runtime/      bootstrap (arming), detect (cloud/DBR/shape/jvm)
+  analysis/     parser · waste · confidence · signal
+  pricing/      price_table.json · provider
+  service/      aggregator · resolver · suppression · teams · onboard
+  jars/         photon_listener.jar  (baked in by the release workflow)
+scala/          the QueryExecutionListener (sbt; mirror of the Python parser)
+harness/        certify.py  (catch-rate + Delta matrix, run on a dedicated cluster)
+tests/          parser / waste / confidence
+```
+
+See [SPEC.md](SPEC.md) for the full design and invariants.
+
+## Release
+
+Push-button: the [Publish workflow](.github/workflows/release-pypi.yml) auto-versions
+(`max(PyPI, tags)+1`), builds the JAR with sbt, bakes it into the wheel, lints+tests,
+gates on *(.pth at wheel root AND JAR bundled)*, then publishes via PyPI Trusted
+Publishing (OIDC — no stored token) and tags the bump.

clusop-0.0.1/SPEC.md

@@ -0,0 +1,117 @@
+# clusop — Design Specification
+
+The Photon Fallback Analyzer. Detect silent Photon→JVM fallbacks, price the waste,
+propose a fix. In-process, fail-open, propose-never-apply.
+
+## 1. Problem
+
+Photon bills a 2–2.9× DBU premium. When it meets an operator it can't vectorize it
+inserts a `ColumnarToRow` transition and runs that subtree on the JVM — a **fallback**.
+The customer keeps paying the premium for JVM work. Fallbacks are invisible in the UI and
+common: Python/Pandas UDFs, struct/array predicates, some Delta features per DBR, RDD
+APIs. clusop surfaces them with a dollar figure and a confidence.
+
+## 2. Delivery — one pip, self-arming
+
+`pip install clusop` ships:
+- `clusop_autoload.pth` force-included at the **wheel root**. Python's `site` module
+  executes it at interpreter startup → `import clusop.runtime.bootstrap` → `activate()`.
+  Importing the package in user code is **not** required and does **not** arm it.
+- `clusop/jars/photon_listener.jar`, baked in by the release workflow and shipped as a
+  package artifact.
+
+Arming path matters: only interpreters that **start** with clusop installed arm. Cluster/job
+**library** = production (every interpreter). `%pip install` + `restartPython()` = dev.
+
+`bootstrap.activate()`: locate the active `SparkSession` → `jvm_available()` (False on
+Spark Connect / USER_ISOLATION) → `sparkContext.addJar(bundled_jar)` →
+`spark._jvm.com.clusop.listener.PhotonFallbackListener(premium, endpoint)` →
+`listenerManager().register(...)`. A watcher polls for a late session. Driver-only
+(`DB_IS_DRIVER`). Every step is wrapped — failure leaves the customer job untouched.
+
+Native alternative: `spark.sql.queryExecutionListeners` conf pointing at the listener
+class (no Py4J), for conf-managed fleets.
+
+## 3. Auto-detection (no per-user config)
+
+[detect.py](src/clusop/runtime/detect.py):
+- **cloud** from instance-type prefix (`m5`→aws, `Standard_D`→azure, `n2`→gcp) + host.
+- **DBR** from `DATABRICKS_RUNTIME_VERSION`.
+- **shape** (driver/worker node types, num_workers, photon_enabled, data_security_mode,
+  compute_type) → `ClusterShape`.
+- **jvm_available(spark)** → False under Spark Connect.
+
+Defaults when unknown: AWS + DBR 14.3/15.4/16.x, all overridable.
+
+## 4. The parser — the moat
+
+[parser.py](src/clusop/analysis/parser.py) / `PlanParser.scala` (kept identical).
+`analyze_plan(plan_text, runtime_seconds, photon_enabled) → PlanAnalysis`.
+
+Invariant: **terminal vs mid-plan `ColumnarToRow`.** A clean Photon plan ends in exactly
+one root `ColumnarToRow` — the result boundary, **not** a fallback. Fallback =
+- `mid_plan_col2row` ≥ 1, or
+- `RowToColumnar` round-trips, or
+- `BatchEvalPython`/`ArrowEvalPython` (UDF), or
+- zero Photon nodes on a Photon-enabled cluster (`fallback_weight = 1.0`).
+
+Outputs `fallback_weight` (0..1, share of the plan off Photon), `photon_ratio`, and a
+`Verdict(kind, cause)`: `fix_fallback` (e.g. cause `python_udf`) vs `disable_photon`
+(round-trip / whole-plan JVM where Photon is pure overhead). Node-count capped
+(`_NODE_CAP`). Empty/garbage plan → safe no-fallback.
+
+## 5. Waste model
+
+[waste.py](src/clusop/analysis/waste.py): `modeled_waste(analysis, shape, provider,
+runtime_seconds) → WasteEstimate`. Dimensionally correct:
+
+```
+waste$ = runtime_hours × Σ(node DBU/hr) × dbu_$rate × (photon_premium − 1) × fallback_weight
+```
+
+Unknown node type → 0 (never crash, never guess). `basis="modeled"` until reconciled.
+
+## 6. Confidence — four legs
+
+[confidence.py](src/clusop/analysis/confidence.py): `parse`, `diagnosis`, `cost`,
+`recommendation`. The recommendation leg never exceeds the legs it depends on:
+- `fix_fallback` depends on parse+diagnosis (not cost) — a UDF rewrite is right whatever
+  the dollar figure.
+- `disable_photon` is a dollar decision — capped by the **cost** leg. Modeled cost
+  (MEDIUM/LOW) caps a disable recommendation accordingly.
+
+## 7. Cost tiers — adaptive, never a dependency
+
+[resolver.py](src/clusop/service/resolver.py): `probe()` tests
+`SELECT 1 FROM system.billing.usage LIMIT 1`.
+- readable → **Tier-1 billed**, cost leg HIGH; `reconcile(signal)` upgrades modeled→billed.
+- not readable → **Tier-0 modeled**, cost leg MEDIUM.
+
+Detection is **always** in-process and never depends on system tables. Fail-open. Never
+present modeled cost as billed.
+
+## 8. Aggregation & action
+
+[aggregator.py](src/clusop/service/aggregator.py): `DriverBatcher` (dedup by
+`fallback_signature`, accumulate, evict, flush) → `CentralAggregator` (idempotent upsert,
+resolver reconcile). [suppression.py](src/clusop/service/suppression.py): `passes_gate`
+(min waste / min confidence / cooldown) + `priority(waste, conf)`.
+[teams.py](src/clusop/service/teams.py): propose-never-apply Adaptive Card.
+[onboard.py](src/clusop/service/onboard.py): creates `clusop_signals` in the customer's own
+schema and probes cost tier. clusop writes only to its own schema; reads system tables
+read-only.
+
+## 9. Certification
+
+[harness/certify.py](harness/certify.py) on a dedicated cluster: catch-rate over known
+clean/fallback fixtures + a **DBR-stamped Delta feature-support matrix**, derived from
+real executed plans through the shipped parser — never hand-maintained.
+
+## 10. Invariants (non-negotiable)
+
+1. Fail-open — clusop never breaks/slows/blocks a customer query.
+2. Propose-never-apply — never mutates production compute or jobs.
+3. Detection in-process; system tables only upgrade cost.
+4. Modeled cost is never labeled billed.
+5. Writes confined to clusop's own schema; system tables read-only.
+6. Terminal `ColumnarToRow` is not a fallback.

clusop-0.0.1/clusop_autoload.pth

@@ -0,0 +1 @@
+import clusop.runtime.bootstrap

clusop-0.0.1/harness/certify.py

@@ -0,0 +1,69 @@
+"""clusop certification harness — run on a DEDICATED (single-user) cluster.
+
+Produces two artifacts the product depends on (no week-by-week phasing — run it on
+each target DBR):
+  1. catch-rate baseline — does the parser flag known-fallback fixtures and stay
+     quiet on clean Photon? (the only question that matters: is the signal real)
+  2. the Delta feature-support matrix — for each (DBR, Delta feature), run a fixture
+     and record whether Photon fell back. The matrix is DERIVED here, never
+     hand-maintained, and stamped with the DBR it was observed on.
+
+Captures the real executed plan via `df._jdf.queryExecution().executedPlan().toString()`
+(needs JVM access — hence dedicated clusters) and feeds it to the SAME structural
+parser the product uses (clusop.analysis.parser), so the harness validates the shipped logic.
+"""
+
+from __future__ import annotations
+
+import json
+
+from clusop.analysis.parser import analyze_plan
+from clusop.runtime.detect import detect_dbr
+
+
+def _plan(df) -> str:
+    # JVM path — dedicated cluster only; this is the certification environment.
+    return df._jdf.queryExecution().executedPlan().toString()
+
+
+def run(spark) -> dict:
+    dbr = detect_dbr()
+    results = {"dbr": dbr, "catch_rate": {}, "delta_matrix": {}}
+
+    # ---- catch-rate fixtures -------------------------------------------------
+    fixtures = {
+        # name -> (builder, expect_fallback)
+        "clean_sql":   (lambda: spark.range(0, 1_000_00).groupBy((spark.range(1).id).alias("k")).count(), False),
+    }
+    # a Python-UDF fixture (should fall back to BatchEvalPython)
+    try:
+        from pyspark.sql.functions import udf
+        from pyspark.sql.types import StringType
+        bucket = udf(lambda v: "hi" if v % 2 else "lo", StringType())
+        fixtures["python_udf"] = (lambda: spark.range(0, 100000).withColumn("b", bucket("id")), True)
+    except Exception:
+        pass
+
+    for name, (build, expect) in fixtures.items():
+        try:
+            df = build()
+            df.collect()
+            a = analyze_plan(_plan(df), photon_enabled=True)
+            results["catch_rate"][name] = {"expected": expect, "detected": a.has_fallback,
+                                           "correct": a.has_fallback == expect,
+                                           "weight": a.fallback_weight, "cause": a.verdict.cause}
+        except Exception as e:
+            results["catch_rate"][name] = {"error": str(e)[:160]}
+
+    # ---- Delta feature matrix (stub fixtures; expand per feature) -------------
+    for feature in ("deletionVectors", "CDF", "generatedColumns", "schemaEvolution"):
+        # production: build a MERGE/UPDATE exercising `feature`, parse, record fellBack.
+        results["delta_matrix"][f"{dbr}::{feature}"] = {"fellBack": None, "status": "untested_stub"}
+
+    print(json.dumps(results, indent=2))
+    return results
+
+
+if __name__ == "__main__":  # pragma: no cover — run inside a Databricks notebook
+    from pyspark.sql import SparkSession
+    run(SparkSession.builder.getOrCreate())

clusop-0.0.1/pyproject.toml

@@ -0,0 +1,42 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "clusop"
+version = "0.0.1"
+description = "Photon Fallback Analyzer — zero-touch Databricks cost forensics (.pth self-arm + bundled JVM listener)"
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "Proprietary" }
+authors = [{ name = "yogasathyandrun" }]
+keywords = ["databricks", "photon", "finops", "spark", "cost"]
+dependencies = []  # in-cluster agent stays dependency-free; service extras below
+
+[project.optional-dependencies]
+# the off-cluster central service (aggregation + cost-tier resolver + Teams)
+service = ["requests>=2.31", "databricks-sdk>=0.30"]
+dev = ["pytest>=7", "ruff>=0.5"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/clusop"]
+# ship the compiled Scala listener JAR inside the wheel so `pip install clusop`
+# delivers everything in one artifact (CI's sbt step drops it into src/clusop/jars/).
+artifacts = ["src/clusop/jars/*.jar"]
+
+# Ship the self-arming hook at the WHEEL ROOT so pip drops it straight into
+# site-packages, where Python's `site` module runs .pth files at interpreter
+# startup -> `import clusop.runtime.bootstrap` arms the listener. See README.
+[tool.hatch.build.targets.wheel.force-include]
+"clusop_autoload.pth" = "clusop_autoload.pth"
+
+[tool.ruff]
+target-version = "py310"
+line-length = 120
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "W"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]

clusop-0.0.1/scala/build.sbt

@@ -0,0 +1,20 @@
+// clusop — Photon fallback listener (the bundled JAR).
+// Built by CI (sbt assembly/package) and copied into src/clusop/jars/ before the wheel
+// is packed, so `pip install clusop` ships everything in one artifact.
+//
+// Spark/Scala are PROVIDED — supplied by the Databricks Runtime at execution time;
+// we never bundle Spark into the JAR.
+
+ThisBuild / scalaVersion := "2.12.18"
+
+lazy val clusopListener = (project in file("."))
+  .settings(
+    name := "clusop-photon-listener",
+    version := "0.1.0",
+    libraryDependencies ++= Seq(
+      "org.apache.spark" %% "spark-sql"  % "3.5.0" % Provided,
+      "org.apache.spark" %% "spark-core" % "3.5.0" % Provided
+    ),
+    // a thin JAR — no shading needed since deps are Provided
+    Compile / packageBin / artifactPath := baseDirectory.value / "target" / "clusop-photon-listener.jar"
+  )

clusop-0.0.1/scala/project/build.properties

@@ -0,0 +1 @@
+sbt.version=1.9.9

clusop-0.0.1/scala/src/main/scala/com/clusop/listener/PhotonFallbackListener.scala

@@ -0,0 +1,36 @@
+package com.clusop.listener
+
+import org.apache.spark.sql.util.QueryExecutionListener
+import org.apache.spark.sql.execution.QueryExecution
+
+/**
+ * Passive QueryExecutionListener — captures the executed physical plan of every
+ * query, with zero query wrappers and zero code change for data engineers. Native
+ * JVM parsing avoids per-query Py4J callback overhead.
+ *
+ * CARDINAL RULE: best-effort, non-blocking, must never slow or break the job.
+ * Everything is wrapped; the webhook/dispatch is async (see SignalDispatcher).
+ *
+ * Registered by the clusop `.pth` via:
+ *   spark._jvm.com.clusop.listener.PhotonFallbackListener(premium, endpoint)
+ *   spark._jsparkSession.listenerManager().register(listener)
+ */
+class PhotonFallbackListener(photonPremiumMultiplier: Double, signalEndpoint: String)
+    extends QueryExecutionListener {
+
+  override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = {
+    try {
+      // executedPlan is post-AQE/post-Catalyst; treeString avoids toString truncation.
+      val plan = qe.executedPlan.treeString
+      val a = PlanParser.analyze(plan)
+      if (a.hasFallback) {
+        WasteCalculator.emit(funcName, durationNs, a, photonPremiumMultiplier, signalEndpoint, plan)
+      }
+    } catch {
+      case _: Throwable => () // never propagate into the customer's job
+    }
+  }
+
+  // failed queries didn't complete -> no DBU waste to attribute
+  override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = ()
+}