clusop 0.0.1__py3-none-any.whl

clusop/__init__.py

@@ -0,0 +1,23 @@
+"""clusop — Photon Fallback Analyzer.
+
+Zero-touch Databricks cost forensics. Installed as one PyPI wheel (``pip install
+clusop``) that bundles a Scala QueryExecutionListener JAR; a ``.pth`` self-arms it at
+interpreter startup. It detects when Photon silently falls back to the JVM, models
+the wasted DBU premium, and proposes a human-approved fix — it never auto-applies.
+
+`import clusop` is intentionally light and does NOT arm anything. The self-arming
+tripwire is installed by ``clusop_autoload.pth`` at interpreter startup (it runs
+``import clusop.runtime.bootstrap``).
+"""
+
+from __future__ import annotations
+
+__version__ = "0.0.1"
+
+
+def diagnose(plan_text: str, *, runtime_seconds: float = 0.0):
+    """Offline helper: analyze a physical-plan string (e.g. from EXPLAIN FORMATTED)
+    and return the fallback verdict + decomposed confidence. Used by the harness and
+    for ad-hoc checks — the live path is the in-process listener."""
+    from clusop.analysis.parser import analyze_plan
+    return analyze_plan(plan_text, runtime_seconds=runtime_seconds)

clusop/analysis/__init__.py

@@ -0,0 +1 @@
+"""clusop — Photon Fallback Analyzer (zero-touch Databricks cost forensics)."""

clusop/analysis/confidence.py

@@ -0,0 +1,71 @@
+"""Decomposed confidence — four independent axes, never one MIN'd number.
+
+A card must never assert more certainty about its *recommendation* than it has on
+the weakest leg supporting it. One number collapses four distinct questions:
+  parse        — did we read the plan correctly?
+  diagnosis    — do we know WHY it fell back?
+  cost         — how trustworthy is the dollar figure?
+  recommendation (DERIVED) — bounded by the legs the action actually depends on.
+
+Example: a FIX-the-UDF rec doesn't depend on the $ figure, so it isn't capped by a
+weak cost leg; a DISABLE-Photon rec IS a dollar decision, so it is.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import IntEnum
+
+
+class Level(IntEnum):
+    LOW = 0
+    MEDIUM = 1
+    HIGH = 2
+
+    def label(self) -> str:
+        return self.name
+
+
+def _min(*levels: Level) -> Level:
+    return Level(min(int(x) for x in levels))
+
+
+@dataclass
+class Confidence:
+    parse: Level
+    diagnosis: Level
+    cost: Level
+    recommendation: Level
+
+    def to_dict(self) -> dict:
+        return {k: getattr(self, k).label() for k in ("parse", "diagnosis", "cost", "recommendation")}
+
+
+def score(analysis, *, version_certified: Level, cost_tier_billed: bool,
+          attribution: Level) -> Confidence:
+    """analysis: PlanAnalysis. version_certified: from the harness cert level for this
+    DBR. cost_tier_billed: True if reconciled from system.billing. attribution:
+    concurrency-aware cost confidence (see service/aggregator concurrency note)."""
+    # parse — did we read the plan cleanly?
+    parse = Level.LOW if (not analysis.aqe_ok or analysis.partial) else Level.HIGH
+
+    # diagnosis — do we know WHY it fell back?
+    clear_signature = Level.HIGH if (analysis.udf_fallback or analysis.round_trips > 0) else Level.MEDIUM
+    diagnosis = _min(version_certified, clear_signature)
+
+    # cost — how trustworthy is the dollar figure?
+    cost = _min(
+        Level.HIGH if cost_tier_billed else Level.MEDIUM,   # billed vs modeled
+        Level.HIGH if analysis.fallback_weight > 0 else Level.LOW,  # had real severity?
+        attribution,                                        # concurrency-aware
+    )
+
+    # recommendation — bounded by exactly the legs the action needs.
+    if analysis.verdict.kind == "fix_fallback":
+        rec = _min(parse, diagnosis)               # not a $ decision
+    elif analysis.verdict.kind == "disable_photon":
+        rec = _min(parse, diagnosis, cost)         # a $ decision -> needs cost too
+    else:
+        rec = Level.LOW
+
+    return Confidence(parse=parse, diagnosis=diagnosis, cost=cost, recommendation=rec)

clusop/analysis/parser.py

@@ -0,0 +1,110 @@
+"""Version-resilient, STRUCTURAL physical-plan parser — the detection moat.
+
+Why structural (not a raw count): a fully-Photon query STILL ends with one
+ColumnarToRow at the root, to hand columnar batches back to the driver as rows.
+That terminal transition is NORMAL, not waste. Real fallback is a *mid-plan*
+ColumnarToRow (Photon -> rows -> JVM op), often with a RowToColumnar round-trip
+back into Photon, or a (Batch|Arrow)EvalPython for a Python UDF. Counting raw
+occurrences false-positives on every healthy Photon query — so we distinguish the
+ROOT transition from interior ones.
+
+Operator names are internal Spark/DBR strings and change across runtimes, so we
+match FAMILIES via regex, never exact class names. # ASSUMPTION: verified per DBR
+by the certification harness.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+_PHOTON = re.compile(r"(?i)\bPhoton[A-Za-z]+")
+_COL2ROW = re.compile(r"(?i)ColumnarToRow")
+_ROW2COL = re.compile(r"(?i)RowToColumnar")
+_UDF = re.compile(r"(?i)(?:Batch|Arrow)EvalPython")
+_AQE_NOT_FINAL = re.compile(r"(?i)AdaptiveSparkPlan\s+isFinalPlan=false")
+_NODE_CAP = 4000  # pathological-plan guard (P99 latency budget)
+
+
+@dataclass
+class Verdict:
+    kind: str          # "none" | "fix_fallback" | "disable_photon"
+    cause: str         # "none" | "python_udf" | "unsupported_op"
+    summary: str = ""
+
+
+@dataclass
+class PlanAnalysis:
+    photon_nodes: int = 0
+    terminal_col2row: bool = False
+    mid_plan_col2row: int = 0
+    round_trips: int = 0          # RowToColumnar = JVM->Photon hop = definite fallback
+    udf_fallback: bool = False
+    aqe_ok: bool = True
+    partial: bool = False         # plan truncated / over the node cap
+    photon_ratio: float = 1.0
+    fallback_weight: float = 0.0  # 0..1 severity (drives $ and cost confidence)
+    verdict: Verdict = field(default_factory=lambda: Verdict("none", "none"))
+
+    @property
+    def has_fallback(self) -> bool:
+        return self.verdict.kind != "none"
+
+
+def analyze_plan(plan_text: str, runtime_seconds: float = 0.0,
+                 photon_enabled: bool | None = None) -> PlanAnalysis:
+    a = PlanAnalysis()
+    if not plan_text:
+        return a
+    lines = [ln for ln in plan_text.splitlines() if ln.strip()]
+    if len(lines) > _NODE_CAP:
+        a.partial = True
+        lines = lines[:_NODE_CAP]
+    text = "\n".join(lines)
+
+    a.aqe_ok = not bool(_AQE_NOT_FINAL.search(text))
+    a.photon_nodes = len(_PHOTON.findall(text))
+    total_col2row = len(_COL2ROW.findall(text))
+    a.round_trips = len(_ROW2COL.findall(text))
+    a.udf_fallback = bool(_UDF.search(text))
+
+    # the ROOT operator is the first (outermost) line; a ColumnarToRow there is the
+    # normal result-boundary transition and must NOT count as fallback.
+    root = lines[0] if lines else ""
+    a.terminal_col2row = bool(_COL2ROW.search(root))
+    a.mid_plan_col2row = max(total_col2row - (1 if a.terminal_col2row else 0), 0)
+
+    photon_seen = a.photon_nodes > 0 or (photon_enabled is True)
+    jvm_fallback = a.mid_plan_col2row + a.round_trips + (1 if a.udf_fallback else 0)
+
+    # no Photon ops at all on a Photon-enabled cluster -> the whole query ran on JVM.
+    full_fallback = (photon_enabled is True) and a.photon_nodes == 0
+
+    if not photon_seen:
+        a.verdict = Verdict("none", "none", "not a Photon plan")
+        a.photon_ratio = 1.0
+        return a
+
+    denom = a.photon_nodes + jvm_fallback
+    a.photon_ratio = (a.photon_nodes / denom) if denom else 1.0
+
+    if jvm_fallback == 0 and not full_fallback:
+        a.verdict = Verdict("none", "none", "clean Photon execution")
+        a.fallback_weight = 0.0
+        return a
+
+    # severity: how much of the work bypassed the C++ engine.
+    weight = 1.0 - a.photon_ratio
+    if full_fallback:
+        weight = 1.0
+    if a.udf_fallback and a.photon_ratio < 0.5:
+        weight = max(weight, 0.9)   # query bottlenecked by the JVM/Python transition
+    a.fallback_weight = round(min(max(weight, 0.0), 1.0), 3)
+
+    if a.udf_fallback:
+        a.verdict = Verdict("fix_fallback", "python_udf",
+                            "Photon premium paid but execution fell back to the JVM via a Python UDF.")
+    else:
+        a.verdict = Verdict("disable_photon", "unsupported_op",
+                            "Photon fell back to the JVM (unsupported operator); premium largely wasted.")
+    return a

clusop/analysis/signal.py

@@ -0,0 +1,38 @@
+"""The compact signal record the driver emits and the central service aggregates.
+
+Driver emits SIGNALS (not cards). The signature is what collapses 500 statements
+hitting the same fallback into one accumulating record (driver-local dedup).
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+
+
+def fallback_signature(plan_shape_hint: str, fallback_family: str, source_table: str) -> str:
+    """hash(normalized plan shape, fallback op family, source table). Same problem ->
+    same signature -> one record, regardless of how many statements hit it."""
+    raw = f"{plan_shape_hint}|{fallback_family}|{source_table}".lower()
+    return hashlib.sha256(raw.encode()).hexdigest()[:16]
+
+
+@dataclass
+class FallbackSignal:
+    signature: str
+    cluster_id: str
+    job_id: str
+    job_run_id: str
+    cloud: str
+    dbr: str
+    cause: str                  # python_udf | unsupported_op
+    verdict_kind: str           # fix_fallback | disable_photon
+    fallback_weight: float
+    runtime_seconds: float
+    wasted_usd_modeled: float
+    occurrences: int = 1
+    confidence: dict = field(default_factory=dict)   # decomposed, from analysis
+    evidence: str = ""          # short plan snippet (NO full plan retained)
+
+    def to_dict(self) -> dict:
+        return self.__dict__.copy()

clusop/analysis/waste.py

@@ -0,0 +1,36 @@
+"""Modeled premium-waste — dimensionally correct, cluster-size aware.
+
+waste$ = runtime_hours
+       x Sum(node DBU/hr)            # cluster size matters (2 nodes != 40 nodes)
+       x dbu_$_rate(compute_type)
+       x (photon_premium_multiplier - 1)   # the PREMIUM, not the whole bill
+       x fallback_weight             # 0..1 severity from the plan
+
+i.e. of the Photon premium you paid for this run, the fraction that bought you
+nothing because execution fell back to the JVM. Modeled (Tier-0); the central
+cost-tier resolver upgrades it to billed when system.billing is readable.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass
+class WasteEstimate:
+    premium_usd: float        # total Photon premium for the run
+    wasted_usd: float         # premium x fallback_weight
+    basis: str                # "modeled" | "billed"
+    fallback_weight: float
+
+
+def modeled_waste(analysis, shape, provider, runtime_seconds: float) -> WasteEstimate:
+    """analysis: PlanAnalysis. shape: detect.ClusterShape. provider: PricingProvider."""
+    hours = max(runtime_seconds, 0.0) / 3600.0
+    premium = provider.photon_premium_usd(
+        shape.driver_node_type, shape.worker_node_type, shape.num_workers,
+        shape.compute_type, hours,
+    )
+    wasted = round(premium * analysis.fallback_weight, 4)
+    return WasteEstimate(premium_usd=premium, wasted_usd=wasted,
+                         basis="modeled", fallback_weight=analysis.fallback_weight)

clusop/jars/.gitkeep

@@ -0,0 +1 @@
+# The compiled Scala listener JAR is built by CI (sbt) and copied here before packaging.

clusop/pricing/__init__.py

@@ -0,0 +1 @@
+"""clusop — Photon Fallback Analyzer (zero-touch Databricks cost forensics)."""

clusop/pricing/price_table.json

@@ -0,0 +1,34 @@
+{
+  "_comment": "EXAMPLE list prices — VERIFY per cloud/region/tier before trusting dollar figures. cloud_usd_per_hour should be auto-refreshed from the public AWS Pricing API / Azure Retail Prices API / GCP Catalog; dbu_per_hour + photon_premium_multiplier come from the Databricks price list. The agent AUTO-DETECTS cloud at runtime and selects the right block.",
+  "currency": "USD",
+  "default_contract_discount": 0.0,
+  "clouds": {
+    "aws": {
+      "dbu_usd_rate": { "jobs": 0.15, "all_purpose": 0.55, "sql_serverless": 0.70 },
+      "photon_premium_multiplier": 2.0,
+      "nodes": {
+        "m5.large":   { "dbu_per_hour": 0.40, "cloud_usd_per_hour": 0.096 },
+        "m5d.large":  { "dbu_per_hour": 0.40, "cloud_usd_per_hour": 0.113 },
+        "m5.xlarge":  { "dbu_per_hour": 0.75, "cloud_usd_per_hour": 0.192 },
+        "m5.2xlarge": { "dbu_per_hour": 1.50, "cloud_usd_per_hour": 0.384 },
+        "m5.4xlarge": { "dbu_per_hour": 3.00, "cloud_usd_per_hour": 0.768 },
+        "i3.xlarge":  { "dbu_per_hour": 1.00, "cloud_usd_per_hour": 0.312 }
+      }
+    },
+    "azure": {
+      "dbu_usd_rate": { "jobs": 0.15, "all_purpose": 0.55, "sql_serverless": 0.70 },
+      "photon_premium_multiplier": 2.0,
+      "nodes": {
+        "standard_ds3_v2": { "dbu_per_hour": 0.75, "cloud_usd_per_hour": 0.229 },
+        "standard_d4ds_v5": { "dbu_per_hour": 1.00, "cloud_usd_per_hour": 0.272 }
+      }
+    },
+    "gcp": {
+      "dbu_usd_rate": { "jobs": 0.15, "all_purpose": 0.55, "sql_serverless": 0.70 },
+      "photon_premium_multiplier": 2.0,
+      "nodes": {
+        "n2-standard-4": { "dbu_per_hour": 1.00, "cloud_usd_per_hour": 0.194 }
+      }
+    }
+  }
+}

clusop/pricing/provider.py

@@ -0,0 +1,61 @@
+"""PricingProvider — modeled cost, cloud-aware, zero external dependency.
+
+Cost = usage (measured) x rate (reference). We measure usage in-process; the rate
+comes from the bundled per-cloud price table (cloud auto-detected). Modeled cost is
+DIRECTIONAL — good enough to rank waste, not to invoice. If system.billing.usage is
+granted, the central service reconciles modeled -> billed (cost-tier resolver).
+
+The Photon angle: the premium is the EXTRA DBUs Photon costs vs standard. Wasted
+premium = the premium you paid for the fraction of work that fell back to the JVM.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+
+_TABLE_PATH = os.getenv("CLSO_PRICE_TABLE") or os.path.join(os.path.dirname(__file__), "price_table.json")
+
+
+@dataclass
+class PricingProvider:
+    cloud: str = "aws"
+    contract_discount: float | None = None
+    _table: dict = None  # type: ignore
+
+    def __post_init__(self):
+        with open(_TABLE_PATH) as f:
+            self._table = json.load(f)
+        if self.contract_discount is None:
+            self.contract_discount = float(self._table.get("default_contract_discount", 0.0))
+
+    def _block(self) -> dict:
+        return (self._table.get("clouds", {}).get(self.cloud)
+                or self._table.get("clouds", {}).get("aws", {}))
+
+    def _node(self, nt: str) -> dict:
+        return self._block().get("nodes", {}).get(nt, {"dbu_per_hour": 0.0, "cloud_usd_per_hour": 0.0})
+
+    def _dbu_rate(self, compute_type: str) -> float:
+        rates = self._block().get("dbu_usd_rate", {})
+        return float(rates.get(compute_type, rates.get("jobs", 0.0)))
+
+    def photon_premium_multiplier(self) -> float:
+        """How much MORE DBU/hr Photon costs vs standard (e.g. 2.0 = +100%)."""
+        return float(self._block().get("photon_premium_multiplier", 1.0))
+
+    def cluster_dbu_per_hour(self, driver: str, worker: str, num_workers: int) -> float:
+        nodes = [driver] + [worker] * max(num_workers, 0)
+        return sum(self._node(n)["dbu_per_hour"] for n in nodes)
+
+    def photon_premium_usd(self, driver: str, worker: str, num_workers: int,
+                           compute_type: str, runtime_hours: float) -> float:
+        """The DOLLARS of Photon premium for a run of this shape/duration — i.e. the
+        extra cost vs running the same cluster on standard compute. This is the pool
+        from which fallback waste is drawn."""
+        dbu_hr = self.cluster_dbu_per_hour(driver, worker, num_workers)
+        rate = self._dbu_rate(compute_type)
+        extra_factor = max(self.photon_premium_multiplier() - 1.0, 0.0)
+        gross = runtime_hours * dbu_hr * rate * extra_factor
+        return round(gross * (1.0 - self.contract_discount), 4)

clusop/runtime/__init__.py

@@ -0,0 +1 @@
+"""clusop — Photon Fallback Analyzer (zero-touch Databricks cost forensics)."""

clusop/runtime/bootstrap.py

@@ -0,0 +1,117 @@
+"""Self-arming tripwire — runs at interpreter startup via clusop_autoload.pth.
+
+CARDINAL RULE: best-effort, non-blocking, must NEVER break or slow the customer's
+job. Every step is wrapped to swallow exceptions and return fast. clusop degrading
+silently is fine; clusop delaying a production run is not.
+
+What activate() does, once a SparkSession exists and the JVM is reachable:
+  1. add the bundled listener JAR to the JVM (sparkContext.addJar) — "one pip" delivery,
+  2. instantiate + register com.clusop.listener.PhotonFallbackListener via py4j.
+
+On Spark Connect / USER_ISOLATION clusters the JVM is sealed -> we no-op cleanly
+(the listener can't run there; that's the documented scope boundary).
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+
+_ARMED = False
+
+
+def _log(msg: str) -> None:
+    try:
+        sys.stderr.write(f"[clusop] {msg}\n")
+    except Exception:
+        pass
+
+
+def _bundled_jar() -> str | None:
+    try:
+        import glob
+        here = os.path.dirname(os.path.abspath(__file__))
+        jars = glob.glob(os.path.join(here, "..", "jars", "*.jar"))
+        return os.path.abspath(jars[0]) if jars else None
+    except Exception:
+        return None
+
+
+def _active_spark():
+    try:
+        from pyspark.sql import SparkSession
+        return SparkSession.getActiveSession() or SparkSession._instantiatedSession
+    except Exception:
+        return None
+
+
+def activate() -> bool:
+    """Arm the listener. Idempotent, fail-open. Returns True only if registered."""
+    global _ARMED
+    if _ARMED or os.environ.get("CLSO_DISABLE") == "1":
+        return False
+    try:
+        spark = _active_spark()
+        if spark is None:
+            # session not up yet — a watcher (below) retries; don't block startup.
+            _install_watcher()
+            return False
+
+        from clusop.runtime.detect import jvm_available
+        if not jvm_available(spark):
+            _log("JVM not reachable (Spark Connect / Shared cluster) — listener not armed. "
+                 "clusop detection requires a dedicated / single-user cluster.")
+            return False
+
+        jar = _bundled_jar()
+        if jar:
+            try:
+                spark.sparkContext.addJar(jar)
+            except Exception as ex:
+                _log(f"addJar failed ({str(ex)[:80]}) — JAR may already be on the classpath")
+
+        premium = float(os.getenv("CLSO_PHOTON_PREMIUM", "1.0"))
+        endpoint = os.getenv("CLSO_SIGNAL_ENDPOINT", "")
+        try:
+            listener = spark._jvm.com.clusop.listener.PhotonFallbackListener(premium, endpoint)
+            spark._jsparkSession.listenerManager().register(listener)
+            _ARMED = True
+            _log("armed — PhotonFallbackListener registered.")
+            return True
+        except Exception as ex:
+            _log(f"listener registration failed: {str(ex)[:120]}")
+            return False
+    except Exception as ex:  # absolute backstop — never propagate into the job
+        _log(f"activate suppressed error: {str(ex)[:120]}")
+        return False
+
+
+def _install_watcher() -> None:
+    """If the session isn't up at import time, poll briefly on a daemon thread and
+    arm once it appears. Never touches the import machinery; never blocks the kernel."""
+    try:
+        import threading
+        import time
+
+        def _poll():
+            for _ in range(150):  # ~30s, then give up (job-task path still covered)
+                try:
+                    if _active_spark() is not None and activate():
+                        return
+                except Exception:
+                    pass
+                time.sleep(0.2)
+
+        threading.Thread(target=_poll, name="clusop-arm", daemon=True).start()
+    except Exception:
+        pass
+
+
+# fire on import (the .pth triggers this module import at interpreter startup)
+try:
+    # never arm Spark executors; only the driver.
+    _is_driver = os.environ.get("DB_IS_DRIVER")
+    if _is_driver is None or _is_driver.upper() == "TRUE":
+        activate()
+except Exception:
+    pass

clusop/runtime/detect.py

@@ -0,0 +1,106 @@
+"""Runtime auto-detection — cloud, DBR, cluster shape, access mode.
+
+Everything here is discovered in-process from the live SparkSession / env, so a NEW
+customer needs zero manual config. Values are only *overridable* (env vars), never
+required. All lookups are best-effort and never raise.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+# instance-prefix -> cloud (extend as needed; verified per cloud).
+_CLOUD_BY_PREFIX = {
+    # AWS EC2 families
+    "m5": "aws", "m5d": "aws", "m6": "aws", "c5": "aws", "r5": "aws", "i3": "aws", "i4": "aws",
+    # Azure
+    "standard_d": "azure", "standard_e": "azure", "standard_f": "azure", "standard_l": "azure",
+    # GCP
+    "n1": "gcp", "n2": "gcp", "n2d": "gcp", "e2": "gcp", "c2": "gcp",
+}
+
+
+def _conf(spark, key, default=None):
+    try:
+        return spark.conf.get(key)
+    except Exception:
+        return default
+
+
+def detect_cloud(spark=None, node_type: str | None = None, host: str | None = None) -> str:
+    """aws | azure | gcp | unknown — from instance-type prefix, then host pattern."""
+    if os.getenv("CLSO_CLOUD"):
+        return os.getenv("CLSO_CLOUD")
+    nt = (node_type or "").lower()
+    for pref, cloud in sorted(_CLOUD_BY_PREFIX.items(), key=lambda kv: -len(kv[0])):
+        if nt.startswith(pref):
+            return cloud
+    h = (host or os.getenv("DATABRICKS_HOST") or "").lower()
+    if "azuredatabricks.net" in h:
+        return "azure"
+    if "gcp.databricks.com" in h:
+        return "gcp"
+    if "cloud.databricks.com" in h:
+        return "aws"
+    return "unknown"
+
+
+def detect_dbr() -> str:
+    """Databricks Runtime version string, e.g. '15.4'. Read in-process; never asks."""
+    return (os.getenv("CLSO_DBR")
+            or os.getenv("DATABRICKS_RUNTIME_VERSION")
+            or "unknown")
+
+
+@dataclass(frozen=True)
+class ClusterShape:
+    cloud: str
+    dbr: str
+    driver_node_type: str
+    worker_node_type: str
+    num_workers: int
+    photon_enabled: bool
+    data_security_mode: str   # SINGLE_USER | USER_ISOLATION | ... (drives JVM availability)
+    compute_type: str         # jobs | all_purpose
+    cluster_id: str
+    cluster_name: str
+
+
+def detect_shape(spark=None) -> ClusterShape:
+    """Best-effort cluster shape from clusterUsageTags / env. JVM-free where possible
+    (works even before the listener registers)."""
+    tag = lambda k, d=None: _conf(spark, f"spark.databricks.clusterUsageTags.{k}", d) if spark else d  # noqa: E731
+    driver = tag("driverNodeType") or tag("clusterNodeType") or os.getenv("CLSO_NODE_TYPE") or "unknown"
+    worker = tag("workerNodeType") or tag("clusterNodeType") or driver
+    try:
+        workers = int(float(tag("clusterWorkers", "0") or 0))
+    except Exception:
+        workers = 0
+    dbr = detect_dbr()
+    photon = "photon" in (dbr.lower()) or (_conf(spark, "spark.databricks.photon.enabled", "false") == "true")
+    dsm = tag("clusterDataSecurityMode") or os.getenv("CLSO_DATA_SECURITY_MODE") or "unknown"
+    return ClusterShape(
+        cloud=detect_cloud(spark, node_type=driver),
+        dbr=dbr,
+        driver_node_type=driver,
+        worker_node_type=worker,
+        num_workers=workers,
+        photon_enabled=bool(photon),
+        data_security_mode=dsm,
+        compute_type=("jobs" if (tag("jobId") or os.getenv("DB_JOB_ID")) else "all_purpose"),
+        cluster_id=tag("clusterId") or os.getenv("DB_CLUSTER_ID") or "unknown",
+        cluster_name=tag("clusterName") or os.getenv("DB_CLUSTER_NAME") or "unknown",
+    )
+
+
+def jvm_available(spark=None) -> bool:
+    """True only on clusters where Python can reach the JVM/py4j (SINGLE_USER /
+    dedicated). USER_ISOLATION (Shared) runs Spark Connect and seals the JVM — the
+    listener cannot register there. This is the scope boundary, checked honestly."""
+    try:
+        _ = spark._jvm  # noqa: B018 — Spark Connect raises here
+        _ = spark.sparkContext
+        return True
+    except Exception:
+        return False

clusop/service/__init__.py

@@ -0,0 +1 @@
+"""clusop — Photon Fallback Analyzer (zero-touch Databricks cost forensics)."""

clusop/service/aggregator.py

@@ -0,0 +1,69 @@
+"""Two-tier aggregator.
+
+Tier 1 — DriverBatcher (ephemeral, in the driver): dedup by signature within the
+session, accumulate waste + occurrences, bound memory, flush compact signals to the
+central sink. Dies with the cluster, holds nothing durable (a driver crash loses at
+most the unflushed batch — acceptable for an estimation product).
+
+Tier 2 — CentralAggregator (durable, off-cluster): window by job_run (closed on an
+idle timeout — job-run completion isn't observable from the listener, an explicit
+approximation), idempotent upsert by {job_run_id, signature}, cross-run suppression,
+prioritization. The real scaling locus.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+# ---- Tier 1: driver-local ---------------------------------------------------
+@dataclass
+class DriverBatcher:
+    max_signatures: int = 1000
+    _acc: dict = field(default_factory=dict)
+    evicted: int = 0
+
+    def add(self, signal) -> None:
+        sig = signal.signature
+        cur = self._acc.get(sig)
+        if cur is None:
+            if len(self._acc) >= self.max_signatures:
+                # evict lowest-waste signature (bounded memory)
+                victim = min(self._acc.values(), key=lambda s: s.wasted_usd_modeled)
+                del self._acc[victim.signature]
+                self.evicted += 1
+            self._acc[sig] = signal
+        else:
+            cur.occurrences += signal.occurrences
+            cur.wasted_usd_modeled = round(cur.wasted_usd_modeled + signal.wasted_usd_modeled, 4)
+            cur.runtime_seconds += signal.runtime_seconds
+
+    def flush(self) -> list:
+        out = list(self._acc.values())
+        self._acc.clear()
+        return out
+
+
+# ---- Tier 2: central, durable ----------------------------------------------
+class CentralAggregator:
+    """Reference in-memory impl of the durable service contract (real deployment =
+    stateful HA service + DB). Demonstrates window / idempotency / suppression."""
+
+    def __init__(self, store=None, resolver=None):
+        self._store = store if store is not None else {}   # {(job_run_id, signature): signal}
+        self._resolver = resolver                            # CostTierResolver | None
+
+    def ingest(self, signal) -> None:
+        key = (signal.job_run_id, signal.signature)
+        existing = self._store.get(key)
+        if existing is None:
+            if self._resolver is not None:
+                self._resolver.reconcile(signal)   # modeled -> billed if granted
+            self._store[key] = signal
+        else:
+            # idempotent upsert: retried/duplicate delivery collapses, no double count
+            existing.occurrences = max(existing.occurrences, signal.occurrences)
+            existing.wasted_usd_modeled = max(existing.wasted_usd_modeled, signal.wasted_usd_modeled)
+
+    def open_for_run(self, job_run_id: str) -> list:
+        return [s for (jr, _), s in self._store.items() if jr == job_run_id]

clusop/service/onboard.py

@@ -0,0 +1,34 @@
+"""clusop onboarding (central service side).
+
+Creates the clusop telemetry schema + signals table, probes the cost tier, and reports
+exactly what the customer is getting. Fail-LOUD only if it can't create its own schema
+or reach the webhook; never fails on missing system.* access (that just keeps cost
+modeled). Pass a sql_runner (a Databricks SQL warehouse client) + teams webhook.
+"""
+
+from __future__ import annotations
+
+from clusop.service.resolver import CostTierResolver
+
+_DDL = """
+CREATE TABLE IF NOT EXISTS {catalog}.{schema}.clusop_signals (
+    signature STRING, cluster_id STRING, job_id STRING, job_run_id STRING,
+    cloud STRING, dbr STRING, cause STRING, verdict_kind STRING,
+    fallback_weight DOUBLE, runtime_seconds DOUBLE,
+    wasted_usd DOUBLE, cost_basis STRING, occurrences INT,
+    confidence STRING, evidence STRING, captured_at TIMESTAMP, dt DATE
+) USING DELTA PARTITIONED BY (dt)
+"""
+
+
+def onboard(sql_runner, *, catalog="main", schema="clusop", teams_webhook=None) -> dict:
+    sql_runner(f"CREATE SCHEMA IF NOT EXISTS {catalog}.{schema}")
+    sql_runner(_DDL.format(catalog=catalog, schema=schema))
+    tier = CostTierResolver(sql_runner).probe()
+    msg = (f"clusop onboarded -> {catalog}.{schema}.clusop_signals. Cost tier: "
+           f"{'BILLED (Tier-1)' if tier.billed else 'MODELED (Tier-0)'} — {tier.reason}.")
+    if not tier.billed:
+        msg += (" To enable billed reconciliation, grant the clusop service principal "
+                "SELECT on system.billing + system.compute.")
+    return {"ok": True, "cost_tier_billed": tier.billed, "message": msg,
+            "webhook_set": bool(teams_webhook)}

clusop/service/resolver.py

@@ -0,0 +1,59 @@
+"""Cost-tier resolver (central service, off-cluster).
+
+Detection is ALWAYS in-process (system tables can't see the physical plan). This
+resolver only upgrades the COST leg: probe read access to system.billing; if present,
+reconcile modeled -> billed and raise cost confidence to HIGH; else stay modeled.
+Read-only, cheap, cached, fail-open. NEVER a detection dependency.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass
+class CostTier:
+    billed: bool
+    reason: str
+
+
+class CostTierResolver:
+    def __init__(self, sql_runner=None):
+        # sql_runner: callable(str)->rows, e.g. a Databricks SQL warehouse client.
+        # None => no system access => modeled only.
+        self._sql = sql_runner
+        self._cached: CostTier | None = None
+
+    def probe(self) -> CostTier:
+        if self._cached is not None:
+            return self._cached
+        if self._sql is None:
+            self._cached = CostTier(False, "no SQL access configured — modeled cost (Tier-0)")
+            return self._cached
+        try:
+            self._sql("SELECT 1 FROM system.billing.usage LIMIT 1")
+            self._cached = CostTier(True, "system.billing readable — billed reconciliation (Tier-1)")
+        except Exception as ex:
+            self._cached = CostTier(False, f"system.billing not readable ({str(ex)[:60]}) — modeled (Tier-0)")
+        return self._cached
+
+    def reconcile(self, signal) -> None:
+        """If billed access exists, replace the modeled waste on a FallbackSignal with
+        billed-derived cost and flip its cost basis. Mutates signal in place."""
+        tier = self.probe()
+        if not tier.billed or self._sql is None:
+            return
+        try:
+            rows = self._sql(
+                "SELECT SUM(usage_quantity) AS dbus FROM system.billing.usage u "
+                "JOIN system.billing.list_prices p USING (sku_name) "
+                f"WHERE u.usage_metadata.cluster_id = '{signal.cluster_id}' "
+                "AND u.sku_name ILIKE '%PHOTON%' "
+                "AND u.usage_date >= current_date() - INTERVAL 1 DAY LIMIT 1"
+            )
+            # (production: derive premium share of billed DBUs and scale by fallback_weight)
+            if rows:
+                signal.confidence["cost"] = "HIGH"
+                signal.evidence += " | billed-reconciled"
+        except Exception:
+            pass  # fail-open: keep modeled

clusop/service/suppression.py

@@ -0,0 +1,40 @@
+"""Suppression + prioritization — turns thousands of signals into the few cards a
+human should actually see. At fleet scale the bottleneck is human attention, not
+parser throughput, so surfacing 10,000 cards == surfacing none.
+
+All parameters are CALIBRATION OUTPUTS of the false-positive study, not constants.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+_CONF_RANK = {"LOW": 0, "MEDIUM": 1, "HIGH": 2}
+
+
+@dataclass(frozen=True)
+class Gate:
+    sustained_runs_n: int = 3          # fallback must persist across N runs
+    min_history_runs: int = 3          # below this -> insufficient_data, no card
+    min_waste_usd_per_month: float = 50.0
+    min_recommendation_conf: str = "MEDIUM"
+    cooldown_hours: float = 168.0      # don't re-surface an accepted/snoozed sig for 7d
+    per_team_daily_cap: int = 25       # alert-storm guard
+
+
+def passes_gate(*, sustained_runs: int, history_runs: int, monthly_waste_usd: float,
+                rec_conf: str, gate: Gate = Gate()) -> tuple[bool, str]:
+    if history_runs < gate.min_history_runs:
+        return False, "insufficient_data"
+    if sustained_runs < gate.sustained_runs_n:
+        return False, "not yet sustained"
+    if monthly_waste_usd < gate.min_waste_usd_per_month:
+        return False, "below dollar floor"
+    if _CONF_RANK.get(rec_conf, 0) < _CONF_RANK[gate.min_recommendation_conf]:
+        return False, "recommendation confidence too low"
+    return True, "ok"
+
+
+def priority(monthly_waste_usd: float, rec_conf: str) -> float:
+    """Rank surfaced candidates by waste x confidence — emit top-N per team/day."""
+    return monthly_waste_usd * (1 + _CONF_RANK.get(rec_conf, 0))

clusop/service/teams.py

@@ -0,0 +1,44 @@
+"""Propose-never-apply Teams Adaptive Card.
+
+Surfaces the fallback, the modeled (or billed) wasted premium, the decomposed
+confidence (headline badge = recommendation leg, never parse), and the action.
+On Approve the control plane flips runtime_engine PHOTON->STANDARD for FUTURE runs
+only, or files the UDF-rewrite — never auto-applied, always reversible + audited.
+"""
+
+from __future__ import annotations
+
+
+def build_card(signal, *, view_url: str = "") -> dict:
+    rec = signal.confidence.get("recommendation", "MEDIUM")
+    action = ("Rewrite the Python UDF in native PySpark/SQL (or register it as a UC "
+              "Python UDF), or disable Photon on this job."
+              if signal.verdict_kind == "fix_fallback"
+              else "Disable Photon on this job cluster — execution fell back to the JVM, "
+                   "so the premium is yielding little/no speedup.")
+    body = [
+        {"type": "TextBlock", "size": "Large", "weight": "Bolder", "wrap": True,
+         "text": "\U0001f9ea Photon Fallback Detected"},
+        {"type": "TextBlock", "isSubtle": True, "wrap": True,
+         "text": f"job_run `{signal.job_run_id}` · cluster `{signal.cluster_id}` · {signal.cloud}/{signal.dbr}"},
+        {"type": "TextBlock", "wrap": True,
+         "text": f"**Wasted Photon premium (~):** ${signal.wasted_usd_modeled} "
+                 f"({'billed' if signal.confidence.get('cost') == 'HIGH' else 'modeled'}) · "
+                 f"fallback severity {int(signal.fallback_weight*100)}%"},
+        {"type": "TextBlock", "wrap": True, "text": f"**Cause:** {signal.cause}"},
+        {"type": "FactSet", "facts": [
+            {"title": "parse", "value": signal.confidence.get("parse", "?")},
+            {"title": "diagnosis", "value": signal.confidence.get("diagnosis", "?")},
+            {"title": "cost", "value": signal.confidence.get("cost", "?")},
+            {"title": "recommendation", "value": rec},
+        ]},
+        {"type": "TextBlock", "wrap": True, "text": f"**Action:** {action}"},
+        {"type": "TextBlock", "wrap": True, "isSubtle": True,
+         "text": "Propose only — nothing changes until you approve; applies to future runs."},
+    ]
+    actions = [{"type": "Action.OpenUrl", "title": "\U0001f50d View analysis",
+                "url": view_url or "#"}]
+    content = {"$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
+               "type": "AdaptiveCard", "version": "1.4", "body": body, "actions": actions}
+    return {"type": "message", "attachments": [
+        {"contentType": "application/vnd.microsoft.card.adaptive", "content": content}]}

clusop-0.0.1.dist-info/METADATA

@@ -0,0 +1,107 @@
+Metadata-Version: 2.4
+Name: clusop
+Version: 0.0.1
+Summary: Photon Fallback Analyzer — zero-touch Databricks cost forensics (.pth self-arm + bundled JVM listener)
+Author: yogasathyandrun
+License: Proprietary
+Keywords: cost,databricks,finops,photon,spark
+Requires-Python: >=3.10
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == 'dev'
+Requires-Dist: ruff>=0.5; extra == 'dev'
+Provides-Extra: service
+Requires-Dist: databricks-sdk>=0.30; extra == 'service'
+Requires-Dist: requests>=2.31; extra == 'service'
+Description-Content-Type: text/markdown
+
+# clusop — Photon Fallback Analyzer
+
+`pip install clusop`
+
+clusop finds the money Databricks Photon quietly burns. When Photon hits an operator it
+can't run (a Python UDF, a struct-IN filter, an unsupported Delta feature), it silently
+**falls back** to the JVM — you keep paying the 2–2.9× Photon DBU premium while getting
+JVM speed. clusop detects those fallbacks from the executed plan, estimates the wasted
+spend, and proposes a fix. It never touches your job.
+
+## One install, two halves, zero config
+
+A single `pip install clusop` ships both halves and arms itself:
+
+- a **Python `.pth`** (`clusop_autoload.pth`) that Python's `site` machinery runs at
+  interpreter startup → imports `clusop.runtime.bootstrap` → arms automatically. You do
+  not `import clusop` anywhere in your job.
+- a bundled **Scala JAR** (`clusop/jars/photon_listener.jar`) — the actual
+  `QueryExecutionListener`. The bootstrap `addJar`s it and registers it over Py4J.
+
+Install it as a **cluster/job library** (production path) so every interpreter that
+starts already has it. `%pip install clusop` + `dbutils.library.restartPython()` works for
+dev.
+
+Everything is **auto-detected** at runtime — cloud (from the instance type), DBR (from
+`DATABRICKS_RUNTIME_VERSION`), cluster shape, and whether the JVM is reachable. No
+per-user setup; a new customer pip-installs and it works.
+
+## What it costs you: nothing if it can't run safely
+
+- **Fail-open everywhere.** Any error in arming, listening, parsing, or dispatch is
+  swallowed. clusop never breaks, slows, or blocks a customer query.
+- **Needs a SINGLE_USER / dedicated cluster** for the JVM listener. On USER_ISOLATION
+  (Shared) clusters the JVM is sealed behind Spark Connect — clusop detects this and stays
+  dormant rather than failing.
+- **Propose-never-apply.** clusop emits a signal and a Teams card. A human decides.
+
+## How a signal is born
+
+1. The JVM listener sees a finished query and reads its **executed plan**.
+2. The **structural parser** ([src/clusop/analysis/parser.py](src/clusop/analysis/parser.py),
+   mirrored in Scala) decides if this is a *real* fallback. The hard part: a clean Photon
+   query always ends in one **terminal** `ColumnarToRow` (the normal result boundary) —
+   that is **not** a fallback. Real fallback is **mid-plan** `ColumnarToRow`,
+   `RowToColumnar` round-trips, or `BatchEvalPython`/`ArrowEvalPython`. Counting raw
+   occurrences false-positives; clusop doesn't.
+3. The **waste model** ([waste.py](src/clusop/analysis/waste.py)) sizes the loss:
+   `runtime × Σ(node DBU/hr) × $rate × (photon_premium−1) × fallback_weight`.
+4. **Confidence** ([confidence.py](src/clusop/analysis/confidence.py)) is decomposed into
+   four legs — parse / diagnosis / cost / recommendation. A *fix* (rewrite the UDF) needs
+   only parse+diagnosis; a *disable-Photon* recommendation is a dollar decision and is
+   capped by the cost leg.
+5. The signal flows to a **driver batcher** (dedup by signature) → **central aggregator**
+   (idempotent upsert, suppression, prioritization) → Teams card.
+
+## Cost: modeled now, billed if granted
+
+clusop always works with a **modeled** cost (public price table → MEDIUM cost confidence).
+If the workspace can read `system.billing.usage`, the
+[CostTierResolver](src/clusop/service/resolver.py) reconciles the estimate to **billed**
+dollars (HIGH confidence). Detection never depends on system tables — they only upgrade
+the cost layer. Prices are reference data, not something clusop invents per-row.
+
+## Certifying it's real
+
+[harness/certify.py](harness/certify.py) runs on a dedicated cluster, captures real
+executed plans for known clean / known-fallback fixtures, and reports catch-rate plus a
+**DBR-stamped Delta feature-support matrix** — derived, never hand-maintained.
+
+## Layout
+
+```
+src/clusop/
+  runtime/      bootstrap (arming), detect (cloud/DBR/shape/jvm)
+  analysis/     parser · waste · confidence · signal
+  pricing/      price_table.json · provider
+  service/      aggregator · resolver · suppression · teams · onboard
+  jars/         photon_listener.jar  (baked in by the release workflow)
+scala/          the QueryExecutionListener (sbt; mirror of the Python parser)
+harness/        certify.py  (catch-rate + Delta matrix, run on a dedicated cluster)
+tests/          parser / waste / confidence
+```
+
+See [SPEC.md](SPEC.md) for the full design and invariants.
+
+## Release
+
+Push-button: the [Publish workflow](.github/workflows/release-pypi.yml) auto-versions
+(`max(PyPI, tags)+1`), builds the JAR with sbt, bakes it into the wheel, lints+tests,
+gates on *(.pth at wheel root AND JAR bundled)*, then publishes via PyPI Trusted
+Publishing (OIDC — no stored token) and tags the bump.

clusop-0.0.1.dist-info/RECORD

@@ -0,0 +1,24 @@
+clusop/__init__.py,sha256=Fb6IqaqvJe1IeYKawOAElXXgTu_joXp78y9Q7QMBLuM,1066
+clusop/analysis/__init__.py,sha256=2HKrWyp5lTZHU921rI2-nVGyxgKEgh5DiMDIyI1pzHo,82
+clusop/analysis/confidence.py,sha256=NmkAiyuNj6GDzRhLEBqtJi6DpK5YXgOtS68VQXVFsJk,2661
+clusop/analysis/parser.py,sha256=j1Yz51uiATlF2i5Z-sbdH99uE9G00XbJHXLw79_cRzo,4390
+clusop/analysis/signal.py,sha256=ZSah6nlCVP2rOIOQaJM-8uTMRtmXtfvKEqzGkjpIvNE,1327
+clusop/analysis/waste.py,sha256=lfmYPWbMOxDWwGPim12SGUuyuij3o8vVt5Q2aVIZKTA,1491
+clusop/jars/.gitkeep,sha256=Sd7zg-Ir9eq3LY53l8RJ9uFiEgt9g7deppu-qtF3NVk,89
+clusop/jars/photon_listener.jar,sha256=TUhbw87uk8Jo6YLuYg6iKxdFa6SVYeXWLqEjcMG7tq8,19171
+clusop/pricing/__init__.py,sha256=2HKrWyp5lTZHU921rI2-nVGyxgKEgh5DiMDIyI1pzHo,82
+clusop/pricing/price_table.json,sha256=YUQq0QIK3zWooOPAO4rY-K9HuoVn9enW3m7cxsnCfM0,1660
+clusop/pricing/provider.py,sha256=ULfovKkvUjDl6q6n-oZsi5niaxvVJn1VXibc5SPi_k4,2749
+clusop/runtime/__init__.py,sha256=2HKrWyp5lTZHU921rI2-nVGyxgKEgh5DiMDIyI1pzHo,82
+clusop/runtime/bootstrap.py,sha256=I3yb4gLfFQLPyd9gmVN8877s4iuhH1iUVqmy7A23oPs,4033
+clusop/runtime/detect.py,sha256=J3hRAg2mCjbDk-l6r9EYZzK9LJ9L_YD4lwMhJNqiTvY,3977
+clusop/service/__init__.py,sha256=2HKrWyp5lTZHU921rI2-nVGyxgKEgh5DiMDIyI1pzHo,82
+clusop/service/aggregator.py,sha256=t0aEWUE-4zocWBXymgxsAAcyuGrAlGNgk_b2eNF3DB0,2955
+clusop/service/onboard.py,sha256=K93zle_MVBqdFth5PNXzpSocI2m3EabB9UBLy8zCNGo,1647
+clusop/service/resolver.py,sha256=jZEkSBdjPzViYa7JQzP3ziIr-2DI6waokVQ7w4ZbgVo,2421
+clusop/service/suppression.py,sha256=RRDzTqReRgJPauc9thMwk8XcR4Orp149dhRmNH97tx0,1680
+clusop/service/teams.py,sha256=TLMRmkUJpoi4dXfnE8s4SKHRdqLXLCcmtbHp7v90M9c,2612
+clusop_autoload.pth,sha256=PzWzzxAddUNX7qpdsYXiWHXcJHkhYb_nPQMAbtERTbU,32
+clusop-0.0.1.dist-info/METADATA,sha256=Pfp-v2-GOQTvJxSe8o9gJ9drtINGarxpUEKW2mfpy34,5296
+clusop-0.0.1.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+clusop-0.0.1.dist-info/RECORD,,

clusop-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

clusop_autoload.pth

@@ -0,0 +1 @@
+import clusop.runtime.bootstrap