shkit 1.2.0__py3-none-any.whl

iic/diagnosis/diagnosis_engine.py

@@ -0,0 +1,183 @@
+"""Stage 9 — the diagnosis engine.
+
+The only place an LLM enters the system, and only after the DNA, impact, and
+routing decision exist. Two paths:
+
+  * ``tier == NONE``  → no LLM. A deterministic narrative is synthesised from the
+    DNA (cache replay, or a derived-dependency pointer, or a templated explanation
+    for a confidently-classified failure). The system is fully useful with zero
+    tokens spent.
+  * LLM tier → a strict-JSON prompt built *from the structured DNA* (not raw
+    text), parsed into a structured :class:`DiagnosisResult`. The model interprets
+    structure; it does not classify or chat.
+"""
+
+from __future__ import annotations
+
+import json
+
+from iic.models.change import ChangeDiffObject
+from iic.models.context import IncidentContextBundle
+from iic.models.diagnosis import DiagnosisResult
+from iic.models.dna import FailureType, IncidentDNA
+from iic.models.event import NormalizedFailureEvent
+from iic.models.routing import RoutingDecision
+
+_SYSTEM_MSG = (
+    "You are a senior data reliability engineer. You are given a STRUCTURED incident "
+    "fingerprint that was already classified deterministically. Do not re-classify; "
+    "explain the root cause and propose a concrete fix. Respond ONLY in valid JSON. "
+    "Never suggest DROP, DELETE, or TRUNCATE."
+)
+
+_JSON_SPEC = (
+    '\n\nReturn ONLY this JSON object:\n'
+    '{"root_cause": "one-sentence root cause", "confidence": 0.0-1.0, '
+    '"reasoning": "why, citing the signals/changes", '
+    '"suggested_fix": "concrete remediation step", '
+    '"alternatives": ["other hypothesis", "..."]}'
+)
+
+# Deterministic narratives used when no LLM is called (tier == NONE) or as a
+# fallback if the model errors. Keyed by failure type.
+_TEMPLATES: dict[FailureType, tuple[str, str]] = {
+    FailureType.SCHEMA_DRIFT: (
+        "The failing task hit a schema mismatch — a column it expects is missing or changed type.",
+        "Reconcile the ingestion/transform mapping with the new source schema "
+        "(add/rename the column or enable schema evolution), then re-run.",
+    ),
+    FailureType.DATA_QUALITY: (
+        "A data-quality rule failed (duplicates, nulls, or a threshold/constraint breach).",
+        "Inspect the offending rows from the evidence query, fix the source/transform, and re-run the quality gate.",
+    ),
+    FailureType.MISSING_DATA: (
+        "An expected input table or file path was empty or absent when the task ran.",
+        "Confirm the upstream load completed and the path/table exists, then re-run; add a presence check upstream.",
+    ),
+    FailureType.PERMISSION: (
+        "The executing identity lacks a required privilege on a table, path, or secret.",
+        "Grant the missing privilege to the run identity (least-privilege) and re-run; no data change needed.",
+    ),
+    FailureType.DEPENDENCY: (
+        "This task did not fail on its own — an upstream task in the same run failed first.",
+        "Resolve the upstream root-cause incident; this task should recover on its re-run.",
+    ),
+    FailureType.RESOURCE: (
+        "The task exhausted compute resources (memory/executors/disk).",
+        "Increase cluster size or partition the workload; re-run with adjusted resources.",
+    ),
+    FailureType.TIMEOUT: (
+        "The task exceeded its time budget.",
+        "Raise the timeout or optimise the slow stage (skew/partitioning), then re-run.",
+    ),
+    FailureType.CONFIG: (
+        "A required configuration value, parameter, or secret was missing or invalid.",
+        "Correct the parameter/secret value in the job/secret scope and re-run.",
+    ),
+    FailureType.CODE_ERROR: (
+        "The notebook raised a code-level exception (syntax/type/import).",
+        "Fix the code defect indicated by the traceback and redeploy/re-run.",
+    ),
+    FailureType.UNKNOWN: (
+        "The failure could not be classified deterministically.",
+        "Review the captured logs and notebook source; escalate to the owning engineer.",
+    ),
+}
+
+
+class DiagnosisEngine:
+    """Produces a structured :class:`DiagnosisResult` for an incident."""
+
+    def __init__(self, client=None):
+        self.client = client
+
+    def diagnose(
+        self,
+        event: NormalizedFailureEvent,
+        dna: IncidentDNA,
+        routing: RoutingDecision,
+        context: IncidentContextBundle | None = None,
+        change_diff: ChangeDiffObject | None = None,
+        evidence: list[str] | None = None,
+    ) -> tuple[DiagnosisResult, int]:
+        """Return ``(DiagnosisResult, tokens_used)``."""
+        if not routing.requires_llm or self.client is None:
+            return self._deterministic(dna, routing, change_diff), 0
+
+        prompt = self._build_prompt(event, dna, context, change_diff, evidence)
+        try:
+            content, usage = self.client.invoke_model_full(
+                routing.model,
+                [{"role": "system", "content": _SYSTEM_MSG}, {"role": "user", "content": prompt}],
+                max_tokens=900,
+            )
+        except Exception as ex:
+            result = self._deterministic(dna, routing, change_diff)
+            result.reasoning = f"LLM unavailable ({str(ex)[:80]}); used deterministic template."
+            return result, 0
+
+        tokens = int((usage or {}).get("total_tokens", 0) or 0)
+        return self._parse(content, dna, routing), tokens
+
+    # ─── deterministic path ───
+
+    def _deterministic(self, dna: IncidentDNA, routing: RoutingDecision,
+                        change_diff: ChangeDiffObject | None) -> DiagnosisResult:
+        root_cause, fix = _TEMPLATES.get(dna.failure_type, _TEMPLATES[FailureType.UNKNOWN])
+        confidence = {"high": 0.85, "medium": 0.6, "low": 0.35}.get(dna.confidence_signature, 0.4)
+        evidence = list(dna.signals)
+        if change_diff and change_diff.has_changes:
+            evidence += [f"change: {s}" for s in change_diff.summaries()[:3]]
+        reasoning = routing.reason or "Deterministic classification; no LLM required."
+        return DiagnosisResult(
+            root_cause=root_cause,
+            confidence=confidence,
+            reasoning=reasoning,
+            suggested_fix=fix,
+            evidence=evidence,
+            produced_by="rules",
+        )
+
+    # ─── LLM path ───
+
+    def _build_prompt(self, event, dna, context, change_diff, evidence) -> str:
+        changes = (change_diff.summaries() if change_diff else []) or ["none detected"]
+        ev = evidence or []
+        nb = (context.notebook_source[:1200] if context and context.notebook_source else "")
+        return (
+            f"INCIDENT FINGERPRINT (pre-classified):\n"
+            f"- failure_type: {dna.failure_type.value}\n"
+            f"- affected_layer: {dna.affected_layer}\n"
+            f"- system_layer: {dna.system_layer.value}\n"
+            f"- root_signal: {dna.root_signal}\n"
+            f"- signals: {', '.join(dna.signals) or 'n/a'}\n"
+            f"- pattern_id: {dna.pattern_id}\n\n"
+            f"PIPELINE: {event.pipeline} · TASK: {event.task}\n"
+            f"ERROR:\n{event.error_message[:1500]}\n\n"
+            f"RECENT CHANGES (since last success):\n- " + "\n- ".join(changes[:8]) + "\n\n"
+            "DATA EVIDENCE:\n- " + ("\n- ".join(ev[:6]) if ev else "none") + "\n\n"
+            + (f"NOTEBOOK (truncated):\n{nb}\n" if nb else "")
+            + _JSON_SPEC
+        )
+
+    def _parse(self, content: str, dna: IncidentDNA, routing: RoutingDecision) -> DiagnosisResult:
+        cleaned = (content or "").strip()
+        if cleaned.startswith("```"):
+            cleaned = cleaned.split("\n", 1)[-1]
+        if cleaned.endswith("```"):
+            cleaned = cleaned[:-3]
+        try:
+            data = json.loads(cleaned.strip())
+        except Exception:
+            result = self._deterministic(dna, routing, None)
+            result.reasoning = "LLM returned unparseable output; used deterministic template."
+            return result
+        return DiagnosisResult(
+            root_cause=str(data.get("root_cause", ""))[:500] or _TEMPLATES[dna.failure_type][0],
+            confidence=data.get("confidence", 0.5),
+            reasoning=str(data.get("reasoning", ""))[:800],
+            suggested_fix=str(data.get("suggested_fix", ""))[:500],
+            alternatives=[str(a)[:200] for a in (data.get("alternatives") or [])][:4],
+            evidence=list(dna.signals),
+            produced_by=f"llm:{routing.model}",
+        )

iic/dna/__init__.py

@@ -0,0 +1,7 @@
+"""Stage 6 — Incident DNA construction (the core intelligence layer)."""
+
+from __future__ import annotations
+
+from iic.dna.dna_builder import IncidentDNABuilder
+
+__all__ = ["IncidentDNABuilder"]

iic/dna/dna_builder.py

@@ -0,0 +1,184 @@
+"""Stage 6 — build the IncidentDNA. The heart of the system.
+
+This is *deterministic, rule-based* classification: it maps the raw error text,
+the gathered context, and the change diff onto the canonical
+:class:`FailureType` taxonomy, and records exactly which signals fired so the
+result is fully traceable. The LLM never runs here — it runs later, on the DNA
+this stage produces.
+
+Why rules and not an LLM for classification:
+  * predictable, testable, free, and instant;
+  * the classification drives cost decisions (model routing), so it must not
+    itself cost an LLM call;
+  * a wrong-but-confident LLM label would poison everything downstream.
+"""
+
+from __future__ import annotations
+
+from iic.models.change import ChangeDiffObject
+from iic.models.context import IncidentContextBundle
+from iic.models.dna import FailureType, IncidentDNA, SystemLayer
+from iic.models.event import NormalizedFailureEvent
+
+# Ordered most-specific → least-specific. The first type with a matching signal
+# (after the dependency override) wins. Each pattern is a lowercased substring.
+_RULES: list[tuple[FailureType, tuple[str, ...]]] = [
+    (FailureType.PERMISSION, (
+        "permission denied", "access denied", "unauthorized", "not authorized",
+        "forbidden", "does not have privilege", "requires permission", "403",
+    )),
+    (FailureType.SCHEMA_DRIFT, (
+        "no such column", "cannot resolve column", "cannot resolve '",
+        "unresolved column", "missing column", "schema mismatch",
+        "incompatible schema", "schema is not compatible", "field not found",
+        "column not found", "mergeschema", "schema drift",
+    )),
+    (FailureType.MISSING_DATA, (
+        "path does not exist", "table or view not found", "no such table",
+        "file not found", "filenotfound", "does not exist", "no files found",
+        "empty input", "0 rows", "returned no rows",
+    )),
+    (FailureType.DATA_QUALITY, (
+        "duplicate", "constraint", "expectation", "check constraint",
+        "not null", "null value", "validation failed", "data quality",
+        "threshold", "violat",
+    )),
+    (FailureType.RESOURCE, (
+        "out of memory", "outofmemory", "java heap", "gc overhead",
+        "executor lost", "container killed", "no space left", "disk is full",
+        "exceeded memory", "lost executor", "exceeds the allowed",
+    )),
+    (FailureType.TIMEOUT, (
+        "timeout", "timed out", "deadline exceeded", "exceeded the timeout",
+        "query has exceeded the time", "cancelled because it exceeded",
+    )),
+    (FailureType.CONFIG, (
+        "not found in scope", "missing required", "invalid argument",
+        "secret", "environment variable", "missing parameter", "no value for",
+        "configuration", "could not find a value",
+    )),
+    (FailureType.CODE_ERROR, (
+        "syntaxerror", "typeerror", "attributeerror", "nameerror",
+        "importerror", "modulenotfounderror", "valueerror", "indexerror",
+        "keyerror", "zerodivisionerror", "traceback (most recent call last)",
+    )),
+]
+
+# System-layer hints (lowercased substrings).
+_LAYER_HINTS: list[tuple[SystemLayer, tuple[str, ...]]] = [
+    (SystemLayer.SPARK, ("spark", "executor", "java heap", "stage failure", "py4j", "analysisexception")),
+    (SystemLayer.STORAGE, ("s3", "abfss", "dbfs", "adls", "blob", "no space", "disk", "path does not exist")),
+    (SystemLayer.NETWORK, ("connection", "timeout", "timed out", "unreachable", "dns", "socket")),
+]
+
+_MEDALLION = ("bronze", "silver", "gold", "raw", "staging", "mart", "clean", "curated")
+
+
+class IncidentDNABuilder:
+    """Constructs an :class:`IncidentDNA` deterministically from structured inputs."""
+
+    def build(
+        self,
+        event: NormalizedFailureEvent,
+        context: IncidentContextBundle | None = None,
+        change_diff: ChangeDiffObject | None = None,
+        upstream_failed: bool = False,
+    ) -> IncidentDNA:
+        text = self._haystack(event, context)
+        signals: list[str] = []
+
+        failure_type = FailureType.UNKNOWN
+        # Dependency override: a task whose upstream also failed is a *derived*
+        # failure — its true root cause is upstream, so we never mis-attribute it.
+        if upstream_failed:
+            failure_type = FailureType.DEPENDENCY
+            signals.append("upstream task in the same run also failed")
+        else:
+            for ftype, patterns in _RULES:
+                hit = [p for p in patterns if p in text]
+                if hit:
+                    failure_type = ftype
+                    signals.extend(hit)
+                    break
+
+        change_diff = change_diff or ChangeDiffObject()
+        likely_change = self._change_corroborates(failure_type, change_diff)
+        if likely_change:
+            signals.append(f"{change_diff.change_count} change(s) since last success")
+
+        return IncidentDNA(
+            failure_type=failure_type,
+            system_layer=self._system_layer(text),
+            affected_layer=self._affected_layer(event, context),
+            root_signal=self._root_signal(event, signals),
+            confidence_signature=self._confidence(failure_type, signals, likely_change),
+            pattern_id=self._pattern_id(failure_type, event, context),
+            signals=signals,
+            likely_caused_by_change=likely_change,
+        )
+
+    # ─── helpers ───
+
+    @staticmethod
+    def _haystack(event: NormalizedFailureEvent, context: IncidentContextBundle | None) -> str:
+        parts = [event.error_message or "", event.error_trace or ""]
+        if context:
+            parts.append(context.error_text)
+        return "\n".join(parts).lower()
+
+    @staticmethod
+    def _system_layer(text: str) -> SystemLayer:
+        for layer, hints in _LAYER_HINTS:
+            if any(h in text for h in hints):
+                return layer
+        return SystemLayer.DATABRICKS
+
+    @staticmethod
+    def _affected_layer(event: NormalizedFailureEvent, context: IncidentContextBundle | None) -> str:
+        candidates = [event.task.lower(), event.notebook_path.lower()]
+        if context:
+            candidates.extend(t.lower() for t in context.referenced_tables)
+        for cand in candidates:
+            for layer in _MEDALLION:
+                if layer in cand:
+                    return layer
+        return "unknown"
+
+    @staticmethod
+    def _root_signal(event: NormalizedFailureEvent, signals: list[str]) -> str:
+        if signals:
+            return signals[0]
+        return event.short_error or "no error text captured"
+
+    def _change_corroborates(self, ftype: FailureType, diff: ChangeDiffObject) -> bool:
+        if not diff.has_changes:
+            return False
+        # A change of the matching category strongly corroborates the type.
+        if ftype == FailureType.SCHEMA_DRIFT and diff.schema_changes:
+            return True
+        if ftype == FailureType.CONFIG and diff.config_changes:
+            return True
+        if ftype == FailureType.CODE_ERROR and (diff.code_changes or diff.deployment_changes):
+            return True
+        if ftype in (FailureType.RESOURCE, FailureType.TIMEOUT) and diff.runtime_changes:
+            return True
+        # Any recent change is at least weak corroboration for an otherwise
+        # unexplained failure.
+        return ftype == FailureType.UNKNOWN
+
+    @staticmethod
+    def _confidence(ftype: FailureType, signals: list[str], likely_change: bool) -> str:
+        if ftype == FailureType.UNKNOWN:
+            return "low"
+        score = len([s for s in signals if "change(s)" not in s])
+        if score >= 2 or (score >= 1 and likely_change):
+            return "high"
+        if score >= 1:
+            return "medium"
+        return "low"
+
+    @staticmethod
+    def _pattern_id(ftype: FailureType, event: NormalizedFailureEvent, context: IncidentContextBundle | None) -> str:
+        layer = IncidentDNABuilder._affected_layer(event, context)
+        base = ftype.value.lower()
+        return f"{base}__{layer}_v1" if layer != "unknown" else f"{base}_v1"

iic/impact/__init__.py

@@ -0,0 +1,7 @@
+"""Stage 7 — deterministic impact scoring (NO LLM)."""
+
+from __future__ import annotations
+
+from iic.impact.impact_engine import ImpactEngine
+
+__all__ = ["ImpactEngine"]

iic/impact/impact_engine.py

@@ -0,0 +1,102 @@
+"""Stage 7 — the deterministic impact engine.
+
+PURE DETERMINISTIC. No LLM, no I/O. Severity and business risk are a transparent
+function of the blast radius and recurrence, so the same situation always yields
+the same score and every point is explained in ``breakdown``.
+
+Scoring formula (weights chosen so dashboard impact dominates — a broken exec
+dashboard is worse than a blocked internal job):
+
+    score = downstream_jobs * 2
+          + affected_tables * 1.5
+          + dashboard_impact * 3
+          + recurrence_score * 2
+
+A critical-layer failure (gold / mart — the business-facing layer) bumps severity
+by one band, since the same blast radius matters more there.
+"""
+
+from __future__ import annotations
+
+from iic.dependency.dependency_analyzer import BlastRadius
+from iic.models.dna import IncidentDNA
+from iic.models.impact import BusinessRisk, ImpactScore, Severity
+
+W_DOWNSTREAM_JOBS = 2.0
+W_AFFECTED_TABLES = 1.5
+W_DASHBOARDS = 3.0
+W_RECURRENCE = 2.0
+
+# raw_score → severity band thresholds (inclusive lower bound).
+_SEVERITY_BANDS = [
+    (20.0, Severity.CRITICAL),
+    (10.0, Severity.HIGH),
+    (4.0, Severity.MEDIUM),
+    (0.0, Severity.LOW),
+]
+
+_BUSINESS_LAYERS = ("gold", "mart", "curated")
+
+
+class ImpactEngine:
+    """Computes an :class:`ImpactScore` from blast radius + recurrence + DNA."""
+
+    def score(self, blast: BlastRadius, dna: IncidentDNA, recurrence: int = 0) -> ImpactScore:
+        downstream_jobs = blast.downstream_jobs
+        affected_tables = blast.affected_tables
+        dashboards = blast.dashboard_impact
+        recurrence = max(0, int(recurrence))
+
+        terms = {
+            "downstream_jobs": downstream_jobs * W_DOWNSTREAM_JOBS,
+            "affected_tables": affected_tables * W_AFFECTED_TABLES,
+            "dashboards": dashboards * W_DASHBOARDS,
+            "recurrence": recurrence * W_RECURRENCE,
+        }
+        raw = sum(terms.values())
+
+        severity = self._band(raw)
+        # Business-facing layer failures are escalated one band.
+        if dna.affected_layer in _BUSINESS_LAYERS:
+            severity = self._bump(severity)
+
+        return ImpactScore(
+            raw_score=raw,
+            blast_radius=blast.total,
+            downstream_jobs=downstream_jobs,
+            affected_tables=affected_tables,
+            dashboard_impact=dashboards,
+            recurrence_score=recurrence,
+            severity=severity,
+            business_risk=self._business_risk(severity, dna),
+            breakdown={k: round(v, 2) for k, v in terms.items()},
+        )
+
+    @staticmethod
+    def _band(raw: float) -> Severity:
+        for threshold, sev in _SEVERITY_BANDS:
+            if raw >= threshold:
+                return sev
+        return Severity.LOW
+
+    @staticmethod
+    def _bump(sev: Severity) -> Severity:
+        order = [Severity.LOW, Severity.MEDIUM, Severity.HIGH, Severity.CRITICAL]
+        idx = min(order.index(sev) + 1, len(order) - 1)
+        return order[idx]
+
+    @staticmethod
+    def _business_risk(sev: Severity, dna: IncidentDNA) -> BusinessRisk:
+        mapping = {
+            Severity.LOW: BusinessRisk.LOW,
+            Severity.MEDIUM: BusinessRisk.MODERATE,
+            Severity.HIGH: BusinessRisk.HIGH,
+            Severity.CRITICAL: BusinessRisk.CRITICAL,
+        }
+        risk = mapping[sev]
+        # A business-layer data-quality / schema problem is a reporting-integrity
+        # risk even at moderate blast radius — never below HIGH there.
+        if dna.affected_layer in _BUSINESS_LAYERS and dna.failure_type.value in ("SCHEMA_DRIFT", "DATA_QUALITY"):
+            if risk in (BusinessRisk.LOW, BusinessRisk.MODERATE):
+                risk = BusinessRisk.HIGH
+        return risk

iic/ingestion/__init__.py

@@ -0,0 +1,14 @@
+"""Stage 1+2 — event ingestion.
+
+Each source implements :class:`FailureSource` and emits
+:class:`~iic.models.event.NormalizedFailureEvent` objects. The core never sees a
+source-specific payload, which is what keeps everything downstream uniform.
+"""
+
+from __future__ import annotations
+
+from iic.ingestion.base import FailureSource
+from iic.ingestion.databricks_source import DatabricksFailureSource
+from iic.ingestion.webhook_source import normalize_log_webhook
+
+__all__ = ["FailureSource", "DatabricksFailureSource", "normalize_log_webhook"]

iic/ingestion/base.py

@@ -0,0 +1,21 @@
+"""The ingestion source contract.
+
+Adding a new failure source (e.g. a different orchestrator) means implementing
+``discover`` to return :class:`NormalizedFailureEvent` objects — nothing else in
+the system changes.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from iic.models.event import NormalizedFailureEvent
+
+
+class FailureSource(ABC):
+    """A source that can surface pipeline failures as normalized events."""
+
+    @abstractmethod
+    def discover(self) -> list[NormalizedFailureEvent]:
+        """Return the failed tasks this source currently knows about."""
+        raise NotImplementedError

iic/ingestion/databricks_source.py

@@ -0,0 +1,98 @@
+"""Databricks ingestion — turn failed Job tasks into NormalizedFailureEvents.
+
+Prefers the exact parent run handed in by the trigger task; otherwise discovers
+the most recent failed run of the protected job. The healing-trigger task itself
+is always excluded so the system never reports on itself.
+"""
+
+from __future__ import annotations
+
+from iic.ingestion.base import FailureSource
+from iic.models.event import EventSource, NormalizedFailureEvent
+
+TRIGGER_TASK_KEY = "trigger_incident_intelligence"
+_LEGACY_TRIGGER_KEY = "trigger_self_healing"
+
+
+class DatabricksFailureSource(FailureSource):
+    """Discovers failed tasks via the Databricks Jobs/Runs API."""
+
+    def __init__(self, client, job_id: str, parent_run_id: str = "", lookback: int = 5):
+        self.client = client
+        self.job_id = job_id
+        self.parent_run_id = (parent_run_id or "").strip()
+        self.lookback = lookback
+        # Populated as a side effect of discover(), consumed by later stages.
+        self.run_info: dict = {}
+
+    def _failed_tasks(self, run: dict) -> list[dict]:
+        out = []
+        for t in run.get("tasks", []):
+            if t.get("state", {}).get("result_state") != "FAILED":
+                continue
+            if t.get("task_key") in (TRIGGER_TASK_KEY, _LEGACY_TRIGGER_KEY):
+                continue
+            out.append(t)
+        return out
+
+    def _locate_run(self) -> tuple[list[dict], dict]:
+        if self.parent_run_id:
+            try:
+                run = self.client._get("/api/2.1/jobs/runs/get", {"run_id": int(self.parent_run_id)})
+                ft = self._failed_tasks(run)
+                if ft:
+                    return ft, run
+            except Exception:
+                pass
+        if not self.job_id:
+            return [], {}
+        try:
+            for run in self.client.list_runs(int(self.job_id), limit=self.lookback):
+                ft = self._failed_tasks(run)
+                if ft:
+                    return ft, run
+        except Exception:
+            pass
+        return [], {}
+
+    def discover(self) -> list[NormalizedFailureEvent]:
+        failed, run = self._locate_run()
+        self.run_info = run or {}
+        run_id = str(run.get("run_id", "")) if run else ""
+        pipeline = run.get("run_name", "") if run else ""
+        events: list[NormalizedFailureEvent] = []
+        for task in failed:
+            task_run_id = str(task.get("run_id", ""))
+            error_message, error_trace = "", ""
+            if task_run_id:
+                try:
+                    out = self.client.get_run_output(int(task_run_id))
+                    error_message = (out.get("error") or "").strip()
+                    error_trace = (out.get("error_trace") or "").strip()
+                except Exception:
+                    pass
+            cluster_id = ""
+            cluster = task.get("cluster_instance", {}) or {}
+            cluster_id = cluster.get("cluster_id", "")
+            events.append(NormalizedFailureEvent(
+                source=EventSource.DATABRICKS,
+                pipeline=pipeline or str(self.job_id),
+                task=task.get("task_key", "unknown"),
+                error_message=error_message or "(no error message returned)",
+                error_trace=error_trace,
+                timestamp=_iso_from_ms(task.get("start_time")),
+                run_id=run_id,
+                job_id=str(self.job_id),
+                cluster_id=cluster_id,
+                notebook_path=task.get("notebook_task", {}).get("notebook_path", ""),
+            ))
+        return events
+
+
+def _iso_from_ms(ms) -> str:
+    """Databricks returns epoch-millis. Render ISO-8601 without importing wall-clock."""
+    try:
+        from datetime import datetime, timezone
+        return datetime.fromtimestamp(int(ms) / 1000.0, tz=timezone.utc).isoformat()
+    except Exception:
+        return ""

iic/ingestion/static_source.py

@@ -0,0 +1,23 @@
+"""A failure source backed by already-known events.
+
+The v4 wrapper (``iic_monitor``) and the Spark listener catch a failure *as it
+happens* — they already hold the error, so there's nothing to "discover" from the
+Jobs API. They hand the engine a pre-built :class:`NormalizedFailureEvent` through
+this source instead of polling.
+"""
+
+from __future__ import annotations
+
+from iic.ingestion.base import FailureSource
+from iic.models.event import NormalizedFailureEvent
+
+
+class StaticFailureSource(FailureSource):
+    """Yields a fixed list of events; ``run_info`` carries the job DAG if known."""
+
+    def __init__(self, events: list[NormalizedFailureEvent], run_info: dict | None = None):
+        self._events = events
+        self.run_info = run_info or {}
+
+    def discover(self) -> list[NormalizedFailureEvent]:
+        return list(self._events)