shkit 1.2.0__py3-none-any.whl

iic/ingestion/webhook_source.py

@@ -0,0 +1,39 @@
+"""Logs-webhook ingestion — normalize an arbitrary JSON failure payload.
+
+A pure function (no I/O) so it is trivially unit-testable and reusable from a
+Lakehouse webhook, a custom poller, or a test. Unknown shapes degrade gracefully
+to a MANUAL event rather than raising.
+"""
+
+from __future__ import annotations
+
+from iic.models.event import EventSource, NormalizedFailureEvent
+
+
+def normalize_log_webhook(payload: dict) -> NormalizedFailureEvent:
+    """Map a loosely-structured webhook body onto a NormalizedFailureEvent.
+
+    Accepts common aliases (``pipeline``/``job``/``job_name``, ``task``/``task_key``,
+    ``error``/``message``/``error_message``) so callers needn't match exactly.
+    """
+    p = payload or {}
+
+    def pick(*keys, default=""):
+        for k in keys:
+            v = p.get(k)
+            if v:
+                return str(v)
+        return default
+
+    return NormalizedFailureEvent(
+        source=EventSource.LOG_WEBHOOK,
+        pipeline=pick("pipeline", "job", "job_name", default="unknown"),
+        task=pick("task", "task_key", "step", default="unknown"),
+        error_message=pick("error_message", "error", "message", default="(no error message)"),
+        error_trace=pick("error_trace", "stack_trace", "trace"),
+        timestamp=pick("timestamp", "time", "ts"),
+        run_id=pick("run_id", "run"),
+        job_id=pick("job_id"),
+        cluster_id=pick("cluster_id", "cluster"),
+        notebook_path=pick("notebook_path", "notebook", "path"),
+    )

iic/models/__init__.py

@@ -0,0 +1,44 @@
+"""Core IIC data models — the structured objects that flow through the pipeline.
+
+The whole system is organised around a small set of immutable-ish dataclasses, in
+the order they are produced by the 11-stage pipeline:
+
+    NormalizedFailureEvent   (stage 2)  raw failure, source-agnostic
+    IncidentContextBundle    (stage 3)  everything we gathered about the failure
+    ImpactGraph              (stage 4)  upstream/downstream blast radius
+    ChangeDiffObject         (stage 5)  what changed since the last success
+    IncidentDNA              (stage 6)  structured failure fingerprint (the heart)
+    ImpactScore              (stage 7)  deterministic severity / business risk
+    RoutingDecision          (stage 8)  which model (if any) to use
+    DiagnosisResult          (stage 9)  LLM root-cause + reasoning + evidence
+    IncidentReport           (stage 10) the final, serialisable product
+"""
+
+from __future__ import annotations
+
+from iic.models.change import ChangeDiffObject, FieldChange
+from iic.models.context import IncidentContextBundle
+from iic.models.diagnosis import DiagnosisResult
+from iic.models.dna import FailureType, IncidentDNA, SystemLayer
+from iic.models.event import EventSource, NormalizedFailureEvent
+from iic.models.impact import BusinessRisk, ImpactScore, Severity
+from iic.models.report import IncidentReport
+from iic.models.routing import ModelTier, RoutingDecision
+
+__all__ = [
+    "NormalizedFailureEvent",
+    "EventSource",
+    "IncidentContextBundle",
+    "ChangeDiffObject",
+    "FieldChange",
+    "IncidentDNA",
+    "FailureType",
+    "SystemLayer",
+    "ImpactScore",
+    "Severity",
+    "BusinessRisk",
+    "RoutingDecision",
+    "ModelTier",
+    "DiagnosisResult",
+    "IncidentReport",
+]

iic/models/change.py

@@ -0,0 +1,77 @@
+"""Stage 5 — the change-diff object.
+
+Captures what changed between the last successful execution and the failed one.
+This is a strong, often-decisive signal for root cause: most production failures
+correlate with a recent change (a new column, a config tweak, a runtime bump).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class FieldChange:
+    """A single before/after change with a category."""
+
+    category: str  # schema | config | code | runtime | deployment
+    field: str
+    before: str = ""
+    after: str = ""
+
+    def describe(self) -> str:
+        if self.before and self.after:
+            return f"{self.field}: {self.before} → {self.after}"
+        if self.after and not self.before:
+            return f"{self.field}: added ({self.after})"
+        if self.before and not self.after:
+            return f"{self.field}: removed (was {self.before})"
+        return self.field
+
+
+@dataclass
+class ChangeDiffObject:
+    """Diff of the failed run vs. the last successful run."""
+
+    has_prior_success: bool = False
+    last_success_run_id: str = ""
+    schema_changes: list[FieldChange] = field(default_factory=list)
+    config_changes: list[FieldChange] = field(default_factory=list)
+    code_changes: list[FieldChange] = field(default_factory=list)
+    runtime_changes: list[FieldChange] = field(default_factory=list)
+    deployment_changes: list[FieldChange] = field(default_factory=list)
+
+    @property
+    def all_changes(self) -> list[FieldChange]:
+        return (
+            self.schema_changes
+            + self.config_changes
+            + self.code_changes
+            + self.runtime_changes
+            + self.deployment_changes
+        )
+
+    @property
+    def has_changes(self) -> bool:
+        return bool(self.all_changes)
+
+    @property
+    def change_count(self) -> int:
+        return len(self.all_changes)
+
+    def summaries(self) -> list[str]:
+        return [c.describe() for c in self.all_changes]
+
+    def to_dict(self) -> dict:
+        def dump(changes: list[FieldChange]) -> list[dict]:
+            return [{"field": c.field, "before": c.before, "after": c.after} for c in changes]
+
+        return {
+            "has_prior_success": self.has_prior_success,
+            "last_success_run_id": self.last_success_run_id,
+            "schema_changes": dump(self.schema_changes),
+            "config_changes": dump(self.config_changes),
+            "code_changes": dump(self.code_changes),
+            "runtime_changes": dump(self.runtime_changes),
+            "deployment_changes": dump(self.deployment_changes),
+        }

iic/models/context.py

@@ -0,0 +1,46 @@
+"""Stage 3 — the gathered context bundle.
+
+Everything the context builder could collect about a failure, in one place. All
+fields are optional: the builder fills in what it can reach and leaves the rest
+empty rather than failing, so a partial context still flows through the pipeline.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class IncidentContextBundle:
+    """All evidence gathered about a single failure."""
+
+    event_id: str
+    logs: list[str] = field(default_factory=list)
+    notebook_source: str = ""
+    cluster_state: dict = field(default_factory=dict)
+    job_metadata: dict = field(default_factory=dict)
+    # Task keys, resolved from the job DAG.
+    upstream_tasks: list[str] = field(default_factory=list)
+    downstream_tasks: list[str] = field(default_factory=list)
+    # Best-effort schema of the table the failing task writes to, if known.
+    schema_snapshot: dict = field(default_factory=dict)
+    # Tables referenced by the failing task (parsed from notebook / metadata).
+    referenced_tables: list[str] = field(default_factory=list)
+
+    @property
+    def error_text(self) -> str:
+        """Joined log text, used by the DNA builder and the LLM prompt."""
+        return "\n".join(self.logs)
+
+    def to_dict(self) -> dict:
+        return {
+            "event_id": self.event_id,
+            "logs": self.logs,
+            "notebook_source_len": len(self.notebook_source),
+            "cluster_state": self.cluster_state,
+            "job_metadata": self.job_metadata,
+            "upstream_tasks": self.upstream_tasks,
+            "downstream_tasks": self.downstream_tasks,
+            "schema_snapshot": self.schema_snapshot,
+            "referenced_tables": self.referenced_tables,
+        }

iic/models/diagnosis.py

@@ -0,0 +1,37 @@
+"""Stage 9 — the diagnosis result.
+
+The *only* place an LLM result enters the system, and even here it is structured
+(never free-form chat). When the router decides no LLM is needed, the diagnosis
+engine still returns one of these, populated deterministically from the DNA.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class DiagnosisResult:
+    """Root-cause analysis, whether produced by rules or the LLM."""
+
+    root_cause: str
+    confidence: float = 0.0            # 0.0–1.0
+    reasoning: str = ""
+    suggested_fix: str = ""
+    evidence: list[str] = field(default_factory=list)
+    alternatives: list[str] = field(default_factory=list)
+    produced_by: str = "rules"         # "rules" | "llm:<model>"
+
+    def __post_init__(self) -> None:
+        self.confidence = max(0.0, min(1.0, float(self.confidence or 0.0)))
+
+    def to_dict(self) -> dict:
+        return {
+            "root_cause": self.root_cause,
+            "confidence": round(self.confidence, 3),
+            "reasoning": self.reasoning,
+            "suggested_fix": self.suggested_fix,
+            "evidence": self.evidence,
+            "alternatives": self.alternatives,
+            "produced_by": self.produced_by,
+        }

iic/models/dna.py

@@ -0,0 +1,77 @@
+"""Stage 6 — IncidentDNA, the core intelligence asset.
+
+This is the structured fingerprint of a failure. Everything before it exists to
+build it; everything after it consumes it. It is produced *deterministically*
+(rules over context + changes) so the system's behaviour is predictable and the
+LLM — which only runs later — operates on structure rather than raw text.
+
+``pattern_id`` is a stable, human-readable key (e.g. ``schema_drift_v1``) that lets
+recurring failures be counted, cached, and trended over time.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+
+
+class FailureType(str, Enum):
+    """Canonical failure taxonomy. The DNA builder maps raw errors onto these."""
+
+    SCHEMA_DRIFT = "SCHEMA_DRIFT"          # column added/removed/retyped
+    DATA_QUALITY = "DATA_QUALITY"          # nulls, duplicates, constraint/threshold
+    MISSING_DATA = "MISSING_DATA"          # empty/absent source table or path
+    PERMISSION = "PERMISSION"              # access denied, unauthorized
+    DEPENDENCY = "DEPENDENCY"              # upstream task/table failed first
+    RESOURCE = "RESOURCE"                  # OOM, executor lost, disk, capacity
+    TIMEOUT = "TIMEOUT"                    # exceeded wall-clock / query timeout
+    CONFIG = "CONFIG"                      # bad/missing parameter, secret, path
+    CODE_ERROR = "CODE_ERROR"              # syntax / type / attribute / import
+    UNKNOWN = "UNKNOWN"                    # could not be classified
+
+
+class SystemLayer(str, Enum):
+    """Which infrastructure layer the failure sits in."""
+
+    DATABRICKS = "databricks"
+    SPARK = "spark"
+    STORAGE = "storage"
+    NETWORK = "network"
+    UNKNOWN = "unknown"
+
+
+# Medallion / pipeline layer inferred from the asset name.
+_DATA_LAYERS = ("bronze", "silver", "gold", "raw", "staging", "mart", "unknown")
+
+
+@dataclass
+class IncidentDNA:
+    """Structured failure representation — deterministic, pre-LLM."""
+
+    failure_type: FailureType = FailureType.UNKNOWN
+    system_layer: SystemLayer = SystemLayer.UNKNOWN
+    affected_layer: str = "unknown"          # bronze | silver | gold | ...
+    root_signal: str = ""                    # the single most telling phrase
+    confidence_signature: str = "low"        # low | medium | high (rule confidence)
+    pattern_id: str = "unknown_v1"           # stable recurrence key
+    # Evidence the rules fired on — keeps the classification fully traceable.
+    signals: list[str] = field(default_factory=list)
+    likely_caused_by_change: bool = False
+
+    def __post_init__(self) -> None:
+        if isinstance(self.failure_type, str):
+            self.failure_type = FailureType(self.failure_type)
+        if isinstance(self.system_layer, str):
+            self.system_layer = SystemLayer(self.system_layer)
+
+    def to_dict(self) -> dict:
+        return {
+            "failure_type": self.failure_type.value,
+            "system_layer": self.system_layer.value,
+            "affected_layer": self.affected_layer,
+            "root_signal": self.root_signal,
+            "confidence_signature": self.confidence_signature,
+            "pattern_id": self.pattern_id,
+            "signals": self.signals,
+            "likely_caused_by_change": self.likely_caused_by_change,
+        }

iic/models/event.py

@@ -0,0 +1,78 @@
+"""Stage 2 — the normalized event model.
+
+Every ingestion source (Databricks Jobs API, a logs webhook, …) is responsible
+for emitting this one shape. Nothing downstream of ingestion ever sees a
+source-specific payload, which is what keeps the core source-agnostic.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import uuid
+from dataclasses import dataclass
+from enum import Enum
+
+
+class EventSource(str, Enum):
+    """Where a failure event originated. Extend here when adding an ingester."""
+
+    DATABRICKS = "databricks"
+    LOG_WEBHOOK = "log_webhook"
+    MANUAL = "manual"
+
+
+@dataclass
+class NormalizedFailureEvent:
+    """A single, source-agnostic pipeline failure.
+
+    ``event_id`` is deterministic by default (derived from source+run+task) so the
+    same failure normalizes to the same id and can be de-duplicated, but a caller
+    may pass an explicit id. Timestamps are ISO-8601 strings to stay
+    serialisation-friendly across the Spark/JSON boundary.
+    """
+
+    source: EventSource
+    pipeline: str
+    task: str
+    error_message: str
+    timestamp: str
+    run_id: str = ""
+    job_id: str = ""
+    cluster_id: str = ""
+    notebook_path: str = ""
+    error_trace: str = ""
+    event_id: str = ""
+
+    def __post_init__(self) -> None:
+        if isinstance(self.source, str):
+            self.source = EventSource(self.source)
+        if not self.event_id:
+            self.event_id = self._derive_id()
+
+    def _derive_id(self) -> str:
+        basis = f"{self.source.value}|{self.run_id}|{self.task}"
+        if self.run_id and self.task:
+            digest = hashlib.sha256(basis.encode()).hexdigest()[:12]
+            return f"EVT-{digest}"
+        return f"EVT-{uuid.uuid4().hex[:12]}"
+
+    @property
+    def short_error(self) -> str:
+        """First line of the error, trimmed — handy for fingerprinting."""
+        first = (self.error_message or "").strip().splitlines()
+        return (first[0] if first else "")[:200]
+
+    def to_dict(self) -> dict:
+        return {
+            "event_id": self.event_id,
+            "source": self.source.value,
+            "pipeline": self.pipeline,
+            "task": self.task,
+            "error_message": self.error_message,
+            "error_trace": self.error_trace,
+            "timestamp": self.timestamp,
+            "run_id": self.run_id,
+            "job_id": self.job_id,
+            "cluster_id": self.cluster_id,
+            "notebook_path": self.notebook_path,
+        }

iic/models/impact.py

@@ -0,0 +1,60 @@
+"""Stage 7 — the deterministic impact score.
+
+Produced with NO LLM. The score is a pure function of the blast radius and
+recurrence so the same situation always yields the same severity — auditable and
+cheap. Severity/business-risk thresholds live with the engine, not here.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+
+
+class Severity(str, Enum):
+    LOW = "LOW"
+    MEDIUM = "MEDIUM"
+    HIGH = "HIGH"
+    CRITICAL = "CRITICAL"
+
+
+class BusinessRisk(str, Enum):
+    LOW = "LOW"
+    MODERATE = "MODERATE"
+    HIGH = "HIGH"
+    CRITICAL = "CRITICAL"
+
+
+@dataclass
+class ImpactScore:
+    """Deterministic blast-radius and business-risk assessment."""
+
+    raw_score: float = 0.0
+    blast_radius: int = 0
+    downstream_jobs: int = 0
+    affected_tables: int = 0
+    dashboard_impact: int = 0
+    recurrence_score: int = 0
+    severity: Severity = Severity.LOW
+    business_risk: BusinessRisk = BusinessRisk.LOW
+    # The exact term contributions, for a fully transparent score.
+    breakdown: dict = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if isinstance(self.severity, str):
+            self.severity = Severity(self.severity)
+        if isinstance(self.business_risk, str):
+            self.business_risk = BusinessRisk(self.business_risk)
+
+    def to_dict(self) -> dict:
+        return {
+            "raw_score": round(self.raw_score, 2),
+            "blast_radius": self.blast_radius,
+            "downstream_jobs": self.downstream_jobs,
+            "affected_tables": self.affected_tables,
+            "dashboard_impact": self.dashboard_impact,
+            "recurrence_score": self.recurrence_score,
+            "severity": self.severity.value,
+            "business_risk": self.business_risk.value,
+            "breakdown": self.breakdown,
+        }

iic/models/report.py

@@ -0,0 +1,88 @@
+"""Stage 10 — the final IncidentReport.
+
+The product the whole pipeline exists to produce: a single object that is both
+machine-consumable (``to_dict``) and human-readable (``to_markdown``). It carries
+the full provenance chain (DNA, impact, diagnosis, evidence, timeline) so any
+conclusion can be traced back to the evidence it rests on.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Optional
+
+from iic.models.diagnosis import DiagnosisResult
+from iic.models.dna import IncidentDNA
+from iic.models.impact import ImpactScore
+from iic.models.routing import RoutingDecision
+
+
+@dataclass
+class IncidentReport:
+    incident_id: str
+    pipeline: str
+    task: str
+    timestamp: str
+    summary: str
+    dna: IncidentDNA
+    impact: ImpactScore
+    diagnosis: DiagnosisResult
+    routing: Optional[RoutingDecision] = None
+    evidence: list[str] = field(default_factory=list)
+    timeline: list[str] = field(default_factory=list)
+    changes: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return {
+            "incident_id": self.incident_id,
+            "pipeline": self.pipeline,
+            "task": self.task,
+            "timestamp": self.timestamp,
+            "summary": self.summary,
+            "root_cause": self.diagnosis.root_cause,
+            "confidence": round(self.diagnosis.confidence, 3),
+            "failure_type": self.dna.failure_type.value,
+            "pattern_id": self.dna.pattern_id,
+            "impact": self.impact.to_dict(),
+            "diagnosis": self.diagnosis.to_dict(),
+            "routing": self.routing.to_dict() if self.routing else None,
+            "evidence": self.evidence,
+            "changes": self.changes,
+            "timeline": self.timeline,
+            "suggested_fix": self.diagnosis.suggested_fix,
+        }
+
+    def to_markdown(self) -> str:
+        """Compact human-readable summary (used by the notifier and logs)."""
+        imp = self.impact
+        lines = [
+            f"# Incident {self.incident_id} — {imp.severity.value}",
+            f"**Pipeline:** {self.pipeline} · **Task:** {self.task} · **{self.timestamp}**",
+            "",
+            f"**Summary:** {self.summary}",
+            "",
+            f"**Root cause** ({int(self.diagnosis.confidence * 100)}% confidence): "
+            f"{self.diagnosis.root_cause}",
+            "",
+            "## Impact",
+            f"- Severity: **{imp.severity.value}** · Business risk: **{imp.business_risk.value}**",
+            f"- Blast radius: {imp.blast_radius} asset(s) · "
+            f"{imp.downstream_jobs} downstream job(s) · {imp.dashboard_impact} dashboard(s)",
+            "",
+            "## Failure DNA",
+            f"- Type: `{self.dna.failure_type.value}` · Layer: `{self.dna.affected_layer}` "
+            f"· Pattern: `{self.dna.pattern_id}`",
+            f"- Root signal: {self.dna.root_signal or 'n/a'}",
+        ]
+        if self.changes:
+            lines += ["", "## Recent changes (since last success)"]
+            lines += [f"- {c}" for c in self.changes]
+        if self.diagnosis.suggested_fix:
+            lines += ["", "## Suggested fix", self.diagnosis.suggested_fix]
+        if self.evidence:
+            lines += ["", "## Evidence"]
+            lines += [f"- {e}" for e in self.evidence]
+        if self.timeline:
+            lines += ["", "## Timeline"]
+            lines += [f"- {t}" for t in self.timeline]
+        return "\n".join(lines)

iic/models/routing.py

@@ -0,0 +1,41 @@
+"""Stage 8 — the routing decision.
+
+Records which model (if any) the diagnosis stage should use and *why*, so the
+choice is auditable. The router is deterministic: impact + cache state decide the
+tier, never the LLM itself.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+class ModelTier(str, Enum):
+    NONE = "none"              # skip the LLM entirely (cache hit / trivial)
+    LIGHTWEIGHT = "lightweight"
+    POWERFUL = "powerful"
+
+
+@dataclass
+class RoutingDecision:
+    tier: ModelTier
+    model: str = ""            # resolved endpoint name ("" when tier is NONE)
+    reason: str = ""
+    cache_hit: bool = False
+
+    def __post_init__(self) -> None:
+        if isinstance(self.tier, str):
+            self.tier = ModelTier(self.tier)
+
+    @property
+    def requires_llm(self) -> bool:
+        return self.tier != ModelTier.NONE
+
+    def to_dict(self) -> dict:
+        return {
+            "tier": self.tier.value,
+            "model": self.model,
+            "reason": self.reason,
+            "cache_hit": self.cache_hit,
+        }

iic/notify/__init__.py

@@ -0,0 +1,7 @@
+"""Stage 11 — notification (optional)."""
+
+from __future__ import annotations
+
+from iic.notify.teams_notifier import TeamsNotifier, build_incident_card
+
+__all__ = ["TeamsNotifier", "build_incident_card"]