shkit 1.2.0__py3-none-any.whl

iic/runtime/constants.py

@@ -0,0 +1,40 @@
+"""Shared constants so the runtime, the publish workflow, and the docs agree.
+
+The publish workflow uploads ``config.yaml`` next to the wheel in the tenant
+Volume; the runtime reads it from the same place (override with ``IIC_CONFIG_PATH``).
+"""
+
+from __future__ import annotations
+
+# Default Volume location of the tenant config (override via IIC_CONFIG_PATH).
+DEFAULT_CONFIG_PATH = "/Volumes/dev_catalog/default/libs/config.yaml"
+
+# A tenant can be muted without republishing by dropping this file next to config.
+DISABLED_SENTINEL_NAME = "DISABLED"
+
+# Antibody Ledger (per-tenant issue memory) — lives next to config.yaml in the
+# Volume. The runtime READS this file; it never writes it (humans edit it in git,
+# workflows sync it). New occurrences are appended as tiny per-process marker
+# files under PENDING_DIRNAME, mirroring the .iic_seen/ dedup mechanism.
+ANTIBODIES_FILENAME = "antibodies.yaml"
+PENDING_DIRNAME = ".iic_pending"
+
+# Handler network budgets (seconds) — keep total handler ≤ ~10s.
+TEAMS_TIMEOUT = 5
+GITHUB_DISPATCH_TIMEOUT = 3
+ENRICH_TIMEOUT = 2
+# Per-call budget for optional in-process enrichment (pat-gated); total added ≤ ~4s.
+ENRICH_BUDGET = 2
+
+# ── Secret-scope configuration (the product-model primary config source) ──
+# A customer creates ONE Databricks secret scope with this conventional name and
+# the fixed keys below. Override the scope name via the IIC_SECRET_SCOPE env var.
+DEFAULT_SECRET_SCOPE = "iic"
+
+SECRET_KEY_TEAMS_WEBHOOK = "teams_webhook"   # required
+SECRET_KEY_VOLUME_PATH = "volume_path"       # required — anchors ledger/markers/sentinel
+SECRET_KEY_HOST = "host"                     # optional — "View Run" links
+SECRET_KEY_PAT = "pat"                       # optional — enables best-effort enrichment
+SECRET_KEY_GITHUB_REPO = "github_repo"       # optional — incident archiving
+SECRET_KEY_GITHUB_TOKEN = "github_dispatch_token"  # optional
+SECRET_KEY_DEDUP_TTL = "dedup_ttl_seconds"   # optional — default 300

iic/runtime/context.py

@@ -0,0 +1,46 @@
+"""Session-scoped context cache (initialized by auto-bootstrap).
+
+On a long-lived cluster, the same notebook source / job metadata is fetched
+repeatedly across failures. ``init_context_cache()`` turns on an in-process cache
+so those lookups are memoized for the cluster session.
+
+Crucially the cache is **inactive until initialized**, so off-cluster (tests,
+local) every fetch stays fresh — no cross-test state leaks.
+"""
+
+from __future__ import annotations
+
+_active = False
+_cache: dict = {}
+
+
+def init_context_cache() -> None:
+    """Activate (and clear) the process-wide context cache. Idempotent."""
+    global _active
+    _cache.clear()
+    _active = True
+
+
+def is_active() -> bool:
+    return _active
+
+
+def reset_context_cache() -> None:
+    """Deactivate + clear — used by tests to guarantee isolation."""
+    global _active
+    _active = False
+    _cache.clear()
+
+
+def cached(key: str, loader):
+    """Return ``_cache[key]`` if the cache is active, else just call ``loader()``.
+
+    ``loader`` is a zero-arg callable that produces the value on a miss.
+    """
+    if not _active:
+        return loader()
+    if key in _cache:
+        return _cache[key]
+    value = loader()
+    _cache[key] = value
+    return value

iic/runtime/detective.py

@@ -0,0 +1,72 @@
+"""Payload-only incident archiver — runs in GitHub Actions on a repository_dispatch
+after a failure and renders a GitHub issue PURELY from the self-contained
+``client_payload`` the agent sent.
+
+In the product model GitHub never reaches into a customer's Databricks workspace:
+there are no Databricks API calls and no customer credentials here. Any post-mortem
+enrichment now happens in-process (pat-gated) before the agent dispatches, so the
+payload is already self-sufficient. Best-effort; the workflow must not fail on a
+rendering error.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+
+def _severity(p: dict) -> str:
+    return (p.get("impact", {}) or {}).get("severity") or p.get("severity") or "LOW"
+
+
+def _issue_body(p: dict) -> str:
+    ab = p.get("antibody") or {}
+    lines = [
+        f"**Incident:** `{p.get('incident_id', '?')}`  ·  **Severity:** {_severity(p)}",
+        f"**Failure type:** {p.get('failure_type', '?')}  ·  **Pattern:** `{p.get('pattern_id', '?')}`",
+        "",
+        f"**Root cause:** {p.get('root_cause', 'n/a')}",
+        f"**Suggested fix:** {p.get('suggested_fix', 'n/a')}",
+    ]
+    if ab.get("state"):
+        seen = f" (seen {ab['times_seen']}×)" if ab.get("times_seen") else ""
+        fix = f" — recorded fix: {ab['resolution']}" if ab.get("resolution") else ""
+        lines += ["", f"**Antibody:** {ab.get('state')}{seen}{fix}"]
+    changes = p.get("changes") or []
+    if changes:
+        lines += ["", "## Recent changes (since last success)"]
+        lines += [f"- {c}" for c in changes[:8]]
+    lines += ["", "## Context", "```json", json.dumps(p.get("context", {}), indent=2)[:2000], "```"]
+    return "\n".join(lines)
+
+
+def _create_issue(gh_repo: str, gh_token: str, payload: dict) -> str:
+    import requests
+    title = f"[IIC] {_severity(payload)} — {payload.get('failure_type', 'incident')}"
+    r = requests.post(
+        f"https://api.github.com/repos/{gh_repo}/issues",
+        headers={"Authorization": f"Bearer {gh_token}", "Accept": "application/vnd.github+json"},
+        json={"title": title, "body": _issue_body(payload), "labels": ["iic-incident"]},
+        timeout=15,
+    )
+    return r.json().get("html_url", "") if r.status_code in (200, 201) else ""
+
+
+def investigate(payload: dict, *, gh_repo: str = "", gh_token: str = "") -> str:
+    """Render + file the GitHub issue from the payload alone. No Databricks access."""
+    url = _create_issue(gh_repo, gh_token, payload) if (gh_repo and gh_token) else ""
+    print(f"[detective] issue: {url or '(skipped)'}")
+    return url
+
+
+def main() -> int:  # pragma: no cover - workflow entry
+    payload = json.loads(os.environ.get("IIC_PAYLOAD", "{}") or "{}")
+    investigate(payload,
+                gh_repo=os.environ.get("GITHUB_REPOSITORY", ""),
+                gh_token=os.environ.get("GH_TOKEN", ""))
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

iic/runtime/hooks.py

@@ -0,0 +1,85 @@
+"""Optional Spark runtime hooks (best-effort ONLY).
+
+This is the v4 spec's "optional enhancement layer". A Py4J-based SparkListener can
+surface task-level failures from inside the driver, but it is **runtime-version
+sensitive and NOT a dependency for correctness** — the wrapper (``iic_monitor``)
+and the reconciler are the reliable paths. Everything here is wrapped so a failure
+to attach simply no-ops.
+
+Caveats (why this is best-effort):
+  * Py4J can only *implement an interface*, so we must declare the full
+    ``SparkListenerInterface`` method set; a Spark version that adds methods can
+    break registration (hence the broad try/except).
+  * Spark retries tasks, so task-level failures are noisy — the default callback
+    only *logs*; it does not auto-run the (potentially costly) IncidentEngine.
+    Opt into analysis explicitly via ``on_failure``.
+"""
+
+from __future__ import annotations
+
+# Representative SparkListenerInterface methods (Spark 3.x). Unhandled callbacks
+# must still exist as no-ops or the JVM raises when it invokes them.
+_NOOP_METHODS = [
+    "onStageCompleted", "onStageSubmitted", "onTaskStart", "onTaskGettingResult",
+    "onJobStart", "onJobEnd", "onEnvironmentUpdate", "onBlockManagerAdded",
+    "onBlockManagerRemoved", "onUnpersistRDD", "onApplicationStart", "onApplicationEnd",
+    "onExecutorMetricsUpdate", "onStageExecutorMetrics", "onExecutorAdded",
+    "onExecutorRemoved", "onExecutorExcluded", "onExecutorExcludedForStage",
+    "onNodeExcludedForStage", "onNodeExcluded", "onExecutorUnexcluded",
+    "onNodeUnexcluded", "onBlockUpdated", "onSpeculativeTaskSubmitted",
+    "onResourceProfileAdded", "onOtherEvent",
+]
+
+
+def register_spark_listener(spark=None, on_failure=None) -> bool:
+    """Attach a best-effort Spark task-failure listener. Returns True if attached.
+
+    ``on_failure(info: dict)`` is called when a task ends in failure; if omitted,
+    failures are only logged (safe default — avoids noisy/expensive triggering on
+    Spark's automatic task retries).
+    """
+    try:
+        spark = spark or _active_spark()
+        if spark is None:
+            print("[iic.hooks] no active Spark session — listener not attached")
+            return False
+
+        callback = on_failure or (lambda info: print(f"[iic.hooks] task failed: {info}"))
+
+        def _on_task_end(task_end):
+            try:
+                reason = task_end.reason()
+                # Success reason class name is "Success"; anything else is a failure.
+                if reason is not None and "Success" not in reason.toString():
+                    info = task_end.taskInfo()
+                    callback({
+                        "stage_id": task_end.stageId(),
+                        "task_id": info.taskId() if info else None,
+                        "reason": reason.toString()[:300],
+                    })
+            except Exception:
+                pass
+
+        methods = {"onTaskEnd": _on_task_end}
+        for name in _NOOP_METHODS:
+            methods[name] = (lambda *a, **k: None)
+
+        listener_cls = type("IICSparkListener", (), methods)
+        listener_cls.Java = type("Java", (), {
+            "implements": ["org.apache.spark.scheduler.SparkListenerInterface"]})
+
+        listener = listener_cls()
+        spark.sparkContext._jsc.sc().addSparkListener(listener)
+        print("[iic.hooks] Spark task-failure listener attached (best-effort)")
+        return True
+    except Exception as ex:
+        print(f"[iic.hooks] could not attach Spark listener (best-effort, ignored): {str(ex)[:160]}")
+        return False
+
+
+def _active_spark():
+    try:
+        from pyspark.sql import SparkSession
+        return SparkSession.getActiveSession()
+    except Exception:
+        return None

iic/runtime/incident_engine.py

@@ -0,0 +1,207 @@
+"""The Incident Intelligence Core orchestrator — the 11-stage pipeline.
+
+    1  ingest            DatabricksFailureSource.discover()
+    2  normalize         → NormalizedFailureEvent (done in ingestion)
+    3  context           ContextBuilder.build()
+    4  dependency        DependencyAnalyzer.analyze()  → blast radius
+    5  change            ChangeDetector.detect()
+    6  dna               IncidentDNABuilder.build()    ← deterministic heart
+    7  impact            ImpactEngine.score()          ← NO LLM
+    8  route             IncidentModelRouter.route()
+    9  diagnose          DiagnosisEngine.diagnose()    ← LLM only if routed
+    10 report            ReportGenerator.build()
+    11 notify            TeamsNotifier.send()          ← optional
+
+Deterministic-first: stages 3–8 never call an LLM. The model is only ever invoked
+at stage 9, and only when stage 8 decides it is worth the cost.
+
+The engine is constructed from injected collaborators so the whole flow is unit
+testable with fakes; :func:`run_from_notebook` wires the real Databricks ones.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from iic.change.change_detector import ChangeDetector
+from iic.context.context_builder import ContextBuilder
+from iic.dependency.dependency_analyzer import DependencyAnalyzer
+from iic.diagnosis.diagnosis_engine import DiagnosisEngine
+from iic.dna.dna_builder import IncidentDNABuilder
+from iic.impact.impact_engine import ImpactEngine
+from iic.ingestion.databricks_source import DatabricksFailureSource
+from iic.models.report import IncidentReport
+from iic.notify.teams_notifier import TeamsNotifier
+from iic.report.report_generator import ReportGenerator
+from iic.routing.router import IncidentModelRouter
+from iic.runtime.pattern_store import NullPatternStore
+
+
+@dataclass
+class EngineConfig:
+    job_id: str = ""
+    parent_run_id: str = ""
+    lightweight_model: str = "databricks-meta-llama-3-3-70b-instruct"
+    powerful_model: str = "databricks-claude-opus-4-8"
+    teams_webhook: str = ""
+    host: str = ""
+    notify: bool = True
+
+
+@dataclass
+class EngineResult:
+    reports: list[IncidentReport] = field(default_factory=list)
+    tokens_used: int = 0
+    notified: bool = False
+    summary: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "summary": self.summary,
+            "tokens_used": self.tokens_used,
+            "notified": self.notified,
+            "incidents": [r.to_dict() for r in self.reports],
+        }
+
+
+class IncidentEngine:
+    def __init__(self, config: EngineConfig, *, source, context_builder=None,
+                 dependency_analyzer=None, change_detector=None, dna_builder=None,
+                 impact_engine=None, router=None, diagnosis_engine=None,
+                 report_generator=None, notifier=None, pattern_store=None):
+        self.config = config
+        self.source = source
+        self.context_builder = context_builder or ContextBuilder()
+        self.dependency_analyzer = dependency_analyzer or DependencyAnalyzer()
+        self.change_detector = change_detector or ChangeDetector()
+        self.dna_builder = dna_builder or IncidentDNABuilder()
+        self.impact_engine = impact_engine or ImpactEngine()
+        self.router = router or IncidentModelRouter(config.lightweight_model, config.powerful_model)
+        self.diagnosis_engine = diagnosis_engine or DiagnosisEngine()
+        self.report_generator = report_generator or ReportGenerator()
+        self.notifier = notifier or TeamsNotifier(config.teams_webhook)
+        self.pattern_store = pattern_store or NullPatternStore()
+
+    def run(self) -> EngineResult:
+        # Stage 1+2 — ingest & normalize.
+        events = self.source.discover()
+        if not events:
+            return EngineResult(summary="No failures found. Nothing to analyze.")
+
+        run_info = getattr(self.source, "run_info", {}) or {}
+        run_id = events[0].run_id
+        job_tasks = (run_info.get("tasks") or [])
+        failed_keys = {e.task for e in events}
+        # Authoritative upstream map from the run DAG (always present on the
+        # Databricks path) — used for the dependency override without an extra
+        # client call.
+        upstream_map = {
+            t.get("task_key", ""): [d.get("task_key", "") for d in t.get("depends_on", [])]
+            for t in job_tasks
+        }
+
+        reports: list[IncidentReport] = []
+        total_tokens = 0
+
+        for i, event in enumerate(events, 1):
+            # Stage 3 — context.
+            context = self.context_builder.build(event)
+
+            # Stage 4 — dependency / blast radius.
+            blast = self.dependency_analyzer.analyze(
+                job_tasks, event.task, referenced_tables=context.referenced_tables)
+
+            # Stage 5 — change detection.
+            change_diff = self.change_detector.detect(
+                self.config.job_id, run_id, failed_run=run_info or None)
+
+            # Stage 6 — Incident DNA (deterministic heart). A task whose upstream
+            # also failed in this run is a derived failure (DAG ∪ context lineage).
+            upstream_keys = set(upstream_map.get(event.task, [])) | set(context.upstream_tasks)
+            upstream_failed = any(up in failed_keys for up in upstream_keys)
+            dna = self.dna_builder.build(event, context, change_diff, upstream_failed=upstream_failed)
+
+            # Stage 7 — deterministic impact (NO LLM).
+            recurrence = self.pattern_store.recurrence(dna.pattern_id)
+            impact = self.impact_engine.score(blast, dna, recurrence=recurrence)
+
+            # Stage 8 — route. A pattern store may expose is_known(): a recurring,
+            # already-understood pattern is treated as a cache hit → LLM skipped.
+            is_known = getattr(self.pattern_store, "is_known", None)
+            cache_hit = bool(is_known(dna.pattern_id)) if callable(is_known) else False
+            routing = self.router.route(dna, impact, cache_hit=cache_hit)
+
+            # Stage 9 — diagnosis (LLM only if routed).
+            evidence = list(dna.signals)
+            diagnosis, tokens = self.diagnosis_engine.diagnose(
+                event, dna, routing, context=context, change_diff=change_diff, evidence=evidence)
+            total_tokens += tokens
+
+            # Stage 10 — report.
+            incident_id = f"INC-{run_id or 'adhoc'}-{i}"
+            report = self.report_generator.build(
+                incident_id, event, dna, impact, diagnosis,
+                routing=routing, context=context, change_diff=change_diff, evidence=evidence)
+            reports.append(report)
+            self.pattern_store.record(dna.pattern_id, incident_id, impact.severity.value)
+
+        # Sort by severity for prioritisation (the product's core promise).
+        from iic.notify.teams_notifier import _SEVERITY_RANK
+        reports.sort(key=lambda r: _SEVERITY_RANK.get(r.impact.severity, 9))
+
+        # Stage 11 — notify (optional).
+        notified = False
+        if self.config.notify:
+            notified = self.notifier.send(reports, run_id=run_id,
+                                          host=self.config.host, job_id=self.config.job_id)
+
+        top = reports[0].impact.severity.value if reports else "n/a"
+        summary = (f"{len(reports)} incident(s) analyzed | top severity {top} | "
+                   f"{total_tokens} tokens | notified={notified}")
+        return EngineResult(reports=reports, tokens_used=total_tokens, notified=notified, summary=summary)
+
+
+def run_from_notebook(spark, dbutils) -> str:
+    """Entry point for the thin Databricks notebook driver.
+
+    Reads widget params, wires the real Databricks collaborators, runs the
+    pipeline, and returns a one-line summary for ``dbutils.notebook.exit``.
+    """
+    from healing_kit.auth import build_client, resolve_auth_from_dbutils
+
+    def w(name, default=""):
+        dbutils.widgets.text(name, default)
+        return dbutils.widgets.get(name)
+
+    secret_scope = w("secret_scope", "iic")
+    config = EngineConfig(
+        job_id=w("pipeline_job_id", ""),
+        parent_run_id=w("parent_run_id", "").strip(),
+        lightweight_model=w("ai_endpoint", "databricks-meta-llama-3-3-70b-instruct"),
+        powerful_model=w("ai_endpoint_powerful", "databricks-claude-opus-4-8"),
+        teams_webhook=w("teams_webhook_url", ""),
+        notify=w("notify", "true").lower() != "false",
+    )
+    catalog = w("catalog", "dev_catalog")
+    schema = w("schema", "iic_schema")
+
+    auth = resolve_auth_from_dbutils(dbutils, secret_scope)
+    client = build_client(auth)
+    config.host = auth.config.host
+    print(f"Auth method: {auth.method}")
+
+    from iic.runtime.pattern_store import DeltaPatternStore
+
+    source = DatabricksFailureSource(client, config.job_id, parent_run_id=config.parent_run_id)
+    engine = IncidentEngine(
+        config,
+        source=source,
+        context_builder=ContextBuilder(client=client, spark=spark),
+        dependency_analyzer=DependencyAnalyzer(client=client),
+        change_detector=ChangeDetector(client=client),
+        diagnosis_engine=DiagnosisEngine(client=client),
+        pattern_store=DeltaPatternStore(spark, catalog, schema),
+    )
+    result = engine.run()
+    print(result.summary)
+    return result.summary