shkit 1.2.0__py3-none-any.whl

iic/_doctor.py

@@ -0,0 +1,143 @@
+"""``iic doctor`` — the #1 support tool. Verifies, in order, with a clear ✅/❌ line:
+secret scope readable → required keys present → volume writable → webhook reachable
+→ optional-key consistency → (optionally) a named principal's READ ACL.
+
+``python -m iic doctor [--check-principal <name>]`` and ``iic.doctor()``. Exit code
+is 0 only when all REQUIRED checks pass. Failure messages are actionable on purpose.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+
+from iic.runtime.constants import (
+    DEFAULT_SECRET_SCOPE,
+    SECRET_KEY_GITHUB_REPO,
+    SECRET_KEY_GITHUB_TOKEN,
+    SECRET_KEY_HOST,
+    SECRET_KEY_PAT,
+    SECRET_KEY_TEAMS_WEBHOOK,
+    SECRET_KEY_VOLUME_PATH,
+    TEAMS_TIMEOUT,
+)
+
+_LOAD = "__load__"
+
+
+def doctor(check_principal=None, *, settings=_LOAD, probes=None) -> int:
+    scope = os.environ.get("IIC_SECRET_SCOPE", DEFAULT_SECRET_SCOPE)
+    probes = probes or _default_probes()
+    ok_all = True
+
+    s = _safe_load_settings() if settings == _LOAD else settings
+    if s is None:
+        print(f"❌ secret scope '{scope}' not readable as the current identity")
+        print(f"   → grant READ:  databricks secrets put-acl --scope {scope} "
+              "--principal <you-or-your-SP> --permission READ")
+        return 1
+    print(f"✅ secret scope '{scope}' readable")
+
+    missing = [k for k in (SECRET_KEY_TEAMS_WEBHOOK, SECRET_KEY_VOLUME_PATH) if not s.get(k)]
+    if missing:
+        print(f"❌ missing required key(s): {missing}")
+        print(f"   → databricks secrets put-secret --scope {scope} --key {missing[0]} ...")
+        return 1
+    print("✅ required keys present (teams_webhook, volume_path)")
+
+    ok, msg = probes["volume_write"](s.get(SECRET_KEY_VOLUME_PATH))
+    print(("✅" if ok else "❌") + f" volume writable: {msg}")
+    if not ok:
+        print(f"   → ensure the run identity can write {s.get(SECRET_KEY_VOLUME_PATH)} "
+              "(WRITE VOLUME / Unity Catalog grants)")
+        ok_all = False
+
+    ok, msg = probes["webhook"](s.get(SECRET_KEY_TEAMS_WEBHOOK))
+    print(("✅" if ok else "❌") + f" webhook reachable: {msg}")
+    if not ok:
+        print("   → check the teams_webhook URL (is the Power Automate flow enabled?)")
+        ok_all = False
+
+    if bool(s.get(SECRET_KEY_PAT)) != bool(s.get(SECRET_KEY_HOST)):
+        print("⚠️ pat/host should be set together — enrichment needs both (non-fatal)")
+    if bool(s.get(SECRET_KEY_GITHUB_REPO)) != bool(s.get(SECRET_KEY_GITHUB_TOKEN)):
+        print("⚠️ github_repo/github_dispatch_token should be set together (non-fatal)")
+
+    if check_principal:
+        ok, msg = probes["principal"](scope, check_principal)
+        print(("✅" if ok else "❌") + f" principal '{check_principal}' can read '{scope}': {msg}")
+        if not ok:
+            print(f"   → databricks secrets put-acl --scope {scope} "
+                  f"--principal {check_principal} --permission READ")
+            ok_all = False
+
+    print("✅ all required checks passed" if ok_all else "❌ doctor found problems (see above)")
+    return 0 if ok_all else 1
+
+
+def _safe_load_settings():
+    try:
+        from iic.runtime.scope_config import load_settings
+        return load_settings()
+    except Exception:
+        return None
+
+
+def _default_probes():
+    return {"volume_write": _probe_volume_write, "webhook": _probe_webhook,
+            "principal": _probe_principal}
+
+
+def _probe_volume_write(volume_path):
+    import uuid
+    if not volume_path:
+        return False, "no volume_path"
+    try:
+        probe = os.path.join(volume_path, f".iic_doctor_{uuid.uuid4().hex[:8]}")
+        with open(probe, "w") as f:
+            f.write("ok")
+        os.remove(probe)
+        return True, f"{volume_path} (probe write+delete ok)"
+    except Exception as ex:
+        return False, f"{volume_path}: {str(ex)[:120]}"
+
+
+def _probe_webhook(webhook):
+    if not webhook:
+        return False, "no teams_webhook"
+    try:
+        import requests
+        card = {"type": "message", "attachments": [{
+            "contentType": "application/vnd.microsoft.card.adaptive",
+            "content": {"type": "AdaptiveCard", "version": "1.4",
+                        "body": [{"type": "TextBlock", "text": "✅ IIC doctor probe"}]}}]}
+        r = requests.post(webhook, json=card, headers={"Content-Type": "application/json"},
+                          timeout=TEAMS_TIMEOUT)
+        return (r.status_code in (200, 202)), f"HTTP {r.status_code}"
+    except Exception as ex:
+        return False, str(ex)[:120]
+
+
+def _probe_principal(scope, principal):
+    try:
+        from databricks.sdk import WorkspaceClient
+        w = WorkspaceClient()
+        for acl in w.secrets.list_acls(scope=scope):
+            if getattr(acl, "principal", None) == principal:
+                return True, str(getattr(acl, "permission", ""))
+        return False, "no ACL entry"
+    except Exception as ex:
+        return False, str(ex)[:120]
+
+
+def main(argv=None) -> int:
+    argv = argv if argv is not None else sys.argv[1:]
+    principal = None
+    if "--check-principal" in argv:
+        i = argv.index("--check-principal")
+        principal = argv[i + 1] if i + 1 < len(argv) else None
+    return doctor(check_principal=principal)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

iic/change/__init__.py

@@ -0,0 +1,7 @@
+"""Stage 5 — change detection."""
+
+from __future__ import annotations
+
+from iic.change.change_detector import ChangeDetector
+
+__all__ = ["ChangeDetector"]

iic/change/change_detector.py

@@ -0,0 +1,154 @@
+"""Stage 5 — diff the failed run against the last successful run.
+
+Most production failures correlate with a recent change, so this is one of the
+highest-value signals. The comparison is split into a pure, fully-testable core
+(``extract_profile`` + ``diff_profiles`` + ``diff_schemas``) and a thin I/O wrapper
+(``ChangeDetector.detect``) that fetches the two runs via the client.
+
+What we can compare from the Jobs/Runs API without extra infrastructure:
+  * config     — task base_parameters
+  * runtime     — spark_version, node_type_id, worker count
+  * deployment  — git commit of the run's source
+  * code        — notebook revision timestamps
+  * schema      — only when before/after snapshots are supplied
+"""
+
+from __future__ import annotations
+
+from iic.models.change import ChangeDiffObject, FieldChange
+
+
+def extract_profile(run: dict) -> dict:
+    """Flatten a Jobs/Runs API run dict into a comparable profile."""
+    run = run or {}
+    settings = run.get("settings", run)  # runs/get nests under nothing; jobs/get under settings
+    tasks = run.get("tasks") or settings.get("tasks") or []
+
+    params: dict = {}
+    notebook_revisions: dict = {}
+    for t in tasks:
+        nb = t.get("notebook_task", {}) or {}
+        for k, v in (nb.get("base_parameters") or {}).items():
+            params[f"{t.get('task_key', '?')}.{k}"] = str(v)
+        if nb.get("source") == "GIT" or nb.get("notebook_path"):
+            rev = t.get("git_source_revision") or nb.get("revision_timestamp")
+            if rev:
+                notebook_revisions[t.get("task_key", "?")] = str(rev)
+
+    cluster = _first_cluster(run, tasks)
+    git = run.get("git_source") or settings.get("git_source") or {}
+
+    return {
+        "params": params,
+        "spark_version": cluster.get("spark_version", ""),
+        "node_type_id": cluster.get("node_type_id", ""),
+        "num_workers": str(cluster.get("num_workers", "")),
+        "git_commit": git.get("git_commit") or git.get("git_branch") or "",
+        "notebook_revisions": notebook_revisions,
+    }
+
+
+def _first_cluster(run: dict, tasks: list) -> dict:
+    for jc in run.get("job_clusters", []) or []:
+        spec = jc.get("new_cluster") or {}
+        if spec:
+            return spec
+    for t in tasks:
+        spec = t.get("new_cluster") or {}
+        if spec:
+            return spec
+    return {}
+
+
+def diff_profiles(failed: dict, success: dict) -> ChangeDiffObject:
+    """Pure diff of two run profiles → ChangeDiffObject (no schema diff)."""
+    diff = ChangeDiffObject(has_prior_success=True)
+
+    # config — parameter values
+    diff.config_changes = _diff_maps(failed.get("params", {}), success.get("params", {}), "config")
+
+    # runtime — cluster shape
+    for key, label in (("spark_version", "spark_version"),
+                       ("node_type_id", "node_type"),
+                       ("num_workers", "num_workers")):
+        before, after = success.get(key, ""), failed.get(key, "")
+        if before != after and (before or after):
+            diff.runtime_changes.append(FieldChange("runtime", label, before, after))
+
+    # deployment — git commit moved
+    bc, ac = success.get("git_commit", ""), failed.get("git_commit", "")
+    if bc != ac and (bc or ac):
+        diff.deployment_changes.append(FieldChange("deployment", "git_commit", bc, ac))
+
+    # code — notebook revisions changed
+    diff.code_changes = _diff_maps(
+        failed.get("notebook_revisions", {}), success.get("notebook_revisions", {}), "code"
+    )
+    return diff
+
+
+def diff_schemas(before: dict, after: dict) -> list[FieldChange]:
+    """Diff two ``{table: [columns]}`` snapshots into schema FieldChanges."""
+    changes: list[FieldChange] = []
+    for table in sorted(set(before) | set(after)):
+        prev_cols = set(before.get(table, []))
+        curr_cols = set(after.get(table, []))
+        for col in sorted(prev_cols - curr_cols):
+            changes.append(FieldChange("schema", f"{table}.{col}", before=col, after=""))
+        for col in sorted(curr_cols - prev_cols):
+            changes.append(FieldChange("schema", f"{table}.{col}", before="", after=col))
+    return changes
+
+
+def _diff_maps(failed: dict, success: dict, category: str) -> list[FieldChange]:
+    changes: list[FieldChange] = []
+    for key in sorted(set(failed) | set(success)):
+        before, after = success.get(key, ""), failed.get(key, "")
+        if before != after:
+            changes.append(FieldChange(category, key, str(before), str(after)))
+    return changes
+
+
+class ChangeDetector:
+    """Fetches the failed run + last successful run and diffs them."""
+
+    def __init__(self, client=None):
+        self.client = client
+
+    def detect(self, job_id: str, failed_run_id: str,
+               failed_run: dict | None = None,
+               prev_schema: dict | None = None,
+               curr_schema: dict | None = None) -> ChangeDiffObject:
+        if not (self.client and job_id):
+            return ChangeDiffObject(has_prior_success=False)
+
+        failed = failed_run or self._get_run(failed_run_id)
+        success = self._last_success(job_id, exclude_run_id=failed_run_id)
+        if not success:
+            diff = ChangeDiffObject(has_prior_success=False)
+        else:
+            diff = diff_profiles(extract_profile(failed), extract_profile(success))
+            diff.last_success_run_id = str(success.get("run_id", ""))
+
+        if prev_schema is not None and curr_schema is not None:
+            diff.schema_changes.extend(diff_schemas(prev_schema, curr_schema))
+        return diff
+
+    def _get_run(self, run_id: str) -> dict:
+        try:
+            return self.client._get("/api/2.1/jobs/runs/get", {"run_id": int(run_id)})
+        except Exception:
+            return {}
+
+    def _last_success(self, job_id: str, exclude_run_id: str) -> dict:
+        try:
+            runs = self.client.list_runs(int(job_id), limit=20)
+        except Exception:
+            return {}
+        for run in runs:
+            if str(run.get("run_id", "")) == str(exclude_run_id):
+                continue
+            if run.get("state", {}).get("result_state") == "SUCCESS":
+                # Re-fetch for full task/cluster detail.
+                return self._get_run(str(run.get("run_id", ""))) or run
+        return {}

iic/context/__init__.py

@@ -0,0 +1,7 @@
+"""Stage 3 — context building."""
+
+from __future__ import annotations
+
+from iic.context.context_builder import ContextBuilder
+
+__all__ = ["ContextBuilder"]

iic/context/context_builder.py

@@ -0,0 +1,117 @@
+"""Stage 3 — gather everything relevant about a failure.
+
+Best-effort by design: each fetch is independently guarded so a missing notebook,
+an unreachable cluster, or a lineage gap degrades the bundle rather than aborting
+the pipeline. Table references are parsed from the notebook source so later stages
+(schema snapshot, dependency analysis) have something to work with even when UC
+lineage is unavailable.
+"""
+
+from __future__ import annotations
+
+import re
+
+from iic.models.context import IncidentContextBundle
+from iic.models.event import NormalizedFailureEvent
+
+# Three-part (catalog.schema.table) or two-part names following FROM/JOIN/INTO/UPDATE.
+_TABLE_RE = re.compile(
+    r"\b(?:FROM|JOIN|INTO|UPDATE|TABLE)\s+([a-zA-Z_][\w]*(?:\.[a-zA-Z_][\w]*){1,2})",
+    re.IGNORECASE,
+)
+_MAX_NOTEBOOK_CHARS = 4000
+
+
+class ContextBuilder:
+    """Builds an :class:`IncidentContextBundle` from a normalized event.
+
+    ``client`` is a DatabricksClient-like object; ``spark`` is optional and only
+    used to snapshot the failing table's schema. Both may be ``None`` in tests.
+    """
+
+    def __init__(self, client=None, spark=None):
+        self.client = client
+        self.spark = spark
+
+    def build(self, event: NormalizedFailureEvent) -> IncidentContextBundle:
+        bundle = IncidentContextBundle(event_id=event.event_id)
+
+        # Logs always include the failure text we already have.
+        logs = []
+        if event.error_message:
+            logs.append(event.error_message)
+        if event.error_trace:
+            logs.append(event.error_trace)
+        bundle.logs = logs
+
+        bundle.notebook_source = self._notebook(event.notebook_path)
+        bundle.referenced_tables = self._tables(bundle.notebook_source)
+        bundle.cluster_state = self._cluster(event.cluster_id)
+        bundle.job_metadata, dag = self._job_metadata(event.job_id, event.task)
+        bundle.upstream_tasks, bundle.downstream_tasks = dag
+        bundle.schema_snapshot = self._schema(bundle.referenced_tables)
+        return bundle
+
+    # ─── individual, independently-guarded fetches ───
+
+    def _notebook(self, path: str) -> str:
+        if not (path and self.client):
+            return ""
+
+        def _fetch() -> str:
+            try:
+                return (self.client.export_notebook(path) or "")[:_MAX_NOTEBOOK_CHARS]
+            except Exception:
+                return ""
+
+        # Memoized by path only when the session cache is active (i.e. on a
+        # bootstrapped cluster). Off-cluster/tests → always a fresh fetch.
+        from iic.runtime.context import cached
+        return cached(f"notebook:{path}", _fetch)
+
+    @staticmethod
+    def _tables(notebook_source: str) -> list[str]:
+        if not notebook_source:
+            return []
+        found = {m.group(1).lower() for m in _TABLE_RE.finditer(notebook_source)}
+        return sorted(found)
+
+    def _cluster(self, cluster_id: str) -> dict:
+        if not (cluster_id and self.client):
+            return {}
+        try:
+            events = self.client.get_cluster_events(cluster_id, limit=5)
+        except Exception:
+            return {}
+        return {"cluster_id": cluster_id, "recent_events": [e.get("type", "") for e in events]}
+
+    def _job_metadata(self, job_id: str, failed_task: str):
+        """Return (metadata_dict, (upstream_task_keys, downstream_task_keys))."""
+        if not (job_id and self.client):
+            return {}, ([], [])
+        try:
+            job = self.client.get_job(int(job_id))
+        except Exception:
+            return {}, ([], [])
+        settings = job.get("settings", {}) or {}
+        tasks = settings.get("tasks", []) or []
+        upstream, downstream = [], []
+        for t in tasks:
+            if t.get("task_key") == failed_task:
+                upstream = [d.get("task_key", "") for d in t.get("depends_on", [])]
+            if any(d.get("task_key") == failed_task for d in t.get("depends_on", [])):
+                downstream.append(t.get("task_key", ""))
+        meta = {"job_name": settings.get("name", ""), "task_count": len(tasks)}
+        return meta, (upstream, downstream)
+
+    def _schema(self, tables: list[str]) -> dict:
+        if not (tables and self.spark):
+            return {}
+        snapshot = {}
+        for table in tables[:3]:  # cap the cost; first few referenced tables
+            try:
+                rows = self.spark.sql(f"DESCRIBE TABLE {table}").collect()
+                snapshot[table] = [r["col_name"] for r in rows if r["col_name"] and not r["col_name"].startswith("#")]
+            except Exception:
+                continue
+        return snapshot

iic/dependency/__init__.py

@@ -0,0 +1,7 @@
+"""Stage 4 — dependency & lineage analysis (blast radius)."""
+
+from __future__ import annotations
+
+from iic.dependency.dependency_analyzer import BlastRadius, DependencyAnalyzer
+
+__all__ = ["DependencyAnalyzer", "BlastRadius"]

iic/dependency/dependency_analyzer.py

@@ -0,0 +1,93 @@
+"""Stage 4 — compute the blast radius of a failure.
+
+Combines two graphs, both deterministic:
+  * the job DAG (task-level downstream tasks that are now blocked), reusing the
+    existing :class:`DependencyGraphBuilder`; and
+  * Unity Catalog table lineage (downstream tables / dashboards), when reachable.
+
+The result feeds the impact engine. No LLM, no Spark.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from healing_kit.services.dependency_graph import DependencyGraphBuilder
+
+
+@dataclass
+class BlastRadius:
+    """Everything downstream of the failed task that is now at risk."""
+
+    downstream_tasks: list[str] = field(default_factory=list)
+    downstream_tables: list[str] = field(default_factory=list)
+    downstream_dashboards: list[str] = field(default_factory=list)
+
+    @property
+    def downstream_jobs(self) -> int:
+        return len(self.downstream_tasks)
+
+    @property
+    def affected_tables(self) -> int:
+        return len(self.downstream_tables)
+
+    @property
+    def dashboard_impact(self) -> int:
+        return len(self.downstream_dashboards)
+
+    @property
+    def total(self) -> int:
+        return self.downstream_jobs + self.affected_tables + self.dashboard_impact
+
+    def to_dict(self) -> dict:
+        return {
+            "downstream_tasks": self.downstream_tasks,
+            "downstream_tables": self.downstream_tables,
+            "downstream_dashboards": self.downstream_dashboards,
+        }
+
+
+class DependencyAnalyzer:
+    """Computes a :class:`BlastRadius` for a failed task.
+
+    ``client`` is optional and only used for UC table lineage; the task-level
+    blast radius is computed purely from the job DAG passed in.
+    """
+
+    def __init__(self, client=None):
+        self.client = client
+        self._builder = DependencyGraphBuilder()
+
+    def analyze(self, tasks: list[dict], failed_task: str, referenced_tables: list[str] | None = None) -> BlastRadius:
+        radius = BlastRadius()
+
+        # Task-level: every transitively downstream task is now blocked.
+        try:
+            graph = self._builder.build_from_tasks(tasks or [])
+            radius.downstream_tasks = self._builder.get_all_downstream(graph, failed_task)
+        except Exception:
+            radius.downstream_tasks = []
+
+        # Table-level: UC lineage for each table the failing task writes/reads.
+        radius.downstream_tables, radius.downstream_dashboards = self._lineage(referenced_tables or [])
+        return radius
+
+    def _lineage(self, tables: list[str]):
+        downstream_tables: set[str] = set()
+        dashboards: set[str] = set()
+        if not self.client:
+            return [], []
+        for table in tables[:5]:
+            try:
+                lineage = self.client.get_table_lineage(table) or {}
+            except Exception:
+                continue
+            for entry in lineage.get("downstreams", []) or []:
+                tbl = (entry.get("tableInfo") or {}).get("name")
+                if tbl:
+                    downstream_tables.add(tbl)
+                for dash in entry.get("dashboards", []) or []:
+                    name = dash.get("name") or dash.get("id")
+                    if name:
+                        dashboards.add(str(name))
+        return sorted(downstream_tables), sorted(dashboards)

iic/diagnosis/__init__.py

@@ -0,0 +1,7 @@
+"""Stage 9 — diagnosis (LLM, structured output only)."""
+
+from __future__ import annotations
+
+from iic.diagnosis.diagnosis_engine import DiagnosisEngine
+
+__all__ = ["DiagnosisEngine"]