epistemic-engine 1.2.0__py3-none-any.whl

epistemic_engine/__init__.py

@@ -0,0 +1,19 @@
+"""The Epistemic Engine — computational epistemology for software engineering.
+
+Implements SRS v1.1. See README.md for the phased build plan.
+"""
+
+__version__ = "1.2.0"
+
+# Component/model versions participate in the reproducibility record (RGE-F10,
+# REL-2). Bump a component version when its computation changes; a change
+# invalidates cached reports (REL-4).
+MODEL_VERSIONS = {
+    "belief_extraction": "1.1",
+    "justification_mining": "1.1",
+    "agm_revision": "1.1",
+    "confidence_model": "noisy_or-1.0",
+    "erosion_model": "change_driven-1.0",
+}
+
+__all__ = ["__version__", "MODEL_VERSIONS"]

epistemic_engine/analyze.py

@@ -0,0 +1,433 @@
+"""`ee analyze` — the Phase-2 epistemic pass (BEM + JRM + RIM-F15).
+
+Runs after ingestion. Reads the HEAD tree from the graph, extracts and
+formalizes beliefs onto PO-1, mines justifications, emits the change-event
+stream, and writes it all back as a deterministic, idempotent subgraph
+(Belief / Justification / ChangeEvent / Predicate / MetricSnapshot nodes and
+their edges). Re-running reconciles to the same result.
+
+Confidence, entrenchment, and falsification are intentionally left to the BRE
+in Phase 3; beliefs are stored with ``confidence = None`` until then.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from .beliefs import BeliefExtractor
+from .beliefs.extract import aggregate
+from .config import Config
+from .ingestion.change_events import events_for_file, is_manifest
+from .ingestion.git_extractor import GitExtractor
+from .justifications import JustificationMiner, build_chain_metrics
+from .ontology import Ontology
+from .storage.graph import GraphStore
+
+# node/edge types (7.1/7.2)
+N_BELIEF, N_JUST, N_PREDICATE = "Belief", "Justification", "Predicate"
+N_CHANGE, N_METRIC = "ChangeEvent", "MetricSnapshot"
+E_ABOUT, E_HASPRED = "BELIEF_ABOUT", "BELIEF_PREDICATE"
+E_SUBSUMES, E_INCOMPAT = "PREDICATE_SUBSUMES", "PREDICATE_INCOMPATIBLE"
+E_JUSTIFIES, E_EVIDENCE = "JUSTIFIES", "JUSTIFICATION_EVIDENCE"
+E_ERODES = "ERODES"
+
+_PHASE2_TYPES = [N_BELIEF, N_JUST, N_PREDICATE, N_CHANGE, N_METRIC, "Falsifier",
+                 "PullRequest", "Issue", "Comment"]
+
+
+@dataclass
+class AnalyzeResult:
+    repo_id: str
+    head_commit: str
+    belief_count: int
+    formalised: int
+    unformalised: int
+    ontology_coverage: float
+    coverage_by_language: dict[str, dict[str, int]]
+    coverage_by_domain: dict[str, dict[str, int]]
+    justification_count: int
+    justification_coverage: float
+    broken_chains: int
+    change_event_count: int
+    belief_density: float
+    forge_mode: str
+    graph_digest: str
+    overall_confidence: float | None = None
+    health: str | None = None
+    epistemic_debt: float | None = None
+    falsifier_count: int = 0
+    band_counts: dict[str, int] = field(default_factory=dict)
+    erosion_patterns: dict[str, int] = field(default_factory=dict)
+    orphaned_count: int = 0
+    epistemic_velocity: float = 0.0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "repo_id": self.repo_id, "head_commit": self.head_commit,
+            "belief_count": self.belief_count, "formalised": self.formalised,
+            "unformalised": self.unformalised,
+            "ontology_coverage": round(self.ontology_coverage, 4),
+            "coverage_by_language": self.coverage_by_language,
+            "coverage_by_domain": self.coverage_by_domain,
+            "justification_count": self.justification_count,
+            "justification_coverage": round(self.justification_coverage, 4),
+            "broken_chains": self.broken_chains,
+            "change_event_count": self.change_event_count,
+            "belief_density": round(self.belief_density, 4),
+            "forge_mode": self.forge_mode,
+            "overall_confidence": round(self.overall_confidence, 4) if self.overall_confidence is not None else None,
+            "health": self.health,
+            "epistemic_debt": round(self.epistemic_debt, 2) if self.epistemic_debt is not None else None,
+            "falsifier_count": self.falsifier_count,
+            "band_counts": self.band_counts,
+            "erosion_patterns": self.erosion_patterns,
+            "orphaned_count": self.orphaned_count,
+            "epistemic_velocity": self.epistemic_velocity,
+            "graph_digest": self.graph_digest,
+        }
+
+
+# Recent-history windows: commit_message justifications saturate within a source
+# type (11.A.3) and erosion saturates by ~exp(-5), so a bounded recent window is
+# lossless yet keeps analysis O(files) rather than O(files × full history).
+_JUST_WINDOW = 12
+_CE_WINDOW = 20
+
+
+class Analyzer:
+    def __init__(self, git: GitExtractor, graph: GraphStore, config: Config, ontology: Ontology,
+                 *, progress: bool = False) -> None:
+        self.git = git
+        self.graph = graph
+        self.config = config
+        self.ontology = ontology
+        self.progress = progress
+        self.extractor = BeliefExtractor(ontology)
+        self.miner = JustificationMiner(config)
+
+    def _read_source(self, path: str, head: str) -> str | None:
+        """Read a HEAD file, preferring the working tree (fast) over `git show`.
+
+        Right after ingest the working tree matches HEAD, so a direct disk read
+        avoids one subprocess per file. Falls back to `git show HEAD:path` when
+        the file is not on disk (bare repo / detached analysis).
+        """
+        try:
+            disk = Path(self.git.repo_path) / path
+            if disk.is_file():
+                return disk.read_text(encoding="utf-8", errors="replace")
+        except OSError:
+            pass
+        try:
+            return self.git.file_content(path, head)
+        except Exception:
+            return None
+
+    def _file_histories(self) -> dict[str, list[tuple[str, int, str]]]:
+        """Build per-file commit history from the graph (no per-file git calls).
+
+        Reuses the COMMIT_CHANGES_FILE edges recorded at ingest time, so a repo
+        with thousands of files is analysed without thousands of `git log`
+        subprocesses. Each list is (hash, author_date, message), oldest→newest.
+        """
+        commit_meta = {n.id: (int(n.props.get("author_date", 0)), str(n.props.get("message", "")))
+                       for n in self.graph.iter_nodes("Commit")}
+        hist: dict[str, list[tuple[str, int, str]]] = defaultdict(list)
+        for e in self.graph.iter_edges("COMMIT_CHANGES_FILE"):
+            if not e.dst.startswith("file:"):
+                continue
+            path = e.dst[len("file:"):]
+            ts, msg = commit_meta.get(e.src, (0, ""))
+            hist[path].append((e.src, ts, msg))
+        for path in hist:
+            hist[path].sort(key=lambda t: (t[1], t[0]))  # (ts, hash) — stable/deterministic
+        return hist
+
+    def analyze(self) -> AnalyzeResult:
+        head = str(self.graph.get_meta("head_commit") or self.git.head_commit())
+        forge_mode = "git-native"  # Mode 2 (RIM-F13); connected sync is a later phase
+        erosion_deltas = dict(self.config.get("epistemology.erosion_deltas", {}))
+
+        self._wipe_phase2()
+        self._add_ontology_subgraph()
+
+        entity_lookup = self._entity_lookup()
+        files = self._head_files()
+        file_history = self._file_histories()
+
+        from .progress import Progress
+        bar = Progress("  analyzing files", len(files), enabled=self.progress)
+
+        all_beliefs = []
+        language_of: dict[str, str | None] = {}
+        total_loc = 0
+        justifications = []
+        change_events = []
+
+        for path, lang, is_test in files:
+            bar.advance()
+            language_of[path] = lang
+            source = self._read_source(path, head)
+            if source is None:
+                continue
+            total_loc += source.count("\n") + 1
+            history = file_history.get(path, [])
+            timestamp = history[-1][1] if history else 0
+            recent = history[-_JUST_WINDOW:]  # bounded window (saturation-lossless)
+
+            beliefs = self.extractor.extract_file(path, source, lang, timestamp)
+            all_beliefs.extend(beliefs)
+
+            # store beliefs + edges
+            for b in beliefs:
+                self._store_belief(b, entity_lookup)
+                # intrinsic justification (JRM-F10)
+                intrinsic = self.miner.mine_intrinsic(b)
+                if intrinsic:
+                    justifications.append(intrinsic)
+                    self._store_justification(intrinsic, evidence_node=f"file:{path}")
+                # commit justifications from recent history (JRM-F1,F7)
+                for chash, cts, msg in recent:
+                    for j in self.miner.mine_commit(b, chash, msg, cts):
+                        justifications.append(j)
+                        self._store_justification(j, evidence_node=chash)
+
+            # change events (RIM-F15) — bounded to the recent window
+            evs = events_for_file(path, is_test, history, erosion_deltas)[-_CE_WINDOW:]
+            for ev in evs:
+                self._store_change_event(ev, [b for b in beliefs if b.timestamp <= ev.timestamp])
+            change_events.extend(evs)
+        bar.done(f"{len(all_beliefs)} beliefs")
+
+        # --- Documentation justifications (JRM-F5): design docs / ADRs / RFCs ---
+        justifications.extend(self._mine_docs(head, all_beliefs))
+
+        # --- Forge Mode 1: connected-sync justifications from cached PRs/issues -
+        forge_justifications = self._mine_forge(head)
+        justifications.extend(forge_justifications)
+        if forge_justifications or self._forge_present():
+            forge_mode = "connected-sync"
+
+        result_ext = aggregate(all_beliefs, language_of)
+        belief_ids = [b.id for b in all_beliefs]
+        chain = build_chain_metrics(belief_ids, justifications, truncated=(forge_mode == "git-native"))
+
+        density = (len(all_beliefs) / total_loc * 100.0) if total_loc else 0.0
+        self._update_repository(head, density, forge_mode)
+        self._store_metric_snapshot(head, all_beliefs, justifications, result_ext, head_ts=self._head_ts(head))
+
+        # --- Belief Revision Engine: entrenchment, confidence, falsification ---
+        from .revision.engine import BeliefRevisionEngine
+        rev = BeliefRevisionEngine(
+            self.git, self.graph, self.config, self.ontology, head, self._head_ts(head),
+            file_history=file_history,
+        ).run()
+
+        # --- Temporal Analysis Engine: trajectories, erosion, orphaned beliefs -
+        from .temporal.engine import TemporalAnalyzer
+        temporal = TemporalAnalyzer(self.graph, self.config, self._head_ts(head)).run()
+        self._temporal = temporal
+
+        self.graph.set_meta("forge_mode", forge_mode)
+        self.graph.set_meta("analyzed", True)
+        if hasattr(self.graph, "_conn"):
+            self.graph._conn.commit()  # type: ignore[attr-defined]
+
+        return AnalyzeResult(
+            repo_id=str(self.graph.get_meta("repo_id") or ""),
+            head_commit=head,
+            belief_count=len(all_beliefs),
+            formalised=result_ext.formalised,
+            unformalised=result_ext.unformalised,
+            ontology_coverage=result_ext.ontology_coverage,
+            coverage_by_language=result_ext.coverage_by_language,
+            coverage_by_domain=result_ext.coverage_by_domain,
+            justification_count=len(justifications),
+            justification_coverage=chain.coverage,
+            broken_chains=len(chain.broken_belief_ids),
+            change_event_count=len(change_events),
+            belief_density=density,
+            forge_mode=forge_mode,
+            overall_confidence=rev.overall_confidence,
+            health=rev.health,
+            epistemic_debt=rev.epistemic_debt,
+            falsifier_count=rev.falsifier_count,
+            band_counts=rev.band_counts,
+            erosion_patterns=temporal["erosion_patterns"],
+            orphaned_count=temporal["orphaned_count"],
+            epistemic_velocity=temporal["epistemic_velocity"],
+            graph_digest=self.graph.digest(),
+        )
+
+    # -- storage helpers ----------------------------------------------------
+    def _store_belief(self, belief, entity_lookup: dict[tuple[str, str], str]) -> None:
+        self.graph.add_node(N_BELIEF, belief.id, belief.to_props())
+        # BELIEF_ABOUT -> entity if known, else the file node
+        subject = entity_lookup.get((belief.path, belief.subject_qname))
+        if subject is None:
+            subject = f"file:{belief.path}"
+        self.graph.add_edge(E_ABOUT, belief.id, subject, {"belief_type": belief.source_kind})
+        if belief.predicate_id:
+            self.graph.add_edge(E_HASPRED, belief.id, belief.predicate_id, {})
+
+    def _store_justification(self, j, evidence_node: str) -> None:
+        self.graph.add_node(N_JUST, j.id, j.to_props())
+        self.graph.add_edge(E_JUSTIFIES, j.id, j.belief_id,
+                            {"weight": j.weight, "validity": j.validity, "source_type": j.source_type})
+        self.graph.add_edge(E_EVIDENCE, j.id, evidence_node, {"source_type": j.source_type})
+
+    def _store_change_event(self, ev, affected_beliefs) -> None:
+        self.graph.add_node(N_CHANGE, ev.id, ev.to_props())
+        for b in affected_beliefs:
+            self.graph.add_edge(E_ERODES, ev.id, b.id, {"erosion_delta": ev.erosion_delta})
+
+    def _repo_id(self) -> str:
+        return str(self.graph.get_meta("repo_id") or "")
+
+    def _mine_docs(self, head: str, beliefs) -> list:
+        """Justifications from documentation: design docs, ADRs, RFCs (JRM-F5).
+
+        A belief whose subject is named in a doc file gains a ``doc``
+        justification — the documentation stands as evidence for the belief.
+        """
+        import hashlib as _h
+        from .justifications import Justification
+
+        doc_paths: list[tuple[str, str]] = []  # (path, name)
+        for n in self.graph.iter_nodes("File"):
+            path, name = n.props.get("path", ""), n.props.get("name", "").lower()
+            ext = Path(name).suffix
+            is_doc = (ext in {".md", ".rst", ".adoc", ".txt"} or "/docs/" in f"/{path}"
+                      or name.startswith(("adr", "rfc")) or "design" in name)
+            if is_doc:
+                doc_paths.append((path, name))
+        if not doc_paths:
+            return []
+
+        # index beliefs by subject last-component for cheap membership tests
+        subjects: dict[str, list] = defaultdict(list)
+        head_ts = self._head_ts(head)
+        for b in beliefs:
+            key = b.subject_qname.split(".")[-1]
+            if key:
+                subjects[key].append(b)
+
+        out = []
+        seen: set[str] = set()
+        for path, _name in sorted(doc_paths):
+            try:
+                text = self.git.file_content(path, head)
+            except Exception:
+                continue
+            for key, bs in subjects.items():
+                if len(key) < 3 or key not in text:
+                    continue
+                for b in bs:
+                    jid = "J:" + _h.sha1(f"{b.id}|doc|{path}".encode()).hexdigest()[:16]
+                    if jid in seen:
+                        continue
+                    seen.add(jid)
+                    out.append(Justification(
+                        id=jid, belief_id=b.id, source_type="doc", source_id=path,
+                        evidence_text=f"documented in {path}", weight=0.6, validity=1.0,
+                        timestamp=head_ts, acquisition_mode="local",
+                    ))
+                    self._store_justification(
+                        out[-1], evidence_node=f"file:{path}")
+        return out
+
+    def _forge_present(self) -> bool:
+        from .forge.cache import forge_cache_path
+        return forge_cache_path(self.config.data_dir, self._repo_id()).is_file()
+
+    def _mine_forge(self, head: str) -> list:
+        """Load cached forge data and mine Mode-1 justifications (JRM-F2-F4)."""
+        from .forge.cache import forge_cache_path, load_forge
+        from .forge.sync import mine_forge_justifications
+
+        data = load_forge(forge_cache_path(self.config.data_dir, self._repo_id()))
+        if data is None:
+            return []
+        # materialize PullRequest / Issue entities (7.1) with acquisition mode
+        for pr in data.prs:
+            self.graph.add_node("PullRequest", f"PR#{pr.number}", {
+                "number": pr.number, "title": pr.title, "acquisition_mode": "connected",
+            })
+        for issue in data.issues:
+            self.graph.add_node("Issue", f"issue#{issue.number}", {
+                "number": issue.number, "title": issue.title, "acquisition_mode": "connected",
+            })
+        justs = mine_forge_justifications(self.graph, data, self.config)
+        for j in justs:
+            evidence = j.source_id.split(":")[0]  # PR#n / issue#n
+            self._store_justification(j, evidence_node=evidence)
+        return justs
+
+    def _add_ontology_subgraph(self) -> None:
+        for pid, pred in self.ontology.predicates.items():
+            self.graph.add_node(N_PREDICATE, pid, {
+                "label": pred.label, "domain": pred.domain,
+                "complement_id": pred.complement, "ontology_version": self.ontology.version,
+            })
+        for sub in self.ontology.predicate_ids():
+            for sup in self.ontology._direct_supers.get(sub, ()):  # type: ignore[attr-defined]
+                self.graph.add_edge(E_SUBSUMES, sub, sup, {})
+        for a, b in self.ontology._declared_incompat:  # type: ignore[attr-defined]
+            self.graph.add_edge(E_INCOMPAT, a, b, {})
+            self.graph.add_edge(E_INCOMPAT, b, a, {})
+
+    def _update_repository(self, head: str, density: float, forge_mode: str) -> None:
+        node = next(iter(self.graph.iter_nodes("Repository")), None)
+        if node is not None:
+            props = dict(node.props)
+            props["belief_density"] = round(density, 4)
+            props["forge_mode"] = forge_mode
+            self.graph.add_node("Repository", node.id, props)
+
+    def _store_metric_snapshot(self, head: str, beliefs, justifications, ext, head_ts: int) -> None:
+        falsifier_count = self.graph.count_nodes("Falsifier")
+        self.graph.add_node(N_METRIC, f"metric:{head}", {
+            "commit_hash": head,
+            "overall_confidence": None,  # Phase 3
+            "belief_count": len(beliefs),
+            "formalised": ext.formalised,
+            "unformalised": ext.unformalised,
+            "justification_count": len(justifications),
+            "falsifier_count": falsifier_count,
+            "epistemic_debt": None,  # Phase 4
+            "ontology_coverage": round(ext.ontology_coverage, 4),
+            "timestamp": head_ts,
+        })
+
+    # -- lookups ------------------------------------------------------------
+    def _entity_lookup(self) -> dict[tuple[str, str], str]:
+        lookup: dict[tuple[str, str], str] = {}
+        for kind in ("function", "method", "class", "variable", "import"):
+            for n in self.graph.iter_nodes(kind):
+                key = (n.props.get("path", ""), n.props.get("qualified_name", ""))
+                lookup.setdefault(key, n.id)
+        return lookup
+
+    def _head_files(self) -> list[tuple[str, str | None, bool]]:
+        files: list[tuple[str, str | None, bool]] = []
+        for n in self.graph.iter_nodes("File"):
+            files.append((n.props.get("path", ""), n.props.get("language") or None,
+                          bool(n.props.get("is_test", False))))
+        return sorted(files)
+
+    def _head_ts(self, head: str) -> int:
+        node = self.graph.get_node(head)
+        return int(node.props.get("author_date", 0)) if node else 0
+
+    def _wipe_phase2(self) -> None:
+        if not hasattr(self.graph, "_conn"):
+            return
+        conn = self.graph._conn  # type: ignore[attr-defined]
+        for ntype in _PHASE2_TYPES:
+            conn.execute(
+                "DELETE FROM edges WHERE src IN (SELECT id FROM nodes WHERE type=?) "
+                "OR dst IN (SELECT id FROM nodes WHERE type=?)", (ntype, ntype))
+            conn.execute("DELETE FROM nodes WHERE type=?", (ntype,))

epistemic_engine/beliefs/__init__.py

@@ -0,0 +1,24 @@
+"""Belief Extraction Module (BEM) — SRS 4.2."""
+
+from .model import (
+    DEFAULT_SCOPE,
+    DOMAINS,
+    SCOPE_VOCAB,
+    Belief,
+    BeliefSignal,
+    belief_id,
+)
+from .mapper import OntologyMapper
+from .extract import BeliefExtractor, ExtractionResult
+
+__all__ = [
+    "DEFAULT_SCOPE",
+    "DOMAINS",
+    "SCOPE_VOCAB",
+    "Belief",
+    "BeliefSignal",
+    "belief_id",
+    "OntologyMapper",
+    "BeliefExtractor",
+    "ExtractionResult",
+]