whykit 0.3.1__py3-none-any.whl

whykit/template/templates/source-ingestion-template.md

@@ -0,0 +1,43 @@
+---
+title: Source ingestion batch XXX
+aliases: []
+type: research
+status: template
+owner: TODO
+created: YYYY-MM-DD
+last_updated: YYYY-MM-DD
+source_of_truth: false
+sensitivity: public
+source_ids: []
+tags: []
+---
+
+# Source ingestion batch XXX
+
+## Purpose
+
+Record how one batch of uploaded material was assessed and normalized.
+
+## Results
+
+| Uploaded file | SHA-256 | Assessment | Evidence ID | Destination in repository |
+|---|---|---|---|---|
+|  |  | useful / duplicate / empty / unsupported / missing attachment |  |  |
+
+## Normalization performed
+
+- [ ] Platform-specific markup removed.
+- [ ] YAML front matter added.
+- [ ] Aliases, tags, source IDs and links to related notes added.
+- [ ] Facts, hypotheses, recommendations and decisions separated.
+- [ ] Missing assets recorded.
+- [ ] Skipped, empty and duplicate files recorded.
+- [ ] Verification questions and review requirements added.
+
+## Missing assets
+
+- None recorded.
+
+## Review required
+
+-

whykit/template/whykit.toml

@@ -0,0 +1,39 @@
+# Versioned repository-local policy. WhyKit has no hidden user-global config.
+format_version = 1
+
+[defaults]
+owner = "TODO"
+sensitivity = "internal"
+decision_review_days = 90
+status_due_days = 30
+require_hub_links = false
+
+[profiles.local]
+strict = false
+orphans = true
+secrets = true
+require_git = false
+require_clean_tree = false
+require_configured = false
+require_hub_links = false
+history = "optional"
+
+[profiles.ci]
+strict = true
+orphans = true
+secrets = true
+require_git = false
+require_clean_tree = false
+require_configured = false
+require_hub_links = false
+history = "optional"
+
+[profiles.release]
+strict = true
+orphans = true
+secrets = true
+require_git = true
+require_clean_tree = true
+require_configured = true
+require_hub_links = false
+history = "required"

whykit/trace.py

@@ -0,0 +1,272 @@
+"""Decision-to-evidence traceability.
+
+`whykit impact` answers "what breaks if I change this?" for one record.  This
+module answers the ledger-wide question in the other direction: for every
+decision, which evidence does it rest on, and is that evidence still usable?
+
+A decision can cite evidence directly (an ``E-NNN`` in its text or
+``source_ids``) or inherit it from a non-decision document it links to, such as
+a research note.  Both are reported; direct citations win when the same ID
+appears twice.
+Evidence is unusable when it is retired, missing from the register, or older
+than the vault's ``[evidence_access_age_days]`` policy for its source type.
+"""
+from __future__ import annotations
+
+from .io import consistent_read
+
+import argparse
+import datetime as dt
+import json
+import sys
+from pathlib import Path
+
+from .config import ConfigError, load_config
+from .contract import emit_error
+from .graph import build_graph
+from .lint import DECISION_ID_RE, EVIDENCE_ID_RE, _parse_date, evidence_register, find_vault_root, is_vault_root, path_cache, strip_markdown_suffix
+from .vault_index import VaultIndex
+from .console import emit_machine
+
+GAP_KINDS = ("no_evidence", "missing_evidence", "retired_evidence", "stale_evidence")
+
+
+def _evidence_state(
+    evidence_id: str,
+    active: dict[str, dict[str, str]],
+    retired: dict[str, dict[str, str]],
+    access_age_days: dict[str, int],
+    default_max_age: int | None,
+    today: dt.date,
+) -> dict:
+    if evidence_id in active:
+        row = active[evidence_id]
+        max_age = access_age_days.get(row.get("type", ""), default_max_age)
+        accessed = _parse_date(row.get("accessed", ""))
+        age = (today - accessed).days if accessed is not None else None
+        # Same rule as the `evidence.access_stale` lint: older than the window,
+        # never in the future, and an unparseable date is reported by lint.
+        stale = max_age is not None and age is not None and age > max_age
+        return {
+            "state": "active",
+            "source": row.get("source", ""),
+            "type": row.get("type", ""),
+            "accessed": row.get("accessed", "") or None,
+            "age_days": age,
+            "max_age_days": max_age,
+            "stale": stale,
+            "replaced_by": None,
+        }
+    if evidence_id in retired:
+        row = retired[evidence_id]
+        replacement = row.get("replaced_by", "").strip()
+        return {
+            "state": "retired",
+            "source": row.get("source", ""),
+            "type": "",
+            "accessed": None,
+            "age_days": None,
+            "max_age_days": None,
+            "stale": False,
+            "replaced_by": replacement if EVIDENCE_ID_RE.fullmatch(replacement) else None,
+        }
+    return {
+        "state": "missing",
+        "source": "",
+        "type": "",
+        "accessed": None,
+        "age_days": None,
+        "max_age_days": None,
+        "stale": False,
+        "replaced_by": None,
+    }
+
+
+@consistent_read
+@path_cache()
+def build_trace(
+    root: Path,
+    *,
+    today: dt.date | None = None,
+    decision: str | None = None,
+    default_max_age: int | None = None,
+    access_age_days: dict[str, int] | None = None,
+    vault: VaultIndex | None = None,
+) -> dict:
+    """Return a traceability report for every decision (or one, by ``D-NNN``)."""
+    today = today or dt.date.today()
+    root = root.resolve()
+    vault_index = vault or VaultIndex.load(root)
+    if access_age_days is None:
+        try:
+            access_age_days = dict(load_config(root)[0]["evidence_access_age_days"])
+        except ConfigError:
+            access_age_days = {}
+    active, retired, _ = evidence_register(root)
+    graph = build_graph(root, vault=vault_index)
+
+    cites: dict[str, set[str]] = {}
+    links: dict[str, set[str]] = {}
+    for edge in graph["edges"]:
+        if edge["type"] == "evidence":
+            cites.setdefault(edge["from"], set()).add(edge["to"].split(":", 1)[1])
+        elif edge["type"] == "wikilink":
+            links.setdefault(edge["from"], set()).add(edge["to"])
+
+    superseded_by: dict[str, list[str]] = {}
+    decision_notes = []
+    for note in vault_index.notes:
+        own = str(note.front.get("decision_id") or "").strip()
+        if not DECISION_ID_RE.fullmatch(own):
+            continue
+        decision_notes.append((own, note))
+        replaced = str(note.front.get("supersedes") or "").strip()
+        if DECISION_ID_RE.fullmatch(replaced):
+            superseded_by.setdefault(replaced, []).append(own)
+
+    decision_nodes = {strip_markdown_suffix(vault_index.relative(note.path)) for _, note in decision_notes}
+    states: dict[str, dict] = {}
+
+    def state_of(evidence_id: str) -> dict:
+        if evidence_id not in states:
+            states[evidence_id] = _evidence_state(evidence_id, active, retired, access_age_days, default_max_age, today)
+        return states[evidence_id]
+
+    records: list[dict] = []
+    for own, note in sorted(decision_notes, key=lambda item: (item[0], vault_index.relative(item[1].path))):
+        if decision and own != decision:
+            continue
+        node = strip_markdown_suffix(vault_index.relative(note.path))
+        evidence: dict[str, dict] = {}
+        for evidence_id in sorted(cites.get(node, ())):
+            evidence[evidence_id] = {"id": evidence_id, "via": None, **state_of(evidence_id)}
+        for linked in sorted(links.get(node, ())):
+            # Another decision's evidence is that decision's case, not this
+            # one's; inheriting it would let a successor hide behind the very
+            # record it superseded.
+            if linked == node or linked in decision_nodes:
+                continue
+            for evidence_id in sorted(cites.get(linked, ())):
+                if evidence_id not in evidence:
+                    evidence[evidence_id] = {"id": evidence_id, "via": linked, **state_of(evidence_id)}
+        items = [evidence[key] for key in sorted(evidence)]
+        gaps: list[str] = []
+        if not items:
+            gaps.append("no_evidence")
+        if any(item["state"] == "missing" for item in items):
+            gaps.append("missing_evidence")
+        if any(item["state"] == "retired" for item in items):
+            gaps.append("retired_evidence")
+        if any(item["stale"] for item in items):
+            gaps.append("stale_evidence")
+        status = str(note.front.get("status") or "")
+        successors = sorted(superseded_by.get(own, []))
+        records.append({
+            "decision_id": own,
+            "path": vault_index.relative(note.path),
+            "title": str(note.front.get("title") or note.path.stem),
+            "status": status,
+            "owner": str(note.front.get("owner") or ""),
+            "sensitivity": str(note.front.get("sensitivity") or "internal"),
+            "superseded_by": successors,
+            # Only an approved decision nobody has superseded is still binding;
+            # historical records may legitimately cite since-retired evidence.
+            "live": status == "approved" and not successors,
+            "evidence": items,
+            "gaps": gaps,
+        })
+
+    live = [record for record in records if record["live"]]
+    gap_counts = {kind: sum(1 for record in live if kind in record["gaps"]) for kind in GAP_KINDS}
+    return {
+        "contract_version": 1,
+        "as_of": today.isoformat(),
+        "decision": decision,
+        "decisions": records,
+        "summary": {
+            "decisions": len(records),
+            "live": len(live),
+            "live_with_gaps": sum(1 for record in live if record["gaps"]),
+            "gaps": gap_counts,
+        },
+    }
+
+
+def _describe(item: dict) -> str:
+    text = f"{item['id']} {item['state']}"
+    if item["state"] == "retired" and item.get("replaced_by"):
+        text += f" -> {item['replaced_by']}"
+    if item["stale"]:
+        text += f" (stale: accessed {item['accessed']}, {item['age_days']}d > {item['max_age_days']}d)"
+    if item.get("via"):
+        text += f"  via {item['via']}"
+    return text
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="whykit trace",
+        description="Trace every decision to the evidence it rests on and flag missing, retired or stale sources.",
+    )
+    parser.add_argument("--root", help="vault root (default: nearest vault)")
+    parser.add_argument("--decision", help="trace only this D-NNN")
+    parser.add_argument("--today", help="evaluate evidence age as of this ISO date")
+    parser.add_argument(
+        "--max-age-days", type=int,
+        help="treat evidence as stale after N days when [evidence_access_age_days] has no entry for its type",
+    )
+    parser.add_argument("--gaps-only", action="store_true", help="list only live decisions with gaps")
+    parser.add_argument("--strict", action="store_true", help="exit 1 when any live decision has a gap")
+    parser.add_argument("--json", action="store_true", help="emit machine-readable JSON")
+    args = parser.parse_args(argv)
+
+    if args.decision and not DECISION_ID_RE.fullmatch(args.decision):
+        return emit_error("invalid_argument", "--decision must be a D-NNN identifier", json_mode=args.json)
+    if args.max_age_days is not None and args.max_age_days < 0:
+        return emit_error("invalid_argument", "--max-age-days must be >= 0", json_mode=args.json)
+    today = None
+    if args.today:
+        try:
+            today = dt.date.fromisoformat(args.today)
+        except ValueError:
+            return emit_error("invalid_argument", f"--today is not a real ISO date: {args.today}", json_mode=args.json)
+    root = Path(args.root).expanduser().resolve() if args.root else find_vault_root()
+    if root is None or not is_vault_root(root):
+        return emit_error("vault_not_found", "no WhyKit vault found", json_mode=args.json)
+    try:
+        config, _ = load_config(root)
+    except ConfigError as exc:
+        return emit_error("invalid_config", str(exc), json_mode=args.json)
+    report = build_trace(
+        root,
+        today=today,
+        decision=args.decision,
+        default_max_age=args.max_age_days,
+        access_age_days=dict(config["evidence_access_age_days"]),
+    )
+    found = bool(report["decisions"])
+    if args.gaps_only:
+        report["decisions"] = [record for record in report["decisions"] if record["live"] and record["gaps"]]
+    if args.json:
+        emit_machine(json.dumps(report, ensure_ascii=False, indent=2))
+    else:
+        print(f"Decision traceability — {report['as_of']}")
+        for record in report["decisions"]:
+            state = "live" if record["live"] else (
+                f"superseded by {', '.join(record['superseded_by'])}" if record["superseded_by"] else record["status"] or "no status"
+            )
+            gaps = f"  [{', '.join(gap.replace('_', ' ') for gap in record['gaps'])}]" if record["gaps"] else ""
+            print(f"  {record['decision_id']}  {record['path']}  ({state}){gaps}")
+            for item in record["evidence"]:
+                print(f"      {_describe(item)}")
+        summary = report["summary"]
+        print(
+            f"\n{summary['decisions']} decision(s), {summary['live']} live, "
+            f"{summary['live_with_gaps']} live with gaps"
+        )
+    if args.decision and not found:
+        print(f"{args.decision}: not found", file=sys.stderr)
+        return 1
+    if args.strict and report["summary"]["live_with_gaps"]:
+        return 1
+    return 0

whykit/vault_index.py

@@ -0,0 +1,190 @@
+"""Request-scoped vault model: one parse, many consumers."""
+from __future__ import annotations
+
+import contextlib
+import contextvars
+import os
+import stat
+import threading
+import time
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from .io import vault_read_lock
+from .lint import Note, _normalise, _build_index, _link_key, _real, _resolve, _within, collect_markdown, load_note, path_cache, rel, strip_markdown_suffix
+
+
+class NoteCache:
+    """Parsed notes kept across requests while their files are unchanged.
+
+    A long-lived reader (the MCP server) would otherwise read and parse every
+    note on every call.  Each entry is keyed by the file's path and checked
+    against its current ``stat`` signature (modification and change time,
+    size, inode), link metadata and content when change time is untrusted,
+    so an edit, an atomic replace, a rename
+    or a deletion is always picked up by the next call.
+
+    A file whose timestamps are within :attr:`racy_ns` of the clock is never
+    cached: a second write in the same timestamp tick, with the same size,
+    would otherwise be invisible (the "racy clean" problem Git also guards
+    against).  Only parsing is reused.  Link resolution, findings and every
+    other derived view are still computed per request, because they depend on
+    files outside the note set (attachments, symlinks, configuration).
+
+    Notes are shared between requests, so callers must treat them as read-only,
+    as every WhyKit consumer already does.
+    """
+
+    racy_ns = 2_000_000_000
+
+    def __init__(self) -> None:
+        self._entries: dict[str, tuple[tuple[int, ...], Note]] = {}
+        self._lock = threading.Lock()
+        self.hits = 0
+        self.misses = 0
+
+    def load_all(self, paths: list[Path]) -> list[Note]:
+        now = time.time_ns()
+        fresh: dict[str, tuple[tuple[int, ...], Note]] = {}
+        notes: list[Note] = []
+        with self._lock:
+            entries = self._entries
+        for path in paths:
+            key = os.fspath(path)
+            try:
+                link = os.lstat(path)
+                info = os.stat(path) if stat.S_ISLNK(link.st_mode) else link
+            except OSError:
+                notes.append(load_note(path))  # raises like an uncached load
+                continue
+            signature = (info.st_mtime_ns, info.st_ctime_ns, info.st_size, info.st_ino, info.st_dev,
+                         link.st_mtime_ns, link.st_ctime_ns, link.st_size, link.st_ino, link.st_dev)
+            cached = entries.get(key)
+            same = cached is not None and cached[0] == signature
+            from .parse_cache import CHANGE_TIME_TRUSTED
+            if same and cached is not None and not CHANGE_TIME_TRUSTED:
+                try:
+                    text = _normalise(path.read_text(encoding="utf-8"))
+                    same = text == cached[1].text
+                except UnicodeError:
+                    same = False
+            if same and cached is not None:
+                self.hits += 1
+                note = cached[1]
+            else:
+                self.misses += 1
+                note = load_note(path)
+            notes.append(note)
+            if now - max(info.st_mtime_ns, info.st_ctime_ns) >= self.racy_ns:
+                fresh[key] = cached if same and cached is not None else (signature, note)
+        # Only the files of this load survive: a deleted note is dropped.
+        with self._lock:
+            self._entries = fresh
+        return notes
+
+
+_NOTE_CACHE: contextvars.ContextVar[NoteCache | None] = contextvars.ContextVar("whykit_note_cache", default=None)
+
+
+@contextlib.contextmanager
+def reuse_notes(cache: NoteCache) -> Iterator[None]:
+    """Let every :meth:`VaultIndex.load` in this context reuse *cache*."""
+    token = _NOTE_CACHE.set(cache)
+    try:
+        yield
+    finally:
+        _NOTE_CACHE.reset(token)
+
+
+@dataclass
+class VaultIndex:
+    """Immutable in-memory view of a vault after one Markdown parse pass."""
+
+    root: Path
+    notes: list[Note]
+    by_path: dict[Path, Note] = field(repr=False)
+    link_index: dict[str, set[Path]] = field(repr=False)
+    # Link resolution and relative paths cost several ``stat``/``realpath``
+    # calls each.  A vault repeats the same targets thousands of times, so both
+    # are memoised for the lifetime of this request-scoped index.
+    _resolved: dict[str, tuple[Path | None, bool]] = field(default_factory=dict, repr=False, compare=False)
+    _relative: dict[Path, str] = field(default_factory=dict, repr=False, compare=False)
+    # Views derived from this parse (e.g. the wikilink adjacency used by impact
+    # analysis), so a multi-record command does not rebuild them per record.
+    # A subset() starts with its own empty dict: derived views never cross views.
+    derived: dict = field(default_factory=dict, repr=False, compare=False)
+    # A confined index resolves links only to its own notes: a file on disk
+    # that is not one of them is treated as if it did not exist.  See subset().
+    confined: bool = field(default=False, repr=False, compare=False)
+
+    @classmethod
+    @path_cache()
+    def load(cls, root: Path) -> VaultIndex:
+        with vault_read_lock(root):
+            root = Path(root).resolve()
+            paths = collect_markdown(root, [])
+            cache = _NOTE_CACHE.get()
+            if cache is not None:
+                notes = cache.load_all(paths)
+            else:
+                from .parse_cache import current
+
+                disk = current(root)
+                notes = disk.load_all(paths) if disk is not None else [load_note(path) for path in paths]
+            return cls(
+                root=root,
+                notes=notes,
+                by_path={_real(note.path): note for note in notes},
+                link_index=_build_index(notes),
+            )
+
+    def subset(self, keep: Callable[[Note], bool]) -> VaultIndex:
+        """Return a confined index over the notes for which *keep* is true.
+
+        Every consumer that takes a ``vault=`` argument then behaves as if the
+        dropped notes did not exist: they are not link targets, not decision ID
+        holders and not graph nodes, so a stem shared with a dropped note
+        resolves to the remaining one instead of being reported as ambiguous.
+        """
+        notes = [note for note in self.notes if keep(note)]
+        return VaultIndex(
+            root=self.root,
+            notes=notes,
+            by_path={_real(note.path): note for note in notes},
+            link_index=_build_index(notes),
+            _relative=self._relative,
+            confined=True,
+        )
+
+    def note_for(self, path: Path) -> Note | None:
+        return self.by_path.get(_real(Path(path)))
+
+    def relative(self, path: Path) -> str:
+        cached = self._relative.get(path)
+        if cached is None:
+            cached = self._relative[path] = rel(self.root, path)
+        return cached
+
+    def resolve_link(self, target: str) -> tuple[Path | None, bool]:
+        """Resolve a wikilink target exactly like the linter, once per target."""
+        cached = self._resolved.get(target)
+        if cached is None:
+            cached = self._resolved[target] = (
+                self._resolve_confined(target) if self.confined else _resolve(self.root, target, self.link_index)
+            )
+        return cached
+
+    def _resolve_confined(self, target: str) -> tuple[Path | None, bool]:
+        path, ambiguous = _resolve(self.root, target, self.link_index)
+        if path is None or self.note_for(path) is not None:
+            return path, ambiguous
+        # `_resolve` accepted a file that exists on disk but is not one of this
+        # index's notes.  Ignore it and fall back to the stem/alias lookup, which
+        # only ever sees this index's notes.
+        normalized = target.strip().replace("\\", "/").lstrip("/")
+        stem = _link_key(strip_markdown_suffix(normalized.rstrip("/").split("/")[-1]))
+        hits = {item for item in self.link_index.get(stem, set()) if _within(self.root, item)}
+        if len(hits) == 1:
+            return next(iter(hits)), False
+        return None, len(hits) > 1