xrefkit 0.3.0__py3-none-any.whl

xrefkit/__init__.py

@@ -0,0 +1,5 @@
+"""Portable XRefKit package namespace."""
+
+__all__ = ["__version__"]
+
+__version__ = "0.3.0"

xrefkit/__main__.py

@@ -0,0 +1,5 @@
+from .cli import main
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

xrefkit/catalog_cli.py

@@ -0,0 +1,57 @@
+"""CLI for list-first catalogs and candidate maintenance."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from .structure_catalog import get_entry, list_findings, list_targets, load_catalog, maintain_catalog, reconcile_receipts
+
+
+DEFAULT_CATALOG = "knowledge/source_analysis/source_structure_catalog.yaml"
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="xrefkit catalog")
+    parser.add_argument("--root", default=".")
+    parser.add_argument("--catalog", default=DEFAULT_CATALOG)
+    sub = parser.add_subparsers(dest="command", required=True)
+    listing = sub.add_parser("list")
+    listing.add_argument("kind", choices=["targets", "findings"])
+    listing.add_argument("--target")
+    listing.add_argument("--json", action="store_true")
+    get = sub.add_parser("get")
+    get.add_argument("xid")
+    get.add_argument("--json", action="store_true")
+    maintain = sub.add_parser("maintain")
+    maintain.add_argument("--inbox", default="work/inbox/source_structure_findings")
+    maintain.add_argument("--apply-safe", action="store_true")
+    maintain.add_argument("--json", action="store_true")
+    reconcile = sub.add_parser("reconcile")
+    reconcile.add_argument("--reports", default="work/reports")
+    reconcile.add_argument("--inbox", default="work/inbox/source_structure_findings")
+    reconcile.add_argument("--json", action="store_true")
+    args = parser.parse_args(argv)
+    root = Path(args.root).resolve()
+    try:
+        if args.command == "list":
+            catalog = load_catalog(root / args.catalog)
+            if args.kind == "targets":
+                result = list_targets(catalog)
+            else:
+                if not args.target:
+                    raise ValueError("--target is required for finding list")
+                result = list_findings(catalog, args.target)
+        elif args.command == "get":
+            result = get_entry(load_catalog(root / args.catalog), args.xid)
+        elif args.command == "maintain":
+            result = maintain_catalog(root, args.catalog, args.inbox, apply_safe=args.apply_safe)
+        else:
+            result = reconcile_receipts(root, args.reports, args.inbox)
+    except (OSError, KeyError, ValueError) as exc:
+        result = {"ok": False, "error": str(exc)}
+        print(json.dumps(result, ensure_ascii=False, indent=2, default=str))
+        return 1
+    print(json.dumps(result, ensure_ascii=False, indent=2, default=str))
+    return 0 if not isinstance(result, dict) or result.get("ok", True) else 1

xrefkit/cli.py

@@ -0,0 +1,71 @@
+"""Unified command dispatcher for the XRefKit package."""
+
+from __future__ import annotations
+
+import sys
+from collections.abc import Sequence
+
+
+_OPERATIONS = {"xref", "ctx", "goal", "gate", "pack", "dashboard", "skill"}
+_V2 = {"package", "show"}
+
+
+def _print_help() -> None:
+    print(
+        "usage: xrefkit <command> ...\n\n"
+        "commands:\n"
+        "  init       initialize or validate an XRefKit instance\n"
+        "  xref       manage XIDs and references\n"
+        "  ctx        build compact context packs\n"
+        "  skill      discover, validate, run, verify, and close Skills\n"
+        "  tools      list and run XID-backed client tools\n"
+        "  catalog    list and maintain Knowledge and structure catalogs\n"
+        "  pack       validate and build runtime/content packs\n"
+        "  gate       evaluate deterministic gates\n"
+        "  goal       manage desired-state Goals and continuation state\n"
+        "  dashboard  inspect runtime state\n"
+        "  mcp        start the integrated MCP server\n"
+        "  package    inspect installed Skill packages\n"
+        "  show       show effective Skill bundles"
+    )
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = list(argv if argv is not None else sys.argv[1:])
+    if not args or args[0] in {"-h", "--help"}:
+        _print_help()
+        return 0
+
+    command = args[0]
+    if command == "pack" and len(args) > 1 and args[1] in {"build-base", "verify-base"}:
+        from .contracts import main as contracts_main
+
+        return contracts_main(args[1:])
+    if command in _OPERATIONS:
+        from .operations_cli import main as operations_main
+
+        return operations_main(args)
+    if command in _V2:
+        from .v2_cli import main as v2_main
+
+        return v2_main(args)
+    if command == "init":
+        from .instance import main as instance_main
+
+        return instance_main(args[1:])
+    if command == "tools":
+        from .tools import main as tools_main
+
+        return tools_main(args[1:])
+    if command == "catalog":
+        from .catalog_cli import main as catalog_main
+
+        return catalog_main(args[1:])
+    if command == "mcp":
+        from .mcp import main as mcp_main
+
+        return mcp_main(args[1:])
+
+    print(f"xrefkit: unknown command: {command}", file=sys.stderr)
+    _print_help()
+    return 2

xrefkit/contracts.py

@@ -0,0 +1,297 @@
+"""Compile and verify model-facing base runtime contract resources."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+SCHEMA = "xrefkit.base_runtime/v1"
+GENERATED_SCHEMA = "xrefkit.compiled_runtime/v1"
+NORMATIVE_RE = re.compile(r"\bMUST(?:\s+NOT)?\b")
+
+
+def _sha256(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def _load_manifest(path: Path) -> dict[str, Any]:
+    data = yaml.safe_load(path.read_text(encoding="utf-8"))
+    if not isinstance(data, dict) or data.get("schema") != SCHEMA:
+        raise ValueError(f"unsupported base runtime manifest: {path}")
+    return data
+
+
+def _estimate_tokens(text: str) -> int:
+    return (len(text) + 3) // 4
+
+
+def _atomic_write_bytes(path: Path, data: bytes) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
+    try:
+        with os.fdopen(fd, "wb") as handle:
+            handle.write(data)
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temporary, path)
+    except BaseException:
+        try:
+            os.unlink(temporary)
+        except FileNotFoundError:
+            pass
+        raise
+
+
+def _publish_generation(output: Path, generated: dict[str, Any], model_body: str) -> None:
+    contracts_bytes = json.dumps(generated, ensure_ascii=False, indent=2, default=str).encode("utf-8") + b"\n"
+    model_bytes = model_body.encode("utf-8")
+    generation_id = hashlib.sha256(contracts_bytes + model_bytes).hexdigest()[:16]
+    generations = output / "generations"
+    generations.mkdir(parents=True, exist_ok=True)
+    generation_path = generations / generation_id
+    if not generation_path.exists():
+        temporary = Path(tempfile.mkdtemp(prefix=".generation-", dir=generations))
+        try:
+            (temporary / "contracts.json").write_bytes(contracts_bytes)
+            (temporary / "model_body.md").write_bytes(model_bytes)
+            os.replace(temporary, generation_path)
+        except FileExistsError:
+            shutil.rmtree(temporary, ignore_errors=True)
+        except BaseException:
+            shutil.rmtree(temporary, ignore_errors=True)
+            raise
+    pointer = {
+        "schema": "xrefkit.base_generation/v1",
+        "generation": generation_id,
+        "contracts": f"generations/{generation_id}/contracts.json",
+        "model_body": f"generations/{generation_id}/model_body.md",
+    }
+    # The pointer is the publication boundary. Fixed files remain compatibility snapshots.
+    _atomic_write_bytes(output / "contracts.json", contracts_bytes)
+    _atomic_write_bytes(output / "model_body.md", model_bytes)
+    _atomic_write_bytes(
+        output / "current.json",
+        json.dumps(pointer, ensure_ascii=False, indent=2).encode("utf-8") + b"\n",
+    )
+
+
+def _published_paths(output: Path) -> tuple[Path, Path]:
+    pointer_path = output / "current.json"
+    if not pointer_path.is_file():
+        raise ValueError("base runtime generation pointer current.json is required")
+    pointer = json.loads(pointer_path.read_text(encoding="utf-8"))
+    generation = str(pointer["generation"])
+    if not re.fullmatch(r"[0-9a-f]{16}", generation):
+        raise ValueError("invalid base runtime generation")
+    generation_path = output / "generations" / generation
+    return generation_path / "contracts.json", generation_path / "model_body.md"
+
+
+def compile_base_runtime(
+    repo_root: str | Path,
+    manifest_path: str | Path,
+    output_dir: str | Path,
+) -> dict[str, Any]:
+    result = _compile_base_runtime(repo_root, manifest_path, output_dir)
+    output = Path(output_dir)
+    if not output.is_absolute():
+        output = Path(repo_root).resolve() / output
+    generated = {key: value for key, value in result.items() if key != "model_body"}
+    _publish_generation(output, generated, result["model_body"])
+    return {key: value for key, value in result.items() if key != "model_body"}
+
+
+def _compile_base_runtime(
+    repo_root: str | Path,
+    manifest_path: str | Path,
+    output_dir: str | Path,
+) -> dict[str, Any]:
+    root = Path(repo_root).resolve()
+    manifest_file = Path(manifest_path)
+    if not manifest_file.is_absolute():
+        manifest_file = root / manifest_file
+    manifest = _load_manifest(manifest_file)
+    sources: dict[str, dict[str, Any]] = {}
+    source_texts: dict[str, str] = {}
+    for source in manifest.get("sources", []):
+        xid = str(source["xid"])
+        path = root / str(source["path"])
+        body = path.read_text(encoding="utf-8")
+        if f"xid: {xid}" not in body:
+            raise ValueError(f"source XID mismatch: {path} expected {xid}")
+        sources[xid] = {
+            "xid": xid,
+            "path": str(source["path"]),
+            "source_hash": _sha256(body.encode("utf-8")),
+        }
+        source_texts[xid] = body
+
+    obligations: list[dict[str, str]] = []
+    seen_ids: set[str] = set()
+    covered_lines: dict[str, set[str]] = {xid: set() for xid in sources}
+    for item in manifest.get("obligations", []):
+        obligation_id = str(item["id"])
+        if obligation_id in seen_ids:
+            raise ValueError(f"duplicate obligation id: {obligation_id}")
+        seen_ids.add(obligation_id)
+        source_xid = str(item["source_xid"])
+        if source_xid not in source_texts:
+            raise ValueError(f"unknown obligation source XID: {source_xid}")
+        source_match = str(item["source_match"])
+        matching = [line.strip() for line in source_texts[source_xid].splitlines() if source_match in line]
+        if not matching:
+            raise ValueError(f"obligation source_match not found: {obligation_id}")
+        covered_lines[source_xid].update(matching)
+        obligations.append(
+            {
+                "id": obligation_id,
+                "source_xid": source_xid,
+                "level": str(item["level"]),
+                "statement": str(item["statement"]),
+            }
+        )
+
+    lint_candidates: list[dict[str, Any]] = []
+    for xid, body in source_texts.items():
+        for line_number, line in enumerate(body.splitlines(), start=1):
+            stripped = line.strip()
+            if NORMATIVE_RE.search(stripped) and stripped not in covered_lines[xid]:
+                lint_candidates.append(
+                    {
+                        "source_xid": xid,
+                        "line": line_number,
+                        "text": stripped,
+                        "status": "pending",
+                    }
+                )
+
+    body_lines = ["# XRefKit Base Runtime Contract", ""]
+    for item in obligations:
+        body_lines.append(f"- [{item['level']}] {item['id']}: {item['statement']}")
+    conditional = manifest.get("conditional_loads", [])
+    if conditional:
+        body_lines.extend(["", "Conditional loads:"])
+        for item in conditional:
+            body_lines.append(f"- {item['when']}: xid {item['xid']}")
+    model_body = "\n".join(body_lines) + "\n"
+    budgets = dict(manifest.get("budgets", {}))
+    estimated_tokens = _estimate_tokens(model_body)
+    approval = dict(manifest.get("approval", {}))
+    result = {
+        "schema": GENERATED_SCHEMA,
+        "profile": manifest.get("profile", "model-compact"),
+        "compiler_version": manifest.get("compiler_version", 1),
+        "manifest_path": str(manifest_file.relative_to(root)),
+        "manifest_hash": _sha256(manifest_file.read_bytes()),
+        "approval": approval,
+        "estimator": manifest.get("estimator", {}),
+        "budgets": budgets,
+        "sources": list(sources.values()),
+        "obligations": obligations,
+        "conditional_loads": conditional,
+        "lint_candidates": lint_candidates,
+        "model_body_hash": _sha256(model_body.encode("utf-8")),
+        "estimated_tokens": estimated_tokens,
+        "release_ready": approval.get("status") == "accepted" and not lint_candidates,
+        "model_body": model_body,
+    }
+    return result
+
+
+def verify_base_runtime(
+    repo_root: str | Path,
+    manifest_path: str | Path,
+    output_dir: str | Path,
+    *,
+    release: bool,
+) -> dict[str, Any]:
+    compiled = _compile_base_runtime(repo_root, manifest_path, output_dir)
+    result = {key: value for key, value in compiled.items() if key != "model_body"}
+    errors: list[str] = []
+    generated_path = Path(output_dir)
+    if not generated_path.is_absolute():
+        generated_path = Path(repo_root).resolve() / generated_path
+    try:
+        contracts_path, model_body_path = _published_paths(generated_path)
+    except (OSError, UnicodeDecodeError, KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
+        errors.append(f"compiled generation pointer cannot be read: {exc}")
+        return {**result, "ok": False, "errors": errors, "release": release}
+    if not contracts_path.is_file() or not model_body_path.is_file():
+        errors.append("compiled runtime output is missing")
+        return {**result, "ok": False, "errors": errors, "release": release}
+    try:
+        generated = json.loads(contracts_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        errors.append(f"compiled contracts cannot be read: {exc}")
+        return {**result, "ok": False, "errors": errors, "release": release}
+    expected = json.loads(
+        json.dumps(
+            {key: value for key, value in compiled.items() if key != "model_body"},
+            ensure_ascii=False,
+            default=str,
+        )
+    )
+    if generated != expected:
+        errors.append("compiled contracts differ from current source and manifest")
+    try:
+        published_model_body = model_body_path.read_text(encoding="utf-8")
+    except (OSError, UnicodeDecodeError) as exc:
+        errors.append(f"compiled model body cannot be read: {exc}")
+        published_model_body = None
+    if published_model_body is not None and published_model_body != compiled["model_body"]:
+        errors.append("compiled model body differs from current source and manifest")
+    l0_budget = int(result["budgets"].get("l0_tokens", 0))
+    if not l0_budget or result["estimated_tokens"] > l0_budget:
+        errors.append(
+            f"L0 token budget exceeded: {result['estimated_tokens']} > {l0_budget}"
+        )
+    if release:
+        if result["approval"].get("status") != "accepted":
+            errors.append("runtime budget and manifest approval is not accepted")
+        if result["lint_candidates"]:
+            errors.append(f"pending normative lint candidates: {len(result['lint_candidates'])}")
+    return {**result, "ok": not errors, "errors": errors, "release": release}
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="xrefkit pack")
+    sub = parser.add_subparsers(dest="command", required=True)
+    for name in ("build-base", "verify-base"):
+        command = sub.add_parser(name)
+        command.add_argument("--root", default=".")
+        command.add_argument("--manifest", default="docs/core/contracts/base_runtime_manifest.yaml")
+        command.add_argument("--output", default="xrefkit/resources/base")
+        command.add_argument("--json", action="store_true")
+        if name == "verify-base":
+            command.add_argument("--draft", action="store_true")
+    args = parser.parse_args(argv)
+    try:
+        if args.command == "build-base":
+            result = compile_base_runtime(args.root, args.manifest, args.output)
+            result = {**result, "ok": True}
+        else:
+            result = verify_base_runtime(
+                args.root,
+                args.manifest,
+                args.output,
+                release=not args.draft,
+            )
+    except (OSError, ValueError, yaml.YAMLError) as exc:
+        result = {"ok": False, "errors": [str(exc)]}
+    if args.json:
+        print(json.dumps(result, ensure_ascii=False, indent=2, default=str))
+    else:
+        print("ok" if result.get("ok") else "failed")
+        for error in result.get("errors", []):
+            print(f"- {error}")
+    return 0 if result.get("ok") else 1

xrefkit/ctx.py

@@ -0,0 +1,160 @@
+from __future__ import annotations
+
+import argparse
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+from xrefkit.xref import MANAGED_FRAGMENT_RE, WIKI_XREF_RE, XrefConfig, build_index
+
+
+@dataclass(frozen=True)
+class PackedDoc:
+    xid: str
+    path: str
+    title: str | None
+    bytes_included: int
+
+
+def _extract_referenced_xids(text: str) -> set[str]:
+    xids = set(MANAGED_FRAGMENT_RE.findall(text))
+    xids.update(WIKI_XREF_RE.findall(text))
+    return xids
+
+
+def _read_text(path: Path) -> str:
+    return path.read_text(encoding="utf-8")
+
+
+def _build_pack_text(
+    *,
+    root: Path,
+    ordered_xids: list[str],
+    index,
+    per_doc_bytes: int,
+    max_bytes: int,
+) -> tuple[str, list[PackedDoc]]:
+    parts: list[str] = []
+    packed: list[PackedDoc] = []
+
+    header = [
+        "# Context Pack",
+        "",
+        f"- root: {root.as_posix()}",
+        f"- docs: {len(ordered_xids)}",
+        "",
+    ]
+    parts.append("\n".join(header))
+
+    total = sum(len(p.encode("utf-8")) for p in parts)
+    for xid in ordered_xids:
+        info = index.get(xid)
+        if info is None:
+            continue
+        rel = str(info.path.relative_to(root)).replace("\\", "/")
+        title = info.title
+
+        text = _read_text(info.path)
+        blob = text.encode("utf-8")
+        truncated = blob[:per_doc_bytes].decode("utf-8", errors="ignore")
+
+        section = [
+            "---",
+            f"## {title or xid}",
+            f"- xid: {xid}",
+            f"- path: {rel}",
+            "",
+            truncated.rstrip("\n"),
+            "",
+        ]
+        section_text = "\n".join(section)
+        section_bytes = len(section_text.encode("utf-8"))
+        if total + section_bytes > max_bytes:
+            break
+        parts.append(section_text)
+        total += section_bytes
+        packed.append(
+            PackedDoc(
+                xid=xid,
+                path=rel,
+                title=title,
+                bytes_included=len(truncated.encode("utf-8")),
+            )
+        )
+
+    return "\n".join(parts).rstrip() + "\n", packed
+
+
+def ctx_pack(
+    cfg: XrefConfig,
+    *,
+    seeds: list[str],
+    depth: int,
+    max_bytes: int,
+    per_doc_bytes: int,
+) -> dict[str, object]:
+    root = cfg.resolved_root()
+    index, issues = build_index(cfg)
+
+    visited: set[str] = set()
+    frontier: set[str] = {s for s in seeds if s in index}
+    ordered: list[str] = []
+
+    for _ in range(max(0, depth) + 1):
+        if not frontier:
+            break
+        next_frontier: set[str] = set()
+        for xid in sorted(frontier):
+            if xid in visited:
+                continue
+            visited.add(xid)
+            ordered.append(xid)
+            text = _read_text(index[xid].path)
+            for ref in _extract_referenced_xids(text):
+                if ref in index and ref not in visited:
+                    next_frontier.add(ref)
+        frontier = next_frontier
+
+    pack_text, packed_docs = _build_pack_text(
+        root=root,
+        ordered_xids=ordered,
+        index=index,
+        per_doc_bytes=per_doc_bytes,
+        max_bytes=max_bytes,
+    )
+
+    return {
+        "issues": issues,
+        "seeds": seeds,
+        "depth": depth,
+        "docs": [d.__dict__ for d in packed_docs],
+        "text": pack_text,
+    }
+
+
+def cmd_ctx(args: argparse.Namespace, cfg: XrefConfig) -> int:
+    if args.ctx_cmd != "pack":
+        raise ValueError(f"Unknown ctx command: {args.ctx_cmd}")
+
+    result = ctx_pack(
+        cfg,
+        seeds=list(args.seed or []),
+        depth=int(args.depth),
+        max_bytes=int(args.max_bytes),
+        per_doc_bytes=int(args.per_doc_bytes),
+    )
+
+    out_path = Path(args.out).resolve() if args.out else None
+    if out_path is not None:
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        out_path.write_text(result["text"], encoding="utf-8", newline="\n")
+
+    if args.json:
+        if out_path is not None:
+            result["out"] = str(out_path)
+        print(json.dumps(result, ensure_ascii=False, indent=2))
+    else:
+        print(result["text"], end="")
+
+    return 0
+