kbforge 0.1.0__py3-none-any.whl

kbforge/__init__.py

@@ -0,0 +1,9 @@
+"""kbforge — agent-first knowledge bases, forged from your systems of record.
+
+The production half of the Open Knowledge Format: connectors, canonicalization,
+diff, provenance, and publish. See docs/architecture.md.
+"""
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__"]

kbforge/__main__.py

@@ -0,0 +1,125 @@
+"""`python -m kbforge ...` — the walking-skeleton entry point.
+
+Connector selection and config are fully generic: the connector is resolved by
+name from the registry (built-in or entry-point-discovered), and its config comes
+from repeatable `--set KEY=VALUE` pairs. Nothing here knows a connector's config
+shape, so a third-party plugin is usable with no change to this file."""
+
+from __future__ import annotations
+
+import argparse
+from typing import cast
+
+import pluggy
+import yaml
+
+from kbforge.pipeline import (
+    Aborted,
+    ConfigError,
+    ConnectorProtocol,
+    NoOp,
+    Published,
+    PublisherProtocol,
+    run,
+)
+from kbforge.registry import build_registry
+
+
+def _connectors(pm: pluggy.PluginManager) -> dict[str, ConnectorProtocol]:
+    """name -> connector instance (a connector implements kbforge_fetch)."""
+    return {
+        p.kbforge_connector_info().name: cast(ConnectorProtocol, p)
+        for p in pm.get_plugins()
+        if hasattr(p, "kbforge_fetch")
+    }
+
+
+def _publisher(pm: pluggy.PluginManager) -> PublisherProtocol:
+    for p in pm.get_plugins():
+        if hasattr(p, "kbforge_publish"):
+            return cast(PublisherProtocol, p)
+    raise SystemExit("no publisher registered")
+
+
+def _parse_settings(pairs: list[str]) -> dict:
+    """`KEY=VALUE` pairs into a config dict; VALUE is YAML-typed so `max_commits=5`
+    is an int, `ref=HEAD` a str, and `ignore_globs=[a, b]` a list."""
+    config: dict = {}
+    for pair in pairs:
+        key, sep, raw = pair.partition("=")
+        if not sep:
+            raise ValueError(f"--set expects KEY=VALUE, got {pair!r}")
+        config[key] = yaml.safe_load(raw)
+    return config
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="kbforge")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    sub.add_parser("list", help="list available connectors")
+
+    r = sub.add_parser("run", help="run the pipeline once")
+    r.add_argument("--connector", required=True)
+    r.add_argument(
+        "--set",
+        action="append",
+        default=[],
+        dest="settings",
+        metavar="KEY=VALUE",
+        help="connector config (repeatable); values are YAML-typed",
+    )
+    r.add_argument("--mirror", required=True)
+    r.add_argument("--out", required=True)
+    r.add_argument("--state", required=True)
+    args = parser.parse_args(argv)
+
+    pm = build_registry()
+    connectors = _connectors(pm)
+
+    if args.cmd == "list":
+        for name in sorted(connectors):
+            info = connectors[name].kbforge_connector_info()
+            print(f"{name}\t{info.source_system}")
+        return 0
+
+    if args.connector not in connectors:
+        available = ", ".join(sorted(connectors)) or "(none)"
+        print(f"unknown connector {args.connector!r}; available: {available}")
+        return 2
+
+    try:
+        config = _parse_settings(args.settings)
+    except ValueError as exc:
+        print(str(exc))
+        return 2
+
+    try:
+        result = run(
+            connectors[args.connector],
+            _publisher(pm),
+            config=config,
+            mirror=args.mirror,
+            state_dir=args.state,
+            publish_config={"out_dir": args.out},
+        )
+    except ConfigError as exc:
+        print(str(exc))
+        return 2
+
+    if isinstance(result, Published):
+        print(f"Published: {result.url}")
+        return 0
+    if isinstance(result, NoOp):
+        print("NoOp: no change detected; no MR opened.")
+        return 0
+    if isinstance(result, Aborted):
+        print(f"Aborted: {len(result.failures)} validation failure(s):")
+        for f in result.failures:
+            print(f"  [{f.law}] {f.concept_path}: {f.message}")
+        return 1
+    return 2
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

kbforge/canonical.py

@@ -0,0 +1,49 @@
+"""Canonicalization: stable content hashing and the §4.3 law-1 stability check."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Callable, Sequence
+
+from kbforge.models import CanonicalDocument, RawRecord
+
+
+class StabilityError(RuntimeError):
+    """normalize() produced different canonical content for identical input."""
+
+
+def content_hash(doc: CanonicalDocument) -> str:
+    """SHA-256 over the canonical CONTENT — everything the diff must react to.
+    The anchor is excluded: `retrieved_at` is volatile (§4.3 law 2) and the
+    anchor's own content_hash would be circular."""
+    payload = {
+        "doc_id": doc.doc_id,
+        "title": doc.title,
+        "text": doc.text,
+        "structured": doc.structured,
+        "relations": sorted(doc.relations),
+        "deleted": doc.deleted,
+    }
+    # default=str keeps determinism while tolerating YAML-parsed date/datetime
+    # values (PyYAML turns bare `2024-05-01` into a date, which json can't dump).
+    blob = json.dumps(
+        payload,
+        sort_keys=True,
+        ensure_ascii=True,
+        separators=(",", ":"),
+        default=str,
+    )
+    return hashlib.sha256(blob.encode("utf-8")).hexdigest()
+
+
+def assert_stability(
+    normalize: Callable[[Sequence[RawRecord]], list[CanonicalDocument]],
+    records: Sequence[RawRecord],
+) -> None:
+    """§4.3 law 1: normalize twice over identical input, require identical content
+    hashes. A connector that fails is not deterministic and must be rejected."""
+    first = [content_hash(d) for d in normalize(records)]
+    second = [content_hash(d) for d in normalize(records)]
+    if first != second:
+        raise StabilityError("normalize() is not deterministic over identical input")

kbforge/connectors/git_commits.py

@@ -0,0 +1,150 @@
+"""A git-history connector: each commit reachable from a ref becomes one canonical
+document. No credentials, no network — reads a local repository with `git log`.
+
+Unlike the feed-less local_files connector, this one uses the cursor for real: the
+watermark is the last-synced commit SHA, so an incremental fetch returns only
+`<last_sha>..<ref>` — the first live exercise of the pipeline's incremental path.
+A commit is immutable, so its content hash never changes and a re-seen commit is a
+clean no-op."""
+
+from __future__ import annotations
+
+import subprocess
+from collections.abc import Sequence
+from datetime import datetime
+from pathlib import Path
+
+from kbforge.canonical import content_hash
+from kbforge.hookspecs import hookimpl
+from kbforge.models import (
+    CanonicalDocument,
+    ConnectorInfo,
+    Cursor,
+    FetchResult,
+    RawRecord,
+    ResourceAnchor,
+)
+
+_SYSTEM = "git_commits"
+_UNIT = "\x1f"  # field separator inside one commit's formatted record
+# sha, author name, author email, author date (ISO), committer date (ISO), subject,
+# body — body is last because it may contain newlines (records are NUL-delimited).
+_FORMAT = _UNIT.join(["%H", "%an", "%ae", "%aI", "%cI", "%s", "%b"])
+_FIELDS = 7
+
+
+def _git(repo: Path, *args: str, check: bool = True) -> str:
+    result = subprocess.run(
+        ["git", *args],
+        cwd=repo,
+        capture_output=True,
+        text=True,
+        check=check,
+    )
+    return result.stdout
+
+
+class GitCommitsConnector:
+    @hookimpl
+    def kbforge_connector_info(self) -> ConnectorInfo:
+        return ConnectorInfo(
+            name=_SYSTEM,
+            version="0.1.0",
+            source_system="git history (local repository)",
+            info_types=["commit"],
+        )
+
+    @hookimpl
+    def kbforge_validate_config(self, config: dict) -> list[str]:
+        problems: list[str] = []
+        repo = config.get("repo")
+        if not repo or not Path(repo).is_dir():
+            problems.append(f"config 'repo' is not a readable directory: {repo!r}")
+        elif not _git(Path(repo), "rev-parse", "--git-dir", check=False).strip():
+            problems.append(f"config 'repo' is not a git repository: {repo!r}")
+        max_commits = config.get("max_commits")
+        if max_commits is not None and (
+            not isinstance(max_commits, int) or isinstance(max_commits, bool)
+        ):
+            problems.append(f"config 'max_commits' must be an int: {max_commits!r}")
+        return problems
+
+    @hookimpl
+    def kbforge_fetch(self, config: dict, cursor: Cursor | None) -> FetchResult:
+        repo = Path(config["repo"])
+        ref = config.get("ref", "HEAD")
+        tip = _git(repo, "rev-parse", ref, check=False).strip()
+        if not tip:
+            # Empty repo / unknown ref: nothing to sync, watermark stays put.
+            return FetchResult(
+                records=[], cursor=Cursor(connector=_SYSTEM, payload={"ref": ref})
+            )
+
+        # cursor=None → backfill everything reachable from ref (bounded by
+        # max_commits); an existing watermark → only commits since it (`last..ref`).
+        last_sha = cursor.payload.get("last_sha") if cursor else None
+        rev_range = f"{last_sha}..{ref}" if last_sha else ref
+        log_args = ["log", rev_range, f"--format={_FORMAT}", "-z"]
+        max_commits = config.get("max_commits")
+        if not last_sha and max_commits is not None:
+            log_args.append(f"--max-count={max_commits}")
+
+        raw = _git(repo, *log_args)
+        records: list[RawRecord] = []
+        for chunk in raw.split("\x00"):
+            if not chunk.strip():
+                continue
+            parts = chunk.split(_UNIT)
+            if len(parts) < _FIELDS:
+                continue
+            sha, author, email, adate, cdate, subject, body = parts[:_FIELDS]
+            records.append(
+                RawRecord(
+                    anchor_hint={
+                        "native_id": sha,
+                        "url": None,
+                        "retrieved_at": cdate,
+                        "author": author,
+                        "author_email": email,
+                        "author_date": adate,
+                        "subject": subject,
+                    },
+                    media_type="text/x-git-commit",
+                    payload=body.encode("utf-8"),
+                )
+            )
+        return FetchResult(
+            records=records,
+            cursor=Cursor(connector=_SYSTEM, payload={"last_sha": tip, "ref": ref}),
+        )
+
+    @hookimpl
+    def kbforge_normalize(
+        self, records: Sequence[RawRecord]
+    ) -> list[CanonicalDocument]:
+        docs: list[CanonicalDocument] = []
+        for rec in records:
+            hint = rec.anchor_hint
+            sha = hint["native_id"]
+            anchor = ResourceAnchor(
+                system=_SYSTEM,
+                native_id=sha,
+                url=hint.get("url"),
+                retrieved_at=datetime.fromisoformat(hint["retrieved_at"]),
+                content_hash="",
+            )
+            doc = CanonicalDocument(
+                anchor=anchor,
+                doc_id=f"{_SYSTEM}:{sha}",
+                title=hint.get("subject") or sha[:12],
+                text=rec.payload.decode("utf-8").strip(),
+                structured={
+                    "author": hint.get("author"),
+                    "author_email": hint.get("author_email"),
+                    "author_date": hint.get("author_date"),
+                },
+                relations=[],
+            )
+            doc.anchor.content_hash = content_hash(doc)
+            docs.append(doc)
+        return docs

kbforge/connectors/local_files.py

@@ -0,0 +1,175 @@
+"""A fixture connector: a folder of markdown-with-frontmatter → canonical docs.
+No credentials, no network — the deterministic source for the walking skeleton."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from datetime import UTC, datetime
+from fnmatch import fnmatch
+from pathlib import Path
+
+import yaml
+
+from kbforge.canonical import content_hash
+from kbforge.hookspecs import hookimpl
+from kbforge.models import (
+    CanonicalDocument,
+    ConnectorInfo,
+    Cursor,
+    FetchResult,
+    RawRecord,
+    ResourceAnchor,
+)
+
+_SYSTEM = "local_files"
+# Keys handled structurally, so they never leak into `structured` (hence facets):
+# title → the concept title; relations → cross-links; type is dropped here because
+# the OKF type comes from synthesis taxonomy (the stub emits "concept"); description,
+# timestamp, resource, and links are emit-side OKF fields the synthesizer owns — a
+# source key of the same name must not collide with them in the rendered frontmatter.
+_RESERVED_KEYS = frozenset(
+    {"type", "title", "relations", "description", "timestamp", "resource", "links"}
+)
+# Dependency, VCS, and tool-cache directories that a blind rglob would sweep into
+# the KB (a real live test pulled 98 vendored `.venv` docs of 132 total). These
+# always apply; a source's `ignore_globs` config ADDS to them, never replaces them —
+# so adding one custom pattern can't silently re-enable `.venv`.
+_DEFAULT_IGNORES = frozenset(
+    {
+        ".git",
+        ".venv",
+        "venv",
+        "node_modules",
+        ".pytest_cache",
+        "__pycache__",
+        ".mypy_cache",
+        ".ruff_cache",
+        ".ipynb_checkpoints",
+    }
+)
+
+
+def _is_ignored(rel: str, patterns: frozenset[str]) -> bool:
+    """True if the posix-relative path is excluded. A pattern matches either any
+    single path segment (a bare dir name like `.venv` skips the whole subtree) or
+    the full relative path (a glob like `drafts/*` or `_draft*`)."""
+    segments = rel.split("/")
+    return any(
+        fnmatch(rel, pat) or any(fnmatch(seg, pat) for seg in segments)
+        for pat in patterns
+    )
+
+
+def _split_frontmatter(text: str) -> tuple[dict, str]:
+    if not text.startswith("---"):
+        return {}, text
+    _, _, rest = text.partition("---")
+    front_raw, sep, body = rest.partition("\n---")
+    if not sep:
+        return {}, text
+    try:
+        data = yaml.safe_load(front_raw) or {}
+    except yaml.YAMLError:
+        # Malformed frontmatter (e.g. an unquoted colon) must not crash the whole
+        # sync: drop the unparseable frontmatter, keep the clean body.
+        data = {}
+    front = data if isinstance(data, dict) else {}
+    return front, body.lstrip("\n")
+
+
+class LocalFilesConnector:
+    @hookimpl
+    def kbforge_connector_info(self) -> ConnectorInfo:
+        return ConnectorInfo(
+            name=_SYSTEM,
+            version="0.1.0",
+            source_system="local filesystem (fixture)",
+            info_types=["fixture"],
+        )
+
+    @hookimpl
+    def kbforge_validate_config(self, config: dict) -> list[str]:
+        problems: list[str] = []
+        path = config.get("path")
+        if not path or not Path(path).is_dir():
+            problems.append(f"config 'path' is not a readable directory: {path!r}")
+        ignore = config.get("ignore_globs")
+        if ignore is not None and (
+            not isinstance(ignore, list) or not all(isinstance(g, str) for g in ignore)
+        ):
+            problems.append(
+                f"config 'ignore_globs' must be a list of strings: {ignore!r}"
+            )
+        return problems
+
+    @hookimpl
+    def kbforge_fetch(self, config: dict, cursor: Cursor | None) -> FetchResult:
+        # cursor is unused: this feed-less source always re-scans. The mirror diff
+        # detects adds and modifies; it does NOT derive deletions — a full-scan
+        # source can't tell a deleted file from an absent one, so a removed file
+        # leaves a stale concept until a tombstone / `complete`-aware diff lands (a
+        # later increment; `FetchResult.complete` is defined but not yet consumed).
+        # retrieved_at is stamped here (fetch may use a clock; normalize may not)
+        # from file mtime, keeping runs reproducible.
+        root = Path(config["path"])
+        ignores = _DEFAULT_IGNORES | frozenset(config.get("ignore_globs") or [])
+        records: list[RawRecord] = []
+        for path in sorted(root.rglob("*.md")):
+            rel = path.relative_to(root).as_posix()
+            if _is_ignored(rel, ignores):
+                continue
+            mtime = datetime.fromtimestamp(path.stat().st_mtime, tz=UTC)
+            records.append(
+                RawRecord(
+                    anchor_hint={
+                        "native_id": rel,
+                        "url": None,
+                        "retrieved_at": mtime.isoformat(),
+                    },
+                    media_type="text/markdown",
+                    payload=path.read_bytes(),
+                )
+            )
+        return FetchResult(records=records, cursor=Cursor(connector=_SYSTEM))
+
+    @hookimpl
+    def kbforge_normalize(
+        self, records: Sequence[RawRecord]
+    ) -> list[CanonicalDocument]:
+        docs: list[CanonicalDocument] = []
+        for rec in records:
+            # utf-8-sig strips a BOM (Windows editors) that would otherwise defeat
+            # the `startswith("---")` check; line-ending normalization keeps CRLF
+            # and LF copies of the same content hashing identically (§4.3 law 1).
+            text = (
+                rec.payload.decode("utf-8-sig")
+                .replace("\r\n", "\n")
+                .replace("\r", "\n")
+            )
+            front, body = _split_frontmatter(text)
+            native_id = rec.anchor_hint["native_id"]
+            doc_id = f"{_SYSTEM}:{native_id}"
+            relations = sorted(
+                f"{_SYSTEM}:{r}"
+                for r in front.get("relations", [])
+                if isinstance(r, str)
+            )
+            structured = {k: v for k, v in front.items() if k not in _RESERVED_KEYS}
+            anchor = ResourceAnchor(
+                system=_SYSTEM,
+                native_id=native_id,
+                url=rec.anchor_hint.get("url"),
+                retrieved_at=datetime.fromisoformat(rec.anchor_hint["retrieved_at"]),
+                content_hash="",
+            )
+            doc = CanonicalDocument(
+                anchor=anchor,
+                doc_id=doc_id,
+                title=str(front.get("title") or native_id),
+                text=body.strip(),
+                structured=structured,
+                relations=relations,
+            )
+            doc.anchor.content_hash = content_hash(doc)
+            docs.append(doc)
+        return docs

kbforge/hookspecs.py

@@ -0,0 +1,68 @@
+"""Pluggy hookspecs. The connector and publisher interfaces ARE the product (§5).
+Kept minimal for the walking skeleton: one connector family, one publisher family."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Sequence
+
+import pluggy
+
+from kbforge.models import (
+    CanonicalDocument,
+    ConnectorInfo,
+    Cursor,
+    FetchResult,
+    ProposedChange,
+    RawRecord,
+)
+
+PROJECT = "kbforge"
+hookspec = pluggy.HookspecMarker(PROJECT)
+hookimpl = pluggy.HookimplMarker(PROJECT)
+
+# Entry-point groups a third-party distribution advertises to be discovered. Kept
+# separate so a package declares intent: a connector plugin vs a publisher plugin.
+CONNECTOR_ENTRYPOINTS = "kbforge.connectors"
+PUBLISHER_ENTRYPOINTS = "kbforge.publishers"
+
+
+class ConnectorSpec(ABC):
+    """One plugin object per system of record. Connectors never see the bundle,
+    never call the LLM, never touch git (§4.1)."""
+
+    @hookspec
+    @abstractmethod
+    def kbforge_connector_info(self) -> ConnectorInfo:
+        """Static self-description."""
+
+    @hookspec
+    @abstractmethod
+    def kbforge_validate_config(self, config: dict) -> list[str]:
+        """Return human-readable problems ([] = ok). No network I/O."""
+
+    @hookspec
+    @abstractmethod
+    def kbforge_fetch(self, config: dict, cursor: Cursor | None) -> FetchResult:
+        """Pull raw records (cursor=None = full backfill / bootstrap)."""
+
+    @hookspec
+    @abstractmethod
+    def kbforge_normalize(
+        self, records: Sequence[RawRecord]
+    ) -> list[CanonicalDocument]:
+        """Deterministic, volatile-free, clock-free (§4.3)."""
+
+
+class PublisherSpec(ABC):
+    """Where proposals go. MUST NOT merge (§5.2)."""
+
+    @hookspec
+    @abstractmethod
+    def kbforge_publisher_info(self) -> ConnectorInfo:
+        """Static self-description."""
+
+    @hookspec
+    @abstractmethod
+    def kbforge_publish(self, change: ProposedChange, config: dict) -> str:
+        """Open a review request; return its URL/path. Never merges."""

kbforge/mirror.py

@@ -0,0 +1,60 @@
+"""The canonical mirror and the read-only diff (architecture §7's mirror_and_diff,
+split into a pure `diff` and a success-only `commit`)."""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+
+from kbforge.models import CanonicalDocument, ChangeSet
+
+
+def _slot(mirror: Path, doc_id: str) -> Path:
+    key = hashlib.sha256(doc_id.encode("utf-8")).hexdigest()
+    return mirror / f"{key}.json"
+
+
+def _load(mirror: Path, doc_id: str) -> CanonicalDocument | None:
+    slot = _slot(mirror, doc_id)
+    if not slot.exists():
+        return None
+    return CanonicalDocument.model_validate_json(slot.read_text("utf-8"))
+
+
+def diff(mirror: Path, docs: list[CanonicalDocument]) -> ChangeSet:
+    """Read-only comparison against the mirror. Deletions are explicit tombstones
+    (`deleted=True`); absence never implies one (§4.2). Never mutates the mirror."""
+    added: list[str] = []
+    modified: list[str] = []
+    removed: list[str] = []
+    unchanged = 0
+    for doc in docs:
+        prev = _load(mirror, doc.doc_id)
+        if doc.deleted:
+            if prev is not None:
+                removed.append(doc.doc_id)
+            continue
+        if prev is None:
+            added.append(doc.doc_id)
+        elif prev.anchor.content_hash != doc.anchor.content_hash:
+            modified.append(doc.doc_id)
+        else:
+            unchanged += 1
+    return ChangeSet(
+        added=sorted(added),
+        modified=sorted(modified),
+        removed=sorted(removed),
+        unchanged_count=unchanged,
+    )
+
+
+def commit(mirror: Path, docs: list[CanonicalDocument]) -> None:
+    """Advance the mirror to the fetched state. Called only after a run fully
+    succeeds, so a failed publish never leaves the mirror ahead of the bundle."""
+    mirror.mkdir(parents=True, exist_ok=True)
+    for doc in docs:
+        slot = _slot(mirror, doc.doc_id)
+        if doc.deleted:
+            slot.unlink(missing_ok=True)
+        else:
+            slot.write_text(doc.model_dump_json(), "utf-8")