rememberstack 0.1.0__py3-none-any.whl

rememberstack/adapters/selfhost/git.py

@@ -0,0 +1,374 @@
+"""Plain local-Git remote plus D74 affected-path history erasure."""
+
+from pathlib import Path
+from pathlib import PurePosixPath
+import shlex
+import subprocess
+from typing import Protocol
+from uuid import UUID
+
+from rememberstack.model import KRevision
+
+
+class KPathCatalog(Protocol):
+    """Resolve synced owner blockers and manifest-nominated Git paths."""
+
+    def blocking_k_paths(self, *, deployment_id: UUID, doc_id: UUID) -> tuple[str, ...]:
+        """Return owner-controlled paths with standing lineage citations."""
+        ...
+
+    def k_paths_for_artifacts(
+        self, *, deployment_id: UUID, artifact_ids: tuple[UUID, ...]
+    ) -> tuple[str, ...]:
+        """Return exact current body and curation paths for affected artifacts."""
+        ...
+
+
+class LocalGitRepository:
+    """Use one local working repository as self-host Plane-K truth."""
+
+    def __init__(
+        self,
+        *,
+        repository: Path,
+        path_catalog: KPathCatalog,
+        author_name: str,
+        author_email: str,
+    ) -> None:
+        """Bind an existing clean repository and explicit commit identity."""
+        self._repository = repository.resolve(strict=True)
+        self._path_catalog = path_catalog
+        self._author_name = author_name
+        self._author_email = author_email
+        if (
+            _output(
+                arguments=(
+                    "git",
+                    "-C",
+                    str(self._repository),
+                    "rev-parse",
+                    "--is-bare-repository",
+                )
+            )
+            == "true"
+        ):
+            raise ValueError("self-host Plane-K truth must be a working repository")
+
+    def checkout(self, *, destination: Path) -> KRevision:
+        """Clone the local truth repository into one driver-owned worktree."""
+        _run(
+            arguments=(
+                "git",
+                "clone",
+                "--quiet",
+                str(self._repository),
+                str(destination),
+            )
+        )
+        return KRevision(
+            root=_output(arguments=("git", "-C", str(destination), "rev-parse", "HEAD"))
+        )
+
+    def publish(self, *, worktree: Path) -> KRevision:
+        """Commit a prepared driver tree and push its current branch to local truth."""
+        _run(arguments=("git", "-C", str(worktree), "add", "-A"))
+        if (
+            _status(
+                arguments=("git", "-C", str(worktree), "diff", "--cached", "--quiet")
+            )
+            != 0
+        ):
+            _run(
+                arguments=(
+                    "git",
+                    "-C",
+                    str(worktree),
+                    "-c",
+                    f"user.name={self._author_name}",
+                    "-c",
+                    f"user.email={self._author_email}",
+                    "commit",
+                    "--quiet",
+                    "-m",
+                    "rememberstack knowledge update",
+                )
+            )
+            _run(
+                arguments=(
+                    "git",
+                    "-C",
+                    str(self._repository),
+                    "fetch",
+                    "--quiet",
+                    str(worktree),
+                    "HEAD:refs/rememberstack/publish",
+                )
+            )
+            _run(
+                arguments=(
+                    "git",
+                    "-C",
+                    str(self._repository),
+                    "reset",
+                    "--hard",
+                    "refs/rememberstack/publish",
+                )
+            )
+            _run(
+                arguments=(
+                    "git",
+                    "-C",
+                    str(self._repository),
+                    "update-ref",
+                    "-d",
+                    "refs/rememberstack/publish",
+                )
+            )
+        return KRevision(
+            root=_output(arguments=("git", "-C", str(worktree), "rev-parse", "HEAD"))
+        )
+
+    def blocking_redaction_paths(
+        self, *, deployment_id: UUID, doc_id: UUID
+    ) -> tuple[str, ...]:
+        """Delegate to current synced K ownership/citation state."""
+        return self._path_catalog.blocking_k_paths(
+            deployment_id=deployment_id, doc_id=doc_id
+        )
+
+    def purge_artifacts(
+        self, *, deployment_id: UUID, forget_id: UUID, artifact_ids: tuple[UUID, ...]
+    ) -> None:
+        """Erase affected paths from every ref and re-add sanitized current files."""
+        current_paths = self._validated_paths(
+            paths=self._path_catalog.k_paths_for_artifacts(
+                deployment_id=deployment_id, artifact_ids=artifact_ids
+            )
+        )
+        if not current_paths or self._honored(forget_id=forget_id, paths=current_paths):
+            return
+        paths = self._historical_paths(current_paths=current_paths)
+        if _output(
+            arguments=("git", "-C", str(self._repository), "status", "--porcelain")
+        ):
+            raise RuntimeError(
+                "Plane-K repository must be clean before history erasure"
+            )
+        current = {
+            path: target.read_bytes() if target.is_file() else None
+            for path in current_paths
+            for target in (self._repository / path,)
+        }
+        removal = "git rm -r --cached --ignore-unmatch -- " + " ".join(
+            shlex.quote(path) for path in paths
+        )
+        _run(
+            arguments=(
+                "git",
+                "-C",
+                str(self._repository),
+                "filter-branch",
+                "--force",
+                "--index-filter",
+                removal,
+                "--tag-name-filter",
+                "cat",
+                "--",
+                "--all",
+            )
+        )
+        self._drop_original_refs()
+        for path, content in current.items():
+            target = self._repository / path
+            if content is None:
+                target.unlink(missing_ok=True)
+                continue
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_bytes(content)
+        _run(
+            arguments=(
+                "git",
+                "-C",
+                str(self._repository),
+                "add",
+                "-A",
+                "--",
+                *current_paths,
+            )
+        )
+        if (
+            _status(
+                arguments=(
+                    "git",
+                    "-C",
+                    str(self._repository),
+                    "diff",
+                    "--cached",
+                    "--quiet",
+                )
+            )
+            != 0
+        ):
+            _run(
+                arguments=(
+                    "git",
+                    "-C",
+                    str(self._repository),
+                    "-c",
+                    f"user.name={self._author_name}",
+                    "-c",
+                    f"user.email={self._author_email}",
+                    "commit",
+                    "--quiet",
+                    "-m",
+                    f"hard-forget {forget_id}",
+                )
+            )
+        head = _output(
+            arguments=("git", "-C", str(self._repository), "rev-parse", "HEAD")
+        )
+        _run(
+            arguments=(
+                "git",
+                "-C",
+                str(self._repository),
+                "update-ref",
+                self._ack_ref(forget_id=forget_id),
+                head,
+            )
+        )
+        _run(
+            arguments=(
+                "git",
+                "-C",
+                str(self._repository),
+                "reflog",
+                "expire",
+                "--expire=now",
+                "--all",
+            )
+        )
+        _run(arguments=("git", "-C", str(self._repository), "gc", "--prune=now"))
+
+    def verify_artifacts_purged(
+        self, *, deployment_id: UUID, forget_id: UUID, artifact_ids: tuple[UUID, ...]
+    ) -> None:
+        """Prove affected current paths have only their sanitized post-purge history."""
+        paths = self._validated_paths(
+            paths=self._path_catalog.k_paths_for_artifacts(
+                deployment_id=deployment_id, artifact_ids=artifact_ids
+            )
+        )
+        if paths and not self._honored(forget_id=forget_id, paths=paths):
+            raise RuntimeError(
+                f"Plane-K purge verification failed for forget_id {forget_id}"
+            )
+
+    def _honored(self, *, forget_id: UUID, paths: tuple[str, ...]) -> bool:
+        """Validate the store-local acknowledgement and single current path history."""
+        if (
+            _status(
+                arguments=(
+                    "git",
+                    "-C",
+                    str(self._repository),
+                    "show-ref",
+                    "--verify",
+                    "--quiet",
+                    self._ack_ref(forget_id=forget_id),
+                )
+            )
+            != 0
+        ):
+            return False
+        commits = _output(
+            arguments=(
+                "git",
+                "-C",
+                str(self._repository),
+                "log",
+                "--all",
+                "--format=%H",
+                "--",
+                *paths,
+            )
+        ).splitlines()
+        return len(set(commits)) <= 1
+
+    def _validated_paths(self, *, paths: tuple[str, ...]) -> tuple[str, ...]:
+        """Reject absolute/traversing Git paths before filesystem or shell use."""
+        result: list[str] = []
+        for value in sorted(set(paths)):
+            path = PurePosixPath(value)
+            if not value or path.is_absolute() or ".." in path.parts:
+                raise ValueError(f"unsafe Plane-K path {value!r}")
+            candidate = (self._repository / path).resolve()
+            if not candidate.is_relative_to(self._repository):
+                raise ValueError(f"Plane-K path {value!r} escapes the repository")
+            result.append(path.as_posix())
+        return tuple(result)
+
+    def _historical_paths(self, *, current_paths: tuple[str, ...]) -> tuple[str, ...]:
+        """Follow renames and return every historical name of affected files."""
+        paths = set(current_paths)
+        for current in current_paths:
+            output = _output(
+                arguments=(
+                    "git",
+                    "-C",
+                    str(self._repository),
+                    "log",
+                    "--all",
+                    "--follow",
+                    "--name-status",
+                    "--format=",
+                    "--",
+                    current,
+                )
+            )
+            for line in output.splitlines():
+                fields = line.split("\t")
+                if len(fields) >= 2:
+                    paths.update(fields[1:])
+        return self._validated_paths(paths=tuple(paths))
+
+    def _drop_original_refs(self) -> None:
+        """Delete filter-branch backup refs so forgotten history is unreachable."""
+        refs = _output(
+            arguments=(
+                "git",
+                "-C",
+                str(self._repository),
+                "for-each-ref",
+                "--format=%(refname)",
+                "refs/original/",
+            )
+        ).splitlines()
+        for ref in refs:
+            _run(
+                arguments=("git", "-C", str(self._repository), "update-ref", "-d", ref)
+            )
+
+    @staticmethod
+    def _ack_ref(*, forget_id: UUID) -> str:
+        """Return the receipt ref that disappears with an independently restored repo."""
+        return f"refs/rememberstack/forget/{forget_id}"
+
+
+def _run(*, arguments: tuple[str, ...]) -> None:
+    """Run one Git command and preserve its full failing process exception."""
+    subprocess.run(arguments, check=True, capture_output=True, text=True)
+
+
+def _output(*, arguments: tuple[str, ...]) -> str:
+    """Run one Git read and return its exact trimmed stdout."""
+    return subprocess.run(
+        arguments, check=True, capture_output=True, text=True
+    ).stdout.strip()
+
+
+def _status(*, arguments: tuple[str, ...]) -> int:
+    """Run a Git predicate whose nonzero status is an expected false result."""
+    return subprocess.run(
+        arguments, check=False, capture_output=True, text=True
+    ).returncode

rememberstack/adapters/selfhost/lance.py

@@ -0,0 +1,328 @@
+"""The embedded-LanceDB P1 chunk index: one table of text + vectors (D8)."""
+
+from datetime import timedelta
+import math
+from pathlib import Path
+from typing import cast
+from typing import Final
+from uuid import UUID
+
+import lancedb
+from lancedb.index import Bitmap
+from lancedb.index import BTree
+from lancedb.index import IvfFlat
+from lancedb.query import LanceVectorQueryBuilder
+from lancedb.table import Table
+
+from rememberstack.model import P1ChunkRow
+from rememberstack.model import P1ClaimRow
+from rememberstack.model import P1EntityRow
+from rememberstack.model import P1FactRow
+
+_CHUNK_TABLE = "chunks"
+_CLAIM_TABLE = "claims"
+_FACT_TABLE = "facts"
+_ENTITY_TABLE = "entities"
+
+LANCE_TARGET_PARTITION_ROWS: Final = 8_192
+"""WP-5.6 IVF_FLAT target: one vector partition per roughly 8k rows."""
+
+LANCE_NPROBES: Final = 20
+"""WP-5.6 query probe count for filtered ANN reads."""
+
+_MIN_VECTOR_INDEX_ROWS: Final = 256
+
+
+class LanceChunkIndex:
+    """The self-host P1 chunk table in an embedded Lance dataset directory."""
+
+    def __init__(self, *, root: Path) -> None:
+        """Bind the index to its dataset directory, creating it if absent."""
+        self._connection = lancedb.connect(str(root))
+
+    def upsert_chunks(self, *, rows: tuple[P1ChunkRow, ...]) -> None:
+        """Insert or replace rows by chunk_id; re-runs are idempotent."""
+        if not rows:
+            return
+        payload = [
+            {
+                "chunk_id": str(row.chunk_id),
+                "deployment_id": str(row.deployment_id),
+                "doc_id": str(row.doc_id),
+                "version_id": str(row.version_id),
+                "section_role": row.section_role,
+                "text": row.text,
+                "vector": list(row.vector),
+            }
+            for row in rows
+        ]
+        self._upsert(table=_CHUNK_TABLE, key="chunk_id", payload=payload)
+
+    def chunk_vectors(
+        self, *, deployment_id: str, chunk_ids: tuple[str, ...]
+    ) -> dict[str, tuple[float, ...]]:
+        """Stored vectors for the requested ids (absent ids are omitted)."""
+        deployment_id = str(UUID(deployment_id))
+        if not chunk_ids or _CHUNK_TABLE not in self._connection.table_names():
+            return {}
+        ids = ", ".join(f"'{UUID(item)}'" for item in chunk_ids)
+        rows = (
+            self._connection.open_table(_CHUNK_TABLE)
+            .search()
+            .where(f"deployment_id = '{deployment_id}' AND chunk_id IN ({ids})")
+            .limit(len(chunk_ids))
+            .to_list()
+        )
+        return {row["chunk_id"]: tuple(row["vector"]) for row in rows}
+
+    def upsert_claims(self, *, rows: tuple[P1ClaimRow, ...]) -> None:
+        """Insert or replace claims-channel rows by claim_id; idempotent."""
+        self._upsert(
+            table=_CLAIM_TABLE,
+            key="claim_id",
+            payload=[
+                {
+                    "claim_id": str(row.claim_id),
+                    "deployment_id": str(row.deployment_id),
+                    "doc_id": str(row.doc_id),
+                    "chunk_id": str(row.chunk_id),
+                    "text": row.text,
+                    "is_current_testimony": row.is_current_testimony,
+                    "is_attributed": row.is_attributed,
+                    "vector": list(row.vector),
+                }
+                for row in rows
+            ],
+        )
+
+    def upsert_facts(self, *, rows: tuple[P1FactRow, ...]) -> None:
+        """Insert or replace facts-channel rows by fact_id; idempotent."""
+        self._upsert(
+            table=_FACT_TABLE,
+            key="fact_id",
+            payload=[
+                {
+                    "fact_id": str(row.fact_id),
+                    "deployment_id": str(row.deployment_id),
+                    "kind": row.kind,
+                    "label": row.label,
+                    "status": row.status,
+                    "vector": list(row.vector),
+                }
+                for row in rows
+            ],
+        )
+
+    def search_claims(
+        self,
+        *,
+        deployment_id: str,
+        vector: tuple[float, ...],
+        k: int,
+        current_only: bool,
+    ) -> tuple[str, ...]:
+        """Nominate claim ids by vector similarity (D48: nomination, not truth).
+
+        The DEFAULT claims channel filters to current testimony via the
+        stored scalar (retrieval §5); hydration against the spine confirms.
+        """
+        deployment_id = str(UUID(deployment_id))  # refuse filter injection
+        if _CLAIM_TABLE not in self._connection.table_names():
+            return ()
+        query = (
+            cast(
+                "LanceVectorQueryBuilder",
+                self._connection.open_table(_CLAIM_TABLE)
+                .search(list(vector))
+                .where(
+                    f"deployment_id = '{deployment_id}'"
+                    + (" AND is_current_testimony" if current_only else ""),
+                    prefilter=True,
+                ),
+            )
+            .nprobes(LANCE_NPROBES)
+            .limit(k)
+        )
+        return tuple(row["claim_id"] for row in query.to_list())
+
+    def search_facts(
+        self, *, deployment_id: str, vector: tuple[float, ...], k: int, kind: str | None
+    ) -> tuple[str, ...]:
+        """Nominate fact ids (relations/observations) by label similarity."""
+        deployment_id = str(UUID(deployment_id))  # refuse filter injection
+        if kind is not None and kind not in ("relation", "observation"):
+            raise ValueError(f"unknown facts-channel kind {kind!r}")
+        if _FACT_TABLE not in self._connection.table_names():
+            return ()
+        where = f"deployment_id = '{deployment_id}'"
+        if kind is not None:
+            where += f" AND kind = '{kind}'"
+        query = (
+            cast(
+                "LanceVectorQueryBuilder",
+                self._connection.open_table(_FACT_TABLE)
+                .search(list(vector))
+                .where(where, prefilter=True),
+            )
+            .nprobes(LANCE_NPROBES)
+            .limit(k)
+        )
+        return tuple(row["fact_id"] for row in query.to_list())
+
+    def build_search_indexes(self) -> None:
+        """Build the measured scalar + IVF_FLAT indexes after a bulk load.
+
+        This is explicit rather than hidden in every upsert: index construction
+        is a maintenance/backfill operation, while inline P1 writes must stay
+        cheap. Lance still searches unindexed tail fragments after the build.
+        """
+        available = set(self._connection.list_tables().tables or ())
+        if _CLAIM_TABLE in available:
+            claims = self._connection.open_table(_CLAIM_TABLE)
+            claims.create_index("deployment_id", config=BTree())
+            claims.create_index("is_current_testimony", config=Bitmap())
+            self._build_vector_index(table=claims)
+        if _FACT_TABLE in available:
+            facts = self._connection.open_table(_FACT_TABLE)
+            facts.create_index("deployment_id", config=BTree())
+            facts.create_index("kind", config=Bitmap())
+            self._build_vector_index(table=facts)
+
+    @staticmethod
+    def _build_vector_index(*, table: Table) -> None:
+        """Build one vector index when the table is large enough to train it."""
+        rows = table.count_rows()
+        if rows < _MIN_VECTOR_INDEX_ROWS:
+            return
+        table.create_index(
+            "vector",
+            config=IvfFlat(
+                distance_type="l2",
+                num_partitions=max(1, math.ceil(rows / LANCE_TARGET_PARTITION_ROWS)),
+                target_partition_size=LANCE_TARGET_PARTITION_ROWS,
+            ),
+        )
+
+    def upsert_entities(self, *, rows: tuple[P1EntityRow, ...]) -> None:
+        """Insert or replace entity-profile rows by entity_id; idempotent."""
+        self._upsert(
+            table=_ENTITY_TABLE,
+            key="entity_id",
+            payload=[
+                {
+                    "entity_id": str(row.entity_id),
+                    "deployment_id": str(row.deployment_id),
+                    "type": row.type,
+                    "canonical_name": row.canonical_name,
+                    "vector": list(row.vector),
+                }
+                for row in rows
+            ],
+        )
+
+    def entity_vectors(
+        self, *, deployment_id: str, entity_ids: tuple[str, ...]
+    ) -> dict[str, tuple[float, ...]]:
+        """Profile vectors for the requested ids (absent ids are omitted)."""
+        deployment_id = str(UUID(deployment_id))
+        if not entity_ids or _ENTITY_TABLE not in self._connection.table_names():
+            return {}
+        ids = ", ".join(f"'{UUID(item)}'" for item in entity_ids)
+        rows = (
+            self._connection.open_table(_ENTITY_TABLE)
+            .search()
+            .where(f"deployment_id = '{deployment_id}' AND entity_id IN ({ids})")
+            .limit(len(entity_ids))
+            .to_list()
+        )
+        return {row["entity_id"]: tuple(row["vector"]) for row in rows}
+
+    def purge_rows(
+        self,
+        *,
+        deployment_id: UUID,
+        chunk_ids: tuple[UUID, ...],
+        claim_ids: tuple[UUID, ...],
+        fact_ids: tuple[UUID, ...],
+        entity_ids: tuple[UUID, ...],
+    ) -> None:
+        """Delete exact deployment-owned rows and prune obsolete Lance versions."""
+        for table, key, ids in (
+            (_CHUNK_TABLE, "chunk_id", chunk_ids),
+            (_CLAIM_TABLE, "claim_id", claim_ids),
+            (_FACT_TABLE, "fact_id", fact_ids),
+            (_ENTITY_TABLE, "entity_id", entity_ids),
+        ):
+            self._purge_table_rows(
+                table=table, key=key, deployment_id=deployment_id, ids=ids
+            )
+
+    def verify_rows_purged(
+        self,
+        *,
+        deployment_id: UUID,
+        chunk_ids: tuple[UUID, ...],
+        claim_ids: tuple[UUID, ...],
+        fact_ids: tuple[UUID, ...],
+        entity_ids: tuple[UUID, ...],
+    ) -> None:
+        """Prove no nominated UUID remains in its deployment-scoped P1 table."""
+        remaining: dict[str, int] = {}
+        for table, key, ids in (
+            (_CHUNK_TABLE, "chunk_id", chunk_ids),
+            (_CLAIM_TABLE, "claim_id", claim_ids),
+            (_FACT_TABLE, "fact_id", fact_ids),
+            (_ENTITY_TABLE, "entity_id", entity_ids),
+        ):
+            if not ids or table not in self._connection.table_names():
+                continue
+            rendered_ids = ", ".join(f"'{item}'" for item in ids)
+            count = self._connection.open_table(table).count_rows(
+                f"deployment_id = '{deployment_id}' AND {key} IN ({rendered_ids})"
+            )
+            if count:
+                remaining[table] = count
+        if remaining:
+            raise RuntimeError(f"P1 purge verification found rows: {remaining!r}")
+
+    def table_count(self, *, table: str) -> int:
+        """Total rows in one P1 table (0 before its first write)."""
+        if table not in self._connection.table_names():
+            return 0
+        return self._connection.open_table(table).count_rows()
+
+    def _upsert(
+        self, *, table: str, key: str, payload: list[dict[str, object]]
+    ) -> None:
+        """Create-or-merge one table's rows by its key column."""
+        if not payload:
+            return
+        if table not in self._connection.table_names():
+            self._connection.create_table(table, data=payload)
+            return
+        (
+            self._connection.open_table(table)
+            .merge_insert(key)
+            .when_matched_update_all()
+            .when_not_matched_insert_all()
+            .execute(payload)
+        )
+
+    def _purge_table_rows(
+        self, *, table: str, key: str, deployment_id: UUID, ids: tuple[UUID, ...]
+    ) -> None:
+        """Delete one exact UUID set and physically prune its obsolete versions."""
+        if not ids or table not in self._connection.table_names():
+            return
+        rendered_ids = ", ".join(f"'{item}'" for item in ids)
+        lance_table = self._connection.open_table(table)
+        lance_table.delete(
+            f"deployment_id = '{deployment_id}' AND {key} IN ({rendered_ids})"
+        )
+        lance_table.optimize(cleanup_older_than=timedelta(0), delete_unverified=True)
+
+    def row_count(self) -> int:
+        """Total rows in the chunk table (0 before the first write)."""
+        if _CHUNK_TABLE not in self._connection.table_names():
+            return 0
+        return self._connection.open_table(_CHUNK_TABLE).count_rows()