zikaron 0.1.0__py3-none-any.whl

zikaron/core/knowledge/vectors.py

@@ -0,0 +1,102 @@
+"""`chunks_vec` maintenance and the batched embedding that fills it.
+
+**A vector is addressed by `chunk_id`, and there is no `rowid` here to fall back on.** The table is
+declared with an explicit `chunk_id INTEGER PRIMARY KEY`, and a `vec0` table declared that way
+exposes no `rowid` column at all — measured: selecting one raises `no such column`. That is the
+opposite of the memory store's vector table, which has no key column and is addressed by `rowid`, so
+the resemblance between the two is a trap rather than a shortcut.
+
+**Nothing cascades.** `vec0` takes no foreign key, so a vector outlives the chunk row it belongs to
+unless something deletes it explicitly — and an unidentified vector is a silent false positive in
+retrieval, matching a query and resolving to nothing. The per-file transaction in `writes.py` is the
+only thing that keeps the two in step.
+"""
+
+import asyncio
+from collections.abc import Sequence
+
+import aiosqlite
+
+from zikaron.core.errors import ErrorCode, IndexStage, ZikaronError
+from zikaron.core.indexing.encoder import Encoder
+from zikaron.core.indexing.vectors import normalize, serialize
+from zikaron.core.knowledge.chunking import FileChunkPlan
+
+
+def _reject_embed() -> ZikaronError:
+    return ZikaronError(ErrorCode.INDEX_FAILED, stage=IndexStage.EMBED)
+
+
+async def embed_chunks(
+    plan: FileChunkPlan, *, encoder: Encoder, embed_dim: int, batch_size: int
+) -> tuple[bytes, ...]:
+    """Embed every chunk of one file and return the vectors ready for `chunks_vec`.
+
+    Runs **outside** any transaction, deliberately: inference is the expensive step of a build by an
+    order of magnitude, and holding SQLite's single write lock across it would make every concurrent
+    reader's `busy_timeout` a function of model speed. Nothing is staged at this point, so a failure
+    here loses nothing.
+
+    **Batched, and the batch size may not change the answer.** Splitting the file's chunks into
+    passes of `batch_size` is a throughput decision; a chunk's vector must be the same whether it
+    was embedded alone or beside thirty-one others, which holds when pooling is mask-aware and
+    fails silently when it is not — the failure being an embedding that depends on what else
+    happened to be in the batch.
+
+    Each pass goes through a worker thread because inference is blocking and CPU-bound, and this is
+    a coroutine: running it inline would hold the event loop for the whole pass.
+
+    Args:
+        plan: the chunks to embed, in `part_index` order, carrying the prefix their sequences were
+            budgeted against — which is why this takes no path of its own to prepend.
+        encoder: the artifact, whose identity the caller has already checked against the corpus.
+        embed_dim: the width this knowledge base's `chunks_vec` column was created at.
+        batch_size: `knowledge_embed_batch`.
+
+    Returns:
+        One serialized unit-length vector per chunk, in `part_index` order.
+
+    Raises:
+        ZikaronError: `INDEX_FAILED` at stage `embed` if the embedder raises, returns a different
+            number of vectors than it was given texts, or returns one of the wrong width — each of
+            which would otherwise be stored as a vector that does not describe its chunk.
+    """
+    texts = plan.embedded_texts()
+    produced: list[tuple[float, ...]] = []
+    for start in range(0, len(texts), batch_size):
+        batch = texts[start : start + batch_size]
+        produced.extend(await _embed_one_pass(batch, encoder=encoder))
+    return tuple(serialize(normalize(vector, embed_dim=embed_dim)) for vector in produced)
+
+
+async def _embed_one_pass(
+    texts: Sequence[str], *, encoder: Encoder
+) -> tuple[tuple[float, ...], ...]:
+    """One forward pass, with the embedder's own failures named as this layer's."""
+    try:
+        produced = await asyncio.to_thread(encoder.embed, texts)
+    except ZikaronError:
+        raise
+    except Exception as error:
+        raise _reject_embed() from error
+    if len(produced) != len(texts):
+        raise _reject_embed()
+    return produced
+
+
+async def insert(db: aiosqlite.Connection, *, chunk_id: int, vector: bytes) -> None:
+    """Store one chunk's vector under the chunk's own id."""
+    await db.execute(
+        "INSERT INTO chunks_vec (chunk_id, embedding) VALUES (?, ?)", (chunk_id, vector)
+    )
+
+
+async def delete(db: aiosqlite.Connection, *, chunk_ids: Sequence[int]) -> None:
+    """Remove the vectors for `chunk_ids`, which may be empty.
+
+    One statement per id rather than an `IN` list: `vec0` is a virtual table, and keeping its
+    deletes to the primary-key form the extension documents is worth more than saving round trips on
+    a set bounded by one file's chunk count.
+    """
+    for chunk_id in chunk_ids:
+        await db.execute("DELETE FROM chunks_vec WHERE chunk_id = ?", (chunk_id,))

zikaron/core/knowledge/walk.py

@@ -0,0 +1,264 @@
+"""The filesystem walk: which files a corpus root offers, before git is asked anything about them.
+
+**The walk is the sole authority on which files exist.** Filtering is deny-by-default at the
+directory level and runs in a fixed order, because the order decides which reason a file's absence
+is reported under and a report nobody can reproduce is worse than no report:
+
+1. **Directory pruning**, at the directory node *before descending*. A named build or tool
+   directory, or any directory whose name begins with a dot. The name `.git` is pruned whether it
+   is a directory or a **file** — a linked worktree's and a submodule's are files holding a
+   pointer, and directory pruning never sees those.
+2. **Symlinks are not followed**, directory or file. A symlink is one refusal, whatever it points
+   at: refusing to follow it is precisely declining to find out.
+3. **`.gitignore`**, and 4. **globs**, and 5. **the size cap** — the first of which needs an answer
+   from git, so all three are applied in one pass after the walk rather than inside it.
+6. **A name-ending deny-list**, the walk-time half of text detection.
+
+Steps 1, 2 and the `stat` for step 5 happen during the walk; steps 3 to 6 are applied by `admitted`
+to what the walk found.
+
+**What this module never does is read a file.** The whole-file UTF-8 decode that settles text
+detection happens on the first read of a candidate's bytes in a scan, which is either the walk
+phase's hashing read or the index phase's — running it here would read every byte of every
+candidate on every scan and collapse the change-detection fast path into the read-everything path
+it exists to beat.
+"""
+
+import fnmatch
+import os
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final
+
+from zikaron.core.knowledge import text
+from zikaron.core.knowledge.counters import ScanCounters, SkipReason
+
+#: Directories never descended into. Several of these also begin with a dot and would be pruned by
+#: that rule alone; they are named anyway, because the list says what it is *for* — version-control
+#: metadata, dependency trees, build output and tool caches — and would survive the dot rule being
+#: narrowed.
+PRUNED_DIRECTORY_NAMES: Final = frozenset(
+    {
+        ".git",
+        ".hg",
+        ".svn",
+        "node_modules",
+        ".venv",
+        "venv",
+        "__pycache__",
+        "target",
+        "dist",
+        "build",
+        ".next",
+        ".tox",
+        ".mypy_cache",
+        ".pytest_cache",
+    }
+)
+
+#: The one name pruned as a file as well as a directory. Every other dot-file is admitted.
+_GIT_DIRECTORY_NAME: Final = ".git"
+
+
+@dataclass(frozen=True, slots=True)
+class WalkRules:
+    """What one corpus's own configuration says about which files it wants.
+
+    Held together rather than passed as three arguments because they are read together, in a fixed
+    order, and a caller that supplied two of the three would silently index a different corpus.
+    """
+
+    include_globs: tuple[str, ...] = ()
+    exclude_globs: tuple[str, ...] = ()
+    max_file_bytes: int = 0
+
+
+@dataclass(frozen=True, slots=True)
+class Candidate:
+    """One file the walk found, with what it cost nothing to learn while finding it.
+
+    `path` is relative to the corpus root in POSIX form, and is the join key against everything
+    else in the system: the `files` table, git's answers, and the globs. It is matched byte-exactly
+    everywhere, so where a filesystem's casing differs from git's the two simply fail to meet and
+    the file falls to the slower path — never to the wrong answer.
+    """
+
+    path: str
+    absolute: Path
+    size: int
+
+
+def is_pruned_directory(name: str) -> bool:
+    """Whether a directory of this name is one the walk refuses to descend into."""
+    return name in PRUNED_DIRECTORY_NAMES or name.startswith(".")
+
+
+def _relative(prefix: str, name: str) -> str:
+    return f"{prefix}/{name}" if prefix else name
+
+
+def _is_directory(entry: os.DirEntry[str]) -> bool:
+    """Whether this entry is a directory in its own right, never through a symlink.
+
+    A failed `stat` answers *no*, which routes the entry to the file path where it is refused and
+    counted — the one place with a number to report it under.
+    """
+    try:
+        return entry.is_dir(follow_symlinks=False)
+    except OSError:
+        return False
+
+
+def _is_symlink(entry: os.DirEntry[str]) -> bool:
+    """Whether this entry is a symlink. A failed `stat` answers *no*, and the entry is then
+    refused as unreadable a few lines later — which is the more accurate of the two words for an
+    entry whose type could not be determined."""
+    try:
+        return entry.is_symlink()
+    except OSError:
+        return False
+
+
+def _child_candidate(entry: os.DirEntry[str], relative_path: str) -> Candidate | None:
+    """One directory entry as a candidate, or `None` if it is not a readable regular file."""
+    try:
+        if not entry.is_file(follow_symlinks=False):
+            return None
+        size = entry.stat(follow_symlinks=False).st_size
+    except OSError:
+        return None
+    return Candidate(path=relative_path, absolute=Path(entry.path), size=size)
+
+
+def entries(root: Path, counters: ScanCounters) -> Iterator[Candidate]:
+    """Every file under `root` that the directory-level rules admit, depth-first and name-ordered.
+
+    Ordered because the order is observable: it fixes which files a partially-completed scan has
+    committed, and an unordered walk makes two runs over one tree incomparable.
+
+    Counts as it goes, and the two it can only count here are the two nothing downstream will ever
+    see: a symlink, which is refused whatever it points at, and an entry that could not be taken as
+    a readable regular file — a device or a pipe, a file whose `stat` failed, or a **directory that
+    could not be listed**. That last one is counted among the skipped files deliberately: the walk
+    cannot know how many files it did not see, so it records the one refusal it can rather than
+    letting the subtree vanish silently. Everything counted here is also counted as seen, so no
+    skip is ever attributed to an entry the walk did not reach.
+
+    Args:
+        root: the corpus root, already resolved.
+        counters: updated in place as the walk proceeds. Mutated rather than returned because the
+            caller flushes it to the database periodically, which is the only evidence a scan is
+            progressing before it has worked out what changed.
+    """
+    stack: list[tuple[Path, str]] = [(root, "")]
+    while stack:
+        directory, prefix = stack.pop()
+        try:
+            with os.scandir(directory) as scanning:
+                found = sorted(scanning, key=lambda entry: entry.name)
+        except OSError:
+            counters.files_seen += 1
+            counters.skip(SkipReason.UNREADABLE)
+            continue
+        descend: list[tuple[Path, str]] = []
+        for entry in found:
+            yield from _visit(entry, prefix, counters, descend)
+        # Reversed, because the stack is popped from the end and the names were sorted ascending.
+        stack.extend(reversed(descend))
+
+
+def _visit(
+    entry: os.DirEntry[str],
+    prefix: str,
+    counters: ScanCounters,
+    descend: list[tuple[Path, str]],
+) -> Iterator[Candidate]:
+    """Classify one directory entry, counting it and queueing it for descent as appropriate."""
+    if entry.name == _GIT_DIRECTORY_NAME:
+        # Pruned whether it is a directory or the pointer file a linked worktree and a submodule
+        # carry. Only the directory shape is counted: the counter it feeds is reported as a number
+        # of directories, and a repository's own control file is not a corpus file whose absence
+        # anyone would ask about.
+        if _is_directory(entry):
+            counters.pruned_directories += 1
+        return
+    if _is_directory(entry):
+        if is_pruned_directory(entry.name):
+            counters.pruned_directories += 1
+        else:
+            descend.append((Path(entry.path), _relative(prefix, entry.name)))
+        return
+    # Everything that is not a directory in its own right is evaluated as a file, symlinks to
+    # directories included: not following one means never learning what it points at.
+    counters.files_seen += 1
+    if _is_symlink(entry):
+        counters.skip(SkipReason.SYMLINK)
+        return
+    candidate = _child_candidate(entry, _relative(prefix, entry.name))
+    if candidate is None:
+        counters.skip(SkipReason.UNREADABLE)
+        return
+    yield candidate
+
+
+def _matches_any(path: str, patterns: Sequence[str]) -> bool:
+    """Whether the relative POSIX path matches any pattern, case-sensitively.
+
+    `fnmatch`'s `*` crosses `/`, which is what makes `*.md` reach `docs/a.md` and `docs/*` mean
+    the whole tree under `docs` rather than its direct children. Case-sensitive because this path
+    is the same byte-exact string the index stores, so a pattern means one thing to the walk and
+    to anyone reading the index back.
+
+    The one surprise, since it is otherwise met as an empty corpus: **`**` carries no meaning of
+    its own here**. It is two stars, so `docs/**/*.md` asks for a file two directories down and
+    gets exactly that, where a reader used to git's globbing expects the whole tree. It fails
+    closed rather than quietly wide, and `docs/*` is the pattern that means what they wanted.
+    """
+    return any(fnmatch.fnmatchcase(path, pattern) for pattern in patterns)
+
+
+def admitted(
+    found: Sequence[Candidate],
+    *,
+    ignored: frozenset[str],
+    rules: WalkRules,
+    counters: ScanCounters,
+) -> list[Candidate]:
+    """The candidates that survive `.gitignore`, the globs, the size cap and the deny-list.
+
+    Applied in that order, and the order is the reported answer rather than an optimisation: a
+    file that is both ignored and over the cap is reported as ignored, and two implementations
+    disagreeing about which would give two different accounts of one corpus.
+
+    Args:
+        found: what the walk yielded, in its order, which is preserved.
+        ignored: the paths `.gitignore` excludes. Empty where git was not consulted, which is the
+            same thing as applying no ignore rules.
+        rules: this corpus's globs and size cap.
+        counters: updated in place with one skip per refused file.
+    """
+    kept: list[Candidate] = []
+    for candidate in found:
+        reason = _refusal(candidate, ignored=ignored, rules=rules)
+        if reason is None:
+            kept.append(candidate)
+        else:
+            counters.skip(reason)
+    return kept
+
+
+def _refusal(
+    candidate: Candidate, *, ignored: frozenset[str], rules: WalkRules
+) -> SkipReason | None:
+    if candidate.path in ignored:
+        return SkipReason.GITIGNORED
+    if _matches_any(candidate.path, rules.exclude_globs):
+        return SkipReason.EXCLUDED_BY_GLOB
+    if rules.include_globs and not _matches_any(candidate.path, rules.include_globs):
+        return SkipReason.EXCLUDED_BY_GLOB
+    if candidate.size > rules.max_file_bytes:
+        return SkipReason.OVER_SIZE_CAP
+    if text.extension_denied(candidate.absolute.name):
+        return SkipReason.DENIED_EXTENSION
+    return None

zikaron/core/knowledge/writes.py

@@ -0,0 +1,127 @@
+"""One file's place in the index, changed or removed in a single transaction.
+
+**Atomicity is per file, not per corpus.** A build is thousands of independent transactions, which
+buys two things at once: a search running alongside it sees whole files or no files, never a
+half-indexed one; and progress is durable, so an interrupted build resumes rather than restarting.
+
+**Three tables have to move together and none of them will do it on its own.** `chunks_fts` is
+external-content, so it observes no delete on `chunks`; `chunks_vec` is a virtual table that takes
+no foreign key. Nothing here is a cascade, and the word does not apply: every deletion is issued by
+hand, in an order chosen so that the values the lexical arm needs are still readable when it needs
+them.
+
+The caller owns the transaction. These functions stage work and never commit, so a file's chunks,
+its `files` row, and whatever else that caller is doing in the same breath — clearing its pending
+row, advancing its counters — commit together or not at all.
+"""
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+import aiosqlite
+
+from zikaron.core.errors import ErrorCode, IndexStage, ZikaronError
+from zikaron.core.knowledge import files, lexical, vectors
+from zikaron.core.knowledge.chunking import FileChunkPlan
+
+_CHUNK_COLUMNS = "path, part_index, start_line, end_line, text"
+
+
+@dataclass(frozen=True, slots=True)
+class _StoredChunk:
+    """One chunk as it stands in the index, with exactly what removing it requires: its id, and the
+    two column values the lexical arm was given when it was indexed."""
+
+    chunk_id: int
+    document: lexical.Document
+
+
+async def _stored_chunks(db: aiosqlite.Connection, path: str) -> tuple[_StoredChunk, ...]:
+    """Every chunk currently indexed for `path`, read before anything is deleted.
+
+    Read first because the lexical arm's delete needs the values it was given at index time, and
+    they exist only in the rows about to be removed.
+    """
+    rows = await db.execute_fetchall(
+        "SELECT id, text, path FROM chunks WHERE path = ? ORDER BY part_index", (path,)
+    )
+    return tuple(
+        _StoredChunk(
+            chunk_id=int(chunk_id),
+            # Both values come out of the row rather than from this function's own argument, even
+            # though the path is what was searched for: what the delete needs is what was *indexed*,
+            # and reading it back is what makes that true by construction instead of by reasoning.
+            document=lexical.Document(text=str(text), path=str(indexed_path)),
+        )
+        for chunk_id, text, indexed_path in rows
+    )
+
+
+async def _clear(db: aiosqlite.Connection, path: str) -> None:
+    """Remove every trace of `path` from all three tables, in the one order that works.
+
+    Postings first, while the rows that say what was indexed are still there — the lexical delete
+    takes the values it was given at index time, and given anything else it removes nothing and says
+    nothing. Then vectors, then the chunk rows: vectors before their rows is the memory store's
+    order too, and for the same reason. A chunk row with no vector is detectable by a join and
+    repairable by reindexing, where a vector with no chunk row matches a query and resolves to
+    nothing.
+    """
+    stored = await _stored_chunks(db, path)
+    for chunk in stored:
+        await lexical.remove(db, chunk_id=chunk.chunk_id, document=chunk.document)
+    await vectors.delete(db, chunk_ids=[chunk.chunk_id for chunk in stored])
+    await db.execute("DELETE FROM chunks WHERE path = ?", (path,))
+
+
+async def index_file(
+    db: aiosqlite.Connection,
+    *,
+    indexed: files.IndexedFile,
+    plan: FileChunkPlan,
+    embeddings: Sequence[bytes],
+) -> None:
+    """Replace everything the index holds for one file with this plan and these vectors.
+
+    Replace rather than update: change detection tells us the file's content moved, and there is no
+    correspondence between its old chunks and its new ones to preserve. A file the corpus has never
+    seen takes the same path, since clearing a path with nothing under it removes nothing.
+
+    Args:
+        indexed: the `files` row to write, whose `chunk_count` is this plan's chunk count — the two
+            are compared here rather than trusted, since a row claiming a count the chunks do not
+            match is the one inconsistency no later reader can detect.
+        plan: the chunks to store, in `part_index` order.
+        embeddings: one serialized vector per chunk, in the same order.
+
+    Raises:
+        ZikaronError: `INDEX_FAILED` at stage `index_write` if the vectors, the plan and the row's
+            count do not describe the same set of chunks, or if a chunk insert reports no id to pair
+            its vector with.
+    """
+    if len(embeddings) != plan.n_chunks or indexed.chunk_count != plan.n_chunks:
+        raise ZikaronError(ErrorCode.INDEX_FAILED, stage=IndexStage.INDEX_WRITE)
+    await _clear(db, indexed.path)
+    for chunk, vector in zip(plan.chunks, embeddings, strict=True):
+        cursor = await db.execute(
+            f"INSERT INTO chunks ({_CHUNK_COLUMNS}) VALUES (?, ?, ?, ?, ?)",  # noqa: S608 — a source-level column list; every value is bound.
+            (indexed.path, chunk.part_index, chunk.start_line, chunk.end_line, chunk.text),
+        )
+        chunk_id = cursor.lastrowid
+        if chunk_id is None:  # pragma: no cover — SQLite reports an id for every successful insert.
+            raise ZikaronError(ErrorCode.INDEX_FAILED, stage=IndexStage.INDEX_WRITE)
+        document = lexical.Document(text=chunk.text, path=indexed.path)
+        await lexical.insert(db, chunk_id=chunk_id, document=document)
+        await vectors.insert(db, chunk_id=chunk_id, vector=vector)
+    await files.record(db, indexed)
+
+
+async def forget_file(db: aiosqlite.Connection, *, path: str) -> None:
+    """Remove one file from the index entirely: its chunks, both arms, and its `files` row.
+
+    One function rather than two calls at each site, because "a chunk's path always names a row in
+    `files`" is maintained by nobody else: a caller that removed the row and left the chunks would
+    leave fragments pointing at a file the index no longer claims to hold, and nothing would raise.
+    """
+    await _clear(db, path)
+    await files.forget(db, path)

zikaron/core/records/__init__.py

@@ -0,0 +1 @@
+"""Memory rows: versioning, soft retirement, supersession edges, and read receipts."""