zikaron 0.1.0__py3-none-any.whl

zikaron/core/knowledge/lifecycle.py

@@ -0,0 +1,418 @@
+"""The verbs that change a knowledge base: add, rename, refresh, unlock, remove.
+
+Looking at one without changing it is `reporting.py`'s half, and the two are separated on exactly
+that line: everything here either writes, or decides whether a write may proceed.
+
+**The registry is always mutated first**, for both `add` and `remove` — the same ordering, not
+mirrored ones. The rule is to leave the recoverable state where one exists and the lesser harm
+where none does, and the two interruptions are not symmetric, so one ordering satisfies both:
+
+- An interrupted `add` leaves a **row without a file**: a dangling name, reported as
+  `reindex_required`, cleared by `remove` and re-created by `add`. Nothing is lost, because
+  nothing was ever indexed under it. A build cannot repair it — everything defining a corpus but
+  its name and description lives in the database that is missing — so a build refuses instead.
+- An interrupted `remove` leaves a **file without a row**: an orphan, invisible to every query,
+  never opened for search, reported by `status`. It leaks disk until someone clears it, which is
+  the lesser harm.
+
+The reverse ordering is worse in both directions. An `add` writing its row last would leave an
+orphan on interruption, and because a retried `add` mints a fresh id that orphan is *permanent*,
+where the dangling name it avoids costs one `remove`. A `remove` unlinking first would leave a
+name still answering for a corpus the caller had just asked to destroy — one no build can rebuild,
+since its definition went with the file, and which only another `remove` could clear.
+
+**Those two are the states this code produces, not the only two physically reachable, and the
+difference is worth stating because an exhaustive-sounding list is how the third one gets
+overlooked.** Creating a knowledge base's database is itself several steps — the file comes into
+existence when it is connected to, before any table is in it — so a failure in between would leave
+a registered name pointing at an empty database, which reads back as *this index cannot be opened*
+rather than as one needing a build. Creation removes what it made on **every failure it can catch**,
+from the connect onwards, which folds those into the first state above. What remains is a **hard
+kill inside that window**, which no in-process handler can catch: it leaves a row and an empty file,
+reported as an unreadable index. Rarer than the two above, repaired by deleting the file, and named
+here rather than left for somebody to meet without warning.
+"""
+
+import asyncio
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from pathlib import Path
+
+import aiosqlite
+
+from zikaron.core.clock import timestamp
+from zikaron.core.config.resolution import EffectiveConfig
+from zikaron.core.knowledge import (
+    builds,
+    database,
+    lock,
+    meta,
+    paths,
+    registry,
+    reporting,
+    roots,
+    scan,
+)
+from zikaron.core.knowledge.disposal import BuildSettings
+from zikaron.core.knowledge.errors import (
+    DanglingKnowledgeBaseError,
+    IndexerBusyError,
+    InvalidSettingError,
+    SettingBounds,
+)
+from zikaron.core.knowledge.registry import KnowledgeBase
+from zikaron.core.store import permissions
+from zikaron.core.store.transactions import in_one_transaction, propagate
+
+
+@dataclass(frozen=True, slots=True)
+class AddRequest:
+    """A request to create a corpus, exactly as its caller stated it.
+
+    One type rather than a long parameter list, because two surfaces construct exactly this — a
+    command's parsed arguments and a tool call's validated payload — and a parameter list would
+    let the two drift into accepting different things. Nothing here is validated yet: `add` does
+    that, so neither surface can skip a check by being written second.
+
+    `home` is a parameter rather than a lookup so a test states the home directory it is checking
+    instead of mutating the process's. `None` means this process's own.
+    """
+
+    name: str
+    root: Path
+    description: str
+    include_globs: tuple[str, ...] = ()
+    exclude_globs: tuple[str, ...] = ()
+    git_mode: meta.GitMode = meta.GitMode.TRACKED
+    max_file_bytes: int | None = None
+    home: Path | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class Created:
+    """What `add` did, and what the caller should expect next.
+
+    `git_mode_effective` comes from `add`'s own probe rather than from any stored key: the stored
+    one is written by a build, which has not run yet, and a caller learning at creation time that
+    its `git_mode` will not take effect is the whole point of probing at all.
+    """
+
+    knowledge_base: KnowledgeBase
+    database_path: Path
+    git_mode: meta.GitMode
+    git_mode_effective: meta.GitMode
+    status: reporting.Status
+
+
+@dataclass(frozen=True, slots=True)
+class Refreshed:
+    """What one build did, and where it left the corpus.
+
+    Both halves, because they answer different questions: the result says what this build changed,
+    and the status says whether the corpus can now be trusted — which a build can leave unchanged.
+    The case that produces it: configuration names a model whose vectors are not the width
+    configuration also states, so the corpus is rebuilt at the width the model actually emits and
+    goes on reporting that it needs rebuilding, because it does.
+    """
+
+    knowledge_base: KnowledgeBase
+    result: scan.ScanResult
+    status: reporting.Status
+
+
+@dataclass(frozen=True, slots=True)
+class Removed:
+    """What `remove` destroyed — a final snapshot, since its corpus no longer exists to poll."""
+
+    knowledge_base: KnowledgeBase
+    status: reporting.Status
+    files_unlinked: tuple[Path, ...]
+
+
+@asynccontextmanager
+async def _open_registered(
+    store_dir: Path, registered: KnowledgeBase
+) -> AsyncIterator[database.KnowledgeDatabase]:
+    """This corpus's own database, open, for a verb that cannot proceed without one.
+
+    The absence check is explicit rather than inferred from a failed open, because the two mean
+    different things: a missing file is a corpus whose whole definition is missing, and a present
+    one that will not open is a corpus that exists and cannot be read.
+
+    Raises:
+        DanglingKnowledgeBaseError: the registry names this corpus and its database is gone. With
+            it goes everything that said what to index — the root, the globs, the size cap and the
+            encoder identity all live there rather than in the registry — so there is nothing to
+            rebuild from and nothing to guess.
+    """
+    db_path = paths.knowledge_db_path(store_dir, registered.id)
+    if not db_path.is_file():
+        raise DanglingKnowledgeBaseError(
+            f"{registered.name!r} has no database, so nothing records what it indexes; "
+            f"remove it and add it again"
+        )
+    async with await database.KnowledgeDatabase.open(store_dir, registered.id) as opened:
+        yield opened
+
+
+async def add(
+    store_dir: Path,
+    db: aiosqlite.Connection,
+    config: EffectiveConfig,
+    request: AddRequest,
+) -> Created:
+    """Register a corpus and create its database, in that order.
+
+    Every per-knowledge-base setting is persisted at creation, `max_file_bytes` included. A
+    setting accepted and not persisted would be silently undone by the next build, which would
+    re-walk under the global default and change the corpus the caller had just defined.
+
+    A successful `add` reports `reindex_required`, and that is the truth rather than a wart: the
+    database exists and its `meta` is sound, but no build has completed, so the honest answer to
+    *can I trust results from this corpus* is not yet.
+
+    Args:
+        store_dir: the `.zikaron` directory this store lives in.
+        db: an open connection to `memory.db`, which carries the registry.
+        config: the effective configuration to seed this corpus's tuning keys from.
+        request: what to create, as supplied. Validated here rather than by the caller, so the
+            command and the tool surface cannot come to accept different things.
+
+    Raises:
+        InvalidNameError: the name is empty or blank.
+        InvalidRootError: the root is absent, is not a directory, or is degenerate.
+        DuplicateNameError: the name is taken. Never an upsert — silently reconfiguring a corpus
+            underneath whoever created it is worse than a failed call.
+        InvalidSettingError: `max_file_bytes` is outside the range configuration declares for it.
+    """
+    home = request.home if request.home is not None else Path.home()
+    resolved_root = roots.validate_root(request.root, home=home)
+    stored_name = registry.normalize_name(request.name)
+    if request.max_file_bytes is not None:
+        _check_supplied_size_cap(request.max_file_bytes)
+    effective = await roots.effective_git_mode(resolved_root, request.git_mode)
+
+    async def _register(connection: aiosqlite.Connection) -> KnowledgeBase:
+        await registry.ensure_table(connection)
+        return await registry.insert(
+            connection,
+            name=stored_name,
+            description=request.description,
+            created_at=timestamp(),
+        )
+
+    registered = await in_one_transaction(db, _register, failure=propagate)
+
+    spec = database.NewKnowledgeBase(
+        name=registered.name,
+        root=resolved_root,
+        description=request.description,
+        include_globs=request.include_globs,
+        exclude_globs=request.exclude_globs,
+        git_mode=request.git_mode,
+        max_file_bytes=request.max_file_bytes,
+    )
+    async with await database.KnowledgeDatabase.create(
+        store_dir, database.seed_identity(registered.id, spec, config)
+    ):
+        pass
+
+    observed = await reporting.observe(store_dir, registered, config)
+    return Created(
+        knowledge_base=registered,
+        database_path=paths.knowledge_db_path(store_dir, registered.id),
+        git_mode=request.git_mode,
+        git_mode_effective=effective,
+        status=observed.status,
+    )
+
+
+def _check_supplied_size_cap(value: int) -> None:
+    """Refuse a size cap a caller chose, in the caller's own terms rather than the store's.
+
+    The range comes from the configuration schema, which is what keeps a corpus created this way
+    reproducible by writing a configuration file — but the *refusal* must not: a caller that typed
+    a number is not being told its configuration is unusable, and pointing it at a file to fix
+    would send it looking for a typo that is in its own request.
+
+    Raises:
+        InvalidSettingError: the value is outside the declared range.
+    """
+    bounds, expected = meta.bounds_for(meta.MAX_FILE_BYTES_KEY)
+    if bounds.permits(value):
+        return
+    raise InvalidSettingError(
+        f"{meta.MAX_FILE_BYTES_KEY} must be {expected}, not {value}",
+        bounds=SettingBounds(key=meta.MAX_FILE_BYTES_KEY, value=value, expected=expected),
+    )
+
+
+async def rename(
+    store_dir: Path, db: aiosqlite.Connection, config: EffectiveConfig, *, name: str, new_name: str
+) -> reporting.Status:
+    """Change a corpus's name. A registry `UPDATE` that touches no file.
+
+    Safe while a build is running, and deliberately does not check the lock: the name lives in the
+    registry alone, so nothing about a rename can disturb a database being written.
+
+    Raises:
+        InvalidNameError: either name is empty or blank.
+        UnknownKnowledgeBaseError: nothing is registered under `name`.
+        DuplicateNameError: `new_name` is taken.
+    """
+
+    async def _work(connection: aiosqlite.Connection) -> KnowledgeBase:
+        await registry.ensure_table(connection)
+        return await registry.rename(connection, name=name, new_name=new_name)
+
+    renamed = await in_one_transaction(db, _work, failure=propagate)
+    observed = await reporting.observe(store_dir, renamed, config)
+    return observed.status
+
+
+async def refresh(
+    store_dir: Path,
+    db: aiosqlite.Connection,
+    config: EffectiveConfig,
+    *,
+    name: str,
+    build: BuildSettings,
+) -> Refreshed:
+    """Build one corpus's index, and report what the build did and where it left the corpus.
+
+    Runs the build to completion before returning, and holds the corpus's own lock for the whole
+    of it, so two of these against one knowledge base do not overlap whichever process they run
+    in.
+
+    Args:
+        store_dir: the `.zikaron` directory this store lives in.
+        db: an open connection to `memory.db`, which carries the registry.
+        config: the effective configuration, used only to report whether the built corpus's
+            encoder identity still agrees with it. The corpus itself is defined entirely by its
+            own stored `meta`.
+        name: which corpus to build.
+        build: the encoder to index with, how many chunks to embed per pass, and whether to
+            reindex everything rather than only what changed. Supplied by the caller rather than
+            loaded here, because loading a model is the expensive part of starting a build and a
+            caller building several corpora loads it once.
+
+    Raises:
+        InvalidNameError: `name` is empty or blank.
+        UnknownKnowledgeBaseError: nothing is registered under `name`.
+        DanglingKnowledgeBaseError: the corpus is registered but its database is gone, and with it
+            everything that said what to index.
+        CorpusRootMissingError: the indexed directory is gone. The index is left as it is.
+        IndexerBusyError: another build holds this corpus's lock.
+        aiosqlite.Error | ZikaronError | OSError: the database is present and will not open. The
+            driver's own failure, propagated rather than renamed, since nothing the caller can do
+            fixes it.
+    """
+    registered = await builds.prepare(store_dir, db, config, name=name)
+    async with await database.KnowledgeDatabase.open(store_dir, registered.id) as opened:
+        result = await scan.run(opened, build)
+    observed = await reporting.observe(store_dir, registered, config)
+    return Refreshed(knowledge_base=registered, result=result, status=observed.status)
+
+
+async def unlock(store_dir: Path, db: aiosqlite.Connection, *, name: str) -> lock.LockHolder | None:
+    """Clear a build lock an operator has judged dead, refusing the one case evidence contradicts.
+
+    The way out of the state nothing clears on its own. A lock recorded on another machine is never
+    reclaimed automatically — a local process probe says nothing about a foreign process — and the
+    rules that follow compose into a knowledge base nothing can manage: every build reports it
+    busy, removal refuses, and no timeout runs out. This puts that judgement where the evidence is,
+    with whoever knows what else is running, and declines only where the holder answers a signal
+    from this machine.
+
+    Returns:
+        The holder that was cleared, or `None` if no lock was recorded at all.
+
+    Raises:
+        InvalidNameError: `name` is empty or blank.
+        UnknownKnowledgeBaseError: nothing is registered under `name`.
+        DanglingKnowledgeBaseError: the corpus is registered but its database is gone, so there is
+            no lock to clear and nothing that could have taken one.
+        IndexerBusyError: a process on this host answers to the recorded pid — which is not proof
+            it is the indexer, and `lock.force_release` says so and names the manual exit.
+    """
+    await registry.ensure(db)
+    registered = await registry.require(db, name)
+    async with _open_registered(store_dir, registered) as opened:
+
+        async def _work(connection: aiosqlite.Connection) -> lock.LockHolder | None:
+            return await lock.force_release(connection, host=lock.this_host())
+
+        return await in_one_transaction(opened.connection, _work, failure=propagate)
+
+
+async def remove(
+    store_dir: Path, db: aiosqlite.Connection, config: EffectiveConfig, *, name: str
+) -> Removed:
+    """Destroy a corpus: delete its registry row, commit, then unlink its three files.
+
+    Refuses whenever the corpus's build lock is held by somebody who cannot be shown to be gone —
+    the same test a build's own acquisition uses, and the conservative half of it: refusing to
+    unlink a database a writer may hold is recoverable, and unlinking one it does hold is not. What
+    that test deliberately does **not** refuse is a lock left behind by a crashed indexer on this
+    host, because those rows survive on purpose and a knowledge base that could never be removed
+    again after one crash would be the worse failure.
+
+    **That refusal is a check followed by an act, not one atomic step, and the window is accepted
+    rather than overlooked.** The lock lives in the knowledge base's own database and the registry
+    row lives in `memory.db`, so no single transaction spans them: a build that takes the lock
+    after the check has its database unlinked underneath it. The harm is bounded — the racing
+    build's writes land on an unlinked inode and evaporate, which is what destroying the corpus was
+    asked for — and closing it properly would mean a lock somewhere both databases can see, which
+    is a larger change than the outcome justifies.
+
+    Unlinks all three of the `.db`, its `-wal` and its `-shm`, because in WAL mode a live database
+    is three files and a journal left behind would be inherited by whatever is created at that path
+    next. In practice it usually removes one: reading the corpus's state opens and closes it, and
+    SQLite reclaims its own `-wal` and `-shm` on the last close, so the other two are typically
+    already gone by the time this runs. `files_unlinked` reports what was actually there rather
+    than what was attempted, which is why it is normally a single path.
+
+    Raises:
+        InvalidNameError: `name` is empty or blank.
+        UnknownKnowledgeBaseError: nothing is registered under `name`.
+        IndexerBusyError: a build that cannot be shown to be dead holds this corpus's lock.
+    """
+    await registry.ensure(db)
+    registered = await registry.require(db, name)
+    observed = await reporting.observe(store_dir, registered, config)
+    if observed.blocker is not None:
+        raise IndexerBusyError(
+            f"{registered.name!r}'s build lock is held by a process this host cannot show is "
+            f"gone ({observed.blocker.describe()}); wait for it, or clear the lock with "
+            f"`refresh --force-unlock` if you are sure it is gone",
+            holder=observed.blocker,
+        )
+
+    async def _work(connection: aiosqlite.Connection) -> KnowledgeBase:
+        return await registry.delete(connection, name=name)
+
+    await in_one_transaction(db, _work, failure=propagate)
+
+    db_path = paths.knowledge_db_path(store_dir, registered.id)
+    unlinked = [
+        path
+        for path in permissions.database_files(db_path)
+        if await asyncio.to_thread(_unlink_if_present, path)
+    ]
+    return Removed(
+        knowledge_base=registered, status=observed.status, files_unlinked=tuple(unlinked)
+    )
+
+
+def _unlink_if_present(path: Path) -> bool:
+    """Remove `path`, reporting whether it was there.
+
+    Absence is an ordinary outcome rather than a fault: a knowledge base that was never built has
+    no `-wal` or `-shm`, and one whose file was deleted by hand has none of the three.
+    """
+    try:
+        path.unlink()
+    except FileNotFoundError:
+        return False
+    return True