theurian 0.1.0.dev0__py3-none-any.whl

theurian/__init__.py

@@ -0,0 +1,14 @@
+"""Theurian: Git-native engineering knowledge for AI agents.
+
+The public surface for external consumers is the ``theurian`` CLI, the MCP
+server, and the JSON Schemas under ``schemas/``. Python modules inside this
+package are internal and carry no stability guarantee -- notably, the Claude Code
+plugin must never import them (ADR-0001, CP-2).
+"""
+
+from theurian.domain.compatibility import CURRENT_PROTOCOL_VERSION
+
+__version__ = "0.1.0.dev0"
+__protocol_version__ = CURRENT_PROTOCOL_VERSION
+
+__all__ = ["__protocol_version__", "__version__"]

theurian/application/__init__.py

@@ -0,0 +1,11 @@
+"""Application layer: use cases and orchestration.
+
+Depends on :mod:`theurian.domain` only. Adapters arrive by constructor
+injection; nothing here names a concrete implementation (ADR-0003).
+
+Milestone 1 onward: ``SetupService``, ``MigrationService``, ``IndexingService``,
+``RetrievalService``, ``ReviewService``, ``TraceabilityService``.
+
+``SetupService`` is shared by ``theurian setup`` and ``/theurian:setup``. There
+is exactly one implementation of setup, because two would drift (FR-L1).
+"""

theurian/application/index_builder.py

@@ -0,0 +1,197 @@
+"""Turning a canonical state into an index build (FR-R2, FR-R3, ADR-0022).
+
+Split out of :mod:`theurian.application.retrieval_service` for the reason
+:mod:`theurian.application.visibility` was split out of it before: that file had
+grown past the size at which it can be read in one sitting, and this is a seam
+rather than a cut. It described itself as "three use cases over one index file",
+and this is the one that *writes*. Everything left there reads.
+
+The seam is real rather than arithmetic. Nothing here consults a
+:class:`~theurian.application.visibility.Visibility`, because at build time there
+is no caller to be visible *to*: the filter that applies is
+:func:`~theurian.domain.enums.may_surface` against the operator's
+``include_unapproved``, and it decides what is written rather than what is shown.
+The equality property the query side exists to hold has no counterpart here.
+
+Takes its collaborators by injection, so a build is testable without a database
+and without an embedding provider.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final, final
+
+from theurian.domain.chunking import IndexableChunk, chunk_document
+from theurian.domain.context import RequestContext
+from theurian.domain.enums import may_surface
+from theurian.domain.identifiers import ProjectId
+from theurian.domain.ports.canonical_store import CanonicalReadSession
+from theurian.domain.ports.embedding import EmbeddingProvider
+from theurian.domain.ports.index_store import IndexStore
+
+#: Chunks per embedding request. An API-backed provider caps request size, and a
+#: local one gains nothing from an unbounded batch -- while an unbounded batch
+#: holds the whole corpus and all its vectors in memory at once.
+EMBED_BATCH: Final = 128
+
+
+@dataclass(frozen=True, slots=True)
+class IndexRequest:
+    """What to index, and where to put it."""
+
+    database: Path
+    index_path: Path
+    project_id: str
+    state_hash: str
+    index_build_id: str
+    #: Whether unapproved revisions are written at all. Off by default, so an
+    #: operator who never opts in has a hard guarantee that no draft is in the
+    #: file — not merely that a query filter is expected to hold.
+    include_unapproved: bool = False
+
+
+@final
+class IndexBuilder:
+    """Turns a canonical state into an index build."""
+
+    def __init__(
+        self,
+        *,
+        store_factory: Callable[[Path], CanonicalReadSession],
+        index_factory: Callable[[Path], IndexStore],
+        embedder: EmbeddingProvider | None = None,
+    ) -> None:
+        self._store_factory = store_factory
+        self._index_factory = index_factory
+        self._embedder = embedder
+
+    def build(self, request: IndexRequest) -> dict[str, object]:
+        """Write a new index file from a canonical state.
+
+        Unapproved revisions are written only when asked for, and `rejected`
+        never is.
+
+        The obvious simplification — index everything, filter at query time —
+        was tried and reverted. It makes `includeUnapproved=True` a single
+        boolean that reaches content the team decided must not be followed, and
+        it removes the operator's ability to guarantee that a draft is not in
+        the file at all. The cost is that `includeUnapproved=True` cannot return
+        rows that were never written, which is reported rather than hidden:
+        `indexesUnapproved` says whether this build can answer such a query.
+
+        Either a whole index file or none: a build that fails part-way deletes
+        what it wrote. That guarantee used to live only in the CLI while
+        :meth:`_embed`'s docstring asserted it here, so any other caller of
+        `build` — a daemon, a test, a future scheduled rebuild — got a
+        half-written file and the promise that it could not happen.
+
+        **What it wrote, and nothing else.** `IndexStore.create` refuses to
+        overwrite an existing file and raises; the cleanup then unlinked the very
+        file it had just been refused permission to touch — which is the file
+        `active-index.json` names, so a build against an already-taken path
+        deleted the published index and left the pointer aimed at nothing. Not
+        reachable from `theurian index build`, which mints a fresh ULID per
+        build, but `build` is a public application-layer API and `create`'s
+        contract says an existing file is left alone.
+        """
+        # Sampled before `create` rather than inferred from the exception: only a
+        # path this call brought into existence may be removed by it.
+        preexisting = request.index_path.exists()
+        try:
+            return self._build(request)
+        except Exception:
+            # A partial index is worse than none. It looks complete, ranks the
+            # fraction it holds, and never surfaces the rest -- which reads as a
+            # relevance problem rather than a build failure, and so does not get
+            # investigated. Nothing this build wrote is published until it
+            # returns, so deleting here loses no index a search could have used.
+            if not preexisting:
+                request.index_path.unlink(missing_ok=True)
+            raise
+
+    def _build(self, request: IndexRequest) -> dict[str, object]:
+        index = self._index_factory(request.index_path)
+        index.create(index_build_id=request.index_build_id, state_hash=request.state_hash)
+
+        context = RequestContext(project_id=ProjectId(request.project_id))
+        indexable: list[IndexableChunk] = []
+
+        with self._store_factory(request.database) as store:
+            for item in store.list_items(context):
+                # The same authority the search paths consult. Inlined here as
+                # two comparisons until `may_surface` moved to the domain, which
+                # is one copy of a security rule too many.
+                if not may_surface(item.status, include_unapproved=request.include_unapproved):
+                    continue
+                if item.current_revision_id is None:
+                    continue
+                revision = store.get_revision(context, item.current_revision_id)
+                if revision is None:  # pragma: no cover - the pointer is a foreign key
+                    continue
+
+                # The title is prepended to the body before splitting so that a
+                # query matching only the title still finds the document. A
+                # separately indexed title field would need its own retriever and
+                # its own fusion weight for the same effect.
+                body = f"{revision.title}\n\n{revision.body}"
+                for chunk in chunk_document(revision.revision_id.value, body):
+                    indexable.append(
+                        IndexableChunk(
+                            chunk=chunk,
+                            project_id=request.project_id,
+                            item_id=item.item_id.value,
+                            revision_id=revision.revision_id.value,
+                            status=item.status.value,
+                            sensitivity=revision.metadata.sensitivity.value,
+                            trust_level=revision.metadata.trust_level.value,
+                        )
+                    )
+
+        index.add_chunks(indexable)
+        embedded = self._embed(index, indexable)
+
+        return {
+            "indexBuildId": request.index_build_id,
+            "stateHash": request.state_hash,
+            "indexPath": str(request.index_path),
+            "chunks": len(indexable),
+            "embeddings": embedded,
+            "embeddingModel": self._embedder.model_id if self._embedder else "",
+            "indexesUnapproved": request.include_unapproved,
+        }
+
+    def _embed(self, index: IndexStore, indexable: Sequence[IndexableChunk]) -> int:
+        """Embed every chunk, or none.
+
+        Batched, because a real provider caps request size and a local one gains
+        nothing from an unbounded batch.
+
+        A partial embedding is worse than none: the dense retriever would rank
+        the embedded half and silently never surface the rest, which looks like
+        a relevance problem rather than a build problem. :meth:`build` discards
+        the whole index file if any batch raises, so a partial one never exists
+        to be published.
+        """
+        if self._embedder is None or not indexable:
+            return 0
+
+        embedded = 0
+        for start in range(0, len(indexable), EMBED_BATCH):
+            batch = indexable[start : start + EMBED_BATCH]
+            vectors = asyncio.run(self._embedder.embed(tuple(c.chunk.text for c in batch)))
+            index.add_embeddings(
+                [(c.chunk.chunk_id, v) for c, v in zip(batch, vectors, strict=True)]
+            )
+            embedded += len(vectors)
+
+        index.record_embedding_model(
+            model_id=self._embedder.model_id, dimension=self._embedder.dimension
+        )
+        return embedded
+
+
+__all__ = ["EMBED_BATCH", "IndexBuilder", "IndexRequest"]

theurian/application/ingestion_service.py

@@ -0,0 +1,255 @@
+"""Source ingestion (FR-S1 .. FR-S6, ADR-0010).
+
+Walks a project's knowledge directory, dispatches each file to a parser, and
+normalizes the result into the Canonical Layer.
+
+Two properties shape the design:
+
+**Failure is per document.** A malformed YAML file among two hundred must not
+make the other 199 unavailable, so a parse failure becomes a value in the report
+rather than an exception that unwinds the walk.
+
+**Unchanged files cost one hash.** Touching a file without changing it must not
+trigger a reparse and a reindex, so content hashes from the previous run are
+compared before any parsing happens.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+from typing import Protocol
+
+from theurian.domain.errors import (
+    InputTooLargeError,
+    PathEscapeError,
+    TheurianError,
+)
+from theurian.domain.ingestion import (
+    IngestedDocument,
+    IngestionReport,
+    ParseFailure,
+    ParseWarning,
+)
+from theurian.domain.knowledge import SourceAnchor
+from theurian.domain.ports import NormalizedDocument, SourceParser
+from theurian.domain.values import ContentHash, MediaType
+from theurian.normalization.projection import project
+from theurian.security.paths import MAX_SOURCE_FILE_BYTES, read_source_file
+
+#: Directories under `.theurian/` that hold ingestible sources. Derived
+#: directories are excluded: ingesting `generated/` would feed Theurian's own
+#: output back into itself as though it were a source (ADR-0004).
+INGESTIBLE_SUBDIRECTORIES: tuple[str, ...] = ("knowledge", "specifications")
+
+
+class ParserResolver(Protocol):
+    """Resolves a document to its media type and parser.
+
+    A Protocol rather than a concrete registry so the application layer never
+    imports an adapter (ADR-0003).
+    """
+
+    def detect(self, path: PurePosixPath, data: bytes) -> MediaType | None: ...
+    def for_media_type(self, media_type: MediaType) -> SourceParser | None: ...
+
+
+class WarningSource(Protocol):
+    """Parsers that surface warnings alongside a normalized document.
+
+    ``NormalizedDocument`` carries no warning field, and widening the
+    ``SourceParser`` port for one parser's benefit would push a Markdown concern
+    into every future adapter. Parsers that have warnings expose them here
+    instead, and the service asks only those that do (ADR-0019).
+    """
+
+    def warnings_for(self, text: str) -> tuple[ParseWarning, ...]: ...
+
+
+@dataclass(frozen=True, slots=True)
+class IngestionRequest:
+    """One ingestion run's inputs."""
+
+    project_root: Path
+    knowledge_dir: Path
+    #: Content hashes from the previous run, keyed by project-relative path.
+    #: Anything matching is reported unchanged and never reparsed.
+    known_hashes: dict[str, str]
+    #: Commit the sources were read at, so every anchor pins an immutable
+    #: object rather than a path that may since have moved (FR-S3).
+    commit_sha: str | None = None
+    repository: str | None = None
+
+
+class IngestionService:
+    """Normalizes a project's sources into canonical documents."""
+
+    def __init__(self, resolver: ParserResolver) -> None:
+        self._resolver = resolver
+
+    def ingest(self, request: IngestionRequest) -> IngestionReport:
+        """Walk and normalize every ingestible source.
+
+        Never raises for a document-level problem. A caller inspects
+        ``report.failures`` and decides; the run itself always completes.
+        """
+        report = IngestionReport()
+
+        for path in self._discover(request.knowledge_dir):
+            relative = path.relative_to(request.project_root).as_posix()
+            try:
+                self._ingest_one(relative, request, report)
+            except TheurianError as exc:
+                # Security and limit refusals are per-document too. One hostile
+                # file must not prevent the rest of the tree from being read.
+                report.failures.append(ParseFailure(path=relative, reason=str(exc)))
+
+        return report
+
+    def _ingest_one(
+        self, relative: str, request: IngestionRequest, report: IngestionReport
+    ) -> None:
+        try:
+            data = read_source_file(request.project_root, PurePosixPath(relative))
+        except (PathEscapeError, InputTooLargeError) as exc:
+            report.failures.append(ParseFailure(path=relative, reason=str(exc)))
+            return
+        except OSError as exc:
+            report.failures.append(ParseFailure(path=relative, reason=f"unreadable: {exc}"))
+            return
+
+        content_hash = ContentHash.of_bytes(data)
+        if request.known_hashes.get(relative) == content_hash.value:
+            # The cheap early exit: touching a file without changing it costs
+            # one hash, not a reparse and a reindex.
+            report.unchanged.append(relative)
+            return
+
+        media_type = self._resolver.detect(PurePosixPath(relative), data)
+        if media_type is None:
+            report.skipped.append(relative)
+            return
+
+        parser = self._resolver.for_media_type(media_type)
+        if parser is None:
+            report.failures.append(
+                ParseFailure(
+                    path=relative,
+                    reason=f"no parser registered for {media_type}",
+                    media_type=media_type.value,
+                )
+            )
+            return
+
+        anchor = _anchor(relative, request)
+
+        try:
+            normalized = parser.parse(data, media_type=media_type, anchor=anchor)
+        except (ValueError, InputTooLargeError) as exc:
+            report.failures.append(
+                ParseFailure(path=relative, reason=str(exc), media_type=media_type.value)
+            )
+            return
+
+        report.documents.append(
+            _to_document(
+                normalized,
+                path=relative,
+                source_hash=content_hash,
+                parser=parser,
+                warnings=_warnings(parser, data),
+            )
+        )
+
+    def _discover(self, knowledge_dir: Path) -> Iterator[Path]:
+        """Yield candidate source files in a stable order.
+
+        Sorted so a failure reports the same first offender on every run rather
+        than whichever the filesystem happened to yield first.
+        """
+        for subdirectory in INGESTIBLE_SUBDIRECTORIES:
+            root = knowledge_dir / subdirectory
+            if not root.is_dir():
+                continue
+            for path in sorted(root.rglob("*")):
+                if not path.is_file() or path.name.startswith("."):
+                    continue
+                # A symlink is refused later by read_source_file if it escapes;
+                # skipping obviously-oversized files here avoids reading them.
+                if path.stat().st_size > MAX_SOURCE_FILE_BYTES:
+                    continue
+                yield path
+
+
+def _warnings(parser: SourceParser, data: bytes) -> tuple[ParseWarning, ...]:
+    """Collect warnings from parsers that produce them."""
+    collector = getattr(parser, "warnings_for", None)
+    if collector is None:
+        return ()
+    try:
+        text = data.decode("utf-8")
+    except UnicodeDecodeError:  # pragma: no cover - parse would have failed first
+        return ()
+    result: tuple[ParseWarning, ...] = collector(text)
+    return result
+
+
+def _anchor(relative: str, request: IngestionRequest) -> SourceAnchor:
+    """Build the anchor that makes this document traceable (FR-S3).
+
+    A commit SHA plus a path pins an immutable Git object, so the anchor still
+    resolves after the file is edited, moved, or deleted.
+    """
+    return SourceAnchor(
+        provider="git" if request.commit_sha else "filesystem",
+        source_uri=f"git://{request.repository or 'local'}/{relative}"
+        if request.commit_sha
+        else f"file://{relative}",
+        repository=request.repository,
+        commit_sha=request.commit_sha,
+        file_path=relative,
+    )
+
+
+def _to_document(
+    normalized: NormalizedDocument,
+    *,
+    path: str,
+    source_hash: ContentHash,
+    parser: SourceParser,
+    warnings: tuple[ParseWarning, ...],
+) -> IngestedDocument:
+    projection: str | None = None
+    if normalized.structured is not None:
+        projection = project(normalized.structured)
+
+    return IngestedDocument(
+        path=path,
+        title=normalized.title,
+        body=normalized.body,
+        content_type=normalized.content_type,
+        content_hash=normalized.content_hash,
+        source_hash=source_hash,
+        anchors=normalized.anchors,
+        parser_id=parser.parser_id,
+        structured=normalized.structured,
+        text_projection=projection,
+        warnings=warnings,
+    )
+
+
+def manifest_from(report: IngestionReport, previous: dict[str, str]) -> dict[str, str]:
+    """Build the content-hash manifest for the next run's early exit.
+
+    Carries forward hashes for unchanged files, because those were never
+    reparsed and so are absent from ``report.documents``. Dropping them would
+    make every second run a full reparse.
+    """
+    manifest = {path: previous[path] for path in report.unchanged if path in previous}
+    for document in report.documents:
+        # The *source* hash, matching what the early exit computes. Storing the
+        # body hash instead makes every Markdown file with front matter reparse
+        # on every run, because its body and its bytes differ.
+        manifest[document.path] = document.source_hash.value
+    return manifest