biblicus 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

biblicus/__init__.py

@@ -25,4 +25,4 @@ __all__ = [
     "RetrievalRun",
 ]
 
-__version__ = "0.3.0"
+__version__ = "0.5.0"

biblicus/cli.py

@@ -13,11 +13,19 @@ from typing import Dict, List, Optional
 from pydantic import ValidationError
 
 from .backends import get_backend
+from .context import (
+    ContextPackPolicy,
+    TokenBudget,
+    build_context_pack,
+    fit_context_pack_to_token_budget,
+)
 from .corpus import Corpus
+from .crawl import CrawlRequest, crawl_into_corpus
 from .errors import ExtractionRunFatalError
 from .evaluation import evaluate_run, load_dataset
+from .evidence_processing import apply_evidence_filter, apply_evidence_reranker
 from .extraction import build_extraction_run
-from .models import QueryBudget
+from .models import QueryBudget, RetrievalResult, parse_extraction_run_reference
 from .uris import corpus_ref_to_path
 
 
@@ -327,7 +335,7 @@ def cmd_build(arguments: argparse.Namespace) -> int:
     return 0
 
 
-def cmd_extract(arguments: argparse.Namespace) -> int:
+def cmd_extract_build(arguments: argparse.Namespace) -> int:
     """
     Build a text extraction run for the corpus using a pipeline of extractors.
 
@@ -359,6 +367,69 @@ def cmd_extract(arguments: argparse.Namespace) -> int:
     return 0
 
 
+def cmd_extract_list(arguments: argparse.Namespace) -> int:
+    """
+    List extraction runs stored under the corpus.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    corpus = (
+        Corpus.open(arguments.corpus)
+        if getattr(arguments, "corpus", None)
+        else Corpus.find(Path.cwd())
+    )
+    runs = corpus.list_extraction_runs(extractor_id=arguments.extractor_id)
+    print(json.dumps([entry.model_dump() for entry in runs], indent=2))
+    return 0
+
+
+def cmd_extract_show(arguments: argparse.Namespace) -> int:
+    """
+    Show an extraction run manifest.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    corpus = (
+        Corpus.open(arguments.corpus)
+        if getattr(arguments, "corpus", None)
+        else Corpus.find(Path.cwd())
+    )
+    reference = parse_extraction_run_reference(arguments.run)
+    manifest = corpus.load_extraction_run_manifest(
+        extractor_id=reference.extractor_id, run_id=reference.run_id
+    )
+    print(manifest.model_dump_json(indent=2))
+    return 0
+
+
+def cmd_extract_delete(arguments: argparse.Namespace) -> int:
+    """
+    Delete an extraction run directory and its derived artifacts.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    corpus = (
+        Corpus.open(arguments.corpus)
+        if getattr(arguments, "corpus", None)
+        else Corpus.find(Path.cwd())
+    )
+    if arguments.confirm != arguments.run:
+        raise ValueError("Refusing to delete extraction run without an exact --confirm match.")
+    reference = parse_extraction_run_reference(arguments.run)
+    corpus.delete_extraction_run(extractor_id=reference.extractor_id, run_id=reference.run_id)
+    print(json.dumps({"deleted": True, "run": arguments.run}, indent=2))
+    return 0
+
+
 def cmd_query(arguments: argparse.Namespace) -> int:
     """
     Execute a retrieval query.
@@ -385,10 +456,62 @@ def cmd_query(arguments: argparse.Namespace) -> int:
     query_text = arguments.query if arguments.query is not None else sys.stdin.read()
     budget = _budget_from_args(arguments)
     result = backend.query(corpus, run=run, query_text=query_text, budget=budget)
+    processed_evidence = result.evidence
+    if getattr(arguments, "reranker_id", None):
+        processed_evidence = apply_evidence_reranker(
+            reranker_id=arguments.reranker_id,
+            query_text=result.query_text,
+            evidence=processed_evidence,
+        )
+    if getattr(arguments, "minimum_score", None) is not None:
+        processed_evidence = apply_evidence_filter(
+            filter_id="filter-minimum-score",
+            query_text=result.query_text,
+            evidence=processed_evidence,
+            config={"minimum_score": float(arguments.minimum_score)},
+        )
+    if processed_evidence is not result.evidence:
+        result = result.model_copy(update={"evidence": processed_evidence})
     print(result.model_dump_json(indent=2))
     return 0
 
 
+def cmd_context_pack_build(arguments: argparse.Namespace) -> int:
+    """
+    Build a context pack from a retrieval result.
+
+    The retrieval result is read from standard input as JavaScript Object Notation.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    input_text = sys.stdin.read()
+    if not input_text.strip():
+        raise ValueError("Context pack build requires a retrieval result JavaScript Object Notation on standard input")
+    retrieval_result = RetrievalResult.model_validate_json(input_text)
+    join_with = bytes(arguments.join_with, "utf-8").decode("unicode_escape")
+    policy = ContextPackPolicy(join_with=join_with)
+    context_pack = build_context_pack(retrieval_result, policy=policy)
+    if arguments.max_tokens is not None:
+        context_pack = fit_context_pack_to_token_budget(
+            context_pack,
+            policy=policy,
+            token_budget=TokenBudget(max_tokens=int(arguments.max_tokens)),
+        )
+    print(
+        json.dumps(
+            {
+                "policy": policy.model_dump(),
+                "context_pack": context_pack.model_dump(),
+            },
+            indent=2,
+        )
+    )
+    return 0
+
+
 def cmd_eval(arguments: argparse.Namespace) -> int:
     """
     Evaluate a retrieval run against a dataset.
@@ -414,6 +537,32 @@ def cmd_eval(arguments: argparse.Namespace) -> int:
     return 0
 
 
+def cmd_crawl(arguments: argparse.Namespace) -> int:
+    """
+    Crawl a website prefix into a corpus.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    corpus = (
+        Corpus.open(arguments.corpus)
+        if getattr(arguments, "corpus", None)
+        else Corpus.find(Path.cwd())
+    )
+    tags = _parse_tags(arguments.tags, arguments.tag)
+    request = CrawlRequest(
+        root_url=arguments.root_url,
+        allowed_prefix=arguments.allowed_prefix,
+        max_items=arguments.max_items,
+        tags=tags,
+    )
+    result = crawl_into_corpus(corpus=corpus, request=request)
+    print(result.model_dump_json(indent=2))
+    return 0
+
+
 def build_parser() -> argparse.ArgumentParser:
     """
     Build the command-line interface argument parser.
@@ -511,16 +660,53 @@ def build_parser() -> argparse.ArgumentParser:
     )
     p_build.set_defaults(func=cmd_build)
 
-    p_extract = sub.add_parser("extract", help="Build a text extraction run for the corpus.")
-    _add_common_corpus_arg(p_extract)
-    p_extract.add_argument("--recipe-name", default="default", help="Human-readable recipe name.")
-    p_extract.add_argument(
+    p_extract = sub.add_parser("extract", help="Work with text extraction runs for the corpus.")
+    extract_sub = p_extract.add_subparsers(dest="extract_command", required=True)
+
+    p_extract_build = extract_sub.add_parser("build", help="Build a text extraction run.")
+    _add_common_corpus_arg(p_extract_build)
+    p_extract_build.add_argument(
+        "--recipe-name", default="default", help="Human-readable recipe name."
+    )
+    p_extract_build.add_argument(
         "--step",
         action="append",
         default=None,
         help="Pipeline step spec in the form extractor_id or extractor_id:key=value,key=value (repeatable).",
     )
-    p_extract.set_defaults(func=cmd_extract)
+    p_extract_build.set_defaults(func=cmd_extract_build)
+
+    p_extract_list = extract_sub.add_parser("list", help="List extraction runs.")
+    _add_common_corpus_arg(p_extract_list)
+    p_extract_list.add_argument(
+        "--extractor-id",
+        default=None,
+        help="Optional extractor identifier filter (for example: pipeline).",
+    )
+    p_extract_list.set_defaults(func=cmd_extract_list)
+
+    p_extract_show = extract_sub.add_parser("show", help="Show an extraction run manifest.")
+    _add_common_corpus_arg(p_extract_show)
+    p_extract_show.add_argument(
+        "--run",
+        required=True,
+        help="Extraction run reference in the form extractor_id:run_id.",
+    )
+    p_extract_show.set_defaults(func=cmd_extract_show)
+
+    p_extract_delete = extract_sub.add_parser("delete", help="Delete an extraction run directory.")
+    _add_common_corpus_arg(p_extract_delete)
+    p_extract_delete.add_argument(
+        "--run",
+        required=True,
+        help="Extraction run reference in the form extractor_id:run_id.",
+    )
+    p_extract_delete.add_argument(
+        "--confirm",
+        required=True,
+        help="Type the exact extractor_id:run_id to confirm deletion.",
+    )
+    p_extract_delete.set_defaults(func=cmd_extract_delete)
 
     p_query = sub.add_parser("query", help="Run a retrieval query.")
     _add_common_corpus_arg(p_query)
@@ -530,8 +716,38 @@ def build_parser() -> argparse.ArgumentParser:
     p_query.add_argument("--max-total-items", type=int, default=5)
     p_query.add_argument("--max-total-characters", type=int, default=2000)
     p_query.add_argument("--max-items-per-source", type=int, default=5)
+    p_query.add_argument(
+        "--reranker-id",
+        default=None,
+        help="Optional reranker identifier to apply after retrieval (for example: rerank-longest-text).",
+    )
+    p_query.add_argument(
+        "--minimum-score",
+        type=float,
+        default=None,
+        help="Optional minimum score threshold to filter evidence after retrieval.",
+    )
     p_query.set_defaults(func=cmd_query)
 
+    p_context_pack = sub.add_parser("context-pack", help="Build context pack text from evidence.")
+    context_pack_sub = p_context_pack.add_subparsers(dest="context_pack_command", required=True)
+
+    p_context_pack_build = context_pack_sub.add_parser(
+        "build", help="Build a context pack from a retrieval result JavaScript Object Notation."
+    )
+    p_context_pack_build.add_argument(
+        "--join-with",
+        default="\\n\\n",
+        help="Separator between evidence blocks (escape sequences supported, default is two newlines).",
+    )
+    p_context_pack_build.add_argument(
+        "--max-tokens",
+        default=None,
+        type=int,
+        help="Optional token budget for the final context pack using the naive-whitespace tokenizer.",
+    )
+    p_context_pack_build.set_defaults(func=cmd_context_pack_build)
+
     p_eval = sub.add_parser("eval", help="Evaluate a run against a dataset.")
     _add_common_corpus_arg(p_eval)
     p_eval.add_argument("--run", default=None, help="Run identifier (defaults to latest run).")
@@ -545,6 +761,19 @@ def build_parser() -> argparse.ArgumentParser:
     p_eval.add_argument("--max-items-per-source", type=int, default=5)
     p_eval.set_defaults(func=cmd_eval)
 
+    p_crawl = sub.add_parser("crawl", help="Crawl a website prefix into the corpus.")
+    _add_common_corpus_arg(p_crawl)
+    p_crawl.add_argument("--root-url", required=True, help="Root uniform resource locator to fetch.")
+    p_crawl.add_argument(
+        "--allowed-prefix",
+        required=True,
+        help="Uniform resource locator prefix that limits which links are eligible for crawl.",
+    )
+    p_crawl.add_argument("--max-items", type=int, default=50, help="Maximum number of items to store.")
+    p_crawl.add_argument("--tags", default=None, help="Comma-separated tags to apply to stored items.")
+    p_crawl.add_argument("--tag", action="append", help="Repeatable tag to apply to stored items.")
+    p_crawl.set_defaults(func=cmd_crawl)
+
     return parser
 
 

biblicus/context.py

@@ -0,0 +1,183 @@
+"""
+Context pack building for Biblicus.
+
+A context pack is the text that your application sends to a large language model.
+Biblicus produces a context pack from structured retrieval results so that evidence remains a
+stable contract while context formatting remains an explicit policy surface.
+"""
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from .models import RetrievalResult
+
+
+class ContextPackPolicy(BaseModel):
+    """
+    Policy that controls how evidence becomes context pack text.
+
+    :ivar join_with: Separator inserted between evidence text blocks.
+    :vartype join_with: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    join_with: str = Field(default="\n\n")
+
+
+class ContextPack(BaseModel):
+    """
+    Context pack derived from retrieval evidence.
+
+    :ivar text: Context pack text suitable for inclusion in a model call.
+    :vartype text: str
+    :ivar evidence_count: Number of evidence blocks included in the context pack.
+    :vartype evidence_count: int
+    :ivar blocks: Structured blocks that produced the context pack.
+    :vartype blocks: list[ContextPackBlock]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    text: str
+    evidence_count: int = Field(ge=0)
+    blocks: List["ContextPackBlock"] = Field(default_factory=list)
+
+
+class ContextPackBlock(BaseModel):
+    """
+    A single context pack block derived from one evidence item.
+
+    :ivar evidence_item_id: Item identifier that produced this block.
+    :vartype evidence_item_id: str
+    :ivar text: Text included in this block.
+    :vartype text: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    evidence_item_id: str = Field(min_length=1)
+    text: str = Field(min_length=1)
+
+
+class TokenCounter(BaseModel):
+    """
+    Token counter configuration for token budget fitting.
+
+    This is a lightweight model wrapper so token fitting remains explicit and testable even when
+    the underlying tokenizer is provided by an optional dependency.
+
+    :ivar tokenizer_id: Tokenizer identifier (for example, naive-whitespace).
+    :vartype tokenizer_id: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    tokenizer_id: str = Field(default="naive-whitespace", min_length=1)
+
+
+class TokenBudget(BaseModel):
+    """
+    Token budget for a context pack.
+
+    :ivar max_tokens: Maximum tokens permitted for the final context pack text.
+    :vartype max_tokens: int
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    max_tokens: int = Field(ge=1)
+
+
+def build_context_pack(result: RetrievalResult, *, policy: ContextPackPolicy) -> ContextPack:
+    """
+    Build a context pack from a retrieval result using an explicit policy.
+
+    :param result: Retrieval result containing ranked evidence.
+    :type result: RetrievalResult
+    :param policy: Policy controlling how evidence text is joined.
+    :type policy: ContextPackPolicy
+    :return: Context pack containing concatenated evidence text.
+    :rtype: ContextPack
+    """
+    selected_blocks: List[ContextPackBlock] = []
+    for evidence in result.evidence:
+        if not isinstance(evidence.text, str):
+            continue
+        trimmed_text = evidence.text.strip()
+        if not trimmed_text:
+            continue
+        selected_blocks.append(
+            ContextPackBlock(evidence_item_id=evidence.item_id, text=trimmed_text)
+        )
+
+    return ContextPack(
+        text=policy.join_with.join([block.text for block in selected_blocks]),
+        evidence_count=len(selected_blocks),
+        blocks=selected_blocks,
+    )
+
+
+def count_tokens(text: str, *, tokenizer_id: str) -> int:
+    """
+    Count tokens in a text using a tokenizer identifier.
+
+    The default tokenizer is naive-whitespace, which counts whitespace-separated tokens.
+
+    :param text: Text payload to count.
+    :type text: str
+    :param tokenizer_id: Tokenizer identifier.
+    :type tokenizer_id: str
+    :return: Token count.
+    :rtype: int
+    :raises KeyError: If the tokenizer identifier is unknown.
+    """
+    tokenizers = {
+        "naive-whitespace": lambda value: len([token for token in value.split() if token]),
+    }
+    tokenizer = tokenizers[tokenizer_id]
+    return int(tokenizer(text))
+
+
+def fit_context_pack_to_token_budget(
+    context_pack: ContextPack,
+    *,
+    policy: ContextPackPolicy,
+    token_budget: TokenBudget,
+    token_counter: Optional[TokenCounter] = None,
+) -> ContextPack:
+    """
+    Fit a context pack to a token budget by dropping trailing blocks.
+
+    This function is deterministic. It never rewrites block text. It only removes blocks from the
+    end of the block list until the token budget is met.
+
+    :param context_pack: Context pack to fit.
+    :type context_pack: ContextPack
+    :param policy: Policy controlling how blocks are joined into text.
+    :type policy: ContextPackPolicy
+    :param token_budget: Token budget to enforce.
+    :type token_budget: TokenBudget
+    :param token_counter: Optional token counter configuration.
+    :type token_counter: TokenCounter or None
+    :return: Fitted context pack.
+    :rtype: ContextPack
+    """
+    token_counter = token_counter or TokenCounter()
+    remaining_blocks: List[ContextPackBlock] = list(context_pack.blocks)
+
+    while remaining_blocks:
+        candidate_text = policy.join_with.join([block.text for block in remaining_blocks])
+        candidate_tokens = count_tokens(candidate_text, tokenizer_id=token_counter.tokenizer_id)
+        if candidate_tokens <= token_budget.max_tokens:
+            return ContextPack(
+                text=candidate_text,
+                evidence_count=len(remaining_blocks),
+                blocks=remaining_blocks,
+            )
+        remaining_blocks = remaining_blocks[:-1]
+
+    return ContextPack(text="", evidence_count=0, blocks=[])

biblicus/corpus.py

@@ -27,7 +27,14 @@ from .frontmatter import parse_front_matter, render_front_matter
 from .hook_manager import HookManager
 from .hooks import HookPoint
 from .ignore import load_corpus_ignore_spec
-from .models import CatalogItem, CorpusCatalog, CorpusConfig, IngestResult, RetrievalRun
+from .models import (
+    CatalogItem,
+    CorpusCatalog,
+    CorpusConfig,
+    ExtractionRunListEntry,
+    IngestResult,
+    RetrievalRun,
+)
 from .sources import load_source
 from .time import utc_now_iso
 from .uris import corpus_ref_to_path, normalize_corpus_uri
@@ -567,6 +574,96 @@ class Corpus:
             return None
         return path.read_text(encoding="utf-8")
 
+    def load_extraction_run_manifest(self, *, extractor_id: str, run_id: str):
+        """
+        Load an extraction run manifest from the corpus.
+
+        :param extractor_id: Extractor plugin identifier.
+        :type extractor_id: str
+        :param run_id: Extraction run identifier.
+        :type run_id: str
+        :return: Parsed extraction run manifest.
+        :rtype: biblicus.extraction.ExtractionRunManifest
+        :raises FileNotFoundError: If the manifest file does not exist.
+        :raises ValueError: If the manifest data is invalid.
+        """
+        from .extraction import ExtractionRunManifest
+
+        manifest_path = (
+            self.extraction_run_dir(extractor_id=extractor_id, run_id=run_id) / "manifest.json"
+        )
+        if not manifest_path.is_file():
+            raise FileNotFoundError(f"Missing extraction run manifest: {manifest_path}")
+        data = json.loads(manifest_path.read_text(encoding="utf-8"))
+        return ExtractionRunManifest.model_validate(data)
+
+    def list_extraction_runs(self, *, extractor_id: Optional[str] = None) -> List[ExtractionRunListEntry]:
+        """
+        List extraction runs stored under the corpus.
+
+        :param extractor_id: Optional extractor identifier filter.
+        :type extractor_id: str or None
+        :return: Summary list entries for each run.
+        :rtype: list[biblicus.models.ExtractionRunListEntry]
+        """
+        runs_root = self.extraction_runs_dir
+        if not runs_root.is_dir():
+            return []
+
+        extractor_dirs: List[Path]
+        if extractor_id is None:
+            extractor_dirs = [path for path in sorted(runs_root.iterdir()) if path.is_dir()]
+        else:
+            extractor_path = runs_root / extractor_id
+            extractor_dirs = [extractor_path] if extractor_path.is_dir() else []
+
+        entries: List[ExtractionRunListEntry] = []
+        for extractor_dir in extractor_dirs:
+            for run_dir in sorted(extractor_dir.iterdir()):
+                if not run_dir.is_dir():
+                    continue
+                manifest_path = run_dir / "manifest.json"
+                if not manifest_path.is_file():
+                    continue
+                try:
+                    manifest = self.load_extraction_run_manifest(
+                        extractor_id=extractor_dir.name,
+                        run_id=run_dir.name,
+                    )
+                except (FileNotFoundError, ValueError):
+                    continue
+                entries.append(
+                    ExtractionRunListEntry(
+                        extractor_id=extractor_dir.name,
+                        run_id=run_dir.name,
+                        recipe_id=manifest.recipe.recipe_id,
+                        recipe_name=manifest.recipe.name,
+                        catalog_generated_at=manifest.catalog_generated_at,
+                        created_at=manifest.created_at,
+                        stats=dict(manifest.stats),
+                    )
+                )
+
+        entries.sort(key=lambda entry: (entry.created_at, entry.extractor_id, entry.run_id), reverse=True)
+        return entries
+
+    def delete_extraction_run(self, *, extractor_id: str, run_id: str) -> None:
+        """
+        Delete an extraction run directory and its derived artifacts.
+
+        :param extractor_id: Extractor plugin identifier.
+        :type extractor_id: str
+        :param run_id: Extraction run identifier.
+        :type run_id: str
+        :return: None.
+        :rtype: None
+        :raises FileNotFoundError: If the extraction run directory does not exist.
+        """
+        run_dir = self.extraction_run_dir(extractor_id=extractor_id, run_id=run_id)
+        if not run_dir.is_dir():
+            raise FileNotFoundError(f"Missing extraction run directory: {run_dir}")
+        shutil.rmtree(run_dir)
+
     def _ensure_runs_dir(self) -> None:
         """
         Ensure the retrieval runs directory exists.
@@ -1185,6 +1282,78 @@ class Corpus:
             raise KeyError(f"Unknown item identifier: {item_id}")
         return item
 
+    def create_crawl_id(self) -> str:
+        """
+        Create a new crawl identifier.
+
+        :return: Crawl identifier.
+        :rtype: str
+        """
+        return str(uuid.uuid4())
+
+    def ingest_crawled_payload(
+        self,
+        *,
+        crawl_id: str,
+        relative_path: str,
+        data: bytes,
+        filename: str,
+        media_type: str,
+        source_uri: str,
+        tags: Sequence[str],
+    ) -> None:
+        """
+        Ingest a crawled payload under a crawl import namespace.
+
+        :param crawl_id: Crawl identifier used to group crawled artifacts.
+        :type crawl_id: str
+        :param relative_path: Relative path within the crawl prefix.
+        :type relative_path: str
+        :param data: Raw payload bytes.
+        :type data: bytes
+        :param filename: Suggested filename from the payload metadata.
+        :type filename: str
+        :param media_type: Internet Assigned Numbers Authority media type.
+        :type media_type: str
+        :param source_uri: Source uniform resource identifier (typically an http or https uniform resource locator).
+        :type source_uri: str
+        :param tags: Tags to attach to the stored item.
+        :type tags: Sequence[str]
+        :return: None.
+        :rtype: None
+        """
+        _ = filename
+        item_id = str(uuid.uuid4())
+        destination_relpath = str(Path(DEFAULT_RAW_DIR) / "imports" / "crawl" / crawl_id / relative_path)
+        destination_path = (self.root / destination_relpath).resolve()
+        destination_path.parent.mkdir(parents=True, exist_ok=True)
+        destination_path.write_bytes(data)
+
+        sha256_digest = _sha256_bytes(data)
+
+        sidecar: Dict[str, Any] = {}
+        sidecar["tags"] = [t.strip() for t in tags if isinstance(t, str) and t.strip()]
+        sidecar["media_type"] = media_type
+        sidecar["biblicus"] = {"id": item_id, "source": source_uri}
+        _write_sidecar(destination_path, sidecar)
+
+        merged_metadata = _merge_metadata({}, sidecar)
+        resolved_tags = _merge_tags([], merged_metadata.get("tags"))
+
+        item_record = CatalogItem(
+            id=item_id,
+            relpath=destination_relpath,
+            sha256=sha256_digest,
+            bytes=len(data),
+            media_type=media_type,
+            title=None,
+            tags=list(resolved_tags),
+            metadata=dict(merged_metadata or {}),
+            created_at=utc_now_iso(),
+            source_uri=source_uri,
+        )
+        self._upsert_catalog_item(item_record)
+
     def reindex(self) -> Dict[str, int]:
         """
         Rebuild/refresh the corpus catalog from the current on-disk corpus contents.