biblicus 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

biblicus/__init__.py

@@ -25,4 +25,4 @@ __all__ = [
     "RetrievalRun",
 ]
 
-__version__ = "0.3.0"
+__version__ = "0.4.0"

biblicus/cli.py

@@ -14,10 +14,11 @@ from pydantic import ValidationError
 
 from .backends import get_backend
 from .corpus import Corpus
+from .crawl import CrawlRequest, crawl_into_corpus
 from .errors import ExtractionRunFatalError
 from .evaluation import evaluate_run, load_dataset
 from .extraction import build_extraction_run
-from .models import QueryBudget
+from .models import QueryBudget, parse_extraction_run_reference
 from .uris import corpus_ref_to_path
 
 
@@ -327,7 +328,7 @@ def cmd_build(arguments: argparse.Namespace) -> int:
     return 0
 
 
-def cmd_extract(arguments: argparse.Namespace) -> int:
+def cmd_extract_build(arguments: argparse.Namespace) -> int:
     """
     Build a text extraction run for the corpus using a pipeline of extractors.
 
@@ -359,6 +360,69 @@ def cmd_extract(arguments: argparse.Namespace) -> int:
     return 0
 
 
+def cmd_extract_list(arguments: argparse.Namespace) -> int:
+    """
+    List extraction runs stored under the corpus.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    corpus = (
+        Corpus.open(arguments.corpus)
+        if getattr(arguments, "corpus", None)
+        else Corpus.find(Path.cwd())
+    )
+    runs = corpus.list_extraction_runs(extractor_id=arguments.extractor_id)
+    print(json.dumps([entry.model_dump() for entry in runs], indent=2))
+    return 0
+
+
+def cmd_extract_show(arguments: argparse.Namespace) -> int:
+    """
+    Show an extraction run manifest.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    corpus = (
+        Corpus.open(arguments.corpus)
+        if getattr(arguments, "corpus", None)
+        else Corpus.find(Path.cwd())
+    )
+    reference = parse_extraction_run_reference(arguments.run)
+    manifest = corpus.load_extraction_run_manifest(
+        extractor_id=reference.extractor_id, run_id=reference.run_id
+    )
+    print(manifest.model_dump_json(indent=2))
+    return 0
+
+
+def cmd_extract_delete(arguments: argparse.Namespace) -> int:
+    """
+    Delete an extraction run directory and its derived artifacts.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    corpus = (
+        Corpus.open(arguments.corpus)
+        if getattr(arguments, "corpus", None)
+        else Corpus.find(Path.cwd())
+    )
+    if arguments.confirm != arguments.run:
+        raise ValueError("Refusing to delete extraction run without an exact --confirm match.")
+    reference = parse_extraction_run_reference(arguments.run)
+    corpus.delete_extraction_run(extractor_id=reference.extractor_id, run_id=reference.run_id)
+    print(json.dumps({"deleted": True, "run": arguments.run}, indent=2))
+    return 0
+
+
 def cmd_query(arguments: argparse.Namespace) -> int:
     """
     Execute a retrieval query.
@@ -414,6 +478,32 @@ def cmd_eval(arguments: argparse.Namespace) -> int:
     return 0
 
 
+def cmd_crawl(arguments: argparse.Namespace) -> int:
+    """
+    Crawl a website prefix into a corpus.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    corpus = (
+        Corpus.open(arguments.corpus)
+        if getattr(arguments, "corpus", None)
+        else Corpus.find(Path.cwd())
+    )
+    tags = _parse_tags(arguments.tags, arguments.tag)
+    request = CrawlRequest(
+        root_url=arguments.root_url,
+        allowed_prefix=arguments.allowed_prefix,
+        max_items=arguments.max_items,
+        tags=tags,
+    )
+    result = crawl_into_corpus(corpus=corpus, request=request)
+    print(result.model_dump_json(indent=2))
+    return 0
+
+
 def build_parser() -> argparse.ArgumentParser:
     """
     Build the command-line interface argument parser.
@@ -511,16 +601,53 @@ def build_parser() -> argparse.ArgumentParser:
     )
     p_build.set_defaults(func=cmd_build)
 
-    p_extract = sub.add_parser("extract", help="Build a text extraction run for the corpus.")
-    _add_common_corpus_arg(p_extract)
-    p_extract.add_argument("--recipe-name", default="default", help="Human-readable recipe name.")
-    p_extract.add_argument(
+    p_extract = sub.add_parser("extract", help="Work with text extraction runs for the corpus.")
+    extract_sub = p_extract.add_subparsers(dest="extract_command", required=True)
+
+    p_extract_build = extract_sub.add_parser("build", help="Build a text extraction run.")
+    _add_common_corpus_arg(p_extract_build)
+    p_extract_build.add_argument(
+        "--recipe-name", default="default", help="Human-readable recipe name."
+    )
+    p_extract_build.add_argument(
         "--step",
         action="append",
         default=None,
         help="Pipeline step spec in the form extractor_id or extractor_id:key=value,key=value (repeatable).",
     )
-    p_extract.set_defaults(func=cmd_extract)
+    p_extract_build.set_defaults(func=cmd_extract_build)
+
+    p_extract_list = extract_sub.add_parser("list", help="List extraction runs.")
+    _add_common_corpus_arg(p_extract_list)
+    p_extract_list.add_argument(
+        "--extractor-id",
+        default=None,
+        help="Optional extractor identifier filter (for example: pipeline).",
+    )
+    p_extract_list.set_defaults(func=cmd_extract_list)
+
+    p_extract_show = extract_sub.add_parser("show", help="Show an extraction run manifest.")
+    _add_common_corpus_arg(p_extract_show)
+    p_extract_show.add_argument(
+        "--run",
+        required=True,
+        help="Extraction run reference in the form extractor_id:run_id.",
+    )
+    p_extract_show.set_defaults(func=cmd_extract_show)
+
+    p_extract_delete = extract_sub.add_parser("delete", help="Delete an extraction run directory.")
+    _add_common_corpus_arg(p_extract_delete)
+    p_extract_delete.add_argument(
+        "--run",
+        required=True,
+        help="Extraction run reference in the form extractor_id:run_id.",
+    )
+    p_extract_delete.add_argument(
+        "--confirm",
+        required=True,
+        help="Type the exact extractor_id:run_id to confirm deletion.",
+    )
+    p_extract_delete.set_defaults(func=cmd_extract_delete)
 
     p_query = sub.add_parser("query", help="Run a retrieval query.")
     _add_common_corpus_arg(p_query)
@@ -545,6 +672,19 @@ def build_parser() -> argparse.ArgumentParser:
     p_eval.add_argument("--max-items-per-source", type=int, default=5)
     p_eval.set_defaults(func=cmd_eval)
 
+    p_crawl = sub.add_parser("crawl", help="Crawl a website prefix into the corpus.")
+    _add_common_corpus_arg(p_crawl)
+    p_crawl.add_argument("--root-url", required=True, help="Root uniform resource locator to fetch.")
+    p_crawl.add_argument(
+        "--allowed-prefix",
+        required=True,
+        help="Uniform resource locator prefix that limits which links are eligible for crawl.",
+    )
+    p_crawl.add_argument("--max-items", type=int, default=50, help="Maximum number of items to store.")
+    p_crawl.add_argument("--tags", default=None, help="Comma-separated tags to apply to stored items.")
+    p_crawl.add_argument("--tag", action="append", help="Repeatable tag to apply to stored items.")
+    p_crawl.set_defaults(func=cmd_crawl)
+
     return parser
 
 

biblicus/corpus.py

@@ -27,7 +27,14 @@ from .frontmatter import parse_front_matter, render_front_matter
 from .hook_manager import HookManager
 from .hooks import HookPoint
 from .ignore import load_corpus_ignore_spec
-from .models import CatalogItem, CorpusCatalog, CorpusConfig, IngestResult, RetrievalRun
+from .models import (
+    CatalogItem,
+    CorpusCatalog,
+    CorpusConfig,
+    ExtractionRunListEntry,
+    IngestResult,
+    RetrievalRun,
+)
 from .sources import load_source
 from .time import utc_now_iso
 from .uris import corpus_ref_to_path, normalize_corpus_uri
@@ -567,6 +574,96 @@ class Corpus:
             return None
         return path.read_text(encoding="utf-8")
 
+    def load_extraction_run_manifest(self, *, extractor_id: str, run_id: str):
+        """
+        Load an extraction run manifest from the corpus.
+
+        :param extractor_id: Extractor plugin identifier.
+        :type extractor_id: str
+        :param run_id: Extraction run identifier.
+        :type run_id: str
+        :return: Parsed extraction run manifest.
+        :rtype: biblicus.extraction.ExtractionRunManifest
+        :raises FileNotFoundError: If the manifest file does not exist.
+        :raises ValueError: If the manifest data is invalid.
+        """
+        from .extraction import ExtractionRunManifest
+
+        manifest_path = (
+            self.extraction_run_dir(extractor_id=extractor_id, run_id=run_id) / "manifest.json"
+        )
+        if not manifest_path.is_file():
+            raise FileNotFoundError(f"Missing extraction run manifest: {manifest_path}")
+        data = json.loads(manifest_path.read_text(encoding="utf-8"))
+        return ExtractionRunManifest.model_validate(data)
+
+    def list_extraction_runs(self, *, extractor_id: Optional[str] = None) -> List[ExtractionRunListEntry]:
+        """
+        List extraction runs stored under the corpus.
+
+        :param extractor_id: Optional extractor identifier filter.
+        :type extractor_id: str or None
+        :return: Summary list entries for each run.
+        :rtype: list[biblicus.models.ExtractionRunListEntry]
+        """
+        runs_root = self.extraction_runs_dir
+        if not runs_root.is_dir():
+            return []
+
+        extractor_dirs: List[Path]
+        if extractor_id is None:
+            extractor_dirs = [path for path in sorted(runs_root.iterdir()) if path.is_dir()]
+        else:
+            extractor_path = runs_root / extractor_id
+            extractor_dirs = [extractor_path] if extractor_path.is_dir() else []
+
+        entries: List[ExtractionRunListEntry] = []
+        for extractor_dir in extractor_dirs:
+            for run_dir in sorted(extractor_dir.iterdir()):
+                if not run_dir.is_dir():
+                    continue
+                manifest_path = run_dir / "manifest.json"
+                if not manifest_path.is_file():
+                    continue
+                try:
+                    manifest = self.load_extraction_run_manifest(
+                        extractor_id=extractor_dir.name,
+                        run_id=run_dir.name,
+                    )
+                except (FileNotFoundError, ValueError):
+                    continue
+                entries.append(
+                    ExtractionRunListEntry(
+                        extractor_id=extractor_dir.name,
+                        run_id=run_dir.name,
+                        recipe_id=manifest.recipe.recipe_id,
+                        recipe_name=manifest.recipe.name,
+                        catalog_generated_at=manifest.catalog_generated_at,
+                        created_at=manifest.created_at,
+                        stats=dict(manifest.stats),
+                    )
+                )
+
+        entries.sort(key=lambda entry: (entry.created_at, entry.extractor_id, entry.run_id), reverse=True)
+        return entries
+
+    def delete_extraction_run(self, *, extractor_id: str, run_id: str) -> None:
+        """
+        Delete an extraction run directory and its derived artifacts.
+
+        :param extractor_id: Extractor plugin identifier.
+        :type extractor_id: str
+        :param run_id: Extraction run identifier.
+        :type run_id: str
+        :return: None.
+        :rtype: None
+        :raises FileNotFoundError: If the extraction run directory does not exist.
+        """
+        run_dir = self.extraction_run_dir(extractor_id=extractor_id, run_id=run_id)
+        if not run_dir.is_dir():
+            raise FileNotFoundError(f"Missing extraction run directory: {run_dir}")
+        shutil.rmtree(run_dir)
+
     def _ensure_runs_dir(self) -> None:
         """
         Ensure the retrieval runs directory exists.
@@ -1185,6 +1282,78 @@ class Corpus:
             raise KeyError(f"Unknown item identifier: {item_id}")
         return item
 
+    def create_crawl_id(self) -> str:
+        """
+        Create a new crawl identifier.
+
+        :return: Crawl identifier.
+        :rtype: str
+        """
+        return str(uuid.uuid4())
+
+    def ingest_crawled_payload(
+        self,
+        *,
+        crawl_id: str,
+        relative_path: str,
+        data: bytes,
+        filename: str,
+        media_type: str,
+        source_uri: str,
+        tags: Sequence[str],
+    ) -> None:
+        """
+        Ingest a crawled payload under a crawl import namespace.
+
+        :param crawl_id: Crawl identifier used to group crawled artifacts.
+        :type crawl_id: str
+        :param relative_path: Relative path within the crawl prefix.
+        :type relative_path: str
+        :param data: Raw payload bytes.
+        :type data: bytes
+        :param filename: Suggested filename from the payload metadata.
+        :type filename: str
+        :param media_type: Internet Assigned Numbers Authority media type.
+        :type media_type: str
+        :param source_uri: Source uniform resource identifier (typically an http or https uniform resource locator).
+        :type source_uri: str
+        :param tags: Tags to attach to the stored item.
+        :type tags: Sequence[str]
+        :return: None.
+        :rtype: None
+        """
+        _ = filename
+        item_id = str(uuid.uuid4())
+        destination_relpath = str(Path(DEFAULT_RAW_DIR) / "imports" / "crawl" / crawl_id / relative_path)
+        destination_path = (self.root / destination_relpath).resolve()
+        destination_path.parent.mkdir(parents=True, exist_ok=True)
+        destination_path.write_bytes(data)
+
+        sha256_digest = _sha256_bytes(data)
+
+        sidecar: Dict[str, Any] = {}
+        sidecar["tags"] = [t.strip() for t in tags if isinstance(t, str) and t.strip()]
+        sidecar["media_type"] = media_type
+        sidecar["biblicus"] = {"id": item_id, "source": source_uri}
+        _write_sidecar(destination_path, sidecar)
+
+        merged_metadata = _merge_metadata({}, sidecar)
+        resolved_tags = _merge_tags([], merged_metadata.get("tags"))
+
+        item_record = CatalogItem(
+            id=item_id,
+            relpath=destination_relpath,
+            sha256=sha256_digest,
+            bytes=len(data),
+            media_type=media_type,
+            title=None,
+            tags=list(resolved_tags),
+            metadata=dict(merged_metadata or {}),
+            created_at=utc_now_iso(),
+            source_uri=source_uri,
+        )
+        self._upsert_catalog_item(item_record)
+
     def reindex(self) -> Dict[str, int]:
         """
         Rebuild/refresh the corpus catalog from the current on-disk corpus contents.

biblicus/crawl.py

@@ -0,0 +1,186 @@
+"""
+Website crawl utilities for Biblicus corpora.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from html.parser import HTMLParser
+from typing import Deque, List, Optional, Set
+from urllib.parse import urldefrag, urljoin
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from .ignore import load_corpus_ignore_spec
+from .sources import load_source
+
+
+class CrawlRequest(BaseModel):
+    """
+    Request describing a website crawl into a corpus.
+
+    :ivar root_url: Initial uniform resource locator to fetch.
+    :vartype root_url: str
+    :ivar allowed_prefix: Uniform resource locator prefix that limits which links are eligible for crawl.
+    :vartype allowed_prefix: str
+    :ivar max_items: Maximum number of items to store during the crawl.
+    :vartype max_items: int
+    :ivar tags: Tags to apply to stored items.
+    :vartype tags: list[str]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    root_url: str = Field(min_length=1)
+    allowed_prefix: str = Field(min_length=1)
+    max_items: int = Field(default=50, ge=1)
+    tags: List[str] = Field(default_factory=list)
+
+
+class CrawlResult(BaseModel):
+    """
+    Summary result for a crawl execution.
+
+    :ivar crawl_id: Crawl identifier used in the corpus raw import namespace.
+    :vartype crawl_id: str
+    :ivar discovered_items: Total number of distinct uniform resource locators discovered.
+    :vartype discovered_items: int
+    :ivar fetched_items: Number of eligible items fetched over hypertext transfer protocol.
+    :vartype fetched_items: int
+    :ivar stored_items: Number of items stored into the corpus.
+    :vartype stored_items: int
+    :ivar skipped_outside_prefix_items: Number of discovered items outside the allowed prefix.
+    :vartype skipped_outside_prefix_items: int
+    :ivar skipped_ignored_items: Number of eligible items skipped due to corpus ignore rules.
+    :vartype skipped_ignored_items: int
+    :ivar errored_items: Number of eligible items that failed to fetch or store.
+    :vartype errored_items: int
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    crawl_id: str
+    discovered_items: int = Field(default=0, ge=0)
+    fetched_items: int = Field(default=0, ge=0)
+    stored_items: int = Field(default=0, ge=0)
+    skipped_outside_prefix_items: int = Field(default=0, ge=0)
+    skipped_ignored_items: int = Field(default=0, ge=0)
+    errored_items: int = Field(default=0, ge=0)
+
+
+class _LinkExtractor(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__()
+        self.links: List[str] = []
+
+    def handle_starttag(self, tag: str, attrs):  # type: ignore[no-untyped-def]
+        _ = tag
+        for key, value in attrs:
+            if key in {"href", "src"} and isinstance(value, str) and value.strip():
+                self.links.append(value.strip())
+
+
+def _normalize_crawl_url(candidate: str, *, base_url: str) -> Optional[str]:
+    joined = urljoin(base_url, candidate)
+    joined, _fragment = urldefrag(joined)
+    joined = joined.strip()
+    if joined.startswith(("mailto:", "javascript:")):
+        return None
+    return joined
+
+
+def _crawl_relative_path(url: str, *, allowed_prefix: str) -> str:
+    relative = url[len(allowed_prefix) :].lstrip("/")
+    if not relative or relative.endswith("/"):
+        relative = relative.rstrip("/") + "/index.html" if relative else "index.html"
+    return relative
+
+
+def _should_parse_links(media_type: str) -> bool:
+    return media_type.startswith("text/html")
+
+
+def _discover_links(html_text: str, *, base_url: str) -> List[str]:
+    parser = _LinkExtractor()
+    parser.feed(html_text)
+    discovered: List[str] = []
+    for raw in parser.links:
+        normalized = _normalize_crawl_url(raw, base_url=base_url)
+        if normalized is not None:
+            discovered.append(normalized)
+    return discovered
+
+
+def crawl_into_corpus(*, corpus, request: CrawlRequest) -> CrawlResult:  # type: ignore[no-untyped-def]
+    """
+    Crawl a website prefix into a corpus.
+
+    :param corpus: Target corpus to receive crawled items.
+    :type corpus: biblicus.corpus.Corpus
+    :param request: Crawl request describing limits and allowed prefix.
+    :type request: CrawlRequest
+    :return: Crawl result summary.
+    :rtype: CrawlResult
+    """
+    ignore_spec = load_corpus_ignore_spec(corpus.root)
+    allowed_prefix = request.allowed_prefix
+    root_url = request.root_url
+
+    crawl_id = corpus.create_crawl_id()
+
+    queue: Deque[str] = deque([root_url])
+    seen: Set[str] = set()
+    stored_count = 0
+    fetched_count = 0
+    skipped_outside_prefix_count = 0
+    skipped_ignored_count = 0
+    errored_count = 0
+    discovered_urls: Set[str] = set()
+
+    while queue and stored_count < request.max_items:
+        url = queue.popleft()
+        if url in seen:
+            continue
+        seen.add(url)
+        discovered_urls.add(url)
+
+        if not url.startswith(allowed_prefix):
+            skipped_outside_prefix_count += 1
+            continue
+
+        relative_path = _crawl_relative_path(url, allowed_prefix=allowed_prefix)
+        if ignore_spec.matches(relative_path):
+            skipped_ignored_count += 1
+            continue
+
+        try:
+            payload = load_source(url)
+            fetched_count += 1
+            corpus.ingest_crawled_payload(
+                crawl_id=crawl_id,
+                relative_path=relative_path,
+                data=payload.data,
+                filename=payload.filename,
+                media_type=payload.media_type,
+                source_uri=payload.source_uri,
+                tags=request.tags,
+            )
+            stored_count += 1
+        except Exception:
+            errored_count += 1
+            continue
+
+        if _should_parse_links(payload.media_type):
+            text = payload.data.decode("utf-8", errors="replace")
+            for discovered in _discover_links(text, base_url=url):
+                queue.append(discovered)
+
+    return CrawlResult(
+        crawl_id=crawl_id,
+        discovered_items=len(discovered_urls),
+        fetched_items=fetched_count,
+        stored_items=stored_count,
+        skipped_outside_prefix_items=skipped_outside_prefix_count,
+        skipped_ignored_items=skipped_ignored_count,
+        errored_items=errored_count,
+    )

biblicus/extraction.py

@@ -7,7 +7,6 @@ from __future__ import annotations
 import json
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
-from uuid import uuid4
 
 from pydantic import BaseModel, ConfigDict, Field
 
@@ -196,8 +195,9 @@ def create_extraction_run_manifest(
     :rtype: ExtractionRunManifest
     """
     catalog = corpus.load_catalog()
+    run_id = hash_text(f"{recipe.recipe_id}:{catalog.generated_at}")
     return ExtractionRunManifest(
-        run_id=str(uuid4()),
+        run_id=run_id,
         recipe=recipe,
         corpus_uri=corpus.uri,
         catalog_generated_at=catalog.generated_at,
@@ -341,6 +341,8 @@ def build_extraction_run(
     )
     manifest = create_extraction_run_manifest(corpus, recipe=recipe)
     run_dir = corpus.extraction_run_dir(extractor_id=extractor_id, run_id=manifest.run_id)
+    if run_dir.exists():
+        return corpus.load_extraction_run_manifest(extractor_id=extractor_id, run_id=manifest.run_id)
     run_dir.mkdir(parents=True, exist_ok=False)
 
     catalog = corpus.load_catalog()

biblicus/models.py

@@ -189,6 +189,37 @@ def parse_extraction_run_reference(value: str) -> ExtractionRunReference:
     return ExtractionRunReference(extractor_id=extractor_id, run_id=run_id)
 
 
+class ExtractionRunListEntry(BaseModel):
+    """
+    Summary entry for an extraction run stored in a corpus.
+
+    :ivar extractor_id: Extractor plugin identifier.
+    :vartype extractor_id: str
+    :ivar run_id: Extraction run identifier.
+    :vartype run_id: str
+    :ivar recipe_id: Deterministic recipe identifier.
+    :vartype recipe_id: str
+    :ivar recipe_name: Human-readable recipe name.
+    :vartype recipe_name: str
+    :ivar catalog_generated_at: Catalog timestamp used for the run.
+    :vartype catalog_generated_at: str
+    :ivar created_at: International Organization for Standardization 8601 timestamp for run creation.
+    :vartype created_at: str
+    :ivar stats: Run statistics.
+    :vartype stats: dict[str, object]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    extractor_id: str = Field(min_length=1)
+    run_id: str = Field(min_length=1)
+    recipe_id: str = Field(min_length=1)
+    recipe_name: str = Field(min_length=1)
+    catalog_generated_at: str = Field(min_length=1)
+    created_at: str = Field(min_length=1)
+    stats: Dict[str, object] = Field(default_factory=dict)
+
+
 class QueryBudget(BaseModel):
     """
     Evidence selection budget for retrieval.

biblicus/time.py

@@ -14,4 +14,4 @@ def utc_now_iso() -> str:
     :return: Current Coordinated Universal Time timestamp in International Organization for Standardization 8601 format.
     :rtype: str
     """
-    return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+    return datetime.now(timezone.utc).isoformat(timespec="microseconds")

{biblicus-0.3.0.dist-info → biblicus-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: biblicus
-Version: 0.3.0
+Version: 0.4.0
 Summary: Command line interface and Python library for corpus ingestion, retrieval, and evaluation.
 License: MIT
 Requires-Python: >=3.9
@@ -77,10 +77,7 @@ flowchart LR
     direction LR
     LegendArtifact[Stored artifact or evidence]
     LegendStep[Step]
-    LegendStable[Stable region]
-    LegendPluggable[Pluggable region]
     LegendArtifact --- LegendStep
-    LegendStable --- LegendPluggable
   end
 
   subgraph Main[" "]
@@ -93,14 +90,14 @@ flowchart LR
       Raw --> Catalog[Catalog file]
     end
 
-    subgraph PluggableExtractionPipeline[Pluggable extraction pipeline]
+    subgraph PluggableExtractionPipeline[Pluggable: extraction pipeline]
       direction TB
       Catalog --> Extract[Extract pipeline]
       Extract --> ExtractedText[Extracted text artifacts]
       ExtractedText --> ExtractionRun[Extraction run manifest]
     end
 
-    subgraph PluggableRetrievalBackend[Pluggable retrieval backend]
+    subgraph PluggableRetrievalBackend[Pluggable: retrieval backend]
       direction LR
 
       subgraph BackendIngestionIndexing[Ingestion and indexing]
@@ -154,8 +151,6 @@ flowchart LR
   style Main fill:#ffffff,stroke:#ffffff,color:#111111
   style LegendArtifact fill:#f3e5f5,stroke:#8e24aa,color:#111111
   style LegendStep fill:#eceff1,stroke:#90a4ae,color:#111111
-  style LegendStable fill:#ffffff,stroke:#8e24aa,stroke-width:2px,color:#111111
-  style LegendPluggable fill:#ffffff,stroke:#1e88e5,stroke-dasharray:6 3,stroke-width:2px,color:#111111
 ```
 
 ## Practical value
@@ -168,6 +163,7 @@ flowchart LR
 
 - Initialize a corpus folder.
 - Ingest items from file paths, web addresses, or text input.
+- Crawl a website section into corpus items when you want a repeatable “import from the web” workflow.
 - Run extraction when you want derived text artifacts from non-text sources.
 - Reindex to refresh the catalog after edits.
 - Build a retrieval run with a backend.
@@ -205,11 +201,22 @@ biblicus init corpora/example
 biblicus ingest --corpus corpora/example notes/example.txt
 echo "A short note" | biblicus ingest --corpus corpora/example --stdin --title "First note"
 biblicus list --corpus corpora/example
-biblicus extract --corpus corpora/example --step pass-through-text --step metadata-text
+biblicus extract build --corpus corpora/example --step pass-through-text --step metadata-text
+biblicus extract list --corpus corpora/example
 biblicus build --corpus corpora/example --backend scan
 biblicus query --corpus corpora/example --query "note"
 ```
 
+If you want to turn a website section into corpus items, crawl a root web address while restricting the crawl to an allowed prefix:
+
+```
+biblicus crawl --corpus corpora/example \\
+  --root-url https://example.com/docs/index.html \\
+  --allowed-prefix https://example.com/docs/ \\
+  --max-items 50 \\
+  --tag crawled
+```
+
 ## Python usage
 
 From Python, the same flow is available through the Corpus class and backend interfaces. The public surface area is small on purpose.
@@ -233,7 +240,7 @@ In an assistant system, retrieval usually produces context for a model call. Thi
 
 ## Learn more
 
-Full documentation is available on [ReadTheDocs](https://biblicus.readthedocs.io/).
+Full documentation is published on GitHub Pages: https://anthusai.github.io/Biblicus/
 
 The documents below are written to be read in order.
 
@@ -262,7 +269,16 @@ corpus/
     config.json
     catalog.json
     runs/
-      run-id.json
+      extraction/
+        pipeline/
+          <run id>/
+            manifest.json
+            text/
+              <item id>.txt
+      retrieval/
+        <backend id>/
+          <run id>/
+            manifest.json
 ```
 
 ## Retrieval backends
@@ -313,7 +329,7 @@ python3 -m pip install -e ".[dev]"
 Build the documentation:
 
 ```
-python3 -m sphinx -b html docs docs/_build
+python3 -m sphinx -b html docs docs/_build/html
 ```
 
 ## License
@@ -333,4 +349,4 @@ License terms are in `LICENSE`.
 
 [continuous-integration-badge]: https://github.com/AnthusAI/Biblicus/actions/workflows/ci.yml/badge.svg?branch=main
 [coverage-badge]: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/AnthusAI/Biblicus/main/coverage_badge.json
-[documentation-badge]: https://readthedocs.org/projects/biblicus/badge/?version=latest
+[documentation-badge]: https://img.shields.io/badge/docs-GitHub%20Pages-blue

{biblicus-0.3.0.dist-info → biblicus-0.4.0.dist-info}/RECORD

@@ -1,20 +1,21 @@
-biblicus/__init__.py,sha256=1vPJokNgr7JcDO9eJ2SRR8VLkFG44ZaSACSaalogvYQ,432
+biblicus/__init__.py,sha256=6TFpzDiMJlFyBfVrHfS6xnGd8P7Zybj6DxpWkCJqyf4,432
 biblicus/__main__.py,sha256=ipfkUoTlocVnrQDM69C7TeBqQxmHVeiWMRaT3G9rtnk,117
-biblicus/cli.py,sha256=k09mMToSawDC7TbetwtK0RItTLO84EOJCZQKDRA-b9Y,19229
+biblicus/cli.py,sha256=MtYTkJh0lVOTrbwY3u6V8ti5WZUsb96fLatqh23cUYg,24289
 biblicus/constants.py,sha256=R6fZDoLVMCwgKvTaxEx7G0CstwHGaUTlW9MsmNLDZ44,269
-biblicus/corpus.py,sha256=oBg5nbDoDBkkXaW180ixtvU9Yh0y9nOiZDEMKomtrVU,47688
+biblicus/corpus.py,sha256=gF1RNl6fdz7wplzpHEIkEBkhYxHgKTKguBR_kD9IgUw,54109
+biblicus/crawl.py,sha256=n8rXBMnziBK9vtKQQCXYOpBzqsPCswj2PzVJUb370KY,6250
 biblicus/errors.py,sha256=uMajd5DvgnJ_-jq5sbeom1GV8DPUc-kojBaECFi6CsY,467
 biblicus/evaluation.py,sha256=5xWpb-8f49Osh9aHzo1ab3AXOmls3Imc5rdnEC0pN-8,8143
-biblicus/extraction.py,sha256=MYaHkhj0NWBKNcaohnLvNiHLwyps9JyZGaTxX5gHR-A,19281
+biblicus/extraction.py,sha256=VEjBjIpaBboftGgEcpDj7z7um41e5uDZpP_7acQg7fw,19448
 biblicus/frontmatter.py,sha256=JOGjIDzbbOkebQw2RzA-3WDVMAMtJta2INjS4e7-LMg,2463
 biblicus/hook_logging.py,sha256=IMvde-JhVWrx9tNz3eDJ1CY_rr5Sj7DZ2YNomYCZbz0,5366
 biblicus/hook_manager.py,sha256=ZCAkE5wLvn4lnQz8jho_o0HGEC9KdQd9qitkAEUQRcw,6997
 biblicus/hooks.py,sha256=OHQOmOi7rUcQqYWVeod4oPe8nVLepD7F_SlN7O_-BsE,7863
 biblicus/ignore.py,sha256=fyjt34E6tWNNrm1FseOhgH2MgryyVBQVzxhKL5s4aio,1800
-biblicus/models.py,sha256=fdpPRtWmtirjEKpOPL_6ZVRY0vpA2WRqMwNrOqPaauM,14204
+biblicus/models.py,sha256=6SWQ2Czg9O3zjuam8a4m8V3LlEgcGLbEctYDB6F1rRs,15317
 biblicus/retrieval.py,sha256=A1SI4WK5cX-WbtN6FJ0QQxqlEOtQhddLrL0LZIuoTC4,4180
 biblicus/sources.py,sha256=EFy8-rQNLsyzz-98mH-z8gEHMYbqigcNFKLaR92KfDE,7241
-biblicus/time.py,sha256=NEHkJLJ3RH1PdJVAWMYbNCBnCb6UW9DVBLo7Qh1zO88,485
+biblicus/time.py,sha256=3BSKOSo7R10K-0Dzrbdtl3fh5_yShTYqfdlKvvdkx7M,485
 biblicus/uris.py,sha256=xXD77lqsT9NxbyzI1spX9Y5a3-U6sLYMnpeSAV7g-nM,2013
 biblicus/user_config.py,sha256=DqO08yLn82DhTiFpmIyyLj_J0nMbrtE8xieTj2Cgd6A,4287
 biblicus/_vendor/dotyaml/__init__.py,sha256=e4zbejeJRwlD4I0q3YvotMypO19lXqmT8iyU1q6SvhY,376
@@ -36,9 +37,9 @@ biblicus/extractors/rapidocr_text.py,sha256=OMAuZealLSSTFVVmBalT-AFJy2pEpHyyvpuW
 biblicus/extractors/select_longest_text.py,sha256=wRveXAfYLdj7CpGuo4RoD7zE6SIfylRCbv40z2azO0k,3702
 biblicus/extractors/select_text.py,sha256=w0ATmDy3tWWbOObzW87jGZuHbgXllUhotX5XyySLs-o,3395
 biblicus/extractors/unstructured_text.py,sha256=l2S_wD_htu7ZHoJQNQtP-kGlEgOeKV_w2IzAC93lePE,3564
-biblicus-0.3.0.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
-biblicus-0.3.0.dist-info/METADATA,sha256=MHE8tAh9jGiMwk5X9jPSnhRFB6uAZa3T8jo_c1zrIZM,13202
-biblicus-0.3.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-biblicus-0.3.0.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
-biblicus-0.3.0.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
-biblicus-0.3.0.dist-info/RECORD,,
+biblicus-0.4.0.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
+biblicus-0.4.0.dist-info/METADATA,sha256=JMZjfIOMEmbWFHgzjUHsQDUUg11jdxudtJdRK8Iu29U,13586
+biblicus-0.4.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+biblicus-0.4.0.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
+biblicus-0.4.0.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
+biblicus-0.4.0.dist-info/RECORD,,