biblicus 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

biblicus/backends/sqlite_full_text_search.py

@@ -12,9 +12,15 @@ from pydantic import BaseModel, ConfigDict, Field
 
 from ..constants import CORPUS_DIR_NAME, RUNS_DIR_NAME
 from ..corpus import Corpus
-from ..extraction import ExtractionRunReference, parse_extraction_run_reference
 from ..frontmatter import parse_front_matter
-from ..models import Evidence, QueryBudget, RetrievalResult, RetrievalRun
+from ..models import (
+    Evidence,
+    ExtractionRunReference,
+    QueryBudget,
+    RetrievalResult,
+    RetrievalRun,
+    parse_extraction_run_reference,
+)
 from ..retrieval import apply_budget, create_recipe_manifest, create_run_manifest, hash_text
 from ..time import utc_now_iso
 
@@ -51,7 +57,9 @@ class SqliteFullTextSearchBackend:
 
     backend_id = "sqlite-full-text-search"
 
-    def build_run(self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]) -> RetrievalRun:
+    def build_run(
+        self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]
+    ) -> RetrievalRun:
         """
         Build a full-text search version five index for the corpus.
 
@@ -64,7 +72,6 @@ class SqliteFullTextSearchBackend:
         :return: Run manifest describing the build.
         :rtype: RetrievalRun
         """
-
         recipe_config = SqliteFullTextSearchRecipeConfig.model_validate(config)
         catalog = corpus.load_catalog()
         recipe = create_recipe_manifest(
@@ -110,7 +117,6 @@ class SqliteFullTextSearchBackend:
         :return: Retrieval results containing evidence.
         :rtype: RetrievalResult
         """
-
         recipe_config = SqliteFullTextSearchRecipeConfig.model_validate(run.recipe.config)
         db_path = _resolve_run_db_path(corpus, run)
         candidates = _query_full_text_search_index(
@@ -156,7 +162,6 @@ def _candidate_limit(max_total_items: int) -> int:
     :return: Candidate limit for backend search.
     :rtype: int
     """
-
     return max_total_items * 5
 
 
@@ -172,7 +177,6 @@ def _resolve_run_db_path(corpus: Corpus, run: RetrievalRun) -> Path:
     :rtype: Path
     :raises FileNotFoundError: If the run does not have artifact paths.
     """
-
     if not run.artifact_paths:
         raise FileNotFoundError("Run has no artifact paths to query")
     return corpus.root / run.artifact_paths[0]
@@ -188,7 +192,6 @@ def _ensure_full_text_search_version_five(conn: sqlite3.Connection) -> None:
     :rtype: None
     :raises RuntimeError: If full-text search version five support is unavailable.
     """
-
     try:
         cursor = conn.execute(
             "CREATE VIRTUAL TABLE IF NOT EXISTS chunks_full_text_search USING fts5(content)"
@@ -210,7 +213,6 @@ def _create_full_text_search_schema(conn: sqlite3.Connection) -> None:
     :return: None.
     :rtype: None
     """
-
     conn.execute(
         """
         CREATE VIRTUAL TABLE chunks_full_text_search USING fts5(
@@ -249,7 +251,6 @@ def _build_full_text_search_index(
     :return: Index statistics.
     :rtype: dict[str, int]
     """
-
     if db_path.exists():
         db_path.unlink()
     connection = sqlite3.connect(str(db_path))
@@ -339,7 +340,6 @@ def _load_text_from_item(
     :return: Text payload or None if not text.
     :rtype: str or None
     """
-
     if extraction_reference:
         extracted_text = corpus.read_extracted_text(
             extractor_id=extraction_reference.extractor_id,
@@ -375,7 +375,6 @@ def _resolve_extraction_reference(
     :rtype: ExtractionRunReference or None
     :raises FileNotFoundError: If an extraction run is referenced but not present.
     """
-
     if not recipe_config.extraction_run:
         return None
     extraction_reference = parse_extraction_run_reference(recipe_config.extraction_run)
@@ -388,7 +387,9 @@ def _resolve_extraction_reference(
     return extraction_reference
 
 
-def _iter_chunks(text: str, *, chunk_size: int, chunk_overlap: int) -> Iterable[Tuple[int, int, str]]:
+def _iter_chunks(
+    text: str, *, chunk_size: int, chunk_overlap: int
+) -> Iterable[Tuple[int, int, str]]:
     """
     Yield overlapping chunks of text for indexing.
 
@@ -402,7 +403,6 @@ def _iter_chunks(text: str, *, chunk_size: int, chunk_overlap: int) -> Iterable[
     :rtype: Iterable[tuple[int, int, str]]
     :raises ValueError: If the overlap is greater than or equal to the chunk size.
     """
-
     if chunk_overlap >= chunk_size:
         raise ValueError("chunk_overlap must be smaller than chunk_size")
     start_offset = 0
@@ -435,7 +435,6 @@ def _query_full_text_search_index(
     :return: Evidence candidates.
     :rtype: list[Evidence]
     """
-
     connection = sqlite3.connect(str(db_path))
     try:
         rows = connection.execute(

biblicus/cli.py

@@ -14,9 +14,11 @@ from pydantic import ValidationError
 
 from .backends import get_backend
 from .corpus import Corpus
-from .extraction import build_extraction_run
+from .crawl import CrawlRequest, crawl_into_corpus
+from .errors import ExtractionRunFatalError
 from .evaluation import evaluate_run, load_dataset
-from .models import QueryBudget
+from .extraction import build_extraction_run
+from .models import QueryBudget, parse_extraction_run_reference
 from .uris import corpus_ref_to_path
 
 
@@ -29,7 +31,6 @@ def _add_common_corpus_arg(parser: argparse.ArgumentParser) -> None:
     :return: None.
     :rtype: None
     """
-
     parser.add_argument(
         "--corpus",
         type=str,
@@ -51,7 +52,6 @@ def cmd_init(arguments: argparse.Namespace) -> int:
     :return: Exit code.
     :rtype: int
     """
-
     corpus_path = corpus_ref_to_path(arguments.path)
     corpus = Corpus.init(corpus_path, force=arguments.force)
     print(f"Initialized corpus at {corpus.root}")
@@ -69,7 +69,6 @@ def _parse_tags(raw: Optional[str], raw_list: Optional[List[str]]) -> List[str]:
     :return: Deduplicated tag list.
     :rtype: list[str]
     """
-
     parsed_tags: List[str] = []
     if raw:
         parsed_tags.extend([tag.strip() for tag in raw.split(",") if tag.strip()])
@@ -94,7 +93,6 @@ def cmd_ingest(arguments: argparse.Namespace) -> int:
     :return: Exit code.
     :rtype: int
     """
-
     corpus = (
         Corpus.open(arguments.corpus)
         if getattr(arguments, "corpus", None)
@@ -135,7 +133,6 @@ def cmd_list(arguments: argparse.Namespace) -> int:
     :return: Exit code.
     :rtype: int
     """
-
     corpus = (
         Corpus.open(arguments.corpus)
         if getattr(arguments, "corpus", None)
@@ -157,7 +154,6 @@ def cmd_show(arguments: argparse.Namespace) -> int:
     :return: Exit code.
     :rtype: int
     """
-
     corpus = (
         Corpus.open(arguments.corpus)
         if getattr(arguments, "corpus", None)
@@ -177,7 +173,6 @@ def cmd_reindex(arguments: argparse.Namespace) -> int:
     :return: Exit code.
     :rtype: int
     """
-
     corpus = (
         Corpus.open(arguments.corpus)
         if getattr(arguments, "corpus", None)
@@ -197,7 +192,6 @@ def cmd_import_tree(arguments: argparse.Namespace) -> int:
     :return: Exit code.
     :rtype: int
     """
-
     corpus = (
         Corpus.open(arguments.corpus)
         if getattr(arguments, "corpus", None)
@@ -218,7 +212,6 @@ def cmd_purge(arguments: argparse.Namespace) -> int:
     :return: Exit code.
     :rtype: int
     """
-
     corpus = (
         Corpus.open(arguments.corpus)
         if getattr(arguments, "corpus", None)
@@ -241,7 +234,6 @@ def _parse_config_pairs(pairs: Optional[List[str]]) -> Dict[str, object]:
     :rtype: dict[str, object]
     :raises ValueError: If any entry is not key=value.
     """
-
     config: Dict[str, object] = {}
     for item in pairs or []:
         if "=" not in item:
@@ -264,7 +256,7 @@ def _parse_config_pairs(pairs: Optional[List[str]]) -> Dict[str, object]:
 
 def _parse_step_spec(raw_step: str) -> tuple[str, Dict[str, object]]:
     """
-    Parse a cascade step specification.
+    Parse a pipeline step specification.
 
     :param raw_step: Step spec in the form extractor_id or extractor_id:key=value,key=value.
     :type raw_step: str
@@ -272,7 +264,6 @@ def _parse_step_spec(raw_step: str) -> tuple[str, Dict[str, object]]:
     :rtype: tuple[str, dict[str, object]]
     :raises ValueError: If the step spec is invalid.
     """
-
     raw_step = raw_step.strip()
     if not raw_step:
         raise ValueError("Step spec must be non-empty")
@@ -309,7 +300,6 @@ def _budget_from_args(arguments: argparse.Namespace) -> QueryBudget:
     :return: Query budget instance.
     :rtype: QueryBudget
     """
-
     return QueryBudget(
         max_total_items=arguments.max_total_items,
         max_total_characters=arguments.max_total_characters,
@@ -326,7 +316,6 @@ def cmd_build(arguments: argparse.Namespace) -> int:
     :return: Exit code.
     :rtype: int
     """
-
     corpus = (
         Corpus.open(arguments.corpus)
         if getattr(arguments, "corpus", None)
@@ -339,33 +328,31 @@ def cmd_build(arguments: argparse.Namespace) -> int:
     return 0
 
 
-def cmd_extract(arguments: argparse.Namespace) -> int:
+def cmd_extract_build(arguments: argparse.Namespace) -> int:
     """
-    Build a text extraction run for the corpus.
+    Build a text extraction run for the corpus using a pipeline of extractors.
 
     :param arguments: Parsed command-line interface arguments.
     :type arguments: argparse.Namespace
     :return: Exit code.
     :rtype: int
     """
-
     corpus = (
         Corpus.open(arguments.corpus)
         if getattr(arguments, "corpus", None)
         else Corpus.find(Path.cwd())
     )
-    config = _parse_config_pairs(arguments.config)
-    if getattr(arguments, "step", None):
-        if arguments.extractor != "cascade":
-            raise ValueError("--step is only supported for the cascade extractor")
-        steps: List[Dict[str, object]] = []
-        for raw_step in arguments.step:
-            extractor_id, step_config = _parse_step_spec(raw_step)
-            steps.append({"extractor_id": extractor_id, "config": step_config})
-        config = {"steps": steps}
+    raw_steps = list(arguments.step or [])
+    if not raw_steps:
+        raise ValueError("Pipeline extraction requires at least one --step")
+    steps: List[Dict[str, object]] = []
+    for raw_step in raw_steps:
+        extractor_id, step_config = _parse_step_spec(raw_step)
+        steps.append({"extractor_id": extractor_id, "config": step_config})
+    config = {"steps": steps}
     manifest = build_extraction_run(
         corpus,
-        extractor_id=arguments.extractor,
+        extractor_id="pipeline",
         recipe_name=arguments.recipe_name,
         config=config,
     )
@@ -373,6 +360,69 @@ def cmd_extract(arguments: argparse.Namespace) -> int:
     return 0
 
 
+def cmd_extract_list(arguments: argparse.Namespace) -> int:
+    """
+    List extraction runs stored under the corpus.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    corpus = (
+        Corpus.open(arguments.corpus)
+        if getattr(arguments, "corpus", None)
+        else Corpus.find(Path.cwd())
+    )
+    runs = corpus.list_extraction_runs(extractor_id=arguments.extractor_id)
+    print(json.dumps([entry.model_dump() for entry in runs], indent=2))
+    return 0
+
+
+def cmd_extract_show(arguments: argparse.Namespace) -> int:
+    """
+    Show an extraction run manifest.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    corpus = (
+        Corpus.open(arguments.corpus)
+        if getattr(arguments, "corpus", None)
+        else Corpus.find(Path.cwd())
+    )
+    reference = parse_extraction_run_reference(arguments.run)
+    manifest = corpus.load_extraction_run_manifest(
+        extractor_id=reference.extractor_id, run_id=reference.run_id
+    )
+    print(manifest.model_dump_json(indent=2))
+    return 0
+
+
+def cmd_extract_delete(arguments: argparse.Namespace) -> int:
+    """
+    Delete an extraction run directory and its derived artifacts.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    corpus = (
+        Corpus.open(arguments.corpus)
+        if getattr(arguments, "corpus", None)
+        else Corpus.find(Path.cwd())
+    )
+    if arguments.confirm != arguments.run:
+        raise ValueError("Refusing to delete extraction run without an exact --confirm match.")
+    reference = parse_extraction_run_reference(arguments.run)
+    corpus.delete_extraction_run(extractor_id=reference.extractor_id, run_id=reference.run_id)
+    print(json.dumps({"deleted": True, "run": arguments.run}, indent=2))
+    return 0
+
+
 def cmd_query(arguments: argparse.Namespace) -> int:
     """
     Execute a retrieval query.
@@ -382,7 +432,6 @@ def cmd_query(arguments: argparse.Namespace) -> int:
     :return: Exit code.
     :rtype: int
     """
-
     corpus = (
         Corpus.open(arguments.corpus)
         if getattr(arguments, "corpus", None)
@@ -413,7 +462,6 @@ def cmd_eval(arguments: argparse.Namespace) -> int:
     :return: Exit code.
     :rtype: int
     """
-
     corpus = (
         Corpus.open(arguments.corpus)
         if getattr(arguments, "corpus", None)
@@ -430,6 +478,32 @@ def cmd_eval(arguments: argparse.Namespace) -> int:
     return 0
 
 
+def cmd_crawl(arguments: argparse.Namespace) -> int:
+    """
+    Crawl a website prefix into a corpus.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    corpus = (
+        Corpus.open(arguments.corpus)
+        if getattr(arguments, "corpus", None)
+        else Corpus.find(Path.cwd())
+    )
+    tags = _parse_tags(arguments.tags, arguments.tag)
+    request = CrawlRequest(
+        root_url=arguments.root_url,
+        allowed_prefix=arguments.allowed_prefix,
+        max_items=arguments.max_items,
+        tags=tags,
+    )
+    result = crawl_into_corpus(corpus=corpus, request=request)
+    print(result.model_dump_json(indent=2))
+    return 0
+
+
 def build_parser() -> argparse.ArgumentParser:
     """
     Build the command-line interface argument parser.
@@ -437,7 +511,6 @@ def build_parser() -> argparse.ArgumentParser:
     :return: Argument parser instance.
     :rtype: argparse.ArgumentParser
     """
-
     parser = argparse.ArgumentParser(
         prog="biblicus",
         description="Biblicus command-line interface (minimum viable product)",
@@ -457,14 +530,18 @@ def build_parser() -> argparse.ArgumentParser:
 
     p_init = sub.add_parser("init", help="Initialize a new corpus at PATH.")
     p_init.add_argument("path", help="Corpus path or file:// uniform resource identifier.")
-    p_init.add_argument("--force", action="store_true", help="Overwrite existing config if present.")
+    p_init.add_argument(
+        "--force", action="store_true", help="Overwrite existing config if present."
+    )
     p_init.set_defaults(func=cmd_init)
 
     p_ingest = sub.add_parser("ingest", help="Ingest file(s) and/or text into the corpus.")
     _add_common_corpus_arg(p_ingest)
     p_ingest.add_argument("files", nargs="*", help="File paths to ingest.")
     p_ingest.add_argument("--note", default=None, help="Ingest a literal note as Markdown text.")
-    p_ingest.add_argument("--stdin", action="store_true", help="Read text to ingest from standard input.")
+    p_ingest.add_argument(
+        "--stdin", action="store_true", help="Read text to ingest from standard input."
+    )
     p_ingest.add_argument("--title", default=None, help="Optional title (for --note/--stdin).")
     p_ingest.add_argument("--tags", default=None, help="Comma-separated tags.")
     p_ingest.add_argument("--tag", action="append", help="Repeatable tag.")
@@ -480,18 +557,26 @@ def build_parser() -> argparse.ArgumentParser:
     p_show.add_argument("id", help="Item identifier (universally unique identifier).")
     p_show.set_defaults(func=cmd_show)
 
-    p_reindex = sub.add_parser("reindex", help="Rebuild/refresh the corpus catalog from the on-disk corpus.")
+    p_reindex = sub.add_parser(
+        "reindex", help="Rebuild/refresh the corpus catalog from the on-disk corpus."
+    )
     _add_common_corpus_arg(p_reindex)
     p_reindex.set_defaults(func=cmd_reindex)
 
     p_import_tree = sub.add_parser("import-tree", help="Import a folder tree into the corpus.")
     _add_common_corpus_arg(p_import_tree)
     p_import_tree.add_argument("path", help="Folder tree root to import.")
-    p_import_tree.add_argument("--tags", default=None, help="Comma-separated tags to apply to imported items.")
-    p_import_tree.add_argument("--tag", action="append", help="Repeatable tag to apply to imported items.")
+    p_import_tree.add_argument(
+        "--tags", default=None, help="Comma-separated tags to apply to imported items."
+    )
+    p_import_tree.add_argument(
+        "--tag", action="append", help="Repeatable tag to apply to imported items."
+    )
     p_import_tree.set_defaults(func=cmd_import_tree)
 
-    p_purge = sub.add_parser("purge", help="Delete all items and derived files (requires confirmation).")
+    p_purge = sub.add_parser(
+        "purge", help="Delete all items and derived files (requires confirmation)."
+    )
     _add_common_corpus_arg(p_purge)
     p_purge.add_argument(
         "--confirm",
@@ -516,27 +601,53 @@ def build_parser() -> argparse.ArgumentParser:
     )
     p_build.set_defaults(func=cmd_build)
 
-    p_extract = sub.add_parser("extract", help="Build a text extraction run for the corpus.")
-    _add_common_corpus_arg(p_extract)
-    p_extract.add_argument(
-        "--extractor",
-        required=True,
-        help="Extractor identifier (for example, pass-through-text, metadata-text, cascade).",
+    p_extract = sub.add_parser("extract", help="Work with text extraction runs for the corpus.")
+    extract_sub = p_extract.add_subparsers(dest="extract_command", required=True)
+
+    p_extract_build = extract_sub.add_parser("build", help="Build a text extraction run.")
+    _add_common_corpus_arg(p_extract_build)
+    p_extract_build.add_argument(
+        "--recipe-name", default="default", help="Human-readable recipe name."
     )
-    p_extract.add_argument("--recipe-name", default="default", help="Human-readable recipe name.")
-    p_extract.add_argument(
+    p_extract_build.add_argument(
         "--step",
         action="append",
         default=None,
-        help="Cascade step spec in the form extractor_id or extractor_id:key=value,key=value (repeatable).",
+        help="Pipeline step spec in the form extractor_id or extractor_id:key=value,key=value (repeatable).",
     )
-    p_extract.add_argument(
-        "--config",
-        action="append",
+    p_extract_build.set_defaults(func=cmd_extract_build)
+
+    p_extract_list = extract_sub.add_parser("list", help="List extraction runs.")
+    _add_common_corpus_arg(p_extract_list)
+    p_extract_list.add_argument(
+        "--extractor-id",
         default=None,
-        help="Extractor config as key=value (repeatable).",
+        help="Optional extractor identifier filter (for example: pipeline).",
     )
-    p_extract.set_defaults(func=cmd_extract)
+    p_extract_list.set_defaults(func=cmd_extract_list)
+
+    p_extract_show = extract_sub.add_parser("show", help="Show an extraction run manifest.")
+    _add_common_corpus_arg(p_extract_show)
+    p_extract_show.add_argument(
+        "--run",
+        required=True,
+        help="Extraction run reference in the form extractor_id:run_id.",
+    )
+    p_extract_show.set_defaults(func=cmd_extract_show)
+
+    p_extract_delete = extract_sub.add_parser("delete", help="Delete an extraction run directory.")
+    _add_common_corpus_arg(p_extract_delete)
+    p_extract_delete.add_argument(
+        "--run",
+        required=True,
+        help="Extraction run reference in the form extractor_id:run_id.",
+    )
+    p_extract_delete.add_argument(
+        "--confirm",
+        required=True,
+        help="Type the exact extractor_id:run_id to confirm deletion.",
+    )
+    p_extract_delete.set_defaults(func=cmd_extract_delete)
 
     p_query = sub.add_parser("query", help="Run a retrieval query.")
     _add_common_corpus_arg(p_query)
@@ -561,6 +672,19 @@ def build_parser() -> argparse.ArgumentParser:
     p_eval.add_argument("--max-items-per-source", type=int, default=5)
     p_eval.set_defaults(func=cmd_eval)
 
+    p_crawl = sub.add_parser("crawl", help="Crawl a website prefix into the corpus.")
+    _add_common_corpus_arg(p_crawl)
+    p_crawl.add_argument("--root-url", required=True, help="Root uniform resource locator to fetch.")
+    p_crawl.add_argument(
+        "--allowed-prefix",
+        required=True,
+        help="Uniform resource locator prefix that limits which links are eligible for crawl.",
+    )
+    p_crawl.add_argument("--max-items", type=int, default=50, help="Maximum number of items to store.")
+    p_crawl.add_argument("--tags", default=None, help="Comma-separated tags to apply to stored items.")
+    p_crawl.add_argument("--tag", action="append", help="Repeatable tag to apply to stored items.")
+    p_crawl.set_defaults(func=cmd_crawl)
+
     return parser
 
 
@@ -573,7 +697,6 @@ def main(argument_list: Optional[List[str]] = None) -> int:
     :return: Exit code.
     :rtype: int
     """
-
     parser = build_parser()
     arguments = parser.parse_args(argument_list)
     try:
@@ -583,6 +706,7 @@ def main(argument_list: Optional[List[str]] = None) -> int:
         FileExistsError,
         KeyError,
         ValueError,
+        ExtractionRunFatalError,
         NotImplementedError,
         ValidationError,
     ) as exception: