biblicus 0.14.0__py3-none-any.whl → 0.15.0__py3-none-any.whl

biblicus/analysis/topic_modeling.py

@@ -13,12 +13,12 @@ from typing import Any, Dict, List, Tuple
 
 from pydantic import BaseModel
 
+from ..ai.llm import generate_completion
 from ..corpus import Corpus
 from ..models import ExtractionRunReference
 from ..retrieval import hash_text
 from ..time import utc_now_iso
 from .base import CorpusAnalysisBackend
-from .llm import generate_completion
 from .models import (
     AnalysisRecipeManifest,
     AnalysisRunInput,
@@ -45,7 +45,18 @@ from .models import (
 
 
 @dataclass
-class _TopicDocument:
+class TopicModelingDocument:
+    """
+    Text document input for topic modeling.
+
+    :ivar document_id: Stable identifier for this document in the topic modeling stage.
+    :vartype document_id: str
+    :ivar source_item_id: Corpus item identifier the text was derived from.
+    :vartype source_item_id: str
+    :ivar text: Document text content.
+    :vartype text: str
+    """
+
     document_id: str
     source_item_id: str
     text: str
@@ -190,6 +201,74 @@ def _run_topic_modeling(
     return output
 
 
+def run_topic_modeling_for_documents(
+    *,
+    documents: List[TopicModelingDocument],
+    config: TopicModelingRecipeConfig,
+) -> TopicModelingReport:
+    """
+    Run topic modeling using caller-provided documents.
+
+    :param documents: Pre-collected documents to model.
+    :type documents: list[TopicModelingDocument]
+    :param config: Topic modeling recipe configuration.
+    :type config: TopicModelingRecipeConfig
+    :return: Topic modeling report with topic assignments.
+    :rtype: TopicModelingReport
+    """
+    text_report = TopicModelingTextCollectionReport(
+        status=TopicModelingStageStatus.COMPLETE,
+        source_items=len({doc.source_item_id for doc in documents}),
+        documents=len(documents),
+        sample_size=config.text_source.sample_size,
+        min_text_characters=config.text_source.min_text_characters,
+        empty_texts=len([doc for doc in documents if not doc.text.strip()]),
+        skipped_items=0,
+        warnings=[],
+        errors=[],
+    )
+
+    llm_extraction_report, extracted_documents = _apply_llm_extraction(
+        documents=documents,
+        config=config.llm_extraction,
+    )
+
+    lexical_report, lexical_documents = _apply_lexical_processing(
+        documents=extracted_documents,
+        config=config.lexical_processing,
+    )
+
+    bertopic_report, topics = _run_bertopic(
+        documents=lexical_documents,
+        config=config.bertopic_analysis,
+    )
+
+    fine_tuning_report, labeled_topics = _apply_llm_fine_tuning(
+        topics=topics,
+        documents=lexical_documents,
+        config=config.llm_fine_tuning,
+    )
+
+    return TopicModelingReport(
+        text_collection=text_report,
+        llm_extraction=llm_extraction_report,
+        lexical_processing=lexical_report,
+        bertopic_analysis=bertopic_report,
+        llm_fine_tuning=fine_tuning_report,
+        topics=labeled_topics,
+        warnings=(
+            text_report.warnings
+            + llm_extraction_report.warnings
+            + bertopic_report.warnings
+            + fine_tuning_report.warnings
+        ),
+        errors=text_report.errors
+        + llm_extraction_report.errors
+        + bertopic_report.errors
+        + fine_tuning_report.errors,
+    )
+
+
 def _create_recipe_manifest(
     *, name: str, config: TopicModelingRecipeConfig
 ) -> AnalysisRecipeManifest:
@@ -226,14 +305,14 @@ def _collect_documents(
     corpus: Corpus,
     extraction_run: ExtractionRunReference,
     config: TopicModelingTextSourceConfig,
-) -> Tuple[List[_TopicDocument], TopicModelingTextCollectionReport]:
+) -> Tuple[List[TopicModelingDocument], TopicModelingTextCollectionReport]:
     manifest = corpus.load_extraction_run_manifest(
         extractor_id=extraction_run.extractor_id,
         run_id=extraction_run.run_id,
     )
     warnings: List[str] = []
     errors: List[str] = []
-    documents: List[_TopicDocument] = []
+    documents: List[TopicModelingDocument] = []
     skipped_items = 0
     empty_texts = 0
 
@@ -256,7 +335,7 @@ def _collect_documents(
             skipped_items += 1
             continue
         documents.append(
-            _TopicDocument(
+            TopicModelingDocument(
                 document_id=item_result.item_id,
                 source_item_id=item_result.item_id,
                 text=text_value,
@@ -286,9 +365,9 @@ def _collect_documents(
 
 def _apply_llm_extraction(
     *,
-    documents: List[_TopicDocument],
+    documents: List[TopicModelingDocument],
     config: TopicModelingLlmExtractionConfig,
-) -> Tuple[TopicModelingLlmExtractionReport, List[_TopicDocument]]:
+) -> Tuple[TopicModelingLlmExtractionReport, List[TopicModelingDocument]]:
     if not config.enabled:
         report = TopicModelingLlmExtractionReport(
             status=TopicModelingStageStatus.SKIPPED,
@@ -300,7 +379,7 @@ def _apply_llm_extraction(
         )
         return report, list(documents)
 
-    extracted_documents: List[_TopicDocument] = []
+    extracted_documents: List[TopicModelingDocument] = []
     errors: List[str] = []
 
     for document in documents:
@@ -315,7 +394,7 @@ def _apply_llm_extraction(
                 errors.append(f"LLM extraction returned empty output for {document.document_id}")
                 continue
             extracted_documents.append(
-                _TopicDocument(
+                TopicModelingDocument(
                     document_id=document.document_id,
                     source_item_id=document.source_item_id,
                     text=response_text,
@@ -328,7 +407,7 @@ def _apply_llm_extraction(
             continue
         for index, item_text in enumerate(items, start=1):
             extracted_documents.append(
-                _TopicDocument(
+                TopicModelingDocument(
                     document_id=f"{document.document_id}:{index}",
                     source_item_id=document.source_item_id,
                     text=item_text,
@@ -381,9 +460,9 @@ def _parse_itemized_response(response_text: str) -> List[str]:
 
 def _apply_lexical_processing(
     *,
-    documents: List[_TopicDocument],
+    documents: List[TopicModelingDocument],
     config: TopicModelingLexicalProcessingConfig,
-) -> Tuple[TopicModelingLexicalProcessingReport, List[_TopicDocument]]:
+) -> Tuple[TopicModelingLexicalProcessingReport, List[TopicModelingDocument]]:
     if not config.enabled:
         report = TopicModelingLexicalProcessingReport(
             status=TopicModelingStageStatus.SKIPPED,
@@ -395,7 +474,7 @@ def _apply_lexical_processing(
         )
         return report, list(documents)
 
-    processed: List[_TopicDocument] = []
+    processed: List[TopicModelingDocument] = []
     for document in documents:
         text_value = document.text
         if config.lowercase:
@@ -405,7 +484,7 @@ def _apply_lexical_processing(
         if config.collapse_whitespace:
             text_value = re.sub(r"\s+", " ", text_value).strip()
         processed.append(
-            _TopicDocument(
+            TopicModelingDocument(
                 document_id=document.document_id,
                 source_item_id=document.source_item_id,
                 text=text_value,
@@ -425,7 +504,7 @@ def _apply_lexical_processing(
 
 def _run_bertopic(
     *,
-    documents: List[_TopicDocument],
+    documents: List[TopicModelingDocument],
     config: TopicModelingBerTopicConfig,
 ) -> Tuple[TopicModelingBerTopicReport, List[TopicModelingTopic]]:
     try:
@@ -496,9 +575,9 @@ def _run_bertopic(
 
 
 def _group_documents_by_topic(
-    documents: List[_TopicDocument], assignments: List[int]
-) -> Dict[int, List[_TopicDocument]]:
-    grouped: Dict[int, List[_TopicDocument]] = {}
+    documents: List[TopicModelingDocument], assignments: List[int]
+) -> Dict[int, List[TopicModelingDocument]]:
+    grouped: Dict[int, List[TopicModelingDocument]] = {}
     for index, topic_id in enumerate(assignments):
         grouped.setdefault(int(topic_id), []).append(documents[index])
     return grouped
@@ -514,7 +593,7 @@ def _resolve_topic_keywords(*, topic_model: Any, topic_id: int) -> List[TopicMod
 def _apply_llm_fine_tuning(
     *,
     topics: List[TopicModelingTopic],
-    documents: List[_TopicDocument],
+    documents: List[TopicModelingDocument],
     config: TopicModelingLlmFineTuningConfig,
 ) -> Tuple[TopicModelingLlmFineTuningReport, List[TopicModelingTopic]]:
     if not config.enabled:

biblicus/backends/sqlite_full_text_search.py

@@ -459,7 +459,8 @@ def _create_full_text_search_schema(conn: sqlite3.Connection) -> None:
     :return: None.
     :rtype: None
     """
-    conn.execute("""
+    conn.execute(
+        """
         CREATE VIRTUAL TABLE chunks_full_text_search USING fts5(
             content,
             item_id UNINDEXED,
@@ -470,7 +471,8 @@ def _create_full_text_search_schema(conn: sqlite3.Connection) -> None:
             start_offset UNINDEXED,
             end_offset UNINDEXED
         )
-        """)
+        """
+    )
 
 
 def _build_full_text_search_index(

biblicus/cli.py

@@ -394,7 +394,7 @@ def cmd_extract_build(arguments: argparse.Namespace) -> int:
     :return: Exit code.
     :rtype: int
     """
-    import yaml
+    from .recipes import load_recipe_view
 
     corpus = (
         Corpus.open(arguments.corpus)
@@ -404,11 +404,11 @@ def cmd_extract_build(arguments: argparse.Namespace) -> int:
 
     # Load recipe from file if --recipe is provided
     if getattr(arguments, "recipe", None):
-        recipe_path = Path(arguments.recipe)
-        if not recipe_path.exists():
-            raise FileNotFoundError(f"Recipe file not found: {recipe_path}")
-        with open(recipe_path, "r", encoding="utf-8") as f:
-            recipe_data = yaml.safe_load(f)
+        recipe_data = load_recipe_view(
+            arguments.recipe,
+            recipe_label="Recipe file",
+            mapping_error_message="Extraction recipe must be a mapping/object",
+        )
         loaded_extractor_id = recipe_data.get("extractor_id", "pipeline")
         loaded_config = recipe_data.get("config", {})
 
@@ -713,19 +713,20 @@ def cmd_analyze_topics(arguments: argparse.Namespace) -> int:
     :return: Exit code.
     :rtype: int
     """
-    import yaml
+    from .recipes import apply_dotted_overrides, load_recipe_view, parse_dotted_overrides
 
     corpus = (
         Corpus.open(arguments.corpus)
         if getattr(arguments, "corpus", None)
         else Corpus.find(Path.cwd())
     )
-    recipe_path = Path(arguments.recipe)
-    if not recipe_path.is_file():
-        raise FileNotFoundError(f"Recipe file not found: {recipe_path}")
-    recipe_data = yaml.safe_load(recipe_path.read_text(encoding="utf-8")) or {}
-    if not isinstance(recipe_data, dict):
-        raise ValueError("Topic modeling recipe must be a mapping/object")
+    recipe_data = load_recipe_view(
+        arguments.recipe,
+        recipe_label="Recipe file",
+        mapping_error_message="Topic modeling recipe must be a mapping/object",
+    )
+    overrides = parse_dotted_overrides(arguments.config)
+    recipe_data = apply_dotted_overrides(recipe_data, overrides)
 
     if arguments.extraction_run:
         extraction_run = parse_extraction_run_reference(arguments.extraction_run)
@@ -761,7 +762,7 @@ def cmd_analyze_profile(arguments: argparse.Namespace) -> int:
     :return: Exit code.
     :rtype: int
     """
-    import yaml
+    from .recipes import apply_dotted_overrides, load_recipe_view, parse_dotted_overrides
 
     corpus = (
         Corpus.open(arguments.corpus)
@@ -771,13 +772,17 @@ def cmd_analyze_profile(arguments: argparse.Namespace) -> int:
 
     recipe_data: dict[str, object] = {}
     if arguments.recipe is not None:
-        recipe_path = Path(arguments.recipe)
-        if not recipe_path.is_file():
-            raise FileNotFoundError(f"Recipe file not found: {recipe_path}")
-        recipe_raw = yaml.safe_load(recipe_path.read_text(encoding="utf-8")) or {}
-        if not isinstance(recipe_raw, dict):
-            raise ValueError("Profiling recipe must be a mapping/object")
-        recipe_data = recipe_raw
+        recipe_data = load_recipe_view(
+            arguments.recipe,
+            recipe_label="Recipe file",
+            mapping_error_message="Profiling recipe must be a mapping/object",
+        )
+        overrides = parse_dotted_overrides(arguments.config)
+        recipe_data = apply_dotted_overrides(recipe_data, overrides)
+    else:
+        overrides = parse_dotted_overrides(arguments.config)
+        if overrides:
+            recipe_data = apply_dotted_overrides(recipe_data, overrides)
 
     if arguments.extraction_run:
         extraction_run = parse_extraction_run_reference(arguments.extraction_run)
@@ -804,6 +809,55 @@ def cmd_analyze_profile(arguments: argparse.Namespace) -> int:
     return 0
 
 
+def cmd_analyze_markov(arguments: argparse.Namespace) -> int:
+    """
+    Run Markov analysis for a corpus.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    from .recipes import apply_dotted_overrides, load_recipe_view, parse_dotted_overrides
+
+    corpus = (
+        Corpus.open(arguments.corpus)
+        if getattr(arguments, "corpus", None)
+        else Corpus.find(Path.cwd())
+    )
+    recipe_data = load_recipe_view(
+        arguments.recipe,
+        recipe_label="Recipe file",
+        mapping_error_message="Markov analysis recipe must be a mapping/object",
+    )
+    overrides = parse_dotted_overrides(arguments.config)
+    recipe_data = apply_dotted_overrides(recipe_data, overrides)
+
+    if arguments.extraction_run:
+        extraction_run = parse_extraction_run_reference(arguments.extraction_run)
+    else:
+        extraction_run = corpus.latest_extraction_run_reference()
+        if extraction_run is None:
+            raise ValueError("Markov analysis requires an extraction run to supply text inputs")
+        print(
+            "Warning: using latest extraction run; pass --extraction-run for reproducibility.",
+            file=sys.stderr,
+        )
+
+    backend = get_analysis_backend("markov")
+    try:
+        output = backend.run_analysis(
+            corpus,
+            recipe_name=arguments.recipe_name,
+            config=recipe_data,
+            extraction_run=extraction_run,
+        )
+    except ValidationError as exc:
+        raise ValueError(f"Invalid Markov analysis recipe: {exc}") from exc
+    print(output.model_dump_json(indent=2))
+    return 0
+
+
 def build_parser() -> argparse.ArgumentParser:
     """
     Build the command-line interface argument parser.
@@ -912,6 +966,7 @@ def build_parser() -> argparse.ArgumentParser:
     p_extract_build.add_argument(
         "--recipe",
         default=None,
+        action="append",
         help="Path to YAML recipe file. If provided, --step arguments are ignored.",
     )
     p_extract_build.add_argument(
@@ -1067,7 +1122,14 @@ def build_parser() -> argparse.ArgumentParser:
     p_analyze_topics.add_argument(
         "--recipe",
         required=True,
-        help="Path to topic modeling recipe YAML.",
+        action="append",
+        help="Path to topic modeling recipe YAML. Repeatable; later recipes override earlier recipes.",
+    )
+    p_analyze_topics.add_argument(
+        "--config",
+        action="append",
+        default=[],
+        help="Override key=value pairs applied after composing recipes (supports dotted keys).",
     )
     p_analyze_topics.add_argument(
         "--recipe-name",
@@ -1086,7 +1148,14 @@ def build_parser() -> argparse.ArgumentParser:
     p_analyze_profile.add_argument(
         "--recipe",
         default=None,
-        help="Optional profiling recipe YAML file.",
+        action="append",
+        help="Optional profiling recipe YAML file. Repeatable; later recipes override earlier recipes.",
+    )
+    p_analyze_profile.add_argument(
+        "--config",
+        action="append",
+        default=[],
+        help="Override key=value pairs applied after composing recipes (supports dotted keys).",
     )
     p_analyze_profile.add_argument(
         "--recipe-name",
@@ -1100,6 +1169,32 @@ def build_parser() -> argparse.ArgumentParser:
     )
     p_analyze_profile.set_defaults(func=cmd_analyze_profile)
 
+    p_analyze_markov = analyze_sub.add_parser("markov", help="Run Markov analysis.")
+    _add_common_corpus_arg(p_analyze_markov)
+    p_analyze_markov.add_argument(
+        "--recipe",
+        required=True,
+        action="append",
+        help="Path to Markov analysis recipe YAML. Repeatable; later recipes override earlier recipes.",
+    )
+    p_analyze_markov.add_argument(
+        "--config",
+        action="append",
+        default=[],
+        help="Override key=value pairs applied after composing recipes (supports dotted keys).",
+    )
+    p_analyze_markov.add_argument(
+        "--recipe-name",
+        default="default",
+        help="Human-readable recipe name.",
+    )
+    p_analyze_markov.add_argument(
+        "--extraction-run",
+        default=None,
+        help="Extraction run reference in the form extractor_id:run_id.",
+    )
+    p_analyze_markov.set_defaults(func=cmd_analyze_markov)
+
     return parser
 
 

biblicus/recipes.py

@@ -0,0 +1,136 @@
+"""
+Recipe loading utilities for Biblicus.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Dict, Iterable, List, Mapping, MutableMapping, Optional
+
+
+def _parse_scalar(value: str) -> object:
+    lowered = value.lower()
+    if lowered == "true":
+        return True
+    if lowered == "false":
+        return False
+    if lowered in {"null", "none"}:
+        return None
+    if value.isdigit():
+        return int(value)
+    try:
+        return float(value)
+    except ValueError:
+        return value
+
+
+def parse_override_value(raw: str) -> object:
+    """
+    Parse a command-line override string into a Python value.
+
+    :param raw: Raw override string.
+    :type raw: str
+    :return: Parsed value.
+    :rtype: object
+    """
+    raw = str(raw)
+    stripped = raw.strip()
+    if not stripped:
+        return ""
+    if stripped[0] in {"{", "["}:
+        try:
+            return json.loads(stripped)
+        except json.JSONDecodeError:
+            return raw
+    return _parse_scalar(stripped)
+
+
+def parse_dotted_overrides(pairs: Optional[List[str]]) -> Dict[str, object]:
+    """
+    Parse repeated key=value pairs into a dotted override mapping.
+
+    :param pairs: Repeated command-line pairs.
+    :type pairs: list[str] or None
+    :return: Override mapping.
+    :rtype: dict[str, object]
+    :raises ValueError: If a pair is not key=value.
+    """
+    overrides: Dict[str, object] = {}
+    for item in pairs or []:
+        if "=" not in item:
+            raise ValueError(f"Config values must be key=value (got {item!r})")
+        key, raw = item.split("=", 1)
+        key = key.strip()
+        if not key:
+            raise ValueError("Config keys must be non-empty")
+        overrides[key] = parse_override_value(raw)
+    return overrides
+
+
+def _set_dotted_key(target: MutableMapping[str, object], dotted_key: str, value: object) -> None:
+    parts = [part.strip() for part in dotted_key.split(".") if part.strip()]
+    if not parts:
+        raise ValueError("Override keys must be non-empty")
+    current: MutableMapping[str, object] = target
+    for part in parts[:-1]:
+        existing = current.get(part)
+        if not isinstance(existing, dict):
+            nested: Dict[str, object] = {}
+            current[part] = nested
+            current = nested
+        else:
+            current = existing
+    current[parts[-1]] = value
+
+
+def apply_dotted_overrides(
+    config: Dict[str, object], overrides: Mapping[str, object]
+) -> Dict[str, object]:
+    """
+    Apply dotted key overrides to a nested configuration mapping.
+
+    :param config: Base configuration mapping.
+    :type config: dict[str, object]
+    :param overrides: Dotted key override mapping.
+    :type overrides: Mapping[str, object]
+    :return: New configuration mapping with overrides applied.
+    :rtype: dict[str, object]
+    """
+    updated: Dict[str, object] = json.loads(json.dumps(config))
+    for key, value in overrides.items():
+        _set_dotted_key(updated, key, value)
+    return updated
+
+
+def load_recipe_view(
+    recipe_paths: Iterable[str],
+    *,
+    recipe_label: str = "Recipe",
+    mapping_error_message: Optional[str] = None,
+) -> Dict[str, object]:
+    """
+    Load a composed recipe view from one or more YAML files.
+
+    :param recipe_paths: Iterable of recipe file paths in precedence order.
+    :type recipe_paths: Iterable[str]
+    :param recipe_label: Label used in error messages (for example: "Recipe file").
+    :type recipe_label: str
+    :return: Composed configuration view.
+    :rtype: dict[str, object]
+    :raises FileNotFoundError: If any recipe file is missing.
+    :raises ValueError: If any recipe file is not a mapping/object.
+    """
+    from biblicus._vendor.dotyaml import load_yaml_view
+
+    paths: List[str] = [str(path) for path in recipe_paths]
+    for raw in paths:
+        candidate = Path(raw)
+        if not candidate.is_file():
+            raise FileNotFoundError(f"{recipe_label} not found: {candidate}")
+    try:
+        view = load_yaml_view(paths)
+    except ValueError as exc:
+        message = mapping_error_message or f"{recipe_label} must be a mapping/object"
+        raise ValueError(message) from exc
+    return view

biblicus/text/__init__.py

@@ -0,0 +1,43 @@
+"""
+Agentic text utilities.
+"""
+
+from .annotate import apply_text_annotate
+from .extract import apply_text_extract
+from .link import apply_text_link
+from .models import (
+    TextAnnotateRequest,
+    TextAnnotateResult,
+    TextExtractRequest,
+    TextExtractResult,
+    TextExtractSpan,
+    TextLinkRequest,
+    TextLinkResult,
+    TextRedactRequest,
+    TextRedactResult,
+    TextSliceRequest,
+    TextSliceResult,
+    TextSliceSegment,
+)
+from .redact import apply_text_redact
+from .slice import apply_text_slice
+
+__all__ = [
+    "TextAnnotateRequest",
+    "TextAnnotateResult",
+    "TextExtractRequest",
+    "TextExtractResult",
+    "TextExtractSpan",
+    "TextLinkRequest",
+    "TextLinkResult",
+    "TextRedactRequest",
+    "TextRedactResult",
+    "TextSliceRequest",
+    "TextSliceResult",
+    "TextSliceSegment",
+    "apply_text_annotate",
+    "apply_text_extract",
+    "apply_text_link",
+    "apply_text_redact",
+    "apply_text_slice",
+]