biblicus 0.1.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

biblicus/__init__.py

@@ -2,6 +2,7 @@
 Biblicus public package interface.
 """
 
+from .corpus import Corpus
 from .models import (
     CorpusConfig,
     Evidence,
@@ -11,7 +12,6 @@ from .models import (
     RetrievalResult,
     RetrievalRun,
 )
-from .corpus import Corpus
 
 __all__ = [
     "__version__",
@@ -25,4 +25,4 @@ __all__ = [
     "RetrievalRun",
 ]
 
-__version__ = "0.1.1"
+__version__ = "0.3.0"

biblicus/_vendor/dotyaml/__init__.py

@@ -0,0 +1,14 @@
+"""
+Vendored dotyaml utilities.
+
+This package vendors the minimal pieces of the `dotyaml` project that Biblicus uses for
+loading and interpolating YAML configuration files.
+"""
+
+from __future__ import annotations
+
+from .interpolation import interpolate_env_vars
+from .loader import ConfigLoader, load_config
+
+__all__ = ["ConfigLoader", "interpolate_env_vars", "load_config"]
+

biblicus/_vendor/dotyaml/interpolation.py

@@ -0,0 +1,63 @@
+"""
+Environment variable interpolation functionality for dotyaml.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from typing import Any, Dict, Union
+
+
+def interpolate_env_vars(data: Union[str, Dict[str, Any], Any]) -> Union[str, Dict[str, Any], Any]:
+    """
+    Recursively interpolate environment variables in YAML data using Jinja-like syntax.
+
+    Supports syntax like: ``{{ ENV_VAR_NAME }}`` or ``{{ ENV_VAR_NAME|default_value }}``
+
+    :param data: Data structure to interpolate (string, dict, list, etc).
+    :type data: str or dict[str, Any] or Any
+    :return: Data structure with environment variables interpolated.
+    :rtype: str or dict[str, Any] or Any
+    """
+
+    if isinstance(data, str):
+        return _interpolate_string(data)
+    if isinstance(data, dict):
+        return {key: interpolate_env_vars(value) for key, value in data.items()}
+    if isinstance(data, list):
+        return [interpolate_env_vars(item) for item in data]
+    return data
+
+
+def _interpolate_string(text: str) -> str:
+    """
+    Interpolate environment variables in a string using Jinja-like syntax.
+
+    Supports:
+    - ``{{ ENV_VAR }}`` required environment variable.
+    - ``{{ ENV_VAR|default_value }}`` environment variable with default.
+
+    :param text: String to interpolate.
+    :type text: str
+    :return: String with environment variables interpolated.
+    :rtype: str
+    :raises ValueError: If a required environment variable is not found.
+    """
+
+    pattern = r"\{\{\s*([A-Z_][A-Z0-9_]*)\s*(?:\|\s*([^}]*?))?\s*\}\}"
+
+    def replace_match(match):  # type: ignore[no-untyped-def]
+        env_var = match.group(1)
+        default_value = match.group(2)
+
+        env_value = os.getenv(env_var)
+
+        if env_value is not None:
+            return env_value
+        if default_value is not None:
+            return default_value.strip()
+        raise ValueError(f"Required environment variable '{env_var}' not found")
+
+    return re.sub(pattern, replace_match, text)
+

biblicus/_vendor/dotyaml/loader.py

@@ -0,0 +1,181 @@
+"""
+Core loading functionality for dotyaml.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
+
+import yaml
+
+try:
+    from dotenv import load_dotenv
+
+    DOTENV_AVAILABLE = True
+except ImportError:
+    DOTENV_AVAILABLE = False
+
+from .interpolation import interpolate_env_vars
+from .transformer import flatten_dict, unflatten_env_vars
+
+
+def load_config(
+    yaml_path: Optional[Union[str, Path]] = None,
+    prefix: str = "",
+    override: bool = False,
+    dotenv_path: Optional[Union[str, Path]] = ".env",
+    load_dotenv_first: bool = True,
+) -> Dict[str, str]:
+    """
+    Load configuration from a YAML file and set environment variables.
+
+    :param yaml_path: Path to YAML configuration file. When None, only reads existing env vars.
+    :type yaml_path: str or Path or None
+    :param prefix: Prefix for environment variable names (for example, ``APP``).
+    :type prefix: str
+    :param override: Whether to override existing environment variables.
+    :type override: bool
+    :param dotenv_path: Optional ``.env`` file path to load first.
+    :type dotenv_path: str or Path or None
+    :param load_dotenv_first: Whether to load ``.env`` before YAML.
+    :type load_dotenv_first: bool
+    :return: Mapping of values that were set.
+    :rtype: dict[str, str]
+    """
+
+    config: Dict[str, str] = {}
+
+    if load_dotenv_first and DOTENV_AVAILABLE and dotenv_path:
+        env_file = Path(dotenv_path)
+        env_locations: list[Path] = []
+
+        if env_file.is_absolute():
+            env_locations.append(env_file)
+        else:
+            env_locations.append(Path.cwd() / dotenv_path)
+            if yaml_path:
+                yaml_dir = Path(yaml_path).parent
+                env_locations.append(yaml_dir / dotenv_path)
+
+        for env_path in env_locations:
+            if env_path.exists():
+                load_dotenv(env_path)
+                break
+
+    if yaml_path and Path(yaml_path).exists():
+        with open(yaml_path, "r", encoding="utf-8") as file:
+            yaml_data = yaml.safe_load(file)
+
+        if yaml_data:
+            yaml_data = interpolate_env_vars(yaml_data)
+            flat_config = flatten_dict(yaml_data, prefix)
+
+            for key, value in flat_config.items():
+                if not override and key in os.environ:
+                    config[key] = os.environ[key]
+                else:
+                    os.environ[key] = value
+                    config[key] = value
+
+    return config
+
+
+class ConfigLoader:
+    """
+    Configuration loader that can read YAML files or environment variables.
+    """
+
+    def __init__(
+        self,
+        prefix: str = "",
+        schema: Optional[Dict[str, Any]] = None,
+        dotenv_path: Optional[Union[str, Path]] = ".env",
+        load_dotenv_first: bool = True,
+    ):
+        self.prefix = prefix
+        self.schema = schema
+        self.dotenv_path = dotenv_path
+        self.load_dotenv_first = load_dotenv_first
+
+        if self.load_dotenv_first and DOTENV_AVAILABLE and self.dotenv_path:
+            env_file = Path(self.dotenv_path)
+            env_locations: list[Path] = []
+
+            if env_file.is_absolute():
+                env_locations.append(env_file)
+            else:
+                env_locations.append(Path.cwd() / self.dotenv_path)
+
+            for env_path in env_locations:
+                if env_path.exists():
+                    load_dotenv(env_path)
+                    break
+
+    def load_from_yaml(self, yaml_path: Union[str, Path]) -> Dict[str, Any]:
+        """
+        Load configuration from a YAML file with environment variable interpolation.
+
+        :param yaml_path: YAML configuration file path.
+        :type yaml_path: str or Path
+        :return: Parsed YAML data.
+        :rtype: dict[str, Any]
+        """
+
+        if not Path(yaml_path).exists():
+            return {}
+
+        if self.load_dotenv_first and DOTENV_AVAILABLE and self.dotenv_path:
+            env_file = Path(self.dotenv_path)
+            env_locations: list[Path] = []
+
+            if env_file.is_absolute():
+                env_locations.append(env_file)
+            else:
+                env_locations.append(Path.cwd() / self.dotenv_path)
+                yaml_dir = Path(yaml_path).parent
+                env_locations.append(yaml_dir / self.dotenv_path)
+
+            for env_path in env_locations:
+                if env_path.exists():
+                    load_dotenv(env_path)
+                    break
+
+        with open(yaml_path, "r", encoding="utf-8") as file:
+            yaml_data = yaml.safe_load(file)
+
+        if yaml_data:
+            yaml_data = interpolate_env_vars(yaml_data)
+
+        return yaml_data or {}
+
+    def load_from_env(self) -> Dict[str, Any]:
+        """
+        Load configuration from environment variables.
+
+        :return: Nested configuration dictionary.
+        :rtype: dict[str, Any]
+        """
+
+        env_vars = dict(os.environ)
+        return unflatten_env_vars(env_vars, self.prefix)
+
+    def set_env_vars(self, config: Dict[str, Any], override: bool = False) -> None:
+        """
+        Set environment variables from a configuration dictionary.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :param override: Whether to override existing environment variables.
+        :type override: bool
+        :return: None.
+        :rtype: None
+        """
+
+        flat_config = flatten_dict(config, self.prefix)
+
+        for key, value in flat_config.items():
+            if override or key not in os.environ:
+                os.environ[key] = value
+

biblicus/_vendor/dotyaml/transformer.py

@@ -0,0 +1,135 @@
+"""
+YAML to environment variable transformation utilities for dotyaml.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Dict
+
+
+def flatten_dict(data: Dict[str, Any], prefix: str = "", separator: str = "_") -> Dict[str, str]:
+    """
+    Flatten a nested dictionary into environment-variable style keys.
+
+    :param data: Nested dictionary to flatten.
+    :type data: dict[str, Any]
+    :param prefix: Prefix to add to all keys.
+    :type prefix: str
+    :param separator: Separator between key parts.
+    :type separator: str
+    :return: Flattened mapping with string values.
+    :rtype: dict[str, str]
+    """
+
+    result: Dict[str, str] = {}
+
+    for key, value in data.items():
+        if prefix:
+            full_key = f"{prefix}{separator}{key.upper()}"
+        else:
+            full_key = key.upper()
+
+        clean_key = full_key.replace("-", "_").replace(".", "_")
+
+        if isinstance(value, dict):
+            result.update(flatten_dict(value, clean_key, separator))
+        else:
+            result[clean_key] = convert_value_to_string(value)
+
+    return result
+
+
+def convert_value_to_string(value: Any) -> str:
+    """
+    Convert a Python value to its environment variable string representation.
+
+    :param value: Value to convert.
+    :type value: Any
+    :return: String representation suitable for environment variables.
+    :rtype: str
+    """
+
+    if value is None:
+        return ""
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (int, float)):
+        return str(value)
+    if isinstance(value, str):
+        return value
+    if isinstance(value, (list, tuple)):
+        return ",".join(convert_value_to_string(item) for item in value)
+    if isinstance(value, dict):
+        return json.dumps(value)
+    return str(value)
+
+
+def unflatten_env_vars(env_vars: Dict[str, str], prefix: str = "") -> Dict[str, Any]:
+    """
+    Convert flat environment variables back to nested dictionary structure.
+
+    :param env_vars: Mapping of environment variables.
+    :type env_vars: dict[str, str]
+    :param prefix: Optional prefix to filter by.
+    :type prefix: str
+    :return: Nested dictionary structure.
+    :rtype: dict[str, Any]
+    """
+
+    result: Dict[str, Any] = {}
+
+    for key, value in env_vars.items():
+        if prefix and not key.startswith(f"{prefix}_"):
+            continue
+
+        clean_key = key
+        if prefix:
+            clean_key = key[len(prefix) + 1 :]
+
+        parts = clean_key.lower().split("_")
+
+        current: Dict[str, Any] = result
+        for part in parts[:-1]:
+            if part not in current:
+                current[part] = {}
+            current = current[part]
+
+        final_key = parts[-1]
+        current[final_key] = convert_string_to_value(value)
+
+    return result
+
+
+def convert_string_to_value(value: str) -> Any:
+    """
+    Convert a string environment variable back to an appropriate Python type.
+
+    :param value: String value from an environment variable.
+    :type value: str
+    :return: Converted Python value.
+    :rtype: Any
+    """
+
+    if value == "":
+        return None
+    lowered = value.lower()
+    if lowered == "true":
+        return True
+    if lowered == "false":
+        return False
+    if value.isdigit():
+        return int(value)
+    if value.replace(".", "").replace("-", "").isdigit():
+        try:
+            return float(value)
+        except ValueError:
+            return value
+    if "," in value:
+        items = [item.strip() for item in value.split(",")]
+        return [convert_string_to_value(item) for item in items]
+    try:
+        return json.loads(value)
+    except (json.JSONDecodeError, ValueError):
+        return value
+

biblicus/backends/__init__.py

@@ -18,7 +18,6 @@ def available_backends() -> Dict[str, Type[RetrievalBackend]]:
     :return: Mapping of backend identifiers to backend classes.
     :rtype: dict[str, Type[RetrievalBackend]]
     """
-
     return {
         ScanBackend.backend_id: ScanBackend,
         SqliteFullTextSearchBackend.backend_id: SqliteFullTextSearchBackend,
@@ -35,7 +34,6 @@ def get_backend(backend_id: str) -> RetrievalBackend:
     :rtype: RetrievalBackend
     :raises KeyError: If the backend identifier is unknown.
     """
-
     registry = available_backends()
     backend_class = registry.get(backend_id)
     if backend_class is None:

biblicus/backends/base.py

@@ -22,7 +22,9 @@ class RetrievalBackend(ABC):
     backend_id: str
 
     @abstractmethod
-    def build_run(self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]) -> RetrievalRun:
+    def build_run(
+        self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]
+    ) -> RetrievalRun:
         """
         Build or register a retrieval run for the backend.
 
@@ -35,7 +37,6 @@ class RetrievalBackend(ABC):
         :return: Run manifest describing the build.
         :rtype: RetrievalRun
         """
-
         raise NotImplementedError
 
     @abstractmethod
@@ -61,5 +62,4 @@ class RetrievalBackend(ABC):
         :return: Retrieval results containing evidence.
         :rtype: RetrievalResult
         """
-
         raise NotImplementedError

biblicus/backends/scan.py

@@ -10,7 +10,14 @@ from pydantic import BaseModel, ConfigDict, Field
 
 from ..corpus import Corpus
 from ..frontmatter import parse_front_matter
-from ..models import Evidence, QueryBudget, RetrievalResult, RetrievalRun
+from ..models import (
+    Evidence,
+    ExtractionRunReference,
+    QueryBudget,
+    RetrievalResult,
+    RetrievalRun,
+    parse_extraction_run_reference,
+)
 from ..retrieval import apply_budget, create_recipe_manifest, create_run_manifest, hash_text
 from ..time import utc_now_iso
 
@@ -21,11 +28,14 @@ class ScanRecipeConfig(BaseModel):
 
     :ivar snippet_characters: Maximum characters to include in evidence snippets.
     :vartype snippet_characters: int
+    :ivar extraction_run: Optional extraction run reference in the form extractor_id:run_id.
+    :vartype extraction_run: str or None
     """
 
     model_config = ConfigDict(extra="forbid")
 
     snippet_characters: int = Field(default=400, ge=1)
+    extraction_run: Optional[str] = None
 
 
 class ScanBackend:
@@ -38,7 +48,9 @@ class ScanBackend:
 
     backend_id = "scan"
 
-    def build_run(self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]) -> RetrievalRun:
+    def build_run(
+        self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]
+    ) -> RetrievalRun:
         """
         Register a scan backend run (no materialization).
 
@@ -51,7 +63,6 @@ class ScanBackend:
         :return: Run manifest describing the build.
         :rtype: RetrievalRun
         """
-
         recipe_config = ScanRecipeConfig.model_validate(config)
         catalog = corpus.load_catalog()
         recipe = create_recipe_manifest(
@@ -59,7 +70,10 @@ class ScanBackend:
             name=recipe_name,
             config=recipe_config.model_dump(),
         )
-        stats = {"items": len(catalog.items), "text_items": _count_text_items(catalog.items.values())}
+        stats = {
+            "items": len(catalog.items),
+            "text_items": _count_text_items(corpus, catalog.items.values(), recipe_config),
+        }
         run = create_run_manifest(corpus, recipe=recipe, stats=stats, artifact_paths=[])
         corpus.write_run(run)
         return run
@@ -86,15 +100,16 @@ class ScanBackend:
         :return: Retrieval results containing evidence.
         :rtype: RetrievalResult
         """
-
         recipe_config = ScanRecipeConfig.model_validate(run.recipe.config)
         catalog = corpus.load_catalog()
+        extraction_reference = _resolve_extraction_reference(corpus, recipe_config)
         query_tokens = _tokenize_query(query_text)
         scored_candidates = _score_items(
             corpus,
             catalog.items.values(),
             query_tokens,
             recipe_config.snippet_characters,
+            extraction_reference=extraction_reference,
         )
         sorted_candidates = sorted(
             scored_candidates,
@@ -124,18 +139,62 @@ class ScanBackend:
         )
 
 
-def _count_text_items(items: Iterable[object]) -> int:
+def _resolve_extraction_reference(
+    corpus: Corpus, recipe_config: ScanRecipeConfig
+) -> Optional[ExtractionRunReference]:
+    """
+    Resolve an extraction run reference from a recipe config.
+
+    :param corpus: Corpus associated with the recipe.
+    :type corpus: Corpus
+    :param recipe_config: Parsed scan recipe configuration.
+    :type recipe_config: ScanRecipeConfig
+    :return: Parsed extraction reference or None.
+    :rtype: ExtractionRunReference or None
+    :raises FileNotFoundError: If an extraction run is referenced but not present.
+    """
+    if not recipe_config.extraction_run:
+        return None
+    extraction_reference = parse_extraction_run_reference(recipe_config.extraction_run)
+    run_dir = corpus.extraction_run_dir(
+        extractor_id=extraction_reference.extractor_id,
+        run_id=extraction_reference.run_id,
+    )
+    if not run_dir.is_dir():
+        raise FileNotFoundError(f"Missing extraction run: {extraction_reference.as_string()}")
+    return extraction_reference
+
+
+def _count_text_items(
+    corpus: Corpus, items: Iterable[object], recipe_config: ScanRecipeConfig
+) -> int:
     """
     Count catalog items that represent text content.
 
+    When an extraction run is configured, extracted artifacts are treated as text.
+
+    :param corpus: Corpus containing the items.
+    :type corpus: Corpus
     :param items: Catalog items to inspect.
     :type items: Iterable[object]
+    :param recipe_config: Parsed scan recipe configuration.
+    :type recipe_config: ScanRecipeConfig
     :return: Number of text items.
     :rtype: int
     """
-
     text_item_count = 0
+    extraction_reference = _resolve_extraction_reference(corpus, recipe_config)
     for catalog_item in items:
+        item_id = str(getattr(catalog_item, "id", ""))
+        if extraction_reference and item_id:
+            extracted_text = corpus.read_extracted_text(
+                extractor_id=extraction_reference.extractor_id,
+                run_id=extraction_reference.run_id,
+                item_id=item_id,
+            )
+            if isinstance(extracted_text, str) and extracted_text.strip():
+                text_item_count += 1
+                continue
         media_type = getattr(catalog_item, "media_type", "")
         if media_type == "text/markdown" or str(media_type).startswith("text/"):
             text_item_count += 1
@@ -151,23 +210,41 @@ def _tokenize_query(query_text: str) -> List[str]:
     :return: Lowercased non-empty tokens.
     :rtype: list[str]
     """
-
     return [token for token in query_text.lower().split() if token]
 
 
-def _load_text_from_item(corpus: Corpus, relpath: str, media_type: str) -> Optional[str]:
+def _load_text_from_item(
+    corpus: Corpus,
+    *,
+    item_id: str,
+    relpath: str,
+    media_type: str,
+    extraction_reference: Optional[ExtractionRunReference],
+) -> Optional[str]:
     """
     Load a text payload from a catalog item.
 
     :param corpus: Corpus containing the item.
     :type corpus: Corpus
+    :param item_id: Item identifier.
+    :type item_id: str
     :param relpath: Relative path to the stored content.
     :type relpath: str
     :param media_type: Media type for the stored content.
     :type media_type: str
+    :param extraction_reference: Optional extraction run reference.
+    :type extraction_reference: ExtractionRunReference or None
     :return: Text payload or None if not decodable as text.
     :rtype: str or None
     """
+    if extraction_reference:
+        extracted_text = corpus.read_extracted_text(
+            extractor_id=extraction_reference.extractor_id,
+            run_id=extraction_reference.run_id,
+            item_id=item_id,
+        )
+        if isinstance(extracted_text, str) and extracted_text.strip():
+            return extracted_text
 
     content_path = corpus.root / relpath
     raw_bytes = content_path.read_bytes()
@@ -191,7 +268,6 @@ def _find_first_match(text: str, tokens: List[str]) -> Optional[Tuple[int, int]]
     :return: Start/end span for the earliest match, or None if no matches.
     :rtype: tuple[int, int] or None
     """
-
     lower_text = text.lower()
     best_start: Optional[int] = None
     best_end: Optional[int] = None
@@ -223,7 +299,6 @@ def _build_snippet(text: str, span: Optional[Tuple[int, int]], *, max_chars: int
     :return: Snippet text.
     :rtype: str
     """
-
     if not text:
         return ""
     if span is None:
@@ -240,6 +315,8 @@ def _score_items(
     items: Iterable[object],
     tokens: List[str],
     snippet_characters: int,
+    *,
+    extraction_reference: Optional[ExtractionRunReference],
 ) -> List[Evidence]:
     """
     Score catalog items by token frequency and return evidence candidates.
@@ -255,12 +332,18 @@ def _score_items(
     :return: Evidence candidates with provisional ranks.
     :rtype: list[Evidence]
     """
-
     evidence_items: List[Evidence] = []
     for catalog_item in items:
         media_type = getattr(catalog_item, "media_type", "")
         relpath = getattr(catalog_item, "relpath", "")
-        item_text = _load_text_from_item(corpus, relpath, media_type)
+        item_id = str(getattr(catalog_item, "id", ""))
+        item_text = _load_text_from_item(
+            corpus,
+            item_id=item_id,
+            relpath=relpath,
+            media_type=str(media_type),
+            extraction_reference=extraction_reference,
+        )
         if item_text is None:
             continue
         lower_text = item_text.lower()