biblicus 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

biblicus/__init__.py

@@ -2,6 +2,7 @@
 Biblicus public package interface.
 """
 
+from .corpus import Corpus
 from .models import (
     CorpusConfig,
     Evidence,
@@ -11,7 +12,6 @@ from .models import (
     RetrievalResult,
     RetrievalRun,
 )
-from .corpus import Corpus
 
 __all__ = [
     "__version__",
@@ -25,4 +25,4 @@ __all__ = [
     "RetrievalRun",
 ]
 
-__version__ = "0.2.0"
+__version__ = "0.4.0"

biblicus/_vendor/dotyaml/__init__.py

@@ -0,0 +1,14 @@
+"""
+Vendored dotyaml utilities.
+
+This package vendors the minimal pieces of the `dotyaml` project that Biblicus uses for
+loading and interpolating YAML configuration files.
+"""
+
+from __future__ import annotations
+
+from .interpolation import interpolate_env_vars
+from .loader import ConfigLoader, load_config
+
+__all__ = ["ConfigLoader", "interpolate_env_vars", "load_config"]
+

biblicus/_vendor/dotyaml/interpolation.py

@@ -0,0 +1,63 @@
+"""
+Environment variable interpolation functionality for dotyaml.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from typing import Any, Dict, Union
+
+
+def interpolate_env_vars(data: Union[str, Dict[str, Any], Any]) -> Union[str, Dict[str, Any], Any]:
+    """
+    Recursively interpolate environment variables in YAML data using Jinja-like syntax.
+
+    Supports syntax like: ``{{ ENV_VAR_NAME }}`` or ``{{ ENV_VAR_NAME|default_value }}``
+
+    :param data: Data structure to interpolate (string, dict, list, etc).
+    :type data: str or dict[str, Any] or Any
+    :return: Data structure with environment variables interpolated.
+    :rtype: str or dict[str, Any] or Any
+    """
+
+    if isinstance(data, str):
+        return _interpolate_string(data)
+    if isinstance(data, dict):
+        return {key: interpolate_env_vars(value) for key, value in data.items()}
+    if isinstance(data, list):
+        return [interpolate_env_vars(item) for item in data]
+    return data
+
+
+def _interpolate_string(text: str) -> str:
+    """
+    Interpolate environment variables in a string using Jinja-like syntax.
+
+    Supports:
+    - ``{{ ENV_VAR }}`` required environment variable.
+    - ``{{ ENV_VAR|default_value }}`` environment variable with default.
+
+    :param text: String to interpolate.
+    :type text: str
+    :return: String with environment variables interpolated.
+    :rtype: str
+    :raises ValueError: If a required environment variable is not found.
+    """
+
+    pattern = r"\{\{\s*([A-Z_][A-Z0-9_]*)\s*(?:\|\s*([^}]*?))?\s*\}\}"
+
+    def replace_match(match):  # type: ignore[no-untyped-def]
+        env_var = match.group(1)
+        default_value = match.group(2)
+
+        env_value = os.getenv(env_var)
+
+        if env_value is not None:
+            return env_value
+        if default_value is not None:
+            return default_value.strip()
+        raise ValueError(f"Required environment variable '{env_var}' not found")
+
+    return re.sub(pattern, replace_match, text)
+

biblicus/_vendor/dotyaml/loader.py

@@ -0,0 +1,181 @@
+"""
+Core loading functionality for dotyaml.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
+
+import yaml
+
+try:
+    from dotenv import load_dotenv
+
+    DOTENV_AVAILABLE = True
+except ImportError:
+    DOTENV_AVAILABLE = False
+
+from .interpolation import interpolate_env_vars
+from .transformer import flatten_dict, unflatten_env_vars
+
+
+def load_config(
+    yaml_path: Optional[Union[str, Path]] = None,
+    prefix: str = "",
+    override: bool = False,
+    dotenv_path: Optional[Union[str, Path]] = ".env",
+    load_dotenv_first: bool = True,
+) -> Dict[str, str]:
+    """
+    Load configuration from a YAML file and set environment variables.
+
+    :param yaml_path: Path to YAML configuration file. When None, only reads existing env vars.
+    :type yaml_path: str or Path or None
+    :param prefix: Prefix for environment variable names (for example, ``APP``).
+    :type prefix: str
+    :param override: Whether to override existing environment variables.
+    :type override: bool
+    :param dotenv_path: Optional ``.env`` file path to load first.
+    :type dotenv_path: str or Path or None
+    :param load_dotenv_first: Whether to load ``.env`` before YAML.
+    :type load_dotenv_first: bool
+    :return: Mapping of values that were set.
+    :rtype: dict[str, str]
+    """
+
+    config: Dict[str, str] = {}
+
+    if load_dotenv_first and DOTENV_AVAILABLE and dotenv_path:
+        env_file = Path(dotenv_path)
+        env_locations: list[Path] = []
+
+        if env_file.is_absolute():
+            env_locations.append(env_file)
+        else:
+            env_locations.append(Path.cwd() / dotenv_path)
+            if yaml_path:
+                yaml_dir = Path(yaml_path).parent
+                env_locations.append(yaml_dir / dotenv_path)
+
+        for env_path in env_locations:
+            if env_path.exists():
+                load_dotenv(env_path)
+                break
+
+    if yaml_path and Path(yaml_path).exists():
+        with open(yaml_path, "r", encoding="utf-8") as file:
+            yaml_data = yaml.safe_load(file)
+
+        if yaml_data:
+            yaml_data = interpolate_env_vars(yaml_data)
+            flat_config = flatten_dict(yaml_data, prefix)
+
+            for key, value in flat_config.items():
+                if not override and key in os.environ:
+                    config[key] = os.environ[key]
+                else:
+                    os.environ[key] = value
+                    config[key] = value
+
+    return config
+
+
+class ConfigLoader:
+    """
+    Configuration loader that can read YAML files or environment variables.
+    """
+
+    def __init__(
+        self,
+        prefix: str = "",
+        schema: Optional[Dict[str, Any]] = None,
+        dotenv_path: Optional[Union[str, Path]] = ".env",
+        load_dotenv_first: bool = True,
+    ):
+        self.prefix = prefix
+        self.schema = schema
+        self.dotenv_path = dotenv_path
+        self.load_dotenv_first = load_dotenv_first
+
+        if self.load_dotenv_first and DOTENV_AVAILABLE and self.dotenv_path:
+            env_file = Path(self.dotenv_path)
+            env_locations: list[Path] = []
+
+            if env_file.is_absolute():
+                env_locations.append(env_file)
+            else:
+                env_locations.append(Path.cwd() / self.dotenv_path)
+
+            for env_path in env_locations:
+                if env_path.exists():
+                    load_dotenv(env_path)
+                    break
+
+    def load_from_yaml(self, yaml_path: Union[str, Path]) -> Dict[str, Any]:
+        """
+        Load configuration from a YAML file with environment variable interpolation.
+
+        :param yaml_path: YAML configuration file path.
+        :type yaml_path: str or Path
+        :return: Parsed YAML data.
+        :rtype: dict[str, Any]
+        """
+
+        if not Path(yaml_path).exists():
+            return {}
+
+        if self.load_dotenv_first and DOTENV_AVAILABLE and self.dotenv_path:
+            env_file = Path(self.dotenv_path)
+            env_locations: list[Path] = []
+
+            if env_file.is_absolute():
+                env_locations.append(env_file)
+            else:
+                env_locations.append(Path.cwd() / self.dotenv_path)
+                yaml_dir = Path(yaml_path).parent
+                env_locations.append(yaml_dir / self.dotenv_path)
+
+            for env_path in env_locations:
+                if env_path.exists():
+                    load_dotenv(env_path)
+                    break
+
+        with open(yaml_path, "r", encoding="utf-8") as file:
+            yaml_data = yaml.safe_load(file)
+
+        if yaml_data:
+            yaml_data = interpolate_env_vars(yaml_data)
+
+        return yaml_data or {}
+
+    def load_from_env(self) -> Dict[str, Any]:
+        """
+        Load configuration from environment variables.
+
+        :return: Nested configuration dictionary.
+        :rtype: dict[str, Any]
+        """
+
+        env_vars = dict(os.environ)
+        return unflatten_env_vars(env_vars, self.prefix)
+
+    def set_env_vars(self, config: Dict[str, Any], override: bool = False) -> None:
+        """
+        Set environment variables from a configuration dictionary.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :param override: Whether to override existing environment variables.
+        :type override: bool
+        :return: None.
+        :rtype: None
+        """
+
+        flat_config = flatten_dict(config, self.prefix)
+
+        for key, value in flat_config.items():
+            if override or key not in os.environ:
+                os.environ[key] = value
+

biblicus/_vendor/dotyaml/transformer.py

@@ -0,0 +1,135 @@
+"""
+YAML to environment variable transformation utilities for dotyaml.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Dict
+
+
+def flatten_dict(data: Dict[str, Any], prefix: str = "", separator: str = "_") -> Dict[str, str]:
+    """
+    Flatten a nested dictionary into environment-variable style keys.
+
+    :param data: Nested dictionary to flatten.
+    :type data: dict[str, Any]
+    :param prefix: Prefix to add to all keys.
+    :type prefix: str
+    :param separator: Separator between key parts.
+    :type separator: str
+    :return: Flattened mapping with string values.
+    :rtype: dict[str, str]
+    """
+
+    result: Dict[str, str] = {}
+
+    for key, value in data.items():
+        if prefix:
+            full_key = f"{prefix}{separator}{key.upper()}"
+        else:
+            full_key = key.upper()
+
+        clean_key = full_key.replace("-", "_").replace(".", "_")
+
+        if isinstance(value, dict):
+            result.update(flatten_dict(value, clean_key, separator))
+        else:
+            result[clean_key] = convert_value_to_string(value)
+
+    return result
+
+
+def convert_value_to_string(value: Any) -> str:
+    """
+    Convert a Python value to its environment variable string representation.
+
+    :param value: Value to convert.
+    :type value: Any
+    :return: String representation suitable for environment variables.
+    :rtype: str
+    """
+
+    if value is None:
+        return ""
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (int, float)):
+        return str(value)
+    if isinstance(value, str):
+        return value
+    if isinstance(value, (list, tuple)):
+        return ",".join(convert_value_to_string(item) for item in value)
+    if isinstance(value, dict):
+        return json.dumps(value)
+    return str(value)
+
+
+def unflatten_env_vars(env_vars: Dict[str, str], prefix: str = "") -> Dict[str, Any]:
+    """
+    Convert flat environment variables back to nested dictionary structure.
+
+    :param env_vars: Mapping of environment variables.
+    :type env_vars: dict[str, str]
+    :param prefix: Optional prefix to filter by.
+    :type prefix: str
+    :return: Nested dictionary structure.
+    :rtype: dict[str, Any]
+    """
+
+    result: Dict[str, Any] = {}
+
+    for key, value in env_vars.items():
+        if prefix and not key.startswith(f"{prefix}_"):
+            continue
+
+        clean_key = key
+        if prefix:
+            clean_key = key[len(prefix) + 1 :]
+
+        parts = clean_key.lower().split("_")
+
+        current: Dict[str, Any] = result
+        for part in parts[:-1]:
+            if part not in current:
+                current[part] = {}
+            current = current[part]
+
+        final_key = parts[-1]
+        current[final_key] = convert_string_to_value(value)
+
+    return result
+
+
+def convert_string_to_value(value: str) -> Any:
+    """
+    Convert a string environment variable back to an appropriate Python type.
+
+    :param value: String value from an environment variable.
+    :type value: str
+    :return: Converted Python value.
+    :rtype: Any
+    """
+
+    if value == "":
+        return None
+    lowered = value.lower()
+    if lowered == "true":
+        return True
+    if lowered == "false":
+        return False
+    if value.isdigit():
+        return int(value)
+    if value.replace(".", "").replace("-", "").isdigit():
+        try:
+            return float(value)
+        except ValueError:
+            return value
+    if "," in value:
+        items = [item.strip() for item in value.split(",")]
+        return [convert_string_to_value(item) for item in items]
+    try:
+        return json.loads(value)
+    except (json.JSONDecodeError, ValueError):
+        return value
+

biblicus/backends/__init__.py

@@ -18,7 +18,6 @@ def available_backends() -> Dict[str, Type[RetrievalBackend]]:
     :return: Mapping of backend identifiers to backend classes.
     :rtype: dict[str, Type[RetrievalBackend]]
     """
-
     return {
         ScanBackend.backend_id: ScanBackend,
         SqliteFullTextSearchBackend.backend_id: SqliteFullTextSearchBackend,
@@ -35,7 +34,6 @@ def get_backend(backend_id: str) -> RetrievalBackend:
     :rtype: RetrievalBackend
     :raises KeyError: If the backend identifier is unknown.
     """
-
     registry = available_backends()
     backend_class = registry.get(backend_id)
     if backend_class is None:

biblicus/backends/base.py

@@ -22,7 +22,9 @@ class RetrievalBackend(ABC):
     backend_id: str
 
     @abstractmethod
-    def build_run(self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]) -> RetrievalRun:
+    def build_run(
+        self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]
+    ) -> RetrievalRun:
         """
         Build or register a retrieval run for the backend.
 
@@ -35,7 +37,6 @@ class RetrievalBackend(ABC):
         :return: Run manifest describing the build.
         :rtype: RetrievalRun
         """
-
         raise NotImplementedError
 
     @abstractmethod
@@ -61,5 +62,4 @@ class RetrievalBackend(ABC):
         :return: Retrieval results containing evidence.
         :rtype: RetrievalResult
         """
-
         raise NotImplementedError

biblicus/backends/scan.py

@@ -9,9 +9,15 @@ from typing import Dict, Iterable, List, Optional, Tuple
 from pydantic import BaseModel, ConfigDict, Field
 
 from ..corpus import Corpus
-from ..extraction import ExtractionRunReference, parse_extraction_run_reference
 from ..frontmatter import parse_front_matter
-from ..models import Evidence, QueryBudget, RetrievalResult, RetrievalRun
+from ..models import (
+    Evidence,
+    ExtractionRunReference,
+    QueryBudget,
+    RetrievalResult,
+    RetrievalRun,
+    parse_extraction_run_reference,
+)
 from ..retrieval import apply_budget, create_recipe_manifest, create_run_manifest, hash_text
 from ..time import utc_now_iso
 
@@ -42,7 +48,9 @@ class ScanBackend:
 
     backend_id = "scan"
 
-    def build_run(self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]) -> RetrievalRun:
+    def build_run(
+        self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]
+    ) -> RetrievalRun:
         """
         Register a scan backend run (no materialization).
 
@@ -55,7 +63,6 @@ class ScanBackend:
         :return: Run manifest describing the build.
         :rtype: RetrievalRun
         """
-
         recipe_config = ScanRecipeConfig.model_validate(config)
         catalog = corpus.load_catalog()
         recipe = create_recipe_manifest(
@@ -63,7 +70,10 @@ class ScanBackend:
             name=recipe_name,
             config=recipe_config.model_dump(),
         )
-        stats = {"items": len(catalog.items), "text_items": _count_text_items(corpus, catalog.items.values(), recipe_config)}
+        stats = {
+            "items": len(catalog.items),
+            "text_items": _count_text_items(corpus, catalog.items.values(), recipe_config),
+        }
         run = create_run_manifest(corpus, recipe=recipe, stats=stats, artifact_paths=[])
         corpus.write_run(run)
         return run
@@ -90,7 +100,6 @@ class ScanBackend:
         :return: Retrieval results containing evidence.
         :rtype: RetrievalResult
         """
-
         recipe_config = ScanRecipeConfig.model_validate(run.recipe.config)
         catalog = corpus.load_catalog()
         extraction_reference = _resolve_extraction_reference(corpus, recipe_config)
@@ -130,7 +139,9 @@ class ScanBackend:
         )
 
 
-def _resolve_extraction_reference(corpus: Corpus, recipe_config: ScanRecipeConfig) -> Optional[ExtractionRunReference]:
+def _resolve_extraction_reference(
+    corpus: Corpus, recipe_config: ScanRecipeConfig
+) -> Optional[ExtractionRunReference]:
     """
     Resolve an extraction run reference from a recipe config.
 
@@ -142,7 +153,6 @@ def _resolve_extraction_reference(corpus: Corpus, recipe_config: ScanRecipeConfi
     :rtype: ExtractionRunReference or None
     :raises FileNotFoundError: If an extraction run is referenced but not present.
     """
-
     if not recipe_config.extraction_run:
         return None
     extraction_reference = parse_extraction_run_reference(recipe_config.extraction_run)
@@ -155,7 +165,9 @@ def _resolve_extraction_reference(corpus: Corpus, recipe_config: ScanRecipeConfi
     return extraction_reference
 
 
-def _count_text_items(corpus: Corpus, items: Iterable[object], recipe_config: ScanRecipeConfig) -> int:
+def _count_text_items(
+    corpus: Corpus, items: Iterable[object], recipe_config: ScanRecipeConfig
+) -> int:
     """
     Count catalog items that represent text content.
 
@@ -170,7 +182,6 @@ def _count_text_items(corpus: Corpus, items: Iterable[object], recipe_config: Sc
     :return: Number of text items.
     :rtype: int
     """
-
     text_item_count = 0
     extraction_reference = _resolve_extraction_reference(corpus, recipe_config)
     for catalog_item in items:
@@ -199,7 +210,6 @@ def _tokenize_query(query_text: str) -> List[str]:
     :return: Lowercased non-empty tokens.
     :rtype: list[str]
     """
-
     return [token for token in query_text.lower().split() if token]
 
 
@@ -227,7 +237,6 @@ def _load_text_from_item(
     :return: Text payload or None if not decodable as text.
     :rtype: str or None
     """
-
     if extraction_reference:
         extracted_text = corpus.read_extracted_text(
             extractor_id=extraction_reference.extractor_id,
@@ -259,7 +268,6 @@ def _find_first_match(text: str, tokens: List[str]) -> Optional[Tuple[int, int]]
     :return: Start/end span for the earliest match, or None if no matches.
     :rtype: tuple[int, int] or None
     """
-
     lower_text = text.lower()
     best_start: Optional[int] = None
     best_end: Optional[int] = None
@@ -291,7 +299,6 @@ def _build_snippet(text: str, span: Optional[Tuple[int, int]], *, max_chars: int
     :return: Snippet text.
     :rtype: str
     """
-
     if not text:
         return ""
     if span is None:
@@ -325,7 +332,6 @@ def _score_items(
     :return: Evidence candidates with provisional ranks.
     :rtype: list[Evidence]
     """
-
     evidence_items: List[Evidence] = []
     for catalog_item in items:
         media_type = getattr(catalog_item, "media_type", "")