biblicus 0.13.0__py3-none-any.whl → 0.15.0__py3-none-any.whl

biblicus/__init__.py

@@ -27,4 +27,4 @@ __all__ = [
     "RetrievalRun",
 ]
 
-__version__ = "0.13.0"
+__version__ = "0.15.0"

biblicus/_vendor/dotyaml/__init__.py

@@ -8,6 +8,6 @@ loading and interpolating YAML configuration files.
 from __future__ import annotations
 
 from .interpolation import interpolate_env_vars
-from .loader import ConfigLoader, load_config
+from .loader import ConfigLoader, load_config, load_yaml_view
 
-__all__ = ["ConfigLoader", "interpolate_env_vars", "load_config"]
+__all__ = ["ConfigLoader", "interpolate_env_vars", "load_config", "load_yaml_view"]

biblicus/_vendor/dotyaml/loader.py

@@ -6,7 +6,7 @@ from __future__ import annotations
 
 import os
 from pathlib import Path
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, Iterable, Optional, Union
 
 import yaml
 
@@ -82,6 +82,45 @@ def load_config(
     return config
 
 
+def _merge_mapping_values(base: Dict[str, Any], overlay: Dict[str, Any]) -> Dict[str, Any]:
+    merged: Dict[str, Any] = dict(base)
+    for key, value in overlay.items():
+        existing = merged.get(key)
+        if isinstance(existing, dict) and isinstance(value, dict):
+            merged[key] = _merge_mapping_values(existing, value)
+        else:
+            merged[key] = value
+    return merged
+
+
+def load_yaml_view(paths: Iterable[Union[str, Path]]) -> Dict[str, Any]:
+    """
+    Load and compose one or more YAML files into a single mapping.
+
+    :param paths: Iterable of YAML file paths in precedence order.
+    :type paths: Iterable[str | Path]
+    :return: Composed YAML mapping.
+    :rtype: dict[str, Any]
+    :raises ValueError: If any YAML file does not contain a mapping.
+    """
+
+    composed: Dict[str, Any] = {}
+    for raw_path in paths:
+        yaml_path = Path(raw_path)
+        with open(yaml_path, "r", encoding="utf-8") as file:
+            yaml_data = yaml.safe_load(file)
+
+        if yaml_data is None:
+            yaml_data = {}
+
+        if not isinstance(yaml_data, dict):
+            raise ValueError(f"YAML content must be a mapping for {yaml_path}")
+
+        yaml_data = interpolate_env_vars(yaml_data)
+        composed = _merge_mapping_values(composed, yaml_data)
+    return composed
+
+
 class ConfigLoader:
     """
     Configuration loader that can read YAML files or environment variables.

biblicus/ai/__init__.py

@@ -0,0 +1,39 @@
+"""
+Provider-backed AI utilities for Biblicus.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+__all__ = [
+    "AiProvider",
+    "EmbeddingsClientConfig",
+    "LlmClientConfig",
+    "generate_completion",
+    "generate_embeddings",
+    "generate_embeddings_batch",
+]
+
+
+def __getattr__(name: str) -> Any:
+    if name in {"AiProvider", "EmbeddingsClientConfig", "LlmClientConfig"}:
+        from .models import AiProvider, EmbeddingsClientConfig, LlmClientConfig
+
+        return {
+            "AiProvider": AiProvider,
+            "EmbeddingsClientConfig": EmbeddingsClientConfig,
+            "LlmClientConfig": LlmClientConfig,
+        }[name]
+    if name in {"generate_completion"}:
+        from .llm import generate_completion
+
+        return generate_completion
+    if name in {"generate_embeddings", "generate_embeddings_batch"}:
+        from .embeddings import generate_embeddings, generate_embeddings_batch
+
+        return {
+            "generate_embeddings": generate_embeddings,
+            "generate_embeddings_batch": generate_embeddings_batch,
+        }[name]
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

biblicus/ai/embeddings.py

@@ -0,0 +1,114 @@
+"""
+Provider-backed text embeddings.
+"""
+
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any, List, Sequence
+
+from .models import EmbeddingsClientConfig
+
+
+def _require_dspy_embedder():
+    try:
+        import dspy
+    except ImportError as import_error:
+        raise ValueError(
+            "DSPy backend requires an optional dependency. "
+            'Install it with pip install "biblicus[dspy]".'
+        ) from import_error
+    if not hasattr(dspy, "Embedder"):
+        raise ValueError(
+            "DSPy backend requires an optional dependency with Embedder support. "
+            'Install it with pip install "biblicus[dspy]".'
+        )
+    return dspy
+
+
+def generate_embeddings(*, client: EmbeddingsClientConfig, text: str) -> List[float]:
+    """
+    Generate a single embedding vector.
+
+    :param client: Embeddings client configuration.
+    :type client: biblicus.ai.models.EmbeddingsClientConfig
+    :param text: Input text to embed.
+    :type text: str
+    :return: Embedding vector.
+    :rtype: list[float]
+    """
+    vectors = generate_embeddings_batch(client=client, texts=[text])
+    return vectors[0]
+
+
+def _chunks(texts: Sequence[str], batch_size: int) -> List[List[str]]:
+    return [list(texts[idx : idx + batch_size]) for idx in range(0, len(texts), batch_size)]
+
+
+def _normalize_embeddings(embeddings: Any) -> List[List[float]]:
+    if hasattr(embeddings, "tolist"):
+        embeddings = embeddings.tolist()
+    if isinstance(embeddings, list) and embeddings and not isinstance(embeddings[0], list):
+        return [[float(value) for value in embeddings]]
+    return [[float(value) for value in row] for row in embeddings]
+
+
+def generate_embeddings_batch(
+    *, client: EmbeddingsClientConfig, texts: Sequence[str]
+) -> List[List[float]]:
+    """
+    Generate embeddings for a batch of texts.
+
+    The implementation performs batched requests and can run requests concurrently.
+
+    :param client: Embeddings client configuration.
+    :type client: biblicus.ai.models.EmbeddingsClientConfig
+    :param texts: Text inputs to embed.
+    :type texts: Sequence[str]
+    :return: Embedding vectors in input order.
+    :rtype: list[list[float]]
+    :raises ValueError: If required dependencies or credentials are missing.
+    """
+    if not texts:
+        return []
+
+    dspy = _require_dspy_embedder()
+
+    model = client.litellm_model()
+    request_kwargs = client.build_litellm_kwargs()
+
+    items = list(texts)
+    if len(items) == 1:
+        embedder = dspy.Embedder(
+            model,
+            batch_size=1,
+            caching=False,
+            **request_kwargs,
+        )
+        embeddings = embedder(items[0])
+        return _normalize_embeddings(embeddings)
+
+    batches = _chunks(items, client.batch_size)
+    results: List[List[List[float]]] = [None for _ in range(len(batches))]  # type: ignore[list-item]
+
+    def _embed_batch(batch_texts: List[str]) -> List[List[float]]:
+        embedder = dspy.Embedder(
+            model,
+            batch_size=len(batch_texts),
+            caching=False,
+            **request_kwargs,
+        )
+        embeddings = embedder(batch_texts)
+        return _normalize_embeddings(embeddings)
+
+    with ThreadPoolExecutor(max_workers=client.parallelism) as executor:
+        futures = {executor.submit(_embed_batch, batch): idx for idx, batch in enumerate(batches)}
+        for future in as_completed(futures):
+            idx = futures[future]
+            results[idx] = future.result()
+
+    flattened: List[List[float]] = []
+    for batch_vectors in results:
+        for vector in batch_vectors:
+            flattened.append(vector)
+    return flattened

biblicus/ai/llm.py

@@ -0,0 +1,138 @@
+"""
+Provider-backed chat completions.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Optional, Sequence
+
+from .models import LlmClientConfig
+
+
+@dataclass
+class ChatCompletionResult:
+    """
+    Normalized response from a chat completion call.
+
+    :param text: Generated assistant text.
+    :type text: str
+    :param tool_calls: Structured tool calls from the provider.
+    :type tool_calls: list[dict[str, Any]]
+    """
+
+    text: str
+    tool_calls: list[dict[str, Any]]
+
+
+def _require_dspy():
+    try:
+        import dspy
+    except ImportError as import_error:
+        raise ValueError(
+            "DSPy backend requires an optional dependency. "
+            'Install it with pip install "biblicus[dspy]".'
+        ) from import_error
+    if not hasattr(dspy, "LM"):
+        raise ValueError(
+            "DSPy backend requires an optional dependency with LM support. "
+            'Install it with pip install "biblicus[dspy]".'
+        )
+    return dspy
+
+
+def _normalize_tool_calls(tool_calls: Sequence[object]) -> list[dict[str, Any]]:
+    normalized: list[dict[str, Any]] = []
+    for tool_call in tool_calls:
+        if isinstance(tool_call, dict):
+            function = tool_call.get("function") or {}
+            normalized.append(
+                {
+                    "id": str(tool_call.get("id") or ""),
+                    "type": str(tool_call.get("type") or "function"),
+                    "function": {
+                        "name": str(function.get("name") or ""),
+                        "arguments": str(function.get("arguments") or ""),
+                    },
+                }
+            )
+            continue
+        function = getattr(tool_call, "function", None)
+        normalized.append(
+            {
+                "id": str(getattr(tool_call, "id", "") or ""),
+                "type": str(getattr(tool_call, "type", "function") or "function"),
+                "function": {
+                    "name": str(getattr(function, "name", "") or ""),
+                    "arguments": str(getattr(function, "arguments", "") or ""),
+                },
+            }
+        )
+    return normalized
+
+
+def chat_completion(
+    *,
+    client: LlmClientConfig,
+    messages: Sequence[dict[str, Any]],
+    tools: Optional[Sequence[dict[str, Any]]] = None,
+    tool_choice: Optional[str] = None,
+) -> ChatCompletionResult:
+    """
+    Execute a chat completion using DSPy (LiteLLM-backed).
+
+    :param client: LLM client configuration.
+    :type client: biblicus.ai.models.LlmClientConfig
+    :param messages: Chat messages payload.
+    :type messages: Sequence[dict[str, Any]]
+    :param tools: Optional tool definitions to pass through.
+    :type tools: Sequence[dict[str, Any]] or None
+    :param tool_choice: Optional tool choice directive.
+    :type tool_choice: str or None
+    :return: Normalized completion result.
+    :rtype: ChatCompletionResult
+    :raises ValueError: If required dependencies or credentials are missing.
+    """
+    dspy = _require_dspy()
+    lm = dspy.LM(client.litellm_model(), **client.build_litellm_kwargs())
+    request_kwargs: dict[str, Any] = {}
+    if tools:
+        request_kwargs["tools"] = list(tools)
+    if tool_choice:
+        request_kwargs["tool_choice"] = tool_choice
+    if client.response_format:
+        request_kwargs["response_format"] = {"type": client.response_format}
+
+    response = lm(messages=list(messages), **request_kwargs)
+    item = response[0] if isinstance(response, list) and response else response
+    if isinstance(item, dict):
+        text = str(item.get("text") or item.get("content") or "")
+        tool_calls = _normalize_tool_calls(item.get("tool_calls") or [])
+        return ChatCompletionResult(text=text, tool_calls=tool_calls)
+    return ChatCompletionResult(text=str(item or ""), tool_calls=[])
+
+
+def generate_completion(
+    *,
+    client: LlmClientConfig,
+    system_prompt: Optional[str],
+    user_prompt: str,
+) -> str:
+    """
+    Generate a completion using the configured provider.
+
+    :param client: LLM client configuration.
+    :type client: biblicus.ai.models.LlmClientConfig
+    :param system_prompt: Optional system prompt content.
+    :type system_prompt: str or None
+    :param user_prompt: User prompt content.
+    :type user_prompt: str
+    :return: Generated completion text.
+    :rtype: str
+    :raises ValueError: If required dependencies or credentials are missing.
+    """
+    messages: list[dict[str, Any]] = []
+    if system_prompt:
+        messages.append({"role": "system", "content": system_prompt})
+    messages.append({"role": "user", "content": user_prompt})
+    return chat_completion(client=client, messages=messages).text

biblicus/ai/models.py

@@ -0,0 +1,226 @@
+"""
+Pydantic models for provider-backed AI clients.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Optional
+
+from pydantic import Field, field_validator
+
+from ..analysis.schema import AnalysisSchemaModel
+from ..user_config import resolve_openai_api_key
+
+
+class AiProvider(str, Enum):
+    """
+    Supported AI providers.
+    """
+
+    OPENAI = "openai"
+    BEDROCK = "bedrock"
+    ANTHROPIC = "anthropic"
+    GEMINI = "gemini"
+    OLLAMA = "ollama"
+    LITELLM = "litellm"
+
+
+def _normalize_provider(value: object, *, error_label: str) -> str:
+    if isinstance(value, AiProvider):
+        return value.value
+    if isinstance(value, str):
+        return value.lower()
+    raise ValueError(f"{error_label} must be a string or AiProvider")
+
+
+def _litellm_model(provider: str, model: str) -> str:
+    normalized_model = model.strip()
+    if "/" in normalized_model:
+        return normalized_model
+    return f"{provider}/{normalized_model}"
+
+
+class LlmClientConfig(AnalysisSchemaModel):
+    """
+    Configuration for a chat completion invocation.
+
+    :ivar provider: Provider identifier.
+    :vartype provider: str or AiProvider
+    :ivar model: Model identifier.
+    :vartype model: str
+    :ivar api_key: Optional API key override.
+    :vartype api_key: str or None
+    :ivar api_base: Optional API base override.
+    :vartype api_base: str or None
+    :ivar temperature: Optional generation temperature.
+    :vartype temperature: float or None
+    :ivar max_tokens: Optional maximum output tokens.
+    :vartype max_tokens: int or None
+    :ivar response_format: Optional response format identifier.
+    :vartype response_format: str or None
+    :ivar max_retries: Optional maximum retry count for transient failures.
+    :vartype max_retries: int
+    :ivar timeout_seconds: Optional request timeout in seconds.
+    :vartype timeout_seconds: float or None
+    :ivar model_type: Optional model type identifier.
+    :vartype model_type: str or None
+    :ivar extra_params: Additional provider-specific parameters to pass through.
+    :vartype extra_params: dict[str, Any]
+    """
+
+    provider: str
+    model: str = Field(min_length=1)
+    api_key: Optional[str] = None
+    api_base: Optional[str] = None
+    temperature: Optional[float] = Field(default=None, ge=0.0)
+    max_tokens: Optional[int] = Field(default=None, ge=1)
+    response_format: Optional[str] = None
+    max_retries: int = Field(default=0, ge=0)
+    timeout_seconds: Optional[float] = Field(default=None, gt=0.0)
+    model_type: Optional[str] = None
+    extra_params: dict[str, Any] = Field(default_factory=dict)
+
+    @field_validator("provider", mode="before")
+    @classmethod
+    def _parse_provider(cls, value: object) -> str:
+        return _normalize_provider(value, error_label="llm client provider")
+
+    def litellm_model(self) -> str:
+        """
+        Resolve the DSPy model identifier for this client.
+
+        :return: DSPy model string (LiteLLM format).
+        :rtype: str
+        """
+        return _litellm_model(self.provider, self.model)
+
+    def resolve_api_key(self) -> Optional[str]:
+        """
+        Resolve an API key for the configured provider.
+
+        :return: API key string or None if not required.
+        :rtype: str or None
+        :raises ValueError: If OpenAI is configured and no key is available.
+        """
+        if self.api_key:
+            return self.api_key
+        if self.provider != AiProvider.OPENAI.value:
+            return None
+        api_key = resolve_openai_api_key()
+        if api_key is None:
+            raise ValueError(
+                "OpenAI provider requires an OpenAI API key. "
+                "Set OPENAI_API_KEY or configure it in ~/.biblicus/config.yml or ./.biblicus/config.yml under "
+                "openai.api_key."
+            )
+        return api_key
+
+    def build_litellm_kwargs(self) -> dict[str, Any]:
+        """
+        Build DSPy keyword arguments for chat completions.
+
+        :return: Keyword arguments for DSPy (LiteLLM-backed).
+        :rtype: dict[str, Any]
+        """
+        api_key = self.resolve_api_key()
+        base_kwargs: dict[str, Any] = {
+            "api_key": api_key,
+            "api_base": self.api_base,
+            "temperature": self.temperature,
+            "max_tokens": self.max_tokens,
+            "model_type": self.model_type,
+            "timeout": self.timeout_seconds,
+            "num_retries": self.max_retries,
+        }
+        for key, value in (self.extra_params or {}).items():
+            base_kwargs[key] = value
+        return {key: value for key, value in base_kwargs.items() if value is not None}
+
+
+class EmbeddingsClientConfig(AnalysisSchemaModel):
+    """
+    Configuration for an embeddings invocation.
+
+    :ivar provider: Provider identifier.
+    :vartype provider: str or AiProvider
+    :ivar model: Model identifier.
+    :vartype model: str
+    :ivar api_key: Optional API key override.
+    :vartype api_key: str or None
+    :ivar api_base: Optional API base override.
+    :vartype api_base: str or None
+    :ivar batch_size: Maximum number of texts per request.
+    :vartype batch_size: int
+    :ivar parallelism: Maximum number of concurrent requests.
+    :vartype parallelism: int
+    :ivar max_retries: Optional maximum retry count for transient failures.
+    :vartype max_retries: int
+    :ivar timeout_seconds: Optional request timeout in seconds.
+    :vartype timeout_seconds: float or None
+    :ivar extra_params: Additional provider-specific parameters to pass through.
+    :vartype extra_params: dict[str, Any]
+    """
+
+    provider: str
+    model: str = Field(min_length=1)
+    api_key: Optional[str] = None
+    api_base: Optional[str] = None
+    batch_size: int = Field(default=64, ge=1)
+    parallelism: int = Field(default=4, ge=1)
+    max_retries: int = Field(default=0, ge=0)
+    timeout_seconds: Optional[float] = Field(default=None, gt=0.0)
+    extra_params: dict[str, Any] = Field(default_factory=dict)
+
+    @field_validator("provider", mode="before")
+    @classmethod
+    def _parse_provider(cls, value: object) -> str:
+        return _normalize_provider(value, error_label="embeddings provider")
+
+    def litellm_model(self) -> str:
+        """
+        Resolve the DSPy model identifier for this client.
+
+        :return: DSPy model string (LiteLLM format).
+        :rtype: str
+        """
+        return _litellm_model(self.provider, self.model)
+
+    def resolve_api_key(self) -> Optional[str]:
+        """
+        Resolve an API key for the configured provider.
+
+        :return: API key string or None if not required.
+        :rtype: str or None
+        :raises ValueError: If OpenAI is configured and no key is available.
+        """
+        if self.api_key:
+            return self.api_key
+        if self.provider != AiProvider.OPENAI.value:
+            return None
+        api_key = resolve_openai_api_key()
+        if api_key is None:
+            raise ValueError(
+                "OpenAI provider requires an OpenAI API key. "
+                "Set OPENAI_API_KEY or configure it in ~/.biblicus/config.yml or ./.biblicus/config.yml under "
+                "openai.api_key."
+            )
+        return api_key
+
+    def build_litellm_kwargs(self) -> dict[str, Any]:
+        """
+        Build DSPy keyword arguments for embeddings calls.
+
+        :return: Keyword arguments for DSPy (LiteLLM-backed).
+        :rtype: dict[str, Any]
+        """
+        api_key = self.resolve_api_key()
+        base_kwargs: dict[str, Any] = {
+            "api_key": api_key,
+            "api_base": self.api_base,
+            "timeout": self.timeout_seconds,
+            "num_retries": self.max_retries,
+        }
+        for key, value in (self.extra_params or {}).items():
+            base_kwargs[key] = value
+        return {key: value for key, value in base_kwargs.items() if value is not None}

biblicus/analysis/__init__.py

@@ -7,8 +7,6 @@ from __future__ import annotations
 from typing import Dict, Type
 
 from .base import CorpusAnalysisBackend
-from .profiling import ProfilingBackend
-from .topic_modeling import TopicModelingBackend
 
 
 def available_analysis_backends() -> Dict[str, Type[CorpusAnalysisBackend]]:
@@ -18,9 +16,14 @@ def available_analysis_backends() -> Dict[str, Type[CorpusAnalysisBackend]]:
     :return: Mapping of analysis identifiers to backend classes.
     :rtype: dict[str, Type[CorpusAnalysisBackend]]
     """
+    from .markov import MarkovBackend
+    from .profiling import ProfilingBackend
+    from .topic_modeling import TopicModelingBackend
+
     return {
         ProfilingBackend.analysis_id: ProfilingBackend,
         TopicModelingBackend.analysis_id: TopicModelingBackend,
+        MarkovBackend.analysis_id: MarkovBackend,
     }