agentforge-core 0.2.1__py3-none-any.whl

agentforge_core/__init__.py

@@ -0,0 +1,228 @@
+"""AgentForge core — stable contracts (ABCs, value types).
+
+Per ADR-0007, this package's public surface is the framework's locked
+contract layer. Adding a method to an ABC is a major version bump.
+
+This module re-exports every public symbol so consumers can import
+from `agentforge_core` directly. Submodules (`agentforge_core.contracts`,
+`agentforge_core.values`, `agentforge_core.production`) remain part of
+the public surface for granular imports.
+"""
+
+from __future__ import annotations
+
+from agentforge_core.config import (
+    AgentConfig,
+    AgentForgeConfig,
+    BudgetConfig,
+    EvaluatorEntry,
+    GraphModuleConfig,
+    LoggingConfig,
+    MemoryModuleConfig,
+    ModuleEntry,
+    ModulesConfig,
+    ObservabilityEntry,
+    OutputConfig,
+    ProviderConfig,
+    RerankerEntry,
+    RetrievalConfig,
+    RetrieverModuleConfig,
+    load_config,
+    parse_overrides,
+)
+from agentforge_core.contracts import (
+    EmbeddingClient,
+    EvalResult,
+    Evaluator,
+    Finding,
+    FindingRenderer,
+    GraphStore,
+    LLMClient,
+    MemoryStore,
+    Migration,
+    MigrationChecksumError,
+    MigrationStatus,
+    Migrator,
+    ReasoningStrategy,
+    Reranker,
+    Tool,
+    VectorStore,
+)
+from agentforge_core.migrations import discover_migrations
+from agentforge_core.observability import SCOPE_NAME as OBSERVABILITY_SCOPE_NAME
+from agentforge_core.observability import get_tracer
+from agentforge_core.production import (
+    AgentForgeError,
+    AuthenticationError,
+    BudgetExceeded,
+    BudgetPolicy,
+    CapabilityNotSupported,
+    GuardrailViolation,
+    JsonFormatter,
+    ModelNotFoundError,
+    ModuleError,
+    ProviderError,
+    RateLimitError,
+    RunContext,
+    RunIdFilter,
+    ServiceError,
+    TimeoutError,
+    bind_run,
+    current_run,
+    install_json_formatter,
+    install_run_id_filter,
+    new_run,
+    reset_run,
+    uninstall_json_formatter,
+    uninstall_run_id_filter,
+)
+
+# Imported from the submodule directly (not via production/__init__)
+# to avoid a circular import — see the comment in production/__init__.py
+# for the chain. This import runs *after* production finishes, so
+# LLMClient is already available.
+from agentforge_core.production.fallback import FallbackChain
+from agentforge_core.resolver import (
+    Resolver,
+    discover_entry_points,
+    parse_model_string,
+    register,
+    register_embedding_provider,
+    register_provider,
+    reset_discovery,
+)
+from agentforge_core.values import (
+    AgentState,
+    AppliedEnvVar,
+    AppliedManifest,
+    AppliedTemplate,
+    Claim,
+    EmbeddingResponse,
+    EnvVarEntry,
+    FinishReason,
+    GraphEdge,
+    GraphNode,
+    GraphPattern,
+    GraphSegment,
+    LLMResponse,
+    Manifest,
+    Message,
+    MessageRole,
+    ModuleInfo,
+    Path,
+    RunResult,
+    Step,
+    StepKind,
+    StopReason,
+    StreamChunk,
+    StreamChunkKind,
+    TemplateFile,
+    TokenUsage,
+    ToolCall,
+    ToolSpec,
+    VectorItem,
+    VectorMatch,
+)
+
+__version__ = "0.2.1"
+
+__all__ = [
+    "OBSERVABILITY_SCOPE_NAME",
+    "AgentConfig",
+    "AgentForgeConfig",
+    "AgentForgeError",
+    "AgentState",
+    "AppliedEnvVar",
+    "AppliedManifest",
+    "AppliedTemplate",
+    "AuthenticationError",
+    "BudgetConfig",
+    "BudgetExceeded",
+    "BudgetPolicy",
+    "CapabilityNotSupported",
+    "Claim",
+    "EmbeddingClient",
+    "EmbeddingResponse",
+    "EnvVarEntry",
+    "EvalResult",
+    "Evaluator",
+    "EvaluatorEntry",
+    "FallbackChain",
+    "Finding",
+    "FindingRenderer",
+    "FinishReason",
+    "GraphEdge",
+    "GraphModuleConfig",
+    "GraphNode",
+    "GraphPattern",
+    "GraphSegment",
+    "GraphStore",
+    "GuardrailViolation",
+    "JsonFormatter",
+    "LLMClient",
+    "LLMResponse",
+    "LoggingConfig",
+    "Manifest",
+    "MemoryModuleConfig",
+    "MemoryStore",
+    "Message",
+    "MessageRole",
+    "Migration",
+    "MigrationChecksumError",
+    "MigrationStatus",
+    "Migrator",
+    "ModelNotFoundError",
+    "ModuleEntry",
+    "ModuleError",
+    "ModuleInfo",
+    "ModulesConfig",
+    "ObservabilityEntry",
+    "OutputConfig",
+    "Path",
+    "ProviderConfig",
+    "ProviderError",
+    "RateLimitError",
+    "ReasoningStrategy",
+    "Reranker",
+    "RerankerEntry",
+    "Resolver",
+    "RetrievalConfig",
+    "RetrieverModuleConfig",
+    "RunContext",
+    "RunIdFilter",
+    "RunResult",
+    "ServiceError",
+    "Step",
+    "StepKind",
+    "StopReason",
+    "StreamChunk",
+    "StreamChunkKind",
+    "TemplateFile",
+    "TimeoutError",
+    "TokenUsage",
+    "Tool",
+    "ToolCall",
+    "ToolSpec",
+    "VectorItem",
+    "VectorMatch",
+    "VectorStore",
+    "__version__",
+    "bind_run",
+    "current_run",
+    "discover_entry_points",
+    "discover_migrations",
+    "get_tracer",
+    "install_json_formatter",
+    "install_run_id_filter",
+    "load_config",
+    "new_run",
+    "parse_model_string",
+    "parse_overrides",
+    "register",
+    "register_embedding_provider",
+    "register_provider",
+    "reset_discovery",
+    "reset_run",
+    "uninstall_json_formatter",
+    "uninstall_run_id_filter",
+]

agentforge_core/_bm25.py

@@ -0,0 +1,132 @@
+"""Private pure-Python BM25 helper (feat-022).
+
+Used by ``VectorStore`` drivers that don't have a native lexical
+path (e.g. the in-memory store) and by future hybrid retrieval
+test fixtures. Public-facing hybrid retrieval is driven via
+``Retriever(mode="hybrid")`` — this module is an implementation
+detail and not part of the framework's stable API.
+
+Tokeniser: lowercase + ``\\W+`` split + drop tokens of length ≤ 1.
+No stemming / stopword removal in v0.2 (keeps the dependency
+surface zero). Defaults follow Robertson: ``k1=1.5`` and ``b=0.75``.
+
+Formula (Okapi BM25):
+
+    score(D, Q) = Σ_t∈Q IDF(t) · TF_norm(t, D)
+
+    IDF(t) = ln( (N - df(t) + 0.5) / (df(t) + 0.5) + 1 )
+
+    TF_norm(t, D) =
+        ( tf(t, D) · (k1 + 1) ) /
+        ( tf(t, D) + k1 · (1 - b + b · |D| / avg_dl) )
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+from typing import Final
+
+_TOKEN_RE: Final = re.compile(r"\W+", re.UNICODE)
+_MIN_TOKEN_LEN: Final = 2
+
+
+def _tokenise(text: str) -> list[str]:
+    """Lowercase, split on non-word characters, drop tokens ≤ 1 char."""
+    return [tok for tok in _TOKEN_RE.split(text.lower()) if len(tok) >= _MIN_TOKEN_LEN]
+
+
+class _BM25Index:
+    """In-memory BM25 index over ``(doc_id, text)`` pairs.
+
+    Maintains per-document term frequencies, document lengths, and a
+    global document frequency table. Designed for small corpora
+    (~thousands of docs) — every ``add`` / ``delete`` is O(|tokens|)
+    and ``score`` is O(|query tokens| · |matching docs|).
+
+    Not thread-safe. Callers wrap with a mutex if needed.
+    """
+
+    def __init__(self, *, k1: float = 1.5, b: float = 0.75) -> None:
+        if k1 < 0:
+            raise ValueError(f"k1 must be >= 0, got {k1}")
+        if not 0.0 <= b <= 1.0:
+            raise ValueError(f"b must be in [0, 1], got {b}")
+        self._k1 = k1
+        self._b = b
+        self._tf: dict[str, Counter[str]] = {}
+        self._doc_len: dict[str, int] = {}
+        self._df: Counter[str] = Counter()
+
+    def add(self, doc_id: str, text: str) -> None:
+        """Insert or replace the document at ``doc_id``."""
+        if doc_id in self._tf:
+            self.delete(doc_id)
+        tokens = _tokenise(text)
+        if not tokens:
+            self._tf[doc_id] = Counter()
+            self._doc_len[doc_id] = 0
+            return
+        tf = Counter(tokens)
+        self._tf[doc_id] = tf
+        self._doc_len[doc_id] = len(tokens)
+        for term in tf:
+            self._df[term] += 1
+
+    def delete(self, doc_id: str) -> bool:
+        """Remove the document. Returns True if it existed."""
+        tf = self._tf.pop(doc_id, None)
+        if tf is None:
+            return False
+        self._doc_len.pop(doc_id, None)
+        for term in tf:
+            self._df[term] -= 1
+            if self._df[term] <= 0:
+                del self._df[term]
+        return True
+
+    def __len__(self) -> int:
+        return len(self._tf)
+
+    def score(self, query: str, *, limit: int) -> list[tuple[str, float]]:
+        """Return up to ``limit`` ``(doc_id, score)`` pairs sorted desc.
+
+        Scores are raw BM25 (unbounded ≥ 0). Empty corpus or empty
+        query returns ``[]``. Callers that want normalised scores
+        divide by the top score.
+        """
+        if limit < 1:
+            raise ValueError(f"limit must be >= 1, got {limit}")
+        query_tokens = _tokenise(query)
+        if not query_tokens or not self._tf:
+            return []
+
+        n_docs = len(self._tf)
+        total_len = sum(self._doc_len.values())
+        avg_dl = total_len / n_docs if n_docs else 0.0
+
+        idf_cache: dict[str, float] = {}
+        for term in set(query_tokens):
+            df = self._df.get(term, 0)
+            # +1 inside the log keeps IDF non-negative even when
+            # df > N/2 (which can happen on tiny corpora).
+            idf_cache[term] = math.log(((n_docs - df + 0.5) / (df + 0.5)) + 1.0)
+
+        scored: list[tuple[str, float]] = []
+        for doc_id, tf in self._tf.items():
+            doc_len = self._doc_len[doc_id]
+            score = 0.0
+            for term in query_tokens:
+                tf_term = tf.get(term, 0)
+                if tf_term == 0:
+                    continue
+                denom = tf_term + self._k1 * (
+                    1.0 - self._b + self._b * (doc_len / avg_dl if avg_dl else 0.0)
+                )
+                score += idf_cache[term] * (tf_term * (self._k1 + 1.0)) / denom
+            if score > 0.0:
+                scored.append((doc_id, score))
+
+        scored.sort(key=lambda kv: kv[1], reverse=True)
+        return scored[:limit]

agentforge_core/config/__init__.py

@@ -0,0 +1,62 @@
+"""Configuration system for AgentForge (feat-012).
+
+`agentforge.yaml` is the single source of truth for an agent's
+runtime wiring. This package ships:
+
+- The locked **root schema** (`AgentForgeConfig` + sub-models).
+- The **loader** (`load_config`) with env-var interpolation,
+  layered env files, dotted-path overrides, and module-side
+  schema validation.
+
+Per ADR-0013, configuration is *data* — no Jinja, no dynamic
+imports, no arbitrary template logic. Env-var interpolation
+(`${VAR}`, `${VAR:default}`, `${VAR:?error}`, `$$`) and
+schema-validated YAML are the only ways content flows into the
+runtime.
+
+The schema is locked under ADR-0007: adding a field is a minor
+bump; removing or renaming requires a major bump.
+"""
+
+from __future__ import annotations
+
+from agentforge_core.config.loader import load_config, parse_overrides
+from agentforge_core.config.module_schemas import validate_module_configs
+from agentforge_core.config.schema import (
+    AgentConfig,
+    AgentForgeConfig,
+    BudgetConfig,
+    EvaluatorEntry,
+    GraphModuleConfig,
+    LoggingConfig,
+    MemoryModuleConfig,
+    ModuleEntry,
+    ModulesConfig,
+    ObservabilityEntry,
+    OutputConfig,
+    ProviderConfig,
+    RerankerEntry,
+    RetrievalConfig,
+    RetrieverModuleConfig,
+)
+
+__all__ = [
+    "AgentConfig",
+    "AgentForgeConfig",
+    "BudgetConfig",
+    "EvaluatorEntry",
+    "GraphModuleConfig",
+    "LoggingConfig",
+    "MemoryModuleConfig",
+    "ModuleEntry",
+    "ModulesConfig",
+    "ObservabilityEntry",
+    "OutputConfig",
+    "ProviderConfig",
+    "RerankerEntry",
+    "RetrievalConfig",
+    "RetrieverModuleConfig",
+    "load_config",
+    "parse_overrides",
+    "validate_module_configs",
+]

agentforge_core/config/loader.py

@@ -0,0 +1,239 @@
+"""YAML loader for `agentforge.yaml` (feat-012).
+
+Resolution order (last wins) per spec §4.3:
+
+    1. Defaults from each Pydantic model.
+    2. agentforge.yaml on disk (if present).
+    3. agentforge.<env>.yaml (if AGENTFORGE_ENV set).
+    4. Env-var interpolation inside YAML values.
+    5. CLI / loader-API `--override agent.budget.usd=10` arguments.
+    6. Constructor kwargs to Agent (handled in `agentforge.agent`).
+
+Env-var interpolation syntax (feat-001):
+- `${VAR}` — required; raises at load if missing.
+- `${VAR:default}` — optional with default.
+- `${VAR:?error message}` — required with custom error.
+- `$$` — literal `$`.
+
+Env-var shortcuts honoured by `load_config`:
+- `AGENTFORGE_CONFIG` — overrides the default `./agentforge.yaml`
+  path (lowest precedence — still beaten by an explicit `path=`).
+- `AGENTFORGE_ENV` — picks the overlay file (e.g. `production` →
+  `agentforge.production.yaml` next to the base file).
+- `AGENTFORGE_LOG_LEVEL` — applied after schema validation to
+  `cfg.logging.level`.
+
+Per ADR-0013, the loader is data only — no Jinja, no dynamic
+imports, no template logic. Behaviour goes in Python code.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from agentforge_core.config.schema import AgentForgeConfig
+from agentforge_core.production.exceptions import ModuleError
+
+_INTERP_RE = re.compile(
+    r"""
+    \$\$                     # $$ -> literal $
+    | \$\{                   # ${
+        (?P<name>[A-Z_][A-Z0-9_]*)
+        (?:
+            :
+            (?:
+                \?(?P<error>[^}]*)
+                | (?P<default>[^}]*)
+            )
+        )?
+      \}
+    """,
+    re.VERBOSE,
+)
+
+
+def _interp(value: str) -> str:
+    """Interpolate env-var references inside a single string."""
+
+    def repl(match: re.Match[str]) -> str:
+        if match.group(0) == "$$":
+            return "$"
+        name = match.group("name")
+        error = match.group("error")
+        default = match.group("default")
+        env_value = os.environ.get(name)
+        if env_value is not None:
+            return env_value
+        if error is not None:
+            raise ModuleError(f"Required env var {name} not set: {error}")
+        if default is not None:
+            return default
+        raise ModuleError(f"Required env var {name} not set (no default provided).")
+
+    return _INTERP_RE.sub(repl, value)
+
+
+def _walk(value: Any) -> Any:
+    """Recursively interpolate strings inside a config tree."""
+    if isinstance(value, str):
+        return _interp(value)
+    if isinstance(value, dict):
+        return {k: _walk(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_walk(v) for v in value]
+    return value
+
+
+def _deep_merge(base: dict[str, Any], overlay: dict[str, Any]) -> dict[str, Any]:
+    """Recursive dict merge — overlay wins; lists replace wholesale.
+
+    Per spec §4.3 the overlay file's lists replace, not append. This
+    keeps the YAML behaviour predictable; users who want to extend
+    a list write the full list in the overlay.
+    """
+    out: dict[str, Any] = dict(base)
+    for key, value in overlay.items():
+        if key in out and isinstance(out[key], dict) and isinstance(value, dict):
+            out[key] = _deep_merge(out[key], value)
+        else:
+            out[key] = value
+    return out
+
+
+def parse_overrides(overrides: list[str]) -> dict[str, Any]:
+    """Parse `["agent.budget.usd=10", ...]` into a nested dict.
+
+    Each entry is `<dotted.path>=<value>`. Values are YAML-parsed via
+    `yaml.safe_load` so numbers, booleans, and inline lists / dicts
+    work without surprise (`agent.tools=[a, b]` -> `["a", "b"]`).
+
+    Raises:
+        ModuleError: malformed override (missing `=`, empty path, etc.)
+    """
+    out: dict[str, Any] = {}
+    for entry in overrides:
+        if "=" not in entry:
+            raise ModuleError(f"Invalid override {entry!r}: expected '<path>=<value>'.")
+        path, _, raw_value = entry.partition("=")
+        path = path.strip()
+        if not path:
+            raise ModuleError(f"Invalid override {entry!r}: empty path before '='.")
+        parts = path.split(".")
+        if any(not p for p in parts):
+            raise ModuleError(f"Invalid override {entry!r}: empty path segment.")
+        try:
+            value = yaml.safe_load(raw_value)
+        except yaml.YAMLError as exc:
+            raise ModuleError(
+                f"Invalid override {entry!r}: value not parseable as YAML ({exc})."
+            ) from exc
+        # Walk down `out`, creating dicts as needed; assign at the leaf.
+        cursor = out
+        for part in parts[:-1]:
+            existing = cursor.get(part)
+            if not isinstance(existing, dict):
+                cursor[part] = {}
+            cursor = cursor[part]
+        cursor[parts[-1]] = value
+    return out
+
+
+def _read_yaml(path: Path) -> dict[str, Any]:
+    """Read a YAML file; require a mapping at the top level."""
+    with path.open() as fh:
+        raw = yaml.safe_load(fh) or {}
+    if not isinstance(raw, dict):
+        raise ModuleError(
+            f"agentforge.yaml at {path} must be a mapping at the top level; "
+            f"got {type(raw).__name__}."
+        )
+    return raw
+
+
+def _env_overlay_path(base: Path, env: str) -> Path:
+    """Compute the overlay path next to `base`: foo.yaml → foo.<env>.yaml."""
+    return base.with_suffix(f".{env}{base.suffix}")
+
+
+def load_config(
+    path: Path | str | None = None,
+    *,
+    env: str | None = None,
+    overrides: list[str] | None = None,
+) -> AgentForgeConfig:
+    """Load + validate `agentforge.yaml` with full feat-012 resolution.
+
+    Args:
+        path: Explicit path to the YAML file. If `None`, falls back to
+            `AGENTFORGE_CONFIG` env var, then `./agentforge.yaml`. If
+            no file exists at any of these, returns the default config.
+        env: Environment name. Selects the overlay file
+            `agentforge.<env>.yaml` next to the base. If `None`, falls
+            back to `AGENTFORGE_ENV`.
+        overrides: List of `"<dotted.path>=<value>"` strings to apply
+            after env-var interpolation and before schema validation.
+
+    Returns:
+        Validated `AgentForgeConfig` with `AGENTFORGE_LOG_LEVEL`
+        applied to `cfg.logging.level` post-validation if set.
+
+    Raises:
+        ModuleError: env-var interpolation, layered-file, or override
+            problem.
+        pydantic.ValidationError: schema validation failed.
+    """
+    resolved_path = _resolve_path(path)
+    if resolved_path is None or not resolved_path.exists():
+        merged: dict[str, Any] = {}
+    else:
+        merged = _read_yaml(resolved_path)
+        # Layered env file overlays the base. Missing overlay is fine
+        # (env-without-file is just "use base").
+        resolved_env = env if env is not None else os.environ.get("AGENTFORGE_ENV")
+        if resolved_env:
+            overlay_path = _env_overlay_path(resolved_path, resolved_env)
+            if overlay_path.exists():
+                merged = _deep_merge(merged, _read_yaml(overlay_path))
+
+    interpolated = _walk(merged)
+    if overrides:
+        interpolated = _deep_merge(interpolated, parse_overrides(overrides))
+
+    config = AgentForgeConfig.model_validate(interpolated)
+    return _apply_env_log_level(config)
+
+
+def _resolve_path(path: Path | str | None) -> Path | None:
+    """Resolve the config-file path with `AGENTFORGE_CONFIG` fallback.
+
+    Order of precedence:
+    1. Explicit `path` argument.
+    2. `AGENTFORGE_CONFIG` env var.
+    3. `./agentforge.yaml` (default).
+    """
+    if path is not None:
+        return Path(path)
+    env_path = os.environ.get("AGENTFORGE_CONFIG")
+    if env_path:
+        return Path(env_path)
+    candidate = Path.cwd() / "agentforge.yaml"
+    return candidate if candidate.exists() else None
+
+
+def _apply_env_log_level(config: AgentForgeConfig) -> AgentForgeConfig:
+    """Apply `AGENTFORGE_LOG_LEVEL` over the validated config.
+
+    This is a post-validation override so users can flip log level
+    without touching the file (debugging, CI). Implemented via
+    `model_copy` to keep the model frozen-friendly.
+    """
+    level = os.environ.get("AGENTFORGE_LOG_LEVEL")
+    if not level:
+        return config
+    new_logging = config.logging.model_copy(update={"level": level})
+    return config.model_copy(update={"logging": new_logging})