lorebase-platform 0.2.3__tar.gz

lorebase_platform-0.2.3/.gitignore

@@ -0,0 +1,79 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+dist/
+*.egg-info/
+.eggs/
+.venv/
+venv/
+.python-eggs/
+pip-wheel-metadata/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.pyre/
+.pytype/
+.hypothesis/
+.coverage
+.coverage.*
+coverage.xml
+htmlcov/
+.tox/
+.nox/
+*.prof
+
+# Node
+node_modules/
+.pnpm-store/
+.npm/
+.yarn/
+!.yarn/releases
+.pnp.*
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+.vite/
+*.tsbuildinfo
+apps/web/dist/
+apps/web/.vitest/
+
+# Environment and secrets
+.env
+.env.*
+!.env.example
+*.pem
+*.key
+secrets/
+
+# Docker
+docker-compose.override.yaml
+docker-compose.override.yml
+.docker/
+
+# Data and local artifacts
+*.sqlite
+*.db
+/tmp/
+/data/
+tests/perf/results/
+
+# Editors and OS
+.idea/
+.vscode/*
+!.vscode/extensions.json
+!.vscode/settings.example.json
+*.swp
+*.swo
+*~
+.DS_Store
+Thumbs.db
+*.code-workspace
+.history/
+
+# Session worktrees
+.claude/

lorebase_platform-0.2.3/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.5
+Name: lorebase-platform
+Version: 0.2.3
+Summary: Lorebase platform: settings, secrets, logging and telemetry
+Requires-Python: >=3.12
+Requires-Dist: opentelemetry-api>=1.25
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc>=1.25
+Requires-Dist: opentelemetry-sdk>=1.25
+Requires-Dist: pydantic-settings>=2.3
+Requires-Dist: pydantic>=2.7
+Requires-Dist: pyyaml>=6.0

lorebase_platform-0.2.3/pyproject.toml

@@ -0,0 +1,13 @@
+[project]
+name = "lorebase-platform"
+version = "0.2.3"
+description = "Lorebase platform: settings, secrets, logging and telemetry"
+requires-python = ">=3.12"
+dependencies = ["pydantic>=2.7", "pydantic-settings>=2.3", "pyyaml>=6.0", "opentelemetry-api>=1.25", "opentelemetry-sdk>=1.25", "opentelemetry-exporter-otlp-proto-grpc>=1.25"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/lorebase_platform"]

lorebase_platform-0.2.3/src/lorebase_platform/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.3"

lorebase_platform-0.2.3/src/lorebase_platform/logging.py

@@ -0,0 +1,160 @@
+"""Structured logging: JSON records with trace correlation and secret redaction.
+
+`configure_logging` installs one handler on the root logger. Every record passes through
+`RedactionFilter`, which masks registered secret values found in messages, exception text and
+extra fields, and masks the value of any field whose name looks sensitive.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import logging
+import sys
+from collections.abc import Iterator, Mapping
+from contextvars import ContextVar
+from datetime import UTC, datetime
+from typing import IO, Any
+
+from opentelemetry import trace
+from pydantic import SecretStr
+
+from lorebase_platform.secrets import registered_secret_values
+from lorebase_platform.settings import LoggingSettings
+
+REDACTED = "***"
+HANDLER_MARK = "_lorebase_handler"
+
+request_id_var: ContextVar[str | None] = ContextVar("lorebase_request_id", default=None)
+
+_STANDARD_ATTRIBUTES = frozenset(
+    logging.LogRecord("", 0, "", 0, "", (), None).__dict__.keys()
+    | {"message", "asctime", "taskName"}
+)
+
+
+@contextlib.contextmanager
+def request_context(request_id: str) -> Iterator[None]:
+    """Bind a request id to every log record emitted inside the block."""
+    token = request_id_var.set(request_id)
+    try:
+        yield
+    finally:
+        request_id_var.reset(token)
+
+
+def is_sensitive_key(key: str, sensitive: tuple[str, ...]) -> bool:
+    """True when `key` equals a sensitive name or contains it as underscore-separated parts."""
+    normalized = f"_{key.lower().replace('-', '_')}_"
+    return any(f"_{name.lower()}_" in normalized for name in sensitive)
+
+
+class RedactionFilter(logging.Filter):
+    """Mask secrets in log records before any formatter sees them."""
+
+    def __init__(self, sensitive_keys: tuple[str, ...], min_length: int) -> None:
+        super().__init__()
+        self._keys = sensitive_keys
+        self._min_length = min_length
+
+    def scrub_text(self, text: str) -> str:
+        """Replace every registered secret value found in `text`."""
+        for secret in sorted(registered_secret_values(), key=len, reverse=True):
+            if len(secret) >= self._min_length and secret in text:
+                text = text.replace(secret, REDACTED)
+        return text
+
+    def scrub(self, value: Any, key: str | None = None) -> Any:
+        """Redact a value; a sensitive key masks the whole value."""
+        if key is not None and is_sensitive_key(key, self._keys):
+            return REDACTED
+        if isinstance(value, SecretStr):
+            return REDACTED
+        if isinstance(value, str):
+            return self.scrub_text(value)
+        if isinstance(value, Mapping):
+            return {str(k): self.scrub(v, str(k)) for k, v in value.items()}
+        if isinstance(value, list | tuple | set | frozenset):
+            return [self.scrub(item) for item in value]
+        return value
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        record.msg = self.scrub_text(record.getMessage())
+        record.args = None
+        if record.exc_info and not record.exc_text:
+            record.exc_text = logging.Formatter().formatException(record.exc_info)
+        if record.exc_text:
+            record.exc_text = self.scrub_text(record.exc_text)
+        if record.stack_info:
+            record.stack_info = self.scrub_text(record.stack_info)
+        for name in set(record.__dict__) - _STANDARD_ATTRIBUTES:
+            record.__dict__[name] = self.scrub(record.__dict__[name], name)
+        return True
+
+
+def _trace_ids() -> dict[str, str]:
+    context = trace.get_current_span().get_span_context()
+    if not context.is_valid:
+        return {}
+    return {"trace_id": f"{context.trace_id:032x}", "span_id": f"{context.span_id:016x}"}
+
+
+class JsonFormatter(logging.Formatter):
+    """One JSON object per line with trace and request correlation."""
+
+    def __init__(self, service: str) -> None:
+        super().__init__()
+        self._service = service
+
+    def format(self, record: logging.LogRecord) -> str:
+        payload: dict[str, Any] = {
+            "timestamp": datetime.fromtimestamp(record.created, UTC).isoformat(),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+            "service": self._service,
+            **_trace_ids(),
+        }
+        request_id = request_id_var.get()
+        if request_id:
+            payload["request_id"] = request_id
+        for name, value in record.__dict__.items():
+            if name not in _STANDARD_ATTRIBUTES:
+                payload[name] = value
+        if record.exc_info and not record.exc_text:
+            record.exc_text = self.formatException(record.exc_info)
+        if record.exc_text:
+            payload["exception"] = record.exc_text
+        return json.dumps(payload, default=str, ensure_ascii=False)
+
+
+class TextFormatter(logging.Formatter):
+    """Human-readable format that keeps the correlation identifiers."""
+
+    def __init__(self, service: str) -> None:
+        super().__init__("%(asctime)s %(levelname)s %(name)s %(message)s")
+        self._service = service
+
+    def format(self, record: logging.LogRecord) -> str:
+        line = super().format(record)
+        ids = {**_trace_ids(), **({"request_id": r} if (r := request_id_var.get()) else {})}
+        suffix = " ".join(f"{k}={v}" for k, v in ids.items())
+        return f"{line} [{self._service}{' ' + suffix if suffix else ''}]"
+
+
+def configure_logging(
+    settings: LoggingSettings, *, service: str, stream: IO[str] | None = None
+) -> logging.Handler:
+    """Install the structured handler on the root logger (idempotent) and return it."""
+    root = logging.getLogger()
+    for existing in [h for h in root.handlers if getattr(h, HANDLER_MARK, False)]:
+        root.removeHandler(existing)
+    handler = logging.StreamHandler(stream if stream is not None else sys.stderr)
+    setattr(handler, HANDLER_MARK, True)
+    handler.addFilter(RedactionFilter(settings.redact_keys, settings.redact_min_length))
+    handler.setFormatter(
+        JsonFormatter(service) if settings.format == "json" else TextFormatter(service)
+    )
+    root.addHandler(handler)
+    root.setLevel(settings.level)
+    return handler

lorebase_platform-0.2.3/src/lorebase_platform/secrets.py

@@ -0,0 +1,103 @@
+"""Secret references and the registry of secret values used for log redaction.
+
+A secret setting holds either a literal value or a reference resolved at load time:
+
+- `secret://env/NAME`: value of the environment variable `NAME`;
+- `secret://file/PATH`: content of the file at `PATH` (trailing newline removed). Use a double
+  slash for an absolute path, for example `secret://file//run/secrets/db_password`.
+
+Resolved values are typed `SecretStr` and are registered so that the logging redaction filter
+can mask them wherever they appear.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import threading
+from collections.abc import Callable
+from pathlib import Path
+from typing import Annotated, Any
+
+from pydantic import BeforeValidator, SecretStr
+
+REFERENCE_PREFIX = "secret://"
+_REFERENCE = re.compile(r"^secret://(?P<provider>[a-z0-9_-]+)/(?P<target>.+)$")
+
+
+class SecretResolutionError(ValueError):
+    """A secret reference cannot be resolved. The message never contains a secret value."""
+
+
+def _from_env(name: str) -> str:
+    value = os.environ.get(name)
+    if value is None:
+        raise SecretResolutionError(f"environment variable {name!r} is not set")
+    return value
+
+
+def _from_file(path: str) -> str:
+    try:
+        content = Path(path).read_text(encoding="utf-8")
+    except OSError as exc:
+        raise SecretResolutionError(f"cannot read secret file {path!r}: {exc.strerror}") from None
+    return content.removesuffix("\n").removesuffix("\r")
+
+
+_PROVIDERS: dict[str, Callable[[str], str]] = {"env": _from_env, "file": _from_file}
+
+_known_values: set[str] = set()
+_lock = threading.Lock()
+
+
+def register_secret_value(value: str) -> None:
+    """Remember a secret value so log redaction can mask it."""
+    if value:
+        with _lock:
+            _known_values.add(value)
+
+
+def registered_secret_values() -> frozenset[str]:
+    """Snapshot of every secret value seen so far."""
+    with _lock:
+        return frozenset(_known_values)
+
+
+def resolve_secret(value: str) -> str:
+    """Return the secret behind a reference; a literal value is returned unchanged."""
+    if not value.startswith(REFERENCE_PREFIX):
+        register_secret_value(value)
+        return value
+    match = _REFERENCE.match(value)
+    if match is None:
+        raise SecretResolutionError(
+            f"malformed secret reference {value!r}; expected secret://env/NAME or "
+            "secret://file/PATH"
+        )
+    provider = _PROVIDERS.get(match["provider"])
+    if provider is None:
+        raise SecretResolutionError(
+            f"unsupported secret provider {match['provider']!r} in {value!r}; "
+            f"available: {', '.join(sorted(_PROVIDERS))}"
+        )
+    try:
+        secret = provider(match["target"])
+    except SecretResolutionError as exc:
+        raise SecretResolutionError(f"{value}: {exc}") from None
+    if not secret:
+        raise SecretResolutionError(f"{value}: resolved to an empty value")
+    register_secret_value(secret)
+    return secret
+
+
+def _validate(value: Any) -> Any:
+    if isinstance(value, SecretStr):
+        register_secret_value(value.get_secret_value())
+        return value
+    if isinstance(value, str):
+        return resolve_secret(value)
+    return value
+
+
+SecretValue = Annotated[SecretStr, BeforeValidator(_validate)]
+"""A `SecretStr` field that accepts a literal or a `secret://` reference."""

lorebase_platform-0.2.3/src/lorebase_platform/settings.py

@@ -0,0 +1,265 @@
+"""Layered settings: code defaults, then a YAML file, then environment variables.
+
+Sources, lowest to highest precedence:
+
+1. defaults declared on the section models;
+2. the YAML file named by `LOREBASE_CONFIG` (optional);
+3. environment variables `LOREBASE__SECTION__KEY` (nested keys use further `__`), then the
+   short aliases a section registers (for example `LLM_BASE_URL`).
+
+Sections owned by this package (`telemetry`, `logging`) are typed fields of `Settings`. Other
+packages declare their section as a model and register it with `register_section`; they read it
+with `Settings.section`. `load_settings` validates every registered section, so invalid
+configuration stops the process at startup with a message naming each key and the reason.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, TypeVar
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
+from pydantic.fields import FieldInfo
+from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict
+
+CONFIG_ENV_VAR = "LOREBASE_CONFIG"
+ENV_PREFIX = "LOREBASE__"
+ENV_DELIMITER = "__"
+
+SectionT = TypeVar("SectionT", bound=BaseModel)
+
+
+class ConfigError(Exception):
+    """Invalid configuration. The message names every offending key and the reason."""
+
+    def __init__(self, problems: list[str]) -> None:
+        self.problems = problems
+        super().__init__("invalid configuration:\n" + "\n".join(f"  - {p}" for p in problems))
+
+    @classmethod
+    def from_validation(cls, exc: ValidationError, prefix: str = "") -> ConfigError:
+        """Build the error from pydantic details, never echoing input values."""
+        return cls(_problems(exc, prefix))
+
+
+def _problems(exc: ValidationError, prefix: str = "") -> list[str]:
+    problems = []
+    for error in exc.errors(include_input=False, include_url=False, include_context=False):
+        key = ".".join(str(part) for part in (*([prefix] if prefix else []), *error["loc"]))
+        problems.append(f"{key or '<root>'}: {error['msg']}")
+    return problems
+
+
+class SectionModel(BaseModel):
+    """Base class for settings sections: immutable, unknown keys rejected."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+
+class TelemetrySettings(SectionModel):
+    """OpenTelemetry export. Disabled unless `otlp_endpoint` is set."""
+
+    otlp_endpoint: str | None = Field(
+        default=None,
+        description="OTLP gRPC collector endpoint, e.g. http://otel-collector:4317. "
+        "Unset or empty disables traces and metrics.",
+    )
+    service_name: str = Field(default="lorebase", description="Value of `service.name`.")
+    service_namespace: str = Field(default="lorebase", description="Value of `service.namespace`.")
+    sample_ratio: float = Field(
+        default=0.1, ge=0, le=1, description="Fraction of root traces that are sampled."
+    )
+    metric_export_interval_s: float = Field(
+        default=60, gt=0, description="Seconds between metric exports."
+    )
+
+    @field_validator("otlp_endpoint")
+    @classmethod
+    def _empty_means_unset(cls, value: str | None) -> str | None:
+        return value.strip() or None if value else None
+
+
+class LoggingSettings(SectionModel):
+    """Structured logging and redaction."""
+
+    level: str = Field(default="INFO", description="Root log level name.")
+    format: str = Field(default="json", pattern="^(json|text)$", description="json or text.")
+    redact_keys: tuple[str, ...] = Field(
+        default=("password", "secret", "token", "api_key", "authorization", "credential"),
+        description="Field names whose values are masked. A name matches a whole "
+        "underscore-separated key or a part of it (access_token matches token).",
+    )
+    redact_min_length: int = Field(
+        default=4,
+        ge=1,
+        description="Secret values shorter than this are not masked inside free text.",
+    )
+
+    @field_validator("level")
+    @classmethod
+    def _known_level(cls, value: str) -> str:
+        name = value.upper()
+        if name not in {"CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"}:
+            raise ValueError("must be one of CRITICAL, ERROR, WARNING, INFO, DEBUG")
+        return name
+
+
+@dataclass(frozen=True)
+class SectionSpec:
+    """A settings section contributed by a package."""
+
+    name: str
+    model: type[BaseModel]
+    env_aliases: Mapping[str, str] = field(default_factory=dict)
+    """Environment variable name -> dotted key inside the section, e.g. LLM_MODEL -> llm.model."""
+
+
+_SECTIONS: dict[str, SectionSpec] = {}
+
+
+def register_section(
+    name: str, model: type[BaseModel], env_aliases: Mapping[str, str] | None = None
+) -> None:
+    """Declare a settings section; repeating the same declaration is harmless."""
+    if name in Settings.model_fields:
+        raise ValueError(f"section {name!r} is owned by lorebase_platform")
+    spec = SectionSpec(name, model, dict(env_aliases or {}))
+    existing = _SECTIONS.get(name)
+    if existing is not None and existing != spec:
+        raise ValueError(f"section {name!r} is already registered with a different model")
+    _SECTIONS[name] = spec
+
+
+def registered_sections() -> Mapping[str, SectionSpec]:
+    """Sections registered so far."""
+    return dict(_SECTIONS)
+
+
+def _parse_value(raw: str) -> Any:
+    stripped = raw.strip()
+    if stripped[:1] in ("[", "{"):
+        try:
+            return json.loads(stripped)
+        except json.JSONDecodeError:
+            return raw
+    return raw
+
+
+def _assign(tree: dict[str, Any], path: list[str], value: Any, origin: str) -> None:
+    node = tree
+    for part in path[:-1]:
+        child = node.setdefault(part, {})
+        if not isinstance(child, dict):
+            raise ConfigError([f"{origin}: conflicts with a value set for {part!r}"])
+        node = child
+    if isinstance(node.get(path[-1]), dict):
+        raise ConfigError([f"{origin}: conflicts with nested keys under {path[-1]!r}"])
+    node[path[-1]] = value
+
+
+def environment_overrides(environ: Mapping[str, str]) -> dict[str, Any]:
+    """Nested mapping from `LOREBASE__A__B` variables and registered section aliases."""
+    tree: dict[str, Any] = {}
+    for name in sorted(environ):
+        if name.startswith(ENV_PREFIX):
+            path = [p.lower() for p in name[len(ENV_PREFIX) :].split(ENV_DELIMITER)]
+            if all(path):
+                _assign(tree, path, _parse_value(environ[name]), name)
+    for spec in _SECTIONS.values():
+        for alias, dotted in sorted(spec.env_aliases.items()):
+            value = environ.get(alias)
+            path = [spec.name, *dotted.split(".")]
+            if value and _lookup(tree, path) is None:
+                _assign(tree, path, value, alias)
+    return tree
+
+
+def _lookup(tree: Mapping[str, Any], path: list[str]) -> Any:
+    node: Any = tree
+    for part in path:
+        if not isinstance(node, Mapping) or part not in node:
+            return None
+        node = node[part]
+    return node
+
+
+class _EnvSource(PydanticBaseSettingsSource):
+    def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
+        return None, field_name, False
+
+    def __call__(self) -> dict[str, Any]:
+        return environment_overrides(os.environ)
+
+
+class _YamlSource(PydanticBaseSettingsSource):
+    def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
+        return None, field_name, False
+
+    def __call__(self) -> dict[str, Any]:
+        location = os.environ.get(CONFIG_ENV_VAR)
+        if not location:
+            return {}
+        path = Path(location)
+        try:
+            content = yaml.safe_load(path.read_text(encoding="utf-8"))
+        except OSError as exc:
+            raise ConfigError(
+                [f"{CONFIG_ENV_VAR}: cannot read {location!r}: {exc.strerror}"]
+            ) from None
+        except yaml.YAMLError as exc:
+            raise ConfigError([f"{location}: invalid YAML: {exc}"]) from None
+        if content is None:
+            return {}
+        if not isinstance(content, dict):
+            raise ConfigError([f"{location}: top level must be a mapping"])
+        return {str(key): value for key, value in content.items()}
+
+
+class Settings(BaseSettings):
+    """Process-wide settings. Build with `load_settings`."""
+
+    model_config = SettingsConfigDict(extra="allow", frozen=True)
+
+    telemetry: TelemetrySettings = Field(default_factory=TelemetrySettings)
+    logging: LoggingSettings = Field(default_factory=LoggingSettings)
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> tuple[PydanticBaseSettingsSource, ...]:
+        return (init_settings, _EnvSource(settings_cls), _YamlSource(settings_cls))
+
+    def section(self, name: str, model: type[SectionT]) -> SectionT:
+        """Validated settings section declared by another package."""
+        raw = (self.model_extra or {}).get(name) or {}
+        try:
+            return model.model_validate(raw)
+        except ValidationError as exc:
+            raise ConfigError.from_validation(exc, prefix=name) from None
+
+
+def load_settings() -> Settings:
+    """Load and fully validate settings; raise `ConfigError` listing every problem."""
+    try:
+        settings = Settings()
+    except ValidationError as exc:
+        raise ConfigError.from_validation(exc) from None
+    problems: list[str] = []
+    for spec in _SECTIONS.values():
+        try:
+            settings.section(spec.name, spec.model)
+        except ConfigError as exc:
+            problems.extend(exc.problems)
+    if problems:
+        raise ConfigError(problems)
+    return settings

lorebase_platform-0.2.3/src/lorebase_platform/telemetry.py

@@ -0,0 +1,92 @@
+"""OpenTelemetry setup for traces and metrics.
+
+Telemetry is off unless `telemetry.otlp_endpoint` is configured: no provider, no exporter, no
+background thread and no network access are created in that case.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from opentelemetry import metrics, trace
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import MetricReader, PeriodicExportingMetricReader
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter
+from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
+
+from lorebase_platform.settings import TelemetrySettings
+
+LATENCY_BUCKETS_S: tuple[float, ...] = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
+"""Bucket boundaries, in seconds, of the duration histograms (request, job and commit
+latency). Without them the SDK defaults (0, 5, 10, 25, ...) put every sub-5-second value in
+one bucket and quantiles cannot be read."""
+
+
+@dataclass
+class Telemetry:
+    """Handle on the configured providers; `enabled` is False when nothing was set up."""
+
+    enabled: bool
+    tracer_provider: TracerProvider | None = None
+    meter_provider: MeterProvider | None = None
+
+    def shutdown(self) -> None:
+        """Flush pending data and stop exporters."""
+        if self.tracer_provider is not None:
+            self.tracer_provider.shutdown()
+        if self.meter_provider is not None:
+            self.meter_provider.shutdown()
+
+
+def setup_telemetry(
+    settings: TelemetrySettings,
+    *,
+    install_global: bool = True,
+    span_exporter: SpanExporter | None = None,
+    metric_reader: MetricReader | None = None,
+) -> Telemetry:
+    """Create providers for the configured endpoint, or return a disabled handle.
+
+    `span_exporter` and `metric_reader` replace the OTLP exporters (used by tests and by
+    deployments that export through another channel). `install_global` registers the
+    providers with the OpenTelemetry API.
+    """
+    if settings.otlp_endpoint is None:
+        return Telemetry(enabled=False)
+
+    resource = Resource.create(
+        {
+            "service.name": settings.service_name,
+            "service.namespace": settings.service_namespace,
+        }
+    )
+    tracer_provider = TracerProvider(
+        resource=resource, sampler=ParentBased(TraceIdRatioBased(settings.sample_ratio))
+    )
+    tracer_provider.add_span_processor(_span_processor(settings, span_exporter))
+    meter_provider = MeterProvider(
+        resource=resource, metric_readers=[metric_reader or _metric_reader(settings)]
+    )
+    if install_global:
+        trace.set_tracer_provider(tracer_provider)
+        metrics.set_meter_provider(meter_provider)
+    return Telemetry(enabled=True, tracer_provider=tracer_provider, meter_provider=meter_provider)
+
+
+def _span_processor(settings: TelemetrySettings, exporter: SpanExporter | None) -> SpanProcessor:
+    if exporter is None:
+        from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+
+        exporter = OTLPSpanExporter(endpoint=settings.otlp_endpoint)
+    return BatchSpanProcessor(exporter)
+
+
+def _metric_reader(settings: TelemetrySettings) -> MetricReader:
+    from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+
+    return PeriodicExportingMetricReader(
+        OTLPMetricExporter(endpoint=settings.otlp_endpoint),
+        export_interval_millis=settings.metric_export_interval_s * 1000,
+    )

lorebase_platform-0.2.3/tests/conftest.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from lorebase_platform import secrets, settings
+
+
+@pytest.fixture(autouse=True)
+def clean_configuration(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Isolate each test from ambient configuration and from other tests' registrations."""
+    for name in list(os.environ):
+        if name.startswith(settings.ENV_PREFIX) or name == settings.CONFIG_ENV_VAR:
+            monkeypatch.delenv(name)
+    monkeypatch.setattr(settings, "_SECTIONS", dict(settings._SECTIONS))
+    monkeypatch.setattr(secrets, "_known_values", set())

lorebase_platform-0.2.3/tests/test_logging.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+import io
+import json
+import logging
+from collections.abc import Iterator
+from typing import Any
+
+import pytest
+from opentelemetry.sdk.trace import TracerProvider
+from pydantic import SecretStr
+
+from lorebase_platform.logging import (
+    REDACTED,
+    configure_logging,
+    is_sensitive_key,
+    request_context,
+)
+from lorebase_platform.secrets import register_secret_value
+from lorebase_platform.settings import LoggingSettings
+
+SECRET = "hunter2-very-secret"
+
+
+@pytest.fixture
+def stream() -> Iterator[io.StringIO]:
+    root = logging.getLogger()
+    saved_handlers, saved_level = list(root.handlers), root.level
+    buffer = io.StringIO()
+    configure_logging(LoggingSettings(level="DEBUG"), service="test-service", stream=buffer)
+    yield buffer
+    root.handlers[:] = saved_handlers
+    root.setLevel(saved_level)
+
+
+def records(stream: io.StringIO) -> list[dict[str, Any]]:
+    return [json.loads(line) for line in stream.getvalue().splitlines()]
+
+
+def test_emits_one_json_object_per_record(stream: io.StringIO) -> None:
+    logging.getLogger("demo").info("hello %s", "world", extra={"order": 3})
+    (record,) = records(stream)
+    assert record["message"] == "hello world"
+    assert record["level"] == "INFO"
+    assert record["logger"] == "demo"
+    assert record["service"] == "test-service"
+    assert record["order"] == 3
+    assert "trace_id" not in record
+
+
+def test_registered_secret_is_masked_in_message_args_and_extra(stream: io.StringIO) -> None:
+    register_secret_value(SECRET)
+    log = logging.getLogger("demo")
+    log.warning("token is %s", SECRET)
+    log.warning(f"inline {SECRET}")
+    log.warning("extra", extra={"detail": f"value={SECRET}", "nested": {"a": [SECRET]}})
+    output = stream.getvalue()
+    assert SECRET not in output
+    assert output.count(REDACTED) >= 4
+
+
+def test_secret_in_exception_text_is_masked(stream: io.StringIO) -> None:
+    register_secret_value(SECRET)
+    try:
+        raise RuntimeError(f"connection failed for {SECRET}")
+    except RuntimeError:
+        logging.getLogger("demo").exception("boom")
+    (record,) = records(stream)
+    assert SECRET not in json.dumps(record)
+    assert "connection failed for ***" in record["exception"]
+
+
+def test_sensitive_field_names_are_masked_even_for_unknown_values(stream: io.StringIO) -> None:
+    logging.getLogger("demo").info(
+        "login",
+        extra={"password": "p4ss", "access_token": "abc", "input_tokens": 12, "user": "ann"},
+    )
+    (record,) = records(stream)
+    assert record["password"] == REDACTED
+    assert record["access_token"] == REDACTED
+    assert record["input_tokens"] == 12
+    assert record["user"] == "ann"
+
+
+def test_secret_str_objects_are_masked(stream: io.StringIO) -> None:
+    logging.getLogger("demo").info("cfg", extra={"key": SecretStr(SECRET)})
+    assert SECRET not in stream.getvalue()
+
+
+def test_short_secret_values_are_not_masked_in_free_text(stream: io.StringIO) -> None:
+    register_secret_value("ab")
+    logging.getLogger("demo").info("a cabinet")
+    assert records(stream)[0]["message"] == "a cabinet"
+
+
+def test_trace_and_request_ids_are_attached(stream: io.StringIO) -> None:
+    tracer = TracerProvider().get_tracer("test")
+    with tracer.start_as_current_span("op") as span, request_context("req-1"):
+        logging.getLogger("demo").info("inside")
+        context = span.get_span_context()
+    (record,) = records(stream)
+    assert record["trace_id"] == f"{context.trace_id:032x}"
+    assert record["span_id"] == f"{context.span_id:016x}"
+    assert record["request_id"] == "req-1"
+    logging.getLogger("demo").info("outside")
+    assert "request_id" not in records(stream)[1]
+
+
+def test_configure_logging_is_idempotent(stream: io.StringIO) -> None:
+    second = io.StringIO()
+    configure_logging(LoggingSettings(), service="again", stream=second)
+    logging.getLogger("demo").warning("once")
+    assert stream.getvalue() == ""
+    assert len(second.getvalue().splitlines()) == 1
+
+
+def test_text_format_keeps_request_id() -> None:
+    root = logging.getLogger()
+    saved_handlers, saved_level = list(root.handlers), root.level
+    buffer = io.StringIO()
+    try:
+        configure_logging(LoggingSettings(format="text"), service="svc", stream=buffer)
+        with request_context("req-9"):
+            logging.getLogger("demo").warning("plain")
+    finally:
+        root.handlers[:] = saved_handlers
+        root.setLevel(saved_level)
+    assert "plain" in buffer.getvalue()
+    assert "request_id=req-9" in buffer.getvalue()
+
+
+@pytest.mark.parametrize(
+    ("key", "expected"),
+    [
+        ("password", True),
+        ("api_key", True),
+        ("X-Api-Key", True),
+        ("access_token", True),
+        ("input_tokens", False),
+        ("monkey", False),
+        ("secretary", False),
+    ],
+)
+def test_sensitive_key_matching(key: str, expected: bool) -> None:
+    keys = ("password", "secret", "token", "api_key")
+    assert is_sensitive_key(key, keys) is expected

lorebase_platform-0.2.3/tests/test_secrets.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+from pydantic import SecretStr
+
+from lorebase_platform.secrets import (
+    SecretResolutionError,
+    SecretValue,
+    registered_secret_values,
+    resolve_secret,
+)
+from lorebase_platform.settings import (
+    CONFIG_ENV_VAR,
+    ConfigError,
+    SectionModel,
+    load_settings,
+    register_section,
+)
+
+
+class Holder(SectionModel):
+    password: SecretValue
+    optional: SecretValue | None = None
+
+
+def test_env_reference_resolves(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("DEMO_PASSWORD", "s3cr3t-value")
+    assert resolve_secret("secret://env/DEMO_PASSWORD") == "s3cr3t-value"
+
+
+def test_file_reference_resolves_and_strips_newline(tmp_path: Path) -> None:
+    path = tmp_path / "pw"
+    path.write_text("file-secret\n")
+    assert resolve_secret(f"secret://file/{path}") == "file-secret"
+
+
+def test_literal_value_is_returned_unchanged() -> None:
+    assert resolve_secret("plain-literal") == "plain-literal"
+
+
+def test_missing_environment_variable_is_a_clear_error(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.delenv("DEMO_MISSING", raising=False)
+    with pytest.raises(SecretResolutionError, match=r"secret://env/DEMO_MISSING.*not set"):
+        resolve_secret("secret://env/DEMO_MISSING")
+
+
+def test_missing_file_is_a_clear_error() -> None:
+    with pytest.raises(SecretResolutionError, match="cannot read secret file"):
+        resolve_secret("secret://file//nonexistent/secret")
+
+
+def test_unsupported_provider_and_malformed_reference() -> None:
+    with pytest.raises(SecretResolutionError, match="unsupported secret provider 'vault'"):
+        resolve_secret("secret://vault/path")
+    with pytest.raises(SecretResolutionError, match="malformed"):
+        resolve_secret("secret://env")
+
+
+def test_empty_secret_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("DEMO_EMPTY", "")
+    with pytest.raises(SecretResolutionError, match="empty"):
+        resolve_secret("secret://env/DEMO_EMPTY")
+
+
+def test_secret_fields_hide_values_in_repr_and_str(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("DEMO_PASSWORD", "s3cr3t-value")
+    holder = Holder(password="secret://env/DEMO_PASSWORD")
+    assert isinstance(holder.password, SecretStr)
+    assert holder.password.get_secret_value() == "s3cr3t-value"
+    assert "s3cr3t-value" not in repr(holder)
+    assert "s3cr3t-value" not in str(holder)
+    assert "s3cr3t-value" not in holder.model_dump_json()
+
+
+def test_resolved_values_are_registered_for_redaction(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("DEMO_PASSWORD", "s3cr3t-value")
+    Holder(password="secret://env/DEMO_PASSWORD")
+    assert "s3cr3t-value" in registered_secret_values()
+
+
+def test_missing_reference_in_settings_names_key_and_reference(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    register_section("holder", Holder)
+    config = tmp_path / "c.yaml"
+    config.write_text("holder:\n  password: secret://env/DEMO_ABSENT\n")
+    monkeypatch.setenv(CONFIG_ENV_VAR, str(config))
+    monkeypatch.delenv("DEMO_ABSENT", raising=False)
+    with pytest.raises(ConfigError) as caught:
+        load_settings()
+    assert "holder.password" in str(caught.value)
+    assert "secret://env/DEMO_ABSENT" in str(caught.value)
+
+
+def test_errors_never_contain_secret_values(monkeypatch: pytest.MonkeyPatch) -> None:
+    register_section("holder", Holder)
+    monkeypatch.setenv("LOREBASE__HOLDER__PASSWORD", "super-secret-literal")
+    monkeypatch.setenv("LOREBASE__HOLDER__OPTIONAL", "[1, 2]")
+    with pytest.raises(ConfigError) as caught:
+        load_settings()
+    assert "super-secret-literal" not in str(caught.value)

lorebase_platform-0.2.3/tests/test_settings.py

@@ -0,0 +1,168 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+from pydantic import Field
+
+from lorebase_platform.secrets import SecretValue
+from lorebase_platform.settings import (
+    CONFIG_ENV_VAR,
+    ConfigError,
+    SectionModel,
+    Settings,
+    load_settings,
+    register_section,
+)
+
+EXAMPLE = Path(__file__).resolve().parents[3] / "config" / "lorebase.example.yaml"
+
+
+class DemoSection(SectionModel):
+    count: int = 1
+    name: str = "base"
+    token: SecretValue | None = None
+    tags: list[str] = Field(default_factory=list)
+
+
+def write_yaml(tmp_path: Path, text: str) -> Path:
+    path = tmp_path / "config.yaml"
+    path.write_text(text)
+    return path
+
+
+def test_defaults_apply_without_any_source() -> None:
+    settings = load_settings()
+    assert settings.telemetry.otlp_endpoint is None
+    assert settings.telemetry.sample_ratio == 0.1
+    assert settings.logging.level == "INFO"
+
+
+def test_yaml_overrides_defaults(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    path = write_yaml(tmp_path, "telemetry:\n  sample_ratio: 0.5\n  service_name: from-yaml\n")
+    monkeypatch.setenv(CONFIG_ENV_VAR, str(path))
+    settings = load_settings()
+    assert settings.telemetry.sample_ratio == 0.5
+    assert settings.telemetry.service_name == "from-yaml"
+    assert settings.telemetry.service_namespace == "lorebase"
+
+
+def test_env_overrides_yaml_overrides_defaults(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    path = write_yaml(tmp_path, "telemetry:\n  sample_ratio: 0.5\n  service_name: from-yaml\n")
+    monkeypatch.setenv(CONFIG_ENV_VAR, str(path))
+    monkeypatch.setenv("LOREBASE__TELEMETRY__SAMPLE_RATIO", "0.25")
+    settings = load_settings()
+    assert settings.telemetry.sample_ratio == 0.25  # env wins
+    assert settings.telemetry.service_name == "from-yaml"  # yaml wins over default
+    assert settings.telemetry.service_namespace == "lorebase"  # default
+
+
+def test_env_accepts_json_for_lists(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("LOREBASE__LOGGING__REDACT_KEYS", '["pin", "otp"]')
+    assert load_settings().logging.redact_keys == ("pin", "otp")
+
+
+def test_empty_endpoint_means_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("LOREBASE__TELEMETRY__OTLP_ENDPOINT", "  ")
+    assert load_settings().telemetry.otlp_endpoint is None
+
+
+def test_invalid_value_names_key_and_reason(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("LOREBASE__TELEMETRY__SAMPLE_RATIO", "3")
+    monkeypatch.setenv("LOREBASE__LOGGING__LEVEL", "LOUD")
+    with pytest.raises(ConfigError) as caught:
+        load_settings()
+    message = str(caught.value)
+    assert "telemetry.sample_ratio: Input should be less than or equal to 1" in message
+    assert "logging.level" in message
+
+
+def test_unknown_key_in_owned_section_is_rejected(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setenv(CONFIG_ENV_VAR, str(write_yaml(tmp_path, "telemetry:\n  endpoint: x\n")))
+    with pytest.raises(ConfigError, match=r"telemetry\.endpoint: Extra inputs"):
+        load_settings()
+
+
+def test_missing_config_file_fails_clearly(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv(CONFIG_ENV_VAR, "/nonexistent/lorebase.yaml")
+    with pytest.raises(ConfigError, match=CONFIG_ENV_VAR):
+        load_settings()
+
+
+def test_yaml_must_be_a_mapping(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv(CONFIG_ENV_VAR, str(write_yaml(tmp_path, "- a\n- b\n")))
+    with pytest.raises(ConfigError, match="top level must be a mapping"):
+        load_settings()
+
+
+def test_registered_section_is_layered_and_validated(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    register_section("demo", DemoSection)
+    path = write_yaml(tmp_path, "demo:\n  count: 5\n  name: yaml\n  tags: [a]\n")
+    monkeypatch.setenv(CONFIG_ENV_VAR, str(path))
+    monkeypatch.setenv("LOREBASE__DEMO__COUNT", "7")
+    section = load_settings().section("demo", DemoSection)
+    assert (section.count, section.name, section.tags) == (7, "yaml", ["a"])
+
+
+def test_registered_section_errors_use_the_section_prefix(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    register_section("demo", DemoSection)
+    monkeypatch.setenv("LOREBASE__DEMO__COUNT", "many")
+    with pytest.raises(ConfigError, match=r"demo\.count"):
+        load_settings()
+
+
+def test_unregistered_sections_are_tolerated(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setenv(CONFIG_ENV_VAR, str(write_yaml(tmp_path, "database:\n  pool_size: 3\n")))
+    assert load_settings().model_extra == {"database": {"pool_size": 3}}
+
+
+def test_env_aliases_override_yaml_but_not_explicit_env(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    register_section("demo", DemoSection, env_aliases={"DEMO_NAME": "name", "DEMO_TOKEN": "token"})
+    monkeypatch.setenv(CONFIG_ENV_VAR, str(write_yaml(tmp_path, "demo:\n  name: yaml\n")))
+    monkeypatch.setenv("DEMO_NAME", "alias")
+    monkeypatch.setenv("DEMO_TOKEN", "")
+    assert load_settings().section("demo", DemoSection).name == "alias"
+    assert load_settings().section("demo", DemoSection).token is None  # empty alias ignored
+    monkeypatch.setenv("LOREBASE__DEMO__NAME", "explicit")
+    assert load_settings().section("demo", DemoSection).name == "explicit"
+
+
+def test_conflicting_env_keys_fail(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("LOREBASE__DEMO", "x")
+    monkeypatch.setenv("LOREBASE__DEMO__COUNT", "1")
+    with pytest.raises(ConfigError, match="conflicts"):
+        load_settings()
+
+
+def test_registering_a_platform_section_name_is_refused() -> None:
+    with pytest.raises(ValueError, match="owned by lorebase_platform"):
+        register_section("telemetry", DemoSection)
+
+
+def test_example_file_loads_and_matches_code_defaults(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    defaults = load_settings()
+    monkeypatch.setenv(CONFIG_ENV_VAR, str(EXAMPLE))
+    from_example = load_settings()
+    assert from_example.telemetry == defaults.telemetry
+    assert from_example.logging == defaults.logging
+
+
+def test_settings_is_immutable() -> None:
+    settings = load_settings()
+    with pytest.raises(ValueError, match="frozen"):
+        settings.telemetry = settings.telemetry.model_copy()
+    assert isinstance(settings, Settings)

lorebase_platform-0.2.3/tests/test_telemetry.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import socket
+
+import pytest
+from opentelemetry import trace
+from opentelemetry.sdk.metrics.export import InMemoryMetricReader
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+
+from lorebase_platform.settings import TelemetrySettings
+from lorebase_platform.telemetry import setup_telemetry
+
+
+def test_no_endpoint_creates_no_exporter_and_no_network_call(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    def fail(*_args: object, **_kwargs: object) -> None:
+        raise AssertionError("telemetry setup must not touch the network")
+
+    monkeypatch.setattr(socket.socket, "connect", fail)
+    monkeypatch.setattr(socket, "getaddrinfo", fail)
+    previous = trace.get_tracer_provider()
+
+    handle = setup_telemetry(TelemetrySettings())
+    handle.shutdown()
+
+    assert handle.enabled is False
+    assert handle.tracer_provider is None
+    assert handle.meter_provider is None
+    assert trace.get_tracer_provider() is previous
+
+
+def test_empty_endpoint_is_treated_as_disabled() -> None:
+    assert setup_telemetry(TelemetrySettings(otlp_endpoint="")).enabled is False
+
+
+def test_endpoint_enables_providers_with_resource_and_sampling() -> None:
+    spans = InMemorySpanExporter()
+    reader = InMemoryMetricReader()
+    settings = TelemetrySettings(
+        otlp_endpoint="http://collector:4317",
+        service_name="svc",
+        service_namespace="ns",
+        sample_ratio=1.0,
+    )
+
+    handle = setup_telemetry(
+        settings, install_global=False, span_exporter=spans, metric_reader=reader
+    )
+    assert handle.enabled
+    assert handle.tracer_provider is not None
+    assert handle.meter_provider is not None
+
+    with handle.tracer_provider.get_tracer("t").start_as_current_span("op"):
+        pass
+    handle.meter_provider.get_meter("m").create_counter("hits").add(2)
+    handle.tracer_provider.force_flush()
+
+    (span,) = spans.get_finished_spans()
+    assert span.name == "op"
+    assert span.resource.attributes["service.name"] == "svc"
+    assert span.resource.attributes["service.namespace"] == "ns"
+    assert reader.get_metrics_data() is not None
+    handle.shutdown()
+
+
+def test_zero_sample_ratio_drops_root_spans() -> None:
+    spans = InMemorySpanExporter()
+    handle = setup_telemetry(
+        TelemetrySettings(otlp_endpoint="http://collector:4317", sample_ratio=0),
+        install_global=False,
+        span_exporter=spans,
+        metric_reader=InMemoryMetricReader(),
+    )
+    assert handle.tracer_provider is not None
+    with handle.tracer_provider.get_tracer("t").start_as_current_span("op"):
+        pass
+    handle.tracer_provider.force_flush()
+    assert spans.get_finished_spans() == ()
+    handle.shutdown()