tunarag-python 0.2.1__py3-none-any.whl

tunarag/errors.py

@@ -0,0 +1,179 @@
+"""Structured public errors with secret-safe diagnostic details."""
+
+from collections.abc import Mapping, Sequence
+from dataclasses import fields, is_dataclass
+from enum import Enum
+from types import MappingProxyType
+from typing import Any, cast
+
+from .domain import Secret
+
+_SENSITIVE_KEYS = {
+    "access_token",
+    "api_key",
+    "authorization",
+    "credential",
+    "credentials",
+    "password",
+    "refresh_token",
+    "secret",
+    "token",
+}
+_REDACTED = "***redacted***"
+
+
+class ErrorCode(str, Enum):
+    """Stable machine-readable error identifiers."""
+
+    CONFIGURATION = "configuration.invalid"
+    SEARCH_SPACE = "search_space.invalid"
+    DATASET = "dataset.invalid"
+    ADAPTER = "adapter.failed"
+    EVALUATION = "evaluation.failed"
+    STORE = "store.failed"
+    RESUME = "resume.incompatible"
+    BUDGET = "budget.exhausted"
+    OPTIONAL_DEPENDENCY = "dependency.optional_missing"
+    SYNC_IN_ASYNC = "runtime.sync_in_async"
+
+
+class TunaRAGError(Exception):
+    """Base class for public package errors."""
+
+    code = ErrorCode.CONFIGURATION
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        stage: str | None = None,
+        retryable: bool = False,
+        details: Mapping[str, Any] | None = None,
+    ) -> None:
+        if not message.strip():
+            raise ValueError("error message must not be empty")
+        if stage is not None and not stage.strip():
+            raise ValueError("error stage must not be empty")
+        self.message = message
+        self.stage = stage
+        self.retryable = retryable
+        safe_details = cast(dict[str, Any], redact(details or {}))
+        self.details: Mapping[str, Any] = MappingProxyType(safe_details)
+        super().__init__(message)
+
+    def as_dict(self) -> dict[str, Any]:
+        """Return a JSON-compatible diagnostic representation."""
+
+        return {
+            "code": self.code.value,
+            "message": self.message,
+            "stage": self.stage,
+            "retryable": self.retryable,
+            "details": dict(self.details),
+        }
+
+
+class ConfigurationError(TunaRAGError):
+    """Configuration could not be parsed or validated."""
+
+    code = ErrorCode.CONFIGURATION
+
+
+class SearchSpaceError(TunaRAGError):
+    """A search-space definition or candidate is invalid."""
+
+    code = ErrorCode.SEARCH_SPACE
+
+
+class DatasetError(TunaRAGError):
+    """An evaluation dataset could not be loaded or validated."""
+
+    code = ErrorCode.DATASET
+
+    def __init__(self, message: str, *, details: Mapping[str, Any] | None = None) -> None:
+        super().__init__(message, stage="dataset", details=details)
+
+
+class AdapterError(TunaRAGError):
+    """The user-owned RAG adapter failed."""
+
+    code = ErrorCode.ADAPTER
+
+
+class EvaluationError(TunaRAGError):
+    """An evaluator failed or returned invalid observations."""
+
+    code = ErrorCode.EVALUATION
+
+
+class StoreError(TunaRAGError):
+    """Durable state could not be read or written."""
+
+    code = ErrorCode.STORE
+
+
+class ResumeError(TunaRAGError):
+    """A study cannot resume under the supplied semantics."""
+
+    code = ErrorCode.RESUME
+
+
+class BudgetError(TunaRAGError):
+    """A configured scheduling budget has been exhausted."""
+
+    code = ErrorCode.BUDGET
+
+
+class MissingOptionalDependencyError(TunaRAGError):
+    """A requested integration extra is not installed."""
+
+    code = ErrorCode.OPTIONAL_DEPENDENCY
+
+    def __init__(self, feature: str, extra: str) -> None:
+        if not feature.strip() or not extra.strip():
+            raise ValueError("feature and extra must not be empty")
+        super().__init__(
+            f"{feature} requires the optional '{extra}' extra",
+            stage="dependency",
+            details={"feature": feature, "extra": extra},
+        )
+
+
+class SyncInAsyncContextError(TunaRAGError):
+    """A synchronous entry point was called from an active event loop."""
+
+    code = ErrorCode.SYNC_IN_ASYNC
+
+    def __init__(self) -> None:
+        super().__init__(
+            "the synchronous API cannot run inside an active event loop; use the async API",
+            stage="runtime",
+        )
+
+
+def redact(value: Any, *, key: str | None = None) -> Any:
+    """Return a recursively redacted, JSON-compatible diagnostic value."""
+
+    if key is not None and _is_sensitive_key(key):
+        return _REDACTED
+    if isinstance(value, Secret):
+        return _REDACTED
+    if is_dataclass(value):
+        return {
+            field.name: redact(getattr(value, field.name), key=field.name)
+            for field in fields(cast(Any, value))
+        }
+    if isinstance(value, Mapping):
+        return {str(item_key): redact(item, key=str(item_key)) for item_key, item in value.items()}
+    if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+        return [redact(item) for item in value]
+    if value is None or isinstance(value, (bool, int, float, str)):
+        return value
+    return f"<{type(value).__name__}>"
+
+
+def _is_sensitive_key(key: str) -> bool:
+    normalized = key.lower().replace("-", "_")
+    return normalized in _SENSITIVE_KEYS or normalized.endswith(
+        ("_api_key", "_credential", "_password", "_secret", "_token")
+    )

tunarag/evaluators.py

@@ -0,0 +1,186 @@
+"""Built-in evaluator helpers and optional third-party integrations."""
+
+from __future__ import annotations
+
+import inspect
+import math
+from collections.abc import Awaitable, Callable, Mapping, Sequence
+from importlib import import_module
+from numbers import Real
+from typing import Any
+
+from .contracts import Fingerprintable
+from .domain import MetricValue, ValueStatus
+from .errors import EvaluationError, MissingOptionalDependencyError
+
+MetricScorer = Callable[[Any, Any], float | None | Awaitable[float | None]]
+RagasSampleMapper = Callable[[Any, Any], Mapping[str, Any]]
+
+
+class MetricEvaluator:
+    """Adapt one synchronous or asynchronous numeric scorer to an evaluator."""
+
+    def __init__(
+        self,
+        name: str,
+        scorer: MetricScorer,
+        *,
+        status: ValueStatus = ValueStatus.EXACT,
+    ) -> None:
+        if not name.strip():
+            raise ValueError("metric name must not be empty")
+        if name == "tunarag.objective":
+            raise ValueError("metric name is reserved: tunarag.objective")
+        if not callable(scorer):
+            raise TypeError("metric scorer must be callable")
+        self._name = name
+        self._scorer = scorer
+        self._status = status
+
+    async def evaluate(self, output: Any, example: Any) -> Sequence[MetricValue]:
+        """Score one adapter output and evaluation example."""
+
+        result = self._scorer(output, example)
+        value = await result if inspect.isawaitable(result) else result
+        normalized = _numeric_score(value, metric=self._name)
+        if normalized is None:
+            return (MetricValue(self._name, None, ValueStatus.UNAVAILABLE, 0.0),)
+        return (MetricValue(self._name, normalized, self._status),)
+
+    def fingerprint(self) -> Any:
+        """Return stable configuration used by study and cache identity."""
+
+        return {
+            "name": self._name,
+            "status": self._status.value,
+            "scorer": _callable_identity(self._scorer),
+        }
+
+
+class RagasEvaluator:
+    """Evaluate one sample with optional RAGAS metrics without hard coupling."""
+
+    def __init__(
+        self,
+        metrics: Sequence[Any],
+        sample_mapper: RagasSampleMapper,
+        *,
+        metric_prefix: str = "ragas.",
+        llm: Any = None,
+        embeddings: Any = None,
+        raise_exceptions: bool = False,
+    ) -> None:
+        if not metrics:
+            raise ValueError("RAGAS evaluator requires at least one metric")
+        if not callable(sample_mapper):
+            raise TypeError("RAGAS sample mapper must be callable")
+        if metric_prefix == "tunarag.objective" or metric_prefix.startswith("tunarag.objective."):
+            raise ValueError("RAGAS metric prefix uses a reserved name")
+        self._metrics = tuple(metrics)
+        self._sample_mapper = sample_mapper
+        self._metric_prefix = metric_prefix
+        self._llm = llm
+        self._embeddings = embeddings
+        self._raise_exceptions = raise_exceptions
+
+    async def evaluate(self, output: Any, example: Any) -> Sequence[MetricValue]:
+        """Map and evaluate exactly one sample through RAGAS's async API."""
+
+        try:
+            sample_data = self._sample_mapper(output, example)
+        except Exception as error:
+            raise EvaluationError(
+                "RAGAS sample mapping failed",
+                stage="evaluation",
+                details={"cause": type(error).__name__},
+            ) from error
+        if not isinstance(sample_data, Mapping):
+            raise EvaluationError("RAGAS sample mapper must return a mapping", stage="evaluation")
+        ragas, dataset_schema = _load_ragas()
+        try:
+            sample = dataset_schema.SingleTurnSample(**dict(sample_data))
+            dataset = ragas.EvaluationDataset(samples=[sample])
+            result = await ragas.aevaluate(
+                dataset=dataset,
+                metrics=list(self._metrics),
+                llm=self._llm,
+                embeddings=self._embeddings,
+                raise_exceptions=self._raise_exceptions,
+                show_progress=False,
+            )
+            scores = result.scores
+        except Exception as error:
+            raise EvaluationError(
+                "RAGAS evaluation failed",
+                stage="evaluation",
+                details={"cause": type(error).__name__},
+            ) from error
+        if not isinstance(scores, Sequence) or len(scores) != 1:
+            raise EvaluationError("RAGAS returned an invalid per-sample result", stage="evaluation")
+        row = scores[0]
+        if not isinstance(row, Mapping) or not row:
+            raise EvaluationError("RAGAS returned no metric scores", stage="evaluation")
+        observations: list[MetricValue] = []
+        names: set[str] = set()
+        for raw_name, raw_value in row.items():
+            name = f"{self._metric_prefix}{raw_name}"
+            if not name.strip() or name == "tunarag.objective":
+                raise EvaluationError("RAGAS returned an invalid metric name", stage="evaluation")
+            if name in names:
+                raise EvaluationError("RAGAS returned duplicate metric names", stage="evaluation")
+            names.add(name)
+            value = _numeric_score(raw_value, metric=name)
+            if value is None:
+                observations.append(MetricValue(name, None, ValueStatus.UNAVAILABLE, 0.0))
+            else:
+                observations.append(MetricValue(name, value))
+        return tuple(observations)
+
+    def fingerprint(self) -> Any:
+        """Return secret-safe semantic integration configuration."""
+
+        return {
+            "metrics": [_component_identity(metric) for metric in self._metrics],
+            "sample_mapper": _callable_identity(self._sample_mapper),
+            "metric_prefix": self._metric_prefix,
+            "llm": _component_identity(self._llm),
+            "embeddings": _component_identity(self._embeddings),
+            "raise_exceptions": self._raise_exceptions,
+        }
+
+
+def _load_ragas() -> tuple[Any, Any]:
+    try:
+        return import_module("ragas"), import_module("ragas.dataset_schema")
+    except ImportError as error:
+        raise MissingOptionalDependencyError("RAGAS evaluation", "ragas") from error
+
+
+def _numeric_score(value: Any, *, metric: str) -> float | None:
+    if value is None:
+        return None
+    if isinstance(value, bool) or not isinstance(value, Real):
+        raise EvaluationError(
+            "evaluator score must be numeric or None",
+            stage="evaluation",
+            details={"metric": metric, "value_type": type(value).__name__},
+        )
+    normalized = float(value)
+    return normalized if math.isfinite(normalized) else None
+
+
+def _callable_identity(function: Callable[..., Any]) -> str:
+    module = getattr(function, "__module__", type(function).__module__)
+    name = getattr(function, "__qualname__", type(function).__qualname__)
+    return f"{module}.{name}"
+
+
+def _component_identity(component: Any) -> Any:
+    if component is None:
+        return None
+    identity: dict[str, Any] = {
+        "type": f"{type(component).__module__}.{type(component).__qualname__}"
+    }
+    if isinstance(component, Fingerprintable):
+        identity["fingerprint"] = component.fingerprint()
+    return identity

tunarag/integrations/__init__.py

@@ -0,0 +1,25 @@
+"""Optional integration adapters."""
+
+from .mlflow import MLflowCallback
+from .runnables import (
+    AsyncRunnable,
+    LangChainAdapter,
+    LangGraphAdapter,
+    RunnableConfigMapper,
+    RunnableFactory,
+    RunnableInputMapper,
+    RunnableOutputMapper,
+    RunnableUsageMapper,
+)
+
+__all__ = [
+    "AsyncRunnable",
+    "LangChainAdapter",
+    "LangGraphAdapter",
+    "MLflowCallback",
+    "RunnableConfigMapper",
+    "RunnableFactory",
+    "RunnableInputMapper",
+    "RunnableOutputMapper",
+    "RunnableUsageMapper",
+]

tunarag/integrations/mlflow.py

@@ -0,0 +1,155 @@
+"""Optional post-commit MLflow tracking mirror."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+from importlib import import_module
+from typing import Any, Protocol
+
+from ..errors import MissingOptionalDependencyError, redact
+from ..store import SQLiteStore, StudyEvent, TrialStatus
+
+
+class MLflowClient(Protocol):
+    """Subset of the low-level MLflow client used by the mirror."""
+
+    def create_run(self, experiment_id: str, *, tags: Mapping[str, str]) -> Any: ...
+
+    def log_metric(
+        self, run_id: str, key: str, value: float, *, step: int | None = None
+    ) -> Any: ...
+
+    def log_param(self, run_id: str, key: str, value: Any) -> Any: ...
+
+    def set_tag(self, run_id: str, key: str, value: Any) -> None: ...
+
+    def set_terminated(self, run_id: str, *, status: str | None = None) -> None: ...
+
+
+@dataclass(frozen=True, slots=True)
+class _TrialMirrorData:
+    sequence: int
+    status: TrialStatus
+    candidate: Mapping[str, Any]
+    metrics: Mapping[str, float]
+
+
+class MLflowCallback:
+    """Mirror committed study events into one low-level MLflow run."""
+
+    def __init__(
+        self,
+        store: SQLiteStore,
+        *,
+        experiment_id: str = "0",
+        tracking_uri: str | None = None,
+        tags: Mapping[str, str] | None = None,
+        client: MLflowClient | None = None,
+    ) -> None:
+        if not experiment_id.strip():
+            raise ValueError("MLflow experiment id must not be empty")
+        self._store = store
+        self._experiment_id = experiment_id
+        self._tracking_uri = tracking_uri
+        safe_tags = redact(tags or {})
+        if not isinstance(safe_tags, Mapping):
+            raise TypeError("MLflow tags must be a mapping")
+        self._tags = {str(key): str(value) for key, value in safe_tags.items()}
+        self._client = client
+        self._run_ids: dict[str, str] = {}
+
+    async def on_event(self, event: StudyEvent) -> None:
+        """Mirror one durable event after gathering committed local state."""
+
+        trial_data = self._trial_data(event)
+        await asyncio.to_thread(self._mirror, event, trial_data)
+
+    def _trial_data(self, event: StudyEvent) -> _TrialMirrorData | None:
+        if event.type not in {"trial.succeeded", "trial.failed", "trial.cancelled"}:
+            return None
+        trial_id = event.payload.get("trial_id")
+        if not isinstance(trial_id, str):
+            return None
+        trial = self._store.get_trial(trial_id)
+        candidate = json.loads(trial.candidate_json)
+        if not isinstance(candidate, Mapping):
+            candidate = {}
+        safe_candidate = redact(candidate)
+        if not isinstance(safe_candidate, Mapping):
+            safe_candidate = {}
+        metrics = {
+            metric.name: float(metric.value)
+            for metric in self._store.list_metrics(trial_id)
+            if metric.value is not None
+        }
+        return _TrialMirrorData(trial.sequence, trial.status, safe_candidate, metrics)
+
+    def _mirror(self, event: StudyEvent, trial: _TrialMirrorData | None) -> None:
+        client = self._client_or_import()
+        run_id = self._run_ids.get(event.study_id)
+        if run_id is None:
+            tags = {
+                **self._tags,
+                "tunarag.study_id": event.study_id,
+                "tunarag.mirror": "event-callback",
+            }
+            run = client.create_run(self._experiment_id, tags=tags)
+            run_id = str(run.info.run_id)
+            self._run_ids[event.study_id] = run_id
+        client.set_tag(run_id, "tunarag.last_event", event.type)
+        client.set_tag(run_id, "tunarag.last_event_sequence", event.sequence)
+        if trial is not None:
+            self._mirror_trial(client, run_id, trial)
+        terminal_status = {
+            "study.completed": "FINISHED",
+            "study.failed": "FAILED",
+            "study.cancelled": "KILLED",
+        }.get(event.type)
+        if terminal_status is not None:
+            client.set_terminated(run_id, status=terminal_status)
+
+    def _mirror_trial(self, client: MLflowClient, run_id: str, trial: _TrialMirrorData) -> None:
+        prefix = f"trial.{trial.sequence}"
+        client.set_tag(run_id, f"{prefix}.status", trial.status.value)
+        for key, value in sorted(trial.candidate.items()):
+            client.log_param(
+                run_id, _tracking_key(f"{prefix}.candidate.{key}"), _parameter_value(value)
+            )
+        for name, value in sorted(trial.metrics.items()):
+            client.log_metric(run_id, _tracking_key(name), value, step=trial.sequence)
+        client.log_metric(
+            run_id,
+            "tunarag.trial_failed",
+            1.0 if trial.status is TrialStatus.FAILED else 0.0,
+            step=trial.sequence,
+        )
+
+    def _client_or_import(self) -> MLflowClient:
+        if self._client is not None:
+            return self._client
+        try:
+            module = import_module("mlflow")
+        except ImportError as error:
+            raise MissingOptionalDependencyError("MLflow tracking", "mlflow") from error
+        self._client = module.MlflowClient(tracking_uri=self._tracking_uri)
+        return self._client
+
+
+def _parameter_value(value: Any) -> str:
+    safe = redact(value)
+    if safe is None or isinstance(safe, (bool, int, float, str)):
+        return str(safe)
+    return json.dumps(safe, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+
+
+def _tracking_key(value: str) -> str:
+    normalized = re.sub(r"[^A-Za-z0-9_.\-/ ]", "_", value)
+    if normalized == value and len(normalized) <= 250:
+        return normalized
+    digest = hashlib.sha256(value.encode("utf-8")).hexdigest()[:12]
+    return f"{normalized[:237]}.{digest}"