tunarag-python 0.2.1__py3-none-any.whl

tunarag/retry.py

@@ -0,0 +1,70 @@
+"""Deterministic retry and timeout policy."""
+
+import asyncio
+import math
+from dataclasses import dataclass
+
+from .errors import TunaRAGError
+
+
+@dataclass(frozen=True, slots=True)
+class RetryPolicy:
+    """Bound retries, exponential backoff, and execution timeouts."""
+
+    max_retries: int = 2
+    sample_timeout_seconds: float | None = 60.0
+    trial_timeout_seconds: float | None = 300.0
+    base_delay_seconds: float = 0.5
+    max_delay_seconds: float = 30.0
+
+    def __post_init__(self) -> None:
+        if (
+            not isinstance(self.max_retries, int)
+            or isinstance(self.max_retries, bool)
+            or self.max_retries < 0
+        ):
+            raise ValueError("max_retries must be a nonnegative integer")
+        _validate_optional_timeout("sample timeout", self.sample_timeout_seconds)
+        _validate_optional_timeout("trial timeout", self.trial_timeout_seconds)
+        _validate_nonnegative_finite("base retry delay", self.base_delay_seconds)
+        _validate_nonnegative_finite("maximum retry delay", self.max_delay_seconds)
+        if self.base_delay_seconds > self.max_delay_seconds:
+            raise ValueError("base retry delay must not exceed maximum retry delay")
+
+    @property
+    def max_attempts(self) -> int:
+        """Return the initial attempt plus configured retries."""
+
+        return self.max_retries + 1
+
+    @property
+    def lease_seconds(self) -> float:
+        """Return a recovery lease longer than the configured trial timeout."""
+
+        return 86_400.0 if self.trial_timeout_seconds is None else self.trial_timeout_seconds + 60.0
+
+    def delay_before_retry(self, failed_attempt_number: int) -> float:
+        """Return deterministic capped exponential backoff."""
+
+        if failed_attempt_number < 0:
+            raise ValueError("failed attempt number must not be negative")
+        return float(
+            min(self.base_delay_seconds * (2**failed_attempt_number), self.max_delay_seconds)
+        )
+
+    def is_retryable(self, error: BaseException) -> bool:
+        """Classify known transient failures without parsing messages."""
+
+        if isinstance(error, TunaRAGError):
+            return error.retryable
+        return isinstance(error, (TimeoutError, asyncio.TimeoutError, ConnectionError))
+
+
+def _validate_optional_timeout(name: str, value: float | None) -> None:
+    if value is not None and (not math.isfinite(value) or value <= 0):
+        raise ValueError(f"{name} must be a positive finite number or None")
+
+
+def _validate_nonnegative_finite(name: str, value: float) -> None:
+    if not math.isfinite(value) or value < 0:
+        raise ValueError(f"{name} must be a nonnegative finite number")

tunarag/search.py

@@ -0,0 +1,286 @@
+"""Typed search spaces and the deterministic random strategy."""
+
+import math
+import random
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any, Protocol, cast
+
+from .contracts import Fingerprintable
+from .domain import Candidate, MetricValue, Secret, UsageRecord
+from .errors import SearchSpaceError, redact
+
+
+class Parameter(Protocol):
+    """Sample one typed parameter value from a random stream."""
+
+    def sample(self, generator: random.Random) -> Any: ...
+
+    def accepts(self, value: Any) -> bool: ...
+
+    def describe(self) -> Mapping[str, Any]: ...
+
+
+@dataclass(frozen=True, slots=True)
+class IntegerRange:
+    """Inclusive integer range."""
+
+    minimum: int
+    maximum: int
+
+    def __post_init__(self) -> None:
+        if any(
+            isinstance(value, bool) or not isinstance(value, int)
+            for value in (self.minimum, self.maximum)
+        ):
+            raise TypeError("integer range bounds must be integers")
+        if self.minimum > self.maximum:
+            raise ValueError("integer range minimum must not exceed maximum")
+
+    def sample(self, generator: random.Random) -> int:
+        return generator.randint(self.minimum, self.maximum)
+
+    def accepts(self, value: Any) -> bool:
+        return (
+            isinstance(value, int)
+            and not isinstance(value, bool)
+            and self.minimum <= value <= self.maximum
+        )
+
+    def describe(self) -> Mapping[str, Any]:
+        return {"type": "integer", "minimum": self.minimum, "maximum": self.maximum}
+
+
+@dataclass(frozen=True, slots=True)
+class FloatRange:
+    """Continuous float range."""
+
+    minimum: float
+    maximum: float
+
+    def __post_init__(self) -> None:
+        if any(
+            isinstance(value, bool)
+            or not isinstance(value, (int, float))
+            or not math.isfinite(value)
+            for value in (self.minimum, self.maximum)
+        ):
+            raise ValueError("float range bounds must be finite numbers")
+        if self.minimum > self.maximum:
+            raise ValueError("float range minimum must not exceed maximum")
+
+    def sample(self, generator: random.Random) -> float:
+        return generator.uniform(self.minimum, self.maximum)
+
+    def accepts(self, value: Any) -> bool:
+        return (
+            isinstance(value, (int, float))
+            and not isinstance(value, bool)
+            and self.minimum <= value <= self.maximum
+        )
+
+    def describe(self) -> Mapping[str, Any]:
+        return {"type": "number", "minimum": self.minimum, "maximum": self.maximum}
+
+
+@dataclass(frozen=True, slots=True)
+class Categorical:
+    """A non-empty collection of candidate values."""
+
+    values: tuple[Any, ...]
+
+    def __post_init__(self) -> None:
+        if not self.values:
+            raise ValueError("categorical values must not be empty")
+
+    def sample(self, generator: random.Random) -> Any:
+        return generator.choice(self.values)
+
+    def accepts(self, value: Any) -> bool:
+        return value in self.values
+
+    def describe(self) -> Mapping[str, Any]:
+        return {
+            "type": "categorical",
+            "values": tuple(value for value in self.values if not isinstance(value, Secret)),
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class SearchSpace:
+    """Named typed parameters from which candidates can be sampled."""
+
+    parameters: dict[str, Parameter]
+
+    def __post_init__(self) -> None:
+        if not self.parameters:
+            raise ValueError("search space must contain at least one parameter")
+        if any(not name.strip() for name in self.parameters):
+            raise ValueError("search-space parameter names must not be empty")
+        object.__setattr__(self, "parameters", dict(self.parameters))
+
+    def sample(self, generator: random.Random) -> Candidate:
+        return Candidate(
+            {name: parameter.sample(generator) for name, parameter in self.parameters.items()}
+        )
+
+    def validate(self, parameters: Mapping[str, Any]) -> Candidate:
+        """Validate an exact candidate mapping against every parameter."""
+
+        if set(parameters) != set(self.parameters):
+            raise ValueError("candidate keys must exactly match the search space")
+        for name, parameter in self.parameters.items():
+            if not parameter.accepts(parameters[name]):
+                raise ValueError(f"candidate value is outside the search space: {name}")
+        return Candidate(dict(parameters))
+
+    def describe(self) -> Mapping[str, Mapping[str, Any]]:
+        """Return a provider-safe structured search-space description."""
+
+        return {name: parameter.describe() for name, parameter in self.parameters.items()}
+
+
+class RandomSearch:
+    """Seeded random strategy implementing the common search contract."""
+
+    def __init__(self, space: SearchSpace, *, seed: int | None = None) -> None:
+        self._space = space
+        self._seed = seed
+        self._generator = random.Random(seed)
+
+    async def suggest(self) -> Candidate:
+        return self._space.sample(self._generator)
+
+    async def observe(self, candidate: Candidate, metrics: Any, usage: Any) -> None:
+        del candidate, metrics, usage
+
+    async def replay(self, candidate: Candidate, metrics: Any, usage: Any) -> None:
+        """Advance and verify deterministic generator state during resume."""
+
+        del metrics, usage
+        expected = self._space.sample(self._generator)
+        if expected != candidate:
+            raise ValueError("persisted candidate does not match seeded random search state")
+
+    def fingerprint(self) -> Any:
+        """Return stable search-space and seed semantics."""
+
+        return {"space": self._space, "seed": self._seed}
+
+
+@dataclass(frozen=True, slots=True)
+class SearchObservation:
+    """A prior reserved candidate and any committed metrics."""
+
+    candidate: Candidate
+    metrics: tuple[MetricValue, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class LLMSuggestionRequest:
+    """Provider-neutral structured input for one suggestion attempt."""
+
+    search_space: Mapping[str, Mapping[str, Any]]
+    observations: tuple[SearchObservation, ...]
+    suggestion_number: int
+    attempt_number: int
+
+
+class LLMSearchProvider(Protocol):
+    """Produce one structured candidate proposal without provider coupling."""
+
+    async def suggest(self, request: LLMSuggestionRequest) -> Mapping[str, Any]: ...
+
+
+class LLMSearch:
+    """Validate bounded provider suggestions with deterministic random fallback."""
+
+    def __init__(
+        self,
+        space: SearchSpace,
+        provider: LLMSearchProvider,
+        *,
+        max_suggestion_attempts: int = 3,
+        fallback_seed: int = 0,
+        fallback: bool = True,
+    ) -> None:
+        if (
+            not isinstance(max_suggestion_attempts, int)
+            or isinstance(max_suggestion_attempts, bool)
+            or max_suggestion_attempts < 1
+        ):
+            raise ValueError("maximum suggestion attempts must be positive")
+        self._space = space
+        self._provider = provider
+        self._max_suggestion_attempts = max_suggestion_attempts
+        self._fallback_seed = fallback_seed
+        self._fallback = fallback
+        self._suggestion_number = 0
+        self._observations: list[SearchObservation] = []
+
+    async def suggest(self) -> Candidate:
+        causes: list[str] = []
+        for attempt_number in range(self._max_suggestion_attempts):
+            request = LLMSuggestionRequest(
+                self._space.describe(),
+                tuple(_provider_observation(item) for item in self._observations),
+                self._suggestion_number,
+                attempt_number,
+            )
+            try:
+                proposal = await self._provider.suggest(request)
+                candidate = self._space.validate(proposal)
+            except Exception as error:
+                causes.append(type(error).__name__)
+                continue
+            self._suggestion_number += 1
+            return candidate
+        if self._fallback:
+            generator = random.Random(f"{self._fallback_seed}:{self._suggestion_number}")
+            candidate = self._space.sample(generator)
+            self._suggestion_number += 1
+            return candidate
+        raise SearchSpaceError(
+            "LLM search exhausted invalid suggestion attempts",
+            stage="strategy.suggest",
+            details={"causes": causes, "attempts": self._max_suggestion_attempts},
+        )
+
+    async def observe(
+        self,
+        candidate: Candidate,
+        metrics: Sequence[MetricValue],
+        usage: Sequence[UsageRecord],
+    ) -> None:
+        del usage
+        self._observations.append(SearchObservation(candidate, tuple(metrics)))
+
+    async def replay(
+        self,
+        candidate: Candidate,
+        metrics: Sequence[MetricValue],
+        usage: Sequence[UsageRecord],
+    ) -> None:
+        del usage
+        self._suggestion_number += 1
+        self._observations.append(SearchObservation(candidate, tuple(metrics)))
+
+    def fingerprint(self) -> Any:
+        provider_type = type(self._provider)
+        provider: dict[str, Any] = {
+            "type": f"{provider_type.__module__}.{provider_type.__qualname__}"
+        }
+        if isinstance(self._provider, Fingerprintable):
+            provider["fingerprint"] = self._provider.fingerprint()
+        return {
+            "space": self._space,
+            "provider": provider,
+            "max_suggestion_attempts": self._max_suggestion_attempts,
+            "fallback_seed": self._fallback_seed,
+            "fallback": self._fallback,
+        }
+
+
+def _provider_observation(observation: SearchObservation) -> SearchObservation:
+    parameters = cast(dict[str, Any], redact(observation.candidate.parameters))
+    return SearchObservation(Candidate(parameters), observation.metrics)

tunarag/serialization.py

@@ -0,0 +1,78 @@
+"""Canonical, secret-safe serialization for identity and cache keys."""
+
+import hashlib
+import json
+import math
+from collections.abc import Mapping, Sequence
+from dataclasses import asdict, is_dataclass
+from typing import Any, cast
+
+from .domain import Secret
+
+
+def canonicalize(value: Any) -> Any:
+    """Convert supported values to deterministic JSON-compatible structures."""
+
+    if isinstance(value, Secret):
+        return {"__secret__": True}
+    if is_dataclass(value):
+        return canonicalize(asdict(cast(Any, value)))
+    if isinstance(value, Mapping):
+        return {
+            str(key): canonicalize(item)
+            for key, item in sorted(value.items(), key=lambda entry: str(entry[0]))
+            if not isinstance(item, Secret)
+        }
+    if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+        return [canonicalize(item) for item in value]
+    if isinstance(value, float):
+        if not math.isfinite(value):
+            raise ValueError("nonfinite numbers are not supported")
+        return format(value, ".15g")
+    if value is None or isinstance(value, (bool, int, str)):
+        return value
+    raise TypeError(f"unsupported canonical value: {type(value).__name__}")
+
+
+def canonical_json(value: Any) -> bytes:
+    """Serialize a value using stable key ordering and schema-independent JSON."""
+
+    return json.dumps(
+        canonicalize(value), ensure_ascii=True, separators=(",", ":"), sort_keys=True
+    ).encode("utf-8")
+
+
+def storage_json(value: Any) -> bytes:
+    """Serialize supported values reversibly while omitting mapping secrets."""
+
+    return json.dumps(
+        _storage_value(value), ensure_ascii=True, separators=(",", ":"), sort_keys=True
+    ).encode("utf-8")
+
+
+def content_hash(value: Any, *, namespace: str = "autorag-tuner:v1") -> str:
+    """Return a stable SHA-256 digest for a canonical value."""
+
+    return hashlib.sha256(namespace.encode() + b"\0" + canonical_json(value)).hexdigest()
+
+
+def _storage_value(value: Any) -> Any:
+    if isinstance(value, Secret):
+        return {"__secret__": True}
+    if is_dataclass(value):
+        return _storage_value(asdict(cast(Any, value)))
+    if isinstance(value, Mapping):
+        return {
+            str(key): _storage_value(item)
+            for key, item in sorted(value.items(), key=lambda entry: str(entry[0]))
+            if not isinstance(item, Secret)
+        }
+    if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+        return [_storage_value(item) for item in value]
+    if isinstance(value, float):
+        if not math.isfinite(value):
+            raise ValueError("nonfinite numbers are not supported")
+        return value
+    if value is None or isinstance(value, (bool, int, str)):
+        return value
+    raise TypeError(f"unsupported storage value: {type(value).__name__}")

tunarag/stopping.py

@@ -0,0 +1,193 @@
+"""Composable stop conditions for study execution."""
+
+import math
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass(frozen=True, slots=True)
+class StopContext:
+    """Current study totals supplied to stop conditions."""
+
+    completed_trials: int
+    exact_cost: float = 0.0
+    estimated_cost: float = 0.0
+    elapsed_seconds: float = 0.0
+    successful_trials: int = 0
+    failed_trials: int = 0
+    objective_scores: tuple[float | None, ...] = ()
+
+    def __post_init__(self) -> None:
+        if (
+            isinstance(self.completed_trials, bool)
+            or not isinstance(self.completed_trials, int)
+            or self.completed_trials < 0
+        ):
+            raise ValueError("completed trials must not be negative")
+        if any(
+            isinstance(value, bool) or not isinstance(value, int) or value < 0
+            for value in (self.successful_trials, self.failed_trials)
+        ):
+            raise ValueError("trial status counts must not be negative")
+        if self.successful_trials + self.failed_trials > self.completed_trials:
+            raise ValueError("trial status counts exceed completed trials")
+        if self.objective_scores and len(self.objective_scores) != self.completed_trials:
+            raise ValueError("objective score history must match completed trials")
+        for name in ("exact_cost", "estimated_cost", "elapsed_seconds"):
+            value = getattr(self, name)
+            if not math.isfinite(value) or value < 0:
+                raise ValueError(f"{name} must be finite and nonnegative")
+        if any(score is not None and not math.isfinite(score) for score in self.objective_scores):
+            raise ValueError("objective scores must be finite or None")
+
+
+class StopCondition(Protocol):
+    """Return whether execution should stop for the current context."""
+
+    def should_stop(self, context: StopContext) -> bool: ...
+
+
+@dataclass(frozen=True, slots=True)
+class MaxTrials:
+    """Stop after a maximum number of completed trials."""
+
+    limit: int
+
+    def __post_init__(self) -> None:
+        if isinstance(self.limit, bool) or not isinstance(self.limit, int) or self.limit < 1:
+            raise ValueError("trial limit must be at least one")
+
+    def should_stop(self, context: StopContext) -> bool:
+        return context.completed_trials >= self.limit
+
+    def fingerprint(self) -> object:
+        return {"limit": self.limit}
+
+
+@dataclass(frozen=True, slots=True)
+class CostBudget:
+    """Stop when exact cost, or exact plus estimated cost, reaches a budget."""
+
+    limit: float
+    count_estimated: bool = True
+
+    def __post_init__(self) -> None:
+        if isinstance(self.limit, bool) or not math.isfinite(self.limit) or self.limit < 0:
+            raise ValueError("cost budget must be finite and nonnegative")
+        if not isinstance(self.count_estimated, bool):
+            raise TypeError("count_estimated must be a boolean")
+
+    def should_stop(self, context: StopContext) -> bool:
+        total = context.exact_cost + (context.estimated_cost if self.count_estimated else 0.0)
+        return total >= self.limit
+
+    def fingerprint(self) -> object:
+        return {"limit": self.limit, "count_estimated": self.count_estimated}
+
+
+@dataclass(frozen=True, slots=True)
+class TimeBudget:
+    """Stop when elapsed wall-clock execution reaches a limit."""
+
+    seconds: float
+
+    def __post_init__(self) -> None:
+        if isinstance(self.seconds, bool) or not math.isfinite(self.seconds) or self.seconds < 0:
+            raise ValueError("time budget must be finite and nonnegative")
+
+    def should_stop(self, context: StopContext) -> bool:
+        return context.elapsed_seconds >= self.seconds
+
+    def fingerprint(self) -> object:
+        return {"seconds": self.seconds}
+
+
+@dataclass(frozen=True, slots=True)
+class TargetScore:
+    """Stop when any successful objective reaches a benefit-oriented target."""
+
+    target: float
+
+    def __post_init__(self) -> None:
+        if isinstance(self.target, bool) or not math.isfinite(self.target):
+            raise ValueError("target score must be finite")
+
+    def should_stop(self, context: StopContext) -> bool:
+        return any(score is not None and score >= self.target for score in context.objective_scores)
+
+    def fingerprint(self) -> object:
+        return {"target": self.target}
+
+
+@dataclass(frozen=True, slots=True)
+class MaxFailures:
+    """Stop after a total or consecutive number of failed trials."""
+
+    limit: int
+    consecutive: bool = False
+
+    def __post_init__(self) -> None:
+        if isinstance(self.limit, bool) or not isinstance(self.limit, int) or self.limit < 1:
+            raise ValueError("failure limit must be a positive integer")
+
+    def should_stop(self, context: StopContext) -> bool:
+        if not self.consecutive:
+            return context.failed_trials >= self.limit
+        failures = 0
+        for score in reversed(context.objective_scores):
+            if score is not None:
+                break
+            failures += 1
+        return failures >= self.limit
+
+    def fingerprint(self) -> object:
+        return {"limit": self.limit, "consecutive": self.consecutive}
+
+
+@dataclass(frozen=True, slots=True)
+class NoImprovement:
+    """Stop after completed trials fail to improve the best objective."""
+
+    patience: int
+    min_delta: float = 0.0
+    warmup_trials: int = 1
+
+    def __post_init__(self) -> None:
+        if (
+            isinstance(self.patience, bool)
+            or not isinstance(self.patience, int)
+            or self.patience < 1
+        ):
+            raise ValueError("no-improvement patience must be a positive integer")
+        if (
+            isinstance(self.min_delta, bool)
+            or not math.isfinite(self.min_delta)
+            or self.min_delta < 0
+        ):
+            raise ValueError("minimum improvement must be finite and nonnegative")
+        if (
+            isinstance(self.warmup_trials, bool)
+            or not isinstance(self.warmup_trials, int)
+            or self.warmup_trials < 1
+        ):
+            raise ValueError("warmup trials must be a positive integer")
+
+    def should_stop(self, context: StopContext) -> bool:
+        if context.completed_trials < self.warmup_trials or not context.objective_scores:
+            return False
+        best: float | None = None
+        since_improvement = 0
+        for score in context.objective_scores:
+            if score is not None and (best is None or score > best + self.min_delta):
+                best = score
+                since_improvement = 0
+            else:
+                since_improvement += 1
+        return best is not None and since_improvement >= self.patience
+
+    def fingerprint(self) -> object:
+        return {
+            "patience": self.patience,
+            "min_delta": self.min_delta,
+            "warmup_trials": self.warmup_trials,
+        }