quantred 2.4.0__py3-none-any.whl

quantred/research/proposer.py

@@ -0,0 +1,234 @@
+"""Proposers: where the *creativity* enters the research loop.
+
+A :class:`Proposer` turns a goal + the history so far into the next
+:class:`Proposal` - either a parameter config to try ("tune") or new strategy
+*code* to author ("code"). The agent loop and all guardrails are proposer-agnostic,
+so the LLM-backed :class:`LLMProposer` and a deterministic test double share one
+contract.
+
+Each proposal must carry a one-paragraph **hypothesis** (the economic/behavioral
+rationale). No rationale -> the agent rejects it unevaluated (a research-hygiene
+rule; see the research-agent guide in the engineering docs).
+
+:class:`LLMProposer` is provider-agnostic - it drives any
+:class:`~quantred.research.llm.LLMClient` (Claude, GPT, or a local Ollama model), so
+the same loop runs against a hosted frontier model or one on your laptop.
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+from quantred.research.llm import LLMClient, build_llm_client
+
+
+@dataclass
+class Proposal:
+    """One experiment the agent should try next."""
+
+    hypothesis: str  # WHY this should work (required by hygiene)
+    kind: str = "tune"  # "tune" (config) | "code" (new subclass)
+    strategy: Optional[str] = None  # registered strategy to tune (kind == "tune")
+    params: Dict[str, Any] = field(default_factory=dict)  # config overrides (kind == "tune")
+    code: Optional[str] = None  # source of a new Strategy subclass (kind == "code")
+    tuned_params: List[str] = field(default_factory=list)  # which params this varies (hygiene cap)
+    parent_id: Optional[str] = None  # lineage: the candidate this mutates, if any
+    tokens_used: int = 0  # reported LLM token cost, for the budget
+    #: Why the reply could not be read as a proposal at all (unparseable text, a field of
+    #: the wrong type). Set instead of returning ``None``, which means "nothing more to
+    #: try": a model that answered badly once still has ideas, and the round is spent.
+    malformed: Optional[str] = None
+
+
+@dataclass
+class ProposalContext:
+    """What the proposer sees each round (never includes holdout data)."""
+
+    goal: str
+    strategy: str
+    param_ranges: Dict[str, Any]
+    history: List[Dict[str, Any]]  # compact summary of prior trials + OOS scores
+    incumbent: Optional[Dict[str, Any]]  # current best candidate, if any
+    round_index: int
+
+
+class Proposer(ABC):
+    """Turns the goal + history into the next proposal."""
+
+    @abstractmethod
+    def propose(self, context: ProposalContext) -> Optional[Proposal]:
+        """Return the next proposal, or ``None`` to signal "nothing more to try"."""
+
+
+class FixedProposer(Proposer):
+    """Deterministic proposer that replays a fixed list - for tests and replays."""
+
+    def __init__(self, proposals: List[Proposal]):
+        self._proposals = list(proposals)
+
+    def propose(self, context: ProposalContext) -> Optional[Proposal]:
+        if context.round_index >= len(self._proposals):
+            return None
+        return self._proposals[context.round_index]
+
+
+class LLMProposer(Proposer):
+    """Proposer backed by any :class:`~quantred.research.llm.LLMClient`.
+
+    Asks the model for a JSON proposal: a hypothesis plus either param overrides
+    within the supplied ``PARAM_RANGES`` (``kind="tune"``) or, when
+    ``allow_code_gen`` is set, the source of a brand-new ``Strategy`` subclass
+    (``kind="code"``). The model id is pinned on the client and logged by the
+    agent's audit journal, so every proposal is replayable.
+
+    Code proposals are *untrusted text* until :mod:`quantred.research.sandbox` has
+    validated them - this class only transports them.
+    """
+
+    def __init__(self, client: LLMClient, max_tokens: int = 1024, *, allow_code_gen: bool = False):
+        self.client = client
+        self.max_tokens = max_tokens
+        self.allow_code_gen = allow_code_gen
+
+    def propose(self, context: ProposalContext) -> Optional[Proposal]:
+        import json
+
+        system = _CODEGEN_SYSTEM_PROMPT if self.allow_code_gen else _SYSTEM_PROMPT
+        response = self.client.complete(system, _build_user_prompt(context), self.max_tokens)
+        try:
+            payload = json.loads(_extract_json(response.text))
+        except (ValueError, KeyError):
+            return self._malformed("unparseable reply", response.tokens, context)
+        if not isinstance(payload, dict):
+            return self._malformed("unparseable reply: expected a JSON object", response.tokens, context)
+
+        hypothesis = payload.get("hypothesis", "")
+        if not isinstance(hypothesis, str):
+            return self._malformed(
+                f"hypothesis must be text, got {_type_name(hypothesis)}", response.tokens, context
+            )
+
+        if payload.get("kind") == "code":
+            code = payload.get("code", "")
+            if not isinstance(code, str):
+                return self._malformed(f"code must be text, got {_type_name(code)}", response.tokens, context)
+            return Proposal(
+                hypothesis=hypothesis,
+                kind="code",
+                strategy=context.strategy,
+                code=code,
+                tokens_used=response.tokens,
+            )
+
+        params = payload.get("params", {})
+        if not isinstance(params, dict):
+            return self._malformed(
+                f"params must be an object of name: value, got {_type_name(params)}",
+                response.tokens,
+                context,
+                hypothesis,
+            )
+        return Proposal(
+            hypothesis=hypothesis,
+            kind="tune",
+            strategy=context.strategy,
+            params=params,
+            tuned_params=list(params),
+            tokens_used=response.tokens,
+        )
+
+    @staticmethod
+    def _malformed(reason: str, tokens: int, context: ProposalContext, hypothesis: str = "") -> Proposal:
+        return Proposal(
+            hypothesis=hypothesis,
+            kind="tune",
+            strategy=context.strategy,
+            tokens_used=tokens,
+            malformed=reason,
+        )
+
+
+def _type_name(value: Any) -> str:
+    """A JSON-flavoured name for a decoded value, for a message a human reads."""
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return "a boolean"
+    if isinstance(value, (int, float)):
+        return "a number"
+    if isinstance(value, str):
+        return "text"
+    return "a list" if isinstance(value, list) else type(value).__name__
+
+
+def build_proposer(
+    provider: str = "anthropic",
+    model: Optional[str] = None,
+    *,
+    client: Optional[LLMClient] = None,
+    max_tokens: int = 1024,
+    allow_code_gen: bool = False,
+    **kwargs,
+) -> LLMProposer:
+    """Build an :class:`LLMProposer` for a provider (anthropic | openai | ollama)."""
+    llm = client or build_llm_client(provider, model, **kwargs)
+    return LLMProposer(llm, max_tokens=max_tokens, allow_code_gen=allow_code_gen)
+
+
+_SYSTEM_PROMPT = (
+    "You are a quantitative research assistant proposing the next backtest experiment. "
+    "Change at most 5 parameters. Give a one-paragraph hypothesis of WHY the change should "
+    "improve OUT-OF-SAMPLE performance - in-sample gains do not count. "
+    'Respond ONLY with JSON: {"hypothesis": "...", "params": {"name": value, ...}}.'
+)
+
+#: System prompt for sessions run with ``--allow-code-gen``. The model may either
+#: tune the incumbent or author a new mechanism; the sandbox enforces the contract
+#: described here, so a violation is a rejected proposal, not a broken run.
+_CODEGEN_SYSTEM_PROMPT = (
+    "You are a quantitative research assistant proposing the next backtest experiment. "
+    "You may respond with EITHER of two proposal kinds.\n\n"
+    '1. Tune the existing strategy: {"kind": "tune", "hypothesis": "...", '
+    '"params": {"name": value, ...}} - at most 5 parameters, each within PARAM_RANGES.\n\n'
+    '2. Author a NEW strategy mechanism: {"kind": "code", "hypothesis": "...", "code": "<python source>"}.\n\n'
+    "Propose 'code' when the history suggests the parameter space is exhausted and a different "
+    "mechanism is needed; propose 'tune' when the incumbent looks refinable.\n\n"
+    "Generated code runs in a restricted sandbox and MUST satisfy this contract or it is rejected:\n"
+    "- Define exactly one concrete subclass of `Strategy` (from quantred.strategies.base).\n"
+    "- Imports are limited to: pandas, numpy, math, quantred.indicators.indicators, quantred.strategies.base. "
+    "No os, sys, io, requests, or file access of any kind.\n"
+    "- Give the class a docstring stating the hypothesis (required).\n"
+    "- Declare a PARAM_RANGES ClassVar dict with AT MOST 5 SEARCHABLE params. A param is "
+    "searchable when its spec has min, max, and step; a spec with only a default is fixed and "
+    "does NOT count against the cap. Each spec needs type ('int'|'float'), default, description.\n"
+    "- PARAM_RANGES MUST include risk_per_trade, stop_loss, and take_profit (fractional distances). "
+    "The execution path reads these on every bar; omitting them is rejected. Pin them with a "
+    "default only if you would rather spend the searchable budget on your mechanism.\n"
+    "- Set TIMEFRAME (e.g. '1Day') and call super().__init__(config) after setting config['timeframe'].\n"
+    "- Implement all four abstract hooks: initialize(), process_data(data) -> DataFrame, "
+    "calculate_scores(data) -> Series, calculate_required_lookback() -> int.\n"
+    "- calculate_scores returns ONE continuous conviction score per bar; the base class derives "
+    "BUY/SELL/HOLD from its sign. Do not emit discrete signals yourself.\n"
+    "- The class must construct from its own declared defaults.\n\n"
+    "Give a one-paragraph hypothesis of WHY this should improve OUT-OF-SAMPLE performance - "
+    "in-sample gains do not count. Respond ONLY with the JSON object."
+)
+
+
+def _build_user_prompt(context: ProposalContext) -> str:
+    import json
+
+    return (
+        f"GOAL: {context.goal}\n"
+        f"STRATEGY: {context.strategy}\n"
+        f"TUNABLE PARAM_RANGES (stay within bounds):\n{json.dumps(context.param_ranges, default=str)}\n"
+        f"HISTORY (prior out-of-sample results):\n{json.dumps(context.history, default=str)}\n"
+    )
+
+
+def _extract_json(text: str) -> str:
+    """Pull the first {...} block out of an LLM response."""
+    start, end = text.find("{"), text.rfind("}")
+    if start == -1 or end == -1 or end < start:
+        raise ValueError("no JSON object in response")
+    return text[start : end + 1]

quantred/research/provider_errors.py

@@ -0,0 +1,242 @@
+"""Naming what went wrong when a model provider refuses or cannot be reached.
+
+One classification, used twice: by the opt-in readiness probe, and by the research
+loop, because a key that passes a probe can still run out of credit twenty minutes
+into a session. Both surfaces must name the same failure the same way, so the
+mapping lives here and imports nothing heavy - the provider SDKs are never imported,
+the exceptions are recognised by their shape (``status_code``, ``code``, ``type``,
+class names).
+
+**Structured fields decide; the message is the last resort.** Both SDKs carry a
+machine-readable error code/type (``insufficient_quota`` against
+``rate_limit_exceeded``; ``billing_error``, ``rate_limit_error``,
+``overloaded_error``). A message is free text that can mention billing for any reason -
+a rate-limit reply that links the billing page is still a rate limit - so wording is
+consulted only when no structured field settles the question.
+
+**Messages never carry the exception's own text.** A vendor error string can echo
+request parameters, and the request carries the key. The detail is composed here from
+the category, the HTTP status and the exception's type name.
+"""
+
+from __future__ import annotations
+
+import http.client
+import socket
+import urllib.error
+from typing import Any, List, Literal, Optional, Tuple
+
+ProbeCategory = Literal[
+    "ok",
+    "auth",
+    "quota",
+    "rate_limited",
+    "unavailable",
+    "entitlement",
+    "unreachable",
+    "timeout",
+    "model_not_found",
+    "unknown",
+]
+
+#: Categories worth sending again: the condition is the provider's or the network's and
+#: passes by itself. ``quota`` (an empty balance), ``auth``, ``entitlement`` and
+#: ``model_not_found`` are not - repeating the request cannot change the answer, only
+#: bill or hammer an account. ``timeout`` is left out too: a provider that did not answer
+#: in the configured time is not helped by waiting that long again, three times.
+RETRYABLE: Tuple[str, ...] = ("rate_limited", "unavailable", "unreachable")
+
+#: Structured ``code`` / ``type`` values (OpenAI and Anthropic) and the category each settles.
+_STRUCTURED = {
+    "insufficient_quota": "quota",
+    "billing_error": "quota",
+    "rate_limit_exceeded": "rate_limited",
+    "rate_limit_error": "rate_limited",
+    "overloaded_error": "unavailable",
+    "server_error": "unavailable",
+    "api_error": "unavailable",
+    "authentication_error": "auth",
+    "invalid_api_key": "auth",
+    "permission_error": "entitlement",
+    "not_found_error": "model_not_found",
+    "model_not_found": "model_not_found",
+}
+
+#: Wording that marks a 400/429 as "out of credit" - consulted only when no structured
+#: field settled it. Anthropic reports an empty balance as a plain 400 with no code.
+_CREDIT_MARKERS = (
+    "insufficient_quota",
+    "credit balance",
+    "exceeded your current quota",
+)
+
+_HTTP_STATUS = {
+    401: ("auth", "the provider rejected the credentials (HTTP 401)"),
+    402: ("quota", "credit exhausted (HTTP 402) - the account has no usable balance"),
+    403: ("entitlement", "the credentials are valid but not permitted to use this model (HTTP 403)"),
+    404: ("model_not_found", "the provider has no such model (HTTP 404)"),
+}
+
+
+class ModelProviderError(RuntimeError):
+    """A provider call failed in a way the user can act on.
+
+    ``category`` is one of :data:`ProbeCategory`; ``detail`` is safe to print.
+    """
+
+    def __init__(
+        self,
+        provider: str,
+        category: ProbeCategory,
+        detail: str,
+        *,
+        attempts: int = 1,
+        advised_wait: Optional[float] = None,
+    ):
+        super().__init__(f"model provider {provider}: {category} - {detail}")
+        self.provider = provider
+        self.category = category
+        self.detail = detail
+        #: Requests actually sent before giving up (1 means no retry happened) and the
+        #: wait the provider asked for, if any. Carried so the message can say what
+        #: happened instead of what the policy usually does.
+        self.attempts = attempts
+        self.advised_wait = advised_wait
+
+
+def _class_names(exc: BaseException) -> set:
+    return {cls.__name__ for cls in type(exc).__mro__}
+
+
+def _status_of(exc: BaseException) -> Optional[int]:
+    for attr in ("status_code", "code", "status"):
+        value = getattr(exc, attr, None)
+        if isinstance(value, int) and not isinstance(value, bool):
+            return value
+    return None
+
+
+def _structured_values(exc: BaseException) -> List[str]:
+    """The machine-readable code/type strings an error carries, lower-cased."""
+    values: List[Any] = [getattr(exc, "code", None), getattr(exc, "type", None)]
+    body = getattr(exc, "body", None)
+    for level in (body, body.get("error") if isinstance(body, dict) else None):
+        if isinstance(level, dict):
+            values.extend((level.get("code"), level.get("type")))
+    return [v.lower() for v in values if isinstance(v, str)]
+
+
+def _message_text(exc: BaseException) -> str:
+    """Free text that might say "credit" - used only to decide, never to print."""
+    parts = [str(exc)]
+    body = getattr(exc, "body", None)
+    if body is not None:
+        parts.append(str(body))
+    return " ".join(parts).lower()
+
+
+def _structured_category(exc: BaseException) -> Optional[str]:
+    for value in _structured_values(exc):
+        if value in _STRUCTURED:
+            return _STRUCTURED[value]
+    return None
+
+
+def is_provider_exception(exc: BaseException) -> bool:
+    """Whether ``exc`` is a vendor/network failure rather than a bug in this code.
+
+    Recognised by shape, like the classifier: an HTTP status carried by the exception,
+    an SDK ``API*Error`` class, or a socket/urllib failure. Anything else - an
+    ``AttributeError`` parsing a reply, a ``KeyError`` - is ours and keeps its traceback.
+    """
+    if isinstance(
+        exc,
+        (
+            urllib.error.URLError,
+            TimeoutError,
+            socket.timeout,
+            socket.gaierror,
+            ConnectionError,
+            http.client.HTTPException,
+        ),
+    ):
+        return True
+    if _status_of(exc) is not None:
+        return True
+    return bool(_class_names(exc) & {"APIError", "APIStatusError", "APIConnectionError", "APITimeoutError"})
+
+
+def retry_after_seconds(exc: BaseException) -> Optional[float]:
+    """The provider's own ``Retry-After`` (seconds), when the error carries one."""
+    headers = getattr(getattr(exc, "response", None), "headers", None) or getattr(exc, "headers", None)
+    if headers is None:
+        return None
+    try:
+        value = headers.get("retry-after")
+    except AttributeError:
+        return None
+    try:
+        return float(value) if value is not None else None
+    except (TypeError, ValueError):
+        return None
+
+
+def classify_provider_error(exc: BaseException) -> Tuple[ProbeCategory, str]:
+    """Map a provider exception to ``(category, safe detail)``.
+
+    Duck-typed so it needs no SDK. Order: transport failures by type, then the
+    structured code/type, then the HTTP status, and only for a 400/429 that nothing
+    structured explained, the message wording. ``quota`` means credit (a billing
+    action); ``rate_limited`` and ``unavailable`` mean wait.
+    """
+    names = _class_names(exc)
+
+    if isinstance(exc, TimeoutError) or isinstance(exc, socket.timeout) or "APITimeoutError" in names:
+        return "timeout", "the provider did not answer in time"
+    if isinstance(exc, urllib.error.URLError) and not isinstance(exc, urllib.error.HTTPError):
+        reason = exc.reason
+        if isinstance(reason, (TimeoutError, socket.timeout)):
+            return "timeout", "the provider did not answer in time"
+        return "unreachable", "the connection failed before any response"
+
+    status = _status_of(exc)
+    suffix = f" (HTTP {status})" if status is not None else ""
+    structured = _structured_category(exc)
+    if structured == "quota":
+        return "quota", f"credit exhausted{suffix} - the account has no usable balance"
+    if structured == "rate_limited":
+        return "rate_limited", f"rate limited{suffix} - too many requests, not an empty balance; wait"
+    if structured == "unavailable":
+        return "unavailable", f"the provider is overloaded or failing{suffix}; try again shortly"
+    if structured in ("auth", "entitlement", "model_not_found") and status in (None, 400, 401, 403, 404):
+        return _HTTP_STATUS[{"auth": 401, "entitlement": 403, "model_not_found": 404}[structured]]
+
+    if status in _HTTP_STATUS:
+        return _HTTP_STATUS[status]
+    if status == 429 or status == 400:
+        # No structured field settled it: the message is all there is.
+        if any(marker in _message_text(exc) for marker in _CREDIT_MARKERS):
+            return "quota", f"credit exhausted{suffix} - the account has no usable balance"
+        if status == 429:
+            return "rate_limited", f"rate limited{suffix} - too many requests, not an empty balance; wait"
+    if status is not None and status >= 500:
+        return "unavailable", f"the provider is overloaded or failing{suffix}; try again shortly"
+
+    if "APIConnectionError" in names or isinstance(exc, (ConnectionError, OSError)):
+        return "unreachable", "the connection failed before any response"
+
+    return "unknown", f"unexpected {type(exc).__name__}{suffix}"
+
+
+def as_provider_error(
+    provider: str, exc: BaseException, *, detail: Optional[Any] = None, attempts: int = 1
+) -> ModelProviderError:
+    """Wrap ``exc`` as a :class:`ModelProviderError`, optionally overriding the detail."""
+    category, classified = classify_provider_error(exc)
+    return ModelProviderError(
+        provider,
+        category,
+        str(detail) if detail else classified,
+        attempts=attempts,
+        advised_wait=retry_after_seconds(exc),
+    )