evalrun 0.4.0__py3-none-any.whl

framework/llms/openai.py

@@ -0,0 +1,94 @@
+"""OpenAI client wrapper implementation."""
+
+from typing import List, Optional
+from framework.llms.base import BaseLLM, LLMResponse, Message
+
+
+class OpenAILLM(BaseLLM):
+    """OpenAI API client wrapper."""
+
+    def __init__(
+        self,
+        model_name: str = "gpt-5.6-terra",
+        api_key: Optional[str] = None,
+        base_url: str = "https://api.openai.com/v1",
+        timeout: float = 60.0,
+        max_retries: int = 2,
+    ):
+        """Initializes the OpenAILLM.
+
+        Validates that the 'openai' library is installed immediately.
+
+        Args:
+            model_name: The target model ID.
+            api_key: The OpenAI API key. Reads from environment if None.
+            base_url: The API base URL to connect to. Defaults to OpenAI API.
+            timeout: The request timeout in seconds. Defaults to 60.0.
+            max_retries: The maximum number of retries for request failures. Defaults to 2.
+
+        Raises:
+            ImportError: If the 'openai' library is not installed.
+        """
+        try:
+            # pyrefly: ignore [missing-import]
+            import openai
+        except ImportError:
+            raise ImportError(
+                "The 'openai' library is required to use OpenAILLM. "
+                "Please install it using 'pip install openai'."
+            )
+
+        self._client = openai.OpenAI(
+            api_key=api_key,
+            base_url=base_url,
+            timeout=timeout,
+            max_retries=max_retries,
+        )
+        self.model_name = model_name
+        self.max_retries = max_retries
+        # GPT-5 family Chat Completions models use the newer
+        # `max_completion_tokens` parameter. NVIDIA's OpenAI-compatible NIM
+        # endpoint continues to expect `max_tokens`.
+        self._token_parameter = (
+            "max_completion_tokens"
+            if base_url.rstrip("/") == "https://api.openai.com/v1" and model_name.startswith("gpt-5")
+            else "max_tokens"
+        )
+
+    def generate(self, messages: List[Message]) -> LLMResponse:
+        # Compatibility layer: prepend system content to user content
+        # since some NIM endpoints (e.g. MiniMax) do not support the system role.
+        formatted_messages = []
+        system_content = ""
+        for msg in messages:
+            if msg.role == "system":
+                system_content += msg.content + "\n"
+            else:
+                role = msg.role
+                content = msg.content
+                if system_content:
+                    content = system_content + "\n" + content
+                    system_content = ""
+                formatted_messages.append({"role": role, "content": content})
+        
+        if system_content and not formatted_messages:
+            formatted_messages.append({"role": "user", "content": system_content})
+
+        request_kwargs = {
+            "model": self.model_name,
+            "messages": formatted_messages,
+            self._token_parameter: 4096,
+        }
+        for attempt in range(self.max_retries + 1):
+            response = self._client.chat.completions.create(
+                **request_kwargs,
+            )
+
+            if not response.choices:
+                raise ValueError(f"API returned empty choices. Full response: {response}")
+
+            response_text = response.choices[0].message.content or ""
+            if response_text.strip():
+                return LLMResponse(text=response_text)
+
+        return LLMResponse(text=response_text)

framework/llms/openai_compatible.py

@@ -0,0 +1,139 @@
+import os
+import time
+from typing import Any, Dict, List, Optional
+from openai import OpenAI
+
+from framework.llms.base import BaseLLM, LLMResponse, Message
+
+
+class OpenAICompatibleLLM(BaseLLM):
+    """Universal OpenAI-compatible completion client adapter.
+
+    Supports cloud hosted APIs (OpenAI, NVIDIA NIM, Gemini OpenAI-compat) and
+    local user-hosted servers (vLLM, LM Studio, Ollama) via a single base_url.
+    """
+
+    def __init__(
+        self,
+        model_name: str,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        timeout: float = 180.0,
+        max_retries: int = 3,
+        retry_delay: float = 1.0,
+        extra_headers: Optional[Dict[str, str]] = None,
+        provider: str = "openai_compatible",
+    ):
+        """Initializes the OpenAICompatibleLLM client.
+
+        Args:
+            model_name: Model identifier (e.g. 'gpt-5.6-terra', 'qwen2.5-72b-instruct', 'llama-3.3-70b').
+            api_key: API key. Defaults to OPENAI_API_KEY environment variable or 'EMPTY' for local endpoints.
+            base_url: Base endpoint URL (e.g. 'http://localhost:8000/v1', 'https://integrate.api.nvidia.com/v1').
+            timeout: Request timeout in seconds.
+            max_retries: Maximum retry attempts on transient errors.
+            retry_delay: Base delay between retries in seconds.
+            extra_headers: Optional custom HTTP headers.
+            provider: Human-readable provider label.
+        """
+        self.model_name = model_name
+        self.provider = provider
+        self.base_url = base_url or "https://api.openai.com/v1"
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.retry_delay = retry_delay
+        self.extra_headers = extra_headers
+
+        resolved_api_key = api_key or os.environ.get("OPENAI_API_KEY") or "EMPTY"
+        self.client = OpenAI(
+            api_key=resolved_api_key,
+            base_url=self.base_url,
+            timeout=self.timeout,
+            default_headers=self.extra_headers,
+        )
+
+        self.request_count: int = 0
+        self.retries_attempted: int = 0
+        self.last_error: Optional[str] = None
+        self.last_token_usage: Optional[Dict[str, int]] = None
+
+    def _is_transient_error(self, e: Exception) -> bool:
+        """Determines whether an exception is a transient network/server error suitable for retries."""
+        if isinstance(e, (TimeoutError, ConnectionError)):
+            return True
+        status_code = getattr(e, "status_code", None)
+        if status_code is not None:
+            if status_code in (429, 500, 502, 503, 504):
+                return True
+            if 400 <= status_code < 500 and status_code != 429:
+                return False
+        err_str = str(e).lower()
+        if any(k in err_str for k in ("timeout", "connection", "rate limit", "500", "502", "503", "504", "deadline exceeded")):
+            return True
+        return False
+
+    def generate(self, messages: List[Message]) -> LLMResponse:
+        """Generates a text response from chat messages with retry logic.
+
+        Args:
+            messages: List of Message instances.
+
+        Returns:
+            An LLMResponse containing the text output and execution metadata.
+        """
+        formatted_messages = [{"role": msg.role, "content": msg.content} for msg in messages]
+
+        response = None
+        last_exception = None
+        retries_this_call = 0
+
+        for attempt in range(1 + self.max_retries):
+            try:
+                response = self.client.chat.completions.create(
+                    model=self.model_name,
+                    messages=formatted_messages,
+                )
+                break
+            except Exception as e:
+                last_exception = e
+                self.last_error = str(e)
+                if attempt < self.max_retries and self._is_transient_error(e):
+                    retries_this_call += 1
+                    self.retries_attempted += 1
+                    time.sleep(self.retry_delay * (2 ** attempt))
+                else:
+                    raise last_exception
+
+        self.request_count += 1
+        self.last_error = None
+
+        if hasattr(response, "usage") and response.usage:
+            self.last_token_usage = {
+                "prompt_tokens": getattr(response.usage, "prompt_tokens", 0),
+                "completion_tokens": getattr(response.usage, "completion_tokens", 0),
+                "total_tokens": getattr(response.usage, "total_tokens", 0),
+            }
+        else:
+            self.last_token_usage = None
+
+        if not getattr(response, "choices", None):
+            return LLMResponse(
+                text="",
+                metadata={
+                    "error": "Empty choices array in model response",
+                    "provider": self.provider,
+                    "model_name": self.model_name,
+                    "retries_attempted": retries_this_call,
+                },
+            )
+
+        content = response.choices[0].message.content or ""
+        return LLMResponse(
+            text=content,
+            metadata={
+                "provider": self.provider,
+                "model_name": self.model_name,
+                "token_usage": self.last_token_usage,
+                "retries_attempted": retries_this_call,
+            },
+        )

framework/mcp/__init__.py

@@ -0,0 +1,20 @@
+"""MCP integrations for deterministic agent validation tools."""
+
+from framework.mcp.constraints import (
+    calculate_savings,
+    get_locked_constraints,
+    validate_revision,
+)
+from framework.mcp.client import TravelValidationMCPClient
+
+__all__ = [
+    "calculate_savings",
+    "get_locked_constraints",
+    "validate_revision",
+    "TravelValidationMCPClient",
+]
+"""MCP-backed deterministic validation components."""
+
+from .revision_summary import RevisionSummary, parse_revision_summary
+
+__all__ = ["RevisionSummary", "parse_revision_summary"]

framework/mcp/client.py

@@ -0,0 +1,62 @@
+"""Async client for the local travel-validation MCP server."""
+
+import os
+import sys
+from typing import Any, Dict, List, Optional
+
+from mcp import ClientSession, StdioServerParameters
+from mcp.client.stdio import stdio_client
+
+
+class TravelValidationMCPClient:
+    """Connects to the local travel constraint server over stdio.
+
+    The client creates a short-lived session per request. This is appropriate for
+    the current local validation workflow and keeps the agent integration
+    explicit; long-lived connection management is unnecessary until deployment.
+    """
+
+    def __init__(self, python_executable: Optional[str] = None):
+        self.python_executable = python_executable or sys.executable
+
+    def _server_parameters(self) -> StdioServerParameters:
+        return StdioServerParameters(
+            command=self.python_executable,
+            args=["-m", "framework.mcp.server"],
+            env=dict(os.environ),
+        )
+
+    async def list_tools(self) -> List[str]:
+        """Returns the names of tools exposed by the validation server."""
+        async with stdio_client(self._server_parameters()) as (read, write):
+            async with ClientSession(read, write) as session:
+                await session.initialize()
+                result = await session.list_tools()
+                return [tool.name for tool in result.tools]
+
+    async def call_tool(
+        self, name: str, arguments: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """Calls a tool and returns its structured JSON response.
+
+        Raises:
+            RuntimeError: If the MCP server reports an error or returns an
+                unexpected non-dictionary payload.
+        """
+        async with stdio_client(self._server_parameters()) as (read, write):
+            async with ClientSession(read, write) as session:
+                await session.initialize()
+                result = await session.call_tool(name, arguments)
+
+        if result.isError:
+            raise RuntimeError(f"MCP tool '{name}' failed: {result.content}")
+        if not isinstance(result.structuredContent, dict):
+            raise RuntimeError(
+                f"MCP tool '{name}' returned an unexpected payload: {result.content}"
+            )
+        payload = dict(result.structuredContent)
+        # FastMCP wraps plain dictionary tool returns under ``result``. Keep the
+        # application-facing client independent from that transport detail.
+        if set(payload) == {"result"} and isinstance(payload["result"], dict):
+            return dict(payload["result"])
+        return payload

framework/mcp/constraints.py

@@ -0,0 +1,125 @@
+"""Deterministic constraint checks used by the travel validation MCP server."""
+
+from copy import deepcopy
+from typing import Any, Dict, List
+
+
+MID_TRIP_REPLANNING_SCENARIO = "travel-mid-trip-replanning"
+
+_SCENARIO_CONSTRAINTS: Dict[str, Dict[str, Any]] = {
+    MID_TRIP_REPLANNING_SCENARIO: {
+        "scenario_id": MID_TRIP_REPLANNING_SCENARIO,
+        "immutable_booking_ids": ["kyoto-hostel", "narita-return-flight"],
+        "locked_bookings": [
+            {
+                "booking_id": "kyoto-hostel",
+                "description": "Kyoto hostel stay on Days 15–18",
+                "start_day": 15,
+                "end_day": 18,
+                "reason": "Pre-paid and non-refundable",
+            },
+            {
+                "booking_id": "narita-return-flight",
+                "description": "Return flight from Narita on Day 28",
+                "start_day": 28,
+                "end_day": 28,
+                "reason": "Non-changeable",
+            },
+        ],
+        "required_savings_inr": 20000.0,
+        "revision_start_day": 13,
+    }
+}
+
+
+def get_locked_constraints(scenario_id: str) -> Dict[str, Any]:
+    """Returns immutable bookings and numeric targets for a supported scenario."""
+    try:
+        return deepcopy(_SCENARIO_CONSTRAINTS[scenario_id])
+    except KeyError as error:
+        raise ValueError(f"Unsupported scenario: '{scenario_id}'") from error
+
+
+def validate_revision(
+    scenario_id: str,
+    booking_actions: List[Dict[str, str]],
+    changed_days: List[int],
+) -> Dict[str, Any]:
+    """Checks whether a proposed revision changes an immutable booking.
+
+    ``booking_actions`` must include one entry for each locked booking with an
+    action of ``preserve``, ``move``, ``cancel``, or ``modify``. The function
+    intentionally validates only explicit, deterministic assertions; the LLM
+    remains responsible for itinerary quality and travel judgment.
+    """
+    constraints = get_locked_constraints(scenario_id)
+    immutable_ids = constraints["immutable_booking_ids"]
+    actions_by_booking = {
+        action.get("booking_id"): action.get("action")
+        for action in booking_actions
+        if action.get("booking_id")
+    }
+    violations = []
+
+    for booking_id in immutable_ids:
+        action = actions_by_booking.get(booking_id)
+        if action is None:
+            violations.append(
+                f"No preservation assertion was supplied for locked booking '{booking_id}'."
+            )
+        elif action != "preserve":
+            violations.append(
+                f"Locked booking '{booking_id}' must be preserved, not '{action}'."
+            )
+
+    invalid_days = [
+        day
+        for day in changed_days
+        if not isinstance(day, int) or day < constraints["revision_start_day"]
+    ]
+    if invalid_days:
+        violations.append(
+            "A replanning revision cannot modify days before Day "
+            f"{constraints['revision_start_day']}: {invalid_days}."
+        )
+
+    return {
+        "valid": not violations,
+        "violations": violations,
+        "immutable_booking_ids": immutable_ids,
+        "changed_days": changed_days,
+    }
+
+
+def calculate_savings(
+    scenario_id: str, savings_items: List[Dict[str, Any]]
+) -> Dict[str, Any]:
+    """Totals explicit savings items against the scenario's required reduction."""
+    constraints = get_locked_constraints(scenario_id)
+    normalized_items = []
+    invalid_items = []
+
+    for item in savings_items:
+        label = str(item.get("label", "")).strip()
+        try:
+            amount_inr = float(item["amount_inr"])
+        except (KeyError, TypeError, ValueError):
+            invalid_items.append(item)
+            continue
+        if not label or amount_inr < 0:
+            invalid_items.append(item)
+            continue
+        normalized_items.append({"label": label, "amount_inr": amount_inr})
+
+    total_savings_inr = sum(item["amount_inr"] for item in normalized_items)
+    target_savings_inr = constraints["required_savings_inr"]
+    remaining_gap_inr = max(target_savings_inr - total_savings_inr, 0.0)
+
+    return {
+        "target_savings_inr": target_savings_inr,
+        "total_savings_inr": total_savings_inr,
+        "remaining_gap_inr": remaining_gap_inr,
+        "target_met": not invalid_items and remaining_gap_inr == 0.0,
+        "accepted_items": normalized_items,
+        "invalid_items": invalid_items,
+    }

framework/mcp/revision_summary.py

@@ -0,0 +1,122 @@
+"""Structured, LLM-produced claims for deterministic revision validation.
+
+The travel itinerary remains free-form text for the traveler.  Before a
+replanning revision is accepted, the model must also provide this small JSON
+summary of the claims that a deterministic tool can verify.
+"""
+
+from dataclasses import asdict, dataclass
+from typing import Any, Dict, List
+
+from framework.utils import parse_json_markdown
+
+
+@dataclass(frozen=True)
+class BookingAction:
+    """A declared treatment of one booking in a proposed itinerary revision."""
+
+    booking_id: str
+    action: str
+
+
+@dataclass(frozen=True)
+class SavingsItem:
+    """One concrete cost-saving substitution claimed by the proposed revision."""
+
+    label: str
+    amount_inr: float
+
+
+@dataclass(frozen=True)
+class RevisionSummary:
+    """The complete deterministic-validation contract for a replanning draft."""
+
+    scenario_id: str
+    changed_days: List[int]
+    booking_actions: List[BookingAction]
+    savings_items: List[SavingsItem]
+
+    def to_tool_arguments(self) -> Dict[str, Any]:
+        """Returns JSON-compatible arguments for the validation MCP tools."""
+        return {
+            "scenario_id": self.scenario_id,
+            "booking_actions": [asdict(action) for action in self.booking_actions],
+            "changed_days": self.changed_days,
+            "savings_items": [asdict(item) for item in self.savings_items],
+        }
+
+
+REVISION_SUMMARY_JSON_TEMPLATE = """{
+  "scenario_id": "travel-mid-trip-replanning",
+  "changed_days": [13, 19, 22],
+  "booking_actions": [
+    {"booking_id": "kyoto-hostel", "action": "preserve"},
+    {"booking_id": "narita-return-flight", "action": "preserve"}
+  ],
+  "savings_items": [
+    {"label": "Kyoto-to-Tokyo night bus instead of Shinkansen", "amount_inr": 12000}
+  ]
+}"""
+
+
+def parse_revision_summary(text: str) -> RevisionSummary:
+    """Parses and validates the JSON contract emitted by an LLM.
+
+    This parser intentionally rejects incomplete or ambiguous declarations. A
+    model must explicitly state every field before its proposal can be passed
+    to the deterministic MCP tools.
+    """
+    payload = parse_json_markdown(text)
+    if not isinstance(payload, dict):
+        raise ValueError("Revision summary must be a JSON object.")
+
+    scenario_id = payload.get("scenario_id")
+    if not isinstance(scenario_id, str) or not scenario_id.strip():
+        raise ValueError("Revision summary requires a non-empty scenario_id.")
+
+    changed_days = payload.get("changed_days")
+    if not isinstance(changed_days, list) or not all(
+        isinstance(day, int) and not isinstance(day, bool) for day in changed_days
+    ):
+        raise ValueError("changed_days must be a list of integer day numbers.")
+
+    raw_booking_actions = payload.get("booking_actions")
+    if not isinstance(raw_booking_actions, list):
+        raise ValueError("booking_actions must be a list.")
+    booking_actions = []
+    for item in raw_booking_actions:
+        if not isinstance(item, dict):
+            raise ValueError("Each booking action must be an object.")
+        booking_id = item.get("booking_id")
+        action = item.get("action")
+        if not isinstance(booking_id, str) or not booking_id.strip():
+            raise ValueError("Each booking action requires a non-empty booking_id.")
+        if action not in {"preserve", "move", "cancel", "modify"}:
+            raise ValueError("Each booking action must be preserve, move, cancel, or modify.")
+        booking_actions.append(BookingAction(booking_id=booking_id, action=action))
+
+    raw_savings_items = payload.get("savings_items")
+    if not isinstance(raw_savings_items, list):
+        raise ValueError("savings_items must be a list.")
+    savings_items = []
+    for item in raw_savings_items:
+        if not isinstance(item, dict):
+            raise ValueError("Each savings item must be an object.")
+        label = item.get("label")
+        amount_inr = item.get("amount_inr")
+        if not isinstance(label, str) or not label.strip():
+            raise ValueError("Each savings item requires a non-empty label.")
+        if (
+            not isinstance(amount_inr, (int, float))
+            or isinstance(amount_inr, bool)
+            or amount_inr < 0
+        ):
+            raise ValueError("Each savings amount_inr must be a non-negative number.")
+        savings_items.append(SavingsItem(label=label, amount_inr=float(amount_inr)))
+
+    return RevisionSummary(
+        scenario_id=scenario_id,
+        changed_days=changed_days,
+        booking_actions=booking_actions,
+        savings_items=savings_items,
+    )

framework/mcp/server.py

@@ -0,0 +1,49 @@
+"""Local stdio MCP server for travel-planning constraint validation."""
+
+from typing import Any, Dict, List
+
+from mcp.server.fastmcp import FastMCP
+
+from framework.mcp.constraints import (
+    calculate_savings as calculate_savings_check,
+    get_locked_constraints as get_locked_constraints_check,
+    validate_revision as validate_revision_check,
+)
+
+
+mcp = FastMCP(
+    "Travel Constraint Validation",
+    instructions=(
+        "Use these tools to verify explicit locked-booking constraints and "
+        "cost-saving arithmetic for travel itinerary revisions."
+    ),
+    json_response=True,
+)
+
+
+@mcp.tool()
+def get_locked_constraints(scenario_id: str) -> Dict[str, Any]:
+    """Get immutable bookings and savings targets for a benchmark scenario."""
+    return get_locked_constraints_check(scenario_id)
+
+
+@mcp.tool()
+def validate_revision(
+    scenario_id: str,
+    booking_actions: List[Dict[str, str]],
+    changed_days: List[int],
+) -> Dict[str, Any]:
+    """Check whether a revision preserves every locked booking."""
+    return validate_revision_check(scenario_id, booking_actions, changed_days)
+
+
+@mcp.tool()
+def calculate_savings(
+    scenario_id: str, savings_items: List[Dict[str, Any]]
+) -> Dict[str, Any]:
+    """Calculate explicit INR savings against a scenario's required reduction."""
+    return calculate_savings_check(scenario_id, savings_items)
+
+
+if __name__ == "__main__":
+    mcp.run()

framework/memory/__init__.py

@@ -0,0 +1,3 @@
+"""Abstract session memory framework interfaces."""
+
+from framework.memory.base import BaseSessionMemory

framework/memory/base.py

@@ -0,0 +1,17 @@
+"""Abstract base class for session memory."""
+
+from abc import ABC, abstractmethod
+
+
+class BaseSessionMemory(ABC):
+    """Abstract base class defining the interface for agent session memory."""
+
+    @abstractmethod
+    def to_yaml(self) -> str:
+        """Serializes memory state to structured YAML string for LLM injection."""
+        pass
+
+    @abstractmethod
+    def to_json(self) -> str:
+        """Serializes memory state to JSON string representation."""
+        pass