remdb 0.2.6__py3-none-any.whl → 0.3.103__py3-none-any.whl

rem/agentic/llm_provider_models.py

@@ -0,0 +1,301 @@
+"""
+LLM Provider Model Registry.
+
+Defines available LLM models across providers (OpenAI, Anthropic, Google, Cerebras).
+Used by the models API endpoint and for validating model requests.
+
+Future: Models will be stored in database for dynamic management.
+"""
+
+from pydantic import BaseModel, Field
+from typing import Literal
+
+
+class ModelInfo(BaseModel):
+    """Information about a single model."""
+
+    id: str = Field(description="Model ID in provider:model format")
+    object: Literal["model"] = "model"
+    created: int = Field(description="Unix timestamp of model availability")
+    owned_by: str = Field(description="Provider name")
+    description: str | None = Field(default=None, description="Model description")
+    context_window: int | None = Field(default=None, description="Max context tokens")
+    max_output_tokens: int | None = Field(default=None, description="Max output tokens")
+
+
+# Model definitions with 2025 releases
+# Using Unix timestamps for created dates (approximate release dates)
+AVAILABLE_MODELS: list[ModelInfo] = [
+    # ==========================================================================
+    # OpenAI Models (2025)
+    # ==========================================================================
+    # GPT-4.1 series (Released April 14, 2025)
+    ModelInfo(
+        id="openai:gpt-4.1",
+        created=1744588800,  # April 14, 2025
+        owned_by="openai",
+        description="Latest GPT-4 iteration, excels at coding and instruction following. 1M context.",
+        context_window=1047576,
+        max_output_tokens=32768,
+    ),
+    ModelInfo(
+        id="openai:gpt-4.1-mini",
+        created=1744588800,
+        owned_by="openai",
+        description="Small model beating GPT-4o in many benchmarks. 83% cost reduction vs GPT-4o.",
+        context_window=1047576,
+        max_output_tokens=32768,
+    ),
+    ModelInfo(
+        id="openai:gpt-4.1-nano",
+        created=1744588800,
+        owned_by="openai",
+        description="Fastest and cheapest OpenAI model. Ideal for classification and autocompletion.",
+        context_window=1047576,
+        max_output_tokens=32768,
+    ),
+    # GPT-4o (legacy but still supported)
+    ModelInfo(
+        id="openai:gpt-4o",
+        created=1715644800,  # May 13, 2024
+        owned_by="openai",
+        description="Previous flagship multimodal model. Being superseded by GPT-4.1.",
+        context_window=128000,
+        max_output_tokens=16384,
+    ),
+    ModelInfo(
+        id="openai:gpt-4o-mini",
+        created=1721347200,  # July 18, 2024
+        owned_by="openai",
+        description="Cost-efficient smaller GPT-4o variant.",
+        context_window=128000,
+        max_output_tokens=16384,
+    ),
+    # o1 reasoning models
+    ModelInfo(
+        id="openai:o1",
+        created=1733961600,  # December 12, 2024
+        owned_by="openai",
+        description="Advanced reasoning model for complex problems. Extended thinking.",
+        context_window=200000,
+        max_output_tokens=100000,
+    ),
+    ModelInfo(
+        id="openai:o1-mini",
+        created=1726099200,  # September 12, 2024
+        owned_by="openai",
+        description="Smaller reasoning model, fast for coding and math.",
+        context_window=128000,
+        max_output_tokens=65536,
+    ),
+    ModelInfo(
+        id="openai:o3-mini",
+        created=1738195200,  # January 30, 2025
+        owned_by="openai",
+        description="Latest mini reasoning model with improved performance.",
+        context_window=200000,
+        max_output_tokens=100000,
+    ),
+    # ==========================================================================
+    # Anthropic Models (2025)
+    # ==========================================================================
+    # Claude 4.5 series (Latest - November 2025)
+    ModelInfo(
+        id="anthropic:claude-opus-4-5-20251124",
+        created=1732406400,  # November 24, 2025
+        owned_by="anthropic",
+        description="Most capable Claude model. World-class coding with 'effort' parameter control.",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    ModelInfo(
+        id="anthropic:claude-sonnet-4-5-20250929",
+        created=1727568000,  # September 29, 2025
+        owned_by="anthropic",
+        description="Best balance of intelligence and speed. Excellent for coding and agents.",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    ModelInfo(
+        id="anthropic:claude-haiku-4-5-20251101",
+        created=1730419200,  # November 1, 2025
+        owned_by="anthropic",
+        description="Fast and affordable. Sonnet 4 performance at 1/3 cost. Safest Claude model.",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    # Claude 4 series
+    ModelInfo(
+        id="anthropic:claude-opus-4-20250514",
+        created=1715644800,  # May 14, 2025
+        owned_by="anthropic",
+        description="World's best coding model. Sustained performance on complex agent workflows.",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    ModelInfo(
+        id="anthropic:claude-sonnet-4-20250514",
+        created=1715644800,  # May 14, 2025
+        owned_by="anthropic",
+        description="Significant upgrade to Sonnet 3.7. Great for everyday tasks.",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    ModelInfo(
+        id="anthropic:claude-opus-4-1-20250805",
+        created=1722816000,  # August 5, 2025
+        owned_by="anthropic",
+        description="Opus 4 upgrade focused on agentic tasks and real-world coding.",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    # Aliases for convenience
+    ModelInfo(
+        id="anthropic:claude-opus-4-5",
+        created=1732406400,
+        owned_by="anthropic",
+        description="Alias for latest Claude Opus 4.5",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    ModelInfo(
+        id="anthropic:claude-sonnet-4-5",
+        created=1727568000,
+        owned_by="anthropic",
+        description="Alias for latest Claude Sonnet 4.5",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    ModelInfo(
+        id="anthropic:claude-haiku-4-5",
+        created=1730419200,
+        owned_by="anthropic",
+        description="Alias for latest Claude Haiku 4.5",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    # ==========================================================================
+    # Google Models (2025)
+    # ==========================================================================
+    # Gemini 3 (Latest)
+    ModelInfo(
+        id="google:gemini-3-pro",
+        created=1730419200,  # November 2025
+        owned_by="google",
+        description="Most advanced Gemini. State-of-the-art reasoning, 35% better than 2.5 Pro.",
+        context_window=2000000,
+        max_output_tokens=65536,
+    ),
+    # Gemini 2.5 series
+    ModelInfo(
+        id="google:gemini-2.5-pro",
+        created=1727568000,  # September 2025
+        owned_by="google",
+        description="High-capability model with adaptive thinking. 1M context window.",
+        context_window=1000000,
+        max_output_tokens=65536,
+    ),
+    ModelInfo(
+        id="google:gemini-2.5-flash",
+        created=1727568000,
+        owned_by="google",
+        description="Fast and capable. Best for large-scale processing and agentic tasks.",
+        context_window=1000000,
+        max_output_tokens=65536,
+    ),
+    ModelInfo(
+        id="google:gemini-2.5-flash-lite",
+        created=1727568000,
+        owned_by="google",
+        description="Optimized for massive scale. Balances cost and performance.",
+        context_window=1000000,
+        max_output_tokens=32768,
+    ),
+    # Gemini 2.0
+    ModelInfo(
+        id="google:gemini-2.0-flash",
+        created=1733875200,  # December 2024
+        owned_by="google",
+        description="Fast multimodal model with native tool use.",
+        context_window=1000000,
+        max_output_tokens=8192,
+    ),
+    # Gemma open models
+    ModelInfo(
+        id="google:gemma-3",
+        created=1727568000,
+        owned_by="google",
+        description="Open model with text/image input, 140+ languages, 128K context.",
+        context_window=128000,
+        max_output_tokens=8192,
+    ),
+    ModelInfo(
+        id="google:gemma-3n",
+        created=1730419200,
+        owned_by="google",
+        description="Efficient open model for low-resource devices. Multimodal input.",
+        context_window=128000,
+        max_output_tokens=8192,
+    ),
+    # ==========================================================================
+    # Cerebras Models (Ultra-fast inference)
+    # ==========================================================================
+    ModelInfo(
+        id="cerebras:llama-3.3-70b",
+        created=1733875200,  # December 2024
+        owned_by="cerebras",
+        description="Llama 3.3 70B on Cerebras. Ultra-fast inference (~2000 tok/s). Fully compatible with structured output.",
+        context_window=128000,
+        max_output_tokens=8192,
+    ),
+    ModelInfo(
+        id="cerebras:qwen-3-32b",
+        created=1733875200,  # December 2024
+        owned_by="cerebras",
+        description="Qwen 3 32B on Cerebras. Ultra-fast inference (~2400 tok/s). Requires strict schema mode.",
+        context_window=32000,
+        max_output_tokens=8192,
+    ),
+]
+
+# Set of valid model IDs for fast O(1) lookup
+ALLOWED_MODEL_IDS: set[str] = {model.id for model in AVAILABLE_MODELS}
+
+
+def is_valid_model(model_id: str | None) -> bool:
+    """Check if a model ID is in the allowed list."""
+    if model_id is None:
+        return False
+    return model_id in ALLOWED_MODEL_IDS
+
+
+def get_valid_model_or_default(model_id: str | None, default_model: str) -> str:
+    """
+    Return the model_id if it's valid, otherwise return the default.
+
+    Args:
+        model_id: The requested model ID (may be None or invalid)
+        default_model: Fallback model from settings
+
+    Returns:
+        Valid model ID to use
+    """
+    if is_valid_model(model_id):
+        return model_id  # type: ignore[return-value]
+    return default_model
+
+
+def get_model_by_id(model_id: str) -> ModelInfo | None:
+    """
+    Get model info by ID.
+
+    Args:
+        model_id: Model identifier in provider:model format
+
+    Returns:
+        ModelInfo if found, None otherwise
+    """
+    for model in AVAILABLE_MODELS:
+        if model.id == model_id:
+            return model
+    return None

rem/agentic/providers/phoenix.py

@@ -128,15 +128,16 @@ def sanitize_tool_name(tool_name: str) -> str:
 
 
 def load_evaluator_schema(evaluator_name: str) -> dict[str, Any]:
-    """Load evaluator schema from schemas/evaluators/ directory.
+    """Load evaluator schema using centralized schema loader.
 
-    Searches for evaluator schema in rem/schemas/evaluators/
-    Supports .json, .yaml, and .yml files.
+    Uses the same unified search logic as agent schemas:
+    - "hello-world/default" → schemas/evaluators/hello-world/default.yaml
+    - "lookup-correctness" → schemas/evaluators/rem/lookup-correctness.yaml
+    - "rem-lookup-correctness" → schemas/evaluators/rem/lookup-correctness.yaml
 
     Args:
-        evaluator_name: Evaluator name (with or without extension)
-                       e.g., "rem-lookup-correctness" or
-                             "rem-lookup-correctness.yaml"
+        evaluator_name: Evaluator name or path
+                       e.g., "hello-world/default", "lookup-correctness"
 
     Returns:
         Evaluator schema dictionary with keys:
@@ -150,43 +151,13 @@ def load_evaluator_schema(evaluator_name: str) -> dict[str, Any]:
         FileNotFoundError: If evaluator schema not found
 
     Example:
-        >>> schema = load_evaluator_schema("rem-lookup-correctness")
+        >>> schema = load_evaluator_schema("hello-world/default")
         >>> print(schema["description"])
     """
-    # Get schemas directory (rem/schemas/evaluators/)
-    # rem.__file__ = rem/src/rem/__init__.py
-    # We need rem/schemas/evaluators/
-    import rem
-    rem_module_dir = Path(rem.__file__).parent  # rem/src/rem
-    rem_package_root = rem_module_dir.parent.parent  # rem/src/rem -> rem/src -> rem
-    schema_dir = rem_package_root / "schemas" / "evaluators"
-
-    # Try .yaml first (preferred format)
-    yaml_path = schema_dir / f"{evaluator_name}.yaml"
-    if yaml_path.exists():
-        logger.debug(f"Loading evaluator schema from {yaml_path}")
-        with open(yaml_path) as f:
-            return yaml.safe_load(f)
-
-    # Try .yml
-    yml_path = schema_dir / f"{evaluator_name}.yml"
-    if yml_path.exists():
-        logger.debug(f"Loading evaluator schema from {yml_path}")
-        with open(yml_path) as f:
-            return yaml.safe_load(f)
-
-    # Try .json
-    json_path = schema_dir / f"{evaluator_name}.json"
-    if json_path.exists():
-        logger.debug(f"Loading evaluator schema from {json_path}")
-        with open(json_path) as f:
-            return json.load(f)
-
-    raise FileNotFoundError(
-        f"Evaluator schema not found: {evaluator_name}\n"
-        f"Searched in: {schema_dir}\n"
-        f"Supported formats: .yaml, .yml, .json"
-    )
+    from ...utils.schema_loader import load_agent_schema
+
+    # Use centralized schema loader (searches evaluator paths too)
+    return load_agent_schema(evaluator_name)
 
 
 # =============================================================================
@@ -338,6 +309,22 @@ def create_evaluator_from_schema(
         # Already a dict
         schema = evaluator_schema_path
 
+    # Extract model from schema's provider_configs if not explicitly provided
+    if model_name is None:
+        json_schema_extra = schema.get("json_schema_extra", {})
+        provider_configs = json_schema_extra.get("provider_configs", [])
+        if provider_configs:
+            # Use first provider config
+            first_provider = provider_configs[0]
+            provider_name = first_provider.get("provider_name", "openai")
+            schema_model_name = first_provider.get("model_name", "gpt-4o-mini")
+            # Format as "provider:model" if not OpenAI (OpenAI is default)
+            if provider_name == "openai":
+                model_name = schema_model_name
+            else:
+                model_name = f"{provider_name}:{schema_model_name}"
+            logger.debug(f"Using model from schema provider_configs: {model_name}")
+
     # Create evaluator config
     evaluator_config = create_phoenix_evaluator(
         evaluator_schema=schema,
@@ -361,7 +348,8 @@ def create_evaluator_from_schema(
         Returns:
             Evaluation result with score, label, explanation
         """
-        logger.debug(f"Evaluating example: {example.get('input', '')[:100]}...")
+        input_preview = str(example.get('input', ''))[:100]
+        logger.debug(f"Evaluating example: {input_preview}...")
 
         # Phoenix llm_classify() expects a flat dict with string values
         # Build evaluation input by flattening nested dicts
@@ -393,6 +381,7 @@ def create_evaluator_from_schema(
 
         try:
             # Create single-row DataFrame for llm_classify
+            # Note: Phoenix's llm_classify requires pandas DataFrame (imported above)
             df = pd.DataFrame([eval_input])
 
             # Call Phoenix llm_classify
@@ -404,7 +393,7 @@ def create_evaluator_from_schema(
                 provide_explanation=True,
             )
 
-            # Extract result
+            # Extract result (results_df is pandas DataFrame from Phoenix)
             if not results_df.empty:
                 row = results_df.iloc[0]
                 label = row.get("label", "error")

rem/agentic/providers/pydantic_ai.py

@@ -303,6 +303,68 @@ def _prepare_schema_for_qwen(schema: dict[str, Any]) -> dict[str, Any]:
     return schema_copy
 
 
+def _convert_properties_to_prompt(properties: dict[str, Any]) -> str:
+    """
+    Convert schema properties to prompt guidance text.
+
+    When structured_output is disabled, this converts the properties
+    definition into natural language guidance that informs the agent
+    about the expected response structure without forcing JSON output.
+
+    Args:
+        properties: JSON Schema properties dict
+
+    Returns:
+        Prompt text describing the expected response elements
+
+    Example:
+        properties = {
+            "answer": {"type": "string", "description": "The answer"},
+            "confidence": {"type": "number", "description": "Confidence 0-1"}
+        }
+        # Returns:
+        # "## Response Structure\n\nYour response should include:\n- **answer**: The answer\n..."
+    """
+    if not properties:
+        return ""
+
+    lines = ["## Response Guidelines", "", "Your response should address the following elements:"]
+
+    for field_name, field_def in properties.items():
+        field_type = field_def.get("type", "any")
+        description = field_def.get("description", "")
+
+        # Format based on type
+        if field_type == "array":
+            type_hint = "list"
+        elif field_type == "number":
+            type_hint = "number"
+            # Include min/max if specified
+            if "minimum" in field_def or "maximum" in field_def:
+                min_val = field_def.get("minimum", "")
+                max_val = field_def.get("maximum", "")
+                if min_val != "" and max_val != "":
+                    type_hint = f"number ({min_val}-{max_val})"
+        elif field_type == "boolean":
+            type_hint = "yes/no"
+        else:
+            type_hint = field_type
+
+        # Build field description
+        field_line = f"- **{field_name}**"
+        if type_hint and type_hint != "string":
+            field_line += f" ({type_hint})"
+        if description:
+            field_line += f": {description}"
+
+        lines.append(field_line)
+
+    lines.append("")
+    lines.append("Respond naturally in prose, addressing these elements where relevant.")
+
+    return "\n".join(lines)
+
+
 def _create_schema_wrapper(
     result_type: type[BaseModel], strip_description: bool = True
 ) -> type[BaseModel]:
@@ -462,10 +524,11 @@ async def create_agent(
         # agent_schema = load_agent_schema(context.agent_schema_uri)
         pass
 
-    # Determine model: override > context.default_model > settings
-    model = (
-        model_override or (context.default_model if context else settings.llm.default_model)
-    )
+    # Determine model: validate override against allowed list, fallback to context or settings
+    from rem.agentic.llm_provider_models import get_valid_model_or_default
+
+    default_model = context.default_model if context else settings.llm.default_model
+    model = get_valid_model_or_default(model_override, default_model)
 
     # Extract schema fields
     system_prompt = agent_schema.get("description", "") if agent_schema else ""
@@ -526,14 +589,25 @@ async def create_agent(
         # TODO: Convert resources to tools (MCP convenience syntax)
         pass
 
+    # Check if structured output is disabled for this schema
+    # When structured_output: false, properties become part of prompt instead of output_type
+    use_structured_output = metadata.get("structured_output", True)
+
     # Create dynamic result_type from schema if not provided
     if result_type is None and agent_schema and "properties" in agent_schema:
-        # Pre-process schema for Qwen compatibility (strips min/max, sets additionalProperties=False)
-        # This ensures the generated Pydantic model doesn't have incompatible constraints
-        sanitized_schema = _prepare_schema_for_qwen(agent_schema)
-        result_type = _create_model_from_schema(sanitized_schema)
-        logger.debug(f"Created dynamic Pydantic model: {result_type.__name__}")
-        logger.debug(f"Created dynamic Pydantic model: {result_type.__name__}")
+        if use_structured_output:
+            # Pre-process schema for Qwen compatibility (strips min/max, sets additionalProperties=False)
+            # This ensures the generated Pydantic model doesn't have incompatible constraints
+            sanitized_schema = _prepare_schema_for_qwen(agent_schema)
+            result_type = _create_model_from_schema(sanitized_schema)
+            logger.debug(f"Created dynamic Pydantic model: {result_type.__name__}")
+        else:
+            # Convert properties to prompt guidance instead of structured output
+            # This informs the agent about expected response structure without forcing it
+            properties_prompt = _convert_properties_to_prompt(agent_schema.get("properties", {}))
+            if properties_prompt:
+                system_prompt = system_prompt + "\n\n" + properties_prompt
+            logger.debug("Structured output disabled - properties converted to prompt guidance")
 
     # Create agent with optional output_type for structured output and tools
     if result_type: