remdb 0.3.0__py3-none-any.whl → 0.3.114__py3-none-any.whl

rem/agentic/llm_provider_models.py

@@ -0,0 +1,301 @@
+"""
+LLM Provider Model Registry.
+
+Defines available LLM models across providers (OpenAI, Anthropic, Google, Cerebras).
+Used by the models API endpoint and for validating model requests.
+
+Future: Models will be stored in database for dynamic management.
+"""
+
+from pydantic import BaseModel, Field
+from typing import Literal
+
+
+class ModelInfo(BaseModel):
+    """Information about a single model."""
+
+    id: str = Field(description="Model ID in provider:model format")
+    object: Literal["model"] = "model"
+    created: int = Field(description="Unix timestamp of model availability")
+    owned_by: str = Field(description="Provider name")
+    description: str | None = Field(default=None, description="Model description")
+    context_window: int | None = Field(default=None, description="Max context tokens")
+    max_output_tokens: int | None = Field(default=None, description="Max output tokens")
+
+
+# Model definitions with 2025 releases
+# Using Unix timestamps for created dates (approximate release dates)
+AVAILABLE_MODELS: list[ModelInfo] = [
+    # ==========================================================================
+    # OpenAI Models (2025)
+    # ==========================================================================
+    # GPT-4.1 series (Released April 14, 2025)
+    ModelInfo(
+        id="openai:gpt-4.1",
+        created=1744588800,  # April 14, 2025
+        owned_by="openai",
+        description="Latest GPT-4 iteration, excels at coding and instruction following. 1M context.",
+        context_window=1047576,
+        max_output_tokens=32768,
+    ),
+    ModelInfo(
+        id="openai:gpt-4.1-mini",
+        created=1744588800,
+        owned_by="openai",
+        description="Small model beating GPT-4o in many benchmarks. 83% cost reduction vs GPT-4o.",
+        context_window=1047576,
+        max_output_tokens=32768,
+    ),
+    ModelInfo(
+        id="openai:gpt-4.1-nano",
+        created=1744588800,
+        owned_by="openai",
+        description="Fastest and cheapest OpenAI model. Ideal for classification and autocompletion.",
+        context_window=1047576,
+        max_output_tokens=32768,
+    ),
+    # GPT-4o (legacy but still supported)
+    ModelInfo(
+        id="openai:gpt-4o",
+        created=1715644800,  # May 13, 2024
+        owned_by="openai",
+        description="Previous flagship multimodal model. Being superseded by GPT-4.1.",
+        context_window=128000,
+        max_output_tokens=16384,
+    ),
+    ModelInfo(
+        id="openai:gpt-4o-mini",
+        created=1721347200,  # July 18, 2024
+        owned_by="openai",
+        description="Cost-efficient smaller GPT-4o variant.",
+        context_window=128000,
+        max_output_tokens=16384,
+    ),
+    # o1 reasoning models
+    ModelInfo(
+        id="openai:o1",
+        created=1733961600,  # December 12, 2024
+        owned_by="openai",
+        description="Advanced reasoning model for complex problems. Extended thinking.",
+        context_window=200000,
+        max_output_tokens=100000,
+    ),
+    ModelInfo(
+        id="openai:o1-mini",
+        created=1726099200,  # September 12, 2024
+        owned_by="openai",
+        description="Smaller reasoning model, fast for coding and math.",
+        context_window=128000,
+        max_output_tokens=65536,
+    ),
+    ModelInfo(
+        id="openai:o3-mini",
+        created=1738195200,  # January 30, 2025
+        owned_by="openai",
+        description="Latest mini reasoning model with improved performance.",
+        context_window=200000,
+        max_output_tokens=100000,
+    ),
+    # ==========================================================================
+    # Anthropic Models (2025)
+    # ==========================================================================
+    # Claude 4.5 series (Latest - November 2025)
+    ModelInfo(
+        id="anthropic:claude-opus-4-5-20251124",
+        created=1732406400,  # November 24, 2025
+        owned_by="anthropic",
+        description="Most capable Claude model. World-class coding with 'effort' parameter control.",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    ModelInfo(
+        id="anthropic:claude-sonnet-4-5-20250929",
+        created=1727568000,  # September 29, 2025
+        owned_by="anthropic",
+        description="Best balance of intelligence and speed. Excellent for coding and agents.",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    ModelInfo(
+        id="anthropic:claude-haiku-4-5-20251101",
+        created=1730419200,  # November 1, 2025
+        owned_by="anthropic",
+        description="Fast and affordable. Sonnet 4 performance at 1/3 cost. Safest Claude model.",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    # Claude 4 series
+    ModelInfo(
+        id="anthropic:claude-opus-4-20250514",
+        created=1715644800,  # May 14, 2025
+        owned_by="anthropic",
+        description="World's best coding model. Sustained performance on complex agent workflows.",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    ModelInfo(
+        id="anthropic:claude-sonnet-4-20250514",
+        created=1715644800,  # May 14, 2025
+        owned_by="anthropic",
+        description="Significant upgrade to Sonnet 3.7. Great for everyday tasks.",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    ModelInfo(
+        id="anthropic:claude-opus-4-1-20250805",
+        created=1722816000,  # August 5, 2025
+        owned_by="anthropic",
+        description="Opus 4 upgrade focused on agentic tasks and real-world coding.",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    # Aliases for convenience
+    ModelInfo(
+        id="anthropic:claude-opus-4-5",
+        created=1732406400,
+        owned_by="anthropic",
+        description="Alias for latest Claude Opus 4.5",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    ModelInfo(
+        id="anthropic:claude-sonnet-4-5",
+        created=1727568000,
+        owned_by="anthropic",
+        description="Alias for latest Claude Sonnet 4.5",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    ModelInfo(
+        id="anthropic:claude-haiku-4-5",
+        created=1730419200,
+        owned_by="anthropic",
+        description="Alias for latest Claude Haiku 4.5",
+        context_window=200000,
+        max_output_tokens=128000,
+    ),
+    # ==========================================================================
+    # Google Models (2025)
+    # ==========================================================================
+    # Gemini 3 (Latest)
+    ModelInfo(
+        id="google:gemini-3-pro",
+        created=1730419200,  # November 2025
+        owned_by="google",
+        description="Most advanced Gemini. State-of-the-art reasoning, 35% better than 2.5 Pro.",
+        context_window=2000000,
+        max_output_tokens=65536,
+    ),
+    # Gemini 2.5 series
+    ModelInfo(
+        id="google:gemini-2.5-pro",
+        created=1727568000,  # September 2025
+        owned_by="google",
+        description="High-capability model with adaptive thinking. 1M context window.",
+        context_window=1000000,
+        max_output_tokens=65536,
+    ),
+    ModelInfo(
+        id="google:gemini-2.5-flash",
+        created=1727568000,
+        owned_by="google",
+        description="Fast and capable. Best for large-scale processing and agentic tasks.",
+        context_window=1000000,
+        max_output_tokens=65536,
+    ),
+    ModelInfo(
+        id="google:gemini-2.5-flash-lite",
+        created=1727568000,
+        owned_by="google",
+        description="Optimized for massive scale. Balances cost and performance.",
+        context_window=1000000,
+        max_output_tokens=32768,
+    ),
+    # Gemini 2.0
+    ModelInfo(
+        id="google:gemini-2.0-flash",
+        created=1733875200,  # December 2024
+        owned_by="google",
+        description="Fast multimodal model with native tool use.",
+        context_window=1000000,
+        max_output_tokens=8192,
+    ),
+    # Gemma open models
+    ModelInfo(
+        id="google:gemma-3",
+        created=1727568000,
+        owned_by="google",
+        description="Open model with text/image input, 140+ languages, 128K context.",
+        context_window=128000,
+        max_output_tokens=8192,
+    ),
+    ModelInfo(
+        id="google:gemma-3n",
+        created=1730419200,
+        owned_by="google",
+        description="Efficient open model for low-resource devices. Multimodal input.",
+        context_window=128000,
+        max_output_tokens=8192,
+    ),
+    # ==========================================================================
+    # Cerebras Models (Ultra-fast inference)
+    # ==========================================================================
+    ModelInfo(
+        id="cerebras:llama-3.3-70b",
+        created=1733875200,  # December 2024
+        owned_by="cerebras",
+        description="Llama 3.3 70B on Cerebras. Ultra-fast inference (~2000 tok/s). Fully compatible with structured output.",
+        context_window=128000,
+        max_output_tokens=8192,
+    ),
+    ModelInfo(
+        id="cerebras:qwen-3-32b",
+        created=1733875200,  # December 2024
+        owned_by="cerebras",
+        description="Qwen 3 32B on Cerebras. Ultra-fast inference (~2400 tok/s). Requires strict schema mode.",
+        context_window=32000,
+        max_output_tokens=8192,
+    ),
+]
+
+# Set of valid model IDs for fast O(1) lookup
+ALLOWED_MODEL_IDS: set[str] = {model.id for model in AVAILABLE_MODELS}
+
+
+def is_valid_model(model_id: str | None) -> bool:
+    """Check if a model ID is in the allowed list."""
+    if model_id is None:
+        return False
+    return model_id in ALLOWED_MODEL_IDS
+
+
+def get_valid_model_or_default(model_id: str | None, default_model: str) -> str:
+    """
+    Return the model_id if it's valid, otherwise return the default.
+
+    Args:
+        model_id: The requested model ID (may be None or invalid)
+        default_model: Fallback model from settings
+
+    Returns:
+        Valid model ID to use
+    """
+    if is_valid_model(model_id):
+        return model_id  # type: ignore[return-value]
+    return default_model
+
+
+def get_model_by_id(model_id: str) -> ModelInfo | None:
+    """
+    Get model info by ID.
+
+    Args:
+        model_id: Model identifier in provider:model format
+
+    Returns:
+        ModelInfo if found, None otherwise
+    """
+    for model in AVAILABLE_MODELS:
+        if model.id == model_id:
+            return model
+    return None

rem/agentic/otel/setup.py

@@ -14,6 +14,7 @@ from loguru import logger
 
 from ...settings import settings
 
+
 # Global flag to track if instrumentation is initialized
 _instrumentation_initialized = False
 
@@ -52,12 +53,94 @@ def setup_instrumentation() -> None:
 
     try:
         from opentelemetry import trace
-        from opentelemetry.sdk.trace import TracerProvider
-        from opentelemetry.sdk.trace.export import BatchSpanProcessor
+        from opentelemetry.sdk.trace import TracerProvider, ReadableSpan
+        from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter, SpanExportResult
         from opentelemetry.sdk.resources import Resource, SERVICE_NAME, DEPLOYMENT_ENVIRONMENT
         from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter as HTTPExporter
         from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter as GRPCExporter
 
+        class SanitizingSpanExporter(SpanExporter):
+            """
+            Wrapper exporter that sanitizes span attributes before export.
+
+            Removes None values that cause OTLP encoding failures like:
+            - llm.input_messages.3.message.content: None
+            """
+
+            def __init__(self, wrapped_exporter: SpanExporter):
+                self._wrapped = wrapped_exporter
+
+            def _sanitize_value(self, value):
+                """Recursively sanitize a value, replacing None with empty string."""
+                if value is None:
+                    return ""  # Replace None with empty string
+                if isinstance(value, dict):
+                    return {k: self._sanitize_value(v) for k, v in value.items()}
+                if isinstance(value, (list, tuple)):
+                    return [self._sanitize_value(v) for v in value]
+                return value
+
+            def export(self, spans: tuple[ReadableSpan, ...]) -> SpanExportResult:
+                # Create sanitized copies of spans
+                sanitized_spans = []
+                for span in spans:
+                    if span.attributes:
+                        # Sanitize all attribute values - replace None with empty string
+                        sanitized_attrs = {}
+                        for k, v in span.attributes.items():
+                            sanitized_attrs[k] = self._sanitize_value(v)
+                        sanitized_spans.append(_SanitizedSpan(span, sanitized_attrs))
+                    else:
+                        sanitized_spans.append(span)
+
+                return self._wrapped.export(tuple(sanitized_spans))
+
+            def shutdown(self) -> None:
+                self._wrapped.shutdown()
+
+            def force_flush(self, timeout_millis: int = 30000) -> bool:
+                return self._wrapped.force_flush(timeout_millis)
+
+        class _SanitizedSpan(ReadableSpan):
+            """ReadableSpan wrapper with sanitized attributes."""
+
+            def __init__(self, original: ReadableSpan, sanitized_attributes: dict):
+                self._original = original
+                self._sanitized_attributes = sanitized_attributes
+
+            @property
+            def name(self): return self._original.name
+            @property
+            def context(self): return self._original.context
+            @property
+            def parent(self): return self._original.parent
+            @property
+            def resource(self): return self._original.resource
+            @property
+            def instrumentation_scope(self): return self._original.instrumentation_scope
+            @property
+            def status(self): return self._original.status
+            @property
+            def start_time(self): return self._original.start_time
+            @property
+            def end_time(self): return self._original.end_time
+            @property
+            def links(self): return self._original.links
+            @property
+            def events(self): return self._original.events
+            @property
+            def kind(self): return self._original.kind
+            @property
+            def attributes(self): return self._sanitized_attributes
+            @property
+            def dropped_attributes(self): return self._original.dropped_attributes
+            @property
+            def dropped_events(self): return self._original.dropped_events
+            @property
+            def dropped_links(self): return self._original.dropped_links
+
+            def get_span_context(self): return self._original.get_span_context()
+
         # Create resource with service metadata
         resource = Resource(
             attributes={
@@ -72,16 +155,19 @@ def setup_instrumentation() -> None:
 
         # Configure OTLP exporter based on protocol
         if settings.otel.protocol == "grpc":
-            exporter = GRPCExporter(
+            base_exporter = GRPCExporter(
                 endpoint=settings.otel.collector_endpoint,
                 timeout=settings.otel.export_timeout,
             )
         else:  # http
-            exporter = HTTPExporter(
+            base_exporter = HTTPExporter(
                 endpoint=f"{settings.otel.collector_endpoint}/v1/traces",
                 timeout=settings.otel.export_timeout,
             )
 
+        # Wrap with sanitizing exporter to handle None values
+        exporter = SanitizingSpanExporter(base_exporter)
+
         # Add span processor
         tracer_provider.add_span_processor(BatchSpanProcessor(exporter))
 
@@ -95,6 +181,8 @@ def setup_instrumentation() -> None:
         # Add OpenInference span processor for Pydantic AI
         # This adds rich attributes (openinference.span.kind, input/output, etc.) to ALL traces
         # Phoenix receives these traces via the OTLP collector - no separate "Phoenix integration" needed
+        # Note: The OTEL exporter may log warnings about None values in tool call messages,
+        # but this is a known limitation in openinference-instrumentation-pydantic-ai
         try:
             from openinference.instrumentation.pydantic_ai import OpenInferenceSpanProcessor as PydanticAISpanProcessor
 

rem/agentic/providers/phoenix.py

@@ -128,15 +128,16 @@ def sanitize_tool_name(tool_name: str) -> str:
 
 
 def load_evaluator_schema(evaluator_name: str) -> dict[str, Any]:
-    """Load evaluator schema from schemas/evaluators/ directory.
+    """Load evaluator schema using centralized schema loader.
 
-    Searches for evaluator schema in rem/schemas/evaluators/
-    Supports .json, .yaml, and .yml files.
+    Uses the same unified search logic as agent schemas:
+    - "hello-world/default" → schemas/evaluators/hello-world/default.yaml
+    - "lookup-correctness" → schemas/evaluators/rem/lookup-correctness.yaml
+    - "rem-lookup-correctness" → schemas/evaluators/rem/lookup-correctness.yaml
 
     Args:
-        evaluator_name: Evaluator name (with or without extension)
-                       e.g., "rem-lookup-correctness" or
-                             "rem-lookup-correctness.yaml"
+        evaluator_name: Evaluator name or path
+                       e.g., "hello-world/default", "lookup-correctness"
 
     Returns:
         Evaluator schema dictionary with keys:
@@ -150,43 +151,13 @@ def load_evaluator_schema(evaluator_name: str) -> dict[str, Any]:
         FileNotFoundError: If evaluator schema not found
 
     Example:
-        >>> schema = load_evaluator_schema("rem-lookup-correctness")
+        >>> schema = load_evaluator_schema("hello-world/default")
         >>> print(schema["description"])
     """
-    # Get schemas directory (rem/schemas/evaluators/)
-    # rem.__file__ = rem/src/rem/__init__.py
-    # We need rem/schemas/evaluators/
-    import rem
-    rem_module_dir = Path(rem.__file__).parent  # rem/src/rem
-    rem_package_root = rem_module_dir.parent.parent  # rem/src/rem -> rem/src -> rem
-    schema_dir = rem_package_root / "schemas" / "evaluators"
-
-    # Try .yaml first (preferred format)
-    yaml_path = schema_dir / f"{evaluator_name}.yaml"
-    if yaml_path.exists():
-        logger.debug(f"Loading evaluator schema from {yaml_path}")
-        with open(yaml_path) as f:
-            return yaml.safe_load(f)
-
-    # Try .yml
-    yml_path = schema_dir / f"{evaluator_name}.yml"
-    if yml_path.exists():
-        logger.debug(f"Loading evaluator schema from {yml_path}")
-        with open(yml_path) as f:
-            return yaml.safe_load(f)
-
-    # Try .json
-    json_path = schema_dir / f"{evaluator_name}.json"
-    if json_path.exists():
-        logger.debug(f"Loading evaluator schema from {json_path}")
-        with open(json_path) as f:
-            return json.load(f)
-
-    raise FileNotFoundError(
-        f"Evaluator schema not found: {evaluator_name}\n"
-        f"Searched in: {schema_dir}\n"
-        f"Supported formats: .yaml, .yml, .json"
-    )
+    from ...utils.schema_loader import load_agent_schema
+
+    # Use centralized schema loader (searches evaluator paths too)
+    return load_agent_schema(evaluator_name)
 
 
 # =============================================================================
@@ -338,6 +309,22 @@ def create_evaluator_from_schema(
         # Already a dict
         schema = evaluator_schema_path
 
+    # Extract model from schema's provider_configs if not explicitly provided
+    if model_name is None:
+        json_schema_extra = schema.get("json_schema_extra", {})
+        provider_configs = json_schema_extra.get("provider_configs", [])
+        if provider_configs:
+            # Use first provider config
+            first_provider = provider_configs[0]
+            provider_name = first_provider.get("provider_name", "openai")
+            schema_model_name = first_provider.get("model_name", "gpt-4o-mini")
+            # Format as "provider:model" if not OpenAI (OpenAI is default)
+            if provider_name == "openai":
+                model_name = schema_model_name
+            else:
+                model_name = f"{provider_name}:{schema_model_name}"
+            logger.debug(f"Using model from schema provider_configs: {model_name}")
+
     # Create evaluator config
     evaluator_config = create_phoenix_evaluator(
         evaluator_schema=schema,
@@ -361,7 +348,8 @@ def create_evaluator_from_schema(
         Returns:
             Evaluation result with score, label, explanation
         """
-        logger.debug(f"Evaluating example: {example.get('input', '')[:100]}...")
+        input_preview = str(example.get('input', ''))[:100]
+        logger.debug(f"Evaluating example: {input_preview}...")
 
         # Phoenix llm_classify() expects a flat dict with string values
         # Build evaluation input by flattening nested dicts
@@ -393,6 +381,7 @@ def create_evaluator_from_schema(
 
         try:
             # Create single-row DataFrame for llm_classify
+            # Note: Phoenix's llm_classify requires pandas DataFrame (imported above)
             df = pd.DataFrame([eval_input])
 
             # Call Phoenix llm_classify
@@ -404,7 +393,7 @@ def create_evaluator_from_schema(
                 provide_explanation=True,
             )
 
-            # Extract result
+            # Extract result (results_df is pandas DataFrame from Phoenix)
             if not results_df.empty:
                 row = results_df.iloc[0]
                 label = row.get("label", "error")