opik 1.9.39__py3-none-any.whl → 1.9.86__py3-none-any.whl

opik/rest_api/types/span_user_defined_metric_python_code_public.py

@@ -0,0 +1,20 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+import pydantic
+from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
+
+
+class SpanUserDefinedMetricPythonCodePublic(UniversalBaseModel):
+    metric: str
+    arguments: typing.Dict[str, str]
+
+    if IS_PYDANTIC_V2:
+        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
+    else:
+
+        class Config:
+            frozen = True
+            smart_union = True
+            extra = pydantic.Extra.allow

opik/rest_api/types/span_user_defined_metric_python_code_write.py

@@ -0,0 +1,20 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+import pydantic
+from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
+
+
+class SpanUserDefinedMetricPythonCodeWrite(UniversalBaseModel):
+    metric: str
+    arguments: typing.Dict[str, str]
+
+    if IS_PYDANTIC_V2:
+        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
+    else:
+
+        class Config:
+            frozen = True
+            smart_union = True
+            extra = pydantic.Extra.allow

opik/types.py

@@ -2,6 +2,7 @@ import enum
 import sys
 from typing import Literal, Optional
 
+from pydantic import StrictStr
 from typing_extensions import TypedDict
 
 if sys.version_info < (3, 11):
@@ -79,6 +80,41 @@ class FeedbackScoreDict(TypedDict):
     """An optional explanation or justification for the given score."""
 
 
+class BatchFeedbackScoreDict(TypedDict):
+    """
+    A TypedDict representing a feedback score for batch operations.
+
+    This class defines the structure for feedback scores used in batch logging
+    operations, with a required id field and optional per-score project_name.
+    """
+
+    id: Required[str]
+    """
+    A unique identifier for the object this score should be assigned to.
+    Refers to either the trace_id, span_id or thread_id depending on how the score is logged.
+    Required for batch operations.
+    """
+
+    name: Required[str]
+    """The name of the feedback metric or criterion."""
+
+    value: Required[float]
+    """The numerical value of the feedback score."""
+
+    project_name: NotRequired[Optional[StrictStr]]
+    """
+    The name of the project for this specific score.
+    If not provided, falls back to the project_name parameter in the method call,
+    or the default project name configured in the Opik instance.
+    """
+
+    category_name: NotRequired[Optional[str]]
+    """An optional category name for the given score."""
+
+    reason: NotRequired[Optional[str]]
+    """An optional explanation or justification for the given score."""
+
+
 class ErrorInfoDict(TypedDict):
     """
     A TypedDict representing the information about the error occurred.

opik/validation/chat_prompt_messages.py

@@ -0,0 +1,241 @@
+from typing import Any, List, Optional
+
+import opik.exceptions as exceptions
+from . import validator, result
+
+
+class ChatPromptMessagesValidator(validator.RaisableValidator):
+    """
+    Validator for ChatPrompt messages list.
+
+    Validates that messages is a list of dicts with:
+    - "role" key with value "system", "user", or "assistant"
+    - "content" key with value either string or list of dicts
+    - If content is list of dicts, each dict must have "type" key
+    """
+
+    VALID_ROLES = {"system", "user", "assistant"}
+    URL_BASED_CONTENT_TYPES = {"image_url", "video_url", "audio_url"}
+
+    def __init__(self, messages: Any):
+        self.messages = messages
+        self.validation_result: Optional[result.ValidationResult] = None
+
+    def validate(self) -> result.ValidationResult:
+        failure_reasons: List[str] = []
+
+        # Validate messages is a list
+        if not self._validate_messages_is_list(failure_reasons):
+            self.validation_result = result.ValidationResult(
+                failed=True, failure_reasons=failure_reasons
+            )
+            return self.validation_result
+
+        # Validate each message in the list
+        for idx, message in enumerate(self.messages):
+            prefix = f"messages[{idx}]"
+            self._validate_message(prefix, message, failure_reasons)
+
+        # Create validation result
+        if len(failure_reasons) > 0:
+            self.validation_result = result.ValidationResult(
+                failed=True, failure_reasons=failure_reasons
+            )
+        else:
+            self.validation_result = result.ValidationResult(failed=False)
+
+        return self.validation_result
+
+    def _validate_messages_is_list(self, failure_reasons: List[str]) -> bool:
+        """Validate that messages is a list. Returns False if validation fails."""
+        if not isinstance(self.messages, list):
+            msg = (
+                f"messages must be a list but {type(self.messages).__name__} was given"
+            )
+            failure_reasons.append(msg)
+            return False
+        return True
+
+    def _validate_message(
+        self, prefix: str, message: Any, failure_reasons: List[str]
+    ) -> None:
+        """Validate a single message structure, role, and content."""
+        if not self._validate_message_structure(prefix, message, failure_reasons):
+            return
+
+        self._validate_role(prefix, message, failure_reasons)
+        self._validate_content(prefix, message, failure_reasons)
+
+    def _validate_message_structure(
+        self, prefix: str, message: Any, failure_reasons: List[str]
+    ) -> bool:
+        """Validate that message is a dict with exactly 'role' and 'content' keys. Returns False if validation fails."""
+        # Validate message is a dict
+        if not isinstance(message, dict):
+            msg = f"{prefix}: must be a dict but {type(message).__name__} was given"
+            failure_reasons.append(msg)
+            return False
+
+        # Validate message has exactly "role" and "content" keys
+        message_keys = set(message.keys())
+        expected_keys = {"role", "content"}
+
+        if message_keys != expected_keys:
+            if not message_keys.issubset(expected_keys):
+                missing_keys = expected_keys - message_keys
+                msg = f"{prefix}: missing required keys: {sorted(missing_keys)}"
+                failure_reasons.append(msg)
+            if not expected_keys.issubset(message_keys):
+                extra_keys = message_keys - expected_keys
+                msg = (
+                    f"{prefix}: unexpected keys: {sorted(extra_keys)}. "
+                    f"Expected only: {sorted(expected_keys)}"
+                )
+                failure_reasons.append(msg)
+            return False
+
+        return True
+
+    def _validate_role(
+        self, prefix: str, message: dict, failure_reasons: List[str]
+    ) -> None:
+        """Validate the role field of a message."""
+        role = message.get("role")
+        if role not in self.VALID_ROLES:
+            valid_roles_str = ", ".join([f"'{r}'" for r in sorted(self.VALID_ROLES)])
+            msg = (
+                f"{prefix}.role: must be one of [{valid_roles_str}] "
+                f"but {repr(role)} was given"
+            )
+            failure_reasons.append(msg)
+
+    def _validate_content(
+        self, prefix: str, message: dict, failure_reasons: List[str]
+    ) -> None:
+        """Validate the content field of a message."""
+        content = message.get("content")
+        if content is None:
+            msg = f"{prefix}.content: must not be None"
+            failure_reasons.append(msg)
+        elif not isinstance(content, (str, list)):
+            msg = (
+                f"{prefix}.content: must be either str or list of dicts "
+                f"but {type(content).__name__} was given"
+            )
+            failure_reasons.append(msg)
+        elif isinstance(content, list):
+            self._validate_content_list(prefix, content, failure_reasons)
+
+    def _validate_content_list(
+        self, prefix: str, content: list, failure_reasons: List[str]
+    ) -> None:
+        """Validate content when it is a list of content parts."""
+        for content_idx, content_part in enumerate(content):
+            content_prefix = f"{prefix}.content[{content_idx}]"
+            self._validate_content_part(content_prefix, content_part, failure_reasons)
+
+    def _validate_content_part(
+        self, content_prefix: str, content_part: Any, failure_reasons: List[str]
+    ) -> None:
+        """Validate a single content part in the content list."""
+        if not isinstance(content_part, dict):
+            msg = (
+                f"{content_prefix}: must be a dict "
+                f"but {type(content_part).__name__} was given"
+            )
+            failure_reasons.append(msg)
+            return
+
+        if "type" not in content_part:
+            msg = f"{content_prefix}: must have 'type' key"
+            failure_reasons.append(msg)
+            return
+
+        # Validate type-specific requirements
+        content_type = content_part.get("type")
+        self._validate_content_type_specific(
+            content_prefix, content_type, content_part, failure_reasons
+        )
+
+    def _validate_content_type_specific(
+        self,
+        content_prefix: str,
+        content_type: Any,
+        content_part: dict,
+        failure_reasons: List[str],
+    ) -> None:
+        """Validate type-specific requirements for content parts."""
+        if content_type in self.URL_BASED_CONTENT_TYPES:
+            self._validate_required_url_object(
+                content_prefix,
+                content_part,
+                content_type,
+                content_type,
+                failure_reasons,
+            )
+        elif content_type == "text":
+            self._validate_required_string_key(
+                content_prefix, content_part, "text", "text", failure_reasons
+            )
+
+    def _validate_required_string_key(
+        self,
+        prefix: str,
+        content_part: dict,
+        key_name: str,
+        type_name: str,
+        failure_reasons: List[str],
+    ) -> None:
+        """Validate that a required key exists and is a string."""
+        if key_name not in content_part:
+            msg = f"{prefix}: must have '{key_name}' key when type is '{type_name}'"
+            failure_reasons.append(msg)
+        elif not isinstance(content_part.get(key_name), str):
+            msg = (
+                f"{prefix}.{key_name}: must be a string "
+                f"but {type(content_part.get(key_name)).__name__} was given"
+            )
+            failure_reasons.append(msg)
+
+    def _validate_required_url_object(
+        self,
+        prefix: str,
+        content_part: dict,
+        key_name: str,
+        type_name: str,
+        failure_reasons: List[str],
+    ) -> None:
+        """Validate that a required key exists and is a dict with a 'url' key that is a string."""
+        if key_name not in content_part:
+            msg = f"{prefix}: must have '{key_name}' key when type is '{type_name}'"
+            failure_reasons.append(msg)
+            return
+
+        url_object = content_part.get(key_name)
+        if not isinstance(url_object, dict):
+            msg = (
+                f"{prefix}.{key_name}: must be a dict "
+                f"but {type(url_object).__name__} was given"
+            )
+            failure_reasons.append(msg)
+            return
+
+        if "url" not in url_object:
+            msg = f"{prefix}.{key_name}: must have 'url' key"
+            failure_reasons.append(msg)
+        elif not isinstance(url_object.get("url"), str):
+            msg = (
+                f"{prefix}.{key_name}.url: must be a string "
+                f"but {type(url_object.get('url')).__name__} was given"
+            )
+            failure_reasons.append(msg)
+
+    def raise_if_validation_failed(self) -> None:
+        if (
+            self.validation_result is not None
+            and len(self.validation_result.failure_reasons) > 0
+        ):
+            raise exceptions.ValidationError(
+                prefix="ChatPrompt.__init__",
+                failure_reasons=self.validation_result.failure_reasons,
+            )

opik/validation/feedback_score.py

@@ -1,16 +1,16 @@
 import pydantic
 
 from typing import Any
-from ..types import FeedbackScoreDict
+from ..types import BatchFeedbackScoreDict
 from . import validator, result
 
 
 class PydanticWrapper(pydantic.BaseModel):
     model_config = pydantic.ConfigDict(extra="forbid")
-    feedback_score: FeedbackScoreDict
+    feedback_score: BatchFeedbackScoreDict
 
 
-EXPECTED_TYPES = "{'id': str, 'name': str, 'value': float, 'reason': NotRequired[str], 'category_name': NotRequired[str]}"
+EXPECTED_TYPES = "{'id': str, 'name': str, 'value': float, 'reason': NotRequired[str], 'category_name': NotRequired[str], 'project_name': NotRequired[str]}"
 
 
 class FeedbackScoreValidator(validator.Validator):

opik/validation/validator.py

@@ -1,4 +1,5 @@
 import abc
+from typing import Any
 
 from . import result
 
@@ -7,3 +8,30 @@ class Validator(abc.ABC):
     @abc.abstractmethod
     def validate(self) -> result.ValidationResult:
         pass
+
+
+class RaisableValidator(Validator):
+    """
+    Abstract validator class that extends Validator and adds raise_if_validation_failed method.
+
+    This is used for validators that need to raise ValidationError exceptions
+    when validation fails, typically used in class initialization.
+    """
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """
+        Initialize the validator.
+
+        Subclasses can override this method with their own initialization signature.
+        """
+        pass
+
+    @abc.abstractmethod
+    def raise_if_validation_failed(self) -> None:
+        """
+        Raise a ValidationError if validation failed.
+
+        This method should check the validation result and raise an appropriate
+        ValidationError exception if validation failed.
+        """
+        pass

{opik-1.9.39.dist-info → opik-1.9.86.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opik
-Version: 1.9.39
+Version: 1.9.86
 Summary: Comet tool for logging and evaluating LLM traces
 Home-page: https://www.comet.com
 Author: Comet ML Inc.
@@ -14,13 +14,12 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Natural Language :: English
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: boto3-stubs[bedrock-runtime]>=1.34.110
@@ -70,9 +69,9 @@ Dynamic: summary
         Opik
     </div>
 </h1>
-<h2 align="center" style="border-bottom: none">Open-source LLM evaluation platform</h2>
+<h2 align="center" style="border-bottom: none">Open-source AI Observability, Evaluation, and Optimization</h2>
 <p align="center">
-Opik helps you build, evaluate, and optimize LLM systems that run better, faster, and cheaper. From RAG chatbots to code assistants to complex agentic pipelines, Opik provides comprehensive tracing, evaluations, dashboards, and powerful features like <b>Opik Agent Optimizer</b> and <b>Opik Guardrails</b> to improve and secure your LLM powered applications in production.
+Opik helps you build, test, and optimize generative AI application that run better, from prototype to production.  From RAG chatbots to code assistants to complex agentic systems, Opik provides comprehensive tracing, evaluation, and automatic prompt and tool optimization to take the guesswork out of AI development.
 </p>
 
 <div align="center">
@@ -215,7 +214,7 @@ For production or larger-scale self-hosted deployments, Opik can be installed on
 
 ## 💻 Opik Client SDK
 
-Opik provides a suite of client libraries and a REST API to interact with the Opik server. This includes SDKs for Python, TypeScript, and Ruby (via OpenTelemetry), allowing for seamless integration into your workflows. For detailed API and SDK references, see the [Opik Client Reference Documentation](apps/opik-documentation/documentation/fern/docs/reference/overview.mdx).
+Opik provides a suite of client libraries and a REST API to interact with the Opik server. This includes SDKs for Python, TypeScript, and Ruby (via OpenTelemetry), allowing for seamless integration into your workflows. For detailed API and SDK references, see the [Opik Client Reference Documentation](https://www.comet.com/docs/opik/reference/overview?from=llm&utm_source=opik&utm_medium=github&utm_content=reference_link&utm_campaign=opik).
 
 ### Python SDK Quick Start
 
@@ -238,7 +237,7 @@ opik configure
 ```
 
 > [!TIP]
-> You can also call `opik.configure(use_local=True)` from your Python code to configure the SDK to run on a local self-hosted installation, or provide API key and workspace details directly for Comet.com. Refer to the [Python SDK documentation](apps/opik-documentation/documentation/fern/docs/reference/python-sdk/) for more configuration options.
+> You can also call `opik.configure(use_local=True)` from your Python code to configure the SDK to run on a local self-hosted installation, or provide API key and workspace details directly for Comet.com. Refer to the [Python SDK documentation](https://www.comet.com/docs/opik/python-sdk-reference/?from=llm&utm_source=opik&utm_medium=github&utm_content=python_sdk_docs_link&utm_campaign=opik) for more configuration options.
 
 You are now ready to start logging traces using the [Python SDK](https://www.comet.com/docs/opik/python-sdk-reference/?from=llm&utm_source=opik&utm_medium=github&utm_content=sdk_link2&utm_campaign=opik).
 
@@ -272,6 +271,7 @@ The easiest way to log traces is to use one of our direct integrations. Opik sup
 | Groq                  | Log traces for Groq LLM calls                           | [Documentation](https://www.comet.com/docs/opik/integrations/groq?utm_source=opik&utm_medium=github&utm_content=groq_link&utm_campaign=opik)                                   |
 | Guardrails            | Log traces for Guardrails AI validations                | [Documentation](https://www.comet.com/docs/opik/integrations/guardrails-ai?utm_source=opik&utm_medium=github&utm_content=guardrails_link&utm_campaign=opik)                    |
 | Haystack              | Log traces for Haystack calls                           | [Documentation](https://www.comet.com/docs/opik/integrations/haystack?utm_source=opik&utm_medium=github&utm_content=haystack_link&utm_campaign=opik)                           |
+| Harbor                | Log traces for Harbor benchmark evaluation trials       | [Documentation](https://www.comet.com/docs/opik/integrations/harbor?utm_source=opik&utm_medium=github&utm_content=harbor_link&utm_campaign=opik)                               |
 | Instructor            | Log traces for LLM calls made with Instructor           | [Documentation](https://www.comet.com/docs/opik/integrations/instructor?utm_source=opik&utm_medium=github&utm_content=instructor_link&utm_campaign=opik)                       |
 | LangChain (Python)    | Log traces for LangChain LLM calls                      | [Documentation](https://www.comet.com/docs/opik/integrations/langchain?utm_source=opik&utm_medium=github&utm_content=langchain_link&utm_campaign=opik)                         |
 | LangChain (JS/TS)     | Log traces for LangChain JavaScript/TypeScript calls    | [Documentation](https://www.comet.com/docs/opik/integrations/langchainjs?utm_source=opik&utm_medium=github&utm_content=langchainjs_link&utm_campaign=opik)                     |