opik 1.6.4__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/validation/chat_prompt_messages.py

@@ -0,0 +1,241 @@
+from typing import Any, List, Optional
+
+import opik.exceptions as exceptions
+from . import validator, result
+
+
+class ChatPromptMessagesValidator(validator.RaisableValidator):
+    """
+    Validator for ChatPrompt messages list.
+
+    Validates that messages is a list of dicts with:
+    - "role" key with value "system", "user", or "assistant"
+    - "content" key with value either string or list of dicts
+    - If content is list of dicts, each dict must have "type" key
+    """
+
+    VALID_ROLES = {"system", "user", "assistant"}
+    URL_BASED_CONTENT_TYPES = {"image_url", "video_url", "audio_url"}
+
+    def __init__(self, messages: Any):
+        self.messages = messages
+        self.validation_result: Optional[result.ValidationResult] = None
+
+    def validate(self) -> result.ValidationResult:
+        failure_reasons: List[str] = []
+
+        # Validate messages is a list
+        if not self._validate_messages_is_list(failure_reasons):
+            self.validation_result = result.ValidationResult(
+                failed=True, failure_reasons=failure_reasons
+            )
+            return self.validation_result
+
+        # Validate each message in the list
+        for idx, message in enumerate(self.messages):
+            prefix = f"messages[{idx}]"
+            self._validate_message(prefix, message, failure_reasons)
+
+        # Create validation result
+        if len(failure_reasons) > 0:
+            self.validation_result = result.ValidationResult(
+                failed=True, failure_reasons=failure_reasons
+            )
+        else:
+            self.validation_result = result.ValidationResult(failed=False)
+
+        return self.validation_result
+
+    def _validate_messages_is_list(self, failure_reasons: List[str]) -> bool:
+        """Validate that messages is a list. Returns False if validation fails."""
+        if not isinstance(self.messages, list):
+            msg = (
+                f"messages must be a list but {type(self.messages).__name__} was given"
+            )
+            failure_reasons.append(msg)
+            return False
+        return True
+
+    def _validate_message(
+        self, prefix: str, message: Any, failure_reasons: List[str]
+    ) -> None:
+        """Validate a single message structure, role, and content."""
+        if not self._validate_message_structure(prefix, message, failure_reasons):
+            return
+
+        self._validate_role(prefix, message, failure_reasons)
+        self._validate_content(prefix, message, failure_reasons)
+
+    def _validate_message_structure(
+        self, prefix: str, message: Any, failure_reasons: List[str]
+    ) -> bool:
+        """Validate that message is a dict with exactly 'role' and 'content' keys. Returns False if validation fails."""
+        # Validate message is a dict
+        if not isinstance(message, dict):
+            msg = f"{prefix}: must be a dict but {type(message).__name__} was given"
+            failure_reasons.append(msg)
+            return False
+
+        # Validate message has exactly "role" and "content" keys
+        message_keys = set(message.keys())
+        expected_keys = {"role", "content"}
+
+        if message_keys != expected_keys:
+            if not message_keys.issubset(expected_keys):
+                missing_keys = expected_keys - message_keys
+                msg = f"{prefix}: missing required keys: {sorted(missing_keys)}"
+                failure_reasons.append(msg)
+            if not expected_keys.issubset(message_keys):
+                extra_keys = message_keys - expected_keys
+                msg = (
+                    f"{prefix}: unexpected keys: {sorted(extra_keys)}. "
+                    f"Expected only: {sorted(expected_keys)}"
+                )
+                failure_reasons.append(msg)
+            return False
+
+        return True
+
+    def _validate_role(
+        self, prefix: str, message: dict, failure_reasons: List[str]
+    ) -> None:
+        """Validate the role field of a message."""
+        role = message.get("role")
+        if role not in self.VALID_ROLES:
+            valid_roles_str = ", ".join([f"'{r}'" for r in sorted(self.VALID_ROLES)])
+            msg = (
+                f"{prefix}.role: must be one of [{valid_roles_str}] "
+                f"but {repr(role)} was given"
+            )
+            failure_reasons.append(msg)
+
+    def _validate_content(
+        self, prefix: str, message: dict, failure_reasons: List[str]
+    ) -> None:
+        """Validate the content field of a message."""
+        content = message.get("content")
+        if content is None:
+            msg = f"{prefix}.content: must not be None"
+            failure_reasons.append(msg)
+        elif not isinstance(content, (str, list)):
+            msg = (
+                f"{prefix}.content: must be either str or list of dicts "
+                f"but {type(content).__name__} was given"
+            )
+            failure_reasons.append(msg)
+        elif isinstance(content, list):
+            self._validate_content_list(prefix, content, failure_reasons)
+
+    def _validate_content_list(
+        self, prefix: str, content: list, failure_reasons: List[str]
+    ) -> None:
+        """Validate content when it is a list of content parts."""
+        for content_idx, content_part in enumerate(content):
+            content_prefix = f"{prefix}.content[{content_idx}]"
+            self._validate_content_part(content_prefix, content_part, failure_reasons)
+
+    def _validate_content_part(
+        self, content_prefix: str, content_part: Any, failure_reasons: List[str]
+    ) -> None:
+        """Validate a single content part in the content list."""
+        if not isinstance(content_part, dict):
+            msg = (
+                f"{content_prefix}: must be a dict "
+                f"but {type(content_part).__name__} was given"
+            )
+            failure_reasons.append(msg)
+            return
+
+        if "type" not in content_part:
+            msg = f"{content_prefix}: must have 'type' key"
+            failure_reasons.append(msg)
+            return
+
+        # Validate type-specific requirements
+        content_type = content_part.get("type")
+        self._validate_content_type_specific(
+            content_prefix, content_type, content_part, failure_reasons
+        )
+
+    def _validate_content_type_specific(
+        self,
+        content_prefix: str,
+        content_type: Any,
+        content_part: dict,
+        failure_reasons: List[str],
+    ) -> None:
+        """Validate type-specific requirements for content parts."""
+        if content_type in self.URL_BASED_CONTENT_TYPES:
+            self._validate_required_url_object(
+                content_prefix,
+                content_part,
+                content_type,
+                content_type,
+                failure_reasons,
+            )
+        elif content_type == "text":
+            self._validate_required_string_key(
+                content_prefix, content_part, "text", "text", failure_reasons
+            )
+
+    def _validate_required_string_key(
+        self,
+        prefix: str,
+        content_part: dict,
+        key_name: str,
+        type_name: str,
+        failure_reasons: List[str],
+    ) -> None:
+        """Validate that a required key exists and is a string."""
+        if key_name not in content_part:
+            msg = f"{prefix}: must have '{key_name}' key when type is '{type_name}'"
+            failure_reasons.append(msg)
+        elif not isinstance(content_part.get(key_name), str):
+            msg = (
+                f"{prefix}.{key_name}: must be a string "
+                f"but {type(content_part.get(key_name)).__name__} was given"
+            )
+            failure_reasons.append(msg)
+
+    def _validate_required_url_object(
+        self,
+        prefix: str,
+        content_part: dict,
+        key_name: str,
+        type_name: str,
+        failure_reasons: List[str],
+    ) -> None:
+        """Validate that a required key exists and is a dict with a 'url' key that is a string."""
+        if key_name not in content_part:
+            msg = f"{prefix}: must have '{key_name}' key when type is '{type_name}'"
+            failure_reasons.append(msg)
+            return
+
+        url_object = content_part.get(key_name)
+        if not isinstance(url_object, dict):
+            msg = (
+                f"{prefix}.{key_name}: must be a dict "
+                f"but {type(url_object).__name__} was given"
+            )
+            failure_reasons.append(msg)
+            return
+
+        if "url" not in url_object:
+            msg = f"{prefix}.{key_name}: must have 'url' key"
+            failure_reasons.append(msg)
+        elif not isinstance(url_object.get("url"), str):
+            msg = (
+                f"{prefix}.{key_name}.url: must be a string "
+                f"but {type(url_object.get('url')).__name__} was given"
+            )
+            failure_reasons.append(msg)
+
+    def raise_if_validation_failed(self) -> None:
+        if (
+            self.validation_result is not None
+            and len(self.validation_result.failure_reasons) > 0
+        ):
+            raise exceptions.ValidationError(
+                prefix="ChatPrompt.__init__",
+                failure_reasons=self.validation_result.failure_reasons,
+            )

opik/validation/feedback_score.py

@@ -1,16 +1,16 @@
 import pydantic
 
 from typing import Any
-from ..types import FeedbackScoreDict
+from ..types import BatchFeedbackScoreDict
 from . import validator, result
 
 
 class PydanticWrapper(pydantic.BaseModel):
     model_config = pydantic.ConfigDict(extra="forbid")
-    feedback_score: FeedbackScoreDict
+    feedback_score: BatchFeedbackScoreDict
 
 
-EXPECTED_TYPES = "{'id': str, 'name': str, 'value': float, 'reason': NotRequired[str], 'category_name': NotRequired[str]}"
+EXPECTED_TYPES = "{'id': str, 'name': str, 'value': float, 'reason': NotRequired[str], 'category_name': NotRequired[str], 'project_name': NotRequired[str]}"
 
 
 class FeedbackScoreValidator(validator.Validator):
@@ -30,8 +30,7 @@ class FeedbackScoreValidator(validator.Validator):
             for e in exception.errors():
                 component_name: str = ".".join(e["loc"])
                 msg: str = (
-                    f"{component_name} - {e['msg']}.\n"
-                    f"Expected dict: {EXPECTED_TYPES}."
+                    f"{component_name} - {e['msg']}.\nExpected dict: {EXPECTED_TYPES}."
                 )
                 failure_reasons.append(msg)
             self.validation_result = result.ValidationResult(

opik/validation/parameter.py

@@ -0,0 +1,122 @@
+import numbers
+from typing import Any, List, Optional, Tuple
+
+
+class Parameter:
+    __slots__ = ["name", "value", "types", "allow_empty", "possible_values"]
+
+    def __init__(
+        self,
+        name: str,
+        value: Any,
+        types: Tuple,
+        possible_values: Optional[List] = None,
+        allow_empty: bool = True,
+    ):
+        self.name = name
+        self.value = value
+        self.types = types
+        self.allow_empty = allow_empty
+        self.possible_values = possible_values
+
+
+def create_str_parameter(
+    name: str,
+    value: Any,
+    possible_values: Optional[List[str]] = None,
+    allow_empty: bool = True,
+) -> Parameter:
+    return Parameter(
+        name=name,
+        value=value,
+        types=(str,),
+        possible_values=possible_values,
+        allow_empty=allow_empty,
+    )
+
+
+def create_int_parameter(
+    name: str,
+    value: Any,
+    possible_values: Optional[List[int]] = None,
+    allow_empty: bool = True,
+) -> Parameter:
+    return Parameter(
+        name=name,
+        value=value,
+        types=(int,),
+        possible_values=possible_values,
+        allow_empty=allow_empty,
+    )
+
+
+def create_float_parameter(
+    name: str,
+    value: Any,
+    possible_values: Optional[List[float]] = None,
+    allow_empty: bool = True,
+) -> Parameter:
+    return Parameter(
+        name=name,
+        value=value,
+        types=(float,),
+        possible_values=possible_values,
+        allow_empty=allow_empty,
+    )
+
+
+def create_numeric_parameter(
+    name: str,
+    value: Any,
+    possible_values: Optional[List[numbers.Number]] = None,
+    allow_empty: bool = True,
+) -> Parameter:
+    return Parameter(
+        name=name,
+        value=value,
+        types=(float, int),
+        possible_values=possible_values,
+        allow_empty=allow_empty,
+    )
+
+
+def create_bool_parameter(
+    name: str,
+    value: Any,
+    allow_empty: bool = True,
+) -> Parameter:
+    return Parameter(
+        name=name,
+        value=value,
+        types=(bool,),
+        possible_values=None,
+        allow_empty=allow_empty,
+    )
+
+
+def create_list_parameter(
+    name: str,
+    value: Any,
+    allow_empty: bool = True,
+) -> Parameter:
+    return Parameter(
+        name=name,
+        value=value,
+        types=(list,),
+        possible_values=None,
+        allow_empty=allow_empty,
+    )
+
+
+def create_dict_parameter(
+    name: str,
+    value: Any,
+    allow_empty: bool = True,
+) -> Parameter:
+    return Parameter(
+        name=name,
+        value=value,
+        types=(dict,),
+        possible_values=None,
+        allow_empty=allow_empty,
+    )

opik/validation/parameters_validator.py

@@ -0,0 +1,175 @@
+import logging
+import numbers
+from collections.abc import Iterable
+from typing import Any, List, Optional
+
+import opik.exceptions as exceptions
+from . import validator_helpers, parameter, result, validator
+
+
+class MethodParametersTypeValidator(validator.Validator):
+    def __init__(self, method_name: str, class_name: Optional[str] = None):
+        if class_name is not None:
+            self.prefix = "%s.%s" % (class_name, method_name)
+        else:
+            self.prefix = method_name
+        self.parameters: List[parameter.Parameter] = []
+        self.validation_result: Optional[result.ValidationResult] = None
+
+    def add_str_parameter(
+        self,
+        value: Any,
+        name: str,
+        possible_values: Optional[List[str]] = None,
+        allow_empty: bool = True,
+    ) -> None:
+        self.parameters.append(
+            parameter.create_str_parameter(
+                name=name,
+                value=value,
+                possible_values=possible_values,
+                allow_empty=allow_empty,
+            )
+        )
+
+    def add_bool_parameter(
+        self, value: Any, name: str, allow_empty: bool = True
+    ) -> None:
+        self.parameters.append(
+            parameter.create_bool_parameter(
+                name=name, value=value, allow_empty=allow_empty
+            )
+        )
+
+    def add_int_parameter(
+        self,
+        value: Any,
+        name: str,
+        possible_values: Optional[List[int]] = None,
+        allow_empty: bool = True,
+    ) -> None:
+        self.parameters.append(
+            parameter.create_int_parameter(
+                name=name,
+                value=value,
+                possible_values=possible_values,
+                allow_empty=allow_empty,
+            )
+        )
+
+    def add_float_parameter(
+        self,
+        value: Any,
+        name: str,
+        possible_values: Optional[List[float]] = None,
+        allow_empty: bool = True,
+    ) -> None:
+        self.parameters.append(
+            parameter.create_float_parameter(
+                name=name,
+                value=value,
+                possible_values=possible_values,
+                allow_empty=allow_empty,
+            )
+        )
+
+    def add_numeric_parameter(
+        self,
+        value: Any,
+        name: str,
+        possible_values: Optional[List[numbers.Number]] = None,
+        allow_empty: bool = True,
+    ) -> None:
+        self.parameters.append(
+            parameter.create_numeric_parameter(
+                name=name,
+                value=value,
+                possible_values=possible_values,
+                allow_empty=allow_empty,
+            )
+        )
+
+    def add_list_parameter(
+        self, value: Any, name: str, allow_empty: bool = True
+    ) -> None:
+        self.parameters.append(
+            parameter.create_list_parameter(
+                name=name, value=value, allow_empty=allow_empty
+            )
+        )
+
+    def add_list_with_strings_parameter(
+        self, value: Any, name: str, allow_empty: bool = True
+    ) -> None:
+        self.parameters.append(
+            parameter.create_list_parameter(
+                name=name, value=value, allow_empty=allow_empty
+            )
+        )
+        if isinstance(value, Iterable) and not isinstance(value, str):
+            for ix, string in enumerate(value):
+                name_ = f"{name}[{ix}]"
+                self.parameters.append(
+                    parameter.create_str_parameter(
+                        name=name_, value=string, allow_empty=allow_empty
+                    )
+                )
+
+    def add_dict_parameter(
+        self, value: Any, name: str, allow_empty: bool = True
+    ) -> None:
+        self.parameters.append(
+            parameter.create_dict_parameter(
+                name=name, value=value, allow_empty=allow_empty
+            )
+        )
+
+    def validate(self) -> result.ValidationResult:
+        try:
+            failures: List[str] = []
+            for param in self.parameters:
+                valid, msg = validator_helpers.validate_parameter(param)
+                if not valid:
+                    assert msg is not None, "msg must be set if valid is False"
+                    failures.append(msg)
+
+            if len(failures) > 0:
+                self.validation_result = result.ValidationResult(
+                    failure_reasons=failures, failed=True
+                )
+            else:
+                self.validation_result = result.ValidationResult(failed=False)
+        except Exception as e:
+            self.validation_result = result.ValidationResult(
+                failure_reasons=["Unexpected validation error: %r" % e], failed=True
+            )
+
+        return self.validation_result
+
+    def print_result(
+        self, logger: logging.Logger, log_level: int = logging.ERROR
+    ) -> None:
+        if self.validation_result is None:
+            logger.log(
+                level=log_level, msg="No validation result, please call validate first"
+            )
+            return
+
+        for msg in self.validation_result.failure_reasons:
+            logger.log(log_level, "%s: %s", self.prefix, msg)
+
+    def raise_validation_error(self) -> None:
+        if (
+            self.validation_result is not None
+            and len(self.validation_result.failure_reasons) > 0
+        ):
+            raise exceptions.ValidationError(
+                prefix=self.prefix,
+                failure_reasons=self.validation_result.failure_reasons,
+            )
+
+
+def create_validator(
+    method_name: str, class_name: Optional[str] = None
+) -> MethodParametersTypeValidator:
+    return MethodParametersTypeValidator(method_name=method_name, class_name=class_name)

opik/validation/validator.py

@@ -1,9 +1,37 @@
 import abc
+from typing import Any
 
-from .result import ValidationResult
+from . import result
 
 
 class Validator(abc.ABC):
     @abc.abstractmethod
-    def validate(self) -> ValidationResult:
+    def validate(self) -> result.ValidationResult:
+        pass
+
+
+class RaisableValidator(Validator):
+    """
+    Abstract validator class that extends Validator and adds raise_if_validation_failed method.
+
+    This is used for validators that need to raise ValidationError exceptions
+    when validation fails, typically used in class initialization.
+    """
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """
+        Initialize the validator.
+
+        Subclasses can override this method with their own initialization signature.
+        """
+        pass
+
+    @abc.abstractmethod
+    def raise_if_validation_failed(self) -> None:
+        """
+        Raise a ValidationError if validation failed.
+
+        This method should check the validation result and raise an appropriate
+        ValidationError exception if validation failed.
+        """
         pass