opik 1.6.4__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/integrations/langchain/google_run_helpers.py

@@ -1,75 +0,0 @@
-import logging
-from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple, Union
-
-from opik import llm_usage, logging_messages
-from opik.types import LLMProvider
-
-if TYPE_CHECKING:
-    from langchain_core.tracers.schemas import Run
-
-LOGGER = logging.getLogger(__name__)
-
-
-def get_llm_usage_info(
-    run_dict: Optional[Dict[str, Any]] = None,
-) -> llm_usage.LLMUsageInfo:
-    if run_dict is None:
-        return llm_usage.LLMUsageInfo()
-
-    usage_dict = _try_get_token_usage(run_dict)
-    provider, model = _get_provider_and_model(run_dict)
-
-    return llm_usage.LLMUsageInfo(provider=provider, model=model, usage=usage_dict)
-
-
-def _try_get_token_usage(run_dict: Dict[str, Any]) -> Optional[llm_usage.OpikUsage]:
-    try:
-        usage_metadata = run_dict["outputs"]["generations"][-1][-1]["generation_info"][
-            "usage_metadata"
-        ]
-
-        opik_usage = llm_usage.OpikUsage.from_google_dict(usage_metadata)
-        return opik_usage
-    except Exception:
-        LOGGER.warning(
-            logging_messages.FAILED_TO_EXTRACT_TOKEN_USAGE_FROM_PRESUMABLY_LANGCHAIN_GOOGLE_LLM_RUN,
-            exc_info=True,
-        )
-        return None
-
-
-def is_google_run(run: "Run") -> bool:
-    try:
-        if run.serialized is None:
-            return False
-
-        invocation_params = run.extra.get("invocation_params", {})
-        provider = invocation_params.get("_type", "").lower()
-        is_google = "vertexai" in provider.lower()
-
-        return is_google
-
-    except Exception:
-        LOGGER.debug(
-            "Failed to check if Run instance is from Google LLM, returning False.",
-            exc_info=True,
-        )
-        return False
-
-
-def _get_provider_and_model(
-    run_dict: Dict[str, Any],
-) -> Tuple[Optional[Union[Literal[LLMProvider.GOOGLE_VERTEXAI], str]], Optional[str]]:
-    """
-    Fetches the provider and model information from a given run dictionary.
-    """
-    provider = None
-    model = None
-
-    if invocation_params := run_dict["extra"].get("invocation_params"):
-        provider = invocation_params.get("_type")
-        if provider == "vertexai":
-            provider = LLMProvider.GOOGLE_VERTEXAI
-        model = invocation_params.get("model_name")
-
-    return provider, model

opik/integrations/langchain/openai_run_helpers.py

@@ -1,122 +0,0 @@
-import logging
-from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union, Literal
-
-from opik import _logging as opik_logging
-from opik import llm_usage, logging_messages
-
-if TYPE_CHECKING:
-    from langchain_core.tracers.schemas import Run
-
-
-LOGGER = logging.getLogger(__name__)
-
-
-def get_llm_usage_info(
-    run_dict: Optional[Dict[str, Any]] = None,
-) -> llm_usage.LLMUsageInfo:
-    if run_dict is None:
-        return llm_usage.LLMUsageInfo()
-
-    opik_usage = _try_get_token_usage(run_dict)
-    provider, model = _get_provider_and_model(run_dict)
-
-    return llm_usage.LLMUsageInfo(provider=provider, model=model, usage=opik_usage)
-
-
-def _try_get_token_usage(run_dict: Dict[str, Any]) -> Optional[llm_usage.OpikUsage]:
-    """
-    Attempts to extract and return the token usage from the given run dictionary.
-
-    Depending on the execution type (invoke, streaming mode, async etc.), or even the model name itself,
-    token usage info might be in different places, different formats or completely missing.
-    """
-    try:
-        if run_dict["outputs"]["llm_output"] is not None:
-            token_usage_dict = run_dict["outputs"]["llm_output"]["token_usage"]
-            return llm_usage.OpikUsage.from_openai_completions_dict(token_usage_dict)
-        # streaming mode handling
-        # token usage data MAY be available at the end of streaming
-        # in async mode may not provide token usage info
-        elif token_usage_dict := run_dict["outputs"]["generations"][-1][-1]["message"][
-            "kwargs"
-        ].get("usage_metadata"):
-            # TODO: provide better support for langchain usage. We probably need to convert it
-            # to the full openai usage dict (with details) or find a way to access raw data
-            openai_formatted_dict = {
-                "completion_tokens": token_usage_dict["output_tokens"],
-                "prompt_tokens": token_usage_dict["input_tokens"],
-                "total_tokens": token_usage_dict["total_tokens"],
-            }
-            opik_usage = llm_usage.OpikUsage.from_openai_completions_dict(
-                openai_formatted_dict
-            )
-            return opik_usage
-        else:
-            opik_logging.log_once_at_level(
-                logging_level=logging.WARNING,
-                message=logging_messages.WARNING_TOKEN_USAGE_DATA_IS_NOT_AVAILABLE,
-                logger=LOGGER,
-            )
-            return None
-
-    except Exception:
-        LOGGER.warning(
-            logging_messages.FAILED_TO_EXTRACT_TOKEN_USAGE_FROM_PRESUMABLY_LANGCHAIN_OPENAI_LLM_RUN,
-            exc_info=True,
-        )
-        return None
-
-
-def is_openai_run(run: "Run") -> bool:
-    try:
-        if run.serialized is None:
-            return False
-
-        serialized_kwargs = run.serialized.get("kwargs", {})
-        has_openai_key = "openai_api_key" in serialized_kwargs
-
-        return has_openai_key
-
-    except Exception:
-        LOGGER.debug(
-            "Failed to check if Run instance is from OpenAI LLM, returning False.",
-            exc_info=True,
-        )
-        return False
-
-
-def _get_provider_and_model(
-    run_dict: Dict[str, Any],
-) -> Tuple[Optional[Union[Literal["openai"], str]], Optional[str]]:
-    """
-    Fetches the provider and model information from a given run dictionary.
-
-    By default, the provider is assumed to be OpenAI (will be available in extra/metadata field).
-    If LLM output is available, the model version is included.
-    If the Client is available, the BaseURL is also checked.
-    """
-    provider = None
-    model = None
-
-    # here will be always provider and model
-    # but model name may not be full (model+version)
-    if metadata := run_dict["extra"].get("metadata"):
-        provider = metadata.get("ls_provider")
-        model = metadata.get("ls_model_name")
-
-    # try to detect model+version more precise way if possible
-    # .invoke() mode
-    if llm_output := run_dict["outputs"].get("llm_output"):
-        model = llm_output.get("model_name", model)
-    # streaming mode
-    elif generation_info := run_dict["outputs"]["generations"][-1][-1][
-        "generation_info"
-    ]:
-        model = generation_info.get("model_name", model)
-
-    # provider: check base url
-    if base_url := run_dict["extra"].get("invocation_params", {}).get("base_url"):
-        if base_url.host != "api.openai.com":
-            provider = base_url.host
-
-    return provider, model

opik/message_processing/message_processors.py

@@ -1,203 +0,0 @@
-import abc
-import logging
-from typing import Callable, Dict, Type, List
-from opik import logging_messages
-from . import messages
-from ..jsonable_encoder import encode
-from .. import dict_utils
-from ..rest_api.types import feedback_score_batch_item, trace_write
-from ..rest_api.types import span_write
-from ..rest_api import core as rest_api_core
-from ..rest_api import client as rest_api_client
-
-from .batching import sequence_splitter
-
-LOGGER = logging.getLogger(__name__)
-
-BATCH_MEMORY_LIMIT_MB = 50
-
-
-class BaseMessageProcessor(abc.ABC):
-    @abc.abstractmethod
-    def process(self, message: messages.BaseMessage) -> None:
-        pass
-
-
-class MessageSender(BaseMessageProcessor):
-    def __init__(self, rest_client: rest_api_client.OpikApi):
-        self._rest_client = rest_client
-
-        self._handlers: Dict[Type, Callable[[messages.BaseMessage], None]] = {
-            messages.CreateSpanMessage: self._process_create_span_message,  # type: ignore
-            messages.CreateTraceMessage: self._process_create_trace_message,  # type: ignore
-            messages.UpdateSpanMessage: self._process_update_span_message,  # type: ignore
-            messages.UpdateTraceMessage: self._process_update_trace_message,  # type: ignore
-            messages.AddTraceFeedbackScoresBatchMessage: self._process_add_trace_feedback_scores_batch_message,  # type: ignore
-            messages.AddSpanFeedbackScoresBatchMessage: self._process_add_span_feedback_scores_batch_message,  # type: ignore
-            messages.CreateSpansBatchMessage: self._process_create_span_batch_message,  # type: ignore
-            messages.CreateTraceBatchMessage: self._process_create_trace_batch_message,  # type: ignore
-        }
-
-    def process(self, message: messages.BaseMessage) -> None:
-        message_type = type(message)
-        handler = self._handlers.get(message_type)
-        if handler is None:
-            LOGGER.debug("Unknown type of message - %s", message_type.__name__)
-            return
-
-        try:
-            handler(message)
-        except rest_api_core.ApiError as exception:
-            if exception.status_code == 409:
-                # sometimes retry mechanism works in a way that it sends the same request 2 times.
-                # second request is rejected by the backend, we don't want users to an error.
-                return
-
-            error_fingerprint = _generate_error_fingerprint(exception, message)
-            LOGGER.error(
-                logging_messages.FAILED_TO_PROCESS_MESSAGE_IN_BACKGROUND_STREAMER,
-                message_type.__name__,
-                str(exception),
-                extra={"error_fingerprint": error_fingerprint},
-            )
-        except Exception as exception:
-            error_fingerprint = _generate_error_fingerprint(exception, message)
-            LOGGER.error(
-                logging_messages.FAILED_TO_PROCESS_MESSAGE_IN_BACKGROUND_STREAMER,
-                message_type.__name__,
-                str(exception),
-                exc_info=True,
-                extra={"error_fingerprint": error_fingerprint},
-            )
-
-    def _process_create_span_message(self, message: messages.CreateSpanMessage) -> None:
-        create_span_kwargs = message.as_payload_dict()
-        cleaned_create_span_kwargs = dict_utils.remove_none_from_dict(
-            create_span_kwargs
-        )
-        cleaned_create_span_kwargs = encode(cleaned_create_span_kwargs)
-        LOGGER.debug("Create span request: %s", cleaned_create_span_kwargs)
-        self._rest_client.spans.create_span(**cleaned_create_span_kwargs)
-
-    def _process_create_trace_message(
-        self, message: messages.CreateTraceMessage
-    ) -> None:
-        create_trace_kwargs = message.as_payload_dict()
-        cleaned_create_trace_kwargs = dict_utils.remove_none_from_dict(
-            create_trace_kwargs
-        )
-        cleaned_create_trace_kwargs = encode(cleaned_create_trace_kwargs)
-        LOGGER.debug("Create trace request: %s", cleaned_create_trace_kwargs)
-        self._rest_client.traces.create_trace(**cleaned_create_trace_kwargs)
-
-    def _process_update_span_message(self, message: messages.UpdateSpanMessage) -> None:
-        update_span_kwargs = message.as_payload_dict()
-
-        cleaned_update_span_kwargs = dict_utils.remove_none_from_dict(
-            update_span_kwargs
-        )
-        cleaned_update_span_kwargs = encode(cleaned_update_span_kwargs)
-        LOGGER.debug("Update span request: %s", cleaned_update_span_kwargs)
-        self._rest_client.spans.update_span(**cleaned_update_span_kwargs)
-
-    def _process_update_trace_message(
-        self, message: messages.UpdateTraceMessage
-    ) -> None:
-        update_trace_kwargs = message.as_payload_dict()
-
-        cleaned_update_trace_kwargs = dict_utils.remove_none_from_dict(
-            update_trace_kwargs
-        )
-        cleaned_update_trace_kwargs = encode(cleaned_update_trace_kwargs)
-        LOGGER.debug("Update trace request: %s", cleaned_update_trace_kwargs)
-        self._rest_client.traces.update_trace(**cleaned_update_trace_kwargs)
-        LOGGER.debug("Sent trace %s", message.trace_id)
-
-    def _process_add_span_feedback_scores_batch_message(
-        self, message: messages.AddSpanFeedbackScoresBatchMessage
-    ) -> None:
-        scores = [
-            feedback_score_batch_item.FeedbackScoreBatchItem(**score_message.__dict__)
-            for score_message in message.batch
-        ]
-
-        LOGGER.debug("Add spans feedbacks scores request of size: %d", len(scores))
-
-        self._rest_client.spans.score_batch_of_spans(
-            scores=scores,
-        )
-        LOGGER.debug("Sent batch of spans feedback scores %d", len(scores))
-
-    def _process_add_trace_feedback_scores_batch_message(
-        self, message: messages.AddTraceFeedbackScoresBatchMessage
-    ) -> None:
-        scores = [
-            feedback_score_batch_item.FeedbackScoreBatchItem(**score_message.__dict__)
-            for score_message in message.batch
-        ]
-
-        LOGGER.debug("Add traces feedbacks scores request: %d", len(scores))
-
-        self._rest_client.traces.score_batch_of_traces(
-            scores=scores,
-        )
-        LOGGER.debug("Sent batch of traces feedbacks scores of size %d", len(scores))
-
-    def _process_create_span_batch_message(
-        self, message: messages.CreateSpansBatchMessage
-    ) -> None:
-        rest_spans: List[span_write.SpanWrite] = []
-
-        for item in message.batch:
-            span_write_kwargs = item.as_payload_dict()
-            cleaned_span_write_kwargs = dict_utils.remove_none_from_dict(
-                span_write_kwargs
-            )
-            cleaned_span_write_kwargs = encode(cleaned_span_write_kwargs)
-            rest_spans.append(span_write.SpanWrite(**cleaned_span_write_kwargs))
-
-        memory_limited_batches = sequence_splitter.split_into_batches(
-            items=rest_spans,
-            max_payload_size_MB=BATCH_MEMORY_LIMIT_MB,
-        )
-
-        for batch in memory_limited_batches:
-            LOGGER.debug("Create spans batch request of size %d", len(batch))
-            self._rest_client.spans.create_spans(spans=batch)
-            LOGGER.debug("Sent spans batch of size %d", len(batch))
-
-    def _process_create_trace_batch_message(
-        self, message: messages.CreateTraceBatchMessage
-    ) -> None:
-        rest_traces: List[trace_write.TraceWrite] = []
-
-        for item in message.batch:
-            trace_write_kwargs = item.as_payload_dict()
-            cleaned_trace_write_kwargs = dict_utils.remove_none_from_dict(
-                trace_write_kwargs
-            )
-            cleaned_trace_write_kwargs = encode(cleaned_trace_write_kwargs)
-            rest_traces.append(trace_write.TraceWrite(**cleaned_trace_write_kwargs))
-
-        memory_limited_batches = sequence_splitter.split_into_batches(
-            items=rest_traces,
-            max_payload_size_MB=BATCH_MEMORY_LIMIT_MB,
-        )
-
-        for batch in memory_limited_batches:
-            LOGGER.debug("Create trace batch request of size %d", len(batch))
-            self._rest_client.traces.create_traces(traces=batch)
-            LOGGER.debug("Sent trace batch of size %d", len(batch))
-
-
-def _generate_error_fingerprint(
-    exception: Exception, message: messages.BaseMessage
-) -> List[str]:
-    fingerprint = [type(message).__name__, type(exception).__name__]
-
-    if isinstance(exception, rest_api_core.ApiError):
-        fingerprint.append(str(exception.status_code))
-
-        return fingerprint
-
-    return fingerprint

opik/rest_api/types/delta_role.py

@@ -1,7 +0,0 @@
-# This file was auto-generated by Fern from our API Definition.
-
-import typing
-
-DeltaRole = typing.Union[
-    typing.Literal["system", "user", "assistant", "tool", "function"], typing.Any
-]

opik/rest_api/types/json_object_schema.py

@@ -1,34 +0,0 @@
-# This file was auto-generated by Fern from our API Definition.
-
-from ..core.pydantic_utilities import UniversalBaseModel
-import typing
-from .json_schema_element import JsonSchemaElement
-import typing_extensions
-from ..core.serialization import FieldMetadata
-from ..core.pydantic_utilities import IS_PYDANTIC_V2
-import pydantic
-
-
-class JsonObjectSchema(UniversalBaseModel):
-    type: typing.Optional[str] = None
-    description: typing.Optional[str] = None
-    properties: typing.Optional[typing.Dict[str, JsonSchemaElement]] = None
-    required: typing.Optional[typing.List[str]] = None
-    additional_properties: typing_extensions.Annotated[
-        typing.Optional[bool], FieldMetadata(alias="additionalProperties")
-    ] = None
-    defs: typing_extensions.Annotated[
-        typing.Optional[typing.Dict[str, JsonSchemaElement]],
-        FieldMetadata(alias="$defs"),
-    ] = None
-
-    if IS_PYDANTIC_V2:
-        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(
-            extra="allow", frozen=True
-        )  # type: ignore # Pydantic v2
-    else:
-
-        class Config:
-            frozen = True
-            smart_union = True
-            extra = pydantic.Extra.allow