arize-phoenix 5.8.0__tar.gz → 5.9.1__tar.gz

{arize_phoenix-5.8.0 → arize_phoenix-5.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: arize-phoenix
-Version: 5.8.0
+Version: 5.9.1
 Summary: AI Observability and Evaluation
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues

{arize_phoenix-5.8.0 → arize_phoenix-5.9.1}/pyproject.toml

@@ -412,6 +412,7 @@ target-version = "py39"
 
 [tool.ruff.lint.per-file-ignores]
 "*.ipynb" = ["E402", "E501"]
+"examples/**/*.py" = ["E501"]
 
 [tool.ruff.lint]
 select = ["E", "F", "W", "I", "NPY201"]

{arize_phoenix-5.8.0 → arize_phoenix-5.9.1}/src/phoenix/config.py

@@ -559,7 +559,19 @@ def get_env_host() -> str:
 
 
 def get_env_host_root_path() -> str:
-    return os.getenv(ENV_PHOENIX_HOST_ROOT_PATH) or HOST_ROOT_PATH
+    if (host_root_path := os.getenv(ENV_PHOENIX_HOST_ROOT_PATH)) is None:
+        return HOST_ROOT_PATH
+    if not host_root_path.startswith("/"):
+        raise ValueError(
+            f"Invalid value for environment variable {ENV_PHOENIX_HOST_ROOT_PATH}: "
+            f"{host_root_path}. Value must start with '/'"
+        )
+    if host_root_path.endswith("/"):
+        raise ValueError(
+            f"Invalid value for environment variable {ENV_PHOENIX_HOST_ROOT_PATH}: "
+            f"{host_root_path}. Value cannot end with '/'"
+        )
+    return host_root_path
 
 
 def get_env_collector_endpoint() -> Optional[str]:

{arize_phoenix-5.8.0 → arize_phoenix-5.9.1}/src/phoenix/db/helpers.py

@@ -7,7 +7,16 @@ from openinference.semconv.trace import (
     RerankerAttributes,
     SpanAttributes,
 )
-from sqlalchemy import Integer, Select, SQLColumnExpression, case, distinct, func, select
+from sqlalchemy import (
+    Integer,
+    Select,
+    SQLColumnExpression,
+    and_,
+    case,
+    distinct,
+    func,
+    select,
+)
 from typing_extensions import assert_never
 
 from phoenix.db import models
@@ -103,3 +112,48 @@ def dedup(
             ans.append(item)
             seen.add(k)
     return ans
+
+
+def get_dataset_example_revisions(
+    dataset_version_id: int,
+) -> Select[tuple[models.DatasetExampleRevision]]:
+    version = (
+        select(
+            models.DatasetVersion.id,
+            models.DatasetVersion.dataset_id,
+        )
+        .filter_by(id=dataset_version_id)
+        .subquery()
+    )
+    table = models.DatasetExampleRevision
+    revision = (
+        select(
+            table.dataset_example_id,
+            func.max(table.dataset_version_id).label("dataset_version_id"),
+        )
+        .join_from(
+            table,
+            models.DatasetExample,
+            table.dataset_example_id == models.DatasetExample.id,
+        )
+        .join_from(
+            models.DatasetExample,
+            version,
+            models.DatasetExample.dataset_id == version.c.dataset_id,
+        )
+        .where(models.DatasetExample.dataset_id == version.c.dataset_id)
+        .where(table.dataset_version_id <= version.c.id)
+        .group_by(table.dataset_example_id)
+        .subquery()
+    )
+    return (
+        select(table)
+        .where(table.revision_kind != "DELETE")
+        .join(
+            revision,
+            onclause=and_(
+                revision.c.dataset_example_id == table.dataset_example_id,
+                revision.c.dataset_version_id == table.dataset_version_id,
+            ),
+        )
+    )

{arize_phoenix-5.8.0 → arize_phoenix-5.9.1}/src/phoenix/server/api/helpers/playground_clients.py

@@ -26,6 +26,7 @@ from phoenix.server.api.input_types.GenerativeModelInput import GenerativeModelI
 from phoenix.server.api.input_types.InvocationParameters import (
     BoundedFloatInvocationParameter,
     CanonicalParameterName,
+    FloatInvocationParameter,
     IntInvocationParameter,
     InvocationParameter,
     InvocationParameterInput,
@@ -44,14 +45,28 @@ from phoenix.server.api.types.GenerativeProvider import GenerativeProviderKey
 
 if TYPE_CHECKING:
     from anthropic.types import MessageParam
+    from google.generativeai.types import ContentType
     from openai.types import CompletionUsage
     from openai.types.chat import ChatCompletionMessageParam, ChatCompletionMessageToolCallParam
 
-DependencyName: TypeAlias = str
 SetSpanAttributesFn: TypeAlias = Callable[[Mapping[str, Any]], None]
 ChatCompletionChunk: TypeAlias = Union[TextChunk, ToolCallChunk]
 
 
+class Dependency:
+    """
+    Set the module_name to the import name if it is different from the install name
+    """
+
+    def __init__(self, name: str, module_name: Optional[str] = None):
+        self.name = name
+        self.module_name = module_name
+
+    @property
+    def import_name(self) -> str:
+        return self.module_name or self.name
+
+
 class KeyedSingleton:
     _instances: dict[Hashable, "KeyedSingleton"] = {}
 
@@ -148,8 +163,8 @@ class PlaygroundStreamingClient(ABC):
 
     @classmethod
     @abstractmethod
-    def dependencies(cls) -> list[DependencyName]:
-        # A list of dependency names this client needs to run
+    def dependencies(cls) -> list[Dependency]:
+        # A list of dependencies this client needs to run
         ...
 
     @classmethod
@@ -194,7 +209,8 @@ class PlaygroundStreamingClient(ABC):
     def dependencies_are_installed(cls) -> bool:
         try:
             for dependency in cls.dependencies():
-                if importlib.util.find_spec(dependency) is None:
+                import_name = dependency.import_name
+                if importlib.util.find_spec(import_name) is None:
                     return False
             return True
         except ValueError:
@@ -244,8 +260,8 @@ class OpenAIStreamingClient(PlaygroundStreamingClient):
         self.rate_limiter = PlaygroundRateLimiter(model.provider_key, OpenAIRateLimitError)
 
     @classmethod
-    def dependencies(cls) -> list[DependencyName]:
-        return ["openai"]
+    def dependencies(cls) -> list[Dependency]:
+        return [Dependency(name="openai")]
 
     @classmethod
     def supported_invocation_parameters(cls) -> list[InvocationParameter]:
@@ -331,6 +347,9 @@ class OpenAIStreamingClient(PlaygroundStreamingClient):
             if (usage := chunk.usage) is not None:
                 token_usage = usage
                 continue
+            if not chunk.choices:
+                # for Azure, initial chunk contains the content filter
+                continue
             choice = chunk.choices[0]
             delta = choice.delta
             if choice.finish_reason is None:
@@ -624,8 +643,8 @@ class AnthropicStreamingClient(PlaygroundStreamingClient):
         self.rate_limiter = PlaygroundRateLimiter(model.provider_key, anthropic.RateLimitError)
 
     @classmethod
-    def dependencies(cls) -> list[DependencyName]:
-        return ["anthropic"]
+    def dependencies(cls) -> list[Dependency]:
+        return [Dependency(name="anthropic")]
 
     @classmethod
     def supported_invocation_parameters(cls) -> list[InvocationParameter]:
@@ -746,6 +765,139 @@ class AnthropicStreamingClient(PlaygroundStreamingClient):
         return anthropic_messages, system_prompt
 
 
+@register_llm_client(
+    provider_key=GenerativeProviderKey.GEMINI,
+    model_names=[
+        PROVIDER_DEFAULT,
+        "gemini-1.5-flash",
+        "gemini-1.5-flash-8b",
+        "gemini-1.5-pro",
+        "gemini-1.0-pro",
+    ],
+)
+class GeminiStreamingClient(PlaygroundStreamingClient):
+    def __init__(
+        self,
+        model: GenerativeModelInput,
+        api_key: Optional[str] = None,
+    ) -> None:
+        import google.generativeai as google_genai
+
+        super().__init__(model=model, api_key=api_key)
+        google_genai.configure(api_key=api_key)
+        self.model_name = model.name
+
+    @classmethod
+    def dependencies(cls) -> list[Dependency]:
+        return [Dependency(name="google-generativeai", module_name="google.generativeai")]
+
+    @classmethod
+    def supported_invocation_parameters(cls) -> list[InvocationParameter]:
+        return [
+            BoundedFloatInvocationParameter(
+                invocation_name="temperature",
+                canonical_name=CanonicalParameterName.TEMPERATURE,
+                label="Temperature",
+                default_value=0.0,
+                min_value=0.0,
+                max_value=2.0,
+            ),
+            IntInvocationParameter(
+                invocation_name="max_output_tokens",
+                canonical_name=CanonicalParameterName.MAX_COMPLETION_TOKENS,
+                label="Max Output Tokens",
+            ),
+            StringListInvocationParameter(
+                invocation_name="stop",
+                canonical_name=CanonicalParameterName.STOP_SEQUENCES,
+                label="Stop Sequences",
+            ),
+            FloatInvocationParameter(
+                invocation_name="presence_penalty",
+                label="Presence Penalty",
+            ),
+            FloatInvocationParameter(
+                invocation_name="frequency_penalty",
+                label="Frequency Penalty",
+            ),
+            BoundedFloatInvocationParameter(
+                invocation_name="top_p",
+                canonical_name=CanonicalParameterName.TOP_P,
+                label="Top P",
+                min_value=0.0,
+                max_value=1.0,
+            ),
+            BoundedFloatInvocationParameter(
+                invocation_name="top_k",
+                label="Top K",
+                min_value=0.0,
+                max_value=1.0,
+            ),
+            IntInvocationParameter(
+                invocation_name="seed",
+                canonical_name=CanonicalParameterName.RANDOM_SEED,
+                label="Seed",
+            ),
+        ]
+
+    async def chat_completion_create(
+        self,
+        messages: list[
+            tuple[ChatCompletionMessageRole, str, Optional[str], Optional[list[JSONScalarType]]]
+        ],
+        tools: list[JSONScalarType],
+        **invocation_parameters: Any,
+    ) -> AsyncIterator[ChatCompletionChunk]:
+        import google.generativeai as google_genai
+
+        gemini_message_history, current_message, system_prompt = self._build_gemini_messages(
+            messages
+        )
+
+        model_args = {"model_name": self.model_name}
+        if system_prompt:
+            model_args["system_instruction"] = system_prompt
+        client = google_genai.GenerativeModel(**model_args)
+
+        gemini_config = google_genai.GenerationConfig(
+            **invocation_parameters,
+        )
+        gemini_params = {
+            "content": current_message,
+            "generation_config": gemini_config,
+            "stream": True,
+        }
+
+        chat = client.start_chat(history=gemini_message_history)
+        stream = await chat.send_message_async(**gemini_params)
+        async for event in stream:
+            yield TextChunk(content=event.text)
+
+    def _build_gemini_messages(
+        self,
+        messages: list[tuple[ChatCompletionMessageRole, str, Optional[str], Optional[list[str]]]],
+    ) -> tuple[list["ContentType"], str, str]:
+        gemini_message_history: list["ContentType"] = []
+        system_prompts = []
+        for role, content, _tool_call_id, _tool_calls in messages:
+            if role == ChatCompletionMessageRole.USER:
+                gemini_message_history.append({"role": "user", "parts": content})
+            elif role == ChatCompletionMessageRole.AI:
+                gemini_message_history.append({"role": "model", "parts": content})
+            elif role == ChatCompletionMessageRole.SYSTEM:
+                system_prompts.append(content)
+            elif role == ChatCompletionMessageRole.TOOL:
+                raise NotImplementedError
+            else:
+                assert_never(role)
+        if gemini_message_history:
+            prompt = gemini_message_history.pop()["parts"]
+        else:
+            prompt = ""
+
+        return gemini_message_history, prompt, "\n".join(system_prompts)
+
+
 def initialize_playground_clients() -> None:
     """
     Ensure that all playground clients are registered at import time.

{arize_phoenix-5.8.0 → arize_phoenix-5.9.1}/src/phoenix/server/api/mutations/chat_mutations.py

@@ -1,9 +1,9 @@
-import json
-from dataclasses import asdict
+import asyncio
+from dataclasses import asdict, field
 from datetime import datetime, timezone
 from itertools import chain
 from traceback import format_exc
-from typing import Any, Iterable, Iterator, List, Optional
+from typing import Any, Iterable, Iterator, List, Optional, Union
 
 import strawberry
 from openinference.instrumentation import safe_json_dumps
@@ -18,14 +18,19 @@ from openinference.semconv.trace import (
 from opentelemetry.sdk.trace.id_generator import RandomIdGenerator as DefaultOTelIDGenerator
 from opentelemetry.trace import StatusCode
 from sqlalchemy import insert, select
+from strawberry.relay import GlobalID
 from strawberry.types import Info
 from typing_extensions import assert_never
 
 from phoenix.datetime_utils import local_now, normalize_datetime
 from phoenix.db import models
+from phoenix.db.helpers import get_dataset_example_revisions
 from phoenix.server.api.context import Context
-from phoenix.server.api.exceptions import BadRequest
-from phoenix.server.api.helpers.playground_clients import initialize_playground_clients
+from phoenix.server.api.exceptions import BadRequest, NotFound
+from phoenix.server.api.helpers.playground_clients import (
+    PlaygroundStreamingClient,
+    initialize_playground_clients,
+)
 from phoenix.server.api.helpers.playground_registry import PLAYGROUND_CLIENT_REGISTRY
 from phoenix.server.api.helpers.playground_spans import (
     input_value_and_mime_type,
@@ -35,17 +40,28 @@ from phoenix.server.api.helpers.playground_spans import (
     llm_span_kind,
     llm_tools,
 )
-from phoenix.server.api.input_types.ChatCompletionInput import ChatCompletionInput
+from phoenix.server.api.input_types.ChatCompletionInput import (
+    ChatCompletionInput,
+    ChatCompletionOverDatasetInput,
+)
 from phoenix.server.api.input_types.TemplateOptions import TemplateOptions
+from phoenix.server.api.subscriptions import (
+    _default_playground_experiment_description,
+    _default_playground_experiment_metadata,
+    _default_playground_experiment_name,
+)
 from phoenix.server.api.types.ChatCompletionMessageRole import ChatCompletionMessageRole
 from phoenix.server.api.types.ChatCompletionSubscriptionPayload import (
     TextChunk,
     ToolCallChunk,
 )
+from phoenix.server.api.types.Dataset import Dataset
+from phoenix.server.api.types.DatasetVersion import DatasetVersion
+from phoenix.server.api.types.node import from_global_id_with_expected_type
 from phoenix.server.api.types.Span import Span, to_gql_span
 from phoenix.server.api.types.TemplateLanguage import TemplateLanguage
 from phoenix.server.dml_event import SpanInsertEvent
-from phoenix.trace.attributes import unflatten
+from phoenix.trace.attributes import get_attribute_value, unflatten
 from phoenix.trace.schemas import SpanException
 from phoenix.utilities.json import jsonify
 from phoenix.utilities.template_formatters import (
@@ -79,21 +95,192 @@ class ChatCompletionMutationPayload:
     error_message: Optional[str]
 
 
+@strawberry.type
+class ChatCompletionMutationError:
+    message: str
+
+
+@strawberry.type
+class ChatCompletionOverDatasetMutationExamplePayload:
+    dataset_example_id: GlobalID
+    experiment_run_id: GlobalID
+    result: Union[ChatCompletionMutationPayload, ChatCompletionMutationError]
+
+
+@strawberry.type
+class ChatCompletionOverDatasetMutationPayload:
+    dataset_id: GlobalID
+    dataset_version_id: GlobalID
+    experiment_id: GlobalID
+    examples: list[ChatCompletionOverDatasetMutationExamplePayload] = field(default_factory=list)
+
+
 @strawberry.type
 class ChatCompletionMutationMixin:
     @strawberry.mutation
+    @classmethod
+    async def chat_completion_over_dataset(
+        cls,
+        info: Info[Context, None],
+        input: ChatCompletionOverDatasetInput,
+    ) -> ChatCompletionOverDatasetMutationPayload:
+        provider_key = input.model.provider_key
+        llm_client_class = PLAYGROUND_CLIENT_REGISTRY.get_client(provider_key, input.model.name)
+        if llm_client_class is None:
+            raise BadRequest(f"No LLM client registered for provider '{provider_key}'")
+        llm_client = llm_client_class(
+            model=input.model,
+            api_key=input.api_key,
+        )
+        dataset_id = from_global_id_with_expected_type(input.dataset_id, Dataset.__name__)
+        dataset_version_id = (
+            from_global_id_with_expected_type(
+                global_id=input.dataset_version_id, expected_type_name=DatasetVersion.__name__
+            )
+            if input.dataset_version_id
+            else None
+        )
+        async with info.context.db() as session:
+            dataset = await session.scalar(select(models.Dataset).filter_by(id=dataset_id))
+            if dataset is None:
+                raise NotFound("Dataset not found")
+            if dataset_version_id is None:
+                resolved_version_id = await session.scalar(
+                    select(models.DatasetVersion.id)
+                    .filter_by(dataset_id=dataset_id)
+                    .order_by(models.DatasetVersion.id.desc())
+                    .limit(1)
+                )
+                if resolved_version_id is None:
+                    raise NotFound("No versions found for the given dataset")
+            else:
+                resolved_version_id = dataset_version_id
+            revisions = [
+                revision
+                async for revision in await session.stream_scalars(
+                    get_dataset_example_revisions(resolved_version_id)
+                )
+            ]
+            if not revisions:
+                raise NotFound("No examples found for the given dataset and version")
+            experiment = models.Experiment(
+                dataset_id=from_global_id_with_expected_type(input.dataset_id, Dataset.__name__),
+                dataset_version_id=resolved_version_id,
+                name=input.experiment_name or _default_playground_experiment_name(),
+                description=input.experiment_description
+                or _default_playground_experiment_description(dataset_name=dataset.name),
+                repetitions=1,
+                metadata_=input.experiment_metadata
+                or _default_playground_experiment_metadata(
+                    dataset_name=dataset.name,
+                    dataset_id=input.dataset_id,
+                    version_id=GlobalID(DatasetVersion.__name__, str(resolved_version_id)),
+                ),
+                project_name=PLAYGROUND_PROJECT_NAME,
+            )
+            session.add(experiment)
+            await session.flush()
+
+        start_time = datetime.now(timezone.utc)
+        results = await asyncio.gather(
+            *(
+                cls._chat_completion(
+                    info,
+                    llm_client,
+                    ChatCompletionInput(
+                        model=input.model,
+                        api_key=input.api_key,
+                        messages=input.messages,
+                        tools=input.tools,
+                        invocation_parameters=input.invocation_parameters,
+                        template=TemplateOptions(
+                            language=input.template_language,
+                            variables=revision.input,
+                        ),
+                    ),
+                )
+                for revision in revisions
+            ),
+            return_exceptions=True,
+        )
+
+        payload = ChatCompletionOverDatasetMutationPayload(
+            dataset_id=GlobalID(models.Dataset.__name__, str(dataset.id)),
+            dataset_version_id=GlobalID(DatasetVersion.__name__, str(resolved_version_id)),
+            experiment_id=GlobalID(models.Experiment.__name__, str(experiment.id)),
+        )
+        experiment_runs = []
+        for revision, result in zip(revisions, results):
+            if isinstance(result, BaseException):
+                experiment_run = models.ExperimentRun(
+                    experiment_id=experiment.id,
+                    dataset_example_id=revision.dataset_example_id,
+                    output={},
+                    repetition_number=1,
+                    start_time=start_time,
+                    end_time=start_time,
+                    error=str(result),
+                )
+            else:
+                db_span = result.span.db_span
+                experiment_run = models.ExperimentRun(
+                    experiment_id=experiment.id,
+                    dataset_example_id=revision.dataset_example_id,
+                    trace_id=str(result.span.context.trace_id),
+                    output=models.ExperimentRunOutput(
+                        task_output=get_attribute_value(db_span.attributes, LLM_OUTPUT_MESSAGES),
+                    ),
+                    prompt_token_count=db_span.cumulative_llm_token_count_prompt,
+                    completion_token_count=db_span.cumulative_llm_token_count_completion,
+                    repetition_number=1,
+                    start_time=result.span.start_time,
+                    end_time=result.span.end_time,
+                    error=str(result.error_message) if result.error_message else None,
+                )
+            experiment_runs.append(experiment_run)
+
+        async with info.context.db() as session:
+            session.add_all(experiment_runs)
+            await session.flush()
+
+        for revision, experiment_run, result in zip(revisions, experiment_runs, results):
+            dataset_example_id = GlobalID(
+                models.DatasetExample.__name__, str(revision.dataset_example_id)
+            )
+            experiment_run_id = GlobalID(models.ExperimentRun.__name__, str(experiment_run.id))
+            example_payload = ChatCompletionOverDatasetMutationExamplePayload(
+                dataset_example_id=dataset_example_id,
+                experiment_run_id=experiment_run_id,
+                result=result
+                if isinstance(result, ChatCompletionMutationPayload)
+                else ChatCompletionMutationError(message=str(result)),
+            )
+            payload.examples.append(example_payload)
+        return payload
+
+    @strawberry.mutation
+    @classmethod
     async def chat_completion(
-        self, info: Info[Context, None], input: ChatCompletionInput
+        cls, info: Info[Context, None], input: ChatCompletionInput
     ) -> ChatCompletionMutationPayload:
         provider_key = input.model.provider_key
         llm_client_class = PLAYGROUND_CLIENT_REGISTRY.get_client(provider_key, input.model.name)
         if llm_client_class is None:
             raise BadRequest(f"No LLM client registered for provider '{provider_key}'")
-        attributes: dict[str, Any] = {}
         llm_client = llm_client_class(
             model=input.model,
             api_key=input.api_key,
         )
+        return await cls._chat_completion(info, llm_client, input)
+
+    @classmethod
+    async def _chat_completion(
+        cls,
+        info: Info[Context, None],
+        llm_client: PlaygroundStreamingClient,
+        input: ChatCompletionInput,
+    ) -> ChatCompletionMutationPayload:
+        attributes: dict[str, Any] = {}
 
         messages = [
             (
@@ -122,7 +309,6 @@ class ChatCompletionMutationMixin:
                 llm_input_messages(messages),
                 llm_invocation_parameters(invocation_parameters),
                 input_value_and_mime_type(input),
-                **llm_client.attributes,
             )
         )
 
@@ -167,6 +353,7 @@ class ChatCompletionMutationMixin:
         else:
             end_time = normalize_datetime(dt=local_now(), tz=timezone.utc)
 
+        attributes.update(llm_client.attributes)
         if text_content or tool_calls:
             attributes.update(
                 chain(
@@ -306,7 +493,7 @@ def _llm_output_messages(
         if arguments := tool_call.function.arguments:
             yield (
                 f"{LLM_OUTPUT_MESSAGES}.0.{MESSAGE_TOOL_CALLS}.{tool_call_index}.{TOOL_CALL_FUNCTION_ARGUMENTS_JSON}",
-                json.dumps(arguments),
+                arguments,
             )
 
 

{arize_phoenix-5.8.0 → arize_phoenix-5.9.1}/src/phoenix/server/api/queries.py

@@ -48,6 +48,7 @@ from phoenix.server.api.input_types.DatasetSort import DatasetSort
 from phoenix.server.api.input_types.InvocationParameters import (
     InvocationParameter,
 )
+from phoenix.server.api.subscriptions import PLAYGROUND_PROJECT_NAME
 from phoenix.server.api.types.Cluster import Cluster, to_gql_clusters
 from phoenix.server.api.types.Dataset import Dataset, to_gql_dataset
 from phoenix.server.api.types.DatasetExample import DatasetExample
@@ -237,7 +238,10 @@ class Query:
             select(models.Project)
             .outerjoin(
                 models.Experiment,
-                models.Project.name == models.Experiment.project_name,
+                and_(
+                    models.Project.name == models.Experiment.project_name,
+                    models.Experiment.project_name != PLAYGROUND_PROJECT_NAME,
+                ),
             )
             .where(models.Experiment.project_name.is_(None))
             .order_by(models.Project.id)