arize-phoenix 10.0.3__tar.gz → 10.1.0__tar.gz

{arize_phoenix-10.0.3 → arize_phoenix-10.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arize-phoenix
-Version: 10.0.3
+Version: 10.1.0
 Summary: AI Observability and Evaluation
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues
@@ -21,8 +21,8 @@ Requires-Dist: aioitertools
 Requires-Dist: aiosqlite
 Requires-Dist: alembic<2,>=1.3.0
 Requires-Dist: arize-phoenix-client
-Requires-Dist: arize-phoenix-evals>=0.13.1
-Requires-Dist: arize-phoenix-otel>=0.5.1
+Requires-Dist: arize-phoenix-evals>=0.20.6
+Requires-Dist: arize-phoenix-otel>=0.9.2
 Requires-Dist: authlib
 Requires-Dist: cachetools
 Requires-Dist: email-validator

{arize_phoenix-10.0.3 → arize_phoenix-10.1.0}/pyproject.toml

@@ -52,8 +52,8 @@ dependencies = [
   "sqlean.py>=3.45.1",
   "cachetools",
   "python-multipart",  # see https://www.starlette.io/#dependencies
-  "arize-phoenix-evals>=0.13.1",
-  "arize-phoenix-otel>=0.5.1",
+  "arize-phoenix-evals>=0.20.6",
+  "arize-phoenix-otel>=0.9.2",
   "fastapi",
   "pydantic>=2.1.0", # exclude 2.0.* since it does not support the `json_encoders` configuration setting
   "authlib",

{arize_phoenix-10.0.3 → arize_phoenix-10.1.0}/src/phoenix/experiments/functions.py

@@ -16,6 +16,7 @@ from urllib.parse import urljoin
 import httpx
 import opentelemetry.sdk.trace as trace_sdk
 import pandas as pd
+from httpx import HTTPStatusError
 from openinference.semconv.resource import ResourceAttributes
 from openinference.semconv.trace import (
     OpenInferenceMimeTypeValues,
@@ -232,8 +233,43 @@ def run_experiment(
         print(f"📺 View dataset experiments: {dataset_experiments_url}")
         print(f"🔗 View this experiment: {experiment_compare_url}")
 
-    def sync_run_experiment(test_case: TestCase) -> ExperimentRun:
+    # Create a cache for task results
+    task_result_cache: dict[tuple[str, int], Any] = {}
+
+    def sync_run_experiment(test_case: TestCase) -> Optional[ExperimentRun]:
         example, repetition_number = test_case.example, test_case.repetition_number
+        cache_key = (example.id, repetition_number)
+
+        # Check if we have a cached result
+        if cache_key in task_result_cache:
+            output = task_result_cache[cache_key]
+            exp_run = ExperimentRun(
+                start_time=datetime.now(
+                    timezone.utc
+                ),  # Use current time since we don't have the original span
+                end_time=datetime.now(timezone.utc),
+                experiment_id=experiment.id,
+                dataset_example_id=example.id,
+                repetition_number=repetition_number,
+                output=output,
+                error=None,
+                trace_id=None,  # No trace ID since we don't have the original span
+            )
+            if not dry_run:
+                try:
+                    # Try to create the run directly
+                    resp = sync_client.post(
+                        f"/v1/experiments/{experiment.id}/runs", json=jsonify(exp_run)
+                    )
+                    resp.raise_for_status()
+                    exp_run = replace(exp_run, id=resp.json()["data"]["id"])
+                except HTTPStatusError as e:
+                    if e.response.status_code == 409:
+                        # Ignore duplicate runs - we'll get the final state from the database
+                        return None
+                    raise
+            return exp_run
+
         output = None
         error: Optional[BaseException] = None
         status = Status(StatusCode.OK)
@@ -283,6 +319,7 @@ def run_experiment(
         assert isinstance(
             output, (dict, list, str, int, float, bool, type(None))
         ), "Output must be JSON serializable"
+
         exp_run = ExperimentRun(
             start_time=_decode_unix_nano(cast(int, span.start_time)),
             end_time=_decode_unix_nano(cast(int, span.end_time)),
@@ -294,13 +331,62 @@ def run_experiment(
             trace_id=_str_trace_id(span.get_span_context().trace_id),  # type: ignore[no-untyped-call]
         )
         if not dry_run:
-            resp = sync_client.post(f"/v1/experiments/{experiment.id}/runs", json=jsonify(exp_run))
-            resp.raise_for_status()
-            exp_run = replace(exp_run, id=resp.json()["data"]["id"])
+            try:
+                # Try to create the run directly
+                resp = sync_client.post(
+                    f"/v1/experiments/{experiment.id}/runs", json=jsonify(exp_run)
+                )
+                resp.raise_for_status()
+                exp_run = replace(exp_run, id=resp.json()["data"]["id"])
+                if error is None:
+                    task_result_cache[cache_key] = output
+            except HTTPStatusError as e:
+                if e.response.status_code == 409:
+                    # 409 conflict errors are caused by submitting duplicate runs
+                    return None
+                raise
         return exp_run
 
-    async def async_run_experiment(test_case: TestCase) -> ExperimentRun:
+    async def async_run_experiment(test_case: TestCase) -> Optional[ExperimentRun]:
         example, repetition_number = test_case.example, test_case.repetition_number
+        cache_key = (example.id, repetition_number)
+
+        # Check if we have a cached result
+        if cache_key in task_result_cache:
+            output = task_result_cache[cache_key]
+            exp_run = ExperimentRun(
+                start_time=datetime.now(
+                    timezone.utc
+                ),  # Use current time since we don't have the original span
+                end_time=datetime.now(timezone.utc),
+                experiment_id=experiment.id,
+                dataset_example_id=example.id,
+                repetition_number=repetition_number,
+                output=output,
+                error=None,
+                trace_id=None,  # No trace ID since we don't have the original span
+            )
+            if not dry_run:
+                try:
+                    # Try to create the run directly
+                    future = asyncio.get_running_loop().run_in_executor(
+                        None,
+                        functools.partial(
+                            sync_client.post,
+                            url=f"/v1/experiments/{experiment.id}/runs",
+                            json=jsonify(exp_run),
+                        ),
+                    )
+                    resp = await future
+                    resp.raise_for_status()
+                    exp_run = replace(exp_run, id=resp.json()["data"]["id"])
+                except HTTPStatusError as e:
+                    if e.response.status_code == 409:
+                        # 409 conflict errors are caused by submitting duplicate runs
+                        return None
+                    raise
+            return exp_run
+
         output = None
         error: Optional[BaseException] = None
         status = Status(StatusCode.OK)
@@ -344,6 +430,7 @@ def run_experiment(
         assert isinstance(
             output, (dict, list, str, int, float, bool, type(None))
         ), "Output must be JSON serializable"
+
         exp_run = ExperimentRun(
             start_time=_decode_unix_nano(cast(int, span.start_time)),
             end_time=_decode_unix_nano(cast(int, span.end_time)),
@@ -355,19 +442,26 @@ def run_experiment(
             trace_id=_str_trace_id(span.get_span_context().trace_id),  # type: ignore[no-untyped-call]
         )
         if not dry_run:
-            # Below is a workaround to avoid timeout errors sometimes
-            # encountered when the task is a synchronous function that
-            # blocks for too long.
-            resp = await asyncio.get_running_loop().run_in_executor(
-                None,
-                functools.partial(
-                    sync_client.post,
-                    url=f"/v1/experiments/{experiment.id}/runs",
-                    json=jsonify(exp_run),
-                ),
-            )
-            resp.raise_for_status()
-            exp_run = replace(exp_run, id=resp.json()["data"]["id"])
+            try:
+                # Try to create the run directly
+                future = asyncio.get_running_loop().run_in_executor(
+                    None,
+                    functools.partial(
+                        sync_client.post,
+                        url=f"/v1/experiments/{experiment.id}/runs",
+                        json=jsonify(exp_run),
+                    ),
+                )
+                resp = await future
+                resp.raise_for_status()
+                exp_run = replace(exp_run, id=resp.json()["data"]["id"])
+                if error is None:
+                    task_result_cache[cache_key] = output
+            except HTTPStatusError as e:
+                if e.response.status_code == 409:
+                    # Ignore duplicate runs - we'll get the final state from the database
+                    return None
+                raise
         return exp_run
 
     _errors: tuple[type[BaseException], ...]
@@ -401,6 +495,26 @@ def run_experiment(
     ]
     task_runs, _execution_details = executor.run(test_cases)
     print("✅ Task runs completed.")
+
+    # Get the final state of runs from the database
+    if not dry_run:
+        all_runs = sync_client.get(f"/v1/experiments/{experiment.id}/runs").json()["data"]
+        task_runs = []
+        for run in all_runs:
+            # Parse datetime strings
+            run["start_time"] = datetime.fromisoformat(run["start_time"])
+            run["end_time"] = datetime.fromisoformat(run["end_time"])
+            task_runs.append(ExperimentRun.from_dict(run))
+
+        # Check if we got all expected runs
+        expected_runs = len(dataset.examples) * repetitions
+        actual_runs = len(task_runs)
+        if actual_runs < expected_runs:
+            print(
+                f"⚠️  Warning: Only {actual_runs} out of {expected_runs} expected runs were "
+                "completed successfully."
+            )
+
     params = ExperimentParameters(n_examples=len(dataset.examples), n_repetitions=repetitions)
     task_summary = TaskSummary.from_task_runs(params, task_runs)
     ran_experiment: RanExperiment = object.__new__(RanExperiment)

{arize_phoenix-10.0.3 → arize_phoenix-10.1.0}/src/phoenix/server/api/helpers/playground_clients.py

@@ -701,6 +701,10 @@ class AzureOpenAIStreamingClient(OpenAIBaseStreamingClient):
     provider_key=GenerativeProviderKey.ANTHROPIC,
     model_names=[
         PROVIDER_DEFAULT,
+        "claude-sonnet-4-0",
+        "claude-sonnet-4-20250514",
+        "claude-opus-4-0",
+        "claude-opus-4-20250514",
         "claude-3-7-sonnet-latest",
         "claude-3-7-sonnet-20250219",
         "claude-3-5-sonnet-latest",

{arize_phoenix-10.0.3 → arize_phoenix-10.1.0}/src/phoenix/server/api/routers/v1/experiment_runs.py

@@ -4,8 +4,10 @@ from typing import Any, Optional
 from fastapi import APIRouter, HTTPException
 from pydantic import Field
 from sqlalchemy import select
+from sqlalchemy.exc import IntegrityError as PostgreSQLIntegrityError
+from sqlean.dbapi2 import IntegrityError as SQLiteIntegrityError  # type: ignore[import-untyped]
 from starlette.requests import Request
-from starlette.status import HTTP_404_NOT_FOUND
+from starlette.status import HTTP_404_NOT_FOUND, HTTP_409_CONFLICT
 from strawberry.relay import GlobalID
 
 from phoenix.db import models
@@ -58,7 +60,11 @@ class CreateExperimentRunResponseBody(ResponseBody[CreateExperimentRunResponseBo
             {
                 "status_code": HTTP_404_NOT_FOUND,
                 "description": "Experiment or dataset example not found",
-            }
+            },
+            {
+                "status_code": HTTP_409_CONFLICT,
+                "description": "This experiment run has already been submitted",
+            },
         ]
     ),
 )
@@ -101,8 +107,14 @@ async def create_experiment_run(
             end_time=end_time,
             error=error,
         )
-        session.add(exp_run)
-        await session.flush()
+        try:
+            session.add(exp_run)
+            await session.flush()
+        except (PostgreSQLIntegrityError, SQLiteIntegrityError):
+            raise HTTPException(
+                detail="This experiment run has already been submitted",
+                status_code=HTTP_409_CONFLICT,
+            )
     request.state.event_queue.put(ExperimentRunInsertEvent((exp_run.id,)))
     run_gid = GlobalID("ExperimentRun", str(exp_run.id))
     return CreateExperimentRunResponseBody(