arize-phoenix 10.0.3__py3-none-any.whl → 10.0.4__py3-none-any.whl

{arize_phoenix-10.0.3.dist-info → arize_phoenix-10.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arize-phoenix
-Version: 10.0.3
+Version: 10.0.4
 Summary: AI Observability and Evaluation
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues

{arize_phoenix-10.0.3.dist-info → arize_phoenix-10.0.4.dist-info}/RECORD

@@ -6,7 +6,7 @@ phoenix/exceptions.py,sha256=n2L2KKuecrdflB9MsCdAYCiSEvGJptIsfRkXMoJle7A,169
 phoenix/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 phoenix/services.py,sha256=ngkyKGVatX3cO2WJdo2hKdaVKP-xJCMvqthvga6kJss,5196
 phoenix/settings.py,sha256=x87BX7hWGQQZbrW_vrYqFR_izCGfO9gFc--JXUG4Tdk,754
-phoenix/version.py,sha256=SBPze3-vOQ7xHI2sABcoDWHzmJDMtJBs9M1wdSz1A3E,23
+phoenix/version.py,sha256=gyWMSD5bIZRum86t3FjwjQ0gz3poNiCaCrUA12HtJp8,23
 phoenix/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/core/embedding_dimension.py,sha256=zKGbcvwOXgLf-yrJBpQyKtd-LEOPRKHnUToyAU8Owis,87
 phoenix/core/model.py,sha256=qBFraOtmwCCnWJltKNP18DDG0mULXigytlFsa6YOz6k,4837
@@ -56,7 +56,7 @@ phoenix/db/types/identifier.py,sha256=Opr3_1di6e5ncrBDn30WfBSr-jN_VGBnkkA4BMuSoy
 phoenix/db/types/model_provider.py,sha256=96UMeqiy5X9PmYMOWA6dZAmI_BSV3yVxt9HEVYGe5Ns,157
 phoenix/db/types/trace_retention.py,sha256=UoLVX4efhv-mcF32zHpMX_hXWe8474cmhEyPmYqeMfI,9573
 phoenix/experiments/__init__.py,sha256=6JGwgUd7xCbGpuHqYZlsmErmYvVgv7N_j43bn3dUqsk,123
-phoenix/experiments/functions.py,sha256=hmTLqNgrWGFWQlQnWvNgOrpeWStGvbzQkD-1Tx-wM94,33270
+phoenix/experiments/functions.py,sha256=6cSS_5O5V1EZ_5CIQ6lGOUwFyNNsADP7Uu__GmuOz4A,37983
 phoenix/experiments/tracing.py,sha256=seNh9rBH-rtQe8_FPI_VJj1rbo3ADcP6wDvERkMoxNc,2858
 phoenix/experiments/types.py,sha256=yntt6fnAny1U4Q9Y5Mm4ZYIb9319OaJovl-kyXFtGQE,23475
 phoenix/experiments/utils.py,sha256=MZ1-OnTcavk_KUtbfGqt55Fk9TGtJpYG_K71WsN-zDk,785
@@ -235,7 +235,7 @@ phoenix/server/api/routers/v1/annotations.py,sha256=oeafR2tCLu-uIwM9J72gN3MX5WDh
 phoenix/server/api/routers/v1/datasets.py,sha256=Wqiy6ZKqn4BZSFyn93gzuhWx3mGn7kOkNncHzCWuBq8,37325
 phoenix/server/api/routers/v1/evaluations.py,sha256=GFTo42aIEX0Htn0EjjoE1JZDYlvryeZ_CK9kowhwzGw,12830
 phoenix/server/api/routers/v1/experiment_evaluations.py,sha256=xSs004jNYsOl3eg-6Zjo2tt9TefTd7WR3twWYrsNQNk,4828
-phoenix/server/api/routers/v1/experiment_runs.py,sha256=jqpquCygtUYNNN7lgSvGvOlXCE7KTleDRFjxJ7bbDfM,6400
+phoenix/server/api/routers/v1/experiment_runs.py,sha256=FreGzzprPpJ_DBHUkdUckca6EGCrnvgVpqk3CLT7wRc,7000
 phoenix/server/api/routers/v1/experiments.py,sha256=V9_sxqLTE1MKGFu9H3FEdGKr70lYMbGZx813MGaavfQ,20430
 phoenix/server/api/routers/v1/models.py,sha256=r0nM2kFJ3mxDqgc5vFr1cjNuyOPs3RIKE_DS2VMdF48,1749
 phoenix/server/api/routers/v1/projects.py,sha256=RVOAWW8RQIqaebQFcIbk1OGxO7B1BITtUewlcSUhasg,12615
@@ -397,9 +397,9 @@ phoenix/utilities/project.py,sha256=auVpARXkDb-JgeX5f2aStyFIkeKvGwN9l7qrFeJMVxI,
 phoenix/utilities/re.py,sha256=6YyUWIkv0zc2SigsxfOWIHzdpjKA_TZo2iqKq7zJKvw,2081
 phoenix/utilities/span_store.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/utilities/template_formatters.py,sha256=gh9PJD6WEGw7TEYXfSst1UR4pWWwmjxMLrDVQ_CkpkQ,2779
-arize_phoenix-10.0.3.dist-info/METADATA,sha256=5iF5zvSMrc2EboqD4A3zQIJZQIzDBs-E8xDk7nvaAb4,25591
-arize_phoenix-10.0.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-arize_phoenix-10.0.3.dist-info/entry_points.txt,sha256=Pgpn8Upxx9P8z8joPXZWl2LlnAlGc3gcQoVchb06X1Q,94
-arize_phoenix-10.0.3.dist-info/licenses/IP_NOTICE,sha256=JBqyyCYYxGDfzQ0TtsQgjts41IJoa-hiwDrBjCb9gHM,469
-arize_phoenix-10.0.3.dist-info/licenses/LICENSE,sha256=HFkW9REuMOkvKRACuwLPT0hRydHb3zNg-fdFt94td18,3794
-arize_phoenix-10.0.3.dist-info/RECORD,,
+arize_phoenix-10.0.4.dist-info/METADATA,sha256=Xywjo53oBnsHEco90Wn_5l0uSpp5XcDVhT2MjjG8VNg,25591
+arize_phoenix-10.0.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+arize_phoenix-10.0.4.dist-info/entry_points.txt,sha256=Pgpn8Upxx9P8z8joPXZWl2LlnAlGc3gcQoVchb06X1Q,94
+arize_phoenix-10.0.4.dist-info/licenses/IP_NOTICE,sha256=JBqyyCYYxGDfzQ0TtsQgjts41IJoa-hiwDrBjCb9gHM,469
+arize_phoenix-10.0.4.dist-info/licenses/LICENSE,sha256=HFkW9REuMOkvKRACuwLPT0hRydHb3zNg-fdFt94td18,3794
+arize_phoenix-10.0.4.dist-info/RECORD,,

phoenix/experiments/functions.py

@@ -16,6 +16,7 @@ from urllib.parse import urljoin
 import httpx
 import opentelemetry.sdk.trace as trace_sdk
 import pandas as pd
+from httpx import HTTPStatusError
 from openinference.semconv.resource import ResourceAttributes
 from openinference.semconv.trace import (
     OpenInferenceMimeTypeValues,
@@ -232,8 +233,43 @@ def run_experiment(
         print(f"📺 View dataset experiments: {dataset_experiments_url}")
         print(f"🔗 View this experiment: {experiment_compare_url}")
 
-    def sync_run_experiment(test_case: TestCase) -> ExperimentRun:
+    # Create a cache for task results
+    task_result_cache: dict[tuple[str, int], Any] = {}
+
+    def sync_run_experiment(test_case: TestCase) -> Optional[ExperimentRun]:
         example, repetition_number = test_case.example, test_case.repetition_number
+        cache_key = (example.id, repetition_number)
+
+        # Check if we have a cached result
+        if cache_key in task_result_cache:
+            output = task_result_cache[cache_key]
+            exp_run = ExperimentRun(
+                start_time=datetime.now(
+                    timezone.utc
+                ),  # Use current time since we don't have the original span
+                end_time=datetime.now(timezone.utc),
+                experiment_id=experiment.id,
+                dataset_example_id=example.id,
+                repetition_number=repetition_number,
+                output=output,
+                error=None,
+                trace_id=None,  # No trace ID since we don't have the original span
+            )
+            if not dry_run:
+                try:
+                    # Try to create the run directly
+                    resp = sync_client.post(
+                        f"/v1/experiments/{experiment.id}/runs", json=jsonify(exp_run)
+                    )
+                    resp.raise_for_status()
+                    exp_run = replace(exp_run, id=resp.json()["data"]["id"])
+                except HTTPStatusError as e:
+                    if e.response.status_code == 409:
+                        # Ignore duplicate runs - we'll get the final state from the database
+                        return None
+                    raise
+            return exp_run
+
         output = None
         error: Optional[BaseException] = None
         status = Status(StatusCode.OK)
@@ -283,6 +319,7 @@ def run_experiment(
         assert isinstance(
             output, (dict, list, str, int, float, bool, type(None))
         ), "Output must be JSON serializable"
+
         exp_run = ExperimentRun(
             start_time=_decode_unix_nano(cast(int, span.start_time)),
             end_time=_decode_unix_nano(cast(int, span.end_time)),
@@ -294,13 +331,62 @@ def run_experiment(
             trace_id=_str_trace_id(span.get_span_context().trace_id),  # type: ignore[no-untyped-call]
         )
         if not dry_run:
-            resp = sync_client.post(f"/v1/experiments/{experiment.id}/runs", json=jsonify(exp_run))
-            resp.raise_for_status()
-            exp_run = replace(exp_run, id=resp.json()["data"]["id"])
+            try:
+                # Try to create the run directly
+                resp = sync_client.post(
+                    f"/v1/experiments/{experiment.id}/runs", json=jsonify(exp_run)
+                )
+                resp.raise_for_status()
+                exp_run = replace(exp_run, id=resp.json()["data"]["id"])
+                if error is None:
+                    task_result_cache[cache_key] = output
+            except HTTPStatusError as e:
+                if e.response.status_code == 409:
+                    # 409 conflict errors are caused by submitting duplicate runs
+                    return None
+                raise
         return exp_run
 
-    async def async_run_experiment(test_case: TestCase) -> ExperimentRun:
+    async def async_run_experiment(test_case: TestCase) -> Optional[ExperimentRun]:
         example, repetition_number = test_case.example, test_case.repetition_number
+        cache_key = (example.id, repetition_number)
+
+        # Check if we have a cached result
+        if cache_key in task_result_cache:
+            output = task_result_cache[cache_key]
+            exp_run = ExperimentRun(
+                start_time=datetime.now(
+                    timezone.utc
+                ),  # Use current time since we don't have the original span
+                end_time=datetime.now(timezone.utc),
+                experiment_id=experiment.id,
+                dataset_example_id=example.id,
+                repetition_number=repetition_number,
+                output=output,
+                error=None,
+                trace_id=None,  # No trace ID since we don't have the original span
+            )
+            if not dry_run:
+                try:
+                    # Try to create the run directly
+                    future = asyncio.get_running_loop().run_in_executor(
+                        None,
+                        functools.partial(
+                            sync_client.post,
+                            url=f"/v1/experiments/{experiment.id}/runs",
+                            json=jsonify(exp_run),
+                        ),
+                    )
+                    resp = await future
+                    resp.raise_for_status()
+                    exp_run = replace(exp_run, id=resp.json()["data"]["id"])
+                except HTTPStatusError as e:
+                    if e.response.status_code == 409:
+                        # 409 conflict errors are caused by submitting duplicate runs
+                        return None
+                    raise
+            return exp_run
+
         output = None
         error: Optional[BaseException] = None
         status = Status(StatusCode.OK)
@@ -344,6 +430,7 @@ def run_experiment(
         assert isinstance(
             output, (dict, list, str, int, float, bool, type(None))
         ), "Output must be JSON serializable"
+
         exp_run = ExperimentRun(
             start_time=_decode_unix_nano(cast(int, span.start_time)),
             end_time=_decode_unix_nano(cast(int, span.end_time)),
@@ -355,19 +442,26 @@ def run_experiment(
             trace_id=_str_trace_id(span.get_span_context().trace_id),  # type: ignore[no-untyped-call]
         )
         if not dry_run:
-            # Below is a workaround to avoid timeout errors sometimes
-            # encountered when the task is a synchronous function that
-            # blocks for too long.
-            resp = await asyncio.get_running_loop().run_in_executor(
-                None,
-                functools.partial(
-                    sync_client.post,
-                    url=f"/v1/experiments/{experiment.id}/runs",
-                    json=jsonify(exp_run),
-                ),
-            )
-            resp.raise_for_status()
-            exp_run = replace(exp_run, id=resp.json()["data"]["id"])
+            try:
+                # Try to create the run directly
+                future = asyncio.get_running_loop().run_in_executor(
+                    None,
+                    functools.partial(
+                        sync_client.post,
+                        url=f"/v1/experiments/{experiment.id}/runs",
+                        json=jsonify(exp_run),
+                    ),
+                )
+                resp = await future
+                resp.raise_for_status()
+                exp_run = replace(exp_run, id=resp.json()["data"]["id"])
+                if error is None:
+                    task_result_cache[cache_key] = output
+            except HTTPStatusError as e:
+                if e.response.status_code == 409:
+                    # Ignore duplicate runs - we'll get the final state from the database
+                    return None
+                raise
         return exp_run
 
     _errors: tuple[type[BaseException], ...]
@@ -401,6 +495,26 @@ def run_experiment(
     ]
     task_runs, _execution_details = executor.run(test_cases)
     print("✅ Task runs completed.")
+
+    # Get the final state of runs from the database
+    if not dry_run:
+        all_runs = sync_client.get(f"/v1/experiments/{experiment.id}/runs").json()["data"]
+        task_runs = []
+        for run in all_runs:
+            # Parse datetime strings
+            run["start_time"] = datetime.fromisoformat(run["start_time"])
+            run["end_time"] = datetime.fromisoformat(run["end_time"])
+            task_runs.append(ExperimentRun.from_dict(run))
+
+        # Check if we got all expected runs
+        expected_runs = len(dataset.examples) * repetitions
+        actual_runs = len(task_runs)
+        if actual_runs < expected_runs:
+            print(
+                f"⚠️  Warning: Only {actual_runs} out of {expected_runs} expected runs were "
+                "completed successfully."
+            )
+
     params = ExperimentParameters(n_examples=len(dataset.examples), n_repetitions=repetitions)
     task_summary = TaskSummary.from_task_runs(params, task_runs)
     ran_experiment: RanExperiment = object.__new__(RanExperiment)

phoenix/server/api/routers/v1/experiment_runs.py

@@ -4,8 +4,10 @@ from typing import Any, Optional
 from fastapi import APIRouter, HTTPException
 from pydantic import Field
 from sqlalchemy import select
+from sqlalchemy.exc import IntegrityError as PostgreSQLIntegrityError
+from sqlean.dbapi2 import IntegrityError as SQLiteIntegrityError  # type: ignore[import-untyped]
 from starlette.requests import Request
-from starlette.status import HTTP_404_NOT_FOUND
+from starlette.status import HTTP_404_NOT_FOUND, HTTP_409_CONFLICT
 from strawberry.relay import GlobalID
 
 from phoenix.db import models
@@ -58,7 +60,11 @@ class CreateExperimentRunResponseBody(ResponseBody[CreateExperimentRunResponseBo
             {
                 "status_code": HTTP_404_NOT_FOUND,
                 "description": "Experiment or dataset example not found",
-            }
+            },
+            {
+                "status_code": HTTP_409_CONFLICT,
+                "description": "This experiment run has already been submitted",
+            },
         ]
     ),
 )
@@ -101,8 +107,14 @@ async def create_experiment_run(
             end_time=end_time,
             error=error,
         )
-        session.add(exp_run)
-        await session.flush()
+        try:
+            session.add(exp_run)
+            await session.flush()
+        except (PostgreSQLIntegrityError, SQLiteIntegrityError):
+            raise HTTPException(
+                detail="This experiment run has already been submitted",
+                status_code=HTTP_409_CONFLICT,
+            )
     request.state.event_queue.put(ExperimentRunInsertEvent((exp_run.id,)))
     run_gid = GlobalID("ExperimentRun", str(exp_run.id))
     return CreateExperimentRunResponseBody(

phoenix/version.py

@@ -1 +1 @@
-__version__ = "10.0.3"
+__version__ = "10.0.4"