arize-phoenix 4.7.1__tar.gz → 4.7.2__tar.gz

{arize_phoenix-4.7.1 → arize_phoenix-4.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: arize-phoenix
-Version: 4.7.1
+Version: 4.7.2
 Summary: AI Observability and Evaluation
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues

{arize_phoenix-4.7.1 → arize_phoenix-4.7.2}/src/phoenix/experiments/functions.py

@@ -1,3 +1,4 @@
+import asyncio
 import functools
 import inspect
 import json
@@ -62,7 +63,6 @@ from phoenix.experiments.types import (
     ExperimentEvaluationRun,
     ExperimentParameters,
     ExperimentRun,
-    ExperimentRunOutput,
     ExperimentTask,
     RanExperiment,
     TaskSummary,
@@ -284,7 +284,7 @@ def run_experiment(
             experiment_id=experiment.id,
             dataset_example_id=example.id,
             repetition_number=repetition_number,
-            experiment_run_output=ExperimentRunOutput(task_output=output),
+            output=output,
             error=repr(error) if error else None,
             trace_id=_str_trace_id(span.get_span_context().trace_id),  # type: ignore[no-untyped-call]
         )
@@ -345,13 +345,21 @@ def run_experiment(
             experiment_id=experiment.id,
             dataset_example_id=example.id,
             repetition_number=repetition_number,
-            experiment_run_output=ExperimentRunOutput(task_output=output),
+            output=output,
             error=repr(error) if error else None,
             trace_id=_str_trace_id(span.get_span_context().trace_id),  # type: ignore[no-untyped-call]
         )
         if not dry_run:
-            resp = await async_client.post(
-                f"/v1/experiments/{experiment.id}/runs", json=jsonify(exp_run)
+            # Below is a workaround to avoid timeout errors sometimes
+            # encountered when the task is a synchronous function that
+            # blocks for too long.
+            resp = await asyncio.get_running_loop().run_in_executor(
+                None,
+                functools.partial(
+                    sync_client.post,
+                    url=f"/v1/experiments/{experiment.id}/runs",
+                    json=jsonify(exp_run),
+                ),
             )
             resp.raise_for_status()
             exp_run = replace(exp_run, id=resp.json()["data"]["id"])
@@ -393,7 +401,7 @@ def run_experiment(
     ran_experiment.__init__(  # type: ignore[misc]
         params=params,
         dataset=dataset,
-        runs={r.id: r for r in task_runs},
+        runs={r.id: r for r in task_runs if r is not None},
         task_summary=task_summary,
         **_asdict(experiment),
     )
@@ -498,7 +506,7 @@ def evaluate_experiment(
             stack.enter_context(capture_spans(resource))
             try:
                 result = evaluator.evaluate(
-                    output=experiment_run.output,
+                    output=deepcopy(experiment_run.output),
                     expected=example.output,
                     reference=example.output,
                     input=example.input,
@@ -550,7 +558,7 @@ def evaluate_experiment(
             stack.enter_context(capture_spans(resource))
             try:
                 result = await evaluator.async_evaluate(
-                    output=experiment_run.output,
+                    output=deepcopy(experiment_run.output),
                     expected=example.output,
                     reference=example.output,
                     input=example.input,
@@ -582,7 +590,17 @@ def evaluate_experiment(
             trace_id=_str_trace_id(span.get_span_context().trace_id),  # type: ignore[no-untyped-call]
         )
         if not dry_run:
-            resp = await async_client.post("/v1/experiment_evaluations", json=jsonify(eval_run))
+            # Below is a workaround to avoid timeout errors sometimes
+            # encountered when the evaluator is a synchronous function
+            # that blocks for too long.
+            resp = await asyncio.get_running_loop().run_in_executor(
+                None,
+                functools.partial(
+                    sync_client.post,
+                    url="/v1/experiment_evaluations",
+                    json=jsonify(eval_run),
+                ),
+            )
             resp.raise_for_status()
             eval_run = replace(eval_run, id=resp.json()["data"]["id"])
         return eval_run

{arize_phoenix-4.7.1 → arize_phoenix-4.7.2}/src/phoenix/experiments/types.py

@@ -198,20 +198,6 @@ class Experiment:
         )
 
 
-@dataclass(frozen=True)
-class ExperimentRunOutput:
-    task_output: TaskOutput
-
-    def __post_init__(self) -> None:
-        object.__setattr__(self, "task_output", _make_read_only(self.task_output))
-
-    @classmethod
-    def from_dict(cls, obj: Optional[Mapping[str, Any]]) -> ExperimentRunOutput:
-        if not obj:
-            return cls(task_output=None)
-        return cls(task_output=obj["task_output"])
-
-
 @dataclass(frozen=True)
 class ExperimentRun:
     start_time: datetime
@@ -219,15 +205,11 @@ class ExperimentRun:
     experiment_id: ExperimentId
     dataset_example_id: ExampleId
     repetition_number: RepetitionNumber
-    experiment_run_output: ExperimentRunOutput
+    output: JSONSerializable
     error: Optional[str] = None
     id: ExperimentRunId = field(default_factory=_dry_run_id)
     trace_id: Optional[TraceId] = None
 
-    @property
-    def output(self) -> Optional[TaskOutput]:
-        return deepcopy(self.experiment_run_output.task_output)
-
     @classmethod
     def from_dict(cls, obj: Mapping[str, Any]) -> ExperimentRun:
         return cls(
@@ -236,14 +218,14 @@ class ExperimentRun:
             experiment_id=obj["experiment_id"],
             dataset_example_id=obj["dataset_example_id"],
             repetition_number=obj.get("repetition_number") or 1,
-            experiment_run_output=ExperimentRunOutput.from_dict(obj["experiment_run_output"]),
+            output=_make_read_only(obj.get("output")),
             error=obj.get("error"),
             id=obj["id"],
             trace_id=obj.get("trace_id"),
         )
 
     def __post_init__(self) -> None:
-        if bool(self.experiment_run_output) == bool(self.error):
+        if bool(self.output) == bool(self.error):
             ValueError("Must specify exactly one of experiment_run_output or error")
 
 
@@ -381,7 +363,7 @@ class EvaluationSummary(_HasStats):
         if df.empty:
             df = pd.DataFrame.from_records(
                 [
-                    {"evaluator": name, "error": True, "score": None, "label": None}
+                    {"evaluator": name, "error": None, "score": None, "label": None}
                     for name in params.eval_names
                 ]
             )
@@ -571,7 +553,7 @@ class RanExperiment(Experiment):
                 {
                     "run_id": run.id,
                     "error": run.error,
-                    "output": deepcopy(run.experiment_run_output.task_output),
+                    "output": deepcopy(run.output),
                     "input": deepcopy((ex := self.dataset.examples[run.dataset_example_id]).input),
                     "expected": deepcopy(ex.output),
                     "metadata": deepcopy(ex.metadata),

{arize_phoenix-4.7.1 → arize_phoenix-4.7.2}/src/phoenix/server/api/routers/v1/experiment_runs.py

@@ -7,9 +7,8 @@ from starlette.status import HTTP_404_NOT_FOUND
 from strawberry.relay import GlobalID
 
 from phoenix.db import models
-from phoenix.experiments.types import ExperimentRun, ExperimentRunOutput
+from phoenix.db.models import ExperimentRunOutput
 from phoenix.server.api.types.node import from_global_id_with_expected_type
-from phoenix.utilities.json import jsonify
 
 
 async def create_experiment_run(request: Request) -> Response:
@@ -39,9 +38,8 @@ async def create_experiment_run(request: Request) -> Response:
               trace_id:
                 type: string
                 description: Optional trace ID for tracking
-              experiment_run_output:
-                type: object
-                description: The output of the experiment run
+              output:
+                description: The output of the experiment task
               repetition_number:
                 type: integer
                 description: The repetition number of the experiment run
@@ -101,7 +99,7 @@ async def create_experiment_run(request: Request) -> Response:
         )
 
     trace_id = payload.get("trace_id", None)
-    output = payload["experiment_run_output"]
+    task_output = payload["output"]
     repetition_number = payload["repetition_number"]
     start_time = payload["start_time"]
     end_time = payload["end_time"]
@@ -112,7 +110,7 @@ async def create_experiment_run(request: Request) -> Response:
             experiment_id=experiment_id,
             dataset_example_id=dataset_example_id,
             trace_id=trace_id,
-            output=output,
+            output=ExperimentRunOutput(task_output=task_output),
             repetition_number=repetition_number,
             start_time=datetime.fromisoformat(start_time),
             end_time=datetime.fromisoformat(end_time),
@@ -170,9 +168,8 @@ async def list_experiment_runs(request: Request) -> Response:
                         type: string
                         format: date-time
                         description: The end time of the experiment run in ISO format
-                      experiment_run_output:
-                        type: object
-                        description: The output of the experiment run
+                      output:
+                        description: The output of the experiment task
                       error:
                         type: string
                         description: Error message if the experiment run encountered an error
@@ -205,16 +202,16 @@ async def list_experiment_runs(request: Request) -> Response:
             experiment_gid = GlobalID("Experiment", str(exp_run.experiment_id))
             example_gid = GlobalID("DatasetExample", str(exp_run.dataset_example_id))
             runs.append(
-                ExperimentRun(
-                    start_time=exp_run.start_time,
-                    end_time=exp_run.end_time,
+                dict(
+                    start_time=exp_run.start_time.isoformat(),
+                    end_time=exp_run.end_time.isoformat(),
                     experiment_id=str(experiment_gid),
                     dataset_example_id=str(example_gid),
                     repetition_number=exp_run.repetition_number,
-                    experiment_run_output=ExperimentRunOutput.from_dict(exp_run.output),
+                    output=exp_run.output.get("task_output"),
                     error=exp_run.error,
                     id=str(run_gid),
                     trace_id=exp_run.trace_id,
                 )
             )
-    return JSONResponse(content={"data": jsonify(runs)}, status_code=200)
+    return JSONResponse(content={"data": runs}, status_code=200)