arize-phoenix 11.37.0__tar.gz → 11.38.0__tar.gz

{arize_phoenix-11.37.0 → arize_phoenix-11.38.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arize-phoenix
-Version: 11.37.0
+Version: 11.38.0
 Summary: AI Observability and Evaluation
 Project-URL: Documentation, https://arize.com/docs/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues

{arize_phoenix-11.37.0 → arize_phoenix-11.38.0}/src/phoenix/server/api/helpers/playground_spans.py

@@ -222,6 +222,7 @@ def get_db_experiment_run(
     *,
     experiment_id: int,
     example_id: int,
+    repetition_number: int,
 ) -> models.ExperimentRun:
     return models.ExperimentRun(
         experiment_id=experiment_id,
@@ -230,7 +231,7 @@ def get_db_experiment_run(
         output=models.ExperimentRunOutput(
             task_output=get_dataset_example_output(db_span),
         ),
-        repetition_number=1,
+        repetition_number=repetition_number,
         start_time=db_span.start_time,
         end_time=db_span.end_time,
         error=db_span.status_message or None,

{arize_phoenix-11.37.0 → arize_phoenix-11.38.0}/src/phoenix/server/api/input_types/ChatCompletionInput.py

@@ -26,6 +26,7 @@ class ChatCompletionInput:
     credentials: Optional[list[GenerativeCredentialInput]] = UNSET
     template: Optional[PromptTemplateOptions] = UNSET
     prompt_name: Optional[Identifier] = None
+    repetitions: int
 
 
 @strawberry.input
@@ -36,6 +37,7 @@ class ChatCompletionOverDatasetInput:
     tools: Optional[list[JSON]] = UNSET
     credentials: Optional[list[GenerativeCredentialInput]] = UNSET
     template_format: PromptTemplateFormat = PromptTemplateFormat.MUSTACHE
+    repetitions: int
     dataset_id: GlobalID
     dataset_version_id: Optional[GlobalID] = None
     experiment_name: Optional[str] = None

{arize_phoenix-11.37.0 → arize_phoenix-11.38.0}/src/phoenix/server/api/mutations/chat_mutations.py

@@ -112,6 +112,7 @@ class ChatCompletionMutationError:
 @strawberry.type
 class ChatCompletionOverDatasetMutationExamplePayload:
     dataset_example_id: GlobalID
+    repetition_number: int
     experiment_run_id: GlobalID
     result: Union[ChatCompletionMutationPayload, ChatCompletionMutationError]
 
@@ -207,7 +208,12 @@ class ChatCompletionMutationMixin:
         results: list[Union[ChatCompletionMutationPayload, BaseException]] = []
         batch_size = 3
         start_time = datetime.now(timezone.utc)
-        for batch in _get_batches(revisions, batch_size):
+        unbatched_items = [
+            (revision, repetition_number)
+            for revision in revisions
+            for repetition_number in range(1, input.repetitions + 1)
+        ]
+        for batch in _get_batches(unbatched_items, batch_size):
             batch_results = await asyncio.gather(
                 *(
                     cls._chat_completion(
@@ -224,10 +230,11 @@ class ChatCompletionMutationMixin:
                                 variables=revision.input,
                             ),
                             prompt_name=input.prompt_name,
+                            repetitions=repetition_number,
                         ),
                         project_name=project_name,
                     )
-                    for revision in batch
+                    for revision, repetition_number in batch
                 ),
                 return_exceptions=True,
             )
@@ -239,13 +246,13 @@ class ChatCompletionMutationMixin:
             experiment_id=GlobalID(models.Experiment.__name__, str(experiment.id)),
         )
         experiment_runs = []
-        for revision, result in zip(revisions, results):
+        for (revision, repetition_number), result in zip(unbatched_items, results):
             if isinstance(result, BaseException):
                 experiment_run = models.ExperimentRun(
                     experiment_id=experiment.id,
                     dataset_example_id=revision.dataset_example_id,
                     output={},
-                    repetition_number=1,
+                    repetition_number=repetition_number,
                     start_time=start_time,
                     end_time=start_time,
                     error=str(result),
@@ -261,7 +268,7 @@ class ChatCompletionMutationMixin:
                     ),
                     prompt_token_count=db_span.cumulative_llm_token_count_prompt,
                     completion_token_count=db_span.cumulative_llm_token_count_completion,
-                    repetition_number=1,
+                    repetition_number=repetition_number,
                     start_time=db_span.start_time,
                     end_time=db_span.end_time,
                     error=str(result.error_message) if result.error_message else None,
@@ -272,13 +279,16 @@ class ChatCompletionMutationMixin:
             session.add_all(experiment_runs)
             await session.flush()
 
-        for revision, experiment_run, result in zip(revisions, experiment_runs, results):
+        for (revision, repetition_number), experiment_run, result in zip(
+            unbatched_items, experiment_runs, results
+        ):
             dataset_example_id = GlobalID(
                 models.DatasetExample.__name__, str(revision.dataset_example_id)
             )
             experiment_run_id = GlobalID(models.ExperimentRun.__name__, str(experiment_run.id))
             example_payload = ChatCompletionOverDatasetMutationExamplePayload(
                 dataset_example_id=dataset_example_id,
+                repetition_number=repetition_number,
                 experiment_run_id=experiment_run_id,
                 result=result
                 if isinstance(result, ChatCompletionMutationPayload)

{arize_phoenix-11.37.0 → arize_phoenix-11.38.0}/src/phoenix/server/api/subscriptions.py

@@ -308,7 +308,7 @@ class Subscription:
                 name=input.experiment_name
                 or _default_playground_experiment_name(input.prompt_name),
                 description=input.experiment_description,
-                repetitions=1,
+                repetitions=input.repetitions,
                 metadata_=input.experiment_metadata or dict(),
                 project_name=project_name,
             )
@@ -327,11 +327,13 @@ class Subscription:
                     llm_client=llm_client,
                     revision=revision,
                     results=results,
+                    repetition_number=repetition_number,
                     experiment_id=experiment.id,
                     project_id=playground_project_id,
                 ),
             )
             for revision in revisions
+            for repetition_number in range(1, input.repetitions + 1)
         ]
         in_progress: list[
             tuple[
@@ -409,6 +411,7 @@ async def _stream_chat_completion_over_dataset_example(
     input: ChatCompletionOverDatasetInput,
     llm_client: PlaygroundStreamingClient,
     revision: models.DatasetExampleRevision,
+    repetition_number: int,
     results: asyncio.Queue[ChatCompletionResult],
     experiment_id: int,
     project_id: int,
@@ -435,7 +438,11 @@ async def _stream_chat_completion_over_dataset_example(
         )
     except TemplateFormatterError as error:
         format_end_time = cast(datetime, normalize_datetime(dt=local_now(), tz=timezone.utc))
-        yield ChatCompletionSubscriptionError(message=str(error), dataset_example_id=example_id)
+        yield ChatCompletionSubscriptionError(
+            message=str(error),
+            dataset_example_id=example_id,
+            repetition_number=repetition_number,
+        )
         await results.put(
             (
                 example_id,
@@ -445,7 +452,7 @@ async def _stream_chat_completion_over_dataset_example(
                     dataset_example_id=revision.dataset_example_id,
                     trace_id=None,
                     output={},
-                    repetition_number=1,
+                    repetition_number=repetition_number,
                     start_time=format_start_time,
                     end_time=format_end_time,
                     error=str(error),
@@ -465,17 +472,24 @@ async def _stream_chat_completion_over_dataset_example(
         ):
             span.add_response_chunk(chunk)
             chunk.dataset_example_id = example_id
+            chunk.repetition_number = repetition_number
             yield chunk
         span.set_attributes(llm_client.attributes)
     db_trace = get_db_trace(span, project_id)
     db_span = get_db_span(span, db_trace)
     db_run = get_db_experiment_run(
-        db_span, db_trace, experiment_id=experiment_id, example_id=revision.dataset_example_id
+        db_span,
+        db_trace,
+        experiment_id=experiment_id,
+        example_id=revision.dataset_example_id,
+        repetition_number=repetition_number,
     )
     await results.put((example_id, db_span, db_run))
     if span.status_message is not None:
         yield ChatCompletionSubscriptionError(
-            message=span.status_message, dataset_example_id=example_id
+            message=span.status_message,
+            dataset_example_id=example_id,
+            repetition_number=repetition_number,
         )
 
 
@@ -511,6 +525,7 @@ async def _chat_completion_result_payloads(
             span=Span(span_rowid=span.id, db_span=span) if span else None,
             experiment_run=to_gql_experiment_run(run),
             dataset_example_id=example_id,
+            repetition_number=run.repetition_number,
         )
 
 

{arize_phoenix-11.37.0 → arize_phoenix-11.38.0}/src/phoenix/server/api/types/ChatCompletionSubscriptionPayload.py

@@ -11,6 +11,7 @@ from .Span import Span
 @strawberry.interface
 class ChatCompletionSubscriptionPayload:
     dataset_example_id: Optional[GlobalID] = None
+    repetition_number: Optional[int] = None
 
 
 @strawberry.type

{arize_phoenix-11.37.0 → arize_phoenix-11.38.0}/src/phoenix/server/static/.vite/manifest.json

@@ -1,22 +1,22 @@
 {
-  "_components-CFzdBkk_.js": {
-    "file": "assets/components-CFzdBkk_.js",
+  "_components-BQPHTBfv.js": {
+    "file": "assets/components-BQPHTBfv.js",
     "name": "components",
     "imports": [
       "_vendor-BdjZxMii.js",
       "_vendor-arizeai-CHYlS8jV.js",
-      "_pages-CvUhOO9h.js",
+      "_pages-C0Y17J0T.js",
       "_vendor-codemirror-Di6t4HnH.js",
       "_vendor-three-BLWp5bic.js"
     ]
   },
-  "_pages-CvUhOO9h.js": {
-    "file": "assets/pages-CvUhOO9h.js",
+  "_pages-C0Y17J0T.js": {
+    "file": "assets/pages-C0Y17J0T.js",
     "name": "pages",
     "imports": [
       "_vendor-BdjZxMii.js",
       "_vendor-arizeai-CHYlS8jV.js",
-      "_components-CFzdBkk_.js",
+      "_components-BQPHTBfv.js",
       "_vendor-codemirror-Di6t4HnH.js",
       "_vendor-recharts-C9wCDYj3.js"
     ]
@@ -75,15 +75,15 @@
     "name": "vendor-three"
   },
   "index.tsx": {
-    "file": "assets/index-DayUA9lQ.js",
+    "file": "assets/index-BL5BMgJU.js",
     "name": "index",
     "src": "index.tsx",
     "isEntry": true,
     "imports": [
       "_vendor-BdjZxMii.js",
       "_vendor-arizeai-CHYlS8jV.js",
-      "_pages-CvUhOO9h.js",
-      "_components-CFzdBkk_.js",
+      "_pages-C0Y17J0T.js",
+      "_components-BQPHTBfv.js",
       "_vendor-three-BLWp5bic.js",
       "_vendor-codemirror-Di6t4HnH.js",
       "_vendor-shiki-MNnmOotP.js",