arize-phoenix 4.4.4rc4__py3-none-any.whl → 4.4.4rc6__py3-none-any.whl

phoenix/session/client.py

@@ -1,6 +1,7 @@
 import csv
 import gzip
 import logging
+import re
 import weakref
 from collections import Counter
 from datetime import datetime
@@ -15,6 +16,7 @@ from typing import (
     Literal,
     Mapping,
     Optional,
+    Sequence,
     Tuple,
     Union,
     cast,
@@ -24,6 +26,7 @@ from urllib.parse import quote, urljoin
 import httpx
 import pandas as pd
 import pyarrow as pa
+from httpx import HTTPStatusError, Response
 from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
 from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue
 from opentelemetry.proto.resource.v1.resource_pb2 import Resource
@@ -38,9 +41,9 @@ from phoenix.config import (
     get_env_port,
     get_env_project_name,
 )
-from phoenix.datasets.types import Dataset, Example
 from phoenix.datetime_utils import normalize_datetime
 from phoenix.db.insertion.dataset import DatasetKeys
+from phoenix.experiments.types import Dataset, Example
 from phoenix.session.data_extractor import DEFAULT_SPAN_LIMIT, TraceDataExtractor
 from phoenix.trace import Evaluations, TraceDataset
 from phoenix.trace.dsl import SpanQuery
@@ -48,6 +51,8 @@ from phoenix.trace.otel import encode_span_to_otlp
 
 logger = logging.getLogger(__name__)
 
+DatasetAction: TypeAlias = Literal["create", "append"]
+
 
 class Client(TraceDataExtractor):
     def __init__(
@@ -88,6 +93,23 @@ class Client(TraceDataExtractor):
         if warn_if_server_not_running:
             self._warn_if_phoenix_is_not_running()
 
+    @property
+    def web_url(self) -> str:
+        """
+        Return the web URL of the Phoenix UI. This is different from the base
+        URL in the cases where there is a proxy like colab
+
+
+        Returns:
+            str: A fully qualified URL to the Phoenix UI.
+        """
+        # Avoid circular import
+        from phoenix.session.session import active_session
+
+        if session := active_session():
+            return session.url
+        return self._base_url
+
     def query_spans(
         self,
         *queries: SpanQuery,
@@ -125,7 +147,10 @@ class Client(TraceDataExtractor):
             end_time = end_time or stop_time
         response = self._client.post(
             url=urljoin(self._base_url, "v1/spans"),
-            params={"project-name": project_name},
+            params={
+                "project_name": project_name,
+                "project-name": project_name,  # for backward-compatibility
+            },
             json={
                 "queries": [q.to_dict() for q in queries],
                 "start_time": _to_iso_format(normalize_datetime(start_time)),
@@ -172,7 +197,10 @@ class Client(TraceDataExtractor):
         project_name = project_name or get_env_project_name()
         response = self._client.get(
             url=urljoin(self._base_url, "v1/evaluations"),
-            params={"project-name": project_name},
+            params={
+                "project_name": project_name,
+                "project-name": project_name,  # for backward-compatibility
+            },
         )
         if response.status_code == 404:
             logger.info("No evaluations found.")
@@ -329,12 +357,12 @@ class Client(TraceDataExtractor):
 
         response = self._client.get(
             urljoin(self._base_url, f"/v1/datasets/{quote(id)}/examples"),
-            params={"version-id": version_id} if version_id else None,
+            params={"version_id": version_id} if version_id else None,
         )
         response.raise_for_status()
         data = response.json()["data"]
-        examples = [
-            Example(
+        examples = {
+            example["id"]: Example(
                 id=example["id"],
                 input=example["input"],
                 output=example["output"],
@@ -342,7 +370,7 @@ class Client(TraceDataExtractor):
                 updated_at=datetime.fromisoformat(example["updated_at"]),
             )
             for example in data["examples"]
-        ]
+        }
         resolved_dataset_id = data["dataset_id"]
         resolved_version_id = data["version_id"]
         return Dataset(
@@ -399,7 +427,7 @@ class Client(TraceDataExtractor):
         url = f"v1/datasets/{dataset_id}/csv"
         response = httpx.get(
             url=urljoin(self._base_url, url),
-            params={"version": dataset_version_id} if dataset_version_id else {},
+            params={"version_id": dataset_version_id} if dataset_version_id else {},
         )
         response.raise_for_status()
         return pd.read_csv(
@@ -408,16 +436,170 @@ class Client(TraceDataExtractor):
         )
 
     def upload_dataset(
+        self,
+        *,
+        dataset_name: str,
+        dataframe: Optional[pd.DataFrame] = None,
+        csv_file_path: Optional[Union[str, Path]] = None,
+        input_keys: Iterable[str] = (),
+        output_keys: Iterable[str] = (),
+        metadata_keys: Iterable[str] = (),
+        inputs: Iterable[Mapping[str, Any]] = (),
+        outputs: Iterable[Mapping[str, Any]] = (),
+        metadata: Iterable[Mapping[str, Any]] = (),
+        dataset_description: Optional[str] = None,
+    ) -> Dataset:
+        """
+        Upload examples as dataset to the Phoenix server. If `dataframe` or
+        `csv_file_path` are provided, must also provide `input_keys` (and
+        optionally with `output_keys` or `metadata_keys` or both), which is a
+        list of strings denoting the column names in the dataframe or the csv
+        file. On the other hand, a sequence of dictionaries can also be provided
+        via `inputs` (and optionally with `outputs` or `metadat` or both), each
+        item of which represents a separate example in the dataset.
+
+        Args:
+            dataset_name: (str): Name of the dataset.
+            dataframe (pd.DataFrame): pandas DataFrame.
+            csv_file_path (str | Path): Location of a CSV text file
+            input_keys (Iterable[str]): List of column names used as input keys.
+                input_keys, output_keys, metadata_keys must be disjoint, and must
+                exist in CSV column headers.
+            output_keys (Iterable[str]): List of column names used as output keys.
+                input_keys, output_keys, metadata_keys must be disjoint, and must
+                exist in CSV column headers.
+            metadata_keys (Iterable[str]): List of column names used as metadata keys.
+                input_keys, output_keys, metadata_keys must be disjoint, and must
+                exist in CSV column headers.
+            inputs (Iterable[Mapping[str, Any]]): List of dictionaries object each
+                corresponding to an example in the dataset.
+            outputs (Iterable[Mapping[str, Any]]): List of dictionaries object each
+                corresponding to an example in the dataset.
+            metadata (Iterable[Mapping[str, Any]]): List of dictionaries object each
+                corresponding to an example in the dataset.
+            dataset_description: (Optional[str]): Description of the dataset.
+
+        Returns:
+            A Dataset object with the uploaded examples.
+        """
+        if dataframe is not None or csv_file_path is not None:
+            if dataframe is not None and csv_file_path is not None:
+                raise ValueError(
+                    "Please provide either `dataframe` or `csv_file_path`, but not both"
+                )
+            if list(inputs) or list(outputs) or list(metadata):
+                option = "dataframe" if dataframe is not None else "csv_file_path"
+                raise ValueError(
+                    f"Please provide only either `{option}` or list of dictionaries "
+                    f"via `inputs` (with `outputs` and `metadata`) but not both."
+                )
+            table = dataframe if dataframe is not None else csv_file_path
+            assert table is not None  # for type-checker
+            return self._upload_tabular_dataset(
+                table,
+                dataset_name=dataset_name,
+                input_keys=input_keys,
+                output_keys=output_keys,
+                metadata_keys=metadata_keys,
+                dataset_description=dataset_description,
+            )
+        return self._upload_json_dataset(
+            dataset_name=dataset_name,
+            inputs=inputs,
+            outputs=outputs,
+            metadata=metadata,
+            dataset_description=dataset_description,
+        )
+
+    def append_to_dataset(
+        self,
+        *,
+        dataset_name: str,
+        dataframe: Optional[pd.DataFrame] = None,
+        csv_file_path: Optional[Union[str, Path]] = None,
+        input_keys: Iterable[str] = (),
+        output_keys: Iterable[str] = (),
+        metadata_keys: Iterable[str] = (),
+        inputs: Iterable[Mapping[str, Any]] = (),
+        outputs: Iterable[Mapping[str, Any]] = (),
+        metadata: Iterable[Mapping[str, Any]] = (),
+        dataset_description: Optional[str] = None,
+    ) -> Dataset:
+        """
+        Append examples to dataset on the Phoenix server. If `dataframe` or
+        `csv_file_path` are provided, must also provide `input_keys` (and
+        optionally with `output_keys` or `metadata_keys` or both), which is a
+        list of strings denoting the column names in the dataframe or the csv
+        file. On the other hand, a sequence of dictionaries can also be provided
+        via `inputs` (and optionally with `outputs` or `metadat` or both), each
+        item of which represents a separate example in the dataset.
+
+        Args:
+            dataset_name: (str): Name of the dataset.
+            dataframe (pd.DataFrame): pandas DataFrame.
+            csv_file_path (str | Path): Location of a CSV text file
+            input_keys (Iterable[str]): List of column names used as input keys.
+                input_keys, output_keys, metadata_keys must be disjoint, and must
+                exist in CSV column headers.
+            output_keys (Iterable[str]): List of column names used as output keys.
+                input_keys, output_keys, metadata_keys must be disjoint, and must
+                exist in CSV column headers.
+            metadata_keys (Iterable[str]): List of column names used as metadata keys.
+                input_keys, output_keys, metadata_keys must be disjoint, and must
+                exist in CSV column headers.
+            inputs (Iterable[Mapping[str, Any]]): List of dictionaries object each
+                corresponding to an example in the dataset.
+            outputs (Iterable[Mapping[str, Any]]): List of dictionaries object each
+                corresponding to an example in the dataset.
+            metadata (Iterable[Mapping[str, Any]]): List of dictionaries object each
+                corresponding to an example in the dataset.
+            dataset_description: (Optional[str]): Description of the dataset.
+
+        Returns:
+            A Dataset object with its examples.
+        """
+        if dataframe is not None or csv_file_path is not None:
+            if dataframe is not None and csv_file_path is not None:
+                raise ValueError(
+                    "Please provide either `dataframe` or `csv_file_path`, but not both"
+                )
+            if list(inputs) or list(outputs) or list(metadata):
+                option = "dataframe" if dataframe is not None else "csv_file_path"
+                raise ValueError(
+                    f"Please provide only either `{option}` or list of dictionaries "
+                    f"via `inputs` (with `outputs` and `metadata`) but not both."
+                )
+            table = dataframe if dataframe is not None else csv_file_path
+            assert table is not None  # for type-checker
+            return self._upload_tabular_dataset(
+                table,
+                dataset_name=dataset_name,
+                input_keys=input_keys,
+                output_keys=output_keys,
+                metadata_keys=metadata_keys,
+                dataset_description=dataset_description,
+                action="append",
+            )
+        return self._upload_json_dataset(
+            dataset_name=dataset_name,
+            inputs=inputs,
+            outputs=outputs,
+            metadata=metadata,
+            dataset_description=dataset_description,
+            action="append",
+        )
+
+    def _upload_tabular_dataset(
         self,
         table: Union[str, Path, pd.DataFrame],
         /,
         *,
-        name: str,
+        dataset_name: str,
         input_keys: Iterable[str],
-        output_keys: Iterable[str],
+        output_keys: Iterable[str] = (),
         metadata_keys: Iterable[str] = (),
-        description: Optional[str] = None,
-        action: Literal["create", "append"] = "create",
+        dataset_description: Optional[str] = None,
+        action: DatasetAction = "create",
     ) -> Dataset:
         """
         Upload examples as dataset to the Phoenix server.
@@ -425,7 +607,7 @@ class Client(TraceDataExtractor):
         Args:
             table (str | Path | pd.DataFrame): Location of a CSV text file, or
                 pandas DataFrame.
-            name: (str): Name of the dataset. Required if action=append.
+            dataset_name: (str): Name of the dataset. Required if action=append.
             input_keys (Iterable[str]): List of column names used as input keys.
                 input_keys, output_keys, metadata_keys must be disjoint, and must
                 exist in CSV column headers.
@@ -435,17 +617,24 @@ class Client(TraceDataExtractor):
             metadata_keys (Iterable[str]): List of column names used as metadata keys.
                 input_keys, output_keys, metadata_keys must be disjoint, and must
                 exist in CSV column headers.
-            description: (Optional[str]): Description of the dataset.
-            action: (Literal["create", "append"): Create new dataset or append to an
-                existing dataset. If action=append, dataset name is required.
+            dataset_description: (Optional[str]): Description of the dataset.
+            action: (Literal["create", "append"]): Create new dataset or append to an
+                existing one. If action="append" and dataset does not exist, it'll
+                be created.
 
         Returns:
             A Dataset object with the uploaded examples.
         """
         if action not in ("create", "append"):
             raise ValueError(f"Invalid action: {action}")
-        if not name:
+        if not dataset_name:
             raise ValueError("Dataset name must not be blank")
+        input_keys, output_keys, metadata_keys = (
+            (keys,) if isinstance(keys, str) else (keys or ())
+            for keys in (input_keys, output_keys, metadata_keys)
+        )
+        if not any(map(bool, (input_keys, output_keys, metadata_keys))):
+            input_keys, output_keys, metadata_keys = _infer_keys(table)
         keys = DatasetKeys(
             frozenset(input_keys),
             frozenset(output_keys),
@@ -457,20 +646,86 @@ class Client(TraceDataExtractor):
             file = _prepare_csv(Path(table), keys)
         else:
             assert_never(table)
+        print("📤 Uploading dataset...")
         response = self._client.post(
             url=urljoin(self._base_url, "v1/datasets/upload"),
             files={"file": file},
             data={
                 "action": action,
-                "name": name,
-                "description": description,
+                "name": dataset_name,
+                "description": dataset_description,
                 "input_keys[]": sorted(keys.input),
                 "output_keys[]": sorted(keys.output),
                 "metadata_keys[]": sorted(keys.metadata),
             },
             params={"sync": True},
         )
-        response.raise_for_status()
+        return self._process_dataset_upload_response(response)
+
+    def _upload_json_dataset(
+        self,
+        *,
+        dataset_name: str,
+        inputs: Iterable[Mapping[str, Any]],
+        outputs: Iterable[Mapping[str, Any]] = (),
+        metadata: Iterable[Mapping[str, Any]] = (),
+        dataset_description: Optional[str] = None,
+        action: DatasetAction = "create",
+    ) -> Dataset:
+        """
+        Upload examples as dataset to the Phoenix server.
+
+        Args:
+            dataset_name: (str): Name of the dataset
+            inputs (Iterable[Mapping[str, Any]]): List of dictionaries object each
+                corresponding to an example in the dataset.
+            outputs (Iterable[Mapping[str, Any]]): List of dictionaries object each
+                corresponding to an example in the dataset.
+            metadata (Iterable[Mapping[str, Any]]): List of dictionaries object each
+                corresponding to an example in the dataset.
+            dataset_description: (Optional[str]): Description of the dataset.
+            action: (Literal["create", "append"]): Create new dataset or append to an
+                existing one. If action="append" and dataset does not exist, it'll
+                be created.
+
+        Returns:
+            A Dataset object with the uploaded examples.
+        """
+        # convert to list to avoid issues with pandas Series
+        inputs, outputs, metadata = list(inputs), list(outputs), list(metadata)
+        if not inputs or not _is_all_dict(inputs):
+            raise ValueError(
+                "`inputs` should be a non-empty sequence containing only dictionary objects"
+            )
+        for name, seq in {"outputs": outputs, "metadata": metadata}.items():
+            if seq and not (len(seq) == len(inputs) and _is_all_dict(seq)):
+                raise ValueError(
+                    f"`{name}` should be a sequence of the same length as `inputs` "
+                    "containing only dictionary objects"
+                )
+        print("📤 Uploading dataset...")
+        response = self._client.post(
+            url=urljoin(self._base_url, "v1/datasets/upload"),
+            headers={"Content-Encoding": "gzip"},
+            json={
+                "action": action,
+                "name": dataset_name,
+                "description": dataset_description,
+                "inputs": inputs,
+                "outputs": outputs,
+                "metadata": metadata,
+            },
+            params={"sync": True},
+        )
+        return self._process_dataset_upload_response(response)
+
+    def _process_dataset_upload_response(self, response: Response) -> Dataset:
+        try:
+            response.raise_for_status()
+        except HTTPStatusError as e:
+            if msg := response.text:
+                raise DatasetUploadError(msg) from e
+            raise
         data = response.json()["data"]
         dataset_id = data["dataset_id"]
         response = self._client.get(
@@ -480,11 +735,14 @@ class Client(TraceDataExtractor):
         data = response.json()["data"]
         version_id = data["version_id"]
         examples = data["examples"]
+        print(f"💾 Examples uploaded: {self.web_url}datasets/{dataset_id}/examples")
+        print(f"🗄️ Dataset version ID: {version_id}")
+
         return Dataset(
             id=dataset_id,
             version_id=version_id,
-            examples=[
-                Example(
+            examples={
+                example["id"]: Example(
                     id=example["id"],
                     input=example["input"],
                     output=example["output"],
@@ -492,7 +750,7 @@ class Client(TraceDataExtractor):
                     updated_at=datetime.fromisoformat(example["updated_at"]),
                 )
                 for example in examples
-            ],
+            },
         )
 
 
@@ -502,20 +760,25 @@ FileType: TypeAlias = str
 FileHeaders: TypeAlias = Dict[str, str]
 
 
-def _prepare_csv(
-    path: Path,
-    keys: DatasetKeys,
-) -> Tuple[FileName, FilePointer, FileType, FileHeaders]:
+def _get_csv_column_headers(path: Path) -> Tuple[str, ...]:
     path = path.resolve()
     if not path.is_file():
         raise FileNotFoundError(f"File does not exist: {path}")
     with open(path, "r") as f:
         rows = csv.reader(f)
         try:
-            column_headers = next(rows)
+            column_headers = tuple(next(rows))
             _ = next(rows)
         except StopIteration:
             raise ValueError("csv file has no data")
+    return column_headers
+
+
+def _prepare_csv(
+    path: Path,
+    keys: DatasetKeys,
+) -> Tuple[FileName, FilePointer, FileType, FileHeaders]:
+    column_headers = _get_csv_column_headers(path)
     (header, freq), *_ = Counter(column_headers).most_common(1)
     if freq > 1:
         raise ValueError(f"Duplicated column header in CSV file: {header}")
@@ -545,5 +808,35 @@ def _prepare_pyarrow(
     return "pandas", file, "application/x-pandas-pyarrow", {}
 
 
+_response_header = re.compile(r"(?i)(response|answer)s*$")
+
+
+def _infer_keys(
+    table: Union[str, Path, pd.DataFrame],
+) -> Tuple[Tuple[str, ...], Tuple[str, ...], Tuple[str, ...]]:
+    column_headers = (
+        tuple(table.columns)
+        if isinstance(table, pd.DataFrame)
+        else _get_csv_column_headers(Path(table))
+    )
+    for i, header in enumerate(column_headers):
+        if _response_header.search(header):
+            break
+    else:
+        i = len(column_headers)
+    return (
+        column_headers[:i],
+        column_headers[i : i + 1],
+        column_headers[i + 1 :],
+    )
+
+
 def _to_iso_format(value: Optional[datetime]) -> Optional[str]:
     return value.isoformat() if value else None
+
+
+def _is_all_dict(seq: Sequence[Any]) -> bool:
+    return all(map(lambda obj: isinstance(obj, dict), seq))
+
+
+class DatasetUploadError(Exception): ...

phoenix/trace/fixtures.py

@@ -244,12 +244,12 @@ def send_dataset_fixtures(
         try:
             if i % 2:
                 client.upload_dataset(
-                    fixture.dataframe,
-                    name=fixture.name,
+                    dataset_name=fixture.name,
+                    dataframe=fixture.dataframe,
                     input_keys=fixture.input_keys,
                     output_keys=fixture.output_keys,
                     metadata_keys=fixture.metadata_keys,
-                    description=fixture.description,
+                    dataset_description=fixture.description,
                 )
             else:
                 with NamedTemporaryFile() as tf:
@@ -257,12 +257,12 @@ def send_dataset_fixtures(
                         shutil.copyfileobj(fixture.csv, f)
                         f.flush()
                     client.upload_dataset(
-                        tf.name,
-                        name=fixture.name,
+                        dataset_name=fixture.name,
+                        csv_file_path=tf.name,
                         input_keys=fixture.input_keys,
                         output_keys=fixture.output_keys,
                         metadata_keys=fixture.metadata_keys,
-                        description=fixture.description,
+                        dataset_description=fixture.description,
                     )
         except HTTPStatusError as e:
             print(e.response.content.decode())

phoenix/utilities/json.py

@@ -2,7 +2,7 @@ import dataclasses
 import datetime
 from enum import Enum
 from pathlib import Path
-from typing import Any, Mapping, Sequence, SupportsFloat, Union, get_args, get_origin
+from typing import Any, Mapping, Sequence, Union, get_args, get_origin
 
 import numpy as np
 
@@ -15,10 +15,10 @@ def jsonify(obj: Any) -> Any:
         return jsonify(obj.value)
     if isinstance(obj, (str, int, float, bool)) or obj is None:
         return obj
-    if isinstance(obj, np.ndarray):
+    if isinstance(obj, (list, set, frozenset, Sequence)):
         return [jsonify(v) for v in obj]
-    if isinstance(obj, SupportsFloat):
-        return float(obj)
+    if isinstance(obj, (dict, Mapping)):
+        return {jsonify(k): jsonify(v) for k, v in obj.items()}
     if dataclasses.is_dataclass(obj):
         return {
             k: jsonify(v)
@@ -29,10 +29,6 @@ def jsonify(obj: Any) -> Any:
                 and type(None) in get_args(field)
             )
         }
-    if isinstance(obj, (Sequence, set, frozenset)):
-        return [jsonify(v) for v in obj]
-    if isinstance(obj, Mapping):
-        return {jsonify(k): jsonify(v) for k, v in obj.items()}
     if isinstance(obj, (datetime.date, datetime.datetime, datetime.time)):
         return obj.isoformat()
     if isinstance(obj, datetime.timedelta):
@@ -41,6 +37,10 @@ def jsonify(obj: Any) -> Any:
         return str(obj)
     if isinstance(obj, BaseException):
         return str(obj)
+    if isinstance(obj, np.ndarray):
+        return [jsonify(v) for v in obj]
+    if hasattr(obj, "__float__"):
+        return float(obj)
     if hasattr(obj, "model_dump") and callable(obj.model_dump):
         # pydantic v2
         try:

phoenix/version.py

@@ -1 +1 @@
-__version__ = "4.4.4rc4"
+__version__ = "4.4.4rc6"

phoenix/datasets/evaluators/__init__.py

@@ -1,18 +0,0 @@
-from phoenix.datasets.evaluators.code_evaluators import ContainsKeyword, JSONParsable
-from phoenix.datasets.evaluators.llm_evaluators import (
-    CoherenceEvaluator,
-    ConcisenessEvaluator,
-    HelpfulnessEvaluator,
-    LLMCriteriaEvaluator,
-    RelevanceEvaluator,
-)
-
-__all__ = [
-    "ContainsKeyword",
-    "JSONParsable",
-    "CoherenceEvaluator",
-    "ConcisenessEvaluator",
-    "LLMCriteriaEvaluator",
-    "HelpfulnessEvaluator",
-    "RelevanceEvaluator",
-]

phoenix/datasets/evaluators/_utils.py

@@ -1,13 +0,0 @@
-from phoenix.datasets.types import JSONSerializable
-
-
-def _unwrap_json(obj: JSONSerializable) -> JSONSerializable:
-    if isinstance(obj, dict):
-        if len(obj) == 1:
-            key = next(iter(obj.keys()))
-            output = obj[key]
-            assert isinstance(
-                output, (dict, list, str, int, float, bool, type(None))
-            ), "Output must be JSON serializable"
-            return output
-    return obj