arize-phoenix 0.0.29rc8__py3-none-any.whl → 0.0.31__py3-none-any.whl

{arize_phoenix-0.0.29rc8.dist-info → arize_phoenix-0.0.31.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arize-phoenix
-Version: 0.0.29rc8
+Version: 0.0.31
 Summary: ML Observability in your notebook
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues
@@ -41,6 +41,9 @@ Requires-Dist: pytest; extra == 'dev'
 Requires-Dist: pytest-cov; extra == 'dev'
 Requires-Dist: pytest-lazy-fixture; extra == 'dev'
 Requires-Dist: strawberry-graphql[debug-server]==0.178.0; extra == 'dev'
+Provides-Extra: experimental
+Requires-Dist: openai; extra == 'experimental'
+Requires-Dist: tenacity; extra == 'experimental'
 Description-Content-Type: text/markdown
 
 <p align="center">

{arize_phoenix-0.0.29rc8.dist-info → arize_phoenix-0.0.31.dist-info}/RECORD

@@ -1,4 +1,4 @@
-phoenix/__init__.py,sha256=GtycPB0XtJn1eIzZcwNUueSxnCun8o6sY827KA_6Fpg,1117
+phoenix/__init__.py,sha256=xUy3vWV6BkKa3ckBCBq0PWsocvw-5YMviTv5yQhF7f8,1189
 phoenix/config.py,sha256=tjNn9oqDxQmeO85sCchLlTsDiRJ6AoK0CTt_Uc_hrKM,1442
 phoenix/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 phoenix/services.py,sha256=BlQF7lTQwhXUHBZBjZOoTnDM2Qni_hziUXsfp1Hux4Q,3978
@@ -10,11 +10,14 @@ phoenix/core/model.py,sha256=vQ6RxpUPlncezJvur5u6xBN0Lkrk2gW0cTyb-qqaSqA,4713
 phoenix/core/model_schema.py,sha256=H8EHEsrGdsieD0grSYejunv3vqlzwM-OIxXx6DDGibA,50994
 phoenix/core/model_schema_adapter.py,sha256=GzJetQALsDUWJuFzLOEBSEC8M0JI0_F7i6CODYK2elI,8292
 phoenix/datasets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-phoenix/datasets/dataset.py,sha256=DvaX2o5er0QuzIPcHpp6ujKYXcPAYvkYmtm1sDHKZrs,21873
+phoenix/datasets/dataset.py,sha256=sH4yLFhJjEJrWCd8LjmCoa3FdfcIXAeDpnXUcUzS7dc,30996
 phoenix/datasets/errors.py,sha256=-Iyk8rsvP_KX-P4gOqjm26slkDq1-9CohK07_LkrYCI,8117
 phoenix/datasets/fixtures.py,sha256=0_PacL3dw49zulKpFpPdhvxJxeGmHTguqIyf2VXkBkk,19158
 phoenix/datasets/schema.py,sha256=HlM0f-pLFul2sYyHZM-Av8OFxLFkn57dkK_BWbMzyJY,6668
 phoenix/datasets/validation.py,sha256=dZ9lCFUV0EY7HCkQkQBrs-GLAEIZdpOqUxwD5l4dp88,8294
+phoenix/experimental/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+phoenix/experimental/evals/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+phoenix/experimental/evals/retrievals.py,sha256=3PBv2yFIcxdn_y-L6We554TsozqCDjTVBE8017RB2mY,3793
 phoenix/metrics/README.md,sha256=5gekqTU-5gGdMwvcfNp2Wlu8p1ul9kGY_jq0XXQusoI,1964
 phoenix/metrics/__init__.py,sha256=sLp7td1GIt_0Z8dPUyP4L0-_4x9c871yAaGX30oMsvg,2433
 phoenix/metrics/binning.py,sha256=CXPPcAkRmmR__IG36a6UGs5RBtgXXPuWQbafPtuG1ww,12787
@@ -84,12 +87,12 @@ phoenix/server/static/apple-touch-icon.png,sha256=fOfpjqGpWYbJ0eAurKsyoZP1EAs6ZV
 phoenix/server/static/favicon.ico,sha256=bY0vvCKRftemZfPShwZtE93DiiQdaYaozkPGwNFr6H8,34494
 phoenix/server/static/index.css,sha256=KKGpx4iwF91VGRm0YN-4cn8oC-oIqC6HecoPf0x3ZM8,1885
 phoenix/server/static/index.html,sha256=xPZZH-y4dWlbDutPEV1k0rhmWJtIV-Db9aYP-dEc7wM,703
-phoenix/server/static/index.js,sha256=Z7vMbIgbNLvLrx-FNUc84R8qvqfFm7XhQ2fCKZ3LvLU,2534545
+phoenix/server/static/index.js,sha256=SJnIFu7ufB_k38YBCI7D1btTSq4mmE7WkUf1iOBxiGw,2573278
 phoenix/server/static/modernizr.js,sha256=mvK-XtkNqjOral-QvzoqsyOMECXIMu5BQwSVN_wcU9c,2564
 phoenix/session/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/session/session.py,sha256=WwGH7qStR4kAhmrRsw35BwXvCQDnEWpGm0crjyrWTvs,9519
-arize_phoenix-0.0.29rc8.dist-info/METADATA,sha256=vx4rnznFhvKF0LjDzgZ5_ZRpeqGeQ7dbKxAahoitZJs,10852
-arize_phoenix-0.0.29rc8.dist-info/WHEEL,sha256=9QBuHhg6FNW7lppboF2vKVbCGTVzsFykgRQjjlajrhA,87
-arize_phoenix-0.0.29rc8.dist-info/licenses/IP_NOTICE,sha256=JBqyyCYYxGDfzQ0TtsQgjts41IJoa-hiwDrBjCb9gHM,469
-arize_phoenix-0.0.29rc8.dist-info/licenses/LICENSE,sha256=HFkW9REuMOkvKRACuwLPT0hRydHb3zNg-fdFt94td18,3794
-arize_phoenix-0.0.29rc8.dist-info/RECORD,,
+arize_phoenix-0.0.31.dist-info/METADATA,sha256=PLrtaGcA6hRk8KhAjAxHXtvLNNfIrH5HhjnPlhpeiSU,10974
+arize_phoenix-0.0.31.dist-info/WHEEL,sha256=hKi7AIIx6qfnsRbr087vpeJnrVUuDokDHZacPPMW7-Y,87
+arize_phoenix-0.0.31.dist-info/licenses/IP_NOTICE,sha256=JBqyyCYYxGDfzQ0TtsQgjts41IJoa-hiwDrBjCb9gHM,469
+arize_phoenix-0.0.31.dist-info/licenses/LICENSE,sha256=HFkW9REuMOkvKRACuwLPT0hRydHb3zNg-fdFt94td18,3794
+arize_phoenix-0.0.31.dist-info/RECORD,,

{arize_phoenix-0.0.29rc8.dist-info → arize_phoenix-0.0.31.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.18.0
+Generator: hatchling 1.12.2
 Root-Is-Purelib: true
 Tag: py3-none-any

phoenix/__init__.py

@@ -2,8 +2,9 @@ from .datasets.dataset import Dataset
 from .datasets.fixtures import ExampleDatasets, load_example
 from .datasets.schema import EmbeddingColumnNames, RetrievalEmbeddingColumnNames, Schema
 from .session.session import Session, active_session, close_app, launch_app
+from .trace.fixtures import load_example_traces
 
-__version__ = "0.0.29rc8"
+__version__ = "0.0.31"
 
 # module level doc-string
 __doc__ = """
@@ -32,4 +33,5 @@ __all__ = [
     "close_app",
     "launch_app",
     "Session",
+    "load_example_traces",
 ]

phoenix/datasets/dataset.py

@@ -1,7 +1,10 @@
 import logging
+import re
 import uuid
 from copy import deepcopy
-from dataclasses import fields, replace
+from dataclasses import dataclass, fields, replace
+from enum import Enum
+from itertools import groupby
 from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
 import numpy as np
@@ -25,6 +28,7 @@ from .schema import (
     SINGLE_COLUMN_SCHEMA_FIELD_NAMES,
     EmbeddingColumnNames,
     EmbeddingFeatures,
+    RetrievalEmbeddingColumnNames,
     Schema,
     SchemaFieldName,
     SchemaFieldValue,
@@ -121,6 +125,160 @@ class Dataset:
         schema = Schema.from_json(schema_json)
         return cls(df, schema, name)
 
+    @classmethod
+    def from_open_inference(cls, dataframe: DataFrame) -> "Dataset":
+        schema = Schema()
+        column_renaming: Dict[str, str] = {}
+        for group_name, group in groupby(
+            sorted(
+                map(_parse_open_inference_column_name, dataframe.columns),
+                key=lambda column: column.name,
+            ),
+            key=lambda column: column.name,
+        ):
+            open_inference_columns = list(group)
+            if group_name == "":
+                column_names_by_category = {
+                    column.category: column.full_name for column in open_inference_columns
+                }
+                schema = replace(
+                    schema,
+                    prediction_id_column_name=column_names_by_category.get(
+                        OpenInferenceCategory.id
+                    ),
+                    timestamp_column_name=column_names_by_category.get(
+                        OpenInferenceCategory.timestamp
+                    ),
+                )
+                continue
+            column_names_by_specifier = {
+                column.specifier: column.full_name for column in open_inference_columns
+            }
+            if group_name == "response":
+                response_vector_column_name = column_names_by_specifier.get(
+                    OpenInferenceSpecifier.embedding
+                )
+                if response_vector_column_name is not None:
+                    column_renaming[response_vector_column_name] = "response"
+                    schema = replace(
+                        schema,
+                        response_column_names=EmbeddingColumnNames(
+                            vector_column_name=column_renaming[response_vector_column_name],
+                            raw_data_column_name=column_names_by_specifier.get(
+                                OpenInferenceSpecifier.default
+                            ),
+                        ),
+                    )
+                else:
+                    response_text_column_name = column_names_by_specifier.get(
+                        OpenInferenceSpecifier.default
+                    )
+                    if response_text_column_name is None:
+                        raise ValueError(
+                            "invalid OpenInference format: missing text column for response"
+                        )
+                    column_renaming[response_text_column_name] = "response"
+                    schema = replace(
+                        schema,
+                        response_column_names=column_renaming[response_text_column_name],
+                    )
+            elif group_name == "prompt":
+                prompt_vector_column_name = column_names_by_specifier.get(
+                    OpenInferenceSpecifier.embedding
+                )
+                if prompt_vector_column_name is None:
+                    raise ValueError(
+                        "invalid OpenInference format: missing embedding vector column for prompt"
+                    )
+                column_renaming[prompt_vector_column_name] = "prompt"
+                schema = replace(
+                    schema,
+                    prompt_column_names=RetrievalEmbeddingColumnNames(
+                        vector_column_name=column_renaming[prompt_vector_column_name],
+                        raw_data_column_name=column_names_by_specifier.get(
+                            OpenInferenceSpecifier.default
+                        ),
+                        context_retrieval_ids_column_name=column_names_by_specifier.get(
+                            OpenInferenceSpecifier.retrieved_document_ids
+                        ),
+                        context_retrieval_scores_column_name=column_names_by_specifier.get(
+                            OpenInferenceSpecifier.retrieved_document_scores
+                        ),
+                    ),
+                )
+            elif OpenInferenceSpecifier.embedding in column_names_by_specifier:
+                vector_column_name = column_names_by_specifier[OpenInferenceSpecifier.embedding]
+                column_renaming[vector_column_name] = group_name
+                embedding_feature_column_names = schema.embedding_feature_column_names or {}
+                embedding_feature_column_names.update(
+                    {
+                        group_name: EmbeddingColumnNames(
+                            vector_column_name=column_renaming[vector_column_name],
+                            raw_data_column_name=column_names_by_specifier.get(
+                                OpenInferenceSpecifier.raw_data
+                            ),
+                            link_to_data_column_name=column_names_by_specifier.get(
+                                OpenInferenceSpecifier.link_to_data
+                            ),
+                        )
+                    }
+                )
+                schema = replace(
+                    schema,
+                    embedding_feature_column_names=embedding_feature_column_names,
+                )
+            elif len(open_inference_columns) == 1:
+                open_inference_column = open_inference_columns[0]
+                raw_column_name = open_inference_column.full_name
+                column_renaming[raw_column_name] = open_inference_column.name
+                if open_inference_column.category is OpenInferenceCategory.feature:
+                    schema = replace(
+                        schema,
+                        feature_column_names=(
+                            (schema.feature_column_names or []) + [column_renaming[raw_column_name]]
+                        ),
+                    )
+                elif open_inference_column.category is OpenInferenceCategory.tag:
+                    schema = replace(
+                        schema,
+                        tag_column_names=(
+                            (schema.tag_column_names or []) + [column_renaming[raw_column_name]]
+                        ),
+                    )
+                elif open_inference_column.category is OpenInferenceCategory.prediction:
+                    if open_inference_column.specifier is OpenInferenceSpecifier.score:
+                        schema = replace(
+                            schema,
+                            prediction_score_column_name=column_renaming[raw_column_name],
+                        )
+                    if open_inference_column.specifier is OpenInferenceSpecifier.label:
+                        schema = replace(
+                            schema,
+                            prediction_label_column_name=column_renaming[raw_column_name],
+                        )
+                elif open_inference_column.category is OpenInferenceCategory.actual:
+                    if open_inference_column.specifier is OpenInferenceSpecifier.score:
+                        schema = replace(
+                            schema,
+                            actual_score_column_name=column_renaming[raw_column_name],
+                        )
+                    if open_inference_column.specifier is OpenInferenceSpecifier.label:
+                        schema = replace(
+                            schema,
+                            actual_label_column_name=column_renaming[raw_column_name],
+                        )
+            else:
+                raise ValueError(f"invalid OpenInference format: duplicated name `{group_name}`")
+
+        return cls(
+            dataframe.rename(
+                column_renaming,
+                axis=1,
+                copy=False,
+            ),
+            schema,
+        )
+
     def to_disc(self) -> None:
         """writes the data and schema to disc"""
         directory = DATASET_DIR / self.name
@@ -528,3 +686,48 @@ def _get_schema_from_unknown_schema_param(schemaLike: SchemaLike) -> Schema:
 
 def _add_prediction_id(num_rows: int) -> List[str]:
     return [str(uuid.uuid4()) for _ in range(num_rows)]
+
+
+class OpenInferenceCategory(Enum):
+    id = "id"
+    timestamp = "timestamp"
+    feature = "feature"
+    tag = "tag"
+    prediction = "prediction"
+    actual = "actual"
+
+
+class OpenInferenceSpecifier(Enum):
+    default = ""
+    score = "score"
+    label = "label"
+    embedding = "embedding"
+    raw_data = "raw_data"
+    link_to_data = "link_to_data"
+    retrieved_document_ids = "retrieved_document_ids"
+    retrieved_document_scores = "retrieved_document_scores"
+
+
+@dataclass(frozen=True)
+class _OpenInferenceColumnName:
+    full_name: str
+    category: OpenInferenceCategory
+    data_type: str
+    specifier: OpenInferenceSpecifier = OpenInferenceSpecifier.default
+    name: str = ""
+
+
+def _parse_open_inference_column_name(column_name: str) -> _OpenInferenceColumnName:
+    pattern = (
+        r"^:(?P<category>\w+)\.(?P<data_type>\[\w+\]|\w+)(\.(?P<specifier>\w+))?:(?P<name>.*)?$"
+    )
+    if match := re.match(pattern, column_name):
+        extract = match.groupdict(default="")
+        return _OpenInferenceColumnName(
+            full_name=column_name,
+            category=OpenInferenceCategory(extract.get("category", "").lower()),
+            data_type=extract.get("data_type", "").lower(),
+            specifier=OpenInferenceSpecifier(extract.get("specifier", "").lower()),
+            name=extract.get("name", ""),
+        )
+    raise ValueError(f"Invalid format for column name: {column_name}")

phoenix/experimental/evals/retrievals.py

@@ -0,0 +1,91 @@
+"""
+Helper functions for evaluating the retrieval step of retrieval-augmented generation.
+"""
+
+from typing import List, Optional
+
+import openai
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_random_exponential,
+)
+
+_EVALUATION_SYSTEM_MESSAGE = (
+    "You will be given a query and a reference text. "
+    "You must determine whether the reference text contains an answer to the input query. "
+    'Your response must be single word, either "relevant" or "irrelevant", '
+    "and should not contain any text or characters aside from that word. "
+    '"irrelevant" means that the reference text does not contain an answer to the query. '
+    '"relevant" means the reference text contains an answer to the query.'
+)
+_QUERY_CONTEXT_PROMPT_TEMPLATE = """# Query: {query}
+
+# Reference: {reference}
+
+# Answer ("relevant" or "irrelevant"): """
+
+
+def compute_precisions_at_k(
+    relevance_classifications: List[Optional[bool]],
+) -> List[Optional[float]]:
+    """Given a list of relevance classifications, computes precision@k for k = 1, 2, ..., n, where
+    n is the length of the input list.
+
+    Args:
+        relevance_classifications (List[Optional[bool]]): A list of relevance classifications for a
+            set of retrieved documents, sorted by order of retrieval (i.e., the first element is the
+            classification for the first retrieved document, the second element is the
+            classification for the second retrieved document, etc.). The list may contain None
+            values, which indicate that the relevance classification for the corresponding document
+            is unknown.
+
+    Returns:
+        List[Optional[float]]: A list of precision@k values for k = 1, 2, ..., n, where n is the
+            length of the input list. The first element is the precision@1 value, the second element
+            is the precision@2 value, etc. If the input list contains any None values, those values
+            are omitted when computing the precision@k values.
+    """
+    precisions_at_k = []
+    num_relevant_classifications = 0
+    num_non_none_classifications = 0
+    for relevance_classification in relevance_classifications:
+        if isinstance(relevance_classification, bool):
+            num_non_none_classifications += 1
+            num_relevant_classifications += int(relevance_classification)
+        precisions_at_k.append(
+            num_relevant_classifications / num_non_none_classifications
+            if num_non_none_classifications > 0
+            else None
+        )
+    return precisions_at_k
+
+
+@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
+def classify_relevance(query: str, document: str, model_name: str) -> Optional[bool]:
+    """Given a query and a document, determines whether the document contains an answer to the
+    query.
+
+    Args:
+        query (str): The query text. document (str): The document text. model_name (str): The name
+        of the OpenAI API model to use for the classification.
+
+    Returns:
+        Optional[bool]: A boolean indicating whether the document contains an answer to the query
+            (True meaning relevant, False meaning irrelevant), or None if the LLM produces an
+            unparseable output.
+    """
+    prompt = _QUERY_CONTEXT_PROMPT_TEMPLATE.format(
+        query=query,
+        reference=document,
+    )
+    response = openai.ChatCompletion.create(
+        messages=[
+            {"role": "system", "content": _EVALUATION_SYSTEM_MESSAGE},
+            {"role": "user", "content": prompt},
+        ],
+        model=model_name,
+    )
+    raw_response_text = str(response["choices"][0]["message"]["content"]).strip()
+    relevance_classification = {"relevant": True, "irrelevant": False}.get(raw_response_text)
+    return relevance_classification