arize-phoenix 0.0.30__tar.gz → 0.0.31rc2__tar.gz

{arize_phoenix-0.0.30 → arize_phoenix-0.0.31rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arize-phoenix
-Version: 0.0.30
+Version: 0.0.31rc2
 Summary: ML Observability in your notebook
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues
@@ -41,6 +41,9 @@ Requires-Dist: pytest; extra == 'dev'
 Requires-Dist: pytest-cov; extra == 'dev'
 Requires-Dist: pytest-lazy-fixture; extra == 'dev'
 Requires-Dist: strawberry-graphql[debug-server]==0.178.0; extra == 'dev'
+Provides-Extra: experimental
+Requires-Dist: openai; extra == 'experimental'
+Requires-Dist: tenacity; extra == 'experimental'
 Description-Content-Type: text/markdown
 
 <p align="center">

{arize_phoenix-0.0.30 → arize_phoenix-0.0.31rc2}/pyproject.toml

@@ -51,6 +51,10 @@ dev = [
   "pre-commit",
   "arize[AutoEmbeddings, LLM_Evaluation]",
 ]
+experimental = [
+  "openai",
+  "tenacity",
+]
 
 [project.urls]
 Documentation = "https://docs.arize.com/phoenix/"
@@ -90,6 +94,8 @@ dependencies = [
   "pytest-cov",
   "pytest-lazy-fixture",
   "arize",
+  "openai",
+  "tenacity",
 ]
 
 [tool.hatch.envs.type]
@@ -98,6 +104,7 @@ dependencies = [
   "pandas-stubs",
   "pytest",
   "types-psutil",
+  "tenacity",
 ]
 
 [tool.hatch.envs.style]
@@ -241,6 +248,7 @@ module = [
   "arize.*",
   "portpicker",
   "wrapt",
+  "openai",
 ]
 ignore_missing_imports = true
 

{arize_phoenix-0.0.30 → arize_phoenix-0.0.31rc2}/src/phoenix/__init__.py

@@ -2,8 +2,9 @@ from .datasets.dataset import Dataset
 from .datasets.fixtures import ExampleDatasets, load_example
 from .datasets.schema import EmbeddingColumnNames, RetrievalEmbeddingColumnNames, Schema
 from .session.session import Session, active_session, close_app, launch_app
+from .trace.fixtures import load_example_traces
 
-__version__ = "0.0.30"
+__version__ = "0.0.31rc2"
 
 # module level doc-string
 __doc__ = """
@@ -32,4 +33,5 @@ __all__ = [
     "close_app",
     "launch_app",
     "Session",
+    "load_example_traces",
 ]

arize_phoenix-0.0.31rc2/src/phoenix/experimental/evals/retrievals.py

@@ -0,0 +1,91 @@
+"""
+Helper functions for evaluating the retrieval step of retrieval-augmented generation.
+"""
+
+from typing import List, Optional
+
+import openai
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_random_exponential,
+)
+
+_EVALUATION_SYSTEM_MESSAGE = (
+    "You will be given a query and a reference text. "
+    "You must determine whether the reference text contains an answer to the input query. "
+    'Your response must be single word, either "relevant" or "irrelevant", '
+    "and should not contain any text or characters aside from that word. "
+    '"irrelevant" means that the reference text does not contain an answer to the query. '
+    '"relevant" means the reference text contains an answer to the query.'
+)
+_QUERY_CONTEXT_PROMPT_TEMPLATE = """# Query: {query}
+
+# Reference: {reference}
+
+# Answer ("relevant" or "irrelevant"): """
+
+
+def compute_precisions_at_k(
+    relevance_classifications: List[Optional[bool]],
+) -> List[Optional[float]]:
+    """Given a list of relevance classifications, computes precision@k for k = 1, 2, ..., n, where
+    n is the length of the input list.
+
+    Args:
+        relevance_classifications (List[Optional[bool]]): A list of relevance classifications for a
+            set of retrieved documents, sorted by order of retrieval (i.e., the first element is the
+            classification for the first retrieved document, the second element is the
+            classification for the second retrieved document, etc.). The list may contain None
+            values, which indicate that the relevance classification for the corresponding document
+            is unknown.
+
+    Returns:
+        List[Optional[float]]: A list of precision@k values for k = 1, 2, ..., n, where n is the
+            length of the input list. The first element is the precision@1 value, the second element
+            is the precision@2 value, etc. If the input list contains any None values, those values
+            are omitted when computing the precision@k values.
+    """
+    precisions_at_k = []
+    num_relevant_classifications = 0
+    num_non_none_classifications = 0
+    for relevance_classification in relevance_classifications:
+        if isinstance(relevance_classification, bool):
+            num_non_none_classifications += 1
+            num_relevant_classifications += int(relevance_classification)
+        precisions_at_k.append(
+            num_relevant_classifications / num_non_none_classifications
+            if num_non_none_classifications > 0
+            else None
+        )
+    return precisions_at_k
+
+
+@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
+def classify_relevance(query: str, document: str, model_name: str) -> Optional[bool]:
+    """Given a query and a document, determines whether the document contains an answer to the
+    query.
+
+    Args:
+        query (str): The query text. document (str): The document text. model_name (str): The name
+        of the OpenAI API model to use for the classification.
+
+    Returns:
+        Optional[bool]: A boolean indicating whether the document contains an answer to the query
+            (True meaning relevant, False meaning irrelevant), or None if the LLM produces an
+            unparseable output.
+    """
+    prompt = _QUERY_CONTEXT_PROMPT_TEMPLATE.format(
+        query=query,
+        reference=document,
+    )
+    response = openai.ChatCompletion.create(
+        messages=[
+            {"role": "system", "content": _EVALUATION_SYSTEM_MESSAGE},
+            {"role": "user", "content": prompt},
+        ],
+        model=model_name,
+    )
+    raw_response_text = str(response["choices"][0]["message"]["content"]).strip()
+    relevance_classification = {"relevant": True, "irrelevant": False}.get(raw_response_text)
+    return relevance_classification