arize-phoenix 4.4.4rc3__tar.gz → 4.4.4rc5__tar.gz

{arize_phoenix-4.4.4rc3 → arize_phoenix-4.4.4rc5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: arize-phoenix
-Version: 4.4.4rc3
+Version: 4.4.4rc5
 Summary: AI Observability and Evaluation
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues
@@ -31,7 +31,7 @@ Requires-Dist: openinference-instrumentation
 Requires-Dist: openinference-instrumentation-langchain>=0.1.12
 Requires-Dist: openinference-instrumentation-llama-index>=1.2.0
 Requires-Dist: openinference-instrumentation-openai>=0.1.4
-Requires-Dist: openinference-semantic-conventions>=0.1.5
+Requires-Dist: openinference-semantic-conventions>=0.1.9
 Requires-Dist: opentelemetry-exporter-otlp
 Requires-Dist: opentelemetry-proto>=1.12.0
 Requires-Dist: opentelemetry-sdk

{arize_phoenix-4.4.4rc3 → arize_phoenix-4.4.4rc5}/pyproject.toml

@@ -46,7 +46,7 @@ dependencies = [
   "opentelemetry-proto>=1.12.0",  # needed to avoid this issue: https://github.com/Arize-ai/phoenix/issues/2695
   "opentelemetry-exporter-otlp",
   "opentelemetry-semantic-conventions",
-  "openinference-semantic-conventions>=0.1.5",
+  "openinference-semantic-conventions>=0.1.9",
   "openinference-instrumentation",
   "openinference-instrumentation-langchain>=0.1.12",
   "openinference-instrumentation-llama-index>=1.2.0",
@@ -206,6 +206,7 @@ dependencies = [
 [tool.hatch.envs.docs]
 detached = true
 dependencies = [
+  "pyment",
   "interrogate",
 ]
 

arize_phoenix-4.4.4rc5/src/phoenix/datasets/evaluators/__init__.py

@@ -0,0 +1,18 @@
+from phoenix.datasets.evaluators.code_evaluators import ContainsKeyword, JSONParsable
+from phoenix.datasets.evaluators.llm_evaluators import (
+    CoherenceEvaluator,
+    ConcisenessEvaluator,
+    HelpfulnessEvaluator,
+    LLMCriteriaEvaluator,
+    RelevanceEvaluator,
+)
+
+__all__ = [
+    "ContainsKeyword",
+    "JSONParsable",
+    "CoherenceEvaluator",
+    "ConcisenessEvaluator",
+    "LLMCriteriaEvaluator",
+    "HelpfulnessEvaluator",
+    "RelevanceEvaluator",
+]

arize_phoenix-4.4.4rc5/src/phoenix/datasets/evaluators/code_evaluators.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, List, Optional, Union
+
+from phoenix.datasets.evaluators.utils import Evaluator
+from phoenix.datasets.types import EvaluationResult, TaskOutput
+
+
+class JSONParsable(Evaluator):
+    def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
+        assert isinstance(output, str), "Experiment run output must be a string"
+        try:
+            json.loads(output)
+            json_parsable = True
+        except BaseException:
+            json_parsable = False
+        return EvaluationResult(
+            score=int(json_parsable),
+        )
+
+
+class ContainsKeyword(Evaluator):
+    def __init__(self, keyword: str, name: Optional[str] = None) -> None:
+        self.keyword = keyword
+        self._name = name or f"Contains({repr(keyword)})"
+
+    def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
+        assert isinstance(output, str), "Experiment run output must be a string"
+        found = self.keyword in output
+        return EvaluationResult(
+            score=float(found),
+            explanation=(
+                f"the string {repr(self.keyword)} was "
+                f"{'found' if found else 'not found'} in the output"
+            ),
+        )
+
+
+class ContainsAnyKeyword(Evaluator):
+    def __init__(self, keywords: List[str], name: Optional[str] = None) -> None:
+        self.keywords = keywords
+        self._name = name or f"ContainsAny({keywords})"
+
+    def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
+        assert isinstance(output, str), "Experiment run output must be a string"
+        found = [keyword for keyword in self.keywords if keyword in output]
+        if found:
+            explanation = f"the keywords {found} were found in the output"
+        else:
+            explanation = f"none of the keywords {self.keywords} were found in the output"
+        return EvaluationResult(
+            score=float(bool(found)),
+            explanation=explanation,
+        )
+
+
+class ContainsAllKeywords(Evaluator):
+    def __init__(self, keywords: List[str], name: Optional[str] = None) -> None:
+        self.keywords = keywords
+        self._name = name or f"ContainsAll({keywords})"
+
+    def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
+        assert isinstance(output, str), "Experiment run output must be a string"
+        not_found = [keyword for keyword in self.keywords if keyword not in output]
+        if not_found:
+            contains_all = False
+            explanation = f"the keywords {not_found} were not found in the output"
+        else:
+            contains_all = True
+            explanation = f"all of the keywords {self.keywords} were found in the output"
+        return EvaluationResult(
+            score=float(contains_all),
+            explanation=explanation,
+        )
+
+
+class MatchesRegex(Evaluator):
+    def __init__(self, pattern: Union[str, re.Pattern[str]], name: Optional[str] = None) -> None:
+        if isinstance(pattern, str):
+            pattern = re.compile(pattern)
+        self.pattern = pattern
+        assert isinstance(pattern, re.Pattern)
+        self._name = name or f"matches_({pattern})"
+
+    def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
+        assert isinstance(output, str), "Experiment run output must be a string"
+        matches = self.pattern.findall(output)
+        if matches:
+            explanation = (
+                f"the substrings {matches} matched the regex pattern {self.pattern.pattern}"
+            )
+        else:
+            explanation = f"no substrings matched the regex pattern {self.pattern.pattern}"
+        return EvaluationResult(
+            score=float(bool(matches)),
+            explanation=explanation,
+        )

arize_phoenix-4.4.4rc3/src/phoenix/datasets/evaluators.py → arize_phoenix-4.4.4rc5/src/phoenix/datasets/evaluators/llm_evaluators.py

@@ -1,72 +1,23 @@
-import json
 import re
-from typing import TYPE_CHECKING, Callable, Optional, Type
+from types import MappingProxyType
+from typing import Any, Callable, Optional, Type
 
+from phoenix.datasets.evaluators.utils import (
+    ExampleInput,
+    ExampleMetadata,
+    ExperimentEvaluator,
+    LLMEvaluator,
+    _unwrap_json,
+)
 from phoenix.datasets.types import (
     EvaluationResult,
-    Example,
-    ExperimentEvaluator,
-    ExperimentRun,
-    JSONSerializable,
+    TaskOutput,
 )
 from phoenix.evals.models.base import BaseModel as LLMBaseModel
 from phoenix.evals.utils import snap_to_rail
 
 
-def _unwrap_json(obj: JSONSerializable) -> JSONSerializable:
-    if isinstance(obj, dict):
-        if len(obj) == 1:
-            key = next(iter(obj.keys()))
-            output = obj[key]
-            assert isinstance(
-                output, (dict, list, str, int, float, bool, type(None))
-            ), "Output must be JSON serializable"
-            return output
-    return obj
-
-
-class JSONParsable:
-    annotator_kind = "CODE"
-    name = "JSONParsable"
-
-    def evaluate(self, example: Example, exp_run: ExperimentRun) -> EvaluationResult:
-        assert exp_run.output is not None
-        output = _unwrap_json(exp_run.output.result)
-        assert isinstance(output, str), "Experiment run output must be a string"
-        try:
-            json.loads(output)
-            json_parsable = True
-        except BaseException:
-            json_parsable = False
-        return EvaluationResult(
-            score=int(json_parsable),
-        )
-
-
-class ContainsKeyword:
-    annotator_kind = "CODE"
-
-    def __init__(self, keyword: str) -> None:
-        super().__init__()
-        self.keyword = keyword
-        self.name = f"ContainsKeyword({keyword})"
-
-    def evaluate(self, example: Example, exp_run: ExperimentRun) -> EvaluationResult:
-        assert exp_run.output is not None
-        result = _unwrap_json(exp_run.output.result)
-        assert isinstance(result, str), "Experiment run output must be a string"
-        found = self.keyword in result
-        return EvaluationResult(
-            score=float(found),
-            explanation=(
-                f"the string {repr(self.keyword)} was "
-                f"{'found' if found else 'not found'} in the output"
-            ),
-        )
-
-
-class LLMCriteriaEvaluator:
-    annotator_kind = "LLM"
+class LLMCriteriaEvaluator(LLMEvaluator):
     _base_template = (
         "Determine if the following text is {criteria}. {description}"
         "First, explain step-by-step why you think the text is or is not {criteria}. Then provide "
@@ -77,7 +28,7 @@ class LLMCriteriaEvaluator:
         "EXPLANATION: *a step by step explanation of your reasoning for whether the text meets "
         "the criteria*\n"
         "LABEL: *true or false*\n\n"
-        "Follow this template for the following text:\n\n"
+        "Follow this template for the following example:\n\n"
         "CRITERIA: the text is '{criteria}'\n"
         "TEXT: {text}\n"
         "EXPLANATION: "
@@ -95,21 +46,23 @@ class LLMCriteriaEvaluator:
         self.criteria = criteria
         self.description = description
         self.template = self._format_base_template(self.criteria, self.description)
-        self.name = name
+        self._name = name
 
-    def evaluate(self, example: Example, exp_run: ExperimentRun) -> EvaluationResult:
-        formatted_template = self._format_eval_template(exp_run)
+    def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
+        formatted_template = self._format_eval_template(output)
         unparsed_response = self.model._generate(formatted_template)
         return self._parse_eval_output(unparsed_response)
 
-    async def async_evaluate(self, example: Example, exp_run: ExperimentRun) -> EvaluationResult:
-        formatted_template = self._format_eval_template(exp_run)
+    async def async_evaluate(
+        self, *, output: Optional[TaskOutput] = None, **_: Any
+    ) -> EvaluationResult:
+        formatted_template = self._format_eval_template(output)
         unparsed_response = await self.model._async_generate(formatted_template)
         return self._parse_eval_output(unparsed_response)
 
-    def _format_eval_template(self, experiment_run: ExperimentRun) -> str:
-        assert experiment_run.output is not None
-        result = _unwrap_json(experiment_run.output.result)
+    def _format_eval_template(self, output: TaskOutput) -> str:
+        assert output is not None
+        result = _unwrap_json(output)
         return self.template.format(text=str(result))
 
     def _parse_eval_output(self, unparsed_response: str) -> EvaluationResult:
@@ -142,40 +95,43 @@ class LLMCriteriaEvaluator:
 
 
 def criteria_evaluator_factory(
-    class_name: str, criteria: str, description: str
+    class_name: str, criteria: str, description: str, default_name: str
 ) -> Type[ExperimentEvaluator]:
+    def _init(self, model: LLMBaseModel, name: str = default_name) -> None:  # type: ignore
+        LLMCriteriaEvaluator.__init__(self, model, criteria, description, name=name)
+
     return type(
         class_name,
         (LLMCriteriaEvaluator,),
         {
-            "__init__": lambda self, model: LLMCriteriaEvaluator.__init__(
-                self, model, criteria, description, name=class_name
-            ),
+            "__init__": _init,
             "__module__": __name__,
-            "name": class_name,
             "template": LLMCriteriaEvaluator._format_base_template(criteria, description),
         },
     )
 
 
-LLMConcisenessEvaluator = criteria_evaluator_factory(
-    class_name="LLMConcisenessEvaluator",
+ConcisenessEvaluator = criteria_evaluator_factory(
+    class_name="ConcisenessEvaluator",
     criteria="concise",
     description="is just a few sentences and easy to follow",
+    default_name="Conciseness",
 )
 
 
-LLMHelpfulnessEvaluator = criteria_evaluator_factory(
-    class_name="LLMHelpfulnessEvaluator",
+HelpfulnessEvaluator = criteria_evaluator_factory(
+    class_name="HelpfulnessEvaluator",
     criteria="helpful",
     description="provides useful information",
+    default_name="Helpfulness",
 )
 
 
-LLMCoherenceEvaluator = criteria_evaluator_factory(
-    class_name="LLMCoherenceEvaluator",
+CoherenceEvaluator = criteria_evaluator_factory(
+    class_name="CoherenceEvaluator",
     criteria="coherent",
-    description="is coherent, well-structured, and organized",
+    description="is coherent, well-structured, and logically sound",
+    default_name="Coherence",
 )
 
 
@@ -192,8 +148,7 @@ def _parse_label_from_explanation(raw_string: str) -> str:
     return raw_string
 
 
-class RelevanceEvaluator:
-    annotator_kind = "LLM"
+class RelevanceEvaluator(LLMEvaluator):
     template = (
         "Determine if the following response is relevant to the query. In this context, "
         "'relevance' means that the response directly addresses the core question or topic of the "
@@ -217,19 +172,24 @@ class RelevanceEvaluator:
     def __init__(
         self,
         model: LLMBaseModel,
-        get_query: Optional[Callable[[Example, ExperimentRun], str]] = None,
-        get_response: Optional[Callable[[Example, ExperimentRun], str]] = None,
+        get_query: Optional[Callable[[ExampleInput, ExampleMetadata], str]] = None,
+        get_response: Optional[Callable[[Optional[TaskOutput], ExampleMetadata], str]] = None,
         name: str = "RelevanceEvaluator",
     ):
         self.model = model
-        self.name = name
+        self._name = name
         self.get_query = get_query or self._default_get_query
         self.get_response = get_response or self._default_get_response
 
-    def _format_eval_template(self, example: Example, experiment_run: ExperimentRun) -> str:
-        assert experiment_run.output is not None
-        query = self.get_query(example, experiment_run)
-        response = self.get_response(example, experiment_run)
+    def _format_eval_template(
+        self,
+        output: Optional[TaskOutput] = None,
+        input: ExampleInput = MappingProxyType({}),
+        metadata: ExampleMetadata = MappingProxyType({}),
+    ) -> str:
+        assert output is not None
+        query = self.get_query(input, metadata)
+        response = self.get_response(output, metadata)
         return self.template.format(query=query, response=response)
 
     def _parse_eval_output(self, unparsed_response: str) -> EvaluationResult:
@@ -250,26 +210,35 @@ class RelevanceEvaluator:
             metadata={},
         )
 
-    def _default_get_query(self, example: Example, experiment_run: ExperimentRun) -> str:
-        return str(example.input)
+    def _default_get_query(self, input: ExampleInput, *args: Any, **kwargs: Any) -> str:
+        return str(input)
 
-    def _default_get_response(self, example: Example, experiment_run: ExperimentRun) -> str:
-        assert experiment_run.output is not None
-        return str(_unwrap_json(experiment_run.output.result))
+    def _default_get_response(
+        self, output: Optional[TaskOutput] = None, *args: Any, **kwargs: Any
+    ) -> str:
+        assert output is not None
+        return str(_unwrap_json(output))
 
-    def evaluate(self, example: Example, exp_run: ExperimentRun) -> EvaluationResult:
-        formatted_template = self._format_eval_template(example, exp_run)
+    def evaluate(
+        self,
+        *,
+        output: Optional[TaskOutput] = None,
+        metadata: ExampleMetadata = MappingProxyType({}),
+        input: ExampleInput = MappingProxyType({}),
+        **_: Any,
+    ) -> EvaluationResult:
+        formatted_template = self._format_eval_template(output, input, metadata)
         unparsed_response = self.model._generate(formatted_template)
         return self._parse_eval_output(unparsed_response)
 
-    async def async_evaluate(self, example: Example, exp_run: ExperimentRun) -> EvaluationResult:
-        formatted_template = self._format_eval_template(example, exp_run)
+    async def async_evaluate(
+        self,
+        *,
+        output: Optional[TaskOutput] = None,
+        metadata: ExampleMetadata = MappingProxyType({}),
+        input: ExampleInput = MappingProxyType({}),
+        **_: Any,
+    ) -> EvaluationResult:
+        formatted_template = self._format_eval_template(output, input, metadata)
         unparsed_response = await self.model._async_generate(formatted_template)
         return self._parse_eval_output(unparsed_response)
-
-
-# Someday we'll do typing checking in unit tests.
-if TYPE_CHECKING:
-    _: ExperimentEvaluator
-    _ = JSONParsable()
-    _ = ContainsKeyword("test")