arize-phoenix 4.4.4rc3__tar.gz → 4.4.4rc4__tar.gz

{arize_phoenix-4.4.4rc3 → arize_phoenix-4.4.4rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: arize-phoenix
-Version: 4.4.4rc3
+Version: 4.4.4rc4
 Summary: AI Observability and Evaluation
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues

arize_phoenix-4.4.4rc4/src/phoenix/datasets/evaluators/__init__.py

@@ -0,0 +1,18 @@
+from phoenix.datasets.evaluators.code_evaluators import ContainsKeyword, JSONParsable
+from phoenix.datasets.evaluators.llm_evaluators import (
+    CoherenceEvaluator,
+    ConcisenessEvaluator,
+    HelpfulnessEvaluator,
+    LLMCriteriaEvaluator,
+    RelevanceEvaluator,
+)
+
+__all__ = [
+    "ContainsKeyword",
+    "JSONParsable",
+    "CoherenceEvaluator",
+    "ConcisenessEvaluator",
+    "LLMCriteriaEvaluator",
+    "HelpfulnessEvaluator",
+    "RelevanceEvaluator",
+]

arize_phoenix-4.4.4rc4/src/phoenix/datasets/evaluators/_utils.py

@@ -0,0 +1,13 @@
+from phoenix.datasets.types import JSONSerializable
+
+
+def _unwrap_json(obj: JSONSerializable) -> JSONSerializable:
+    if isinstance(obj, dict):
+        if len(obj) == 1:
+            key = next(iter(obj.keys()))
+            output = obj[key]
+            assert isinstance(
+                output, (dict, list, str, int, float, bool, type(None))
+            ), "Output must be JSON serializable"
+            return output
+    return obj

arize_phoenix-4.4.4rc4/src/phoenix/datasets/evaluators/code_evaluators.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+import json
+import re
+from typing import TYPE_CHECKING, List, Optional, Union
+
+from phoenix.datasets.evaluators._utils import _unwrap_json
+from phoenix.datasets.types import EvaluationResult, Example, ExperimentEvaluator, ExperimentRun
+
+
+class JSONParsable:
+    annotator_kind = "CODE"
+    name = "JSONParsable"
+
+    def evaluate(self, example: Example, exp_run: ExperimentRun) -> EvaluationResult:
+        assert exp_run.output is not None
+        output = _unwrap_json(exp_run.output.result)
+        assert isinstance(output, str), "Experiment run output must be a string"
+        try:
+            json.loads(output)
+            json_parsable = True
+        except BaseException:
+            json_parsable = False
+        return EvaluationResult(
+            score=int(json_parsable),
+        )
+
+
+class ContainsKeyword:
+    annotator_kind = "CODE"
+
+    def __init__(self, keyword: str, name: Optional[str] = None) -> None:
+        self.keyword = keyword
+        self.name = name or f"Contains({repr(keyword)})"
+
+    def evaluate(self, example: Example, exp_run: ExperimentRun) -> EvaluationResult:
+        assert exp_run.output is not None
+        result = _unwrap_json(exp_run.output.result)
+        assert isinstance(result, str), "Experiment run output must be a string"
+        found = self.keyword in result
+        return EvaluationResult(
+            score=float(found),
+            explanation=(
+                f"the string {repr(self.keyword)} was "
+                f"{'found' if found else 'not found'} in the output"
+            ),
+        )
+
+
+class ContainsAnyKeyword:
+    annotator_kind = "CODE"
+
+    def __init__(self, keywords: List[str], name: Optional[str] = None) -> None:
+        self.keywords = keywords
+        self.name = name or f"ContainsAny({keywords})"
+
+    def evaluate(self, example: Example, exp_run: ExperimentRun) -> EvaluationResult:
+        assert exp_run.output is not None
+        result = _unwrap_json(exp_run.output.result)
+        assert isinstance(result, str), "Experiment run output must be a string"
+        found = [keyword for keyword in self.keywords if keyword in result]
+        if found:
+            explanation = f"the keywords {found} were found in the output"
+        else:
+            explanation = f"none of the keywords {self.keywords} were found in the output"
+        return EvaluationResult(
+            score=float(bool(found)),
+            explanation=explanation,
+        )
+
+
+class ContainsAllKeywords:
+    annotator_kind = "CODE"
+
+    def __init__(self, keywords: List[str], name: Optional[str] = None) -> None:
+        self.keywords = keywords
+        self.name = name or f"ContainsAll({keywords})"
+
+    def evaluate(self, example: Example, exp_run: ExperimentRun) -> EvaluationResult:
+        assert exp_run.output is not None
+        result = _unwrap_json(exp_run.output.result)
+        assert isinstance(result, str), "Experiment run output must be a string"
+        not_found = [keyword for keyword in self.keywords if keyword not in result]
+        if not_found:
+            contains_all = False
+            explanation = f"the keywords {not_found} were not found in the output"
+        else:
+            contains_all = True
+            explanation = f"all of the keywords {self.keywords} were found in the output"
+        return EvaluationResult(
+            score=float(contains_all),
+            explanation=explanation,
+        )
+
+
+class MatchesRegex:
+    annotator_kind = "CODE"
+
+    def __init__(self, pattern: Union[str, re.Pattern[str]], name: Optional[str] = None) -> None:
+        if isinstance(pattern, str):
+            pattern = re.compile(pattern)
+        self.pattern = pattern
+        assert isinstance(pattern, re.Pattern)
+        self.name = name or f"matches_({pattern})"
+
+    def evaluate(self, example: Example, exp_run: ExperimentRun) -> EvaluationResult:
+        assert exp_run.output is not None
+        result = _unwrap_json(exp_run.output.result)
+        assert isinstance(result, str), "Experiment run output must be a string"
+        matches = self.pattern.findall(result)
+        if matches:
+            explanation = (
+                f"the substrings {matches} matched the regex pattern {self.pattern.pattern}"
+            )
+        else:
+            explanation = f"no substrings matched the regex pattern {self.pattern.pattern}"
+        return EvaluationResult(
+            score=float(bool(matches)),
+            explanation=explanation,
+        )
+
+
+# Someday we'll do typing checking in unit tests.
+if TYPE_CHECKING:
+    _: ExperimentEvaluator
+    _ = JSONParsable()
+    _ = ContainsKeyword("test")

arize_phoenix-4.4.4rc3/src/phoenix/datasets/evaluators.py → arize_phoenix-4.4.4rc4/src/phoenix/datasets/evaluators/llm_evaluators.py

@@ -1,70 +1,12 @@
-import json
 import re
-from typing import TYPE_CHECKING, Callable, Optional, Type
+from typing import Callable, Optional, Type
 
-from phoenix.datasets.types import (
-    EvaluationResult,
-    Example,
-    ExperimentEvaluator,
-    ExperimentRun,
-    JSONSerializable,
-)
+from phoenix.datasets.evaluators._utils import _unwrap_json
+from phoenix.datasets.types import EvaluationResult, Example, ExperimentEvaluator, ExperimentRun
 from phoenix.evals.models.base import BaseModel as LLMBaseModel
 from phoenix.evals.utils import snap_to_rail
 
 
-def _unwrap_json(obj: JSONSerializable) -> JSONSerializable:
-    if isinstance(obj, dict):
-        if len(obj) == 1:
-            key = next(iter(obj.keys()))
-            output = obj[key]
-            assert isinstance(
-                output, (dict, list, str, int, float, bool, type(None))
-            ), "Output must be JSON serializable"
-            return output
-    return obj
-
-
-class JSONParsable:
-    annotator_kind = "CODE"
-    name = "JSONParsable"
-
-    def evaluate(self, example: Example, exp_run: ExperimentRun) -> EvaluationResult:
-        assert exp_run.output is not None
-        output = _unwrap_json(exp_run.output.result)
-        assert isinstance(output, str), "Experiment run output must be a string"
-        try:
-            json.loads(output)
-            json_parsable = True
-        except BaseException:
-            json_parsable = False
-        return EvaluationResult(
-            score=int(json_parsable),
-        )
-
-
-class ContainsKeyword:
-    annotator_kind = "CODE"
-
-    def __init__(self, keyword: str) -> None:
-        super().__init__()
-        self.keyword = keyword
-        self.name = f"ContainsKeyword({keyword})"
-
-    def evaluate(self, example: Example, exp_run: ExperimentRun) -> EvaluationResult:
-        assert exp_run.output is not None
-        result = _unwrap_json(exp_run.output.result)
-        assert isinstance(result, str), "Experiment run output must be a string"
-        found = self.keyword in result
-        return EvaluationResult(
-            score=float(found),
-            explanation=(
-                f"the string {repr(self.keyword)} was "
-                f"{'found' if found else 'not found'} in the output"
-            ),
-        )
-
-
 class LLMCriteriaEvaluator:
     annotator_kind = "LLM"
     _base_template = (
@@ -77,7 +19,7 @@ class LLMCriteriaEvaluator:
         "EXPLANATION: *a step by step explanation of your reasoning for whether the text meets "
         "the criteria*\n"
         "LABEL: *true or false*\n\n"
-        "Follow this template for the following text:\n\n"
+        "Follow this template for the following example:\n\n"
         "CRITERIA: the text is '{criteria}'\n"
         "TEXT: {text}\n"
         "EXPLANATION: "
@@ -142,40 +84,43 @@ class LLMCriteriaEvaluator:
 
 
 def criteria_evaluator_factory(
-    class_name: str, criteria: str, description: str
+    class_name: str, criteria: str, description: str, default_name: str
 ) -> Type[ExperimentEvaluator]:
+    def _init(self, model: LLMBaseModel, name: str = default_name) -> None:  # type: ignore
+        LLMCriteriaEvaluator.__init__(self, model, criteria, description, name=name)
+
     return type(
         class_name,
         (LLMCriteriaEvaluator,),
         {
-            "__init__": lambda self, model: LLMCriteriaEvaluator.__init__(
-                self, model, criteria, description, name=class_name
-            ),
+            "__init__": _init,
             "__module__": __name__,
-            "name": class_name,
             "template": LLMCriteriaEvaluator._format_base_template(criteria, description),
         },
     )
 
 
-LLMConcisenessEvaluator = criteria_evaluator_factory(
-    class_name="LLMConcisenessEvaluator",
+ConcisenessEvaluator = criteria_evaluator_factory(
+    class_name="ConcisenessEvaluator",
     criteria="concise",
     description="is just a few sentences and easy to follow",
+    default_name="Conciseness",
 )
 
 
-LLMHelpfulnessEvaluator = criteria_evaluator_factory(
-    class_name="LLMHelpfulnessEvaluator",
+HelpfulnessEvaluator = criteria_evaluator_factory(
+    class_name="HelpfulnessEvaluator",
     criteria="helpful",
     description="provides useful information",
+    default_name="Helpfulness",
 )
 
 
-LLMCoherenceEvaluator = criteria_evaluator_factory(
-    class_name="LLMCoherenceEvaluator",
+CoherenceEvaluator = criteria_evaluator_factory(
+    class_name="CoherenceEvaluator",
     criteria="coherent",
-    description="is coherent, well-structured, and organized",
+    description="is coherent, well-structured, and logically sound",
+    default_name="Coherence",
 )
 
 
@@ -266,10 +211,3 @@ class RelevanceEvaluator:
         formatted_template = self._format_eval_template(example, exp_run)
         unparsed_response = await self.model._async_generate(formatted_template)
         return self._parse_eval_output(unparsed_response)
-
-
-# Someday we'll do typing checking in unit tests.
-if TYPE_CHECKING:
-    _: ExperimentEvaluator
-    _ = JSONParsable()
-    _ = ContainsKeyword("test")

{arize_phoenix-4.4.4rc3 → arize_phoenix-4.4.4rc4}/src/phoenix/datasets/experiments.py

@@ -458,6 +458,7 @@ def _evaluate_experiment(
         max_retries=0,
         exit_on_error=False,
         fallback_return_value=None,
+        tqdm_bar_format=get_tqdm_progress_bar_formatter("running experiment evaluations"),
     )
     evaluation_payloads, _execution_details = executor.run(evaluation_inputs)
     for payload in evaluation_payloads:

{arize_phoenix-4.4.4rc3 → arize_phoenix-4.4.4rc4}/src/phoenix/server/api/types/Span.py

@@ -59,6 +59,7 @@ class SpanKind(Enum):
     embedding = "EMBEDDING"
     agent = "AGENT"
     reranker = "RERANKER"
+    evaluator = "EVALUATOR"
     unknown = "UNKNOWN"
 
     @classmethod