arize-phoenix 4.4.4rc5__tar.gz → 4.4.4rc6__tar.gz

{arize_phoenix-4.4.4rc5 → arize_phoenix-4.4.4rc6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: arize-phoenix
-Version: 4.4.4rc5
+Version: 4.4.4rc6
 Summary: AI Observability and Evaluation
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues
@@ -20,7 +20,7 @@ Requires-Python: <3.13,>=3.8
 Requires-Dist: aioitertools
 Requires-Dist: aiosqlite
 Requires-Dist: alembic<2,>=1.3.0
-Requires-Dist: arize-phoenix-evals>=0.3.0
+Requires-Dist: arize-phoenix-evals>=0.13.1
 Requires-Dist: cachetools
 Requires-Dist: grpcio
 Requires-Dist: hdbscan>=0.8.33
@@ -36,7 +36,7 @@ Requires-Dist: opentelemetry-exporter-otlp
 Requires-Dist: opentelemetry-proto>=1.12.0
 Requires-Dist: opentelemetry-sdk
 Requires-Dist: opentelemetry-semantic-conventions
-Requires-Dist: pandas
+Requires-Dist: pandas>=1.0
 Requires-Dist: protobuf<6.0,>=3.20
 Requires-Dist: psutil
 Requires-Dist: pyarrow
@@ -79,6 +79,7 @@ Requires-Dist: llama-index>=0.10.3; extra == 'dev'
 Requires-Dist: nbqa; extra == 'dev'
 Requires-Dist: pandas-stubs==2.0.3.230814; (python_version < '3.9') and extra == 'dev'
 Requires-Dist: pandas-stubs==2.2.2.240603; (python_version >= '3.9') and extra == 'dev'
+Requires-Dist: pandas>=1.0; extra == 'dev'
 Requires-Dist: pre-commit; extra == 'dev'
 Requires-Dist: prometheus-client; extra == 'dev'
 Requires-Dist: psycopg[binary]; extra == 'dev'
@@ -88,10 +89,15 @@ Requires-Dist: pytest-postgresql; extra == 'dev'
 Requires-Dist: pytest==8.2.2; extra == 'dev'
 Requires-Dist: ruff==0.4.9; extra == 'dev'
 Requires-Dist: strawberry-graphql[debug-server,opentelemetry]==0.235.0; extra == 'dev'
+Requires-Dist: tabulate; extra == 'dev'
+Requires-Dist: types-tabulate; extra == 'dev'
 Provides-Extra: evals
 Provides-Extra: experimental
 Provides-Extra: llama-index
-Requires-Dist: llama-index==0.10.44; extra == 'llama-index'
+Requires-Dist: llama-index-embeddings-openai; extra == 'llama-index'
+Requires-Dist: llama-index-llms-openai; extra == 'llama-index'
+Requires-Dist: llama-index-readers-file; extra == 'llama-index'
+Requires-Dist: llama-index==0.10.51; extra == 'llama-index'
 Provides-Extra: pg
 Requires-Dist: asyncpg; extra == 'pg'
 Description-Content-Type: text/markdown
@@ -127,7 +133,7 @@ Description-Content-Type: text/markdown
 
 Phoenix is an open-source AI observability platform designed for experimentation, evaluation, and troubleshooting. It provides:
 
--   **_Tracing_** - Trace your LLM application's runtime using using OpenTelemetry-based instrumentation.
+-   **_Tracing_** - Trace your LLM application's runtime using OpenTelemetry-based instrumentation.
 -   **_Evaluation_** - Leverage LLMs to benchmark your application's performance using response and retrieval evals.
 -   **_Inference Analysis_** - Visualize inferences and embeddings using dimensionality reduction and clustering to identify drift and performance degradation.
 

{arize_phoenix-4.4.4rc5 → arize_phoenix-4.4.4rc6}/README.md

@@ -29,7 +29,7 @@
 
 Phoenix is an open-source AI observability platform designed for experimentation, evaluation, and troubleshooting. It provides:
 
--   **_Tracing_** - Trace your LLM application's runtime using using OpenTelemetry-based instrumentation.
+-   **_Tracing_** - Trace your LLM application's runtime using OpenTelemetry-based instrumentation.
 -   **_Evaluation_** - Leverage LLMs to benchmark your application's performance using response and retrieval evals.
 -   **_Inference Analysis_** - Visualize inferences and embeddings using dimensionality reduction and clustering to identify drift and performance degradation.
 

{arize_phoenix-4.4.4rc5 → arize_phoenix-4.4.4rc6}/examples/manually-instrumented-chatbot/frontend/requirements.txt

@@ -87,7 +87,7 @@ referencing==0.34.0
     # via
     #   jsonschema
     #   jsonschema-specifications
-requests==2.31.0
+requests==2.32.2
     # via streamlit
 rich==13.7.1
     # via streamlit
@@ -110,7 +110,7 @@ toml==0.10.2
     # via streamlit
 toolz==0.12.1
     # via altair
-tornado==6.4
+tornado==6.4.1
     # via streamlit
 typing-extensions==4.11.0
     # via

{arize_phoenix-4.4.4rc5 → arize_phoenix-4.4.4rc6}/pyproject.toml

@@ -24,7 +24,7 @@ classifiers = [
 dependencies = [
   "scikit-learn",
   "numpy<2",  # https://github.com/scikit-learn-contrib/hdbscan/issues/642
-  "pandas",
+  "pandas>=1.0",
   "jinja2",
   "umap-learn",
   "hdbscan>=0.8.33",
@@ -58,7 +58,7 @@ dependencies = [
   "sqlean.py>=3.45.1",
   "cachetools",
   "python-multipart",  # see https://www.starlette.io/#dependencies
-  "arize-phoenix-evals>=0.3.0",
+  "arize-phoenix-evals>=0.13.1",
 ]
 dynamic = ["version"]
 
@@ -69,6 +69,9 @@ dev = [
   "jupyter",
   "nbqa",
   "ruff==0.4.9",
+  "pandas>=1.0",
+  "tabulate",  # used by DataFrame.to_markdown()
+  "types-tabulate",
   "pandas-stubs==2.2.2.240603; python_version>='3.9'",
   "pandas-stubs==2.0.3.230814; python_version<'3.9'",
   "pytest==8.2.2",
@@ -90,7 +93,10 @@ dev = [
 evals = []
 experimental = []
 llama-index = [
-  "llama-index==0.10.44",  # always pin to a version that keeps our notebooks working
+  "llama-index==0.10.51",  # always pin to a version that keeps our notebooks working
+  "llama-index-readers-file",
+  "llama-index-llms-openai",
+  "llama-index-embeddings-openai",
 ]
 pg = [
   "asyncpg",
@@ -163,7 +169,9 @@ dependencies = [
 dependencies = [
   "mypy==1.10.0",
   "tenacity",
+  "pandas>=1.0",
   "pandas-stubs==2.0.3.230814",
+  "types-tabulate",
   "types-psutil",
   "types-tqdm",
   "types-protobuf",
@@ -337,6 +345,7 @@ disallow_untyped_defs = true
 disallow_incomplete_defs  = true
 strict = true
 exclude = [
+  "api_reference",
   "packages/",
   "src/phoenix/evals/",
   "dist/",
@@ -358,6 +367,7 @@ module = [
   "wrapt",
   "langchain.*",
   "litellm",
+  "litellm.*",
   "nest_asyncio",
   "opentelemetry.*",
   "pyarrow",

{arize_phoenix-4.4.4rc5 → arize_phoenix-4.4.4rc6}/src/phoenix/config.py

@@ -233,4 +233,25 @@ def get_env_client_headers() -> Optional[Dict[str, str]]:
     return None
 
 
+def get_base_url() -> str:
+    host = get_env_host()
+    if host == "0.0.0.0":
+        host = "127.0.0.1"
+    base_url = get_env_collector_endpoint() or f"http://{host}:{get_env_port()}"
+    return base_url if base_url.endswith("/") else base_url + "/"
+
+
+def get_web_base_url() -> str:
+    """Return the web UI base URL.
+
+    Returns:
+        str: the web UI base URL
+    """
+    from phoenix.session.session import active_session
+
+    if session := active_session():
+        return session.url
+    return get_base_url()
+
+
 DEFAULT_PROJECT_NAME = "default"

{arize_phoenix-4.4.4rc5 → arize_phoenix-4.4.4rc6}/src/phoenix/datetime_utils.py

@@ -14,6 +14,10 @@ from pandas.core.dtypes.common import (
 _LOCAL_TIMEZONE = datetime.now(timezone.utc).astimezone().tzinfo
 
 
+def local_now() -> datetime:
+    return datetime.now(timezone.utc).astimezone(tz=_LOCAL_TIMEZONE)
+
+
 def normalize_datetime(
     dt: Optional[datetime],
     tz: Optional[tzinfo] = None,

{arize_phoenix-4.4.4rc5 → arize_phoenix-4.4.4rc6}/src/phoenix/db/insertion/evaluation.py

@@ -6,7 +6,7 @@ from typing_extensions import assert_never
 
 from phoenix.db import models
 from phoenix.db.helpers import SupportedSQLDialect, num_docs_col
-from phoenix.db.insertion.helpers import OnConflict, insert_stmt
+from phoenix.db.insertion.helpers import OnConflict, insert_on_conflict
 from phoenix.exceptions import PhoenixException
 from phoenix.trace import v1 as pb
 
@@ -91,7 +91,7 @@ async def _insert_trace_evaluation(
     set_.pop("metadata_")
     set_["metadata"] = values["metadata_"]  # `metadata` must match database
     await session.execute(
-        insert_stmt(
+        insert_on_conflict(
             dialect=dialect,
             table=models.TraceAnnotation,
             values=values,
@@ -139,7 +139,7 @@ async def _insert_span_evaluation(
     set_.pop("metadata_")
     set_["metadata"] = values["metadata_"]  # `metadata` must match database
     await session.execute(
-        insert_stmt(
+        insert_on_conflict(
             dialect=dialect,
             table=models.SpanAnnotation,
             values=values,
@@ -196,7 +196,7 @@ async def _insert_document_evaluation(
     set_.pop("metadata_")
     set_["metadata"] = values["metadata_"]  # `metadata` must match database
     await session.execute(
-        insert_stmt(
+        insert_on_conflict(
             dialect=dialect,
             table=models.DocumentAnnotation,
             values=values,

{arize_phoenix-4.4.4rc5 → arize_phoenix-4.4.4rc6}/src/phoenix/db/insertion/helpers.py

@@ -2,7 +2,7 @@ from abc import ABC
 from enum import Enum, auto
 from typing import Any, Awaitable, Callable, Mapping, Optional, Sequence
 
-from sqlalchemy import Insert, insert
+from sqlalchemy import Insert
 from sqlalchemy.dialects.postgresql import insert as insert_postgresql
 from sqlalchemy.dialects.sqlite import insert as insert_sqlite
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -25,26 +25,18 @@ class OnConflict(Enum):
     DO_UPDATE = auto()
 
 
-def insert_stmt(
+def insert_on_conflict(
     dialect: SupportedSQLDialect,
     table: Any,
     values: Mapping[str, Any],
-    constraint: Optional[str] = None,
-    column_names: Sequence[str] = (),
+    constraint: str,
+    column_names: Sequence[str],
     on_conflict: OnConflict = OnConflict.DO_NOTHING,
     set_: Optional[Mapping[str, Any]] = None,
 ) -> Insert:
     """
     Dialect specific insertion statement using ON CONFLICT DO syntax.
     """
-    if bool(constraint) != bool(column_names):
-        raise ValueError(
-            "Both `constraint` and `column_names` must be provided or omitted at the same time."
-        )
-    if (dialect is SupportedSQLDialect.POSTGRESQL and constraint is None) or (
-        dialect is SupportedSQLDialect.SQLITE and not column_names
-    ):
-        return insert(table).values(values)
     if dialect is SupportedSQLDialect.POSTGRESQL:
         stmt_postgresql = insert_postgresql(table).values(values)
         if on_conflict is OnConflict.DO_NOTHING or not set_:

{arize_phoenix-4.4.4rc5 → arize_phoenix-4.4.4rc6}/src/phoenix/db/insertion/span.py

@@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from phoenix.db import models
 from phoenix.db.helpers import SupportedSQLDialect
-from phoenix.db.insertion.helpers import OnConflict, insert_stmt
+from phoenix.db.insertion.helpers import OnConflict, insert_on_conflict
 from phoenix.trace.attributes import get_attribute_value
 from phoenix.trace.schemas import Span, SpanStatusCode
 
@@ -27,7 +27,7 @@ async def insert_span(
 ) -> Optional[SpanInsertionEvent]:
     dialect = SupportedSQLDialect(session.bind.dialect.name)
     project_rowid = await session.scalar(
-        insert_stmt(
+        insert_on_conflict(
             dialect=dialect,
             table=models.Project,
             constraint="uq_projects_name",
@@ -87,7 +87,7 @@ async def insert_span(
         cumulative_llm_token_count_prompt += cast(int, accumulation[1] or 0)
         cumulative_llm_token_count_completion += cast(int, accumulation[2] or 0)
     span_rowid = await session.scalar(
-        insert_stmt(
+        insert_on_conflict(
             dialect=dialect,
             table=models.Span,
             constraint="uq_spans_span_id",

{arize_phoenix-4.4.4rc5 → arize_phoenix-4.4.4rc6}/src/phoenix/db/models.py

@@ -92,7 +92,7 @@ class UtcTimeStamp(TypeDecorator[datetime]):
 
 
 class ExperimentResult(TypedDict, total=False):
-    result: Dict[str, Any]
+    result: Any
 
 
 class Base(DeclarativeBase):

arize_phoenix-4.4.4rc6/src/phoenix/experiments/__init__.py

@@ -0,0 +1,6 @@
+from .functions import evaluate_experiment, run_experiment
+
+__all__ = [
+    "evaluate_experiment",
+    "run_experiment",
+]

arize_phoenix-4.4.4rc6/src/phoenix/experiments/evaluators/__init__.py

@@ -0,0 +1,29 @@
+from phoenix.experiments.evaluators.code_evaluators import (
+    ContainsAllKeywords,
+    ContainsAnyKeyword,
+    ContainsKeyword,
+    JSONParsable,
+    MatchesRegex,
+)
+from phoenix.experiments.evaluators.llm_evaluators import (
+    CoherenceEvaluator,
+    ConcisenessEvaluator,
+    HelpfulnessEvaluator,
+    LLMCriteriaEvaluator,
+    RelevanceEvaluator,
+)
+from phoenix.experiments.evaluators.utils import create_evaluator
+
+__all__ = [
+    "create_evaluator",
+    "ContainsAllKeywords",
+    "ContainsAnyKeyword",
+    "ContainsKeyword",
+    "JSONParsable",
+    "MatchesRegex",
+    "CoherenceEvaluator",
+    "ConcisenessEvaluator",
+    "LLMCriteriaEvaluator",
+    "HelpfulnessEvaluator",
+    "RelevanceEvaluator",
+]

arize_phoenix-4.4.4rc6/src/phoenix/experiments/evaluators/base.py

@@ -0,0 +1,153 @@
+import functools
+import inspect
+from abc import ABC
+from types import MappingProxyType
+from typing import Any, Awaitable, Callable, Optional, Union
+
+from typing_extensions import TypeAlias
+
+from phoenix.experiments.evaluators.utils import validate_signature
+from phoenix.experiments.types import (
+    AnnotatorKind,
+    EvaluationResult,
+    EvaluatorKind,
+    EvaluatorName,
+    EvaluatorOutput,
+    ExampleInput,
+    ExampleMetadata,
+    ExampleOutput,
+    TaskOutput,
+)
+
+
+class Evaluator(ABC):
+    """
+    A helper super class to guide the implementation of an `Evaluator` object.
+    Subclasses must implement either the `evaluate` or `async_evaluate` method.
+    Implementing both methods is recommended, but not required.
+
+    This Class is intended to be subclassed, and should not be instantiated directly.
+    """
+
+    _kind: AnnotatorKind
+    _name: EvaluatorName
+
+    @functools.cached_property
+    def name(self) -> EvaluatorName:
+        if hasattr(self, "_name"):
+            return self._name
+        return self.__class__.__name__
+
+    @functools.cached_property
+    def kind(self) -> EvaluatorKind:
+        if hasattr(self, "_kind"):
+            return self._kind.value
+        return AnnotatorKind.CODE.value
+
+    def __new__(cls, *args: Any, **kwargs: Any) -> "Evaluator":
+        if cls is Evaluator:
+            raise TypeError(f"{cls.__name__} is an abstract class and should not be instantiated.")
+        return object.__new__(cls)
+
+    def evaluate(
+        self,
+        *,
+        output: Optional[TaskOutput] = None,
+        expected: Optional[ExampleOutput] = None,
+        metadata: ExampleMetadata = MappingProxyType({}),
+        input: ExampleInput = MappingProxyType({}),
+        **kwargs: Any,
+    ) -> EvaluationResult:
+        # For subclassing, one should implement either this sync method or the
+        # async version. Implementing both is recommended but not required.
+        raise NotImplementedError
+
+    async def async_evaluate(
+        self,
+        *,
+        output: Optional[TaskOutput] = None,
+        expected: Optional[ExampleOutput] = None,
+        metadata: ExampleMetadata = MappingProxyType({}),
+        input: ExampleInput = MappingProxyType({}),
+        **kwargs: Any,
+    ) -> EvaluationResult:
+        # For subclassing, one should implement either this async method or the
+        # sync version. Implementing both is recommended but not required.
+        return self.evaluate(
+            output=output,
+            expected=expected,
+            metadata=metadata,
+            input=input,
+            **kwargs,
+        )
+
+    def __init_subclass__(cls, is_abstract: bool = False, **kwargs: Any) -> None:
+        super().__init_subclass__(**kwargs)
+        if is_abstract:
+            return
+        evaluate_fn_signature = inspect.signature(Evaluator.evaluate)
+        for super_cls in inspect.getmro(cls):
+            if super_cls in (LLMEvaluator, Evaluator):
+                break
+            if evaluate := super_cls.__dict__.get(Evaluator.evaluate.__name__):
+                assert callable(evaluate), "`evaluate()` method should be callable"
+                # need to remove the first param, i.e. `self`
+                _validate_sig(functools.partial(evaluate, None), "evaluate")
+                return
+            if async_evaluate := super_cls.__dict__.get(Evaluator.async_evaluate.__name__):
+                assert callable(async_evaluate), "`async_evaluate()` method should be callable"
+                # need to remove the first param, i.e. `self`
+                _validate_sig(functools.partial(async_evaluate, None), "async_evaluate")
+                return
+        raise ValueError(
+            f"Evaluator must implement either "
+            f"`def evaluate{evaluate_fn_signature}` or "
+            f"`async def async_evaluate{evaluate_fn_signature}`"
+        )
+
+
+def _validate_sig(fn: Callable[..., Any], fn_name: str) -> None:
+    sig = inspect.signature(fn)
+    validate_signature(sig)
+    for param in sig.parameters.values():
+        if param.kind is inspect.Parameter.VAR_KEYWORD:
+            return
+    else:
+        raise ValueError(f"`{fn_name}` should allow variadic keyword arguments `**kwargs`")
+
+
+class CodeEvaluator(Evaluator, ABC, is_abstract=True):
+    """
+    A convenience super class for defining code evaluators.
+
+    This class is intended to be subclassed, and should not be instantiated directly.
+    """
+
+    _kind = AnnotatorKind.CODE
+
+    def __new__(cls, *args: Any, **kwargs: Any) -> "CodeEvaluator":
+        if cls is CodeEvaluator:
+            raise TypeError(f"{cls.__name__} is an abstract class and should not be instantiated.")
+        return object.__new__(cls)
+
+
+class LLMEvaluator(Evaluator, ABC, is_abstract=True):
+    """
+    A convenience super class for defining LLM evaluators.
+
+    This class is intended to be subclassed, and should not be instantiated directly.
+    """
+
+    _kind = AnnotatorKind.LLM
+
+    def __new__(cls, *args: Any, **kwargs: Any) -> "LLMEvaluator":
+        if cls is LLMEvaluator:
+            raise TypeError(f"{cls.__name__} is an abstract class and should not be instantiated.")
+        return object.__new__(cls)
+
+
+ExperimentEvaluator: TypeAlias = Union[
+    Evaluator,
+    Callable[..., EvaluatorOutput],
+    Callable[..., Awaitable[EvaluatorOutput]],
+]

{arize_phoenix-4.4.4rc5/src/phoenix/datasets → arize_phoenix-4.4.4rc6/src/phoenix/experiments}/evaluators/code_evaluators.py

@@ -4,11 +4,11 @@ import json
 import re
 from typing import Any, List, Optional, Union
 
-from phoenix.datasets.evaluators.utils import Evaluator
-from phoenix.datasets.types import EvaluationResult, TaskOutput
+from phoenix.experiments.evaluators.base import CodeEvaluator
+from phoenix.experiments.types import EvaluationResult, TaskOutput
 
 
-class JSONParsable(Evaluator):
+class JSONParsable(CodeEvaluator):
     def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
         assert isinstance(output, str), "Experiment run output must be a string"
         try:
@@ -21,7 +21,7 @@ class JSONParsable(Evaluator):
         )
 
 
-class ContainsKeyword(Evaluator):
+class ContainsKeyword(CodeEvaluator):
     def __init__(self, keyword: str, name: Optional[str] = None) -> None:
         self.keyword = keyword
         self._name = name or f"Contains({repr(keyword)})"
@@ -38,7 +38,7 @@ class ContainsKeyword(Evaluator):
         )
 
 
-class ContainsAnyKeyword(Evaluator):
+class ContainsAnyKeyword(CodeEvaluator):
     def __init__(self, keywords: List[str], name: Optional[str] = None) -> None:
         self.keywords = keywords
         self._name = name or f"ContainsAny({keywords})"
@@ -56,7 +56,7 @@ class ContainsAnyKeyword(Evaluator):
         )
 
 
-class ContainsAllKeywords(Evaluator):
+class ContainsAllKeywords(CodeEvaluator):
     def __init__(self, keywords: List[str], name: Optional[str] = None) -> None:
         self.keywords = keywords
         self._name = name or f"ContainsAll({keywords})"
@@ -76,7 +76,7 @@ class ContainsAllKeywords(Evaluator):
         )
 
 
-class MatchesRegex(Evaluator):
+class MatchesRegex(CodeEvaluator):
     def __init__(self, pattern: Union[str, re.Pattern[str]], name: Optional[str] = None) -> None:
         if isinstance(pattern, str):
             pattern = re.compile(pattern)

{arize_phoenix-4.4.4rc5/src/phoenix/datasets → arize_phoenix-4.4.4rc6/src/phoenix/experiments}/evaluators/llm_evaluators.py

@@ -2,19 +2,19 @@ import re
 from types import MappingProxyType
 from typing import Any, Callable, Optional, Type
 
-from phoenix.datasets.evaluators.utils import (
-    ExampleInput,
-    ExampleMetadata,
+from phoenix.evals.models.base import BaseModel as LLMBaseModel
+from phoenix.evals.utils import snap_to_rail
+from phoenix.experiments.evaluators.base import (
     ExperimentEvaluator,
     LLMEvaluator,
-    _unwrap_json,
 )
-from phoenix.datasets.types import (
+from phoenix.experiments.evaluators.utils import unwrap_json
+from phoenix.experiments.types import (
     EvaluationResult,
+    ExampleInput,
+    ExampleMetadata,
     TaskOutput,
 )
-from phoenix.evals.models.base import BaseModel as LLMBaseModel
-from phoenix.evals.utils import snap_to_rail
 
 
 class LLMCriteriaEvaluator(LLMEvaluator):
@@ -62,7 +62,7 @@ class LLMCriteriaEvaluator(LLMEvaluator):
 
     def _format_eval_template(self, output: TaskOutput) -> str:
         assert output is not None
-        result = _unwrap_json(output)
+        result = unwrap_json(output)
         return self.template.format(text=str(result))
 
     def _parse_eval_output(self, unparsed_response: str) -> EvaluationResult:
@@ -217,7 +217,7 @@ class RelevanceEvaluator(LLMEvaluator):
         self, output: Optional[TaskOutput] = None, *args: Any, **kwargs: Any
     ) -> str:
         assert output is not None
-        return str(_unwrap_json(output))
+        return str(unwrap_json(output))
 
     def evaluate(
         self,