arize-phoenix 2.0.0__tar.gz → 2.2.0rc0__tar.gz

{arize_phoenix-2.0.0 → arize_phoenix-2.2.0rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arize-phoenix
-Version: 2.0.0
+Version: 2.2.0rc0
 Summary: ML Observability in your notebook
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues
@@ -20,6 +20,8 @@ Requires-Dist: ddsketch
 Requires-Dist: hdbscan<1.0.0,>=0.8.33
 Requires-Dist: jinja2
 Requires-Dist: numpy
+Requires-Dist: opentelemetry-proto
+Requires-Dist: opentelemetry-sdk
 Requires-Dist: pandas
 Requires-Dist: protobuf<5.0,>=3.20
 Requires-Dist: psutil
@@ -36,8 +38,10 @@ Requires-Dist: umap-learn
 Requires-Dist: uvicorn
 Requires-Dist: wrapt
 Provides-Extra: dev
+Requires-Dist: anthropic; extra == 'dev'
 Requires-Dist: arize[autoembeddings,llm-evaluation]; extra == 'dev'
 Requires-Dist: gcsfs; extra == 'dev'
+Requires-Dist: google-cloud-aiplatform>=1.3; extra == 'dev'
 Requires-Dist: hatch; extra == 'dev'
 Requires-Dist: jupyter; extra == 'dev'
 Requires-Dist: langchain>=0.0.334; extra == 'dev'

{arize_phoenix-2.0.0 → arize_phoenix-2.2.0rc0}/pyproject.toml

@@ -40,6 +40,8 @@ dependencies = [
   "ddsketch",
   "tqdm",
   "requests",
+  "opentelemetry-sdk",
+  "opentelemetry-proto",
 ]
 dynamic = ["version"]
 
@@ -60,7 +62,9 @@ dev = [
   "arize[AutoEmbeddings, LLM_Evaluation]",
   "llama-index>=0.9.14",
   "langchain>=0.0.334",
-  "litellm>=1.0.3"
+  "litellm>=1.0.3",
+  "google-cloud-aiplatform>=1.3",
+  "anthropic",
 ]
 experimental = [
   "tenacity",
@@ -75,7 +79,7 @@ Issues = "https://github.com/Arize-ai/phoenix/issues"
 Source = "https://github.com/Arize-ai/phoenix"
 
 [tool.hatch.version]
-path = "src/phoenix/__init__.py"
+path = "src/phoenix/version.py"
 
 [build-system]
 requires = ["hatchling"]
@@ -242,7 +246,6 @@ dependencies = [
 
 [tool.hatch.envs.proto.scripts]
 recompile = """
-python -m grpc_tools.protoc -I src/phoenix/proto --python_out=src/phoenix --mypy_out=src/phoenix src/phoenix/proto/trace/v1/trace.proto &&
 python -m grpc_tools.protoc -I src/phoenix/proto --python_out=src/phoenix --mypy_out=src/phoenix src/phoenix/proto/trace/v1/evaluation.proto
 """
 
@@ -288,6 +291,7 @@ module = [
   "langchain.*",
   "litellm",
   "nest_asyncio",
+  "opentelemetry.*",
 ]
 ignore_missing_imports = true
 

{arize_phoenix-2.0.0 → arize_phoenix-2.2.0rc0}/src/phoenix/__init__.py

@@ -5,8 +5,7 @@ from .session.evaluation import log_evaluations
 from .session.session import NotebookEnvironment, Session, active_session, close_app, launch_app
 from .trace.fixtures import load_example_traces
 from .trace.trace_dataset import TraceDataset
-
-__version__ = "2.0.0"
+from .version import __version__
 
 # module level doc-string
 __doc__ = """
@@ -25,6 +24,7 @@ Here are just a few of the things that phoenix does well:
 """
 
 __all__ = [
+    "__version__",
     "Dataset",
     "EmbeddingColumnNames",
     "RetrievalEmbeddingColumnNames",

{arize_phoenix-2.0.0 → arize_phoenix-2.2.0rc0}/src/phoenix/core/evals.py

@@ -1,6 +1,7 @@
 import logging
 import weakref
 from collections import defaultdict
+from datetime import datetime, timezone
 from queue import SimpleQueue
 from threading import RLock, Thread
 from types import MethodType
@@ -46,6 +47,7 @@ class Evals:
         self._document_evaluations_by_name: DefaultDict[
             EvaluationName, DefaultDict[SpanID, Dict[DocumentPosition, pb.Evaluation]]
         ] = defaultdict(lambda: defaultdict(dict))
+        self._last_updated_at: Optional[datetime] = None
         self._start_consumer()
 
     def put(self, evaluation: pb.Evaluation) -> None:
@@ -92,10 +94,16 @@ class Evals:
             )
         else:
             assert_never(subject_id_kind)
+        self._last_updated_at = datetime.now(timezone.utc)
+
+    @property
+    def last_updated_at(self) -> Optional[datetime]:
+        return self._last_updated_at
 
     def get_span_evaluation(self, span_id: SpanID, name: str) -> Optional[pb.Evaluation]:
         with self._lock:
-            return self._evaluations_by_span_id[span_id].get(name)
+            span_evaluations = self._evaluations_by_span_id.get(span_id)
+            return span_evaluations.get(name) if span_evaluations else None
 
     def get_span_evaluation_names(self) -> List[EvaluationName]:
         with self._lock:
@@ -108,28 +116,36 @@ class Evals:
         with self._lock:
             if span_id is None:
                 return list(self._document_evaluations_by_name)
-            return list(self._document_evaluations_by_span_id[span_id])
+            document_evaluations = self._document_evaluations_by_span_id.get(span_id)
+            return list(document_evaluations) if document_evaluations else []
 
     def get_span_evaluation_labels(self, name: EvaluationName) -> Tuple[str, ...]:
         with self._lock:
-            return tuple(self._span_evaluation_labels[name])
+            labels = self._span_evaluation_labels.get(name)
+            return tuple(labels) if labels else ()
 
     def get_span_evaluation_span_ids(self, name: EvaluationName) -> Tuple[SpanID, ...]:
         with self._lock:
-            return tuple(self._span_evaluations_by_name[name].keys())
+            span_evaluations = self._span_evaluations_by_name.get(name)
+            return tuple(span_evaluations.keys()) if span_evaluations else ()
 
     def get_evaluations_by_span_id(self, span_id: SpanID) -> List[pb.Evaluation]:
         with self._lock:
-            return list(self._evaluations_by_span_id[span_id].values())
+            evaluations = self._evaluations_by_span_id.get(span_id)
+            return list(evaluations.values()) if evaluations else []
 
     def get_document_evaluation_span_ids(self, name: EvaluationName) -> Tuple[SpanID, ...]:
         with self._lock:
-            return tuple(self._document_evaluations_by_name[name].keys())
+            document_evaluations = self._document_evaluations_by_name.get(name)
+            return tuple(document_evaluations.keys()) if document_evaluations else ()
 
     def get_document_evaluations_by_span_id(self, span_id: SpanID) -> List[pb.Evaluation]:
         all_evaluations: List[pb.Evaluation] = []
         with self._lock:
-            for evaluations in self._document_evaluations_by_span_id[span_id].values():
+            document_evaluations = self._document_evaluations_by_span_id.get(span_id)
+            if not document_evaluations:
+                return all_evaluations
+            for evaluations in document_evaluations.values():
                 all_evaluations.extend(evaluations.values())
         return all_evaluations
 
@@ -144,7 +160,12 @@ class Evals:
         # of one trillion, we would not want to create a result that large.
         scores: List[float] = [np.nan] * num_documents
         with self._lock:
-            evaluations = self._document_evaluations_by_span_id[span_id][evaluation_name]
+            document_evaluations = self._document_evaluations_by_span_id.get(span_id)
+            if not document_evaluations:
+                return scores
+            evaluations = document_evaluations.get(evaluation_name)
+            if not evaluations:
+                return scores
             for document_position, evaluation in evaluations.items():
                 result = evaluation.result
                 if result.HasField("score") and document_position < num_documents:

{arize_phoenix-2.0.0 → arize_phoenix-2.2.0rc0}/src/phoenix/core/traces.py

@@ -13,20 +13,21 @@ from typing import (
     Iterator,
     List,
     Optional,
+    Set,
     SupportsFloat,
     Tuple,
-    Union,
     cast,
 )
 
+import opentelemetry.proto.trace.v1.trace_pb2 as otlp
 from ddsketch import DDSketch
 from sortedcontainers import SortedKeyList
 from typing_extensions import TypeAlias
 from wrapt import ObjectProxy
 
-import phoenix.trace.v1 as pb
 from phoenix.datetime_utils import right_open_time_range
 from phoenix.trace import semantic_conventions
+from phoenix.trace.otel import decode
 from phoenix.trace.schemas import (
     ATTRIBUTE_PREFIX,
     COMPUTED_PREFIX,
@@ -34,9 +35,10 @@ from phoenix.trace.schemas import (
     Span,
     SpanAttributes,
     SpanID,
+    SpanStatusCode,
     TraceID,
 )
-from phoenix.trace.v1.utils import decode, encode
+from phoenix.trace.semantic_conventions import RETRIEVAL_DOCUMENTS
 
 END_OF_QUEUE = None  # sentinel value for queue termination
 
@@ -74,15 +76,15 @@ class ReadableSpan(ObjectProxy):  # type: ignore
     are ingested, and would need to be re-computed on the fly.
     """
 
-    __wrapped__: pb.Span
-
-    def __init__(self, span: pb.Span) -> None:
+    def __init__(self, otlp_span: otlp.Span) -> None:
+        span = decode(otlp_span)
         super().__init__(span)
+        self._self_otlp_span = otlp_span
         self._self_computed_values: Dict[str, SupportsFloat] = {}
 
     @property
     def span(self) -> Span:
-        span = decode(self.__wrapped__)
+        span = decode(self._self_otlp_span)
         span.attributes.update(cast(SpanAttributes, self._self_computed_values))
         # TODO: compute latency rank percent (which can change depending on how
         # many spans already ingested).
@@ -96,9 +98,7 @@ class ReadableSpan(ObjectProxy):  # type: ignore
             return getattr(self.__wrapped__.context, suffix_key, None)
         if key.startswith(ATTRIBUTE_PREFIX):
             suffix_key = key[len(ATTRIBUTE_PREFIX) :]
-            if suffix_key not in self.__wrapped__.attributes:
-                return None
-            return self.__wrapped__.attributes[suffix_key]
+            return self.__wrapped__.attributes.get(suffix_key)
         return getattr(self.__wrapped__, key, None)
 
     def __setitem__(self, key: str, value: Any) -> None:
@@ -113,21 +113,21 @@ ChildSpanID: TypeAlias = SpanID
 
 class Traces:
     def __init__(self) -> None:
-        self._queue: "SimpleQueue[Optional[pb.Span]]" = SimpleQueue()
+        self._queue: "SimpleQueue[Optional[otlp.Span]]" = SimpleQueue()
         # Putting `None` as the sentinel value for queue termination.
         weakref.finalize(self, self._queue.put, END_OF_QUEUE)
         self._lock = RLock()
         self._spans: Dict[SpanID, ReadableSpan] = {}
         self._parent_span_ids: Dict[SpanID, ParentSpanID] = {}
-        self._traces: Dict[TraceID, List[SpanID]] = defaultdict(list)
-        self._child_span_ids: DefaultDict[SpanID, List[ChildSpanID]] = defaultdict(list)
-        self._orphan_spans: DefaultDict[ParentSpanID, List[pb.Span]] = defaultdict(list)
+        self._traces: DefaultDict[TraceID, List[SpanID]] = defaultdict(list)
+        self._child_span_ids: DefaultDict[SpanID, Set[ChildSpanID]] = defaultdict(set)
+        self._orphan_spans: DefaultDict[ParentSpanID, List[otlp.Span]] = defaultdict(list)
         self._num_documents: DefaultDict[SpanID, int] = defaultdict(int)
         self._start_time_sorted_span_ids: SortedKeyList[SpanID] = SortedKeyList(
-            key=lambda span_id: self._spans[span_id].start_time.ToDatetime(timezone.utc),
+            key=lambda span_id: self._spans[span_id].start_time,
         )
         self._start_time_sorted_root_span_ids: SortedKeyList[SpanID] = SortedKeyList(
-            key=lambda span_id: self._spans[span_id].start_time.ToDatetime(timezone.utc),
+            key=lambda span_id: self._spans[span_id].start_time,
         )
         self._latency_sorted_root_span_ids: SortedKeyList[SpanID] = SortedKeyList(
             key=lambda span_id: self._spans[span_id][ComputedAttributes.LATENCY_MS.value],
@@ -136,15 +136,18 @@ class Traces:
         self._min_start_time: Optional[datetime] = None
         self._max_start_time: Optional[datetime] = None
         self._token_count_total: int = 0
+        self._last_updated_at: Optional[datetime] = None
         self._start_consumer()
 
-    def put(self, span: Optional[Union[Span, pb.Span]] = None) -> None:
-        self._queue.put(encode(span) if isinstance(span, Span) else span)
+    def put(self, span: Optional[otlp.Span] = None) -> None:
+        self._queue.put(span)
 
     def get_trace(self, trace_id: TraceID) -> Iterator[Span]:
         with self._lock:
             # make a copy because source data can mutate during iteration
-            span_ids = tuple(self._traces[trace_id])
+            if not (trace := self._traces.get(trace_id)):
+                return
+            span_ids = tuple(trace)
         for span_id in span_ids:
             if span := self[span_id]:
                 yield span
@@ -194,7 +197,7 @@ class Traces:
 
     def get_num_documents(self, span_id: SpanID) -> int:
         with self._lock:
-            return self._num_documents[span_id]
+            return self._num_documents.get(span_id) or 0
 
     def latency_rank_percent(self, latency_ms: float) -> Optional[float]:
         """
@@ -221,11 +224,17 @@ class Traces:
     def get_descendant_span_ids(self, span_id: SpanID) -> Iterator[SpanID]:
         with self._lock:
             # make a copy because source data can mutate during iteration
-            span_ids = tuple(self._child_span_ids[span_id])
+            if not (child_span_ids := self._child_span_ids.get(span_id)):
+                return
+            span_ids = tuple(child_span_ids)
         for child_span_id in span_ids:
             yield child_span_id
             yield from self.get_descendant_span_ids(child_span_id)
 
+    @property
+    def last_updated_at(self) -> Optional[datetime]:
+        return self._last_updated_at
+
     @property
     def span_count(self) -> int:
         """Total number of spans (excluding orphan spans if any)"""
@@ -259,24 +268,24 @@ class Traces:
             with self._lock:
                 self._process_span(item)
 
-    def _process_span(self, span: pb.Span) -> None:
-        span_id = SpanID(span.context.span_id)
+    def _process_span(self, span: otlp.Span) -> None:
+        new_span = ReadableSpan(span)
+        span_id = new_span.context.span_id
         existing_span = self._spans.get(span_id)
-        if existing_span and existing_span.HasField("end_time"):
+        if existing_span and existing_span.end_time:
             # Reject updates if span has ended.
             return
-        is_root_span = not span.HasField("parent_span_id")
+        is_root_span = not new_span.parent_id
         if not is_root_span:
-            parent_span_id = SpanID(span.parent_span_id.value)
+            parent_span_id = new_span.parent_id
             if parent_span_id not in self._spans:
                 # Span can't be processed before its parent.
                 self._orphan_spans[parent_span_id].append(span)
                 return
-            self._child_span_ids[parent_span_id].append(span_id)
+            self._child_span_ids[parent_span_id].add(span_id)
             self._parent_span_ids[span_id] = parent_span_id
-        new_span = ReadableSpan(span)
-        start_time = span.start_time.ToDatetime(timezone.utc)
-        end_time = span.end_time.ToDatetime(timezone.utc) if span.HasField("end_time") else None
+        start_time = new_span.start_time
+        end_time = new_span.end_time
         if end_time:
             new_span[ComputedAttributes.LATENCY_MS.value] = latency = (
                 end_time - start_time
@@ -287,7 +296,7 @@ class Traces:
         if is_root_span and end_time:
             self._latency_sorted_root_span_ids.add(span_id)
         if not existing_span:
-            trace_id = TraceID(span.context.trace_id)
+            trace_id = new_span.context.trace_id
             self._traces[trace_id].append(span_id)
             if is_root_span:
                 self._start_time_sorted_root_span_ids.add(span_id)
@@ -303,7 +312,7 @@ class Traces:
                 else max(self._max_start_time, start_time)
             )
         new_span[ComputedAttributes.ERROR_COUNT.value] = int(
-            span.status.code is pb.Span.Status.Code.ERROR
+            new_span.status_code is SpanStatusCode.ERROR
         )
         # Update cumulative values for span's ancestors.
         for attribute_name, cumulative_attribute_name in (
@@ -336,14 +345,16 @@ class Traces:
             self._token_count_total -= existing_span[LLM_TOKEN_COUNT_TOTAL] or 0
         self._token_count_total += new_span[LLM_TOKEN_COUNT_TOTAL] or 0
         # Update number of documents
-        num_documents_update = len(span.retrieval.documents)
+        num_documents_update = len(new_span.attributes.get(RETRIEVAL_DOCUMENTS) or ())
         if existing_span:
-            num_documents_update -= len(existing_span.retrieval.documents)
+            num_documents_update -= len(existing_span.attributes.get(RETRIEVAL_DOCUMENTS) or ())
         if num_documents_update:
             self._num_documents[span_id] += num_documents_update
         # Process previously orphaned spans, if any.
         for orphan_span in self._orphan_spans.pop(span_id, ()):
             self._process_span(orphan_span)
+        # Update last updated timestamp
+        self._last_updated_at = datetime.now(timezone.utc)
 
     def _add_value_to_span_ancestors(
         self,

{arize_phoenix-2.0.0 → arize_phoenix-2.2.0rc0}/src/phoenix/experimental/evals/__init__.py

@@ -1,4 +1,4 @@
-from .evaluators import LLMEvaluator
+from .evaluators import InvalidEvalCriteriaError, LLMEvaluator
 from .functions import llm_classify, llm_generate, run_relevance_eval
 from .models import BedrockModel, LiteLLMModel, OpenAIModel, VertexAIModel
 from .retrievals import compute_precisions_at_k
@@ -16,11 +16,13 @@ from .templates import (
     TOXICITY_PROMPT_RAILS_MAP,
     TOXICITY_PROMPT_TEMPLATE,
     ClassificationTemplate,
+    EvalCriteria,
     PromptTemplate,
 )
 from .utils import NOT_PARSABLE, download_benchmark_dataset
 
 __all__ = [
+    "EvalCriteria",
     "compute_precisions_at_k",
     "download_benchmark_dataset",
     "llm_classify",
@@ -46,4 +48,5 @@ __all__ = [
     "QA_PROMPT_TEMPLATE",
     "NOT_PARSABLE",
     "run_relevance_eval",
+    "InvalidEvalCriteriaError",
 ]

{arize_phoenix-2.0.0 → arize_phoenix-2.2.0rc0}/src/phoenix/experimental/evals/evaluators.py

@@ -1,15 +1,26 @@
 from typing import List, Mapping, Optional, Tuple
 
+from phoenix.exceptions import PhoenixException
 from phoenix.experimental.evals.models import set_verbosity
-from phoenix.experimental.evals.utils import parse_openai_function_call, snap_to_rail
+from phoenix.experimental.evals.templates.default_templates import (
+    EvalCriteria,
+)
+from phoenix.experimental.evals.utils import (
+    NOT_PARSABLE,
+    openai_function_call_kwargs,
+    parse_openai_function_call,
+    snap_to_rail,
+)
 from phoenix.utilities.logging import printif
 
-from .models import BaseEvalModel
+from .models import BaseEvalModel, OpenAIModel
 from .templates import ClassificationTemplate, PromptOptions, PromptTemplate
 
 Record = Mapping[str, str]
 
-NOT_PARSABLE = "NOT_PARSABLE"
+
+class InvalidEvalCriteriaError(PhoenixException):
+    pass
 
 
 class LLMEvaluator:
@@ -35,6 +46,7 @@ class LLMEvaluator:
         self,
         record: Record,
         provide_explanation: bool = False,
+        use_function_calling_if_available: bool = True,
         verbose: bool = False,
     ) -> Tuple[str, Optional[str]]:
         """
@@ -46,27 +58,53 @@ class LLMEvaluator:
             provide_explanation (bool, optional): Whether to provide an
             explanation.
 
+            use_function_calling_if_available (bool, optional): If True, use
+            function calling (if available) as a means to constrain the LLM
+            outputs. With function calling, the LLM is instructed to provide its
+            response as a structured JSON object, which is easier to parse.
+
+            use_function_calling_if_available (bool, optional): If True, use
+            function calling (if available) as a means to constrain the LLM
+            outputs. With function calling, the LLM is instructed to provide its
+            response as a structured JSON object, which is easier to parse.
+
             verbose (bool, optional): Whether to print verbose output.
 
         Returns:
             Tuple[str, Optional[str]]: The label and explanation (if provided).
         """
+        use_openai_function_call = (
+            use_function_calling_if_available
+            and isinstance(self._model, OpenAIModel)
+            and self._model.supports_function_calling
+        )
         prompt = self._template.format(
             record, options=PromptOptions(provide_explanation=provide_explanation)
         )
         with set_verbosity(self._model, verbose) as verbose_model:
-            unparsed_output = verbose_model(prompt)
+            unparsed_output = verbose_model(
+                prompt,
+                **(
+                    openai_function_call_kwargs(self._template.rails, provide_explanation)
+                    if use_openai_function_call
+                    else {}
+                ),
+            )
         label, explanation = _extract_label_and_explanation(
             unparsed_output=unparsed_output,
             template=self._template,
-            use_openai_function_call=False,
             provide_explanation=provide_explanation,
+            use_openai_function_call=use_openai_function_call,
             verbose=verbose,
         )
         return label, explanation
 
     async def aevaluate(
-        self, record: Record, provide_explanation: bool = False, verbose: bool = False
+        self,
+        record: Record,
+        provide_explanation: bool = False,
+        use_function_calling_if_available: bool = True,
+        verbose: bool = False,
     ) -> Tuple[str, Optional[str]]:
         """
         Evaluates a single record.
@@ -77,25 +115,64 @@ class LLMEvaluator:
             provide_explanation (bool, optional): Whether to provide an
             explanation.
 
+            use_function_calling_if_available (bool, optional): If True, use
+            function calling (if available) as a means to constrain the LLM
+            outputs. With function calling, the LLM is instructed to provide its
+            response as a structured JSON object, which is easier to parse.
+
             verbose (bool, optional): Whether to print verbose output.
 
         Returns:
             Tuple[str, Optional[str]]: The label and explanation (if provided).
         """
+        use_openai_function_call = (
+            use_function_calling_if_available
+            and isinstance(self._model, OpenAIModel)
+            and self._model.supports_function_calling
+        )
         prompt = self._template.format(
             record, options=PromptOptions(provide_explanation=provide_explanation)
         )
         with set_verbosity(self._model, verbose) as verbose_model:
-            unparsed_output = await verbose_model._async_generate(prompt)
+            unparsed_output = await verbose_model._async_generate(
+                prompt,
+                **(
+                    openai_function_call_kwargs(self._template.rails, provide_explanation)
+                    if use_openai_function_call
+                    else {}
+                ),
+            )
         label, explanation = _extract_label_and_explanation(
             unparsed_output=unparsed_output,
             template=self._template,
-            use_openai_function_call=False,
             provide_explanation=provide_explanation,
+            use_openai_function_call=use_openai_function_call,
             verbose=verbose,
         )
         return label, explanation
 
+    @classmethod
+    def from_criteria(
+        cls,
+        criteria: EvalCriteria,
+        model: BaseEvalModel,
+    ) -> "LLMEvaluator":
+        """
+        Instantiates an LLMEvaluator from an eval criteria.
+
+        Args:
+            criteria (EvalCriteria): The eval criteria.
+
+            model (BaseEvalModel): The model to use for evaluation.
+
+        Returns:
+            LLMEvaluator: The instantiate evaluator.
+        """
+        return cls(
+            model=model,
+            template=criteria.value,
+        )
+
 
 class MapReducer:
     """