arize-phoenix 2.1.0__py3-none-any.whl → 2.2.0rc0__py3-none-any.whl

{arize_phoenix-2.1.0.dist-info → arize_phoenix-2.2.0rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arize-phoenix
-Version: 2.1.0
+Version: 2.2.0rc0
 Summary: ML Observability in your notebook
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues
@@ -38,8 +38,10 @@ Requires-Dist: umap-learn
 Requires-Dist: uvicorn
 Requires-Dist: wrapt
 Provides-Extra: dev
+Requires-Dist: anthropic; extra == 'dev'
 Requires-Dist: arize[autoembeddings,llm-evaluation]; extra == 'dev'
 Requires-Dist: gcsfs; extra == 'dev'
+Requires-Dist: google-cloud-aiplatform>=1.3; extra == 'dev'
 Requires-Dist: hatch; extra == 'dev'
 Requires-Dist: jupyter; extra == 'dev'
 Requires-Dist: langchain>=0.0.334; extra == 'dev'

{arize_phoenix-2.1.0.dist-info → arize_phoenix-2.2.0rc0.dist-info}/RECORD

@@ -1,9 +1,10 @@
-phoenix/__init__.py,sha256=VtlQlins6lDwQzHwmklebwg43hXwQA8wl9Qcu9iQ67E,1373
+phoenix/__init__.py,sha256=EEh0vZGRQS8686h34GQ64OjQoZ7neKYO_iO5j6Oa9Jw,1402
 phoenix/config.py,sha256=ErvGg22SSiuqPJtIX1WZE5KcM2lt6XOGZ__HwRg3JqA,2390
 phoenix/datetime_utils.py,sha256=D955QLrkgrrSdUM6NyqbCeAu2SMsjhR5rHVQEsVUdng,2773
 phoenix/exceptions.py,sha256=igIWGAg3m8jm5YwQDeCY1p8ml_60A7zaGVXJ1yZhY9s,44
 phoenix/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 phoenix/services.py,sha256=slL4Uu___QQSKEssgD738-WAld-kzVQnpW92uKLxV4E,4886
+phoenix/version.py,sha256=pG4VqwySwU54SQ_mHFbajVD0oK3-38mb_fLPwWYLdoc,25
 phoenix/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/core/embedding_dimension.py,sha256=zKGbcvwOXgLf-yrJBpQyKtd-LEOPRKHnUToyAU8Owis,87
 phoenix/core/evals.py,sha256=OrHeYlh804rpcZIXTA6kan2mzSZMfgpphNNQdPMpNoM,7597
@@ -23,15 +24,17 @@ phoenix/experimental/evals/evaluators.py,sha256=rLvvXBK2H_cjJyRMBQStTlMYntTJI3Rt
 phoenix/experimental/evals/retrievals.py,sha256=o3fqrsYbYZjyGj_jWkN_9VQVyXjLkDKDw5Ws7l8bwdI,3828
 phoenix/experimental/evals/functions/__init__.py,sha256=3FMGrjmgxegXAwgDV_RpaN-73cFVyBiO8YwZvml5P9c,156
 phoenix/experimental/evals/functions/classify.py,sha256=uCTZR_ctQorzS0Abcwxzsza0g-4q_91DHiObjJISIXE,18177
-phoenix/experimental/evals/functions/executor.py,sha256=TSw2lVhkl6-VBYcXSlUl3E0U4OuJWnhwhed7NmFgbF8,13376
+phoenix/experimental/evals/functions/executor.py,sha256=bM7PI2rcPukQQzZ2rWqN_-Kfo_a935YJj0bh1Red8Ps,13406
 phoenix/experimental/evals/functions/generate.py,sha256=sdr6TeXn5JLEKM0NqYtvq01Lq48Q7uatb0fsq5zQgVY,5310
 phoenix/experimental/evals/functions/processing.py,sha256=F4xtLsulLV4a8CkuLldRddsCim75dSTIShEJUYN6I6w,1823
 phoenix/experimental/evals/models/__init__.py,sha256=j1N7DhiOPbcaemtVBONcQ0miNnGQwEXz4u3P3Vwe6-4,320
+phoenix/experimental/evals/models/anthropic.py,sha256=Tcv8R-vTyY8sLAv1wIHeZdMCBtqhyayqMPJXRDc7blI,6267
 phoenix/experimental/evals/models/base.py,sha256=aSE3Al3MsLvzNKuN2e-z6O-RB5mgpisH4UQqwNQcqp0,7734
 phoenix/experimental/evals/models/bedrock.py,sha256=CRPmBuSLc_nRnKKWLHhGMxdWEISIKUJM1tzIlOQ_qWM,7927
 phoenix/experimental/evals/models/litellm.py,sha256=jrRlph22xWxMXMUabUWjIO2e-sHxQzlQwSM-SnAACFQ,4714
 phoenix/experimental/evals/models/openai.py,sha256=Kl2uES3HRcZGFqblfBQZ6D1BpDffuLZDAqVTjhrSXXQ,17101
 phoenix/experimental/evals/models/rate_limiters.py,sha256=5GVN0RQKt36Przg3-9jLgocRmyg-tbeO-cdbuLIx89w,10160
+phoenix/experimental/evals/models/vertex.py,sha256=nwTIjVn4gGFfoKfGqUGwPD9GLJaBM4HLXDnMNs9hSrw,5407
 phoenix/experimental/evals/models/vertexai.py,sha256=NfBpQq0l7XzP-wDEDsK27IRiQBzA1GXEdfwlAf8leX4,5609
 phoenix/experimental/evals/templates/__init__.py,sha256=GSJSoWJ4jwyoUANniidmWMUtXQhNQYbTJbfFqCvuYuo,1470
 phoenix/experimental/evals/templates/default_templates.py,sha256=_VVxuhPsY8fkasA9XMNNM_fvftltkIfVCfElSdFbsQY,21056
@@ -133,7 +136,7 @@ phoenix/trace/__init__.py,sha256=lnuxATMemAqjURYqOfIo_HyCo5oIWIVTy98XAsiS1d8,215
 phoenix/trace/evaluation_conventions.py,sha256=t8jydM3U0-T5YpiQKRJ3tWdWGlHtzKyttYdw-ddvPOk,1048
 phoenix/trace/exporter.py,sha256=z3xrGJhIRh7XMy4Q1FkR3KmFZym-GX0XxLTZ6eSnN0Q,4347
 phoenix/trace/fixtures.py,sha256=lFuhPe-di54BmCT-RxS95m3e2-z1YBOo8CxsqYSgDD8,6341
-phoenix/trace/otel.py,sha256=cYD93sW2otPcfwMWMaAn7QeWgwn2o4Y0fdo7IyEpkZY,13838
+phoenix/trace/otel.py,sha256=4cJ85O_y_S6C_kkVvYh8d1d1mzKpGk_eQKjg688PI0k,13899
 phoenix/trace/schemas.py,sha256=m1wVlYFT6qL3FovD3TtTYsEgN6OHvv52gNdJkoPCmuY,5400
 phoenix/trace/semantic_conventions.py,sha256=u6NG85ZhbreriZr8cqJaddldM_jUcew7JilszY7JUk8,4652
 phoenix/trace/span_evaluations.py,sha256=9RTJ8BFhXDJNtqErWRlMj65FG7wJiI41YTgB7vYLqcY,8429
@@ -162,8 +165,8 @@ phoenix/trace/v1/evaluation_pb2.pyi,sha256=cCbbx06gwQmaH14s3J1X25TtaARh-k1abbxQd
 phoenix/utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/utilities/error_handling.py,sha256=7b5rpGFj9EWZ8yrZK1IHvxB89suWk3lggDayUQcvZds,1946
 phoenix/utilities/logging.py,sha256=lDXd6EGaamBNcQxL4vP1au9-i_SXe0OraUDiJOcszSw,222
-arize_phoenix-2.1.0.dist-info/METADATA,sha256=GHS_5V3vwFzP_F--H0UJ9YNnvDcyH79gLiTJ_lG6CMk,26378
-arize_phoenix-2.1.0.dist-info/WHEEL,sha256=mRYSEL3Ih6g5a_CVMIcwiF__0Ae4_gLYh01YFNwiq1k,87
-arize_phoenix-2.1.0.dist-info/licenses/IP_NOTICE,sha256=JBqyyCYYxGDfzQ0TtsQgjts41IJoa-hiwDrBjCb9gHM,469
-arize_phoenix-2.1.0.dist-info/licenses/LICENSE,sha256=HFkW9REuMOkvKRACuwLPT0hRydHb3zNg-fdFt94td18,3794
-arize_phoenix-2.1.0.dist-info/RECORD,,
+arize_phoenix-2.2.0rc0.dist-info/METADATA,sha256=cV0tw8Sq1yms7yncqdyAkIoUOMVBIfDQkS_X9wlmgKY,26482
+arize_phoenix-2.2.0rc0.dist-info/WHEEL,sha256=mRYSEL3Ih6g5a_CVMIcwiF__0Ae4_gLYh01YFNwiq1k,87
+arize_phoenix-2.2.0rc0.dist-info/licenses/IP_NOTICE,sha256=JBqyyCYYxGDfzQ0TtsQgjts41IJoa-hiwDrBjCb9gHM,469
+arize_phoenix-2.2.0rc0.dist-info/licenses/LICENSE,sha256=HFkW9REuMOkvKRACuwLPT0hRydHb3zNg-fdFt94td18,3794
+arize_phoenix-2.2.0rc0.dist-info/RECORD,,

phoenix/__init__.py

@@ -5,8 +5,7 @@ from .session.evaluation import log_evaluations
 from .session.session import NotebookEnvironment, Session, active_session, close_app, launch_app
 from .trace.fixtures import load_example_traces
 from .trace.trace_dataset import TraceDataset
-
-__version__ = "2.1.0"
+from .version import __version__
 
 # module level doc-string
 __doc__ = """
@@ -25,6 +24,7 @@ Here are just a few of the things that phoenix does well:
 """
 
 __all__ = [
+    "__version__",
     "Dataset",
     "EmbeddingColumnNames",
     "RetrievalEmbeddingColumnNames",

phoenix/experimental/evals/functions/executor.py

@@ -275,6 +275,7 @@ class SyncExecutor(Executor):
                         result = self.generate(input)
                         outputs[index] = result
                         progress_bar.update()
+                        break
                     except Exception as exc:
                         is_phoenix_exception = isinstance(exc, PhoenixException)
                         if attempt >= self.max_retries or is_phoenix_exception:

phoenix/experimental/evals/models/anthropic.py

@@ -0,0 +1,171 @@
+import logging
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from phoenix.experimental.evals.models.base import BaseEvalModel
+from phoenix.experimental.evals.models.rate_limiters import RateLimiter
+
+if TYPE_CHECKING:
+    from tiktoken import Encoding
+
+logger = logging.getLogger(__name__)
+
+MODEL_TOKEN_LIMIT_MAPPING = {
+    "claude-2.1": 200000,
+    "claude-2.0": 100000,
+    "claude-instant-1.2": 100000,
+}
+
+
+@dataclass
+class AnthropicModel(BaseEvalModel):
+    model: str = "claude-2.1"
+    """The model name to use."""
+    temperature: float = 0.0
+    """What sampling temperature to use."""
+    max_tokens: int = 256
+    """The maximum number of tokens to generate in the completion."""
+    top_p: float = 1
+    """Total probability mass of tokens to consider at each step."""
+    top_k: int = 256
+    """The cutoff where the model no longer selects the words"""
+    stop_sequences: List[str] = field(default_factory=list)
+    """If the model encounters a stop sequence, it stops generating further tokens. """
+    max_retries: int = 6
+    """Maximum number of retries to make when generating."""
+    retry_min_seconds: int = 10
+    """Minimum number of seconds to wait when retrying."""
+    retry_max_seconds: int = 60
+    """Maximum number of seconds to wait when retrying."""
+    extra_parameters: Dict[str, Any] = field(default_factory=dict)
+    """Any extra parameters to add to the request body (e.g., countPenalty for a21 models)"""
+    max_content_size: Optional[int] = None
+    """If you're using a fine-tuned model, set this to the maximum content size"""
+
+    def __post_init__(self) -> None:
+        self._init_environment()
+        self._init_client()
+        self._init_tiktoken()
+        self._init_rate_limiter()
+        self.retry = self._retry(
+            error_types=[],  # default to catching all errors
+            min_seconds=self.retry_min_seconds,
+            max_seconds=self.retry_max_seconds,
+            max_retries=self.max_retries,
+        )
+
+    def _init_environment(self) -> None:
+        try:
+            import tiktoken
+
+            self._tiktoken = tiktoken
+        except ImportError:
+            self._raise_import_error(
+                package_name="tiktoken",
+            )
+
+    def _init_client(self) -> None:
+        try:
+            import anthropic  # type:ignore
+
+            self._anthropic = anthropic
+            self.client = self._anthropic.Anthropic()
+            self.async_client = self._anthropic.AsyncAnthropic()
+        except ImportError:
+            self._raise_import_error(
+                package_name="anthropic",
+            )
+
+    def _init_tiktoken(self) -> None:
+        try:
+            encoding = self._tiktoken.encoding_for_model(self.model)
+        except KeyError:
+            logger.warning("Warning: model not found. Using cl100k_base encoding.")
+            encoding = self._tiktoken.get_encoding("cl100k_base")
+        self._tiktoken_encoding = encoding
+
+    def _init_rate_limiter(self) -> None:
+        self._rate_limiter = RateLimiter(
+            rate_limit_error=self._anthropic.RateLimitError,
+            max_rate_limit_retries=10,
+            initial_per_second_request_rate=1,
+            maximum_per_second_request_rate=20,
+            enforcement_window_minutes=1,
+        )
+
+    def invocation_parameters(self) -> Dict[str, Any]:
+        return {
+            "max_tokens_to_sample": self.max_tokens,
+            "stop_sequences": self.stop_sequences,
+            "temperature": self.temperature,
+            "top_p": self.top_p,
+            "top_k": self.top_k,
+        }
+
+    @property
+    def encoder(self) -> "Encoding":
+        return self._tiktoken_encoding
+
+    def get_tokens_from_text(self, text: str) -> List[int]:
+        return self.encoder.encode(text)
+
+    def get_text_from_tokens(self, tokens: List[int]) -> str:
+        return self.encoder.decode(tokens)
+
+    @property
+    def max_context_size(self) -> int:
+        context_size = self.max_content_size or MODEL_TOKEN_LIMIT_MAPPING.get(self.model, None)
+
+        if context_size is None:
+            raise ValueError(
+                "Can't determine maximum context size. An unknown model name was "
+                + f"used: {self.model}. Please set the `max_content_size` argument"
+                + "when using fine-tuned models. "
+            )
+
+        return context_size
+
+    def _generate(self, prompt: str, **kwargs: Dict[str, Any]) -> str:
+        # instruction is an invalid input to Anthropic models, it is passed in by
+        # BaseEvalModel.__call__ and needs to be removed
+        kwargs.pop("instruction", None)
+        invocation_parameters = self.invocation_parameters()
+        invocation_parameters.update(kwargs)
+        response = self._generate_with_retry(
+            model=self.model,
+            prompt=self._format_prompt_for_claude(prompt),
+            **invocation_parameters,
+        )
+
+        return str(response)
+
+    def _generate_with_retry(self, **kwargs: Any) -> Any:
+        @self.retry
+        @self._rate_limiter.limit
+        def _completion_with_retry(**kwargs: Any) -> Any:
+            response = self.client.completions.create(**kwargs)
+            return response.completion
+
+        return _completion_with_retry(**kwargs)
+
+    async def _async_generate(self, prompt: str, **kwargs: Dict[str, Any]) -> str:
+        invocation_parameters = self.invocation_parameters()
+        invocation_parameters.update(kwargs)
+        response = await self._async_generate_with_retry(
+            model=self.model, prompt=self._format_prompt_for_claude(prompt), **invocation_parameters
+        )
+
+        return str(response)
+
+    async def _async_generate_with_retry(self, **kwargs: Any) -> Any:
+        @self.retry
+        @self._rate_limiter.alimit
+        async def _async_completion_with_retry(**kwargs: Any) -> Any:
+            response = await self.async_client.completions.create(**kwargs)
+            return response.completion
+
+        return await _async_completion_with_retry(**kwargs)
+
+    def _format_prompt_for_claude(self, prompt: str) -> str:
+        # Claude requires prompt in the format of Human: ... Assistant:
+        return f"{self._anthropic.HUMAN_PROMPT} {prompt} {self._anthropic.AI_PROMPT}"

phoenix/experimental/evals/models/vertex.py

@@ -0,0 +1,155 @@
+import logging
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Dict, List
+
+from phoenix.experimental.evals.models.base import BaseEvalModel
+from phoenix.experimental.evals.models.rate_limiters import RateLimiter
+
+if TYPE_CHECKING:
+    from tiktoken import Encoding
+
+logger = logging.getLogger(__name__)
+
+
+# https://cloud.google.com/vertex-ai/docs/generative-ai/learn/models
+MODEL_TOKEN_LIMIT_MAPPING = {
+    "gemini-pro": 32760,
+    "gemini-pro-vision": 16384,
+}
+
+
+@dataclass
+class GeminiModel(BaseEvalModel):
+    model: str = "gemini-pro"
+    """The model name to use."""
+    temperature: float = 0.0
+    """What sampling temperature to use."""
+    max_tokens: int = 256
+    """The maximum number of tokens to generate in the completion."""
+    top_p: float = 1
+    """Total probability mass of tokens to consider at each step."""
+    top_k: int = 32
+    """The cutoff where the model no longer selects the words"""
+    stop_sequences: List[str] = field(default_factory=list)
+    """If the model encounters a stop sequence, it stops generating further tokens. """
+    max_retries: int = 6
+    """Maximum number of retries to make when generating."""
+    retry_min_seconds: int = 10
+    """Minimum number of seconds to wait when retrying."""
+    retry_max_seconds: int = 60
+    """Maximum number of seconds to wait when retrying."""
+
+    def __post_init__(self) -> None:
+        self._init_client()
+        self._init_rate_limiter()
+        self.retry = self._retry(
+            error_types=[],  # default to catching all errors
+            min_seconds=self.retry_min_seconds,
+            max_seconds=self.retry_max_seconds,
+            max_retries=self.max_retries,
+        )
+
+    def _init_client(self) -> None:
+        try:
+            from google.api_core import exceptions  # type:ignore
+            from vertexai.preview import generative_models as vertex  # type:ignore
+
+            self._vertex = vertex
+            self._gcp_exceptions = exceptions
+            self._model = self._vertex.GenerativeModel(self.model)
+        except ImportError:
+            self._raise_import_error(
+                package_name="vertexai",
+            )
+
+    def _init_rate_limiter(self) -> None:
+        self._rate_limiter = RateLimiter(
+            rate_limit_error=self._gcp_exceptions.ResourceExhausted,
+            max_rate_limit_retries=10,
+            initial_per_second_request_rate=1,
+            maximum_per_second_request_rate=20,
+            enforcement_window_minutes=1,
+        )
+
+    @property
+    def encoder(self) -> "Encoding":
+        raise TypeError("Gemini models contain their own token counting")
+
+    def get_tokens_from_text(self, text: str) -> List[int]:
+        raise NotImplementedError
+
+    def get_text_from_tokens(self, tokens: List[int]) -> str:
+        raise NotImplementedError
+
+    @property
+    def max_context_size(self) -> int:
+        context_size = MODEL_TOKEN_LIMIT_MAPPING.get(self.model, None)
+
+        if context_size is None:
+            raise ValueError(
+                "Can't determine maximum context size. An unknown model name was "
+                + f"used: {self.model}. Please set the `max_content_size` argument"
+                + "when using fine-tuned models. "
+            )
+
+        return context_size
+
+    @property
+    def generation_config(self) -> Dict[str, Any]:
+        return {
+            "temperature": self.temperature,
+            "max_output_tokens": self.max_tokens,
+            "top_p": self.top_p,
+            "top_k": self.top_k,
+            "stop_sequences": self.stop_sequences,
+        }
+
+    def _generate(self, prompt: str, **kwargs: Dict[str, Any]) -> str:
+        # instruction is an invalid input to Gemini models, it is passed in by
+        # BaseEvalModel.__call__ and needs to be removed
+        kwargs.pop("instruction", None)
+        response = self._generate_with_retry(
+            prompt=prompt,
+            generation_config=self.generation_config,
+            **kwargs,
+        )
+
+        return str(response)
+
+    def _generate_with_retry(
+        self, prompt: str, generation_config: Dict[str, Any], **kwargs: Any
+    ) -> Any:
+        @self.retry
+        @self._rate_limiter.limit
+        def _completion_with_retry(**kwargs: Any) -> Any:
+            response = self._model.generate_content(
+                contents=prompt, generation_config=generation_config, **kwargs
+            )
+            candidate = response.candidates[0]
+            return candidate.text
+
+        return _completion_with_retry(**kwargs)
+
+    async def _async_generate(self, prompt: str, **kwargs: Dict[str, Any]) -> str:
+        kwargs.pop("instruction", None)
+        response = await self._async_generate_with_retry(
+            prompt=prompt,
+            generation_config=self.generation_config,
+            **kwargs,
+        )
+
+        return str(response)
+
+    async def _async_generate_with_retry(
+        self, prompt: str, generation_config: Dict[str, Any], **kwargs: Any
+    ) -> Any:
+        @self.retry
+        @self._rate_limiter.limit
+        async def _completion_with_retry(**kwargs: Any) -> Any:
+            response = await self._model.generate_content_async(
+                contents=prompt, generation_config=generation_config, **kwargs
+            )
+            candidate = response.candidates[0]
+            return candidate.text
+
+        return await _completion_with_retry(**kwargs)

phoenix/trace/otel.py

@@ -396,11 +396,11 @@ def _encode_value(value: AttributeValue) -> AnyValue:
         return AnyValue(int_value=value)
     if isinstance(value, float):
         return AnyValue(double_value=value)
-    if isinstance(value, Sequence):
-        return AnyValue(array_value=ArrayValue(values=map(_encode_value, value)))
     if isinstance(value, bytes):
         return AnyValue(bytes_value=value)
-    assert_never(value)
+    if isinstance(value, Sequence):
+        return AnyValue(array_value=ArrayValue(values=map(_encode_value, value)))
+    raise ValueError(f"Unexpected attribute value {value} with type {type(value)}.")
 
 
 __all__ = [

phoenix/version.py

@@ -0,0 +1 @@
+__version__ = "2.2.0rc0"