arize-phoenix 2.1.0__tar.gz → 2.2.0rc0__tar.gz

{arize_phoenix-2.1.0 → arize_phoenix-2.2.0rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arize-phoenix
-Version: 2.1.0
+Version: 2.2.0rc0
 Summary: ML Observability in your notebook
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues
@@ -38,8 +38,10 @@ Requires-Dist: umap-learn
 Requires-Dist: uvicorn
 Requires-Dist: wrapt
 Provides-Extra: dev
+Requires-Dist: anthropic; extra == 'dev'
 Requires-Dist: arize[autoembeddings,llm-evaluation]; extra == 'dev'
 Requires-Dist: gcsfs; extra == 'dev'
+Requires-Dist: google-cloud-aiplatform>=1.3; extra == 'dev'
 Requires-Dist: hatch; extra == 'dev'
 Requires-Dist: jupyter; extra == 'dev'
 Requires-Dist: langchain>=0.0.334; extra == 'dev'

{arize_phoenix-2.1.0 → arize_phoenix-2.2.0rc0}/pyproject.toml

@@ -62,7 +62,9 @@ dev = [
   "arize[AutoEmbeddings, LLM_Evaluation]",
   "llama-index>=0.9.14",
   "langchain>=0.0.334",
-  "litellm>=1.0.3"
+  "litellm>=1.0.3",
+  "google-cloud-aiplatform>=1.3",
+  "anthropic",
 ]
 experimental = [
   "tenacity",
@@ -77,7 +79,7 @@ Issues = "https://github.com/Arize-ai/phoenix/issues"
 Source = "https://github.com/Arize-ai/phoenix"
 
 [tool.hatch.version]
-path = "src/phoenix/__init__.py"
+path = "src/phoenix/version.py"
 
 [build-system]
 requires = ["hatchling"]

{arize_phoenix-2.1.0 → arize_phoenix-2.2.0rc0}/src/phoenix/__init__.py

@@ -5,8 +5,7 @@ from .session.evaluation import log_evaluations
 from .session.session import NotebookEnvironment, Session, active_session, close_app, launch_app
 from .trace.fixtures import load_example_traces
 from .trace.trace_dataset import TraceDataset
-
-__version__ = "2.1.0"
+from .version import __version__
 
 # module level doc-string
 __doc__ = """
@@ -25,6 +24,7 @@ Here are just a few of the things that phoenix does well:
 """
 
 __all__ = [
+    "__version__",
     "Dataset",
     "EmbeddingColumnNames",
     "RetrievalEmbeddingColumnNames",

{arize_phoenix-2.1.0 → arize_phoenix-2.2.0rc0}/src/phoenix/experimental/evals/functions/executor.py

@@ -275,6 +275,7 @@ class SyncExecutor(Executor):
                         result = self.generate(input)
                         outputs[index] = result
                         progress_bar.update()
+                        break
                     except Exception as exc:
                         is_phoenix_exception = isinstance(exc, PhoenixException)
                         if attempt >= self.max_retries or is_phoenix_exception:

arize_phoenix-2.2.0rc0/src/phoenix/experimental/evals/models/anthropic.py

@@ -0,0 +1,171 @@
+import logging
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from phoenix.experimental.evals.models.base import BaseEvalModel
+from phoenix.experimental.evals.models.rate_limiters import RateLimiter
+
+if TYPE_CHECKING:
+    from tiktoken import Encoding
+
+logger = logging.getLogger(__name__)
+
+MODEL_TOKEN_LIMIT_MAPPING = {
+    "claude-2.1": 200000,
+    "claude-2.0": 100000,
+    "claude-instant-1.2": 100000,
+}
+
+
+@dataclass
+class AnthropicModel(BaseEvalModel):
+    model: str = "claude-2.1"
+    """The model name to use."""
+    temperature: float = 0.0
+    """What sampling temperature to use."""
+    max_tokens: int = 256
+    """The maximum number of tokens to generate in the completion."""
+    top_p: float = 1
+    """Total probability mass of tokens to consider at each step."""
+    top_k: int = 256
+    """The cutoff where the model no longer selects the words"""
+    stop_sequences: List[str] = field(default_factory=list)
+    """If the model encounters a stop sequence, it stops generating further tokens. """
+    max_retries: int = 6
+    """Maximum number of retries to make when generating."""
+    retry_min_seconds: int = 10
+    """Minimum number of seconds to wait when retrying."""
+    retry_max_seconds: int = 60
+    """Maximum number of seconds to wait when retrying."""
+    extra_parameters: Dict[str, Any] = field(default_factory=dict)
+    """Any extra parameters to add to the request body (e.g., countPenalty for a21 models)"""
+    max_content_size: Optional[int] = None
+    """If you're using a fine-tuned model, set this to the maximum content size"""
+
+    def __post_init__(self) -> None:
+        self._init_environment()
+        self._init_client()
+        self._init_tiktoken()
+        self._init_rate_limiter()
+        self.retry = self._retry(
+            error_types=[],  # default to catching all errors
+            min_seconds=self.retry_min_seconds,
+            max_seconds=self.retry_max_seconds,
+            max_retries=self.max_retries,
+        )
+
+    def _init_environment(self) -> None:
+        try:
+            import tiktoken
+
+            self._tiktoken = tiktoken
+        except ImportError:
+            self._raise_import_error(
+                package_name="tiktoken",
+            )
+
+    def _init_client(self) -> None:
+        try:
+            import anthropic  # type:ignore
+
+            self._anthropic = anthropic
+            self.client = self._anthropic.Anthropic()
+            self.async_client = self._anthropic.AsyncAnthropic()
+        except ImportError:
+            self._raise_import_error(
+                package_name="anthropic",
+            )
+
+    def _init_tiktoken(self) -> None:
+        try:
+            encoding = self._tiktoken.encoding_for_model(self.model)
+        except KeyError:
+            logger.warning("Warning: model not found. Using cl100k_base encoding.")
+            encoding = self._tiktoken.get_encoding("cl100k_base")
+        self._tiktoken_encoding = encoding
+
+    def _init_rate_limiter(self) -> None:
+        self._rate_limiter = RateLimiter(
+            rate_limit_error=self._anthropic.RateLimitError,
+            max_rate_limit_retries=10,
+            initial_per_second_request_rate=1,
+            maximum_per_second_request_rate=20,
+            enforcement_window_minutes=1,
+        )
+
+    def invocation_parameters(self) -> Dict[str, Any]:
+        return {
+            "max_tokens_to_sample": self.max_tokens,
+            "stop_sequences": self.stop_sequences,
+            "temperature": self.temperature,
+            "top_p": self.top_p,
+            "top_k": self.top_k,
+        }
+
+    @property
+    def encoder(self) -> "Encoding":
+        return self._tiktoken_encoding
+
+    def get_tokens_from_text(self, text: str) -> List[int]:
+        return self.encoder.encode(text)
+
+    def get_text_from_tokens(self, tokens: List[int]) -> str:
+        return self.encoder.decode(tokens)
+
+    @property
+    def max_context_size(self) -> int:
+        context_size = self.max_content_size or MODEL_TOKEN_LIMIT_MAPPING.get(self.model, None)
+
+        if context_size is None:
+            raise ValueError(
+                "Can't determine maximum context size. An unknown model name was "
+                + f"used: {self.model}. Please set the `max_content_size` argument"
+                + "when using fine-tuned models. "
+            )
+
+        return context_size
+
+    def _generate(self, prompt: str, **kwargs: Dict[str, Any]) -> str:
+        # instruction is an invalid input to Anthropic models, it is passed in by
+        # BaseEvalModel.__call__ and needs to be removed
+        kwargs.pop("instruction", None)
+        invocation_parameters = self.invocation_parameters()
+        invocation_parameters.update(kwargs)
+        response = self._generate_with_retry(
+            model=self.model,
+            prompt=self._format_prompt_for_claude(prompt),
+            **invocation_parameters,
+        )
+
+        return str(response)
+
+    def _generate_with_retry(self, **kwargs: Any) -> Any:
+        @self.retry
+        @self._rate_limiter.limit
+        def _completion_with_retry(**kwargs: Any) -> Any:
+            response = self.client.completions.create(**kwargs)
+            return response.completion
+
+        return _completion_with_retry(**kwargs)
+
+    async def _async_generate(self, prompt: str, **kwargs: Dict[str, Any]) -> str:
+        invocation_parameters = self.invocation_parameters()
+        invocation_parameters.update(kwargs)
+        response = await self._async_generate_with_retry(
+            model=self.model, prompt=self._format_prompt_for_claude(prompt), **invocation_parameters
+        )
+
+        return str(response)
+
+    async def _async_generate_with_retry(self, **kwargs: Any) -> Any:
+        @self.retry
+        @self._rate_limiter.alimit
+        async def _async_completion_with_retry(**kwargs: Any) -> Any:
+            response = await self.async_client.completions.create(**kwargs)
+            return response.completion
+
+        return await _async_completion_with_retry(**kwargs)
+
+    def _format_prompt_for_claude(self, prompt: str) -> str:
+        # Claude requires prompt in the format of Human: ... Assistant:
+        return f"{self._anthropic.HUMAN_PROMPT} {prompt} {self._anthropic.AI_PROMPT}"

arize_phoenix-2.2.0rc0/src/phoenix/experimental/evals/models/vertex.py

@@ -0,0 +1,155 @@
+import logging
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Dict, List
+
+from phoenix.experimental.evals.models.base import BaseEvalModel
+from phoenix.experimental.evals.models.rate_limiters import RateLimiter
+
+if TYPE_CHECKING:
+    from tiktoken import Encoding
+
+logger = logging.getLogger(__name__)
+
+
+# https://cloud.google.com/vertex-ai/docs/generative-ai/learn/models
+MODEL_TOKEN_LIMIT_MAPPING = {
+    "gemini-pro": 32760,
+    "gemini-pro-vision": 16384,
+}
+
+
+@dataclass
+class GeminiModel(BaseEvalModel):
+    model: str = "gemini-pro"
+    """The model name to use."""
+    temperature: float = 0.0
+    """What sampling temperature to use."""
+    max_tokens: int = 256
+    """The maximum number of tokens to generate in the completion."""
+    top_p: float = 1
+    """Total probability mass of tokens to consider at each step."""
+    top_k: int = 32
+    """The cutoff where the model no longer selects the words"""
+    stop_sequences: List[str] = field(default_factory=list)
+    """If the model encounters a stop sequence, it stops generating further tokens. """
+    max_retries: int = 6
+    """Maximum number of retries to make when generating."""
+    retry_min_seconds: int = 10
+    """Minimum number of seconds to wait when retrying."""
+    retry_max_seconds: int = 60
+    """Maximum number of seconds to wait when retrying."""
+
+    def __post_init__(self) -> None:
+        self._init_client()
+        self._init_rate_limiter()
+        self.retry = self._retry(
+            error_types=[],  # default to catching all errors
+            min_seconds=self.retry_min_seconds,
+            max_seconds=self.retry_max_seconds,
+            max_retries=self.max_retries,
+        )
+
+    def _init_client(self) -> None:
+        try:
+            from google.api_core import exceptions  # type:ignore
+            from vertexai.preview import generative_models as vertex  # type:ignore
+
+            self._vertex = vertex
+            self._gcp_exceptions = exceptions
+            self._model = self._vertex.GenerativeModel(self.model)
+        except ImportError:
+            self._raise_import_error(
+                package_name="vertexai",
+            )
+
+    def _init_rate_limiter(self) -> None:
+        self._rate_limiter = RateLimiter(
+            rate_limit_error=self._gcp_exceptions.ResourceExhausted,
+            max_rate_limit_retries=10,
+            initial_per_second_request_rate=1,
+            maximum_per_second_request_rate=20,
+            enforcement_window_minutes=1,
+        )
+
+    @property
+    def encoder(self) -> "Encoding":
+        raise TypeError("Gemini models contain their own token counting")
+
+    def get_tokens_from_text(self, text: str) -> List[int]:
+        raise NotImplementedError
+
+    def get_text_from_tokens(self, tokens: List[int]) -> str:
+        raise NotImplementedError
+
+    @property
+    def max_context_size(self) -> int:
+        context_size = MODEL_TOKEN_LIMIT_MAPPING.get(self.model, None)
+
+        if context_size is None:
+            raise ValueError(
+                "Can't determine maximum context size. An unknown model name was "
+                + f"used: {self.model}. Please set the `max_content_size` argument"
+                + "when using fine-tuned models. "
+            )
+
+        return context_size
+
+    @property
+    def generation_config(self) -> Dict[str, Any]:
+        return {
+            "temperature": self.temperature,
+            "max_output_tokens": self.max_tokens,
+            "top_p": self.top_p,
+            "top_k": self.top_k,
+            "stop_sequences": self.stop_sequences,
+        }
+
+    def _generate(self, prompt: str, **kwargs: Dict[str, Any]) -> str:
+        # instruction is an invalid input to Gemini models, it is passed in by
+        # BaseEvalModel.__call__ and needs to be removed
+        kwargs.pop("instruction", None)
+        response = self._generate_with_retry(
+            prompt=prompt,
+            generation_config=self.generation_config,
+            **kwargs,
+        )
+
+        return str(response)
+
+    def _generate_with_retry(
+        self, prompt: str, generation_config: Dict[str, Any], **kwargs: Any
+    ) -> Any:
+        @self.retry
+        @self._rate_limiter.limit
+        def _completion_with_retry(**kwargs: Any) -> Any:
+            response = self._model.generate_content(
+                contents=prompt, generation_config=generation_config, **kwargs
+            )
+            candidate = response.candidates[0]
+            return candidate.text
+
+        return _completion_with_retry(**kwargs)
+
+    async def _async_generate(self, prompt: str, **kwargs: Dict[str, Any]) -> str:
+        kwargs.pop("instruction", None)
+        response = await self._async_generate_with_retry(
+            prompt=prompt,
+            generation_config=self.generation_config,
+            **kwargs,
+        )
+
+        return str(response)
+
+    async def _async_generate_with_retry(
+        self, prompt: str, generation_config: Dict[str, Any], **kwargs: Any
+    ) -> Any:
+        @self.retry
+        @self._rate_limiter.limit
+        async def _completion_with_retry(**kwargs: Any) -> Any:
+            response = await self._model.generate_content_async(
+                contents=prompt, generation_config=generation_config, **kwargs
+            )
+            candidate = response.candidates[0]
+            return candidate.text
+
+        return await _completion_with_retry(**kwargs)

{arize_phoenix-2.1.0 → arize_phoenix-2.2.0rc0}/src/phoenix/trace/otel.py

@@ -396,11 +396,11 @@ def _encode_value(value: AttributeValue) -> AnyValue:
         return AnyValue(int_value=value)
     if isinstance(value, float):
         return AnyValue(double_value=value)
-    if isinstance(value, Sequence):
-        return AnyValue(array_value=ArrayValue(values=map(_encode_value, value)))
     if isinstance(value, bytes):
         return AnyValue(bytes_value=value)
-    assert_never(value)
+    if isinstance(value, Sequence):
+        return AnyValue(array_value=ArrayValue(values=map(_encode_value, value)))
+    raise ValueError(f"Unexpected attribute value {value} with type {type(value)}.")
 
 
 __all__ = [

arize_phoenix-2.2.0rc0/src/phoenix/version.py

@@ -0,0 +1 @@
+__version__ = "2.2.0rc0"