arize-phoenix 3.16.1__py3-none-any.whl → 7.7.0__py3-none-any.whl

phoenix/experimental/evals/models/rate_limiters.py

@@ -1,246 +0,0 @@
-import asyncio
-import time
-from functools import wraps
-from math import exp
-from typing import Any, Callable, Coroutine, Optional, Tuple, Type, TypeVar
-
-from typing_extensions import ParamSpec
-
-from phoenix.exceptions import PhoenixException
-from phoenix.utilities.logging import printif
-
-ParameterSpec = ParamSpec("ParameterSpec")
-GenericType = TypeVar("GenericType")
-AsyncCallable = Callable[ParameterSpec, Coroutine[Any, Any, GenericType]]
-
-
-class UnavailableTokensError(PhoenixException):
-    pass
-
-
-class AdaptiveTokenBucket:
-    """
-    An adaptive rate-limiter that adjusts the rate based on the number of rate limit errors.
-
-    This rate limiter does not need to know the exact rate limit. Instead, it starts with a high
-    rate and reduces it whenever a rate limit error occurs. The rate is increased slowly over time
-    if no further errors occur.
-
-    Args:
-    initial_per_second_request_rate (float): The allowed request rate.
-    maximum_per_second_request_rate (float): The maximum allowed request rate.
-    enforcement_window_minutes (float): The time window over which the rate limit is enforced.
-    rate_reduction_factor (float): Multiplier used to reduce the rate limit after an error.
-    rate_increase_factor (float): Exponential factor increasing the rate limit over time.
-    cooldown_seconds (float): The minimum time before allowing the rate limit to decrease again.
-    """
-
-    def __init__(
-        self,
-        initial_per_second_request_rate: float,
-        maximum_per_second_request_rate: float = 1000,
-        minimum_per_second_request_rate: float = 0.1,
-        enforcement_window_minutes: float = 1,
-        rate_reduction_factor: float = 0.5,
-        rate_increase_factor: float = 0.01,
-        cooldown_seconds: float = 5,
-    ):
-        now = time.time()
-        self._initial_rate = initial_per_second_request_rate
-        self.rate = initial_per_second_request_rate
-        self.maximum_rate = maximum_per_second_request_rate
-        self.minimum_rate = minimum_per_second_request_rate
-        self.rate_reduction_factor = rate_reduction_factor
-        self.enforcement_window = enforcement_window_minutes * 60
-        self.rate_increase_factor = rate_increase_factor
-        self.cooldown = cooldown_seconds
-        self.last_rate_update = now
-        self.last_checked = now
-        self.last_error = now - self.cooldown
-        self.tokens = 0.0
-
-    def increase_rate(self) -> None:
-        time_since_last_update = time.time() - self.last_rate_update
-        if time_since_last_update > self.enforcement_window:
-            self.rate = self._initial_rate
-        else:
-            self.rate *= exp(self.rate_increase_factor * time_since_last_update)
-            self.rate = min(self.rate, self.maximum_rate)
-        self.last_rate_update = time.time()
-
-    def on_rate_limit_error(self, request_start_time: float, verbose: bool = False) -> None:
-        now = time.time()
-        if request_start_time < (self.last_error + self.cooldown):
-            # do not reduce the rate for concurrent requests
-            return
-
-        original_rate = self.rate
-
-        self.rate = original_rate * self.rate_reduction_factor
-        printif(
-            verbose, f"Reducing rate from {original_rate} to {self.rate} after rate limit error"
-        )
-
-        self.rate = max(self.rate, self.minimum_rate)
-
-        # reset request tokens on a rate limit error
-        self.tokens = 0
-        self.last_checked = now
-        self.last_rate_update = now
-        self.last_error = now
-        time.sleep(self.cooldown)  # block for a bit to let the rate limit reset
-
-    def max_tokens(self) -> float:
-        return self.rate * self.enforcement_window
-
-    def available_requests(self) -> float:
-        now = time.time()
-        time_since_last_checked = time.time() - self.last_checked
-        self.tokens = min(self.max_tokens(), self.rate * time_since_last_checked + self.tokens)
-        self.last_checked = now
-        return self.tokens
-
-    def make_request_if_ready(self) -> None:
-        if self.available_requests() <= 1:
-            raise UnavailableTokensError
-        self.tokens -= 1
-
-    def wait_until_ready(
-        self,
-        max_wait_time: float = 300,
-    ) -> None:
-        start = time.time()
-        while (time.time() - start) < max_wait_time:
-            try:
-                self.increase_rate()
-                self.make_request_if_ready()
-                break
-            except UnavailableTokensError:
-                time.sleep(0.1 / self.rate)
-                continue
-
-    async def async_wait_until_ready(
-        self,
-        max_wait_time: float = 10,  # defeat the token bucket rate limiter at low rates (<.1 req/s)
-    ) -> None:
-        start = time.time()
-        while (time.time() - start) < max_wait_time:
-            try:
-                self.increase_rate()
-                self.make_request_if_ready()
-                break
-            except UnavailableTokensError:
-                await asyncio.sleep(0.1 / self.rate)
-                continue
-
-
-class RateLimitError(PhoenixException): ...
-
-
-class RateLimiter:
-    def __init__(
-        self,
-        rate_limit_error: Optional[Type[BaseException]] = None,
-        max_rate_limit_retries: int = 3,
-        initial_per_second_request_rate: float = 1,
-        maximum_per_second_request_rate: float = 50,
-        enforcement_window_minutes: float = 1,
-        rate_reduction_factor: float = 0.5,
-        rate_increase_factor: float = 0.01,
-        cooldown_seconds: float = 5,
-        verbose: bool = False,
-    ) -> None:
-        self._rate_limit_error: Tuple[Type[BaseException], ...]
-        self._rate_limit_error = (rate_limit_error,) if rate_limit_error is not None else tuple()
-
-        self._max_rate_limit_retries = max_rate_limit_retries
-        self._throttler = AdaptiveTokenBucket(
-            initial_per_second_request_rate=initial_per_second_request_rate,
-            maximum_per_second_request_rate=maximum_per_second_request_rate,
-            enforcement_window_minutes=enforcement_window_minutes,
-            rate_reduction_factor=rate_reduction_factor,
-            rate_increase_factor=rate_increase_factor,
-            cooldown_seconds=cooldown_seconds,
-        )
-        self._rate_limit_handling: Optional[asyncio.Event] = None
-        self._rate_limit_handling_lock: Optional[asyncio.Lock] = None
-        self._current_loop: Optional[asyncio.AbstractEventLoop] = None
-        self._verbose = verbose
-
-    def limit(
-        self, fn: Callable[ParameterSpec, GenericType]
-    ) -> Callable[ParameterSpec, GenericType]:
-        @wraps(fn)
-        def wrapper(*args: Any, **kwargs: Any) -> GenericType:
-            try:
-                self._throttler.wait_until_ready()
-                request_start_time = time.time()
-                return fn(*args, **kwargs)
-            except self._rate_limit_error:
-                self._throttler.on_rate_limit_error(request_start_time, verbose=self._verbose)
-                for _attempt in range(self._max_rate_limit_retries):
-                    try:
-                        request_start_time = time.time()
-                        self._throttler.wait_until_ready()
-                        return fn(*args, **kwargs)
-                    except self._rate_limit_error:
-                        self._throttler.on_rate_limit_error(
-                            request_start_time, verbose=self._verbose
-                        )
-                        continue
-            raise RateLimitError(f"Exceeded max ({self._max_rate_limit_retries}) retries")
-
-        return wrapper
-
-    def _initialize_async_primitives(self) -> None:
-        """
-        Lazily initialize async primitives to ensure they are created in the correct event loop.
-        """
-
-        loop = asyncio.get_running_loop()
-        if loop is not self._current_loop:
-            self._current_loop = loop
-            self._rate_limit_handling = asyncio.Event()
-            self._rate_limit_handling.set()
-            self._rate_limit_handling_lock = asyncio.Lock()
-
-    def alimit(
-        self, fn: AsyncCallable[ParameterSpec, GenericType]
-    ) -> AsyncCallable[ParameterSpec, GenericType]:
-        @wraps(fn)
-        async def wrapper(*args: Any, **kwargs: Any) -> GenericType:
-            self._initialize_async_primitives()
-            assert self._rate_limit_handling_lock is not None and isinstance(
-                self._rate_limit_handling_lock, asyncio.Lock
-            )
-            assert self._rate_limit_handling is not None and isinstance(
-                self._rate_limit_handling, asyncio.Event
-            )
-            try:
-                try:
-                    await asyncio.wait_for(self._rate_limit_handling.wait(), 120)
-                except asyncio.TimeoutError:
-                    self._rate_limit_handling.set()  # Set the event as a failsafe
-                await self._throttler.async_wait_until_ready()
-                request_start_time = time.time()
-                return await fn(*args, **kwargs)
-            except self._rate_limit_error:
-                async with self._rate_limit_handling_lock:
-                    self._rate_limit_handling.clear()  # prevent new requests from starting
-                    self._throttler.on_rate_limit_error(request_start_time, verbose=self._verbose)
-                    try:
-                        for _attempt in range(self._max_rate_limit_retries):
-                            try:
-                                request_start_time = time.time()
-                                await self._throttler.async_wait_until_ready()
-                                return await fn(*args, **kwargs)
-                            except self._rate_limit_error:
-                                self._throttler.on_rate_limit_error(
-                                    request_start_time, verbose=self._verbose
-                                )
-                                continue
-                    finally:
-                        self._rate_limit_handling.set()  # allow new requests to start
-            raise RateLimitError(f"Exceeded max ({self._max_rate_limit_retries}) retries")
-
-        return wrapper

phoenix/experimental/evals/models/vertex.py

@@ -1,173 +0,0 @@
-import logging
-from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, Dict, List
-
-from phoenix.experimental.evals.models.base import BaseEvalModel
-from phoenix.experimental.evals.models.rate_limiters import RateLimiter
-from phoenix.utilities.logging import printif
-
-if TYPE_CHECKING:
-    from tiktoken import Encoding
-
-logger = logging.getLogger(__name__)
-
-
-# https://cloud.google.com/vertex-ai/docs/generative-ai/learn/models
-MODEL_TOKEN_LIMIT_MAPPING = {
-    "gemini-pro": 32760,
-    "gemini-pro-vision": 16384,
-}
-
-
-@dataclass
-class GeminiModel(BaseEvalModel):
-    # The vertex SDK runs into connection pool limits at high concurrency
-    default_concurrency: int = 5
-
-    model: str = "gemini-pro"
-    """The model name to use."""
-    temperature: float = 0.0
-    """What sampling temperature to use."""
-    max_tokens: int = 256
-    """The maximum number of tokens to generate in the completion."""
-    top_p: float = 1
-    """Total probability mass of tokens to consider at each step."""
-    top_k: int = 32
-    """The cutoff where the model no longer selects the words"""
-    stop_sequences: List[str] = field(default_factory=list)
-    """If the model encounters a stop sequence, it stops generating further tokens. """
-    max_retries: int = 6
-    """Maximum number of retries to make when generating."""
-    retry_min_seconds: int = 10
-    """Minimum number of seconds to wait when retrying."""
-    retry_max_seconds: int = 60
-    """Maximum number of seconds to wait when retrying."""
-
-    def __post_init__(self) -> None:
-        self._init_client()
-        self._init_rate_limiter()
-
-    def reload_client(self) -> None:
-        self._init_client()
-
-    def _init_client(self) -> None:
-        try:
-            from google.api_core import exceptions  # type:ignore
-            from vertexai.preview import generative_models as vertex  # type:ignore
-
-            self._vertex = vertex
-            self._gcp_exceptions = exceptions
-            self._model = self._vertex.GenerativeModel(self.model)
-        except ImportError:
-            self._raise_import_error(
-                package_name="vertexai",
-            )
-
-    def _init_rate_limiter(self) -> None:
-        self._rate_limiter = RateLimiter(
-            rate_limit_error=self._gcp_exceptions.ResourceExhausted,
-            max_rate_limit_retries=10,
-            initial_per_second_request_rate=1,
-            maximum_per_second_request_rate=20,
-            enforcement_window_minutes=1,
-        )
-
-    @property
-    def encoder(self) -> "Encoding":
-        raise TypeError("Gemini models contain their own token counting")
-
-    def get_tokens_from_text(self, text: str) -> List[int]:
-        raise NotImplementedError
-
-    def get_text_from_tokens(self, tokens: List[int]) -> str:
-        raise NotImplementedError
-
-    @property
-    def max_context_size(self) -> int:
-        context_size = MODEL_TOKEN_LIMIT_MAPPING.get(self.model, None)
-
-        if context_size is None:
-            raise ValueError(
-                "Can't determine maximum context size. An unknown model name was "
-                + f"used: {self.model}. Please set the `max_content_size` argument"
-                + "when using fine-tuned models. "
-            )
-
-        return context_size
-
-    @property
-    def generation_config(self) -> Dict[str, Any]:
-        return {
-            "temperature": self.temperature,
-            "max_output_tokens": self.max_tokens,
-            "top_p": self.top_p,
-            "top_k": self.top_k,
-            "stop_sequences": self.stop_sequences,
-        }
-
-    def _generate(self, prompt: str, **kwargs: Dict[str, Any]) -> str:
-        # instruction is an invalid input to Gemini models, it is passed in by
-        # BaseEvalModel.__call__ and needs to be removed
-        kwargs.pop("instruction", None)
-
-        @self._rate_limiter.limit
-        def _rate_limited_completion(
-            prompt: str, generation_config: Dict[str, Any], **kwargs: Any
-        ) -> Any:
-            response = self._model.generate_content(
-                contents=prompt, generation_config=generation_config, **kwargs
-            )
-            return self._parse_response_candidates(response)
-
-        response = _rate_limited_completion(
-            prompt=prompt,
-            generation_config=self.generation_config,
-            **kwargs,
-        )
-
-        return str(response)
-
-    async def _async_generate(self, prompt: str, **kwargs: Dict[str, Any]) -> str:
-        # instruction is an invalid input to Gemini models, it is passed in by
-        # BaseEvalModel.__call__ and needs to be removed
-        kwargs.pop("instruction", None)
-
-        @self._rate_limiter.alimit
-        async def _rate_limited_completion(
-            prompt: str, generation_config: Dict[str, Any], **kwargs: Any
-        ) -> Any:
-            response = await self._model.generate_content_async(
-                contents=prompt, generation_config=generation_config, **kwargs
-            )
-            return self._parse_response_candidates(response)
-
-        response = await _rate_limited_completion(
-            prompt=prompt,
-            generation_config=self.generation_config,
-            **kwargs,
-        )
-
-        return str(response)
-
-    def _parse_response_candidates(self, response: Any) -> Any:
-        if hasattr(response, "candidates"):
-            if isinstance(response.candidates, list) and len(response.candidates) > 0:
-                try:
-                    candidate = response.candidates[0].text
-                except ValueError:
-                    printif(
-                        self._verbose, "The 'candidates' object does not have a 'text' attribute."
-                    )
-                    printif(self._verbose, str(response.candidates[0]))
-                    candidate = ""
-            else:
-                printif(
-                    self._verbose,
-                    "The 'candidates' attribute of 'response' is either not a list or is empty.",
-                )
-                printif(self._verbose, str(response))
-                candidate = ""
-        else:
-            printif(self._verbose, "The 'response' object does not have a 'candidates' attribute.")
-            candidate = ""
-        return candidate

phoenix/experimental/evals/models/vertexai.py

@@ -1,186 +0,0 @@
-import logging
-import warnings
-from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
-
-from phoenix.experimental.evals.models.base import BaseEvalModel
-
-if TYPE_CHECKING:
-    from google.auth.credentials import Credentials  # type:ignore
-
-logger = logging.getLogger(__name__)
-
-MINIMUM_VERTEX_AI_VERSION = "1.33.0"
-
-
-@dataclass
-class VertexAIModel(BaseEvalModel):
-    project: Optional[str] = None
-    "project (str): The default project to use when making API calls."
-    location: Optional[str] = None
-    "location (str): The default location to use when making API calls. If not "
-    "set defaults to us-central-1."
-    credentials: Optional["Credentials"] = None
-    model: str = "text-bison"
-    tuned_model: Optional[str] = None
-    "The name of a tuned model. If provided, model is ignored."
-    max_retries: int = 6
-    """Maximum number of retries to make when generating."""
-    retry_min_seconds: int = 10
-    """Minimum number of seconds to wait when retrying."""
-    retry_max_seconds: int = 60
-    """Maximum number of seconds to wait when retrying."""
-    temperature: float = 0.0
-    """What sampling temperature to use."""
-    max_tokens: int = 256
-    """The maximum number of tokens to generate in the completion.
-    -1 returns as many tokens as possible given the prompt and
-    the models maximal context size."""
-    top_p: float = 0.95
-    "Tokens are selected from most probable to least until the sum of their "
-    "probabilities equals the top-p value. Top-p is ignored for Codey models."
-    top_k: int = 40
-    "How the model selects tokens for output, the next token is selected from "
-    "among the top-k most probable tokens. Top-k is ignored for Codey models."
-
-    # Deprecated fields
-    model_name: Optional[str] = None
-    """
-    .. deprecated:: 3.0.0
-       use `model` instead. This will be removed in a future release.
-    """
-    tuned_model_name: Optional[str] = None
-    """
-    .. deprecated:: 3.0.0
-       use `tuned_model` instead. This will be removed in a future release.
-    """
-
-    def __post_init__(self) -> None:
-        self._migrate_model_name()
-        self._init_environment()
-        self._init_vertex_ai()
-        self._instantiate_model()
-
-    def _migrate_model_name(self) -> None:
-        if self.model_name is not None:
-            warning_message = (
-                "The `model_name` field is deprecated. Use `model` instead. "
-                + "This will be removed in a future release."
-            )
-            warnings.warn(
-                warning_message,
-                DeprecationWarning,
-            )
-            print(warning_message)
-            self.model = self.model_name
-            self.model_name = None
-        if self.tuned_model_name is not None:
-            warning_message = (
-                "`tuned_model_name` field is deprecated. Use `tuned_model` instead. "
-                + "This will be removed in a future release."
-            )
-            warnings.warn(
-                warning_message,
-                DeprecationWarning,
-            )
-            print(warning_message)
-            self.tuned_model = self.tuned_model_name
-            self.tuned_model_name = None
-
-    def _init_environment(self) -> None:
-        try:
-            import google.api_core.exceptions as google_exceptions  # type:ignore
-            import vertexai  # type:ignore
-
-            self._vertexai = vertexai
-            self._google_exceptions = google_exceptions
-        except ImportError:
-            self._raise_import_error(
-                package_display_name="VertexAI",
-                package_name="google-cloud-aiplatform",
-                package_min_version=MINIMUM_VERTEX_AI_VERSION,
-            )
-
-    def _init_vertex_ai(self) -> None:
-        self._vertexai.init(**self._init_params)
-
-    def _instantiate_model(self) -> None:
-        if self.is_codey_model:
-            from vertexai.preview.language_models import CodeGenerationModel  # type:ignore
-
-            model = CodeGenerationModel
-        else:
-            from vertexai.preview.language_models import TextGenerationModel
-
-            model = TextGenerationModel
-
-        if self.tuned_model:
-            self._model = model.get_tuned_model(self.tuned_model)
-        else:
-            self._model = model.from_pretrained(self.model)
-
-    def verbose_generation_info(self) -> str:
-        return f"VertexAI invocation parameters: {self.invocation_params}"
-
-    async def _async_generate(self, prompt: str, **kwargs: Dict[str, Any]) -> str:
-        return self._generate(prompt, **kwargs)
-
-    def _generate(self, prompt: str, **kwargs: Dict[str, Any]) -> str:
-        invoke_params = self.invocation_params
-        response = self._model.predict(
-            prompt=prompt,
-            **invoke_params,
-        )
-        return str(response.text)
-
-    @property
-    def is_codey_model(self) -> bool:
-        return is_codey_model(self.tuned_model or self.model)
-
-    @property
-    def _init_params(self) -> Dict[str, Any]:
-        return {
-            "project": self.project,
-            "location": self.location,
-            "credentials": self.credentials,
-        }
-
-    @property
-    def invocation_params(self) -> Dict[str, Any]:
-        params = {
-            "temperature": self.temperature,
-            "max_output_tokens": self.max_tokens,
-        }
-        if self.is_codey_model:
-            return params
-        else:
-            return {
-                **params,
-                "top_k": self.top_k,
-                "top_p": self.top_p,
-            }
-
-    def get_tokens_from_text(self, text: str) -> List[int]:
-        raise NotImplementedError
-
-    def get_text_from_tokens(self, tokens: List[int]) -> str:
-        raise NotImplementedError
-
-    @property
-    def max_context_size(self) -> int:
-        raise NotImplementedError
-
-    @property
-    def encoder(self):  # type:ignore
-        raise NotImplementedError
-
-
-def is_codey_model(model_name: str) -> bool:
-    """Returns True if the model name is a Codey model.
-
-    Args:
-        model_name: The model name to check.
-
-    Returns: True if the model name is a Codey model.
-    """
-    return "code" in model_name

phoenix/experimental/evals/retrievals.py

@@ -1,96 +0,0 @@
-"""
-Helper functions for evaluating the retrieval step of retrieval-augmented generation.
-"""
-
-from typing import List, Optional
-
-from tenacity import (
-    retry,
-    stop_after_attempt,
-    wait_random_exponential,
-)
-
-_EVALUATION_SYSTEM_MESSAGE = (
-    "You will be given a query and a reference text. "
-    "You must determine whether the reference text contains an answer to the input query. "
-    'Your response must be single word, either "relevant" or "irrelevant", '
-    "and should not contain any text or characters aside from that word. "
-    '"irrelevant" means that the reference text does not contain an answer to the query. '
-    '"relevant" means the reference text contains an answer to the query.'
-)
-_QUERY_CONTEXT_PROMPT_TEMPLATE = """# Query: {query}
-
-# Reference: {reference}
-
-# Answer ("relevant" or "irrelevant"): """
-
-
-def compute_precisions_at_k(
-    relevance_classifications: List[Optional[bool]],
-) -> List[Optional[float]]:
-    """Given a list of relevance classifications, computes precision@k for k = 1, 2, ..., n, where
-    n is the length of the input list.
-
-    Args:
-        relevance_classifications (List[Optional[bool]]): A list of relevance classifications for a
-            set of retrieved documents, sorted by order of retrieval (i.e., the first element is the
-            classification for the first retrieved document, the second element is the
-            classification for the second retrieved document, etc.). The list may contain None
-            values, which indicate that the relevance classification for the corresponding document
-            is unknown.
-
-    Returns:
-        List[Optional[float]]: A list of precision@k values for k = 1, 2, ..., n, where n is the
-            length of the input list. The first element is the precision@1 value, the second element
-            is the precision@2 value, etc. If the input list contains any None values, those values
-            are omitted when computing the precision@k values.
-    """
-    precisions_at_k = []
-    num_relevant_classifications = 0
-    num_non_none_classifications = 0
-    for relevance_classification in relevance_classifications:
-        if isinstance(relevance_classification, bool):
-            num_non_none_classifications += 1
-            num_relevant_classifications += int(relevance_classification)
-        precisions_at_k.append(
-            num_relevant_classifications / num_non_none_classifications
-            if num_non_none_classifications > 0
-            else None
-        )
-    return precisions_at_k
-
-
-@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
-def classify_relevance(query: str, document: str, model_name: str) -> Optional[bool]:
-    """Given a query and a document, determines whether the document contains an answer to the
-    query.
-
-    Args:
-        query (str): The query text. document (str): The document text. model_name (str): The name
-        of the OpenAI API model to use for the classification.
-
-    Returns:
-        Optional[bool]: A boolean indicating whether the document contains an answer to the query
-            (True meaning relevant, False meaning irrelevant), or None if the LLM produces an
-            unparseable output.
-    """
-
-    from openai import OpenAI
-
-    client = OpenAI()
-
-    prompt = _QUERY_CONTEXT_PROMPT_TEMPLATE.format(
-        query=query,
-        reference=document,
-    )
-    response = client.chat.completions.create(
-        messages=[
-            {"role": "system", "content": _EVALUATION_SYSTEM_MESSAGE},
-            {"role": "user", "content": prompt},
-        ],
-        model=model_name,
-    )
-
-    raw_response_text = str(response.choices[0].message.content).strip()
-    relevance_classification = {"relevant": True, "irrelevant": False}.get(raw_response_text)
-    return relevance_classification