lionagi 0.5.5__py3-none-any.whl → 0.6.1__py3-none-any.whl

lionagi/integrations/pandas_/search_keywords.py

@@ -1,69 +0,0 @@
-# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
-#
-# SPDX-License-Identifier: Apache-2.0
-import pandas as pd
-
-from .to_df import to_df
-
-
-def search_dataframe_keywords(
-    df: pd.DataFrame,
-    keywords: str | list[str],
-    *,
-    column: str | list[str] = "content",
-    case_sensitive: bool = False,
-    reset_index: bool = False,
-    dropna: bool = False,
-    regex: bool = False,
-    match_all: bool = False,
-) -> pd.DataFrame:
-    """
-    Filters a DataFrame for rows where specified column(s) contain given keywords.
-
-    Args:
-        df: The DataFrame to search through.
-        keywords: A keyword or list of keywords to search for.
-        column: The column(s) to perform the search in. Can be a string or list of strings.
-        case_sensitive: Whether the search should be case-sensitive.
-        reset_index: Whether to reset the DataFrame's index after filtering.
-        dropna: Whether to drop rows with NA values before searching.
-        regex: If True, treat keywords as regular expressions.
-        match_all: If True, all keywords must be present for a match.
-
-    Returns:
-        A filtered DataFrame containing only rows where the specified column(s) contain
-        any (or all) of the provided keywords.
-
-    Example:
-        >>> df = pd.DataFrame({'content': ['apple pie', 'banana split', 'cherry pie']})
-        >>> search_keywords(df, ['pie', 'cherry'], match_all=True)
-           content
-        2  cherry pie
-    """
-    if dropna:
-        df = df.dropna(subset=[column] if isinstance(column, str) else column)
-
-    if isinstance(keywords, str):
-        keywords = [keywords]
-
-    if isinstance(column, str):
-        column = [column]
-
-    mask = pd.DataFrame(index=df.index)
-    for col in column:
-        col_mask = pd.Series(False, index=df.index)
-        for keyword in keywords:
-            keyword_mask = df[col].str.contains(
-                keyword, case=case_sensitive, regex=regex, na=False
-            )
-            col_mask = col_mask | keyword_mask
-        mask[col] = col_mask
-
-    final_mask = mask.all(axis=1) if match_all else mask.any(axis=1)
-
-    result = df[final_mask]
-
-    if reset_index:
-        result = result.reset_index(drop=True)
-
-    return to_df(result)

lionagi/integrations/pandas_/to_df.py

@@ -1,196 +0,0 @@
-# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
-#
-# SPDX-License-Identifier: Apache-2.0
-from typing import Any, Literal
-
-from pandas import DataFrame, Series, concat
-from pandas.core.generic import NDFrame
-
-from lionagi.libs.parse import to_dict, to_list
-
-
-def to_df(
-    input_: Any,
-    /,
-    *,
-    drop_how: Literal["any", "all"] = "all",
-    drop_kwargs: dict[str, Any] | None = None,
-    reset_index: bool = True,
-    concat_kwargs: dict[str, Any] | None = None,
-    **kwargs: Any,
-) -> DataFrame:
-    """
-    Convert various input types to a pandas DataFrame.
-
-    This function attempts to convert the input to a DataFrame, handling
-    both single objects and lists of objects. It can process various input
-    types including dictionaries, lists, and existing pandas objects.
-
-    Args:
-        input_: The input data to convert to a DataFrame.
-        drop_how: How to drop NA values. Either "any" or "all".
-        drop_kwargs: Additional keyword arguments for dropna().
-        reset_index: Whether to reset the index of the resulting DataFrame.
-        concat_kwargs: Keyword arguments for pandas.concat() when dealing with lists.
-        **kwargs: Additional keyword arguments passed to DataFrame constructor.
-
-    Returns:
-        A pandas DataFrame created from the input data.
-
-    Raises:
-        ValueError: If the input cannot be converted to a DataFrame.
-
-    Example:
-        >>> data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
-        >>> df = to_df(data)
-        >>> print(df)
-           a  b
-        0  1  2
-        1  3  4
-    """
-    if not isinstance(input_, list):
-        try:
-            return general_to_df(
-                input_,
-                drop_how=drop_how,
-                drop_kwargs=drop_kwargs,
-                reset_index=reset_index,
-                **kwargs,
-            )
-        except ValueError:
-            input_ = [input_]
-
-    if isinstance(input_, list):
-        try:
-            return _list_to_df(
-                input_,
-                drop_how=drop_how,
-                drop_kwargs=drop_kwargs,
-                reset_index=reset_index,
-                concat_kwargs=concat_kwargs or {},
-                **kwargs,
-            )
-        except ValueError:
-            try:
-                _d = [to_dict(i) for i in input_]
-                return _list_to_df(
-                    _d,
-                    drop_how=drop_how,
-                    drop_kwargs=drop_kwargs,
-                    reset_index=reset_index,
-                    concat_kwargs=concat_kwargs or {},
-                    **kwargs,
-                )
-            except ValueError:
-                raise ValueError(
-                    "Error converting input_ to DataFrame"
-                ) from None
-
-
-def general_to_df(
-    input_: Any,
-    /,
-    *,
-    drop_how: str = "all",
-    drop_kwargs: dict[str, Any] | None = None,
-    reset_index: bool = True,
-    **kwargs: Any,
-) -> DataFrame:
-
-    try:
-        return _general_to_df(
-            input_,
-            drop_how=drop_how,
-            drop_kwargs=drop_kwargs,
-            reset_index=reset_index,
-            **kwargs,
-        )
-    except ValueError:
-        try:
-            _d = to_dict(input_)
-            return _general_to_df(
-                _d,
-                drop_how=drop_how,
-                drop_kwargs=drop_kwargs,
-                reset_index=reset_index,
-                **kwargs,
-            )
-        except ValueError:
-            raise ValueError("Error converting input_ to DataFrame") from None
-
-
-def _general_to_df(
-    input_: Any,
-    /,
-    *,
-    drop_how: str = "all",
-    drop_kwargs: dict[str, Any] | None = None,
-    reset_index: bool = True,
-    **kwargs: Any,
-) -> DataFrame:
-    if drop_kwargs is None:
-        drop_kwargs = {}
-
-    try:
-        df: DataFrame = DataFrame(input_, **kwargs)
-
-        if "thresh" not in drop_kwargs:
-            drop_kwargs["how"] = drop_how
-        df = df.dropna(**drop_kwargs)
-        return df.reset_index(drop=True) if reset_index else df
-    except Exception as e:
-        raise ValueError(f"Error converting input_ to DataFrame: {e}") from e
-
-
-def _list_to_df(
-    input_: list,
-    /,
-    *,
-    drop_how: str = "all",
-    drop_kwargs: dict | None = None,
-    reset_index: bool = True,
-    concat_kwargs: dict | None = None,
-    **kwargs,
-) -> DataFrame:
-    if not input_:
-        return DataFrame()
-
-    if not isinstance(input_[0], (DataFrame, Series, NDFrame)):
-        if drop_kwargs is None:
-            drop_kwargs = {}
-        try:
-            df: DataFrame = DataFrame(input_, **kwargs)
-            if "thresh" not in drop_kwargs:
-                drop_kwargs["how"] = drop_how
-            df = df.dropna(**drop_kwargs)
-            return df.reset_index(drop=True) if reset_index else df
-        except Exception as e:
-            raise ValueError(
-                f"Error converting input_ to DataFrame: {e}"
-            ) from e
-
-    if drop_kwargs is None:
-        drop_kwargs = {}
-    try:
-        config = concat_kwargs.copy()
-        config.pop("axis", None)
-        df = concat(
-            input_,
-            axis=1 if all(isinstance(i, Series) for i in input_) else 0,
-            **concat_kwargs,
-        )
-    except Exception as e1:
-        try:
-            input_ = to_list(input_, dropna=True, flatten=True)
-            df = input_[0]
-            if len(input_) > 1:
-                for i in input_[1:]:
-                    df = concat([df, i], **concat_kwargs)
-        except Exception as e2:
-            raise ValueError(
-                f"Error converting input_ to DataFrame: {e1}, {e2}"
-            ) from e2
-
-    drop_kwargs["how"] = drop_how
-    df.dropna(**drop_kwargs, inplace=True)
-    return df.reset_index(drop=True) if reset_index else df

lionagi/integrations/pandas_/update_cells.py

@@ -1,54 +0,0 @@
-# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
-#
-# SPDX-License-Identifier: Apache-2.0
-from typing import Any
-
-import pandas as pd
-
-from .to_df import to_df
-
-
-def update_cells(
-    df: pd.DataFrame,
-    updates: dict[tuple[int | str, int | str], Any],
-    create_missing: bool = False,
-) -> pd.DataFrame:
-    """
-    Updates multiple cells in a DataFrame based on a dictionary of updates.
-
-    Args:
-        df: The DataFrame to update.
-        updates: A dictionary where keys are (row, column) tuples and values are the new values.
-                 Rows and columns can be specified by integer position or label.
-        create_missing: If True, create new columns if they don't exist.
-
-    Returns:
-        The updated DataFrame.
-
-    Raises:
-        KeyError: If a specified row or column doesn't exist and create_missing is False.
-
-    Example:
-        >>> df = pd.DataFrame({'A': [1, 2], 'B': [3, 4]})
-        >>> updates = {(0, 'A'): 10, (1, 'B'): 40, (0, 'C'): 100}
-        >>> update_cells(df, updates, create_missing=True)
-            A   B    C
-        0  10   3  100
-        1   2  40  NaN
-    """
-    df_copy = df.copy()
-
-    for (row, col), value in updates.items():
-        try:
-            if isinstance(col, str) and col not in df_copy.columns:
-                if create_missing:
-                    df_copy[col] = None
-                else:
-                    raise KeyError(f"Column '{col}' does not exist.")
-
-            df_copy.loc[row, col] = value
-        except KeyError as e:
-            if not create_missing:
-                raise KeyError(f"Invalid row or column: {e}") from e
-
-    return to_df(df_copy)

lionagi/integrations/perplexity_/PerplexityModel.py

@@ -1,274 +0,0 @@
-# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-import os
-from pathlib import Path
-
-from dotenv import load_dotenv
-from pydantic import (
-    BaseModel,
-    ConfigDict,
-    Field,
-    field_serializer,
-    model_validator,
-)
-
-from lionagi.integrations.perplexity_.api_endpoints.api_request import (
-    PerplexityRequest,
-)
-from lionagi.integrations.perplexity_.api_endpoints.chat_completions.request.request_body import (
-    PerplexityChatCompletionRequestBody,
-)
-from lionagi.integrations.perplexity_.api_endpoints.match_response import (
-    match_response,
-)
-from lionagi.service.rate_limiter import RateLimiter, RateLimitError
-from lionagi.service.service_util import invoke_retry
-from lionagi.service.token_calculator import TiktokenCalculator
-
-load_dotenv()
-path = Path(__file__).parent
-
-price_config_file_name = path / "perplexity_price_data.yaml"
-max_output_token_file_name = path / "perplexity_max_output_token_data.yaml"
-
-
-class _ModuleImportClass:
-    from lionagi.libs.package.imports import check_import
-
-    yaml = check_import("yaml", pip_name="pyyaml")
-
-
-class PerplexityModel(BaseModel):
-    model: str = Field(description="ID of the model to use.")
-
-    request_model: PerplexityRequest = Field(description="Making requests")
-
-    rate_limiter: RateLimiter = Field(
-        description="Rate Limiter to track usage"
-    )
-
-    text_token_calculator: TiktokenCalculator = Field(
-        default=None, description="Token Calculator"
-    )
-
-    estimated_output_len: int = Field(
-        default=0, description="Expected output len before making request"
-    )
-
-    model_config = ConfigDict(extra="forbid")
-
-    @model_validator(mode="before")
-    @classmethod
-    def parse_input(cls, data: dict):
-        if not isinstance(data, dict):
-            raise ValueError("Invalid init param")
-
-        # parse request model
-        request_model_params = {
-            "api_key": data.pop("api_key", None),
-            "endpoint": data.pop("endpoint", None),
-            "method": data.pop("method", None),
-            "content_type": data.pop("content_type", None),
-        }
-        try:
-            api_key = os.getenv(request_model_params["api_key"], None)
-            if api_key:
-                request_model_params["api_key"] = api_key
-        except Exception:
-            pass
-
-        data["request_model"] = PerplexityRequest(**request_model_params)
-
-        # parse rate limiter
-        if "rate_limiter" not in data:
-            rate_limiter_params = {}
-            if limit_tokens := data.pop("limit_tokens", None):
-                rate_limiter_params["limit_tokens"] = limit_tokens
-            if limit_requests := data.pop("limit_requests", None):
-                rate_limiter_params["limit_requests"] = limit_requests
-
-            data["rate_limiter"] = RateLimiter(**rate_limiter_params)
-
-        # parse token calculator
-        try:
-            # Perplexity uses cl100k_base encoding like OpenAI
-            text_calc = TiktokenCalculator(encoding_name="cl100k_base")
-            data["text_token_calculator"] = text_calc
-        except Exception:
-            pass
-
-        return data
-
-    @field_serializer("request_model")
-    def serialize_request_model(self, value: PerplexityRequest):
-        return value.model_dump(exclude_unset=True)
-
-    @invoke_retry(max_retries=3, base_delay=1, max_delay=60)
-    async def invoke(
-        self,
-        request_body: PerplexityChatCompletionRequestBody,
-        estimated_output_len: int = 0,
-        output_file=None,
-        parse_response=True,
-    ):
-        if request_model := getattr(request_body, "model"):
-            if request_model != self.model:
-                raise ValueError(
-                    f"Request model does not match. Model is {self.model}, but request is made for {request_model}."
-                )
-
-        # check remaining rate limit
-        input_token_len = await self.get_input_token_len(request_body)
-
-        if getattr(request_body, "max_tokens", None):
-            estimated_output_len = request_body.max_tokens
-
-        invoke_viability_result = self.verify_invoke_viability(
-            input_tokens_len=input_token_len,
-            estimated_output_len=estimated_output_len,
-        )
-        if not invoke_viability_result:
-            raise RateLimitError(
-                message="Rate limit reached for requests",
-                input_token_len=input_token_len,
-                estimated_output_len=estimated_output_len,
-            )
-
-        try:
-            if getattr(request_body, "stream", None):
-                return await self.stream(
-                    request_body,
-                    output_file=output_file,
-                    parse_response=parse_response,
-                )
-
-            response_body, response_headers = await self.request_model.invoke(
-                json_data=request_body,
-                output_file=output_file,
-                with_response_header=True,
-                parse_response=False,
-            )
-
-            if response_body:
-                # Update rate limit based on usage
-                if response_body.get("usage"):
-                    total_token_usage = response_body["usage"]["total_tokens"]
-                    if date_str := response_headers.get("date"):
-                        self.rate_limiter.update_rate_limit(
-                            date_str, total_token_usage
-                        )
-                    else:
-                        self.rate_limiter.update_rate_limit(
-                            None, total_token_usage
-                        )
-                else:
-                    self.rate_limiter.update_rate_limit(None)
-
-            if parse_response:
-                return match_response(self.request_model, response_body)
-            else:
-                return response_body
-
-        except Exception as e:
-            raise e
-
-    async def stream(
-        self,
-        request_body: PerplexityChatCompletionRequestBody,
-        output_file=None,
-        parse_response=True,
-        verbose=True,
-    ):
-        response_chunks = []
-        response_headers = None
-
-        async for chunk in self.request_model.stream(
-            json_data=request_body,
-            output_file=output_file,
-            with_response_header=True,
-            verbose=verbose,
-        ):
-            if isinstance(chunk, dict):
-                if "headers" in chunk:
-                    response_headers = chunk["headers"]
-                else:
-                    response_chunks.append(chunk)
-
-        # Update rate limit if we have usage information
-        if response_chunks and response_chunks[-1].get("usage"):
-            total_token_usage = response_chunks[-1]["usage"]["total_tokens"]
-            if date_str := response_headers.get("date"):
-                self.rate_limiter.update_rate_limit(
-                    date_str, total_token_usage
-                )
-            else:
-                self.rate_limiter.update_rate_limit(None, total_token_usage)
-
-        if parse_response:
-            return match_response(self.request_model, response_chunks)
-        else:
-            return response_chunks
-
-    async def get_input_token_len(
-        self, request_body: PerplexityChatCompletionRequestBody
-    ):
-        if request_model := getattr(request_body, "model"):
-            if request_model != self.model:
-                raise ValueError(
-                    f"Request model does not match. Model is {self.model}, but request is made for {request_model}."
-                )
-
-        total_tokens = 0
-        for message in request_body.messages:
-            total_tokens += self.text_token_calculator.calculate(
-                message.content
-            )
-
-        return total_tokens
-
-    def verify_invoke_viability(
-        self, input_tokens_len: int = 0, estimated_output_len: int = 0
-    ):
-        self.rate_limiter.release_tokens()
-
-        estimated_output_len = (
-            estimated_output_len
-            if estimated_output_len != 0
-            else self.estimated_output_len
-        )
-        if estimated_output_len == 0:
-            with open(max_output_token_file_name) as file:
-                output_token_config = _ModuleImportClass.yaml.safe_load(file)
-                estimated_output_len = output_token_config.get(self.model, 0)
-                self.estimated_output_len = estimated_output_len
-
-        if self.rate_limiter.check_availability(
-            input_tokens_len, estimated_output_len
-        ):
-            return True
-        else:
-            return False
-
-    def estimate_text_price(
-        self,
-        input_text: str,
-        estimated_num_of_output_tokens: int = 0,
-    ):
-        if self.text_token_calculator is None:
-            raise ValueError("Token calculator not available")
-
-        num_of_input_tokens = self.text_token_calculator.calculate(input_text)
-
-        with open(price_config_file_name) as file:
-            price_config = _ModuleImportClass.yaml.safe_load(file)
-
-        model_price_info_dict = price_config["model"][self.model]
-        estimated_price = (
-            model_price_info_dict["input_tokens"] * num_of_input_tokens
-            + model_price_info_dict["output_tokens"]
-            * estimated_num_of_output_tokens
-        )
-
-        return estimated_price