lionagi 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

lionagi/integrations/openai_/api_endpoints/moderations/__init__.py

@@ -0,0 +1,3 @@
+from .request_body import OpenAIModerationRequestBody
+
+__all__ = ["OpenAIModerationRequestBody"]

lionagi/integrations/openai_/api_endpoints/moderations/request_body.py

@@ -0,0 +1,20 @@
+from enum import Enum
+from typing import Optional
+
+from pydantic import Field
+
+from ..data_models import OpenAIEndpointRequestBody
+
+
+class Model(str, Enum):
+    LATEST = "text-moderation-latest"
+    STABLE = "text-moderation-stable"
+
+
+class OpenAIModerationRequestBody(OpenAIEndpointRequestBody):
+    input: list | str = Field(description="The input text to classify")
+
+    model: Model | None = Field(
+        "text-moderation-latest",
+        description="Two content moderations models are available: text-moderation-stable and text-moderation-latest.",
+    )

lionagi/integrations/openai_/api_endpoints/moderations/response_body.py

@@ -0,0 +1,139 @@
+from typing import List
+
+from pydantic import BaseModel, Field
+
+from ..data_models import OpenAIEndpointResponseBody
+
+
+class Categories(BaseModel):
+    hate: bool = Field(
+        description="Content that expresses, incites, or promotes hate based on race, gender, ethnicity, "
+        "religion, nationality, sexual orientation, disability status, or caste. Hateful "
+        "content aimed at non-protected groups (e.g., chess players) is harassment."
+    )
+
+    hate_threatening: bool = Field(
+        alias="hate/threatening",
+        description="Hateful content that also includes violence or serious harm towards the targeted "
+        "group based on race, gender, ethnicity, religion, nationality, sexual orientation, "
+        "disability status, or caste.",
+    )
+
+    harassment: bool = Field(
+        description="Content that expresses, incites, or promotes harassing language towards any target."
+    )
+
+    harassment_threatening: bool = Field(
+        alias="harassment/threatening",
+        description="Harassment content that also includes violence or serious harm towards any target.",
+    )
+
+    self_harm: bool = Field(
+        alias="self-harm",
+        description="Content that promotes, encourages, or depicts acts of self-harm, such as suicide, "
+        "cutting, and eating disorders.",
+    )
+
+    self_harm_intent: bool = Field(
+        alias="self-harm/intent",
+        description="Content where the speaker expresses that they are engaging or intend to engage in "
+        "acts of self-harm, such as suicide, cutting, and eating disorders.",
+    )
+
+    self_harm_instructions: bool = Field(
+        alias="self-harm/instructions",
+        description="Content that encourages performing acts of self-harm, such as suicide, cutting, "
+        "and eating disorders, or that gives instructions or advice on how to commit such acts.",
+    )
+
+    sexual: bool = Field(
+        description="Content meant to arouse sexual excitement, such as the description of sexual activity, "
+        "or that promotes sexual services (excluding sex education and wellness)."
+    )
+
+    sexual_minors: bool = Field(
+        alias="sexual/minors",
+        description="Sexual content that includes an individual who is under 18 years old.",
+    )
+
+    violence: bool = Field(
+        description="Content that depicts death, violence, or physical injury."
+    )
+
+    violence_graphic: bool = Field(
+        alias="violence/graphic",
+        description="Content that depicts death, violence, or physical injury in graphic detail.",
+    )
+
+
+class CategoryScores(BaseModel):
+    hate: float = Field(description="The score for the category 'hate'.")
+
+    hate_threatening: float = Field(
+        alias="hate/threatening",
+        description="The score for the category 'hate/threatening'.",
+    )
+
+    harassment: float = Field(
+        description="The score for the category 'harassment'."
+    )
+
+    harassment_threatening: float = Field(
+        alias="harassment/threatening",
+        description="The score for the category 'harassment/threatening'.",
+    )
+
+    self_harm: float = Field(
+        alias="self-harm",
+        description="The score for the category 'self-harm'.",
+    )
+
+    self_harm_intent: float = Field(
+        alias="self-harm/intent",
+        description="The score for the category 'self-harm/intent'.",
+    )
+
+    self_harm_instructions: float = Field(
+        alias="self-harm/instructions",
+        description="The score for the category 'self-harm/instructions'.",
+    )
+
+    sexual: float = Field(description="The score for the category 'sexual'.")
+
+    sexual_minors: float = Field(
+        alias="sexual/minors",
+        description="The score for the category 'sexual/minors'.",
+    )
+
+    violence: float = Field(
+        description="The score for the category 'violence'."
+    )
+
+    violence_graphic: float = Field(
+        alias="violence/graphic",
+        description="The score for the category 'violence/graphic'.",
+    )
+
+
+class Result(BaseModel):
+    flagged: bool = Field(
+        description="Whether the content violates OpenAI's usage policies."
+    )
+
+    categories: Categories = Field(
+        description="A dictionary of per-category binary content policy violation flags.",
+    )
+
+    category_scores: CategoryScores = Field(
+        description="A dictionary of per-category raw scores output by the model."
+    )
+
+
+class OpenAIModerationResponseBody(OpenAIEndpointResponseBody):
+    id: str = Field(
+        description="The unique identifier for the moderation request."
+    )
+    model: str = Field(
+        description="The model used to generate the moderation results."
+    )
+    results: list[Result] = Field(description="A list of moderation objects.")

lionagi/integrations/openai_/api_endpoints/uploads/__init__.py

@@ -0,0 +1,19 @@
+from .add_upload_part import (
+    OpenAIUploadPartPathParam,
+    OpenAIUploadPartRequestBody,
+)
+from .cancel_upload import OpenAICancelUploadPathParam
+from .complete_upload import (
+    OpenAICompleteUploadPathParam,
+    OpenAICompleteUploadRequestBody,
+)
+from .create_upload import OpenAIUploadRequestBody
+
+__all__ = [
+    "OpenAIUploadRequestBody",
+    "OpenAIUploadPartPathParam",
+    "OpenAIUploadPartRequestBody",
+    "OpenAICompleteUploadPathParam",
+    "OpenAICompleteUploadRequestBody",
+    "OpenAICancelUploadPathParam",
+]

lionagi/integrations/openai_/api_endpoints/uploads/add_upload_part.py

@@ -0,0 +1,11 @@
+from pydantic import Field
+
+from ..data_models import OpenAIEndpointPathParam, OpenAIEndpointRequestBody
+
+
+class OpenAIUploadPartPathParam(OpenAIEndpointPathParam):
+    upload_id: str = Field(description="The ID of the Upload.")
+
+
+class OpenAIUploadPartRequestBody(OpenAIEndpointRequestBody):
+    data: bytes = Field(description="The chunk of bytes for this Part.")

lionagi/integrations/openai_/api_endpoints/uploads/cancel_upload.py

@@ -0,0 +1,7 @@
+from pydantic import Field
+
+from ..data_models import OpenAIEndpointPathParam
+
+
+class OpenAICancelUploadPathParam(OpenAIEndpointPathParam):
+    upload_id: str = Field(description="The ID of the Upload.")

lionagi/integrations/openai_/api_endpoints/uploads/complete_upload.py

@@ -0,0 +1,18 @@
+from typing import List, Optional
+
+from pydantic import Field
+
+from ..data_models import OpenAIEndpointPathParam, OpenAIEndpointRequestBody
+
+
+class OpenAICompleteUploadPathParam(OpenAIEndpointPathParam):
+    upload_id: str = Field(description="The ID of the Upload.")
+
+
+class OpenAICompleteUploadRequestBody(OpenAIEndpointRequestBody):
+    part_ids: list[str] = Field(description="The ordered list of Part IDs.")
+
+    md5: str | None = Field(
+        description="The optional md5 checksum for the file contents to verify "
+        "if the bytes uploaded matches what you expect."
+    )

lionagi/integrations/openai_/api_endpoints/uploads/create_upload.py

@@ -0,0 +1,17 @@
+from pydantic import Field
+
+from ..data_models import OpenAIEndpointRequestBody
+
+
+class OpenAIUploadRequestBody(OpenAIEndpointRequestBody):
+    filename: str = Field(description="The name of the file to upload.")
+
+    purpose: str = Field(
+        description="The intended purpose of the uploaded file."
+    )
+
+    bytes: int = Field(
+        description="The intended number of bytes to be uploaded."
+    )
+
+    mime_type: str = Field(description="The MIME type of the file.")

lionagi/integrations/openai_/api_endpoints/uploads/uploads_models.py

@@ -0,0 +1,52 @@
+from typing import Optional
+
+from pydantic import Field
+
+from ..data_models import OpenAIEndpointResponseBody
+from ..files.file_models import OpenAIFileResponseBody, Purpose
+
+
+class OpenAIUploadResponseBody(OpenAIEndpointResponseBody):
+    id: str = Field(description="The Upload unique identifier.")
+
+    created_at: int = Field(
+        description="The Unix timestamp (in seconds) for when the Upload was created.",
+    )
+
+    filename: str = Field(description="The name of the file to be uploaded.")
+
+    bytes: int = Field(
+        description="The intended number of bytes to be uploaded."
+    )
+
+    purpose: Purpose = Field(description="The intended purpose of the file.")
+
+    status: str = Field(description="The status of the Upload.")
+
+    expires_at: int = Field(
+        description="The Unix timestamp (in seconds) for when the Upload expires."
+    )
+
+    object: str = Field(
+        description="The object type, which is always 'upload'."
+    )
+
+    file: OpenAIFileResponseBody | None = Field(
+        None, description="The File object created after completion."
+    )
+
+
+class OpenAIUploadPartResponseBody(OpenAIUploadResponseBody):
+    id: str = Field(description="The upload Part unique identifier.")
+
+    created_at: int = Field(
+        description="The Unix timestamp (in seconds) for when the Part was created.",
+    )
+
+    upload_id: str = Field(
+        description="The ID of the Upload object that this Part was added to."
+    )
+
+    object: str = Field(
+        description="The object type, which is always 'upload.part'."
+    )

lionagi/integrations/openai_/image_token_calculator/image_token_calculator.py

@@ -0,0 +1,92 @@
+import base64
+import math
+import os
+from io import BytesIO
+from typing import Literal
+
+import aiohttp
+import yaml
+from PIL import Image
+from pydantic import BaseModel, Field, field_validator
+
+image_token_config_file_name = os.path.join(
+    os.path.dirname(__file__), "openai_image_token_data.yaml"
+)
+
+
+class OpenAIImageTokenCalculator(BaseModel):
+    model: str = Field(description="ID of the model to use.")
+
+    @field_validator("model")
+    @classmethod
+    def validate_model_image_function(cls, value: str):
+        with open(image_token_config_file_name) as file:
+            token_config = yaml.safe_load(file)
+
+        token_config = token_config.get(value, None)
+
+        if token_config is None:
+            raise ValueError("The model does not have vision capabilities.")
+        return value
+
+    @staticmethod
+    async def get_image_size(url: str):
+        if url.startswith("data:image/jpeg;base64,"):
+            # base64 encoded format
+            image_base64 = url.split("data:image/jpeg;base64,")[1]
+            image_base64.strip("{}")
+            image_data = base64.b64decode(image_base64)
+            image = Image.open(BytesIO(image_data))
+        else:
+            # image url
+            async with aiohttp.ClientSession() as client:
+                async with client.get(url=url) as response:
+                    response.raise_for_status()
+
+                    content = await response.read()
+
+                    image = Image.open(BytesIO(content))
+
+        return image.size
+
+    async def calculate(
+        self, url: str, detail: Literal["low", "high", "auto"] = "auto"
+    ):
+        if detail not in ["low", "high", "auto"]:
+            raise ValueError(
+                "Invalid detail option. Valid options are: low, high, auto"
+            )
+
+        with open(image_token_config_file_name) as file:
+            token_config = yaml.safe_load(file)
+
+        token_config = token_config.get(self.model, None)
+
+        if token_config is None:
+            raise ValueError("The model does not have vision capabilities.")
+
+        width, height = await self.get_image_size(url)
+
+        if detail == "low":
+            return token_config["base"]
+        if detail == "auto":
+            # TODO: check "auto" option condition
+            if width <= 512 and height <= 512:
+                return token_config["base"]
+
+        # Scale to fit within a 2048 x 2048 square
+        if max(width, height) > 2048:
+            scale_factor = 2048 / max(width, height)
+            width = int(width * scale_factor)
+            height = int(height * scale_factor)
+
+        # Scale such that the shortest side is 768px
+        if min(width, height) > 768:
+            scale_factor = 768 / min(width, height)
+            width = int(width * scale_factor)
+            height = int(height * scale_factor)
+
+        # Calculate the number of 512px squares
+        num_squares = math.ceil(width / 512) * math.ceil(height / 512)
+
+        return num_squares * token_config["tile"] + token_config["base"]

lionagi/integrations/openai_/image_token_calculator/openai_image_token_data.yaml

@@ -0,0 +1,15 @@
+gpt-4o:
+  base: 85
+  tile: 170
+gpt-4o-2024-05-13:
+  base: 85
+  tile: 170
+gpt-4o-2024-08-06:
+  base: 85
+  tile: 170
+gpt-4o-mini:
+  base: 2833
+  tile: 5667
+gpt-4o-mini-2024-07-18:
+  base: 2833
+  tile: 5667

lionagi/integrations/openai_/openai_max_output_token_data.yaml

@@ -0,0 +1,12 @@
+gpt-4o: 4096
+gpt-4o-2024-05-13: 4096
+gpt-4o-2024-08-06: 16384
+gpt-4o-mini: 16384
+gpt-4o-mini-2024-07-18: 16384
+gpt-4-turbo: 4096
+gpt-4-turbo-2024-04-09: 4096
+gpt-4-turbo-preview: 4096
+gpt-4-0125-preview: 4096
+gpt-4-1106-preview: 4096
+gpt-4: 8192
+gpt-4-0613: 8192

lionagi/integrations/openai_/openai_price_data.yaml

@@ -0,0 +1,26 @@
+model:
+  gpt-4o:
+    input_tokens: 0.000005
+    output_tokens: 0.000015
+    input_tokens_with_batch: 0.0000025
+    output_tokens_with_batch: 0.0000075
+  gpt-4o-2024-08-06:
+    input_tokens: 0.0000025
+    output_tokens: 0.00001
+    input_tokens_with_batch: 0.00000125
+    output_tokens_with_batch: 0.000005
+  gpt-4o-2024-05-13:
+    input_tokens: 0.000005
+    output_tokens: 0.000015
+    input_tokens_with_batch: 0.0000025
+    output_tokens_with_batch: 0.0000075
+  gpt-4o-mini:
+    input_tokens: 0.00000015
+    output_tokens: 0.0000006
+    input_tokens_with_batch: 0.000000075
+    output_tokens_with_batch: 0.0000003
+  gpt-4o-mini-2024-07-18:
+    input_tokens: 0.00000015
+    output_tokens: 0.0000006
+    input_tokens_with_batch: 0.000000075
+    output_tokens_with_batch: 0.0000003

lionagi/integrations/openai_/version.py

@@ -0,0 +1 @@
+__version__ = "1.0.2"

lionagi/integrations/pandas_/__init__.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+from .extend_df import extend_dataframe
+from .read import read_csv, read_json
+from .remove_rows import remove_rows
+from .replace_keywords import replace_keywords
+from .save import to_csv, to_excel
+from .search_keywords import search_dataframe_keywords
+from .to_df import to_df
+from .update_cells import update_cells
+
+__all__ = [
+    "to_df",
+    "replace_keywords",
+    "search_dataframe_keywords",
+    "read_csv",
+    "read_json",
+    "to_csv",
+    "to_excel",
+    "extend_dataframe",
+    "remove_rows",
+    "update_cells",
+]

lionagi/integrations/pandas_/extend_df.py

@@ -0,0 +1,61 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+from typing import Any, Literal
+
+import pandas as pd
+
+from lionagi.integrations.pandas_.to_df import to_df
+
+
+def extend_dataframe(
+    dataframes: list[pd.DataFrame],
+    unique_col: str = "node_id",
+    keep: Literal["first", "last", False] = "first",
+    ignore_index: bool = False,
+    sort: bool = False,
+    **kwargs: Any,
+) -> pd.DataFrame:
+    """
+    Merges multiple DataFrames while ensuring no duplicate entries based on a specified unique column.
+
+    Args:
+        dataframes: List of DataFrames to merge, with the first being the primary DataFrame.
+        unique_col: The column name to check for duplicate entries. Defaults to 'node_id'.
+        keep: Determines which duplicates to keep. 'first' keeps the first occurrence,
+              'last' keeps the last occurrence, and False drops all duplicates.
+        ignore_index: If True, the resulting axis will be labeled 0, 1, …, n - 1.
+        sort: If True, sort the join keys lexicographically in the result DataFrame.
+        **kwargs: Additional keyword arguments for `drop_duplicates`.
+
+    Returns:
+        A DataFrame combined from all input DataFrames with duplicates removed based on the unique column.
+
+    Raises:
+        ValueError: If all DataFrames are empty or if there's an error in extending.
+
+    Example:
+        >>> df1 = pd.DataFrame({'node_id': [1, 2], 'value': ['a', 'b']})
+        >>> df2 = pd.DataFrame({'node_id': [2, 3], 'value': ['c', 'd']})
+        >>> extend_dataframe([df1, df2], keep='last', ignore_index=True)
+           node_id value
+        0       1     a
+        1       2     c
+        2       3     d
+    """
+    try:
+        if all(df.empty for df in dataframes):
+            raise ValueError("All input DataFrames are empty.")
+
+        combined = pd.concat(dataframes, ignore_index=ignore_index, sort=sort)
+        result = combined.drop_duplicates(
+            subset=[unique_col], keep=keep, **kwargs
+        )
+
+        if result.empty:
+            raise ValueError("No data left after removing duplicates.")
+
+        return to_df(result)
+
+    except Exception as e:
+        raise ValueError(f"Error in extending DataFrames: {e}") from e

lionagi/integrations/pandas_/read.py

@@ -0,0 +1,103 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+from typing import Any, Literal
+
+import pandas as pd
+
+from .to_df import to_df
+
+
+def read_csv(
+    filepath: str,
+    chunk_size: int | None = None,
+    low_memory: bool = False,
+    return_as: Literal["dataframe", "json", "jsonl", "dict"] = "dataframe",
+    **kwargs: Any,
+) -> pd.DataFrame | str | list | dict:
+    """
+    Reads a CSV file into a DataFrame with optional chunking for large files.
+
+    Args:
+        filepath: The path to the CSV file to read.
+        chunk_size: Number of rows to read at a time. If specified, returns an iterable.
+        low_memory: Internally process the file in chunks to conserve memory (slower).
+        **kwargs: Additional keyword arguments to pass to pandas.read_csv function.
+
+    Returns:
+        A DataFrame containing the data read from the CSV file, or a TextFileReader
+        if chunk_size is specified.
+
+    Example:
+        >>> df = read_csv('large_file.csv', chunk_size=10000)
+        >>> for chunk in df:
+        ...     process_chunk(chunk)
+    """
+    try:
+        if chunk_size:
+            return pd.read_csv(
+                filepath, chunksize=chunk_size, low_memory=low_memory, **kwargs
+            )
+        df = pd.read_csv(filepath, low_memory=low_memory, **kwargs)
+        df = to_df(df)
+        if return_as == "dataframe":
+            return df
+        if return_as == "json":
+            return df.to_json(orient="records")
+        if return_as == "jsonl":
+            return df.to_json(orient="records", lines=True)
+        if return_as == "dict":
+            return df.to_dict(orient="records")
+
+    except Exception as e:
+        raise OSError(f"Error reading CSV file: {e}") from e
+
+
+def read_json(
+    filepath: str,
+    orient: str | None = None,
+    lines: bool = False,
+    chunk_size: int | None = None,
+    return_as: Literal["dataframe", "json", "jsonl", "dict"] = "dataframe",
+    **kwargs: Any,
+) -> pd.DataFrame | pd.io.json._json.JsonReader:
+    """
+    Reads a JSON file into a DataFrame with options for different JSON formats.
+
+    Args:
+        filepath: The path to the JSON file to read.
+        orient: Indication of expected JSON string format.
+                Allowed values are: 'split', 'records', 'index', 'columns', 'values', 'table'.
+        lines: Read the file as a json object per line.
+        chunk_size: Number of lines to read at a time. If specified, returns an iterable.
+        **kwargs: Additional keyword arguments to pass to pandas.read_json function.
+
+    Returns:
+        A DataFrame containing the data read from the JSON file, or a JsonReader
+        if chunk_size is specified.
+
+    Example:
+        >>> df = read_json('data.json', orient='records', lines=True)
+    """
+    try:
+        if chunk_size:
+            return pd.read_json(
+                filepath,
+                orient=orient,
+                lines=lines,
+                chunksize=chunk_size,
+                **kwargs,
+            )
+        df = pd.read_json(filepath, orient=orient, lines=lines, **kwargs)
+        df = to_df(df)
+        if return_as == "dataframe":
+            return df
+        if return_as == "json":
+            return df.to_json(orient="records")
+        if return_as == "jsonl":
+            return df.to_json(orient="records", lines=True)
+        if return_as == "dict":
+            return df.to_dict(orient="records")
+
+    except Exception as e:
+        raise OSError(f"Error reading JSON file: {e}") from e