lionagi 0.3.8__py3-none-any.whl → 0.5.0__py3-none-any.whl

lionagi/libs/parse/string_parse/code_block.py

@@ -0,0 +1,73 @@
+import re
+
+
+def extract_code_block(
+    str_to_parse: str,
+    return_as_list: bool = False,
+    languages: list[str] | None = None,
+    categorize: bool = False,
+) -> str | list[str] | dict[str, list[str]]:
+    """
+    Extract code blocks from a given string containing
+    Markdown-formatted text.
+
+    This function identifies code blocks enclosed by triple
+    backticks (```) or
+    tildes (~~~), extracts their content, and can filter
+    them based on specified
+    programming languages. It provides options to return
+    the extracted code
+    blocks as a single concatenated string, a list, or a
+    dictionary categorized
+    by language.
+
+    Args:
+        str_to_parse: The input string containing Markdown
+        code blocks.
+        return_as_list: If True, returns a list of code
+        blocks; otherwise, returns
+            them as a single concatenated string separated
+            by two newlines.
+        languages: A list of languages to filter the code
+        blocks. If None,
+            extracts code blocks of all languages.
+        categorize: If True, returns a dictionary mapping
+        languages to lists of
+            code blocks.
+
+    Returns:
+        Depending on the parameters:
+            - A concatenated string of code blocks.
+            - A list of code blocks.
+            - A dictionary mapping languages to lists of
+            code blocks.
+    """
+    code_blocks = []
+    code_dict = {}
+
+    pattern = re.compile(
+        r"""
+        ^(?P<fence>```|~~~)[ \t]*     # Opening fence ``` or ~~~
+        (?P<lang>[\w+-]*)[ \t]*\n     # Optional language identifier
+        (?P<code>.*?)(?<=\n)          # Code content
+        ^(?P=fence)[ \t]*$            # Closing fence matching the opening
+        """,
+        re.MULTILINE | re.DOTALL | re.VERBOSE,
+    )
+
+    for match in pattern.finditer(str_to_parse):
+        lang = match.group("lang") or "plain"
+        code = match.group("code")
+
+        if languages is None or lang in languages:
+            if categorize:
+                code_dict.setdefault(lang, []).append(code)
+            else:
+                code_blocks.append(code)
+
+    if categorize:
+        return code_dict
+    elif return_as_list:
+        return code_blocks
+    else:
+        return "\n\n".join(code_blocks)

lionagi/libs/parse/string_parse/docstring.py

@@ -0,0 +1,179 @@
+import inspect
+from collections.abc import Callable
+from typing import Literal
+
+
+def extract_docstring(
+    func: Callable, style: Literal["google", "rest"] = "google"
+) -> tuple[str | None, dict[str, str]]:
+    """
+    Extract function description and parameter descriptions from docstring.
+
+    Args:
+        func: The function from which to extract docstring details.
+        style: The style of docstring to parse ('google' or 'rest').
+
+    Returns:
+        A tuple containing the function description and a dictionary with
+        parameter names as keys and their descriptions as values.
+
+    Raises:
+        ValueError: If an unsupported style is provided.
+
+    Examples:
+        >>> def example_function(param1: int, param2: str):
+        ...     '''Example function.
+        ...
+        ...     Args:
+        ...         param1 (int): The first parameter.
+        ...         param2 (str): The second parameter.
+        ...     '''
+        ...     pass
+        >>> description, params = extract_docstring_details(example_function)
+        >>> description
+        'Example function.'
+        >>> params == {'param1': 'The first parameter.',
+        ...            'param2': 'The second parameter.'}
+        True
+    """
+    style = str(style).strip().lower()
+
+    if style == "google":
+        func_description, params_description = (
+            _extract_docstring_details_google(func)
+        )
+    elif style == "rest":
+        func_description, params_description = _extract_docstring_details_rest(
+            func
+        )
+    else:
+        raise ValueError(
+            f'{style} is not supported. Please choose either "google" or'
+            ' "reST".'
+        )
+    return func_description, params_description
+
+
+def _extract_docstring_details_google(
+    func: Callable,
+) -> tuple[str | None, dict[str, str]]:
+    """
+    Extract details from Google-style docstring.
+
+    Args:
+        func: The function from which to extract docstring details.
+
+    Returns:
+        A tuple containing the function description and a dictionary with
+        parameter names as keys and their descriptions as values.
+
+    Examples:
+        >>> def example_function(param1: int, param2: str):
+        ...     '''Example function.
+        ...
+        ...     Args:
+        ...         param1 (int): The first parameter.
+        ...         param2 (str): The second parameter.
+        ...     '''
+        ...     pass
+        >>> description, params = _extract_docstring_details_google(
+        ...     example_function)
+        >>> description
+        'Example function.'
+        >>> params == {'param1': 'The first parameter.',
+        ...            'param2': 'The second parameter.'}
+        True
+    """
+    docstring = inspect.getdoc(func)
+    if not docstring:
+        return None, {}
+    lines = docstring.split("\n")
+    func_description = lines[0].strip()
+
+    lines_len = len(lines)
+
+    params_description = {}
+    param_start_pos = next(
+        (
+            i + 1
+            for i in range(1, lines_len)
+            if (
+                (a := str(lines[i]).strip().lower()).startswith("args")
+                or a.startswith("parameters")
+                or a.startswith("params")
+                or a.startswith("arguments")
+            )
+        ),
+        0,
+    )
+    current_param = None
+    for i in range(param_start_pos, lines_len):
+        if lines[i] == "":
+            continue
+        elif lines[i].startswith(" "):
+            param_desc = lines[i].split(":", 1)
+            if len(param_desc) == 1:
+                params_description[
+                    current_param
+                ] += f" {param_desc[0].strip()}"
+                continue
+            param, desc = param_desc
+            param = param.split("(")[0].strip()
+            params_description[param] = desc.strip()
+            current_param = param
+        else:
+            break
+    return func_description, params_description
+
+
+def _extract_docstring_details_rest(
+    func: Callable,
+) -> tuple[str | None, dict[str, str]]:
+    """
+    Extract details from reStructuredText-style docstring.
+
+    Args:
+        func: The function from which to extract docstring details.
+
+    Returns:
+        A tuple containing the function description and a dictionary with
+        parameter names as keys and their descriptions as values.
+
+    Examples:
+        >>> def example_function(param1: int, param2: str):
+        ...     '''Example function.
+        ...
+        ...     :param param1: The first parameter.
+        ...     :type param1: int
+        ...     :param param2: The second parameter.
+        ...     :type param2: str
+        ...     '''
+        ...     pass
+        >>> description, params = _extract_docstring_details_rest(
+        ...     example_function)
+        >>> description
+        'Example function.'
+        >>> params == {'param1': 'The first parameter.',
+        ...            'param2': 'The second parameter.'}
+        True
+    """
+    docstring = inspect.getdoc(func)
+    if not docstring:
+        return None, {}
+    lines = docstring.split("\n")
+    func_description = lines[0].strip()
+
+    params_description = {}
+    current_param = None
+    for line in lines[1:]:
+        line = line.strip()
+        if line.startswith(":param"):
+            param_desc = line.split(":", 2)
+            _, param, desc = param_desc
+            param = param.split()[-1].strip()
+            params_description[param] = desc.strip()
+            current_param = param
+        elif line.startswith(" "):
+            params_description[current_param] += f" {line}"
+
+    return func_description, params_description

lionagi/libs/parse/string_parse/function_.py

@@ -0,0 +1,92 @@
+import inspect
+from typing import Any, Literal
+
+from lionagi.libs.constants import py_json_msp
+
+from .docstring import extract_docstring
+
+
+def function_to_schema(
+    f_,
+    style: Literal["google", "rest"] = "google",
+    *,
+    f_description=None,
+    p_description=None,
+) -> dict:
+    """
+    Generate a schema description for a given function. in openai format
+
+    This function generates a schema description for the given function
+    using typing hints and docstrings. The schema includes the function's
+    name, description, and parameter details.
+
+    Args:
+        func (Callable): The function to generate a schema for.
+        style (str): The docstring format. Can be 'google' (default) or
+            'reST'.
+        func_description (str, optional): A custom description for the
+            function. If not provided, the description will be extracted
+            from the function's docstring.
+        params_description (dict, optional): A dictionary mapping
+            parameter names to their descriptions. If not provided, the
+            parameter descriptions will be extracted from the function's
+            docstring.
+
+    Returns:
+        dict: A schema describing the function, including its name,
+        description, and parameter details.
+
+    Example:
+        >>> def example_func(param1: int, param2: str) -> bool:
+        ...     '''Example function.
+        ...
+        ...     Args:
+        ...         param1 (int): The first parameter.
+        ...         param2 (str): The second parameter.
+        ...     '''
+        ...     return True
+        >>> schema = function_to_schema(example_func)
+        >>> schema['function']['name']
+        'example_func'
+    """
+    # Extract function name
+    func_name = f_.__name__
+
+    # Extract function description and parameter descriptions
+    if not f_description or not p_description:
+        func_desc, params_desc = extract_docstring(f_, style)
+        f_description = f_description or func_desc
+        p_description = p_description or params_desc
+
+    # Extract parameter details using typing hints
+    sig = inspect.signature(f_)
+    parameters: dict[str, Any] = {
+        "type": "object",
+        "properties": {},
+        "required": [],
+    }
+
+    for name, param in sig.parameters.items():
+        # Default type to string and update if type hint is available
+        param_type = "string"
+        if param.annotation is not inspect.Parameter.empty:
+            param_type = py_json_msp[param.annotation.__name__]
+
+        # Extract parameter description from docstring, if available
+        param_description = p_description.get(name)
+
+        # Assuming all parameters are required for simplicity
+        parameters["required"].append(name)
+        parameters["properties"][name] = {
+            "type": param_type,
+            "description": param_description,
+        }
+
+    return {
+        "type": "function",
+        "function": {
+            "name": func_name,
+            "description": f_description,
+            "parameters": parameters,
+        },
+    }

lionagi/libs/parse/type_convert/__init__.py

@@ -0,0 +1,19 @@
+"""Type conversion utilities for various data formats."""
+
+from .params import ToDictParams, ToListParams, ToNumParams, ToStrParams
+from .to_dict import to_dict
+from .to_list import to_list
+from .to_num import to_num
+from .to_str import strip_lower, to_str
+
+__all__ = [
+    "to_dict",
+    "to_str",
+    "strip_lower",
+    "to_list",
+    "to_num",
+    "ToDictParams",
+    "ToListParams",
+    "ToNumParams",
+    "ToStrParams",
+]

lionagi/libs/parse/type_convert/params.py

@@ -0,0 +1,145 @@
+from collections.abc import Callable
+from typing import Any, Literal, TypeVar
+
+from pydantic import BaseModel, Field, field_validator
+
+from lionagi.libs.constants import NUM_TYPES
+
+T = TypeVar("T")
+
+
+class ToDictParams(BaseModel):
+    """Parameters for to_dict conversion function."""
+
+    str_type: Literal["json", "xml"] | None = Field(
+        default="json",
+        description="Type of string to parse",
+    )
+    fuzzy_parse: bool = Field(
+        default=False,
+        description="Whether to use fuzzy parsing for JSON-like strings",
+    )
+    suppress: bool = Field(
+        default=False,
+        description="Whether to suppress errors and return empty dict",
+    )
+    parser: Callable[[str], Any] | None = Field(
+        default=None,
+        description="Custom parser function for string inputs",
+    )
+    recursive: bool = Field(
+        default=False,
+        description="Enable recursive conversion of nested structures",
+    )
+    max_recursive_depth: int | None = Field(
+        default=None,
+        description="Maximum recursion depth (default 5, max 10)",
+        ge=0,
+        le=10,
+    )
+    recursive_python_only: bool = Field(
+        default=True,
+        description="If False, attempts to convert custom types recursively",
+    )
+    use_model_dump: bool = Field(
+        default=True,
+        description="Whether to use model_dump for pydantic models",
+    )
+
+
+class ToNumParams(BaseModel):
+    """Parameters for to_num conversion function."""
+
+    upper_bound: int | float | None = Field(
+        default=None,
+        description="Maximum allowed value (inclusive)",
+    )
+    lower_bound: int | float | None = Field(
+        default=None,
+        description="Minimum allowed value (inclusive)",
+    )
+    num_type: NUM_TYPES = Field(
+        default=float,
+        description="Target numeric type (int, float, complex)",
+    )
+    precision: int | None = Field(
+        default=None,
+        description="Number of decimal places for rounding (float only)",
+        ge=0,
+    )
+    num_count: int = Field(
+        default=1,
+        description="Number of numeric values to extract",
+        gt=0,
+    )
+
+    @field_validator("upper_bound")
+    def validate_bounds(
+        cls, v: float | None, values: dict[str, Any]
+    ) -> float | None:
+        if v is not None and values.get("lower_bound") is not None:
+            if v < values["lower_bound"]:
+                raise ValueError(
+                    "upper_bound must be greater than lower_bound"
+                )
+        return v
+
+
+class ToStrParams(BaseModel):
+    """Parameters for to_str conversion function."""
+
+    strip_lower: bool = Field(
+        default=False,
+        description="Whether to strip and convert to lowercase",
+    )
+    chars: str | None = Field(
+        default=None,
+        description="Characters to strip from the result",
+    )
+    str_type: Literal["json", "xml"] | None = Field(
+        default=None,
+        description="Type of string input if applicable",
+    )
+    serialize_as: Literal["json", "xml"] | None = Field(
+        default=None,
+        description="Output serialization format",
+    )
+    use_model_dump: bool = Field(
+        default=False,
+        description="Whether to use model_dump for pydantic models",
+    )
+    str_parser: Callable[[str], Any] | None = Field(
+        default=None,
+        description="Custom parser function for string inputs",
+    )
+    parser_kwargs: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Additional keyword arguments for the parser",
+    )
+
+
+class ToListParams(BaseModel):
+    """Parameters for to_list conversion function."""
+
+    flatten: bool = Field(
+        default=False,
+        description="Whether to flatten nested structures",
+    )
+    dropna: bool = Field(
+        default=False,
+        description="Whether to drop None values",
+    )
+    unique: bool = Field(
+        default=False,
+        description="Whether to return only unique values (requires flatten=True)",
+    )
+    use_values: bool = Field(
+        default=False,
+        description="Whether to use .values() for dict-like inputs",
+    )
+
+    @field_validator("unique")
+    def validate_unique(cls, v: bool, values: dict[str, Any]) -> bool:
+        if v and not values.get("flatten", False):
+            raise ValueError("unique=True requires flatten=True")
+        return v