lionagi 0.12.2__py3-none-any.whl → 0.12.4__py3-none-any.whl

lionagi/config.py

@@ -0,0 +1,123 @@
+# Copyright (c) 2025, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, Field, SecretStr
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class CacheConfig(BaseModel):
+    """Configuration for aiocache."""
+
+    ttl: int = 300
+    key: str | None = None
+    namespace: str | None = None
+    key_builder: Any = None
+    skip_cache_func: Any = lambda _: False
+    serializer: dict[str, Any] | None = None
+    plugins: Any = None
+    alias: str | None = None
+    noself: Any = lambda _: False
+
+    def as_kwargs(self) -> dict[str, Any]:
+        """Convert config to kwargs dict for @cached decorator.
+
+        Removes unserialisable callables that aiocache can't pickle.
+        """
+        raw = self.model_dump(exclude_none=True)
+        # Remove all unserialisable callables
+        unserialisable_keys = (
+            "key_builder",
+            "skip_cache_func",
+            "noself",
+            "serializer",
+            "plugins",
+        )
+        for key in unserialisable_keys:
+            raw.pop(key, None)
+        return raw
+
+
+class AppSettings(BaseSettings, frozen=True):
+    """Application settings with environment variable support."""
+
+    model_config = SettingsConfigDict(
+        env_file=(".env", ".env.local", ".secrets.env"),
+        env_file_encoding="utf-8",
+        case_sensitive=False,
+        extra="ignore",
+    )
+
+    aiocache_config: CacheConfig = Field(
+        default_factory=CacheConfig, description="Cache settings for aiocache"
+    )
+
+    # secrets
+    OPENAI_API_KEY: SecretStr | None = None
+    OPENROUTER_API_KEY: SecretStr | None = None
+    OLLAMA_API_KEY: SecretStr | None = None
+    EXA_API_KEY: SecretStr | None = None
+    PERPLEXITY_API_KEY: SecretStr | None = None
+    GROQ_API_KEY: SecretStr | None = None
+    ANTHROPIC_API_KEY: SecretStr | None = None
+
+    # defaults models
+    LIONAGI_EMBEDDING_PROVIDER: str = "openai"
+    LIONAGI_EMBEDDING_MODEL: str = "text-embedding-3-small"
+
+    LIONAGI_CHAT_PROVIDER: str = "openai"
+    LIONAGI_CHAT_MODEL: str = "gpt-4o"
+
+    # default storage
+    LIONAGI_AUTO_STORE_EVENT: bool = False
+    LIONAGI_STORAGE_PROVIDER: str = "async_qdrant"
+
+    LIONAGI_AUTO_EMBED_LOG: bool = False
+
+    # specific storage
+    LIONAGI_QDRANT_URL: str = "http://localhost:6333"
+    LIONAGI_DEFAULT_QDRANT_COLLECTION: str = "event_logs"
+
+    # Class variable to store the singleton instance
+    _instance: ClassVar[Any] = None
+
+    def get_secret(self, key_name: str) -> str:
+        """
+        Get the secret value for a given key name.
+
+        Args:
+            key_name: The name of the secret key (e.g., "OPENAI_API_KEY")
+
+        Returns:
+            The secret value as a string
+
+        Raises:
+            AttributeError: If the key doesn't exist
+            ValueError: If the key exists but is None
+        """
+        if not hasattr(self, key_name):
+            if "ollama" in key_name.lower():
+                return "ollama"
+            raise AttributeError(
+                f"Secret key '{key_name}' not found in settings"
+            )
+
+        secret = getattr(self, key_name)
+        if secret is None:
+            # Special case for Ollama - return "ollama" even if key exists but is None
+            if "ollama" in key_name.lower():
+                return "ollama"
+            raise ValueError(f"Secret key '{key_name}' is not set")
+
+        if isinstance(secret, SecretStr):
+            return secret.get_secret_value()
+
+        return str(secret)
+
+
+# Create a singleton instance
+settings = AppSettings()
+# Store the instance in the class variable for singleton pattern
+AppSettings._instance = settings

lionagi/fields/file.py

@@ -71,7 +71,7 @@ class File(HashableModel):
         header: str | None = None,
         footer: str | None = None,
     ) -> Path:
-        from lionagi.libs.file.create_path import create_path
+        from lionagi.utils import create_path
 
         fp = create_path(
             directory=directory,

lionagi/fields/reason.py

@@ -4,8 +4,8 @@
 
 from pydantic import Field, field_validator
 
-from lionagi.libs.parse.to_num import to_num
 from lionagi.models import FieldModel, HashableModel
+from lionagi.utils import to_num
 
 __all__ = ("Reason",)
 

lionagi/libs/file/concat.py

@@ -1,12 +1,7 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
-# SPDX-License-Identifier: Apache-2.0
-
 from pathlib import Path
 from typing import Any
 
-from lionagi.libs.file.create_path import create_path
-from lionagi.utils import lcall
+from lionagi.utils import create_path, lcall
 
 from .process import dir_to_files
 

lionagi/libs/file/concat_files.py

@@ -1,10 +1,6 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
-# SPDX-License-Identifier: Apache-2.0
-
 from pathlib import Path
 
-from lionagi.libs.file.create_path import create_path
+from lionagi.utils import create_path
 
 from .process import dir_to_files
 

lionagi/libs/file/save.py

@@ -7,7 +7,7 @@ import logging
 from pathlib import Path
 from typing import Any
 
-from .create_path import create_path
+from lionagi.utils import create_path
 
 
 def save_to_file(

lionagi/libs/package/imports.py

@@ -2,184 +2,15 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-import logging
-import shutil
-import subprocess
-import sys
-from collections.abc import Sequence
-from typing import Any
-
-from lionagi.utils import is_import_installed
-
-
-def run_package_manager_command(
-    args: Sequence[str],
-) -> subprocess.CompletedProcess[bytes]:
-    """Run a package manager command, using uv if available, otherwise falling back to pip."""
-    # Check if uv is available in PATH
-    uv_path = shutil.which("uv")
-
-    if uv_path:
-        # Use uv if available
-        try:
-            return subprocess.run(
-                [uv_path] + list(args),
-                check=True,
-                capture_output=True,
-            )
-        except subprocess.CalledProcessError:
-            # If uv fails, fall back to pip
-            print("uv command failed, falling back to pip...")
-
-    # Fall back to pip
-    return subprocess.run(
-        [sys.executable, "-m", "pip"] + list(args),
-        check=True,
-        capture_output=True,
-    )
-
-
-def check_import(
-    package_name: str,
-    module_name: str | None = None,
-    import_name: str | None = None,
-    pip_name: str | None = None,
-    attempt_install: bool = True,
-    error_message: str = "",
-):
-    """
-    Check if a package is installed, attempt to install if not.
-
-    Args:
-        package_name: The name of the package to check.
-        module_name: The specific module to import (if any).
-        import_name: The specific name to import from the module (if any).
-        pip_name: The name to use for pip installation (if different).
-        attempt_install: Whether to attempt installation if not found.
-        error_message: Custom error message to use if package not found.
-
-    Raises:
-        ImportError: If the package is not found and not installed.
-        ValueError: If the import fails after installation attempt.
-    """
-    if not is_import_installed(package_name):
-        if attempt_install:
-            logging.info(
-                f"Package {package_name} not found. Attempting " "to install.",
-            )
-            try:
-                return install_import(
-                    package_name=package_name,
-                    module_name=module_name,
-                    import_name=import_name,
-                    pip_name=pip_name,
-                )
-            except ImportError as e:
-                raise ValueError(
-                    f"Failed to install {package_name}: {e}"
-                ) from e
-        else:
-            logging.info(
-                f"Package {package_name} not found. {error_message}",
-            )
-            raise ImportError(
-                f"Package {package_name} not found. {error_message}",
-            )
-
-    return import_module(
-        package_name=package_name,
-        module_name=module_name,
-        import_name=import_name,
-    )
-
-
-def import_module(
-    package_name: str,
-    module_name: str = None,
-    import_name: str | list = None,
-) -> Any:
-    """
-    Import a module by its path.
-
-    Args:
-        module_path: The path of the module to import.
-
-    Returns:
-        The imported module.
-
-    Raises:
-        ImportError: If the module cannot be imported.
-    """
-    try:
-        full_import_path = (
-            f"{package_name}.{module_name}" if module_name else package_name
-        )
-
-        if import_name:
-            import_name = (
-                [import_name]
-                if not isinstance(import_name, list)
-                else import_name
-            )
-            a = __import__(
-                full_import_path,
-                fromlist=import_name,
-            )
-            if len(import_name) == 1:
-                return getattr(a, import_name[0])
-            return [getattr(a, name) for name in import_name]
-        else:
-            return __import__(full_import_path)
-
-    except ImportError as e:
-        raise ImportError(
-            f"Failed to import module {full_import_path}: {e}"
-        ) from e
-
-
-def install_import(
-    package_name: str,
-    module_name: str | None = None,
-    import_name: str | None = None,
-    pip_name: str | None = None,
-):
-    """
-    Attempt to import a package, installing it if not found.
-
-    Args:
-        package_name: The name of the package to import.
-        module_name: The specific module to import (if any).
-        import_name: The specific name to import from the module (if any).
-        pip_name: The name to use for pip installation (if different).
-
-    Raises:
-        ImportError: If the package cannot be imported or installed.
-        subprocess.CalledProcessError: If pip installation fails.
-    """
-    pip_name = pip_name or package_name
-
-    try:
-        return import_module(
-            package_name=package_name,
-            module_name=module_name,
-            import_name=import_name,
-        )
-    except ImportError:
-        logging.info(f"Installing {pip_name}...")
-        try:
-            run_package_manager_command(["install", pip_name])
-            return import_module(
-                package_name=package_name,
-                module_name=module_name,
-                import_name=import_name,
-            )
-        except subprocess.CalledProcessError as e:
-            raise ImportError(f"Failed to install {pip_name}: {e}") from e
-        except ImportError as e:
-            raise ImportError(
-                f"Failed to import {pip_name} after installation: {e}"
-            ) from e
+from lionagi.utils import (
+    check_import,
+    import_module,
+    install_import,
+    is_import_installed,
+    run_package_manager_command,
+)
 
+# backward compatibility
 
 __all__ = (
     "run_package_manager_command",

lionagi/libs/parse.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from lionagi.libs.schema.as_readable import as_readable
+from lionagi.libs.schema.extract_code_block import extract_code_block
+from lionagi.libs.schema.function_to_schema import function_to_schema
+from lionagi.libs.validate.fuzzy_match_keys import fuzzy_match_keys
+from lionagi.libs.validate.fuzzy_validate_mapping import fuzzy_validate_mapping
+from lionagi.libs.validate.string_similarity import string_similarity
+from lionagi.utils import fuzzy_parse_json, to_dict, to_json, to_num
+
+validate_keys = fuzzy_match_keys  # for backward compatibility
+validate_mapping = fuzzy_validate_mapping  # for backward compatibility
+
+
+__all__ = (
+    "as_readable",
+    "extract_code_block",
+    "function_to_schema",
+    "fuzzy_match_keys",
+    "fuzzy_validate_mapping",
+    "string_similarity",
+    "validate_keys",
+    "validate_mapping",
+    "to_dict",
+    "to_json",
+    "to_num",
+    "fuzzy_parse_json",
+)

lionagi/libs/schema/load_pydantic_model_from_schema.py

@@ -0,0 +1,259 @@
+# Copyright (c) 2025, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import importlib.util
+import json
+import string
+import tempfile
+from pathlib import Path
+from typing import Any, TypeVar
+
+from pydantic import BaseModel, PydanticUserError
+
+from lionagi.utils import is_import_installed
+
+_HAS_DATAMODEL_CODE_GENERATOR = is_import_installed("datamodel-code-generator")
+
+# Import at module level for easier mocking in tests
+if _HAS_DATAMODEL_CODE_GENERATOR:
+    from datamodel_code_generator import (
+        DataModelType,
+        InputFileType,
+        PythonVersion,
+        generate,
+    )
+else:
+    # Create dummy objects for when package is not installed
+    DataModelType = None
+    InputFileType = None
+    PythonVersion = None
+    generate = None
+
+
+B = TypeVar("B", bound=BaseModel)
+
+
+def load_pydantic_model_from_schema(
+    schema: str | dict[str, Any],
+    model_name: str = "DynamicModel",
+    /,
+    pydantic_version=None,
+    python_version=None,
+) -> type[BaseModel]:
+    """
+    Generates a Pydantic model class dynamically from a JSON schema string or dict,
+    and ensures it's fully resolved using model_rebuild() with the correct namespace.
+
+    Args:
+        schema: The JSON schema as a string or a Python dictionary.
+        model_name: The desired base name for the generated Pydantic model.
+            If the schema has a 'title', that will likely be used.
+        pydantic_version: The Pydantic model type to generate.
+        python_version: The target Python version for generated code syntax.
+
+    Returns:
+        The dynamically created and resolved Pydantic BaseModel class.
+
+    Raises:
+        ValueError: If the schema is invalid.
+        FileNotFoundError: If the generated model file is not found.
+        AttributeError: If the expected model class cannot be found.
+        RuntimeError: For errors during generation, loading, or rebuilding.
+        Exception: For other potential errors.
+    """
+    if not _HAS_DATAMODEL_CODE_GENERATOR:
+        error_msg = "`datamodel-code-generator` is not installed. Please install with `pip install datamodel-code-generator`."
+        raise ImportError(error_msg)
+
+    if DataModelType is not None:
+        pydantic_version = (
+            pydantic_version or DataModelType.PydanticV2BaseModel
+        )
+        python_version = python_version or PythonVersion.PY_312
+    else:
+        # These won't be used since we'll raise ImportError above
+        pydantic_version = None
+        python_version = None
+
+    schema_input_data: str
+    schema_dict: dict[str, Any]
+    resolved_model_name = (
+        model_name  # Keep track of the potentially updated name
+    )
+
+    # --- 1. Prepare Schema Input ---
+    if isinstance(schema, dict):
+        try:
+            model_name_from_title = schema.get("title")
+            if model_name_from_title and isinstance(
+                model_name_from_title, str
+            ):
+                valid_chars = string.ascii_letters + string.digits + "_"
+                sanitized_title = "".join(
+                    c
+                    for c in model_name_from_title.replace(" ", "")
+                    if c in valid_chars
+                )
+                if sanitized_title and sanitized_title[0].isalpha():
+                    resolved_model_name = (
+                        sanitized_title  # Update the name to use
+                    )
+            schema_dict = schema
+            schema_input_data = json.dumps(schema)
+        except TypeError as e:
+            error_msg = "Invalid dictionary provided for schema"
+            raise ValueError(error_msg) from e
+    elif isinstance(schema, str):
+        try:
+            schema_dict = json.loads(schema)
+            model_name_from_title = schema_dict.get("title")
+            if model_name_from_title and isinstance(
+                model_name_from_title, str
+            ):
+                valid_chars = string.ascii_letters + string.digits + "_"
+                sanitized_title = "".join(
+                    c
+                    for c in model_name_from_title.replace(" ", "")
+                    if c in valid_chars
+                )
+                if sanitized_title and sanitized_title[0].isalpha():
+                    resolved_model_name = (
+                        sanitized_title  # Update the name to use
+                    )
+            schema_input_data = schema
+        except json.JSONDecodeError as e:
+            error_msg = "Invalid JSON schema string provided"
+            raise ValueError(error_msg) from e
+    else:
+        error_msg = "Schema must be a JSON string or a dictionary."
+        raise TypeError(error_msg)
+
+    # --- 2. Generate Code to Temporary File ---
+    with tempfile.TemporaryDirectory() as temporary_directory_name:
+        temporary_directory = Path(temporary_directory_name)
+        # Use a predictable but unique-ish filename
+        output_file = (
+            temporary_directory
+            / f"{resolved_model_name.lower()}_model_{hash(schema_input_data)}.py"
+        )
+        module_name = output_file.stem  # e.g., "userprofile_model_12345"
+
+        try:
+            generate(
+                schema_input_data,
+                input_file_type=InputFileType.JsonSchema,
+                input_filename="schema.json",
+                output=output_file,
+                output_model_type=pydantic_version,
+                target_python_version=python_version,
+                # Ensure necessary base models are imported in the generated code
+                base_class="pydantic.BaseModel",
+            )
+        except Exception as e:
+            # Optional: Print generated code on failure for debugging
+            # if output_file.exists():
+            #     print(f"--- Generated Code (Error) ---\n{output_file.read_text()}\n--------------------------")
+            error_msg = "Failed to generate model code"
+            raise RuntimeError(error_msg) from e
+
+        if not output_file.exists():
+            error_msg = f"Generated model file was not created: {output_file}"
+            raise FileNotFoundError(error_msg)
+
+        def get_modules():
+            spec = importlib.util.spec_from_file_location(
+                module_name, str(output_file)
+            )
+
+            if spec is None or spec.loader is None:
+                error_msg = f"Could not create module spec for {output_file}"
+                raise ImportError(error_msg)
+
+            return spec, importlib.util.module_from_spec(spec)
+
+        # --- 3. Import the Generated Module Dynamically ---
+        try:
+            spec, generated_module = get_modules()
+            # Important: Make pydantic available within the executed module's globals
+            # if it's not explicitly imported by the generated code for some reason.
+            # Usually, datamodel-code-generator handles imports well.
+            # generated_module.__dict__['BaseModel'] = BaseModel
+            spec.loader.exec_module(generated_module)
+
+        except Exception as e:
+            # Optional: Print generated code on failure for debugging
+            # print(f"--- Generated Code (Import Error) ---\n{output_file.read_text()}\n--------------------------")
+            error_msg = f"Failed to load generated module ({output_file})"
+            raise RuntimeError(error_msg) from e
+
+        def validate_base_model_class(m):
+            if not isinstance(m, type) or not issubclass(m, BaseModel):
+                error_msg = f"Found attribute '{resolved_model_name}' is not a Pydantic BaseModel class."
+                raise TypeError(error_msg)
+
+        # --- 4. Find the Model Class ---
+        model_class: type[BaseModel]
+        try:
+            # Use the name potentially derived from the schema title
+            model_class = getattr(generated_module, resolved_model_name)
+            validate_base_model_class(model_class)
+
+        except AttributeError:
+            # Fallback attempt (less likely now with title extraction)
+            try:
+                model_class = generated_module.Model  # Default fallback name
+                validate_base_model_class(model_class)
+                print(
+                    f"Warning: Model name '{resolved_model_name}' not found, falling back to 'Model'."
+                )
+            except AttributeError as e:
+                # List available Pydantic models found in the module for debugging
+                available_attrs = [
+                    attr
+                    for attr in dir(generated_module)
+                    if isinstance(getattr(generated_module, attr, None), type)
+                    and issubclass(
+                        getattr(generated_module, attr, object), BaseModel
+                    )  # Check inheritance safely
+                    and getattr(generated_module, attr, None)
+                    is not BaseModel  # Exclude BaseModel itself
+                ]
+                # Optional: Print generated code on failure for debugging
+                # print(f"--- Generated Code (AttributeError) ---\n{output_file.read_text()}\n--------------------------")
+                error_msg = (
+                    f"Could not find expected model class '{resolved_model_name}' or fallback 'Model' "
+                    f"in the generated module {output_file}. "
+                    f"Found Pydantic models: {available_attrs}"
+                )
+                raise AttributeError(error_msg) from e
+        except TypeError as e:
+            error_msg = (
+                f"Error validating found model class '{resolved_model_name}'"
+            )
+            raise TypeError(error_msg) from e
+
+        # --- 5. Rebuild the Model (Providing Namespace) ---
+        try:
+            # Pass the generated module's dictionary as the namespace
+            # for resolving type hints like 'Status', 'ProfileDetails', etc.
+            model_class.model_rebuild(
+                _types_namespace=generated_module.__dict__,
+                force=True,  # Force rebuild even if Pydantic thinks it's okay
+            )
+        except (
+            PydanticUserError,
+            NameError,
+        ) as e:  # Catch NameError explicitly here
+            # Optional: Print generated code on failure for debugging
+            # print(f"--- Generated Code (Rebuild Error) ---\n{output_file.read_text()}\n--------------------------")
+            error_msg = f"Error during model_rebuild for {resolved_model_name}"
+            raise RuntimeError(error_msg) from e
+        except Exception as e:
+            # Optional: Print generated code on failure for debugging
+            # print(f"--- Generated Code (Rebuild Error) ---\n{output_file.read_text()}\n--------------------------")
+            error_msg = f"Unexpected error during model_rebuild for {resolved_model_name}"
+            raise RuntimeError(error_msg) from e
+
+        # --- 6. Return the Resolved Model Class ---
+        return model_class

lionagi/libs/token_transform/perplexity.py

@@ -7,7 +7,7 @@ from pydantic import BaseModel
 
 from lionagi.protocols.generic.event import EventStatus
 from lionagi.protocols.generic.log import Log
-from lionagi.service.endpoints.base import APICalling
+from lionagi.service.connections.api_calling import APICalling
 from lionagi.service.imodel import iModel
 from lionagi.utils import alcall, lcall, to_dict, to_list
 
@@ -208,9 +208,7 @@ class LLMCompressor:
         Tokenize text. If no custom tokenizer, use the default from lionagi.
         """
         if not self.tokenizer:
-            from lionagi.service.endpoints.token_calculator import (
-                TokenCalculator,
-            )
+            from lionagi.service.token_calculator import TokenCalculator
 
             return TokenCalculator.tokenize(
                 text,