lionagi 0.12.3__py3-none-any.whl → 0.12.4__py3-none-any.whl

lionagi/config.py

@@ -0,0 +1,123 @@
+# Copyright (c) 2025, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, Field, SecretStr
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class CacheConfig(BaseModel):
+    """Configuration for aiocache."""
+
+    ttl: int = 300
+    key: str | None = None
+    namespace: str | None = None
+    key_builder: Any = None
+    skip_cache_func: Any = lambda _: False
+    serializer: dict[str, Any] | None = None
+    plugins: Any = None
+    alias: str | None = None
+    noself: Any = lambda _: False
+
+    def as_kwargs(self) -> dict[str, Any]:
+        """Convert config to kwargs dict for @cached decorator.
+
+        Removes unserialisable callables that aiocache can't pickle.
+        """
+        raw = self.model_dump(exclude_none=True)
+        # Remove all unserialisable callables
+        unserialisable_keys = (
+            "key_builder",
+            "skip_cache_func",
+            "noself",
+            "serializer",
+            "plugins",
+        )
+        for key in unserialisable_keys:
+            raw.pop(key, None)
+        return raw
+
+
+class AppSettings(BaseSettings, frozen=True):
+    """Application settings with environment variable support."""
+
+    model_config = SettingsConfigDict(
+        env_file=(".env", ".env.local", ".secrets.env"),
+        env_file_encoding="utf-8",
+        case_sensitive=False,
+        extra="ignore",
+    )
+
+    aiocache_config: CacheConfig = Field(
+        default_factory=CacheConfig, description="Cache settings for aiocache"
+    )
+
+    # secrets
+    OPENAI_API_KEY: SecretStr | None = None
+    OPENROUTER_API_KEY: SecretStr | None = None
+    OLLAMA_API_KEY: SecretStr | None = None
+    EXA_API_KEY: SecretStr | None = None
+    PERPLEXITY_API_KEY: SecretStr | None = None
+    GROQ_API_KEY: SecretStr | None = None
+    ANTHROPIC_API_KEY: SecretStr | None = None
+
+    # defaults models
+    LIONAGI_EMBEDDING_PROVIDER: str = "openai"
+    LIONAGI_EMBEDDING_MODEL: str = "text-embedding-3-small"
+
+    LIONAGI_CHAT_PROVIDER: str = "openai"
+    LIONAGI_CHAT_MODEL: str = "gpt-4o"
+
+    # default storage
+    LIONAGI_AUTO_STORE_EVENT: bool = False
+    LIONAGI_STORAGE_PROVIDER: str = "async_qdrant"
+
+    LIONAGI_AUTO_EMBED_LOG: bool = False
+
+    # specific storage
+    LIONAGI_QDRANT_URL: str = "http://localhost:6333"
+    LIONAGI_DEFAULT_QDRANT_COLLECTION: str = "event_logs"
+
+    # Class variable to store the singleton instance
+    _instance: ClassVar[Any] = None
+
+    def get_secret(self, key_name: str) -> str:
+        """
+        Get the secret value for a given key name.
+
+        Args:
+            key_name: The name of the secret key (e.g., "OPENAI_API_KEY")
+
+        Returns:
+            The secret value as a string
+
+        Raises:
+            AttributeError: If the key doesn't exist
+            ValueError: If the key exists but is None
+        """
+        if not hasattr(self, key_name):
+            if "ollama" in key_name.lower():
+                return "ollama"
+            raise AttributeError(
+                f"Secret key '{key_name}' not found in settings"
+            )
+
+        secret = getattr(self, key_name)
+        if secret is None:
+            # Special case for Ollama - return "ollama" even if key exists but is None
+            if "ollama" in key_name.lower():
+                return "ollama"
+            raise ValueError(f"Secret key '{key_name}' is not set")
+
+        if isinstance(secret, SecretStr):
+            return secret.get_secret_value()
+
+        return str(secret)
+
+
+# Create a singleton instance
+settings = AppSettings()
+# Store the instance in the class variable for singleton pattern
+AppSettings._instance = settings

lionagi/libs/schema/load_pydantic_model_from_schema.py

@@ -0,0 +1,259 @@
+# Copyright (c) 2025, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import importlib.util
+import json
+import string
+import tempfile
+from pathlib import Path
+from typing import Any, TypeVar
+
+from pydantic import BaseModel, PydanticUserError
+
+from lionagi.utils import is_import_installed
+
+_HAS_DATAMODEL_CODE_GENERATOR = is_import_installed("datamodel-code-generator")
+
+# Import at module level for easier mocking in tests
+if _HAS_DATAMODEL_CODE_GENERATOR:
+    from datamodel_code_generator import (
+        DataModelType,
+        InputFileType,
+        PythonVersion,
+        generate,
+    )
+else:
+    # Create dummy objects for when package is not installed
+    DataModelType = None
+    InputFileType = None
+    PythonVersion = None
+    generate = None
+
+
+B = TypeVar("B", bound=BaseModel)
+
+
+def load_pydantic_model_from_schema(
+    schema: str | dict[str, Any],
+    model_name: str = "DynamicModel",
+    /,
+    pydantic_version=None,
+    python_version=None,
+) -> type[BaseModel]:
+    """
+    Generates a Pydantic model class dynamically from a JSON schema string or dict,
+    and ensures it's fully resolved using model_rebuild() with the correct namespace.
+
+    Args:
+        schema: The JSON schema as a string or a Python dictionary.
+        model_name: The desired base name for the generated Pydantic model.
+            If the schema has a 'title', that will likely be used.
+        pydantic_version: The Pydantic model type to generate.
+        python_version: The target Python version for generated code syntax.
+
+    Returns:
+        The dynamically created and resolved Pydantic BaseModel class.
+
+    Raises:
+        ValueError: If the schema is invalid.
+        FileNotFoundError: If the generated model file is not found.
+        AttributeError: If the expected model class cannot be found.
+        RuntimeError: For errors during generation, loading, or rebuilding.
+        Exception: For other potential errors.
+    """
+    if not _HAS_DATAMODEL_CODE_GENERATOR:
+        error_msg = "`datamodel-code-generator` is not installed. Please install with `pip install datamodel-code-generator`."
+        raise ImportError(error_msg)
+
+    if DataModelType is not None:
+        pydantic_version = (
+            pydantic_version or DataModelType.PydanticV2BaseModel
+        )
+        python_version = python_version or PythonVersion.PY_312
+    else:
+        # These won't be used since we'll raise ImportError above
+        pydantic_version = None
+        python_version = None
+
+    schema_input_data: str
+    schema_dict: dict[str, Any]
+    resolved_model_name = (
+        model_name  # Keep track of the potentially updated name
+    )
+
+    # --- 1. Prepare Schema Input ---
+    if isinstance(schema, dict):
+        try:
+            model_name_from_title = schema.get("title")
+            if model_name_from_title and isinstance(
+                model_name_from_title, str
+            ):
+                valid_chars = string.ascii_letters + string.digits + "_"
+                sanitized_title = "".join(
+                    c
+                    for c in model_name_from_title.replace(" ", "")
+                    if c in valid_chars
+                )
+                if sanitized_title and sanitized_title[0].isalpha():
+                    resolved_model_name = (
+                        sanitized_title  # Update the name to use
+                    )
+            schema_dict = schema
+            schema_input_data = json.dumps(schema)
+        except TypeError as e:
+            error_msg = "Invalid dictionary provided for schema"
+            raise ValueError(error_msg) from e
+    elif isinstance(schema, str):
+        try:
+            schema_dict = json.loads(schema)
+            model_name_from_title = schema_dict.get("title")
+            if model_name_from_title and isinstance(
+                model_name_from_title, str
+            ):
+                valid_chars = string.ascii_letters + string.digits + "_"
+                sanitized_title = "".join(
+                    c
+                    for c in model_name_from_title.replace(" ", "")
+                    if c in valid_chars
+                )
+                if sanitized_title and sanitized_title[0].isalpha():
+                    resolved_model_name = (
+                        sanitized_title  # Update the name to use
+                    )
+            schema_input_data = schema
+        except json.JSONDecodeError as e:
+            error_msg = "Invalid JSON schema string provided"
+            raise ValueError(error_msg) from e
+    else:
+        error_msg = "Schema must be a JSON string or a dictionary."
+        raise TypeError(error_msg)
+
+    # --- 2. Generate Code to Temporary File ---
+    with tempfile.TemporaryDirectory() as temporary_directory_name:
+        temporary_directory = Path(temporary_directory_name)
+        # Use a predictable but unique-ish filename
+        output_file = (
+            temporary_directory
+            / f"{resolved_model_name.lower()}_model_{hash(schema_input_data)}.py"
+        )
+        module_name = output_file.stem  # e.g., "userprofile_model_12345"
+
+        try:
+            generate(
+                schema_input_data,
+                input_file_type=InputFileType.JsonSchema,
+                input_filename="schema.json",
+                output=output_file,
+                output_model_type=pydantic_version,
+                target_python_version=python_version,
+                # Ensure necessary base models are imported in the generated code
+                base_class="pydantic.BaseModel",
+            )
+        except Exception as e:
+            # Optional: Print generated code on failure for debugging
+            # if output_file.exists():
+            #     print(f"--- Generated Code (Error) ---\n{output_file.read_text()}\n--------------------------")
+            error_msg = "Failed to generate model code"
+            raise RuntimeError(error_msg) from e
+
+        if not output_file.exists():
+            error_msg = f"Generated model file was not created: {output_file}"
+            raise FileNotFoundError(error_msg)
+
+        def get_modules():
+            spec = importlib.util.spec_from_file_location(
+                module_name, str(output_file)
+            )
+
+            if spec is None or spec.loader is None:
+                error_msg = f"Could not create module spec for {output_file}"
+                raise ImportError(error_msg)
+
+            return spec, importlib.util.module_from_spec(spec)
+
+        # --- 3. Import the Generated Module Dynamically ---
+        try:
+            spec, generated_module = get_modules()
+            # Important: Make pydantic available within the executed module's globals
+            # if it's not explicitly imported by the generated code for some reason.
+            # Usually, datamodel-code-generator handles imports well.
+            # generated_module.__dict__['BaseModel'] = BaseModel
+            spec.loader.exec_module(generated_module)
+
+        except Exception as e:
+            # Optional: Print generated code on failure for debugging
+            # print(f"--- Generated Code (Import Error) ---\n{output_file.read_text()}\n--------------------------")
+            error_msg = f"Failed to load generated module ({output_file})"
+            raise RuntimeError(error_msg) from e
+
+        def validate_base_model_class(m):
+            if not isinstance(m, type) or not issubclass(m, BaseModel):
+                error_msg = f"Found attribute '{resolved_model_name}' is not a Pydantic BaseModel class."
+                raise TypeError(error_msg)
+
+        # --- 4. Find the Model Class ---
+        model_class: type[BaseModel]
+        try:
+            # Use the name potentially derived from the schema title
+            model_class = getattr(generated_module, resolved_model_name)
+            validate_base_model_class(model_class)
+
+        except AttributeError:
+            # Fallback attempt (less likely now with title extraction)
+            try:
+                model_class = generated_module.Model  # Default fallback name
+                validate_base_model_class(model_class)
+                print(
+                    f"Warning: Model name '{resolved_model_name}' not found, falling back to 'Model'."
+                )
+            except AttributeError as e:
+                # List available Pydantic models found in the module for debugging
+                available_attrs = [
+                    attr
+                    for attr in dir(generated_module)
+                    if isinstance(getattr(generated_module, attr, None), type)
+                    and issubclass(
+                        getattr(generated_module, attr, object), BaseModel
+                    )  # Check inheritance safely
+                    and getattr(generated_module, attr, None)
+                    is not BaseModel  # Exclude BaseModel itself
+                ]
+                # Optional: Print generated code on failure for debugging
+                # print(f"--- Generated Code (AttributeError) ---\n{output_file.read_text()}\n--------------------------")
+                error_msg = (
+                    f"Could not find expected model class '{resolved_model_name}' or fallback 'Model' "
+                    f"in the generated module {output_file}. "
+                    f"Found Pydantic models: {available_attrs}"
+                )
+                raise AttributeError(error_msg) from e
+        except TypeError as e:
+            error_msg = (
+                f"Error validating found model class '{resolved_model_name}'"
+            )
+            raise TypeError(error_msg) from e
+
+        # --- 5. Rebuild the Model (Providing Namespace) ---
+        try:
+            # Pass the generated module's dictionary as the namespace
+            # for resolving type hints like 'Status', 'ProfileDetails', etc.
+            model_class.model_rebuild(
+                _types_namespace=generated_module.__dict__,
+                force=True,  # Force rebuild even if Pydantic thinks it's okay
+            )
+        except (
+            PydanticUserError,
+            NameError,
+        ) as e:  # Catch NameError explicitly here
+            # Optional: Print generated code on failure for debugging
+            # print(f"--- Generated Code (Rebuild Error) ---\n{output_file.read_text()}\n--------------------------")
+            error_msg = f"Error during model_rebuild for {resolved_model_name}"
+            raise RuntimeError(error_msg) from e
+        except Exception as e:
+            # Optional: Print generated code on failure for debugging
+            # print(f"--- Generated Code (Rebuild Error) ---\n{output_file.read_text()}\n--------------------------")
+            error_msg = f"Unexpected error during model_rebuild for {resolved_model_name}"
+            raise RuntimeError(error_msg) from e
+
+        # --- 6. Return the Resolved Model Class ---
+        return model_class

lionagi/libs/token_transform/perplexity.py

@@ -7,7 +7,7 @@ from pydantic import BaseModel
 
 from lionagi.protocols.generic.event import EventStatus
 from lionagi.protocols.generic.log import Log
-from lionagi.service.endpoints.base import APICalling
+from lionagi.service.connections.api_calling import APICalling
 from lionagi.service.imodel import iModel
 from lionagi.utils import alcall, lcall, to_dict, to_list
 
@@ -208,9 +208,7 @@ class LLMCompressor:
         Tokenize text. If no custom tokenizer, use the default from lionagi.
         """
         if not self.tokenizer:
-            from lionagi.service.endpoints.token_calculator import (
-                TokenCalculator,
-            )
+            from lionagi.service.token_calculator import TokenCalculator
 
             return TokenCalculator.tokenize(
                 text,

lionagi/libs/token_transform/synthlang_/translate_to_synthlang.py

@@ -93,7 +93,7 @@ async def translate_to_synthlang(
     else:
         branch = Branch(system=final_prompt, chat_model=chat_model)
 
-    from lionagi.service.endpoints.token_calculator import TokenCalculator
+    from lionagi.service.token_calculator import TokenCalculator
 
     calculator = TokenCalculator()
 

lionagi/operations/chat/chat.py

@@ -122,8 +122,8 @@ async def chat(
                     _msgs.append(i)
         messages = _msgs
 
-    imodel = imodel or branch.chat_model
-    if branch.msgs.system and imodel.sequential_exchange:
+    # All endpoints now assume sequential exchange (system message embedded in first user message)
+    if branch.msgs.system:
         messages = [msg for msg in messages if msg.role != "system"]
         first_instruction = None
 

lionagi/operations/communicate/communicate.py

@@ -91,12 +91,27 @@ async def communicate(
         return res.response
 
     if response_format is not None:
-        return await branch.parse(
-            text=res.response,
-            request_type=response_format,
-            max_retries=num_parse_retries,
+        # Default to raising errors unless explicitly set in fuzzy_match_kwargs
+        parse_kwargs = {
+            "handle_validation": "raise",  # Default to raising errors
             **(fuzzy_match_kwargs or {}),
-        )
+        }
+
+        try:
+            return await branch.parse(
+                text=res.response,
+                request_type=response_format,
+                max_retries=num_parse_retries,
+                **parse_kwargs,
+            )
+        except ValueError as e:
+            # Re-raise with more context
+            logging.error(
+                f"Failed to parse response '{res.response}' into {response_format}: {e}"
+            )
+            raise ValueError(
+                f"Failed to parse model response into {response_format.__name__}: {e}"
+            ) from e
 
     if request_fields is not None:
         _d = fuzzy_validate_mapping(

lionagi/operations/parse/parse.py

@@ -35,35 +35,117 @@ async def parse(
     suppress_conversion_errors: bool = False,
     response_format=None,
 ):
-    _should_try = True
-    num_try = 0
-    response_model = text
     if operative is not None:
         max_retries = operative.max_retries
-        response_format = operative.request_type
+        response_format = operative.request_type or response_format
+        request_type = request_type or operative.request_type
 
-    while (
-        _should_try
-        and num_try < max_retries
-        and not isinstance(response_model, BaseModel)
-    ):
-        num_try += 1
-        if num_try == max_retries:
-            _should_try = False
-        _, res = await branch.chat(
-            instruction="reformat text into specified model",
-            guidane="follow the required response format, using the model schema as a guide",
-            context=[{"text_to_format": text}],
-            response_format=response_format or request_type,
-            sender=branch.user,
-            recipient=branch.id,
-            imodel=branch.parse_model,
-            return_ins_res_message=True,
+    if not request_type and not response_format:
+        raise ValueError(
+            "Either request_type or response_format must be provided"
+        )
+
+    request_type = request_type or response_format
+
+    # First attempt: try to parse the text directly
+    import logging
+
+    initial_error = None
+    parsed_data = None  # Initialize to avoid scoping issues
+
+    try:
+        # Try fuzzy validation first
+        parsed_data = fuzzy_validate_mapping(
+            text,
+            breakdown_pydantic_annotation(request_type),
+            similarity_algo=similarity_algo,
+            similarity_threshold=similarity_threshold,
+            fuzzy_match=fuzzy_match,
+            handle_unmatched=handle_unmatched,
+            fill_value=fill_value,
+            fill_mapping=fill_mapping,
+            strict=strict,
+            suppress_conversion_errors=False,  # Don't suppress on first attempt
         )
+
+        logging.debug(f"Parsed data from fuzzy validation: {parsed_data}")
+
+        # Validate with pydantic
         if operative is not None:
-            response_model = operative.update_response_model(res.response)
+            response_model = operative.update_response_model(parsed_data)
         else:
-            response_model = fuzzy_validate_mapping(
+            response_model = request_type.model_validate(parsed_data)
+
+        # If successful, return immediately
+        if isinstance(response_model, BaseModel):
+            return response_model
+
+    except Exception as e:
+        initial_error = e
+        # Log the initial parsing error for debugging
+        logging.debug(
+            f"Initial parsing failed for text '{text[:100]}...': {e}"
+        )
+        logging.debug(
+            f"Parsed data was: {locals().get('parsed_data', 'not set')}"
+        )
+
+        # Only continue if we have retries left
+        if max_retries <= 0:
+            if handle_validation == "raise":
+                raise ValueError(f"Failed to parse response: {e}") from e
+            elif handle_validation == "return_none":
+                return None
+            else:  # return_value
+                return text
+
+    # If direct parsing failed, try using the parse model
+    num_try = 0
+    last_error = initial_error
+
+    # Check if the parsed_data exists but just failed validation
+    # This might mean we have the right structure but wrong values
+    if parsed_data is not None and isinstance(parsed_data, dict):
+        logging.debug(
+            f"Have parsed_data dict, checking if it's close to valid..."
+        )
+        # If we got a dict with the right keys, maybe we just need to clean it up
+        expected_fields = set(request_type.model_fields.keys())
+        parsed_fields = set(parsed_data.keys())
+        if expected_fields == parsed_fields and all(
+            parsed_data.get(k) is not None for k in expected_fields
+        ):
+            # We have the right structure with non-None values, don't retry with parse model
+            logging.debug(
+                "Structure matches with valid values, returning original error"
+            )
+            if handle_validation == "raise":
+                raise ValueError(
+                    f"Failed to parse response: {initial_error}"
+                ) from initial_error
+            elif handle_validation == "return_none":
+                return None
+            else:
+                return text
+
+    while num_try < max_retries:
+        num_try += 1
+
+        try:
+            logging.debug(f"Retry {num_try}: Using parse model to reformat")
+            _, res = await branch.chat(
+                instruction="reformat text into specified model",
+                guidance="follow the required response format, using the model schema as a guide",
+                context=[{"text_to_format": text}],
+                response_format=request_type,
+                sender=branch.user,
+                recipient=branch.id,
+                imodel=branch.parse_model,
+                return_ins_res_message=True,
+            )
+
+            # Try to parse the reformatted response
+            parsed_data = fuzzy_validate_mapping(
                 res.response,
                 breakdown_pydantic_annotation(request_type),
                 similarity_algo=similarity_algo,
@@ -75,25 +157,31 @@ async def parse(
                 strict=strict,
                 suppress_conversion_errors=suppress_conversion_errors,
             )
-            try:
-                response_model = request_type.model_validate(response_model)
-            except InterruptedError as e:
-                raise e
-            except Exception:
-                if _should_try:
-                    continue
-                else:
-                    break
-
-    if not isinstance(response_model, BaseModel):
-        match handle_validation:
-            case "return_value":
+
+            if operative is not None:
+                response_model = operative.update_response_model(parsed_data)
+            else:
+                response_model = request_type.model_validate(parsed_data)
+
+            # If successful, return
+            if isinstance(response_model, BaseModel):
                 return response_model
-            case "return_none":
-                return None
-            case "raise":
-                raise ValueError(
-                    "Failed to parse response into request format"
-                )
 
-    return response_model
+        except InterruptedError as e:
+            raise e
+        except Exception as e:
+            last_error = e
+            # Continue to next retry
+            continue
+
+    # All retries exhausted
+    match handle_validation:
+        case "return_value":
+            return text
+        case "return_none":
+            return None
+        case "raise":
+            error_msg = "Failed to parse response into request format"
+            if last_error:
+                error_msg += f": {last_error}"
+            raise ValueError(error_msg) from last_error

lionagi/protocols/generic/pile.py

@@ -19,7 +19,7 @@ from pathlib import Path
 from typing import Any, ClassVar, Generic, TypeVar
 
 import pandas as pd
-from pydantic import Field, field_serializer
+from pydantic import Field
 from pydantic.fields import FieldInfo
 from typing_extensions import Self, override
 
@@ -910,9 +910,23 @@ class Pile(Element, Collective[E], Generic[E]):
         self.progression.insert(index, item_order)
         self.collections.update(item_dict)
 
-    @field_serializer("collections")
-    def _(self, value: dict[str, T]):
-        return [i.to_dict() for i in value.values()]
+    def to_dict(self) -> dict[str, Any]:
+        """Convert pile to dictionary, properly handling collections."""
+        # Get base dict from parent class
+        dict_ = super().to_dict()
+
+        # Manually serialize collections
+        collections_list = []
+        for item in self.collections.values():
+            if hasattr(item, "to_dict"):
+                collections_list.append(item.to_dict())
+            elif hasattr(item, "model_dump"):
+                collections_list.append(item.model_dump())
+            else:
+                collections_list.append(str(item))
+
+        dict_["collections"] = collections_list
+        return dict_
 
     class AsyncPileIterator:
         def __init__(self, pile: Pile):