lionagi 0.15.3__py3-none-any.whl → 0.15.5__py3-none-any.whl

lionagi/adapters/_utils.py

@@ -0,0 +1,26 @@
+def check_async_postgres_available():
+    try:
+        import sqlalchemy as sa
+        from pydapter.extras.async_postgres_ import AsyncPostgresAdapter
+        from sqlalchemy.ext.asyncio import create_async_engine
+
+        return True
+    except Exception:
+        return ImportError(
+            "This adapter requires postgres option to be installed. "
+            'Please install them using `uv pip install "lionagi[postgres]"`.'
+        )
+
+
+def check_postgres_available():
+    try:
+        from pydapter.model_adapters.postgres_model import PostgresModelAdapter
+        from sqlalchemy import String
+        from sqlalchemy.orm import DeclarativeBase
+
+        return True
+    except Exception:
+        return ImportError(
+            "This adapter requires postgres option to be installed. "
+            'Please install them using `uv pip install "lionagi[postgres]"`.'
+        )

lionagi/adapters/async_postgres_adapter.py

@@ -12,16 +12,16 @@ from typing import Any, ClassVar, TypeVar
 
 from pydapter.exceptions import QueryError
 
-try:
-    import sqlalchemy as sa
-    from pydapter.extras.async_postgres_ import AsyncPostgresAdapter
-    from sqlalchemy.ext.asyncio import create_async_engine
-except ImportError:
-    raise ImportError(
-        "This adapter requires postgres option to be installed. "
-        'Please install them using `uv pip install "lionagi[postgres]"`.'
-    )
+from ._utils import check_async_postgres_available
 
+_ASYNC_POSTGRES_AVAILABLE = check_async_postgres_available()
+
+if isinstance(_ASYNC_POSTGRES_AVAILABLE, ImportError):
+    raise _ASYNC_POSTGRES_AVAILABLE
+
+import sqlalchemy as sa
+from pydapter.extras.async_postgres_ import AsyncPostgresAdapter
+from sqlalchemy.ext.asyncio import create_async_engine
 
 T = TypeVar("T")
 

lionagi/adapters/postgres_model_adapter.py

@@ -7,19 +7,19 @@ PostgreSQL persistence for lionagi Nodes.
 
 from __future__ import annotations
 
-from typing import Any, Union, get_args, get_origin
+from typing import Union, get_args, get_origin
 
 from pydantic import BaseModel
 
-try:
-    from pydapter.model_adapters.postgres_model import PostgresModelAdapter
-    from sqlalchemy import String
-    from sqlalchemy.orm import DeclarativeBase
-except ImportError:
-    raise ImportError(
-        "This adapter requires postgres option to be installed. "
-        'Please install them using `uv pip install "lionagi[postgres]"`.'
-    )
+from ._utils import check_postgres_available
+
+_POSTGRES_AVAILABLE = check_postgres_available()
+if isinstance(_POSTGRES_AVAILABLE, ImportError):
+    raise _POSTGRES_AVAILABLE
+
+from pydapter.model_adapters.postgres_model import PostgresModelAdapter
+from sqlalchemy import String
+from sqlalchemy.orm import DeclarativeBase
 
 
 class LionAGIPostgresAdapter(PostgresModelAdapter):

lionagi/protocols/generic/pile.py

@@ -16,17 +16,23 @@ from collections.abc import (
 )
 from functools import wraps
 from pathlib import Path
-from typing import Any, Generic, TypeVar
+from typing import Any, ClassVar, Generic, Literal, TypeVar
 
 import pandas as pd
-from pydantic import Field
+from pydantic import Field, field_serializer, field_validator, model_validator
 from pydantic.fields import FieldInfo
 from pydapter import Adaptable, AsyncAdaptable
-from typing_extensions import Self, override
+from typing_extensions import Self, deprecated, override
 
-from lionagi._errors import ItemExistsError, ItemNotFoundError
+from lionagi._errors import ItemExistsError, ItemNotFoundError, ValidationError
 from lionagi.libs.concurrency import Lock as ConcurrencyLock
-from lionagi.utils import UNDEFINED, is_same_dtype, to_list
+from lionagi.utils import (
+    UNDEFINED,
+    is_same_dtype,
+    is_union_type,
+    to_list,
+    union_members,
+)
 
 from .._concepts import Observable
 from .element import ID, Collective, E, Element, IDType, validate_order
@@ -35,6 +41,8 @@ from .progression import Progression
 D = TypeVar("D")
 T = TypeVar("T", bound=E)
 
+_ADAPATER_REGISTERED = False
+
 
 def synchronized(func: Callable):
     @wraps(func)
@@ -54,6 +62,125 @@ def async_synchronized(func: Callable):
     return wrapper
 
 
+def _validate_item_type(value, /) -> set[type[T]] | None:
+    if value is None:
+        return None
+
+    value = to_list_type(value)
+    out = set()
+
+    from lionagi.utils import import_module
+
+    for i in value:
+        subcls = i
+        if isinstance(i, str):
+            try:
+                mod, imp = i.rsplit(".", 1)
+                subcls = import_module(mod, import_name=imp)
+            except Exception as e:
+                raise ValidationError.from_value(
+                    i,
+                    expected="A subclass of Observable.",
+                    cause=e,
+                ) from e
+        if isinstance(subcls, type):
+            if is_union_type(subcls):
+                members = union_members(subcls)
+                for m in members:
+                    if not issubclass(m, Observable):
+                        raise ValidationError.from_value(
+                            m, expected="A subclass of Observable."
+                        )
+                    out.add(m)
+            elif not issubclass(subcls, Observable):
+                raise ValidationError.from_value(
+                    subcls, expected="A subclass of Observable."
+                )
+            else:
+                out.add(subcls)
+        else:
+            raise ValidationError.from_value(
+                i, expected="A subclass of Observable."
+            )
+
+    if len(value) != len(set(value)):
+        raise ValidationError("Detected duplicated item types in item_type.")
+
+    if len(value) > 0:
+        return out
+
+
+def _validate_progression(
+    value: Any, collections: dict[IDType, T], /
+) -> Progression:
+    if not value:
+        return Progression(order=list(collections.keys()))
+
+    prog = None
+    if isinstance(value, dict):
+        try:
+            prog = Progression.from_dict(value)
+            value = list(prog)
+        except Exception:
+            # If we can't create Progression from dict, try to extract order field
+            value = to_list_type(value.get("order", []))
+    elif isinstance(value, Progression):
+        prog = value
+        value = list(prog)
+    else:
+        value = to_list_type(value)
+
+    value_set = set(value)
+    if len(value_set) != len(value):
+        raise ValueError("There are duplicate elements in the order")
+    if len(value_set) != len(collections.keys()):
+        raise ValueError(
+            "The length of the order does not match the length of the pile"
+        )
+
+    for i in value_set:
+        if ID.get_id(i) not in collections.keys():
+            raise ValueError(
+                f"The order does not match the pile. {i} not found"
+            )
+    return prog or Progression(order=value)
+
+
+def _validate_collections(
+    value: Any, item_type: set | None, strict_type: bool, /
+) -> dict[str, T]:
+    if not value:
+        return {}
+
+    value = to_list_type(value)
+
+    result = {}
+    for i in value:
+        if isinstance(i, dict):
+            i = Element.from_dict(i)
+
+        if item_type:
+            if strict_type:
+                if type(i) not in item_type:
+                    raise ValidationError.from_value(
+                        i,
+                        expected=f"One of {item_type}, no subclasses allowed.",
+                    )
+            else:
+                if not any(issubclass(type(i), t) for t in item_type):
+                    raise ValidationError.from_value(
+                        i,
+                        expected=f"One of {item_type} or the subclasses",
+                    )
+        else:
+            if not isinstance(i, Observable):
+                raise ValueError(f"Invalid pile item {i}")
+
+        result[i.id] = i
+
+    return result
+
+
 class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
     """Thread-safe async-compatible, ordered collection of elements.
 
@@ -79,7 +206,6 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
     progression: Progression = Field(
         default_factory=Progression,
         description="Progression specifying the order of items in the pile.",
-        exclude=True,
     )
     strict_type: bool = Field(
         default=False,
@@ -87,6 +213,13 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
         frozen=True,
     )
 
+    _EXTRA_FIELDS: ClassVar[set[str]] = {
+        "collections",
+        "item_type",
+        "progression",
+        "strict_type",
+    }
+
     def __pydantic_extra__(self) -> dict[str, FieldInfo]:
         return {
             "_lock": Field(default_factory=threading.Lock),
@@ -96,6 +229,29 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
     def __pydantic_private__(self) -> dict[str, FieldInfo]:
         return self.__pydantic_extra__()
 
+    @classmethod
+    def _validate_before(cls, data: dict[str, Any]) -> dict[str, Any]:
+        item_type = _validate_item_type(data.get("item_type"))
+        strict_type = data.get("strict_type", False)
+        collections = _validate_collections(
+            data.get("collections"), item_type, strict_type
+        )
+        progression = None
+        if "order" in data:
+            progression = _validate_progression(data["order"], collections)
+        else:
+            progression = _validate_progression(
+                data.get("progression"), collections
+            )
+
+        return {
+            "collections": collections,
+            "item_type": item_type,
+            "progression": progression,
+            "strict_type": strict_type,
+            **{k: v for k, v in data.items() if k not in cls._EXTRA_FIELDS},
+        }
+
     @override
     def __init__(
         self,
@@ -113,22 +269,33 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
             order: Initial order of items (as Progression).
             strict_type: If True, enforce strict type checking.
         """
-        _config = {}
-        if "id" in kwargs:
-            _config["id"] = kwargs["id"]
-        if "created_at" in kwargs:
-            _config["created_at"] = kwargs["created_at"]
-
-        super().__init__(strict_type=strict_type, **_config)
-        self.item_type = self._validate_item_type(item_type)
-
-        if isinstance(collections, list) and is_same_dtype(collections, dict):
-            collections = [Element.from_dict(i) for i in collections]
-
-        self.collections = self._validate_pile(
-            collections or kwargs.get("collections", {})
+        data = Pile._validate_before(
+            {
+                "collections": collections,
+                "item_type": item_type,
+                "progression": order,
+                "strict_type": strict_type,
+                **kwargs,
+            }
         )
-        self.progression = self._validate_order(order)
+        super().__init__(**data)
+
+    @field_serializer("collections")
+    def _serialize_collections(
+        self, v: dict[IDType, T]
+    ) -> list[dict[str, Any]]:
+        return [i.to_dict() for i in v.values()]
+
+    @field_serializer("progression")
+    def _serialize_progression(self, v: Progression) -> dict[str, Any]:
+        return v.to_dict()
+
+    @field_serializer("item_type")
+    def _serialize_item_type(self, v: set[type[T]] | None) -> list[str] | None:
+        """Serialize item_type to a list of class names."""
+        if v is None:
+            return None
+        return [c.class_name(full=True) for c in v]
 
     # Sync Interface methods
     @override
@@ -147,11 +314,9 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
             A new Pile instance created from the provided data.
 
         Raises:
-            ValueError: If the dictionary format is invalid.
+            ValidationError: If the dictionary format is invalid.
         """
-        items = data.pop("collections", [])
-        items = [Element.from_dict(i) for i in items]
-        return cls(collections=items, **data)
+        return cls(**data)
 
     def __setitem__(
         self,
@@ -204,22 +369,32 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
         Raises:
             ValueError: If item not found.
         """
-        self._remove(item)
+        if isinstance(item, int | slice):
+            raise TypeError(
+                "Invalid item type for remove, should be ID or Item(s)"
+            )
+        if item in self:
+            self.pop(item)
+            return
+        raise ItemNotFoundError(f"{item}")
 
-    def include(
-        self,
-        item: ID.ItemSeq | ID.Item,
-        /,
-    ) -> None:
+    def include(self, item: ID.ItemSeq | ID.Item, /) -> None:
         """Include item(s) if not present.
 
         Args:
             item: Item(s) to include.
-
-        Raises:
-            TypeError: If item type not allowed.
         """
-        self._include(item)
+        item_dict = _validate_collections(
+            item, self.item_type, self.strict_type
+        )
+
+        item_order = []
+        for i in item_dict.keys():
+            if i not in self.progression:
+                item_order.append(i)
+
+        self.progression.append(item_order)
+        self.collections.update(item_dict)
 
     def exclude(
         self,
@@ -231,7 +406,13 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
         Args:
             item: Item(s) to exclude.
         """
-        self._exclude(item)
+        item = to_list_type(item)
+        exclude_list = []
+        for i in item:
+            if i in self:
+                exclude_list.append(i)
+        if exclude_list:
+            self.pop(exclude_list)
 
     @synchronized
     def clear(self) -> None:
@@ -251,7 +432,12 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
         Raises:
             TypeError: If item types not allowed.
         """
-        self._update(other)
+        others = _validate_collections(other, self.item_type, self.strict_type)
+        for i in others.keys():
+            if i in self.collections:
+                self.collections[i] = others[i]
+            else:
+                self.include(others[i])
 
     @synchronized
     def insert(self, index: int, item: T, /) -> None:
@@ -370,7 +556,9 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
             raise TypeError(
                 f"Invalid type for Pile operation. expected <Pile>, got {type(other)}"
             )
-        other = self._validate_pile(list(other))
+        other = _validate_collections(
+            list(other), self.item_type, self.strict_type
+        )
         self.include(other)
         return self
 
@@ -526,7 +714,7 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
         /,
     ) -> None:
         """Async remove item."""
-        self._remove(item)
+        self.remove(item)
 
     @async_synchronized
     async def ainclude(
@@ -535,7 +723,7 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
         /,
     ) -> None:
         """Async include item(s)."""
-        self._include(item)
+        self.include(item)
         if item not in self:
             raise TypeError(f"Item {item} is not of allowed types")
 
@@ -546,7 +734,7 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
         /,
     ) -> None:
         """Async exclude item(s)."""
-        self._exclude(item)
+        self.exclude(item)
 
     @async_synchronized
     async def aclear(self) -> None:
@@ -560,7 +748,7 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
         /,
     ) -> None:
         """Async update with items."""
-        self._update(other)
+        self.update(other)
 
     @async_synchronized
     async def aget(
@@ -635,7 +823,9 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
         key: ID.Ref | ID.RefSeq | int | slice,
         item: ID.Item | ID.ItemSeq,
     ) -> None:
-        item_dict = self._validate_pile(item)
+        item_dict = _validate_collections(
+            item, self.item_type, self.strict_type
+        )
 
         item_order = []
         for i in item_dict.keys():
@@ -745,128 +935,14 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
                     raise ItemNotFoundError(f"Item not found. Error: {e}")
                 return default
 
-    def _remove(self, item: ID.Ref | ID.RefSeq):
-        if isinstance(item, int | slice):
-            raise TypeError(
-                "Invalid item type for remove, should be ID or Item(s)"
-            )
-        if item in self:
-            self.pop(item)
-            return
-        raise ItemNotFoundError(f"{item}")
-
-    def _include(self, item: ID.ItemSeq | ID.Item):
-        item_dict = self._validate_pile(item)
-
-        item_order = []
-        for i in item_dict.keys():
-            if i not in self.progression:
-                item_order.append(i)
-
-        self.progression.append(item_order)
-        self.collections.update(item_dict)
-
-    def _exclude(self, item: ID.Ref | ID.RefSeq):
-        item = to_list_type(item)
-        exclude_list = []
-        for i in item:
-            if i in self:
-                exclude_list.append(i)
-        if exclude_list:
-            self.pop(exclude_list)
-
     def _clear(self) -> None:
         self.collections.clear()
         self.progression.clear()
 
-    def _update(self, other: ID.ItemSeq | ID.Item):
-        others = self._validate_pile(other)
-        for i in others.keys():
-            if i in self.collections:
-                self.collections[i] = others[i]
-            else:
-                self.include(others[i])
-
-    def _validate_item_type(self, value) -> set[type[T]] | None:
-        if value is None:
-            return None
-
-        value = to_list_type(value)
-
-        for i in value:
-            if not issubclass(i, Observable):
-                raise TypeError(
-                    f"Item type must be a subclass of Observable. Got {i}"
-                )
-
-        if len(value) != len(set(value)):
-            raise ValueError(
-                "Detected duplicated item types in item_type.",
-            )
-
-        if len(value) > 0:
-            return set(value)
-
-    def _validate_pile(self, value: Any) -> dict[str, T]:
-        if not value:
-            return {}
-
-        value = to_list_type(value)
-
-        result = {}
-        for i in value:
-            if isinstance(i, dict):
-                i = Element.from_dict(i)
-
-            if self.item_type:
-                if self.strict_type:
-                    if type(i) not in self.item_type:
-                        raise TypeError(
-                            f"Invalid item type in pile. Expected {self.item_type}",
-                        )
-                else:
-                    if not any(issubclass(type(i), t) for t in self.item_type):
-                        raise TypeError(
-                            "Invalid item type in pile. Expected "
-                            f"{self.item_type} or the subclasses",
-                        )
-            else:
-                if not isinstance(i, Observable):
-                    raise ValueError(f"Invalid pile item {i}")
-
-            result[i.id] = i
-
-        return result
-
-    def _validate_order(self, value: Any) -> Progression:
-        if not value:
-            return self.progression.__class__(
-                order=list(self.collections.keys())
-            )
-
-        if isinstance(value, Progression):
-            value = list(value)
-        else:
-            value = to_list_type(value)
-
-        value_set = set(value)
-        if len(value_set) != len(value):
-            raise ValueError("There are duplicate elements in the order")
-        if len(value_set) != len(self.collections.keys()):
-            raise ValueError(
-                "The length of the order does not match the length of the pile"
-            )
-
-        for i in value_set:
-            if ID.get_id(i) not in self.collections.keys():
-                raise ValueError(
-                    f"The order does not match the pile. {i} not found"
-                )
-
-        return self.progression.__class__(order=value)
-
     def _insert(self, index: int, item: ID.Item):
-        item_dict = self._validate_pile(item)
+        item_dict = _validate_collections(
+            item, self.item_type, self.strict_type
+        )
 
         item_order = []
         for i in item_dict.keys():
@@ -876,24 +952,6 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
         self.progression.insert(index, item_order)
         self.collections.update(item_dict)
 
-    def to_dict(self) -> dict[str, Any]:
-        """Convert pile to dictionary, properly handling collections."""
-        # Get base dict from parent class
-        dict_ = super().to_dict()
-
-        # Manually serialize collections
-        collections_list = []
-        for item in self.collections.values():
-            if hasattr(item, "to_dict"):
-                collections_list.append(item.to_dict())
-            elif hasattr(item, "model_dump"):
-                collections_list.append(item.model_dump())
-            else:
-                collections_list.append(str(item))
-
-        dict_["collections"] = collections_list
-        return dict_
-
     class AsyncPileIterator:
         def __init__(self, pile: Pile):
             self.pile = pile
@@ -930,58 +988,86 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
             is_same_dtype(self.collections.values())
         )
 
-    def adapt_to(self, obj_key: str, many=False, **kwargs: Any) -> Any:
-        kwargs["adapt_meth"] = "to_dict"
-        return super().adapt_to(obj_key, many=many, **kwargs)
+    def adapt_to(self, obj_key: str, many=False, **kw: Any) -> Any:
+        """Adapt to another format.
+
+        Args:
+            obj_key: Key indicating the format (e.g., 'json', 'csv').
+            many: If True, interpret to receive list of items in the collection.
+            **kw: Additional keyword arguments for adaptation.
+
+        Example:
+            >>> str_ = pile.adapt_to('json')
+            >>> df = pile.adapt_to('pd.DataFrame', many=True)
+            >>> csv_str = pile.adapt_to('csv', many=True)
+
+        Pile built-in with `json`, `csv`, `pd.DataFrame` adapters. You can add more
+        from pydapter, such as `qdrant`, `neo4j`, `postgres`, etc.
+        please visit https://khive-ai.github.io/pydapter/ for more details.
+        """
+        kw["adapt_meth"] = "to_dict"
+        return super().adapt_to(obj_key=obj_key, many=many, **kw)
 
     @classmethod
-    def adapt_from(cls, obj: Any, obj_key: str, many=False, **kwargs: Any):
-        """Create from another format."""
-        kwargs["adapt_meth"] = "from_dict"
-        return super().adapt_from(obj, obj_key, many=many, **kwargs)
+    def adapt_from(cls, obj: Any, obj_key: str, many=False, **kw: Any):
+        """Create from another format.
+
+        Args:
+            obj: Object to adapt from.
+            obj_key: Key indicating the format (e.g., 'json', 'csv').
+            many: If True, interpret to receive list of items in the collection.
+            **kw: Additional keyword arguments for adaptation.
+
+        Example:
+            >>> pile = Pile.adapt_from(str_, 'json')
+            >>> pile = Pile.adapt_from(df, 'pd.DataFrame', many=True)
+        Pile built-in with `json`, `csv`, `pd.DataFrame` adapters. You can add more
+        from pydapter, such as `qdrant`, `neo4j`, `postgres`, etc.
+        please visit https://khive-ai.github.io/pydapter/ for more details.
+        """
+        kw["adapt_meth"] = "from_dict"
+        return super().adapt_from(obj, obj_key, many=many, **kw)
 
-    async def adapt_to_async(
-        self, obj_key: str, many=False, **kwargs: Any
-    ) -> Any:
-        kwargs["adapt_meth"] = "to_dict"
-        return await super().adapt_to_async(obj_key, many=many, **kwargs)
+    async def adapt_to_async(self, obj_key: str, many=False, **kw: Any) -> Any:
+        """Asynchronously adapt to another format."""
+        kw["adapt_meth"] = "to_dict"
+        return await super().adapt_to_async(obj_key=obj_key, many=many, **kw)
 
     @classmethod
     async def adapt_from_async(
-        cls, obj: Any, obj_key: str, many=False, **kwargs: Any
+        cls, obj: Any, obj_key: str, many=False, **kw: Any
     ):
-        kwargs["adapt_meth"] = "from_dict"
-        return await super().adapt_from_async(
-            obj, obj_key, many=many, **kwargs
-        )
+        """Asynchronously create from another format."""
+        kw["adapt_meth"] = "from_dict"
+        return await super().adapt_from_async(obj, obj_key, many=many, **kw)
 
     def to_df(
-        self,
-        columns: list[str] | None = None,
-        **kwargs: Any,
+        self, columns: list[str] | None = None, **kw: Any
     ) -> pd.DataFrame:
         """Convert to DataFrame."""
         from pydapter.extras.pandas_ import DataFrameAdapter
 
         df = DataFrameAdapter.to_obj(
-            list(self.collections.values()), adapt_meth="to_dict", **kwargs
+            list(self.collections.values()), adapt_meth="to_dict", **kw
         )
         if columns:
             return df[columns]
         return df
 
-    def to_csv_file(self, fp: str | Path, **kwargs: Any) -> None:
+    @deprecated(
+        "to_csv_file is deprecated, use `pile.dump(fp, 'csv')` instead"
+    )
+    def to_csv_file(self, fp: str | Path, **kw: Any) -> None:
         """Save to CSV file."""
-        from pydapter.adapters import CsvAdapter
-
-        csv_str = CsvAdapter.to_obj(
-            list(self.collections.values()), adapt_meth="to_dict", **kwargs
-        )
+        csv_str = self.adapt_to("csv", many=True, **kw)
         with open(fp, "w") as f:
             f.write(csv_str)
 
+    @deprecated(
+        "to_json_file is deprecated, use `pile.dump(fp, 'json')` instead"
+    )
     def to_json_file(
-        self, fp: str | Path, mode: str = "w", many: bool = False, **kwargs
+        self, fp: str | Path, mode: str = "w", many: bool = False, **kw
     ):
         """Export collection to JSON file.
 
@@ -991,14 +1077,61 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
             mode: File mode ('w' for write, 'a' for append).
             **kwargs: Additional arguments for json.dump() or DataFrame.to_json().
         """
-        from pydapter.adapters import JsonAdapter
-
-        json_str = JsonAdapter.to_obj(
-            self, many=many, adapt_meth="to_dict", **kwargs
-        )
+        json_str = self.adapt_to("json", many=many, **kw)
         with open(fp, mode) as f:
             f.write(json_str)
 
+    def dump(
+        self,
+        fp: str | Path | None,
+        obj_key: Literal["json", "csv", "parquet"] = "json",
+        *,
+        mode: Literal["w", "a"] = "w",
+        clear=False,
+        **kw,
+    ) -> None:
+        """Export collection to file in specified format.
+
+        Args:
+            fp: File path or buffer to write to. If None, returns string.
+                Cannot be None if obj_key is 'parquet'.
+            obj_key: Format to export ('json', 'csv', 'parquet').
+            mode: File mode ('w' for write, 'a' for append).
+            clear: If True, clear the collection after export.
+            **kw: Additional arguments for the export method, pandas kwargs
+        """
+        df = self.to_df()
+        match obj_key:
+            case "parquet":
+                df.to_parquet(fp, engine="pyarrow", index=False, **kw)
+            case "json":
+                out = df.to_json(
+                    fp, orient="records", lines=True, mode=mode, **kw
+                )
+                return out if out is not None else None
+            case "csv":
+                out = df.to_csv(fp, index=False, mode=mode, **kw)
+                return out if out is not None else None
+            case _:
+                raise ValueError(
+                    f"Unsupported obj_key: {obj_key}. Supported keys are 'json', 'csv', 'parquet'."
+                )
+
+        if clear:
+            self.clear()
+
+    @async_synchronized
+    async def adump(
+        self,
+        fp: str | Path,
+        *,
+        obj_key: Literal["json", "csv", "parquet"] = "json",
+        mode: Literal["w", "a"] = "w",
+        clear=False,
+        **kw,
+    ) -> None:
+        return self.dump(fp, obj_key=obj_key, mode=mode, clear=clear, **kw)
+
 
 def to_list_type(value: Any, /) -> list[Any]:
     """Convert input to a list format"""
@@ -1017,4 +1150,17 @@ def to_list_type(value: Any, /) -> list[Any]:
     return [value]
 
 
+if not _ADAPATER_REGISTERED:
+    from pydapter.adapters import CsvAdapter, JsonAdapter
+    from pydapter.extras.pandas_ import DataFrameAdapter
+
+    Pile.register_adapter(CsvAdapter)
+    Pile.register_adapter(JsonAdapter)
+    Pile.register_adapter(DataFrameAdapter)
+    _ADAPATER_REGISTERED = True
+
+Pile = Pile
+
+__all__ = ("Pile",)
+
 # File: lionagi/protocols/generic/pile.py

lionagi/protocols/graph/node.py

@@ -9,22 +9,13 @@ from typing import Any
 
 from pydantic import field_validator
 from pydapter import Adaptable, AsyncAdaptable
-from pydapter.adapters import JsonAdapter, TomlAdapter
-from pydapter.extras.pandas_ import SeriesAdapter
 
 from lionagi._class_registry import LION_CLASS_REGISTRY
 
 from .._concepts import Relational
 from ..generic.element import Element
 
-NODE_DEFAULT_ADAPTERS = (
-    JsonAdapter,
-    SeriesAdapter,
-    TomlAdapter,
-)
-
-
-__all__ = ("Node",)
+_ADAPATER_REGISTERED = False
 
 
 class Node(Element, Relational, AsyncAdaptable, Adaptable):
@@ -113,4 +104,27 @@ class Node(Element, Relational, AsyncAdaptable, Adaptable):
         return super().adapt_from(obj, obj_key=obj_key, many=many, **kwargs)
 
 
+if not _ADAPATER_REGISTERED:
+    from pydapter.adapters import JsonAdapter, TomlAdapter
+    from pydapter.extras.pandas_ import SeriesAdapter
+
+    Node.register_adapter(JsonAdapter)
+    Node.register_adapter(TomlAdapter)
+    Node.register_adapter(SeriesAdapter)
+
+    from lionagi.adapters._utils import check_async_postgres_available
+
+    if check_async_postgres_available() is True:
+        from lionagi.adapters.async_postgres_adapter import (
+            LionAGIAsyncPostgresAdapter,
+        )
+
+        Node.register_async_adapter(LionAGIAsyncPostgresAdapter)
+
+    _ADAPATER_REGISTERED = True
+
+Node = Node
+
+__all__ = ("Node",)
+
 # File: lionagi/protocols/graph/node.py

lionagi/session/branch.py

@@ -8,7 +8,7 @@ from typing import Any, Literal
 
 import pandas as pd
 from jinja2 import Template
-from pydantic import BaseModel, Field, JsonValue, PrivateAttr
+from pydantic import BaseModel, Field, JsonValue, PrivateAttr, field_serializer
 
 from lionagi.config import settings
 from lionagi.fields import Instruct
@@ -394,6 +394,10 @@ class Branch(Element, Communicatable, Relational):
         for tool in tools:
             self._register_tool(tool, update=update)
 
+    @field_serializer("user")
+    def _serialize_user(self, v):
+        return str(v) if v else None
+
     # -------------------------------------------------------------------------
     # Conversion / Serialization
     # -------------------------------------------------------------------------

lionagi/utils.py

@@ -14,6 +14,7 @@ import re
 import shutil
 import subprocess
 import sys
+import types
 import uuid
 from collections.abc import (
     AsyncGenerator,
@@ -28,7 +29,15 @@ from enum import Enum as _Enum
 from functools import partial
 from inspect import isclass
 from pathlib import Path
-from typing import Any, Literal, TypeVar, get_args, get_origin
+from typing import (
+    Annotated,
+    Any,
+    Literal,
+    TypeVar,
+    Union,
+    get_args,
+    get_origin,
+)
 
 from pydantic import BaseModel
 from pydantic_core import PydanticUndefinedType
@@ -88,6 +97,8 @@ __all__ = (
     "StringEnum",
     "Enum",
     "hash_dict",
+    "is_union_type",
+    "union_members",
 )
 
 
@@ -191,6 +202,40 @@ def is_same_dtype(
     return (result, dtype) if return_dtype else result
 
 
+def is_union_type(tp) -> bool:
+    """True for typing.Union[...] and PEP 604 unions (A | B)."""
+    origin = get_origin(tp)
+    return origin is Union or origin is getattr(
+        types, "UnionType", object()
+    )  # Py3.10+
+
+
+NoneType = type(None)
+_UnionType = getattr(types, "UnionType", None)  # for A | B (PEP 604)
+
+
+def _unwrap_annotated(tp):
+    while get_origin(tp) is Annotated:
+        tp = get_args(tp)[0]
+    return tp
+
+
+def union_members(
+    tp, *, unwrap_annotated: bool = True, drop_none: bool = False
+) -> tuple[type, ...]:
+    """Return the member types of a Union (typing.Union or A|B). Empty tuple if not a Union."""
+    tp = _unwrap_annotated(tp) if unwrap_annotated else tp
+    origin = get_origin(tp)
+    if origin is not Union and origin is not _UnionType:
+        return ()
+    members = get_args(tp)
+    if unwrap_annotated:
+        members = tuple(_unwrap_annotated(m) for m in members)
+    if drop_none:
+        members = tuple(m for m in members if m is not NoneType)
+    return members
+
+
 async def custom_error_handler(
     error: Exception, error_map: dict[type, Callable[[Exception], None]]
 ) -> None:

lionagi/version.py

@@ -1 +1 @@
-__version__ = "0.15.3"
+__version__ = "0.15.5"

{lionagi-0.15.3.dist-info → lionagi-0.15.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lionagi
-Version: 0.15.3
+Version: 0.15.5
 Summary: An Intelligence Operating System.
 Author-email: HaiyangLi <quantocean.li@gmail.com>
 License:            Apache License
@@ -220,6 +220,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.10
 Requires-Dist: aiocache>=0.12.0
+Requires-Dist: aiofiles>=24.1.0
 Requires-Dist: aiohttp>=3.11.0
 Requires-Dist: anyio>=4.7.0
 Requires-Dist: backoff>=2.2.1

{lionagi-0.15.3.dist-info → lionagi-0.15.5.dist-info}/RECORD

@@ -5,11 +5,12 @@ lionagi/_types.py,sha256=j8XwSGeGrYwfmSJ8o-80bsfoalLWJgQH41ZkVevc4wk,75
 lionagi/config.py,sha256=W3JOC_TFad8hFkpTG8yv0-GNupa7x3wX4NAUfWpB59U,3763
 lionagi/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lionagi/settings.py,sha256=HDuKCEJCpc4HudKodBnhoQUGuTGhRHdlIFhbtf3VBtY,1633
-lionagi/utils.py,sha256=Adtr1wyrU9Ra-HfHDoHLWasD6V88Z8sqkg2CQ8i8nzI,38686
-lionagi/version.py,sha256=WbcBdqaNCRHsRqs6rGd3pRLP4kQSII2td9dR0Fl6HcU,23
+lionagi/utils.py,sha256=LxsMXyXbj5DC64y7QTmg8XzjE6hogxaed5FHw2PyK_M,39811
+lionagi/version.py,sha256=axkldpLAi2TFk3heerDJg_kSegHyvqyTI3ogd991rpE,23
 lionagi/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lionagi/adapters/async_postgres_adapter.py,sha256=Kf2YCzwRqKpEHY3GQCXEiMl201CCIkDvXcvddwZNkkE,12723
-lionagi/adapters/postgres_model_adapter.py,sha256=e_wfJNyihbpLCXuAs_W9tbLoMXAXbAXtkQDaHfqWz3o,4555
+lionagi/adapters/_utils.py,sha256=n4DS27CZfC-0O_UFaYtlUdjiMx9IeYsGpP7MVaFO5ZA,885
+lionagi/adapters/async_postgres_adapter.py,sha256=QKzEnzGR5_HNxxmnHhyuPELHv6yvQ-p7cfaX1bWRAQU,12722
+lionagi/adapters/postgres_model_adapter.py,sha256=uwWrbnihtYsCIttHExmtAIZwyohFtKoxnHU1N1M2NvQ,4519
 lionagi/fields/__init__.py,sha256=yrn9NDAM6_v73kK7aJeb-Pvqigeu8WASaV-My-6CDsc,939
 lionagi/fields/action.py,sha256=OziEpbaUeEVo34KdtbzDxXJBgkf3QLxlcKIQAfHe4O0,5791
 lionagi/fields/base.py,sha256=mvgqxLonCROszMjnG8QWt00l-MvIr_mnGvCtaH-SQ_k,3814
@@ -158,13 +159,13 @@ lionagi/protocols/generic/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGv
 lionagi/protocols/generic/element.py,sha256=Eaij2YpTWsGk28Tqjazmjmc_tOnalH7_iGFZrL6QJb4,14420
 lionagi/protocols/generic/event.py,sha256=cAkj6hiStPPJNlaYpmIXEgnix3LVAYYyDDjoubuT0ks,6602
 lionagi/protocols/generic/log.py,sha256=Y06zAQewkNlaIWjut_c6c45KY_LJfLHwzUaDGLULaas,8212
-lionagi/protocols/generic/pile.py,sha256=rzhKytyizb5xoyBOeAtDwvLZQhGR04NkgLyr3O_XzS8,30418
+lionagi/protocols/generic/pile.py,sha256=vTBxA40mhhDhoifQm1qQkVF-VuPACVhOt90a4wRHZYk,35461
 lionagi/protocols/generic/processor.py,sha256=c_a7HB9WAaCY-HoI19YyStef8WOXcDj9UeiQb5bz_TM,11759
 lionagi/protocols/generic/progression.py,sha256=qlITq1qzV119iR5qR__fBAzV489S7d4t20E8uDRicEw,15189
 lionagi/protocols/graph/__init__.py,sha256=UPu3OmUpjSgX2aBuBJUdG2fppGlfqAH96hU0qIMBMp0,253
 lionagi/protocols/graph/edge.py,sha256=YxSGj4w_fG7khm-zpKduuK5fJzhJDx23JhU1dZp29d8,5241
 lionagi/protocols/graph/graph.py,sha256=u7qoEPXh4Tpp-O6ciEGJkzzQyA7weJmE2spnWrEYZqs,10698
-lionagi/protocols/graph/node.py,sha256=cfPay0iPdt8PKAptpMXdCOqpv6EWgp_ZKNsvlrDEJwE,3460
+lionagi/protocols/graph/node.py,sha256=vW4z08wyW4YXvBch3DMteO2gVOV4j3Qa8SRd-4cCL50,3912
 lionagi/protocols/mail/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/protocols/mail/exchange.py,sha256=P1PcrFylIBeiQa8kox9H1qyJ4kjhUlbLiTUT8rs1OXg,7041
 lionagi/protocols/mail/mail.py,sha256=RB5CUft_4J85H9nM9g6aRXomTaqKwF5xVjJacPAhoa8,1356
@@ -225,7 +226,7 @@ lionagi/service/third_party/exa_models.py,sha256=G_hnekcy-DillPLzMoDQ8ZisVAL8Mp7
 lionagi/service/third_party/openai_model_names.py,sha256=C44tnqexgc4ZU2-3I_sn5d688hf3WWx-25xBd50bvas,5121
 lionagi/service/third_party/pplx_models.py,sha256=-EhyJgOWR6rzSv3zczUtk80X6c19p18Dg9KC6l8BFRQ,6473
 lionagi/session/__init__.py,sha256=kDypY6L3kGPnatAw7YNQAykgg-9MlIBnlhHExaXvt-c,202
-lionagi/session/branch.py,sha256=yfOtMITbtKzGAcxwZ5JjkXrKtb2PBGRPt53xqbR0lAs,68329
+lionagi/session/branch.py,sha256=79l014dCsyOdI4daBLOzyTTLnfYv89Bg7uf9qcg_Bwg,68449
 lionagi/session/prompts.py,sha256=GPr0jibyAAqS3awDzGC8SoCL6aWJLLCCbXY0JUuxOC0,3170
 lionagi/session/session.py,sha256=FzsUsqEQ6cnGd57E4HmEucftz5nMKsfj9kemRDqsXwU,13257
 lionagi/tools/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
@@ -234,7 +235,7 @@ lionagi/tools/types.py,sha256=XtJLY0m-Yi_ZLWhm0KycayvqMCZd--HxfQ0x9vFUYDE,230
 lionagi/tools/file/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/tools/file/reader.py,sha256=jnSHVSQ66AHZXQrgRuGmlbwKT5JHYoo-1zv1hKgTEfc,9544
 lionagi/tools/memory/tools.py,sha256=earYkKxSOz_iXkqVZYTEDfE3dwZYIWPXZrqQ1DYGz4I,15941
-lionagi-0.15.3.dist-info/METADATA,sha256=vGts1USZhbyGGcLRd2eF7t2iDpzDl9gDLoVh_aEnrqs,22895
-lionagi-0.15.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-lionagi-0.15.3.dist-info/licenses/LICENSE,sha256=VXFWsdoN5AAknBCgFqQNgPWYx7OPp-PFEP961zGdOjc,11288
-lionagi-0.15.3.dist-info/RECORD,,
+lionagi-0.15.5.dist-info/METADATA,sha256=nj6TquGq_LpwcN7NE9pMzLAraegXI4rkN5MEQViLofc,22927
+lionagi-0.15.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+lionagi-0.15.5.dist-info/licenses/LICENSE,sha256=VXFWsdoN5AAknBCgFqQNgPWYx7OPp-PFEP961zGdOjc,11288
+lionagi-0.15.5.dist-info/RECORD,,