lionagi 0.5.5__py3-none-any.whl → 0.6.0__py3-none-any.whl

lionagi/{core → protocols}/generic/pile.py

@@ -2,49 +2,49 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from __future__ import annotations
+
 import asyncio
+import json
 import threading
-from collections.abc import AsyncIterator, Callable, Iterator, Sequence
+from collections import deque
+from collections.abc import (
+    AsyncIterator,
+    Callable,
+    Generator,
+    Iterator,
+    Sequence,
+)
 from functools import wraps
 from pathlib import Path
+from typing import Any, ClassVar, Generic, Self, TypeVar
 
 import pandas as pd
+from pydantic import Field, field_serializer
+from pydantic.fields import FieldInfo
 from typing_extensions import override
 
-from lionagi.core.typing import (
-    ID,
-    UNDEFINED,
-    Any,
-    ClassVar,
-    Field,
-    FieldInfo,
-    Generic,
-    ItemExistsError,
-    ItemNotFoundError,
-    Observable,
-    Self,
-    TypeVar,
-    field_serializer,
-)
-from lionagi.core.typing._concepts import IDType
-from lionagi.libs.parse import is_same_dtype, to_list
-from lionagi.protocols.adapters.adapter import Adapter, AdapterRegistry
-from lionagi.protocols.registries._pile_registry import PileAdapterRegistry
+from lionagi._errors import ItemExistsError, ItemNotFoundError
+from lionagi.utils import UNDEFINED, is_same_dtype, to_list
 
-from .element import Element
+from .._adapter import Adapter, AdapterRegistry, PileAdapterRegistry
+from .._concepts import Observable
+from .element import ID, Collective, E, Element, IDType, validate_order
 from .progression import Progression
-from .utils import to_list_type, validate_order
 
-T = TypeVar("T", bound=Element)
 D = TypeVar("D")
+T = TypeVar("T", bound=E)
 
 
-__all__ = ("Pile", "pile")
+__all__ = (
+    "Pile",
+    "pile",
+)
 
 
 def synchronized(func: Callable):
     @wraps(func)
-    def wrapper(self: "Pile", *args, **kwargs):
+    def wrapper(self: Pile, *args, **kwargs):
         with self.lock:
             return func(self, *args, **kwargs)
 
@@ -53,14 +53,14 @@ def synchronized(func: Callable):
 
 def async_synchronized(func: Callable):
     @wraps(func)
-    async def wrapper(self: "Pile", *args, **kwargs):
+    async def wrapper(self: Pile, *args, **kwargs):
         async with self.async_lock:
             return await func(self, *args, **kwargs)
 
     return wrapper
 
 
-class Pile(Element, Generic[T]):
+class Pile(Element, Collective[E], Generic[E]):
     """Thread-safe async-compatible, ordered collection of elements.
 
     The Pile class provides a thread-safe, async-compatible collection with:
@@ -76,13 +76,13 @@ class Pile(Element, Generic[T]):
         strict_type (bool): Whether to enforce strict type checking
     """
 
-    pile_: dict[IDType, T] = Field(default_factory=dict)
-    item_type: set[type[T]] | None = Field(
+    collections: dict[IDType, T] = Field(default_factory=dict)
+    item_type: set | None = Field(
         default=None,
         description="Set of allowed types for items in the pile.",
         exclude=True,
     )
-    progress: Progression = Field(
+    progression: Progression = Field(
         default_factory=Progression,
         description="Progression specifying the order of items in the pile.",
         exclude=True,
@@ -107,7 +107,7 @@ class Pile(Element, Generic[T]):
     @override
     def __init__(
         self,
-        items: ID.ItemSeq = None,
+        collections: ID.ItemSeq = None,
         item_type: set[type[T]] = None,
         order: ID.RefSeq = None,
         strict_type: bool = False,
@@ -122,15 +122,21 @@ class Pile(Element, Generic[T]):
             strict_type: If True, enforce strict type checking.
         """
         _config = {}
-        if "ln_id" in kwargs:
-            _config["ln_id"] = kwargs["ln_id"]
-        if "created_timestamp" in kwargs:
-            _config["created_timestamp"] = kwargs["created_timestamp"]
+        if "id" in kwargs:
+            _config["id"] = kwargs["id"]
+        if "created_at" in kwargs:
+            _config["created_at"] = kwargs["created_at"]
 
         super().__init__(strict_type=strict_type, **_config)
         self.item_type = self._validate_item_type(item_type)
-        self.pile_ = self._validate_pile(items or kwargs.get("pile_", {}))
-        self.progress = self._validate_order(order)
+
+        if isinstance(collections, list) and is_same_dtype(collections, dict):
+            collections = [Element.from_dict(i) for i in collections]
+
+        self.collections = self._validate_pile(
+            collections or kwargs.get("collections", {})
+        )
+        self.progression = self._validate_order(order)
 
     # Sync Interface methods
     @override
@@ -139,7 +145,7 @@ class Pile(Element, Generic[T]):
         cls,
         data: dict[str, Any],
         /,
-    ) -> "Pile":
+    ) -> Pile:
         """Create a Pile instance from a dictionary.
 
         Args:
@@ -151,9 +157,9 @@ class Pile(Element, Generic[T]):
         Raises:
             ValueError: If the dictionary format is invalid.
         """
-        items = data.pop("pile_", [])
+        items = data.pop("collections", [])
         items = [Element.from_dict(i) for i in items]
-        return cls(items=items, **data)
+        return cls(collections=items, **data)
 
     def __setitem__(
         self,
@@ -178,7 +184,7 @@ class Pile(Element, Generic[T]):
         key: ID.Ref | ID.RefSeq | int | slice,
         default: D = UNDEFINED,
         /,
-    ) -> T | "Pile" | D:
+    ) -> T | Pile | D:
         """Remove and return item(s) from the Pile.
 
         Args:
@@ -287,7 +293,7 @@ class Pile(Element, Generic[T]):
         key: ID.Ref | ID.RefSeq | int | slice,
         default: D = UNDEFINED,
         /,
-    ) -> T | "Pile" | D:
+    ) -> T | Pile | D:
         """Get item(s) by key with default.
 
         Args:
@@ -301,31 +307,31 @@ class Pile(Element, Generic[T]):
 
     def keys(self) -> Sequence[str]:
         """Get all Lion IDs in order."""
-        return list(self.progress)
+        return list(self.progression)
 
     def values(self) -> Sequence[T]:
         """Get all items in order."""
-        return [self.pile_[key] for key in self.progress]
+        return [self.collections[key] for key in self.progression]
 
     def items(self) -> Sequence[tuple[IDType, T]]:
         """Get all (ID, item) pairs in order."""
-        return [(key, self.pile_[key]) for key in self.progress]
+        return [(key, self.collections[key]) for key in self.progression]
 
     def is_empty(self) -> bool:
         """Check if empty."""
-        return len(self.progress) == 0
+        return len(self.progression) == 0
 
     def size(self) -> int:
         """Get number of items."""
-        return len(self.progress)
+        return len(self.progression)
 
     def __iter__(self) -> Iterator[T]:
         """Iterate over items safely."""
         with self.lock:
-            current_order = list(self.progress)
+            current_order = list(self.progression)
 
         for key in current_order:
-            yield self.pile_[key]
+            yield self.collections[key]
 
     def __next__(self) -> T:
         """Get next item."""
@@ -352,11 +358,11 @@ class Pile(Element, Generic[T]):
 
     def __contains__(self, item: ID.RefSeq | ID.Ref) -> bool:
         """Check if item exists."""
-        return item in self.progress
+        return item in self.progression
 
     def __len__(self) -> int:
         """Get number of items."""
-        return len(self.pile_)
+        return len(self.collections)
 
     @override
     def __bool__(self) -> bool:
@@ -367,42 +373,36 @@ class Pile(Element, Generic[T]):
         """Convert to list."""
         return self.values()
 
-    def __ior__(self, other: "Pile") -> Self:
+    def __ior__(self, other: Pile) -> Self:
         """In-place union."""
         if not isinstance(other, Pile):
             raise TypeError(
-                "Invalid type for Pile operation.",
-                expected_type=Pile,
-                actual_type=type(other),
+                f"Invalid type for Pile operation. expected <Pile>, got {type(other)}"
             )
         other = self._validate_pile(list(other))
         self.include(other)
         return self
 
-    def __or__(self, other: "Pile") -> "Pile":
+    def __or__(self, other: Pile) -> Pile:
         """Union."""
         if not isinstance(other, Pile):
             raise TypeError(
-                "Invalid type for Pile operation.",
-                expected_type=Pile,
-                actual_type=type(other),
+                f"Invalid type for Pile operation. expected <Pile>, got {type(other)}"
             )
 
         result = self.__class__(
             items=self.values(),
             item_type=self.item_type,
-            order=self.progress,
+            order=self.progression,
         )
         result.include(list(other))
         return result
 
-    def __ixor__(self, other: "Pile") -> Self:
+    def __ixor__(self, other: Pile) -> Self:
         """In-place symmetric difference."""
         if not isinstance(other, Pile):
             raise TypeError(
-                "Invalid type for Pile operation.",
-                expected_type=Pile,
-                actual_type=type(other),
+                f"Invalid type for Pile operation. expected <Pile>, got {type(other)}"
             )
 
         to_exclude = []
@@ -415,13 +415,11 @@ class Pile(Element, Generic[T]):
         self.include(other)
         return self
 
-    def __xor__(self, other: "Pile") -> "Pile":
+    def __xor__(self, other: Pile) -> Pile:
         """Symmetric difference."""
         if not isinstance(other, Pile):
             raise TypeError(
-                "Invalid type for Pile operation.",
-                expected_type=Pile,
-                actual_type=type(other),
+                f"Invalid type for Pile operation. expected <Pile>, got {type(other)}"
             )
 
         to_exclude = []
@@ -439,13 +437,11 @@ class Pile(Element, Generic[T]):
         )
         return result
 
-    def __iand__(self, other: "Pile") -> Self:
+    def __iand__(self, other: Pile) -> Self:
         """In-place intersection."""
         if not isinstance(other, Pile):
             raise TypeError(
-                "Invalid type for Pile operation.",
-                expected_type=Pile,
-                actual_type=type(other),
+                f"Invalid type for Pile operation. expected <Pile>, got {type(other)}"
             )
 
         to_exclude = []
@@ -455,13 +451,11 @@ class Pile(Element, Generic[T]):
         self.exclude(to_exclude)
         return self
 
-    def __and__(self, other: "Pile") -> "Pile":
+    def __and__(self, other: Pile) -> Pile:
         """Intersection."""
         if not isinstance(other, Pile):
             raise TypeError(
-                "Invalid type for Pile operation.",
-                expected_type=Pile,
-                actual_type=type(other),
+                f"Invalid type for Pile operation. expected <Pile>, got {type(other)}"
             )
 
         values = [i for i in self if i in other]
@@ -482,7 +476,7 @@ class Pile(Element, Generic[T]):
         if length == 0:
             return "Pile()"
         elif length == 1:
-            return f"Pile({next(iter(self.pile_.values())).__repr__()})"
+            return f"Pile({next(iter(self.collections.values())).__repr__()})"
         else:
             return f"Pile({length})"
 
@@ -590,10 +584,10 @@ class Pile(Element, Generic[T]):
     async def __aiter__(self) -> AsyncIterator[T]:
         """Async iterate over items."""
         async with self.async_lock:
-            current_order = list(self.progress)
+            current_order = list(self.progression)
 
         for key in current_order:
-            yield self.pile_[key]
+            yield self.collections[key]
             await asyncio.sleep(0)  # Yield control to the event loop
 
     async def __anext__(self) -> T:
@@ -610,7 +604,7 @@ class Pile(Element, Generic[T]):
 
         if isinstance(key, int | slice):
             try:
-                result_ids = self.progress[key]
+                result_ids = self.progression[key]
                 result_ids = (
                     [result_ids]
                     if not isinstance(result_ids, list)
@@ -618,14 +612,14 @@ class Pile(Element, Generic[T]):
                 )
                 result = []
                 for i in result_ids:
-                    result.append(self.pile_[i])
+                    result.append(self.collections[i])
                 return result[0] if len(result) == 1 else result
             except Exception as e:
                 raise ItemNotFoundError(f"index {key}. Error: {e}")
 
         elif isinstance(key, IDType):
             try:
-                return self.pile_[key]
+                return self.collections[key]
             except Exception as e:
                 raise ItemNotFoundError(f"key {key}. Error: {e}")
 
@@ -635,7 +629,7 @@ class Pile(Element, Generic[T]):
             try:
                 for k in key:
                     result_id = ID.get_id(k)
-                    result.append(self.pile_[result_id])
+                    result.append(self.collections[result_id])
 
                 if len(result) == 0:
                     raise ItemNotFoundError(f"key {key} item not found")
@@ -654,20 +648,20 @@ class Pile(Element, Generic[T]):
 
         item_order = []
         for i in item_dict.keys():
-            if i in self.progress:
+            if i in self.progression:
                 raise ItemExistsError(f"item {i} already exists in the pile")
             item_order.append(i)
         if isinstance(key, int | slice):
             try:
                 delete_order = (
-                    list(self.progress[key])
-                    if isinstance(self.progress[key], Progression)
-                    else [self.progress[key]]
+                    list(self.progression[key])
+                    if isinstance(self.progression[key], Progression)
+                    else [self.progression[key]]
                 )
-                self.progress[key] = item_order
+                self.progression[key] = item_order
                 for i in to_list(delete_order, flatten=True):
-                    self.pile_.pop(i)
-                self.pile_.update(item_dict)
+                    self.collections.pop(i)
+                self.collections.update(item_dict)
             except Exception as e:
                 raise ValueError(f"Failed to set pile. Error: {e}")
         else:
@@ -684,10 +678,10 @@ class Pile(Element, Generic[T]):
                     raise KeyError(
                         f"Invalid key {id_}. Key and item does not match.",
                     )
-            self.progress += key
-            self.pile_.update(item_dict)
+            self.progression += key
+            self.collections.update(item_dict)
 
-    def _get(self, key: Any, default: D = UNDEFINED) -> T | "Pile" | D:
+    def _get(self, key: Any, default: D = UNDEFINED) -> T | Pile | D:
         if isinstance(key, int | slice):
             try:
                 return self[key]
@@ -724,15 +718,15 @@ class Pile(Element, Generic[T]):
         self,
         key: ID.Ref | ID.RefSeq | int | slice,
         default: D = UNDEFINED,
-    ) -> T | "Pile" | D:
+    ) -> T | Pile | D:
         if isinstance(key, int | slice):
             try:
-                pops = self.progress[key]
+                pops = self.progression[key]
                 pops = [pops] if isinstance(pops, IDType) else pops
                 result = []
                 for i in pops:
-                    self.progress.remove(i)
-                    result.append(self.pile_.pop(i))
+                    self.progression.remove(i)
+                    result.append(self.collections.pop(i))
                 result = (
                     self.__class__(items=result, item_type=self.item_type)
                     if len(result) > 1
@@ -748,8 +742,8 @@ class Pile(Element, Generic[T]):
                 key = validate_order(key)
                 result = []
                 for k in key:
-                    self.progress.remove(k)
-                    result.append(self.pile_.pop(k))
+                    self.progression.remove(k)
+                    result.append(self.collections.pop(k))
                 if len(result) == 0:
                     raise ItemNotFoundError(f"key {key} item not found")
                 elif len(result) == 1:
@@ -775,11 +769,11 @@ class Pile(Element, Generic[T]):
 
         item_order = []
         for i in item_dict.keys():
-            if i not in self.progress:
+            if i not in self.progression:
                 item_order.append(i)
 
-        self.progress.append(item_order)
-        self.pile_.update(item_dict)
+        self.progression.append(item_order)
+        self.collections.update(item_dict)
 
     def _exclude(self, item: ID.Ref | ID.RefSeq):
         item = to_list_type(item)
@@ -791,14 +785,14 @@ class Pile(Element, Generic[T]):
             self.pop(exclude_list)
 
     def _clear(self) -> None:
-        self.pile_.clear()
-        self.progress.clear()
+        self.collections.clear()
+        self.progression.clear()
 
     def _update(self, other: ID.ItemSeq | ID.Item):
         others = self._validate_pile(other)
         for i in others.keys():
-            if i in self.pile_:
-                self.pile_[i] = others[i]
+            if i in self.collections:
+                self.collections[i] = others[i]
             else:
                 self.include(others[i])
 
@@ -811,9 +805,7 @@ class Pile(Element, Generic[T]):
         for i in value:
             if not issubclass(i, Observable):
                 raise TypeError(
-                    message="Item type must be a subclass of T.",
-                    expected_type=T,
-                    actual_type=type(i),
+                    f"Item type must be a subclass of Observable. Got {i}"
                 )
 
         if len(value) != len(set(value)):
@@ -832,11 +824,14 @@ class Pile(Element, Generic[T]):
 
         result = {}
         for i in value:
+            if isinstance(i, dict):
+                i = Element.from_dict(i)
+
             if self.item_type:
                 if self.strict_type:
                     if type(i) not in self.item_type:
                         raise TypeError(
-                            message="Invalid item type in pile."
+                            "Invalid item type in pile."
                             f" Expected {self.item_type}",
                         )
                 else:
@@ -849,13 +844,15 @@ class Pile(Element, Generic[T]):
                 if not isinstance(i, Observable):
                     raise ValueError(f"Invalid pile item {i}")
 
-            result[i.ln_id] = i
+            result[i.id] = i
 
         return result
 
     def _validate_order(self, value: Any) -> Progression:
         if not value:
-            return self.progress.__class__(order=list(self.pile_.keys()))
+            return self.progression.__class__(
+                order=list(self.collections.keys())
+            )
 
         if isinstance(value, Progression):
             value = list(value)
@@ -865,36 +862,36 @@ class Pile(Element, Generic[T]):
         value_set = set(value)
         if len(value_set) != len(value):
             raise ValueError("There are duplicate elements in the order")
-        if len(value_set) != len(self.pile_.keys()):
+        if len(value_set) != len(self.collections.keys()):
             raise ValueError(
                 "The length of the order does not match the length of the pile"
             )
 
         for i in value_set:
-            if ID.get_id(i) not in self.pile_.keys():
+            if ID.get_id(i) not in self.collections.keys():
                 raise ValueError(
                     f"The order does not match the pile. {i} not found"
                 )
 
-        return self.progress.__class__(order=value)
+        return self.progression.__class__(order=value)
 
     def _insert(self, index: int, item: ID.Item):
         item_dict = self._validate_pile(item)
 
         item_order = []
         for i in item_dict.keys():
-            if i in self.progress:
+            if i in self.progression:
                 raise ItemExistsError(f"item {i} already exists in the pile")
             item_order.append(i)
-        self.progress.insert(index, item_order)
-        self.pile_.update(item_dict)
+        self.progression.insert(index, item_order)
+        self.collections.update(item_dict)
 
-    @field_serializer("pile_")
+    @field_serializer("collections")
     def _(self, value: dict[str, T]):
         return [i.to_dict() for i in value.values()]
 
     class AsyncPileIterator:
-        def __init__(self, pile: "Pile"):
+        def __init__(self, pile: Pile):
             self.pile = pile
             self.index = 0
 
@@ -904,7 +901,7 @@ class Pile(Element, Generic[T]):
         async def __anext__(self) -> T:
             if self.index >= len(self.pile):
                 raise StopAsyncIteration
-            item = self.pile[self.pile.progress[self.index]]
+            item = self.pile[self.pile.progression[self.index]]
             self.index += 1
             await asyncio.sleep(0)  # Yield control to the event loop
             return item
@@ -925,7 +922,9 @@ class Pile(Element, Generic[T]):
 
     def is_homogenous(self) -> bool:
         """Check if all items are same type."""
-        return len(self.pile_) < 2 or all(is_same_dtype(self.pile_.values()))
+        return len(self.collections) < 2 or all(
+            is_same_dtype(self.collections.values())
+        )
 
     def adapt_to(self, obj_key: str, /, **kwargs: Any) -> Any:
         """Convert to another format."""
@@ -954,27 +953,55 @@ class Pile(Element, Generic[T]):
             cls, obj, obj_key, **kwargs
         )
         if isinstance(dict_, list):
-            dict_ = {"pile_": dict_}
+            dict_ = {"collections": dict_}
         return cls.from_dict(dict_)
 
     def to_df(
         self,
         columns: list[str] | None = None,
         **kwargs: Any,
-    ):
+    ) -> pd.DataFrame:
         """Convert to DataFrame."""
         return self.adapt_to("pd_dataframe", columns=columns, **kwargs)
 
-    def to_csv(self, fp: str | Path, **kwargs: Any) -> None:
+    def to_csv_file(self, fp: str | Path, **kwargs: Any) -> None:
         """Save to CSV file."""
         self.adapt_to(".csv", fp=fp, **kwargs)
 
+    def to_json_file(
+        self,
+        path_or_buf,
+        *,
+        use_pd: bool = False,
+        mode="w",
+        verbose=False,
+        **kwargs,
+    ):
+        """Export collection to JSON file.
+
+        Args:
+            path_or_buf: File path or buffer to write to.
+            use_pd: Use pandas JSON export if True.
+            mode: File open mode.
+            verbose: Print confirmation message.
+            **kwargs: Additional arguments for json.dump() or DataFrame.to_json().
+        """
+
+        if use_pd:
+            return self.to_df().to_json(mode=mode, **kwargs)
+        dict_ = self.to_dict()
+        with open(path_or_buf, mode) as f:
+            json.dump(dict_, f, **kwargs)
+
+        if verbose:
+            print(f"Saved Pile to {path_or_buf}")
+
 
 def pile(
-    items: Any = None,
+    collections: Any = None,
     /,
     item_type: type[T] | set[type[T]] | None = None,
-    order: list[str] | None = None,
+    progression: list[str] | None = None,
     strict_type: bool = False,
     df: pd.DataFrame | None = None,  # priority 1
     fp: str | Path | None = None,  # priority 2
@@ -1001,16 +1028,30 @@ def pile(
             return Pile.adapt_from(fp, ".csv", **kwargs)
         if fp.suffix == ".xlsx":
             return Pile.adapt_from(fp, ".xlsx", **kwargs)
-        if fp.suffix == ".json":
+        if fp.suffix in [".json", ".jsonl"]:
             return Pile.adapt_from(fp, ".json", **kwargs)
 
     return Pile(
-        items,
+        collections,
         item_type=item_type,
-        order=order,
+        order=progression,
         strict=strict_type,
         **kwargs,
     )
 
 
-# File: autoos/generic/pile.py
+def to_list_type(value: Any, /) -> list[Any]:
+    """Convert input to a list format"""
+    if value is None:
+        return []
+    if isinstance(value, IDType):
+        return [value]
+    if isinstance(value, str):
+        return ID.get_id(value) if ID.is_id(value) else []
+    if isinstance(value, Element):
+        return [value]
+    if hasattr(value, "values") and callable(value.values):
+        return list(value.values())
+    if isinstance(value, list | tuple | set | deque | Generator):
+        return list(value)
+    return [value]