lionagi 0.15.9__py3-none-any.whl → 0.15.13__py3-none-any.whl

lionagi/ln/_fuzzy_json.py

@@ -0,0 +1,116 @@
+import contextlib
+import re
+from typing import Any
+
+import orjson
+
+
+def fuzzy_json(str_to_parse: str, /) -> dict[str, Any] | list[dict[str, Any]]:
+    """
+    Attempt to parse a JSON string, trying a few minimal "fuzzy" fixes if needed.
+
+    Steps:
+    1. Parse directly with json.loads.
+    2. Replace single quotes with double quotes, normalize spacing, and try again.
+    3. Attempt to fix unmatched brackets using fix_json_string.
+    4. If all fail, raise ValueError.
+
+    Args:
+        str_to_parse: The JSON string to parse
+
+    Returns:
+        Parsed JSON (dict or list of dicts)
+
+    Raises:
+        ValueError: If the string cannot be parsed as valid JSON
+        TypeError: If the input is not a string
+    """
+    _check_valid_str(str_to_parse)
+
+    # 1. Direct attempt
+    with contextlib.suppress(Exception):
+        return orjson.loads(str_to_parse)
+
+    # 2. Try cleaning: replace single quotes with double and normalize
+    cleaned = _clean_json_string(str_to_parse.replace("'", '"'))
+    with contextlib.suppress(Exception):
+        return orjson.loads(cleaned)
+
+    # 3. Try fixing brackets
+    fixed = fix_json_string(cleaned)
+    with contextlib.suppress(Exception):
+        return orjson.loads(fixed)
+
+    # If all attempts fail
+    raise ValueError("Invalid JSON string")
+
+
+def _check_valid_str(str_to_parse: str, /):
+    if not isinstance(str_to_parse, str):
+        raise TypeError("Input must be a string")
+    if not str_to_parse.strip():
+        raise ValueError("Input string is empty")
+
+
+def _clean_json_string(s: str) -> str:
+    """Basic normalization: replace unescaped single quotes, trim spaces, ensure keys are quoted."""
+    # Replace unescaped single quotes with double quotes
+    # '(?<!\\)'" means a single quote not preceded by a backslash
+    s = re.sub(r"(?<!\\)'", '"', s)
+    # Collapse multiple whitespaces
+    s = re.sub(r"\s+", " ", s)
+    # Ensure keys are quoted
+    # This attempts to find patterns like { key: value } and turn them into {"key": value}
+    s = re.sub(r'([{,])\s*([^"\s]+)\s*:', r'\1"\2":', s)
+    return s.strip()
+
+
+def fix_json_string(str_to_parse: str, /) -> str:
+    """Try to fix JSON string by ensuring brackets are matched properly."""
+    if not str_to_parse:
+        raise ValueError("Input string is empty")
+
+    brackets = {"{": "}", "[": "]"}
+    open_brackets = []
+    pos = 0
+    length = len(str_to_parse)
+
+    while pos < length:
+        char = str_to_parse[pos]
+
+        if char == "\\":
+            pos += 2  # Skip escaped chars
+            continue
+
+        if char == '"':
+            pos += 1
+            # skip string content
+            while pos < length:
+                if str_to_parse[pos] == "\\":
+                    pos += 2
+                    continue
+                if str_to_parse[pos] == '"':
+                    pos += 1
+                    break
+                pos += 1
+            continue
+
+        if char in brackets:
+            open_brackets.append(brackets[char])
+        elif char in brackets.values():
+            if not open_brackets:
+                # Extra closing bracket
+                # Better to raise error than guess
+                raise ValueError("Extra closing bracket found.")
+            if open_brackets[-1] != char:
+                # Mismatched bracket
+                raise ValueError("Mismatched brackets.")
+            open_brackets.pop()
+
+        pos += 1
+
+    # Add missing closing brackets if any
+    if open_brackets:
+        str_to_parse += "".join(reversed(open_brackets))
+
+    return str_to_parse

lionagi/models/field_model.py

@@ -16,7 +16,6 @@ from typing import Annotated, Any
 from typing_extensions import Self, override
 
 from .._errors import ValidationError
-from ..utils import UNDEFINED
 
 # Cache of valid Pydantic Field parameters
 _PYDANTIC_FIELD_PARAMS: set[str] | None = None
@@ -660,13 +659,16 @@ def to_dict(self) -> dict[str, Any]:
 
     # Convert metadata to dictionary
     for meta in self.metadata:
-        if meta.key not in ("nullable", "listable", "validator"):
+        if meta.key not in (
+            "nullable",
+            "listable",
+            "validator",
+            "name",
+            "validator_kwargs",
+            "annotation",
+        ):
             result[meta.key] = meta.value
 
-    # Add annotation if available
-    if hasattr(self, "annotation"):
-        result["annotation"] = self.base_type
-
     return result
 
 

lionagi/operations/__init__.py

@@ -8,6 +8,8 @@ from .flow import flow
 from .node import BranchOperations, Operation
 from .plan.plan import PlanOperation, plan
 
+Builder = OperationGraphBuilder
+
 __all__ = (
     "ExpansionStrategy",
     "OperationGraphBuilder",
@@ -19,4 +21,5 @@ __all__ = (
     "PlanOperation",
     "brainstorm",
     "BrainstormOperation",
+    "Builder",
 )

lionagi/operations/builder.py

@@ -453,6 +453,16 @@ def visualize_graph(
     figsize=(14, 10),
 ):
     """Visualization with improved layout for complex graphs."""
+    from lionagi.protocols.graph.graph import (
+        _MATPLIB_AVAILABLE,
+        _NETWORKX_AVAILABLE,
+    )
+
+    if _MATPLIB_AVAILABLE is not True:
+        raise _MATPLIB_AVAILABLE
+    if _NETWORKX_AVAILABLE is not True:
+        raise _NETWORKX_AVAILABLE
+
     import matplotlib.pyplot as plt
     import networkx as nx
     import numpy as np

lionagi/protocols/generic/element.py

@@ -6,7 +6,7 @@ from __future__ import annotations
 
 import datetime as dt
 from collections.abc import Mapping, Sequence
-from typing import Any, Generic, TypeAlias, TypeVar
+from typing import Any, Generic, Literal, TypeAlias, TypeVar
 from uuid import UUID, uuid4
 
 import orjson
@@ -22,7 +22,7 @@ from lionagi import ln
 from lionagi._class_registry import get_class
 from lionagi._errors import IDError
 from lionagi.settings import Settings
-from lionagi.utils import time, to_dict
+from lionagi.utils import import_module, time, to_dict
 
 from .._concepts import Collective, Observable, Ordering
 
@@ -190,11 +190,13 @@ class Element(BaseModel, Observable):
         return val
 
     @field_validator("created_at", mode="before")
-    def _coerce_created_at(cls, val: float | dt.datetime | None) -> float:
+    def _coerce_created_at(
+        cls, val: float | dt.datetime | str | None
+    ) -> float:
         """Coerces `created_at` to a float-based timestamp.
 
         Args:
-            val (float | datetime | None): The initial creation time value.
+            val (float | datetime | str | None): The initial creation time value.
 
         Returns:
             float: A float representing Unix epoch time in seconds.
@@ -208,6 +210,27 @@ class Element(BaseModel, Observable):
             return val
         if isinstance(val, dt.datetime):
             return val.timestamp()
+        if isinstance(val, str):
+            # Parse datetime string from database
+            try:
+                # Handle datetime strings like "2025-08-30 10:54:59.310329"
+                # Convert space to T for ISO format, but handle timezone properly
+                iso_string = val.replace(" ", "T")
+                parsed_dt = dt.datetime.fromisoformat(iso_string)
+
+                # If parsed as naive datetime (no timezone), treat as UTC to avoid local timezone issues
+                if parsed_dt.tzinfo is None:
+                    parsed_dt = parsed_dt.replace(tzinfo=dt.timezone.utc)
+
+                return parsed_dt.timestamp()
+            except ValueError:
+                # Try parsing as float string as fallback
+                try:
+                    return float(val)
+                except ValueError:
+                    raise ValueError(
+                        f"Invalid datetime string: {val}"
+                    ) from None
         try:
             return float(val)  # type: ignore
         except Exception:
@@ -245,7 +268,7 @@ class Element(BaseModel, Observable):
         Returns:
             datetime: The creation time in UTC.
         """
-        return dt.datetime.fromtimestamp(self.created_at)
+        return dt.datetime.fromtimestamp(self.created_at, tz=dt.timezone.utc)
 
     def __eq__(self, other: Any) -> bool:
         """Compares two Element instances by their ID."""
@@ -274,14 +297,31 @@ class Element(BaseModel, Observable):
             return str(cls).split("'")[1]
         return cls.__name__
 
-    def to_dict(self) -> dict:
-        """Converts this Element to a dictionary."""
+    def _to_dict(self) -> dict:
         dict_ = self.model_dump()
         dict_["metadata"].update({"lion_class": self.class_name(full=True)})
         return {k: v for k, v in dict_.items() if ln.not_sentinel(v)}
 
+    def to_dict(
+        self, mode: Literal["python", "json", "db"] = "python"
+    ) -> dict:
+        """Converts this Element to a dictionary."""
+        if mode == "python":
+            return self._to_dict()
+        if mode == "json":
+            return orjson.loads(self.to_json(decode=False))
+        if mode == "db":
+            dict_ = orjson.loads(self.to_json(decode=False))
+            dict_["node_metadata"] = dict_.pop("metadata", {})
+            dict_["created_at"] = self.created_datetime.isoformat(sep=" ")
+            return dict_
+
+    def as_jsonable(self) -> dict:
+        """Converts this Element to a JSON-serializable dictionary."""
+        return self.to_dict(mode="json")
+
     @classmethod
-    def from_dict(cls, data: dict, /) -> Element:
+    def from_dict(cls, data: dict, /, mode: str = "python") -> Element:
         """Deserializes a dictionary into an Element or subclass of Element.
 
         If `lion_class` in `metadata` refers to a subclass, this method
@@ -289,8 +329,17 @@ class Element(BaseModel, Observable):
 
         Args:
             data (dict): A dictionary of field data.
+            mode (str): Format mode - "python" for normal dicts, "db" for database format.
         """
-        metadata = data.pop("metadata", {})
+        # Preprocess database format if needed
+        if mode == "db":
+            data = cls._preprocess_db_data(data.copy())
+        metadata = {}
+
+        if "node_metadata" in data:
+            metadata = data.pop("node_metadata")
+        elif "metadata" in data:
+            metadata = data.pop("metadata")
         if "lion_class" in metadata:
             subcls: str = metadata.pop("lion_class")
             if subcls != Element.class_name(full=True):
@@ -308,9 +357,6 @@ class Element(BaseModel, Observable):
                         return subcls_type.from_dict(data)
 
                 except Exception:
-                    # Fallback attempt: direct import if not in registry
-                    from lionagi.libs.package.imports import import_module
-
                     mod, imp = subcls.rsplit(".", 1)
                     subcls_type = import_module(mod, import_name=imp)
                     data["metadata"] = metadata
@@ -321,15 +367,72 @@ class Element(BaseModel, Observable):
         data["metadata"] = metadata
         return cls.model_validate(data)
 
-    def to_json(self) -> str:
+    @classmethod
+    def _preprocess_db_data(cls, data: dict) -> dict:
+        """Preprocess raw database data for Element compatibility."""
+        import datetime as dt
+        import json
+
+        # Handle created_at field - convert datetime string to timestamp
+        if "created_at" in data and isinstance(data["created_at"], str):
+            try:
+                # Parse datetime string and convert to timestamp
+                dt_obj = dt.datetime.fromisoformat(
+                    data["created_at"].replace(" ", "T")
+                )
+                # Treat as UTC if naive
+                if dt_obj.tzinfo is None:
+                    dt_obj = dt_obj.replace(tzinfo=dt.timezone.utc)
+                data["created_at"] = dt_obj.timestamp()
+            except (ValueError, TypeError):
+                # Keep as string if parsing fails
+                pass
+
+        # Handle JSON string fields - parse to dict/list
+        json_fields = ["content", "node_metadata", "embedding"]
+        for field in json_fields:
+            if field in data and isinstance(data[field], str):
+                if data[field] in ("null", ""):
+                    data[field] = None if field == "embedding" else {}
+                else:
+                    try:
+                        data[field] = json.loads(data[field])
+                    except (json.JSONDecodeError, TypeError):
+                        # Keep as empty dict for metadata fields, None for embedding
+                        data[field] = {} if field != "embedding" else None
+
+        # Handle node_metadata -> metadata mapping
+        if "node_metadata" in data:
+            if (
+                data["node_metadata"] == "null"
+                or data["node_metadata"] is None
+            ):
+                data["metadata"] = {}
+            else:
+                data["metadata"] = (
+                    data["node_metadata"] if data["node_metadata"] else {}
+                )
+            # Remove node_metadata to avoid Pydantic validation error
+            data.pop("node_metadata", None)
+
+        return data
+
+    def to_json(self, decode: bool = True) -> str:
         """Converts this Element to a JSON string."""
-        dict_ = self.to_dict()
-        return orjson.dumps(dict_, default=DEFAULT_ELEMENT_SERIALIZER).decode()
+        dict_ = self._to_dict()
+        if decode:
+            return orjson.dumps(
+                dict_,
+                default=DEFAULT_ELEMENT_SERIALIZER,
+                option=ln.DEFAULT_SERIALIZER_OPTION,
+            ).decode()
+        return orjson.dumps(dict_, default=DEFAULT_ELEMENT_SERIALIZER)
 
-    def from_json(cls, json_str: str) -> Element:
+    @classmethod
+    def from_json(cls, json_str: str, mode: str = "python") -> Element:
         """Deserializes a JSON string into an Element or subclass of Element."""
         data = orjson.loads(json_str)
-        return cls.from_dict(data)
+        return cls.from_dict(data, mode=mode)
 
 
 DEFAULT_ELEMENT_SERIALIZER = ln.get_orjson_default(

lionagi/protocols/generic/pile.py

@@ -19,7 +19,7 @@ from pathlib import Path
 from typing import Any, ClassVar, Generic, Literal, TypeVar
 
 import pandas as pd
-from pydantic import Field, field_serializer, field_validator, model_validator
+from pydantic import Field, field_serializer
 from pydantic.fields import FieldInfo
 from pydapter import Adaptable, AsyncAdaptable
 from typing_extensions import Self, deprecated, override
@@ -988,6 +988,12 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
             is_same_dtype(self.collections.values())
         )
 
+    @classmethod
+    def list_adapters(cls) -> list[str]:
+        syn_ = cls._adapter_registry._reg.keys()
+        asy_ = cls._async_registry._reg.keys()
+        return list(set(syn_) | set(asy_))
+
     def adapt_to(self, obj_key: str, many=False, **kw: Any) -> Any:
         """Adapt to another format.
 
@@ -1132,6 +1138,55 @@ class Pile(Element, Collective[T], Generic[T], Adaptable, AsyncAdaptable):
     ) -> None:
         return self.dump(fp, obj_key=obj_key, mode=mode, clear=clear, **kw)
 
+    def filter_by_type(
+        self,
+        item_type: type[T] | list | set,
+        strict_type: bool = False,
+        as_pile: bool = False,
+        reverse: bool = False,
+        num_items: int | None = None,
+    ) -> list[T]:
+        if isinstance(item_type, type):
+            if is_union_type(item_type):
+                item_type = set(union_members(item_type))
+            else:
+                item_type = {item_type}
+
+        if isinstance(item_type, list | tuple):
+            item_type = set(item_type)
+
+        if not isinstance(item_type, set):
+            raise TypeError("item_type must be a type or a list/set of types")
+
+        meth = None
+
+        if strict_type:
+            meth = lambda item: type(item) in item_type
+        else:
+            meth = (
+                lambda item: any(isinstance(item, t) for t in item_type)
+                is True
+            )
+
+        out = []
+        prog = (
+            list(self.progression)
+            if not reverse
+            else reversed(list(self.progression))
+        )
+        for i in prog:
+            item = self.collections[i]
+            if meth(item):
+                out.append(item)
+            if num_items is not None and len(out) == num_items:
+                break
+
+        if as_pile:
+            return self.__class__(
+                collections=out, item_type=item_type, strict_type=strict_type
+            )
+        return out
+
 
 def to_list_type(value: Any, /) -> list[Any]:
     """Convert input to a list format"""

lionagi/protocols/generic/progression.py

@@ -14,7 +14,7 @@ from lionagi._errors import ItemNotFoundError
 from .._concepts import Ordering
 from .element import ID, Element, IDError, IDType, validate_order
 
-E = TypeVar("E", bound=Element)
+T = TypeVar("T", bound=Element)
 
 
 __all__ = (
@@ -23,7 +23,7 @@ __all__ = (
 )
 
 
-class Progression(Element, Ordering[E], Generic[E]):
+class Progression(Element, Ordering[T], Generic[T]):
     """Tracks an ordered sequence of item IDs, with optional naming.
 
     This class extends `Element` and implements `Ordering`, providing
@@ -39,7 +39,7 @@ class Progression(Element, Ordering[E], Generic[E]):
             An optional human-readable identifier for the progression.
     """
 
-    order: list[ID[E].ID] = Field(
+    order: list[ID[T].ID] = Field(
         default_factory=list,
         title="Order",
         description="A sequence of IDs representing the progression.",
@@ -358,7 +358,7 @@ class Progression(Element, Ordering[E], Generic[E]):
             raise ValueError("Can only extend with another Progression.")
         self.order.extend(other.order)
 
-    def __add__(self, other: Any) -> Progression[E]:
+    def __add__(self, other: Any) -> Progression[T]:
         """Returns a new Progression with IDs from both this and `other`.
 
         Args:
@@ -371,7 +371,7 @@ class Progression(Element, Ordering[E], Generic[E]):
         new_refs = validate_order(other)
         return Progression(order=self.order + new_refs)
 
-    def __radd__(self, other: Any) -> Progression[E]:
+    def __radd__(self, other: Any) -> Progression[T]:
         """Returns a new Progression with IDs from `other` + this.
 
         Args:
@@ -396,7 +396,7 @@ class Progression(Element, Ordering[E], Generic[E]):
         self.append(other)
         return self
 
-    def __sub__(self, other: Any) -> Progression[E]:
+    def __sub__(self, other: Any) -> Progression[T]:
         """Returns a new Progression excluding specified IDs.
 
         Args:
@@ -435,7 +435,7 @@ class Progression(Element, Ordering[E], Generic[E]):
         for i in reversed(item_):
             self.order.insert(index, ID.get_id(i))
 
-    def __reverse__(self) -> Progression[E]:
+    def __reversed__(self) -> Progression[T]:
         """Returns a new reversed Progression.
 
         Returns:
@@ -456,19 +456,19 @@ class Progression(Element, Ordering[E], Generic[E]):
             return NotImplemented
         return (self.order == other.order) and (self.name == other.name)
 
-    def __gt__(self, other: Progression[E]) -> bool:
+    def __gt__(self, other: Progression[T]) -> bool:
         """Compares if this progression is "greater" by ID order."""
         return self.order > other.order
 
-    def __lt__(self, other: Progression[E]) -> bool:
+    def __lt__(self, other: Progression[T]) -> bool:
         """Compares if this progression is "less" by ID order."""
         return self.order < other.order
 
-    def __ge__(self, other: Progression[E]) -> bool:
+    def __ge__(self, other: Progression[T]) -> bool:
         """Compares if this progression is >= the other by ID order."""
         return self.order >= other.order
 
-    def __le__(self, other: Progression[E]) -> bool:
+    def __le__(self, other: Progression[T]) -> bool:
         """Compares if this progression is <= the other by ID order."""
         return self.order <= other.order
 

lionagi/protocols/graph/_utils.py

@@ -0,0 +1,22 @@
+def check_networkx_available():
+    try:
+        from networkx import DiGraph  # noqa: F401
+
+        return True
+    except Exception:
+        return ImportError(
+            "The 'networkx' package is required for this feature. "
+            "Please install `networkx` or `'lionagi[graph]'`."
+        )
+
+
+def check_matplotlib_available():
+    try:
+        import matplotlib.pyplot as plt
+
+        return True
+    except Exception:
+        return ImportError(
+            "The 'matplotlib' package is required for this feature. "
+            "Please install `matplotlib` or `'lionagi[graph]'`."
+        )

lionagi/protocols/graph/graph.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from collections import deque
-from typing import Any, Literal
+from typing import Any, Generic, Literal, TypeVar
 
 from pydantic import Field, field_serializer, field_validator, model_validator
 from typing_extensions import Self
@@ -17,11 +17,17 @@ from ..generic.pile import Pile
 from .edge import Edge
 from .node import Node
 
+T = TypeVar("T", bound=Node)
+
+from ._utils import check_matplotlib_available, check_networkx_available
+
+_NETWORKX_AVAILABLE = check_networkx_available()
+_MATPLIB_AVAILABLE = check_matplotlib_available()
 __all__ = ("Graph",)
 
 
-class Graph(Element, Relational):
-    internal_nodes: Pile[Node] = Field(
+class Graph(Element, Relational, Generic[T]):
+    internal_nodes: Pile[T] = Field(
         default_factory=lambda: Pile(item_type={Node}, strict_type=False),
         title="Internal Nodes",
         description="A collection of nodes in the graph.",
@@ -214,13 +220,10 @@ class Graph(Element, Relational):
 
     def to_networkx(self, **kwargs) -> Any:
         """Convert the graph to a NetworkX graph object."""
-        try:
-            from networkx import DiGraph  # type: ignore
+        if _NETWORKX_AVAILABLE is not True:
+            raise _NETWORKX_AVAILABLE
 
-        except ImportError:
-            from lionagi.libs.package.imports import check_import
-
-            DiGraph = check_import("networkx", import_name="DiGraph")
+        from networkx import DiGraph  # type: ignore
 
         g = DiGraph(**kwargs)
         for node in self.internal_nodes:
@@ -245,20 +248,13 @@ class Graph(Element, Relational):
         **kwargs,
     ):
         """Display the graph using NetworkX and Matplotlib."""
+        g = self.to_networkx(**kwargs)
+        if _MATPLIB_AVAILABLE is not True:
+            raise _MATPLIB_AVAILABLE
 
-        try:
-            import matplotlib.pyplot as plt  # type: ignore
-            import networkx as nx  # type: ignore
-        except ImportError:
-            from lionagi.libs.package.imports import check_import
-
-            check_import("matplotlib")
-            check_import("networkx")
-
-            import matplotlib.pyplot as plt  # type: ignore
-            import networkx as nx  # type: ignore
+        import matplotlib.pyplot as plt  # type: ignore
+        import networkx as nx  # type: ignore
 
-        g = self.to_networkx(**kwargs)
         pos = nx.spring_layout(g)
         nx.draw(
             g,