lionagi 0.16.1__py3-none-any.whl → 0.16.2__py3-none-any.whl

lionagi/libs/file/save.py

@@ -89,7 +89,14 @@ def save_chunks(
             random_hash_digits=random_hash_digits,
         )
         save_to_file(
-            ln.json_dumps(chunk),
+            ln.json_dumps(
+                chunk,
+                pretty=True,
+                sort_keys=True,
+                append_newline=True,
+                deterministic_sets=True,
+                decimal_as_float=True,
+            ),
             directory=file_path.parent,
             filename=file_path.name,
             verbose=verbose,

lionagi/ln/__init__.py

@@ -4,7 +4,10 @@ from ._json_dump import (
     DEFAULT_SERIALIZER,
     DEFAULT_SERIALIZER_OPTION,
     get_orjson_default,
+    json_dumpb,
     json_dumps,
+    json_lines_iter,
+    make_options,
 )
 from ._list_call import lcall
 from ._to_list import to_list
@@ -47,6 +50,9 @@ __all__ = (
     "DEFAULT_SERIALIZER_OPTION",
     "get_orjson_default",
     "json_dumps",
+    "make_options",
+    "json_dumpb",
+    "json_lines_iter",
     "lcall",
     "to_list",
     "acreate_path",

lionagi/ln/_json_dump.py

@@ -1,75 +1,348 @@
+from __future__ import annotations
+
 import datetime as dt
 import decimal
-from collections.abc import Callable
+import re
+from collections.abc import Callable, Iterable, Mapping
+from enum import Enum
+from functools import lru_cache
 from pathlib import Path
+from textwrap import shorten
+from typing import Any
 from uuid import UUID
 
 import orjson
 
+__all__ = [
+    "get_orjson_default",
+    "DEFAULT_SERIALIZER",
+    "DEFAULT_SERIALIZER_OPTION",
+    "make_options",
+    "json_dumpb",
+    "json_dumps",
+    "json_lines_iter",
+]
+
+# Types orjson already serializes natively at C/Rust speed.
+# (We only route them through default() when passthrough is requested.)
+_NATIVE = (dt.datetime, dt.date, dt.time, UUID)
+
+# --------- helpers ------------------------------------------------------------
+
+_ADDR_PAT = re.compile(r" at 0x[0-9A-Fa-f]+")
+
+
+def _clip(s: str, limit: int = 2048) -> str:
+    return shorten(s, width=limit, placeholder=f"...(+{len(s) - limit} chars)")  # type: ignore[arg-type]
+
+
+def _normalize_for_sorting(x: Any) -> str:
+    """Normalize repr/str to remove process-specific addresses."""
+    s = str(x)
+    return _ADDR_PAT.sub(" at 0x?", s)
+
+
+def _stable_sorted_iterable(o: Iterable[Any]) -> list[Any]:
+    """
+    Deterministic ordering for sets (including mixed types).
+    Key: (class name, normalized str) avoids comparisons across unlike types
+    and removes memory address variance in default reprs.
+    """
+    return sorted(
+        o, key=lambda x: (x.__class__.__name__, _normalize_for_sorting(x))
+    )
+
+
+def _safe_exception_payload(ex: Exception) -> dict[str, str]:
+    return {"type": ex.__class__.__name__, "message": str(ex)}
 
-def _get_default_serializers():
-    return {
-        dt.datetime: lambda o: o.isoformat(),
-        Path: lambda o: str(o),
-        UUID: lambda o: str(o),
-        decimal.Decimal: lambda o: str(o),
-        set: lambda o: list(o),
-        frozenset: lambda o: list(o),
+
+def _default_serializers(
+    deterministic_sets: bool,
+    decimal_as_float: bool,
+    enum_as_name: bool,
+    passthrough_datetime: bool,
+) -> dict[type, Callable[[Any], Any]]:
+    ser: dict[type, Callable[[Any], Any]] = {
+        Path: str,
+        decimal.Decimal: (float if decimal_as_float else str),
+        set: (_stable_sorted_iterable if deterministic_sets else list),
+        frozenset: (_stable_sorted_iterable if deterministic_sets else list),
     }
+    if enum_as_name:
+        ser[Enum] = lambda e: e.name
+    # Only needed if you also set OPT_PASSTHROUGH_DATETIME via options.
+    if passthrough_datetime:
+        ser[dt.datetime] = lambda o: o.isoformat()
+    return ser
 
 
-def _get_default_serializer_order():
-    return [dt.datetime, Path, UUID, decimal.Decimal, set, frozenset]
+# --------- default() factory --------------------------------------------------
 
 
 def get_orjson_default(
-    order: list[type] = None,
-    additional: dict[type, Callable] = None,
+    *,
+    order: list[type] | None = None,
+    additional: Mapping[type, Callable[[Any], Any]] | None = None,
     extend_default: bool = True,
-) -> Callable:
-    """get the default function for orjson.dumps
-    Args:
-        order: order of types to check. Defaults to None.
-        additional: additional serializers
-        extend_default: when order is provided, whether to extend the default order or replace it.
+    deterministic_sets: bool = False,
+    decimal_as_float: bool = False,
+    enum_as_name: bool = False,
+    passthrough_datetime: bool = False,
+    safe_fallback: bool = False,
+    fallback_clip: int = 2048,
+) -> Callable[[Any], Any]:
     """
-    dict_ = _get_default_serializers()
-    dict_.update(additional or {})
-    order_ = _get_default_serializer_order()
+    Build a fast, extensible `default=` callable for orjson.dumps.
+
+    - deterministic_sets: sort set/frozenset deterministically (slower).
+    - decimal_as_float: serialize Decimal as float (faster/smaller; precision loss).
+    - enum_as_name: serialize Enum as .name (else orjson uses .value by default).
+    - passthrough_datetime: if True, also pass OPT_PASSTHROUGH_DATETIME in options.
+    - safe_fallback: if True, unknown objects never raise (for logs);
+      Exceptions become a tiny dict; all else becomes clipped repr(str).
+
+    'order' and 'additional' preserve your override semantics.
+    """
+    ser = _default_serializers(
+        deterministic_sets=deterministic_sets,
+        decimal_as_float=decimal_as_float,
+        enum_as_name=enum_as_name,
+        passthrough_datetime=passthrough_datetime,
+    )
+    if additional:
+        ser.update(additional)
+
+    base_order: list[type] = [Path, decimal.Decimal, set, frozenset]
+    if enum_as_name:
+        base_order.insert(0, Enum)
+    if passthrough_datetime:
+        base_order.insert(0, dt.datetime)
 
     if order:
-        if len(additional or {}) > 0 and extend_default:
-            order_.extend([k for k in order if k not in order_])
-        else:
-            order_ = list(order)
+        order_ = (
+            (base_order + [t for t in order if t not in base_order])
+            if extend_default
+            else list(order)
+        )
     else:
-        if len(additional or {}) > 0:
-            order_.extend([k for k in additional.keys() if k not in order_])
+        order_ = base_order.copy()
+
+    if not passthrough_datetime:
+        # Avoid checks for types already on the orjson native fast path.
+        order_ = [t for t in order_ if t not in _NATIVE]
+
+    order_tuple = tuple(order_)
+    cache: dict[type, Callable[[Any], Any]] = {}
 
-    def default(obj):
-        for t in order_:
-            if isinstance(obj, t) and t in dict_:
-                return dict_[t](obj)
-        raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
+    def default(obj: Any) -> Any:
+        typ = obj.__class__
+        func = cache.get(typ)
+        if func is None:
+            for T in order_tuple:
+                if issubclass(typ, T):
+                    f = ser.get(T)
+                    if f:
+                        cache[typ] = f
+                        func = f
+                        break
+            else:
+                # Duck-typed support for common data holders
+                md = getattr(obj, "model_dump", None)
+                if callable(md):
+                    try:
+                        return md()
+                    except Exception:
+                        pass
+                dd = getattr(obj, "dict", None)
+                if callable(dd):
+                    try:
+                        return dd()
+                    except Exception:
+                        pass
+                if safe_fallback:
+                    if isinstance(obj, Exception):
+                        return _safe_exception_payload(obj)
+                    return _clip(repr(obj), fallback_clip)
+                raise TypeError(
+                    f"Type is not JSON serializable: {typ.__name__}"
+                )
+        return func(obj)
 
     return default
 
 
+@lru_cache(maxsize=128)
+def _cached_default(
+    deterministic_sets: bool,
+    decimal_as_float: bool,
+    enum_as_name: bool,
+    passthrough_datetime: bool,
+    safe_fallback: bool,
+    fallback_clip: int,
+):
+    return get_orjson_default(
+        deterministic_sets=deterministic_sets,
+        decimal_as_float=decimal_as_float,
+        enum_as_name=enum_as_name,
+        passthrough_datetime=passthrough_datetime,
+        safe_fallback=safe_fallback,
+        fallback_clip=fallback_clip,
+    )
+
+
+# --------- defaults & options -------------------------------------------------
+
+# Compact, no newline, no sorting: neutral default for most use-cases.
+DEFAULT_SERIALIZER_OPTION = 0
 DEFAULT_SERIALIZER = get_orjson_default()
-DEFAULT_SERIALIZER_OPTION = (
-    orjson.OPT_INDENT_2
-    | orjson.OPT_SORT_KEYS
-    | orjson.OPT_APPEND_NEWLINE
-    | orjson.OPT_SERIALIZE_DATACLASS
-)
-
-
-def json_dumps(d_, decode=True, /) -> str:
-    by_ = orjson.dumps(
-        d_,
-        default=DEFAULT_SERIALIZER,
-        option=DEFAULT_SERIALIZER_OPTION,
+
+
+def make_options(
+    *,
+    pretty: bool = False,
+    sort_keys: bool = False,
+    naive_utc: bool = False,
+    utc_z: bool = False,
+    append_newline: bool = False,
+    passthrough_datetime: bool = False,
+    allow_non_str_keys: bool = False,
+) -> int:
+    """
+    Compose orjson 'option' bit flags succinctly.
+    """
+    opt = 0
+    if append_newline:
+        opt |= orjson.OPT_APPEND_NEWLINE
+    if pretty:
+        opt |= orjson.OPT_INDENT_2
+    if sort_keys:
+        opt |= orjson.OPT_SORT_KEYS
+    if naive_utc:
+        opt |= orjson.OPT_NAIVE_UTC
+    if utc_z:
+        opt |= orjson.OPT_UTC_Z
+    if passthrough_datetime:
+        opt |= orjson.OPT_PASSTHROUGH_DATETIME
+    if allow_non_str_keys:
+        opt |= orjson.OPT_NON_STR_KEYS
+    return opt
+
+
+# --------- dump helpers -------------------------------------------------------
+
+
+def json_dumpb(
+    obj: Any,
+    *,
+    pretty: bool = False,
+    sort_keys: bool = False,
+    naive_utc: bool = False,
+    utc_z: bool = False,
+    append_newline: bool = False,
+    allow_non_str_keys: bool = False,
+    deterministic_sets: bool = False,
+    decimal_as_float: bool = False,
+    enum_as_name: bool = False,
+    passthrough_datetime: bool = False,
+    safe_fallback: bool = False,
+    fallback_clip: int = 2048,
+    default: Callable[[Any], Any] | None = None,
+    options: int | None = None,
+) -> bytes:
+    """
+    Serialize to **bytes** (fast path). Prefer this in hot code.
+
+    Notes:
+      - If you set passthrough_datetime=True, you likely also want it in options.
+      - safe_fallback=True is recommended for LOGGING ONLY.
+    """
+    if default is None:
+        default = _cached_default(
+            deterministic_sets=deterministic_sets,
+            decimal_as_float=decimal_as_float,
+            enum_as_name=enum_as_name,
+            passthrough_datetime=passthrough_datetime,
+            safe_fallback=safe_fallback,
+            fallback_clip=fallback_clip,
+        )
+    opt = (
+        options
+        if options is not None
+        else make_options(
+            pretty=pretty,
+            sort_keys=sort_keys,
+            naive_utc=naive_utc,
+            utc_z=utc_z,
+            append_newline=append_newline,
+            passthrough_datetime=passthrough_datetime,
+            allow_non_str_keys=allow_non_str_keys,
+        )
     )
-    if decode:
-        return by_.decode("utf-8")
-    return by_
+    return orjson.dumps(obj, default=default, option=opt)
+
+
+def json_dumps(
+    obj: Any,
+    /,
+    *,
+    decode: bool = True,
+    **kwargs: Any,
+) -> str | bytes:
+    """
+    Serialize to str by default (decode=True), or bytes if decode=False.
+    """
+    out = json_dumpb(obj, **kwargs)
+    return out.decode("utf-8") if decode else out
+
+
+# --------- streaming for very large outputs ----------------------------------
+
+
+def json_lines_iter(
+    it: Iterable[Any],
+    *,
+    # default() configuration for each line
+    deterministic_sets: bool = False,
+    decimal_as_float: bool = False,
+    enum_as_name: bool = False,
+    passthrough_datetime: bool = False,
+    safe_fallback: bool = False,
+    fallback_clip: int = 2048,
+    # options
+    naive_utc: bool = False,
+    utc_z: bool = False,
+    allow_non_str_keys: bool = False,
+    # advanced
+    default: Callable[[Any], Any] | None = None,
+    options: int | None = None,
+) -> Iterable[bytes]:
+    """
+    Stream an iterable as **NDJSON** (one JSON object per line) in **bytes**.
+
+    Always ensures a trailing newline per line (OPT_APPEND_NEWLINE).
+    """
+    if default is None:
+        default = _cached_default(
+            deterministic_sets=deterministic_sets,
+            decimal_as_float=decimal_as_float,
+            enum_as_name=enum_as_name,
+            passthrough_datetime=passthrough_datetime,
+            safe_fallback=safe_fallback,
+            fallback_clip=fallback_clip,
+        )
+    if options is None:
+        opt = make_options(
+            pretty=False,
+            sort_keys=False,
+            naive_utc=naive_utc,
+            utc_z=utc_z,
+            append_newline=True,  # enforce newline for NDJSON
+            passthrough_datetime=passthrough_datetime,
+            allow_non_str_keys=allow_non_str_keys,
+        )
+    else:
+        opt = options | orjson.OPT_APPEND_NEWLINE
+
+    for item in it:
+        yield orjson.dumps(item, default=default, option=opt)

lionagi/models/note.py

@@ -9,10 +9,6 @@ from pydantic import BaseModel, ConfigDict, Field, field_serializer
 from typing_extensions import override
 
 from lionagi.libs.nested.flatten import flatten
-from lionagi.libs.nested.nget import nget
-from lionagi.libs.nested.ninsert import ninsert
-from lionagi.libs.nested.npop import npop
-from lionagi.libs.nested.nset import nset
 from lionagi.utils import UNDEFINED, copy, to_list
 
 IndiceType: TypeAlias = str | list[str | int]
@@ -120,6 +116,8 @@ class Note(BaseModel):
         Raises:
             KeyError: If the path is not found and no default value is provided.
         """
+        from lionagi.libs.nested.npop import npop
+
         indices = to_list(indices, flatten=True, dropna=True)
         return npop(self.content, indices, default)
 
@@ -130,6 +128,8 @@ class Note(BaseModel):
             indices: Path where to insert
             value: Value to insert
         """
+        from lionagi.libs.nested.ninsert import ninsert
+
         indices = to_list(indices, flatten=True, dropna=True)
         ninsert(self.content, indices, value)
 
@@ -144,6 +144,8 @@ class Note(BaseModel):
         if self.get(indices, None) is None:
             self.insert(indices, value)
         else:
+            from lionagi.libs.nested.nset import nset
+
             nset(self.content, indices, value)
 
     def get(
@@ -170,6 +172,8 @@ class Note(BaseModel):
         Raises:
             KeyError: If the path is not found and no default value is provided.
         """
+        from lionagi.libs.nested.nget import nget
+
         indices = to_list(indices, flatten=True, dropna=True)
         return nget(self.content, indices, default)
 

lionagi/operations/brainstorm/brainstorm.py

@@ -309,8 +309,8 @@ async def brainstormStream(
                             )
                             print(f"\n-----Exploring Idea-----\n{snippet}")
                         new_branch = session.split(branch)
-                        resp = await new_branch.instruct(
-                            ins_, **(explore_kwargs or {})
+                        resp = await new_branch.operate(
+                            **ins_.to_dict(), **(explore_kwargs or {})
                         )
                         return InstructResponse(instruct=ins_, response=resp)
 
@@ -357,8 +357,8 @@ async def brainstormStream(
                                 else i.guidance
                             )
                             print(f"\n-----Exploring Idea-----\n{snippet}")
-                        seq_res = await branch.instruct(
-                            i, **(explore_kwargs or {})
+                        seq_res = await branch.operate(
+                            **i.to_dict(), **(explore_kwargs or {})
                         )
                         ins_res = InstructResponse(
                             instruct=i, response=seq_res
@@ -389,8 +389,8 @@ async def brainstormStream(
 
                         async def _explore(ins_: Instruct):
                             child_branch = session.split(base_branch)
-                            child_resp = await child_branch.instruct(
-                                ins_, **(explore_kwargs or {})
+                            child_resp = await child_branch.operate(
+                                **ins_.to_dict(), **(explore_kwargs or {})
                             )
                             return InstructResponse(
                                 instruct=ins_, response=child_resp
@@ -453,8 +453,8 @@ async def brainstormStream(
                                     f"\n-----Exploring Idea (sequential in chunk)-----\n{snippet}"
                                 )
 
-                            seq_resp = await local_branch.instruct(
-                                ins_, **(explore_kwargs or {})
+                            seq_resp = await local_branch.operate(
+                                **ins_.to_dict(), **(explore_kwargs or {})
                             )
                             chunk_results.append(
                                 InstructResponse(
@@ -468,8 +468,8 @@ async def brainstormStream(
                     all_responses = await alcall(
                         all_chunks,
                         explore_chunk_sequentially,
-                        flatten=True,
-                        dropna=True,
+                        output_flatten=True,
+                        output_dropna=True,
                     )
                     out.explore = all_responses
 

lionagi/protocols/generic/element.py

@@ -315,10 +315,6 @@ class Element(BaseModel, Observable):
             dict_["node_metadata"] = dict_.pop("metadata", {})
             return dict_
 
-    def as_jsonable(self) -> dict:
-        """Converts this Element to a JSON-serializable dictionary."""
-        return self.to_dict(mode="json")
-
     @classmethod
     def from_dict(cls, data: dict) -> Element:
         """Deserializes a dictionary into an Element or subclass of Element.
@@ -363,19 +359,14 @@ class Element(BaseModel, Observable):
     def to_json(self, decode: bool = True) -> str:
         """Converts this Element to a JSON string."""
         dict_ = self._to_dict()
-        if decode:
-            return orjson.dumps(
-                dict_,
-                default=DEFAULT_ELEMENT_SERIALIZER,
-                option=ln.DEFAULT_SERIALIZER_OPTION,
-            ).decode()
-        return orjson.dumps(dict_, default=DEFAULT_ELEMENT_SERIALIZER)
+        return ln.json_dumps(
+            dict_, default=DEFAULT_ELEMENT_SERIALIZER, decode=decode
+        )
 
     @classmethod
-    def from_json(cls, json_str: str, mode: str = "python") -> Element:
+    def from_json(cls, json_str: str) -> Element:
         """Deserializes a JSON string into an Element or subclass of Element."""
-        data = orjson.loads(json_str)
-        return cls.from_dict(data, mode=mode)
+        return cls.from_dict(orjson.loads(json_str))
 
 
 DEFAULT_ELEMENT_SERIALIZER = ln.get_orjson_default(

lionagi/protocols/generic/log.py

@@ -176,9 +176,9 @@ class DataLogger:
         suffix = fp.suffix.lower()
         try:
             if suffix == ".csv":
-                self.logs.to_csv_file(fp)
+                self.logs.dump(fp, "csv")
             elif suffix == ".json":
-                self.logs.to_json_file(fp)
+                self.logs.dump(fp, "json")
             else:
                 raise ValueError(f"Unsupported file extension: {suffix}")
 

lionagi/protocols/messages/message.py

@@ -116,7 +116,14 @@ class RoledMessage(Node, Sendable):
             if isinstance(self.template, Template):
                 return self.template.render(**self.content)
         except Exception:
-            return ln.json_dumps(self.content)
+            return ln.json_dumps(
+                self.content,
+                pretty=True,
+                sort_keys=True,
+                append_newline=True,
+                deterministic_sets=True,
+                decimal_as_float=True,
+            )
 
     @classmethod
     def create(cls, **kwargs):

lionagi/service/connections/endpoint.py

@@ -150,6 +150,13 @@ class Endpoint:
                 "branch",
                 "aggregation_sources",
                 "aggregation_count",
+                "action_strategy",
+                "parse_model",
+                "reason",
+                "actions",
+                "return_operative",
+                "operative_model",
+                "request_model",
             }
             payload = {
                 k: v for k, v in payload.items() if k not in non_api_params

lionagi/service/connections/providers/claude_code_.py

@@ -266,7 +266,12 @@ def _verbose_output(res) -> str:
                 inp_ = None
 
                 if isinstance(block.input, dict | list):
-                    inp_ = ln.json_dumps(block.input)
+                    inp_ = ln.json_dumps(
+                        block.input,
+                        pretty=True,
+                        sort_keys=True,
+                        append_newline=True,
+                    )
                 else:
                     inp_ = str(block.input)