lionagi 0.14.4__tar.gz → 0.14.5__tar.gz

{lionagi-0.14.4 → lionagi-0.14.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lionagi
-Version: 0.14.4
+Version: 0.14.5
 Summary: An Intelligence Operating System.
 Author-email: HaiyangLi <quantocean.li@gmail.com>, Liangbingyan Luo <llby_luo@outlook.com>
 License:            Apache License
@@ -225,7 +225,6 @@ Requires-Dist: anyio>=4.8.0
 Requires-Dist: backoff>=2.2.1
 Requires-Dist: jinja2>=3.1.0
 Requires-Dist: json-repair>=0.40.0
-Requires-Dist: matplotlib>=3.9.0
 Requires-Dist: pillow>=11.0.0
 Requires-Dist: psutil>=7.0.0
 Requires-Dist: pydantic-settings>=2.8.0
@@ -239,6 +238,8 @@ Requires-Dist: claude-code-sdk>=0.0.14; extra == 'all'
 Requires-Dist: datamodel-code-generator>=0.31.2; extra == 'all'
 Requires-Dist: docling>=2.15.1; extra == 'all'
 Requires-Dist: fastmcp>=2.10.5; extra == 'all'
+Requires-Dist: matplotlib>=3.7.0; extra == 'all'
+Requires-Dist: networkx>=3.0.0; extra == 'all'
 Requires-Dist: ollama>=0.4.0; extra == 'all'
 Requires-Dist: pydapter[postgres]; extra == 'all'
 Requires-Dist: rich>=13.0.0; extra == 'all'
@@ -248,6 +249,9 @@ Provides-Extra: docs
 Requires-Dist: furo>=2024.8.6; extra == 'docs'
 Requires-Dist: sphinx-autobuild>=2024.10.3; extra == 'docs'
 Requires-Dist: sphinx>=8.1.3; extra == 'docs'
+Provides-Extra: graph
+Requires-Dist: matplotlib>=3.7.0; extra == 'graph'
+Requires-Dist: networkx>=3.0.0; extra == 'graph'
 Provides-Extra: lint
 Requires-Dist: black[jupyter]>=24.10.0; extra == 'lint'
 Requires-Dist: isort>=5.13.2; extra == 'lint'

{lionagi-0.14.4 → lionagi-0.14.5}/docs/modules/branch_operations.rst

@@ -64,7 +64,7 @@ of action requests. Optionally retries, logs errors, or merges results.
 This is best used for **explicit function calls** triggered from user 
 instructions or system logic.
 
-.. method:: Branch.act(action_request, *, suppress_errors=True, sanitize_input=False, unique_input=False, num_retries=0, initial_delay=0, retry_delay=0, backoff_factor=1, retry_default=UNDEFINED, retry_timeout=None, retry_timing=False, max_concurrent=None, throttle_period=None, flatten=True, dropna=True, unique_output=False, flatten_tuple_set=False)
+.. method:: Branch.act(action_request, *, suppress_errors=True, sanitize_input=False, unique_input=False, num_retries=0, initial_delay=0, retry_delay=0, backoff_factor=1, retry_default=UNDEFINED, retry_timeout=None, max_concurrent=None, throttle_period=None, flatten=True, dropna=True, unique_output=False, flatten_tuple_set=False)
 
     Public, potentially batched, asynchronous interface to run one or multiple action requests.
 
@@ -90,8 +90,6 @@ instructions or system logic.
         Fallback value if all retries fail (if suppressing errors)
     retry_timeout : float | None, default=None
         Overall timeout for all attempts (None = no limit)
-    retry_timing : bool, default=False
-        If True, track time used for retries
     max_concurrent : int | None, default=None
         Maximum concurrent tasks (if batching)
     throttle_period : float | None, default=None

{lionagi-0.14.4 → lionagi-0.14.5}/lionagi/libs/concurrency/__init__.py

@@ -1,4 +1,4 @@
-"""Structured concurrency primitives.
+"""Structured concurrency primitives for pynector.
 
 This module provides structured concurrency primitives using AnyIO,
 which allows for consistent behavior across asyncio and trio backends.
@@ -6,7 +6,21 @@ which allows for consistent behavior across asyncio and trio backends.
 
 from .cancel import CancelScope, fail_after, move_on_after
 from .errors import get_cancelled_exc_class, shield
+from .patterns import (
+    ConnectionPool,
+    WorkerPool,
+    parallel_requests,
+    retry_with_timeout,
+)
 from .primitives import CapacityLimiter, Condition, Event, Lock, Semaphore
+from .resource_tracker import (
+    ResourceTracker,
+    cleanup_check,
+    get_global_tracker,
+    resource_leak_detector,
+    track_resource,
+    untrack_resource,
+)
 from .task import TaskGroup, create_task_group
 
 __all__ = [
@@ -15,6 +29,10 @@ __all__ = [
     "CancelScope",
     "move_on_after",
     "fail_after",
+    "ConnectionPool",
+    "WorkerPool",
+    "parallel_requests",
+    "retry_with_timeout",
     "Lock",
     "Semaphore",
     "CapacityLimiter",
@@ -22,4 +40,10 @@ __all__ = [
     "Condition",
     "get_cancelled_exc_class",
     "shield",
+    "ResourceTracker",
+    "resource_leak_detector",
+    "track_resource",
+    "untrack_resource",
+    "cleanup_check",
+    "get_global_tracker",
 ]

lionagi-0.14.5/lionagi/libs/concurrency/patterns.py

@@ -0,0 +1,259 @@
+"""Common concurrency patterns for structured concurrency."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Awaitable, Callable
+from types import TracebackType
+from typing import Any, TypeVar
+
+import anyio
+
+from .cancel import move_on_after
+from .primitives import CapacityLimiter, Lock
+from .resource_tracker import track_resource, untrack_resource
+from .task import create_task_group
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+R = TypeVar("R")
+Response = TypeVar("Response")
+
+
+class ConnectionPool:
+    """A pool of reusable connections."""
+
+    def __init__(
+        self,
+        max_connections: int,
+        connection_factory: Callable[[], Awaitable[T]],
+    ):
+        """Initialize a new connection pool."""
+        if max_connections < 1:
+            raise ValueError("max_connections must be >= 1")
+        if not callable(connection_factory):
+            raise ValueError("connection_factory must be callable")
+
+        self._connection_factory = connection_factory
+        self._limiter = CapacityLimiter(max_connections)
+        self._connections: list[T] = []
+        self._lock = Lock()
+
+        track_resource(self, f"ConnectionPool-{id(self)}", "ConnectionPool")
+
+    def __del__(self):
+        """Clean up resource tracking."""
+        try:
+            untrack_resource(self)
+        except Exception:
+            pass
+
+    async def acquire(self) -> T:
+        """Acquire a connection from the pool."""
+        await self._limiter.acquire()
+
+        try:
+            async with self._lock:
+                if self._connections:
+                    return self._connections.pop()
+
+            # No pooled connection available, create new one
+            return await self._connection_factory()
+        except Exception:
+            self._limiter.release()
+            raise
+
+    async def release(self, connection: T) -> None:
+        """Release a connection back to the pool."""
+        try:
+            async with self._lock:
+                self._connections.append(connection)
+        finally:
+            self._limiter.release()
+
+    async def __aenter__(self) -> ConnectionPool[T]:
+        """Enter the connection pool context."""
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        """Exit the connection pool context."""
+        # Clean up any remaining connections
+        async with self._lock:
+            self._connections.clear()
+
+
+async def parallel_requests(
+    inputs: list[str],
+    func: Callable[[str], Awaitable[Response]],
+    max_concurrency: int = 10,
+) -> list[Response]:
+    """Execute requests in parallel with controlled concurrency.
+
+    Args:
+        inputs: List of inputs
+        fetch_func: Async function
+        max_concurrency: Maximum number of concurrent requests
+
+    Returns:
+        List of responses in the same order as inputs
+    """
+    if not inputs:
+        return []
+
+    results: list[Response | None] = [None] * len(inputs)
+
+    async def bounded_fetch(
+        semaphore: anyio.Semaphore, idx: int, url: str
+    ) -> None:
+        async with semaphore:
+            results[idx] = await func(url)
+
+    try:
+        async with create_task_group() as tg:
+            semaphore = anyio.Semaphore(max_concurrency)
+
+            for i, inp in enumerate(inputs):
+                await tg.start_soon(bounded_fetch, semaphore, i, inp)
+    except BaseException as e:
+        # Re-raise the first exception directly instead of ExceptionGroup
+        if hasattr(e, "exceptions") and e.exceptions:
+            raise e.exceptions[0]
+        else:
+            raise
+
+    return results  # type: ignore
+
+
+async def retry_with_timeout(
+    func: Callable[[], Awaitable[T]],
+    max_retries: int = 3,
+    timeout: float = 30.0,
+    backoff_factor: float = 1.0,
+) -> T:
+    """Retry an async function with exponential backoff and timeout.
+
+    Args:
+        func: The async function to retry
+        max_retries: Maximum number of retries
+        timeout: Timeout for each attempt
+        backoff_factor: Multiplier for exponential backoff
+
+    Returns:
+        The result of the successful function call
+
+    Raises:
+        Exception: The last exception raised by the function
+    """
+    last_exception = None
+
+    for attempt in range(max_retries):
+        try:
+            with move_on_after(timeout) as cancel_scope:
+                result = await func()
+                if not cancel_scope.cancelled_caught:
+                    return result
+                else:
+                    raise TimeoutError(f"Function timed out after {timeout}s")
+        except Exception as e:
+            last_exception = e
+            if attempt < max_retries - 1:
+                delay = backoff_factor * (2**attempt)
+                await anyio.sleep(delay)
+            continue
+
+    if last_exception:
+        raise last_exception
+    else:
+        raise RuntimeError("Retry failed without capturing exception")
+
+
+class WorkerPool:
+    """A pool of worker tasks that process items from a queue."""
+
+    def __init__(
+        self, num_workers: int, worker_func: Callable[[Any], Awaitable[None]]
+    ):
+        """Initialize a new worker pool."""
+        if num_workers < 1:
+            raise ValueError("num_workers must be >= 1")
+        if not callable(worker_func):
+            raise ValueError("worker_func must be callable")
+
+        self._num_workers = num_workers
+        self._worker_func = worker_func
+        self._queue = anyio.create_memory_object_stream(1000)
+        self._task_group = None
+
+        track_resource(self, f"WorkerPool-{id(self)}", "WorkerPool")
+
+    def __del__(self):
+        """Clean up resource tracking."""
+        try:
+            untrack_resource(self)
+        except Exception:
+            pass
+
+    async def start(self) -> None:
+        """Start the worker pool."""
+        if self._task_group is not None:
+            raise RuntimeError("Worker pool is already started")
+
+        self._task_group = create_task_group()
+        await self._task_group.__aenter__()
+
+        # Start worker tasks
+        for i in range(self._num_workers):
+            await self._task_group.start_soon(self._worker_loop)
+
+    async def stop(self) -> None:
+        """Stop the worker pool."""
+        if self._task_group is None:
+            return
+
+        # Close the queue to signal workers to stop
+        await self._queue[0].aclose()
+
+        # Wait for all workers to finish
+        try:
+            await self._task_group.__aexit__(None, None, None)
+        finally:
+            self._task_group = None
+
+    async def submit(self, item: Any) -> None:
+        """Submit an item for processing."""
+        if self._task_group is None:
+            raise RuntimeError("Worker pool is not started")
+        await self._queue[0].send(item)
+
+    async def _worker_loop(self) -> None:
+        """Main loop for worker tasks."""
+        try:
+            async with self._queue[1]:
+                async for item in self._queue[1]:
+                    try:
+                        await self._worker_func(item)
+                    except Exception as e:
+                        logger.error(f"Worker error processing item: {e}")
+        except anyio.ClosedResourceError:
+            # Queue was closed, worker should exit gracefully
+            pass
+
+    async def __aenter__(self) -> WorkerPool:
+        """Enter the worker pool context."""
+        await self.start()
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        """Exit the worker pool context."""
+        await self.stop()

lionagi-0.14.5/lionagi/libs/concurrency/primitives.py

@@ -0,0 +1,290 @@
+"""Resource management primitives for structured concurrency.
+
+Pure async primitives focused on correctness and simplicity.
+"""
+
+import math
+from types import TracebackType
+
+import anyio
+
+from .resource_tracker import track_resource, untrack_resource
+
+
+class Lock:
+    """A mutex lock for controlling access to a shared resource."""
+
+    def __init__(self):
+        """Initialize a new lock."""
+        self._lock = anyio.Lock()
+        self._acquired = False
+        track_resource(self, f"Lock-{id(self)}", "Lock")
+
+    def __del__(self):
+        """Clean up resource tracking when lock is destroyed."""
+        try:
+            untrack_resource(self)
+        except Exception:
+            pass
+
+    async def __aenter__(self) -> None:
+        """Acquire the lock."""
+        await self._lock.acquire()
+        self._acquired = True
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        """Release the lock."""
+        self._lock.release()
+        self._acquired = False
+
+    async def acquire(self) -> None:
+        """Acquire the lock directly."""
+        await self._lock.acquire()
+        self._acquired = True
+
+    def release(self) -> None:
+        """Release the lock directly."""
+        if not self._acquired:
+            raise RuntimeError(
+                "Attempted to release lock that was not acquired by this task"
+            )
+        self._lock.release()
+        self._acquired = False
+
+
+class Semaphore:
+    """A semaphore preventing excessive releases."""
+
+    def __init__(self, initial_value: int):
+        """Initialize a new semaphore."""
+        if initial_value < 0:
+            raise ValueError("The initial value must be >= 0")
+        self._initial_value = initial_value
+        self._current_acquisitions = 0
+        self._semaphore = anyio.Semaphore(initial_value)
+        track_resource(self, f"Semaphore-{id(self)}", "Semaphore")
+
+    def __del__(self):
+        """Clean up resource tracking when semaphore is destroyed."""
+        try:
+            untrack_resource(self)
+        except Exception:
+            pass
+
+    async def __aenter__(self) -> None:
+        """Acquire the semaphore."""
+        await self.acquire()
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        """Release the semaphore."""
+        self.release()
+
+    async def acquire(self) -> None:
+        """Acquire the semaphore."""
+        await self._semaphore.acquire()
+        self._current_acquisitions += 1
+
+    def release(self) -> None:
+        """Release the semaphore."""
+        if self._current_acquisitions <= 0:
+            raise RuntimeError(
+                "Cannot release semaphore: no outstanding acquisitions"
+            )
+        self._semaphore.release()
+        self._current_acquisitions -= 1
+
+    @property
+    def current_acquisitions(self) -> int:
+        """Get the current number of outstanding acquisitions."""
+        return self._current_acquisitions
+
+    @property
+    def initial_value(self) -> int:
+        """Get the initial semaphore value."""
+        return self._initial_value
+
+
+class CapacityLimiter:
+    """A context manager for limiting the number of concurrent operations."""
+
+    def __init__(self, total_tokens: int | float):
+        """Initialize a new capacity limiter."""
+        if total_tokens == math.inf:
+            processed_tokens = math.inf
+        elif isinstance(total_tokens, (int, float)) and total_tokens >= 1:
+            processed_tokens = (
+                int(total_tokens) if total_tokens != math.inf else math.inf
+            )
+        else:
+            raise ValueError(
+                "The total number of tokens must be >= 1 (int or math.inf)"
+            )
+
+        self._limiter = anyio.CapacityLimiter(processed_tokens)
+        self._borrower_counter = 0
+        self._active_borrowers = {}
+        track_resource(self, f"CapacityLimiter-{id(self)}", "CapacityLimiter")
+
+    def __del__(self):
+        """Clean up resource tracking when limiter is destroyed."""
+        try:
+            untrack_resource(self)
+        except Exception:
+            pass
+
+    async def __aenter__(self) -> None:
+        """Acquire a token."""
+        await self.acquire()
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        """Release the token."""
+        self.release()
+
+    async def acquire(self) -> None:
+        """Acquire a token."""
+        # Create a unique borrower identity for each acquisition
+        self._borrower_counter += 1
+        borrower = f"borrower-{self._borrower_counter}"
+        await self._limiter.acquire_on_behalf_of(borrower)
+        self._active_borrowers[borrower] = True
+
+    def release(self) -> None:
+        """Release a token."""
+        # Find and release the first active borrower
+        if not self._active_borrowers:
+            raise RuntimeError("No tokens to release")
+
+        borrower = next(iter(self._active_borrowers))
+        self._limiter.release_on_behalf_of(borrower)
+        del self._active_borrowers[borrower]
+
+    @property
+    def total_tokens(self) -> int | float:
+        """The total number of tokens."""
+        return float(self._limiter.total_tokens)
+
+    @total_tokens.setter
+    def total_tokens(self, value: int | float) -> None:
+        """Set the total number of tokens."""
+        if value == math.inf:
+            processed_value = math.inf
+        elif isinstance(value, (int, float)) and value >= 1:
+            processed_value = int(value) if value != math.inf else math.inf
+        else:
+            raise ValueError(
+                "The total number of tokens must be >= 1 (int or math.inf)"
+            )
+
+        current_borrowed = self._limiter.borrowed_tokens
+        if processed_value != math.inf and processed_value < current_borrowed:
+            raise ValueError(
+                f"Cannot set total_tokens to {processed_value}: {current_borrowed} tokens "
+                f"are currently borrowed. Wait for tokens to be released or "
+                f"set total_tokens to at least {current_borrowed}."
+            )
+
+        self._limiter.total_tokens = processed_value
+
+    @property
+    def borrowed_tokens(self) -> int:
+        """The number of tokens currently borrowed."""
+        return self._limiter.borrowed_tokens
+
+    @property
+    def available_tokens(self) -> int | float:
+        """The number of tokens currently available."""
+        return self._limiter.available_tokens
+
+
+class Event:
+    """An event object for task synchronization."""
+
+    def __init__(self):
+        """Initialize a new event in the unset state."""
+        self._event = anyio.Event()
+        track_resource(self, f"Event-{id(self)}", "Event")
+
+    def __del__(self):
+        """Clean up resource tracking when event is destroyed."""
+        try:
+            untrack_resource(self)
+        except Exception:
+            pass
+
+    def is_set(self) -> bool:
+        """Check if the event is set."""
+        return self._event.is_set()
+
+    def set(self) -> None:
+        """Set the event, allowing all waiting tasks to proceed."""
+        self._event.set()
+
+    async def wait(self) -> None:
+        """Wait until the event is set."""
+        await self._event.wait()
+
+
+class Condition:
+    """A condition variable for task synchronization."""
+
+    def __init__(self, lock: Lock | None = None):
+        """Initialize a new condition."""
+        self._lock = lock or Lock()
+        self._condition = anyio.Condition(self._lock._lock)
+        track_resource(self, f"Condition-{id(self)}", "Condition")
+
+    def __del__(self):
+        """Clean up resource tracking when condition is destroyed."""
+        try:
+            untrack_resource(self)
+        except Exception:
+            pass
+
+    async def __aenter__(self) -> "Condition":
+        """Acquire the underlying lock."""
+        await self._lock.__aenter__()
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        """Release the underlying lock."""
+        await self._lock.__aexit__(exc_type, exc_val, exc_tb)
+
+    async def wait(self) -> None:
+        """Wait for a notification.
+
+        This releases the underlying lock, waits for a notification, and then
+        reacquires the lock.
+        """
+        await self._condition.wait()
+
+    async def notify(self, n: int = 1) -> None:
+        """Notify waiting tasks.
+
+        Args:
+            n: The number of tasks to notify
+        """
+        await self._condition.notify(n)
+
+    async def notify_all(self) -> None:
+        """Notify all waiting tasks."""
+        await self._condition.notify_all()