async-graph-data-flow 1.5.1__tar.gz → 1.6.0__tar.gz

{async_graph_data_flow-1.5.1 → async_graph_data_flow-1.6.0}/CHANGELOG.md

@@ -19,6 +19,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Security
 
+## [1.6.0] - 2025-03-19
+
+### Added
+- Added support for custom edge behaviors, through the new `queue` argument at `add_node`
+  for a custom queue object. (#14)
+
+### Deprecated
+- Deprecated the `queue_size` argument at `add_node` -- use the new `queue` instead. (#14)
+
+### Security
+- Resolved CVE-2024-56326, CVE-2024-56201, and CVE-2025-27516
+  by updating jinja2 to v3.1.6 in requirements for Sphinx docs. (#14)
+
 ## [1.5.1] - 2024-11-07
 
 ### Added

{async_graph_data_flow-1.5.1/src/async_graph_data_flow.egg-info → async_graph_data_flow-1.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: async-graph-data-flow
-Version: 1.5.1
+Version: 1.6.0
 Summary: Asynchronous functions that pass data along a directed acyclic graph
 Author-email: Civis Analytics <opensource@civisanalytics.com>
 License: BSD 3-Clause
@@ -24,14 +24,14 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Provides-Extra: dev
-Requires-Dist: black==24.10.0; extra == "dev"
-Requires-Dist: flake8==7.1.1; extra == "dev"
-Requires-Dist: pytest==8.3.3; extra == "dev"
-Requires-Dist: mypy==1.13.0; extra == "dev"
+Requires-Dist: black==25.1.0; extra == "dev"
+Requires-Dist: flake8==7.1.2; extra == "dev"
+Requires-Dist: pytest==8.3.5; extra == "dev"
+Requires-Dist: mypy==1.15.0; extra == "dev"
 Requires-Dist: build==1.2.2; extra == "dev"
-Requires-Dist: twine==5.1.1; extra == "dev"
+Requires-Dist: twine==6.1.0; extra == "dev"
 Provides-Extra: docs
-Requires-Dist: Sphinx==8.1.3; extra == "docs"
+Requires-Dist: Sphinx==8.2.3; extra == "docs"
 Requires-Dist: furo==2024.8.6; extra == "docs"
 Requires-Dist: numpydoc==1.8.0; extra == "docs"
 Requires-Dist: sphinx-copybutton==0.5.2; extra == "docs"
@@ -77,7 +77,7 @@ black --check src tests examples
 
 ## Building Documentation
 
-We use the Sphinx framework. The documentation source files are in `docs/source/`.
+We use the Sphinx framework. The documentation source files are in `docs/`.
 These files can be updated as necessary.
 
 The public documentation is accessible at https://async-graph-data-flow.readthedocs.io.

{async_graph_data_flow-1.5.1 → async_graph_data_flow-1.6.0}/README.md

@@ -38,7 +38,7 @@ black --check src tests examples
 
 ## Building Documentation
 
-We use the Sphinx framework. The documentation source files are in `docs/source/`.
+We use the Sphinx framework. The documentation source files are in `docs/`.
 These files can be updated as necessary.
 
 The public documentation is accessible at https://async-graph-data-flow.readthedocs.io.

{async_graph_data_flow-1.5.1 → async_graph_data_flow-1.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "async-graph-data-flow"
-version = "1.5.1"
+version = "1.6.0"
 description = "Asynchronous functions that pass data along a directed acyclic graph"
 readme = "README.md"
 requires-python = ">= 3.10"
@@ -40,17 +40,17 @@ Source = "https://github.com/civisanalytics/async-graph-data-flow"
 [project.optional-dependencies]
 dev = [
     # Running tests and linters
-    "black == 24.10.0",
-    "flake8 == 7.1.1",
-    "pytest == 8.3.3",
-    "mypy == 1.13.0",
+    "black == 25.1.0",
+    "flake8 == 7.1.2",
+    "pytest == 8.3.5",
+    "mypy == 1.15.0",
 
     # Managing source distributions
     "build == 1.2.2",
-    "twine == 5.1.1",
+    "twine == 6.1.0",
 ]
 docs = [
-    "Sphinx == 8.1.3",
+    "Sphinx == 8.2.3",
     "furo == 2024.8.6",
     "numpydoc == 1.8.0",
     "sphinx-copybutton == 0.5.2",

{async_graph_data_flow-1.5.1 → async_graph_data_flow-1.6.0}/src/async_graph_data_flow/executor.py

@@ -4,7 +4,8 @@ import logging
 import time
 import traceback
 from collections import deque
-from typing import Any, Iterable
+from collections.abc import Iterable
+from typing import Any
 
 from .graph import AsyncGraph, InvalidAsyncGraphError
 
@@ -284,7 +285,10 @@ class AsyncExecutor:
         self._exceptions = {}
 
         for node_name, node in self._graph._nodes.items():
-            queue = asyncio.Queue(maxsize=node.queue_size)
+            if node.queue is None:
+                queue = asyncio.Queue(maxsize=node.queue_size)
+            else:
+                queue = node.queue
             self._node_queues[node_name] = queue
             self._data_flow_stats[node_name] = {"in": 0, "out": 0, "err": 0}
             self._exceptions[node_name] = deque(maxlen=self._max_exceptions)

{async_graph_data_flow-1.5.1 → async_graph_data_flow-1.6.0}/src/async_graph_data_flow/graph.py

@@ -1,6 +1,8 @@
+import asyncio
 import inspect
 from collections import OrderedDict
-from typing import Any, Callable, NamedTuple
+from collections.abc import AsyncGenerator, Callable
+from typing import Any, NamedTuple
 
 
 class InvalidAsyncGraphError(Exception):
@@ -8,8 +10,9 @@ class InvalidAsyncGraphError(Exception):
 
 
 class _Node(NamedTuple):
-    func: Callable
+    func: Callable[..., AsyncGenerator]
     name: str
+    queue: asyncio.Queue | None
     queue_size: int
     max_tasks: int
     halt_on_exception: bool
@@ -32,12 +35,13 @@ class AsyncGraph:
 
     def add_node(
         self,
-        func: Callable,
+        func: Callable[..., AsyncGenerator],
         *,
         name: str | None = None,
         halt_on_exception: bool = False,
         unpack_input: bool = True,
         max_tasks: int = 1,
+        queue: asyncio.Queue | None = None,
         queue_size: int = 10_000,
         check_async_gen: bool = True,
     ) -> None:
@@ -45,8 +49,8 @@ class AsyncGraph:
 
         Parameters
         ----------
-        func : Callable
-            The function that this node runs.
+        func : Callable[..., AsyncGenerator]
+            The asynchronous generator function that this node runs.
             See notes below for the function's requirements.
         name : str, optional
             The name of this node. If not provided, the ``__name__`` attribute
@@ -62,10 +66,23 @@ class AsyncGraph:
             See notes below for more details.
         max_tasks : int, optional
             The number of tasks that this node runs concurrently.
+        queue : asyncio.Queue, optional
+            The queue object that collects items from this node's source nodes,
+            via ``await queue.put(item)``, and then feed into this node
+            with items retrieved by ``await queue.get()``.
+            This queue object must be an instance of either :class:`~asyncio.Queue` or
+            a subclass of :class:`~asyncio.Queue`.
+            If ``None`` or not given, it defaults to an ``asyncio.Queue()`` with max
+            size set by ``queue_size``.
         queue_size : int, optional
             The maximum number of data items allowed to be
-            in the :class:`~asyncio.Queue` object between this node as a source
-            and its destination node(s).
+            in the queue object between this node as a destination node
+            and its source node(s).
+
+            .. deprecated:: 1.6.0
+                The argument ``queue_size`` is deprecated and will be removed in
+                v2.0.0. To configure the queue size, please use the argument ``queue``
+                for a queue object whose queue size is set.
         check_async_gen : bool, optional
             If ``True`` (the default), the callable ``func`` is verified to be an async
             generator function by :func:`inspect.isasyncgenfunction`.
@@ -129,9 +146,12 @@ class AsyncGraph:
             raise TypeError(f"node '{name}' isn't an async generator function")
         if name in self._nodes:
             raise ValueError(f"node '{name}' already exists in the graph")
+        if queue is not None and not isinstance(queue, asyncio.Queue):
+            raise TypeError(f"queue must be an instance of asyncio.Queue: {queue}")
         self._nodes[name] = _Node(
             func=func,
             name=name,
+            queue=queue,
             queue_size=queue_size,
             max_tasks=max_tasks,
             halt_on_exception=halt_on_exception,
@@ -139,14 +159,18 @@ class AsyncGraph:
         )
         self._nodes_to_edges[name] = set()
 
-    def add_edge(self, src_node: str | Callable, dst_node: str | Callable) -> None:
+    def add_edge(
+        self,
+        src_node: str | Callable[..., AsyncGenerator],
+        dst_node: str | Callable[..., AsyncGenerator],
+    ) -> None:
         """Add an edge.
 
         Parameters
         ----------
-        src_node : str | Callable
+        src_node : str | Callable[..., AsyncGenerator]
             The source node, either the function name or the function itself.
-        dst_node : str | Callable
+        dst_node : str | Callable[..., AsyncGenerator]
             The destination node, either the function name or the function itself.
         """
         if not isinstance(src_node, str):

{async_graph_data_flow-1.5.1 → async_graph_data_flow-1.6.0/src/async_graph_data_flow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: async-graph-data-flow
-Version: 1.5.1
+Version: 1.6.0
 Summary: Asynchronous functions that pass data along a directed acyclic graph
 Author-email: Civis Analytics <opensource@civisanalytics.com>
 License: BSD 3-Clause
@@ -24,14 +24,14 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Provides-Extra: dev
-Requires-Dist: black==24.10.0; extra == "dev"
-Requires-Dist: flake8==7.1.1; extra == "dev"
-Requires-Dist: pytest==8.3.3; extra == "dev"
-Requires-Dist: mypy==1.13.0; extra == "dev"
+Requires-Dist: black==25.1.0; extra == "dev"
+Requires-Dist: flake8==7.1.2; extra == "dev"
+Requires-Dist: pytest==8.3.5; extra == "dev"
+Requires-Dist: mypy==1.15.0; extra == "dev"
 Requires-Dist: build==1.2.2; extra == "dev"
-Requires-Dist: twine==5.1.1; extra == "dev"
+Requires-Dist: twine==6.1.0; extra == "dev"
 Provides-Extra: docs
-Requires-Dist: Sphinx==8.1.3; extra == "docs"
+Requires-Dist: Sphinx==8.2.3; extra == "docs"
 Requires-Dist: furo==2024.8.6; extra == "docs"
 Requires-Dist: numpydoc==1.8.0; extra == "docs"
 Requires-Dist: sphinx-copybutton==0.5.2; extra == "docs"
@@ -77,7 +77,7 @@ black --check src tests examples
 
 ## Building Documentation
 
-We use the Sphinx framework. The documentation source files are in `docs/source/`.
+We use the Sphinx framework. The documentation source files are in `docs/`.
 These files can be updated as necessary.
 
 The public documentation is accessible at https://async-graph-data-flow.readthedocs.io.

{async_graph_data_flow-1.5.1 → async_graph_data_flow-1.6.0}/src/async_graph_data_flow.egg-info/requires.txt

@@ -1,14 +1,14 @@
 
 [dev]
-black==24.10.0
-flake8==7.1.1
-pytest==8.3.3
-mypy==1.13.0
+black==25.1.0
+flake8==7.1.2
+pytest==8.3.5
+mypy==1.15.0
 build==1.2.2
-twine==5.1.1
+twine==6.1.0
 
 [docs]
-Sphinx==8.1.3
+Sphinx==8.2.3
 furo==2024.8.6
 numpydoc==1.8.0
 sphinx-copybutton==0.5.2

{async_graph_data_flow-1.5.1 → async_graph_data_flow-1.6.0}/tests/test_graph.py

@@ -84,6 +84,7 @@ class TestAsyncGraphAddNode:
                 "halt_on_exception": False,
                 "max_tasks": 1,
                 "name": "extract_node",
+                "queue": None,
                 "queue_size": 10_000,
                 "unpack_input": True,
             },
@@ -92,6 +93,7 @@ class TestAsyncGraphAddNode:
                 "halt_on_exception": False,
                 "max_tasks": 1,
                 "name": "transform_node",
+                "queue": None,
                 "queue_size": 10_000,
                 "unpack_input": True,
             },
@@ -100,6 +102,7 @@ class TestAsyncGraphAddNode:
                 "halt_on_exception": False,
                 "max_tasks": 1,
                 "name": "load_node",
+                "queue": None,
                 "queue_size": 10_000,
                 "unpack_input": True,
             },