async-graph-data-flow 1.5.0__tar.gz → 1.6.0__tar.gz

{async_graph_data_flow-1.5.0 → async_graph_data_flow-1.6.0}/CHANGELOG.md

@@ -19,6 +19,27 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Security
 
+## [1.6.0] - 2025-03-19
+
+### Added
+- Added support for custom edge behaviors, through the new `queue` argument at `add_node`
+  for a custom queue object. (#14)
+
+### Deprecated
+- Deprecated the `queue_size` argument at `add_node` -- use the new `queue` instead. (#14)
+
+### Security
+- Resolved CVE-2024-56326, CVE-2024-56201, and CVE-2025-27516
+  by updating jinja2 to v3.1.6 in requirements for Sphinx docs. (#14)
+
+## [1.5.1] - 2024-11-07
+
+### Added
+- Added static type checking using mypy. (#12)
+
+### Fixed
+- Fixed documentation to actually show example code. (#12)
+
 ## [1.5.0] - 2024-10-30
 
 ### Added

{async_graph_data_flow-1.5.0/src/async_graph_data_flow.egg-info → async_graph_data_flow-1.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: async-graph-data-flow
-Version: 1.5.0
+Version: 1.6.0
 Summary: Asynchronous functions that pass data along a directed acyclic graph
 Author-email: Civis Analytics <opensource@civisanalytics.com>
 License: BSD 3-Clause
@@ -24,13 +24,14 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Provides-Extra: dev
-Requires-Dist: black==24.10.0; extra == "dev"
-Requires-Dist: flake8==7.1.1; extra == "dev"
-Requires-Dist: pytest==8.3.3; extra == "dev"
+Requires-Dist: black==25.1.0; extra == "dev"
+Requires-Dist: flake8==7.1.2; extra == "dev"
+Requires-Dist: pytest==8.3.5; extra == "dev"
+Requires-Dist: mypy==1.15.0; extra == "dev"
 Requires-Dist: build==1.2.2; extra == "dev"
-Requires-Dist: twine==5.1.1; extra == "dev"
+Requires-Dist: twine==6.1.0; extra == "dev"
 Provides-Extra: docs
-Requires-Dist: Sphinx==8.1.3; extra == "docs"
+Requires-Dist: Sphinx==8.2.3; extra == "docs"
 Requires-Dist: furo==2024.8.6; extra == "docs"
 Requires-Dist: numpydoc==1.8.0; extra == "docs"
 Requires-Dist: sphinx-copybutton==0.5.2; extra == "docs"
@@ -76,7 +77,7 @@ black --check src tests examples
 
 ## Building Documentation
 
-We use the Sphinx framework. The documentation source files are in `docs/source/`.
+We use the Sphinx framework. The documentation source files are in `docs/`.
 These files can be updated as necessary.
 
 The public documentation is accessible at https://async-graph-data-flow.readthedocs.io.

{async_graph_data_flow-1.5.0 → async_graph_data_flow-1.6.0}/README.md

@@ -38,7 +38,7 @@ black --check src tests examples
 
 ## Building Documentation
 
-We use the Sphinx framework. The documentation source files are in `docs/source/`.
+We use the Sphinx framework. The documentation source files are in `docs/`.
 These files can be updated as necessary.
 
 The public documentation is accessible at https://async-graph-data-flow.readthedocs.io.

{async_graph_data_flow-1.5.0 → async_graph_data_flow-1.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "async-graph-data-flow"
-version = "1.5.0"
+version = "1.6.0"
 description = "Asynchronous functions that pass data along a directed acyclic graph"
 readme = "README.md"
 requires-python = ">= 3.10"
@@ -40,16 +40,17 @@ Source = "https://github.com/civisanalytics/async-graph-data-flow"
 [project.optional-dependencies]
 dev = [
     # Running tests and linters
-    "black == 24.10.0",
-    "flake8 == 7.1.1",
-    "pytest == 8.3.3",
+    "black == 25.1.0",
+    "flake8 == 7.1.2",
+    "pytest == 8.3.5",
+    "mypy == 1.15.0",
 
     # Managing source distributions
     "build == 1.2.2",
-    "twine == 5.1.1",
+    "twine == 6.1.0",
 ]
 docs = [
-    "Sphinx == 8.1.3",
+    "Sphinx == 8.2.3",
     "furo == 2024.8.6",
     "numpydoc == 1.8.0",
     "sphinx-copybutton == 0.5.2",

{async_graph_data_flow-1.5.0 → async_graph_data_flow-1.6.0}/src/async_graph_data_flow/executor.py

@@ -4,7 +4,8 @@ import logging
 import time
 import traceback
 from collections import deque
-from typing import Any, Iterable
+from collections.abc import Iterable
+from typing import Any
 
 from .graph import AsyncGraph, InvalidAsyncGraphError
 
@@ -20,7 +21,7 @@ class AsyncExecutor:
         self,
         graph: AsyncGraph,
         *,
-        logger: logging.Logger = None,
+        logger: logging.Logger | None = None,
         max_exceptions: int = 1_000,
     ):
         """Initialize an executor.
@@ -41,23 +42,23 @@ class AsyncExecutor:
         if not isinstance(self._graph, AsyncGraph):
             raise TypeError(f"{self._graph} must be an AsyncGraph instance")
 
-        self._node_queues = {}
-        self._consumer_tasks = {}
+        self._node_queues: dict[str, asyncio.Queue] = {}
+        self._consumer_tasks: dict[str, asyncio.Task] = {}
         self._halt_pipeline_execution = False
         self._logger = logger if logger else _LOG
         self._max_exceptions = max_exceptions
 
-        self._data_flow_stats = None
+        self._data_flow_stats: dict[str, dict[str, int]] | None = None
         self._data_flow_logging_lock = asyncio.Lock()
         self._data_flow_logging = False
         self._data_flow_logging_node_format = _DEFAULT_DATA_FLOW_LOGGING_NODE_FORMAT
         self._data_flow_logging_time_interval = _DEFAULT_DATA_FLOW_LOGGING_TIME_INTERVAL
-        self._data_flow_logging_node_filter = self._graph._nodes.keys()
-        self._data_flow_logging_last_timestamp = 0
+        self._data_flow_logging_node_filter: Iterable[str] = self._graph._nodes.keys()
+        self._data_flow_logging_last_timestamp = 0.0
 
-        self._start_node_args = None
+        self._start_node_args: dict[str, tuple] | None = None
 
-        self._exceptions = None
+        self._exceptions: dict[str, deque[Exception]] | None = None
 
     @property
     def graph(self) -> AsyncGraph:
@@ -72,7 +73,7 @@ class AsyncExecutor:
         raised from the node.
         """
         if self._exceptions is None:
-            return
+            return None
         from_deque_to_list = {}
         for node_name, excs in self._exceptions.items():
             # `excs` is a deque. Turning it into a list for user-friendliness.
@@ -152,7 +153,7 @@ class AsyncExecutor:
         """Turn off data flow logging."""
         self._data_flow_logging = False
 
-    async def _log_data_flow_nodes(self):
+    def _log_data_flow_nodes(self):
         for node, flow in self._data_flow_stats.items():
             if (
                 self._data_flow_logging_node_filter
@@ -186,7 +187,7 @@ class AsyncExecutor:
                             current_timestamp - self._data_flow_logging_last_timestamp
                             > self._data_flow_logging_time_interval
                         ):
-                            await self._log_data_flow_nodes()
+                            self._log_data_flow_nodes()
                             self._data_flow_logging_last_timestamp = current_timestamp
 
                 queue = self._node_queues[node_name]
@@ -202,17 +203,17 @@ class AsyncExecutor:
                 try:
                     if len(params) == 0:
                         coro = node.func()
-                    elif node.unpack_input and (
-                        (is_tuple := isinstance(data, tuple)) or isinstance(data, dict)
-                    ):
-                        coro = node.func(*data) if is_tuple else node.func(**data)
+                    elif node.unpack_input and isinstance(data, tuple):
+                        coro = node.func(*data)
+                    elif node.unpack_input and isinstance(data, dict):
+                        coro = node.func(**data)
                     else:
                         coro = node.func(data)
                 except asyncio.CancelledError:
                     continue
                 except Exception as exc:
-                    await self._update_data_flow_error_stats(node_name)
-                    await self._update_exceptions(node_name, exc)
+                    self._update_data_flow_error_stats(node_name)
+                    self._update_exceptions(node_name, exc)
                     self._logger.error(traceback.format_exc())
                     if self._graph.halt_on_exception or node.halt_on_exception:
                         self._logger.error(
@@ -238,8 +239,8 @@ class AsyncExecutor:
                     except asyncio.CancelledError:
                         break
                     except Exception as exc:
-                        await self._update_data_flow_error_stats(node_name)
-                        await self._update_exceptions(node_name, exc)
+                        self._update_data_flow_error_stats(node_name)
+                        self._update_exceptions(node_name, exc)
                         self._logger.error(traceback.format_exc())
                         if self._graph.halt_on_exception or node.halt_on_exception:
                             # close current agen
@@ -255,30 +256,39 @@ class AsyncExecutor:
                             continue
                     else:
                         node_edges = self._graph._nodes_to_edges[node_name]
-                        await self._update_data_flow_in_out_stats(node_name, node_edges)
+                        self._update_data_flow_in_out_stats(node_name, node_edges)
                         await self._add_to_node_queue(node_edges, next_data_item)
 
                 queue.task_done()
             except asyncio.CancelledError:
                 break
 
-    async def _update_data_flow_in_out_stats(self, in_node: str, out_nodes: set[str]):
+    def _update_data_flow_in_out_stats(self, in_node: str, out_nodes: set[str]):
+        if self._data_flow_stats is None:
+            return None
         self._data_flow_stats[in_node]["out"] += 1
         for node in out_nodes:
             self._data_flow_stats[node]["in"] += 1
 
-    async def _update_data_flow_error_stats(self, node: str):
+    def _update_data_flow_error_stats(self, node: str):
+        if self._data_flow_stats is None:
+            return None
         self._data_flow_stats[node]["err"] += 1
 
-    async def _update_exceptions(self, node: str, exc: Exception):
+    def _update_exceptions(self, node: str, exc: Exception):
+        if self._exceptions is None:
+            return None
         self._exceptions[node].append(exc)
 
     async def _pipeline_execution(self):
-        self._data_flow_stats: dict[str, dict[str, int]] = {}
-        self._exceptions: dict[str, deque[Exception]] = {}
+        self._data_flow_stats = {}
+        self._exceptions = {}
 
         for node_name, node in self._graph._nodes.items():
-            queue = asyncio.Queue(maxsize=node.queue_size)
+            if node.queue is None:
+                queue = asyncio.Queue(maxsize=node.queue_size)
+            else:
+                queue = node.queue
             self._node_queues[node_name] = queue
             self._data_flow_stats[node_name] = {"in": 0, "out": 0, "err": 0}
             self._exceptions[node_name] = deque(maxlen=self._max_exceptions)
@@ -299,7 +309,7 @@ class AsyncExecutor:
         await asyncio.gather(*self._consumer_tasks.values())
 
         if self._data_flow_logging:
-            await self._log_data_flow_nodes()
+            self._log_data_flow_nodes()
 
     def _get_start_node_args(self, start_node_args) -> dict[str, tuple]:
         if start_node_args is None:
@@ -313,7 +323,7 @@ class AsyncExecutor:
                 raise TypeError(f"args for the node '{node}' isn't a tuple: {args}")
         return start_node_args
 
-    def execute(self, start_nodes: dict[str, tuple] = None) -> None:
+    def execute(self, start_nodes: dict[str, tuple] | None = None) -> None:
         """Start executing the functions along the graph.
 
         Parameters

{async_graph_data_flow-1.5.0 → async_graph_data_flow-1.6.0}/src/async_graph_data_flow/graph.py

@@ -1,9 +1,8 @@
+import asyncio
 import inspect
 from collections import OrderedDict
-from typing import Any, Callable, NamedTuple
-
-
-_DEFAULT_NUM_OF_WORKERS = 1
+from collections.abc import AsyncGenerator, Callable
+from typing import Any, NamedTuple
 
 
 class InvalidAsyncGraphError(Exception):
@@ -11,8 +10,9 @@ class InvalidAsyncGraphError(Exception):
 
 
 class _Node(NamedTuple):
-    func: Callable
+    func: Callable[..., AsyncGenerator]
     name: str
+    queue: asyncio.Queue | None
     queue_size: int
     max_tasks: int
     halt_on_exception: bool
@@ -35,12 +35,13 @@ class AsyncGraph:
 
     def add_node(
         self,
-        func: Callable,
+        func: Callable[..., AsyncGenerator],
         *,
         name: str | None = None,
         halt_on_exception: bool = False,
         unpack_input: bool = True,
         max_tasks: int = 1,
+        queue: asyncio.Queue | None = None,
         queue_size: int = 10_000,
         check_async_gen: bool = True,
     ) -> None:
@@ -48,8 +49,8 @@ class AsyncGraph:
 
         Parameters
         ----------
-        func : Callable
-            The function that this node runs.
+        func : Callable[..., AsyncGenerator]
+            The asynchronous generator function that this node runs.
             See notes below for the function's requirements.
         name : str, optional
             The name of this node. If not provided, the ``__name__`` attribute
@@ -65,10 +66,23 @@ class AsyncGraph:
             See notes below for more details.
         max_tasks : int, optional
             The number of tasks that this node runs concurrently.
+        queue : asyncio.Queue, optional
+            The queue object that collects items from this node's source nodes,
+            via ``await queue.put(item)``, and then feed into this node
+            with items retrieved by ``await queue.get()``.
+            This queue object must be an instance of either :class:`~asyncio.Queue` or
+            a subclass of :class:`~asyncio.Queue`.
+            If ``None`` or not given, it defaults to an ``asyncio.Queue()`` with max
+            size set by ``queue_size``.
         queue_size : int, optional
             The maximum number of data items allowed to be
-            in the :class:`~asyncio.Queue` object between this node as a source
-            and its destination node(s).
+            in the queue object between this node as a destination node
+            and its source node(s).
+
+            .. deprecated:: 1.6.0
+                The argument ``queue_size`` is deprecated and will be removed in
+                v2.0.0. To configure the queue size, please use the argument ``queue``
+                for a queue object whose queue size is set.
         check_async_gen : bool, optional
             If ``True`` (the default), the callable ``func`` is verified to be an async
             generator function by :func:`inspect.isasyncgenfunction`.
@@ -132,9 +146,12 @@ class AsyncGraph:
             raise TypeError(f"node '{name}' isn't an async generator function")
         if name in self._nodes:
             raise ValueError(f"node '{name}' already exists in the graph")
+        if queue is not None and not isinstance(queue, asyncio.Queue):
+            raise TypeError(f"queue must be an instance of asyncio.Queue: {queue}")
         self._nodes[name] = _Node(
             func=func,
             name=name,
+            queue=queue,
             queue_size=queue_size,
             max_tasks=max_tasks,
             halt_on_exception=halt_on_exception,
@@ -142,14 +159,18 @@ class AsyncGraph:
         )
         self._nodes_to_edges[name] = set()
 
-    def add_edge(self, src_node: str | Callable, dst_node: str | Callable) -> None:
+    def add_edge(
+        self,
+        src_node: str | Callable[..., AsyncGenerator],
+        dst_node: str | Callable[..., AsyncGenerator],
+    ) -> None:
         """Add an edge.
 
         Parameters
         ----------
-        src_node : str | Callable
+        src_node : str | Callable[..., AsyncGenerator]
             The source node, either the function name or the function itself.
-        dst_node : str | Callable
+        dst_node : str | Callable[..., AsyncGenerator]
             The destination node, either the function name or the function itself.
         """
         if not isinstance(src_node, str):

{async_graph_data_flow-1.5.0 → async_graph_data_flow-1.6.0/src/async_graph_data_flow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: async-graph-data-flow
-Version: 1.5.0
+Version: 1.6.0
 Summary: Asynchronous functions that pass data along a directed acyclic graph
 Author-email: Civis Analytics <opensource@civisanalytics.com>
 License: BSD 3-Clause
@@ -24,13 +24,14 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Provides-Extra: dev
-Requires-Dist: black==24.10.0; extra == "dev"
-Requires-Dist: flake8==7.1.1; extra == "dev"
-Requires-Dist: pytest==8.3.3; extra == "dev"
+Requires-Dist: black==25.1.0; extra == "dev"
+Requires-Dist: flake8==7.1.2; extra == "dev"
+Requires-Dist: pytest==8.3.5; extra == "dev"
+Requires-Dist: mypy==1.15.0; extra == "dev"
 Requires-Dist: build==1.2.2; extra == "dev"
-Requires-Dist: twine==5.1.1; extra == "dev"
+Requires-Dist: twine==6.1.0; extra == "dev"
 Provides-Extra: docs
-Requires-Dist: Sphinx==8.1.3; extra == "docs"
+Requires-Dist: Sphinx==8.2.3; extra == "docs"
 Requires-Dist: furo==2024.8.6; extra == "docs"
 Requires-Dist: numpydoc==1.8.0; extra == "docs"
 Requires-Dist: sphinx-copybutton==0.5.2; extra == "docs"
@@ -76,7 +77,7 @@ black --check src tests examples
 
 ## Building Documentation
 
-We use the Sphinx framework. The documentation source files are in `docs/source/`.
+We use the Sphinx framework. The documentation source files are in `docs/`.
 These files can be updated as necessary.
 
 The public documentation is accessible at https://async-graph-data-flow.readthedocs.io.

{async_graph_data_flow-1.5.0 → async_graph_data_flow-1.6.0}/src/async_graph_data_flow.egg-info/requires.txt

@@ -1,13 +1,14 @@
 
 [dev]
-black==24.10.0
-flake8==7.1.1
-pytest==8.3.3
+black==25.1.0
+flake8==7.1.2
+pytest==8.3.5
+mypy==1.15.0
 build==1.2.2
-twine==5.1.1
+twine==6.1.0
 
 [docs]
-Sphinx==8.1.3
+Sphinx==8.2.3
 furo==2024.8.6
 numpydoc==1.8.0
 sphinx-copybutton==0.5.2

{async_graph_data_flow-1.5.0 → async_graph_data_flow-1.6.0}/tests/test_graph.py

@@ -84,6 +84,7 @@ class TestAsyncGraphAddNode:
                 "halt_on_exception": False,
                 "max_tasks": 1,
                 "name": "extract_node",
+                "queue": None,
                 "queue_size": 10_000,
                 "unpack_input": True,
             },
@@ -92,6 +93,7 @@ class TestAsyncGraphAddNode:
                 "halt_on_exception": False,
                 "max_tasks": 1,
                 "name": "transform_node",
+                "queue": None,
                 "queue_size": 10_000,
                 "unpack_input": True,
             },
@@ -100,6 +102,7 @@ class TestAsyncGraphAddNode:
                 "halt_on_exception": False,
                 "max_tasks": 1,
                 "name": "load_node",
+                "queue": None,
                 "queue_size": 10_000,
                 "unpack_input": True,
             },