fpstreams 2.2.1__tar.gz → 2.3.0__tar.gz

{fpstreams-2.2.1 → fpstreams-2.3.0}/CHANGELOG.md

@@ -3,7 +3,36 @@
 This file records user-visible and compatibility-relevant changes in fpstreams 2,
 including changed defaults.
 
-## Unreleased
+## 2.3.0 - 2026-10-15
+
+### Fixed
+
+- Async callback return types resolve correctly for `AsyncFlow.map()`,
+  `map_async()`, and `map_batches()`. Mixed immediate and awaitable returns
+  retain the resolved element type.
+
+### Changed
+
+- Supported single-stage Python `with_columns()` and `select()` queries reuse a
+  loop template with separate callback bindings for each execution, reducing
+  fixed setup cost while retaining computed-column live selectors and projection
+  snapshots for each row.
+- Explicit sorted grouping with one key avoids a temporary selector generator
+  and wrapper tuple per row while preserving live callbacks, key validation,
+  lookahead, and callback error behavior.
+
+### Added
+
+- `Flow.map_batches(function, size=...)` and `AsyncFlow.map_batches(...)` call a
+  mapper once per tuple batch, including a shorter final batch. Each call emits
+  one result. Sync batches inherit `parallel()` settings; async batches support
+  concurrency, ordering, per-batch timeouts, and bounded buffering.
+- `Flow.map_parallel()` and `Flow.parallel()` accept a caller-owned `executor`.
+  Queries cancel their queued work and wait for their own running mappers while
+  leaving the shared pool open. Existing worker/backend defaults remain in place
+  when no executor is supplied.
+- Streaming analytics examples show generator input, valid weighted denominators,
+  parameterized SQLite selection, and atomic writes across multiple batches.
 
 ## 2.2.1 - 2026-10-07
 

{fpstreams-2.2.1 → fpstreams-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fpstreams
-Version: 2.2.1
+Version: 2.3.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
@@ -66,6 +66,15 @@ Lazy Python pipelines for iterables and records, with bounded async concurrency.
 
 Python 3.11 or newer is required.
 
+The current stable release is [2.3.0](https://github.com/steventimes/fpstreams/releases/tag/v2.3.0).
+It adds explicit `map_batches()` for sync and async pipelines, reusable mapping
+executors, and async callback typing fixes. See the
+[changelog](https://github.com/steventimes/fpstreams/blob/master/CHANGELOG.md) for its scope and the
+[performance guide](https://steventimes.github.io/fpstreams/user-guide/performance/#process-mapping-and-explicit-batches)
+for batch and executor examples. The
+[roadmap](https://steventimes.github.io/fpstreams/roadmap/#next-development-steps)
+lists the next optimization candidates and their acceptance criteria.
+
 ## Install
 
 Install the latest stable release:

{fpstreams-2.2.1 → fpstreams-2.3.0}/README.md

@@ -11,6 +11,15 @@ Lazy Python pipelines for iterables and records, with bounded async concurrency.
 
 Python 3.11 or newer is required.
 
+The current stable release is [2.3.0](https://github.com/steventimes/fpstreams/releases/tag/v2.3.0).
+It adds explicit `map_batches()` for sync and async pipelines, reusable mapping
+executors, and async callback typing fixes. See the
+[changelog](https://github.com/steventimes/fpstreams/blob/master/CHANGELOG.md) for its scope and the
+[performance guide](https://steventimes.github.io/fpstreams/user-guide/performance/#process-mapping-and-explicit-batches)
+for batch and executor examples. The
+[roadmap](https://steventimes.github.io/fpstreams/roadmap/#next-development-steps)
+lists the next optimization candidates and their acceptance criteria.
+
 ## Install
 
 Install the latest stable release:

{fpstreams-2.2.1 → fpstreams-2.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "fpstreams"
-version = "2.2.1"
+version = "2.3.0"
 description = "Lazy Python pipelines for iterables and records, with bounded async concurrency."
 readme = "README.md"
 authors = [

{fpstreams-2.2.1 → fpstreams-2.3.0}/rust/Cargo.lock

@@ -4,7 +4,7 @@ version = 4
 
 [[package]]
 name = "fpstreams-native"
-version = "2.2.1"
+version = "2.3.0"
 dependencies = [
  "pyo3",
  "pyo3-build-config",

{fpstreams-2.2.1 → fpstreams-2.3.0}/rust/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "fpstreams-native"
-version = "2.2.1"
+version = "2.3.0"
 edition = "2024"
 rust-version = "1.85"
 description = "Native fused execution kernels for fpstreams."

{fpstreams-2.2.1 → fpstreams-2.3.0}/src/fpstreams/__init__.py

@@ -42,7 +42,7 @@ from .streams import (
 from .tabular import Rows, SpillLimits
 from .tabular.factory import rows as rows
 
-__version__ = "2.2.1"
+__version__ = "2.3.0"
 Stream = Flow
 ParallelStream = Flow
 

{fpstreams-2.2.1 → fpstreams-2.3.0}/src/fpstreams/execution/_rows_fusion.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import ast
 from collections.abc import Callable, Iterator
 from itertools import chain
+from types import FunctionType
 from typing import Any, cast
 
 from ..errors import SelectionError
@@ -74,6 +75,33 @@ RowsFilterSink = Callable[[list[Any], Iterator[Any]], tuple[Any | None, bool]]
 I64RowFilterPlan = tuple[str, tuple[tuple[int, int], ...], bool]
 
 
+def _row_map_loop_template(
+    _source: Iterator[Any],
+    _fpstreams_slot_0: Callable[[Any], tuple[bool, Any]],
+    _fpstreams_slot_1: Callable[[Any], type[Any]],
+    _fpstreams_slot_2: type[dict[str, Any]],
+    _fpstreams_slot_3: type[StopIteration],
+    _fpstreams_slot_4: Callable[[Any], Any],
+    _fpstreams_slot_5: type[StopIteration],
+) -> Iterator[Any]:
+    """Keep the one-stage generated loop's dispatch, bindings, and row lifetimes."""
+    for _row in _source:
+        if _fpstreams_slot_1(_row) is not _fpstreams_slot_2:
+            try:
+                _fallback = _fpstreams_slot_0(_row)
+            except _fpstreams_slot_3:
+                return
+            if _fallback[0]:
+                yield _fallback[1]
+            continue
+        _current = _row
+        try:
+            _current = _fpstreams_slot_4(_current)
+        except _fpstreams_slot_5:
+            return
+        yield _current
+
+
 def execute_rows_fusion(
     iterator: Iterator[Any],
     operations: tuple[RowsOperation, ...],
@@ -391,6 +419,28 @@ class _RowsLoopBuilder:
 
     def compile(self) -> RowsLoop:
         """Compile a generator whose globals expose no builtins or source-text interpolation."""
+        if (
+            len(self._operations) == 1
+            and isinstance(operation := self._operations[0], MapOp)
+            and self._descriptors[0].kind in {"select", "with_columns"}
+        ):
+            slots = (
+                _fallback_processor(self._operations),
+                type,
+                dict,
+                StopIteration,
+                operation.function,
+                StopIteration,
+            )
+            namespace: dict[str, Any] = {"__builtins__": {}, "_fpstreams_slots": slots}
+            loop = FunctionType(
+                _row_map_loop_template.__code__,
+                namespace,
+                name="_fpstreams_rows_loop",
+                argdefs=slots,
+            )
+            namespace["_fpstreams_rows_loop"] = loop
+            return cast(RowsLoop, loop)
         body = self._loop_body()
         return cast(
             RowsLoop,

{fpstreams-2.2.1 → fpstreams-2.3.0}/src/fpstreams/execution/sorted_streams.py

@@ -21,6 +21,11 @@ class _End:
 
 _EOF = _End()
 _UNSET = object()
+_BUILTIN_TUPLE: type[Any] = tuple
+_BUILTIN_LEN = len
+_BUILTIN_TYPE = type
+_BUILTIN_STOP_ITERATION = StopIteration
+_BUILTIN_RUNTIME_ERROR = RuntimeError
 
 
 class _KeyShape:
@@ -146,8 +151,24 @@ def aggregate_sorted_rows(
     """Finish adjacent groups without buffering their rows or future groups."""
     source = values if isinstance(values, Source) else Source.from_iterable(values)
 
+    single = (
+        _BUILTIN_TYPE(keys) is cast(Any, _BUILTIN_TUPLE)
+        and _BUILTIN_LEN(keys) == 1
+        and _BUILTIN_TYPE(keys[0]) is cast(Any, _BUILTIN_TUPLE)
+        and _BUILTIN_LEN(keys[0]) == 2
+    )
+
     def select_key(row: Any) -> Any:
-        selected = tuple(selector(row) for _name, selector in keys)
+        if single and tuple is _BUILTIN_TUPLE:
+            try:
+                selected_key = keys[0][1](row)
+            except _BUILTIN_STOP_ITERATION as error:
+                raise _BUILTIN_RUNTIME_ERROR("generator raised StopIteration") from error
+            if len is _BUILTIN_LEN:
+                return selected_key
+            selected = (selected_key,)
+        else:
+            selected = tuple(selector(row) for _name, selector in keys)
         return selected[0] if len(selected) == 1 else selected
 
     cursor = _SortedCursor(source, select_key, side="input")

{fpstreams-2.2.1 → fpstreams-2.3.0}/src/fpstreams/execution/sync_ops.py

@@ -8,9 +8,10 @@ up those local resources themselves.
 from __future__ import annotations
 
 from collections import deque
-from collections.abc import Callable, Iterator
+from collections.abc import Callable, Iterable, Iterator
 from concurrent.futures import (
     FIRST_COMPLETED,
+    Executor,
     Future,
     ProcessPoolExecutor,
     ThreadPoolExecutor,
@@ -86,29 +87,30 @@ def _map_parallel(iterator: Iterator[Any], operation: ParallelMapOp) -> Iterator
     At most buffer futures remain queued. Ordered mode awaits futures in submission
     order; unordered mode yields completed sets as they become available. Callback
     failures propagate through Future.result, and early exit cancels queued work
-    before waiting for executor shutdown.
+    before waiting for this map's running callbacks. Only an owned pool is shut down.
     """
-    executor = (
-        ThreadPoolExecutor(max_workers=operation.workers)
-        if operation.backend == "thread"
-        else ProcessPoolExecutor(
-            max_workers=operation.workers,
-            mp_context=get_context("spawn"),
+    executor = operation.executor
+    owned = executor is None
+    if executor is None:
+        executor = (
+            ThreadPoolExecutor(max_workers=operation.workers)
+            if operation.backend == "thread"
+            else ProcessPoolExecutor(
+                max_workers=operation.workers,
+                mp_context=get_context("spawn"),
+            )
         )
-    )
     if operation.ordered:
         pending: deque[Future[Any]] = deque()
         try:
             for item in iterator:
                 pending.append(executor.submit(operation.function, item))
                 if len(pending) >= operation.buffer:
-                    yield pending.popleft().result()
+                    yield _pop_parallel_result(pending)
             while pending:
-                yield pending.popleft().result()
+                yield _pop_parallel_result(pending)
         finally:
-            for future in pending:
-                future.cancel()
-            executor.shutdown(wait=True, cancel_futures=True)
+            _close_parallel_work(executor, pending, owned=owned)
         return
 
     pending_set: set[Future[Any]] = set()
@@ -124,9 +126,30 @@ def _map_parallel(iterator: Iterator[Any], operation: ParallelMapOp) -> Iterator
             for future in done:
                 yield future.result()
     finally:
-        for future in pending_set:
-            future.cancel()
+        _close_parallel_work(executor, pending_set, owned=owned)
+
+
+def _pop_parallel_result(pending: deque[Future[Any]]) -> Any:
+    """Keep the awaited future registered until its result settles."""
+    result = pending[0].result()
+    pending.popleft()
+    return result
+
+
+def _close_parallel_work(
+    executor: Executor, futures: Iterable[Future[Any]], *, owned: bool
+) -> None:
+    """Cancel this map's queued work and wait without taking ownership of a borrowed pool."""
+    pending = tuple(futures)
+    for future in pending:
+        future.cancel()
+    if owned:
         executor.shutdown(wait=True, cancel_futures=True)
+        return
+    running = tuple(future for future in pending if not future.cancelled())
+    if running:
+        # Waiting does not surface unconsumed callback errors or swallow caller interrupts.
+        wait(running)
 
 
 def _tap(iterator: Iterator[Any], operation: TapOp) -> Iterator[Any]:

{fpstreams-2.2.1 → fpstreams-2.3.0}/src/fpstreams/planning/explain.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
 from dataclasses import dataclass
 from typing import Any
 
@@ -24,6 +25,17 @@ from .sync import FilterOp, MapOp, Operation, ParallelMapOp, TapOp
 _FUSABLE = (MapOp, FilterOp, TapOp)
 
 
+def _parallel_backend(operation: ParallelMapOp) -> str:
+    """Describe the executor actually used, including caller-owned pools."""
+    if operation.executor is None:
+        return operation.backend
+    if isinstance(operation.executor, ThreadPoolExecutor):
+        return "thread"
+    if isinstance(operation.executor, ProcessPoolExecutor):
+        return "process"
+    return "executor"
+
+
 def _python_stages(operations: tuple[Operation, ...]) -> list[dict[str, Any]]:
     """Return Python stages, fusing adjacent map/filter/tap nodes."""
     stages: list[dict[str, Any]] = []
@@ -49,7 +61,9 @@ def _python_stages(operations: tuple[Operation, ...]) -> list[dict[str, Any]]:
             stages.append(
                 {
                     "engine": (
-                        operation.backend if isinstance(operation, ParallelMapOp) else "python"
+                        _parallel_backend(operation)
+                        if isinstance(operation, ParallelMapOp)
+                        else "python"
                     ),
                     "operations": [operation.name],
                     "fused": False,

{fpstreams-2.2.1 → fpstreams-2.3.0}/src/fpstreams/planning/sync.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from collections.abc import Callable
+from concurrent.futures import Executor
 from dataclasses import dataclass
 from os import PathLike
 from typing import Any, Literal
@@ -22,6 +23,7 @@ class ParallelSettings:
     backend: ParallelBackend
     ordered: bool
     buffer: int
+    executor: Executor | None = None
 
 
 @dataclass(frozen=True, slots=True)
@@ -42,6 +44,7 @@ class ParallelMapOp:
     ordered: bool
     buffer: int
     name: str = "map_parallel"
+    executor: Executor | None = None
 
 
 @dataclass(frozen=True, slots=True)

{fpstreams-2.2.1 → fpstreams-2.3.0}/src/fpstreams/storage/spill_store.py

@@ -271,7 +271,11 @@ class SpillStore:
         yield from self.read_path(run.path)
 
     def read_path(self, path: Path) -> Iterator[object]:
-        """Read a registered framed path through the query's tracked I/O boundary."""
+        """Read a query-owned framed path through tracked I/O.
+
+        Callers supply paths produced by this store. This reader does not validate
+        registration or make pickle safe for externally supplied files.
+        """
         from ..runtime.failpoints import hit
         from ..runtime.resources import _add_cleanup_failure
 

{fpstreams-2.2.1 → fpstreams-2.3.0}/src/fpstreams/streams/async_flow.py

@@ -6,7 +6,7 @@ import asyncio
 import operator
 import os
 from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Iterable
-from typing import Any, Generic, TypeVar, cast
+from typing import Any, Generic, TypeVar, cast, overload
 
 from ..expressions.selectors import Selector, compile_selector
 from ..planning.async_ import (
@@ -340,6 +340,28 @@ class AsyncFlow(AsyncFlowTerminalsMixin[T], Generic[T]):
             )
         )
 
+    @overload
+    def map_async(
+        self,
+        function: Callable[[T], Awaitable[R]],
+        *,
+        concurrency: int = 8,
+        ordered: bool = True,
+        timeout: float | None = None,
+        buffer: int | None = None,
+    ) -> AsyncFlow[R]: ...
+
+    @overload
+    def map_async(
+        self,
+        function: Callable[[T], R | Awaitable[R]],
+        *,
+        concurrency: int = 8,
+        ordered: bool = True,
+        timeout: float | None = None,
+        buffer: int | None = None,
+    ) -> AsyncFlow[R]: ...
+
     def map_async(
         self,
         function: Callable[[T], R | Awaitable[R]],
@@ -353,12 +375,14 @@ class AsyncFlow(AsyncFlowTerminalsMixin[T], Generic[T]):
 
         Concurrency is bounded. Ordered mode delays later completed results until earlier inputs
         finish, while the buffer permits completed work to refill active mapper slots.
+        Callbacks run on the event loop. Timeout and cancellation are cooperative and cannot
+        interrupt a callback that blocks the loop.
 
         Args:
             function: Sync or async mapper called once for each source item.
             concurrency: Maximum mapper calls in flight.
             ordered: Emit in source order when true, otherwise in task-completion order.
-            timeout: Optional per-item deadline covering mapper invocation and awaiting its result.
+            timeout: Optional cooperative per-item timeout while awaiting the mapper result.
             buffer: Maximum submitted results not yet emitted, or twice ``concurrency`` by default.
 
         Returns:
@@ -379,6 +403,12 @@ class AsyncFlow(AsyncFlowTerminalsMixin[T], Generic[T]):
         operation = _MapAsync(function, concurrency, ordered, timeout, buffer)
         return cast(AsyncFlow[R], self._append(operation))
 
+    @overload
+    def map(self, function: Callable[[T], Awaitable[R]]) -> AsyncFlow[R]: ...
+
+    @overload
+    def map(self, function: Callable[[T], R | Awaitable[R]]) -> AsyncFlow[R]: ...
+
     def map(self, function: Callable[[T], R | Awaitable[R]]) -> AsyncFlow[R]:
         """Apply a synchronous or asynchronous function to each item in order.
 
@@ -912,6 +942,68 @@ class AsyncFlow(AsyncFlowTerminalsMixin[T], Generic[T]):
 
     batch = chunk
 
+    @overload
+    def map_batches(
+        self,
+        function: Callable[[tuple[T, ...]], Awaitable[R]],
+        *,
+        size: int,
+        concurrency: int = 1,
+        ordered: bool = True,
+        timeout: float | None = None,
+        buffer: int | None = None,
+    ) -> AsyncFlow[R]: ...
+
+    @overload
+    def map_batches(
+        self,
+        function: Callable[[tuple[T, ...]], R | Awaitable[R]],
+        *,
+        size: int,
+        concurrency: int = 1,
+        ordered: bool = True,
+        timeout: float | None = None,
+        buffer: int | None = None,
+    ) -> AsyncFlow[R]: ...
+
+    def map_batches(
+        self,
+        function: Callable[[tuple[T, ...]], R | Awaitable[R]],
+        *,
+        size: int,
+        concurrency: int = 1,
+        ordered: bool = True,
+        timeout: float | None = None,
+        buffer: int | None = None,
+    ) -> AsyncFlow[R]:
+        """Apply a sync or async function to consecutive tuples, one result per batch.
+
+        Batches are lazy and include a shorter final batch. Empty input does not call the
+        function. Results stay intact; use `flat_map(iter)` to expand returned iterables.
+        Awaitable results are resolved before emission. Early exit cancels pending callbacks
+        and closes the source through the existing async map scheduler.
+        Callbacks run on the event loop: timeout and cancellation are cooperative and cannot
+        interrupt a callback that blocks the loop.
+
+        Args:
+            function: Receives up to `size` consecutive items and returns a result or awaitable.
+            size: Maximum input items per batch.
+            concurrency: Maximum batch callbacks in flight; one by default.
+            ordered: Emit in batch order when true, otherwise in completion order.
+            timeout: Optional cooperative per-batch timeout while awaiting the callback.
+            buffer: Maximum submitted batches not yet emitted; twice concurrency by default.
+
+        Returns:
+            A lazy async flow containing one resolved result per batch.
+
+        Raises:
+            TypeError: If size does not support the integer index protocol.
+            ValueError: If size, concurrency, buffer, or timeout is not positive.
+        """
+        return self.chunk(size).map_async(
+            function, concurrency=concurrency, ordered=ordered, timeout=timeout, buffer=buffer
+        )
+
     def window(self, size: int, *, step: int = 1) -> AsyncFlow[tuple[T, ...]]:
         """Emit sliding tuples of size with the requested step.