vcti-dataflow 2.0.0__tar.gz → 3.0.0__tar.gz

vcti_dataflow-3.0.0/PKG-INFO

@@ -0,0 +1,167 @@
+Metadata-Version: 2.4
+Name: vcti-dataflow
+Version: 3.0.0
+Summary: vcti-flow bindings for array data: a DataNode payload binding and a FieldSet table binding (sources, transformers, reducers, sinks).
+Author: Visual Collaboration Technologies Inc.
+Requires-Python: <3.15,>=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: vcti-flow>=2.0.0
+Requires-Dist: vcti-datanode>=2.0.0
+Requires-Dist: vcti-fieldset[datanode]>=2.0.0
+Requires-Dist: numpy>=1.24
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+Provides-Extra: lint
+Requires-Dist: ruff; extra == "lint"
+Provides-Extra: typecheck
+Requires-Dist: mypy; extra == "typecheck"
+Dynamic: license-file
+
+# Data Flow
+
+vcti-flow bindings for array data: a **DataNode** payload binding and a
+**FieldSet** table binding — sources, transformers, reducers, and sinks.
+
+## Overview
+
+[vcti-flow](https://github.com/vcollab/vcti-python-flow) is a payload-agnostic
+framework for composing flow graphs — it never inspects the values flowing
+through it. This package binds that framework to two concrete payloads so you
+subclass ready-bound node kinds instead of writing `Source[...]` everywhere:
+
+- **`vcti.flow.data`** — the [vcti-datanode](https://github.com/vcollab/vcti-python-datanode)
+  `DataNode` binding: one array plus layered attributes behind a data source.
+  Node kinds `Source` / `Transformer` / `Reducer` / `Sink` / `Observer`, plus
+  `from_array` (eager) and `ArraySource` (lazy) payload builders.
+- **`vcti.flow.fieldset`** — the [vcti-fieldset](https://github.com/vcollab/vcti-python-fieldset)
+  `FieldSet` binding: a **table** of named columns. This is where column and row
+  reshaping lives — select / drop / rename / cast / compute, filter / sort /
+  slice, and the N→1 combiners merge / concat. The nodes are thin wrappers over
+  FieldSet's own copy-free `project()` / `rows()` builders.
+
+Use the DataNode binding for single-array flows; use the FieldSet binding to
+reshape multi-column tables. `DataNodesSource` / `WriteDataNodes` bridge between
+them (a group of DataNodes ⇆ a FieldSet).
+
+## Installation
+
+```bash
+pip install vcti-dataflow
+```
+
+### In `pyproject.toml` dependencies
+
+```toml
+dependencies = [
+    "vcti-dataflow>=3.0.0",
+]
+```
+
+---
+
+## Quick Start — DataNode binding
+
+```python
+import numpy as np
+from vcti.flow.data import Source, Transformer, DataNode, from_array
+
+# A source produces a DataNode
+class Stress(Source):
+    def load(self) -> DataNode:
+        return from_array(np.array([1.0, 2.0, 3.0]), {"units": "MPa"})
+
+# A transformer maps one DataNode to another
+class Scale(Transformer):
+    def __init__(self, factor: float) -> None:
+        super().__init__()
+        self.factor = factor
+
+    def transform(self, record: DataNode) -> DataNode:
+        return from_array(record.load() * self.factor, record.attributes)
+
+result = Scale(2.0).connect(Stress()).execute()
+result.load()               # array([2., 4., 6.])
+result.attributes["units"]  # "MPa"
+```
+
+Reach for the array only when you need it (`record.load()`); a leaf source can
+hand back a `LazyDataSource`-backed node (or use `ArraySource`) to defer a heavy
+read.
+
+## Quick Start — FieldSet binding (tables)
+
+The `vcti.flow.fieldset` nodes carry a `FieldSet` (named columns) and reshape it.
+Column ops (`select` / `drop` / `rename` / `cast`) share columns by reference —
+no data copy; `cast` is a lazy per-column cast. Row ops fold to a single index.
+
+```python
+import numpy as np
+from vcti.datanode import DataNode, EagerDataSource
+from vcti.flow.fieldset import (
+    DataNodesSource, SelectFields, RenameFields, CastFields, ComputeFields,
+    FilterRows, SortRows, MergeFields, ConcatRows, WriteDataNodes,
+)
+
+# Read a group of DataNodes as one FieldSet (lazy — nothing materialises here)
+src = DataNodesSource([
+    DataNode(name="id",     data_source=EagerDataSource(np.array([1, 2, 3]))),
+    DataNode(name="stress", data_source=EagerDataSource(np.array([10.0, 20.0, 30.0]))),
+])
+
+# Reshape: rename, add a computed column, filter and sort rows
+flow = RenameFields({"stress": "s"}).connect(src)
+flow = ComputeFields("s_kpa = s * 1000").connect(flow)
+flow = FilterRows("s > 10").connect(flow)
+flow = SortRows("s", descending=True).connect(flow)
+
+result = flow.execute()                 # a FieldSet
+result.get_values("s_kpa")              # array([30000., 20000.])
+
+# Combine tables — N→1 reducers own the policy (collision / schema)
+wide = MergeFields().connect(flowA).connect(flowB)          # union columns
+tall = ConcatRows().connect(flowA).connect(flowB)           # stack rows
+
+# Write out: FieldSet -> DataNodes -> your writer (I/O stays in your app)
+WriteDataNodes(lambda node: write_to_h5(node)).connect(flow).execute()
+```
+
+Column reshaping is `project()`-backed; row reshaping is `rows()`-backed. See
+[docs/patterns.md](docs/patterns.md) for the full recipe set.
+
+---
+
+## API
+
+### `vcti.flow.data` — DataNode binding
+
+| Symbol | Purpose |
+|--------|---------|
+| `Source` / `Transformer` / `Reducer` / `Sink` / `Observer` | `vcti.flow` kinds bound to `DataNode` |
+| `from_array(array, attributes=None)` | Build a `DataNode` from an in-memory array (eager) |
+| `ArraySource(load_fn, attributes=None)` | Lazy leaf source over a callable returning an array |
+| `DataNode` / `EagerDataSource` / `LazyDataSource` | Re-exported from `vcti-datanode` |
+
+### `vcti.flow.fieldset` — FieldSet binding
+
+| Symbol | Purpose |
+|--------|---------|
+| `Source` / `Transformer` / `Reducer` / `Sink` / `Observer` | `vcti.flow` kinds bound to `FieldSet` |
+| `SelectFields` / `DropFields` / `RenameFields` | Keep / remove / rename columns (share sources, no copy) |
+| `CastFields` / `ComputeFields` / `FreezeFields` | Cast dtypes (lazy), add expression columns, bake an expression to data |
+| `FilterRows` / `SortRows` / `HeadRows` / `SliceRows` | Row selection (fold to one index; slice stays a view) |
+| `MergeFields` / `ConcatRows` | N→1: union columns / stack rows (own collision & schema policy) |
+| `FieldSetSource` | Leaf source over an existing `FieldSet` (or a callable returning one) |
+| `DataNodesSource` / `WriteDataNodes` | Boundary: DataNodes → FieldSet / FieldSet → DataNodes |
+| `for_each_group(source, key_field, factory)` | Fan out into one flow per group |
+| `from_datanodes` / `to_datanodes` | Re-exported from `vcti-fieldset` |
+
+---
+
+## Dependencies
+
+- [vcti-flow](https://github.com/vcollab/vcti-python-flow) (>=2.0.0) — the generic framework
+- [vcti-datanode](https://github.com/vcollab/vcti-python-datanode) (>=2.0.0) — the `DataNode` payload
+- [vcti-fieldset](https://github.com/vcollab/vcti-python-fieldset) `[datanode]` (>=2.0.0) — the `FieldSet` table payload and its copy-free reshaping
+- [numpy](https://numpy.org/) (>=1.24)

vcti_dataflow-3.0.0/README.md

@@ -0,0 +1,146 @@
+# Data Flow
+
+vcti-flow bindings for array data: a **DataNode** payload binding and a
+**FieldSet** table binding — sources, transformers, reducers, and sinks.
+
+## Overview
+
+[vcti-flow](https://github.com/vcollab/vcti-python-flow) is a payload-agnostic
+framework for composing flow graphs — it never inspects the values flowing
+through it. This package binds that framework to two concrete payloads so you
+subclass ready-bound node kinds instead of writing `Source[...]` everywhere:
+
+- **`vcti.flow.data`** — the [vcti-datanode](https://github.com/vcollab/vcti-python-datanode)
+  `DataNode` binding: one array plus layered attributes behind a data source.
+  Node kinds `Source` / `Transformer` / `Reducer` / `Sink` / `Observer`, plus
+  `from_array` (eager) and `ArraySource` (lazy) payload builders.
+- **`vcti.flow.fieldset`** — the [vcti-fieldset](https://github.com/vcollab/vcti-python-fieldset)
+  `FieldSet` binding: a **table** of named columns. This is where column and row
+  reshaping lives — select / drop / rename / cast / compute, filter / sort /
+  slice, and the N→1 combiners merge / concat. The nodes are thin wrappers over
+  FieldSet's own copy-free `project()` / `rows()` builders.
+
+Use the DataNode binding for single-array flows; use the FieldSet binding to
+reshape multi-column tables. `DataNodesSource` / `WriteDataNodes` bridge between
+them (a group of DataNodes ⇆ a FieldSet).
+
+## Installation
+
+```bash
+pip install vcti-dataflow
+```
+
+### In `pyproject.toml` dependencies
+
+```toml
+dependencies = [
+    "vcti-dataflow>=3.0.0",
+]
+```
+
+---
+
+## Quick Start — DataNode binding
+
+```python
+import numpy as np
+from vcti.flow.data import Source, Transformer, DataNode, from_array
+
+# A source produces a DataNode
+class Stress(Source):
+    def load(self) -> DataNode:
+        return from_array(np.array([1.0, 2.0, 3.0]), {"units": "MPa"})
+
+# A transformer maps one DataNode to another
+class Scale(Transformer):
+    def __init__(self, factor: float) -> None:
+        super().__init__()
+        self.factor = factor
+
+    def transform(self, record: DataNode) -> DataNode:
+        return from_array(record.load() * self.factor, record.attributes)
+
+result = Scale(2.0).connect(Stress()).execute()
+result.load()               # array([2., 4., 6.])
+result.attributes["units"]  # "MPa"
+```
+
+Reach for the array only when you need it (`record.load()`); a leaf source can
+hand back a `LazyDataSource`-backed node (or use `ArraySource`) to defer a heavy
+read.
+
+## Quick Start — FieldSet binding (tables)
+
+The `vcti.flow.fieldset` nodes carry a `FieldSet` (named columns) and reshape it.
+Column ops (`select` / `drop` / `rename` / `cast`) share columns by reference —
+no data copy; `cast` is a lazy per-column cast. Row ops fold to a single index.
+
+```python
+import numpy as np
+from vcti.datanode import DataNode, EagerDataSource
+from vcti.flow.fieldset import (
+    DataNodesSource, SelectFields, RenameFields, CastFields, ComputeFields,
+    FilterRows, SortRows, MergeFields, ConcatRows, WriteDataNodes,
+)
+
+# Read a group of DataNodes as one FieldSet (lazy — nothing materialises here)
+src = DataNodesSource([
+    DataNode(name="id",     data_source=EagerDataSource(np.array([1, 2, 3]))),
+    DataNode(name="stress", data_source=EagerDataSource(np.array([10.0, 20.0, 30.0]))),
+])
+
+# Reshape: rename, add a computed column, filter and sort rows
+flow = RenameFields({"stress": "s"}).connect(src)
+flow = ComputeFields("s_kpa = s * 1000").connect(flow)
+flow = FilterRows("s > 10").connect(flow)
+flow = SortRows("s", descending=True).connect(flow)
+
+result = flow.execute()                 # a FieldSet
+result.get_values("s_kpa")              # array([30000., 20000.])
+
+# Combine tables — N→1 reducers own the policy (collision / schema)
+wide = MergeFields().connect(flowA).connect(flowB)          # union columns
+tall = ConcatRows().connect(flowA).connect(flowB)           # stack rows
+
+# Write out: FieldSet -> DataNodes -> your writer (I/O stays in your app)
+WriteDataNodes(lambda node: write_to_h5(node)).connect(flow).execute()
+```
+
+Column reshaping is `project()`-backed; row reshaping is `rows()`-backed. See
+[docs/patterns.md](docs/patterns.md) for the full recipe set.
+
+---
+
+## API
+
+### `vcti.flow.data` — DataNode binding
+
+| Symbol | Purpose |
+|--------|---------|
+| `Source` / `Transformer` / `Reducer` / `Sink` / `Observer` | `vcti.flow` kinds bound to `DataNode` |
+| `from_array(array, attributes=None)` | Build a `DataNode` from an in-memory array (eager) |
+| `ArraySource(load_fn, attributes=None)` | Lazy leaf source over a callable returning an array |
+| `DataNode` / `EagerDataSource` / `LazyDataSource` | Re-exported from `vcti-datanode` |
+
+### `vcti.flow.fieldset` — FieldSet binding
+
+| Symbol | Purpose |
+|--------|---------|
+| `Source` / `Transformer` / `Reducer` / `Sink` / `Observer` | `vcti.flow` kinds bound to `FieldSet` |
+| `SelectFields` / `DropFields` / `RenameFields` | Keep / remove / rename columns (share sources, no copy) |
+| `CastFields` / `ComputeFields` / `FreezeFields` | Cast dtypes (lazy), add expression columns, bake an expression to data |
+| `FilterRows` / `SortRows` / `HeadRows` / `SliceRows` | Row selection (fold to one index; slice stays a view) |
+| `MergeFields` / `ConcatRows` | N→1: union columns / stack rows (own collision & schema policy) |
+| `FieldSetSource` | Leaf source over an existing `FieldSet` (or a callable returning one) |
+| `DataNodesSource` / `WriteDataNodes` | Boundary: DataNodes → FieldSet / FieldSet → DataNodes |
+| `for_each_group(source, key_field, factory)` | Fan out into one flow per group |
+| `from_datanodes` / `to_datanodes` | Re-exported from `vcti-fieldset` |
+
+---
+
+## Dependencies
+
+- [vcti-flow](https://github.com/vcollab/vcti-python-flow) (>=2.0.0) — the generic framework
+- [vcti-datanode](https://github.com/vcollab/vcti-python-datanode) (>=2.0.0) — the `DataNode` payload
+- [vcti-fieldset](https://github.com/vcollab/vcti-python-fieldset) `[datanode]` (>=2.0.0) — the `FieldSet` table payload and its copy-free reshaping
+- [numpy](https://numpy.org/) (>=1.24)

{vcti_dataflow-2.0.0 → vcti_dataflow-3.0.0}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vcti-dataflow"
-version = "2.0.0"
-description = "The DataNode binding of vcti-flow: sources, transformers, reducers, and combiners for vcti-datanode payloads."
+version = "3.0.0"
+description = "vcti-flow bindings for array data: a DataNode payload binding and a FieldSet table binding (sources, transformers, reducers, sinks)."
 readme = "README.md"
 authors = [
     {name = "Visual Collaboration Technologies Inc."}
@@ -14,16 +14,18 @@ requires-python = ">=3.12,<3.15"
 dependencies = [
     "vcti-flow>=2.0.0",
     "vcti-datanode>=2.0.0",
+    "vcti-fieldset[datanode]>=2.0.0",
     "numpy>=1.24",
 ]
 
 [tool.setuptools.packages.find]
 where = ["src"]
-include = ["vcti.flow.data", "vcti.flow.data.*"]
+include = ["vcti.flow.data", "vcti.flow.data.*", "vcti.flow.fieldset", "vcti.flow.fieldset.*"]
 namespaces = true
 
 [tool.setuptools.package-data]
 "vcti.flow.data" = ["py.typed"]
+"vcti.flow.fieldset" = ["py.typed"]
 
 [project.optional-dependencies]
 test = ["pytest", "pytest-cov"]
@@ -34,7 +36,7 @@ typecheck = ["mypy"]
 zip-safe = true
 
 [tool.pytest.ini_options]
-addopts = "--cov=vcti.flow.data --cov-report=term-missing --cov-fail-under=95"
+addopts = "--cov=vcti.flow.data --cov=vcti.flow.fieldset --cov-report=term-missing --cov-fail-under=95"
 
 [tool.mypy]
 python_version = "3.12"

{vcti_dataflow-2.0.0 → vcti_dataflow-3.0.0}/src/vcti/flow/data/__init__.py

@@ -4,9 +4,9 @@
 
 DataNode-bound node kinds (``Source``, ``Transformer``, ``Reducer``, ``Sink``,
 ``Observer``) over the generic ``vcti.flow`` framework, plus ``from_array`` (eager)
-and ``ArraySource`` (lazy) for building ``DataNode`` payloads. Structured-array-
-specific nodes (field merge, row iteration, field shaping) live in the
-``vcti.flow.data.fields`` submodule.
+and ``ArraySource`` (lazy) for building ``DataNode`` payloads. This binding is for
+single-array flows; table (named-column) reshaping lives in the sibling
+``vcti.flow.fieldset`` binding over a ``FieldSet`` payload.
 """
 
 from importlib.metadata import version

vcti_dataflow-3.0.0/src/vcti/flow/fieldset/__init__.py

@@ -0,0 +1,73 @@
+# Copyright Visual Collaboration Technologies Inc. All Rights Reserved.
+# See LICENSE for details.
+"""vcti.flow.fieldset — the FieldSet binding of vcti.flow.
+
+FieldSet-bound node kinds (``Source``, ``Transformer``, ``Reducer``, ``Sink``,
+``Observer``) over the generic ``vcti.flow`` framework, plus flow nodes that wrap
+``vcti.fieldset.FieldSet``'s copy-free reshaping:
+
+- Column transformers (1->1, over ``FieldSet.project()`` / ``expressions``):
+  ``SelectFields``, ``DropFields``, ``RenameFields``, ``CastFields``,
+  ``ComputeFields``, ``FreezeFields``.
+- Row transformers (1->1, over ``FieldSet.rows()``): ``FilterRows``,
+  ``SortRows``, ``HeadRows``, ``SliceRows``.
+- Reducers (N->1): ``MergeFields`` (union columns), ``ConcatRows`` (stack rows).
+- Sources: ``FieldSetSource`` (an existing FieldSet) and ``DataNodesSource``
+  (DataNodes -> FieldSet).
+- Sink / boundary: ``WriteDataNodes`` (FieldSet -> DataNodes), plus re-exported
+  ``from_datanodes`` / ``to_datanodes``.
+- Group fan-out: ``for_each_group``.
+"""
+
+from importlib.metadata import version
+
+from vcti.fieldset import FieldSet
+from vcti.fieldset.datanode import from_datanodes, to_datanodes
+
+from .aliases import Observer, Reducer, Sink, Source, Transformer
+from .iterate import for_each_group
+from .reducers import ConcatRows, MergeFields
+from .sinks import WriteDataNodes
+from .sources import DataNodesSource, FieldSetSource
+from .transforms import (
+    CastFields,
+    ComputeFields,
+    DropFields,
+    FilterRows,
+    FreezeFields,
+    HeadRows,
+    RenameFields,
+    SelectFields,
+    SliceRows,
+    SortRows,
+)
+
+__version__ = version("vcti-dataflow")
+
+__all__ = [
+    "__version__",
+    "CastFields",
+    "ComputeFields",
+    "ConcatRows",
+    "DataNodesSource",
+    "DropFields",
+    "FieldSet",
+    "FieldSetSource",
+    "FilterRows",
+    "FreezeFields",
+    "HeadRows",
+    "MergeFields",
+    "Observer",
+    "Reducer",
+    "RenameFields",
+    "SelectFields",
+    "Sink",
+    "SliceRows",
+    "SortRows",
+    "Source",
+    "Transformer",
+    "WriteDataNodes",
+    "for_each_group",
+    "from_datanodes",
+    "to_datanodes",
+]

vcti_dataflow-3.0.0/src/vcti/flow/fieldset/aliases.py

@@ -0,0 +1,26 @@
+# Copyright Visual Collaboration Technologies Inc. All Rights Reserved.
+# See LICENSE for details.
+"""FieldSet-bound aliases for the vcti.flow node kinds.
+
+These bind the generic ``vcti.flow`` framework to the ``vcti.fieldset.FieldSet``
+payload, so authors subclass ``Source`` / ``Transformer`` / … — the
+``vcti.flow.fieldset`` spelling of ``Source[FieldSet]`` /
+``Transformer[FieldSet, FieldSet]`` — instead of repeating the type parameter.
+They are plain assignments (not PEP 695 ``type`` aliases) so they remain usable
+as base classes.
+
+The names intentionally shadow the generic ``vcti.flow.core`` kinds: within this
+binding ``Transformer`` *is* ``Transformer[FieldSet, FieldSet]``. This is the
+FieldSet counterpart of the ``DataNode`` binding in ``vcti.flow.data``.
+"""
+
+from __future__ import annotations
+
+import vcti.flow.core as core
+from vcti.fieldset import FieldSet
+
+Source = core.Source[FieldSet]
+Transformer = core.Transformer[FieldSet, FieldSet]
+Reducer = core.Reducer[FieldSet, FieldSet]
+Sink = core.Sink[FieldSet]
+Observer = core.Observer[FieldSet]

vcti_dataflow-3.0.0/src/vcti/flow/fieldset/iterate.py

@@ -0,0 +1,36 @@
+# Copyright Visual Collaboration Technologies Inc. All Rights Reserved.
+# See LICENSE for details.
+"""Group fan-out over a FieldSet, as a flow combinator."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator
+from typing import Any
+
+from vcti.fieldset import FieldSet
+from vcti.flow.core import Node
+
+
+def for_each_group[U](
+    source: Node[FieldSet],
+    key_field: str,
+    factory: Callable[[FieldSet], Node[U]],
+) -> Iterator[tuple[Any, Node[U]]]:
+    """Fan a FieldSet out into one flow per group, keyed by *key_field*.
+
+    Executes *source* once, partitions its rows with ``rows().group_by(key_field)``
+    (each group is a row-view FieldSet sharing the base's columns), and yields
+    ``(key, factory(group))`` per distinct key. The FieldSet-specific analogue of
+    ``vcti.flow.core.for_each`` for the grouped case.
+
+    Args:
+        source: A node producing the FieldSet to partition.
+        key_field: Field (or expression) name whose distinct values define groups.
+        factory: Builds a downstream node from each group's FieldSet.
+
+    Yields:
+        ``(key_value, node)`` pairs, one per group, key order ascending.
+    """
+    base = source.execute()
+    for key, group in base.rows().group_by(key_field):
+        yield key, factory(group)

vcti_dataflow-3.0.0/src/vcti/flow/fieldset/reducers.py

@@ -0,0 +1,116 @@
+# Copyright Visual Collaboration Technologies Inc. All Rights Reserved.
+# See LICENSE for details.
+"""N->1 reducers over FieldSet payloads.
+
+These combine *multiple* FieldSets into one — the arity FieldSet itself does not
+cover (its ``project`` / ``rows`` are 1->1). ``MergeFields`` unions columns
+(horizontal); ``ConcatRows`` stacks rows (vertical). Combination policy —
+name-collision handling, schema matching — lives here, in the pipeline, not in
+FieldSet.
+
+Only stored fields are combined; registered **expressions are not carried** (they
+are per-input and may reference dropped columns). Freeze an expression
+(``FreezeFields``) before combining if you need it in the output. Per-field
+properties (components, user attributes) of the combined fields are carried over.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+from vcti.fieldset import ArrayFieldSource, FieldSet
+
+from .aliases import Reducer
+
+
+def _carry_field_properties(src: FieldSet, dst: FieldSet, names: set[str]) -> None:
+    """Copy per-field properties (components + user attributes) for ``names``.
+
+    Reaches into the property store via the same ``to_dict`` / ``key_to_parts`` /
+    ``set`` path FieldSet's own projection uses. Keys missing an expected part are
+    skipped rather than raising, so a future store-schema change degrades to
+    "metadata not carried" instead of a hard error.
+    """
+    store = src.properties.store
+    mapper = store.key_mapper
+    for key, value in store.to_dict().items():
+        _, parts = mapper.key_to_parts(key)
+        column = parts.get("column")
+        scope = parts.get("scope")
+        name = parts.get("name")
+        if column in names and scope is not None and name is not None:
+            dst.properties.store.set(scope=scope, column=column, name=name, value=value)
+
+
+class MergeFields(Reducer):
+    """Union the columns of several FieldSets into one (same row count).
+
+    Fields are shared by reference (zero-copy); row counts must match (enforced by
+    ``FieldSet``). ``on_conflict`` decides duplicate output names:
+
+    - ``"error"`` (default) — raise on a name that already exists.
+    - ``"first"`` — keep the first occurrence, ignore later ones.
+    - ``"last"`` — later occurrences replace earlier ones.
+    """
+
+    def __init__(self, *, on_conflict: str = "error", name: str | None = None) -> None:
+        super().__init__(name=name)
+        if on_conflict not in ("error", "first", "last"):
+            raise ValueError(
+                f"on_conflict must be 'error', 'first', or 'last', got {on_conflict!r}"
+            )
+        self._on_conflict = on_conflict
+
+    def reduce(self, records: list[FieldSet]) -> FieldSet:
+        result = FieldSet()
+        origin: dict[str, FieldSet] = {}
+        for fs in records:
+            for field in fs.fields.names:
+                if field in result.fields:
+                    if self._on_conflict == "error":
+                        raise ValueError(f"MergeFields: duplicate field name {field!r}")
+                    if self._on_conflict == "first":
+                        continue
+                    result.fields.remove(field)  # "last" wins
+                result.fields.add(field, fs.fields.source(field))
+                origin[field] = fs
+        # Carry properties once per source (deep-copies the store once per source,
+        # not once per field): group each source with the fields it contributed.
+        grouped: dict[int, tuple[FieldSet, set[str]]] = {}
+        for field, src in origin.items():
+            grouped.setdefault(id(src), (src, set()))[1].add(field)
+        for src, fields in grouped.values():
+            _carry_field_properties(src, result, fields)
+        return result
+
+
+class ConcatRows(Reducer):
+    """Stack the rows of several same-schema FieldSets into one.
+
+    Each stored field is concatenated across inputs (``numpy.concatenate`` — one
+    copy, unavoidable). All inputs must expose the **same stored field names**
+    (order-independent); the output takes the first input's field order. An input
+    with no stored fields therefore matches only when *every* input is field-less,
+    in which case an empty FieldSet is returned — a field-less input mixed with
+    data-bearing ones is a schema mismatch, not silently dropped. Registered
+    expressions are not concatenated; freeze them first (``FreezeFields``).
+
+    Raises:
+        ValueError: If inputs have different stored-field schemas.
+    """
+
+    def reduce(self, records: list[FieldSet]) -> FieldSet:
+        if not records:
+            return FieldSet()
+        names = records[0].fields.names
+        schema = set(names)
+        for fs in records[1:]:
+            if set(fs.fields.names) != schema:
+                raise ValueError(f"ConcatRows: inconsistent schema {fs.fields.names} != {names}")
+        if not names:
+            return FieldSet()
+        result = FieldSet()
+        for field in names:
+            stacked = np.concatenate([fs.get_values(field) for fs in records])
+            result.fields.add(field, ArrayFieldSource(stacked))
+        _carry_field_properties(records[0], result, schema)
+        return result

vcti_dataflow-3.0.0/src/vcti/flow/fieldset/sinks.py

@@ -0,0 +1,45 @@
+# Copyright Visual Collaboration Technologies Inc. All Rights Reserved.
+# See LICENSE for details.
+"""Sinks that export a FieldSet payload."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from vcti.datanode import DataNode
+from vcti.fieldset import FieldSet
+from vcti.fieldset.datanode import to_datanodes
+
+from .aliases import Sink
+
+
+class WriteDataNodes(Sink):
+    """Export a FieldSet's fields as DataNodes and hand each to a writer.
+
+    Wraps ``vcti.fieldset.datanode.to_datanodes``: each field (and, by default,
+    each expression) becomes an eager ``DataNode`` — components map to
+    ``node.intrinsic['components']`` and per-field user attributes to the node's
+    enriched attributes. Each node is passed to *writer*; the actual I/O is the
+    caller's (this package does no I/O itself). Being a ``Sink``, it passes the
+    FieldSet through unchanged.
+
+    Args:
+        writer: Called once per exported node, ``writer(node) -> None``.
+        fields: Optional subset of field/expression names to export (default: all).
+        name: Optional node name.
+    """
+
+    def __init__(
+        self,
+        writer: Callable[[DataNode], None],
+        *,
+        fields: list[str] | None = None,
+        name: str | None = None,
+    ) -> None:
+        super().__init__(name=name)
+        self._writer = writer
+        self._fields = fields
+
+    def save(self, record: FieldSet) -> None:
+        for node in to_datanodes(record, fields=self._fields):
+            self._writer(node)