synaflow 0.1.0__py3-none-any.whl

synaflow/__init__.py

@@ -0,0 +1,21 @@
+"""
+Pipeline Engine
+
+A lightweight, robust engine for defining and executing typed Directed Acyclic Graphs (DAGs).
+This module defines the public interface for clients.
+"""
+
+from .executor import run
+from .pipeline import PipelineDef, pipeline
+from .step import Step, step
+from .types import OnError, StepParams
+
+__all__ = [
+    "PipelineDef",
+    "pipeline",
+    "Step",
+    "step",
+    "OnError",
+    "StepParams",
+    "run",
+]

synaflow/executor.py

@@ -0,0 +1,416 @@
+import inspect
+import itertools
+from collections.abc import Callable, Generator, Iterator
+from typing import Any
+
+from .iterator_utils import InterleavedIterator
+from .pipeline import PipelineDef
+from .type_compatibility import is_iterable_type, is_scalar
+
+
+class PipelineStopException(Exception):
+    """Raised to stop the pipeline execution early."""
+
+    pass
+
+
+class PipelineExecutor:
+    """Executes a compiled Directed Acyclic Graph (DAG) for a pipeline."""
+
+    def __init__(self, dag: dict[str, dict], materialize_fn: Callable = list):
+        self.dag = dag
+        self.materialize_fn = materialize_fn
+        self.context: dict[str, Any] = {}
+        self.executed_steps: set[str] = set()
+
+    def execute(self, params: Any) -> None:
+        self._initialize_context_with_params(params)
+
+        try:
+            levels = self._compute_topological_levels()
+            for level in levels:
+                self._execute_level(level)
+        except PipelineStopException:
+            pass
+
+    def _initialize_context_with_params(self, params: Any) -> None:
+        for field, value in params._asdict().items():
+            node = self.dag.get(field, {})
+            needs_materialization = node.get("needs_materialize", False)
+
+            if needs_materialization and isinstance(value, Iterator):
+                value = self.materialize_fn(value)
+            elif isinstance(value, Iterator):
+                value = self._tee_iterator_for_consumers(field, value)
+
+            self.context[field] = value
+
+    def _tee_iterator_for_consumers(
+        self, producer_name: str, iterator_value: Iterator
+    ) -> Any:
+        consumers = [
+            consumer_name
+            for consumer_name, node in self.dag.items()
+            if producer_name in node.get("deps", {})
+        ]
+        if len(consumers) > 1:
+            tees = itertools.tee(iterator_value, len(consumers))
+            return {"__tees__": dict(zip(consumers, tees))}
+        return iterator_value
+
+    def _compute_topological_levels(self) -> list[list[str]]:
+        in_degree: dict[str, int] = {name: 0 for name in self.dag}
+        for name, node in self.dag.items():
+            for dep in node.get("deps", {}):
+                if dep in in_degree:
+                    in_degree[name] += 1
+
+        levels: list[list[str]] = []
+        processed: set[str] = set()
+
+        while len(processed) < len(self.dag):
+            level = [
+                name
+                for name, degree in in_degree.items()
+                if degree == 0 and name not in processed
+            ]
+            if not level:
+                break
+            levels.append(level)
+            processed.update(level)
+
+            for name in level:
+                for other_name, node in self.dag.items():
+                    if name in node.get("deps", {}):
+                        in_degree[other_name] -= 1
+
+        return levels
+
+    def _execute_level(self, level: list[str]) -> None:
+        dep_each_nodes, dep_all_nodes, independent_nodes = (
+            self._group_nodes_by_execution_mode(level)
+        )
+
+        all_dependencies = set(dep_each_nodes.keys()) | set(dep_all_nodes.keys())
+
+        for dep_name in all_dependencies:
+            each_names = dep_each_nodes.get(dep_name, [])
+            all_names = dep_all_nodes.get(dep_name, [])
+            self._process_grouped_dependencies(
+                dep_name, each_names, all_names, independent_nodes
+            )
+
+        for name in independent_nodes:
+            self._execute_independent_node(name)
+
+    def _group_nodes_by_execution_mode(
+        self, level: list[str]
+    ) -> tuple[dict, dict, list]:
+        dep_each_nodes: dict[str, list[str]] = {}
+        dep_all_nodes: dict[str, list[str]] = {}
+        independent_nodes: list[str] = []
+
+        for name in level:
+            if name in self.executed_steps:
+                continue
+
+            node = self.dag.get(name)
+            if not node or node.get("fn") is None:
+                continue
+
+            deps = node.get("deps", {})
+            if not deps:
+                independent_nodes.append(name)
+                continue
+
+            first_dep_name = next(iter(deps))
+            if self._is_each_mode_execution(deps, first_dep_name):
+                dep_each_nodes.setdefault(first_dep_name, []).append(name)
+            else:
+                consumer_type = (
+                    inspect.signature(node["fn"]).parameters[first_dep_name].annotation
+                )
+                if self._is_lazy_iterator_type(consumer_type):
+                    dep_all_nodes.setdefault(first_dep_name, []).append(name)
+                else:
+                    independent_nodes.append(name)
+
+        return dep_each_nodes, dep_all_nodes, independent_nodes
+
+    def _process_grouped_dependencies(
+        self,
+        dep_name: str,
+        each_names: list[str],
+        all_names: list[str],
+        independent_nodes: list[str],
+    ) -> None:
+        eager_each_names, lazy_each_names = self._split_eager_and_lazy_each_nodes(
+            each_names
+        )
+
+        eager_callbacks = self._create_eager_callbacks(eager_each_names, dep_name)
+
+        independent_nodes.extend(lazy_each_names)
+
+        if eager_callbacks:
+            self._execute_eager_callbacks(
+                eager_callbacks, dep_name, all_names, independent_nodes
+            )
+        else:
+            independent_nodes.extend(all_names)
+
+    def _split_eager_and_lazy_each_nodes(
+        self, each_names: list[str]
+    ) -> tuple[list[str], list[str]]:
+        eager = []
+        lazy = []
+        for name in each_names:
+            consumers = [
+                cn for cn, cnode in self.dag.items() if name in cnode.get("deps", {})
+            ]
+            if name.startswith("_") or not consumers:
+                eager.append(name)
+            else:
+                lazy.append(name)
+        return eager, lazy
+
+    def _create_eager_callbacks(
+        self, eager_names: list[str], dep_name: str
+    ) -> list[Callable]:
+        callbacks = []
+        for name in eager_names:
+            node = self.dag[name]
+            fn = node["fn"]
+            kwargs = self._resolve_node_arguments(name, node)
+            on_error = node.get("on_error")
+
+            def make_callback(fn, kwargs, dep_name, on_error):
+                def cb(item):
+                    try:
+                        item_kwargs = dict(kwargs)
+                        item_kwargs[dep_name] = item
+                        fn(**item_kwargs)
+                    except Exception as e:
+                        if on_error and on_error.value == "stop":
+                            raise PipelineStopException() from e
+
+                return cb
+
+            callbacks.append(make_callback(fn, kwargs, dep_name, on_error))
+            self.executed_steps.add(name)
+
+        return callbacks
+
+    def _execute_eager_callbacks(
+        self,
+        callbacks: list[Callable],
+        dep_name: str,
+        all_names: list[str],
+        independent_nodes: list[str],
+    ) -> None:
+        items_source = self.context.get(dep_name)
+
+        if all_names:
+            first_all = all_names[0]
+            self._execute_interleaved_node(first_all, dep_name, callbacks)
+            independent_nodes.extend(all_names[1:])
+        else:
+            self._execute_lockstep_loop(items_source, callbacks, dep_name)
+
+    def _execute_interleaved_node(
+        self, node_name: str, dep_name: str, callbacks: list[Callable]
+    ) -> None:
+        node = self.dag[node_name]
+        fn = node["fn"]
+        kwargs = self._resolve_node_arguments(node_name, node)
+
+        dep_val = kwargs.get(dep_name)
+        if isinstance(dep_val, dict) and "__tees__" in dep_val:
+            dep_val = dep_val["__tees__"][node_name]
+
+        kwargs[dep_name] = InterleavedIterator(dep_val, callbacks)
+
+        try:
+            output = fn(**kwargs)
+            if node_name and not node_name.startswith("_"):
+                self.context[node_name] = output
+        except Exception:
+            if node.get("on_error") and node["on_error"].value == "stop":
+                raise PipelineStopException()
+
+        self.executed_steps.add(node_name)
+
+    def _execute_lockstep_loop(
+        self, items_source: Any, callbacks: list[Callable], dep_name: str
+    ) -> None:
+        dep_val = items_source
+        if isinstance(dep_val, dict) and "__tees__" in dep_val:
+            consumers = [
+                cn
+                for cn, cnode in self.dag.items()
+                if dep_name in cnode.get("deps", {})
+            ]
+            first_tee_name = next(c for c in consumers if c in dep_val["__tees__"])
+            dep_val = dep_val["__tees__"][first_tee_name]
+
+        for item in dep_val:
+            for cb in callbacks:
+                cb(item)
+
+    def _execute_independent_node(self, name: str) -> None:
+        if name in self.executed_steps:
+            return
+
+        node = self.dag.get(name)
+        if not node or node.get("fn") is None:
+            return
+
+        fn = node["fn"]
+        deps = node.get("deps", {})
+        kwargs = self._resolve_node_arguments(name, node)
+
+        if deps and self._is_each_mode_execution(deps, next(iter(deps))):
+            self._execute_independent_each_node(name, fn, deps, kwargs, node)
+        else:
+            self._execute_standard_node(name, fn, kwargs, node)
+
+        self.executed_steps.add(name)
+
+    def _execute_independent_each_node(
+        self, name: str, fn: Callable, deps: dict, kwargs: dict, node: dict
+    ) -> None:
+        first_dep = next(iter(deps))
+        items = self.context.get(first_dep)
+
+        if isinstance(items, dict) and "__tees__" in items:
+            items = items["__tees__"][name]
+
+        consumers = [
+            cn for cn, cnode in self.dag.items() if name in cnode.get("deps", {})
+        ]
+        is_sink = name.startswith("_") or len(consumers) == 0
+
+        if is_sink:
+            for item in items:
+                try:
+                    item_kwargs = dict(kwargs)
+                    item_kwargs[first_dep] = item
+                    fn(**item_kwargs)
+                except Exception:
+                    if node.get("on_error") and node["on_error"].value == "stop":
+                        raise PipelineStopException()
+        else:
+
+            def each_generator(items, kwargs, first_dep, fn, on_error):
+                for item in items:
+                    try:
+                        item_kwargs = dict(kwargs)
+                        item_kwargs[first_dep] = item
+                        yield fn(**item_kwargs)
+                    except Exception as e:
+                        if on_error and on_error.value == "stop":
+                            raise PipelineStopException() from e
+
+            output = each_generator(items, kwargs, first_dep, fn, node.get("on_error"))
+
+            if len(consumers) > 1:
+                tees = itertools.tee(output, len(consumers))
+                output = {"__tees__": dict(zip(consumers, tees))}
+
+            self.context[name] = output
+
+    def _execute_standard_node(
+        self, name: str, fn: Callable, kwargs: dict, node: dict
+    ) -> None:
+        try:
+            output = fn(**kwargs)
+
+            if name and not name.startswith("_"):
+                if isinstance(output, Iterator) and node.get("needs_materialize"):
+                    output = self.materialize_fn(output)
+                elif isinstance(output, Iterator):
+                    output = self._tee_iterator_for_consumers(name, output)
+
+                self.context[name] = output
+
+        except Exception:
+            if node.get("on_error") and node["on_error"].value == "stop":
+                raise PipelineStopException()
+
+    def _resolve_node_arguments(self, consumer_name: str, node: dict) -> dict[str, Any]:
+        sig = inspect.signature(node["fn"])
+        deps = node.get("deps", {})
+        kwargs: dict[str, Any] = {}
+
+        for param_name in sig.parameters:
+            if param_name in self.context:
+                value = self.context.get(param_name)
+
+                if isinstance(value, dict) and "__tees__" in value:
+                    value = value["__tees__"][consumer_name]
+
+                if param_name in deps:
+                    consumer_type = deps[param_name]
+                    value = self._adapt_argument_to_consumer_type(value, consumer_type)
+
+                kwargs[param_name] = value
+
+        return kwargs
+
+    def _adapt_argument_to_consumer_type(self, value: Any, consumer_type: Any) -> Any:
+        is_lazy_iterator = self._is_lazy_iterator_type(consumer_type)
+        needs_materialization = self._needs_materialize_for(consumer_type)
+
+        if is_lazy_iterator or needs_materialization:
+            if not isinstance(value, (list, set, tuple, Iterator, Generator)):
+                value = [value]
+
+            if isinstance(value, Iterator) and needs_materialization:
+                value = self.materialize_fn(value)
+
+            origin = getattr(consumer_type, "__origin__", consumer_type)
+            if origin is set:
+                value = set(value)
+            elif origin is tuple:
+                value = tuple(value)
+            elif origin in (Iterator, Generator):
+                value = iter(value)
+
+        return value
+
+    def _is_each_mode_execution(self, deps: dict, first_dep_name: str) -> bool:
+        if not deps:
+            return False
+
+        first_type = deps[first_dep_name]
+        producer = self.dag.get(first_dep_name)
+        if not producer or producer.get("output") is None:
+            return False
+
+        producer_output = producer.get("output")
+        return is_iterable_type(producer_output) and is_scalar(first_type)
+
+    def _is_lazy_iterator_type(self, tp: Any) -> bool:
+        if tp is Iterator:
+            return True
+        origin = getattr(tp, "__origin__", tp)
+        return origin in (Iterator, Generator)
+
+    def _needs_materialize_for(self, tp: Any) -> bool:
+        if tp is None:
+            return False
+        if tp in (list, set, tuple):
+            return True
+        origin = getattr(tp, "__origin__", None)
+        return origin in (list, set, tuple)
+
+
+def run(
+    pipeline: PipelineDef,
+    params: Any,
+    *,
+    materialize: Callable = list,
+) -> None:
+    """Executes a pipeline definition."""
+    executor = PipelineExecutor(pipeline._dag, materialize)
+    executor.execute(params)

synaflow/iterator_utils.py

@@ -0,0 +1,24 @@
+from typing import Any, Callable, Iterable
+
+
+class InterleavedIterator:
+    """
+    An iterator that yields items from a source, while simultaneously
+    executing a set of callbacks on each item.
+
+    This is useful for lockstep processing where eager consumers must execute
+    on items before lazy downstream consumers iterate over them.
+    """
+
+    def __init__(self, source: Iterable[Any], callbacks: list[Callable[[Any], None]]):
+        self.source = iter(source)
+        self.callbacks = callbacks
+
+    def __next__(self):
+        item = next(self.source)
+        for callback in self.callbacks:
+            callback(item)
+        return item
+
+    def __iter__(self):
+        return self

synaflow/pipeline.py

@@ -0,0 +1,39 @@
+from dataclasses import dataclass
+from typing import Any
+
+from .step import Step
+
+
+@dataclass
+class PipelineDef:
+    """
+    Defines a Pipeline workflow.
+    """
+
+    name: str
+    params: Any
+    steps: list[Step]
+    description: str = ""
+
+    def __post_init__(self) -> None:
+        from .validator import validate_and_build_dag
+
+        self._dag = validate_and_build_dag(self.name, self.steps, self.params)
+
+    def to_dict(self) -> dict:
+        """Exports the compiled DAG structure to a JSON-serializable dictionary."""
+        from .type_compatibility import get_type_name
+
+        serialized = {}
+        for name, node in self._dag.items():
+            serialized[name] = {
+                "deps": {k: get_type_name(v) for k, v in node["deps"].items()},
+                "output": get_type_name(node["output"]),
+                "fn": node["fn"].__name__ if node["fn"] else None,
+                "on_error": node["on_error"].value if node["on_error"] else None,
+                "needs_materialize": node["needs_materialize"],
+            }
+        return serialized
+
+
+pipeline = PipelineDef

synaflow/step.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+from .types import OnError, StepParams
+
+
+@dataclass
+class Step:
+    name: str
+    fn: Callable
+    on_error: OnError = OnError.STOP
+    params: StepParams | None = None
+    description: str = ""
+
+
+step = Step

synaflow/type_compatibility.py

@@ -0,0 +1,191 @@
+import types
+from collections.abc import Generator, Iterable, Iterator
+from typing import Any, get_args, get_origin
+
+SCALAR_TYPES = {int, float, str, bool, bytes, type(None)}
+COLLECTION_ORIGINS = {list, set, tuple, Generator, Iterator, Iterable}
+
+
+class ListType:
+    """Wrapper to represent a runtime-resolved list of a specific type."""
+
+    def __init__(self, inner_type: Any):
+        self.inner_type = inner_type
+
+    def __repr__(self):
+        return f"ListType({self.inner_type})"
+
+
+def is_type_compatible(producer_type: Any, consumer_type: Any) -> bool:
+    """Checks if a producer output type satisfies a consumer input type."""
+    if producer_type is None or consumer_type is None:
+        return True
+
+    producer_origin = get_origin(producer_type)
+    consumer_origin = get_origin(consumer_type)
+
+    if _is_union(producer_type, producer_origin) and _is_union(
+        consumer_type, consumer_origin
+    ):
+        return _all_producer_types_match_any_consumer_type(producer_type, consumer_type)
+
+    if _is_union(producer_type, producer_origin):
+        return _all_producer_types_match_consumer(producer_type, consumer_type)
+
+    is_producer_iterable = _is_iterable(producer_type, producer_origin)
+    is_consumer_iterable = _is_iterable(consumer_type, consumer_origin)
+
+    if is_producer_iterable:
+        return _check_iterable_producer_compatibility(
+            producer_type, consumer_type, is_consumer_iterable
+        )
+
+    if _is_union(consumer_type, consumer_origin):
+        return _producer_matches_any_consumer_type(producer_type, consumer_type)
+
+    if is_consumer_iterable:
+        return _check_scalar_producer_to_iterable_consumer(producer_type, consumer_type)
+
+    if is_scalar(consumer_type):
+        return _check_scalar_compatibility(producer_type, consumer_type)
+
+    return True
+
+
+def _all_producer_types_match_any_consumer_type(
+    producer_type: Any, consumer_type: Any
+) -> bool:
+    return all(
+        any(is_type_compatible(p, c) for c in get_args(consumer_type))
+        for p in get_args(producer_type)
+    )
+
+
+def _all_producer_types_match_consumer(producer_type: Any, consumer_type: Any) -> bool:
+    return all(is_type_compatible(p, consumer_type) for p in get_args(producer_type))
+
+
+def _producer_matches_any_consumer_type(producer_type: Any, consumer_type: Any) -> bool:
+    return any(is_type_compatible(producer_type, c) for c in get_args(consumer_type))
+
+
+def _check_iterable_producer_compatibility(
+    producer_type: Any, consumer_type: Any, is_consumer_iterable: bool
+) -> bool:
+    producer_inner = get_inner_type(producer_type)
+    if producer_inner is None:
+        return False
+
+    consumer_origin = get_origin(consumer_type)
+
+    if _is_union(consumer_type, consumer_origin):
+        return is_type_compatible(producer_inner, consumer_type)
+
+    if is_consumer_iterable:
+        consumer_inner = get_inner_type(consumer_type)
+        if consumer_inner is not None:
+            return is_type_compatible(producer_inner, consumer_inner)
+        return True
+
+    if is_scalar(consumer_type):
+        return is_type_compatible(producer_inner, consumer_type)
+
+    return False
+
+
+def _check_scalar_producer_to_iterable_consumer(
+    producer_type: Any, consumer_type: Any
+) -> bool:
+    producer_inner = get_inner_type(producer_type)
+    consumer_inner = get_inner_type(consumer_type)
+
+    if consumer_inner is None:
+        return True
+
+    if producer_inner is None:
+        return is_type_compatible(producer_type, consumer_inner)
+
+    return is_type_compatible(producer_inner, consumer_inner)
+
+
+def _check_scalar_compatibility(producer_type: Any, consumer_type: Any) -> bool:
+    producer_inner = get_inner_type(producer_type)
+    if producer_inner is not None:
+        return is_type_compatible(producer_inner, consumer_type)
+    return is_scalar(producer_type) and producer_type == consumer_type
+
+
+def _is_union(tp: Any, origin: Any) -> bool:
+    return origin is types.UnionType or origin is __import__("typing").Union
+
+
+def _is_iterable(tp: Any, origin: Any) -> bool:
+    if isinstance(tp, ListType):
+        return True
+    if origin is not None:
+        return origin in COLLECTION_ORIGINS
+    return tp in COLLECTION_ORIGINS
+
+
+def is_iterable_type(tp: Any) -> bool:
+    if tp is None:
+        return False
+    if isinstance(tp, ListType):
+        return True
+    return _is_iterable(tp, get_origin(tp))
+
+
+def is_scalar(tp: Any) -> bool:
+    if tp is None:
+        return False
+    if tp in SCALAR_TYPES:
+        return True
+    origin = get_origin(tp)
+    if origin is not None:
+        import types
+        from typing import Union
+
+        if origin is types.UnionType or origin is Union:
+            return all(is_scalar(a) for a in get_args(tp))
+        return False
+    return tp not in COLLECTION_ORIGINS
+
+
+def get_inner_type(tp: Any) -> Any:
+    if isinstance(tp, ListType):
+        return tp.inner_type
+    args = get_args(tp)
+    if args:
+        return args[0]
+    return None
+
+
+def get_type_name(tp: Any) -> str:
+    if tp is None:
+        return "None"
+    origin = get_origin(tp)
+    if origin is not None:
+        arg_names = ", ".join(get_type_name(a) for a in get_args(tp))
+        origin_name = getattr(origin, "__name__", str(origin))
+        return f"{origin_name}[{arg_names}]"
+    return getattr(tp, "__name__", str(tp))
+
+
+def is_materialized_consumer(tp: Any) -> bool:
+    """Checks if a consumer type requires an eagerly materialized collection."""
+    if tp is None:
+        return False
+    if tp in (list, set, tuple):
+        return True
+
+    origin = get_origin(tp)
+    if origin in (list, set, tuple):
+        return True
+    if origin in (Iterator, Generator, Iterable):
+        return False
+    if is_scalar(tp):
+        return False
+    if _is_union(tp, origin):
+        return any(is_materialized_consumer(a) for a in get_args(tp))
+
+    return False

synaflow/types.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+
+class OnError(Enum):
+    CONTINUE = "continue"
+    STOP = "stop"
+
+
+@dataclass(frozen=True)
+class StepParams:
+    values: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class StepResult:
+    step_name: str
+    status: str  # "ok" | "error" | "skipped"
+    output: Any = None
+    error: Exception | None = None
+    metrics: dict[str, int] = field(default_factory=dict)

synaflow/validator.py

@@ -0,0 +1,203 @@
+import inspect
+from typing import Any, NamedTuple
+
+from .step import Step
+from .type_compatibility import (
+    ListType,
+    get_type_name,
+    is_iterable_type,
+    is_materialized_consumer,
+    is_scalar,
+    is_type_compatible,
+)
+
+
+def validate_and_build_dag(
+    name: str, steps: list[Step], params: type[NamedTuple]
+) -> dict[str, dict]:
+    """
+    Validates a list of steps and pipeline parameters and compiles them into a
+    Directed Acyclic Graph (DAG) represented as a dictionary.
+    """
+    dag: dict[str, dict] = {}
+
+    _validate_params_is_namedtuple(params, name)
+
+    produced: dict[str, dict] = _initialize_parameters(params)
+
+    for step in steps:
+        _validate_step_is_callable(step, name)
+        _validate_unique_step_name(step.name, dag, pipeline_name=name)
+
+        node = _validate_and_compile_step(step, produced, pipeline_name=name)
+
+        dag[step.name] = node
+        produced[step.name] = node
+
+    _add_parameter_nodes_to_dag(dag, produced)
+    _compute_needs_materialize(dag)
+
+    return dag
+
+
+def _validate_params_is_namedtuple(params: Any, pipeline_name: str) -> None:
+    if not hasattr(params, "_fields"):
+        raise ValueError(
+            f"Pipeline '{pipeline_name}': 'params' must be a NamedTuple, got {type(params).__name__}"
+        )
+
+
+def _validate_step_is_callable(step: Step, pipeline_name: str) -> None:
+    if not callable(step.fn):
+        raise ValueError(
+            f"Pipeline '{pipeline_name}': step '{step.name}' must have a callable 'fn', got {type(step.fn).__name__}"
+        )
+
+
+def _initialize_parameters(params: type[NamedTuple]) -> dict[str, dict]:
+    produced: dict[str, dict] = {}
+    for field in params._fields:
+        tp = getattr(params, "__annotations__", {}).get(field)
+        produced[field] = {"output": tp}
+    return produced
+
+
+def _validate_unique_step_name(step_name: str, dag: dict, pipeline_name: str) -> None:
+    if step_name in dag:
+        raise ValueError(
+            f"Pipeline '{pipeline_name}': duplicate step name '{step_name}'"
+        )
+
+
+def _validate_and_compile_step(
+    step: Step, produced: dict[str, dict], pipeline_name: str
+) -> dict[str, Any]:
+    sig = inspect.signature(step.fn)
+    hints = _get_safe_type_hints(step.fn)
+
+    deps = _validate_and_resolve_dependencies(step, sig, hints, produced, pipeline_name)
+    output_type = _resolve_step_output_type(sig, hints, deps, produced)
+
+    return {
+        "deps": deps,
+        "output": output_type,
+        "fn": step.fn,
+        "on_error": step.on_error,
+        "needs_materialize": _any_dependency_needs_materialization(deps),
+    }
+
+
+def _get_safe_type_hints(fn: Any) -> dict[str, Any]:
+    try:
+        return getattr(fn, "__annotations__", {})
+    except Exception:
+        return {}
+
+
+def _validate_and_resolve_dependencies(
+    step: Step,
+    sig: inspect.Signature,
+    hints: dict[str, Any],
+    produced: dict[str, dict],
+    pipeline_name: str,
+) -> dict[str, Any]:
+    deps: dict[str, Any] = {}
+
+    for param_name, param in sig.parameters.items():
+        consumer_type = hints.get(param_name, param.annotation)
+        if consumer_type is inspect.Parameter.empty:
+            consumer_type = None
+
+        if param_name not in produced:
+            raise ValueError(
+                f"Pipeline '{pipeline_name}': step '{step.name}' depends on '{param_name}' "
+                "but no prior step or param produces it"
+            )
+
+        producer_type = produced[param_name]["output"]
+
+        # Explicit validation for NoneType to strict types
+        if (
+            producer_type is type(None)
+            and consumer_type is not None
+            and not _is_optional_or_any(consumer_type)
+        ):
+            raise ValueError(
+                f"Pipeline '{pipeline_name}': step '{step.name}' param '{param_name}': "
+                f"expects {get_type_name(consumer_type)} "
+                f"but '{param_name}' produces explicit NoneType"
+            )
+
+        if not is_type_compatible(producer_type, consumer_type):
+            raise ValueError(
+                f"Pipeline '{pipeline_name}': step '{step.name}' param '{param_name}': "
+                f"expects {get_type_name(consumer_type)} "
+                f"but '{param_name}' produces {get_type_name(producer_type)}"
+            )
+
+        deps[param_name] = consumer_type
+
+    return deps
+
+
+def _is_optional_or_any(tp: Any) -> bool:
+    if tp is Any:
+        return True
+    # If it's a union containing NoneType
+    origin = getattr(tp, "__origin__", None)
+    import types
+    from typing import Union
+
+    if origin is types.UnionType or origin is Union:
+        return type(None) in getattr(tp, "__args__", [])
+    return False
+
+
+def _resolve_step_output_type(
+    sig: inspect.Signature,
+    hints: dict[str, Any],
+    deps: dict[str, Any],
+    produced: dict[str, dict],
+) -> Any:
+    return_type = hints.get("return", sig.return_annotation)
+    if return_type is inspect.Parameter.empty:
+        return_type = None
+
+    if is_scalar(return_type) and deps:
+        first_dep_name = next(iter(deps))
+        first_dep_output = produced[first_dep_name]["output"]
+        if first_dep_output is not None and is_iterable_type(first_dep_output):
+            return ListType(return_type)
+
+    return return_type
+
+
+def _any_dependency_needs_materialization(deps: dict[str, Any]) -> bool:
+    return any(is_materialized_consumer(t) for t in deps.values())
+
+
+def _add_parameter_nodes_to_dag(dag: dict, produced: dict) -> None:
+    """Nodes that are parameters need to be explicitly added to the final DAG so they can be processed."""
+    for name, info in list(produced.items()):
+        if name not in dag:
+            info["fn"] = None
+            info["deps"] = {}
+            info["on_error"] = None
+            info["needs_materialize"] = False
+            dag[name] = info
+
+
+def _compute_needs_materialize(dag: dict) -> None:
+    """Updates each node to indicate if any downstream consumer needs its output materialized."""
+    for name, node in dag.items():
+        consumers = [
+            other_name
+            for other_name, other_node in dag.items()
+            if name in other_node.get("deps", {})
+        ]
+
+        node["needs_materialize"] = any(
+            is_materialized_consumer(dag[consumer_name]["deps"][name])
+            for consumer_name in consumers
+            if consumer_name in dag and name in dag[consumer_name].get("deps", {})
+        )

synaflow-0.1.0.dist-info/METADATA

@@ -0,0 +1,93 @@
+Metadata-Version: 2.4
+Name: synaflow
+Version: 0.1.0
+Summary: A lightweight, type-hint driven engine for executing Directed Acyclic Graphs (DAGs) and lockstep pipelines in Python.
+Project-URL: Homepage, https://github.com/humansoftware/synaflow
+Project-URL: Repository, https://github.com/humansoftware/synaflow.git
+Author-email: mvallebr <mvallebr@example.com>
+License: MIT
+License-File: LICENSE
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# SynaFlow 🌊🧠
+
+**SynaFlow** is a lightweight, pure-Python pipeline engine that uses **Type Hints** to magically wire and execute Directed Acyclic Graphs (DAGs). 
+
+It solves the "dependency hell" and boilerplate associated with building data pipelines by automatically inferring the flow of data based exclusively on Python's static type annotations. 
+
+## The Problem It Solves
+
+Building data pipelines usually involves two headaches:
+1. **Explicit Wiring:** You have to manually define which function outputs go to which function inputs (e.g., `A >> B >> C`), creating verbose and fragile architectures.
+2. **Memory Explosions vs. Lazy Evaluation:** Passing large datasets around usually means holding them entirely in memory (Lists) or dealing with complex generator management. If you have multiple consumers for a single generator, you usually have to write clunky `itertools.tee` boilerplate yourself.
+
+## The SynaFlow Solution
+
+SynaFlow looks at the **Type Hints** of your functions and automatically wires everything together for you. If `Step A` outputs an `int` and `Step B` requires an `int`, SynaFlow connects them instantly.
+
+Furthermore, SynaFlow has a **smart lockstep streaming engine**:
+- If a producer yields a `Generator` and a consumer expects an `Iterator`, SynaFlow streams the data lazily without ever holding it in memory.
+- If multiple consumers want that same generator, SynaFlow automatically forks it (`tee`) and drives them in parallel (lockstep).
+- If one consumer explicitly asks for a `list`, SynaFlow automatically materializes the data only for that specific branch.
+
+## How is it different from other frameworks?
+
+There are many amazing orchestration frameworks out there, but SynaFlow fills a very specific gap: **In-process Streaming Micro-Orchestration**.
+
+### vs. Hamilton
+[Hamilton](https://github.com/DAGWorks-Inc/hamilton) is a fantastic tool that also uses Python function signatures to build DAGs. However, Hamilton is heavily geared towards DataFrames and feature engineering, generally expecting functions to return concrete values (columns/scalars). **SynaFlow**, on the other hand, is built from the ground up to support **Native Generators and Lazy Streaming**. While Hamilton maps functions to columns, SynaFlow maps functions to continuous data streams, automatically interleaving multiple consumers in lockstep without memory spikes.
+
+### vs. Airflow / Prefect / Dagster
+These are **Macro-Orchestrators**. They are designed to orchestrate heavy, distributed tasks across clusters, Docker containers, and different machines. They rely on state databases and massive IO overhead. **SynaFlow is a Micro-Orchestrator**. It runs entirely within a single Python process. You would use Airflow to trigger a daily job, but you would use SynaFlow *inside* that job to smartly route and stream millions of rows between your Python functions.
+
+## Quickstart
+
+```python
+from typing import NamedTuple
+from collections.abc import Generator, Iterator
+from synaflow import pipeline, step, run
+
+# Define the data required to start your pipeline
+class MyParams(NamedTuple):
+    count: int
+
+# 1. Producer outputs a stream
+def producer(count: int) -> Generator[int, None, None]:
+    yield from range(count)
+
+# 2. Transformer consumes the stream lazily
+def transformer(producer: Iterator[int]) -> Generator[int, None, None]:
+    for val in producer:
+        yield val * 10
+
+# 3. Consumer automatically gets the stream!
+def consumer(transformer: Iterator[int]) -> None:
+    for x in transformer:
+        print(f"Consumed: {x}")
+
+# SynaFlow reads the Type Hints and wires the DAG automatically!
+my_pipeline = pipeline(
+    name="example",
+    params=MyParams,
+    steps=[
+        step("producer", fn=producer),
+        step("transformer", fn=transformer),
+        step("consumer", fn=consumer)
+    ]
+)
+
+# Run it
+run(my_pipeline, MyParams(count=5))
+```
+
+## Advanced Features
+- **Auto-DAG compilation and validation** before execution.
+- Strict type-checking: Pipeline refuses to run if type annotations are incompatible.
+- Easily export DAG structures as JSON (`my_pipeline.to_dict()`) for snapshot testing or UI rendering.
+
+## License
+MIT License

synaflow-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+synaflow/__init__.py,sha256=lc79G8-8BKwq_vHc2gggd6CERAMBEopgq0OhMFF4j3I,430
+synaflow/executor.py,sha256=AysW8lKff4_Ks66IqlRrypQ9DIV2YHU6vb9txUbpY0w,14400
+synaflow/iterator_utils.py,sha256=vgqS9SUxATAYVAJUKeiPmNkqAATp-ynTiAeaSnux6RU,697
+synaflow/pipeline.py,sha256=zJzJIOxWaZFOv1DDin9zzMC-U6Z4dB_r9-BVCrxTGwk,1101
+synaflow/step.py,sha256=GNxvDVki-7EODlDU2pifYTvgKNDyHEypDsC2vTpd8x0,320
+synaflow/type_compatibility.py,sha256=WpnLsxjeeybk7YhQKBiVZ_05NDwUrf7Fl3Lwbk9rJ2s,5855
+synaflow/types.py,sha256=lviJnTEjeZ_ygqfH_gcvmUAQUb6hp8fIUM_tWwPDD6o,502
+synaflow/validator.py,sha256=BpjqYTSIUFqIwSr2di2M8iAAHxXS5j_8EViNcXuDdfc,6425
+synaflow-0.1.0.dist-info/METADATA,sha256=NlaWpgcTxueXCG0Uv99to0JYU0l73AYTJ2tEmOfcCOs,4787
+synaflow-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+synaflow-0.1.0.dist-info/licenses/LICENSE,sha256=XpDpkAL4Vwe8Mb6Uflgq6k-S7UkmcujQDar7Fx8T46E,1071
+synaflow-0.1.0.dist-info/RECORD,,

synaflow-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

synaflow-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Human Software
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.