synaflow 0.1.0__tar.gz

synaflow-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,41 @@
+name: CI
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
+      with:
+        enable-cache: true
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+
+    - name: Install dependencies
+      run: |
+        uv pip install --system hatchling pytest pre-commit
+
+    - name: Check Code Formatting
+      run: |
+        pre-commit run --all-files
+
+    - name: Validate Packing and Installation
+      run: |
+        # Builds the wheel and installs it the way a user would
+        uv pip install --system .
+        
+    - name: Run Tests
+      run: |
+        # Runs pytest, verifying that the installed package works
+        uv run pytest tests/

synaflow-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,40 @@
+name: Release & Publish
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write  # MANDATORY for Trusted Publishing
+      contents: write  # to create github release and tag
+    
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+
+    - name: Install dependencies
+      run: pip install build
+
+    - name: Python Semantic Release
+      id: release
+      uses: python-semantic-release/python-semantic-release@v9.8.1
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Build wheel and sdist
+      if: steps.release.outputs.released == 'true'
+      run: python -m build
+
+    - name: Publish to PyPI
+      if: steps.release.outputs.released == 'true'
+      uses: pypa/gh-action-pypi-publish@release/v1

synaflow-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+-   repo: https://github.com/psf/black
+    rev: 23.3.0
+    hooks:
+    -   id: black
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+    -   id: isort

synaflow-0.1.0/CHANGELOG.md

@@ -0,0 +1,27 @@
+# CHANGELOG
+
+
+
+## v0.1.0 (2026-06-11)
+
+### Documentation
+
+* docs: Add MIT License ([`5bbed31`](https://github.com/humansoftware/synaflow/commit/5bbed317dbfdee421e2c29630b5de029dbd652af))
+
+* docs: Add framework comparisons to README ([`d8f2c53`](https://github.com/humansoftware/synaflow/commit/d8f2c53e3440ee470760e3276c0f8c894a63cba3))
+
+### Feature
+
+* feat: Add CI/CD workflows for testing and semantic release ([`30f4218`](https://github.com/humansoftware/synaflow/commit/30f4218b2b265c39ce56f17b33572c24b928ab4f))
+
+### Fix
+
+* fix: remove build_command from semantic release ([`3232a3e`](https://github.com/humansoftware/synaflow/commit/3232a3e5d33621803955ed341ee5bd3a29c10d8c))
+
+* fix: bump python-semantic-release to v9.8.1 to fix debian repository error ([`e691d1c`](https://github.com/humansoftware/synaflow/commit/e691d1cc19905a6d81f0634911540ddc337fd97c))
+
+### Unknown
+
+* Add pre-commit, improve README, update org in pyproject ([`6c8fbe8`](https://github.com/humansoftware/synaflow/commit/6c8fbe884d0f7f812980f554cbe22a58056faada))
+
+* Initial commit for SynaFlow open source release ([`1cd3c94`](https://github.com/humansoftware/synaflow/commit/1cd3c94f77314898ebb5834e5e2f5d1e8780c224))

synaflow-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Human Software
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

synaflow-0.1.0/PKG-INFO

@@ -0,0 +1,93 @@
+Metadata-Version: 2.4
+Name: synaflow
+Version: 0.1.0
+Summary: A lightweight, type-hint driven engine for executing Directed Acyclic Graphs (DAGs) and lockstep pipelines in Python.
+Project-URL: Homepage, https://github.com/humansoftware/synaflow
+Project-URL: Repository, https://github.com/humansoftware/synaflow.git
+Author-email: mvallebr <mvallebr@example.com>
+License: MIT
+License-File: LICENSE
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# SynaFlow 🌊🧠
+
+**SynaFlow** is a lightweight, pure-Python pipeline engine that uses **Type Hints** to magically wire and execute Directed Acyclic Graphs (DAGs). 
+
+It solves the "dependency hell" and boilerplate associated with building data pipelines by automatically inferring the flow of data based exclusively on Python's static type annotations. 
+
+## The Problem It Solves
+
+Building data pipelines usually involves two headaches:
+1. **Explicit Wiring:** You have to manually define which function outputs go to which function inputs (e.g., `A >> B >> C`), creating verbose and fragile architectures.
+2. **Memory Explosions vs. Lazy Evaluation:** Passing large datasets around usually means holding them entirely in memory (Lists) or dealing with complex generator management. If you have multiple consumers for a single generator, you usually have to write clunky `itertools.tee` boilerplate yourself.
+
+## The SynaFlow Solution
+
+SynaFlow looks at the **Type Hints** of your functions and automatically wires everything together for you. If `Step A` outputs an `int` and `Step B` requires an `int`, SynaFlow connects them instantly.
+
+Furthermore, SynaFlow has a **smart lockstep streaming engine**:
+- If a producer yields a `Generator` and a consumer expects an `Iterator`, SynaFlow streams the data lazily without ever holding it in memory.
+- If multiple consumers want that same generator, SynaFlow automatically forks it (`tee`) and drives them in parallel (lockstep).
+- If one consumer explicitly asks for a `list`, SynaFlow automatically materializes the data only for that specific branch.
+
+## How is it different from other frameworks?
+
+There are many amazing orchestration frameworks out there, but SynaFlow fills a very specific gap: **In-process Streaming Micro-Orchestration**.
+
+### vs. Hamilton
+[Hamilton](https://github.com/DAGWorks-Inc/hamilton) is a fantastic tool that also uses Python function signatures to build DAGs. However, Hamilton is heavily geared towards DataFrames and feature engineering, generally expecting functions to return concrete values (columns/scalars). **SynaFlow**, on the other hand, is built from the ground up to support **Native Generators and Lazy Streaming**. While Hamilton maps functions to columns, SynaFlow maps functions to continuous data streams, automatically interleaving multiple consumers in lockstep without memory spikes.
+
+### vs. Airflow / Prefect / Dagster
+These are **Macro-Orchestrators**. They are designed to orchestrate heavy, distributed tasks across clusters, Docker containers, and different machines. They rely on state databases and massive IO overhead. **SynaFlow is a Micro-Orchestrator**. It runs entirely within a single Python process. You would use Airflow to trigger a daily job, but you would use SynaFlow *inside* that job to smartly route and stream millions of rows between your Python functions.
+
+## Quickstart
+
+```python
+from typing import NamedTuple
+from collections.abc import Generator, Iterator
+from synaflow import pipeline, step, run
+
+# Define the data required to start your pipeline
+class MyParams(NamedTuple):
+    count: int
+
+# 1. Producer outputs a stream
+def producer(count: int) -> Generator[int, None, None]:
+    yield from range(count)
+
+# 2. Transformer consumes the stream lazily
+def transformer(producer: Iterator[int]) -> Generator[int, None, None]:
+    for val in producer:
+        yield val * 10
+
+# 3. Consumer automatically gets the stream!
+def consumer(transformer: Iterator[int]) -> None:
+    for x in transformer:
+        print(f"Consumed: {x}")
+
+# SynaFlow reads the Type Hints and wires the DAG automatically!
+my_pipeline = pipeline(
+    name="example",
+    params=MyParams,
+    steps=[
+        step("producer", fn=producer),
+        step("transformer", fn=transformer),
+        step("consumer", fn=consumer)
+    ]
+)
+
+# Run it
+run(my_pipeline, MyParams(count=5))
+```
+
+## Advanced Features
+- **Auto-DAG compilation and validation** before execution.
+- Strict type-checking: Pipeline refuses to run if type annotations are incompatible.
+- Easily export DAG structures as JSON (`my_pipeline.to_dict()`) for snapshot testing or UI rendering.
+
+## License
+MIT License

synaflow-0.1.0/README.md

@@ -0,0 +1,78 @@
+# SynaFlow 🌊🧠
+
+**SynaFlow** is a lightweight, pure-Python pipeline engine that uses **Type Hints** to magically wire and execute Directed Acyclic Graphs (DAGs). 
+
+It solves the "dependency hell" and boilerplate associated with building data pipelines by automatically inferring the flow of data based exclusively on Python's static type annotations. 
+
+## The Problem It Solves
+
+Building data pipelines usually involves two headaches:
+1. **Explicit Wiring:** You have to manually define which function outputs go to which function inputs (e.g., `A >> B >> C`), creating verbose and fragile architectures.
+2. **Memory Explosions vs. Lazy Evaluation:** Passing large datasets around usually means holding them entirely in memory (Lists) or dealing with complex generator management. If you have multiple consumers for a single generator, you usually have to write clunky `itertools.tee` boilerplate yourself.
+
+## The SynaFlow Solution
+
+SynaFlow looks at the **Type Hints** of your functions and automatically wires everything together for you. If `Step A` outputs an `int` and `Step B` requires an `int`, SynaFlow connects them instantly.
+
+Furthermore, SynaFlow has a **smart lockstep streaming engine**:
+- If a producer yields a `Generator` and a consumer expects an `Iterator`, SynaFlow streams the data lazily without ever holding it in memory.
+- If multiple consumers want that same generator, SynaFlow automatically forks it (`tee`) and drives them in parallel (lockstep).
+- If one consumer explicitly asks for a `list`, SynaFlow automatically materializes the data only for that specific branch.
+
+## How is it different from other frameworks?
+
+There are many amazing orchestration frameworks out there, but SynaFlow fills a very specific gap: **In-process Streaming Micro-Orchestration**.
+
+### vs. Hamilton
+[Hamilton](https://github.com/DAGWorks-Inc/hamilton) is a fantastic tool that also uses Python function signatures to build DAGs. However, Hamilton is heavily geared towards DataFrames and feature engineering, generally expecting functions to return concrete values (columns/scalars). **SynaFlow**, on the other hand, is built from the ground up to support **Native Generators and Lazy Streaming**. While Hamilton maps functions to columns, SynaFlow maps functions to continuous data streams, automatically interleaving multiple consumers in lockstep without memory spikes.
+
+### vs. Airflow / Prefect / Dagster
+These are **Macro-Orchestrators**. They are designed to orchestrate heavy, distributed tasks across clusters, Docker containers, and different machines. They rely on state databases and massive IO overhead. **SynaFlow is a Micro-Orchestrator**. It runs entirely within a single Python process. You would use Airflow to trigger a daily job, but you would use SynaFlow *inside* that job to smartly route and stream millions of rows between your Python functions.
+
+## Quickstart
+
+```python
+from typing import NamedTuple
+from collections.abc import Generator, Iterator
+from synaflow import pipeline, step, run
+
+# Define the data required to start your pipeline
+class MyParams(NamedTuple):
+    count: int
+
+# 1. Producer outputs a stream
+def producer(count: int) -> Generator[int, None, None]:
+    yield from range(count)
+
+# 2. Transformer consumes the stream lazily
+def transformer(producer: Iterator[int]) -> Generator[int, None, None]:
+    for val in producer:
+        yield val * 10
+
+# 3. Consumer automatically gets the stream!
+def consumer(transformer: Iterator[int]) -> None:
+    for x in transformer:
+        print(f"Consumed: {x}")
+
+# SynaFlow reads the Type Hints and wires the DAG automatically!
+my_pipeline = pipeline(
+    name="example",
+    params=MyParams,
+    steps=[
+        step("producer", fn=producer),
+        step("transformer", fn=transformer),
+        step("consumer", fn=consumer)
+    ]
+)
+
+# Run it
+run(my_pipeline, MyParams(count=5))
+```
+
+## Advanced Features
+- **Auto-DAG compilation and validation** before execution.
+- Strict type-checking: Pipeline refuses to run if type annotations are incompatible.
+- Easily export DAG structures as JSON (`my_pipeline.to_dict()`) for snapshot testing or UI rendering.
+
+## License
+MIT License

synaflow-0.1.0/pyproject.toml

@@ -0,0 +1,38 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "synaflow"
+version = "0.1.0"
+description = "A lightweight, type-hint driven engine for executing Directed Acyclic Graphs (DAGs) and lockstep pipelines in Python."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [
+    { name = "mvallebr", email = "mvallebr@example.com" }
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = []
+
+[project.urls]
+Homepage = "https://github.com/humansoftware/synaflow"
+Repository = "https://github.com/humansoftware/synaflow.git"
+
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["synaflow"]
+
+[tool.semantic_release]
+version_toml = [
+    "pyproject.toml:project.version"
+]
+commit_parser = "angular"
+major_on_zero = false

synaflow-0.1.0/synaflow/__init__.py

@@ -0,0 +1,21 @@
+"""
+Pipeline Engine
+
+A lightweight, robust engine for defining and executing typed Directed Acyclic Graphs (DAGs).
+This module defines the public interface for clients.
+"""
+
+from .executor import run
+from .pipeline import PipelineDef, pipeline
+from .step import Step, step
+from .types import OnError, StepParams
+
+__all__ = [
+    "PipelineDef",
+    "pipeline",
+    "Step",
+    "step",
+    "OnError",
+    "StepParams",
+    "run",
+]