dagrun 0.1.0__tar.gz

dagrun-0.1.0/.gitignore

@@ -0,0 +1,4 @@
+# Python
+__pycache__
+dist/
+

dagrun-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Pierre Hugo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dagrun-0.1.0/PKG-INFO

@@ -0,0 +1,87 @@
+Metadata-Version: 2.4
+Name: dagrun
+Version: 0.1.0
+Summary: Declarative DAG runner with pipelining and concurrency control.
+Author-email: Pierre Hugo <pierrekin@househugo.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Typing :: Typed
+Requires-Python: >=3.14
+Description-Content-Type: text/markdown
+
+# dagrun
+
+A small, dependency-free DAG runner for Python where you never declare an edge.
+Annotate your functions with the columns they read and produce, and dagrun will
+figure out the rest.
+
+## Example
+
+```python
+from collections.abc import Iterable
+from dagrun import Dag, DagRunner
+from dagrun.model import PK, FK, Column, Entity
+
+
+class Site(Entity):
+    site_id = PK[str]
+    name = Column[str | None]
+
+
+class File(Entity):
+    file_id = PK[str]
+    site = FK[Site]
+    name = Column[str | None]
+    title = Column[str | None]
+
+
+dag = Dag()
+
+
+@dag.fn
+def discover_sites(sp: Sharepoint) -> Iterable[Site[Site.name]]:
+    for raw in sp.list_sites():
+        yield Site(site_id=raw["id"], name=raw["name"])
+
+
+@dag.fn
+def discover_files(site: Site, sp: Sharepoint) -> Iterable[File[File.name]]:
+    for raw in sp.list_files(site.site_id):
+        yield File(file_id=raw["id"], site=site, name=raw["name"])
+
+
+@dag.fn
+def extract_title(file: File[File.name]) -> File[File.title]:
+    return File(file_id=file.file_id, title=file.name.removesuffix(".txt"))
+
+
+runner = DagRunner()
+runner.provide(Sharepoint, Sharepoint)
+runner.execute(dag)
+```
+
+Parameter annotations declare what a function reads (`File[File.name]`), return
+annotations declare what it produces, and the dependency graph is built from
+those. Reads and writes are keyed per column, so functions that enrich different
+columns of the same entity run independently.
+
+## What you get
+
+- **Compile-time checks.** `compile()` fails on a missing dependency, a column
+  produced twice, or a cycle, before anything runs.
+- **Incremental reruns.** The store records when each `(function, input)` pair
+  last ran. Pass `max_age=timedelta(hours=6)` to skip anything still fresh.
+- **Deferred nodes.** A node can `raise Defer()` to skip the current input at
+  runtime, writing and recording nothing so the next run reconsiders it.
+- **Pipelining.** Tasks are scheduled and gated per input, so `extract_title`
+  starts on the first file while `discover_files` is still finding the rest.
+- **Concurrency control.** Each function declares a `cost` against named pools to
+  bound resource usage and respect rate limits.
+- **Blob columns.** A `Blob` column stashes bytes in a dedicated backend and
+  keeps only a reference in the row.
+

dagrun-0.1.0/README.md

@@ -0,0 +1,71 @@
+# dagrun
+
+A small, dependency-free DAG runner for Python where you never declare an edge.
+Annotate your functions with the columns they read and produce, and dagrun will
+figure out the rest.
+
+## Example
+
+```python
+from collections.abc import Iterable
+from dagrun import Dag, DagRunner
+from dagrun.model import PK, FK, Column, Entity
+
+
+class Site(Entity):
+    site_id = PK[str]
+    name = Column[str | None]
+
+
+class File(Entity):
+    file_id = PK[str]
+    site = FK[Site]
+    name = Column[str | None]
+    title = Column[str | None]
+
+
+dag = Dag()
+
+
+@dag.fn
+def discover_sites(sp: Sharepoint) -> Iterable[Site[Site.name]]:
+    for raw in sp.list_sites():
+        yield Site(site_id=raw["id"], name=raw["name"])
+
+
+@dag.fn
+def discover_files(site: Site, sp: Sharepoint) -> Iterable[File[File.name]]:
+    for raw in sp.list_files(site.site_id):
+        yield File(file_id=raw["id"], site=site, name=raw["name"])
+
+
+@dag.fn
+def extract_title(file: File[File.name]) -> File[File.title]:
+    return File(file_id=file.file_id, title=file.name.removesuffix(".txt"))
+
+
+runner = DagRunner()
+runner.provide(Sharepoint, Sharepoint)
+runner.execute(dag)
+```
+
+Parameter annotations declare what a function reads (`File[File.name]`), return
+annotations declare what it produces, and the dependency graph is built from
+those. Reads and writes are keyed per column, so functions that enrich different
+columns of the same entity run independently.
+
+## What you get
+
+- **Compile-time checks.** `compile()` fails on a missing dependency, a column
+  produced twice, or a cycle, before anything runs.
+- **Incremental reruns.** The store records when each `(function, input)` pair
+  last ran. Pass `max_age=timedelta(hours=6)` to skip anything still fresh.
+- **Deferred nodes.** A node can `raise Defer()` to skip the current input at
+  runtime, writing and recording nothing so the next run reconsiders it.
+- **Pipelining.** Tasks are scheduled and gated per input, so `extract_title`
+  starts on the first file while `discover_files` is still finding the rest.
+- **Concurrency control.** Each function declares a `cost` against named pools to
+  bound resource usage and respect rate limits.
+- **Blob columns.** A `Blob` column stashes bytes in a dedicated backend and
+  keeps only a reference in the row.
+

dagrun-0.1.0/pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "dagrun"
+version = "0.1.0"
+description = "Declarative DAG runner with pipelining and concurrency control."
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.14"
+authors = [{ name = "Pierre Hugo", email = "pierrekin@househugo.com" }]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.14",
+    "Typing :: Typed",
+]
+dependencies = []
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dagrun"]
+
+[dependency-groups]
+dev = [
+    "mypy>=1.20.1",
+    "pytest>=8.0.0",
+    "ruff>=0.15.11",
+]
+
+[tool.mypy]
+strict = true
+
+[tool.ruff]
+line-length = 100
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

dagrun-0.1.0/src/dagrun/__init__.py

@@ -0,0 +1,6 @@
+from dagrun import model
+from dagrun.control import Defer
+from dagrun.dag import Dag, DagValidationError
+from dagrun.dag_runner import DagRunner
+
+__all__ = ["Dag", "DagRunner", "DagValidationError", "Defer", "model"]

dagrun-0.1.0/src/dagrun/chain.py

@@ -0,0 +1,71 @@
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+
+def member(fn: Callable[..., Any]) -> Callable[..., Any]:
+    fn.__dagrun_member__ = True  # type: ignore[attr-defined]
+    return fn
+
+
+class Combine(Protocol):
+    def reduce(self, column: str, values: list[Any]) -> Any: ...
+
+
+@dataclass(frozen=True)
+class LastWins:
+    def reduce(self, column: str, values: list[Any]) -> Any:
+        return values[-1]
+
+
+@dataclass(frozen=True)
+class FirstWins:
+    def reduce(self, column: str, values: list[Any]) -> Any:
+        return values[0]
+
+
+@dataclass(frozen=True)
+class Error:
+    def reduce(self, column: str, values: list[Any]) -> Any:
+        distinct = {_hash(v) for v in values}
+        if len(distinct) > 1:
+            raise CombineConflict(column, values)
+        return values[0]
+
+
+class CombineConflict(RuntimeError):
+    def __init__(self, column: str, values: list[Any]) -> None:
+        super().__init__(f"Conflicting values for column {column!r}: {values!r}")
+        self.column = column
+        self.values = values
+
+
+def _hash(value: Any) -> Any:
+    try:
+        hash(value)
+    except TypeError:
+        return repr(value)
+    return value
+
+
+@dataclass(frozen=True)
+class PerColumn:
+    """Per-column combine override map. '*' means default for unlisted columns."""
+
+    rules: tuple[tuple[str, Combine], ...]
+
+    def __init__(self, rules: Mapping[str, Combine] | "tuple[tuple[str, Combine], ...]") -> None:
+        if isinstance(rules, Mapping):
+            items = tuple(rules.items())
+        else:
+            items = tuple(rules)
+        object.__setattr__(self, "rules", items)
+
+    def reduce(self, column: str, values: list[Any]) -> Any:
+        lookup = dict(self.rules)
+        rule = lookup.get(column, lookup.get("*"))
+        if rule is None:
+            return values[-1]
+        return rule.reduce(column, values)
+
+

dagrun-0.1.0/src/dagrun/control.py

@@ -0,0 +1,9 @@
+class Defer(Exception):
+    """Signal from a node that the current input should not be processed now.
+
+    When a node raises Defer the runner discards any output produced for that
+    input and records no invocation, so the input is evaluated again from
+    scratch on the next run rather than being treated as done. Use it for
+    decisions that depend on runtime configuration (e.g. a token budget)
+    rather than on an intrinsic, persistable property of the input.
+    """