ntask 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ntask-1.0.0/LICENSE +28 -0
- ntask-1.0.0/PKG-INFO +172 -0
- ntask-1.0.0/README.md +131 -0
- ntask-1.0.0/pyproject.toml +76 -0
- ntask-1.0.0/setup.cfg +4 -0
- ntask-1.0.0/src/ntask/__init__.py +21 -0
- ntask-1.0.0/src/ntask/__main__.py +6 -0
- ntask-1.0.0/src/ntask/_cache/__init__.py +178 -0
- ntask-1.0.0/src/ntask/_cache/body.py +39 -0
- ntask-1.0.0/src/ntask/_cache/diff.py +127 -0
- ntask-1.0.0/src/ntask/_cache/hash.py +43 -0
- ntask-1.0.0/src/ntask/_cache/key.py +68 -0
- ntask-1.0.0/src/ntask/_cache/manifest.py +58 -0
- ntask-1.0.0/src/ntask/_cache/outputs.py +86 -0
- ntask-1.0.0/src/ntask/_cache/store.py +116 -0
- ntask-1.0.0/src/ntask/_cli.py +475 -0
- ntask-1.0.0/src/ntask/_cli_args.py +98 -0
- ntask-1.0.0/src/ntask/_cli_docstring.py +55 -0
- ntask-1.0.0/src/ntask/_cli_format.py +76 -0
- ntask-1.0.0/src/ntask/_cli_why.py +122 -0
- ntask-1.0.0/src/ntask/_config.py +62 -0
- ntask-1.0.0/src/ntask/_coordinator.py +56 -0
- ntask-1.0.0/src/ntask/_dag.py +120 -0
- ntask-1.0.0/src/ntask/_depends.py +119 -0
- ntask-1.0.0/src/ntask/_discovery.py +52 -0
- ntask-1.0.0/src/ntask/_errors.py +27 -0
- ntask-1.0.0/src/ntask/_executor.py +221 -0
- ntask-1.0.0/src/ntask/_registry.py +62 -0
- ntask-1.0.0/src/ntask/_remote/__init__.py +48 -0
- ntask-1.0.0/src/ntask/_remote/_tar.py +61 -0
- ntask-1.0.0/src/ntask/_remote/base.py +20 -0
- ntask-1.0.0/src/ntask/_remote/gcs.py +64 -0
- ntask-1.0.0/src/ntask/_remote/http.py +89 -0
- ntask-1.0.0/src/ntask/_remote/local_fs.py +48 -0
- ntask-1.0.0/src/ntask/_remote/s3.py +101 -0
- ntask-1.0.0/src/ntask/_render/__init__.py +5 -0
- ntask-1.0.0/src/ntask/_render/base.py +14 -0
- ntask-1.0.0/src/ntask/_render/log.py +38 -0
- ntask-1.0.0/src/ntask/_render/rich.py +49 -0
- ntask-1.0.0/src/ntask/_render/tui.py +191 -0
- ntask-1.0.0/src/ntask/_shell.py +238 -0
- ntask-1.0.0/src/ntask/_task.py +159 -0
- ntask-1.0.0/src/ntask/_watch.py +147 -0
- ntask-1.0.0/src/ntask.egg-info/PKG-INFO +172 -0
- ntask-1.0.0/src/ntask.egg-info/SOURCES.txt +67 -0
- ntask-1.0.0/src/ntask.egg-info/dependency_links.txt +1 -0
- ntask-1.0.0/src/ntask.egg-info/entry_points.txt +2 -0
- ntask-1.0.0/src/ntask.egg-info/requires.txt +24 -0
- ntask-1.0.0/src/ntask.egg-info/top_level.txt +1 -0
- ntask-1.0.0/tests/test_cli.py +419 -0
- ntask-1.0.0/tests/test_cli_args.py +78 -0
- ntask-1.0.0/tests/test_cli_docstring.py +37 -0
- ntask-1.0.0/tests/test_cli_format.py +54 -0
- ntask-1.0.0/tests/test_cli_why.py +107 -0
- ntask-1.0.0/tests/test_coordinator.py +105 -0
- ntask-1.0.0/tests/test_dag.py +71 -0
- ntask-1.0.0/tests/test_depends.py +73 -0
- ntask-1.0.0/tests/test_discovery.py +60 -0
- ntask-1.0.0/tests/test_executor.py +533 -0
- ntask-1.0.0/tests/test_group.py +46 -0
- ntask-1.0.0/tests/test_integration.py +112 -0
- ntask-1.0.0/tests/test_property.py +62 -0
- ntask-1.0.0/tests/test_registry.py +23 -0
- ntask-1.0.0/tests/test_render.py +77 -0
- ntask-1.0.0/tests/test_render_tui.py +196 -0
- ntask-1.0.0/tests/test_shell.py +170 -0
- ntask-1.0.0/tests/test_task.py +107 -0
- ntask-1.0.0/tests/test_watch.py +180 -0
- ntask-1.0.0/tests/test_watch_integration.py +75 -0
ntask-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Sean Nieuwoudt
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
16
|
+
contributors may be used to endorse or promote products derived from
|
|
17
|
+
this software without specific prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
22
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
23
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
25
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
27
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
ntask-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ntask
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: A Python-native task runner with content-hash caching and DAG execution.
|
|
5
|
+
Author-email: Sean Nieuwoudt <sean@underwulf.com>
|
|
6
|
+
License: BSD-3-Clause
|
|
7
|
+
Keywords: task-runner,build-system,make,just,ci
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Topic :: Software Development :: Build Tools
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: rich>=13.7
|
|
21
|
+
Requires-Dist: xxhash>=3.4
|
|
22
|
+
Requires-Dist: pathspec>=0.12
|
|
23
|
+
Requires-Dist: anyio>=4.3
|
|
24
|
+
Requires-Dist: watchfiles>=0.22
|
|
25
|
+
Requires-Dist: textual>=0.80
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
|
|
29
|
+
Requires-Dist: hypothesis>=6.98; extra == "dev"
|
|
30
|
+
Requires-Dist: ruff>=0.3; extra == "dev"
|
|
31
|
+
Requires-Dist: mypy>=1.9; extra == "dev"
|
|
32
|
+
Requires-Dist: moto>=5.0; extra == "dev"
|
|
33
|
+
Provides-Extra: s3
|
|
34
|
+
Requires-Dist: boto3>=1.34; extra == "s3"
|
|
35
|
+
Provides-Extra: gcs
|
|
36
|
+
Requires-Dist: google-cloud-storage>=2.14; extra == "gcs"
|
|
37
|
+
Provides-Extra: all
|
|
38
|
+
Requires-Dist: boto3>=1.34; extra == "all"
|
|
39
|
+
Requires-Dist: google-cloud-storage>=2.14; extra == "all"
|
|
40
|
+
Dynamic: license-file
|
|
41
|
+
|
|
42
|
+
# ntask
|
|
43
|
+
|
|
44
|
+
A Python-native task runner with content-hash caching and DAG execution.
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install ntask
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Why
|
|
51
|
+
|
|
52
|
+
Your Makefile runs everything every time. Your Justfile has no dependency graph. Your `tasks.py` for Invoke is five years old, has no types, and you still have to write `ctx.run`. This is what a task runner looks like when you start over with caching, types, and a DAG.
|
|
53
|
+
|
|
54
|
+
## Quickstart
|
|
55
|
+
|
|
56
|
+
Create a `tasks.py`:
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
from ntask import task, cached, depends, shell
|
|
60
|
+
|
|
61
|
+
@task
|
|
62
|
+
def install():
|
|
63
|
+
shell("pip install -e .")
|
|
64
|
+
|
|
65
|
+
@task
|
|
66
|
+
@cached(inputs=["src/**/*.py", "tests/**/*.py"])
|
|
67
|
+
def test(pattern: str = "", verbose: bool = False):
|
|
68
|
+
"""Run the test suite."""
|
|
69
|
+
flags = "-v" if verbose else ""
|
|
70
|
+
k = f"-k {pattern}" if pattern else ""
|
|
71
|
+
shell(f"pytest {flags} {k}")
|
|
72
|
+
|
|
73
|
+
@task
|
|
74
|
+
@cached(inputs=["src/**/*.py"])
|
|
75
|
+
def lint():
|
|
76
|
+
shell("ruff check src/")
|
|
77
|
+
|
|
78
|
+
@task
|
|
79
|
+
def check():
|
|
80
|
+
"""All quality checks."""
|
|
81
|
+
depends(lint, test)
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Run:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
ntask --list # show every registered task
|
|
88
|
+
ntask test --pattern=auth --verbose
|
|
89
|
+
ntask check # lint + test in order; both cache
|
|
90
|
+
ntask check -j # all CPU cores; bare -j picks the count
|
|
91
|
+
ntask watch test # rerun on every src/ or tests/ change
|
|
92
|
+
ntask --why test # explain why the last cache lookup missed
|
|
93
|
+
ntask --graph check # ASCII DAG (mermaid / dot also available)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
The second time you run `ntask check`, both `lint` and `test` are content-hash cache hits and finish in milliseconds. Change one file in `src/` and only the affected tasks rerun, transitively.
|
|
97
|
+
|
|
98
|
+
## Team cache
|
|
99
|
+
|
|
100
|
+
Share cache hits across machines via S3 (or GCS, HTTP, NFS):
|
|
101
|
+
|
|
102
|
+
```toml
|
|
103
|
+
# pyproject.toml
|
|
104
|
+
[tool.ntask.remote_cache]
|
|
105
|
+
type = "s3"
|
|
106
|
+
bucket = "my-team-cache"
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
pip install ntask[s3]
|
|
111
|
+
ntask check # first person populates, the rest hit instantly
|
|
112
|
+
ntask check --offline # skip the remote for fast dev loops
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
S3-compatibles (MinIO, R2, B2) take an `endpoint_url`. GCS and HTTP backends work the same way; the HTTP backend is plain `GET`/`PUT`/`HEAD` over stdlib `urllib`, so any object store with PUT enabled is fair game.
|
|
116
|
+
|
|
117
|
+
## Live DAG display
|
|
118
|
+
|
|
119
|
+
Run `ntask check` from an interactive terminal and a Textual TUI shows a live tree of the DAG with per-task state icons and durations. Pipe the output, set `tui = false` in `[tool.ntask]`, or pass `--no-tui` to fall back to the line-based renderer.
|
|
120
|
+
|
|
121
|
+
## Other things you'll reach for
|
|
122
|
+
|
|
123
|
+
- **Exclusive tasks.** `@task(parallel=False)` makes a task a DAG-wide barrier: it waits for everything in flight to drain, then runs alone. Use it for releases, migrations, anything that mutates shared state.
|
|
124
|
+
- **Monorepos.** `@group("api")` over a class namespaces every task method as `api.<name>`. Cross-group dependencies via `@task(deps=[Other.task, ...])` or string fqns.
|
|
125
|
+
- **Capture output.** `shell("git rev-parse HEAD", capture=True)` returns a `ShellResult` with `.stdout`, `.stderr`, `.returncode`, `.duration`, `.ok`.
|
|
126
|
+
- **Force a rerun.** `ntask --force <task>` bypasses the cache for that one task. `ntask --no-cache` ignores the cache entirely. `ntask clean` wipes entry manifests; `ntask clean --all` wipes the whole `.ntask/` directory.
|
|
127
|
+
|
|
128
|
+
## Examples
|
|
129
|
+
|
|
130
|
+
Six runnable, self-contained examples under [`examples/`](examples/):
|
|
131
|
+
|
|
132
|
+
| # | Demonstrates |
|
|
133
|
+
|----------------------------------------------|------------------------------------------------|
|
|
134
|
+
| [01-hello](examples/01-hello/) | Smallest possible cached task |
|
|
135
|
+
| [02-python-lib](examples/02-python-lib/) | install / lint / typecheck / test / build |
|
|
136
|
+
| [03-parallel](examples/03-parallel/) | `-j N` fan-out and `parallel=False` barrier |
|
|
137
|
+
| [04-watch](examples/04-watch/) | `ntask watch` rerun-on-change loop |
|
|
138
|
+
| [05-remote-cache](examples/05-remote-cache/) | local-fs remote backend shared between clones |
|
|
139
|
+
| [06-monorepo](examples/06-monorepo/) | `@group(...)` namespacing and cross-group deps |
|
|
140
|
+
|
|
141
|
+
`cd` into any directory and run `ntask --list`.
|
|
142
|
+
|
|
143
|
+
## Features
|
|
144
|
+
|
|
145
|
+
| feature | ntask | make | just | invoke | doit | poe |
|
|
146
|
+
|----------------------------------|:-----:|:-------:|:-------:|:-------:|:-------:|:-------:|
|
|
147
|
+
| Typed Python tasks | yes | no | no | partial | no | partial |
|
|
148
|
+
| Content-hash input caching | yes | partial | no | no | partial | no |
|
|
149
|
+
| Transitive cache-key propagation | yes | no | no | no | partial | no |
|
|
150
|
+
| Remote cache (S3/GCS/HTTP) | yes | no | no | no | no | no |
|
|
151
|
+
| DAG dependency resolution | yes | yes | no | partial | yes | no |
|
|
152
|
+
| Type-hint to CLI args | yes | no | partial | partial | no | yes |
|
|
153
|
+
| Parallel DAG execution (`-j`) | yes | yes | no | no | yes | no |
|
|
154
|
+
| Live DAG TUI | yes | no | no | no | no | no |
|
|
155
|
+
| Windows first-class | yes | partial | yes | yes | yes | yes |
|
|
156
|
+
|
|
157
|
+
Cache miss messages name the specific file or env change that caused the invalidation. `ntask --why <task>` prints the full breakdown.
|
|
158
|
+
|
|
159
|
+
## Docs
|
|
160
|
+
|
|
161
|
+
- [Technical handbook](docs/guide.md) - the comprehensive reference, every flag and config key
|
|
162
|
+
- [Tutorial: replace your Makefile in 10 minutes](docs/tutorial.md)
|
|
163
|
+
- [Caching: the full contract](docs/caching.md)
|
|
164
|
+
- [Migrating from Make / just / Invoke / Poe](docs/migration.md)
|
|
165
|
+
- [Short API reference](docs/reference.md)
|
|
166
|
+
- [Roadmap](docs/roadmap.md)
|
|
167
|
+
|
|
168
|
+
Requirements: Python 3.11 or newer. BSD-3-Clause.
|
|
169
|
+
|
|
170
|
+
## License
|
|
171
|
+
|
|
172
|
+
BSD-3-Clause. Copyright © 2026 Sean Nieuwoudt.
|
ntask-1.0.0/README.md
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# ntask
|
|
2
|
+
|
|
3
|
+
A Python-native task runner with content-hash caching and DAG execution.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install ntask
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
## Why
|
|
10
|
+
|
|
11
|
+
Your Makefile runs everything every time. Your Justfile has no dependency graph. Your `tasks.py` for Invoke is five years old, has no types, and you still have to write `ctx.run`. This is what a task runner looks like when you start over with caching, types, and a DAG.
|
|
12
|
+
|
|
13
|
+
## Quickstart
|
|
14
|
+
|
|
15
|
+
Create a `tasks.py`:
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
from ntask import task, cached, depends, shell
|
|
19
|
+
|
|
20
|
+
@task
|
|
21
|
+
def install():
|
|
22
|
+
shell("pip install -e .")
|
|
23
|
+
|
|
24
|
+
@task
|
|
25
|
+
@cached(inputs=["src/**/*.py", "tests/**/*.py"])
|
|
26
|
+
def test(pattern: str = "", verbose: bool = False):
|
|
27
|
+
"""Run the test suite."""
|
|
28
|
+
flags = "-v" if verbose else ""
|
|
29
|
+
k = f"-k {pattern}" if pattern else ""
|
|
30
|
+
shell(f"pytest {flags} {k}")
|
|
31
|
+
|
|
32
|
+
@task
|
|
33
|
+
@cached(inputs=["src/**/*.py"])
|
|
34
|
+
def lint():
|
|
35
|
+
shell("ruff check src/")
|
|
36
|
+
|
|
37
|
+
@task
|
|
38
|
+
def check():
|
|
39
|
+
"""All quality checks."""
|
|
40
|
+
depends(lint, test)
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Run:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
ntask --list # show every registered task
|
|
47
|
+
ntask test --pattern=auth --verbose
|
|
48
|
+
ntask check # lint + test in order; both cache
|
|
49
|
+
ntask check -j # all CPU cores; bare -j picks the count
|
|
50
|
+
ntask watch test # rerun on every src/ or tests/ change
|
|
51
|
+
ntask --why test # explain why the last cache lookup missed
|
|
52
|
+
ntask --graph check # ASCII DAG (mermaid / dot also available)
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
The second time you run `ntask check`, both `lint` and `test` are content-hash cache hits and finish in milliseconds. Change one file in `src/` and only the affected tasks rerun, transitively.
|
|
56
|
+
|
|
57
|
+
## Team cache
|
|
58
|
+
|
|
59
|
+
Share cache hits across machines via S3 (or GCS, HTTP, NFS):
|
|
60
|
+
|
|
61
|
+
```toml
|
|
62
|
+
# pyproject.toml
|
|
63
|
+
[tool.ntask.remote_cache]
|
|
64
|
+
type = "s3"
|
|
65
|
+
bucket = "my-team-cache"
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
pip install ntask[s3]
|
|
70
|
+
ntask check # first person populates, the rest hit instantly
|
|
71
|
+
ntask check --offline # skip the remote for fast dev loops
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
S3-compatibles (MinIO, R2, B2) take an `endpoint_url`. GCS and HTTP backends work the same way; the HTTP backend is plain `GET`/`PUT`/`HEAD` over stdlib `urllib`, so any object store with PUT enabled is fair game.
|
|
75
|
+
|
|
76
|
+
## Live DAG display
|
|
77
|
+
|
|
78
|
+
Run `ntask check` from an interactive terminal and a Textual TUI shows a live tree of the DAG with per-task state icons and durations. Pipe the output, set `tui = false` in `[tool.ntask]`, or pass `--no-tui` to fall back to the line-based renderer.
|
|
79
|
+
|
|
80
|
+
## Other things you'll reach for
|
|
81
|
+
|
|
82
|
+
- **Exclusive tasks.** `@task(parallel=False)` makes a task a DAG-wide barrier: it waits for everything in flight to drain, then runs alone. Use it for releases, migrations, anything that mutates shared state.
|
|
83
|
+
- **Monorepos.** `@group("api")` over a class namespaces every task method as `api.<name>`. Cross-group dependencies via `@task(deps=[Other.task, ...])` or string fqns.
|
|
84
|
+
- **Capture output.** `shell("git rev-parse HEAD", capture=True)` returns a `ShellResult` with `.stdout`, `.stderr`, `.returncode`, `.duration`, `.ok`.
|
|
85
|
+
- **Force a rerun.** `ntask --force <task>` bypasses the cache for that one task. `ntask --no-cache` ignores the cache entirely. `ntask clean` wipes entry manifests; `ntask clean --all` wipes the whole `.ntask/` directory.
|
|
86
|
+
|
|
87
|
+
## Examples
|
|
88
|
+
|
|
89
|
+
Six runnable, self-contained examples under [`examples/`](examples/):
|
|
90
|
+
|
|
91
|
+
| # | Demonstrates |
|
|
92
|
+
|----------------------------------------------|------------------------------------------------|
|
|
93
|
+
| [01-hello](examples/01-hello/) | Smallest possible cached task |
|
|
94
|
+
| [02-python-lib](examples/02-python-lib/) | install / lint / typecheck / test / build |
|
|
95
|
+
| [03-parallel](examples/03-parallel/) | `-j N` fan-out and `parallel=False` barrier |
|
|
96
|
+
| [04-watch](examples/04-watch/) | `ntask watch` rerun-on-change loop |
|
|
97
|
+
| [05-remote-cache](examples/05-remote-cache/) | local-fs remote backend shared between clones |
|
|
98
|
+
| [06-monorepo](examples/06-monorepo/) | `@group(...)` namespacing and cross-group deps |
|
|
99
|
+
|
|
100
|
+
`cd` into any directory and run `ntask --list`.
|
|
101
|
+
|
|
102
|
+
## Features
|
|
103
|
+
|
|
104
|
+
| feature | ntask | make | just | invoke | doit | poe |
|
|
105
|
+
|----------------------------------|:-----:|:-------:|:-------:|:-------:|:-------:|:-------:|
|
|
106
|
+
| Typed Python tasks | yes | no | no | partial | no | partial |
|
|
107
|
+
| Content-hash input caching | yes | partial | no | no | partial | no |
|
|
108
|
+
| Transitive cache-key propagation | yes | no | no | no | partial | no |
|
|
109
|
+
| Remote cache (S3/GCS/HTTP) | yes | no | no | no | no | no |
|
|
110
|
+
| DAG dependency resolution | yes | yes | no | partial | yes | no |
|
|
111
|
+
| Type-hint to CLI args | yes | no | partial | partial | no | yes |
|
|
112
|
+
| Parallel DAG execution (`-j`) | yes | yes | no | no | yes | no |
|
|
113
|
+
| Live DAG TUI | yes | no | no | no | no | no |
|
|
114
|
+
| Windows first-class | yes | partial | yes | yes | yes | yes |
|
|
115
|
+
|
|
116
|
+
Cache miss messages name the specific file or env change that caused the invalidation. `ntask --why <task>` prints the full breakdown.
|
|
117
|
+
|
|
118
|
+
## Docs
|
|
119
|
+
|
|
120
|
+
- [Technical handbook](docs/guide.md) - the comprehensive reference, every flag and config key
|
|
121
|
+
- [Tutorial: replace your Makefile in 10 minutes](docs/tutorial.md)
|
|
122
|
+
- [Caching: the full contract](docs/caching.md)
|
|
123
|
+
- [Migrating from Make / just / Invoke / Poe](docs/migration.md)
|
|
124
|
+
- [Short API reference](docs/reference.md)
|
|
125
|
+
- [Roadmap](docs/roadmap.md)
|
|
126
|
+
|
|
127
|
+
Requirements: Python 3.11 or newer. BSD-3-Clause.
|
|
128
|
+
|
|
129
|
+
## License
|
|
130
|
+
|
|
131
|
+
BSD-3-Clause. Copyright © 2026 Sean Nieuwoudt.
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ntask"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "A Python-native task runner with content-hash caching and DAG execution."
|
|
9
|
+
authors = [{name = "Sean Nieuwoudt", email = "sean@underwulf.com"}]
|
|
10
|
+
license = {text = "BSD-3-Clause"}
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
requires-python = ">=3.11"
|
|
13
|
+
keywords = ["task-runner", "build-system", "make", "just", "ci"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"License :: OSI Approved :: BSD License",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3.11",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Programming Language :: Python :: 3.13",
|
|
23
|
+
"Topic :: Software Development :: Build Tools",
|
|
24
|
+
]
|
|
25
|
+
dependencies = [
|
|
26
|
+
"rich>=13.7",
|
|
27
|
+
"xxhash>=3.4",
|
|
28
|
+
"pathspec>=0.12",
|
|
29
|
+
"anyio>=4.3",
|
|
30
|
+
"watchfiles>=0.22",
|
|
31
|
+
"textual>=0.80",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[project.optional-dependencies]
|
|
35
|
+
dev = [
|
|
36
|
+
"pytest>=8.0",
|
|
37
|
+
"pytest-asyncio>=0.23",
|
|
38
|
+
"hypothesis>=6.98",
|
|
39
|
+
"ruff>=0.3",
|
|
40
|
+
"mypy>=1.9",
|
|
41
|
+
"moto>=5.0",
|
|
42
|
+
]
|
|
43
|
+
s3 = ["boto3>=1.34"]
|
|
44
|
+
gcs = ["google-cloud-storage>=2.14"]
|
|
45
|
+
all = ["boto3>=1.34", "google-cloud-storage>=2.14"]
|
|
46
|
+
|
|
47
|
+
[project.scripts]
|
|
48
|
+
ntask = "ntask._cli:main"
|
|
49
|
+
|
|
50
|
+
[tool.setuptools]
|
|
51
|
+
package-dir = {"" = "src"}
|
|
52
|
+
|
|
53
|
+
[tool.setuptools.packages.find]
|
|
54
|
+
where = ["src"]
|
|
55
|
+
|
|
56
|
+
[tool.pytest.ini_options]
|
|
57
|
+
testpaths = ["tests"]
|
|
58
|
+
pythonpath = ["src"]
|
|
59
|
+
asyncio_mode = "auto"
|
|
60
|
+
|
|
61
|
+
[tool.ruff]
|
|
62
|
+
line-length = 100
|
|
63
|
+
target-version = "py311"
|
|
64
|
+
|
|
65
|
+
[tool.ruff.lint]
|
|
66
|
+
select = ["E", "F", "W", "I", "N", "UP", "B", "S", "A", "C4", "RUF"]
|
|
67
|
+
ignore = ["S101"]
|
|
68
|
+
|
|
69
|
+
[tool.mypy]
|
|
70
|
+
python_version = "3.11"
|
|
71
|
+
strict = true
|
|
72
|
+
files = ["src/ntask"]
|
|
73
|
+
|
|
74
|
+
[[tool.mypy.overrides]]
|
|
75
|
+
module = ["boto3", "botocore.*", "google.*"]
|
|
76
|
+
ignore_missing_imports = true
|
ntask-1.0.0/setup.cfg
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""ntask: a Python-native task runner with content-hash caching and DAG execution."""
|
|
2
|
+
|
|
3
|
+
from ._depends import depends
|
|
4
|
+
from ._errors import CycleError, DiscoveryError, NtaskError, ShellError
|
|
5
|
+
from ._shell import ShellResult, shell
|
|
6
|
+
from ._task import cached, group, task
|
|
7
|
+
|
|
8
|
+
__version__ = "1.0.0"
|
|
9
|
+
__all__ = [
|
|
10
|
+
"CycleError",
|
|
11
|
+
"DiscoveryError",
|
|
12
|
+
"NtaskError",
|
|
13
|
+
"ShellError",
|
|
14
|
+
"ShellResult",
|
|
15
|
+
"__version__",
|
|
16
|
+
"cached",
|
|
17
|
+
"depends",
|
|
18
|
+
"group",
|
|
19
|
+
"shell",
|
|
20
|
+
"task",
|
|
21
|
+
]
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import platform
|
|
5
|
+
import sys
|
|
6
|
+
import time
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from .._remote.base import RemoteBackend
|
|
10
|
+
from .._task import Task
|
|
11
|
+
from .body import hash_task_body
|
|
12
|
+
from .key import CacheBreakdown, CacheKeyInputs, InputRecord, compute_cache_key
|
|
13
|
+
from .manifest import compute_input_manifest
|
|
14
|
+
from .outputs import OutputStore
|
|
15
|
+
from .store import CacheEntry, CacheStore
|
|
16
|
+
|
|
17
|
+
# Module-level flag for warn-once pattern.
|
|
18
|
+
_remote_warn_fired = False
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _warn_once_remote_failed(e: BaseException) -> None:
|
|
22
|
+
"""Print a single warning the first time a remote call fails in this process."""
|
|
23
|
+
global _remote_warn_fired
|
|
24
|
+
if not _remote_warn_fired:
|
|
25
|
+
print(
|
|
26
|
+
f"warning: remote cache unreachable: {type(e).__name__}: {e}. "
|
|
27
|
+
f"Falling back to local.",
|
|
28
|
+
file=sys.stderr,
|
|
29
|
+
)
|
|
30
|
+
_remote_warn_fired = True
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _python_version_str() -> str:
|
|
34
|
+
return ".".join(str(x) for x in sys.version_info[:3])
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _platform_tag_str() -> str:
|
|
38
|
+
return f"{platform.system()}-{platform.machine()}".lower()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class CacheEngine:
|
|
42
|
+
def __init__(self, root: Path, remote: RemoteBackend | None = None):
|
|
43
|
+
self.root = root
|
|
44
|
+
self.store = CacheStore(root=root)
|
|
45
|
+
self.outputs = OutputStore(root=root)
|
|
46
|
+
self._remote = remote
|
|
47
|
+
|
|
48
|
+
def compute_key_and_breakdown(
|
|
49
|
+
self,
|
|
50
|
+
t: Task,
|
|
51
|
+
*,
|
|
52
|
+
workspace: Path,
|
|
53
|
+
upstream_keys_by_dep: dict[str, str],
|
|
54
|
+
) -> tuple[str, CacheBreakdown]:
|
|
55
|
+
cfg = t.cached_config
|
|
56
|
+
assert cfg is not None
|
|
57
|
+
manifest = compute_input_manifest(cfg.inputs, root=workspace)
|
|
58
|
+
body = hash_task_body(t.func)
|
|
59
|
+
env_values = {
|
|
60
|
+
name: os.environ[name] if name in os.environ else "<unset>"
|
|
61
|
+
for name in cfg.env
|
|
62
|
+
}
|
|
63
|
+
py_version = _python_version_str()
|
|
64
|
+
plat_tag = _platform_tag_str()
|
|
65
|
+
|
|
66
|
+
# Tuple ordering for cache key matches insertion order of upstream_keys_by_dep.
|
|
67
|
+
upstream_tuple = tuple(upstream_keys_by_dep.values())
|
|
68
|
+
key_inputs = CacheKeyInputs(
|
|
69
|
+
task_fqn=t.fqn,
|
|
70
|
+
task_body_hash=body,
|
|
71
|
+
env={name: ("" if v == "<unset>" else v) for name, v in env_values.items()},
|
|
72
|
+
env_names=cfg.env,
|
|
73
|
+
input_patterns=cfg.inputs,
|
|
74
|
+
input_manifest_digest=manifest.digest,
|
|
75
|
+
root=workspace,
|
|
76
|
+
upstream_keys=upstream_tuple if cfg.propagate else (),
|
|
77
|
+
strict=cfg.strict,
|
|
78
|
+
)
|
|
79
|
+
key = compute_cache_key(key_inputs)
|
|
80
|
+
|
|
81
|
+
input_records = tuple(
|
|
82
|
+
InputRecord(
|
|
83
|
+
path=fh.path.relative_to(workspace).as_posix(),
|
|
84
|
+
digest=fh.digest,
|
|
85
|
+
mode=fh.mode,
|
|
86
|
+
)
|
|
87
|
+
for fh in manifest.files
|
|
88
|
+
)
|
|
89
|
+
breakdown = CacheBreakdown(
|
|
90
|
+
input_patterns=cfg.inputs,
|
|
91
|
+
inputs=input_records,
|
|
92
|
+
env_values=env_values,
|
|
93
|
+
task_body_hash=body,
|
|
94
|
+
python_version=py_version,
|
|
95
|
+
platform_tag=plat_tag,
|
|
96
|
+
upstream_keys_by_dep=dict(upstream_keys_by_dep),
|
|
97
|
+
)
|
|
98
|
+
return key, breakdown
|
|
99
|
+
|
|
100
|
+
def check(self, t: Task, key: str) -> CacheEntry | None:
|
|
101
|
+
return self.store.get(t.fqn, key)
|
|
102
|
+
|
|
103
|
+
def store_entry(
|
|
104
|
+
self,
|
|
105
|
+
t: Task,
|
|
106
|
+
key: str,
|
|
107
|
+
*,
|
|
108
|
+
workspace: Path,
|
|
109
|
+
duration: float,
|
|
110
|
+
breakdown: CacheBreakdown,
|
|
111
|
+
upstream_keys: tuple[str, ...],
|
|
112
|
+
) -> CacheEntry:
|
|
113
|
+
cfg = t.cached_config
|
|
114
|
+
assert cfg is not None
|
|
115
|
+
outputs_hash = (
|
|
116
|
+
self.outputs.capture(cfg.outputs, root=workspace) if cfg.outputs else None
|
|
117
|
+
)
|
|
118
|
+
entry = CacheEntry(
|
|
119
|
+
key=key,
|
|
120
|
+
outputs_hash=outputs_hash,
|
|
121
|
+
duration=duration,
|
|
122
|
+
completed_at=time.time(),
|
|
123
|
+
upstream_keys=upstream_keys,
|
|
124
|
+
breakdown=breakdown,
|
|
125
|
+
)
|
|
126
|
+
self.store.put(t.fqn, entry)
|
|
127
|
+
return entry
|
|
128
|
+
|
|
129
|
+
def restore_outputs(self, entry: CacheEntry, *, workspace: Path) -> None:
|
|
130
|
+
if entry.outputs_hash:
|
|
131
|
+
self.outputs.restore(entry.outputs_hash, root=workspace)
|
|
132
|
+
|
|
133
|
+
def check_remote(self, t: Task, key: str) -> CacheEntry | None:
|
|
134
|
+
"""Try remote: if hit, download entry + outputs and populate local store."""
|
|
135
|
+
if self._remote is None:
|
|
136
|
+
return None
|
|
137
|
+
try:
|
|
138
|
+
entry_dict = self._remote.get_entry(t.fqn, key)
|
|
139
|
+
except BaseException as e:
|
|
140
|
+
_warn_once_remote_failed(e)
|
|
141
|
+
return None
|
|
142
|
+
if entry_dict is None:
|
|
143
|
+
return None
|
|
144
|
+
try:
|
|
145
|
+
entry = CacheEntry.from_dict(entry_dict)
|
|
146
|
+
except Exception as e:
|
|
147
|
+
_warn_once_remote_failed(e)
|
|
148
|
+
return None
|
|
149
|
+
# Download outputs into the content-addressed store if needed.
|
|
150
|
+
if entry.outputs_hash is not None:
|
|
151
|
+
target_dir = self.outputs._hash_dir(entry.outputs_hash)
|
|
152
|
+
if not target_dir.exists():
|
|
153
|
+
try:
|
|
154
|
+
self._remote.get_output(entry.outputs_hash, target_dir)
|
|
155
|
+
except BaseException as e:
|
|
156
|
+
_warn_once_remote_failed(e)
|
|
157
|
+
return None
|
|
158
|
+
# Mirror the entry into local store so future runs short-circuit.
|
|
159
|
+
self.store.put(t.fqn, entry)
|
|
160
|
+
return entry
|
|
161
|
+
|
|
162
|
+
def push_remote(self, t: Task, entry: CacheEntry) -> None:
|
|
163
|
+
"""Upload entry + outputs to remote. Non-fatal on error."""
|
|
164
|
+
if self._remote is None:
|
|
165
|
+
return
|
|
166
|
+
try:
|
|
167
|
+
self._remote.put_entry(t.fqn, entry.key, entry.to_dict())
|
|
168
|
+
except BaseException as e:
|
|
169
|
+
_warn_once_remote_failed(e)
|
|
170
|
+
return
|
|
171
|
+
if entry.outputs_hash is not None:
|
|
172
|
+
source = self.outputs._hash_dir(entry.outputs_hash)
|
|
173
|
+
if source.is_dir():
|
|
174
|
+
try:
|
|
175
|
+
self._remote.put_output(entry.outputs_hash, source)
|
|
176
|
+
except BaseException as e:
|
|
177
|
+
_warn_once_remote_failed(e)
|
|
178
|
+
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import ast
|
|
4
|
+
import inspect
|
|
5
|
+
import textwrap
|
|
6
|
+
from collections.abc import Callable
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from .hash import hash_bytes
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _strip_docstring(tree: ast.AST) -> None:
|
|
13
|
+
"""Remove docstring nodes from functions/classes/modules in-place."""
|
|
14
|
+
for node in ast.walk(tree):
|
|
15
|
+
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef,
|
|
16
|
+
ast.ClassDef, ast.Module)):
|
|
17
|
+
continue
|
|
18
|
+
body = node.body
|
|
19
|
+
if (body and isinstance(body[0], ast.Expr)
|
|
20
|
+
and isinstance(body[0].value, ast.Constant)
|
|
21
|
+
and isinstance(body[0].value.value, str)):
|
|
22
|
+
node.body = body[1:]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def hash_task_body_source(source: str) -> str:
|
|
26
|
+
"""Hash source code structurally, ignoring docstrings/comments/whitespace."""
|
|
27
|
+
tree = ast.parse(textwrap.dedent(source))
|
|
28
|
+
_strip_docstring(tree)
|
|
29
|
+
dumped = ast.dump(tree, annotate_fields=False, include_attributes=False)
|
|
30
|
+
return hash_bytes(dumped.encode())
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def hash_task_body(func: Callable[..., Any]) -> str:
|
|
34
|
+
"""Structural hash of a function's source. Stable under doc/whitespace/comment noise."""
|
|
35
|
+
try:
|
|
36
|
+
source = inspect.getsource(func)
|
|
37
|
+
except (OSError, TypeError):
|
|
38
|
+
return hash_bytes(f"<no-source:{func.__qualname__}>".encode())
|
|
39
|
+
return hash_task_body_source(source)
|