ntask 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. ntask-1.0.0/LICENSE +28 -0
  2. ntask-1.0.0/PKG-INFO +172 -0
  3. ntask-1.0.0/README.md +131 -0
  4. ntask-1.0.0/pyproject.toml +76 -0
  5. ntask-1.0.0/setup.cfg +4 -0
  6. ntask-1.0.0/src/ntask/__init__.py +21 -0
  7. ntask-1.0.0/src/ntask/__main__.py +6 -0
  8. ntask-1.0.0/src/ntask/_cache/__init__.py +178 -0
  9. ntask-1.0.0/src/ntask/_cache/body.py +39 -0
  10. ntask-1.0.0/src/ntask/_cache/diff.py +127 -0
  11. ntask-1.0.0/src/ntask/_cache/hash.py +43 -0
  12. ntask-1.0.0/src/ntask/_cache/key.py +68 -0
  13. ntask-1.0.0/src/ntask/_cache/manifest.py +58 -0
  14. ntask-1.0.0/src/ntask/_cache/outputs.py +86 -0
  15. ntask-1.0.0/src/ntask/_cache/store.py +116 -0
  16. ntask-1.0.0/src/ntask/_cli.py +475 -0
  17. ntask-1.0.0/src/ntask/_cli_args.py +98 -0
  18. ntask-1.0.0/src/ntask/_cli_docstring.py +55 -0
  19. ntask-1.0.0/src/ntask/_cli_format.py +76 -0
  20. ntask-1.0.0/src/ntask/_cli_why.py +122 -0
  21. ntask-1.0.0/src/ntask/_config.py +62 -0
  22. ntask-1.0.0/src/ntask/_coordinator.py +56 -0
  23. ntask-1.0.0/src/ntask/_dag.py +120 -0
  24. ntask-1.0.0/src/ntask/_depends.py +119 -0
  25. ntask-1.0.0/src/ntask/_discovery.py +52 -0
  26. ntask-1.0.0/src/ntask/_errors.py +27 -0
  27. ntask-1.0.0/src/ntask/_executor.py +221 -0
  28. ntask-1.0.0/src/ntask/_registry.py +62 -0
  29. ntask-1.0.0/src/ntask/_remote/__init__.py +48 -0
  30. ntask-1.0.0/src/ntask/_remote/_tar.py +61 -0
  31. ntask-1.0.0/src/ntask/_remote/base.py +20 -0
  32. ntask-1.0.0/src/ntask/_remote/gcs.py +64 -0
  33. ntask-1.0.0/src/ntask/_remote/http.py +89 -0
  34. ntask-1.0.0/src/ntask/_remote/local_fs.py +48 -0
  35. ntask-1.0.0/src/ntask/_remote/s3.py +101 -0
  36. ntask-1.0.0/src/ntask/_render/__init__.py +5 -0
  37. ntask-1.0.0/src/ntask/_render/base.py +14 -0
  38. ntask-1.0.0/src/ntask/_render/log.py +38 -0
  39. ntask-1.0.0/src/ntask/_render/rich.py +49 -0
  40. ntask-1.0.0/src/ntask/_render/tui.py +191 -0
  41. ntask-1.0.0/src/ntask/_shell.py +238 -0
  42. ntask-1.0.0/src/ntask/_task.py +159 -0
  43. ntask-1.0.0/src/ntask/_watch.py +147 -0
  44. ntask-1.0.0/src/ntask.egg-info/PKG-INFO +172 -0
  45. ntask-1.0.0/src/ntask.egg-info/SOURCES.txt +67 -0
  46. ntask-1.0.0/src/ntask.egg-info/dependency_links.txt +1 -0
  47. ntask-1.0.0/src/ntask.egg-info/entry_points.txt +2 -0
  48. ntask-1.0.0/src/ntask.egg-info/requires.txt +24 -0
  49. ntask-1.0.0/src/ntask.egg-info/top_level.txt +1 -0
  50. ntask-1.0.0/tests/test_cli.py +419 -0
  51. ntask-1.0.0/tests/test_cli_args.py +78 -0
  52. ntask-1.0.0/tests/test_cli_docstring.py +37 -0
  53. ntask-1.0.0/tests/test_cli_format.py +54 -0
  54. ntask-1.0.0/tests/test_cli_why.py +107 -0
  55. ntask-1.0.0/tests/test_coordinator.py +105 -0
  56. ntask-1.0.0/tests/test_dag.py +71 -0
  57. ntask-1.0.0/tests/test_depends.py +73 -0
  58. ntask-1.0.0/tests/test_discovery.py +60 -0
  59. ntask-1.0.0/tests/test_executor.py +533 -0
  60. ntask-1.0.0/tests/test_group.py +46 -0
  61. ntask-1.0.0/tests/test_integration.py +112 -0
  62. ntask-1.0.0/tests/test_property.py +62 -0
  63. ntask-1.0.0/tests/test_registry.py +23 -0
  64. ntask-1.0.0/tests/test_render.py +77 -0
  65. ntask-1.0.0/tests/test_render_tui.py +196 -0
  66. ntask-1.0.0/tests/test_shell.py +170 -0
  67. ntask-1.0.0/tests/test_task.py +107 -0
  68. ntask-1.0.0/tests/test_watch.py +180 -0
  69. ntask-1.0.0/tests/test_watch_integration.py +75 -0
ntask-1.0.0/LICENSE ADDED
@@ -0,0 +1,28 @@
1
+ BSD 3-Clause License
2
+
3
+ Copyright (c) 2026, Sean Nieuwoudt
4
+
5
+ Redistribution and use in source and binary forms, with or without
6
+ modification, are permitted provided that the following conditions are met:
7
+
8
+ 1. Redistributions of source code must retain the above copyright notice, this
9
+ list of conditions and the following disclaimer.
10
+
11
+ 2. Redistributions in binary form must reproduce the above copyright notice,
12
+ this list of conditions and the following disclaimer in the documentation
13
+ and/or other materials provided with the distribution.
14
+
15
+ 3. Neither the name of the copyright holder nor the names of its
16
+ contributors may be used to endorse or promote products derived from
17
+ this software without specific prior written permission.
18
+
19
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ntask-1.0.0/PKG-INFO ADDED
@@ -0,0 +1,172 @@
1
+ Metadata-Version: 2.4
2
+ Name: ntask
3
+ Version: 1.0.0
4
+ Summary: A Python-native task runner with content-hash caching and DAG execution.
5
+ Author-email: Sean Nieuwoudt <sean@underwulf.com>
6
+ License: BSD-3-Clause
7
+ Keywords: task-runner,build-system,make,just,ci
8
+ Classifier: Development Status :: 4 - Beta
9
+ Classifier: Intended Audience :: Developers
10
+ Classifier: License :: OSI Approved :: BSD License
11
+ Classifier: Operating System :: OS Independent
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Topic :: Software Development :: Build Tools
17
+ Requires-Python: >=3.11
18
+ Description-Content-Type: text/markdown
19
+ License-File: LICENSE
20
+ Requires-Dist: rich>=13.7
21
+ Requires-Dist: xxhash>=3.4
22
+ Requires-Dist: pathspec>=0.12
23
+ Requires-Dist: anyio>=4.3
24
+ Requires-Dist: watchfiles>=0.22
25
+ Requires-Dist: textual>=0.80
26
+ Provides-Extra: dev
27
+ Requires-Dist: pytest>=8.0; extra == "dev"
28
+ Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
29
+ Requires-Dist: hypothesis>=6.98; extra == "dev"
30
+ Requires-Dist: ruff>=0.3; extra == "dev"
31
+ Requires-Dist: mypy>=1.9; extra == "dev"
32
+ Requires-Dist: moto>=5.0; extra == "dev"
33
+ Provides-Extra: s3
34
+ Requires-Dist: boto3>=1.34; extra == "s3"
35
+ Provides-Extra: gcs
36
+ Requires-Dist: google-cloud-storage>=2.14; extra == "gcs"
37
+ Provides-Extra: all
38
+ Requires-Dist: boto3>=1.34; extra == "all"
39
+ Requires-Dist: google-cloud-storage>=2.14; extra == "all"
40
+ Dynamic: license-file
41
+
42
+ # ntask
43
+
44
+ A Python-native task runner with content-hash caching and DAG execution.
45
+
46
+ ```bash
47
+ pip install ntask
48
+ ```
49
+
50
+ ## Why
51
+
52
+ Your Makefile runs everything every time. Your Justfile has no dependency graph. Your `tasks.py` for Invoke is five years old, has no types, and you still have to write `ctx.run`. This is what a task runner looks like when you start over with caching, types, and a DAG.
53
+
54
+ ## Quickstart
55
+
56
+ Create a `tasks.py`:
57
+
58
+ ```python
59
+ from ntask import task, cached, depends, shell
60
+
61
+ @task
62
+ def install():
63
+ shell("pip install -e .")
64
+
65
+ @task
66
+ @cached(inputs=["src/**/*.py", "tests/**/*.py"])
67
+ def test(pattern: str = "", verbose: bool = False):
68
+ """Run the test suite."""
69
+ flags = "-v" if verbose else ""
70
+ k = f"-k {pattern}" if pattern else ""
71
+ shell(f"pytest {flags} {k}")
72
+
73
+ @task
74
+ @cached(inputs=["src/**/*.py"])
75
+ def lint():
76
+ shell("ruff check src/")
77
+
78
+ @task
79
+ def check():
80
+ """All quality checks."""
81
+ depends(lint, test)
82
+ ```
83
+
84
+ Run:
85
+
86
+ ```bash
87
+ ntask --list # show every registered task
88
+ ntask test --pattern=auth --verbose
89
+ ntask check # lint + test in order; both cache
90
+ ntask check -j # all CPU cores; bare -j picks the count
91
+ ntask watch test # rerun on every src/ or tests/ change
92
+ ntask --why test # explain why the last cache lookup missed
93
+ ntask --graph check # ASCII DAG (mermaid / dot also available)
94
+ ```
95
+
96
+ The second time you run `ntask check`, both `lint` and `test` are content-hash cache hits and finish in milliseconds. Change one file in `src/` and only the affected tasks rerun, transitively.
97
+
98
+ ## Team cache
99
+
100
+ Share cache hits across machines via S3 (or GCS, HTTP, NFS):
101
+
102
+ ```toml
103
+ # pyproject.toml
104
+ [tool.ntask.remote_cache]
105
+ type = "s3"
106
+ bucket = "my-team-cache"
107
+ ```
108
+
109
+ ```bash
110
+ pip install ntask[s3]
111
+ ntask check # first person populates, the rest hit instantly
112
+ ntask check --offline # skip the remote for fast dev loops
113
+ ```
114
+
115
+ S3-compatibles (MinIO, R2, B2) take an `endpoint_url`. GCS and HTTP backends work the same way; the HTTP backend is plain `GET`/`PUT`/`HEAD` over stdlib `urllib`, so any object store with PUT enabled is fair game.
116
+
117
+ ## Live DAG display
118
+
119
+ Run `ntask check` from an interactive terminal and a Textual TUI shows a live tree of the DAG with per-task state icons and durations. Pipe the output, set `tui = false` in `[tool.ntask]`, or pass `--no-tui` to fall back to the line-based renderer.
120
+
121
+ ## Other things you'll reach for
122
+
123
+ - **Exclusive tasks.** `@task(parallel=False)` makes a task a DAG-wide barrier: it waits for everything in flight to drain, then runs alone. Use it for releases, migrations, anything that mutates shared state.
124
+ - **Monorepos.** `@group("api")` over a class namespaces every task method as `api.<name>`. Cross-group dependencies via `@task(deps=[Other.task, ...])` or string fqns.
125
+ - **Capture output.** `shell("git rev-parse HEAD", capture=True)` returns a `ShellResult` with `.stdout`, `.stderr`, `.returncode`, `.duration`, `.ok`.
126
+ - **Force a rerun.** `ntask --force <task>` bypasses the cache for that one task. `ntask --no-cache` ignores the cache entirely. `ntask clean` wipes entry manifests; `ntask clean --all` wipes the whole `.ntask/` directory.
127
+
128
+ ## Examples
129
+
130
+ Six runnable, self-contained examples under [`examples/`](examples/):
131
+
132
+ | # | Demonstrates |
133
+ |----------------------------------------------|------------------------------------------------|
134
+ | [01-hello](examples/01-hello/) | Smallest possible cached task |
135
+ | [02-python-lib](examples/02-python-lib/) | install / lint / typecheck / test / build |
136
+ | [03-parallel](examples/03-parallel/) | `-j N` fan-out and `parallel=False` barrier |
137
+ | [04-watch](examples/04-watch/) | `ntask watch` rerun-on-change loop |
138
+ | [05-remote-cache](examples/05-remote-cache/) | local-fs remote backend shared between clones |
139
+ | [06-monorepo](examples/06-monorepo/) | `@group(...)` namespacing and cross-group deps |
140
+
141
+ `cd` into any directory and run `ntask --list`.
142
+
143
+ ## Features
144
+
145
+ | feature | ntask | make | just | invoke | doit | poe |
146
+ |----------------------------------|:-----:|:-------:|:-------:|:-------:|:-------:|:-------:|
147
+ | Typed Python tasks | yes | no | no | partial | no | partial |
148
+ | Content-hash input caching | yes | partial | no | no | partial | no |
149
+ | Transitive cache-key propagation | yes | no | no | no | partial | no |
150
+ | Remote cache (S3/GCS/HTTP) | yes | no | no | no | no | no |
151
+ | DAG dependency resolution | yes | yes | no | partial | yes | no |
152
+ | Type-hint to CLI args | yes | no | partial | partial | no | yes |
153
+ | Parallel DAG execution (`-j`) | yes | yes | no | no | yes | no |
154
+ | Live DAG TUI | yes | no | no | no | no | no |
155
+ | Windows first-class | yes | partial | yes | yes | yes | yes |
156
+
157
+ Cache miss messages name the specific file or env change that caused the invalidation. `ntask --why <task>` prints the full breakdown.
158
+
159
+ ## Docs
160
+
161
+ - [Technical handbook](docs/guide.md) - the comprehensive reference, every flag and config key
162
+ - [Tutorial: replace your Makefile in 10 minutes](docs/tutorial.md)
163
+ - [Caching: the full contract](docs/caching.md)
164
+ - [Migrating from Make / just / Invoke / Poe](docs/migration.md)
165
+ - [Short API reference](docs/reference.md)
166
+ - [Roadmap](docs/roadmap.md)
167
+
168
+ Requirements: Python 3.11 or newer. BSD-3-Clause.
169
+
170
+ ## License
171
+
172
+ BSD-3-Clause. Copyright © 2026 Sean Nieuwoudt.
ntask-1.0.0/README.md ADDED
@@ -0,0 +1,131 @@
1
+ # ntask
2
+
3
+ A Python-native task runner with content-hash caching and DAG execution.
4
+
5
+ ```bash
6
+ pip install ntask
7
+ ```
8
+
9
+ ## Why
10
+
11
+ Your Makefile runs everything every time. Your Justfile has no dependency graph. Your `tasks.py` for Invoke is five years old, has no types, and you still have to write `ctx.run`. This is what a task runner looks like when you start over with caching, types, and a DAG.
12
+
13
+ ## Quickstart
14
+
15
+ Create a `tasks.py`:
16
+
17
+ ```python
18
+ from ntask import task, cached, depends, shell
19
+
20
+ @task
21
+ def install():
22
+ shell("pip install -e .")
23
+
24
+ @task
25
+ @cached(inputs=["src/**/*.py", "tests/**/*.py"])
26
+ def test(pattern: str = "", verbose: bool = False):
27
+ """Run the test suite."""
28
+ flags = "-v" if verbose else ""
29
+ k = f"-k {pattern}" if pattern else ""
30
+ shell(f"pytest {flags} {k}")
31
+
32
+ @task
33
+ @cached(inputs=["src/**/*.py"])
34
+ def lint():
35
+ shell("ruff check src/")
36
+
37
+ @task
38
+ def check():
39
+ """All quality checks."""
40
+ depends(lint, test)
41
+ ```
42
+
43
+ Run:
44
+
45
+ ```bash
46
+ ntask --list # show every registered task
47
+ ntask test --pattern=auth --verbose
48
+ ntask check # lint + test in order; both cache
49
+ ntask check -j # all CPU cores; bare -j picks the count
50
+ ntask watch test # rerun on every src/ or tests/ change
51
+ ntask --why test # explain why the last cache lookup missed
52
+ ntask --graph check # ASCII DAG (mermaid / dot also available)
53
+ ```
54
+
55
+ The second time you run `ntask check`, both `lint` and `test` are content-hash cache hits and finish in milliseconds. Change one file in `src/` and only the affected tasks rerun, transitively.
56
+
57
+ ## Team cache
58
+
59
+ Share cache hits across machines via S3 (or GCS, HTTP, NFS):
60
+
61
+ ```toml
62
+ # pyproject.toml
63
+ [tool.ntask.remote_cache]
64
+ type = "s3"
65
+ bucket = "my-team-cache"
66
+ ```
67
+
68
+ ```bash
69
+ pip install ntask[s3]
70
+ ntask check # first person populates, the rest hit instantly
71
+ ntask check --offline # skip the remote for fast dev loops
72
+ ```
73
+
74
+ S3-compatibles (MinIO, R2, B2) take an `endpoint_url`. GCS and HTTP backends work the same way; the HTTP backend is plain `GET`/`PUT`/`HEAD` over stdlib `urllib`, so any object store with PUT enabled is fair game.
75
+
76
+ ## Live DAG display
77
+
78
+ Run `ntask check` from an interactive terminal and a Textual TUI shows a live tree of the DAG with per-task state icons and durations. Pipe the output, set `tui = false` in `[tool.ntask]`, or pass `--no-tui` to fall back to the line-based renderer.
79
+
80
+ ## Other things you'll reach for
81
+
82
+ - **Exclusive tasks.** `@task(parallel=False)` makes a task a DAG-wide barrier: it waits for everything in flight to drain, then runs alone. Use it for releases, migrations, anything that mutates shared state.
83
+ - **Monorepos.** `@group("api")` over a class namespaces every task method as `api.<name>`. Cross-group dependencies via `@task(deps=[Other.task, ...])` or string fqns.
84
+ - **Capture output.** `shell("git rev-parse HEAD", capture=True)` returns a `ShellResult` with `.stdout`, `.stderr`, `.returncode`, `.duration`, `.ok`.
85
+ - **Force a rerun.** `ntask --force <task>` bypasses the cache for that one task. `ntask --no-cache` ignores the cache entirely. `ntask clean` wipes entry manifests; `ntask clean --all` wipes the whole `.ntask/` directory.
86
+
87
+ ## Examples
88
+
89
+ Six runnable, self-contained examples under [`examples/`](examples/):
90
+
91
+ | # | Demonstrates |
92
+ |----------------------------------------------|------------------------------------------------|
93
+ | [01-hello](examples/01-hello/) | Smallest possible cached task |
94
+ | [02-python-lib](examples/02-python-lib/) | install / lint / typecheck / test / build |
95
+ | [03-parallel](examples/03-parallel/) | `-j N` fan-out and `parallel=False` barrier |
96
+ | [04-watch](examples/04-watch/) | `ntask watch` rerun-on-change loop |
97
+ | [05-remote-cache](examples/05-remote-cache/) | local-fs remote backend shared between clones |
98
+ | [06-monorepo](examples/06-monorepo/) | `@group(...)` namespacing and cross-group deps |
99
+
100
+ `cd` into any directory and run `ntask --list`.
101
+
102
+ ## Features
103
+
104
+ | feature | ntask | make | just | invoke | doit | poe |
105
+ |----------------------------------|:-----:|:-------:|:-------:|:-------:|:-------:|:-------:|
106
+ | Typed Python tasks | yes | no | no | partial | no | partial |
107
+ | Content-hash input caching | yes | partial | no | no | partial | no |
108
+ | Transitive cache-key propagation | yes | no | no | no | partial | no |
109
+ | Remote cache (S3/GCS/HTTP) | yes | no | no | no | no | no |
110
+ | DAG dependency resolution | yes | yes | no | partial | yes | no |
111
+ | Type-hint to CLI args | yes | no | partial | partial | no | yes |
112
+ | Parallel DAG execution (`-j`) | yes | yes | no | no | yes | no |
113
+ | Live DAG TUI | yes | no | no | no | no | no |
114
+ | Windows first-class | yes | partial | yes | yes | yes | yes |
115
+
116
+ Cache miss messages name the specific file or env change that caused the invalidation. `ntask --why <task>` prints the full breakdown.
117
+
118
+ ## Docs
119
+
120
+ - [Technical handbook](docs/guide.md) - the comprehensive reference, every flag and config key
121
+ - [Tutorial: replace your Makefile in 10 minutes](docs/tutorial.md)
122
+ - [Caching: the full contract](docs/caching.md)
123
+ - [Migrating from Make / just / Invoke / Poe](docs/migration.md)
124
+ - [Short API reference](docs/reference.md)
125
+ - [Roadmap](docs/roadmap.md)
126
+
127
+ Requirements: Python 3.11 or newer. BSD-3-Clause.
128
+
129
+ ## License
130
+
131
+ BSD-3-Clause. Copyright © 2026 Sean Nieuwoudt.
@@ -0,0 +1,76 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "ntask"
7
+ version = "1.0.0"
8
+ description = "A Python-native task runner with content-hash caching and DAG execution."
9
+ authors = [{name = "Sean Nieuwoudt", email = "sean@underwulf.com"}]
10
+ license = {text = "BSD-3-Clause"}
11
+ readme = "README.md"
12
+ requires-python = ">=3.11"
13
+ keywords = ["task-runner", "build-system", "make", "just", "ci"]
14
+ classifiers = [
15
+ "Development Status :: 4 - Beta",
16
+ "Intended Audience :: Developers",
17
+ "License :: OSI Approved :: BSD License",
18
+ "Operating System :: OS Independent",
19
+ "Programming Language :: Python :: 3",
20
+ "Programming Language :: Python :: 3.11",
21
+ "Programming Language :: Python :: 3.12",
22
+ "Programming Language :: Python :: 3.13",
23
+ "Topic :: Software Development :: Build Tools",
24
+ ]
25
+ dependencies = [
26
+ "rich>=13.7",
27
+ "xxhash>=3.4",
28
+ "pathspec>=0.12",
29
+ "anyio>=4.3",
30
+ "watchfiles>=0.22",
31
+ "textual>=0.80",
32
+ ]
33
+
34
+ [project.optional-dependencies]
35
+ dev = [
36
+ "pytest>=8.0",
37
+ "pytest-asyncio>=0.23",
38
+ "hypothesis>=6.98",
39
+ "ruff>=0.3",
40
+ "mypy>=1.9",
41
+ "moto>=5.0",
42
+ ]
43
+ s3 = ["boto3>=1.34"]
44
+ gcs = ["google-cloud-storage>=2.14"]
45
+ all = ["boto3>=1.34", "google-cloud-storage>=2.14"]
46
+
47
+ [project.scripts]
48
+ ntask = "ntask._cli:main"
49
+
50
+ [tool.setuptools]
51
+ package-dir = {"" = "src"}
52
+
53
+ [tool.setuptools.packages.find]
54
+ where = ["src"]
55
+
56
+ [tool.pytest.ini_options]
57
+ testpaths = ["tests"]
58
+ pythonpath = ["src"]
59
+ asyncio_mode = "auto"
60
+
61
+ [tool.ruff]
62
+ line-length = 100
63
+ target-version = "py311"
64
+
65
+ [tool.ruff.lint]
66
+ select = ["E", "F", "W", "I", "N", "UP", "B", "S", "A", "C4", "RUF"]
67
+ ignore = ["S101"]
68
+
69
+ [tool.mypy]
70
+ python_version = "3.11"
71
+ strict = true
72
+ files = ["src/ntask"]
73
+
74
+ [[tool.mypy.overrides]]
75
+ module = ["boto3", "botocore.*", "google.*"]
76
+ ignore_missing_imports = true
ntask-1.0.0/setup.cfg ADDED
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,21 @@
1
+ """ntask: a Python-native task runner with content-hash caching and DAG execution."""
2
+
3
+ from ._depends import depends
4
+ from ._errors import CycleError, DiscoveryError, NtaskError, ShellError
5
+ from ._shell import ShellResult, shell
6
+ from ._task import cached, group, task
7
+
8
+ __version__ = "1.0.0"
9
+ __all__ = [
10
+ "CycleError",
11
+ "DiscoveryError",
12
+ "NtaskError",
13
+ "ShellError",
14
+ "ShellResult",
15
+ "__version__",
16
+ "cached",
17
+ "depends",
18
+ "group",
19
+ "shell",
20
+ "task",
21
+ ]
@@ -0,0 +1,6 @@
1
+ import sys
2
+
3
+ from ._cli import main
4
+
5
+ if __name__ == "__main__":
6
+ sys.exit(main())
@@ -0,0 +1,178 @@
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ import platform
5
+ import sys
6
+ import time
7
+ from pathlib import Path
8
+
9
+ from .._remote.base import RemoteBackend
10
+ from .._task import Task
11
+ from .body import hash_task_body
12
+ from .key import CacheBreakdown, CacheKeyInputs, InputRecord, compute_cache_key
13
+ from .manifest import compute_input_manifest
14
+ from .outputs import OutputStore
15
+ from .store import CacheEntry, CacheStore
16
+
17
+ # Module-level flag for warn-once pattern.
18
+ _remote_warn_fired = False
19
+
20
+
21
+ def _warn_once_remote_failed(e: BaseException) -> None:
22
+ """Print a single warning the first time a remote call fails in this process."""
23
+ global _remote_warn_fired
24
+ if not _remote_warn_fired:
25
+ print(
26
+ f"warning: remote cache unreachable: {type(e).__name__}: {e}. "
27
+ f"Falling back to local.",
28
+ file=sys.stderr,
29
+ )
30
+ _remote_warn_fired = True
31
+
32
+
33
+ def _python_version_str() -> str:
34
+ return ".".join(str(x) for x in sys.version_info[:3])
35
+
36
+
37
+ def _platform_tag_str() -> str:
38
+ return f"{platform.system()}-{platform.machine()}".lower()
39
+
40
+
41
+ class CacheEngine:
42
+ def __init__(self, root: Path, remote: RemoteBackend | None = None):
43
+ self.root = root
44
+ self.store = CacheStore(root=root)
45
+ self.outputs = OutputStore(root=root)
46
+ self._remote = remote
47
+
48
+ def compute_key_and_breakdown(
49
+ self,
50
+ t: Task,
51
+ *,
52
+ workspace: Path,
53
+ upstream_keys_by_dep: dict[str, str],
54
+ ) -> tuple[str, CacheBreakdown]:
55
+ cfg = t.cached_config
56
+ assert cfg is not None
57
+ manifest = compute_input_manifest(cfg.inputs, root=workspace)
58
+ body = hash_task_body(t.func)
59
+ env_values = {
60
+ name: os.environ[name] if name in os.environ else "<unset>"
61
+ for name in cfg.env
62
+ }
63
+ py_version = _python_version_str()
64
+ plat_tag = _platform_tag_str()
65
+
66
+ # Tuple ordering for cache key matches insertion order of upstream_keys_by_dep.
67
+ upstream_tuple = tuple(upstream_keys_by_dep.values())
68
+ key_inputs = CacheKeyInputs(
69
+ task_fqn=t.fqn,
70
+ task_body_hash=body,
71
+ env={name: ("" if v == "<unset>" else v) for name, v in env_values.items()},
72
+ env_names=cfg.env,
73
+ input_patterns=cfg.inputs,
74
+ input_manifest_digest=manifest.digest,
75
+ root=workspace,
76
+ upstream_keys=upstream_tuple if cfg.propagate else (),
77
+ strict=cfg.strict,
78
+ )
79
+ key = compute_cache_key(key_inputs)
80
+
81
+ input_records = tuple(
82
+ InputRecord(
83
+ path=fh.path.relative_to(workspace).as_posix(),
84
+ digest=fh.digest,
85
+ mode=fh.mode,
86
+ )
87
+ for fh in manifest.files
88
+ )
89
+ breakdown = CacheBreakdown(
90
+ input_patterns=cfg.inputs,
91
+ inputs=input_records,
92
+ env_values=env_values,
93
+ task_body_hash=body,
94
+ python_version=py_version,
95
+ platform_tag=plat_tag,
96
+ upstream_keys_by_dep=dict(upstream_keys_by_dep),
97
+ )
98
+ return key, breakdown
99
+
100
+ def check(self, t: Task, key: str) -> CacheEntry | None:
101
+ return self.store.get(t.fqn, key)
102
+
103
+ def store_entry(
104
+ self,
105
+ t: Task,
106
+ key: str,
107
+ *,
108
+ workspace: Path,
109
+ duration: float,
110
+ breakdown: CacheBreakdown,
111
+ upstream_keys: tuple[str, ...],
112
+ ) -> CacheEntry:
113
+ cfg = t.cached_config
114
+ assert cfg is not None
115
+ outputs_hash = (
116
+ self.outputs.capture(cfg.outputs, root=workspace) if cfg.outputs else None
117
+ )
118
+ entry = CacheEntry(
119
+ key=key,
120
+ outputs_hash=outputs_hash,
121
+ duration=duration,
122
+ completed_at=time.time(),
123
+ upstream_keys=upstream_keys,
124
+ breakdown=breakdown,
125
+ )
126
+ self.store.put(t.fqn, entry)
127
+ return entry
128
+
129
+ def restore_outputs(self, entry: CacheEntry, *, workspace: Path) -> None:
130
+ if entry.outputs_hash:
131
+ self.outputs.restore(entry.outputs_hash, root=workspace)
132
+
133
+ def check_remote(self, t: Task, key: str) -> CacheEntry | None:
134
+ """Try remote: if hit, download entry + outputs and populate local store."""
135
+ if self._remote is None:
136
+ return None
137
+ try:
138
+ entry_dict = self._remote.get_entry(t.fqn, key)
139
+ except BaseException as e:
140
+ _warn_once_remote_failed(e)
141
+ return None
142
+ if entry_dict is None:
143
+ return None
144
+ try:
145
+ entry = CacheEntry.from_dict(entry_dict)
146
+ except Exception as e:
147
+ _warn_once_remote_failed(e)
148
+ return None
149
+ # Download outputs into the content-addressed store if needed.
150
+ if entry.outputs_hash is not None:
151
+ target_dir = self.outputs._hash_dir(entry.outputs_hash)
152
+ if not target_dir.exists():
153
+ try:
154
+ self._remote.get_output(entry.outputs_hash, target_dir)
155
+ except BaseException as e:
156
+ _warn_once_remote_failed(e)
157
+ return None
158
+ # Mirror the entry into local store so future runs short-circuit.
159
+ self.store.put(t.fqn, entry)
160
+ return entry
161
+
162
+ def push_remote(self, t: Task, entry: CacheEntry) -> None:
163
+ """Upload entry + outputs to remote. Non-fatal on error."""
164
+ if self._remote is None:
165
+ return
166
+ try:
167
+ self._remote.put_entry(t.fqn, entry.key, entry.to_dict())
168
+ except BaseException as e:
169
+ _warn_once_remote_failed(e)
170
+ return
171
+ if entry.outputs_hash is not None:
172
+ source = self.outputs._hash_dir(entry.outputs_hash)
173
+ if source.is_dir():
174
+ try:
175
+ self._remote.put_output(entry.outputs_hash, source)
176
+ except BaseException as e:
177
+ _warn_once_remote_failed(e)
178
+
@@ -0,0 +1,39 @@
1
+ from __future__ import annotations
2
+
3
+ import ast
4
+ import inspect
5
+ import textwrap
6
+ from collections.abc import Callable
7
+ from typing import Any
8
+
9
+ from .hash import hash_bytes
10
+
11
+
12
+ def _strip_docstring(tree: ast.AST) -> None:
13
+ """Remove docstring nodes from functions/classes/modules in-place."""
14
+ for node in ast.walk(tree):
15
+ if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef,
16
+ ast.ClassDef, ast.Module)):
17
+ continue
18
+ body = node.body
19
+ if (body and isinstance(body[0], ast.Expr)
20
+ and isinstance(body[0].value, ast.Constant)
21
+ and isinstance(body[0].value.value, str)):
22
+ node.body = body[1:]
23
+
24
+
25
+ def hash_task_body_source(source: str) -> str:
26
+ """Hash source code structurally, ignoring docstrings/comments/whitespace."""
27
+ tree = ast.parse(textwrap.dedent(source))
28
+ _strip_docstring(tree)
29
+ dumped = ast.dump(tree, annotate_fields=False, include_attributes=False)
30
+ return hash_bytes(dumped.encode())
31
+
32
+
33
+ def hash_task_body(func: Callable[..., Any]) -> str:
34
+ """Structural hash of a function's source. Stable under doc/whitespace/comment noise."""
35
+ try:
36
+ source = inspect.getsource(func)
37
+ except (OSError, TypeError):
38
+ return hash_bytes(f"<no-source:{func.__qualname__}>".encode())
39
+ return hash_task_body_source(source)