ntask 1.0.0__py3-none-any.whl

ntask/__init__.py

@@ -0,0 +1,21 @@
+"""ntask: a Python-native task runner with content-hash caching and DAG execution."""
+
+from ._depends import depends
+from ._errors import CycleError, DiscoveryError, NtaskError, ShellError
+from ._shell import ShellResult, shell
+from ._task import cached, group, task
+
+__version__ = "1.0.0"
+__all__ = [
+    "CycleError",
+    "DiscoveryError",
+    "NtaskError",
+    "ShellError",
+    "ShellResult",
+    "__version__",
+    "cached",
+    "depends",
+    "group",
+    "shell",
+    "task",
+]

ntask/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from ._cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

ntask/_cache/__init__.py

@@ -0,0 +1,178 @@
+from __future__ import annotations
+
+import os
+import platform
+import sys
+import time
+from pathlib import Path
+
+from .._remote.base import RemoteBackend
+from .._task import Task
+from .body import hash_task_body
+from .key import CacheBreakdown, CacheKeyInputs, InputRecord, compute_cache_key
+from .manifest import compute_input_manifest
+from .outputs import OutputStore
+from .store import CacheEntry, CacheStore
+
+# Module-level flag for warn-once pattern.
+_remote_warn_fired = False
+
+
+def _warn_once_remote_failed(e: BaseException) -> None:
+    """Print a single warning the first time a remote call fails in this process."""
+    global _remote_warn_fired
+    if not _remote_warn_fired:
+        print(
+            f"warning: remote cache unreachable: {type(e).__name__}: {e}. "
+            f"Falling back to local.",
+            file=sys.stderr,
+        )
+        _remote_warn_fired = True
+
+
+def _python_version_str() -> str:
+    return ".".join(str(x) for x in sys.version_info[:3])
+
+
+def _platform_tag_str() -> str:
+    return f"{platform.system()}-{platform.machine()}".lower()
+
+
+class CacheEngine:
+    def __init__(self, root: Path, remote: RemoteBackend | None = None):
+        self.root = root
+        self.store = CacheStore(root=root)
+        self.outputs = OutputStore(root=root)
+        self._remote = remote
+
+    def compute_key_and_breakdown(
+        self,
+        t: Task,
+        *,
+        workspace: Path,
+        upstream_keys_by_dep: dict[str, str],
+    ) -> tuple[str, CacheBreakdown]:
+        cfg = t.cached_config
+        assert cfg is not None
+        manifest = compute_input_manifest(cfg.inputs, root=workspace)
+        body = hash_task_body(t.func)
+        env_values = {
+            name: os.environ[name] if name in os.environ else "<unset>"
+            for name in cfg.env
+        }
+        py_version = _python_version_str()
+        plat_tag = _platform_tag_str()
+
+        # Tuple ordering for cache key matches insertion order of upstream_keys_by_dep.
+        upstream_tuple = tuple(upstream_keys_by_dep.values())
+        key_inputs = CacheKeyInputs(
+            task_fqn=t.fqn,
+            task_body_hash=body,
+            env={name: ("" if v == "<unset>" else v) for name, v in env_values.items()},
+            env_names=cfg.env,
+            input_patterns=cfg.inputs,
+            input_manifest_digest=manifest.digest,
+            root=workspace,
+            upstream_keys=upstream_tuple if cfg.propagate else (),
+            strict=cfg.strict,
+        )
+        key = compute_cache_key(key_inputs)
+
+        input_records = tuple(
+            InputRecord(
+                path=fh.path.relative_to(workspace).as_posix(),
+                digest=fh.digest,
+                mode=fh.mode,
+            )
+            for fh in manifest.files
+        )
+        breakdown = CacheBreakdown(
+            input_patterns=cfg.inputs,
+            inputs=input_records,
+            env_values=env_values,
+            task_body_hash=body,
+            python_version=py_version,
+            platform_tag=plat_tag,
+            upstream_keys_by_dep=dict(upstream_keys_by_dep),
+        )
+        return key, breakdown
+
+    def check(self, t: Task, key: str) -> CacheEntry | None:
+        return self.store.get(t.fqn, key)
+
+    def store_entry(
+        self,
+        t: Task,
+        key: str,
+        *,
+        workspace: Path,
+        duration: float,
+        breakdown: CacheBreakdown,
+        upstream_keys: tuple[str, ...],
+    ) -> CacheEntry:
+        cfg = t.cached_config
+        assert cfg is not None
+        outputs_hash = (
+            self.outputs.capture(cfg.outputs, root=workspace) if cfg.outputs else None
+        )
+        entry = CacheEntry(
+            key=key,
+            outputs_hash=outputs_hash,
+            duration=duration,
+            completed_at=time.time(),
+            upstream_keys=upstream_keys,
+            breakdown=breakdown,
+        )
+        self.store.put(t.fqn, entry)
+        return entry
+
+    def restore_outputs(self, entry: CacheEntry, *, workspace: Path) -> None:
+        if entry.outputs_hash:
+            self.outputs.restore(entry.outputs_hash, root=workspace)
+
+    def check_remote(self, t: Task, key: str) -> CacheEntry | None:
+        """Try remote: if hit, download entry + outputs and populate local store."""
+        if self._remote is None:
+            return None
+        try:
+            entry_dict = self._remote.get_entry(t.fqn, key)
+        except BaseException as e:
+            _warn_once_remote_failed(e)
+            return None
+        if entry_dict is None:
+            return None
+        try:
+            entry = CacheEntry.from_dict(entry_dict)
+        except Exception as e:
+            _warn_once_remote_failed(e)
+            return None
+        # Download outputs into the content-addressed store if needed.
+        if entry.outputs_hash is not None:
+            target_dir = self.outputs._hash_dir(entry.outputs_hash)
+            if not target_dir.exists():
+                try:
+                    self._remote.get_output(entry.outputs_hash, target_dir)
+                except BaseException as e:
+                    _warn_once_remote_failed(e)
+                    return None
+        # Mirror the entry into local store so future runs short-circuit.
+        self.store.put(t.fqn, entry)
+        return entry
+
+    def push_remote(self, t: Task, entry: CacheEntry) -> None:
+        """Upload entry + outputs to remote. Non-fatal on error."""
+        if self._remote is None:
+            return
+        try:
+            self._remote.put_entry(t.fqn, entry.key, entry.to_dict())
+        except BaseException as e:
+            _warn_once_remote_failed(e)
+            return
+        if entry.outputs_hash is not None:
+            source = self.outputs._hash_dir(entry.outputs_hash)
+            if source.is_dir():
+                try:
+                    self._remote.put_output(entry.outputs_hash, source)
+                except BaseException as e:
+                    _warn_once_remote_failed(e)
+

ntask/_cache/body.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import ast
+import inspect
+import textwrap
+from collections.abc import Callable
+from typing import Any
+
+from .hash import hash_bytes
+
+
+def _strip_docstring(tree: ast.AST) -> None:
+    """Remove docstring nodes from functions/classes/modules in-place."""
+    for node in ast.walk(tree):
+        if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef,
+                                 ast.ClassDef, ast.Module)):
+            continue
+        body = node.body
+        if (body and isinstance(body[0], ast.Expr)
+                and isinstance(body[0].value, ast.Constant)
+                and isinstance(body[0].value.value, str)):
+            node.body = body[1:]
+
+
+def hash_task_body_source(source: str) -> str:
+    """Hash source code structurally, ignoring docstrings/comments/whitespace."""
+    tree = ast.parse(textwrap.dedent(source))
+    _strip_docstring(tree)
+    dumped = ast.dump(tree, annotate_fields=False, include_attributes=False)
+    return hash_bytes(dumped.encode())
+
+
+def hash_task_body(func: Callable[..., Any]) -> str:
+    """Structural hash of a function's source. Stable under doc/whitespace/comment noise."""
+    try:
+        source = inspect.getsource(func)
+    except (OSError, TypeError):
+        return hash_bytes(f"<no-source:{func.__qualname__}>".encode())
+    return hash_task_body_source(source)

ntask/_cache/diff.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+from .key import CacheBreakdown, InputRecord
+
+MissKind = Literal[
+    "first-run",
+    "input-modified", "input-added", "input-removed",
+    "env-changed", "env-added", "env-removed",
+    "body-changed",
+    "upstream-invalidated",
+    "python-changed", "platform-changed",
+]
+
+
+@dataclass(frozen=True, slots=True)
+class MissItem:
+    kind: MissKind
+    detail: str
+
+
+@dataclass(frozen=True, slots=True)
+class MissReport:
+    items: tuple[MissItem, ...]
+
+    @property
+    def is_hit(self) -> bool:
+        return not self.items
+
+    def summary(self) -> str:
+        if not self.items:
+            return "no changes"
+        first = self.items[0]
+        rest = len(self.items) - 1
+        label = f"{first.kind}: {first.detail}" if first.detail else first.kind
+        return f"{label}" + (f" (+{rest} more)" if rest else "")
+
+
+def _diff_inputs(
+    current: tuple[InputRecord, ...],
+    prior: tuple[InputRecord, ...],
+) -> list[MissItem]:
+    prior_by_path = {r.path: r for r in prior}
+    current_by_path = {r.path: r for r in current}
+    items: list[MissItem] = []
+
+    all_paths = sorted(set(prior_by_path) | set(current_by_path))
+    for path in all_paths:
+        p = prior_by_path.get(path)
+        c = current_by_path.get(path)
+        if p is None and c is not None:
+            items.append(MissItem(kind="input-added", detail=path))
+        elif c is None and p is not None:
+            items.append(MissItem(kind="input-removed", detail=path))
+        elif p is not None and c is not None:
+            if p.digest != c.digest or p.mode != c.mode:
+                items.append(MissItem(kind="input-modified", detail=path))
+    return items
+
+
+def _diff_env(
+    current: dict[str, str],
+    prior: dict[str, str],
+) -> list[MissItem]:
+    items: list[MissItem] = []
+    all_keys = sorted(set(prior) | set(current))
+    for name in all_keys:
+        if name in current and name not in prior:
+            items.append(MissItem(kind="env-added", detail=name))
+        elif name in prior and name not in current:
+            items.append(MissItem(kind="env-removed", detail=name))
+        elif current[name] != prior[name]:
+            items.append(MissItem(
+                kind="env-changed",
+                detail=f"{name}: {prior[name]!r} → {current[name]!r}",
+            ))
+    return items
+
+
+def _diff_upstream(
+    current: dict[str, str],
+    prior: dict[str, str],
+) -> list[MissItem]:
+    items: list[MissItem] = []
+    all_deps = sorted(set(prior) | set(current))
+    for dep in all_deps:
+        if prior.get(dep) != current.get(dep):
+            items.append(MissItem(kind="upstream-invalidated", detail=dep))
+    return items
+
+
+def diff_cache_state(
+    current: CacheBreakdown,
+    prior: CacheBreakdown | None,
+) -> MissReport:
+    """Compute a MissReport describing what changed between prior and current.
+
+    Items are emitted in priority order:
+        1. input-* (alphabetical by path)
+        2. env-* (alphabetical by name)
+        3. body-changed
+        4. upstream-invalidated (alphabetical by dep name)
+        5. python-changed, platform-changed
+    """
+    if prior is None:
+        return MissReport(items=(MissItem(kind="first-run", detail=""),))
+
+    items: list[MissItem] = []
+    items.extend(_diff_inputs(current.inputs, prior.inputs))
+    items.extend(_diff_env(current.env_values, prior.env_values))
+    if current.task_body_hash != prior.task_body_hash:
+        items.append(MissItem(kind="body-changed", detail=""))
+    items.extend(_diff_upstream(current.upstream_keys_by_dep, prior.upstream_keys_by_dep))
+    if current.python_version != prior.python_version:
+        items.append(MissItem(
+            kind="python-changed",
+            detail=f"{prior.python_version!r} → {current.python_version!r}",
+        ))
+    if current.platform_tag != prior.platform_tag:
+        items.append(MissItem(
+            kind="platform-changed",
+            detail=f"{prior.platform_tag!r} → {current.platform_tag!r}",
+        ))
+
+    return MissReport(items=tuple(items))

ntask/_cache/hash.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from pathlib import Path
+
+import xxhash
+
+
+def hash_bytes(data: bytes) -> str:
+    return xxhash.xxh3_128(data).hexdigest()
+
+
+def hash_many(parts: list[bytes]) -> str:
+    h = xxhash.xxh3_128()
+    for p in parts:
+        h.update(len(p).to_bytes(8, "little"))
+        h.update(p)
+    return h.hexdigest()
+
+
+@dataclass(frozen=True, slots=True)
+class FileHash:
+    path: Path
+    digest: str
+    mode: int
+
+
+def hash_file(path: Path) -> FileHash:
+    h = xxhash.xxh3_128()
+    with path.open("rb") as f:
+        while chunk := f.read(1 << 20):  # 1 MiB
+            h.update(chunk)
+    st = path.stat()
+    return FileHash(path=path, digest=h.hexdigest(), mode=st.st_mode & 0o777)
+
+
+def hash_many_files(paths: list[Path]) -> list[FileHash]:
+    if not paths:
+        return []
+    sorted_paths = sorted(paths)
+    with ThreadPoolExecutor() as ex:
+        return list(ex.map(hash_file, sorted_paths))

ntask/_cache/key.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import platform
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+from .hash import hash_many
+
+FORMAT_VERSION = b"ntask/v1"
+
+
+@dataclass(frozen=True, slots=True)
+class CacheKeyInputs:
+    task_fqn: str
+    task_body_hash: str
+    env: dict[str, str]
+    env_names: tuple[str, ...]
+    input_patterns: tuple[str, ...]
+    input_manifest_digest: str
+    root: Path
+    upstream_keys: tuple[str, ...]
+    strict: bool = True
+
+
+def _python_version_tuple() -> bytes:
+    return ".".join(str(x) for x in sys.version_info[:3]).encode()
+
+
+def _platform_tag() -> bytes:
+    return f"{platform.system()}-{platform.machine()}".lower().encode()
+
+
+def compute_cache_key(inp: CacheKeyInputs) -> str:
+    parts: list[bytes] = [
+        FORMAT_VERSION,
+        inp.task_fqn.encode(),
+        inp.task_body_hash.encode() if inp.strict else b"<body-unstrict>",
+        _python_version_tuple() if inp.strict else b"<py-unstrict>",
+        _platform_tag() if inp.strict else b"<plat-unstrict>",
+    ]
+    env_parts: list[str] = []
+    for name in sorted(inp.env_names):
+        val = inp.env.get(name)
+        env_parts.append(f"{name}={'<unset>' if val is None else val}")
+    parts.append("\n".join(env_parts).encode())
+    parts.extend(p.encode() for p in sorted(inp.input_patterns))
+    parts.append(inp.input_manifest_digest.encode())
+    parts.extend(k.encode() for k in inp.upstream_keys)
+    return hash_many(parts)
+
+
+@dataclass(frozen=True, slots=True)
+class InputRecord:
+    path: str
+    digest: str
+    mode: int
+
+
+@dataclass(frozen=True, slots=True)
+class CacheBreakdown:
+    input_patterns: tuple[str, ...]
+    inputs: tuple[InputRecord, ...]
+    env_values: dict[str, str]
+    task_body_hash: str
+    python_version: str
+    platform_tag: str
+    upstream_keys_by_dep: dict[str, str]

ntask/_cache/manifest.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+import pathspec
+from pathspec.pattern import Pattern
+
+from .hash import FileHash, hash_many, hash_many_files
+
+
+@dataclass(frozen=True, slots=True)
+class InputManifest:
+    files: list[FileHash]
+    digest: str
+
+
+def _load_gitignore(root: Path) -> pathspec.PathSpec[Pattern] | None:
+    gi = root / ".gitignore"
+    if not gi.is_file():
+        return None
+    return pathspec.PathSpec.from_lines("gitwildmatch", gi.read_text().splitlines())
+
+
+def compute_input_manifest(
+    patterns: list[str] | tuple[str, ...],
+    *,
+    root: Path,
+    respect_gitignore: bool = True,
+) -> InputManifest:
+    if not patterns:
+        return InputManifest(files=[], digest=hash_many([b"<empty-manifest>"]))
+
+    spec = pathspec.PathSpec.from_lines("gitwildmatch", patterns)
+    candidates: set[Path] = set()
+    for p in root.rglob("*"):
+        if not p.is_file():
+            continue
+        rel = p.relative_to(root).as_posix()
+        if spec.match_file(rel):
+            candidates.add(p)
+
+    if respect_gitignore:
+        ig = _load_gitignore(root)
+        if ig is not None:
+            candidates = {
+                p for p in candidates if not ig.match_file(p.relative_to(root).as_posix())
+            }
+
+    file_hashes = hash_many_files(sorted(candidates))
+    parts: list[bytes] = []
+    for fh in file_hashes:
+        rel_str: str = fh.path.relative_to(root).as_posix()
+        parts.append(rel_str.encode())
+        parts.append(fh.digest.encode())
+        parts.append(str(fh.mode).encode())
+    digest = hash_many(parts) if parts else hash_many([b"<empty-manifest>"])
+    return InputManifest(files=file_hashes, digest=digest)

ntask/_cache/outputs.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import os
+import shutil
+from pathlib import Path
+
+import pathspec
+
+from .hash import hash_many, hash_many_files
+
+
+class OutputStore:
+    """Content-addressed output blob store.
+
+    Layout: ``<root>/outputs/<outputs-hash>/...`` - preserving relative paths.
+    Restore uses hardlinks on POSIX, falls back to copy on Windows or cross-fs.
+    """
+
+    def __init__(self, root: Path):
+        self.root = root
+        self._outputs_dir = root / "outputs"
+
+    def _hash_dir(self, digest: str) -> Path:
+        return self._outputs_dir / digest
+
+    def capture(
+        self,
+        patterns: list[str] | tuple[str, ...],
+        *,
+        root: Path,
+    ) -> str | None:
+        """Capture matching files into the content-addressed store, return the hash."""
+        if not patterns:
+            return None
+        spec = pathspec.PathSpec.from_lines("gitwildmatch", patterns)
+        matches: list[Path] = []
+        for p in root.rglob("*"):
+            if not p.is_file():
+                continue
+            rel = p.relative_to(root).as_posix()
+            if spec.match_file(rel):
+                matches.append(p)
+        if not matches:
+            return None
+
+        file_hashes = hash_many_files(sorted(matches))
+        parts: list[bytes] = []
+        for fh in file_hashes:
+            rel_encoded = fh.path.relative_to(root).as_posix().encode()
+            parts.append(rel_encoded)
+            parts.append(fh.digest.encode())
+        digest = hash_many(parts)
+
+        target = self._hash_dir(digest)
+        if target.exists():
+            return digest
+
+        target.mkdir(parents=True, exist_ok=True)
+        for fh in file_hashes:
+            rel_path = fh.path.relative_to(root)
+            dest = target / rel_path
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            self._link_or_copy(fh.path, dest)
+        return digest
+
+    def restore(self, digest: str, *, root: Path) -> None:
+        """Restore a previously captured output tree into ``root``."""
+        src = self._hash_dir(digest)
+        if not src.is_dir():
+            raise FileNotFoundError(f"no output blob for hash {digest}")
+        for p in src.rglob("*"):
+            if not p.is_file():
+                continue
+            rel = p.relative_to(src)
+            dest = root / rel
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            if dest.exists():
+                dest.unlink()
+            self._link_or_copy(p, dest)
+
+    @staticmethod
+    def _link_or_copy(src: Path, dest: Path) -> None:
+        try:
+            os.link(src, dest)
+        except (OSError, NotImplementedError):
+            shutil.copy2(src, dest)