PyTaskWireCheck 0.1.0__py3-none-any.whl

pytaskwirecheck/__init__.py

@@ -0,0 +1,32 @@
+"""PyTaskWireCheck: local-first, dependency-aware task orchestration.
+
+PyTaskWireCheck orchestrates work. It does not own the internals of the tools it runs.
+
+The public API is intentionally small: load a workflow, inspect its plan, and
+run it. See :mod:`pytaskwirecheck.api` for details.
+"""
+
+__version__ = "0.1.0"
+
+from pytaskwirecheck.api import (  # noqa: E402
+    Workflow,
+    load_workflow,
+    plan,
+    run,
+    validate,
+)
+from pytaskwirecheck.engine.scheduler import RunResult  # noqa: E402
+from pytaskwirecheck.engine.state import TaskStatus  # noqa: E402
+from pytaskwirecheck.graph.planner import ExecutionPlan  # noqa: E402
+
+__all__ = [
+    "__version__",
+    "Workflow",
+    "ExecutionPlan",
+    "RunResult",
+    "TaskStatus",
+    "load_workflow",
+    "validate",
+    "plan",
+    "run",
+]

pytaskwirecheck/api.py

@@ -0,0 +1,314 @@
+"""High-level Python API for embedding PyTaskWireCheck.
+
+The command-line interface is a thin shell over this module. Library users can
+load a workflow, inspect its plan, and execute it in-process::
+
+    from pytaskwirecheck import Workflow
+
+    workflow = Workflow.from_file("pytaskwirecheck.toml")
+    result = workflow.run()
+    if not result.succeeded:
+        raise SystemExit(1)
+
+Everything here is local-only. Importing this module never executes tasks,
+touches the network, or reads configuration implicitly; work begins only when
+:meth:`Workflow.run` is called.
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from pathlib import Path
+from typing import Any, Callable, List, Optional, Sequence, Set, Union
+
+from pytaskwirecheck.cache.policy import build_store
+from pytaskwirecheck.config.loader import find_config, load_config, parse_toml
+from pytaskwirecheck.config.schema import ExecutionConfig, WorkflowConfig
+from pytaskwirecheck.config.validation import build_workflow
+from pytaskwirecheck.diagnostics.errors import ConfigError, HistoryError
+from pytaskwirecheck.engine.cancellation import CancellationToken
+from pytaskwirecheck.engine.events import EventBus
+from pytaskwirecheck.engine.executor import TaskExecutor
+from pytaskwirecheck.engine.scheduler import RunResult, Scheduler
+from pytaskwirecheck.graph.builder import build_graph
+from pytaskwirecheck.graph.model import TaskGraph
+from pytaskwirecheck.graph.planner import ExecutionPlan, build_plan, resolve_targets
+from pytaskwirecheck.graph.validator import validate_graph
+from pytaskwirecheck.history.store import HistoryStore
+from pytaskwirecheck.notifications.base import RunNotification
+from pytaskwirecheck.notifications.webhook import WebhookNotifier
+
+_log = logging.getLogger("pytaskwirecheck")
+
+EventCallback = Callable[[Any], None]
+
+
+class Workflow:
+    """A loaded, validated workflow that can be planned and executed."""
+
+    def __init__(self, config: WorkflowConfig) -> None:
+        self._config = config
+        self._graph: Optional[TaskGraph] = None
+
+    # -- construction ------------------------------------------------------
+
+    @classmethod
+    def from_file(
+        cls,
+        path: Optional[Union[str, Path]] = None,
+        *,
+        root: Optional[Union[str, Path]] = None,
+    ) -> "Workflow":
+        """Load a workflow from ``path`` or the nearest ``pytaskwirecheck.toml``."""
+        if path is None:
+            found = find_config(Path.cwd())
+            if found is None:
+                raise ConfigError(
+                    "no pytaskwirecheck.toml found in %s or any parent directory"
+                    % Path.cwd()
+                )
+            config_path = found
+        else:
+            config_path = Path(path).resolve()
+        root_override = str(root) if root is not None else None
+        return cls(load_config(config_path, root_override=root_override))
+
+    @classmethod
+    def from_text(
+        cls,
+        text: str,
+        *,
+        root: Optional[Union[str, Path]] = None,
+        name: str = "pytaskwirecheck.toml",
+    ) -> "Workflow":
+        """Parse a workflow from an in-memory TOML document."""
+        base = Path(root).resolve() if root is not None else Path.cwd()
+        raw = parse_toml(text, location=name)
+        config = build_workflow(raw, base, base / name)
+        return cls(config)
+
+    # -- inspection --------------------------------------------------------
+
+    @property
+    def config(self) -> WorkflowConfig:
+        """The validated workflow configuration."""
+        return self._config
+
+    @property
+    def graph(self) -> TaskGraph:
+        """The validated task graph, built once and cached."""
+        if self._graph is None:
+            graph = build_graph(self._config)
+            validate_graph(graph)
+            self._graph = graph
+        return self._graph
+
+    def validate(self) -> ExecutionPlan:
+        """Validate the workflow and return the full execution plan."""
+        return build_plan(self.graph)
+
+    def plan(self, targets: Optional[Sequence[str]] = None) -> ExecutionPlan:
+        """Return the execution plan for ``targets`` (or everything)."""
+        graph = self.graph
+        subset: Optional[Set[str]] = None
+        if targets:
+            subset = resolve_targets(graph, list(targets))
+        return build_plan(graph, subset)
+
+    # -- execution ---------------------------------------------------------
+
+    def run(
+        self,
+        targets: Optional[Sequence[str]] = None,
+        *,
+        use_cache: bool = True,
+        max_workers: Optional[int] = None,
+        fail_fast: Optional[bool] = None,
+        on_event: Optional[EventCallback] = None,
+        cancel_token: Optional[CancellationToken] = None,
+        record_history: bool = True,
+        notify: bool = True,
+    ) -> RunResult:
+        """Execute the workflow and return the aggregate :class:`RunResult`.
+
+        ``on_event`` receives every engine event, enabling custom progress
+        reporting. ``cancel_token`` allows cooperative cancellation from
+        another thread. History and notifications are best-effort and never
+        alter the returned result.
+        """
+        graph = self.graph
+        selected: Set[str] = (
+            resolve_targets(graph, list(targets)) if targets else set(graph.nodes)
+        )
+        execution = self._resolve_execution(max_workers, fail_fast)
+        run_id = uuid.uuid4().hex[:12]
+
+        events = EventBus()
+        if on_event is not None:
+            events.subscribe(on_event)
+        token = cancel_token if cancel_token is not None else CancellationToken()
+
+        cache_enabled = self._config.cache.enabled and use_cache
+        store = build_store(self._config) if cache_enabled else None
+
+        executor = TaskExecutor(
+            root=self._config.project.root,
+            default_timeout=execution.default_timeout_seconds,
+            events=events,
+            cancel_token=token,
+            cache=store,
+        )
+        scheduler = Scheduler(
+            graph=graph,
+            execution=execution,
+            executor=executor,
+            events=events,
+            cancel_token=token,
+            use_cache=cache_enabled,
+        )
+        result = scheduler.run(selected, run_id=run_id)
+
+        if record_history:
+            self._record_history(result)
+        if notify:
+            self._notify(result)
+        return result
+
+    # -- helpers -----------------------------------------------------------
+
+    def _resolve_execution(
+        self, max_workers: Optional[int], fail_fast: Optional[bool]
+    ) -> ExecutionConfig:
+        execution = self._config.execution
+        if max_workers is None and fail_fast is None:
+            return execution
+        if max_workers is not None and max_workers < 1:
+            raise ConfigError("max_workers must be >= 1")
+        return ExecutionConfig(
+            max_workers=(
+                max_workers if max_workers is not None else execution.max_workers
+            ),
+            default_timeout_seconds=execution.default_timeout_seconds,
+            fail_fast=(fail_fast if fail_fast is not None else execution.fail_fast),
+        )
+
+    def _record_history(self, result: RunResult) -> None:
+        if not self._config.history.enabled:
+            return
+        store = HistoryStore(
+            self._config.project.root, max_runs=self._config.history.max_runs
+        )
+        try:
+            store.record_run(
+                result,
+                project=self._config.project.name,
+                config_version=self._config.project.version,
+            )
+        except HistoryError as exc:
+            _log.warning("could not record history: %s", exc)
+        finally:
+            store.close()
+
+    def _notify(self, result: RunResult) -> None:
+        settings = self._config.notifications
+        if not settings.enabled:
+            return
+        event = "success" if result.succeeded else "failure"
+        notifier = WebhookNotifier(
+            settings.url,
+            timeout_seconds=settings.timeout_seconds,
+            max_attempts=settings.max_attempts,
+            events=settings.on,
+        )
+        if not notifier.wants(event):
+            return
+        notification = RunNotification(
+            event=event,
+            run_id=result.run_id,
+            project=self._config.project.name,
+            succeeded=result.succeeded,
+            cancelled=result.cancelled,
+            duration_seconds=result.duration_seconds,
+            counts=result.counts,
+            failed_tasks=self._failed_tasks(result),
+        )
+        try:
+            delivered = notifier.send(notification)
+        except Exception as exc:  # defensive: never break the run
+            _log.warning("notification failed: %s", type(exc).__name__)
+            return
+        if not delivered:
+            _log.warning("notification was not delivered")
+
+    @staticmethod
+    def _failed_tasks(result: RunResult) -> List[dict]:
+        failed = []
+        for task_id, record in sorted(result.records.items()):
+            if record.status.is_failure:
+                failed.append(
+                    {
+                        "task": task_id,
+                        "status": record.status.value,
+                        "exit_code": record.exit_code,
+                    }
+                )
+        return failed
+
+
+# ---------------------------------------------------------------------------
+# Functional conveniences
+# ---------------------------------------------------------------------------
+
+
+def load_workflow(
+    path: Optional[Union[str, Path]] = None,
+    *,
+    root: Optional[Union[str, Path]] = None,
+) -> Workflow:
+    """Load a :class:`Workflow` from ``path`` or the nearest config file."""
+    return Workflow.from_file(path, root=root)
+
+
+def validate(
+    path: Optional[Union[str, Path]] = None,
+    *,
+    root: Optional[Union[str, Path]] = None,
+) -> ExecutionPlan:
+    """Validate a workflow and return its full :class:`ExecutionPlan`."""
+    return load_workflow(path, root=root).validate()
+
+
+def plan(
+    targets: Optional[Sequence[str]] = None,
+    *,
+    path: Optional[Union[str, Path]] = None,
+    root: Optional[Union[str, Path]] = None,
+) -> ExecutionPlan:
+    """Return the plan for ``targets`` without executing anything."""
+    return load_workflow(path, root=root).plan(targets)
+
+
+def run(
+    targets: Optional[Sequence[str]] = None,
+    *,
+    path: Optional[Union[str, Path]] = None,
+    root: Optional[Union[str, Path]] = None,
+    use_cache: bool = True,
+    max_workers: Optional[int] = None,
+    fail_fast: Optional[bool] = None,
+    on_event: Optional[EventCallback] = None,
+    record_history: bool = True,
+    notify: bool = True,
+) -> RunResult:
+    """Load and execute a workflow in one call."""
+    workflow = load_workflow(path, root=root)
+    return workflow.run(
+        targets,
+        use_cache=use_cache,
+        max_workers=max_workers,
+        fail_fast=fail_fast,
+        on_event=on_event,
+        record_history=record_history,
+        notify=notify,
+    )

pytaskwirecheck/cache/fingerprint.py

@@ -0,0 +1,176 @@
+"""Content-addressed cache fingerprinting.
+
+The fingerprint captures everything PyTaskWireCheck knows can affect a task's result:
+its command, shell mode, working directory, declared inputs (by content), the
+cache keys of its declared dependencies, relevant environment variables (hashed,
+never stored in plaintext), and a small runtime identity.
+
+Anything *not* declared is, by definition, invisible to the cache. Tasks that
+depend on undeclared files, the network, the clock, or other nondeterminism may
+receive incorrect cache hits; see ``docs/caching.md``.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import platform
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
+
+from pytaskwirecheck.config.schema import DEFAULT_RELEVANT_ENV, TaskConfig
+
+#: Bump when the fingerprint algorithm or payload shape changes incompatibly.
+CACHE_SCHEMA_VERSION = 1
+
+_CHUNK = 1024 * 1024
+
+
+@dataclass(frozen=True)
+class InputFile:
+    path: str
+    sha256: str
+    size: int
+
+
+@dataclass(frozen=True)
+class Fingerprint:
+    """The cache key plus the information needed to explain a hit or miss."""
+
+    key: str
+    payload: dict
+    inputs: Tuple[InputFile, ...]
+    missing_patterns: Tuple[str, ...]
+    env_names: Tuple[str, ...]
+
+
+def hash_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        while True:
+            chunk = handle.read(_CHUNK)
+            if not chunk:
+                break
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _safe_iter_files(root: Path, base: Path) -> Iterable[Path]:
+    """Yield regular files under ``base`` (a file or directory)."""
+    if base.is_file():
+        yield base
+        return
+    if base.is_dir():
+        for dirpath, _dirnames, filenames in os.walk(base):
+            for name in sorted(filenames):
+                candidate = Path(dirpath) / name
+                if candidate.is_file() and not candidate.is_symlink():
+                    yield candidate
+
+
+def expand_inputs(
+    root: Path, patterns: Sequence[str]
+) -> Tuple[List[InputFile], List[str]]:
+    """Expand input globs into a sorted list of hashed files.
+
+    Returns the hashed files and the patterns that matched nothing.
+    """
+    root = root.resolve()
+    seen: Dict[str, InputFile] = {}
+    missing: List[str] = []
+    for pattern in patterns:
+        matches = sorted(root.glob(pattern))
+        matched_any = False
+        for match in matches:
+            if match.is_dir():
+                files = list(_safe_iter_files(root, match))
+                if files:
+                    matched_any = True
+            else:
+                files = [match]
+                matched_any = True
+            for file_path in files:
+                try:
+                    rel = file_path.resolve().relative_to(root).as_posix()
+                except ValueError:
+                    continue
+                if rel in seen:
+                    continue
+                try:
+                    size = file_path.stat().st_size
+                    digest = hash_file(file_path)
+                except OSError:
+                    continue
+                seen[rel] = InputFile(path=rel, sha256=digest, size=size)
+        if not matched_any:
+            missing.append(pattern)
+    return [seen[key] for key in sorted(seen)], missing
+
+
+def _hash_env_values(values: Mapping[str, str]) -> Dict[str, str]:
+    return {
+        name: hashlib.sha256(value.encode("utf-8")).hexdigest()
+        for name, value in sorted(values.items())
+    }
+
+
+def build_effective_env(
+    task: TaskConfig, base_env: Mapping[str, str]
+) -> Dict[str, str]:
+    env = dict(base_env) if task.inherit_env else {}
+    env.update(task.env)
+    return env
+
+
+def compute_fingerprint(
+    task: TaskConfig,
+    root: Path,
+    dependency_keys: Mapping[str, Optional[str]],
+    *,
+    base_env: Optional[Mapping[str, str]] = None,
+    relevant_env: Sequence[str] = DEFAULT_RELEVANT_ENV,
+) -> Fingerprint:
+    """Compute the cache fingerprint for ``task``."""
+    base_env = base_env if base_env is not None else os.environ
+    effective_env = build_effective_env(task, base_env)
+
+    relevant_names = sorted(set(relevant_env) | set(task.env.keys()))
+    selected_env = {
+        name: effective_env[name] for name in relevant_names if name in effective_env
+    }
+    hashed_env = _hash_env_values(selected_env)
+
+    inputs, missing = expand_inputs(root, task.inputs)
+
+    payload = {
+        "schema": CACHE_SCHEMA_VERSION,
+        "task_id": task.id,
+        "command": list(task.command),
+        "shell": task.shell,
+        "cwd": task.cwd or ".",
+        "timeout_seconds": task.timeout_seconds,
+        "env": hashed_env,
+        "input_patterns": list(task.inputs),
+        "inputs": [
+            {"path": f.path, "sha256": f.sha256, "size": f.size} for f in inputs
+        ],
+        "outputs": list(task.outputs),
+        "dependencies": {name: key for name, key in sorted(dependency_keys.items())},
+        "runtime": {
+            "python": "%d.%d.%d" % sys.version_info[:3],
+            "platform": sys.platform,
+            "machine": platform.machine(),
+        },
+    }
+    canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+    key = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+    return Fingerprint(
+        key=key,
+        payload=payload,
+        inputs=tuple(inputs),
+        missing_patterns=tuple(missing),
+        env_names=tuple(sorted(selected_env.keys())),
+    )

pytaskwirecheck/cache/policy.py

@@ -0,0 +1,49 @@
+"""Cache policy decisions and construction helpers.
+
+Policy is deliberately simple: the global config may disable the cache, the
+CLI may disable it for a single run (``--no-cache``), and individual tasks may
+opt out. A task with no declared inputs or outputs is never cached because its
+side effects are unknown.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Mapping, Optional
+
+from pytaskwirecheck.cache.store import CacheStore
+from pytaskwirecheck.config.schema import TaskConfig, WorkflowConfig
+
+
+@dataclass(frozen=True)
+class CacheDecision:
+    use_cache: bool
+    reason: str
+
+
+def decide(
+    task: TaskConfig, *, global_enabled: bool, cli_disabled: bool
+) -> CacheDecision:
+    """Decide whether cache reads/writes apply to ``task`` in this run."""
+    if cli_disabled:
+        return CacheDecision(False, "disabled for this run (--no-cache)")
+    if not global_enabled:
+        return CacheDecision(False, "disabled in configuration")
+    if not task.cache:
+        return CacheDecision(False, "opted out by task")
+    if not task.cache_eligible:
+        return CacheDecision(False, "no declared inputs or outputs")
+    return CacheDecision(True, "eligible")
+
+
+def build_store(
+    config: WorkflowConfig,
+    *,
+    base_env: Optional[Mapping[str, str]] = None,
+) -> CacheStore:
+    """Construct a :class:`CacheStore` for ``config``."""
+    return CacheStore(
+        config.project.root,
+        base_env=base_env,
+        relevant_env=config.cache.relevant_env,
+    )