PyTaskWireCheck 0.1.0__py3-none-any.whl

pytaskwirecheck/graph/planner.py

@@ -0,0 +1,155 @@
+"""Topological planning for the task graph.
+
+The planner produces deterministic orderings and parallelism opportunities. It
+is pure: it never launches processes, touches the cache, or renders output.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Sequence, Set, Tuple
+
+from pytaskwirecheck.diagnostics.errors import GraphError
+from pytaskwirecheck.graph.model import TaskGraph
+
+
+@dataclass(frozen=True)
+class PlannedTask:
+    """A task as it appears in a plan."""
+
+    id: str
+    dependencies: Tuple[str, ...]
+    level: int
+    cache_eligible: bool
+
+
+@dataclass(frozen=True)
+class ExecutionPlan:
+    """A deterministic execution plan for a (sub)set of tasks."""
+
+    order: Tuple[str, ...]
+    levels: Tuple[Tuple[str, ...], ...]
+    tasks: Tuple[PlannedTask, ...]
+
+    @property
+    def max_parallelism(self) -> int:
+        return max((len(level) for level in self.levels), default=0)
+
+
+def _ordered_ids(graph: TaskGraph, subset: Optional[Set[str]]) -> List[str]:
+    ids = [node.id for node in graph]
+    if subset is None:
+        return ids
+    return [task_id for task_id in ids if task_id in subset]
+
+
+def topological_order(graph: TaskGraph, subset: Optional[Set[str]] = None) -> List[str]:
+    """Return a deterministic topological order of the selected tasks.
+
+    Ties are broken by declaration order, so the same configuration always
+    yields the same plan.
+
+    Raises:
+        GraphError: if the selected subgraph contains a cycle.
+    """
+    ordered = _ordered_ids(graph, subset)
+    selected = set(ordered)
+    rank = {task_id: index for index, task_id in enumerate(ordered)}
+
+    indegree: Dict[str, int] = {}
+    for task_id in ordered:
+        deps = [d for d in graph.dependencies(task_id) if d in selected]
+        indegree[task_id] = len(deps)
+
+    ready: List[str] = sorted(
+        (t for t in ordered if indegree[t] == 0), key=rank.__getitem__
+    )
+    result: List[str] = []
+    while ready:
+        current = ready.pop(0)
+        result.append(current)
+        newly_ready: List[str] = []
+        for dependent in graph.dependents(current):
+            if dependent not in selected:
+                continue
+            indegree[dependent] -= 1
+            if indegree[dependent] == 0:
+                newly_ready.append(dependent)
+        if newly_ready:
+            ready.extend(newly_ready)
+            ready.sort(key=rank.__getitem__)
+
+    if len(result) != len(ordered):
+        raise GraphError("cycle detected while planning")
+    return result
+
+
+def execution_levels(
+    graph: TaskGraph, subset: Optional[Set[str]] = None
+) -> List[List[str]]:
+    """Group selected tasks into levels that may run in parallel.
+
+    Level 0 contains tasks with no selected dependencies; each subsequent level
+    contains tasks whose dependencies all live in earlier levels.
+    """
+    ordered = _ordered_ids(graph, subset)
+    selected = set(ordered)
+    level_of: Dict[str, int] = {}
+    levels: List[List[str]] = []
+    for task_id in topological_order(graph, subset):
+        deps = [d for d in graph.dependencies(task_id) if d in selected]
+        level = 0
+        if deps:
+            level = max(level_of[d] for d in deps) + 1
+        level_of[task_id] = level
+        while len(levels) <= level:
+            levels.append([])
+        levels[level].append(task_id)
+    return levels
+
+
+def build_plan(graph: TaskGraph, subset: Optional[Set[str]] = None) -> ExecutionPlan:
+    """Build a full :class:`ExecutionPlan` for the selected tasks."""
+    order = topological_order(graph, subset)
+    levels = execution_levels(graph, subset)
+    level_of: Dict[str, int] = {}
+    for index, level in enumerate(levels):
+        for task_id in level:
+            level_of[task_id] = index
+
+    def sort_key(task_id: str) -> int:
+        return order.index(task_id)
+
+    tasks = tuple(
+        PlannedTask(
+            id=task_id,
+            dependencies=tuple(
+                d for d in graph.dependencies(task_id) if d in set(order)
+            ),
+            level=level_of[task_id],
+            cache_eligible=graph.get(task_id).config.cache_eligible,
+        )
+        for task_id in order
+    )
+    normalized_levels = tuple(tuple(sorted(level, key=sort_key)) for level in levels)
+    return ExecutionPlan(order=tuple(order), levels=normalized_levels, tasks=tasks)
+
+
+def resolve_targets(
+    graph: TaskGraph, targets: Sequence[str], *, include_dependents: bool = False
+) -> Set[str]:
+    """Resolve the set of tasks to run for the given target names.
+
+    With no targets, the entire graph is selected. Otherwise the dependency
+    closure of the targets is selected, and optionally every task that depends
+    on them.
+    """
+    if not targets:
+        return set(graph.nodes.keys())
+    for target in targets:
+        if target not in graph.nodes:
+            raise GraphError("unknown task: %s" % target, task=target)
+    selected = graph.closure(list(targets))
+    if include_dependents:
+        selected |= graph.reverse_closure(list(targets))
+    return selected

pytaskwirecheck/graph/validator.py

@@ -0,0 +1,92 @@
+"""Validation of task graphs: missing dependencies and cycles.
+
+Validation is read-only and never executes tasks. It raises
+:class:`~pytaskwirecheck.diagnostics.errors.GraphError` subclasses with precise,
+actionable messages.
+"""
+
+from __future__ import annotations
+
+from typing import Dict, Iterator, List, Tuple
+
+from pytaskwirecheck.diagnostics.errors import CycleError, GraphError
+from pytaskwirecheck.graph.model import TaskGraph
+
+_WHITE = 0
+_GRAY = 1
+_BLACK = 2
+
+
+def find_missing_dependencies(graph: TaskGraph) -> Dict[str, Tuple[str, ...]]:
+    """Return a mapping of task id -> missing dependency ids."""
+    known = set(graph.nodes.keys())
+    missing: Dict[str, List[str]] = {}
+    for node in graph:
+        bad = [dep for dep in node.depends_on if dep not in known]
+        if bad:
+            missing[node.id] = bad
+    return {k: tuple(v) for k, v in missing.items()}
+
+
+def validate_references(graph: TaskGraph) -> None:
+    """Raise if any task references a dependency that does not exist."""
+    missing = find_missing_dependencies(graph)
+    if missing:
+        details = "; ".join(
+            "%s -> %s" % (task, ", ".join(deps))
+            for task, deps in sorted(missing.items())
+        )
+        first_task = sorted(missing.keys())[0]
+        raise GraphError("unknown task dependencies: %s" % details, task=first_task)
+
+
+def find_cycle(graph: TaskGraph) -> List[str]:
+    """Return one dependency cycle as an ordered path, or ``[]`` if acyclic.
+
+    The returned path repeats the first node at the end, e.g.
+    ``["a", "b", "c", "a"]``.
+    """
+    known = set(graph.nodes.keys())
+    color: Dict[str, int] = {task_id: _WHITE for task_id in known}
+
+    def deps_iter(task_id: str) -> Iterator[str]:
+        return iter(d for d in graph.dependencies(task_id) if d in known)
+
+    for start in known:
+        if color[start] != _WHITE:
+            continue
+        color[start] = _GRAY
+        path: List[str] = [start]
+        stack: List[Tuple[str, Iterator[str]]] = [(start, deps_iter(start))]
+        while stack:
+            node, iterator = stack[-1]
+            advanced = False
+            for dep in iterator:
+                if color[dep] == _GRAY:
+                    index = path.index(dep)
+                    return path[index:] + [dep]
+                if color[dep] == _WHITE:
+                    color[dep] = _GRAY
+                    path.append(dep)
+                    stack.append((dep, deps_iter(dep)))
+                    advanced = True
+                    break
+            if not advanced:
+                color[node] = _BLACK
+                stack.pop()
+                if path and path[-1] == node:
+                    path.pop()
+    return []
+
+
+def validate_acyclic(graph: TaskGraph) -> None:
+    """Raise :class:`CycleError` if the graph contains a cycle."""
+    cycle = find_cycle(graph)
+    if cycle:
+        raise CycleError(cycle)
+
+
+def validate_graph(graph: TaskGraph) -> None:
+    """Run all structural validations against a graph."""
+    validate_references(graph)
+    validate_acyclic(graph)

pytaskwirecheck/history/store.py

@@ -0,0 +1,164 @@
+"""Local execution history backed by SQLite.
+
+The history is a convenience, never a dependency: a write failure is surfaced
+to the caller but must never change a run's outcome. The database lives under
+``<root>/.pytaskwirecheck/history.db`` and uses WAL mode for safe concurrent readers.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import threading
+import time
+from pathlib import Path
+from typing import List, Optional
+
+from pytaskwirecheck.config.loader import state_dir
+from pytaskwirecheck.diagnostics.errors import HistoryError
+from pytaskwirecheck.engine.scheduler import RunResult
+
+SCHEMA_VERSION = 1
+
+
+class HistoryStore:
+    """A minimal, local execution-history store."""
+
+    def __init__(self, root: Path, *, max_runs: int = 500) -> None:
+        self._root = root.resolve()
+        self._path = state_dir(self._root) / "history.db"
+        self._max_runs = max_runs
+        self._lock = threading.Lock()
+        self._conn: Optional[sqlite3.Connection] = None
+
+    def _connect(self) -> sqlite3.Connection:
+        if self._conn is not None:
+            return self._conn
+        try:
+            self._path.parent.mkdir(parents=True, exist_ok=True)
+            conn = sqlite3.connect(str(self._path), timeout=10.0)
+            conn.execute("PRAGMA journal_mode=WAL")
+            conn.execute("PRAGMA synchronous=NORMAL")
+            self._create_schema(conn)
+            self._conn = conn
+        except sqlite3.Error as exc:
+            raise HistoryError("cannot open history database: %s" % exc) from exc
+        return conn
+
+    def _create_schema(self, conn: sqlite3.Connection) -> None:
+        conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS meta (
+                key TEXT PRIMARY KEY,
+                value TEXT NOT NULL
+            )
+            """
+        )
+        conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS runs (
+                run_id TEXT PRIMARY KEY,
+                project TEXT,
+                config_version INTEGER,
+                started_at REAL,
+                ended_at REAL,
+                duration_seconds REAL,
+                succeeded INTEGER,
+                cancelled INTEGER,
+                summary_json TEXT
+            )
+            """
+        )
+        conn.execute(
+            "INSERT OR IGNORE INTO meta(key, value) VALUES('schema_version', ?)",
+            (str(SCHEMA_VERSION),),
+        )
+        conn.commit()
+
+    def record_run(
+        self,
+        result: RunResult,
+        *,
+        project: str,
+        config_version: int,
+    ) -> None:
+        """Persist a run summary. Raises :class:`HistoryError` on failure."""
+        summary = result.summary()
+        ended_at = time.time()
+        started_at = ended_at - result.duration_seconds
+        with self._lock:
+            try:
+                conn = self._connect()
+                conn.execute(
+                    """
+                    INSERT OR REPLACE INTO runs
+                    (run_id, project, config_version, started_at, ended_at,
+                     duration_seconds, succeeded, cancelled, summary_json)
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    """,
+                    (
+                        result.run_id,
+                        project,
+                        config_version,
+                        started_at,
+                        ended_at,
+                        result.duration_seconds,
+                        1 if result.succeeded else 0,
+                        1 if result.cancelled else 0,
+                        json.dumps(summary, separators=(",", ":")),
+                    ),
+                )
+                conn.commit()
+                self._prune(conn)
+            except sqlite3.Error as exc:
+                raise HistoryError("cannot record run: %s" % exc) from exc
+
+    def _prune(self, conn: sqlite3.Connection) -> None:
+        conn.execute(
+            """
+            DELETE FROM runs WHERE run_id NOT IN (
+                SELECT run_id FROM runs ORDER BY ended_at DESC LIMIT ?
+            )
+            """,
+            (self._max_runs,),
+        )
+        conn.commit()
+
+    def recent(self, limit: int = 20) -> List[dict]:
+        with self._lock:
+            try:
+                conn = self._connect()
+                rows = conn.execute(
+                    """
+                    SELECT run_id, project, started_at, ended_at,
+                           duration_seconds, succeeded, cancelled, summary_json
+                    FROM runs ORDER BY ended_at DESC LIMIT ?
+                    """,
+                    (limit,),
+                ).fetchall()
+            except sqlite3.Error as exc:
+                raise HistoryError("cannot read history: %s" % exc) from exc
+        results = []
+        for row in rows:
+            results.append(
+                {
+                    "run_id": row[0],
+                    "project": row[1],
+                    "started_at": row[2],
+                    "ended_at": row[3],
+                    "duration_seconds": row[4],
+                    "succeeded": bool(row[5]),
+                    "cancelled": bool(row[6]),
+                    "summary": json.loads(row[7]) if row[7] else {},
+                }
+            )
+        return results
+
+    def close(self) -> None:
+        with self._lock:
+            if self._conn is not None:
+                try:
+                    self._conn.close()
+                except sqlite3.Error:
+                    pass
+                self._conn = None

pytaskwirecheck/logging/artifacts.py

@@ -0,0 +1,95 @@
+"""Streaming task output artifacts.
+
+Output is appended to per-task files under the state directory as it is
+produced. Writes are bounded so a runaway process cannot exhaust disk during a
+run; truncation is recorded in the log.
+"""
+
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass
+from pathlib import Path
+from typing import BinaryIO, Dict, Optional
+
+from pytaskwirecheck.config.loader import state_dir
+from pytaskwirecheck.engine.events import Event, TaskOutput
+
+
+@dataclass
+class _TaskLog:
+    handle: Optional[BinaryIO] = None
+    path: Optional[Path] = None
+    written: int = 0
+    truncated: bool = False
+
+
+class ArtifactWriter:
+    """Persists streamed task output to files, bounded per task."""
+
+    def __init__(
+        self,
+        root: Path,
+        run_id: str,
+        *,
+        max_bytes_per_task: Optional[int] = 64 * 1024 * 1024,
+    ) -> None:
+        self._base = state_dir(root) / "logs" / run_id
+        self._max_bytes = max_bytes_per_task
+        self._lock = threading.Lock()
+        self._logs: Dict[str, _TaskLog] = {}
+
+    def _open(self, task_id: str) -> _TaskLog:
+        log = self._logs.get(task_id)
+        if log is not None:
+            return log
+        log = _TaskLog()
+        try:
+            self._base.mkdir(parents=True, exist_ok=True)
+            path = self._base / ("%s.log" % task_id)
+            log.handle = open(path, "ab")
+            log.path = path
+        except OSError:
+            log.handle = None
+        self._logs[task_id] = log
+        return log
+
+    def path_for(self, task_id: str) -> Optional[Path]:
+        log = self._logs.get(task_id)
+        return log.path if log else None
+
+    def handle(self, event: Event) -> None:
+        if not isinstance(event, TaskOutput):
+            return
+        with self._lock:
+            log = self._open(event.task_id)
+            if log.handle is None or log.truncated:
+                return
+            data = event.data
+            if self._max_bytes is not None:
+                remaining = self._max_bytes - log.written
+                if remaining <= 0:
+                    log.truncated = True
+                    try:
+                        log.handle.write(b"\n[pytaskwirecheck] output truncated\n")
+                    except OSError:
+                        pass
+                    return
+                if len(data) > remaining:
+                    data = data[:remaining]
+                    log.truncated = True
+            try:
+                log.handle.write(data)
+                log.written += len(data)
+            except OSError:
+                pass
+
+    def close(self) -> None:
+        with self._lock:
+            for log in self._logs.values():
+                if log.handle is not None:
+                    try:
+                        log.handle.close()
+                    except OSError:
+                        pass
+                    log.handle = None

pytaskwirecheck/logging/structured.py

@@ -0,0 +1,83 @@
+"""Newline-delimited JSON event logging for a run.
+
+This produces a machine-readable trace alongside human output. Task output
+events are summarised (never inlined) to avoid unbounded log growth.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import threading
+from pathlib import Path
+from typing import Any, Optional
+
+from pytaskwirecheck.config.loader import state_dir
+from pytaskwirecheck.engine.events import Event, TaskOutput
+
+
+def _encode(value: Any) -> Any:
+    if isinstance(value, bytes):
+        return value.decode("utf-8", errors="replace")
+    if hasattr(value, "value"):  # Enum
+        return value.value
+    if dataclasses.is_dataclass(value) and not isinstance(value, type):
+        return {
+            f.name: _encode(getattr(value, f.name)) for f in dataclasses.fields(value)
+        }
+    return value
+
+
+def event_to_dict(event: Event) -> dict:
+    payload = _encode(event)
+    payload["event"] = type(event).__name__
+    return payload
+
+
+class StructuredLogger:
+    """Writes JSONL events to ``.pytaskwirecheck/runs/<run_id>.jsonl``."""
+
+    def __init__(self, root: Path, run_id: str) -> None:
+        self._path: Optional[Path] = None
+        self._handle = None
+        self._lock = threading.Lock()
+        try:
+            base = state_dir(root) / "runs"
+            base.mkdir(parents=True, exist_ok=True)
+            self._path = base / ("%s.jsonl" % run_id)
+            self._handle = open(self._path, "a", encoding="utf-8")
+        except OSError:
+            self._handle = None
+
+    @property
+    def path(self) -> Optional[Path]:
+        return self._path
+
+    def handle(self, event: Event) -> None:
+        if self._handle is None:
+            return
+        if isinstance(event, TaskOutput):
+            payload = {
+                "event": "TaskOutput",
+                "task_id": event.task_id,
+                "stream": event.stream,
+                "bytes": len(event.data),
+            }
+        else:
+            payload = event_to_dict(event)
+        line = json.dumps(payload, separators=(",", ":"), default=str)
+        with self._lock:
+            try:
+                self._handle.write(line + "\n")
+                self._handle.flush()
+            except OSError:
+                pass
+
+    def close(self) -> None:
+        with self._lock:
+            if self._handle is not None:
+                try:
+                    self._handle.close()
+                except OSError:
+                    pass
+                self._handle = None

pytaskwirecheck/notifications/base.py

@@ -0,0 +1,42 @@
+"""Notification interfaces.
+
+Notifications are best-effort observability. A notification failure must never
+change a run's outcome, and payloads must never include full environments,
+command output, or file contents.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Protocol
+
+
+@dataclass(frozen=True)
+class RunNotification:
+    """A sanitised, minimal description of a completed run."""
+
+    event: str
+    run_id: str
+    project: str
+    succeeded: bool
+    cancelled: bool
+    duration_seconds: float
+    counts: Dict[str, int] = field(default_factory=dict)
+    failed_tasks: List[Dict[str, Optional[object]]] = field(default_factory=list)
+
+    def to_payload(self) -> dict:
+        return {
+            "event": self.event,
+            "run_id": self.run_id,
+            "project": self.project,
+            "succeeded": self.succeeded,
+            "cancelled": self.cancelled,
+            "duration_seconds": round(self.duration_seconds, 4),
+            "counts": self.counts,
+            "failed_tasks": self.failed_tasks,
+        }
+
+
+class Notifier(Protocol):
+    def send(self, notification: RunNotification) -> bool:
+        """Deliver a notification; return True on success."""