ww-agentic-workflows 1.0.0.dev3__py3-none-any.whl

ww/item_passes.py

@@ -0,0 +1,167 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""What an expanded ``items`` pass requires of its items when it ends.
+
+A workflow has one item collection and any number of sequential ``items``
+passes over it.  Each pass declares what its stages do with an item through
+``item_phase``, and only that is required when the run leaves the pass: an
+analyze stage needs the item's analysis, a resolve stage its actual solution
+and ``resolved``, a report stage ``reported``, and a phase stage's declared
+item saves their values.  The built-in ``handle-item`` stage keeps its whole
+lifecycle: ``resolved`` and ``reported``.  A stage without ``item_phase``
+has only its ordinary completion contract.  A stage that never ran, because
+an assessment, a break, or a stop skipped it, requires nothing.  When the
+pass ends with an assessment, it is left only once the outcome is chosen.
+
+A linked item (``reference_to_id``) shares its canonical item's analysis and
+solution, so a duplicate comment needs no duplicate fix, but it is reported,
+and resolved, for its own source.
+"""
+
+from __future__ import annotations
+
+from ww.execution_models import PlanItemExecution
+from ww.items import WorkItem
+from ww.plan import PlanItem, WorkflowPlan
+
+
+def item_collection(plan: WorkflowPlan) -> PlanItem | None:
+    """The first ``items`` declaration, which holds the collection's settings.
+
+    Every pass works on the same collection; its ``persistent``,
+    ``identity``, and ``unique`` settings are the first declaration's.  The
+    shared validator rejects a later declaration that sets them differently.
+    """
+    return next(
+        (
+            item
+            for item in plan.items
+            if item.item_operation == "collect" and item.child_operation is None
+        ),
+        None,
+    )
+
+
+def is_pass_stage(item: PlanItem) -> bool:
+    """Whether ``item`` is a concrete stage, hook, or verifier of an item pass."""
+    return (
+        item.item_pass is not None
+        and item.item_id is not None
+        and item.child_stage is None
+        and not item.item_template
+    )
+
+
+def reports_item_on_completion(plan: WorkflowPlan, cursor: int) -> bool:
+    """Whether completing ``plan.items[cursor]`` finishes an automatic report.
+
+    A report stage's lifecycle is its step, its handler-group members and its
+    completion hooks: the plan items of one item and pass with the report
+    operation.  When ww runs any of them, the item is reported by the last one
+    to complete, whoever owns that last item, so no agent bookkeeping is
+    needed.  A report stage that ww runs none of is reported by its agent
+    with ``update-item --reported=true``, as before.
+    """
+    current = plan.items[cursor]
+    if current.item_operation != "report_item" or current.item_id is None:
+        return False
+    lifecycle = [
+        other
+        for other in plan.items
+        if other.item_id == current.item_id
+        and other.item_pass == current.item_pass
+        and other.item_operation == "report_item"
+    ]
+    return lifecycle[-1].id == current.id and any(
+        other.owner == "ww" for other in lifecycle
+    )
+
+
+def leaving_pass(plan: WorkflowPlan, cursor: int) -> str | None:
+    """The pass whose expanded stages end right before ``cursor``, if any.
+
+    Assessment outcomes inside a per-item stage carry the stage's item and
+    pass, so they count as stages of the pass.  Every workflow without a
+    handoff ends with its built-in summary, so the run never leaves a pass by
+    reaching the end of the plan; a stopping outcome completes the run.
+    """
+    if not 0 < cursor < len(plan.items):
+        return None
+    previous, following = plan.items[cursor - 1], plan.items[cursor]
+    if not is_pass_stage(previous) or (
+        is_pass_stage(following) and following.item_pass == previous.item_pass
+    ):
+        return None
+    return previous.item_pass
+
+
+def pass_gate_failures(
+    plan: WorkflowPlan,
+    records: tuple[PlanItemExecution, ...],
+    pass_id: str,
+    items: tuple[WorkItem, ...],
+) -> tuple[str, ...]:
+    """What each item of ``pass_id`` still lacks for the stages that ran."""
+    by_id = {item.id: item for item in items}
+    failures: list[str] = []
+    for stage, record in zip(plan.items, records, strict=True):
+        if (
+            not is_pass_stage(stage)
+            or stage.item_pass != pass_id
+            or stage.phase != "step"
+            or stage.verifies is not None
+            or stage.item_operation is None
+            or record.status != "completed"
+            or record.started_at is None
+        ):
+            continue
+        item = by_id.get(str(stage.item_id))
+        if item is None:
+            failures.append(f"{stage.item_id} ({stage.name}): the item is gone")
+            continue
+        missing = _missing(stage, item, by_id)
+        if missing:
+            failures.append(f"{item.id} ({stage.name}): " + ", ".join(missing))
+    return tuple(dict.fromkeys(failures))
+
+
+def _missing(
+    stage: PlanItem, item: WorkItem, items: dict[str, WorkItem]
+) -> tuple[str, ...]:
+    canonical = _canonical(item, items)
+    missing: list[str] = []
+    operation = stage.item_operation
+    if operation == "process_item" and not (
+        item.processed_item or canonical.processed_item
+    ):
+        missing.append("processed_item")
+    if operation == "resolve_item":
+        if not (item.actual_solution or canonical.actual_solution):
+            missing.append("actual_solution")
+        if not (item.resolved or canonical.resolved):
+            missing.append("resolved=true")
+    if operation == "report_item" and not item.reported:
+        missing.append("reported=true")
+    if operation == "handle_item":
+        if not item.resolved:
+            missing.append("resolved=true")
+        if not item.reported:
+            missing.append("reported=true")
+    missing.extend(
+        f"field {field.name}"
+        for field in stage.update_item
+        if not item.field(field.name)
+    )
+    return tuple(missing)
+
+
+def _canonical(item: WorkItem, items: dict[str, WorkItem]) -> WorkItem:
+    """The item a linked item refers to, following links; itself otherwise."""
+    seen = {item.id}
+    current = item
+    while current.reference_to_id is not None:
+        target = items.get(current.reference_to_id)
+        if target is None or target.id in seen:
+            break
+        seen.add(target.id)
+        current = target
+    return current

ww/items.py

@@ -0,0 +1,99 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""Durable, run-local work items used by item-aware workflow steps."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, fields, replace
+
+# An item field name, e.g. "acceptance_criteria" or "due-date"; "2nd" does not
+# match.
+FIELD_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_-]*")
+
+
+@dataclass(frozen=True)
+class WorkItem:
+    """One collected unit of work inside a run."""
+
+    id: str
+    item: str
+    processed_item: str = ""
+    proposed_solution: str = ""
+    actual_solution: str = ""
+    resolved: bool = False
+    reported: bool = False
+    reference_to_id: str | None = None
+    # Custom fields a workflow declares for its items, such as the ID of the
+    # source comment or of the reply posted for it; string values only.
+    fields: tuple[tuple[str, str], ...] = ()
+
+    def field(self, name: str) -> str | None:
+        return dict(self.fields).get(name)
+
+    def with_fields(self, values: dict[str, str]) -> WorkItem:
+        merged = {**dict(self.fields), **values}
+        return replace(self, fields=tuple(merged.items()))
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "id": self.id,
+            "item": self.item,
+            "processed_item": self.processed_item,
+            "proposed_solution": self.proposed_solution,
+            "actual_solution": self.actual_solution,
+            "resolved": self.resolved,
+            "reported": self.reported,
+            "reference_to_id": self.reference_to_id,
+            "fields": dict(self.fields),
+        }
+
+    @classmethod
+    def from_dict(cls, data: object) -> WorkItem:
+        if not isinstance(data, dict):
+            raise ValueError("item must be a mapping")
+
+        if not isinstance(data.get("id"), str) or not data["id"].strip():
+            raise ValueError("item ID must be a non-empty string")
+        if not isinstance(data.get("item"), str) or not data["item"].strip():
+            raise ValueError("item text must be a non-empty string")
+        processed = data.get("processed_item", "")
+        proposed = data.get("proposed_solution", "")
+        actual = data.get("actual_solution", "")
+        if not all(isinstance(value, str) for value in (processed, proposed, actual)):
+            raise ValueError("item text fields must be strings")
+        resolved = data.get("resolved", False)
+        reported = data.get("reported", False)
+        if not isinstance(resolved, bool) or not isinstance(reported, bool):
+            raise ValueError("item resolved and reported fields must be booleans")
+        reference = data.get("reference_to_id")
+        if reference is not None and (not isinstance(reference, str) or not reference):
+            raise ValueError("item reference_to_id must be a non-empty string or null")
+        return cls(
+            id=data["id"],
+            item=data["item"],
+            processed_item=processed,
+            proposed_solution=proposed,
+            actual_solution=actual,
+            resolved=resolved,
+            reported=reported,
+            reference_to_id=reference,
+            fields=validate_item_fields(data.get("fields", {})),
+        )
+
+
+def validate_item_fields(data: object) -> tuple[tuple[str, str], ...]:
+    """Custom fields are a mapping of valid names to strings."""
+    if not isinstance(data, dict):
+        raise ValueError("item fields must be a mapping")
+    for name, value in data.items():
+        if not isinstance(name, str) or not FIELD_NAME.fullmatch(name):
+            raise ValueError(f"invalid item field name: {name!r}")
+        if not isinstance(value, str):
+            raise ValueError(f"item field {name!r} must be a string")
+    return tuple(data.items())
+
+
+# Fields an agent may update after collection; identity and the item text are fixed.
+EDITABLE_WORK_ITEM_FIELDS = frozenset(
+    field.name for field in fields(WorkItem) if field.name not in {"id", "item"}
+)

ww/locking.py

@@ -0,0 +1,207 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""Exclusive file locking and atomic replacement for ww's writes.
+
+Two ww invocations in one project are unrelated processes competing for the
+same task files. Atomic replacement alone is not enough: it stops a reader
+seeing half a write, but not two processes reading one ``state.json``, each
+deciding the next item, and one overwriting the other. The fix is a single
+exclusive lock per task, held by ``WorkflowService`` for the whole of ``start``,
+``next``, ``complete`` or ``reset`` — the span, not the individual write.
+
+The task scope is the primary coordination boundary; the project scope wraps
+``ww init``, project metadata has a dedicated merge lock, the execution log has
+its own lock, and aggregate commits use a separate per-aggregate CAS lock for
+direct storage-adapter callers. Everything else writes inside one of those scopes, so
+``atomic_write`` takes no lock of its own. A shared activity gate wraps those
+locks; maintenance takes it exclusively before pruning sidecars.
+
+Reads are deliberately unlocked. Replacement is atomic, so a reader always sees
+a complete aggregate document. ``RunCoordinator.load`` selects the requested
+run, execution state, and plan snapshot from one decoded revision rather than
+combining independently read files. Read-only commands therefore do not wait
+behind a mid-flight writer.
+
+Locks are advisory POSIX locks on sidecar files under ``.ww/locks/``. The
+kernel releases them when a process exits, so a killed run leaves nothing
+stale. Waiting order is unspecified — no operating system promises FIFO — and
+every wait is bounded by ``WW_LOCK_TIMEOUT`` (seconds, default 30; ``0`` waits
+indefinitely) so a pathological wait fails loudly instead of hanging.
+"""
+
+from __future__ import annotations
+
+import errno
+import fcntl
+import hashlib
+import os
+import sys
+import tempfile
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from typing import TextIO
+
+from ww.errors import LockError
+
+TIMEOUT_VARIABLE = "WW_LOCK_TIMEOUT"
+DEFAULT_TIMEOUT_SECONDS = 30.0
+_NOTICE_AFTER_SECONDS = 0.25
+_MAXIMUM_POLL_SECONDS = 0.05
+
+
+class FileLocks:
+    """Exclusive locks and atomic writes for one project root."""
+
+    def __init__(self, root: Path) -> None:
+        self.root = Path(root)
+        self.directory = self.root / ".ww" / "locks"
+
+    def lock_path(self, target: Path) -> Path:
+        """Return the sidecar lock file that guards ``target``.
+
+        The lock never sits next to the file it guards: atomic replacement
+        swaps the target's inode and `reset` deletes whole task directories, so
+        a sidecar inside the task tree would be destroyed by the very writes it
+        is meant to protect.
+        """
+        absolute = os.path.normpath(Path(target).absolute())
+        digest = hashlib.sha256(absolute.encode("utf-8")).hexdigest()[:32]
+        return self.directory / f"{digest}.lock"
+
+    @property
+    def _activity_path(self) -> Path:
+        return self.directory / ".activity.lock"
+
+    @contextmanager
+    def lock(self, target: Path, *, purpose: str | None = None) -> Iterator[None]:
+        """Hold an exclusive lock on ``target`` for the duration of the block.
+
+        Not reentrant: ww takes each lock at exactly one place.
+        """
+        path = self.lock_path(target)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        # Closing a handle releases its lock, including when the block raised.
+        with self._activity_path.open("a+", encoding="utf-8") as activity:
+            # Cleanup takes the activity gate exclusively before unlinking
+            # sidecars.  Do not open the task sidecar until the shared gate is
+            # held: otherwise cleanup can unlink it between open(2) and the
+            # shared-lock acquisition, leaving this process locking an orphan
+            # inode while a later caller locks the replacement path.
+            _acquire(activity, "ww activity gate", fcntl.LOCK_SH)
+            with path.open("a+", encoding="utf-8") as handle:
+                _acquire(handle, purpose or self._describe(target))
+                yield
+
+    def cleanup(self) -> int:
+        """Remove unused lock sidecars without racing active or waiting users."""
+        self.directory.mkdir(parents=True, exist_ok=True)
+        with self._activity_path.open("a+", encoding="utf-8") as activity:
+            _acquire(activity, "ww activity gate", fcntl.LOCK_EX)
+            removed = 0
+            for path in self.directory.glob("*.lock"):
+                if path == self._activity_path:
+                    continue
+                path.unlink(missing_ok=True)
+                removed += 1
+            return removed
+
+    def atomic_write(self, target: Path, content: str) -> None:
+        """Replace ``target`` with ``content`` in one step.
+
+        This takes no lock. Callers write inside a scope their command already
+        holds; the atomicity here is what keeps a concurrent *reader* from ever
+        seeing a partial file.
+        """
+        target = Path(target)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        descriptor, temporary_name = tempfile.mkstemp(
+            prefix=f".{target.name}.", dir=target.parent
+        )
+        temporary = Path(temporary_name)
+        try:
+            with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+                handle.write(content)
+                handle.flush()
+                os.fsync(handle.fileno())
+            temporary.replace(target)
+            directory_descriptor = os.open(target.parent, os.O_RDONLY)
+            try:
+                os.fsync(directory_descriptor)
+            finally:
+                os.close(directory_descriptor)
+        except BaseException:
+            temporary.unlink(missing_ok=True)
+            raise
+
+    def append_line(self, target: Path, line: str) -> None:
+        """Append one newline-terminated record under ``target``'s own lock."""
+        target = Path(target)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        with self.lock(target), target.open("a", encoding="utf-8") as handle:
+            handle.write(line if line.endswith("\n") else line + "\n")
+            handle.flush()
+            os.fsync(handle.fileno())
+
+    def _describe(self, target: Path) -> str:
+        try:
+            return str(Path(target).relative_to(self.root))
+        except ValueError:
+            return str(target)
+
+
+def timeout_seconds() -> float | None:
+    """Return the configured wait bound, or ``None`` to wait indefinitely."""
+    raw = os.environ.get(TIMEOUT_VARIABLE)
+    if raw is None:
+        return DEFAULT_TIMEOUT_SECONDS
+    try:
+        value = float(raw)
+    except ValueError as error:
+        raise LockError(
+            f"invalid {TIMEOUT_VARIABLE}: {raw!r} is not a number of seconds"
+        ) from error
+    return None if value <= 0 else value
+
+
+_CONTENDED_ERRNOS = {
+    errno.EAGAIN,
+    errno.EWOULDBLOCK,
+    errno.EACCES,
+    errno.EINTR,
+}
+
+
+def _is_contended_error(error: OSError) -> bool:
+    return isinstance(error, BlockingIOError) or error.errno in _CONTENDED_ERRNOS
+
+
+def _acquire(handle: TextIO, description: str, mode: int = fcntl.LOCK_EX) -> None:
+    limit = timeout_seconds()
+    started = time.monotonic()
+    deadline = None if limit is None else started + limit
+    announced = False
+    delay = 0.001
+    last_contention_error: OSError | None = None
+    while True:
+        try:
+            fcntl.flock(handle.fileno(), mode | fcntl.LOCK_NB)
+            return
+        except OSError as error:
+            if not _is_contended_error(error):
+                raise LockError(f"cannot lock {description}: {error}") from error
+            last_contention_error = error
+        if deadline is not None and time.monotonic() >= deadline:
+            raise LockError(
+                f"timed out after {limit:g}s waiting for another ww process to "
+                f"release {description}; set {TIMEOUT_VARIABLE} to wait longer "
+                "(or 0 to wait indefinitely)"
+            ) from last_contention_error
+        if not announced and time.monotonic() - started >= _NOTICE_AFTER_SECONDS:
+            sys.stderr.write(
+                f"ww: waiting for another ww process to release {description}\n"
+            )
+            sys.stderr.flush()
+            announced = True
+        time.sleep(delay)
+        delay = min(delay * 2, _MAXIMUM_POLL_SECONDS)