ww-agentic-workflows 1.0.0.dev3__py3-none-any.whl

ww/metadata_publication.py

@@ -0,0 +1,230 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""Durable task and project metadata publication.
+
+A completing item declares metadata values.  They are validated while the item
+is still in progress, recorded on the run as an intent together with the
+completion, and only then projected into the metadata stores.  A crash between
+the commit and the projection leaves a retriable intent rather than a metadata
+value that claims a completion which did not happen.
+"""
+
+from __future__ import annotations
+
+from dataclasses import replace
+
+from ww.errors import StateError
+from ww.execution_models import (
+    ExecutionState,
+    PlanSnapshot,
+    ProjectMetadataPublication,
+)
+from ww.plan import PlanItem
+from ww.run_coordination import RunLifecycle
+from ww.storage_adapters import (
+    ProjectMetadata,
+    ProjectMetadataStorage,
+    TaskMetadata,
+    TaskStorageAdapter,
+)
+from ww.storage_adapters.base import MetadataLeaf, append_metadata_leaf
+from ww.workflow_config import SavedMetadata
+
+
+class MetadataPublisher:
+    def __init__(
+        self,
+        tasks: TaskStorageAdapter,
+        project_store: ProjectMetadataStorage,
+        lifecycle: RunLifecycle,
+    ) -> None:
+        self.tasks = tasks
+        self.project_store = project_store
+        self.lifecycle = lifecycle
+
+    def values(self, task_id: str) -> dict[str, str]:
+        """Return task and project metadata as interpolation values."""
+        task = self.tasks.read_task_metadata(task_id)
+        project = self.project_store.read_project_metadata()
+        return {
+            **(task.interpolation_values if task is not None else {}),
+            **(project.interpolation_values if project is not None else {}),
+        }
+
+    def prepare(
+        self,
+        task_id: str,
+        state: ExecutionState,
+        item: PlanItem,
+        task_metadata: dict[str, MetadataLeaf],
+        project_metadata: dict[str, MetadataLeaf],
+    ) -> tuple[TaskMetadata | None, ProjectMetadataPublication | None]:
+        """Validate metadata and build the durable intents for one completion."""
+        updated_task_metadata = None
+        if task_metadata:
+            current = self.tasks.read_task_metadata(task_id) or TaskMetadata(task_id)
+            merged = _merge_metadata(dict(current.values), task_metadata)
+            try:
+                updated_task_metadata = TaskMetadata(task_id, tuple(merged.items()))
+            except ValueError as error:
+                raise StateError(str(error)) from error
+        if not project_metadata:
+            return updated_task_metadata, None
+        # This snapshot is a conflict precondition, not a lock held across the
+        # task commit.  Holding the project lock would reintroduce an
+        # early-publication window and block unrelated producers.
+        current_project = (
+            self.project_store.read_project_metadata() or ProjectMetadata()
+        )
+        project_values = tuple(project_metadata.items())
+        try:
+            # Reject an invalid shape while the producing item is still in
+            # progress; malformed input must not become a durable intent that
+            # no normal completion can correct.
+            ProjectMetadata(
+                tuple(
+                    _merge_metadata(
+                        dict(current_project.values), project_metadata
+                    ).items()
+                )
+            )
+        except ValueError as error:
+            raise StateError(str(error)) from error
+        existing_project = dict(current_project.values)
+        return (
+            updated_task_metadata,
+            ProjectMetadataPublication(
+                state.item_executions[state.cursor].operation_id or item.id,
+                project_values,
+                tuple(
+                    # An append key merges onto whatever exists at publication
+                    # time, so it records no snapshot to compare against.
+                    (
+                        key,
+                        None
+                        if isinstance(value, tuple)
+                        else _scalar(existing_project.get(key)),
+                    )
+                    for key, value in project_values
+                ),
+            ),
+        )
+
+    def reconcile(
+        self, state: ExecutionState, snapshot: PlanSnapshot
+    ) -> tuple[ExecutionState, PlanSnapshot]:
+        """Project committed metadata intents without exposing uncommitted data."""
+        if state.pending_task_metadata:
+            metadata = TaskMetadata(state.task_id, state.pending_task_metadata)
+            self.tasks.write_task_metadata(metadata)
+            state = replace(state, pending_task_metadata=())
+            self.lifecycle.commit(state, snapshot)
+
+        publication = state.pending_project_metadata
+        if publication is None:
+            return state, snapshot
+        with self.project_store.lock_project_metadata():
+            current = self.project_store.read_project_metadata() or ProjectMetadata()
+            existing = dict(current.values)
+            expected = dict(publication.expected_values)
+            supplied = dict(publication.values)
+            desired = {
+                key: (
+                    append_metadata_leaf(existing.get(key), value)
+                    if isinstance(value, tuple)
+                    else value
+                )
+                for key, value in supplied.items()
+            }
+            # An append key merges onto whatever is there, so it cannot conflict.
+            conflicts = [
+                key
+                for key, value in desired.items()
+                if not isinstance(supplied[key], tuple)
+                and existing.get(key) != value
+                and existing.get(key) != expected[key]
+            ]
+            if conflicts:
+                raise StateError(
+                    "project metadata publication conflict for operation "
+                    f"{publication.operation_id}: " + ", ".join(sorted(conflicts))
+                )
+            if any(existing.get(key) != value for key, value in desired.items()):
+                try:
+                    self.project_store.write_project_metadata(
+                        ProjectMetadata(tuple({**existing, **desired}.items()))
+                    )
+                except ValueError as error:
+                    # The source operation is already durable, so a shape
+                    # change made by another producer cannot be rolled back.
+                    # Keep the intent for a retry after the conflicting
+                    # project metadata has been resolved.
+                    raise StateError(
+                        "project metadata publication conflict for operation "
+                        f"{publication.operation_id}: incompatible metadata shape; "
+                        "resolve the conflicting project metadata and retry"
+                    ) from error
+        # A crash after the write but before this commit is harmless: the next
+        # reconciliation recognizes the desired values and only clears intent.
+        state = replace(state, pending_project_metadata=None)
+        self.lifecycle.commit(state, snapshot)
+        return state, snapshot
+
+
+def validate_metadata_values(
+    values: dict[str, tuple[str, ...]], requested: tuple[SavedMetadata, ...]
+) -> tuple[dict[str, MetadataLeaf], dict[str, MetadataLeaf]]:
+    """Split supplied metadata by scope after checking it against the request.
+
+    A scalar key takes exactly one value.  An ``append`` key may be omitted
+    or repeated; its values are appended to the stored list on publication.
+    """
+    by_name = {item.name: item for item in requested}
+    required = {name for name, item in by_name.items() if not item.append}
+    unknown, missing = set(values) - set(by_name), required - set(values)
+    scopes = {item.scope for item in requested}
+    if not scopes:
+        label = "task metadata"
+    elif len(scopes) == 1:
+        label = f"{next(iter(scopes))} metadata"
+    else:
+        label = "metadata"
+    if unknown:
+        raise StateError(f"unexpected {label} value(s): " + ", ".join(sorted(unknown)))
+    if missing:
+        raise StateError(
+            f"missing required {label} value(s): " + ", ".join(sorted(missing))
+        )
+    repeated = sorted(
+        name
+        for name, supplied in values.items()
+        if len(supplied) > 1 and name in required
+    )
+    if repeated:
+        raise StateError(
+            f"{label} value(s) supplied more than once: " + ", ".join(repeated)
+        )
+    task: dict[str, MetadataLeaf] = {}
+    project: dict[str, MetadataLeaf] = {}
+    for name, supplied in values.items():
+        declared = by_name[name]
+        target = project if declared.scope == "project" else task
+        target[declared.key] = supplied if declared.append else supplied[0]
+    return task, project
+
+
+def _merge_metadata(
+    current: dict[str, MetadataLeaf], supplied: dict[str, MetadataLeaf]
+) -> dict[str, MetadataLeaf]:
+    """Replace scalar keys; append list keys onto what is stored."""
+    merged = dict(current)
+    for key, value in supplied.items():
+        merged[key] = (
+            append_metadata_leaf(current.get(key), value)
+            if isinstance(value, tuple)
+            else value
+        )
+    return merged
+
+
+def _scalar(value: MetadataLeaf | None) -> str | None:
+    return value if isinstance(value, str) else None

ww/onboarding.py

@@ -0,0 +1,229 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""What ww knows about how far the operator and the project are set up.
+
+Two places hold it, each for what it describes:
+
+- the user level, ``state.json`` in the user configuration directory:
+  ``explain``, whether the operator wants the agent to narrate what ww does
+  while it learns (absent until they say so);
+- the project, in ``.ww/metadata.json`` under ww's own ``ww.`` namespace, which
+  no workflow can save into: ``setup.done``, and when ww last learned about
+  the project (``learned.project``).
+
+``ww onboarding`` shows both and sets a known key; it records the operator's
+stated preference, so it asks for no confirmation. ``discover`` reads it to
+tell an agent whether to mention setup on a first use.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Sequence
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+
+from ww.config_files import display_path, user_directory
+from ww.errors import StateError
+from ww.locking import FileLocks
+from ww.storage_adapters.base import ProjectMetadata, ProjectMetadataStorage
+from ww.workflow_config import WW_METADATA_NAMESPACE
+
+USER_STATE_FILE = "state.json"
+EXPLAIN = "explain"
+SETUP_DONE = "setup.done"
+LEARNED = "learned"
+# What ww learns about, and at which level it records when it did.
+LEARNED_USER: tuple[str, ...] = ()
+LEARNED_PROJECT = ("project",)
+USER_KEYS = (EXPLAIN, *(f"{LEARNED}.{name}" for name in LEARNED_USER))
+PROJECT_KEYS = (SETUP_DONE, *(f"{LEARNED}.{name}" for name in LEARNED_PROJECT))
+KEYS = (*USER_KEYS, *PROJECT_KEYS)
+NOW = "now"
+_BOOLEANS = {"true": True, "false": False}
+
+
+@dataclass(frozen=True)
+class OnboardingState:
+    """Both levels' onboarding keys; ``None`` is a key never set."""
+
+    user_file: Path
+    project_file: Path
+    explain: bool | None
+    setup_done: bool
+    # When ww last learned about each subject, as an ISO timestamp.
+    learned: dict[str, str | None]
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "user": {
+                "file": str(self.user_file),
+                EXPLAIN: self.explain,
+                **{f"{LEARNED}.{name}": self.learned[name] for name in LEARNED_USER},
+            },
+            "project": {
+                "file": str(self.project_file),
+                SETUP_DONE: self.setup_done,
+                **{f"{LEARNED}.{name}": self.learned[name] for name in LEARNED_PROJECT},
+            },
+        }
+
+
+class Onboarding:
+    """Read and set the onboarding keys of one project and its user."""
+
+    def __init__(self, root: Path, project_metadata: ProjectMetadataStorage) -> None:
+        self.root = root
+        self.project_metadata = project_metadata
+
+    @property
+    def user_file(self) -> Path:
+        return user_directory() / USER_STATE_FILE
+
+    @property
+    def project_file(self) -> Path:
+        return self.root / ".ww" / "metadata.json"
+
+    def read(self) -> OnboardingState:
+        user = self._read_user()
+        project = self._read_project()
+        explain = user.get(EXPLAIN)
+        learned_user = user.get(LEARNED)
+        learned_user = learned_user if isinstance(learned_user, dict) else {}
+        return OnboardingState(
+            self.user_file,
+            self.project_file,
+            explain if isinstance(explain, bool) else None,
+            project.get(SETUP_DONE) == "true",
+            {
+                **{name: _text(learned_user.get(name)) for name in LEARNED_USER},
+                **{name: project.get(f"{LEARNED}.{name}") for name in LEARNED_PROJECT},
+            },
+        )
+
+    def set(self, assignments: Sequence[str]) -> OnboardingState:
+        """Apply ``KEY=VALUE`` assignments, all checked before any is written."""
+        values = dict(parse_assignment(item) for item in assignments)
+        user = {key: value for key, value in values.items() if key in USER_KEYS}
+        project = {key: value for key, value in values.items() if key in PROJECT_KEYS}
+        if user:
+            self._write_user(user)
+        if project:
+            self._write_project(project)
+        return self.read()
+
+    def _read_user(self) -> dict[str, object]:
+        path = self.user_file
+        if not path.is_file():
+            return {}
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError) as error:
+            raise StateError(f"invalid onboarding state {path}: {error}") from error
+        if not isinstance(data, dict):
+            raise StateError(f"invalid onboarding state {path}: not an object")
+        return data
+
+    def _write_user(self, values: dict[str, bool | str]) -> None:
+        data = self._read_user()
+        for key, value in values.items():
+            if key == EXPLAIN:
+                data[EXPLAIN] = value
+                continue
+            learned = data.get(LEARNED)
+            learned = dict(learned) if isinstance(learned, dict) else {}
+            learned[key.removeprefix(f"{LEARNED}.")] = value
+            data[LEARNED] = learned
+        # The user directory is shared by every project; the write only has
+        # to be atomic for a concurrent reader.
+        FileLocks(self.root).atomic_write(
+            self.user_file, json.dumps(data, indent=2, sort_keys=True) + "\n"
+        )
+
+    def _read_project(self) -> dict[str, str]:
+        metadata = self.project_metadata.read_project_metadata()
+        prefix = f"{WW_METADATA_NAMESPACE}."
+        return {
+            key.removeprefix(prefix): value
+            for key, value in (metadata.values if metadata is not None else ())
+            if key.startswith(prefix) and isinstance(value, str)
+        }
+
+    def _write_project(self, values: dict[str, bool | str]) -> None:
+        with self.project_metadata.lock_project_metadata():
+            metadata = self.project_metadata.read_project_metadata()
+            current = dict(metadata.values if metadata is not None else ())
+            for key, value in values.items():
+                current[f"{WW_METADATA_NAMESPACE}.{key}"] = (
+                    ("true" if value else "false") if isinstance(value, bool) else value
+                )
+            try:
+                updated = ProjectMetadata(tuple(current.items()))
+            except ValueError as error:
+                raise StateError(
+                    f"cannot record onboarding in {self.project_file}: {error}"
+                ) from error
+            self.project_metadata.write_project_metadata(updated)
+
+
+def render_onboarding(state: OnboardingState, root: Path) -> str:
+    """The keys of both levels as text, each file named where it lives."""
+
+    def when(name: str) -> str:
+        return state.learned[name] or "never"
+
+    explain = {True: "yes", False: "no", None: "not asked yet"}[state.explain]
+    return "\n".join(
+        [
+            "# ww onboarding",
+            "",
+            f"User ({display_path(state.user_file, root)}):",
+            f"- {EXPLAIN}: {explain}",
+            *(f"- {LEARNED}.{name}: {when(name)}" for name in LEARNED_USER),
+            "",
+            f"Project ({display_path(state.project_file, root)}, ww's own keys):",
+            f"- {SETUP_DONE}: {'yes' if state.setup_done else 'no'}",
+            *(f"- {LEARNED}.{name}: {when(name)}" for name in LEARNED_PROJECT),
+            "",
+            "Record a key with `ww onboarding --set KEY=VALUE`.",
+            "",
+        ]
+    )
+
+
+def parse_assignment(text: str) -> tuple[str, bool | str]:
+    """One ``KEY=VALUE``: a known key and its value, checked."""
+    key, separator, value = text.partition("=")
+    key, value = key.strip(), value.strip()
+    if not separator:
+        raise StateError(f"--set takes KEY=VALUE, not {text!r}")
+    if key not in KEYS:
+        raise StateError(
+            f"unknown onboarding key {key!r}; the keys are " + ", ".join(KEYS)
+        )
+    if key in {EXPLAIN, SETUP_DONE}:
+        if value not in _BOOLEANS:
+            raise StateError(f"{key} takes true or false, not {value!r}")
+        return key, _BOOLEANS[value]
+    if value == NOW:
+        return key, _now()
+    try:
+        datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError as error:
+        raise StateError(
+            f"{key} takes `now` or an ISO timestamp, not {value!r}"
+        ) from error
+    return key, value
+
+
+def _text(value: object) -> str | None:
+    return value if isinstance(value, str) else None
+
+
+def _now() -> str:
+    return (
+        datetime.now(timezone.utc)
+        .replace(microsecond=0)
+        .isoformat()
+        .replace("+00:00", "Z")
+    )

ww/open_work.py

@@ -0,0 +1,236 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""Read-only view of the work still open in a project root.
+
+Agent hooks ask two questions of ww's state many times a session: which
+tasks are unfinished, and whether an agent-owned step is being worked on
+right now.  Both are answered here from the persisted runs alone, without
+rendering instructions, compiling plans, or loading extensions, so a hook
+stays fast and its answer is exactly what ``instruction`` would report.
+
+A task whose record cannot be read is reported beside the others, never
+raised: one broken task must not hide every other task from a scan.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+
+from ww.contracts import OperatorReason, run_is_open
+from ww.errors import StateError
+from ww.instructions.policy import operator_reason
+from ww.storage_adapters import TaskStorageAdapter
+from ww.workspace import resolve_workspace
+
+
+@dataclass(frozen=True)
+class OpenTask:
+    """One task whose selected run is neither completed nor abandoned."""
+
+    task_id: str
+    run_id: str | None
+    workflow: str
+    # The agent integration the run was started with, such as ``codex``.
+    agent: str
+    # The plan item at the cursor, when the run has not run past its end.
+    item_id: str | None
+    item_name: str | None
+    step: str | None
+    # ``step`` for the step itself; a hook phase such as
+    # ``before_complete_workflow`` for work attached to that step.
+    phase: str | None
+    owner: str | None
+    item_status: str | None
+    attempt: int
+    run_status: str
+    # Why the task waits for the operator, or ``None`` when it does not.
+    operator_reason: OperatorReason | None
+    # The directory the task works in: its worktree, project, or the root.
+    workspace: Path
+    updated_at: str
+    # When the attempt at the cursor's item started, if it has.
+    started_at: str | None = None
+    # The work item of a per-item stage, which its conversation is filed under.
+    work_item_id: str | None = None
+    # The step is one the manager hands to a worker (``--runtime auto``), so
+    # the session that started the run waits on it rather than holds it.
+    delegated: bool = False
+    # The step is interactive and its conversation has not ended: the session
+    # stops to let the operator speak, so a stop is not a step left open.
+    in_conversation: bool = False
+
+    @property
+    def label(self) -> str:
+        """How messages name the work: a hook by its own name and its step.
+
+        A hook item records the step it is attached to, so naming only the
+        step would point at a step whose own work may be long finished.
+        """
+        if self.phase in (None, "step") or not self.item_name:
+            return self.step or self.item_name or "no active step"
+        if not self.step:
+            return self.item_name
+        return f"{self.item_name} (a hook of {self.step})"
+
+    def waiting_on_another(self, open_tasks: tuple[OpenTask, ...]) -> bool:
+        """A manager legitimately waits: a child task or a worker is at work.
+
+        The child is any open task below this one; the worker holds a step
+        this task's manager delegated and has not yet finished.
+        """
+        return (
+            self.delegated
+            and self.run_status == "in_progress"
+            and self.item_status == "in_progress"
+            and self.operator_reason is None
+        ) or any(task.task_id.startswith(f"{self.task_id}/") for task in open_tasks)
+
+    @property
+    def agent_step_in_progress(self) -> bool:
+        """An agent-owned step was dispatched and is neither done nor waiting.
+
+        Waiting for input, or for the operator, is not work in progress: the
+        agent is right to stop and hand the conversation back.
+        """
+        return (
+            self.run_status == "in_progress"
+            and self.owner == "agent"
+            and self.item_status == "in_progress"
+            and self.operator_reason is None
+        )
+
+
+@dataclass(frozen=True)
+class UnreadableTask:
+    """A task whose persisted record cannot be read by this build of ww.
+
+    Commands addressing the task keep failing with ``reason``; scans across
+    tasks skip it and name it, so every other task stays usable.
+    """
+
+    task_id: str
+    reason: str
+
+    def to_dict(self) -> dict[str, str]:
+        return {"task_id": self.task_id, "reason": self.reason}
+
+
+@dataclass(frozen=True)
+class OpenWork:
+    """The unfinished tasks of a root, and the tasks that could not be read."""
+
+    tasks: tuple[OpenTask, ...]
+    unreadable: tuple[UnreadableTask, ...] = ()
+    # Tasks left unread because they were last written before the scan's
+    # ``since``, finished or not.
+    skipped: int = 0
+
+
+def open_work(
+    tasks: TaskStorageAdapter, root: Path, since: datetime | None = None
+) -> OpenWork:
+    """Every unfinished task in ``root``, children included, newest first.
+
+    With ``since``, a task last written before it is skipped unread, which
+    keeps a scan of a long history cheap.
+    """
+    found: list[OpenTask] = []
+    unreadable: list[UnreadableTask] = []
+    skipped = 0
+    for task_id in tasks.task_ids():
+        for candidate in (task_id, *tasks.child_task_ids(task_id)):
+            if since is not None:
+                written = tasks.task_written_at(candidate)
+                if written is not None and written < since:
+                    skipped += 1
+                    continue
+            try:
+                task = _open_task(tasks, root, candidate)
+            except StateError as error:
+                unreadable.append(UnreadableTask(candidate, str(error)))
+                continue
+            if task is not None:
+                found.append(task)
+    return OpenWork(
+        tuple(sorted(found, key=lambda task: task.updated_at, reverse=True)),
+        tuple(unreadable),
+        skipped,
+    )
+
+
+def _open_task(tasks: TaskStorageAdapter, root: Path, task_id: str) -> OpenTask | None:
+    runs, _, _ = tasks.read_task_record(task_id)
+    run = next((run for run in reversed(runs) if run_is_open(run.state.status)), None)
+    if run is None:
+        return None
+    state, plan = run.state, run.snapshot.plan
+    item = plan.items[state.cursor] if state.cursor < len(plan.items) else None
+    record = (
+        state.item_executions[state.cursor]
+        if state.cursor < len(state.item_executions)
+        else None
+    )
+    active = state.active_item_id is not None and item is not None
+    return OpenTask(
+        task_id=task_id,
+        run_id=state.run_id,
+        workflow=state.workflow,
+        agent=state.agent,
+        item_id=item.id if item is not None else None,
+        item_name=item.name if item is not None else None,
+        step=item.step if item is not None else None,
+        phase=item.phase if item is not None else None,
+        owner=item.owner if item is not None else None,
+        item_status=(record.status if record is not None and active else None),
+        attempt=record.attempts if record is not None else 0,
+        run_status=state.status,
+        operator_reason=operator_reason(state, plan),
+        workspace=resolve_workspace(root, state.working_directory) or root.resolve(),
+        updated_at=state.updated_at,
+        started_at=record.started_at if record is not None else None,
+        work_item_id=item.item_id if item is not None else None,
+        delegated=(
+            state.workflow_runtime == "auto"
+            and item is not None
+            and item.role == "worker"
+        ),
+        in_conversation=(
+            item is not None
+            and item.interactive
+            and record is not None
+            and not record.interaction_ended
+        ),
+    )
+
+
+def tasks_for_session(
+    open_tasks: tuple[OpenTask, ...],
+    root: Path,
+    directory: Path | None,
+    agent: str,
+) -> tuple[OpenTask, ...]:
+    """The tasks a session of ``agent`` working in ``directory`` concerns.
+
+    A session inside a task's own workspace, a worktree or a project
+    checkout, works on that task whichever agent started it; the most
+    specific workspace wins, since worktrees usually sit inside the root.
+    The root itself is shared by every session and every task without a
+    worktree, so it selects nothing: a session there, anywhere else, or in no
+    known directory concerns only the tasks its own agent started. Another
+    agent's open step is that agent's to close.
+    """
+    shared = root.resolve()
+    if directory is not None:
+        resolved = directory.resolve()
+        inside = [
+            task
+            for task in open_tasks
+            if task.workspace != shared and resolved.is_relative_to(task.workspace)
+        ]
+        if inside:
+            deepest = max(len(task.workspace.parts) for task in inside)
+            return tuple(
+                task for task in inside if len(task.workspace.parts) == deepest
+            )
+    return tuple(task for task in open_tasks if task.agent == agent)