ww-agentic-workflows 1.0.0.dev3__py3-none-any.whl

ww/storage_adapters/filesystem.py

@@ -0,0 +1,370 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""Filesystem implementation of task persistence."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import shutil
+from collections.abc import Iterator
+from contextlib import ExitStack, contextmanager
+from datetime import datetime, timezone
+from pathlib import Path
+
+from ww.amendments import Amendment
+from ww.contracts import BOOTSTRAP_REQUEST_PREFIX
+from ww.errors import StateError
+from ww.execution_models import TaskRunAggregate, validate_task_runs
+from ww.items import WorkItem
+from ww.locking import FileLocks
+from ww.storage_adapters.base import (
+    ArtifactAddress,
+    CommandOutputAddress,
+    TaskMetadata,
+    TaskStorageAdapter,
+    flatten_metadata,
+)
+from ww.storage_adapters.task_document import (
+    decode_task_document,
+    encode_task_document,
+)
+
+
+class FileTaskStorageAdapter(TaskStorageAdapter):
+    """Store task state and artifacts beneath a project's ``.ww/tasks/`` directory.
+
+    Every write goes through :mod:`ww.locking`. Mutating methods do not lock:
+    `lock_task` is the boundary and `WorkflowService` holds it around the whole
+    command, which is the span that actually needs protecting.
+    """
+
+    def __init__(self, root: Path) -> None:
+        self.root = root
+        self.tasks_path = root / ".ww" / "tasks"
+        self.locks = FileLocks(root)
+
+    @contextmanager
+    def lock_task(self, task_id: str) -> Iterator[None]:
+        """Lock a task and its parent, so parent reset excludes child writes."""
+        parent = task_id.rsplit("/", 1)[0] if "/" in task_id else None
+        with ExitStack() as stack:
+            if parent is not None:
+                stack.enter_context(
+                    self.locks.lock(
+                        self.tasks_path / parent, purpose=f"task {parent!r}"
+                    )
+                )
+            stack.enter_context(
+                self.locks.lock(self.tasks_path / task_id, purpose=f"task {task_id!r}")
+            )
+            yield
+
+    def read_task_record(
+        self, task_id: str
+    ) -> tuple[tuple[TaskRunAggregate, ...], str | None, int]:
+        runs, handoff, revision, _ = self._read_task_document(task_id)
+        return runs, handoff, revision
+
+    def task_written_at(self, task_id: str) -> datetime | None:
+        # Every commit replaces the state file, so its modification time is
+        # the last commit's.
+        try:
+            modified = self._state_path(task_id).stat().st_mtime
+        except OSError:
+            return None
+        return datetime.fromtimestamp(modified, timezone.utc)
+
+    def _read_task_document(
+        self, task_id: str
+    ) -> tuple[
+        tuple[TaskRunAggregate, ...],
+        str | None,
+        int,
+        dict[str, list[dict[str, object]]],
+    ]:
+        path = self._state_path(task_id)
+        if not path.exists():
+            return (), None, 0, {}
+        try:
+            raw = json.loads(path.read_text(encoding="utf-8"))
+            decoded = decode_task_document(raw, task_id)
+            decoded = (*decoded[:3], self._validate_ledger(decoded[3]))
+            validate_task_runs(task_id, decoded[0])
+            return decoded
+        except (OSError, json.JSONDecodeError, StateError, ValueError) as error:
+            raise StateError(f"invalid task state {path}: {error}") from error
+
+    def commit_task_aggregate(
+        self,
+        task_id: str,
+        runs: tuple[TaskRunAggregate, ...],
+        handoff: str | None = None,
+        expected_revision: int | None = None,
+    ) -> int:
+        # The service normally already holds the task lock.  This separate
+        # aggregate lock also makes direct storage-adapter callers safe: the revision
+        # check and replacement cannot race another storage-adapter commit, while the
+        # lock ordering remains task -> aggregate everywhere in the service.
+        with self.locks.lock(
+            self._state_path(task_id), purpose=f"task state {task_id!r}"
+        ):
+            return self._commit_task_aggregate(
+                task_id, runs, handoff, expected_revision
+            )
+
+    def _commit_task_aggregate(
+        self,
+        task_id: str,
+        runs: tuple[TaskRunAggregate, ...],
+        handoff: str | None = None,
+        expected_revision: int | None = None,
+    ) -> int:
+        previous, _, revision, ledger = self._read_task_document(task_id)
+        if revision and expected_revision is None:
+            raise StateError("existing task aggregate requires expected_revision")
+        if expected_revision is not None and revision != expected_revision:
+            raise StateError(
+                "task aggregate revision conflict: "
+                f"expected {expected_revision}, got {revision}"
+            )
+        validate_task_runs(task_id, runs)
+        previous_ids = {run.run_id for run in previous}
+        previous_status = {run.run_id: run.state.status for run in previous}
+        for run in runs:
+            changed = run.run_id not in previous_ids or (
+                previous_status.get(run.run_id) != run.state.status
+                and not (
+                    previous_status.get(run.run_id) == "in_progress"
+                    and run.state.status == "pending"
+                )
+            )
+            if changed:
+                if (
+                    ledger.get(run.run_id)
+                    and ledger[run.run_id][-1].get("status") == run.state.status
+                ):
+                    continue
+                ledger.setdefault(run.run_id, []).append(
+                    {
+                        "workflow": run.workflow,
+                        "status": run.state.status,
+                        "summary": dict(run.state.workflow_values).get("summary")
+                        if run.state.status == "completed"
+                        else None,
+                    }
+                )
+        ledger = self._validate_ledger(ledger)
+        try:
+            payload = encode_task_document(task_id, runs, handoff, revision + 1, ledger)
+            # Validate the exact compact representation before publishing it.
+            decoded = decode_task_document(payload, task_id)
+        except ValueError as error:
+            raise StateError(
+                f"task {task_id!r} aggregate cannot be encoded: {error}"
+            ) from error
+        validate_task_runs(task_id, decoded[0])
+        if not self._metadata_path(task_id).exists():
+            self._write_task_metadata_payload(TaskMetadata(task_id))
+        self.locks.atomic_write(
+            self._state_path(task_id), json.dumps(payload, indent=2) + "\n"
+        )
+        return revision + 1
+
+    def write_command_output(self, address: CommandOutputAddress, content: str) -> str:
+        path = self._run_path(address.task_id, address.run_id).joinpath(
+            "command-output", *address.segments()
+        )
+        self.locks.atomic_write(path, content)
+        return str(path.relative_to(self.root))
+
+    def read_command_output(self, reference: str) -> str:
+        return self._read_task_file(reference, "command output")
+
+    def _read_task_file(self, reference: str, kind: str) -> str:
+        path = (self.root / reference).resolve()
+        task_root = self.tasks_path.resolve()
+        if not path.is_relative_to(task_root):
+            raise StateError(f"invalid {kind} reference: {reference!r}")
+        try:
+            return path.read_text(encoding="utf-8")
+        except OSError as error:
+            raise StateError(f"cannot read {kind} {reference!r}: {error}") from error
+
+    def task_ids(self) -> tuple[str, ...]:
+        if not self.tasks_path.is_dir():
+            return ()
+        return tuple(
+            sorted(
+                path.name
+                for path in self.tasks_path.iterdir()
+                if path.is_dir()
+                and not path.is_symlink()
+                and not path.name.startswith(BOOTSTRAP_REQUEST_PREFIX)
+            )
+        )
+
+    def task_exists(self, task_id: str) -> bool:
+        # Any task directory claims the ID, even one holding only artifacts or
+        # a child task, so a generated ID never adopts a partially written task.
+        return (self.tasks_path / task_id).exists()
+
+    def read_task_metadata(self, task_id: str) -> TaskMetadata | None:
+        path = self._metadata_path(task_id)
+        if path.exists():
+            return self._read_task_metadata_path(path, task_id)
+        return None
+
+    def _read_task_metadata_path(self, path: Path, task_id: str) -> TaskMetadata:
+        try:
+            raw = json.loads(path.read_text(encoding="utf-8"))
+            if not isinstance(raw, dict) or raw.get("task_id") != task_id:
+                raise ValueError("task metadata has a mismatched task ID")
+            values = flatten_metadata(raw.get("metadata", {}), "task metadata")
+            return TaskMetadata(task_id, values)
+        except (OSError, json.JSONDecodeError, ValueError) as error:
+            raise StateError(f"invalid task metadata {path}: {error}") from error
+
+    def write_task_metadata(self, metadata: TaskMetadata) -> None:
+        self._write_task_metadata_payload(metadata)
+
+    def read_shared_items(self, task_id: str) -> tuple[WorkItem, ...]:
+        path = self._shared_items_path(task_id)
+        if not path.exists():
+            return ()
+        try:
+            raw = json.loads(path.read_text(encoding="utf-8"))
+            if not isinstance(raw, dict) or raw.get("task_id") != task_id:
+                raise ValueError("shared items have a mismatched task ID")
+            entries = raw.get("items", [])
+            if not isinstance(entries, list):
+                raise ValueError("shared items must be a list")
+            return tuple(WorkItem.from_dict(entry) for entry in entries)
+        except (OSError, ValueError) as error:
+            raise StateError(
+                f"cannot read shared items of {task_id!r}: {error}"
+            ) from error
+
+    def write_shared_items(self, task_id: str, items: tuple[WorkItem, ...]) -> None:
+        payload = {"task_id": task_id, "items": [item.to_dict() for item in items]}
+        self.locks.atomic_write(
+            self._shared_items_path(task_id), json.dumps(payload, indent=2) + "\n"
+        )
+
+    def read_amendments(self, task_id: str) -> tuple[Amendment, ...]:
+        path = self._amendments_path(task_id)
+        if not path.exists():
+            return ()
+        try:
+            raw = json.loads(path.read_text(encoding="utf-8"))
+            if not isinstance(raw, dict) or raw.get("task_id") != task_id:
+                raise ValueError("amendments have a mismatched task ID")
+            entries = raw.get("amendments", [])
+            if not isinstance(entries, list):
+                raise ValueError("amendments must be a list")
+            return tuple(Amendment.from_dict(entry) for entry in entries)
+        except (OSError, ValueError) as error:
+            raise StateError(
+                f"cannot read amendments of {task_id!r}: {error}"
+            ) from error
+
+    def append_amendment(self, task_id: str, amendment: Amendment) -> None:
+        amendments = (*self.read_amendments(task_id), amendment)
+        payload = {
+            "task_id": task_id,
+            "amendments": [entry.to_dict() for entry in amendments],
+        }
+        self.locks.atomic_write(
+            self._amendments_path(task_id), json.dumps(payload, indent=2) + "\n"
+        )
+
+    def _write_task_metadata_payload(self, metadata: TaskMetadata) -> None:
+        payload: dict[str, object] = {"task_id": metadata.task_id}
+        if metadata.values:
+            payload["metadata"] = metadata.to_dict()
+        self.locks.atomic_write(
+            self._metadata_path(metadata.task_id),
+            json.dumps(payload, indent=2) + "\n",
+        )
+
+    def remove_task(self, task_id: str) -> bool:
+        path = self.tasks_path / task_id
+        if not path.exists():
+            return False
+        if path.is_symlink() or not path.is_dir():
+            raise StateError(f"task path is not a removable directory: {path}")
+        # Do not recursively remove nested task directories.  Service-level
+        # reset rejects them; direct storage-adapter callers still get exact ownership.
+        owned_paths = (
+            self._state_path(task_id),
+            self._metadata_path(task_id),
+            self._shared_items_path(task_id),
+            self._amendments_path(task_id),
+        )
+        for owned in owned_paths:
+            if owned.exists() and not owned.is_symlink() and owned.is_file():
+                owned.unlink()
+        runs = path / "runs"
+        if runs.exists() and not runs.is_symlink() and runs.is_dir():
+            shutil.rmtree(runs)
+        # A child task or unrecognized file may remain; it stays intact.
+        with contextlib.suppress(OSError):
+            path.rmdir()
+        return True
+
+    def child_task_ids(self, task_id: str) -> tuple[str, ...]:
+        path = self.tasks_path / task_id
+        if not path.is_dir() or path.is_symlink():
+            return ()
+        descendants = []
+        for state in path.rglob("state.json"):
+            relative = state.parent.relative_to(self.tasks_path)
+            candidate = relative.as_posix()
+            if candidate != task_id:
+                descendants.append(candidate)
+        return tuple(sorted(set(descendants)))
+
+    def write_execution_artifact(self, address: ArtifactAddress, content: str) -> str:
+        path = self._run_path(address.task_id, address.run_namespace).joinpath(
+            "steps", *address.segments()
+        )
+        self.locks.atomic_write(path, content)
+        return str(path.relative_to(self.root))
+
+    def _run_path(self, task_id: str, run_id: str) -> Path:
+        return self.tasks_path / task_id / "runs" / run_id
+
+    def read_execution_artifact(self, reference: str) -> str:
+        return self._read_task_file(reference, "execution artifact")
+
+    def _state_path(self, task_id: str) -> Path:
+        return self.tasks_path / task_id / "state.json"
+
+    def _metadata_path(self, task_id: str) -> Path:
+        return self.tasks_path / task_id / "metadata.json"
+
+    def _amendments_path(self, task_id: str) -> Path:
+        return self.tasks_path / task_id / "amendments.json"
+
+    def _shared_items_path(self, task_id: str) -> Path:
+        return self.tasks_path / task_id / "items.json"
+
+    @staticmethod
+    def _validate_ledger(value: object) -> dict[str, list[dict[str, object]]]:
+        if not isinstance(value, dict):
+            raise StateError("task aggregate ledger must be a mapping")
+        if any(
+            not isinstance(run_id, str)
+            or not isinstance(events, list)
+            or not events
+            or any(
+                not isinstance(event, dict)
+                or not isinstance(event.get("workflow"), str)
+                or not isinstance(event.get("status"), str)
+                or event.get("summary") is not None
+                and not isinstance(event.get("summary"), str)
+                for event in events
+            )
+            for run_id, events in value.items()
+        ):
+            raise StateError("task aggregate ledger entries are invalid")
+        return value

ww/storage_adapters/memory.py

@@ -0,0 +1,195 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""In-memory task persistence for tests and embedded callers."""
+
+from __future__ import annotations
+
+import threading
+from collections.abc import Iterator
+from contextlib import ExitStack, contextmanager
+from datetime import datetime, timezone
+
+from ww.amendments import Amendment
+from ww.errors import StateError
+from ww.execution_models import TaskRunAggregate, validate_task_runs
+from ww.items import WorkItem
+from ww.storage_adapters.base import (
+    ArtifactAddress,
+    CommandOutputAddress,
+    TaskMetadata,
+    TaskStorageAdapter,
+)
+
+
+class MemoryTaskStorageAdapter(TaskStorageAdapter):
+    """Store task aggregates, metadata, and artifacts in dictionaries.
+
+    This storage adapter has no filesystem side effects. Artifact keys use stable
+    ``memory://`` references, mirroring the run-aware addressing contract of
+    the filesystem storage adapter.
+    """
+
+    def __init__(self) -> None:
+        self.artifacts: dict[str, str] = {}
+        self._artifact_owners: dict[str, str] = {}
+        self.metadata: dict[str, TaskMetadata] = {}
+        self.shared_items: dict[str, tuple[WorkItem, ...]] = {}
+        self.amendments: dict[str, tuple[Amendment, ...]] = {}
+        self.aggregates: dict[str, tuple[tuple[TaskRunAggregate, ...], str | None]] = {}
+        self.aggregate_revisions: dict[str, int] = {}
+        # When each task's runs were last committed.
+        self.written_at: dict[str, datetime] = {}
+        self._locks: dict[str, threading.RLock] = {}
+        self._locks_guard = threading.Lock()
+
+    @contextmanager
+    def lock_task(self, task_id: str) -> Iterator[None]:
+        """Match the filesystem storage adapter's whole-command task lock in memory."""
+        # A parent reset must exclude concurrent writes to any child task.
+        task_ids = tuple(
+            part
+            for part in (task_id.rsplit("/", 1)[0] if "/" in task_id else None, task_id)
+            if part is not None
+        )
+        with self._locks_guard:
+            locks = tuple(
+                self._locks.setdefault(value, threading.RLock()) for value in task_ids
+            )
+        with ExitStack() as stack:
+            for lock in locks:
+                stack.enter_context(lock)
+            yield
+
+    def read_task_record(
+        self, task_id: str
+    ) -> tuple[tuple[TaskRunAggregate, ...], str | None, int]:
+        aggregate = self.aggregates.get(task_id)
+        if aggregate is None:
+            return (), None, 0
+        runs, handoff = aggregate
+        try:
+            validate_task_runs(task_id, runs)
+        except (StateError, ValueError) as error:
+            # The same error as the filesystem adapter's, so callers that
+            # tolerate one unreadable task treat both adapters alike.
+            raise StateError(f"invalid task state {task_id}: {error}") from error
+        return runs, handoff, self.aggregate_revisions.get(task_id, 0)
+
+    def commit_task_aggregate(
+        self,
+        task_id: str,
+        runs: tuple[TaskRunAggregate, ...],
+        handoff: str | None = None,
+        expected_revision: int | None = None,
+    ) -> int:
+        revision = self.task_aggregate_revision(task_id)
+        if revision and expected_revision is None:
+            raise StateError("existing task aggregate requires expected_revision")
+        if expected_revision is not None and revision != expected_revision:
+            raise StateError(
+                "task aggregate revision conflict: "
+                f"expected {expected_revision}, got {revision}"
+            )
+        validate_task_runs(task_id, runs)
+        # Match the filesystem boundary: snapshots are encoded and decoded at
+        # commit time, retaining plain action payloads for later inspection if
+        # a registration disappears from the process.
+        try:
+            persisted_runs = tuple(
+                TaskRunAggregate.from_dict(run.to_dict()) for run in runs
+            )
+        except ValueError as error:
+            raise StateError(
+                f"task {task_id!r} aggregate cannot be encoded: {error}"
+            ) from error
+        self.aggregates[task_id] = (persisted_runs, handoff)
+        self.aggregate_revisions[task_id] = revision + 1
+        self.written_at[task_id] = datetime.now(timezone.utc)
+        self.metadata.setdefault(task_id, TaskMetadata(task_id))
+        return revision + 1
+
+    def task_written_at(self, task_id: str) -> datetime | None:
+        return self.written_at.get(task_id)
+
+    def task_ids(self) -> tuple[str, ...]:
+        owners = {*self.aggregates, *self.metadata, *self._artifact_owners.values()}
+        return tuple(sorted({owner.split("/")[0] for owner in owners}))
+
+    def task_exists(self, task_id: str) -> bool:
+        # Stored artifacts also claim the ID; see the port docstring.
+        return super().task_exists(task_id) or task_id in self._artifact_owners.values()
+
+    def read_task_metadata(self, task_id: str) -> TaskMetadata | None:
+        return self.metadata.get(task_id)
+
+    def write_task_metadata(self, metadata: TaskMetadata) -> None:
+        self.metadata[metadata.task_id] = metadata
+
+    def read_shared_items(self, task_id: str) -> tuple[WorkItem, ...]:
+        return self.shared_items.get(task_id, ())
+
+    def write_shared_items(self, task_id: str, items: tuple[WorkItem, ...]) -> None:
+        self.shared_items[task_id] = items
+
+    def read_amendments(self, task_id: str) -> tuple[Amendment, ...]:
+        return self.amendments.get(task_id, ())
+
+    def append_amendment(self, task_id: str, amendment: Amendment) -> None:
+        self.amendments[task_id] = (*self.read_amendments(task_id), amendment)
+
+    def remove_task(self, task_id: str) -> bool:
+        existed = bool(
+            task_id in self.aggregates
+            or task_id in self.metadata
+            or task_id in self._artifact_owners.values()
+        )
+        self.metadata.pop(task_id, None)
+        self.shared_items.pop(task_id, None)
+        self.amendments.pop(task_id, None)
+        self.aggregates.pop(task_id, None)
+        self.aggregate_revisions.pop(task_id, None)
+        self.written_at.pop(task_id, None)
+        for reference, owner in tuple(self._artifact_owners.items()):
+            if owner == task_id:
+                del self.artifacts[reference]
+                del self._artifact_owners[reference]
+        return existed
+
+    def child_task_ids(self, task_id: str) -> tuple[str, ...]:
+        prefix = f"{task_id}/"
+        task_ids = set(self.aggregates) | set(self.metadata)
+        task_ids.update(self._artifact_owners.values())
+        return tuple(
+            sorted(candidate for candidate in task_ids if candidate.startswith(prefix))
+        )
+
+    def write_execution_artifact(self, address: ArtifactAddress, content: str) -> str:
+        reference = self._reference(
+            address.task_id, address.run_namespace, "steps", *address.segments()
+        )
+        self.artifacts[reference] = content
+        self._artifact_owners[reference] = address.task_id
+        return reference
+
+    def read_execution_artifact(self, reference: str) -> str:
+        try:
+            return self.artifacts[reference]
+        except KeyError as error:
+            raise StateError(f"missing execution artifact: {reference}") from error
+
+    def write_command_output(self, address: CommandOutputAddress, content: str) -> str:
+        reference = self._reference(
+            address.task_id, address.run_id, "command-output", *address.segments()
+        )
+        self.artifacts[reference] = content
+        self._artifact_owners[reference] = address.task_id
+        return reference
+
+    @staticmethod
+    def _reference(task_id: str, run_id: str, *segments: str) -> str:
+        return f"memory://{task_id}/runs/{run_id}/" + "/".join(segments)
+
+    def read_command_output(self, reference: str) -> str:
+        try:
+            return self.artifacts[reference]
+        except KeyError as error:
+            raise StateError(f"missing command output: {reference}") from error

ww/storage_adapters/project_metadata.py

@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""Project-scoped metadata storage adapters."""
+
+from __future__ import annotations
+
+import json
+import threading
+from collections.abc import Iterator
+from contextlib import AbstractContextManager, contextmanager
+from pathlib import Path
+
+from ww.errors import StateError
+from ww.locking import FileLocks
+from ww.storage_adapters.base import (
+    ProjectMetadata,
+    ProjectMetadataStorage,
+    flatten_metadata,
+)
+
+
+def _decode_project_metadata(raw: object) -> ProjectMetadata:
+    return ProjectMetadata(flatten_metadata(raw, "project metadata"))
+
+
+class FileProjectMetadataStorageAdapter(ProjectMetadataStorage):
+    """Store project metadata in ``.ww/metadata.json``."""
+
+    def __init__(self, root: Path) -> None:
+        self.path = root / ".ww" / "metadata.json"
+        self.locks = FileLocks(root)
+
+    def lock_project_metadata(self) -> AbstractContextManager[None]:
+        return self.locks.lock(self.path, purpose="project metadata")
+
+    def read_project_metadata(self) -> ProjectMetadata | None:
+        if not self.path.exists():
+            return None
+        try:
+            return _decode_project_metadata(
+                json.loads(self.path.read_text(encoding="utf-8"))
+            )
+        except (OSError, json.JSONDecodeError, ValueError) as error:
+            raise StateError(
+                f"invalid project metadata {self.path}: {error}"
+            ) from error
+
+    def write_project_metadata(self, metadata: ProjectMetadata) -> None:
+        self.locks.atomic_write(
+            self.path, json.dumps(metadata.to_dict(), indent=2) + "\n"
+        )
+
+
+class MemoryProjectMetadataStorageAdapter(ProjectMetadataStorage):
+    """Keep project metadata in memory for tests and embedded callers."""
+
+    def __init__(self) -> None:
+        self.metadata: ProjectMetadata | None = None
+        self._lock = threading.RLock()
+
+    @contextmanager
+    def lock_project_metadata(self) -> Iterator[None]:
+        with self._lock:
+            yield
+
+    def read_project_metadata(self) -> ProjectMetadata | None:
+        return self.metadata
+
+    def write_project_metadata(self, metadata: ProjectMetadata) -> None:
+        self.metadata = metadata