pal-code 0.1.0__py3-none-any.whl

pal/storage_manager/components/vector_store_backend.py

@@ -0,0 +1,122 @@
+"""
+Vector store for agent knowledge bases -- backed by chromadb
+(pip install chromadb) when it's installed, one collection per agent so
+different agents' knowledge bases stay isolated. Falls back to a small
+in-memory store, scored via the same embedding/keyword-overlap
+similarity helpers context_manager and memory already use, when
+chromadb isn't available. Functionally equivalent at the scale this
+project targets -- the real difference is chromadb's store is disk-
+backed (survives a restart), the fallback isn't.
+"""
+import logging
+from typing import Optional
+
+from pal.context_manager.utils.relevance import embedding_similarity
+from pal.memory.utils.keyword_overlap import keyword_overlap
+
+log = logging.getLogger(__name__)
+
+try:
+    import chromadb
+    _CHROMADB_AVAILABLE = True
+except ImportError:
+    _CHROMADB_AVAILABLE = False
+
+
+class _NaiveVectorStore:
+    """In-process fallback: one dict of doc_id -> (text, metadata) per
+    agent, scored at query time rather than indexed ahead of time."""
+
+    def __init__(self):
+        self._collections: dict[str, dict[str, tuple[str, dict]]] = {}
+
+    def _collection(self, agent_id: str) -> dict:
+        return self._collections.setdefault(agent_id, {})
+
+    def upsert(self, agent_id: str, doc_id: str, text: str, metadata: dict) -> None:
+        self._collection(agent_id)[doc_id] = (text, metadata)
+
+    def delete(self, agent_id: str, doc_id: str) -> None:
+        self._collection(agent_id).pop(doc_id, None)
+
+    def query(self, agent_id: str, query_text: str, top_k: int) -> list[dict]:
+        collection = self._collection(agent_id)
+        if not collection:
+            return []
+        doc_ids = list(collection.keys())
+        texts = [collection[d][0] for d in doc_ids]
+
+        sims = embedding_similarity(query_text, texts)
+        if all(s == 0.0 for s in sims):
+            sims = [keyword_overlap(query_text, t) for t in texts]
+
+        scored = sorted(zip(doc_ids, sims), key=lambda x: x[1], reverse=True)
+        return [
+            {"id": doc_id, "text": collection[doc_id][0], "metadata": collection[doc_id][1], "score": score}
+            for doc_id, score in scored[:top_k]
+        ]
+
+
+class VectorStoreBackend:
+    def __init__(self, persist_directory: Optional[str] = None):
+        self.backend = "chromadb" if _CHROMADB_AVAILABLE else "naive_fallback"
+        if _CHROMADB_AVAILABLE:
+            self._client = (
+                chromadb.PersistentClient(path=persist_directory)
+                if persist_directory
+                else chromadb.EphemeralClient()
+            )
+            self._collections = {}
+        else:
+            self._naive = _NaiveVectorStore()
+
+    def _chroma_collection(self, agent_id: str):
+        if agent_id not in self._collections:
+            # chromadb collection names have length/charset
+            # restrictions -- prefix and truncate to stay valid while
+            # keeping per-agent namespacing.
+            safe_name = f"agent_{agent_id}".replace(" ", "_")[:63]
+            self._collections[agent_id] = self._client.get_or_create_collection(safe_name)
+        return self._collections[agent_id]
+
+    def delete_agent(self, agent_id: str) -> None:
+        """Drop an agent's whole collection."""
+        if self.backend == "chromadb":
+            self._collections.pop(agent_id, None)
+            try:
+                self._client.delete_collection(f"agent_{agent_id}".replace(" ", "_")[:63])
+            except Exception:  # noqa: BLE001  (collection was never created)
+                log.debug("no collection to delete for %s", agent_id)
+        else:
+            self._naive._collections.pop(agent_id, None)
+
+    def upsert(self, agent_id: str, doc_id: str, text: str, metadata: dict = None) -> None:
+        metadata = metadata or {}
+        if self.backend == "chromadb":
+            self._chroma_collection(agent_id).upsert(
+                ids=[doc_id], documents=[text], metadatas=[metadata] if metadata else None
+            )
+        else:
+            self._naive.upsert(agent_id, doc_id, text, metadata)
+
+    def delete(self, agent_id: str, doc_id: str) -> None:
+        if self.backend == "chromadb":
+            self._chroma_collection(agent_id).delete(ids=[doc_id])
+        else:
+            self._naive.delete(agent_id, doc_id)
+
+    def query(self, agent_id: str, query_text: str, top_k: int = 3) -> list[dict]:
+        if self.backend == "chromadb":
+            collection = self._chroma_collection(agent_id)
+            if collection.count() == 0:
+                return []
+            result = collection.query(query_texts=[query_text], n_results=min(top_k, collection.count()))
+            ids = result.get("ids", [[]])[0]
+            docs = result.get("documents", [[]])[0]
+            metas = result.get("metadatas", [[]])[0] or [{}] * len(ids)
+            dists = result.get("distances", [[]])[0] or [None] * len(ids)
+            return [
+                {"id": doc_id, "text": text, "metadata": meta, "score": (1 - dist) if dist is not None else None}
+                for doc_id, text, meta, dist in zip(ids, docs, metas, dists)
+            ]
+        return self._naive.query(agent_id, query_text, top_k)

pal/storage_manager/pipeline.py

@@ -0,0 +1,153 @@
+"""
+Storage Manager (AIOS section 3.6): persistent storage for agents --
+files/knowledge bases they depend on, and durable storage for the
+memory manager's RAM-to-disk swaps -- backed by local files
+(LocalFileBackend) and a vector database (VectorStoreBackend, chromadb
+when installed).
+
+Two entry points into the same underlying storage, matching the two
+callers section 3.6 describes:
+
+- **Direct calls** (read/write/delete/list_keys/vector_upsert/
+  vector_query) -- what the memory manager uses when it swaps a note to
+  disk, scoped by the agent_id the memory manager passes in.
+- **execute(StorageSyscall)**, reachable either directly or via the
+  queue (submit/tick/drain) -- what an agent's own SDK calls
+  (utils/sdk.py) go through: "the agent calls the storage API in the
+  SDK, which is further converted into storage-related system calls
+  and put into the storage queue... the storage manager then processes
+  the requests in the queue."
+
+Both paths converge on execute()/the same backends -- there's exactly
+one storage manager per process, not a separate implementation per
+caller.
+
+When an AccessManagerPipeline (AIOS section 3.8) is supplied, execute()
+authorizes every request before performing it: a cross-agent request
+(syscall.requesting_agent_id differs from syscall.agent_id, the
+resource owner) is checked against the privilege-group hashmap, and any
+destructive operation -- DELETE, or a WRITE that overwrites an existing
+key -- goes through user intervention regardless of who's making the
+call. Without an access_manager, every request is permitted (the
+pre-3.8 behavior), matching how the direct read/write/etc. calls above
+still bypass access control entirely -- those are for the memory
+manager's own swaps, which are not the cross-agent case 3.8 regulates.
+"""
+from pathlib import Path
+from typing import Any, Optional
+
+from pal.access_manager.components.base import OperationType
+from pal.access_manager.pipeline import AccessManagerPipeline
+from pal.paths import data_dir
+from pal.storage_manager.components.base import StorageOp, StorageResponse, StorageSyscall
+from pal.storage_manager.components.local_file_backend import LocalFileBackend
+from pal.storage_manager.components.storage_queue import StorageSyscallQueue
+from pal.storage_manager.components.vector_store_backend import VectorStoreBackend
+
+
+class StorageManagerPipeline:
+    def __init__(
+        self,
+        root: Optional[str] = None,
+        vector_persist_directory: Optional[str] = None,
+        access_manager: Optional[AccessManagerPipeline] = None,
+    ):
+        root_path = Path(root) if root else data_dir("storage")
+        self.files = LocalFileBackend(root_path / "files")
+        self.vectors = VectorStoreBackend(persist_directory=vector_persist_directory)
+        self.queue = StorageSyscallQueue(self)
+        self.access_manager = access_manager
+
+    # -- direct calls (memory manager swaps, or any in-process caller) --------
+
+    def read(self, agent_id: str, key: str) -> Optional[Any]:
+        return self.files.read(agent_id, key)
+
+    def write(self, agent_id: str, key: str, data: Any) -> None:
+        self.files.write(agent_id, key, data)
+
+    def delete(self, agent_id: str, key: str) -> None:
+        self.files.delete(agent_id, key)
+
+    def exists(self, agent_id: str, key: str) -> bool:
+        return self.files.exists(agent_id, key)
+
+    def list_keys(self, agent_id: str, prefix: str = "") -> list[str]:
+        return self.files.list_keys(agent_id, prefix=prefix)
+
+    def vector_upsert(self, agent_id: str, doc_id: str, text: str, metadata: dict = None) -> None:
+        self.vectors.upsert(agent_id, doc_id, text, metadata or {})
+
+    def vector_query(self, agent_id: str, query_text: str, top_k: int = 3) -> list[dict]:
+        return self.vectors.query(agent_id, query_text, top_k=top_k)
+
+    # -- system-call entry point (agent SDK calls, direct or via queue) -------
+
+    def _operation_type(self, syscall: StorageSyscall) -> OperationType:
+        if syscall.op == StorageOp.DELETE:
+            return OperationType.DELETE
+        if syscall.op in (StorageOp.READ, StorageOp.LIST, StorageOp.VECTOR_QUERY):
+            return OperationType.READ
+        if syscall.op == StorageOp.WRITE:
+            # Overwriting an existing key destroys what was there --
+            # treat it as destructive (user-intervention-gated) the same
+            # as an explicit delete; a write to a fresh key is not.
+            return OperationType.OVERWRITE if self.exists(syscall.agent_id, syscall.key) else OperationType.WRITE
+        return OperationType.WRITE  # VECTOR_UPSERT
+
+    def execute(self, syscall: StorageSyscall) -> StorageResponse:
+        if self.access_manager is not None:
+            requesting = syscall.requesting_agent_id or syscall.agent_id
+            decision = self.access_manager.authorize(
+                requesting_agent_id=requesting,
+                target_agent_id=syscall.agent_id,
+                operation=self._operation_type(syscall),
+                resource=syscall.key or syscall.query_text or "",
+            )
+            if not decision.allowed:
+                return StorageResponse(ok=False, error=f"access denied: {decision.reason}")
+
+        try:
+            if syscall.op == StorageOp.READ:
+                return StorageResponse(ok=True, data=self.read(syscall.agent_id, syscall.key))
+            if syscall.op == StorageOp.WRITE:
+                self.write(syscall.agent_id, syscall.key, syscall.data)
+                return StorageResponse(ok=True)
+            if syscall.op == StorageOp.DELETE:
+                self.delete(syscall.agent_id, syscall.key)
+                return StorageResponse(ok=True)
+            if syscall.op == StorageOp.LIST:
+                return StorageResponse(ok=True, data=self.list_keys(syscall.agent_id, prefix=syscall.key or ""))
+            if syscall.op == StorageOp.VECTOR_UPSERT:
+                self.vector_upsert(syscall.agent_id, syscall.key, syscall.data, syscall.metadata)
+                return StorageResponse(ok=True)
+            if syscall.op == StorageOp.VECTOR_QUERY:
+                return StorageResponse(
+                    ok=True,
+                    data=self.vector_query(syscall.agent_id, syscall.query_text, syscall.top_k),
+                )
+            return StorageResponse(ok=False, error=f"unknown storage op: {syscall.op}")
+        except Exception as e:
+            return StorageResponse(ok=False, error=str(e))
+
+    # -- queue-driven path -- "put into the storage queue by the scheduler" --
+
+    def submit(self, syscall: StorageSyscall) -> str:
+        return self.queue.submit(syscall)
+
+    def tick(self):
+        return self.queue.process_next()
+
+    def drain(self, max_iterations: int = 10_000):
+        return self.queue.drain(max_iterations=max_iterations)
+
+    def get_result(self, request_id: str):
+        return self.queue.get_result(request_id)
+
+    def forget(self, request_id: str) -> None:
+        self.queue.forget(request_id)
+
+    def purge_agent(self, agent_id: str) -> None:
+        """Delete every file and vector stored for an agent (used when a chat session ends)."""
+        self.files.purge(agent_id)
+        self.vectors.delete_agent(agent_id)

pal/storage_manager/utils/__init__.py

@@ -0,0 +1,4 @@
+from pal.storage_manager.utils.memory_adapter import AgentScopedMemoryStorage
+from pal.storage_manager.utils.sdk import StorageSDK
+
+__all__ = ["AgentScopedMemoryStorage", "StorageSDK"]

pal/storage_manager/utils/memory_adapter.py

@@ -0,0 +1,60 @@
+"""
+Adapter presenting the same narrow interface
+pal.memory.components.storage_manager.StorageManager exposes
+(write/read/delete/exists/list_ids/read_all, all keyed by MemoryNote.id)
+but backed by the general-purpose StorageManagerPipeline, scoped to one
+agent_id.
+
+This is the actual integration point for "when the agent's memory
+usage exceeds the allocated limit, the memory manager calls the storage
+manager to swap the data into the disk... the storage manager reads and
+writes data based on the agent ID passed from the memory manager":
+MemoryManager doesn't know or care which storage implementation sits
+behind this interface, only that it has one -- so it works identically
+whether it's given this adapter (backed by the shared, agent-scoped
+StorageManagerPipeline) or the standalone pal.memory.components.
+storage_manager.StorageManager (a private, unscoped JSON-per-note
+store, still the default when no agent_id/storage_manager is supplied).
+"""
+from typing import Optional
+
+from pal.memory.components.note import MemoryNote
+from pal.storage_manager.pipeline import StorageManagerPipeline
+
+_MEMORY_PREFIX = "memory/"
+
+
+class AgentScopedMemoryStorage:
+    def __init__(self, storage_manager: StorageManagerPipeline, agent_id: str):
+        self.storage_manager = storage_manager
+        self.agent_id = agent_id
+
+    def _key(self, note_id: str) -> str:
+        return f"{_MEMORY_PREFIX}{note_id}.json"
+
+    def write(self, note: MemoryNote) -> None:
+        self.storage_manager.write(self.agent_id, self._key(note.id), note.to_dict())
+
+    def read(self, note_id: str) -> Optional[MemoryNote]:
+        data = self.storage_manager.read(self.agent_id, self._key(note_id))
+        if data is None:
+            return None
+        return MemoryNote.from_dict(data)
+
+    def delete(self, note_id: str) -> None:
+        self.storage_manager.delete(self.agent_id, self._key(note_id))
+
+    def exists(self, note_id: str) -> bool:
+        return self.storage_manager.exists(self.agent_id, self._key(note_id))
+
+    def list_ids(self) -> list[str]:
+        keys = self.storage_manager.list_keys(self.agent_id, prefix=_MEMORY_PREFIX)
+        return [k[len(_MEMORY_PREFIX):-len(".json")] for k in keys if k.endswith(".json")]
+
+    def read_all(self) -> list[MemoryNote]:
+        notes = []
+        for note_id in self.list_ids():
+            note = self.read(note_id)
+            if note is not None:
+                notes.append(note)
+        return notes

pal/storage_manager/utils/sdk.py

@@ -0,0 +1,93 @@
+"""
+Storage SDK: what an agent calls to read/write its own persistent data
+during runtime -- distinct from the memory manager's own direct calls
+into the same storage manager (see utils/memory_adapter.py).
+
+"The agent calls the storage API in the SDK, which is further
+converted into storage-related system calls and put into the storage
+queue by the scheduler. The storage manager then processes the
+requests in the queue." read()/write()/etc. here build a StorageSyscall
+and submit it to the shared StorageManagerPipeline's queue, then drain
+that queue until this specific request completes -- so the call still
+looks synchronous to the agent even though it went through the same
+centralized queue every other storage request goes through, the same
+way Pipeline._dispatch_chat drains the LLM syscall scheduler for its
+own request in orchestrator/components/syscall_scheduler.py.
+
+read_from()/write_from()/delete_from() are the cross-agent path AIOS
+3.8's Access Manager regulates: they set requesting_agent_id to this
+SDK's own agent_id while targeting a DIFFERENT agent's resource, so if
+the StorageManagerPipeline has an access_manager wired in, that request
+gets checked against the privilege-group hashmap (and, for delete/
+overwrite, gated behind user intervention) before it's allowed to
+proceed. Plain read()/write()/delete() never trigger a cross-agent
+check -- they always act on this SDK's own agent_id as both requester
+and target.
+"""
+from typing import Any, Optional
+
+from pal.storage_manager.components.base import StorageOp, StorageSyscall
+from pal.storage_manager.pipeline import StorageManagerPipeline
+
+
+class StorageSDK:
+    def __init__(self, agent_id: str, storage_manager: StorageManagerPipeline):
+        self.agent_id = agent_id
+        self.storage_manager = storage_manager
+
+    def _submit_and_wait(self, syscall: StorageSyscall):
+        request_id = self.storage_manager.submit(syscall)
+        try:
+            req = self.storage_manager.get_result(request_id)
+            while req.status.value == "queued":
+                self.storage_manager.tick()
+                req = self.storage_manager.get_result(request_id)
+            if req.status.value == "error":
+                error = req.response.error if req.response else "unknown error"
+                raise RuntimeError(f"storage syscall failed: {error}")
+            return req.response
+        finally:
+            self.storage_manager.forget(request_id)
+
+    def read(self, key: str) -> Optional[Any]:
+        resp = self._submit_and_wait(StorageSyscall(agent_id=self.agent_id, op=StorageOp.READ, key=key))
+        return resp.data
+
+    def write(self, key: str, data: Any) -> None:
+        self._submit_and_wait(StorageSyscall(agent_id=self.agent_id, op=StorageOp.WRITE, key=key, data=data))
+
+    def delete(self, key: str) -> None:
+        self._submit_and_wait(StorageSyscall(agent_id=self.agent_id, op=StorageOp.DELETE, key=key))
+
+    def list_keys(self, prefix: str = "") -> list[str]:
+        resp = self._submit_and_wait(StorageSyscall(agent_id=self.agent_id, op=StorageOp.LIST, key=prefix))
+        return resp.data
+
+    def vector_upsert(self, doc_id: str, text: str, metadata: dict = None) -> None:
+        self._submit_and_wait(StorageSyscall(
+            agent_id=self.agent_id, op=StorageOp.VECTOR_UPSERT, key=doc_id, data=text, metadata=metadata or {},
+        ))
+
+    def vector_query(self, query_text: str, top_k: int = 3) -> list[dict]:
+        resp = self._submit_and_wait(StorageSyscall(
+            agent_id=self.agent_id, op=StorageOp.VECTOR_QUERY, query_text=query_text, top_k=top_k,
+        ))
+        return resp.data
+
+    # -- cross-agent access, subject to the Access Manager (AIOS 3.8) ---------
+
+    def read_from(self, target_agent_id: str, key: str) -> Optional[Any]:
+        resp = self._submit_and_wait(StorageSyscall(
+            agent_id=target_agent_id, requesting_agent_id=self.agent_id, op=StorageOp.READ, key=key,
+        ))
+        return resp.data
+
+    def write_to(self, target_agent_id: str, key: str, data: Any) -> None:
+        self._submit_and_wait(StorageSyscall(
+            agent_id=target_agent_id, requesting_agent_id=self.agent_id, op=StorageOp.WRITE, key=key, data=data,
+        ))
+
+    def delete_from(self, target_agent_id: str, key: str) -> None:
+        self._submit_and_wait(StorageSyscall(
+            agent_id=target_agent_id, requesting_agent_id=self.agent_id, op=StorageOp.DELETE, key=key,
+        ))

pal/studio/builder.py

@@ -0,0 +1,166 @@
+"""One build: the plan's steps, one after another, each a request to the model, each reply read for files, checked, and saved under the policy.
+
+Everything that happens is reported as an event (`emit(kind, text, **data)`), so the page can show the work as it happens. Nothing the model wrote is
+ever run: files are read, checked for mistakes, and written, if the policy allows.
+"""
+from __future__ import annotations
+
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from pal.studio.catalog import ROLES
+from pal.studio.checks import check_project
+from pal.studio.policy import Policy
+from pal.terminal.files import parse_files
+
+FORMAT = ("Reply with the files you create or change. Write each file as a line `### FILE: path/name.ext` followed by its COMPLETE content in a code block. "
+          "No partial files, no '...' placeholders. After the files you may add one line starting with NOTE: if something needs saying.")
+KIND_RULES = {
+    "web_app": "It is a web page: plain HTML, CSS and JavaScript that works when index.html is opened in a browser. No build step, no frameworks to install, no external URLs or CDNs. Start with index.html.",
+    "cli_tool": "It is a command-line program in Python 3 using only the standard library. Start with main.py.",
+    "script": "It is a Python 3 script using only the standard library. Start with main.py.",
+    "library": "It is a small Python 3 library using only the standard library, with a clear public API.",
+    "document": "It is a document: write it as Markdown files.",
+    "other": "Use plain files that need nothing to be installed.",
+}
+MAX_REPAIRS = 1
+
+
+def default_name(plan: dict, step: dict) -> str:
+    if step["role"] == "docs":
+        return "README.md"
+    if step["role"] == "tester":
+        return "test_main.py"
+    return {"web_app": "index.html", "cli_tool": "main.py", "script": "main.py", "library": "main.py", "document": "README.md"}.get(plan["product"]["kind"], "main.txt")
+
+
+def first_message(plan: dict, step: dict, with_references: bool) -> str:
+    team = ", ".join(ROLES[r].name for r in plan["roles"])
+    p = plan["product"]
+    return "\n".join([
+        f"We are building: {p['name']}. {p['summary']}",
+        KIND_RULES.get(p["kind"], KIND_RULES["other"]),
+        f"The team: {team}. Each step is done by one of them, and every file must work with the files already written (same names, same functions).",
+        "The reference files the person attached are available to you." if with_references else "",
+        FORMAT, "",
+        f"Step 1 ({ROLES[step['role']].name}): {step['task']}", ROLES[step["role"]].prompt])
+
+
+def step_message(step: dict, i: int, n: int) -> str:
+    return (f"Step {i} of {n} ({ROLES[step['role']].name}): {step['task']}\n{ROLES[step['role']].prompt}\n"
+            "Keep file names and functions consistent with the files already written. " + FORMAT)
+
+
+class Build:
+    def __init__(self, plan: dict, client, workspace: Path | None, policy: Policy, emit: Callable, cancelled: Callable[[], bool] = lambda: False,
+                 references: list[dict] | None = None):
+        self.plan, self.client, self.workspace, self.policy, self.emit, self.cancelled = plan, client, workspace, policy, emit, cancelled
+        self.references = references if plan["permissions"].get("read_references") else []
+        self.files: dict[str, str] = {}                 # everything the build wrote (or would have written, in a dry run)
+        self.totals: dict[str, Any] = {"calls": 0, "prompt_tokens": 0, "completion_tokens": 0, "main_tokens": 0, "helper_tokens": 0, "seconds": 0.0,
+                       "files": 0, "blocked": 0, "problems_left": 0, "steps_done": 0, "steps": len(plan["steps"]), "stopped": ""}
+
+    # -- one request to the model ----------------------------------------------------------------------------------------
+    def _ask(self, message: str, label: str, refs: list[dict] | None = None) -> str:
+        reply = self.client.ask(message, refs)
+        t = self.totals
+        t["calls"] += max(reply.calls, 1)
+        t["prompt_tokens"] += reply.prompt_tokens
+        t["completion_tokens"] += reply.completion_tokens
+        t["main_tokens"] += reply.prompt_tokens + reply.completion_tokens
+        t["helper_tokens"] += reply.helper_tokens
+        t["seconds"] = round(t["seconds"] + reply.seconds, 3)
+        self.emit("model_call", f"{label}: the model read {reply.prompt_tokens:,} tokens and wrote {reply.completion_tokens:,} in {reply.seconds:.1f}s"
+                  + (f" (Pal's helper used {reply.helper_tokens:,} more)" if reply.helper_tokens else ""),
+                  prompt_tokens=reply.prompt_tokens, completion_tokens=reply.completion_tokens, helper_tokens=reply.helper_tokens, seconds=round(reply.seconds, 2))
+        return reply.text
+
+    # -- reading and saving the files in a reply -----------------------------------------------------------------------
+    def _apply(self, reply: str, step: dict) -> set[str]:
+        parsed, unnamed = parse_files(reply, default_name(self.plan, step))
+        changed: set[str] = set()
+        if unnamed:
+            self.emit("warning", f"{unnamed} code block(s) had no file name and were skipped")
+        for pf in parsed:
+            d = self.policy.check_write(pf.path, pf.content)
+            if d.allowed:
+                existed = d.path.lower() in {p.lower() for p in self.files}
+                self.files[d.path] = pf.content
+                if self.workspace is not None:
+                    target = self.workspace / d.path
+                    target.parent.mkdir(parents=True, exist_ok=True)
+                    target.write_text(pf.content, encoding="utf-8", newline="\n")
+                changed.add(d.path)
+                self.totals["files"] = len(self.files)
+                self.emit("file", f"{'changed' if existed else 'wrote'} {d.path} ({len(pf.content.encode('utf-8')):,} bytes)", path=d.path, action="changed" if existed else "created")
+            elif d.path and "dry run" in d.reason:
+                self.files[d.path] = pf.content
+                changed.add(d.path)
+                self.totals["files"] = len(self.files)
+                self.emit("file", f"would write {d.path} (dry run: not saved)", path=d.path, action="dry")
+            else:
+                self.totals["blocked"] += 1
+                self.emit("blocked", f"did not save {d.path or 'a file'}: {d.reason}", path=d.path, reason=d.reason)
+        return changed
+
+    @staticmethod
+    def _describe(problems: dict[str, list[str]]) -> str:
+        return "\n".join(f"- {path}: {'; '.join(p)}" for path, p in problems.items())
+
+    # -- the build ---------------------------------------------------------------------------------------------------------
+    def run(self) -> dict:
+        steps, n = self.plan["steps"], len(self.plan["steps"])
+        for i, step in enumerate(steps, 1):
+            if self.cancelled():
+                self.totals["stopped"] = "cancelled"
+                self.emit("cancelled", "stopped by you")
+                return self.totals
+            role = ROLES[step["role"]]
+            self.emit("step_start", f"Step {i} of {n}: {role.name}: {step['task']}", step=i, total=n, role=step["role"])
+            first = i == 1
+            msg = first_message(self.plan, step, bool(self.references)) if first else step_message(step, i, n)
+            try:
+                reply = self._ask(msg, f"step {i}", self.references if first else None)
+            except Exception as e:  # noqa: BLE001  (the model failed: say so, and stop cleanly)
+                self.totals["stopped"] = "error"
+                self.emit("error", f"the model failed on step {i}: {type(e).__name__}: {str(e)[:200]}")
+                return self.totals
+            changed = self._apply(reply, step)
+            problems = check_project(self.files, only=changed, links=False)
+            repairs = 0
+            while problems and repairs < MAX_REPAIRS and not self.cancelled():
+                self.emit("check", "found problems: " + "; ".join(f"{p}: {', '.join(v)}" for p, v in list(problems.items())[:3]), problems=problems)
+                repairs += 1
+                try:
+                    fix = self._ask("These problems were found in the files:\n" + self._describe(problems) + "\nReply with the corrected COMPLETE files only. " + FORMAT, f"step {i} fix")
+                except Exception as e:  # noqa: BLE001
+                    self.emit("error", f"the fix request failed: {type(e).__name__}")
+                    break
+                changed |= self._apply(fix, step)
+                problems = check_project(self.files, only=changed, links=False)
+            if problems:
+                self.emit("check", "problems still there after one fix: " + "; ".join(f"{p}: {', '.join(v)}" for p, v in list(problems.items())[:3]), problems=problems)
+            else:
+                self.emit("check", "no problems found in the files")
+            self.totals["steps_done"] = i
+            self.emit("step_done", f"Step {i} done: {len(self.files)} file(s) so far", step=i)
+        self._final_check(steps[-1] if steps else None)
+        return self.totals
+
+    def _final_check(self, step: dict | None) -> None:
+        """When every step is done: do the files fit together (does every page link to files that exist)? One fix is allowed, then the result is recorded."""
+        if step is None or self.cancelled() or self.totals["stopped"]:
+            return
+        problems = check_project(self.files)
+        if problems:
+            self.emit("check", "project check, found: " + "; ".join(f"{p}: {', '.join(v)}" for p, v in list(problems.items())[:3]), problems=problems)
+            try:
+                fix = self._ask("These problems were found in the finished project:\n" + self._describe(problems) + "\nReply with the missing or corrected COMPLETE files only. " + FORMAT, "final fix")
+                self._apply(fix, step)
+            except Exception as e:  # noqa: BLE001
+                self.emit("error", f"the final fix request failed: {type(e).__name__}")
+            problems = check_project(self.files)
+        self.totals["problems_left"] = sum(len(v) for v in problems.values())
+        self.emit("check", "project check: the files fit together" if not problems else f"project check: {self.totals['problems_left']} problem(s) left: "
+                  + "; ".join(f"{p}: {', '.join(v)}" for p, v in list(problems.items())[:3]), problems=problems)