ecosemantic-core 0.1.8__py3-none-any.whl

ecosemantic/backends/filesystem/activity.py

@@ -0,0 +1,103 @@
+"""The activity port over a directory: one JSON Lines file per agent under the systems
+directory."""
+
+from __future__ import annotations
+
+import asyncio
+import uuid
+from datetime import datetime
+from pathlib import Path
+
+from filelock import FileLock
+from pydantic import ValidationError
+
+from ecosemantic.errors import NotFound, Unavailable
+from ecosemantic.ports.types import (
+    AgentSummary,
+    Call,
+    CallDraft,
+    agents_of,
+    checked_agent,
+    selected,
+)
+
+ACTIVITY_DIR = "activity"
+SUFFIX = ".jsonl"
+
+
+class FilesystemActivity:
+    """Recorded calls as one line each, appended to the file of the agent that made them.
+
+    Two processes over one directory may serve agents of one name, so a file is appended to and
+    read under a lock both take.
+    """
+
+    def __init__(self, root: Path) -> None:
+        self.root = root
+
+    def _folder(self) -> Path:
+        return self.root / ACTIVITY_DIR
+
+    def _lock(self, agent: str) -> FileLock:
+        return FileLock(self._folder() / f"{agent}.lock")
+
+    def _append(self, call: Call) -> Call:
+        self._folder().mkdir(parents=True, exist_ok=True)
+        with self._lock(call.agent):
+            with (self._folder() / f"{call.agent}{SUFFIX}").open("a", encoding="utf-8") as handle:
+                handle.write(call.model_dump_json() + "\n")
+        return call
+
+    def _read(self, agent: str | None) -> list[Call]:
+        if not self._folder().is_dir():
+            return []
+        if agent is not None:
+            files = [self._folder() / f"{checked_agent(agent)}{SUFFIX}"]
+        else:
+            files = sorted(self._folder().glob(f"*{SUFFIX}"))
+        calls: list[Call] = []
+        for file in files:
+            if file.is_file():
+                with self._lock(file.name.removesuffix(SUFFIX)):
+                    calls.extend(_parsed(file))
+        return calls
+
+    async def record(self, draft: CallDraft) -> Call:
+        call = Call(id=uuid.uuid4(), **draft.model_dump())
+        return await asyncio.to_thread(self._append, call)
+
+    async def list_calls(
+        self,
+        agent: str | None = None,
+        product_system_id: uuid.UUID | None = None,
+        after: datetime | None = None,
+    ) -> list[Call]:
+        calls = await asyncio.to_thread(self._read, agent)
+        return selected(calls, agent, product_system_id, after)
+
+    async def get_call(self, call_id: uuid.UUID) -> Call:
+        for call in await asyncio.to_thread(self._read, None):
+            if call.id == call_id:
+                return call
+        raise NotFound(f"no call {call_id}")
+
+    async def list_agents(self) -> list[AgentSummary]:
+        return agents_of(await asyncio.to_thread(self._read, None))
+
+
+def _parsed(file: Path) -> list[Call]:
+    """A file's calls, or Unavailable naming the file, the line and the first field this version
+    cannot read in it."""
+    calls: list[Call] = []
+    for number, line in enumerate(file.read_text(encoding="utf-8").splitlines(), start=1):
+        try:
+            calls.append(Call.model_validate_json(line))
+        except ValidationError as error:
+            first = error.errors(include_url=False)[0]
+            field = ".".join(str(part) for part in first["loc"])
+            raise Unavailable(
+                f"{file.name} line {number} cannot be read by this version: "
+                f"{field}: {first['msg']}",
+                hint=f"remove {file} to read the activity again",
+            ) from error
+    return calls

ecosemantic/backends/filesystem/installs.py

@@ -0,0 +1,81 @@
+"""The install port over a directory: one folder per install under the data directory."""
+
+from __future__ import annotations
+
+import asyncio
+import uuid
+from collections.abc import AsyncIterator
+from datetime import datetime
+from pathlib import Path
+
+from filelock import AsyncFileLock
+
+from ecosemantic.backends.filesystem.records import RecordFolder, newest_first
+from ecosemantic.ports.types import Install, InstallDraft, InstallPatch
+
+INSTALLS_DIR = "installs"
+INSTALL_FILE = "install.json"
+UPLOADS_DIR = "uploads"
+
+
+def install_dir(root: Path, install_id: uuid.UUID) -> Path:
+    """The directory of one install under the data directory, whether or not it exists yet."""
+    return root / INSTALLS_DIR / str(install_id)
+
+
+def uploads_dir(root: Path) -> Path:
+    """The directory a package uploaded for an install is written to before that install reads
+    it."""
+    return root / INSTALLS_DIR / UPLOADS_DIR
+
+
+class FilesystemInstalls:
+    """Install records as one JSON file per install, newest first by the time they were posted.
+
+    The install process writes a record from its own process while the server reads it, so a
+    record is replaced whole and every change is read and written under a lock both take.
+    """
+
+    def __init__(self, root: Path) -> None:
+        self.root = root
+        self.records = RecordFolder(root, INSTALLS_DIR, INSTALL_FILE, Install, "install")
+
+    def read_record(self, install_id: uuid.UUID) -> Install:
+        """One install, read synchronously; the install process's callback runs off the event
+        loop."""
+        return self.records.read(install_id)
+
+    def apply_patch(self, install_id: uuid.UUID, patch: InstallPatch) -> Install:
+        """The record with the patch's named fields applied, written synchronously."""
+        return self.records.patch(install_id, patch.applied_to)
+
+    async def post_install(self, draft: InstallDraft) -> Install:
+        def construct(posted_at: datetime) -> Install:
+            return Install(id=uuid.uuid4(), draft=draft, posted_at=posted_at)
+
+        return await asyncio.to_thread(self.records.post, construct)
+
+    async def get_install(self, install_id: uuid.UUID) -> Install:
+        return await asyncio.to_thread(self.read_record, install_id)
+
+    async def list_installs(self) -> list[Install]:
+        return newest_first(await asyncio.to_thread(self.records.all))
+
+    async def cancel_install(self, install_id: uuid.UUID) -> Install:
+        def ask() -> Install:
+            return self.records.patch(
+                install_id, lambda install: install.cancellation().applied_to(install)
+            )
+
+        return await asyncio.to_thread(ask)
+
+    async def update_install(self, install_id: uuid.UUID, patch: InstallPatch) -> Install:
+        return await asyncio.to_thread(self.apply_patch, install_id, patch)
+
+    def slot(self) -> AsyncFileLock:
+        """The workspace's one-install slot, which every process over this directory shares."""
+        return self.records.slot()
+
+    def changes(self) -> AsyncIterator[uuid.UUID]:
+        """Every install any process wrote, from the moment this is called."""
+        return self.records.changed()

ecosemantic/backends/filesystem/records.py

@@ -0,0 +1,183 @@
+"""A record folder: one directory per record under its kind's directory, the record replaced
+whole under a lock both processes take."""
+
+from __future__ import annotations
+
+import asyncio
+import uuid
+from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterable
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Protocol, cast
+
+from filelock import AsyncFileLock, FileLock
+from pydantic import BaseModel, ValidationError
+
+from ecosemantic.backends.filesystem.watching import touched
+from ecosemantic.backends.retry import read_bytes, replace
+from ecosemantic.errors import NotFound, Unavailable
+
+SLOT_FILE = "slot.lock"
+POST_LOCK = "post.lock"
+
+
+class Posted(Protocol):
+    """What a record carries for a listing: its id, and the time it was posted."""
+
+    id: uuid.UUID
+    posted_at: datetime
+
+
+def newest_first[T: Posted](records: Iterable[T]) -> list[T]:
+    """Records newest first, the id breaking a tie between two posted at once."""
+    return sorted(records, key=lambda record: (record.posted_at, str(record.id)), reverse=True)
+
+
+def now() -> datetime:
+    """The current instant in UTC, the clock a record's posted time is measured from."""
+    return datetime.now(UTC)
+
+
+def next_posted_at(now: datetime, newest: datetime | None) -> datetime:
+    """A new record's posted time, newest-first independent of the clock's resolution: the
+    clock, or one microsecond past the store's newest when the clock would tie two."""
+    if newest is None:
+        return now
+    return max(now, newest + timedelta(microseconds=1))
+
+
+class RecordFolder[T: BaseModel]:
+    """The records of one kind, as one JSON file per record in a folder of its own.
+
+    A record is written from another process while this one reads it, so it is replaced whole,
+    never rewritten in place, and every change is read and written under a lock both take.
+    """
+
+    def __init__(self, root: Path, kind: str, record_file: str, model: type[T], noun: str) -> None:
+        self.root = root
+        self.kind = kind
+        self.record_file = record_file
+        self.model = model
+        self.noun = noun
+
+    def folder(self, record_id: uuid.UUID) -> Path:
+        """The directory of one record under its kind's directory, whether or not it exists."""
+        return self.root / self.kind / str(record_id)
+
+    def lock(self, record_id: uuid.UUID) -> FileLock:
+        """The record's file lock, taken by every read-and-write of it, from either process."""
+        folder = self.root / self.kind
+        folder.mkdir(parents=True, exist_ok=True)
+        return FileLock(folder / f"{record_id}.lock")
+
+    def post_lock(self) -> FileLock:
+        """The lock every poster of this kind takes, so two cannot mint the same posted time."""
+        folder = self.root / self.kind
+        folder.mkdir(parents=True, exist_ok=True)
+        return FileLock(folder / POST_LOCK)
+
+    def read(self, record_id: uuid.UUID) -> T:
+        """One record, or NotFound when no folder holds it."""
+        record = self.folder(record_id) / self.record_file
+        if not record.is_file():
+            raise NotFound(f"no {self.noun} {record_id}")
+        return self._parsed(record)
+
+    def write(self, record_id: uuid.UUID, record: T) -> T:
+        """The record written, staged beside its file and moved over it within the bounded
+        retry, so a reader sees the whole of one version or the whole of the other."""
+        folder = self.folder(record_id)
+        folder.mkdir(parents=True, exist_ok=True)
+        staged = folder / f"{self.record_file}.tmp"
+        staged.write_bytes(record.model_dump_json(indent=1).encode("utf-8"))
+        replace(staged, folder / self.record_file)
+        return record
+
+    def patch(self, record_id: uuid.UUID, change: Callable[[T], T]) -> T:
+        """The record with the change applied, read and written under its lock."""
+        with self.lock(record_id):
+            return self.write(record_id, change(self.read(record_id)))
+
+    def post(self, construct: Callable[[datetime], T]) -> T:
+        """A new record posted under the kind's lock, its posted time advanced past the newest
+        stored so two posters cannot mint the same instant."""
+        with self.post_lock():
+            record = construct(self._next_posted_at())
+            return self.write(cast(Posted, record).id, record)
+
+    def _next_posted_at(self) -> datetime:
+        """The posted time of the next record of this kind: the clock, advanced past the newest."""
+        newest = max(
+            (record.posted_at for record in cast(Iterable[Posted], self.all())), default=None
+        )
+        return next_posted_at(now(), newest)
+
+    def all(self) -> list[T]:
+        """Every record of this kind."""
+        folder = self.root / self.kind
+        if not folder.is_dir():
+            return []
+        return [
+            self._parsed(path / self.record_file)
+            for path in folder.iterdir()
+            if (path / self.record_file).is_file()
+        ]
+
+    def changed(self) -> AsyncGenerator[uuid.UUID]:
+        """Every record any process wrote under this kind's directory, from the moment this is
+        called: the watch is armed eagerly, so a write between this call and the first read is
+        announced."""
+        queue: asyncio.Queue[uuid.UUID] = asyncio.Queue()
+        watching = asyncio.create_task(self._watch(touched(self.root / self.kind), queue))
+
+        async def iterate() -> AsyncGenerator[uuid.UUID]:
+            try:
+                while True:
+                    yield await queue.get()
+            finally:
+                watching.cancel()
+
+        return iterate()
+
+    async def _watch(
+        self, batches: AsyncIterator[set[Path]], queue: asyncio.Queue[uuid.UUID]
+    ) -> None:
+        async for batch in batches:
+            found = {record for path in batch if (record := self._record_of(path)) is not None}
+            for record_id in sorted(found, key=str):
+                if (self.folder(record_id) / self.record_file).is_file():
+                    queue.put_nowait(record_id)
+
+    def _record_of(self, path: Path) -> uuid.UUID | None:
+        """The record a written path belongs to, or none for a path that names no record."""
+        folder = self.root / self.kind
+        try:
+            first = path.resolve().relative_to(folder.resolve()).parts[0]
+        except (ValueError, IndexError):
+            return None
+        try:
+            return uuid.UUID(first)
+        except ValueError:
+            return None
+
+    def slot(self) -> AsyncFileLock:
+        """The single slot every process over this directory takes, released when its holder's
+        process ends. Each try is non-blocking on the event loop, so a caller cancelled while it
+        waits holds nothing."""
+        folder = self.root / self.kind
+        folder.mkdir(parents=True, exist_ok=True)
+        return AsyncFileLock(folder / SLOT_FILE, run_in_executor=False)
+
+    def _parsed(self, record: Path) -> T:
+        """The record parsed, or Unavailable naming it and the first field this version cannot
+        read in it."""
+        try:
+            return self.model.model_validate_json(read_bytes(record))
+        except ValidationError as error:
+            first = error.errors(include_url=False)[0]
+            field = ".".join(str(part) for part in first["loc"])
+            raise Unavailable(
+                f"{self.noun} {record.parent.name} cannot be read by this version: "
+                f"{field}: {first['msg']}",
+                hint=f"remove {record.parent} to list and start {self.noun}s again",
+            ) from error

ecosemantic/backends/filesystem/run_folder.py

@@ -0,0 +1,55 @@
+"""Where a run lives on disk: one directory per run under the systems directory, and the
+heartbeat its worker beats there."""
+
+from __future__ import annotations
+
+import time
+import uuid
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from threading import Event, Thread
+
+RUNS_DIR = "runs"
+RECORD_FILE = "run.json"
+LOG_FILE = "worker.log"
+HEARTBEAT_FILE = "heartbeat"
+
+
+def run_dir(root: Path, run_id: uuid.UUID) -> Path:
+    """The directory of one run under a systems directory, whether or not it exists yet."""
+    return root / RUNS_DIR / str(run_id)
+
+
+def beat(folder: Path) -> None:
+    """Mark the worker carrying the run in this folder as alive now, by the heartbeat file's
+    modification time."""
+    (folder / HEARTBEAT_FILE).touch()
+
+
+def heartbeat_age(folder: Path) -> float | None:
+    """Seconds since the worker carrying the run in this folder last beat, or None when it never
+    has."""
+    heartbeat = folder / HEARTBEAT_FILE
+    if not heartbeat.is_file():
+        return None
+    return time.time() - heartbeat.stat().st_mtime
+
+
+@contextmanager
+def beating(folder: Path, every: float) -> Iterator[None]:
+    """Beat this folder's heartbeat every `every` seconds until the block ends, so a step that
+    reports nothing does not look like a process that is gone."""
+    stop = Event()
+
+    def keep_beating() -> None:
+        while not stop.wait(every):
+            beat(folder)
+
+    beater = Thread(target=keep_beating, daemon=True)
+    beater.start()
+    try:
+        yield
+    finally:
+        stop.set()
+        beater.join()

ecosemantic/backends/filesystem/runs.py

@@ -0,0 +1,95 @@
+"""The run port over a directory: one folder per run under the systems directory."""
+
+from __future__ import annotations
+
+import asyncio
+import shutil
+import uuid
+from collections.abc import AsyncIterator
+from datetime import datetime
+from pathlib import Path
+
+from filelock import AsyncFileLock
+
+from ecosemantic.backends.filesystem.records import RecordFolder, newest_first
+from ecosemantic.backends.filesystem.run_folder import RECORD_FILE, RUNS_DIR
+from ecosemantic.ports.types import Run, RunDraft, RunPatch, StoreChange, refuse_open
+
+
+class FilesystemRuns:
+    """Run records as one JSON file per run, newest first by the time they were posted.
+
+    The worker writes a record from its own process while the server reads it, so a record is
+    replaced whole, never rewritten in place, and every change is read and written under a lock
+    both processes take.
+    """
+
+    def __init__(self, root: Path) -> None:
+        self.root = root
+        self.records = RecordFolder(root, RUNS_DIR, RECORD_FILE, Run, "run")
+
+    def read_record(self, run_id: uuid.UUID) -> Run:
+        """One run, read synchronously; the worker's callback runs off the event loop."""
+        return self.records.read(run_id)
+
+    def apply_patch(self, run_id: uuid.UUID, patch: RunPatch) -> Run:
+        """The record with the patch's named fields applied, written synchronously."""
+        with self.records.lock(run_id):
+            return self.records.write(run_id, patch.applied_to(self.read_record(run_id)))
+
+    async def post_run(self, draft: RunDraft) -> Run:
+        def construct(posted_at: datetime) -> Run:
+            return Run(id=uuid.uuid4(), posted_at=posted_at, **draft.model_dump())
+
+        return await asyncio.to_thread(self.records.post, construct)
+
+    async def get_run(self, run_id: uuid.UUID) -> Run:
+        return await asyncio.to_thread(self.read_record, run_id)
+
+    async def list_runs(self, product_system_id: uuid.UUID | None = None) -> list[Run]:
+        runs = newest_first(await asyncio.to_thread(self.records.all))
+        if product_system_id is None:
+            return runs
+        return [run for run in runs if run.product_system_id == product_system_id]
+
+    async def cancel_run(self, run_id: uuid.UUID) -> Run:
+        def ask() -> Run:
+            return self.records.patch(run_id, lambda run: run.cancellation().applied_to(run))
+
+        return await asyncio.to_thread(ask)
+
+    async def update_run(self, run_id: uuid.UUID, patch: RunPatch) -> Run:
+        return await asyncio.to_thread(self.apply_patch, run_id, patch)
+
+    async def delete_runs(self, product_system_id: uuid.UUID) -> None:
+        """Remove every run of a product system, record and folder; Conflict while one is
+        open, since its worker still writes it."""
+
+        def remove() -> None:
+            owned = [r for r in self.records.all() if r.product_system_id == product_system_id]
+            refuse_open(owned)
+            for run in owned:
+                with self.records.lock(run.id):
+                    shutil.rmtree(self.records.folder(run.id))
+
+        await asyncio.to_thread(remove)
+
+    def changes(self) -> AsyncIterator[StoreChange]:
+        """What any process wrote under the runs directory, from the moment this is called: the
+        watch is armed eagerly, so a write between this call and the first read is announced."""
+        found = self.records.changed()
+
+        async def iterate() -> AsyncIterator[StoreChange]:
+            try:
+                async for run_id in found:
+                    run = await asyncio.to_thread(self.read_record, run_id)
+                    yield StoreChange(product_system_id=run.product_system_id, run_id=run.id)
+            finally:
+                await found.aclose()
+
+        return iterate()
+
+    def slot(self) -> AsyncFileLock:
+        """The workspace's one-run slot: a file lock every process over this directory takes,
+        released when its holder's process ends."""
+        return self.records.slot()