simscope 0.1.1__py3-none-any.whl

simscope/server/__init__.py

@@ -0,0 +1,9 @@
+"""The simscope viewer server: Starlette over a library directory.
+
+``create_app(root, author=...)`` builds the ASGI app and ``serve`` runs it
+with uvicorn. The routes read the library folder on disk.
+"""
+
+from simscope.server.app import create_app, serve
+
+__all__ = ["create_app", "serve"]

simscope/server/app.py

@@ -0,0 +1,149 @@
+"""The Starlette app and the ``serve`` entry point (contracts 2)."""
+
+import contextlib
+import logging
+import pathlib
+import secrets
+from collections.abc import AsyncIterator, Sequence
+
+import uvicorn
+from starlette.applications import Starlette
+from starlette.middleware import Middleware
+from starlette.requests import Request
+from starlette.responses import Response
+from starlette.routing import Route
+
+from simscope.server import blocks, jobs, routes, security
+from simscope.server import state as state_mod
+
+logger = logging.getLogger(__name__)
+
+
+def create_app(
+    root: pathlib.Path | str,
+    *,
+    author: str | None = None,
+    token: str | None = None,
+    allowed_hosts: Sequence[str] | None = None,
+    watch: bool = True,
+    warm: bool = True,
+) -> Starlette:
+    """Builds the app that serves one library to the browser.
+
+    The library is scanned before this returns. The file watcher and the
+    highlight warmer start with the app's lifespan (``uvicorn`` runs it; a
+    test client runs it inside ``with``).
+
+    Args:
+        root: The library folder.
+        author: Author of annotations made through the API; the OS user name
+            if ``None``.
+        token: The token that writes must send in ``X-Simscope-Token``; a
+            random one if ``None``. The page's boot block carries it.
+        allowed_hosts: ``Host`` header values to accept (DNS-rebinding
+            guard); loopback names by default.
+        watch: Watch the folder for changes. Turn off to drive
+            ``app.state.services.state.rescan()`` by hand.
+        warm: Compute highlights of every run in the background.
+
+    Returns:
+        The ASGI app. Its services are on ``app.state.services``.
+
+    Raises:
+        FileNotFoundError: If ``root`` is not a directory.
+    """
+    path = pathlib.Path(root)
+    if not path.is_dir():
+        raise FileNotFoundError(f"no library folder at {path}")
+    state = state_mod.LibraryState(
+        path, author=author, token=token or secrets.token_urlsafe(24)
+    )
+    services = routes.Services(state, jobs.Jobs(state), blocks.BlockStore())
+
+    @contextlib.asynccontextmanager
+    async def lifespan(_app: Starlette) -> AsyncIterator[None]:
+        state.start(watch=watch)
+        if warm:
+            services.jobs.start_warmer()
+        try:
+            yield
+        finally:
+            services.jobs.stop()
+            services.blocks.close()
+            state.stop()
+
+    async def not_found(_request: Request, _exc: Exception) -> Response:
+        return routes.error(404, "not found")
+
+    async def not_allowed(_request: Request, _exc: Exception) -> Response:
+        return routes.error(405, "method not allowed")
+
+    app = Starlette(
+        routes=[
+            Route("/", routes.index),
+            Route("/assets/{name}", routes.asset),
+            Route("/api/library", routes.library_info),
+            Route("/api/runs", routes.runs),
+            Route("/api/changes", routes.changes),
+            Route("/api/blk", routes.blk),
+            Route("/api/blocks", routes.blocks_route),
+            Route("/api/export", routes.export_route),
+            Route("/api/groups", routes.groups, methods=["GET"]),
+            Route("/api/groups", routes.change_groups, methods=["POST"]),
+            Route(
+                "/api/runs/{name}/annotations",
+                routes.annotate,
+                methods=["POST"],
+            ),
+            Route(
+                "/api/runs/{name}/rename",
+                routes.rename_run,
+                methods=["POST"],
+            ),
+            Route("/files/{path:path}", routes.files),
+        ],
+        middleware=[
+            Middleware(
+                security.HostGuard,
+                allowed=allowed_hosts or security.LOOPBACK_HOSTS,
+            ),
+        ],
+        exception_handlers={404: not_found, 405: not_allowed},
+        lifespan=lifespan,
+    )
+    app.state.services = services
+    return app
+
+
+def serve(
+    root: pathlib.Path | str,
+    host: str = "127.0.0.1",
+    port: int = 8080,
+    author: str | None = None,
+    *,
+    token: str | None = None,
+) -> None:
+    """Serves a library until interrupted.
+
+    Binds ``host`` (loopback by default). A loopback bind answers only
+    loopback ``Host`` headers, which stops DNS rebinding; a wildcard bind
+    answers any.
+
+    Args:
+        root: The library folder.
+        host: Address to listen on.
+        port: Port to listen on (0 picks a free one).
+        author: Author of annotations made through the API.
+        token: The write token; random if ``None``.
+    """
+    app = create_app(
+        root,
+        author=author,
+        token=token,
+        allowed_hosts=security.allowed_hosts_for(host),
+    )
+    config = uvicorn.Config(
+        app, host=host, port=port, log_level="warning", access_log=False
+    )
+    server = uvicorn.Server(config)
+    server.run()

simscope/server/blocks.py

@@ -0,0 +1,191 @@
+"""Raw block access for ``/api/blk`` and ``/api/blocks`` (contracts 2).
+
+The server never decodes a block. A :class:`simscope.io.blockfile.BlockReader`
+supplies the header and directory (and follows a file that is still being
+written, scanning only the bytes added since the last request); the block
+bytes themselves are sliced out of the file with ``pread``.
+"""
+
+import collections
+import os
+import pathlib
+import threading
+from collections.abc import Sequence
+
+import numpy as np
+
+from simscope.io import blockfile, cas, errors
+
+MAX_ENVS = 256
+"""Most envs one ``/api/blocks`` request may ask for."""
+_MAX_OPEN = 64
+_BLOCK_HEAD = 32
+
+Signature = tuple[int, int, int]
+"""``(inode, size, mtime_ns)`` of a block file."""
+
+
+def signature(path: pathlib.Path) -> Signature:
+    """Returns ``(inode, size, mtime_ns)``, raising if the file is missing."""
+    st = os.stat(path)
+    return st.st_ino, st.st_size, st.st_mtime_ns
+
+
+class _Entry:
+    """An open reader and the file signature it last saw."""
+
+    def __init__(self, reader: blockfile.BlockReader, sig: Signature) -> None:
+        self.reader = reader
+        self.sig = sig
+        self.lock = threading.Lock()
+
+
+class BlockStore:
+    """Keeps block files open so a window request costs one ``stat``."""
+
+    def __init__(self) -> None:
+        """Creates an empty store."""
+        self._entries: collections.OrderedDict[str, _Entry] = (
+            collections.OrderedDict()
+        )
+        self._lock = threading.Lock()
+
+    def reader(self, path: pathlib.Path) -> tuple[blockfile.BlockReader, str]:
+        """Returns an up-to-date reader of a block file.
+
+        A finished file is opened once and reopened if the file is replaced.
+        An unfinished one is refreshed in place, which costs time
+        proportional to what the recorder appended.
+
+        Args:
+            path: The block file.
+
+        Returns:
+            ``(reader, etag)``. The etag changes whenever the readable
+            content does.
+
+        Raises:
+            FileNotFoundError: If the file is missing or has no header yet.
+            errors.FormatError: If the file is corrupt.
+        """
+        key = str(path)
+        sig = signature(path)
+        if sig[1] < blockfile.HEADER_SIZE:
+            # The recorder has created the file but not written its header.
+            raise FileNotFoundError(path)
+        with self._lock:
+            entry = self._entries.get(key)
+            if entry is not None:
+                self._entries.move_to_end(key)
+        if entry is None or (
+            entry.sig != sig
+            and (entry.reader.finished or sig[0] != entry.sig[0])
+        ):
+            entry = self._open(key, path, sig)
+        elif entry.sig != sig:
+            with entry.lock:
+                try:
+                    entry.reader.refresh()
+                except errors.FormatError:
+                    entry = self._open(key, path, sig)
+                else:
+                    entry.sig = sig
+        reader = entry.reader
+        etag = f'W/"{sig[0]:x}-{sig[1]:x}-{sig[2]:x}-{reader.n_blocks}"'
+        return reader, etag
+
+    def _open(self, key: str, path: pathlib.Path, sig: Signature) -> _Entry:
+        """Opens (or reopens) a file and stores the entry."""
+        reader = blockfile.BlockReader(
+            path, cache_blocks=0, verify=False, partial=True
+        )
+        entry = _Entry(reader, sig)
+        with self._lock:
+            old = self._entries.pop(key, None)
+            self._entries[key] = entry
+            while len(self._entries) > _MAX_OPEN:
+                self._entries.popitem(last=False)
+        if old is not None and old.reader is not reader:
+            old.reader.close()
+        return entry
+
+    def close(self) -> None:
+        """Closes every open file."""
+        with self._lock:
+            entries, self._entries = (
+                list(self._entries.values()),
+                (collections.OrderedDict()),
+            )
+        for entry in entries:
+            entry.reader.close()
+
+
+def index_bytes(reader: blockfile.BlockReader) -> bytes:
+    """Builds the body of ``/api/blk``: the header, then the directory.
+
+    The reply is shaped like a finished block file that has had its blocks
+    cut out: a valid 64-byte header (``dir_offset`` is 64, so the directory
+    follows at once) and ``n_blocks`` 32-byte directory entries whose offsets
+    still point into the real file. For a file that is still being written
+    the counts cover complete windows only. ``fmt.parseBlk`` reads it as is.
+
+    Args:
+        reader: An up-to-date reader.
+
+    Returns:
+        The bytes.
+    """
+    d = reader.directory
+    n_blocks = min(reader.n_blocks, len(d))
+    head = blockfile.Header(
+        reader.item_shape,
+        reader.n_envs,
+        reader.n_frames,
+        reader.block_frames,
+        n_blocks,
+        blockfile.HEADER_SIZE,
+        32 * n_blocks,
+    )
+    return head.pack() + d[:n_blocks].tobytes()
+
+
+def read_blocks(
+    reader: blockfile.BlockReader,
+    path: pathlib.Path,
+    window: int,
+    envs: Sequence[int],
+) -> list[bytes] | None:
+    """Slices the raw ``SSBB`` blocks of one window out of the file.
+
+    Args:
+        reader: An up-to-date reader of ``path``.
+        path: The block file.
+        window: Window index (frames ``window * block_frames`` onward).
+        envs: Env indices, in the order wanted.
+
+    Returns:
+        One ``bytes`` per env (the 32-byte block header plus payload), or
+        ``None`` if the window is not complete yet.
+
+    Raises:
+        IndexError: If an env is out of range.
+    """
+    n_envs = reader.n_envs
+    d = reader.directory
+    n_blocks = min(reader.n_blocks, len(d))
+    if window < 0 or (window + 1) * n_envs > n_blocks:
+        return None
+    ids = np.asarray(envs, np.int64)
+    if ids.size and (ids.min() < 0 or ids.max() >= n_envs):
+        raise IndexError("env out of range")
+    rows = d[window * n_envs + ids]
+    offsets = rows["offset"].tolist()
+    lengths = (rows["clen"].astype(np.int64) + _BLOCK_HEAD).tolist()
+    fd = cas.open_read(path)
+    try:
+        return [
+            cas.read_at(fd, n, off)
+            for off, n in zip(offsets, lengths, strict=True)
+        ]
+    finally:
+        os.close(fd)

simscope/server/jobs.py

@@ -0,0 +1,166 @@
+"""Background computation of derived files (contracts 2 and 5).
+
+A request for ``derived/<run>/...`` never blocks: it is answered at once from
+the cache, or a job is queued on a small pool and the request gets "pending"
+until the file exists. Jobs for one file are never queued twice, and a job
+that failed is remembered (for that manifest) so a broken run is reported
+instead of retried on every poll.
+"""
+
+import concurrent.futures
+import enum
+import logging
+import pathlib
+import threading
+
+from simscope import derived
+from simscope.io import errors
+from simscope.server import state as state_mod
+
+logger = logging.getLogger(__name__)
+
+WORKERS = 2
+"""Derived passes that may run at once."""
+
+
+class Status(enum.Enum):
+    """Outcome of asking for a derived file."""
+
+    READY = "ready"
+    PENDING = "pending"
+    MISSING = "missing"  # no such run
+    NOT_APPLICABLE = "not_applicable"
+    FAILED = "failed"
+
+
+class Jobs:
+    """Runs derived-data passes on a thread pool."""
+
+    def __init__(self, state: state_mod.LibraryState, workers: int = WORKERS):
+        """Creates the pool.
+
+        Args:
+            state: The library state, told when highlights are counted.
+            workers: Pool size (at most 2 is sensible: a pass is already
+                multi-threaded inside).
+        """
+        self._state = state
+        self._pool = concurrent.futures.ThreadPoolExecutor(
+            min(workers, WORKERS), thread_name_prefix="simscope-derived"
+        )
+        self._pending: dict[
+            tuple[str, str, str], concurrent.futures.Future
+        ] = {}
+        self._failed: dict[tuple[str, str, str], str] = {}
+        self._lock = threading.Lock()
+        self._stop = threading.Event()
+        self._warmer: threading.Thread | None = None
+
+    def request(
+        self, run: str, what: str
+    ) -> tuple[Status, pathlib.Path | str | None]:
+        """Returns a derived file if it is ready, else queues its job.
+
+        Args:
+            run: Run name.
+            what: Path relative to ``derived/<run>/``.
+
+        Returns:
+            ``(status, value)``: the file path for ``READY``, an error
+            message for ``FAILED``, otherwise ``None``.
+        """
+        try:
+            rollout = self._state.lib.open(run)
+        except FileNotFoundError:
+            return Status.MISSING, None
+        except errors.FormatError as exc:
+            return Status.FAILED, str(exc)
+        try:
+            if not derived.applicable(rollout, what):
+                return Status.NOT_APPLICABLE, None
+            cache = derived.cache_dir(self._state.root, rollout)
+            path = derived.fresh(rollout, cache, what)
+            if path is not None:
+                return Status.READY, path
+            key = (rollout.manifest.id, what, derived.manifest_digest(rollout))
+        finally:
+            rollout.close()
+        with self._lock:
+            if key in self._failed:
+                return Status.FAILED, self._failed[key]
+            if key not in self._pending:
+                future = self._pool.submit(self._run, run, what, key)
+                self._pending[key] = future
+        return Status.PENDING, None
+
+    def _run(self, run: str, what: str, key: tuple[str, str, str]) -> None:
+        """Computes one derived file (worker thread)."""
+        try:
+            self.compute(run, what)
+        except FileNotFoundError:
+            logger.info("run %s is gone (renamed or deleted)", run)
+        except Exception as exc:
+            logger.exception("deriving %s for %s failed", what, run)
+            with self._lock:
+                self._failed[key] = f"{type(exc).__name__}: {exc}"
+        finally:
+            with self._lock:
+                self._pending.pop(key, None)
+
+    def compute(self, run: str, what: str) -> pathlib.Path | None:
+        """Computes a derived file now, in the calling thread.
+
+        Args:
+            run: Run name.
+            what: Path relative to ``derived/<run>/``.
+
+        Returns:
+            The file path, or ``None`` if not applicable.
+        """
+        rollout = self._state.lib.open(run)
+        try:
+            cache = derived.cache_dir(self._state.root, rollout)
+            path = derived.ensure(rollout, cache, what)
+            if what == derived.HIGHLIGHTS and path is not None:
+                count = derived.highlight_count(cache)
+                if count is not None:
+                    self._state.set_highlights(run, count)
+            return path
+        finally:
+            rollout.close()
+
+    def start_warmer(self) -> None:
+        """Computes highlights of every complete run in the background.
+
+        Newest first, one run at a time, so the library's highlight counts
+        fill in without competing with what the user is looking at.
+        """
+        if self._warmer is None:
+            self._warmer = threading.Thread(
+                target=self._warm, name="simscope-warm", daemon=True
+            )
+            self._warmer.start()
+
+    def _warm(self) -> None:
+        """Worker for :meth:`start_warmer`."""
+        names = [
+            name
+            for name, info in self._state.snapshot()
+            if info.status == "complete"
+        ]
+        for name in names:
+            if self._stop.is_set():
+                return
+            try:
+                self.compute(name, derived.HIGHLIGHTS)
+            except FileNotFoundError:
+                continue  # renamed or deleted since the list was made
+            except Exception:
+                logger.exception("highlights for %s failed", name)
+
+    def stop(self) -> None:
+        """Stops the warmer and waits for running passes."""
+        self._stop.set()
+        if self._warmer is not None:
+            self._warmer.join(timeout=2)
+        self._pool.shutdown(wait=False, cancel_futures=True)