simview 6.2.3__tar.gz → 6.3.1__tar.gz

{simview-6.2.3/simview.egg-info → simview-6.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: simview
-Version: 6.2.3
+Version: 6.3.1
 Summary: THREE.js-based simulation visualization tool
 Author-email: Jan Vlk <vlkjan6@fel.cvut.cz>, Ales Kucera <kuceral4@fel.cvut.cz>, David Korcak <david.korcak@gmail.com>
 License-Expression: BSD-3-Clause

{simview-6.2.3 → simview-6.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "simview"
-version = "6.2.3"
+version = "6.3.1"
 readme = "README.md"
 requires-python = ">=3.12"
 license = "BSD-3-Clause"

{simview-6.2.3 → simview-6.3.1}/simview/__main__.py

@@ -82,8 +82,14 @@ def export_static_viewer(directory: Path) -> None:
 
     package = files("simview")
     directory.mkdir(parents=True, exist_ok=True)
-    shutil.copy(str(package / "templates" / "index.html"), directory / "index.html")
-    shutil.copytree(str(package / "static"), directory / "static", dirs_exist_ok=True)
+    from simview.preload import render_index
+
+    index = package / "templates" / "index.html"
+    static = package / "static"
+    (directory / "index.html").write_text(
+        render_index(str(index), str(package)), encoding="utf-8"
+    )
+    shutil.copytree(str(static), directory / "static", dirs_exist_ok=True)
     logger.info("Static viewer written to %s", directory)
 
 
@@ -297,9 +303,18 @@ def save_merged(
         logger.error("Error: --save-merged requires at least 2 input files to merge.")
         sys.exit(1)
 
+    from simview.columnar import columnarize_states, inline_blob
     from simview.merge import merge_simulation_files
 
     merged = merge_simulation_files(paths, selections)
+    # Pack the merged states the same way SimulationScene.save does: merge
+    # decodes every blob to plain lists, so dumping it directly would write
+    # the fat legacy per-frame layout. Falls back silently when numpy is
+    # unavailable or the states are too irregular (columnarize_states
+    # returns None).
+    columnar_states = columnarize_states(merged["states"], merged["model"], inline_blob)
+    if columnar_states is not None:
+        merged["states"] = columnar_states
     payload = json.dumps(merged).encode("utf-8")
     if out_path.suffix == ".gz":
         payload = gzip.compress(payload, compresslevel=1)
@@ -372,7 +387,12 @@ def _add_host_port_flags(sp: argparse.ArgumentParser) -> None:
         "--host",
         type=str,
         default="127.0.0.1",
-        help="Host/interface for the server to bind to (default: 127.0.0.1).",
+        help=(
+            "Host/interface for the server to bind to (default: 127.0.0.1). "
+            "Note: the server has no authentication, so binding to a "
+            "non-loopback address exposes the scene data to everyone who can "
+            "reach it."
+        ),
     )
     sp.add_argument(
         "--port",

{simview-6.2.3 → simview-6.3.1}/simview/info.py

@@ -1,8 +1,7 @@
 """Structural summary of a scene JSON file, for `simview info` and for coding
 agents inspecting simview-generated data files.
 
-Deliberately dependency-free (stdlib only: json, gzip via
-`simview.utils.read_maybe_gzipped_bytes`, base64) so it works on a base
+Deliberately dependency-free (stdlib only: json, gzip, base64) so it works on a base
 install without the `authoring` extra (torch/numpy) -- see "Lazy imports" in
 docs/dev/architecture.md. Does not import `simview.model`/`simview.server`:
 the former pulls in torch at module scope, the latter fastapi/uvicorn --
@@ -11,6 +10,7 @@ both wrong layering for a lightweight inspection tool. Shares its blob-decoding/
 which are stdlib-only on this read path.
 """
 
+import gzip
 import json
 from pathlib import Path
 from typing import Any
@@ -23,7 +23,7 @@ from simview.columnar import (
     is_blob,
     is_columnar,
 )
-from simview.utils import body_label, cap, human_bytes, read_maybe_gzipped_bytes
+from simview.utils import body_label, cap, human_bytes
 
 _MAX_DETAIL_ITEMS = 20
 _LARGE_FILE_BYTES = 100_000_000
@@ -218,9 +218,12 @@ def _summarize_states(states, model: dict | None, warnings: list[str]) -> dict:
 
     scalars_summary = {}
     for name in scalar_names:
-        missing = [i for i, s in enumerate(states) if name not in s]
+        # One pass per scalar: collect missing-frame indices and check
+        # lengths in the same walk.
+        missing = []
         for i, s in enumerate(states):
             if name not in s:
+                missing.append(i)
                 continue
             value = s[name]
             length = len(value) if isinstance(value, list) else 1
@@ -395,7 +398,10 @@ def summarize_scene(path: str | Path) -> dict:
     path = Path(path)
     raw = path.read_bytes()
     gzipped = raw[:2] == b"\x1f\x8b"
-    data = json.loads(read_maybe_gzipped_bytes(path))
+    # Decompress from the bytes already in hand rather than calling
+    # read_maybe_gzipped_bytes(path), which would read the whole file a
+    # second time -- scene files are often hundreds of MB.
+    data = json.loads(gzip.decompress(raw) if gzipped else raw)
     if not isinstance(data, dict):
         raise ValueError(
             "scene file must contain a JSON object with 'model'/'states' keys"

simview-6.3.1/simview/preload.py

@@ -0,0 +1,69 @@
+"""`<link rel="modulepreload">` tags for the viewer page.
+
+The browser only discovers an ES module's imports after it has fetched and
+parsed the importer, so the ~50-file module graph downloads one dependency
+level at a time. Listing every module in the page head lets them all start at
+once (a ~50 ms round trip each otherwise adds up level by level).
+"""
+
+import json
+import re
+from functools import cache
+from pathlib import Path
+
+# `import ... from "x"`, `export ... from "x"` and side-effect `import "x"`.
+# Dynamic import() is skipped: those modules are lazy on purpose.
+_STATIC_IMPORT = re.compile(
+    r"""(?:^|[;\n])\s*(?:import|export)\b[^;'"]*?(?:\bfrom\s*)?["']([^"']+)["']\s*;?""",
+)
+_IMPORTMAP = re.compile(r'<script type="importmap">(.*?)</script>', re.S)
+PLACEHOLDER = "<!-- @modulepreload -->"
+
+
+def _resolve(
+    spec: str, importer: Path, root: Path, imports: dict[str, str]
+) -> Path | None:
+    """Filesystem path `spec` points at, or None for anything outside `root`."""
+    if spec.startswith("."):
+        target = importer.parent / spec
+    else:
+        # Bare specifier -> import map; "prefix/" entries map whole directories.
+        for key, value in imports.items():
+            if spec == key or (key.endswith("/") and spec.startswith(key)):
+                target = root / (value + spec[len(key) :])
+                break
+        else:
+            return None
+    target = target.resolve()
+    return target if target.is_file() and root in target.parents else None
+
+
+def module_graph(html: str, root: Path, entry: str = "static/js/main.js") -> list[str]:
+    """Page-relative URLs (`./static/...`) of every module reachable from `entry`."""
+    match = _IMPORTMAP.search(html)
+    imports = json.loads(match.group(1))["imports"] if match else {}
+    imports = {k: v.removeprefix("./") for k, v in imports.items()}
+    root = root.resolve()
+    seen: dict[Path, None] = {}
+    todo = [(root / entry).resolve()]
+    while todo:
+        path = todo.pop()
+        if path in seen:
+            continue
+        seen[path] = None
+        for spec in _STATIC_IMPORT.findall(path.read_text(encoding="utf-8")):
+            dep = _resolve(spec, path, root, imports)
+            if dep is not None:
+                todo.append(dep)
+    return ["./" + p.relative_to(root).as_posix() for p in seen]
+
+
+@cache
+def render_index(index_path: str, root: str) -> str:
+    """index.html with its modulepreload placeholder filled in."""
+    html = Path(index_path).read_text(encoding="utf-8")
+    links = "\n    ".join(
+        f'<link rel="modulepreload" href="{url}" />'
+        for url in module_graph(html, Path(root))
+    )
+    return html.replace(PLACEHOLDER, links)

{simview-6.2.3 → simview-6.3.1}/simview/server.py

@@ -7,6 +7,7 @@ import hashlib
 import json
 import logging
 import os
+import re
 import secrets
 import threading
 import time
@@ -20,12 +21,15 @@ import orjson
 import uvicorn
 from fastapi import FastAPI, Request, Response, WebSocket, WebSocketDisconnect
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import FileResponse, JSONResponse
+from fastapi.responses import HTMLResponse, JSONResponse
 from fastapi.staticfiles import StaticFiles
 from pydantic import BaseModel
+from starlette.datastructures import Headers
+from starlette.middleware.gzip import GZipMiddleware
 from starlette.types import Scope
 
 from simview.columnar import columnarize_states, is_columnar, rewrite_blobs
+from simview.preload import render_index
 from simview.utils import dialable_host, find_free_port, read_maybe_gzipped_bytes
 
 logger = logging.getLogger("simview.server")
@@ -36,7 +40,31 @@ STATIC = str(files("simview").joinpath("static"))
 # Local-only viewer: CORS is restricted to localhost/127.0.0.1 on any port so a
 # browser tab open on another local dev server can't be silently allowed, while
 # still letting the bundled UI (served from the same host) talk to the API.
-_ALLOWED_ORIGIN_REGEX = r"^https?://(localhost|127\.0\.0\.1)(:\d+)?$"
+# The host list is shared with the /ws/states origin check (see
+# _LOCAL_ORIGIN_RE below), so the two can't drift apart.
+_LOCAL_HOSTS = r"(localhost|127\.0\.0\.1|\[::1\])"
+_ALLOWED_ORIGIN_REGEX = rf"^https?://{_LOCAL_HOSTS}(:\d+)?$"
+
+# The same host list, compiled, for the /ws/states origin check (see
+# _ws_origin_allowed): CORS does not apply to WebSockets, so the endpoint has
+# to check the browser's Origin header itself.
+_LOCAL_ORIGIN_RE = re.compile(rf"^{_LOCAL_HOSTS}(:\d+)?$")
+# A bare address (IPv4 or IPv6 literal, optionally bracketed) -- what a Host
+# header holds when the browser dialed the server by address rather than by a
+# (rebindable) name. Also used by that check.
+_IP_LITERAL_RE = re.compile(r"^\[?[0-9A-Fa-f:.]+\]?$")
+
+# Hosts run()/ViewerHandle bind to without a warning (see _warn_public_bind).
+_LOOPBACK_HOSTS = frozenset({"127.0.0.1", "localhost", "::1"})
+
+# Hard cap on the declared size of any request body. /batch-names is the only
+# endpoint that accepts one, and a scene's batch names are a handful of short
+# strings, so 1 MiB is orders of magnitude above legitimate use while keeping a
+# rogue POST (or a runaway client) from ballooning memory or the on-disk
+# sidecar.
+_MAX_REQUEST_BODY_BYTES = 1 << 20
+# Per-name cap for POST /batch-names, same reasoning per string.
+_MAX_BATCH_NAME_CHARS = 256
 
 # Subdirectories of simview/static that hold vendored, version-pinned third-party
 # libraries. These never change for a given release, so they get a long-lived,
@@ -101,6 +129,99 @@ def _not_found(what: str) -> JSONResponse:
     return JSONResponse({"message": f"{what} data not available"}, status_code=404)
 
 
+def _warn_public_bind(host: str) -> None:
+    """Warn when the server is bound where other machines can reach it.
+
+    The server has no authentication: anyone who can connect can read the
+    scene data and persist batch names to disk next to the scene file. That is
+    fine on loopback (the default, and all a personal viewer needs) and a
+    deliberate choice on e.g. an HPC login node -- hence a warning, not an
+    error.
+    """
+    if host not in _LOOPBACK_HOSTS:
+        logger.warning(
+            "Serving on %s with NO authentication: anyone who can reach this "
+            "address can read the scene data and write batch names. Only bind "
+            "to a shared interface on networks you trust.",
+            host,
+        )
+
+
+def _host_of(authority: str) -> str:
+    """The host part of a Host-header-style `host[:port]` authority, keeping
+    IPv6 brackets intact ('[::1]:5420' -> '[::1]')."""
+    if authority.startswith("["):
+        return authority.partition("]")[0] + "]"
+    return authority.partition(":")[0]
+
+
+def _ws_origin_allowed(websocket: WebSocket) -> bool:
+    """Whether /ws/states should serve this connection.
+
+    Browsers always send an Origin header on WebSocket connections, and CORS
+    does not apply to them, so without this check any site open in the user's
+    browser could connect to a running live viewer and read every pushed
+    frame. Two origins are allowed:
+
+    * a localhost one (the viewer itself, another local dev server, the
+      Jupyter iframe), matching the CORS list above;
+    * a same-origin one -- the Origin's host:port equals the Host header, i.e.
+      a page this very server served, connecting back to the address it was
+      loaded from. The Host must additionally name an address or 'localhost':
+      a hostname that Origin and Host agree on is the DNS-rebinding signature,
+      while an address the browser actually dialed is by construction this
+      machine. (The trade-off: a live viewer reached by hostname over the LAN
+      is refused -- reach it by address, e.g. http://10.0.0.5:5420.)
+
+    A missing Origin (tests, non-browser tooling) is allowed; browsers never
+    omit it.
+    """
+    origin = websocket.headers.get("origin")
+    if origin is None:
+        return True
+    match = re.match(r"^https?://([^/?#]+)", origin)
+    if not match:
+        return False
+    if _LOCAL_ORIGIN_RE.match(match.group(1)):
+        return True
+    host_header = websocket.headers.get("host", "")
+    if match.group(1) != host_header:
+        return False
+    host = _host_of(host_header)
+    return host == "localhost" or _IP_LITERAL_RE.match(host) is not None
+
+
+class _BodySizeLimitMiddleware:
+    """Rejects requests whose declared body exceeds _MAX_REQUEST_BODY_BYTES.
+
+    Checks Content-Length and answers 413 before FastAPI/uvicorn buffers and
+    parses anything. A chunked request with no Content-Length slips past this
+    -- acceptable for a local single-user tool, where the point is bounding
+    what a casual or runaway client can push, not defeating a deliberate
+    attacker with full protocol control.
+    """
+
+    def __init__(self, app):
+        self.app = app
+
+    async def __call__(self, scope: Scope, receive, send):
+        if scope["type"] == "http":
+            length = Headers(scope=scope).get("content-length")
+            if length is not None and length.isdigit():
+                if int(length) > _MAX_REQUEST_BODY_BYTES:
+                    response = JSONResponse(
+                        {
+                            "message": (
+                                f"Request body exceeds {_MAX_REQUEST_BODY_BYTES} bytes"
+                            )
+                        },
+                        status_code=413,
+                    )
+                    await response(scope, receive, send)
+                    return
+        await self.app(scope, receive, send)
+
+
 class BatchNamesRequest(BaseModel):
     names: list[str]
 
@@ -168,10 +289,15 @@ class SimViewServer:
         # SimViewServer doesn't need a reference to it. Bounded: on a run
         # longer than the cap the oldest frames are forgotten, so a late viewer
         # replays the most recent window instead of the entire history.
-        self.frame_buffer: deque[dict] = deque(maxlen=frame_buffer_size)
+        # Each entry is (absolute frame index, frame): the index rides with
+        # the frame so a catch-up never has to reconstruct it from
+        # `frames_pushed - len(buffer)`, whose two halves are updated by
+        # different threads and could be read mid-update.
+        self.frame_buffer: deque[tuple[int, dict]] = deque(maxlen=frame_buffer_size)
         # Absolute index (in the producer's scene.states) of the next frame to
-        # enter frame_buffer, so a catch-up can tell the viewer where its
-        # window starts -- episode startIndex values are absolute too.
+        # enter frame_buffer. Maintained by push_frame on the producer's thread
+        # and read there only (live.py); catch-up reads the indices stored in
+        # the buffer instead.
         self.frames_pushed = 0
         self._batch_selections = (
             None if batch_selections is None else [str(s) for s in batch_selections]
@@ -213,6 +339,10 @@ class SimViewServer:
             allow_methods=["GET", "POST"],
             allow_headers=["*"],
         )
+        # Body-size cap for the one endpoint with a request body. Added after
+        # CORS above, so it wraps it and rejects oversized payloads before any
+        # middleware or handler runs.
+        self.app.add_middleware(_BodySizeLimitMiddleware)
 
         # Mount static files. StaticFiles adds ETag/Last-Modified headers so
         # unchanged assets (vendored libs, textures, our own JS) are served
@@ -220,8 +350,17 @@ class SimViewServer:
         # libs as immutable and everything else short-lived (max-age=60), so
         # index.html's fixed ./static/... paths never need a cache-busting
         # query param.
+        # Gzipped on the fly (three.js alone is ~1.1 MB raw, ~230 KB gzipped);
+        # applied to /static only -- /blob is float data that barely
+        # compresses and is served with Range support.
         self.app.mount(
-            "/static", CacheControlStaticFiles(directory=STATIC), name="static"
+            "/static",
+            GZipMiddleware(
+                CacheControlStaticFiles(directory=STATIC),
+                minimum_size=1024,
+                compresslevel=6,
+            ),
+            name="static",
         )
 
         # Pre-serialized, gzipped payloads for HTTP serving. The parsed dicts are
@@ -286,9 +425,13 @@ class SimViewServer:
         # write dead "/blob/..." URLs, and a second show() would serve stale
         # blob references). Deep-copying just the model is cheap: the big
         # payloads are immutable blob strings, which deepcopy shares rather
-        # than duplicating. `states` is never mutated, so it isn't copied.
+        # than duplicating. A columnar `states` doc gets the same treatment
+        # below (rewrite_blobs rewrites its inline blobs in place too); the
+        # legacy per-frame list is only ever read.
         if preloaded and model_data is not None:
             model_data = copy.deepcopy(model_data)
+        if preloaded and is_columnar(states_data):
+            states_data = copy.deepcopy(states_data)
 
         # Taken once, at load: the sidecar must describe the batches that were
         # actually loaded, not whatever the file is by the time of a rename.
@@ -323,8 +466,11 @@ class SimViewServer:
         # Random per-load token folded into every blob URL so it's safe to cache
         # them forever: a later server restart serving a different scene on the
         # same port gets a different token, so it can never collide with a
-        # stale cached response for blob id N from a previous load.
-        self._blob_token = secrets.token_hex(4)
+        # stale cached response for blob id N from a previous load. It doubles
+        # as the only thing gating /blob responses (the /model payload that
+        # lists the URLs is CORS-restricted), so it is 128-bit rather than the
+        # few bits a pure cache-buster would need.
+        self._blob_token = secrets.token_hex(16)
 
         def register_blob(raw: bytes) -> str:
             blob_id = len(self.blobs)
@@ -386,7 +532,7 @@ class SimViewServer:
     def setup_routes(self):
         @self.app.get("/")
         async def index():
-            return FileResponse(INDEX_HTML, media_type="text/html")
+            return HTMLResponse(render_index(INDEX_HTML, str(Path(STATIC).parent)))
 
         _gzip_headers = {"Content-Encoding": "gzip"}
 
@@ -461,6 +607,19 @@ class SimViewServer:
                     media_type="application/json",
                     status_code=400,
                 )
+            if any(len(name) > _MAX_BATCH_NAME_CHARS for name in names):
+                return Response(
+                    content=json.dumps(
+                        {
+                            "message": (
+                                "Batch names must each be at most "
+                                f"{_MAX_BATCH_NAME_CHARS} characters"
+                            )
+                        }
+                    ),
+                    media_type="application/json",
+                    status_code=400,
+                )
 
             self.model_data["batchNames"] = names
             self._pack_model()
@@ -490,6 +649,23 @@ class SimViewServer:
             # so a viewer opened mid-run still sees the recent timeline.
             @self.app.websocket("/ws/states")
             async def ws_states(websocket: WebSocket):
+                if not _ws_origin_allowed(websocket):
+                    # CORS doesn't apply to WebSockets, so the endpoint checks
+                    # the browser's Origin itself (see _ws_origin_allowed).
+                    # Rejected during the handshake, before any frame is sent.
+                    # A warning, not info: the hostname case is a formerly
+                    # working setup now refused, and this line is the only
+                    # server-side clue.
+                    logger.warning(
+                        "Rejected /ws/states connection with Origin %r: only "
+                        "localhost and same-origin pages may connect to the "
+                        "live stream. A viewer opened by hostname must be "
+                        "opened by its address instead (e.g. "
+                        "http://10.0.0.5:5420).",
+                        websocket.headers.get("origin"),
+                    )
+                    await websocket.close(code=1008)
+                    return
                 await websocket.accept()
                 try:
                     await self._send_catchup(websocket)
@@ -524,20 +700,22 @@ class SimViewServer:
             if not pending:
                 self.ws_clients.add(websocket)
                 return
-            # Absolute index of buffered[0]: everything pushed minus what the
-            # (bounded) buffer still holds.
-            offset = self.frames_pushed - len(buffered) + replayed
             for start in range(0, len(pending), _CATCHUP_CHUNK_SIZE):
                 chunk = pending[start : start + _CATCHUP_CHUNK_SIZE]
                 await websocket.send_text(
-                    json.dumps({"states": chunk, "frameOffset": offset + start})
+                    json.dumps(
+                        {
+                            "states": [frame for _, frame in chunk],
+                            "frameOffset": pending[start][0],
+                        }
+                    )
                 )
             replayed += len(pending)
 
     def push_frame(self, frame: dict) -> None:
         """Record one new frame for catch-up (live mode). The caller
         broadcasts it separately via `broadcast_frame`."""
-        self.frame_buffer.append(frame)
+        self.frame_buffer.append((self.frames_pushed, frame))
         self.frames_pushed += 1
 
     def set_episodes(self, episodes: list[dict]) -> None:
@@ -599,6 +777,7 @@ class SimViewServer:
         open_browser: bool = False,
     ):
         """Serve on `host:port`, blocking until the server stops."""
+        _warn_public_bind(host)
         logger.info("SimView server running on http://%s:%s", host, port)
         if open_browser:
             import webbrowser
@@ -693,6 +872,7 @@ class ViewerHandle:
         self.port = find_free_port(host, preferred_port)
         self.view = view
         self.iframe_height = iframe_height
+        _warn_public_bind(host)
 
         # Prefer uvicorn's modern sansio websocket implementation when the
         # `websockets` package is available -- the default "auto" still selects