visin 0.1.0__py3-none-any.whl

visin/__init__.py

@@ -0,0 +1,42 @@
+"""Report training runs to a Visin instance, and read them back.
+
+    import visin
+
+    with visin.init("unet baseline", project="road-seg") as run:
+        for epoch in range(1, epochs + 1):
+            train_loss, val_loss, val_iou = train_one_epoch()
+            run.log_epoch(epoch, train={"loss": train_loss}, val={"loss": val_loss, "mean_iou": val_iou})
+
+``init`` reads ``VISIN_URL`` and ``VISIN_TOKEN``. With neither set it returns a
+run that reports nothing, so the same script still runs on a laptop.
+"""
+
+from ._internal.config import Settings, read_settings
+from ._internal.console import enable_console_logging
+from ._version import __version__
+from .api import Api, flatten
+from .errors import ApiError, ConfigurationError, TransportError, VisinError
+from .offline import SyncResult, pending, sync
+from .run import Run, epoch_uuid_for, init
+from .system import system_info, system_metrics
+
+__all__ = [
+    "init",
+    "Run",
+    "Api",
+    "sync",
+    "pending",
+    "SyncResult",
+    "epoch_uuid_for",
+    "flatten",
+    "system_info",
+    "system_metrics",
+    "enable_console_logging",
+    "Settings",
+    "read_settings",
+    "VisinError",
+    "ApiError",
+    "TransportError",
+    "ConfigurationError",
+    "__version__",
+]

visin/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

visin/_internal/__init__.py

@@ -0,0 +1,9 @@
+"""How the package works: not part of its public API, and free to change in any release.
+
+config      settings from the environment and arguments
+transport   HTTP: retries that know which requests may repeat, errors, signed uploads
+serialize   NumPy scalars, tensors and NaN into JSON Visin accepts
+reports     each kind of report as data, and how it is delivered
+sender      the background thread reports are sent from, in order
+spool       reports kept on disk, and sending them later
+"""

visin/_internal/config.py

@@ -0,0 +1,112 @@
+"""Where configuration comes from.
+
+There is no default API URL, on purpose. The integrations this package replaces
+each hard-coded one instance's hostname, so a copy taken into a new project
+reported into somebody else's Visin until the line was noticed. Unset means
+disabled, which is loud in the log and harmless, rather than wrong and silent.
+"""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import Any
+
+from ..errors import ConfigurationError
+
+# The first name of each is the one Visin's documentation uses. The others are
+# what earlier drafts of this package read, and scripts already export them.
+ENV_URL = ("VISIN_URL", "VISIN_API_URL")
+ENV_TOKEN = ("VISIN_TOKEN", "VISIN_API_TOKEN")
+ENV_TRAINING_UUID = ("VISIN_TRAINING_UUID",)
+ENV_PROJECT = ("VISIN_PROJECT", "VISIN_PROJECT_ID")
+ENV_VERIFY = "VISIN_VERIFY_SSL"
+ENV_MODE = "VISIN_MODE"
+ENV_DIR = "VISIN_DIR"
+
+MODES = ("online", "offline", "disabled")
+
+
+def _first(names: tuple[str, ...]) -> str | None:
+    for name in names:
+        value = os.getenv(name)
+        if value and value.strip():
+            return value.strip()
+    return None
+
+
+def _flag(name: str, default: bool) -> bool:
+    raw = os.getenv(name)
+    if raw is None:
+        return default
+    return raw.strip().lower() not in {"0", "false", "no", "off"}
+
+
+def default_directory() -> Path:
+    """Where reports wait when they cannot be sent: ``~/.visin``.
+
+    Home rather than the working directory because on a cluster the login node,
+    where ``visin sync`` runs, shares home with the compute nodes and usually
+    shares nothing else.
+    """
+    try:
+        return Path.home() / ".visin"
+    except RuntimeError:  # no HOME, as in some minimal containers
+        return Path(tempfile.gettempdir()) / "visin"
+
+
+@dataclass(frozen=True)
+class Settings:
+    url: str | None
+    token: str | None
+    training_uuid: str | None
+    project: str | None
+    verify_ssl: bool
+    mode: str
+    directory: Path
+
+    @property
+    def configured(self) -> bool:
+        """Enough to talk to a server at all.
+
+        The training UUID is not part of this: a run that registers itself has
+        none yet, and that is the normal case when a script is launched by hand.
+        """
+        return bool(self.url and self.token)
+
+    @property
+    def effective_mode(self) -> str:
+        """What a run will actually do: online without a server is disabled."""
+        if self.mode == "online" and not self.configured:
+            return "disabled"
+        return self.mode
+
+
+def read_settings(**overrides: Any) -> Settings:
+    """Read the environment. Keyword arguments that are not ``None`` win over it."""
+    directory = os.getenv(ENV_DIR)
+    settings = Settings(
+        url=_first(ENV_URL),
+        token=_first(ENV_TOKEN),
+        training_uuid=_first(ENV_TRAINING_UUID),
+        project=_first(ENV_PROJECT),
+        # Verification stays on unless someone deliberately turns it off for a
+        # self-signed dev instance. The scripts this replaces disabled it
+        # globally and silenced the warning, which also disabled it in
+        # production.
+        verify_ssl=_flag(ENV_VERIFY, True),
+        mode=(os.getenv(ENV_MODE) or "online").strip().lower(),
+        directory=Path(directory).expanduser() if directory else default_directory(),
+    )
+    given = {key: value for key, value in overrides.items() if value is not None}
+    if "directory" in given:
+        given["directory"] = Path(given["directory"]).expanduser()
+    if "mode" in given:
+        given["mode"] = str(given["mode"]).strip().lower()
+    if given:
+        settings = replace(settings, **given)
+    if settings.mode not in MODES:
+        raise ConfigurationError(f"{ENV_MODE}={settings.mode!r}; expected one of: {', '.join(MODES)}")
+    return settings

visin/_internal/console.py

@@ -0,0 +1,31 @@
+"""Showing visin's log lines on the console."""
+
+from __future__ import annotations
+
+import logging
+import sys
+from typing import TextIO
+
+_MARK = "_visin_console"
+
+
+def enable_console_logging(level: int | str = logging.INFO, stream: TextIO | None = None) -> None:
+    """Print visin's log lines: the run it created or resumed, anything it could
+    not send, and its summary at ``finish``.
+
+    visin logs through :mod:`logging` under the ``visin`` logger and, as a
+    library should, adds no handler of its own, so without this only warnings
+    reach the console. Pass ``stream=sys.stdout`` to keep the lines in order
+    with a script's own ``print`` output. Calling it again replaces the handler
+    rather than printing every line twice.
+    """
+    logger = logging.getLogger("visin")
+    for handler in list(logger.handlers):
+        if getattr(handler, _MARK, False):
+            logger.removeHandler(handler)
+    handler = logging.StreamHandler(stream or sys.stderr)
+    handler.setFormatter(logging.Formatter("%(message)s"))
+    setattr(handler, _MARK, True)
+    logger.addHandler(handler)
+    logger.setLevel(level)
+    logger.propagate = False

visin/_internal/reports.py

@@ -0,0 +1,187 @@
+"""What a run writes, as data, and how each kind of write is delivered.
+
+Every write a run makes is first an *op*: a small JSON-able dict naming its
+kind and carrying its request body. A live run delivers ops as it goes; an
+offline run, or a live one that lost the server, writes them to disk and
+``visin sync`` delivers them later through this same function. One code path
+means an op replayed a week later lands exactly as it would have at the time.
+
+Delivery is made safe to repeat wherever the API allows it. A run, an epoch and
+a test result carry a caller-generated UUID, so a repeat is answered 409 and
+counted as delivered. Benchmarks and configs carry none, so those are the only
+ops a lost answer can duplicate.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+from urllib.parse import quote
+
+from ..errors import ApiError, ConfigurationError
+from .transport import HttpClient
+
+logger = logging.getLogger("visin")
+
+KINDS = frozenset({"create_run", "epoch", "test_result", "benchmark", "config", "visualization", "update"})
+
+
+@dataclass
+class DeliveryContext:
+    """What delivering one run's ops needs to remember between ops."""
+
+    files: Path | None = None
+    training_ids: dict[str, str] = field(default_factory=dict)
+    # Attempts per request, when a caller cannot afford the client's full
+    # budget: a run's blocking creation, or a catch-up that will be retried
+    # in a minute anyway. None means the client's own.
+    retries: int | None = None
+    # Runs a create_run found already registered: resumed, not new.
+    existing: set[str] = field(default_factory=set)
+    # Epochs the server answered 409 for: recorded before this delivery.
+    repeated: set[str] = field(default_factory=set)
+
+
+def training_id(client: HttpClient, context: DeliveryContext, training_uuid: str) -> str:
+    """The database id of a run, which ``PUT /trainings/:id`` needs."""
+    known = context.training_ids.get(training_uuid)
+    if known:
+        return known
+    found = client.request("GET", f"/trainings/uuid/{quote(training_uuid, safe='')}", retries=context.retries)
+    ident = (found or {}).get("_id")
+    if not ident:
+        raise ApiError(f"no training with uuid {training_uuid}", status=404)
+    context.training_ids[training_uuid] = ident
+    return str(ident)
+
+
+def _already_there(exc: ApiError) -> bool:
+    return exc.status == 409
+
+
+def deliver(client: HttpClient, op: dict[str, Any], context: DeliveryContext) -> Any:
+    kind = op.get("op")
+    body = op.get("body") or {}
+    if kind == "create_run":
+        return _create_run(client, body, context)
+    if kind == "epoch":
+        return _post_once(client, "/epochs/upload", body, context)
+    if kind == "test_result":
+        return _post_once(client, "/test-results/upload", body, context)
+    if kind == "benchmark":
+        return client.request("POST", "/benchmarks/upload", json=body, retries=context.retries)
+    if kind == "config":
+        return _config(client, op, body, context)
+    if kind == "visualization":
+        return _visualization(client, op, body, context)
+    if kind == "update":
+        ident = training_id(client, context, op["training_uuid"])
+        return client.request("PUT", f"/trainings/{ident}", json=body, retries=context.retries)
+    raise ValueError(f"unknown report kind {kind!r}; was it written by a newer visin?")
+
+
+def _post_once(client: HttpClient, path: str, body: dict[str, Any], context: DeliveryContext) -> Any:
+    try:
+        return client.request("POST", path, json=body, idempotent=True, retries=context.retries)
+    except ApiError as exc:
+        # The server refuses a repeat of a UUID it already has. That is exactly
+        # the outcome a retry should produce, so it counts as delivered.
+        if _already_there(exc):
+            logger.debug("visin: %s already recorded", path)
+            if body.get("epoch_uuid") and path == "/epochs/upload":
+                context.repeated.add(str(body["epoch_uuid"]))
+            return None
+        raise
+
+
+def _create_run(client: HttpClient, body: dict[str, Any], context: DeliveryContext) -> Any:
+    uuid = body["uuid"]
+    try:
+        training = client.request("POST", "/trainings", json=body, idempotent=True, retries=context.retries)
+    except ApiError as exc:
+        if not _already_there(exc):
+            raise
+        # Created by an earlier attempt, or by whatever launched this process.
+        context.existing.add(uuid)
+        training = client.request("GET", f"/trainings/uuid/{quote(uuid, safe='')}", retries=context.retries)
+    ident = (training or {}).get("_id")
+    if ident:
+        context.training_ids[uuid] = str(ident)
+    return training
+
+
+def _config(client: HttpClient, op: dict[str, Any], body: dict[str, Any], context: DeliveryContext) -> Any:
+    config = client.request("POST", "/configs/upload", json=body, retries=context.retries)
+    config_id = (config or {}).get("_id")
+    training_uuid = op.get("training_uuid")
+    if config_id and training_uuid:
+        # A config nobody points at is a row in a library. Linking it is what
+        # makes it this run's config, shown on the run's page.
+        ident = training_id(client, context, training_uuid)
+        client.request("PUT", f"/trainings/{ident}", json={"configId": config_id}, retries=context.retries)
+    return config
+
+
+def _visualization(
+    client: HttpClient, op: dict[str, Any], body: dict[str, Any], context: DeliveryContext
+) -> Any:
+    path = staged_path(op, context)
+    if not path.exists():
+        raise ConfigurationError(f"the file for visualization {body.get('filename')!r} is gone: {path}")
+    grant = client.request(
+        "POST",
+        "/visualizations/upload-url",
+        json={
+            "epoch_uuid": body["epoch_uuid"],
+            "filename": body["filename"],
+            "type": body["type"],
+            "mimetype": body["mimetype"],
+        },
+        retries=context.retries,
+    )
+    try:
+        upload_url, visualization_uuid, file_id = (
+            grant["uploadUrl"],
+            grant["visualization_uuid"],
+            grant["fileId"],
+        )
+    except (KeyError, TypeError) as exc:
+        raise ApiError(f"upload-url answered without {exc}", status=None) from exc
+    client.put_file(upload_url, str(path), body["mimetype"], retries=context.retries)
+    record = {
+        "epoch_uuid": body["epoch_uuid"],
+        "visualization_uuid": visualization_uuid,
+        "filename": body["filename"],
+        "type": body["type"],
+        "fileId": file_id,
+        "mimetype": body["mimetype"],
+        # The exact byte count: the server refuses a record whose size differs
+        # from the file it received.
+        "size": os.path.getsize(path),
+    }
+    if body.get("metadata") is not None:
+        record["metadata"] = body["metadata"]
+    return _post_once(client, "/visualizations", record, context)
+
+
+def staged_path(op: dict[str, Any], context: DeliveryContext) -> Path:
+    """Where the file an op uploads lives: its staged copy, or the original."""
+    staged = op.get("staged")
+    if staged and context.files is not None:
+        return context.files / str(staged)
+    return Path(str(op.get("path") or ""))
+
+
+def discard_staged(op: dict[str, Any], context: DeliveryContext) -> None:
+    """Remove an op's staged copy once nothing will need it again."""
+    if op.get("op") != "visualization" or not op.get("staged") or context.files is None:
+        return
+    try:
+        (context.files / op["staged"]).unlink()
+    except FileNotFoundError:
+        pass
+    except OSError as exc:  # pragma: no cover - a read-only or vanished directory
+        logger.debug("visin: could not remove staged file: %s", exc)

visin/_internal/sender.py

@@ -0,0 +1,143 @@
+"""A background sender, so ``log_epoch`` costs the training loop nothing.
+
+An epoch POST across a university network takes anywhere from 20ms to several
+seconds, and it happens at exactly the moment the loop wants to start the next
+epoch. Doing it inline makes reporting a tax on training time; worse, a
+degraded Visin instance would slow the run down rather than merely losing
+metrics.
+
+So work is queued and a daemon thread drains it. The cost of that choice is
+that failures surface late, which is why `flush` exists and why `Run.finish`
+calls it.
+"""
+
+from __future__ import annotations
+
+import logging
+import queue
+import threading
+from typing import Callable
+
+logger = logging.getLogger("visin")
+
+
+class _Barrier:
+    """A marker the worker signals when it reaches this point in the queue."""
+
+    __slots__ = ("event",)
+
+    def __init__(self) -> None:
+        self.event = threading.Event()
+
+
+class Sender:
+    """Serialises queued work onto one thread.
+
+    One thread, not a pool: the writes are ordered (a run is created before its
+    epochs, an epoch exists before its visualizations) and ordering is cheaper
+    to guarantee than to recover from.
+    """
+
+    def __init__(self, max_queue: int = 10_000):
+        self._queue: queue.Queue[Callable[[], None] | _Barrier | None] = queue.Queue(maxsize=max_queue)
+        self._thread = threading.Thread(target=self._drain, name="visin-sender", daemon=True)
+        self._stopping = threading.Event()
+        self._abandoned = threading.Event()
+        self._started = False
+        self._lock = threading.Lock()
+        self.dropped = 0
+        self.failed = 0
+        # Work handed back by a stop that could not wait for it: never attempted.
+        self.leftover: list[Callable[[], None]] = []
+
+    def _ensure_started(self) -> None:
+        with self._lock:
+            if not self._started:
+                self._thread.start()
+                self._started = True
+
+    def submit(self, work: Callable[[], None]) -> bool:
+        """Queue one unit of work. Never blocks the caller.
+
+        A full queue drops the item rather than blocking the training loop.
+        Losing a metric is a bad outcome; stalling training because a metrics
+        server is unreachable is a worse one, and the drop is counted so
+        `finish` can report it. Returns whether the work was queued.
+        """
+        if self._stopping.is_set():
+            return False
+        self._ensure_started()
+        try:
+            self._queue.put_nowait(work)
+        except queue.Full:
+            self.dropped += 1
+            logger.warning("visin: queue full, dropped a report (%d dropped so far)", self.dropped)
+            return False
+        return True
+
+    def _drain(self) -> None:
+        while True:
+            item = self._queue.get()
+            try:
+                if item is None:
+                    return
+                if isinstance(item, _Barrier):
+                    item.event.set()
+                    continue
+                if self._abandoned.is_set():
+                    self.leftover.append(item)
+                    continue
+                item()
+            except Exception as exc:  # noqa: BLE001 - a reporting failure must not kill the thread
+                self.failed += 1
+                logger.warning("visin: report failed: %s", exc)
+            finally:
+                self._queue.task_done()
+
+    def flush(self, timeout: float = 30.0) -> bool:
+        """Block until everything queued so far has been attempted.
+
+        Implemented with a barrier rather than ``Queue.join`` because join has
+        no timeout, and a hung flush at the end of a training run would turn a
+        finished job into one the scheduler eventually kills.
+        """
+        if not self._started or not self._thread.is_alive():
+            return True  # nothing queued will ever run, so there is nothing to wait for
+        barrier = _Barrier()
+        try:
+            self._queue.put_nowait(barrier)
+        except queue.Full:
+            return False
+        if not barrier.event.wait(timeout):
+            logger.warning("visin: flush timed out after %.0fs; some reports may be unsent", timeout)
+            return False
+        return True
+
+    def stop(self, timeout: float = 30.0) -> bool:
+        """Flush, then retire the thread.
+
+        When the flush times out, whatever is still queued is not attempted: it
+        is moved to ``leftover``, so the caller can keep it for later rather
+        than lose it with the process. The item in flight, if any, finishes on
+        its own.
+        """
+        flushed = self.flush(timeout)
+        self._stopping.set()
+        if not self._started:
+            return flushed
+        if not flushed:
+            self._abandoned.set()
+            while True:
+                try:
+                    item = self._queue.get_nowait()
+                except queue.Empty:
+                    break
+                if item is not None and not isinstance(item, _Barrier):
+                    self.leftover.append(item)
+                self._queue.task_done()
+        try:
+            self._queue.put_nowait(None)
+        except queue.Full:  # pragma: no cover - just drained
+            pass
+        self._thread.join(timeout=5.0 if flushed else 0.1)
+        return flushed

visin/_internal/serialize.py

@@ -0,0 +1,85 @@
+"""Turning what a training loop has in hand into JSON Visin will accept.
+
+A loop's metrics are rarely plain floats. They are NumPy scalars, zero-dimensional
+tensors, the odd ``Decimal``; ``json.dumps`` refuses every one of them, and in
+a background sender that refusal turns into a metric that silently never
+arrives. Converting here, at the call, keeps the mistake visible and the
+common cases working.
+
+NaN and infinity have no JSON spelling. Sent anyway, the body is not valid JSON
+and the server refuses the whole epoch, so they become ``null``, which Visin
+already reads as "not measured" and charts as a gap.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import enum
+import math
+import uuid
+from collections.abc import Mapping
+from datetime import date, datetime
+from decimal import Decimal
+from pathlib import PurePath
+from typing import Any, Callable
+
+
+def to_jsonable(
+    value: Any,
+    nonfinite: list[str] | None = None,
+    path: str = "",
+    fallback: Callable[[Any], Any] | None = None,
+) -> Any:
+    """Return ``value`` as plain JSON types.
+
+    ``nonfinite``, when given, collects the dotted path of every NaN or
+    infinity that was replaced by ``None``, so the caller can say so.
+    ``fallback`` converts what nothing else can; without it, that raises.
+    """
+    if value is None or isinstance(value, (bool, str)):
+        return value
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float):
+        if math.isfinite(value):
+            return value
+        if nonfinite is not None:
+            nonfinite.append(path or "<value>")
+        return None
+    if isinstance(value, Mapping):
+        return {
+            str(key): to_jsonable(item, nonfinite, f"{path}.{key}" if path else str(key), fallback)
+            for key, item in value.items()
+        }
+    if isinstance(value, (list, tuple, set, frozenset)):
+        return [
+            to_jsonable(item, nonfinite, f"{path}[{index}]", fallback) for index, item in enumerate(value)
+        ]
+    if isinstance(value, (datetime, date)):
+        return value.isoformat()
+    if isinstance(value, Decimal):
+        return to_jsonable(float(value), nonfinite, path, fallback)
+    if isinstance(value, (uuid.UUID, PurePath)):
+        return str(value)
+    if isinstance(value, enum.Enum):
+        return to_jsonable(value.value, nonfinite, path, fallback)
+    if dataclasses.is_dataclass(value) and not isinstance(value, type):
+        return to_jsonable(dataclasses.asdict(value), nonfinite, path, fallback)
+    # NumPy scalars and arrays, and PyTorch/TensorFlow tensors, all offer
+    # ``tolist``: a scalar comes back as a Python number, an array as nested
+    # lists. Duck-typed so none of those libraries has to be installed.
+    tolist = getattr(value, "tolist", None)
+    if callable(tolist):
+        return to_jsonable(tolist(), nonfinite, path, fallback)
+    item = getattr(value, "item", None)
+    if callable(item):
+        return to_jsonable(item(), nonfinite, path, fallback)
+    numpy = getattr(value, "numpy", None)  # an eager TensorFlow tensor
+    if callable(numpy):
+        return to_jsonable(numpy(), nonfinite, path, fallback)
+    if fallback is not None:
+        return fallback(value)
+    raise TypeError(
+        f"visin cannot send a {type(value).__name__}"
+        f"{f' (at {path})' if path else ''}; convert it to a number, string, list or dict"
+    )