PyLV200 0.1.1__py3-none-any.whl

pylv200/__init__.py

@@ -0,0 +1,100 @@
+"""pylv200 - pull images out of a running Olympus/Evident cellSens experiment.
+
+cellSens does not finalise its image container until a run ends, but the pixel
+data is already on disk in the ``_<experiment>_`` folder as ``.ets`` tile files.
+This package reads those directly and assembles ImageJ hyperstacks of everything
+acquired so far, without writing to, renaming or locking anything in the
+experiment folder.
+
+The channel names come from the small ``.vsi`` cellSens writes beside the tiles
+at the START of the run, and the requested cycle and exposures from the ``.oex``
+acquisition plan. Neither has to be guessed.
+"""
+
+import os
+from pathlib import Path
+
+from .channels import channel_stats, classify_channels, cross_check
+from .client import LV200Client, scan_roots
+from .discover import (experiment_name, find_dataset, group_by_dataset,
+                       looks_like_parts)
+from .engine import (STAGES, assign_axes, build_stacks, decode_tile,
+                     derive_interval, is_open_part, live_length, merge_parts,
+                     parse_ets, pull_experiment, pull_position,
+                     select_channels, select_timepoints, select_window,
+                     write_hyperstack)
+from .errors import (AmbiguousDataset, GeometryMismatch, InvalidSelection,
+                     LV200Error, NoDatasetFound, SourceUnreachable,
+                     UnreadableFormat)
+from .models import (DownloadResult, Experiment, ExportPlan, OutputFile, Part,
+                     Position, ProgressEvent, PullPlan, PullResult, Source,
+                     human_bytes)
+from .manifest import MANIFEST_FILENAME, load_manifest, write_manifest
+from .oex import find_oex, read_oex
+from .options import (LAYOUTS, LAYOUT_AXES, LAYOUT_DESCRIPTIONS, LAYOUT_LABELS,
+                      MOMENT_HELP, STATE_SCOPES, ExportOptions, resolve_layout)
+from .preview import PreviewImage, PreviewSet, encode_png
+from .state import NullStateStore, StateStore, output_key
+from .timeline import TimelineFrame, TimelineSource
+from .vsi import find_sidecar, read_vsi_channels
+from .watch import PollResult, Watcher, format_age
+from .config import home
+from . import compat
+
+# Registers the retired import names, of which there are currently none - see
+# `compat.py`. Called here rather than left to the caller so the mechanism is
+# in place whenever this package is, which is the only way an old script keeps
+# working without being edited first.
+compat.install()
+
+
+def connect(source=None):
+    """A ready :class:`LV200Client` - the PyIncucyte idiom, on this package.
+
+        with pylv200.connect() as lv200:
+            result = lv200.fetch(output="./run-01", last_n=24)
+
+    `source` is the name of a source saved by ``pylv200 setup``, a folder, or
+    None for the only saved source there is. A configured name wins over a
+    folder of the same name: naming a microscope is the deliberate act, and a
+    stray folder in the working directory should not shadow one.
+
+    Nothing here touches the filesystem until a method is called, and nothing
+    imports Tk - `pylv200 list` on a headless machine must not depend on a
+    window system.
+    """
+    from .client import LV200Client
+    if source is None or isinstance(source, Source):
+        return LV200Client.from_saved() if source is None else LV200Client(source)
+    text = str(source)
+    from . import config
+    if text in config.load_sources():
+        return LV200Client.from_saved(text)
+    if os.sep in text or "/" in text or Path(text).is_dir():
+        return LV200Client(text)
+    return LV200Client.from_saved(text)      # raises, listing what it knows
+
+__version__ = "0.1.1"
+__all__ = [
+    "parse_ets", "assign_axes", "build_stacks", "decode_tile", "merge_parts",
+    "select_channels", "select_timepoints", "select_window", "derive_interval",
+    "is_open_part", "live_length", "write_hyperstack",
+    "pull_position", "pull_experiment", "STAGES",
+    "channel_stats", "classify_channels", "cross_check",
+    "LV200Client", "connect", "scan_roots", "ExportOptions",
+    "Watcher", "PollResult", "format_age",
+    "StateStore", "NullStateStore", "output_key",
+    "PreviewSet", "PreviewImage", "encode_png",
+    "TimelineSource", "TimelineFrame",
+    "load_manifest", "write_manifest", "MANIFEST_FILENAME",
+    "LAYOUTS", "LAYOUT_LABELS", "LAYOUT_DESCRIPTIONS", "LAYOUT_AXES",
+    "resolve_layout", "STATE_SCOPES", "MOMENT_HELP",
+    "Experiment", "Position", "Part", "Source", "PullPlan", "PullResult",
+    "DownloadResult", "ExportPlan",
+    "OutputFile", "ProgressEvent", "human_bytes", "home", "compat",
+    "find_dataset", "group_by_dataset", "looks_like_parts", "experiment_name",
+    "find_sidecar", "read_vsi_channels", "find_oex", "read_oex",
+    "LV200Error", "SourceUnreachable", "NoDatasetFound", "AmbiguousDataset",
+    "UnreadableFormat", "GeometryMismatch", "InvalidSelection",
+    "__version__",
+]

pylv200/__main__.py

@@ -0,0 +1,7 @@
+"""``python -m pylv200`` - the same entry point as the console script."""
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

pylv200/cache.py

@@ -0,0 +1,365 @@
+"""The incremental byte mirror: a local copy of remote ``.ets`` files.
+
+This is what turns "extract" into "sync". Tile data is appended at a fixed
+stride, so a mirror only ever needs the bytes past what it already holds, and a
+repeated pull costs the new data and nothing else.
+
+The copy matters more than the speed. During acquisition the tiles are the only
+copy of the run that exists - cellSens writes nothing else until the run ends -
+so a mirror that keeps pace is what stands between a crashed nine-day experiment
+and total loss.
+
+**This is the one module where silent corruption is possible.** Everything else
+fails loudly; a mirror that appends at the wrong offset produces a file that
+parses perfectly and contains garbage. Three things keep that from happening and
+none of them may be weakened:
+
+* the resume point is whatever length of the local copy still agrees with the
+  source at its tail, so an interrupted sync continues where it stopped while a
+  half-written or corrupted tail is dropped rather than appended to;
+* the source is read to EOF rather than to a length measured beforehand, because
+  it is being written to while it is read;
+* the header region is re-copied after every sync, because closing a part
+  rewrites it - and comparing what identifies the file excludes the timepoint
+  count, which legitimately changes when that happens.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+
+from .engine import geometry_key, live_length, probe_geometry
+from .errors import MirrorError
+
+#: Written into each mirrored folder. Records what was copied and what the
+#: source looked like, so a change of file behind the same name is noticed.
+MIRROR_SIDECAR = ".pylv200-mirror.json"
+
+#: Read size for the copy loop. Big enough that SMB round trips do not dominate.
+CHUNK_BYTES = 8 << 20
+
+#: Kept free after any sync, so a full mirror does not take the machine with it.
+FREE_MARGIN = 256 << 20
+
+#: How much of the local copy's end is checked against the source before the
+#: rest of it is believed. The cost is one small read; what it buys is that a
+#: torn tail from a killed process is never appended to.
+TAIL_CHECK = 64 << 10
+
+
+@dataclass(frozen=True)
+class SyncResult:
+    """What one file's sync did."""
+    source: Path
+    local: Path
+    copied: int = 0             # bytes that crossed the wire
+    held: int = 0               # bytes already local before this sync
+    length: int = 0             # local length afterwards
+    reset_reason: str = ""      # non-empty when the whole file was re-copied
+    header_refreshed: bool = False
+    closed: bool = False        # gained its chunk table during this sync
+
+    @property
+    def up_to_date(self):
+        return self.copied == 0 and not self.reset_reason
+
+    def __str__(self):
+        if self.reset_reason:
+            what = "re-copied %.1f MB (%s)" % (self.copied / 1e6, self.reset_reason)
+        elif self.copied:
+            what = "+%.1f MB of %.1f MB" % (self.copied / 1e6, self.length / 1e6)
+        else:
+            what = "up to date (%.1f MB)" % (self.length / 1e6)
+        return "%s  %s%s" % (self.source.name, what,
+                             "  [part closed]" if self.closed else "")
+
+
+def _safe(name):
+    return re.sub(r"[^A-Za-z0-9._-]+", "_", str(name)).strip("_") or "root"
+
+
+class Mirror:
+    """A local directory that grows as the remote files do."""
+
+    def __init__(self, root, free_margin=FREE_MARGIN):
+        self.root = Path(root)
+        self.free_margin = int(free_margin)
+
+    # -- where things land -------------------------------------------------
+
+    def local_path(self, part):
+        """Where this remote file is mirrored to.
+
+        The remote folder is hashed into the directory name: two microscopes can
+        hold a position called ``ois7041`` and the copies must not land on top
+        of each other, while the same file on the same machine must always come
+        back to the same place.
+        """
+        src = Path(getattr(part, "path", part))
+        folder = src.parent
+        key = hashlib.sha1(str(folder).lower().encode("utf-8")).hexdigest()[:8]
+        return self.root / ("%s-%s" % (_safe(folder.name), key)) / src.name
+
+    def resolver(self):
+        """A ``resolve(path)`` callable for ``build_stacks`` / ``channel_stats``.
+
+        Reads come from the mirror when it holds the file and from the source
+        when it does not, so extraction works whether or not a sync has run.
+        """
+        def resolve(src):
+            local = self.local_path(src)
+            return local if local.exists() else Path(src)
+        return resolve
+
+    def held(self, part):
+        """Bytes currently held locally for this part."""
+        local = self.local_path(part)
+        try:
+            return local.stat().st_size
+        except OSError:
+            return 0
+
+    # -- the sidecar -------------------------------------------------------
+
+    def _sidecar_path(self, part):
+        return self.local_path(part).parent / MIRROR_SIDECAR
+
+    def _sidecar(self, part):
+        try:
+            return json.loads(self._sidecar_path(part).read_text())
+        except (OSError, ValueError):
+            return {}
+
+    def _remember(self, part, record):
+        path = self._sidecar_path(part)
+        data = self._sidecar(part)
+        data[Path(getattr(part, "path", part)).name] = record
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(data, indent=2))
+
+    # -- syncing -----------------------------------------------------------
+
+    def _must_reset(self, source, have, geom, remembered):
+        """Why the local copy cannot simply be appended to, or ``""``.
+
+        Closure is deliberately not in here. When cellSens finishes a part it
+        appends the chunk table and rewrites the header; the body already
+        mirrored is still valid, so that is handled by copying the tail and
+        refreshing the header, not by fetching a gigabyte again.
+        """
+        if have <= 0:
+            return ""
+        length = geom.get("size", -1)
+        if 0 <= length < have:
+            return "the source is shorter than the local copy, so it was replaced"
+        was = remembered.get("geometry_key")
+        now = geometry_key(geom)
+        if was and now and list(was) != list(now):
+            return ("the source's geometry changed from %s to %s"
+                    % (tuple(was), tuple(now)))
+        return ""
+
+    def _tail_matches(self, source, local, length, window=TAIL_CHECK):
+        """Does the local copy still agree with the source at this length?"""
+        start = max(length - window, 0)
+        want = length - start
+        try:
+            with open(source, "rb") as src, open(local, "rb") as dst:
+                src.seek(start)
+                dst.seek(start)
+                return src.read(want) == dst.read(want)
+        except OSError:
+            return False
+
+    def _trusted_length(self, source, local, have, recorded):
+        """How much of the local copy can be believed, and appended to.
+
+        The file's own length comes first, so a sync that died half way through
+        resumes rather than starting again. It is believed only if the source
+        still says the same thing there - a process killed mid-write, or a disk
+        that filled, can leave a tail that is not source bytes at all, and
+        appending past it would produce a file that parses and holds garbage.
+        The last recorded length is the fallback, and zero is the floor.
+        """
+        if have <= 0:
+            return 0
+        for cand in sorted({have, min(recorded, have)}, reverse=True):
+            if cand > 0 and self._tail_matches(source, local, cand):
+                return cand
+        return 0
+
+    def sync(self, part, chunk_bytes=CHUNK_BYTES, progress=None):
+        """Bring the local copy up to the source's current length.
+
+        Returns a :class:`SyncResult`. Nothing already held crosses the wire.
+        """
+        source = Path(getattr(part, "path", part))
+        local = self.local_path(source)
+        local.parent.mkdir(parents=True, exist_ok=True)
+
+        try:
+            geom = probe_geometry(source)
+        except OSError as exc:
+            raise MirrorError("cannot read %s: %s"
+                              % (source, exc.strerror or exc)) from None
+        remembered = self._sidecar(source).get(source.name, {})
+
+        have = self.held(source)
+        reason = self._must_reset(source, have, geom, remembered)
+        if reason:
+            have = 0
+        else:
+            have = self._trusted_length(source, local, have,
+                                        int(remembered.get("length") or 0))
+
+        expected = max(geom.get("size", 0) - have, 0)
+        self._check_space(expected, local)
+
+        copied = 0
+        mode = "r+b" if (have and local.exists()) else "wb"
+        with open(source, "rb") as src, open(local, mode) as dst:
+            src.seek(have)
+            dst.seek(have)
+            dst.truncate(have)           # drop any tail left by an interrupted run
+            while True:
+                block = src.read(chunk_bytes)
+                if not block:
+                    break                # read to EOF: the file grows as it is read
+                dst.write(block)
+                copied += len(block)
+                if progress:
+                    progress(copied, expected)
+            dst.flush()
+            os.fsync(dst.fileno())
+
+        length = have + copied
+        refreshed = self._refresh_header(source, local, geom, length)
+
+        # A part that had no chunk table last time and has one now has just been
+        # closed by cellSens. Worth reporting - it is the moment the run's data
+        # stops moving - but it changes nothing about what was copied.
+        open_now = _looks_open(local)
+        closed = bool(remembered.get("open")) and not open_now
+
+        self._remember(source, {
+            "source": str(source),
+            "length": length,
+            "geometry_key": list(geometry_key(geom) or ()),
+            "header_bytes": geom.get("header_bytes"),
+            "open": open_now,
+            "modified": _mtime(source),
+        })
+        return SyncResult(source=source, local=local, copied=copied, held=have,
+                          length=length, reset_reason=reason,
+                          header_refreshed=refreshed, closed=closed)
+
+    def _refresh_header(self, source, local, geom, length):
+        """Re-copy the header region, which closing a part rewrites.
+
+        Done after the tail, not before: a crash between the two leaves a local
+        file whose header describes less than it holds, which the parser already
+        copes with. The other order would leave one describing more.
+        """
+        n = geom.get("header_bytes") or 512
+        if length < n:
+            return False
+        with open(source, "rb") as src:
+            head = src.read(n)
+        with open(local, "r+b") as dst:
+            if dst.read(len(head)) == head:
+                return False
+            dst.seek(0)
+            dst.write(head)
+        return True
+
+    def _check_space(self, need, local):
+        if need <= 0:
+            return
+        try:
+            free = shutil.disk_usage(local.parent).free
+        except OSError:
+            return
+        if free < need + self.free_margin:
+            raise MirrorError(
+                "%s needs %.1f GB and %.1f GB is free. Mirror somewhere with "
+                "room, or pull a narrower time range."
+                % (local.parent, need / 1e9, free / 1e9))
+
+    # -- whole experiments -------------------------------------------------
+
+    def space_needed(self, experiment):
+        """Bytes a full sync of this experiment would still have to fetch."""
+        total = 0
+        for pos in experiment.positions:
+            for part in pos.parts:
+                size = part.size if part.size >= 0 else live_length(part.path)
+                total += max(size - self.held(part.path), 0)
+        return total
+
+    def sync_experiment(self, experiment, progress=None, chunk_bytes=CHUNK_BYTES):
+        """Sync every part of every position. Returns a list of SyncResults."""
+        out = []
+        for pos in experiment.positions:
+            for part in pos.parts:
+                out.append(self.sync(part.path, chunk_bytes=chunk_bytes,
+                                     progress=progress))
+        return out
+
+    # -- checking ----------------------------------------------------------
+
+    def verify(self, part, samples=16, window=4096):
+        """Compare sampled stretches of the copy against the source.
+
+        For when a sync looks wrong. Returns the offsets that disagree; an empty
+        list means every sample matched. Evenly spaced rather than random, so a
+        failure can be reproduced.
+        """
+        source = Path(getattr(part, "path", part))
+        local = self.local_path(source)
+        held = self.held(source)
+        if not held:
+            return [(0, "nothing is mirrored")]
+        bad = []
+        step = max(held // max(samples, 1), 1)
+        offsets = list(range(0, held, step))[:samples]
+        if held > window:
+            offsets.append(held - window)
+        with open(source, "rb") as src, open(local, "rb") as dst:
+            for off in offsets:
+                src.seek(off)
+                dst.seek(off)
+                a, b = src.read(window), dst.read(window)
+                if a != b:
+                    bad.append((off, "%d bytes differ" % sum(1 for x, y
+                                                             in zip(a, b) if x != y)))
+        return bad
+
+
+def _looks_open(path):
+    """Is cellSens still writing this part - has it no usable chunk table yet?
+
+    Asked of the local copy, and asked by parsing it properly. A cheaper probe
+    was tried and was wrong: a header field can look like a plausible table
+    pointer while pointing at nothing, and only the scoring inside parse_ets
+    settles it. The parse costs local CPU and no network, which is the trade
+    worth making. Never used to decide what to copy - only to report that a run
+    has finished.
+    """
+    from .engine import is_open_part, parse_ets
+
+    try:
+        return is_open_part(parse_ets(Path(path)))
+    except (OSError, ValueError):
+        return False
+
+
+def _mtime(path):
+    try:
+        return os.stat(path).st_mtime
+    except OSError:
+        return 0.0

pylv200/channels.py

@@ -0,0 +1,133 @@
+"""Per-channel pixel statistics, and the classifier built on them.
+
+The classifier is a cross-check, not a source of truth. Channel names come from
+the ``.vsi`` written beside the tiles at the start of the run; these statistics
+only say what the pixels look like. On a real four-channel LV200 run the
+classifier named every channel wrongly, because it assumes a luminescence
+channel is photon-starved and a bright luminescent sample inverts that.
+"""
+from __future__ import annotations
+
+from collections import defaultdict
+
+import numpy as np
+
+from .engine import decode_tile
+
+
+def _safe_corr(a, b):
+    """Correlation between two frames, defined even when one is flat.
+
+    A frame with no variance has an undefined correlation. Two identical flat
+    frames are perfectly alike, so score them 1.0; otherwise 0.0. Returning NaN
+    would let a degenerate channel be mistaken for the photon-starved one.
+    """
+    if a.std() == 0 or b.std() == 0:
+        return 1.0 if np.array_equal(a, b) else 0.0
+    return float(np.corrcoef(a.ravel(), b.ravel())[0, 1])
+
+
+def channel_stats(meta, axes, max_pairs=4, resolve=None):
+    """Per-channel statistics, read straight from the tiles.
+
+    Enough to tell the channels apart without decoding the whole run: signal
+    level, how much a frame resembles the next one, and how much of the frame is
+    saturated.
+    """
+    ix = {n: i for i, n in enumerate(axes["names"])}
+    chunks = meta["chunks"]
+
+    def key(rec, ax):
+        return rec[0][ix[ax]] if ax in ix else 0
+
+    by_c = defaultdict(dict)
+    for rec in chunks:
+        by_c[key(rec, "c")].setdefault(key(rec, "t"), rec)
+
+    out = {}
+    for c, byt in sorted(by_c.items()):
+        ts = sorted(byt)[:max_pairs + 1]
+        frames = []
+        for t in ts:
+            coords, off, nbytes, src = byt[t]
+            with open(resolve(src) if resolve else src, "rb") as fh:
+                fh.seek(off)
+                frames.append(decode_tile(fh.read(nbytes), meta).astype(np.float32))
+        if not frames:
+            continue
+        rs = [_safe_corr(a, b) for a, b in zip(frames, frames[1:])]
+        f0 = frames[0]
+        dx = np.diff(f0, axis=1)
+        out[c] = {
+            "median": float(np.median(f0)),
+            "p99": float(np.percentile(f0, 99)),
+            "temporal_r": float(np.mean(rs)) if rs else float("nan"),
+            "saturated": float(np.mean(f0 >= np.iinfo(np.uint16).max)),
+            "smoothness": float(f0.var() / dx.var()) if dx.var() > 0 else float("inf"),
+        }
+    return out
+
+
+def cross_check(stats, names=None):
+    """Facts about the pixels that are worth saying whatever the channel is called.
+
+    Deliberately not a second opinion on identity. The classifier below was
+    wrong on all four channels of a real run, so it disagreeing with a sidecar
+    would say nothing about which is right - and the sidecar is right. What the
+    pixels can state without appealing to identity is that a channel carries no
+    signal, or that it is clipping, and both change what a frame is worth.
+
+    Returns a list of notes; an empty list means nothing looked wrong.
+    """
+    notes = []
+    for i, c in enumerate(sorted(stats)):
+        st = stats[c]
+        label = "C%d" % c
+        if names and i < len(names):
+            label = "C%d (%s)" % (c, names[i])
+        if st["p99"] <= st["median"]:
+            notes.append("%s is flat - no range between its median and its 99th "
+                         "percentile, so nothing was collected on it" % label)
+        elif st["saturated"] > 0.01:
+            notes.append("%s is clipping - %.1f%% of its pixels sit at full scale"
+                         % (label, 100.0 * st["saturated"]))
+    return notes
+
+
+def classify_channels(stats):
+    """Name channels from their statistics. A LAST RESORT, not a source of truth.
+
+    Channel names come from the ``.vsi`` written at the start of the run. This
+    runs only when there is no sidecar and no override, and its output must be
+    reported as inferred: on a real four-channel run it named every channel
+    wrongly, and the output gave no sign anything was amiss.
+
+    Two are identifiable with confidence. A luminescence channel is
+    photon-starved: consecutive frames barely resemble each other because the
+    signal is shot noise, and cosmic-ray hits saturate stray pixels. Brightfield
+    is the opposite - bright, nearly identical frame to frame, and spatially
+    smooth because it is mostly the illumination profile.
+
+    The rest are fluorescence, and nothing in the pixels says which fluorophore
+    is which. They are numbered rather than guessed at.
+    """
+    if not stats:
+        return {}
+    names, remaining = {}, dict(stats)
+
+    lum = min(remaining, key=lambda c: remaining[c]["temporal_r"])
+    if remaining[lum]["temporal_r"] < 0.4:
+        names[lum] = "Bioluminescence"
+        remaining.pop(lum)
+
+    if remaining:
+        bf = max(remaining, key=lambda c: (remaining[c]["temporal_r"],
+                                           remaining[c]["median"]))
+        s = remaining[bf]
+        if s["temporal_r"] > 0.95 and s["median"] == max(v["median"] for v in stats.values()):
+            names[bf] = "Brightfield"
+            remaining.pop(bf)
+
+    for i, c in enumerate(sorted(remaining), start=1):
+        names[c] = "Fluorescence %d" % i if len(remaining) > 1 else "Fluorescence"
+    return names