tofscope 0.1.0__py3-none-any.whl

tofscope/api/schemas.py

@@ -0,0 +1,91 @@
+from typing import Literal, Annotated
+from pydantic import Field, StrictInt, StrictBool
+from tofscope.application.commands import StrictModel
+
+
+class OpenAcquisition(StrictModel):
+    relative_path: str = Field(min_length=1, max_length=4096)
+    location: Literal["acquisition", "saved"] = "acquisition"
+
+
+class PrepareAcquisition(OpenAcquisition):
+    generation: str = Field(min_length=1, max_length=128)
+    filename: str = Field(min_length=1, max_length=240)
+    accept_assumptions: StrictBool = False
+
+
+class RevisionEnvelope(StrictModel):
+    request_id: str = Field(min_length=1, max_length=128)
+    expected_revision: StrictInt = Field(ge=0)
+    dataset_generation: str = Field(min_length=1, max_length=128)
+
+
+class ReplaceAcquisition(RevisionEnvelope):
+    relative_path: str = Field(min_length=1, max_length=4096)
+    location: Literal["acquisition", "saved"] = "acquisition"
+
+
+FiniteStrictFloat = Annotated[float, Field(strict=True, allow_inf_nan=False)]
+
+
+class IntegrationInput(StrictModel):
+    intervals: list[tuple[FiniteStrictFloat, FiniteStrictFloat]] = Field(
+        min_length=1, max_length=128
+    )
+    frames: tuple[StrictInt, StrictInt]
+    name: str = Field(min_length=1, max_length=128)
+    signal_id: str | None = None
+    expected_recipe_version: StrictInt | None = Field(default=None, ge=0)
+    candidate_id: str | None = Field(default=None, max_length=64)
+    measured_mz: FiniteStrictFloat | None = Field(default=None, gt=0)
+
+
+class SubmitJob(RevisionEnvelope):
+    spec: IntegrationInput
+
+
+class Bootstrap(StrictModel):
+    secret: str = Field(max_length=256)
+
+
+class SessionInfo(StrictModel):
+    owner: str
+    csrf: str
+    initial_file: str | None = None
+
+
+class ErrorResponse(StrictModel):
+    code: str
+    message: str
+    request_id: str | None = None
+    current_revision: int | None = None
+
+
+class SaveCopy(RevisionEnvelope):
+    filename: str = Field(min_length=1, max_length=240)
+    directory: str = Field(default='', max_length=4096)
+    destination_location: Literal['acquisition', 'saved'] = 'saved'
+
+
+class CreateSaveFolder(StrictModel):
+    name: str = Field(min_length=1, max_length=240)
+    directory: str = Field(default='', max_length=4096)
+    location: Literal['acquisition', 'saved'] = 'saved'
+
+
+class FolderLocations(StrictModel):
+    acquisitions: str
+    saved_copies: str
+    save_destination: str
+    can_save_to_acquisitions: bool = False
+
+
+class CandidateInput(StrictModel):
+    mz: FiniteStrictFloat = Field(gt=0)
+    tolerance: FiniteStrictFloat = Field(ge=0)
+    unit: Literal['Th', 'ppm']
+    charges: tuple[StrictInt, ...] = Field(min_length=1, max_length=6)
+
+
+class MeasureIsotopes(RevisionEnvelope):
+    signal_ids: tuple[str, ...] = Field(min_length=1, max_length=128)

tofscope/application/analyses.py

@@ -0,0 +1,237 @@
+from collections import OrderedDict
+from dataclasses import dataclass, field, replace
+from threading import RLock
+from uuid import uuid4
+from tofscope.storage.acquisitions import SourceChanged
+from tofscope.application.models import (
+    AnalysisSnapshot,
+    DisplaySettings,
+    SignalSnapshot,
+    Peak,
+)
+from tofscope.application.commands import AllDepth
+from tofscope.science.signals import SignalRecipe
+
+
+class CapacityError(ValueError):
+    pass
+
+
+class Conflict(ValueError):
+    def __init__(self, message, revision=None):
+        super().__init__(message)
+        self.revision = revision
+
+
+@dataclass
+class AnalysisEntry:
+    snapshot: AnalysisSnapshot
+    lock: RLock = field(default_factory=RLock)
+    receipts: OrderedDict = field(default_factory=OrderedDict)
+    tokens: dict = field(default_factory=dict)
+    leases: dict = field(default_factory=dict)
+    closed: bool = False
+
+
+class AnalysisService:
+    def __init__(self, repository, settings):
+        self.repository = repository
+        self.settings = settings
+        self.entries = {}
+        self.lock = RLock()
+        self.signal_lock = RLock()
+        self.on_close = []
+        self.on_remove = []
+        self.prepare = lambda s: None
+
+    def list(self, owner):
+        with self.lock:
+            aids = [e.snapshot.id for e in self.entries.values() if not e.closed and e.snapshot.owner == owner]
+        result = []
+        for aid in aids:
+            try:
+                result.append(self.get(owner, aid))
+            except KeyError:
+                pass
+        return result
+
+    def entry(self, owner, aid):
+        with self.lock:
+            e = self.entries.get(aid)
+        if e is None or e.closed or e.snapshot.owner != owner:
+            raise KeyError("Analysis unavailable.")
+        return e
+
+    def get(self, owner, aid):
+        e = self.entry(owner, aid)
+        try:
+            self.repository.assert_current(owner, e.snapshot.dataset)
+        except (SourceChanged, OSError):
+            self.mark_stale(e.snapshot.dataset)
+        with e.lock:
+            return e.snapshot
+
+    def mark_stale(self, dataset):
+        with self.lock:
+            entries = list(self.entries.values())
+        for entry in entries:
+            with entry.lock:
+                s = entry.snapshot
+                if (
+                    not entry.closed
+                    and s.dataset.key == dataset.key
+                    and s.dataset.generation == dataset.generation
+                    and not s.stale
+                ):
+                    entry.snapshot = replace(s, stale=True, revision=s.revision + 1)
+                    entry.tokens = {k: v + 1 for k, v in entry.tokens.items()}
+                    self._release(entry)
+
+    def initial(self, owner, dataset, aid=None, revision=0):
+        with self.repository.metadata(owner, dataset, aid) as m:
+            nz, ny, nx, _ = m.shape
+            default = min(m.peaks, key=lambda p: abs(p["mass"] - 27))
+            frames = (0, nz - 1)
+            recipe = SignalRecipe(
+                label=f"{default['label']} · {default['mass']:g} Th",
+                source="stored",
+                stored_index=default["index"],
+                intervals=((default["low"], default["high"]),),
+                frames=frames,
+            )
+            snapshot = AnalysisSnapshot(
+                aid or str(uuid4()),
+                owner,
+                dataset,
+                revision,
+                (0, nx - 1, 0, ny - 1),
+                AllDepth(),
+                (SignalSnapshot(recipe.id, recipe.label, 0, recipe),),
+                recipe.id,
+                (recipe.id,),
+                1,
+                DisplaySettings(),
+                m.shape,
+                tuple(Peak(**p) for p in m.peaks),
+                m.fov_um,
+                (float(m.mass_axis.min()), float(m.mass_axis.max())),
+                has_se=bool(m.image_names),
+                opened_revision=revision,
+            )
+
+        from tofscope.storage.analysis_document import read_document, restore_snapshot, ROOT, MAX_DOCUMENT_BYTES
+        from tofscope.storage.integration_store import saved_result_info, iter_saved_frames, metadata_size, ROOT as RESULT_ROOT
+        from contextlib import nullcontext
+        import hashlib
+        import h5py
+        with self.repository.access(owner, dataset) as path:
+            # Plain acquisitions need no document allocation. Reject oversized metadata first.
+            with h5py.File(path, 'r') as h:
+                if ROOT not in h:
+                    return snapshot
+                try:
+                    stored = h[f'{ROOT}/state_json']
+                    if stored.ndim != 1 or stored.size > MAX_DOCUMENT_BYTES:
+                        raise ValueError('Saved analysis metadata exceeds limits.')
+                    estimated = stored.size * 16 + snapshot.shape[1] * snapshot.shape[2] * 8
+                except (KeyError, TypeError, AttributeError) as exc:
+                    raise ValueError('Saved analysis metadata is invalid.') from exc
+            budget = self.repository.budget
+            with budget.reserve(owner, snapshot.id, estimated) if budget else nullcontext():
+                document = read_document(path, self.settings)
+                if document is None:
+                    return snapshot
+                try:
+                    with h5py.File(path, 'r') as handle:
+                        result_metadata_bytes = sum(metadata_size(handle[f'{RESULT_ROOT}/{row.materialization_id}'])
+                                                    for row in document.signals if row.materialization_id)
+                except (KeyError, TypeError) as exc:
+                    raise ValueError('Saved result is missing or invalid.') from exc
+                with budget.reserve(owner, snapshot.id, result_metadata_bytes * 16) if budget else nullcontext():
+                    for row in document.signals:
+                        if row.materialization_id:
+                            info = saved_result_info(path, row.materialization_id)
+                            if tuple(info['spatial_shape']) != snapshot.shape[1:3] or tuple(info['derivation']['frames']) != row.recipe.frames or tuple(tuple(w) for w in info['derivation']['intervals']) != row.recipe.intervals:
+                                raise ValueError('Saved result does not match the document.')
+                            digest = hashlib.sha256()
+                            for frame in iter_saved_frames(path, row.materialization_id, row.recipe.frames):
+                                digest.update(frame.tobytes(order='C'))
+                            with h5py.File(path, 'r') as h:
+                                expected = h[f'ToFScope/Integrations/{row.materialization_id}'].attrs.get('sha256')
+                            if digest.hexdigest() != expected:
+                                raise ValueError('Saved result checksum is invalid.')
+                    return restore_snapshot(document, snapshot)
+
+    def admit_collection(self, snapshot, replacing=None):
+        with self.lock:
+            entries = [e.snapshot for e in self.entries.values() if not e.closed and e.snapshot.id != replacing]
+            count = len(snapshot.signals)
+            if (count > self.settings.signals_per_analysis or
+                    count + sum(len(s.signals) for s in entries if s.owner == snapshot.owner) > self.settings.signals_per_user or
+                    count + sum(len(s.signals) for s in entries) > self.settings.signals_global):
+                raise CapacityError('Saved analysis exceeds signal limits.')
+
+    def admit_signal(self, owner, analysis_count):
+        # Called under signal_lock by every insertion, including pending jobs.
+        with self.lock:
+            entries = [e for e in self.entries.values() if not e.closed]
+            user_count = sum(len(e.snapshot.signals) for e in entries if e.snapshot.owner == owner)
+            global_count = sum(len(e.snapshot.signals) for e in entries)
+        if (analysis_count >= self.settings.signals_per_analysis or
+                user_count >= self.settings.signals_per_user or global_count >= self.settings.signals_global):
+            raise CapacityError('Signal limit reached; remove an unused signal first.')
+
+    def create_analysis(self, owner, dataset):
+        s = self.initial(owner, dataset)
+        with self.signal_lock, self.lock:
+            if (
+                len(self.entries) >= self.settings.analyses_global
+                or sum(e.snapshot.owner == owner for e in self.entries.values())
+                >= self.settings.analyses_per_user
+            ):
+                raise CapacityError("Open analysis limit reached; close an analysis.")
+            self.admit_collection(s)
+            self.prepare(s)
+            self.entries[s.id] = AnalysisEntry(s)
+            return s
+
+    def replace(self, owner, aid, dataset, expected_revision):
+        e = self.entry(owner, aid)
+        with self.signal_lock, e.lock:
+            if e.closed:
+                raise KeyError("Analysis unavailable.")
+            if e.snapshot.revision != expected_revision:
+                raise Conflict("Analysis revision changed.", e.snapshot.revision)
+            s = self.initial(
+                owner, dataset, aid, e.snapshot.revision + 1
+            )
+            self.admit_collection(s, replacing=aid)
+            self.prepare(s)
+            self._release(e)
+            e.snapshot = s
+            e.tokens = {k: v + 1 for k, v in e.tokens.items()}
+            return s
+
+    def _release(self, e):
+        for lease in e.leases.values():
+            lease.release()
+        e.leases.clear()
+
+    def close(self, owner, aid, expected_revision=None):
+        e = self.entry(owner, aid)
+        with self.signal_lock, e.lock:
+            if expected_revision is not None and e.snapshot.revision != expected_revision:
+                raise Conflict("Analysis changed in another view; review it before closing.", e.snapshot.revision)
+            e.closed = True
+            self._release(e)
+        for callback in self.on_close:
+            callback(aid)
+        with self.lock:
+            self.entries.pop(aid, None)
+
+    def close_all(self):
+        for e in list(self.entries.values()):
+            try:
+                self.close(e.snapshot.owner, e.snapshot.id)
+            except KeyError:
+                pass

tofscope/application/cache.py

@@ -0,0 +1,166 @@
+from collections import OrderedDict
+from dataclasses import dataclass, field
+from threading import Condition, RLock
+import numpy as np
+from tofscope.application.analyses import CapacityError
+
+
+@dataclass
+class CachedArray:
+    array: np.ndarray
+    physical: object
+    consumers: dict = field(default_factory=dict)
+
+
+class ArrayLease:
+    def __init__(self, cache, key, consumer, array):
+        self.cache = cache
+        self.key = key
+        self.consumer = consumer
+        self.array = array
+        self.released = False
+
+    def release(self):
+        with self.cache.condition:
+            if self.released:
+                return
+            self.released = True
+            entry = self.cache.entries.get(self.key)
+            if entry is None:
+                return
+            count, reservation = entry.consumers[self.consumer]
+            if count == 1:
+                reservation.release()
+                del entry.consumers[self.consumer]
+            else:
+                entry.consumers[self.consumer] = (count - 1, reservation)
+            self.cache.condition.notify_all()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.release()
+
+
+class ArrayCache:
+    def __init__(self, repository, budget):
+        self.repository = repository
+        self.budget = budget
+        self.entries = OrderedDict()
+        self.loading = set()
+        self.condition = Condition(RLock())
+
+    def key(self, dataset, recipe, materialization_id=None):
+        return (
+            dataset.key,
+            dataset.generation,
+            recipe.source,
+            recipe.stored_index,
+            recipe.saved_id,
+            recipe.intervals,
+            recipe.frames,
+            materialization_id,
+        )
+
+    def _evict(self, needed=None):
+        for key, entry in list(self.entries.items()):
+            if (
+                needed is not None
+                and self.budget.physical_bytes + needed
+                <= self.budget.settings.global_bytes
+            ):
+                break
+            if not entry.consumers:
+                entry.physical.release()
+                del self.entries[key]
+
+    def clear(self):
+        with self.condition:
+            self._evict()
+
+    def acquire(self, owner, analysis_id, dataset, recipe, loader, estimated_bytes, materialization_id=None):
+        self.repository.assert_current(owner, dataset)
+        key = self.key(dataset, recipe, materialization_id)
+        consumer = (owner, analysis_id)
+        with self.condition:
+            while key in self.loading:
+                self.condition.wait()
+            if key in self.entries:
+                return self._lease(owner, analysis_id, dataset, key, consumer)
+            try:
+                physical = self.budget.reserve(None, None, estimated_bytes)
+            except CapacityError:
+                self._evict(estimated_bytes)
+                physical = self.budget.reserve(None, None, estimated_bytes)
+            try:
+                logical = self.budget.reserve(
+                    owner, analysis_id, estimated_bytes, physical=False
+                )
+            except BaseException:
+                physical.release()
+                raise
+            self.loading.add(key)
+        try:
+            array = loader()
+            if not isinstance(array, np.ndarray) or array.nbytes != estimated_bytes:
+                raise CapacityError(
+                    "Reader allocation did not match its preflight shape/dtype."
+                )
+            self.repository.assert_current(owner, dataset)
+            array.setflags(write=False)
+            with self.condition:
+                self.entries[key] = CachedArray(
+                    array, physical, {consumer: (1, logical)}
+                )
+                return ArrayLease(self, key, consumer, array)
+        except BaseException:
+            physical.release()
+            logical.release()
+            raise
+        finally:
+            with self.condition:
+                self.loading.remove(key)
+                self.condition.notify_all()
+
+    def _lease(self, owner, aid, dataset, key, consumer):
+        self.repository.assert_current(owner, dataset)
+        entry = self.entries[key]
+        if consumer in entry.consumers:
+            n, res = entry.consumers[consumer]
+            entry.consumers[consumer] = (n + 1, res)
+        else:
+            entry.consumers[consumer] = (
+                1,
+                self.budget.reserve(
+                    owner, aid, int(entry.array.nbytes), physical=False
+                ),
+            )
+        self.entries.move_to_end(key)
+        return ArrayLease(self, key, consumer, entry.array)
+
+    def load(self, owner, aid, dataset, recipe, materialization_id=None, **kwargs):
+        with self.repository.access(owner, dataset) as path:
+            import h5py
+
+            with h5py.File(path, "r") as f:
+                shape = f["PeakData/PeakData"].shape
+        n = (recipe.frames[1] - recipe.frames[0] + 1) * shape[1] * shape[2] * 4
+        if recipe.source == "custom":
+            kwargs["reserve"] = lambda size: self.budget.reserve(owner, aid, size)
+        return self.acquire(
+            owner,
+            aid,
+            dataset,
+            recipe,
+            lambda: self.repository.read(owner, dataset, recipe, materialization_id=materialization_id, **kwargs),
+            n,
+            materialization_id=materialization_id,
+        )
+
+    def invalidate(self, dataset_key, generation):
+        with self.condition:
+            for key, entry in list(self.entries.items()):
+                if key[:2] == (dataset_key, generation) and not entry.consumers:
+                    entry.physical.release()
+                    del self.entries[key]

tofscope/application/commands.py

@@ -0,0 +1,153 @@
+from tofscope.application.spatial import SpatialView
+from typing import Annotated, Literal, Union
+from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictBool, TypeAdapter
+
+
+class StrictModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+
+class AllDepth(StrictModel):
+    mode: Literal["all"] = "all"
+
+
+class SingleFrame(StrictModel):
+    mode: Literal["frame"]
+    index: StrictInt
+
+
+class DepthWindow(StrictModel):
+    mode: Literal["window"]
+    first: StrictInt
+    last: StrictInt
+
+
+DepthSelection = Annotated[
+    Union[AllDepth, SingleFrame, DepthWindow], Field(discriminator="mode")
+]
+
+
+class AddStoredSignal(StrictModel):
+    type: Literal['add_stored_signal']
+    index: StrictInt
+    duplicate: StrictBool = False
+    frames: tuple[StrictInt, StrictInt] | None = None
+
+
+class SignalId(StrictModel):
+    signal_id: str
+
+
+class RenameSignal(SignalId):
+    type: Literal['rename_signal']
+    name: str
+
+
+class AnnotateSignal(SignalId):
+    type: Literal['annotate_signal']
+    candidate_id: str | None = Field(max_length=64)
+    measured_mz: Annotated[float, Field(strict=True, allow_inf_nan=False, gt=0)] | None = None
+
+
+class SelectImageSignal(SignalId):
+    type: Literal['select_image_signal']
+
+
+class SetProfileVisible(SignalId):
+    type: Literal['set_profile_visible']
+    visible: StrictBool
+
+
+class RemoveSignal(SignalId):
+    type: Literal['remove_signal']
+
+
+class SetStoredCoverage(SignalId):
+    type: Literal['set_stored_coverage']
+    frames: tuple[StrictInt, StrictInt]
+
+
+class SetROI(StrictModel):
+    type: Literal["set_roi"]
+    roi: tuple[StrictInt, StrictInt, StrictInt, StrictInt]
+
+
+class SetDepth(StrictModel):
+    type: Literal["set_depth"]
+    depth: DepthSelection
+
+
+class SetDisplay(StrictModel):
+    type: Literal["set_display"]
+    log: StrictBool
+    relative: StrictBool
+    image: Literal["sims", "se"] = "sims"
+    spectrum_log: StrictBool | None = None
+    colormap: Literal["ion", "gray", "viridis", "magma", "cividis"] | None = None
+
+
+class AnchorInput(StrictModel):
+    frame: StrictInt
+    depth_nm: Annotated[float, Field(strict=True, allow_inf_nan=False)]
+
+
+class SetDepthAnchors(StrictModel):
+    type: Literal['set_depth_anchors']
+    anchors: tuple[AnchorInput, ...] = Field(max_length=4096)
+
+
+class SetDepthDisplay(StrictModel):
+    type: Literal['set_depth_display']
+    unit: Literal['nm', 'um', 'mm']
+    coordinates: Literal['frame', 'depth']
+
+
+class SetSpatial(StrictModel):
+    type: Literal['set_spatial']
+    spatial: SpatialView
+
+
+Command = Annotated[
+    Union[
+        AddStoredSignal,
+        RenameSignal,
+        AnnotateSignal,
+        SelectImageSignal,
+        SetProfileVisible,
+        RemoveSignal,
+        SetStoredCoverage,
+        SetROI,
+        SetDepth,
+        SetDisplay,
+        SetDepthAnchors,
+        SetDepthDisplay,
+        SetSpatial,
+    ],
+    Field(discriminator="type"),
+]
+command_adapter = TypeAdapter(Command)
+
+
+class CommandEnvelope(StrictModel):
+    request_id: str = Field(min_length=1, max_length=128)
+    expected_revision: StrictInt = Field(ge=0)
+    dataset_generation: str = Field(min_length=1, max_length=128)
+    command: Command
+
+    def __init__(self, *args, **kwargs):
+        if args:
+            kwargs = (
+                dict(
+                    zip(
+                        (
+                            "request_id",
+                            "expected_revision",
+                            "dataset_generation",
+                            "command",
+                        ),
+                        args,
+                    )
+                )
+                | kwargs
+            )
+        super().__init__(**kwargs)