tofscope 0.1.0__py3-none-any.whl

tofscope/application/jobs.py

@@ -0,0 +1,347 @@
+"""Bounded round-robin user scheduling, with one active job per user."""
+
+from collections import OrderedDict, deque
+from dataclasses import dataclass, replace, field
+from threading import Condition, Event, RLock, Thread
+import json
+import time
+from uuid import uuid4
+from typing import Literal
+from tofscope.storage.saved_files import SavedFile
+from tofscope.application.analyses import CapacityError
+from tofscope.application.models import SignalSnapshot
+from tofscope.application.signals import get_signal, normalize_view, valid_name
+from tofscope.science.integration import canonical_intervals
+from tofscope.science.signals import SignalRecipe
+
+TERMINAL = {"succeeded", "failed", "cancelled", "obsolete"}
+
+
+@dataclass(frozen=True)
+class IntegrationSpec:
+    intervals: tuple[tuple[float, float], ...]
+    frames: tuple[int, int]
+    name: str
+    signal_id: str | None = None
+    expected_recipe_version: int | None = None
+    candidate_id: str | None = None
+    measured_mz: float | None = None
+
+
+@dataclass(frozen=True)
+class JobSnapshot:
+    id: str
+    analysis_id: str
+    status: str
+    signal_id: str = ""
+    recipe_version: int = 0
+    done: int = 0
+    total: int = 0
+    safe_error: str | None = None
+    kind: Literal["integration", "save", "preparation", "measurement"] = "integration"
+    stage: str = ""
+    saved_file: SavedFile | None = None
+    saved_revision: int | None = None
+
+
+@dataclass
+class Job:
+    snapshot: JobSnapshot
+    owner: str
+    source: object
+    recipe: SignalRecipe
+    signal_id: str
+    token: int
+    name_revision: int
+    referenced: bool = True
+    cancelled: Event = field(default_factory=Event)
+    task: object = None
+
+
+class JobService:
+    def __init__(self, workspaces, explore, cache, settings):
+        self.analyses = workspaces
+        self.explore = explore
+        self.cache = cache
+        self.settings = settings
+        self.jobs = OrderedDict()
+        self.pending = {}
+        self.users = deque()
+        self.running = set()
+        self.condition = Condition(RLock())
+        self.stopping = False
+        self.save_handler = None
+        self.preparation_handler = None
+        self.measurement_handler = None
+        self.threads = [
+            Thread(target=self._worker, daemon=True, name=f"tofscope-job-{i}")
+            for i in range(settings.workers)
+        ]
+        workspaces.on_close.append(self.close_analysis)
+        workspaces.on_remove.append(self.remove_signal)
+        for thread in self.threads:
+            thread.start()
+
+    def submit(self, owner, analysis_id, envelope, spec):
+        e = self.analyses.entry(owner, analysis_id)
+        fingerprint = json.dumps(
+            {"envelope": envelope.model_dump(mode="json"), "spec": spec.__dict__},
+            sort_keys=True,
+        )
+        with self.analyses.signal_lock, e.lock:
+            old = self.explore.validate_envelope(e, envelope, fingerprint)
+            if old is not None:
+                return old
+            s = e.snapshot
+            name = valid_name(spec.name)
+            windows = canonical_intervals(spec.intervals, s.mass_limits)
+            from tofscope.application.identification import assignment
+            atomic_assignment = assignment(spec.candidate_id, spec.measured_mz) if spec.candidate_id else None
+            if spec.signal_id is not None:
+                signal = get_signal(s, spec.signal_id)
+                if signal.recipe is not None and signal.recipe.source != 'custom':
+                    raise ValueError('Stored channels require a custom copy before raw integration.')
+                if spec.expected_recipe_version != signal.recipe_version:
+                    raise ValueError('Signal recipe changed; refresh before calculating.')
+            else:
+                self.analyses.admit_signal(owner, len(s.signals))
+                signal = SignalSnapshot(str(uuid4()), name, s.next_style_index, None, 0)
+            first, last = spec.frames
+            if (
+                type(first) is not int
+                or type(last) is not int
+                or not 0 <= first <= last < s.shape[0]
+            ):
+                raise ValueError("Integration frames outside stored domain.")
+            with self.condition:
+                active = [
+                    j for j in self.jobs.values() if j.snapshot.status not in TERMINAL
+                ]
+                if (
+                    len(active) >= self.settings.jobs_global
+                    or sum(j.owner == owner for j in active)
+                    >= self.settings.jobs_per_user
+                    or sum(j.snapshot.analysis_id == analysis_id for j in active)
+                    >= self.settings.jobs_per_analysis
+                ):
+                    raise CapacityError("Integration queue full; wait or cancel a job.")
+                token = e.tokens.get(signal.id, 0) + 1
+                e.tokens[signal.id] = token
+                recipe = SignalRecipe(id=signal.id, intervals=windows, frames=tuple(spec.frames), label=name)
+                snap = JobSnapshot(str(uuid4()), analysis_id, 'queued', signal.id,
+                                   signal.recipe_version+1, total=last-first+1)
+                self.jobs[snap.id] = Job(snap, owner, s.dataset, recipe, signal.id, token, signal.name_revision)
+                if signal.job_id in self.jobs:
+                    self.jobs[signal.job_id].referenced = False
+                updated = replace(signal, job_id=snap.id, assignment=atomic_assignment if spec.candidate_id else signal.assignment)
+                signals = tuple(updated if r.id == signal.id else r for r in s.signals)
+                if spec.signal_id is None:
+                    signals = (*signals, updated)
+                e.snapshot = replace(s, signals=signals, revision=s.revision+1,
+                                     next_style_index=s.next_style_index+(spec.signal_id is None))
+                self.pending.setdefault(owner, deque()).append(snap.id)
+                if owner not in self.users:
+                    self.users.append(owner)
+                self.explore.receipt(e, envelope.request_id, fingerprint, snap)
+                self.condition.notify_all()
+                # Keep a bounded terminal receipt history; live jobs are never removed.
+                terminal = [
+                    id
+                    for id, j in self.jobs.items()
+                    if j.owner == owner and j.snapshot.status in TERMINAL and not j.referenced
+                ]
+                for id in terminal[:-256]:
+                    del self.jobs[id]
+                return snap
+
+    def enqueue_save(self, owner, task):
+        return self.enqueue_task(owner, task, 'save', task.snapshot.id, task.snapshot.revision)
+
+    def enqueue_task(self, owner, task, kind, aid, saved_revision=None):
+        with self.condition:
+            active = [j for j in self.jobs.values() if j.snapshot.status not in TERMINAL]
+            if (len(active) >= self.settings.jobs_global or
+                    sum(j.owner == owner for j in active) >= self.settings.jobs_per_user or
+                    sum(j.snapshot.analysis_id == aid for j in active) >= self.settings.jobs_per_analysis):
+                raise CapacityError('Work queue full; wait or cancel a job.')
+            snap = JobSnapshot(str(uuid4()), aid, 'queued', kind=kind, saved_revision=saved_revision)
+            source = task.snapshot.dataset if kind in {'save', 'measurement'} else task.source
+            self.jobs[snap.id] = Job(snap, owner, source, None, '', 0, 0, referenced=False, task=task)
+            self.pending.setdefault(owner, deque()).append(snap.id)
+            if owner not in self.users:
+                self.users.append(owner)
+            terminal = [id for id, j in self.jobs.items() if j.owner == owner and j.snapshot.status in TERMINAL and not j.referenced]
+            for id in terminal[:-256]:
+                del self.jobs[id]
+            self.condition.notify_all()
+            return snap
+
+    def get(self, owner, job_id):
+        with self.condition:
+            j = self.jobs.get(job_id)
+            if j is None or j.owner != owner:
+                raise KeyError("Job unavailable.")
+            return j.snapshot
+
+    def cancel(self, owner, job_id):
+        with self.condition:
+            snap = self.get(owner, job_id)
+            j = self.jobs[job_id]
+            if snap.status not in TERMINAL:
+                j.cancelled.set()
+                if snap.status == "queued":
+                    j.snapshot = replace(snap, status="cancelled")
+            self.condition.notify_all()
+            return j.snapshot
+
+    def remove_signal(self, analysis_id, signal_id):
+        with self.condition:
+            for jid, job in self.jobs.items():
+                if job.snapshot.analysis_id == analysis_id and job.signal_id == signal_id:
+                    job.referenced = False
+                    self.cancel(job.owner, jid)
+
+    def close_analysis(self, analysis_id):
+        with self.condition:
+            for jid, job in self.jobs.items():
+                if job.snapshot.analysis_id == analysis_id:
+                    job.referenced = False
+                    self.cancel(job.owner, jid)
+
+    def _next(self):
+        for _ in range(len(self.users)):
+            owner = self.users.popleft()
+            q = self.pending[owner]
+            while q and (q[0] not in self.jobs or self.jobs[q[0]].snapshot.status in TERMINAL):
+                q.popleft()
+            if not q:
+                continue
+            self.users.append(owner)
+            if owner in self.running:
+                continue
+            id = q.popleft()
+            self.running.add(owner)
+            j = self.jobs[id]
+            j.snapshot = replace(j.snapshot, status="running")
+            return j
+        return None
+
+    def _worker(self):
+        while True:
+            with self.condition:
+                j = self._next()
+                while j is None and not self.stopping:
+                    self.condition.wait()
+                    j = self._next()
+                if self.stopping and j is None:
+                    return
+            lease = None
+            try:
+                if j.snapshot.kind in {'save', 'preparation', 'measurement'}:
+                    if j.cancelled.is_set():
+                        raise InterruptedError()
+                    handler = {'save': self.save_handler, 'preparation': self.preparation_handler, 'measurement': self.measurement_handler}[j.snapshot.kind]
+                    result = handler(j)
+                    with self.condition:
+                        j.snapshot = replace(j.snapshot, status='succeeded', saved_file=result, done=j.snapshot.total)
+                    continue
+                aid = j.snapshot.analysis_id
+                e = self.analyses.entry(j.owner, aid)
+
+                def current():
+                    return (
+                        not e.closed
+                        and e.snapshot.dataset == j.source
+                        and e.tokens.get(j.signal_id) == j.token
+                        and any(r.id == j.signal_id for r in e.snapshot.signals)
+                    )
+
+                def cancelled():
+                    return j.cancelled.is_set() or not current()
+
+                if cancelled():
+                    raise InterruptedError()
+
+                def progress(done, total):
+                    with self.condition:
+                        j.snapshot = replace(j.snapshot, done=done, total=total)
+
+                lease = self.cache.load(
+                    j.owner,
+                    aid,
+                    j.source,
+                    j.recipe,
+                    progress=progress,
+                    cancel=cancelled,
+                )
+                with e.lock:
+                    if cancelled():
+                        raise InterruptedError()
+                    self.cache.repository.assert_current(j.owner, j.source)
+                    s = e.snapshot
+                    old = get_signal(s, j.signal_id)
+                    name = old.name if old.name_revision != j.name_revision else j.recipe.label
+                    updated = replace(old, name=name, recipe=j.recipe, recipe_version=j.snapshot.recipe_version, materialization_id=None)
+                    result = normalize_view(replace(s, revision=s.revision+1,
+                        signals=tuple(updated if r.id == old.id else r for r in s.signals),
+                        profile_signal_ids=tuple(dict.fromkeys((*s.profile_signal_ids, old.id))) if old.recipe is None else s.profile_signal_ids))
+                    if result.isotope_measurement:
+                        from tofscope.application.identification import current_result
+                        result = replace(result, isotope_measurement=current_result(result))
+                    previous = e.leases.pop(old.id, None)
+                    if previous:
+                        previous.release()
+                    if old.id in result.profile_signal_ids or old.id == result.image_signal_id:
+                        e.leases[old.id] = lease
+                        lease = None
+                    e.snapshot = result
+                    with self.condition:
+                        j.snapshot = replace(
+                            j.snapshot, status="succeeded", done=j.snapshot.total
+                        )
+            except (InterruptedError, KeyError):
+                with self.condition:
+                    j.snapshot = replace(
+                        j.snapshot,
+                        status="cancelled" if j.cancelled.is_set() else "obsolete",
+                    )
+            except Exception as exc:
+                with self.condition:
+                    j.snapshot = replace(
+                        j.snapshot,
+                        status="cancelled" if j.cancelled.is_set() else "obsolete"
+                        if "changed" in str(exc).lower()
+                        else "failed",
+                        safe_error=("Isotope measurement failed; check source, coverage and memory limits." if j.snapshot.kind == "measurement" else "Preparing raw data failed; check free space, source/calibration and output name." if j.snapshot.kind == "preparation" else "Saving failed; check free space, output name and current analysis revision." if j.snapshot.kind == "save" else "Integration failed; check source/calibration or narrow the frame range.")
+                        if not isinstance(exc, CapacityError)
+                        else str(exc),
+                    )
+            finally:
+                if lease:
+                    lease.release()
+                with self.condition:
+                    self.running.discard(j.owner)
+                    self.condition.notify_all()
+
+    def wait(self, owner, job_id, timeout=30):
+        deadline = time.monotonic() + timeout
+        with self.condition:
+            while True:
+                s = self.get(owner, job_id)
+                if s.status in TERMINAL:
+                    return s
+                left = deadline - time.monotonic()
+                if left <= 0:
+                    raise TimeoutError("Job did not complete.")
+                self.condition.wait(left)
+
+    def shutdown(self):
+        with self.condition:
+            self.stopping = True
+            for j in self.jobs.values():
+                if j.snapshot.status not in TERMINAL:
+                    self.cancel(j.owner, j.snapshot.id)
+            self.condition.notify_all()
+        for thread in self.threads:
+            thread.join(timeout=35)
+        self.cache.clear()

tofscope/application/lifecycle.py

@@ -0,0 +1,80 @@
+from dataclasses import dataclass
+from threading import RLock
+import time
+from uuid import uuid4
+
+
+@dataclass(frozen=True)
+class ViewLease:
+    id: str
+    analysis_id: str
+    expires_at: float
+
+
+class LifecycleService:
+    def __init__(self, workspaces, jobs, settings, clock=time.monotonic):
+        self.analyses = workspaces
+        self.jobs = jobs
+        self.settings = settings
+        self.clock = clock
+        self.views = {}
+        self.last_seen = {}
+        self.lock = RLock()
+        workspaces.on_close.append(self.closed)
+
+    def attach(self, owner, analysis_id):
+        self.analyses.get(owner, analysis_id)
+        with self.lock:
+            now = self.clock()
+            lease = ViewLease(
+                str(uuid4()), analysis_id, now + self.settings.lease_seconds
+            )
+            self.views[lease.id] = (owner, lease)
+            self.last_seen[analysis_id] = now
+            return lease
+
+    def heartbeat(self, owner, vid):
+        with self.lock:
+            o, lease = self.views.get(vid, (None, None))
+            if o != owner:
+                raise KeyError("View unavailable.")
+            self.analyses.get(owner, lease.analysis_id)
+            now = self.clock()
+            lease = ViewLease(vid, lease.analysis_id, now + self.settings.lease_seconds)
+            self.views[vid] = (owner, lease)
+            self.last_seen[lease.analysis_id] = now
+            return lease
+
+    def detach(self, owner, vid):
+        with self.lock:
+            o, lease = self.views.get(vid, (None, None))
+            if o != owner:
+                raise KeyError("View unavailable.")
+            self.last_seen[lease.analysis_id] = self.clock()
+            del self.views[vid]
+
+    def closed(self, aid):
+        with self.lock:
+            self.views = {
+                v: pair for v, pair in self.views.items() if pair[1].analysis_id != aid
+            }
+            self.last_seen.pop(aid, None)
+
+    def sweep(self):
+        now = self.clock()
+        with self.lock:
+            live = {v.analysis_id for _, v in self.views.values() if v.expires_at > now}
+            for e in list(self.analyses.entries.values()):
+                s = e.snapshot
+                if s.id in live:
+                    self.last_seen[s.id] = now
+                    continue
+                absence = now - self.last_seen.setdefault(s.id, now)
+                if absence >= self.settings.reconnect_seconds:
+                    self.jobs.close_analysis(s.id)
+                if absence >= self.settings.idle_seconds:
+                    try:
+                        self.analyses.close(s.owner, s.id)
+                    except KeyError:
+                        pass
+            self.views = {k: v for k, v in self.views.items() if v[1].expires_at > now}

tofscope/application/models.py

@@ -0,0 +1,74 @@
+from tofscope.application.spatial import SpatialView
+from tofscope.application.identification_models import AtomicAssignment, MeasurementResult
+from dataclasses import dataclass
+from typing import Literal
+from tofscope.science.depth_calibration import DepthAnchor
+from tofscope.storage.acquisitions import DatasetRef
+from tofscope.science.signals import SignalRecipe
+from tofscope.application.commands import DepthSelection
+
+
+@dataclass(frozen=True)
+class DisplaySettings:
+    log: bool = True
+    relative: bool = True
+    image: str = "sims"
+    spectrum_log: bool = True
+    colormap: Literal["ion", "gray", "viridis", "magma", "cividis"] = "viridis"
+
+
+    def __post_init__(self):
+        # Saved analyses and older clients may still use the retired palette name.
+        if self.colormap == "ion":
+            object.__setattr__(self, "colormap", "viridis")
+
+
+@dataclass(frozen=True)
+class Peak:
+    index: int
+    label: str
+    mass: float
+    low: float
+    high: float
+
+
+@dataclass(frozen=True)
+class SignalSnapshot:
+    id: str
+    name: str
+    style_index: int
+    recipe: SignalRecipe | None
+    recipe_version: int = 1
+    name_revision: int = 0
+    job_id: str | None = None
+    materialization_id: str | None = None
+    assignment: AtomicAssignment | None = None
+
+
+@dataclass(frozen=True)
+class AnalysisSnapshot:
+    id: str
+    owner: str
+    dataset: DatasetRef
+    revision: int
+    roi: tuple[int, int, int, int]
+    depth: DepthSelection
+    signals: tuple[SignalSnapshot, ...]
+    image_signal_id: str
+    profile_signal_ids: tuple[str, ...]
+    next_style_index: int
+    display: DisplaySettings
+    shape: tuple[int, int, int, int]
+    peaks: tuple[Peak, ...]
+    fov_um: float | None
+    mass_limits: tuple[float, float]
+    stale: bool = False
+    has_se: bool = False
+    saved_revision: int | None = None
+    opened_revision: int = 0
+    depth_anchors: tuple[DepthAnchor, ...] = (DepthAnchor(0, 0.),)
+    depth_unit: Literal['nm', 'um', 'mm'] = 'nm'
+    depth_coordinates: Literal['frame', 'depth'] = 'frame'
+    isotope_measurement: MeasurementResult | None = None
+    isotope_job_id: str | None = None
+    spatial: SpatialView = SpatialView()

tofscope/application/preparations.py

@@ -0,0 +1,46 @@
+"""Explicit generation-bound raw preparations on the shared owner work queue."""
+from dataclasses import dataclass, replace
+from uuid import uuid4
+import os
+
+from tofscope.storage.raw_preparation import inspect_source, prepare_copy
+from tofscope.storage.saved_files import filename
+
+
+@dataclass(frozen=True)
+class PreparationTask:
+    id: str
+    source: object
+    name: str
+    required_bytes: int
+
+
+class PreparationService:
+    def __init__(self, repository, jobs, files):
+        self.repository, self.jobs, self.files = repository, jobs, files
+        jobs.preparation_handler = self.run
+
+    def inspect(self, owner, path, location='acquisition'):
+        return inspect_source(self.repository, owner, path, location)[1]
+
+    def submit(self, owner, path, location, generation, name, *, accept_assumptions=False):
+        name = filename(name)
+        source, preview = inspect_source(self.repository, owner, path, location)
+        if preview.generation != generation:
+            raise ValueError('Acquisition changed; inspect it again before preparation.')
+        if not preview.requires_preparation:
+            raise ValueError('This file already contains integrated peak data.')
+        if preview.assumptions and not accept_assumptions:
+            raise ValueError('Missing acquisition metadata requires explicit acceptance of the reconstruction assumptions.')
+        if os.path.lexists(self.files.owner_root(owner) / name):
+            raise FileExistsError('Output already exists; choose a new name.')
+        task = PreparationTask(str(uuid4()), source, name, preview.required_bytes)
+        return self.jobs.enqueue_task(owner, task, 'preparation', task.id)
+
+    def run(self, job):
+        def progress(stage, done, total):
+            with self.jobs.condition:
+                job.snapshot = replace(job.snapshot, stage=stage, done=done, total=total)
+        task = job.task
+        return prepare_copy(self.repository, self.files, job.owner, task.source, task.name,
+                            task.required_bytes, job.cancelled, progress, task.id)

tofscope/application/resources.py

@@ -0,0 +1,63 @@
+"""Admission for managed arrays and transport buffers; allocator RSS is separate."""
+
+from collections import defaultdict
+from threading import RLock
+from tofscope.application.analyses import CapacityError
+
+
+class Reservation:
+    def __init__(self, budget, owner, analysis_id, n, physical):
+        self.budget = budget
+        self.owner = owner
+        self.analysis_id = analysis_id
+        self.n = n
+        self.physical = physical
+        self.released = False
+
+    def release(self):
+        with self.budget.lock:
+            if self.released:
+                return
+            self.released = True
+            if self.physical:
+                self.budget.physical_bytes -= self.n
+            if self.owner is not None:
+                self.budget.users[self.owner] -= self.n
+                self.budget.analyses[self.analysis_id] -= self.n
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.release()
+
+
+class ResourceBudget:
+    def __init__(self, settings):
+        self.settings = settings
+        self.lock = RLock()
+        self.physical_bytes = 0
+        self.users = defaultdict(int)
+        self.analyses = defaultdict(int)
+
+    def reserve(self, owner, analysis_id, bytes, physical=True):
+        if type(bytes) is not int or bytes < 0:
+            raise ValueError("An exact nonnegative allocation size is required.")
+        with self.lock:
+            s = self.settings
+            if (physical and self.physical_bytes + bytes > s.global_bytes) or (
+                owner is not None
+                and (
+                    self.users[owner] + bytes > s.user_bytes
+                    or self.analyses[analysis_id] + bytes > s.analysis_bytes
+                )
+            ):
+                raise CapacityError(
+                    "Memory budget reached; hide profiles or close an analysis."
+                )
+            if physical:
+                self.physical_bytes += bytes
+            if owner is not None:
+                self.users[owner] += bytes
+                self.analyses[analysis_id] += bytes
+            return Reservation(self, owner, analysis_id, bytes, physical)

tofscope/application/saves.py

@@ -0,0 +1,76 @@
+"""Revision-bound saved copies run through the shared fair job scheduler."""
+from contextlib import contextmanager
+from dataclasses import dataclass, replace
+import json
+
+from tofscope.application.analyses import Conflict
+from tofscope.storage.analysis_document import encode_document
+from tofscope.storage.saved_files import SavedFileStore, COPY_BYTES, filename
+
+
+@dataclass(frozen=True)
+class SaveTask:
+    snapshot: object
+    document: object
+    name: str
+    directory: str = ''
+    destination_location: str = 'saved'
+
+
+class SaveService:
+    def __init__(self, workspaces, jobs, cache, repository, settings):
+        self.analyses, self.jobs, self.cache, self.repository = workspaces, jobs, cache, repository
+        self.files = SavedFileStore(settings)
+        jobs.save_handler = self.run
+
+    def submit(self, owner, analysis_id, envelope, name):
+        name = filename(name)
+        directory = getattr(envelope, 'directory', '')
+        location = getattr(envelope, 'destination_location', 'saved')
+        e = self.analyses.entry(owner, analysis_id)
+        fingerprint = json.dumps({'kind': 'save', 'envelope': envelope.model_dump(mode='json'), 'filename': name}, sort_keys=True)
+        with e.lock:
+            old = self.jobs.explore.validate_envelope(e, envelope, fingerprint)
+            if old is not None:
+                return old
+            try:
+                self.files.directory(owner, directory, location, writable=True)
+            except FileNotFoundError:
+                raise ValueError('Save folder is unavailable.') from None
+            with self.jobs.condition:
+                if any(j.snapshot.analysis_id == analysis_id and j.snapshot.kind in {'integration', 'measurement'} and j.snapshot.status not in {'succeeded', 'failed', 'cancelled', 'obsolete'} for j in self.jobs.jobs.values()):
+                    raise ValueError('Finish or cancel calculations before saving.')
+                task = SaveTask(e.snapshot, encode_document(e.snapshot), name, directory, location)
+                result = self.jobs.enqueue_save(owner, task)
+                self.jobs.explore.receipt(e, envelope.request_id, fingerprint, result)
+                return result
+
+    def run(self, job):
+        task = job.task
+        snapshot = task.snapshot
+        entry = self.analyses.entry(job.owner, snapshot.id)
+        def progress(stage, done, total):
+            from dataclasses import replace
+            with self.jobs.condition:
+                job.snapshot = replace(job.snapshot, stage=stage, done=done, total=total)
+        def frames(signal_id):
+            row = next(x for x in snapshot.signals if x.id == signal_id)
+            with self.cache.load(job.owner, snapshot.id, snapshot.dataset, row.recipe, materialization_id=row.materialization_id, cancel=job.cancelled.is_set) as lease:
+                yield from lease.array
+        @contextmanager
+        def publish_guard():
+            with entry.lock:
+                if entry.closed or job.cancelled.is_set():
+                    raise InterruptedError()
+                if entry.snapshot.revision != snapshot.revision or entry.snapshot.dataset != snapshot.dataset:
+                    raise Conflict('Analysis changed during save; save the current revision again.')
+                self.repository.assert_current(job.owner, snapshot.dataset)
+                yield
+                entry.snapshot = replace(entry.snapshot, saved_revision=snapshot.revision)
+        self.repository.assert_current(job.owner, snapshot.dataset)
+        path = self.repository.path(job.owner, snapshot.dataset)
+        # Native mass-bin validation processes bounded 65,536-element blocks.
+        scratch = COPY_BYTES + snapshot.shape[1] * snapshot.shape[2] * 12 + 4 * 1024**2
+        with self.cache.budget.reserve(job.owner, snapshot.id, scratch):
+            return self.files.publish_copy(job.owner, path, task.document, frames, task.name,
+                                           job.cancelled, progress, publish_guard, task.directory, task.destination_location)