PyLV200 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pylv200/cli.py ADDED
@@ -0,0 +1,2073 @@
1
+ #!/usr/bin/env python3
2
+ """Command line for pylv200 - pull a cellSens experiment that is still running.
3
+
4
+ cellSens does not finalise its image container until a run ends, so a nine-day
5
+ recording is unreadable for nine days. The pixels are already on disk though, as
6
+ `.ets` tile files, and this reads them where they lie.
7
+
8
+ pylv200 list left-lv200
9
+ pylv200 plan left-lv200:ois351A --last-n 24
10
+ pylv200 sync left-lv200:ois351A -o D:/pulled
11
+ pylv200 --json list left-lv200
12
+
13
+ Strictly read-only on the microscope: files are opened for reading and nothing
14
+ is written, renamed or locked inside an experiment folder. Safe to run
15
+ mid-acquisition.
16
+
17
+ The `.ets` layout is self-describing but its field offsets vary between cellSens
18
+ versions, so the parser probes and scores rather than trusting a layout.
19
+ `pull --dry-run` prints exactly what it worked out; check that before trusting
20
+ the pixels.
21
+
22
+ This is the only module that prints or sets an exit code. Everything it calls
23
+ raises instead.
24
+ """
25
+ from __future__ import annotations
26
+
27
+ import argparse
28
+ import contextlib
29
+ import io
30
+ import json
31
+ import logging
32
+ import re
33
+ import sys
34
+ import threading
35
+ from datetime import datetime
36
+ from dataclasses import replace
37
+ from pathlib import Path
38
+
39
+ from . import __version__
40
+ from .client import LV200Client
41
+ from . import config
42
+ from . import engine
43
+ from .engine import derive_interval
44
+ from .errors import (AmbiguousDataset, InvalidSelection, LV200Error,
45
+ NoDatasetFound)
46
+ from .manifest import MANIFEST_FILENAME
47
+ from .models import Source, human_bytes
48
+ from .options import (DEFAULT_BATCH_FRAMES, LAYOUTS, LAYOUT_DESCRIPTIONS,
49
+ MOMENT_HELP, STATE_SCOPES, parse_index,
50
+ resolve_layout)
51
+ from . import preview as lv200preview
52
+ from . import setup as lv200setup
53
+ from .state import StateStore
54
+ from . import watch as lv200watch
55
+ from .timeline import DEFAULT_BYTE_BUDGET, DEFAULT_PROJECTION_FRAMES
56
+
57
+
58
+ #: Exit code for work a caller stopped. Distinct from 1 ("nothing to do") and
59
+ #: 2 ("it failed"), because a scheduled task treats all three differently.
60
+ STOPPED = 3
61
+
62
+ #: `scans` lists things every other part of this package calls timepoints, and
63
+ #: that mismatch has to be explained where somebody meets it rather than only
64
+ #: in a planning document.
65
+ SCANS_HELP = """\
66
+ List what a run has acquired so far, one line per timepoint.
67
+
68
+ Two words for one thing, deliberately: the command is `scans`, which is
69
+ what PyIncucyte calls it and what a microscope operator says, but
70
+ everything this package writes calls them timepoints - the number
71
+ `--from-t` takes, the manifest column, the resume ledger key. So
72
+ `pylv200 timepoints` works too, and --json says "timepoints".
73
+
74
+ Times are DERIVED from the run's start and its frame interval. An .ets
75
+ records no per-tile timestamp, and nothing here reads a tile.
76
+ """
77
+
78
+
79
+ def _cancelled(args):
80
+ """Has the caller asked to stop?
81
+
82
+ ``args.cancel`` is a ``threading.Event`` the desktop app sets from its
83
+ Cancel button; nothing sets it on the command line. It is checked between
84
+ positions and again before a stack is built - not inside the tile loop, so
85
+ a pull already writing one position finishes that position. Saying it stops
86
+ sooner than it does would be worse than the wait.
87
+ """
88
+ event = getattr(args, "cancel", None)
89
+ return event is not None and event.is_set()
90
+
91
+
92
+ def _recipe(args, exp_name=None):
93
+ """The resolved recipe: command line > pinned > preset > source > built-in.
94
+
95
+ Called once before anything is parsed, so the mirror and the channel
96
+ selection are known before a tile is read, and again with the experiment
97
+ name once the sidecar has given one - a per-experiment pin is keyed on that
98
+ name, and nothing can know it until the `.vsi` has been opened.
99
+ """
100
+ source = config.get_source(args.source) if getattr(args, "source", None) \
101
+ else None
102
+ key = config.experiment_key(source.name if source else "local", exp_name) \
103
+ if exp_name else None
104
+ return config.resolve_options(cli=_given(args), key=key,
105
+ preset=getattr(args, "preset", None),
106
+ source=source), source, key
107
+
108
+
109
+ # --------------------------------------------------------------------------
110
+ # machine setup: three commands that are about the microscope, not the data
111
+ #
112
+ # These arrive as subcommands while everything else is still flags. Stage 10
113
+ # turns the whole front end into subparsers; until then a name in this set is
114
+ # routed here and anything else falls through to the flat parser, so nothing
115
+ # that already works changes.
116
+ # --------------------------------------------------------------------------
117
+
118
+ def _cmd_setup(args):
119
+ say("walking %s" % (", ".join(args.drives) if args.drives
120
+ else "every fixed drive"))
121
+ seen = [0]
122
+
123
+ def progress(path):
124
+ seen[0] += 1
125
+ if not args.quiet and seen[0] % 500 == 0:
126
+ say(" %d folders... %s" % (seen[0], path))
127
+
128
+ report = lv200setup.inspect(account=args.account, drives=args.drives,
129
+ max_depth=args.max_depth, progress=progress)
130
+ say(" searched %d folders on %s"
131
+ % (seen[0], " ".join(report.drives) or "nothing"))
132
+
133
+ if report.datasets:
134
+ say("\ncellSens data found in %d folder(s):" % len(report.datasets))
135
+ for path in report.datasets[:20]:
136
+ say(" %s" % path)
137
+ if len(report.datasets) > 20:
138
+ say(" ... and %d more" % (len(report.datasets) - 20))
139
+ say("\nroots to record:")
140
+ for root in report.roots:
141
+ say(" %s" % root)
142
+ else:
143
+ say("\nno cellSens data was found. It is recognised by the first four "
144
+ "bytes of\na file, never by a folder name, so a run in an oddly "
145
+ "named folder still\ncounts - but one on a drive that was not "
146
+ "searched does not. Try --drives.")
147
+
148
+ if report.registry:
149
+ agrees = report.agrees_with_registry()
150
+ say("\ncellSens's own record of an output path:")
151
+ for where, name, value in report.registry[:6]:
152
+ say(" %s\\%s = %s" % (where, name, value))
153
+ say(" -> %s the folders found above"
154
+ % ("agrees with" if agrees else "DISAGREES with"))
155
+ else:
156
+ say("\nno Olympus/Evident registry entry naming an output path. That "
157
+ "is a normal\nresult - nobody has confirmed cellSens records one - "
158
+ "and it changes nothing.")
159
+
160
+ for note in report.notes:
161
+ say(" ** %s" % note)
162
+
163
+ blockers = report.blockers()
164
+ if blockers:
165
+ say("\nwhat is stopping a pull from this machine:")
166
+ for key, detail in blockers:
167
+ say(" %-16s %s" % (key.replace("_", " "), detail))
168
+ say(" %s" % lv200setup.SYMPTOMS[key])
169
+ else:
170
+ say("\nnothing is stopping a pull: the profile is Private, the "
171
+ "firewall rules are\non, %s exists, and a share covers the roots."
172
+ % args.account)
173
+
174
+ if report.account_is_admin:
175
+ say("\n ** %s is in the local Administrators group. A broad "
176
+ "read-only share is\n safe only because the account is an "
177
+ "ordinary user; as an administrator\n the same share reads "
178
+ "every profile and the registry hives. No share will\n be "
179
+ "created while that is true." % args.account)
180
+
181
+ plan = lv200setup.build_plan(report, grant_scope=args.grant_scope)
182
+ if not args.create_share:
183
+ if plan:
184
+ say("\n%d change(s) would fix that. Re-run with --create-share to "
185
+ "see the plan." % len(plan))
186
+ return 0 if not blockers else 1
187
+
188
+ say("\n" + lv200setup.format_plan(report, plan, args.grant_scope))
189
+ if not args.yes:
190
+ return 1
191
+
192
+ undo = Path(lv200setup.UNDO_FILENAME)
193
+ undo.write_text(lv200setup.undo_text(plan, args.account), encoding="utf-8")
194
+ say("\nwrote %s BEFORE making any change, so an interrupted run is still "
195
+ "reversible." % undo)
196
+
197
+ results = lv200setup.apply_plan(plan, report=report, yes=True)
198
+ failed = [(a, c) for a, c, ok, _ in results if not ok]
199
+ for action, command, ok, _ in results:
200
+ say(" %-4s %s" % ("ok" if ok else "FAIL", command))
201
+ if failed:
202
+ say("\n%d command(s) failed. Run them yourself in an elevated "
203
+ "PowerShell." % len(failed))
204
+ return 1
205
+
206
+ source = lv200setup.source_block(report, host=args.host)
207
+ block = json.dumps({"sources": {source.name: {
208
+ "host": source.host, "address": source.address, "share": source.share,
209
+ "roots": list(source.roots), "account": source.account,
210
+ "output": ""}}}, indent=2)
211
+ say("\nrecord this machine on the analysis PC - paste into %s:\n"
212
+ % config._path(config.SOURCES_FILE))
213
+ say(block)
214
+ if args.save_to:
215
+ Path(args.save_to).write_text(block, encoding="utf-8")
216
+ say("\nalso written to %s" % args.save_to)
217
+ say("\nthen, once, on the analysis PC: pylv200 login %s" % source.name)
218
+ return 0
219
+
220
+
221
+ def _cmd_probe(args):
222
+ source = source_for(args.source)
223
+ result = lv200setup.probe(source)
224
+ for step in result["steps"]:
225
+ say(" %-4s %-6s %s" % ("ok" if step["ok"] else "FAIL",
226
+ step["layer"], step["detail"]))
227
+ for note in result.get("notes", ()):
228
+ say(" ** %s" % note)
229
+ if result["ok"]:
230
+ return 0
231
+ if result["failed"] in ("name", "host", "share"):
232
+ return 2
233
+ return 1
234
+
235
+
236
+ def _cmd_login(args):
237
+ source = config.get_source(args.source)
238
+ command = config.credential_command(source)
239
+ # Worded without the obvious noun on purpose: tests/test_config.py greps
240
+ # this package for it, and a guard that has to reason about which lines are
241
+ # prose is a guard that eventually lets a real one through.
242
+ say("Windows will now prompt for the credential for %s.\nIt is stored in "
243
+ "Credential Manager and never reaches pylv200, so it cannot appear in "
244
+ "a\nconfig file, a log line or a traceback." % source.account)
245
+ config.store_credential(source)
246
+ say("\nstored for %s. Check with: cmdkey /list:%s"
247
+ % (source.target, source.target))
248
+ return 0
249
+
250
+
251
+ def _cmd_rescan(args):
252
+ source = config.get_source(args.source)
253
+ previous = config.get_root_counts(source.name)
254
+ result = lv200setup.rescan(source, previous=previous)
255
+ for warning in result["warnings"]:
256
+ say(" ** %s" % warning)
257
+ for note in result["notes"]:
258
+ say(" -- %s" % note)
259
+ say(result["message"])
260
+ if not result["roots"]:
261
+ return 1
262
+ if result["moved"] and not args.yes:
263
+ say("\nre-run with --yes to record these roots.")
264
+ return 1
265
+ if result["moved"]:
266
+ config.save_source(replace(source, roots=tuple(result["roots"])))
267
+ say("recorded.")
268
+ config.record_root_counts(source.name, result.get("counts", {}))
269
+ return 0
270
+
271
+
272
+ # --------------------------------------------------------------------------
273
+ # the front end
274
+ #
275
+ # Shaped like PyIncucyte's so the two tools are driven the same way. Every
276
+ # command here formats and prints; everything it calls raises instead.
277
+ # --------------------------------------------------------------------------
278
+
279
+ #: Set by main() before anything runs. Under --json the human commentary moves
280
+ #: to stderr so stdout holds the payload and nothing else - a friendly line on
281
+ #: stdout corrupts the output of whatever is parsing it.
282
+ _JSON = [False]
283
+
284
+ #: Where output goes when something other than a terminal is driving this.
285
+ #: The desktop app installs itself here and then calls the very same command
286
+ #: functions, so there is one pull, not one per front end - which is the only
287
+ #: way two front ends can be guaranteed to write the same bytes.
288
+ _CONSOLE = [None]
289
+
290
+
291
+ @contextlib.contextmanager
292
+ def console(on_say=None, on_emit=None):
293
+ """Route this module's output to callables instead of stdout.
294
+
295
+ Nested and restored on the way out, so a caller that installs one cannot
296
+ leave the command line silent afterwards.
297
+ """
298
+ previous = _CONSOLE[0]
299
+ _CONSOLE[0] = (on_say, on_emit)
300
+ try:
301
+ yield
302
+ finally:
303
+ _CONSOLE[0] = previous
304
+
305
+
306
+ #: The progress bar currently on screen, if any. `say` wipes it before
307
+ #: printing: the bar is one rewritten line on stderr and a `say` is a fresh
308
+ #: line on stdout, so without this a line lands on top of the bar and leaves
309
+ #: the tail of it behind, on screen, forever.
310
+ _BAR = [None]
311
+
312
+
313
+ def say(text=""):
314
+ """Human-readable progress. stderr under --json, stdout otherwise."""
315
+ installed = _CONSOLE[0]
316
+ if installed and installed[0]:
317
+ installed[0](text)
318
+ return
319
+ if _BAR[0] is not None:
320
+ _BAR[0].clear()
321
+ print(text, file=sys.stderr if _JSON[0] else sys.stdout)
322
+
323
+
324
+ def result(args, payload):
325
+ """One position's outcome. Collected rather than printed when a run covers
326
+ several, so a multi-position pull emits one JSON document, not one per
327
+ stack - two documents on stdout is not JSON."""
328
+ sink = getattr(args, "sink", None)
329
+ if sink is None:
330
+ emit(payload)
331
+ else:
332
+ sink.append(payload)
333
+
334
+
335
+ def emit(payload, lines=()):
336
+ """The one place both output modes are produced, so they cannot disagree."""
337
+ installed = _CONSOLE[0]
338
+ if installed and installed[1]:
339
+ installed[1](payload, list(lines))
340
+ return
341
+ if _JSON[0]:
342
+ print(json.dumps(payload, indent=2, default=str))
343
+ else:
344
+ for line in lines:
345
+ print(line)
346
+
347
+
348
+ def print_table(rows, headers, align=""):
349
+ """An aligned plain-text table, **returned rather than printed**.
350
+
351
+ PyIncucyte's prints. Here it cannot: `emit` is the one place both output
352
+ modes are produced, and a table that printed itself would put stray columns
353
+ on stdout in `--json` mode, where stdout is a document something else is
354
+ parsing. So this hands lines back and the caller passes them to `emit`.
355
+
356
+ ``align`` is one letter per column, "l" or "r"; short strings pad with "l".
357
+ Counts read wrong left-aligned - the eye compares the wrong digits.
358
+ """
359
+ if not rows:
360
+ return []
361
+ body = [[("" if cell is None else str(cell)) for cell in row]
362
+ for row in rows]
363
+ widths = [max(len(cell) for cell in column)
364
+ for column in zip(*([list(headers)] + body))]
365
+ align = (align + "l" * len(widths))[:len(widths)]
366
+
367
+ def lay(cells):
368
+ return " " + " ".join(
369
+ cell.rjust(w) if a == "r" else cell.ljust(w)
370
+ for cell, w, a in zip(cells, widths, align)).rstrip()
371
+
372
+ return [lay(headers), " " + " ".join("-" * w for w in widths)] + [lay(row) for row in body]
373
+
374
+
375
+ class ConsoleProgress:
376
+ """A one-line rewriting progress bar, for a terminal and nothing else.
377
+
378
+ Three things switch it off, and all three matter:
379
+
380
+ * not a TTY - `\r` and padding written into a redirected file are noise a
381
+ pipeline then has to strip;
382
+ * `--json` or `--quiet` - the first has a document on stdout to keep clean,
383
+ the second was asked for silence;
384
+ * the desktop app capturing output through `console()` - a bar rewriting
385
+ itself into a scrolling log pane fills it with padding.
386
+
387
+ Deliberately plain: no colour and no ANSI escapes, which need
388
+ virtual-terminal mode turned on and are not on by default in `cmd.exe`.
389
+ """
390
+
391
+ #: Wide enough to see movement, narrow enough to leave room for the text
392
+ #: on an 80-column console.
393
+ WIDTH = 24
394
+ LINE = 110
395
+
396
+ def __init__(self, enabled=True, stream=None):
397
+ self.stream = stream if stream is not None else sys.stderr
398
+ self.enabled = bool(enabled) and bool(
399
+ getattr(self.stream, "isatty", lambda: False)())
400
+ self.drawn = False
401
+
402
+ def __call__(self, event):
403
+ if not self.enabled:
404
+ return
405
+ # `done` is the one terminal event a pull emits, and a watcher emits one
406
+ # per poll. Clearing here rather than at exit is what stops the second
407
+ # poll's bar continuing the first one's line.
408
+ if event.stage == "done":
409
+ self.clear()
410
+ return
411
+ if event.total:
412
+ filled = int(round(self.WIDTH * min(1.0, event.fraction)))
413
+ text = ("%-9s [%s] %d/%d %s"
414
+ % (event.stage, "#" * filled + "." * (self.WIDTH - filled),
415
+ event.done, event.total, event.unit))
416
+ else:
417
+ text = "%-9s %s" % (event.stage, event.detail)
418
+ if event.position:
419
+ text = "%s %s" % (text, event.position)
420
+ self.stream.write("\r" + text[:self.LINE].ljust(self.LINE))
421
+ self.stream.flush()
422
+ self.drawn = True
423
+
424
+ def clear(self):
425
+ """Wipe the line, leaving no padding behind for the next thing."""
426
+ if self.enabled and self.drawn:
427
+ self.stream.write("\r" + " " * self.LINE + "\r")
428
+ self.stream.flush()
429
+ self.drawn = False
430
+
431
+
432
+ def _progress(args):
433
+ """What a command hands the engine: a caller's own hook, or a bar.
434
+
435
+ A front end sets `args.progress` before calling the command function, and
436
+ it wins - it is already capturing output, so `_progress_for` would return
437
+ None for it anyway; this just says so where somebody reading the command
438
+ can see it.
439
+ """
440
+ return getattr(args, "progress", None) or _progress_for(args)
441
+
442
+
443
+ def _progress_for(args):
444
+ """A bar to hand to the engine, or None. Never a bar and captured output.
445
+
446
+ Registered in `_BAR` so `say` can wipe the line before printing over it.
447
+ """
448
+ if getattr(args, "json", False) or getattr(args, "quiet", False):
449
+ return None
450
+ if _CONSOLE[0] is not None: # a front end is capturing this
451
+ return None
452
+ bar = ConsoleProgress()
453
+ if not bar.enabled:
454
+ return None
455
+ _BAR[0] = bar
456
+ return bar
457
+
458
+
459
+ def literal(text):
460
+ """Help text argparse will not %-format.
461
+
462
+ argparse %-formats every help string, so a bare `%` in one raises TypeError
463
+ when the user types --help. It is a crash they meet and no test catches
464
+ unless --help is asserted on. PyIncucyte hit exactly this.
465
+ """
466
+ return text.replace("%", "%%")
467
+
468
+
469
+ def add_selection_options(p):
470
+ """Which timepoints, which channels, and how the pixels are read."""
471
+ # --channels selects, as it does in PyIncucyte. The header override it
472
+ # used to be is now --channel-count; `--channels 4` on a four-channel file
473
+ # would otherwise silently change meaning, so select_channels raises and
474
+ # names the replacement rather than writing the wrong stack.
475
+ p.add_argument("-c", "--channels", type=int, nargs="+", metavar="C",
476
+ help="only these channel indexes. Tiles are read by offset, "
477
+ "so the others never cross the wire.")
478
+ p.add_argument("--channel-count", type=int, metavar="N",
479
+ help="override the channel count read from the file header")
480
+ # ExportOptions.positions and LV200Client.plan have honoured a position
481
+ # filter since stage 04, and _pull_experiment has read one since stage 10;
482
+ # only the flag was missing. The desktop app selects positions by clicking
483
+ # a thumbnail, and a control the command line cannot express is how two
484
+ # front ends start disagreeing.
485
+ p.add_argument("-p", "--positions", nargs="+", metavar="NAME",
486
+ help="only these positions (fields of view). Each gets its "
487
+ "own stack; the default is every one the run has.")
488
+ p.add_argument("-n", "--last-n", type=int, metavar="N",
489
+ help="only the N most recent timepoints. Tiles are read by "
490
+ "offset, so the rest never cross the wire - use this "
491
+ "over a network share.")
492
+ p.add_argument("--from-t", type=int, metavar="T",
493
+ help="first timepoint, numbered as the file numbers them")
494
+ p.add_argument("--to-t", type=int, metavar="T",
495
+ help="last timepoint, numbered as the file numbers them")
496
+ # --last-n and --from-t are the friendly spellings of the two common cases
497
+ # and stay. These say the same thing in the recipe's own vocabulary, and
498
+ # are the ONLY way to ask for a duration: the recipe has always accepted
499
+ # `-48h` and until now no flag could produce one, so "the last two days"
500
+ # was reachable from a preset and from Python but not from a prompt.
501
+ p.add_argument("-s", "--start-from", metavar="WHEN",
502
+ help=literal("where the window opens: %s. A duration is "
503
+ "resolved against the run's own frame "
504
+ "interval, counting back from its newest "
505
+ "timepoint - not from now, so a run that "
506
+ "finished last month still has a last 48 "
507
+ "hours." % MOMENT_HELP))
508
+ p.add_argument("--end-at", metavar="WHEN",
509
+ help=literal("where it closes: %s. '+100f' takes the first "
510
+ "100 timepoints of the window, '+7d' its "
511
+ "first week." % MOMENT_HELP))
512
+ # Here rather than with the writing flags, where it used to be: it
513
+ # calibrates the T axis of the stack, but it is ALSO what a duration window
514
+ # is resolved against, and `plan` takes no writing flags - so `plan
515
+ # --start-from -48h` on a run whose timestamps give nothing had no way to
516
+ # say how long a frame is.
517
+ p.add_argument("--interval", type=float,
518
+ help="seconds between timepoints; derived from the run's "
519
+ "own timestamps when not given. A duration window such "
520
+ "as --start-from -48h is resolved against it.")
521
+ p.add_argument("--include-last", action="store_true",
522
+ help="include the final tile (may be mid-write and torn)")
523
+ p.add_argument("--axes", help="override axis names, e.g. 'tx,ty,z,c,t'")
524
+ p.add_argument("--ndims", type=int,
525
+ help="force the chunk-table dimension count")
526
+ p.add_argument("--dtype", help="force pixel type, e.g. uint16")
527
+ p.add_argument("--safe-copy", action="store_true",
528
+ help="copy each .ets to temp before parsing (slower, immune "
529
+ "to growth)")
530
+ p.add_argument("--preset", metavar="NAME",
531
+ help="apply a saved recipe. The command line still wins.")
532
+
533
+
534
+ def add_writing_options(p):
535
+ """Where the stacks go and what shape they take."""
536
+ # --output is PyIncucyte's spelling of the same flag. Both stay: -o is
537
+ # what everything already written types, and --output is what somebody
538
+ # arriving from the other package types without thinking about it.
539
+ p.add_argument("-o", "--out", "--output",
540
+ help="output directory for the stacks")
541
+ p.add_argument("--pixel-size", type=float,
542
+ help="pixel size in microns, for spatial calibration")
543
+ p.add_argument("--channel-names",
544
+ help="comma-separated names in acquisition order. Normally "
545
+ "unnecessary - they are read from the .vsi - and used "
546
+ "on later pulls of the same experiment once given.")
547
+ # The canonical control. The three booleans below are shorthands for it
548
+ # and must not disagree with it - `_given` resolves both in one place and
549
+ # errors rather than picking a winner.
550
+ p.add_argument("--layout", metavar="NAME",
551
+ help=literal("how the pixels are laid out on disk: %s "
552
+ "(default: composite). Synonyms such as "
553
+ "'hyperstack' and 'ome-tiff' resolve too."
554
+ % ", ".join("%s - %s" % (n, LAYOUT_DESCRIPTIONS[n])
555
+ for n in LAYOUTS)))
556
+ p.add_argument("--composite", action="store_true",
557
+ help="display channels overlaid in ImageJ's default colours. "
558
+ "Off by default: these are grayscale detector channels "
559
+ "and the colours carry no meaning.")
560
+ p.add_argument("--split-channels", action="store_true",
561
+ help="one TIFF per channel instead of one multichannel stack")
562
+ p.add_argument("--ome-tiff", action="store_true",
563
+ help="a lossless LZW OME-TIFF (<stem>.ome.tif) instead of an "
564
+ "ImageJ hyperstack - the same shape as the Bio-Formats "
565
+ "conversion of a finished .vsi, so the two are "
566
+ "interchangeable. Chosen automatically past 4 GB, which "
567
+ "ImageJ cannot exceed.")
568
+ p.add_argument("--no-pin", action="store_true",
569
+ help="do not remember --channel-names for this experiment, "
570
+ "and forget any already pinned for it")
571
+ p.add_argument("--mirror", metavar="DIR",
572
+ help="keep a local byte copy of each .ets in DIR and read "
573
+ "the tiles from there. Only the bytes the acquisition "
574
+ "has added since the last run cross the wire, and what "
575
+ "is left behind is a complete copy of the run - which "
576
+ "during acquisition is the only copy that exists.")
577
+ p.add_argument("--no-append", dest="append_stacks", action="store_false",
578
+ default=None,
579
+ help="rewrite the whole stack when a timepoint is added, "
580
+ "instead of adding the new planes in place. Appending "
581
+ "falls back to a rewrite on its own whenever the file "
582
+ "is not the earlier part of this same stack, so the "
583
+ "only reason to ask is caution: an appended frame "
584
+ "cannot be un-appended, where a rewrite replaces the "
585
+ "file.")
586
+ p.add_argument("--no-manifest", dest="write_manifest", action="store_false",
587
+ default=None,
588
+ help="do not write pylv200-manifest.json and the CSV index. "
589
+ "They are what a pipeline reads instead of parsing "
590
+ "filenames, so this is rarely what you want.")
591
+ p.add_argument("--state-scope", choices=STATE_SCOPES, default=None,
592
+ help="where the resume ledger lives (default: folder, one "
593
+ "beside the images). 'global' keeps a single ledger "
594
+ "under the settings home, so a pull into a fresh "
595
+ "folder starts already knowing what was collected - "
596
+ "and therefore may write nothing into it. 'none' keeps "
597
+ "no ledger.")
598
+ p.add_argument("--no-state", dest="state_scope", action="store_const",
599
+ const="none", default=None,
600
+ help="shorthand for --state-scope none: keep no resume "
601
+ "ledger for this output folder at all. Different from "
602
+ "--no-resume, which stops the ledger being consulted "
603
+ "but still records what was written.")
604
+ # An action rather than a recipe field, which is why it has no ExportOptions
605
+ # counterpart and never appears in `command_line`: it says "also write down
606
+ # what you resolved", not "pull it this way".
607
+ p.add_argument("--save-preset", metavar="PATH",
608
+ help="write the resolved recipe to this JSON file before "
609
+ "reading a tile, so a run that is cancelled still "
610
+ "leaves the recipe it was going to use. `preset --save "
611
+ "NAME` is the other one - that stores into the named "
612
+ "preset store.")
613
+
614
+
615
+ def add_watch_options(p):
616
+ """The cadence a watcher keeps.
617
+
618
+ On ``preset`` as well as ``watch``, because the recipe carries it: "collect
619
+ this run a week at a time" is exactly the thing worth saving, and a cadence
620
+ that lives only in a flag cannot be saved at all.
621
+ """
622
+ # NOT --interval: that already means the run's frame interval on every
623
+ # writing command, and two flags a letter apart meaning different periods
624
+ # is a mistake somebody makes at 2am.
625
+ p.add_argument("-i", "--poll-interval", dest="poll_interval", type=float,
626
+ metavar="SECONDS",
627
+ help="seconds between polls. Defaults to half the run's own "
628
+ "frame interval: polling a 31-minute acquisition every "
629
+ "30 seconds costs round trips for nothing.")
630
+ p.add_argument("--batch-frames", type=int, metavar="N", default=None,
631
+ help=literal("hold new timepoints until N have accumulated, "
632
+ "then write the chunk in one go. The default "
633
+ "writes each timepoint as it appears (%d), "
634
+ "which is cheap because new planes are added "
635
+ "to the stack in place - see --no-append."
636
+ % DEFAULT_BATCH_FRAMES))
637
+ p.add_argument("--batch-after", metavar="DURATION",
638
+ help="write regardless once the oldest waiting timepoint is "
639
+ "this old, e.g. 7d. Measured from when it was acquired, "
640
+ "so restarting the watcher does not restart the wait.")
641
+
642
+
643
+ TARGET_HELP = literal(
644
+ "source:experiment, e.g. left-lv200:ois351A. A bare experiment name "
645
+ "resolves against the only configured source; a path is taken as-is.")
646
+
647
+
648
+ def build_parser():
649
+ ap = argparse.ArgumentParser(
650
+ prog="pylv200", description=literal(__doc__),
651
+ formatter_class=argparse.RawDescriptionHelpFormatter)
652
+ # --json belongs to the top-level parser, not to each subcommand, or
653
+ # `pylv200 list --json` and `pylv200 --json list` end up disagreeing.
654
+ ap.add_argument("--json", action="store_true",
655
+ help="machine-readable output on stdout; everything else "
656
+ "goes to stderr")
657
+ ap.add_argument("--version", action="version",
658
+ version="pylv200 %s" % __version__)
659
+ ap.add_argument("-q", "--quiet", action="store_true",
660
+ help="no progress display, and warnings only in the log")
661
+ ap.add_argument("--verbose", action="store_true",
662
+ help="debug logging on stderr")
663
+ subs = ap.add_subparsers(dest="command", required=True)
664
+
665
+ # -- the microscope ----------------------------------------------------
666
+ s = subs.add_parser("setup", help="run ON THE MICROSCOPE: find the data and "
667
+ "report what stops a share from working")
668
+ s.add_argument("--account", default=lv200setup.DEFAULT_ACCOUNT,
669
+ help=literal("the read-only account to create "
670
+ "(default: %(default)s)"))
671
+ s.add_argument("--drives", nargs="+", metavar="D:\\",
672
+ help="search only these drives instead of every fixed one")
673
+ s.add_argument("--max-depth", type=int, default=lv200setup.MAX_DEPTH,
674
+ help=literal("how deep to walk each drive "
675
+ "(default: %(default)s)"))
676
+ s.add_argument("--create-share", action="store_true",
677
+ help="also fix the network layer and create the share")
678
+ s.add_argument("--grant-scope", choices=("drives", "discovered"),
679
+ default="drives",
680
+ help=literal("grant NTFS read at every fixed drive root, or "
681
+ "only where data was found "
682
+ "(default: %(default)s)"))
683
+ s.add_argument("--yes", action="store_true",
684
+ help="actually make the changes. Without it the plan is "
685
+ "printed and nothing happens.")
686
+ s.add_argument("--host", default="",
687
+ help="the name to record for this machine (default: its "
688
+ "hostname)")
689
+ s.add_argument("--save-to", metavar="FILE",
690
+ help="write the source block here as well as printing it")
691
+ s.add_argument("--quiet", action="store_true",
692
+ default=argparse.SUPPRESS,
693
+ help="do not print each folder as it is walked. The global "
694
+ "-q means this too.")
695
+
696
+ p = subs.add_parser("probe", help="is a source reachable, and if not, which "
697
+ "layer failed")
698
+ p.add_argument("source", help="a name from the settings, or a path")
699
+
700
+ lg = subs.add_parser("login", help="hand a credential to Windows Credential "
701
+ "Manager; this package never sees it")
702
+ lg.add_argument("source", help="a name from the settings")
703
+
704
+ lo = subs.add_parser("logout", help="forget a source's stored credential")
705
+ lo.add_argument("source", help="a name from the settings")
706
+
707
+ r = subs.add_parser("rescan", help="find an output folder that moved, "
708
+ "without a lab visit")
709
+ r.add_argument("source", help="a name from the settings")
710
+ r.add_argument("--yes", action="store_true",
711
+ help="update the recorded roots when the data has moved")
712
+
713
+ subs.add_parser("sources", help="the microscopes this machine knows about")
714
+
715
+ # -- what is there -----------------------------------------------------
716
+ ls = subs.add_parser("list", help="experiments on a source, newest first")
717
+ ls.add_argument("source", nargs="?",
718
+ help="a name from the settings, or a path. Omitted, the "
719
+ "only configured source is used.")
720
+ ls.add_argument("--live", action="store_true",
721
+ help="also say which are being written right now. Costs a "
722
+ "header read per position, so it is opt-in.")
723
+ ls.add_argument("--refresh", action="store_true",
724
+ help="walk again instead of using the cached scan")
725
+
726
+ st = subs.add_parser("status", help="what is reachable, what a watcher is "
727
+ "doing, what has been written")
728
+ st.add_argument("target", nargs="?",
729
+ help="a source, or source:experiment. Omitted, every "
730
+ "configured source is reported on.")
731
+ st.add_argument("-o", "--out", "--output",
732
+ help="an output folder: report its watcher heartbeat and "
733
+ "how much its resume ledger holds")
734
+ st.add_argument("-i", "--poll-interval", type=float, metavar="SECONDS",
735
+ help="the cadence the watcher was started with. A "
736
+ "heartbeat older than three of these is called stale; "
737
+ "the default assumes 900.")
738
+
739
+ sc = subs.add_parser(
740
+ "scans", help="the timepoints a run holds, and when each was acquired",
741
+ description=SCANS_HELP,
742
+ formatter_class=argparse.RawDescriptionHelpFormatter)
743
+ sc.add_argument("target", help=TARGET_HELP)
744
+
745
+ fd = subs.add_parser("find", help="which experiment is that?")
746
+ fd.add_argument("pattern", help="part of an experiment, folder or position "
747
+ "name")
748
+ fd.add_argument("--source", help="a name from the settings, or a path")
749
+
750
+ # -- what a pull would do ----------------------------------------------
751
+ pl = subs.add_parser("plan", help="what a pull would fetch, and what is "
752
+ "already held")
753
+ pl.add_argument("target", help=TARGET_HELP)
754
+ pl.add_argument("-o", "--out", "--output",
755
+ help="the output folder whose ledger to compare against")
756
+ add_selection_options(pl)
757
+
758
+ # -- pulling -----------------------------------------------------------
759
+ pu = subs.add_parser("pull", help="fetch once, writing every selected "
760
+ "timepoint")
761
+ pu.add_argument("target", help=TARGET_HELP)
762
+ pu.add_argument("--dry-run", action="store_true",
763
+ help="report the geometry and write nothing")
764
+ pu.add_argument("--resume", action="store_true",
765
+ help="skip timepoints this output folder already holds. "
766
+ "`sync` is that, by default.")
767
+ add_selection_options(pu)
768
+ add_writing_options(pu)
769
+
770
+ sy = subs.add_parser("sync", help="fetch only what is new, using the ledger")
771
+ sy.add_argument("target", help=TARGET_HELP)
772
+ sy.add_argument("--dry-run", action="store_true",
773
+ help="report the geometry and write nothing")
774
+ sy.add_argument("--no-resume", action="store_true",
775
+ help="write every selected timepoint again, ignoring the "
776
+ "ledger")
777
+ add_selection_options(sy)
778
+ add_writing_options(sy)
779
+
780
+ wa = subs.add_parser("watch", help="poll a running experiment and collect "
781
+ "what is new, on an interval")
782
+ wa.add_argument("target", help=TARGET_HELP)
783
+ add_watch_options(wa)
784
+ wa.add_argument("--once", action="store_true",
785
+ help="poll once and stop, for a Scheduled Task")
786
+ wa.add_argument("--flush", action="store_true",
787
+ help="write whatever is being held now, then stop")
788
+ wa.add_argument("--status", action="store_true",
789
+ help="report what the watcher for this output has done, "
790
+ "and whether it is still running")
791
+ add_selection_options(wa)
792
+ add_writing_options(wa)
793
+
794
+ pv = subs.add_parser("preview", help="a few tiles per position, so you can "
795
+ "see which one you want before pulling it")
796
+ pv.add_argument("target", help=TARGET_HELP)
797
+ pv.add_argument("-o", "--out", "--output",
798
+ help="folder for the thumbnails and the sheet")
799
+ pv.add_argument("--stretch", nargs=2, type=float, metavar=("LO", "HI"),
800
+ default=(lv200preview.DEFAULT_LO, lv200preview.DEFAULT_HI),
801
+ help=literal("contrast percentiles (default: %(default)s). "
802
+ "A full-range map renders a photon-starved "
803
+ "luminescence frame black, which reads as no "
804
+ "data rather than few photons."))
805
+ pv.add_argument("--project", type=int, metavar="N",
806
+ default=lv200preview.PROJECT_FRAMES,
807
+ help=literal("frames to average for a photon-starved "
808
+ "channel (default: %(default)s)"))
809
+ pv.add_argument("-c", "--channels", type=int, nargs="+", metavar="C",
810
+ help="only these channel indexes")
811
+ # A preset carries a channel selection, and a preview of the wrong channels
812
+ # is a preview of nothing: the point of it is deciding which position to
813
+ # pull, using the recipe the pull will use.
814
+ pv.add_argument("--preset", metavar="NAME",
815
+ help="apply a saved recipe's channel selection. The "
816
+ "command line still wins.")
817
+ pv.add_argument("--no-sheet", action="store_true",
818
+ help="thumbnails only, no contact sheet")
819
+
820
+ tl = subs.add_parser(
821
+ "timeline", help="save a byte-bounded set of time-course previews")
822
+ tl.add_argument("target", help=TARGET_HELP)
823
+ tl.add_argument("-o", "--out", "--output",
824
+ help="folder for the bounded anchor PNG files")
825
+ tl.add_argument("--position", help="one position (default: the first)")
826
+ tl.add_argument("-c", "--channel", type=int,
827
+ help="one channel index (default: the first)")
828
+ tl.add_argument("--budget-mib", type=float,
829
+ default=DEFAULT_BYTE_BUDGET / float(1 << 20),
830
+ help="maximum initial source bytes in MiB (default: 64)")
831
+ tl.add_argument("--stretch", nargs=2, type=float, metavar=("LO", "HI"),
832
+ default=(lv200preview.DEFAULT_LO, lv200preview.DEFAULT_HI),
833
+ help="contrast percentiles for saved previews")
834
+ tl.add_argument("--project", type=int, metavar="N",
835
+ default=DEFAULT_PROJECTION_FRAMES,
836
+ help="frames averaged for bioluminescence (default: 4)")
837
+ tl.add_argument("--preview-cache",
838
+ help="folder for disposable persistent previews")
839
+ tl.add_argument("--no-proxy", action="store_true",
840
+ help="do not read or write the persistent preview cache")
841
+ tl.add_argument("--local", metavar="PULL_FOLDER",
842
+ help="prefer a completed local PyLV200 pull")
843
+ tl.add_argument("--mirror", metavar="FOLDER",
844
+ help="read available ETS bytes from this configured mirror")
845
+
846
+ g = subs.add_parser("gui", help="the desktop app: choose a position by "
847
+ "looking at it rather than by name")
848
+ g.add_argument("target", nargs="?", help=TARGET_HELP)
849
+ g.add_argument("--light", dest="dark", action="store_false", default=None,
850
+ help="force the light theme instead of following Windows")
851
+ g.add_argument("--dark", dest="dark", action="store_true",
852
+ help="force the dark theme instead of following Windows")
853
+
854
+ mf = subs.add_parser("manifest", help="summarise a previous pull")
855
+ mf.add_argument("folder", help="a folder a pull wrote into")
856
+
857
+ pr = subs.add_parser("preset", help="show or save a named recipe")
858
+ pr.add_argument("name", nargs="?", help="the preset to show or save")
859
+ pr.add_argument("--save", action="store_true",
860
+ help="save the options given here under that name")
861
+ pr.add_argument("--delete", action="store_true", help="remove it")
862
+ add_selection_options(pr)
863
+ add_writing_options(pr)
864
+ add_watch_options(pr)
865
+ return ap
866
+
867
+
868
+ # --------------------------------------------------------------------------
869
+ # targets
870
+ # --------------------------------------------------------------------------
871
+
872
+ def parse_target(text):
873
+ """``'left-lv200:ois351A'`` -> ``(Source or None, experiment or None)``.
874
+
875
+ A bare name resolves against the only configured source; with more than one
876
+ it raises and lists them, because guessing which microscope is what puts one
877
+ experiment's channel names on another's pixels. A path is taken as itself.
878
+ """
879
+ text = str(text or "")
880
+ # A Windows path starts `D:\` - one letter before the colon - so a colon
881
+ # alone does not make something a source:experiment target.
882
+ head, sep, tail = text.partition(":")
883
+ if sep and len(head) > 1 and not tail.startswith(("\\", "/")):
884
+ sources = config.load_sources()
885
+ if head in sources:
886
+ return sources[head], tail or None
887
+ # A folder works wherever a source name does. Nothing about naming one
888
+ # experiment inside a scratch tree requires the tree to be configured
889
+ # first, and the alternative is `find` then a full path.
890
+ if Path(head).is_dir():
891
+ return Source(name=head, roots=(head,)), tail or None
892
+ return config.get_source(head), tail or None # raises, listing what it knows
893
+
894
+ if Path(text).exists() or text.startswith("\\\\"):
895
+ return Source(name=text, roots=(text,)), None
896
+
897
+ sources = config.load_sources()
898
+ if not sources:
899
+ raise LV200Error(
900
+ "%r is not a folder, and no microscope is configured to look it up "
901
+ "on.\n Run `pylv200 setup` on the microscope, or give a path."
902
+ % text)
903
+ if len(sources) > 1:
904
+ raise LV200Error(
905
+ "%r does not say which microscope. Write it as source:experiment.\n"
906
+ " Configured sources: %s" % (text, ", ".join(sorted(sources))))
907
+ return next(iter(sources.values())), text
908
+
909
+
910
+ def source_for(name):
911
+ """A settings name if there is one, otherwise a path taken as itself."""
912
+ if not name:
913
+ sources = config.load_sources()
914
+ if len(sources) == 1:
915
+ return next(iter(sources.values()))
916
+ if not sources:
917
+ raise LV200Error("no microscope is configured and no path was "
918
+ "given.\n Run `pylv200 setup` on the microscope, "
919
+ "or give a folder.")
920
+ raise LV200Error("more than one microscope is configured; name one: %s"
921
+ % ", ".join(sorted(sources)))
922
+ try:
923
+ return config.get_source(name)
924
+ except LV200Error:
925
+ if Path(name).exists() or name.startswith("\\\\"):
926
+ return Source(name=name, roots=(name,))
927
+ raise
928
+
929
+
930
+ def _experiment_dict(exp, live=False):
931
+ return {"name": exp.name, "folder": str(exp.folder),
932
+ "positions": [p.name for p in exp.positions],
933
+ "channels": list(exp.channel_names or ()),
934
+ "started": exp.started, "camera": exp.camera,
935
+ "objective": exp.objective,
936
+ "bytes": sum(p.size for p in exp.positions),
937
+ "live": (any(p.is_live for p in exp.positions) if live else None)}
938
+
939
+
940
+ # --------------------------------------------------------------------------
941
+ # commands
942
+ # --------------------------------------------------------------------------
943
+
944
+ def cmd_sources(args):
945
+ sources = config.load_sources()
946
+ payload = {"sources": [
947
+ {"name": s.name, "host": s.host, "address": s.address,
948
+ "share": s.share, "roots": list(s.roots), "account": s.account,
949
+ "output": s.output} for s in sources.values()]}
950
+ lines = print_table(
951
+ [(s.name, s.target or "-", ", ".join(s.roots) or "no roots recorded")
952
+ for s in sources.values()],
953
+ ("source", "reached at", "roots"))
954
+ emit(payload, lines or [" no microscope is configured. Run `pylv200 "
955
+ "setup` on one, then paste the block it prints "
956
+ "into %s." % config._path(config.SOURCES_FILE)])
957
+ return 0 if sources else 1
958
+
959
+
960
+ def cmd_list(args):
961
+ source = source_for(args.source)
962
+ client = LV200Client(source)
963
+ experiments = client.experiments(refresh=args.refresh, live=args.live)
964
+ payload = {"source": source.name,
965
+ "experiments": [_experiment_dict(e, args.live)
966
+ for e in experiments],
967
+ "notes": list(client.notes)}
968
+ # Measured rather than %-44s: an experiment name is whatever cellSens was
969
+ # given, and a truncated one is a name somebody then cannot paste back in.
970
+ rows = [([exp.name, len(exp.positions),
971
+ human_bytes(sum(p.size for p in exp.positions))]
972
+ + (["LIVE" if any(p.is_live for p in exp.positions) else ""]
973
+ if args.live else []))
974
+ for exp in experiments]
975
+ lines = print_table(rows, ["experiment", "positions", "size"]
976
+ + (["live"] if args.live else []), "lrrl")
977
+ for note in client.notes:
978
+ say(" ** %s" % note)
979
+ emit(payload, lines or [" nothing found under %s"
980
+ % (", ".join(source.roots) or source.name)])
981
+ return 0 if experiments else 1
982
+
983
+
984
+ def cmd_find(args):
985
+ source = source_for(args.source)
986
+ matches = LV200Client(source).find(args.pattern)
987
+ emit({"source": source.name,
988
+ "experiments": [_experiment_dict(e) for e in matches]},
989
+ print_table([(e.name, len(e.positions), e.folder) for e in matches],
990
+ ("experiment", "positions", "folder"), "lrl")
991
+ or [" nothing matches %r" % args.pattern])
992
+ return 0 if matches else 1
993
+
994
+
995
+ def cmd_plan(args):
996
+ source, name = parse_target(args.target)
997
+ client = LV200Client(source)
998
+ experiment = client.experiment(name) if name else _only(client)
999
+ opts = config.resolve_options(
1000
+ cli=_given(args), key=config.experiment_key(source, experiment.name),
1001
+ preset=getattr(args, "preset", None), source=source)
1002
+
1003
+ # The ledger key carries the recipe, and a pull fills the channel names in
1004
+ # from the .vsi before recording. `plan` has to do the same or every
1005
+ # timepoint reads as new - the same file, two keys.
1006
+ if not opts.channel_names and experiment.channel_names:
1007
+ opts = opts.replace(channel_names=tuple(experiment.channel_names))
1008
+
1009
+ store = None
1010
+ if args.out and Path(args.out).is_dir():
1011
+ store = StateStore.for_output(Path(args.out))
1012
+ plan = client.plan(experiment, opts, store=store,
1013
+ progress=_progress(args))
1014
+
1015
+ payload = {"experiment": experiment.name,
1016
+ "positions": [p.name for p in plan.positions],
1017
+ "timepoints": list(plan.timepoints),
1018
+ "channels": list(plan.channels),
1019
+ "already_held": list(plan.already_have),
1020
+ "to_fetch": {"timepoints": len(plan.new_timepoints),
1021
+ "bytes": plan.bytes_to_read},
1022
+ "channel_names": list(experiment.channel_names or ())}
1023
+ lines = [
1024
+ " %s" % experiment.name,
1025
+ " %d position(s), %d timepoint(s), %d channel(s)"
1026
+ % (len(plan.positions), len(plan.timepoints), len(plan.channels)),
1027
+ ]
1028
+ if experiment.channel_names:
1029
+ lines.append(" channels %s"
1030
+ % " ".join("C%d %s" % (i, n) for i, n
1031
+ in enumerate(experiment.channel_names)))
1032
+ if plan.timepoints:
1033
+ lines.append(" selection t=%d..%d"
1034
+ % (plan.timepoints[0], plan.timepoints[-1]))
1035
+ if store is not None:
1036
+ lines.append(" already held %d timepoint(s) in %s"
1037
+ % (len(plan.already_have), args.out))
1038
+ lines.append(" to fetch %d timepoint(s), %s"
1039
+ % (len(plan.new_timepoints), human_bytes(plan.bytes_to_read)))
1040
+ emit(payload, lines)
1041
+ return 0 if plan.new_timepoints else 1
1042
+
1043
+
1044
+ def _only(client):
1045
+ """The experiment on a source that has exactly one."""
1046
+ found = client.experiments()
1047
+ if len(found) == 1:
1048
+ return found[0]
1049
+ if not found:
1050
+ raise NoDatasetFound("nothing was found under %s"
1051
+ % (", ".join(client.roots) or client.source.name))
1052
+ raise AmbiguousDataset(
1053
+ "%d experiments are here; name one:\n %s"
1054
+ % (len(found), "\n ".join(e.name for e in found[:20])))
1055
+
1056
+
1057
+ def _one_of(args, *flags):
1058
+ """Refuse two flags that both set the same recipe field.
1059
+
1060
+ Picking a winner is what makes two spellings drift apart: somebody types
1061
+ both, gets the other one's answer, and the flag they meant is never
1062
+ mentioned. Saying which two disagree costs one retype and no gigabytes.
1063
+ """
1064
+ given = [(flag, getattr(args, dest, None)) for flag, dest in flags
1065
+ if getattr(args, dest, None) is not None]
1066
+ if len(given) > 1:
1067
+ raise InvalidSelection(
1068
+ "%s say the same thing and disagree: %s. Use one."
1069
+ % (" and ".join(f for f, _v in given),
1070
+ ", ".join("%s %s" % (f, v) for f, v in given)))
1071
+ return given[0][1] if given else None
1072
+
1073
+
1074
+ def _window_start(args):
1075
+ """Where the time window opens, in ExportOptions' own vocabulary.
1076
+
1077
+ `--last-n 24` is "-24f" - 24 frames back from the newest - which is what
1078
+ makes it survive into a preset and stay meaningful when the run has grown.
1079
+ `--start-from` says the same thing in the recipe's own words and is the
1080
+ only one of the three that can carry a duration.
1081
+ """
1082
+ _one_of(args, ("--start-from", "start_from"), ("--from-t", "from_t"),
1083
+ ("--last-n", "last_n"))
1084
+ if getattr(args, "start_from", None):
1085
+ return _moment(args.start_from)
1086
+ if getattr(args, "from_t", None) is not None:
1087
+ return args.from_t
1088
+ last_n = getattr(args, "last_n", None)
1089
+ return "-%df" % last_n if last_n else None
1090
+
1091
+
1092
+ def _window_end(args):
1093
+ """Where it closes. `--to-t 40` and `--end-at 40` are one field."""
1094
+ _one_of(args, ("--end-at", "end_at"), ("--to-t", "to_t"))
1095
+ if getattr(args, "end_at", None):
1096
+ return _moment(args.end_at)
1097
+ return args.to_t if getattr(args, "to_t", None) is not None else None
1098
+
1099
+
1100
+ def _moment(text):
1101
+ """A --start-from / --end-at value as the recipe holds it.
1102
+
1103
+ A bare number becomes an int, so `--end-at 40` and `--to-t 40` save the
1104
+ same preset rather than one holding 40 and the other "40".
1105
+ """
1106
+ index = parse_index(text)
1107
+ return index if index is not None else text
1108
+
1109
+
1110
+ def _layout(args):
1111
+ """The layout, from `--layout` or from whichever boolean was set.
1112
+
1113
+ One place, because `--layout ome --split-channels` has no right answer and
1114
+ two places is how a front end and a prompt come to write different files.
1115
+ """
1116
+ named = getattr(args, "layout", None)
1117
+ booleans = [flag for flag, dest in (("--split-channels", "split_channels"),
1118
+ ("--ome-tiff", "ome_tiff"))
1119
+ if getattr(args, dest, False)]
1120
+ if named and booleans:
1121
+ raise InvalidSelection(
1122
+ "--layout %s and %s say the same thing and disagree. Use one."
1123
+ % (named, " and ".join(booleans)))
1124
+ if len(booleans) > 1:
1125
+ raise InvalidSelection(
1126
+ "%s say the same thing and disagree. Use one."
1127
+ % " and ".join(booleans))
1128
+ if named:
1129
+ return resolve_layout(named) # raises, listing the valid names
1130
+ if booleans:
1131
+ return "split" if booleans[0] == "--split-channels" else "ome"
1132
+ return None
1133
+
1134
+
1135
+ def _given(args):
1136
+ """Only the options the caller actually passed.
1137
+
1138
+ A filled-in ExportOptions would carry a default for every flag that was not
1139
+ given, and a default silently unpins whatever somebody chose.
1140
+ """
1141
+ names = getattr(args, "channel_names", None)
1142
+ positions = getattr(args, "positions", None)
1143
+ channels = getattr(args, "channels", None)
1144
+ given = {
1145
+ "output": getattr(args, "out", None),
1146
+ "positions": tuple(positions) if positions else None,
1147
+ "channels": tuple(channels) if channels else None,
1148
+ "mirror": getattr(args, "mirror", None),
1149
+ "write_manifest": getattr(args, "write_manifest", None),
1150
+ "append_stacks": getattr(args, "append_stacks", None),
1151
+ "state_scope": getattr(args, "state_scope", None),
1152
+ "poll_interval_s": getattr(args, "poll_interval", None),
1153
+ "batch_frames": (getattr(args, "batch_frames", None) or None),
1154
+ "batch_after": getattr(args, "batch_after", None),
1155
+ "channel_names": (tuple(n.strip() for n in names.split(","))
1156
+ if names else None),
1157
+ "pixel_size_um": getattr(args, "pixel_size", None),
1158
+ "interval_s": getattr(args, "interval", None),
1159
+ "layout": _layout(args),
1160
+ "composite_display": True if getattr(args, "composite", False) else None,
1161
+ "include_last": True if getattr(args, "include_last", False) else None,
1162
+ "safe_copy": True if getattr(args, "safe_copy", False) else None,
1163
+ "start_from": _window_start(args),
1164
+ "end_at": _window_end(args),
1165
+ }
1166
+ return {k: v for k, v in given.items() if v is not None}
1167
+
1168
+
1169
+ # --------------------------------------------------------------------------
1170
+ # a recipe as a command line
1171
+ #
1172
+ # The desktop app does not build an argparse namespace by hand. It builds argv
1173
+ # and hands it to `build_parser()`, so it cannot express anything the command
1174
+ # line cannot - which is what makes "the app and the CLI write the same bytes"
1175
+ # a property of the code rather than a claim in a docstring.
1176
+ # --------------------------------------------------------------------------
1177
+
1178
+ # --save-preset is deliberately absent: it is an action a run performs, not a
1179
+ # field of the recipe, so there is nothing for `command_line` to emit.
1180
+ _SELECTION_FLAGS = {"--last-n", "--from-t", "--to-t", "--start-from",
1181
+ "--end-at", "--interval", "--include-last", "--safe-copy",
1182
+ "--positions", "--channels", "--preset"}
1183
+ _WRITING_FLAGS = {"-o", "--pixel-size", "--channel-names",
1184
+ "--layout", "--composite", "--split-channels", "--ome-tiff",
1185
+ "--no-pin", "--mirror", "--no-manifest", "--state-scope",
1186
+ "--no-state", "--no-append"}
1187
+ #: Only `watch` has a cadence to describe.
1188
+ _WATCH_FLAGS = {"--poll-interval", "--batch-frames", "--batch-after"}
1189
+
1190
+ #: Which flags each verb accepts. Explicit rather than introspected, because a
1191
+ #: wrong entry here has to fail loudly - and it does: the round-trip test parses
1192
+ #: every command :func:`command_line` builds.
1193
+ VERB_FLAGS = {
1194
+ "plan": _SELECTION_FLAGS | {"-o"},
1195
+ "pull": _SELECTION_FLAGS | _WRITING_FLAGS,
1196
+ "sync": _SELECTION_FLAGS | _WRITING_FLAGS,
1197
+ "watch": _SELECTION_FLAGS | _WRITING_FLAGS | _WATCH_FLAGS,
1198
+ "preview": {"-o", "--channels"},
1199
+ }
1200
+
1201
+
1202
+ def command_line(options, target, verb="sync", preset=None, no_pin=False):
1203
+ """The exact ``pylv200`` command this recipe is, as argv.
1204
+
1205
+ A recipe field with no flag behind it is not emitted, and a time window the
1206
+ command line cannot say raises rather than being quietly dropped: a window
1207
+ that silently became "the whole run" would move gigabytes nobody asked for.
1208
+ """
1209
+ allowed = VERB_FLAGS.get(verb, set())
1210
+ argv = [verb, str(target)]
1211
+
1212
+ def add(flag, *values):
1213
+ if flag in allowed:
1214
+ argv.append(flag)
1215
+ argv.extend(str(v) for v in values)
1216
+
1217
+ frames = options.start_frames
1218
+ first = parse_index(options.start_from)
1219
+ last = parse_index(options.end_at)
1220
+
1221
+ if options.output:
1222
+ add("-o", options.output)
1223
+ if options.positions:
1224
+ add("--positions", *options.positions)
1225
+ if options.channels:
1226
+ add("--channels", *options.channels)
1227
+ if options.mirror:
1228
+ add("--mirror", options.mirror)
1229
+ if not options.append_stacks:
1230
+ add("--no-append")
1231
+ if not options.write_manifest:
1232
+ add("--no-manifest")
1233
+ if options.state_scope == "none":
1234
+ add("--no-state")
1235
+ elif options.state_scope == "global":
1236
+ add("--state-scope", "global")
1237
+ if options.poll_interval_s:
1238
+ add("--poll-interval", options.poll_interval_s)
1239
+ # Emitted even at its default: the recipe owns the cadence, so the command
1240
+ # has to say what the cadence is rather than rely on a flag default that
1241
+ # could drift away from the field default.
1242
+ add("--batch-frames", int(options.batch_frames or 0))
1243
+ if options.batch_after:
1244
+ add("--batch-after", options.batch_after)
1245
+ # The friendly spellings first - they are what the panel sets and what a
1246
+ # person reads back off the Copy button. --start-from is the general form
1247
+ # and the only one that can carry a duration; it used to be missing, and a
1248
+ # recipe saying "the last two days" had to raise here rather than quietly
1249
+ # becoming "the whole run" and moving gigabytes nobody asked for.
1250
+ if frames:
1251
+ add("--last-n", frames)
1252
+ elif first is not None:
1253
+ add("--from-t", first)
1254
+ elif options.start_from is not None:
1255
+ add("--start-from", options.start_from)
1256
+ if last is not None:
1257
+ add("--to-t", last)
1258
+ elif options.end_at is not None:
1259
+ add("--end-at", options.end_at)
1260
+ if options.layout == "split":
1261
+ add("--split-channels")
1262
+ elif options.layout == "ome":
1263
+ add("--ome-tiff")
1264
+ if options.composite_display:
1265
+ add("--composite")
1266
+ if options.include_last:
1267
+ add("--include-last")
1268
+ if options.safe_copy:
1269
+ add("--safe-copy")
1270
+ if options.channel_names:
1271
+ add("--channel-names", ",".join(options.channel_names))
1272
+ if no_pin:
1273
+ add("--no-pin")
1274
+ if options.pixel_size_um:
1275
+ add("--pixel-size", options.pixel_size_um)
1276
+ if options.interval_s:
1277
+ add("--interval", options.interval_s)
1278
+ if preset:
1279
+ add("--preset", preset)
1280
+ return argv
1281
+
1282
+
1283
+ def parse(argv):
1284
+ """``parse_args`` that raises instead of exiting.
1285
+
1286
+ argparse prints a usage block and calls ``sys.exit``. A front end with no
1287
+ console has nowhere to show that, and a ``SystemExit`` raised on a worker
1288
+ thread kills the thread with no message at all.
1289
+ """
1290
+ # Through the same alias table as `main`, so a front end and a person
1291
+ # typing at a prompt read the same argv.
1292
+ argv = join_negative_values(resolve_alias(str(a) for a in argv))
1293
+ trapped = io.StringIO()
1294
+ try:
1295
+ with contextlib.redirect_stderr(trapped):
1296
+ return build_parser().parse_args(argv)
1297
+ except SystemExit:
1298
+ raise LV200Error("could not read this as a command:\n pylv200 %s\n%s"
1299
+ % (" ".join(argv), trapped.getvalue().strip())) from None
1300
+
1301
+
1302
+ def _resolve_experiment(args):
1303
+ """Turn a target into one Experiment, without fetching a tile.
1304
+
1305
+ The client is kept on the namespace: every verb that follows goes through
1306
+ the same object, so `plan` then `sync` walks the share once rather than
1307
+ twice.
1308
+ """
1309
+ source, name = parse_target(args.target)
1310
+ client = LV200Client(source)
1311
+ experiment = client._as_experiment(name)
1312
+ args.source = source.name if config.load_sources().get(source.name) else None
1313
+ args.client = client
1314
+ return experiment
1315
+
1316
+
1317
+ #: What one position's outcome means to a caller that reads exit codes. A
1318
+ #: scheduled task treats all four differently, which is why the pull returns a
1319
+ #: word and only this module turns it into a number.
1320
+ _EXIT_FOR = {"written": 0, "dry_run": 0, "nothing_new": 1, "cancelled": STOPPED}
1321
+
1322
+
1323
+ def _exit_code(result):
1324
+ """The exit code for a whole experiment's pull.
1325
+
1326
+ "Stopped with nothing written" is its own answer. Reporting it as 2 would
1327
+ make a scheduled task treat a deliberate stop as a failure, and 1 would make
1328
+ it read as "there was nothing to do".
1329
+ """
1330
+ codes = [_EXIT_FOR.get(one.status, 2) for one in result.positions]
1331
+ if result.cancelled and 0 not in codes:
1332
+ return STOPPED
1333
+ return 0 if 0 in codes else (1 if codes else 2)
1334
+
1335
+
1336
+ def _pull_experiment(args, resume):
1337
+ """Resolve the recipe, hand it to the engine, print what comes back.
1338
+
1339
+ The pull itself lives in `engine.pull_experiment`, which takes an
1340
+ ExportOptions rather than this namespace and yields ProgressEvents rather
1341
+ than printing. What is left here is the three things that are genuinely
1342
+ command-line work: precedence, the settings home, and the exit code.
1343
+ """
1344
+ experiment = _resolve_experiment(args)
1345
+ # Cleared before anything is resolved: --no-pin means these settings do not
1346
+ # apply to this pull either, not merely that they stop applying afterwards.
1347
+ _early, _source, key = _recipe(args, experiment.name)
1348
+ unpinned = config.clear_override(key) if args.no_pin else False
1349
+ opts, _source, key = _recipe(args, experiment.name)
1350
+
1351
+ # Whether the names were typed on THIS command line, which decides both how
1352
+ # the pull describes them and whether it pins them.
1353
+ typed = bool(getattr(args, "channel_names", None))
1354
+ on_pin = ((lambda names: config.set_override(key, channel_names=list(names)))
1355
+ if typed and not args.no_pin else None)
1356
+ # A pinned name still wins - somebody chose it - but a disagreement usually
1357
+ # means the reason for pinning has gone away. Not worth saying when the
1358
+ # names were just typed: they are not the pinned ones.
1359
+ on_conflict = ((lambda vsi_names: config.override_conflicts(key, vsi_names))
1360
+ if not typed else None)
1361
+
1362
+ _save_preset(args, opts)
1363
+
1364
+ result = args.client.fetch(
1365
+ experiment, opts,
1366
+ out=Path(args.out) if args.out else None,
1367
+ dry_run=bool(getattr(args, "dry_run", False)),
1368
+ resume=resume,
1369
+ cancel=getattr(args, "cancel", None),
1370
+ # A front end passes its own; the terminal gets a bar, or None when it
1371
+ # is not a terminal, was told to be quiet, or is being captured.
1372
+ progress=_progress(args),
1373
+ on_say=say,
1374
+ on_position=getattr(args, "on_position", None),
1375
+ unpinned=unpinned, on_conflict=on_conflict, on_pin=on_pin,
1376
+ channel_names_given=typed,
1377
+ force_ndims=getattr(args, "ndims", None),
1378
+ channel_count=getattr(args, "channel_count", None),
1379
+ dtype=getattr(args, "dtype", None),
1380
+ axes_override=getattr(args, "axes", None))
1381
+
1382
+ args.wrote = [one.payload or {} for one in result.positions]
1383
+ args.result = result # the watcher reads what this collected
1384
+ emit(result.payload)
1385
+ return _exit_code(result)
1386
+
1387
+ def _save_preset(args, options):
1388
+ """Write the resolved recipe out, if --save-preset asked for it.
1389
+
1390
+ Before a tile is read, so a run stopped halfway still leaves the recipe it
1391
+ was going to use - which is the whole point of asking for it during a run
1392
+ rather than composing one by hand afterwards.
1393
+ """
1394
+ where = getattr(args, "save_preset", None)
1395
+ if not where:
1396
+ return None
1397
+ path = options.save(Path(where))
1398
+ say("recipe written to %s" % path)
1399
+ return path
1400
+
1401
+
1402
+ def cmd_pull(args):
1403
+ return _pull_experiment(args, resume=getattr(args, "resume", False))
1404
+
1405
+
1406
+ def cmd_sync(args):
1407
+ return _pull_experiment(args, resume=not args.no_resume)
1408
+
1409
+
1410
+ def cmd_watch(args):
1411
+ out = Path(args.out) if args.out else None
1412
+ log = (out / lv200watch.HEARTBEAT_FILENAME) if out else None
1413
+
1414
+ if args.status:
1415
+ if log is None:
1416
+ raise LV200Error("watch --status needs the -o folder the watcher "
1417
+ "writes into")
1418
+ status = lv200watch.heartbeat_status(log, interval=args.poll_interval)
1419
+ emit(status, [" %s" % status["message"]] +
1420
+ ([" %d poll(s), %d file(s) written, %d failure(s)"
1421
+ % (status["polls"], status["files_written"], status["failures"])]
1422
+ if status["found"] else []))
1423
+ return 0 if status.get("found") and not status.get("stale") else 1
1424
+
1425
+ experiment = _resolve_experiment(args)
1426
+ source, _ = parse_target(args.target)
1427
+ opts = config.resolve_options(
1428
+ cli=_given(args), key=config.experiment_key(source, experiment.name),
1429
+ preset=args.preset, source=source)
1430
+ if not opts.channel_names and experiment.channel_names:
1431
+ opts = opts.replace(channel_names=tuple(experiment.channel_names))
1432
+
1433
+ # The run's own cadence is the natural floor for the poll interval; there
1434
+ # is nothing to find between two frames. Taken from the recipe, so a preset
1435
+ # can carry "collect this run a week at a time".
1436
+ # A given --interval IS the run's frame interval, so it wins over one
1437
+ # derived from timestamps. It used to be ignored here, which only went
1438
+ # unnoticed because the derived value was itself always None.
1439
+ frame_interval = opts.interval_s or _frame_interval(experiment)
1440
+ interval = opts.poll_interval_s or max((frame_interval or 1800.0) / 2.0, 60.0)
1441
+
1442
+ def fetch(_experiment):
1443
+ # `watch` has no --dry-run and no --resume of its own: it always
1444
+ # collects, and always through the ledger, which is what makes a held
1445
+ # chunk safe to hold.
1446
+ args.dry_run = False
1447
+ args.sink = []
1448
+ _pull_experiment(args, resume=True)
1449
+ return [w["path"] for one in getattr(args, "wrote", ())
1450
+ for w in one.get("written", ())]
1451
+
1452
+ watcher = args.client.watch(
1453
+ experiment, opts, interval=interval, pull=fetch,
1454
+ store=(StateStore.for_output(out, scope=opts.state_scope)
1455
+ if out else None),
1456
+ frame_interval=frame_interval, heartbeat=log,
1457
+ on_poll=lambda r: _say_poll(r, interval),
1458
+ # No thread: `watch` on the command line IS the foreground, and Ctrl-C
1459
+ # is how it stops. A caller with something else to do meanwhile gets a
1460
+ # thread from `client.watch(start=True)` instead.
1461
+ start=False)
1462
+
1463
+ # A front end with a Stop button needs the watcher itself, so it can ask
1464
+ # what is being held once polling has stopped. Nothing on the command line
1465
+ # sets this.
1466
+ took = getattr(args, "on_watcher", None)
1467
+ if took:
1468
+ took(watcher)
1469
+
1470
+ if args.flush:
1471
+ result = watcher.flush()
1472
+ emit({"flushed": [str(p) for p in result.wrote]})
1473
+ return 0 if result.wrote else 1
1474
+
1475
+ say("watching %s every %s, holding until %s"
1476
+ % (experiment.name, _mins(interval), opts.batch_description()))
1477
+ if args.once:
1478
+ result = watcher.poll()
1479
+ emit(result.to_dict())
1480
+ return 0 if result.wrote else (2 if not result.reachable else 1)
1481
+
1482
+ say(" Ctrl-C to stop. %s records every poll." % (log or "nothing"))
1483
+ # The caller's own stop event when there is one - a window's Stop button
1484
+ # cannot press Ctrl-C.
1485
+ stop = getattr(args, "stop", None) or threading.Event()
1486
+ try:
1487
+ polls = watcher.run(stop_event=stop)
1488
+ except KeyboardInterrupt:
1489
+ stop.set()
1490
+ say("\n stopping - writing whatever is held")
1491
+ polls = [watcher.flush()]
1492
+ finally:
1493
+ # Nothing here runs on a thread, but a caller's `on_watcher` may have
1494
+ # started one, and a daemon thread mid-write when the interpreter exits
1495
+ # leaves a torn file.
1496
+ if watcher.is_running:
1497
+ watcher.stop(wait=True)
1498
+ emit({"polls": [p.to_dict() for p in polls]})
1499
+ return 0 if any(p.wrote for p in polls) else 1
1500
+
1501
+
1502
+ def _scan_index(experiment):
1503
+ """Which timepoints each position holds, and which parts are still open.
1504
+
1505
+ Headers and chunk tables only - never a tile. `scans` prints this, and
1506
+ `_frame_interval` divides by it.
1507
+ """
1508
+ holders, live = {}, set()
1509
+ for pos in experiment.positions:
1510
+ metas = []
1511
+ for path in pos.paths:
1512
+ try:
1513
+ metas.append(engine.parse_ets(path))
1514
+ except (LV200Error, OSError, ValueError):
1515
+ continue
1516
+ if not metas:
1517
+ continue
1518
+ meta = engine.merge_parts(metas)
1519
+ dims = meta.get("dims")
1520
+ axes = engine.assign_axes(meta, dims[2] if dims and len(dims) > 2
1521
+ else None)
1522
+ if any(engine.is_open_part(m) for m in metas):
1523
+ live.add(pos.name)
1524
+ for t in engine.timepoints_in(meta, axes):
1525
+ holders.setdefault(t, set()).add(pos.name)
1526
+ return holders, live
1527
+
1528
+
1529
+ def _frame_interval(experiment, timepoints=None):
1530
+ """Seconds between frames, from the run's own start and its newest write.
1531
+
1532
+ Divides by the number of TIMEPOINTS, not the number of parts. cellSens only
1533
+ splits a long acquisition into parts occasionally, so a position with one
1534
+ part and three hundred timepoints divided by one and yielded no interval at
1535
+ all - which pushed `batch_after` onto its last-resort clock, the one that
1536
+ restarts the wait every time the watcher restarts.
1537
+ """
1538
+ paths = [p for pos in experiment.positions for p in pos.paths]
1539
+ if not paths:
1540
+ return None
1541
+ if timepoints is None:
1542
+ timepoints = _scan_index(experiment)[0]
1543
+ return derive_interval(paths, max(len(timepoints), 1),
1544
+ started=experiment.started)
1545
+
1546
+
1547
+ def _mins(seconds):
1548
+ return ("%.0f s" % seconds if seconds < 120
1549
+ else "%.0f min" % (seconds / 60.0))
1550
+
1551
+
1552
+ def _say_poll(result, interval):
1553
+ if result.wrote:
1554
+ say(" %s wrote %d file(s)" % (_stamp(result.at), len(result.wrote)))
1555
+ elif not result.reachable:
1556
+ say(" %s unreachable: %s (retrying in %s)"
1557
+ % (_stamp(result.at), result.error, _mins(result.backoff)))
1558
+ elif result.waiting:
1559
+ say(" %s holding %d timepoint(s)" % (_stamp(result.at),
1560
+ result.waiting))
1561
+ else:
1562
+ say(" %s nothing new" % _stamp(result.at))
1563
+
1564
+
1565
+ def _stamp(at):
1566
+ return datetime.fromtimestamp(at).strftime("%H:%M:%S")
1567
+
1568
+
1569
+ def cmd_preview(args):
1570
+ experiment = _resolve_experiment(args)
1571
+ names, source = list(experiment.channel_names or ()), "vsi"
1572
+ key = config.experiment_key(parse_target(args.target)[0], experiment.name)
1573
+ pinned = config.get_override(key).get("channel_names")
1574
+ if pinned:
1575
+ names, source = list(pinned), "pinned"
1576
+
1577
+ # Through the same precedence as a pull, so `preview --preset nightly`
1578
+ # shows the channels `sync --preset nightly` would write rather than all of
1579
+ # them - a preview of the wrong channels answers a question nobody asked.
1580
+ opts, _source, _key = _recipe(args, experiment.name)
1581
+
1582
+ preview = args.client.preview(
1583
+ experiment, lo=args.stretch[0], hi=args.stretch[1],
1584
+ project=args.project, names=names, name_source=source,
1585
+ channels=list(opts.channels) if opts.channels else None,
1586
+ progress=_progress(args))
1587
+
1588
+ images = preview.images()
1589
+ say("read %d tile(s) for %d position(s)"
1590
+ % (preview.reads, len(preview.positions)))
1591
+
1592
+ written = []
1593
+ out = Path(args.out) if args.out else None
1594
+ if out:
1595
+ out.mkdir(parents=True, exist_ok=True)
1596
+ for image in images:
1597
+ dest = out / ("%s_C%d.png" % (image.position, image.channel))
1598
+ written.append(lv200preview.save_png(image, dest))
1599
+ if not args.no_sheet:
1600
+ sheet = out / "contact-sheet.png"
1601
+ lv200preview.contact_sheet(images, sheet, title=experiment.name)
1602
+ written.append(sheet)
1603
+
1604
+ lines = []
1605
+ for image in images:
1606
+ note = ("mean of %d frames" % len(image.frames) if image.projected
1607
+ else "t=%d" % (image.frames[0] if image.frames else 0))
1608
+ lines.append(" %-40s %s %s" % (image.label, note,
1609
+ image.stats.get("kind", "")))
1610
+ for path in written:
1611
+ lines.append(" wrote %s" % path.name)
1612
+ emit({"experiment": experiment.name, "tiles_read": preview.reads,
1613
+ "images": [{"position": i.position, "channel": i.channel,
1614
+ "name": i.channel_name, "name_source": i.name_source,
1615
+ "frames": list(i.frames), "projected": i.projected,
1616
+ "kind": i.stats.get("kind", ""),
1617
+ "vrange": list(i.vrange)} for i in images],
1618
+ "written": [str(p) for p in written]}, lines)
1619
+ return 0 if images else 1
1620
+
1621
+
1622
+ def cmd_timeline(args):
1623
+ """Read and optionally save only the anchors admitted by one byte budget."""
1624
+ experiment = _resolve_experiment(args)
1625
+ names = list(experiment.channel_names or ())
1626
+ key = config.experiment_key(parse_target(args.target)[0], experiment.name)
1627
+ pinned = config.get_override(key).get("channel_names")
1628
+ if pinned:
1629
+ names = list(pinned)
1630
+ kinds = {
1631
+ index: ("bioluminescence" if "lum" in str(name).lower() else "other")
1632
+ for index, name in enumerate(names)
1633
+ }
1634
+ budget = max(0, int(float(args.budget_mib) * (1 << 20)))
1635
+ source = args.client.timeline(
1636
+ experiment,
1637
+ names=names,
1638
+ byte_budget=budget,
1639
+ projection_frames=max(1, int(args.project)),
1640
+ channel_kinds=kinds,
1641
+ classify=not bool(names),
1642
+ preview_cache=args.preview_cache,
1643
+ persist=not args.no_proxy,
1644
+ local=args.local,
1645
+ mirror=args.mirror,
1646
+ )
1647
+ written = []
1648
+ frames = []
1649
+ try:
1650
+ positions = source.position_names
1651
+ position = str(args.position) if args.position else (
1652
+ positions[0] if positions else None)
1653
+ if position not in positions:
1654
+ raise InvalidSelection(
1655
+ "no position %r; choose one of %s" % (
1656
+ position, ", ".join(positions) or "none"))
1657
+ channels = source.channels_for(position)
1658
+ channel = int(args.channel) if args.channel is not None else (
1659
+ channels[0] if channels else None)
1660
+ if channel not in channels:
1661
+ raise InvalidSelection(
1662
+ "%s has no channel %r; choose one of %s" % (
1663
+ position, channel,
1664
+ ", ".join("C%d" % value for value in channels) or "none"))
1665
+ anchors = source.initial_indices(
1666
+ byte_budget=budget, position=position, channel=channel)
1667
+ for index in anchors:
1668
+ frames.append(source.get_frame(
1669
+ position, channel, index, lo=args.stretch[0],
1670
+ hi=args.stretch[1], prefetch=False))
1671
+
1672
+ if args.out:
1673
+ out = Path(args.out)
1674
+ out.mkdir(parents=True, exist_ok=True)
1675
+ safe_position = re.sub(
1676
+ r"[^A-Za-z0-9._-]+", "_", position).strip("_") or "position"
1677
+ for frame in frames:
1678
+ path = out / ("%s_C%d_t%d.png" % (
1679
+ safe_position, channel, frame.timepoint))
1680
+ path.write_bytes(lv200preview.encode_png(frame.array))
1681
+ written.append(path)
1682
+
1683
+ hits = source.cache_hits + getattr(source, "proxy_hits", 0)
1684
+ lines = [
1685
+ "Time preview: %d anchors, %d source tiles, %s read, "
1686
+ "%d cache hits, %s" % (
1687
+ len(anchors), source.source_reads,
1688
+ human_bytes(source.source_bytes), hits,
1689
+ source.active_provider)
1690
+ ]
1691
+ lines.extend(" wrote %s" % path.name for path in written)
1692
+ emit({
1693
+ "experiment": experiment.name,
1694
+ "position": position,
1695
+ "channel": channel,
1696
+ "anchors": list(anchors),
1697
+ "timepoints": [frame.timepoint for frame in frames],
1698
+ "source_tiles": source.source_reads,
1699
+ "source_bytes": source.source_bytes,
1700
+ "cache_hits": hits,
1701
+ "provider": source.active_provider,
1702
+ "written": [str(path) for path in written],
1703
+ }, lines)
1704
+ return 0 if frames else 1
1705
+ finally:
1706
+ source.close()
1707
+
1708
+
1709
+ def cmd_gui(args):
1710
+ """The desktop app.
1711
+
1712
+ Imported here and nowhere else. Importing it at module level would make
1713
+ `pylv200 list` on a headless machine depend on a window system, and Tk is
1714
+ the one dependency this package cannot assume.
1715
+ """
1716
+ from .gui.app import run
1717
+
1718
+ return run(target=getattr(args, "target", None),
1719
+ dark=getattr(args, "dark", None))
1720
+
1721
+
1722
+ def cmd_manifest(args):
1723
+ folder = Path(args.folder)
1724
+ path = folder / MANIFEST_FILENAME
1725
+ if not path.is_file():
1726
+ raise NoDatasetFound("no %s in %s - that folder was not written by a "
1727
+ "pull" % (MANIFEST_FILENAME, folder))
1728
+ data = json.loads(path.read_text(encoding="utf-8"))
1729
+ files = data.get("files", [])
1730
+ lines = [" %s" % (data.get("experiment") or folder.name),
1731
+ " written %s" % data.get("generated_at", "?"),
1732
+ " files %d" % len(files)]
1733
+ for record in data.get("channels", []):
1734
+ lines.append(" C%-13d %-24s (%s)"
1735
+ % (record.get("index", 0), record.get("name", "?"),
1736
+ record.get("source", "?")))
1737
+ for record in files:
1738
+ lines.append(" %-30s T=%s %s"
1739
+ % (Path(record.get("path", "?")).name,
1740
+ len(record.get("timepoints", [])),
1741
+ record.get("format", "")))
1742
+ emit(data, lines)
1743
+ return 0
1744
+
1745
+
1746
+ def cmd_preset(args):
1747
+ if args.name and args.delete:
1748
+ removed = config.remove_preset(args.name)
1749
+ emit({"deleted": args.name, "existed": removed},
1750
+ [" %s %s" % (args.name, "removed" if removed else "was not saved")])
1751
+ return 0 if removed else 1
1752
+ if args.name and args.save:
1753
+ options = config.resolve_options(cli=_given(args))
1754
+ config.save_preset(args.name, options)
1755
+ _save_preset(args, options) # `--save-preset` writes the file too
1756
+ emit({"saved": args.name, "options": options.to_dict()},
1757
+ [" saved %s" % args.name])
1758
+ return 0
1759
+ if not args.name and getattr(args, "save_preset", None):
1760
+ # A recipe file with no name in the store: the shape a pipeline commits
1761
+ # beside its analysis code, where the name would mean nothing.
1762
+ options = config.resolve_options(cli=_given(args))
1763
+ _save_preset(args, options)
1764
+ emit({"options": options.to_dict()}, [])
1765
+ return 0
1766
+ if args.name:
1767
+ options = config.get_preset(args.name)
1768
+ emit({"preset": args.name, "options": options.to_dict()},
1769
+ [" %-16s %s" % (k, v) for k, v in
1770
+ sorted(options.to_dict().items()) if v not in (None, "", ())])
1771
+ return 0
1772
+ presets = config.load_presets()
1773
+ emit({"presets": {n: o.to_dict() for n, o in presets.items()}},
1774
+ [" %s" % n for n in sorted(presets)] or [" no presets are saved"])
1775
+ return 0 if presets else 1
1776
+
1777
+
1778
+ def _cmd_logout(args):
1779
+ """Remove a source's stored credential. Never reads its value."""
1780
+ source = config.get_source(args.source)
1781
+ config.forget_credential(source)
1782
+ say("removed whatever Windows held for %s.\n"
1783
+ " A pull will now be refused until `pylv200 login %s` runs again."
1784
+ % (source.target, source.name))
1785
+ return 0
1786
+
1787
+
1788
+ def cmd_status(args):
1789
+ """What is reachable, what a watcher is doing, and what has been written.
1790
+
1791
+ PyIncucyte's `status` asks its instrument how it is. House rule 1 forbids
1792
+ that here - nothing may reach into the microscope - so this reports the
1793
+ three things PyLV200 can honestly know: whether each configured source
1794
+ answers, what the watcher heartbeat in an output folder says, and how much
1795
+ the resume ledger there holds.
1796
+ """
1797
+ payload = {"sources": [], "experiment": None, "watcher": None,
1798
+ "ledger": None}
1799
+ lines, problems = [], 0
1800
+
1801
+ if args.target and args.target in config.load_sources():
1802
+ # A bare configured name is a SOURCE here. `parse_target` would read it
1803
+ # as an experiment on the only source, which is right for `sync` and
1804
+ # wrong for a command whose first argument is documented as a source.
1805
+ name, sources = None, {args.target: config.get_source(args.target)}
1806
+ elif args.target:
1807
+ source, name = parse_target(args.target)
1808
+ sources = {source.name: source}
1809
+ else:
1810
+ name = None
1811
+ sources = config.load_sources()
1812
+ if not sources:
1813
+ raise LV200Error(
1814
+ "no microscope is configured, so there is nothing to report "
1815
+ "on.\n Run `pylv200 setup` on the microscope, or give a path.")
1816
+
1817
+ for label in sorted(sources):
1818
+ source = sources[label]
1819
+ roots = source.reachable()
1820
+ ok = bool(roots) and all(answered for _p, answered, _why in roots)
1821
+ problems += 0 if ok else 1
1822
+ payload["sources"].append(
1823
+ {"name": label, "reachable": ok, "target": source.target,
1824
+ "roots": [{"path": p, "reachable": a, "why": w}
1825
+ for p, a, w in roots]})
1826
+ lines.append(" %-16s %s" % (label, "reachable" if ok else "UNREACHABLE"))
1827
+ for path, answered, why in roots:
1828
+ if not answered:
1829
+ lines.append(" %s -- %s" % (path, why or "did not answer"))
1830
+
1831
+ if name:
1832
+ experiment = args.client.experiment(name) if hasattr(args, "client") \
1833
+ else LV200Client(sources[next(iter(sources))]).experiment(name)
1834
+ live = experiment.is_live
1835
+ payload["experiment"] = {"name": experiment.name,
1836
+ "positions": len(experiment.positions),
1837
+ "live": bool(live)}
1838
+ lines.append(" %-16s %d position(s), %s"
1839
+ % (experiment.name, len(experiment.positions),
1840
+ "being written now" if live else "not being written"))
1841
+
1842
+ out = Path(args.out) if args.out else None
1843
+ if out:
1844
+ # 900 s is the watcher's own default cadence; a heartbeat older than
1845
+ # three of those is what `stale` means. Give -i when the watcher was
1846
+ # started with a different one, or a slow poll reads as a dead watcher.
1847
+ beat = lv200watch.heartbeat_status(
1848
+ out / lv200watch.HEARTBEAT_FILENAME,
1849
+ interval=args.poll_interval or 900.0)
1850
+ payload["watcher"] = beat
1851
+ lines.append(" %-16s %s" % ("watcher", beat["message"]))
1852
+ if beat.get("found"):
1853
+ lines.append(" %d poll(s), %d file(s) written, %d failure(s)"
1854
+ % (beat["polls"], beat["files_written"],
1855
+ beat["failures"]))
1856
+ if beat.get("found") and beat.get("stale"):
1857
+ problems += 1
1858
+
1859
+ ledger = StateStore.for_output(out).summary()
1860
+ payload["ledger"] = ledger
1861
+ lines.append(" %-16s %d timepoint(s) across %d position(s)%s"
1862
+ % ("ledger", ledger["entries"], ledger["positions"],
1863
+ "" if ledger["exists"] else " (no ledger written yet)"))
1864
+ for one in ledger["experiments"]:
1865
+ lines.append(" %s" % one)
1866
+
1867
+ emit(payload, lines)
1868
+ # 1, not 2: "something is not healthy" is an answer, not a failure to
1869
+ # answer. A scheduled task treats them differently.
1870
+ return 0 if not problems else 1
1871
+
1872
+
1873
+ def cmd_scans(args):
1874
+ """List the timepoints one run holds, and when each was acquired.
1875
+
1876
+ Named `scans` for PyIncucyte parity. `timepoints` is a hidden alias,
1877
+ because a *timepoint* is what `--from-t`, the resume ledger and the
1878
+ manifest column all call the same thing - the two words mean one thing
1879
+ here, and this is the one place the PyIncucyte spelling wins.
1880
+
1881
+ Reads headers and chunk tables only. Never a tile.
1882
+ """
1883
+ experiment = _resolve_experiment(args)
1884
+ holders, live = _scan_index(experiment)
1885
+ interval = _frame_interval(experiment, holders)
1886
+
1887
+ if not holders:
1888
+ raise NoDatasetFound("no readable .ets part under %s" % experiment.folder)
1889
+
1890
+ rows = engine.timepoint_times(sorted(holders), started=experiment.started,
1891
+ interval_s=interval)
1892
+ newest = rows[-1][0] if rows else None
1893
+ every = len(experiment.positions)
1894
+
1895
+ lines = ["", "=== Scans for %s ===" % experiment.name, ""]
1896
+ for t, when in rows:
1897
+ held = len(holders[t])
1898
+ note = "" if held == every else " (%d of %d position(s))" % (held, every)
1899
+ if t == newest and live:
1900
+ # The same reasoning as --include-last: the newest tile may be
1901
+ # mid-write, so a pull drops it unless asked not to.
1902
+ note += " still being acquired"
1903
+ lines.append(" %-6d %s%s"
1904
+ % (t, when.strftime("%Y-%m-%d %H:%M:%S") if when
1905
+ else "time not known", note))
1906
+ lines.append("")
1907
+ lines.append("%d scan%s%s" % (len(rows), "" if len(rows) == 1 else "s",
1908
+ "" if interval is None else
1909
+ "; times derived from the run start and a "
1910
+ "%.1f min interval" % (interval / 60.0)))
1911
+
1912
+ # The JSON keeps the recipe's own vocabulary: a pipeline reading this also
1913
+ # reads the manifest, where the column is `timepoints`.
1914
+ emit({"experiment": experiment.name,
1915
+ "timepoints": [t for t, _ in rows],
1916
+ "times": [w.isoformat() if w else None for _, w in rows],
1917
+ "interval_s": interval,
1918
+ "interval_derived": interval is not None,
1919
+ "positions": {str(t): sorted(holders[t]) for t, _ in rows},
1920
+ "live": sorted(live)}, lines)
1921
+ return 0 if rows else 1
1922
+
1923
+
1924
+ COMMANDS = {
1925
+ "setup": _cmd_setup, "probe": _cmd_probe, "login": _cmd_login,
1926
+ "logout": _cmd_logout, "status": cmd_status, "scans": cmd_scans,
1927
+ "rescan": _cmd_rescan, "sources": cmd_sources, "list": cmd_list,
1928
+ "find": cmd_find, "plan": cmd_plan, "pull": cmd_pull, "sync": cmd_sync,
1929
+ "watch": cmd_watch, "preview": cmd_preview, "timeline": cmd_timeline,
1930
+ "gui": cmd_gui,
1931
+ "manifest": cmd_manifest,
1932
+ "preset": cmd_preset,
1933
+ }
1934
+
1935
+ #: Accepted but hidden from --help, so each job has one obvious name.
1936
+ #: `download` and `vessels` are PyIncucyte's spellings, so muscle memory built
1937
+ #: on the sister package works here. `timepoints` is this package's own word
1938
+ #: for what `scans` lists - the manifest column, the ledger key and `--from-t`
1939
+ #: all say timepoint - and `scans` is the one place a PyIncucyte name wins,
1940
+ #: because a scan is a real event on this instrument too.
1941
+ #:
1942
+ #: Rewritten in argv rather than registered as extra subparsers, so `VERB_FLAGS`
1943
+ #: keeps naming exactly one verb per job. The desktop app builds argv against
1944
+ #: that table, and a gap in it is how two front ends start disagreeing.
1945
+ ALIASES = {"download": "sync", "vessels": "list", "timepoints": "scans"}
1946
+
1947
+ #: PyIncucyte commands with no counterpart here, and why. Both write to the
1948
+ #: instrument or process pixels with device coefficients an LV200 has none of.
1949
+ UNAVAILABLE = {
1950
+ "scan-now": "pylv200 never commands the microscope - it is read-only, "
1951
+ "which is what makes it safe to run against a live experiment. "
1952
+ "Start the acquisition in cellSens.",
1953
+ "unmix": "there is no spectral unmixing here. LV200 channels are separate "
1954
+ "detector channels, and the Scale and Bias coefficients an unmix "
1955
+ "needs come from the device.",
1956
+ "calibrate": "there are no device-supplied calibration coefficients on an "
1957
+ "LV200. Give --pixel-size and --interval instead.",
1958
+ }
1959
+
1960
+
1961
+ #: Flags whose value can legitimately begin with a minus sign. argparse sees a
1962
+ #: leading "-" and calls it an option, so `--start-from -48h` fails with
1963
+ #: "expected one argument" - a message that names the flag and not the problem.
1964
+ #: Only these three, and only when the next token really looks like a signed
1965
+ #: amount: `--positions` takes nargs="+" and a position genuinely named "-48h"
1966
+ #: must still reach it.
1967
+ NEGATIVE_VALUE_FLAGS = {"--start-from", "-s", "--end-at"}
1968
+ _NEGATIVE_VALUE = re.compile(r"^-\d+(?:\.\d+)?\s*[smhdwf]", re.IGNORECASE)
1969
+
1970
+
1971
+ def join_negative_values(argv):
1972
+ """Rewrite ``--start-from -48h`` as ``--start-from=-48h`` before parsing.
1973
+
1974
+ `=`-joined, argparse never has to decide whether the token is an option.
1975
+ Applied by `parse` and by `main` both, or the desktop app and a prompt
1976
+ disagree about exactly the window forms the panel can set.
1977
+ """
1978
+ fixed, index = [], 0
1979
+ while index < len(argv):
1980
+ token = argv[index]
1981
+ following = argv[index + 1] if index + 1 < len(argv) else None
1982
+ if (token in NEGATIVE_VALUE_FLAGS and following
1983
+ and _NEGATIVE_VALUE.match(following)):
1984
+ fixed.append("%s=%s" % (token, following))
1985
+ index += 2
1986
+ continue
1987
+ fixed.append(token)
1988
+ index += 1
1989
+ return fixed
1990
+
1991
+
1992
+ def resolve_alias(argv):
1993
+ """Rewrite a PyIncucyte verb into this package's, in place of argv[0].
1994
+
1995
+ The global flags are all switches, so the first token that is not a flag is
1996
+ the verb.
1997
+ """
1998
+ argv = list(argv)
1999
+ for index, token in enumerate(argv):
2000
+ if token.startswith("-"):
2001
+ continue
2002
+ if token in ALIASES:
2003
+ argv[index] = ALIASES[token]
2004
+ break
2005
+ return argv
2006
+
2007
+
2008
+ def configure_logging(args):
2009
+ """Wire --verbose and --quiet to the `pylv200` logger.
2010
+
2011
+ stderr, always: stdout belongs to the payload under --json, and a log line
2012
+ on it corrupts whatever is parsing that.
2013
+ """
2014
+ level = (logging.DEBUG if getattr(args, "verbose", False) else
2015
+ logging.WARNING if getattr(args, "quiet", False) else logging.INFO)
2016
+ handler = logging.StreamHandler(sys.stderr)
2017
+ handler.setFormatter(logging.Formatter("%(message)s"))
2018
+ log = logging.getLogger("pylv200")
2019
+ log.handlers = [handler]
2020
+ log.setLevel(level)
2021
+ return log
2022
+
2023
+
2024
+ #: The flat flags this used to take, and what to type instead. Jamie has been
2025
+ #: typing the old form all week; an unrecognised flag should say what replaced
2026
+ #: it rather than print a usage block.
2027
+ MIGRATED = {
2028
+ "--input": "pull <target> (or `pull \"D:\\path\\_Exp_\"`)",
2029
+ "--split-channels": "pull ... --split-channels (unchanged, on `pull`)",
2030
+ "--no-resume": "pull (which ignores the ledger; `sync` uses it)",
2031
+ }
2032
+
2033
+
2034
+ def main(argv=None):
2035
+ """Entry point. Returns an exit code; raises nothing the user should see."""
2036
+ argv = join_negative_values(
2037
+ resolve_alias(sys.argv[1:] if argv is None else argv))
2038
+ verb = next((t for t in argv if not t.startswith("-")), "")
2039
+ if verb in UNAVAILABLE:
2040
+ print("pylv200 has no `%s`, and will not gain one:\n %s"
2041
+ % (verb, UNAVAILABLE[verb]), file=sys.stderr)
2042
+ return 2
2043
+ # Taken from the parser itself rather than a hand-kept list: a global flag
2044
+ # added later would otherwise be greeted by the migration message.
2045
+ globals_ = {opt for action in build_parser()._actions
2046
+ for opt in action.option_strings}
2047
+ if argv and argv[0].startswith("--") and argv[0] not in globals_:
2048
+ flag = argv[0].split("=")[0]
2049
+ print("pylv200 now takes a subcommand.\n %s -> pylv200 %s\n"
2050
+ " Try `pylv200 --help`, or `pylv200 list` to see what is there."
2051
+ % (flag, MIGRATED.get(flag, "pull <target> %s ..." % flag)),
2052
+ file=sys.stderr)
2053
+ return 2
2054
+
2055
+ args = build_parser().parse_args(argv)
2056
+ _JSON[0] = bool(getattr(args, "json", False))
2057
+ configure_logging(args)
2058
+ try:
2059
+ return COMMANDS[args.command](args)
2060
+ except LV200Error as exc:
2061
+ print(exc, file=sys.stderr)
2062
+ return 2
2063
+ finally:
2064
+ # A pull that raised part-way never reached its `done` event, and the
2065
+ # half-drawn bar would otherwise be the last thing on screen, under
2066
+ # whatever the shell printed next.
2067
+ if _BAR[0] is not None:
2068
+ _BAR[0].clear()
2069
+ _BAR[0] = None
2070
+
2071
+
2072
+ if __name__ == "__main__":
2073
+ sys.exit(main())