mosca 2.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. mosca/__init__.py +11 -0
  2. mosca/common/__init__.py +9 -0
  3. mosca/common/enums.py +79 -0
  4. mosca/common/logger.py +139 -0
  5. mosca/common/threads.py +199 -0
  6. mosca/common/utils.py +70 -0
  7. mosca/control/__init__.py +4 -0
  8. mosca/control/buffer.py +526 -0
  9. mosca/control/controller.py +278 -0
  10. mosca/control/datahandler.py +261 -0
  11. mosca/control/detectors.py +116 -0
  12. mosca/control/mosca.py +693 -0
  13. mosca/control/spectra.py +36 -0
  14. mosca/control/stats.py +43 -0
  15. mosca/control/tasks/__init__.py +3 -0
  16. mosca/control/tasks/background.py +118 -0
  17. mosca/control/tasks/rois.py +466 -0
  18. mosca/control/tasks/saving.py +391 -0
  19. mosca/control/tasks/spectrasums.py +185 -0
  20. mosca/control/tasks/task.py +201 -0
  21. mosca/control/tasks/taskmanager.py +133 -0
  22. mosca/devices/__init__.py +28 -0
  23. mosca/devices/hamamatsu/__init__.py +0 -0
  24. mosca/devices/hamamatsu/hamamatsu.py +88 -0
  25. mosca/devices/hamamatsu/hamamatsuhw.py +1298 -0
  26. mosca/devices/oceanoptics/__init__.py +0 -0
  27. mosca/devices/oceanoptics/oceanoptics.py +59 -0
  28. mosca/devices/oceanoptics/oceanopticshw.py +379 -0
  29. mosca/devices/simulation/__init__.py +0 -0
  30. mosca/devices/simulation/dante/__init__.py +1 -0
  31. mosca/devices/simulation/dante/simdante.py +33 -0
  32. mosca/devices/simulation/dante/simdantehw.py +506 -0
  33. mosca/devices/simulation/dummy/__init__.py +1 -0
  34. mosca/devices/simulation/dummy/dummy.py +299 -0
  35. mosca/devices/simulation/generator.py +607 -0
  36. mosca/devices/simulation/xia/__init__.py +1 -0
  37. mosca/devices/simulation/xia/iniparser.py +99 -0
  38. mosca/devices/simulation/xia/simfalconx.py +26 -0
  39. mosca/devices/simulation/xia/simfalconxhw.py +599 -0
  40. mosca/devices/xglab/__init__.py +0 -0
  41. mosca/devices/xglab/dante.py +126 -0
  42. mosca/devices/xglab/dantehw.py +1178 -0
  43. mosca/devices/xia/__init__.py +0 -0
  44. mosca/devices/xia/falconx.py +232 -0
  45. mosca/devices/xia/handel/__init__.py +14 -0
  46. mosca/devices/xia/handel/_cffi.py +62 -0
  47. mosca/devices/xia/handel/error.py +272 -0
  48. mosca/devices/xia/handel/handel_inc/handel_constants.h +127 -0
  49. mosca/devices/xia/handel/handel_inc/handel_errors.h +254 -0
  50. mosca/devices/xia/handel/handel_inc/handel_generic.h +66 -0
  51. mosca/devices/xia/handel/interface.py +1025 -0
  52. mosca/devices/xia/handel/mapping.py +107 -0
  53. mosca/devices/xia/handel/parser.py +62 -0
  54. mosca/devices/xia/handel/stats.py +85 -0
  55. mosca/devices/xia/mercury.py +160 -0
  56. mosca/devices/xia/utils.py +47 -0
  57. mosca/devices/xia/xia.py +575 -0
  58. mosca/devices/xia/xiads.py +161 -0
  59. mosca/mosca.py +0 -0
  60. mosca/tango/__init__.py +2 -0
  61. mosca/tango/controller.py +80 -0
  62. mosca/tango/main.py +163 -0
  63. mosca/tango/mosca.py +513 -0
  64. mosca/tango/moscav1.py +408 -0
  65. mosca/tango/shared.py +23 -0
  66. mosca/tango/tasks/__init__.py +0 -0
  67. mosca/tango/tasks/background.py +55 -0
  68. mosca/tango/utils.py +109 -0
  69. mosca-2.0.2.dist-info/METADATA +32 -0
  70. mosca-2.0.2.dist-info/RECORD +73 -0
  71. mosca-2.0.2.dist-info/WHEEL +4 -0
  72. mosca-2.0.2.dist-info/entry_points.txt +2 -0
  73. mosca-2.0.2.dist-info/licenses/LICENSE +166 -0
@@ -0,0 +1,391 @@
1
+ import logging
2
+ logger = logging.getLogger(__name__)
3
+
4
+ import os
5
+ import h5py
6
+ import hdf5plugin
7
+ import time
8
+ import contextlib
9
+ from dataclasses import dataclass, asdict
10
+
11
+ import numpy as np
12
+
13
+
14
+ from .task import SinkTask, Task
15
+
16
+
17
+ @dataclass
18
+ class SavingConfiguration:
19
+ directory: str = None
20
+ prefix: str = None
21
+ points_per_file: int = 10
22
+ hdf5_path: str = None
23
+
24
+
25
+ # TODO: prevent changing config while the task is not finished.
26
+ # TODO: optimize (buffer points to that we write points_per_file points at once)
27
+
28
+
29
+ class Saving(SinkTask):
30
+ def __init__(self, *args, **kwargs):
31
+ super().__init__(*args, **kwargs)
32
+ self._configuration = SavingConfiguration()
33
+ self._mosca_file = None
34
+ self._enabled = False
35
+
36
+ def metadata(self):
37
+ metadata = asdict(self._configuration)
38
+ metadata["enabled"] = self.is_active()
39
+ return metadata
40
+
41
+
42
+ def saving_off(self):
43
+ self._enabled = False
44
+
45
+ def saving_on(self):
46
+ self._enabled = True
47
+
48
+ def is_active(self):
49
+ return self._enabled
50
+
51
+ @property
52
+ def directory(self):
53
+ return self._configuration.directory or ""
54
+
55
+ @directory.setter
56
+ def directory(self, directory):
57
+ self._configuration.directory = directory
58
+
59
+ @property
60
+ def prefix(self):
61
+ return self._configuration.prefix or ""
62
+
63
+ @prefix.setter
64
+ def prefix(self, prefix):
65
+ self._configuration.prefix = prefix
66
+
67
+ @property
68
+ def points_per_file(self):
69
+ return self._configuration.points_per_file
70
+
71
+ @points_per_file.setter
72
+ def points_per_file(self, points_per_file):
73
+ self._configuration.points_per_file = points_per_file
74
+
75
+ @property
76
+ def hdf5_path(self):
77
+ return self._configuration.hdf5_path or ""
78
+
79
+ @hdf5_path.setter
80
+ def hdf5_path(self, hdf5_path):
81
+ self._configuration.hdf5_path = hdf5_path
82
+
83
+ def _n_items(self):
84
+ """ Size of the generated data, per point, per detector.
85
+ e.g:
86
+ - a task summing spectras for X detectors would return spectrum_size
87
+ => buffer will have a shape of (<num detectors>, <num points (buffer size)>, spectrum_size)
88
+ - a task computing a roi would return 1
89
+ => buffer will have a shape of (<num rois>, <num points>, 1)
90
+ """
91
+ return 0
92
+ # raise NotImplementedError
93
+
94
+ def _data_size(self):
95
+ return 0
96
+ # raise NotImplementedError
97
+
98
+ def _notify_start(self):
99
+ pass
100
+
101
+ def _notify_end(self):
102
+ if self._mosca_file:
103
+ self._mosca_file.finalize()
104
+ n_files = self._mosca_file.n_written_files
105
+ logger.info("Saving done: %s records in %s files.",
106
+ self.n_points, n_files)
107
+
108
+ def _n_items(self):
109
+ return 0
110
+
111
+ @property
112
+ def n_points(self):
113
+ return self._mosca_file.n_saved_total if self._mosca_file else 0
114
+
115
+ def _prepare(self, *args, **kwargs):
116
+ """ Initializes the writer with the current parameters. """
117
+ self._n_written = 0
118
+ self._mosca_file = MoscaFile(self.detectors,
119
+ self.spectrum_size,
120
+ self._configuration,
121
+ stats_names=self.stats_names)
122
+
123
+ def _detectors_changed(self):
124
+ pass
125
+
126
+ def _new_data(self, spectra, stats):
127
+ self._n_written = self._mosca_file.write(spectra,
128
+ stats,
129
+ overwrite=True)
130
+
131
+
132
+ class MoscaFile:
133
+ """ HDF5 file writer.
134
+
135
+ IMPORTANT: this object's finalize() method needs
136
+ to be called explicitly after the last points
137
+ have been processed. """
138
+
139
+ def __init__(self,
140
+ detectors,
141
+ spectrum_size,
142
+ configuration: SavingConfiguration,
143
+ stats_names=None,
144
+ chunks=None,
145
+ compression=None):
146
+
147
+ self._finalized = False
148
+ # misc info about the expected data
149
+ self._detectors = detectors
150
+ self._spectrum_size = spectrum_size
151
+ self._stats_names = stats_names
152
+
153
+ self._configuration = SavingConfiguration(**asdict(configuration))
154
+
155
+ # current file
156
+ self._filename = None
157
+
158
+ # number of records per detector per files
159
+ # self._records_per_file = records_per_file
160
+
161
+ # number of records on disk (total across all CLOSED files)
162
+ # always <= n_received_total
163
+ self._n_saved_total = 0
164
+
165
+ # number of records processed
166
+ self._n_received_total = 0
167
+
168
+ # hdf5 options (see h5py Dataset::require_dataset documentation)
169
+ # dataset chunking => None == auto
170
+ self._chunks = chunks
171
+ # compression
172
+ self._compression = compression or hdf5plugin.Bitshuffle()
173
+
174
+ # number of records written to the current file
175
+ # obviously cant be greater than records_per_file
176
+ self._dataset_index = 0
177
+
178
+ # -> file names stored as dict keys == ordered set
179
+ self._written_files = {}
180
+
181
+ # TODO: limit the list to a few filters
182
+ self._compressions = {comp.filter_name:comp for comp in hdf5plugin.get_filters()}
183
+
184
+ logger.info("Saving to file: \n"
185
+ f" - dir = %s,\n"
186
+ f" - prefix = %s,\n"
187
+ f" - h5_path = %s,\n"
188
+ f" - pts per files = %s\n"
189
+ f" - compresion = %s.\n",
190
+ self.directory, self.prefix,
191
+ self.hdf5_path, self.points_per_file,
192
+ self._compression.filter_name)
193
+
194
+ @property
195
+ def directory(self):
196
+ """ Directory whithin which the file(s) will be written. """
197
+ return self._configuration.directory
198
+
199
+ @property
200
+ def points_per_file(self):
201
+ """ Max. number of points within the file(s). """
202
+ return self._configuration.points_per_file
203
+
204
+ @property
205
+ def hdf5_path(self):
206
+ """ HDF5 group path within the file(s) """
207
+ return self._configuration.hdf5_path
208
+
209
+ @property
210
+ def prefix(self):
211
+ """ File prefix """
212
+ return self._configuration.prefix
213
+
214
+ def compressions(self):
215
+ """ Available compressions """
216
+ return tuple(self._compressions.keys())
217
+
218
+ @property
219
+ def files(self):
220
+ """ Written file(s).
221
+ WARNING: the last file is not guaranteed to be
222
+ finalized, i.e: it might be reoponed
223
+ if it is not full and more points are
224
+ processed. Open it at your own risk.
225
+ See: closed_files. """
226
+ return tuple(self._written_files.keys())
227
+
228
+ def closed_files(self):
229
+ """ Files that are safe to open. """
230
+ # copy so that it doesn't get updated
231
+ return self.files[:]
232
+
233
+ @property
234
+ def n_saved_total(self):
235
+ """ Number of points saved that can be safely read.
236
+ i.e: points that are in files that won't be
237
+ reopened later by this writer. """
238
+ return self._n_saved_total
239
+
240
+ @property
241
+ def n_written_files(self):
242
+ """ Number of written files. """
243
+ return len(self._written_files)
244
+
245
+ def finalize(self):
246
+ self._finalized = True
247
+ self._close_current_file()
248
+
249
+ def _close_current_file(self):
250
+ """ * Finalizes the current file:
251
+ - resize the datasets if necessary
252
+ (scan ended before the file could be filled)
253
+ - creates a virtual dataset referencing all spectra from all detectors
254
+
255
+ * Updates the n_saved_total counter.
256
+ """
257
+
258
+ # if the counter is already updated -> method was already called
259
+ if self._n_saved_total == self._n_received_total:
260
+ return
261
+
262
+ entry = self.hdf5_path
263
+ filename = self._filename
264
+ if filename:
265
+ n_det = len(self._detectors)
266
+ written = self._written_files[filename]
267
+
268
+ # resize needed
269
+ resize = written != self.points_per_file
270
+
271
+ with h5py.File(self._filename, "a") as h5f:
272
+ for i_det, det in enumerate(self._detectors):
273
+ det_entry = f"{entry}/spectra_{det:02d}"
274
+ sp_dset = h5f[det_entry]
275
+ dtype = sp_dset.dtype
276
+ if resize:
277
+ try:
278
+ sp_dset.resize(written, axis=0)
279
+ except TypeError as ex:
280
+ # resize is only available for chunked datasets
281
+ # we assume that all datasets have the same options
282
+ logger.exception(ex)
283
+ written = sp_dset.shape[0]
284
+ # with h5py.File(self._filename, "a") as h5f:
285
+ # virtual dataset of shape (n detectors, n points, spectrum size)
286
+ with h5f.build_virtual_dataset(f"{entry}/spectra",
287
+ (n_det, written, self._spectrum_size),
288
+ dtype=dtype) as layout:
289
+ for i_det, det in enumerate(self._detectors):
290
+ det_entry = f"{entry}/spectra_{det:02d}"
291
+ layout[i_det] = h5py.VirtualSource(self._filename,
292
+ det_entry,
293
+ shape=(written, self._spectrum_size),
294
+ dtype=dtype)
295
+ # update the counter
296
+ # so that listeners know that a new file is available for reading
297
+ # keep this at the end, for sync reasons, obviously
298
+ self._n_saved_total = self._n_received_total
299
+
300
+ def write(self, spectra, stats=None, rois=None, overwrite=False):
301
+ """ Writes data to files.
302
+ Only writes spectra at the moment. """
303
+
304
+ if self._finalized:
305
+ raise RuntimeError("This writer has already been finalized. Can't add new points to it.")
306
+
307
+ t_start = time.perf_counter()
308
+
309
+ os.makedirs(self.directory, exist_ok=True)
310
+
311
+ n_points = spectra.shape[1]
312
+ spectrum_size = self._spectrum_size
313
+ n_det = len(self._detectors)
314
+
315
+ logger.info("Writing %d.", n_points)
316
+
317
+ assert spectra.shape[0] == n_det
318
+ assert spectra.shape[2] == spectrum_size
319
+
320
+ # number of input records written
321
+ n_data_written = 0
322
+
323
+ # hdf5 group base name
324
+ entry = self.hdf5_path
325
+
326
+ dset_index = self._dataset_index
327
+
328
+ # loop until all incoming records have been written
329
+ while n_data_written != n_points:
330
+ # file open mode => append
331
+ mode = "a"
332
+ # number of records left to write
333
+ remaining = n_points - n_data_written
334
+
335
+ # either the first entry to be written
336
+ # or the previous file was filled and a new one needs to be created
337
+ if dset_index == 0:
338
+ file_index = self.n_written_files
339
+ self._filename = os.path.join(self.directory,
340
+ f"{self.prefix}spectra_{file_index:04d}.h5")
341
+ logger.info("New file: %s.", self._filename)
342
+ self._written_files[self._filename] = 0
343
+ if overwrite:
344
+ mode = "w"
345
+ # writing at most points_per_file records
346
+ n_to_write = min(remaining, self.points_per_file)
347
+
348
+ elif remaining + dset_index > self.points_per_file:
349
+ # current file is not full, but we have more records
350
+ # than what can fit in the file
351
+ n_to_write = self.points_per_file - dset_index
352
+ else:
353
+ # everything can fit in the current file
354
+ n_to_write = remaining
355
+
356
+ with h5py.File(self._filename, mode) as h5f:
357
+ # TODO decide if we want to:
358
+ # * write spectra as a 3D array (det, entry, spectra)
359
+ # and at a ref per detector to their respective spectra in that cube
360
+ # OR
361
+ # * one 2D array (entry, spectra) per detector
362
+ for i_det, det in enumerate(self._detectors):
363
+ sp_dset = h5f.require_dataset(f"{entry}/spectra_{det:02d}",
364
+ (self.points_per_file, self._spectrum_size),
365
+ dtype=spectra.dtype,
366
+ compression=self._compression)
367
+ sp_dset[dset_index:dset_index+n_to_write, :] = spectra[i_det, n_data_written:n_data_written+n_to_write, :]
368
+
369
+ self._written_files[self._filename] += n_to_write
370
+
371
+ n_data_written += n_to_write
372
+
373
+ # Total number of points stored in files at this point,
374
+ # NOT the number of points that can be read by potential observers
375
+ # as the current file might be reopened to add more points
376
+ # until it is full.
377
+ self._n_received_total += n_to_write
378
+
379
+ # next position in the dataset
380
+ dset_index = (dset_index + n_to_write) % self.points_per_file
381
+
382
+ if dset_index == 0:
383
+ # dataset index is back to 0 -> file is full
384
+ self._close_current_file()
385
+
386
+ # storing the current position in the current file
387
+ self._dataset_index = dset_index
388
+
389
+ t_end = time.perf_counter()
390
+ logger.info("Took %ss to write %s records.", t_end - t_start, n_points)
391
+ return n_points
@@ -0,0 +1,185 @@
1
+ import logging
2
+ logger = logging.getLogger(__name__)
3
+
4
+ import time
5
+
6
+ import numpy as np
7
+
8
+ from typing import NamedTuple
9
+
10
+ from .task import SinkTask
11
+
12
+
13
+ class SumParams(NamedTuple):
14
+ detectors: tuple[int]
15
+ sum_id: int
16
+ name: str = None
17
+
18
+
19
+ class InvalidSumException(Exception):
20
+ pass
21
+
22
+ # TODO: need complete rewrite
23
+
24
+ class Sum:
25
+ def __init__(self, sum, spectrum_size: int, detector_detectors: list[int], dtype=np.uint32):
26
+ self._dtype = dtype
27
+ self.sum_id = sum.sum_id
28
+
29
+ if not detector_detectors:
30
+ raise InvalidSumException(f"Sum {self.sum_id}: "
31
+ "missing detector detectors.")
32
+
33
+ if spectrum_size <= 0:
34
+ raise InvalidSumException(f"Sum {self.sum_id}: "
35
+ f"invalid spectrum_size (={spectrum_size}).")
36
+
37
+ schans = sum.detectors
38
+ detectors_slice = None
39
+ if schans == [-1] or not schans:
40
+ detectors_slice = np.s_[0:]
41
+ else:
42
+ index = [i_chan for i_chan, chan in enumerate(detector_detectors)
43
+ if chan in schans]
44
+ if index:
45
+ # checking if detectors are consecutive => use np.s_ to get a view instead of a copy
46
+ imin = min(index)
47
+ imax = max(index)
48
+ if np.array_equal(sorted(index), np.arange(imin, imax + 1)):
49
+ detectors_slice = np.s_[imin:imax + 1]
50
+ else:
51
+ detectors_slice = np.array(index, dtype=np.int32)
52
+ else:
53
+ raise InvalidSumException(f"No valid detectors available for sum {self.sum_id}."
54
+ f"sum chans={sum.detectors}, "
55
+ f"det chans={detector_detectors}.")
56
+ self._detectors_slice = detectors_slice
57
+
58
+ def __call__(self, spectra):
59
+ return np.sum(spectra[self._detectors_slice, :, :], axis=(0,), dtype=self._dtype)
60
+
61
+
62
+ class SpectraSums(SinkTask):
63
+ sums = property(lambda self: self._sums)
64
+ """ sums definitions. """
65
+
66
+ n_sums = property(lambda self: len(self.sums))
67
+ """ Number of sum definitions. """
68
+
69
+ sum_names = property(lambda self: [sum.name for sum in self.sums.values()])
70
+ """ Names of sum definitions. """
71
+
72
+ sum_ids = property(lambda self: list(self.sums.keys()))
73
+ """ Names of sum definitions. """
74
+
75
+ active_sum_ids = property(lambda self: list(self.active_sums.keys()))
76
+ """ Names of active sums. (i.e : sums that have at least one detector enabled. """
77
+
78
+ def __init__(self, *args, **kwargs):
79
+ super().__init__(*args, **kwargs)
80
+ self._last_sum_id = -1
81
+ self._sums = {}
82
+ self._active_sums = None
83
+
84
+ def _clear(self):
85
+ self._active_sums = None
86
+
87
+ def is_active(self):
88
+ return self.n_sums > 0
89
+
90
+ def _n_items(self):
91
+ return len(self.active_sums)
92
+
93
+ def _data_size(self):
94
+ return self._spectrum_size
95
+
96
+ def _prepare(self, *args, **kwargs):
97
+ self._update_sums()
98
+
99
+ def _new_data(self, spectra, stats):
100
+ self._calc_sums(spectra, stats)
101
+
102
+ @property
103
+ def active_sums(self):
104
+ if self._active_sums is None:
105
+ self._update_sums()
106
+ return self._active_sums
107
+
108
+ def _update_sums(self):
109
+ self._active_sums = {}
110
+ dtype = self.dtype
111
+ spectrum_size = self.spectrum_size
112
+ for sum_id, sum in self.sums.items():
113
+ try:
114
+ self._active_sums[sum_id] = Sum(sum, spectrum_size, self.detectors, dtype)
115
+ except InvalidSumException:
116
+ logger.warning("Ignoring invalid sum %d (detectors=%s",
117
+ sum_id,
118
+ sum.detectors)
119
+
120
+ def get_sum_by_name(self, name):
121
+ for sum in self.sums.values():
122
+ if sum.name == name:
123
+ return sum
124
+ return None
125
+
126
+ def get_sum(self, sum_id):
127
+ try:
128
+ return self.sums[sum_id]
129
+ except KeyError:
130
+ raise RuntimeError(f"Unknown sum id {sum_id}.")
131
+
132
+ def remove_all(self):
133
+ self._sums = {}
134
+ self._last_sum_id = -1
135
+ self.clear()
136
+
137
+ def remove_sum(self, sum_id):
138
+ self._sums.pop(sum_id, None)
139
+ self._update_sums()
140
+
141
+ def remove_sum_by_name(self, sum_name):
142
+ sum = self.get_sum_by_name(sum_name)
143
+ if sum is None:
144
+ return
145
+ self.remove_sum(sum.sum_id)
146
+
147
+ def _check_name_unicity(self, name, sum_id=None):
148
+ if name is None:
149
+ return True
150
+ other = self.get_sum_by_name(name)
151
+ if other is None:
152
+ return True
153
+ if sum_id is not None and sum_id == other.sum_id:
154
+ return True
155
+ raise RuntimeError(f"A sum with that name ({name}) "
156
+ f"already exists (with id={other.sum_id}).")
157
+
158
+ def add_sum(self, sum_id:int=None, name:str=None, detectors=None):
159
+ if not self._sums:
160
+ self._last_sum_id = -1
161
+
162
+ if sum_id in self._sums:
163
+ raise RuntimeError(f"Sum id {sum_id} already exists.")
164
+
165
+ self._check_name_unicity(name)
166
+
167
+ sum_id = self._last_sum_id + 1
168
+ self._sums[sum_id] = SumParams(tuple(detectors), sum_id, name)
169
+ self._last_sum_id = sum_id
170
+ return sum_id
171
+
172
+ def _calc_sums(self, spectra, stats):
173
+ t0 = time.perf_counter()
174
+ sums = self.active_sums
175
+ new_points = spectra.shape[1]
176
+ if sums:
177
+ buffer = self._data
178
+ sum_data = np.ndarray((self.n_sums, new_points, spectra.shape[2]), dtype=buffer.dtype)
179
+ for i_sum, sum in enumerate(sums.values()):
180
+ sum_data[i_sum] = sum(spectra)
181
+ buffer.add_data(sum_data)
182
+ logger.info("Took %ss to process %s spectra sums on %s points.",
183
+ time.perf_counter() - t0, len(sums), new_points)
184
+ else:
185
+ logger.debug("No sums defined (new points=%s)", new_points)