pymodaq_data 5.3.9__tar.gz → 5.3.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/PKG-INFO +12 -12
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/README.rst +11 -11
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/backends.py +21 -3
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/data_saving.py +63 -38
- pymodaq_data-5.3.11/src/pymodaq_data/h5modules/migration.py +90 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/.gitignore +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/LICENSE +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/pyproject.toml +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/__init__.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/config.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/data.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/__init__.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/browsing.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/exporter.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/exporters/__init__.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/exporters/base.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/exporters/flimj.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/exporters/hyperspy.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/saving.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/swmr.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/utils.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/icon.ico +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/numpy_func.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/plotting/__init__.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/plotting/plotter/plotter.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/plotting/plotter/plotters/__init__.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/plotting/plotter/plotters/matplotlib_plotters.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/plotting/utils.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/post_treatment/__init__.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/post_treatment/process_to_scalar.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/py.typed +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/resources/__init__.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/resources/config_template.toml +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/slicing.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/splash.png +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: pymodaq_data
|
|
3
|
-
Version: 5.3.
|
|
3
|
+
Version: 5.3.11
|
|
4
4
|
Summary: Modular Data Acquisition with Python
|
|
5
5
|
Project-URL: Homepage, http://pymodaq.cnrs.fr
|
|
6
6
|
Project-URL: Source, https://github.com/PyMoDAQ/PyMoDAQ
|
|
@@ -71,11 +71,11 @@ PyMoDAQ Data
|
|
|
71
71
|
:target: https://pypi.org/project/pymodaq_data/
|
|
72
72
|
:alt: Latest Version
|
|
73
73
|
|
|
74
|
-
.. image:: https://readthedocs.org/projects/pymodaq/badge/?version=
|
|
75
|
-
:target: https://pymodaq.readthedocs.io/en/stable/?badge=
|
|
74
|
+
.. image:: https://readthedocs.org/projects/pymodaq/badge/?version=5.3.x
|
|
75
|
+
:target: https://pymodaq.readthedocs.io/en/stable/?badge=5.3.x
|
|
76
76
|
:alt: Documentation Status
|
|
77
77
|
|
|
78
|
-
.. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/
|
|
78
|
+
.. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/5.3.x/graph/badge.svg?token=IQNJRCQDM2
|
|
79
79
|
:target: https://codecov.io/gh/PyMoDAQ/PyMoDAQ
|
|
80
80
|
|
|
81
81
|
+-------------+-------------+---------------+
|
|
@@ -94,28 +94,28 @@ PyMoDAQ Data
|
|
|
94
94
|
|
|
95
95
|
|
|
96
96
|
|
|
97
|
-
.. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
97
|
+
.. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.10.svg
|
|
98
98
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
99
99
|
|
|
100
|
-
.. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
100
|
+
.. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.11.svg
|
|
101
101
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
102
102
|
|
|
103
|
-
.. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
103
|
+
.. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.12.svg
|
|
104
104
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
105
105
|
|
|
106
|
-
.. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
106
|
+
.. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.13.svg
|
|
107
107
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
108
108
|
|
|
109
|
-
.. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
109
|
+
.. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.10.svg
|
|
110
110
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
111
111
|
|
|
112
|
-
.. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
112
|
+
.. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.11.svg
|
|
113
113
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
114
114
|
|
|
115
|
-
.. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
115
|
+
.. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.12.svg
|
|
116
116
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
117
117
|
|
|
118
|
-
.. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
118
|
+
.. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.13.svg
|
|
119
119
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
120
120
|
|
|
121
121
|
|
|
@@ -11,11 +11,11 @@ PyMoDAQ Data
|
|
|
11
11
|
:target: https://pypi.org/project/pymodaq_data/
|
|
12
12
|
:alt: Latest Version
|
|
13
13
|
|
|
14
|
-
.. image:: https://readthedocs.org/projects/pymodaq/badge/?version=
|
|
15
|
-
:target: https://pymodaq.readthedocs.io/en/stable/?badge=
|
|
14
|
+
.. image:: https://readthedocs.org/projects/pymodaq/badge/?version=5.3.x
|
|
15
|
+
:target: https://pymodaq.readthedocs.io/en/stable/?badge=5.3.x
|
|
16
16
|
:alt: Documentation Status
|
|
17
17
|
|
|
18
|
-
.. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/
|
|
18
|
+
.. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/5.3.x/graph/badge.svg?token=IQNJRCQDM2
|
|
19
19
|
:target: https://codecov.io/gh/PyMoDAQ/PyMoDAQ
|
|
20
20
|
|
|
21
21
|
+-------------+-------------+---------------+
|
|
@@ -34,28 +34,28 @@ PyMoDAQ Data
|
|
|
34
34
|
|
|
35
35
|
|
|
36
36
|
|
|
37
|
-
.. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
37
|
+
.. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.10.svg
|
|
38
38
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
39
39
|
|
|
40
|
-
.. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
40
|
+
.. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.11.svg
|
|
41
41
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
42
42
|
|
|
43
|
-
.. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
43
|
+
.. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.12.svg
|
|
44
44
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
45
45
|
|
|
46
|
-
.. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
46
|
+
.. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.13.svg
|
|
47
47
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
48
48
|
|
|
49
|
-
.. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
49
|
+
.. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.10.svg
|
|
50
50
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
51
51
|
|
|
52
|
-
.. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
52
|
+
.. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.11.svg
|
|
53
53
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
54
54
|
|
|
55
|
-
.. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
55
|
+
.. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.12.svg
|
|
56
56
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
57
57
|
|
|
58
|
-
.. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
58
|
+
.. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.13.svg
|
|
59
59
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
60
60
|
|
|
61
61
|
|
|
@@ -10,11 +10,11 @@ from pathlib import Path
|
|
|
10
10
|
import numpy as np
|
|
11
11
|
import importlib
|
|
12
12
|
from importlib import metadata
|
|
13
|
-
import pickle
|
|
14
13
|
|
|
15
14
|
from pymodaq_utils.logger import set_logger, get_module_name
|
|
16
15
|
from pymodaq_utils.config import GlobalConfig as Config
|
|
17
16
|
from pymodaq_utils.utils import capitalize, JsonConverter
|
|
17
|
+
from pymodaq_utils.safe_pickle import restricted_loads, warn_legacy_format
|
|
18
18
|
from pymodaq_utils import utils
|
|
19
19
|
from pymodaq_utils.enums import BaseEnum, enum_checker
|
|
20
20
|
|
|
@@ -475,10 +475,28 @@ class StringARRAY(VLARRAY):
|
|
|
475
475
|
super().append(data)
|
|
476
476
|
|
|
477
477
|
def array_to_string(self, array):
|
|
478
|
-
|
|
478
|
+
"""Decode a stored string
|
|
479
|
+
|
|
480
|
+
New files hold the plain UTF-8 bytes of the string. Files written by older versions hold a pickle
|
|
481
|
+
(always starting with the 0x80 protocol byte, which can never start a valid UTF-8 text): these are
|
|
482
|
+
read by a restricted loader that only rebuilds plain data and never runs code.
|
|
483
|
+
"""
|
|
484
|
+
raw = bytes(array)
|
|
485
|
+
if raw[:1] == b'\x80':
|
|
486
|
+
warn_legacy_format(f'The string node {self.path} of the file {self._file_name()}')
|
|
487
|
+
return restricted_loads(raw)
|
|
488
|
+
return raw.decode('utf-8')
|
|
479
489
|
|
|
480
490
|
def string_to_array(self, string):
|
|
481
|
-
return np.frombuffer(
|
|
491
|
+
return np.frombuffer(string.encode('utf-8'), np.uint8)
|
|
492
|
+
|
|
493
|
+
def _file_name(self) -> str:
|
|
494
|
+
try:
|
|
495
|
+
if self.backend == 'tables':
|
|
496
|
+
return str(self.node._v_file.filename)
|
|
497
|
+
return str(self.node.file.filename)
|
|
498
|
+
except Exception:
|
|
499
|
+
return '(unknown)'
|
|
482
500
|
|
|
483
501
|
|
|
484
502
|
class Attributes(object):
|
|
@@ -16,6 +16,7 @@ import numpy as np
|
|
|
16
16
|
|
|
17
17
|
from pymodaq_utils.abstract import ABCMeta, abstract_attribute
|
|
18
18
|
from pymodaq_utils.enums import enum_checker
|
|
19
|
+
from pymodaq_utils.warnings import deprecation_msg
|
|
19
20
|
from pymodaq_data.data import (Axis, DataDim, DataWithAxes, DataToExport, DataDistribution,
|
|
20
21
|
DataDimError, squeeze)
|
|
21
22
|
from .saving import DataType, H5SaverLowLevel
|
|
@@ -36,7 +37,7 @@ class DataBundle:
|
|
|
36
37
|
"""Convenience class to hold data to be saved or plotted"""
|
|
37
38
|
dte: DataToExport
|
|
38
39
|
node: str = None
|
|
39
|
-
settings: str = ''
|
|
40
|
+
settings: str = '' # deprecated and ignored: settings are written when the group is created
|
|
40
41
|
indexes: list[int] = None # indexes within an eventual Extended array (see DAQ_Scan)
|
|
41
42
|
axis_values: list[float | np.ndarray] = None # axis values within an eventual Enlargeable array (see Optimizers)
|
|
42
43
|
distribution: DataDistribution = field(
|
|
@@ -337,6 +338,31 @@ class DataSaverLoader(DataManagement):
|
|
|
337
338
|
""" Get the opened status of the underlying hdf5 file"""
|
|
338
339
|
return self._h5saver.isopen()
|
|
339
340
|
|
|
341
|
+
@staticmethod
|
|
342
|
+
def _get_data_metadata(data: DataWithAxes, ind_data: int, **metadata) -> dict:
|
|
343
|
+
"""Metadata saved as attributes of the h5 array holding the ind_data-th array of a DataWithAxes
|
|
344
|
+
|
|
345
|
+
Common to all the savers so that they all write the same attributes: the ones listed below, the
|
|
346
|
+
``units`` of the data and the values of its ``extra_attributes`` (see
|
|
347
|
+
:meth:`DataWithAxes.add_extra_attribute`).
|
|
348
|
+
|
|
349
|
+
Parameters
|
|
350
|
+
----------
|
|
351
|
+
data: DataWithAxes
|
|
352
|
+
ind_data: int
|
|
353
|
+
index of the array within the data
|
|
354
|
+
metadata: dict
|
|
355
|
+
specific entries (``nav_indexes``, ``distribution``...), they take precedence over the defaults
|
|
356
|
+
"""
|
|
357
|
+
meta = dict(timestamp=data.timestamp, label=data.labels[ind_data],
|
|
358
|
+
source=data.source.name, distribution=data.distribution.name,
|
|
359
|
+
origin=data.origin, units=data.units,
|
|
360
|
+
nav_indexes=tuple(data.nav_indexes) if data.nav_indexes is not None else None)
|
|
361
|
+
meta.update(metadata)
|
|
362
|
+
for name in data.extra_attributes:
|
|
363
|
+
meta[name] = getattr(data, name)
|
|
364
|
+
return meta
|
|
365
|
+
|
|
340
366
|
def add_data(self, where: Union[Node, str], data: DataWithAxes, save_axes=True, **kwargs):
|
|
341
367
|
"""Adds Array nodes to a given location adding eventually axes as others nodes and metadata
|
|
342
368
|
|
|
@@ -349,15 +375,7 @@ class DataSaverLoader(DataManagement):
|
|
|
349
375
|
"""
|
|
350
376
|
|
|
351
377
|
for ind_data in range(len(data)):
|
|
352
|
-
metadata =
|
|
353
|
-
source=data.source.name, distribution=data.distribution.name,
|
|
354
|
-
origin=data.origin,
|
|
355
|
-
units=data.units,
|
|
356
|
-
nav_indexes=tuple(data.nav_indexes)
|
|
357
|
-
if data.nav_indexes is not None else None)
|
|
358
|
-
metadata.update(kwargs)
|
|
359
|
-
for name in data.extra_attributes:
|
|
360
|
-
metadata[name] = getattr(data, name)
|
|
378
|
+
metadata = self._get_data_metadata(data, ind_data, **kwargs)
|
|
361
379
|
self._h5saver.add_array(where, self._get_next_node_name(where), self.data_type,
|
|
362
380
|
title=data.name, array_to_save=data[ind_data],
|
|
363
381
|
data_dimension=data.dim.name, metadata=metadata)
|
|
@@ -628,12 +646,9 @@ class DataEnlargeableSaver(DataSaverLoader):
|
|
|
628
646
|
array_type=data[ind_data].dtype,
|
|
629
647
|
enlargeable=True,
|
|
630
648
|
data_dimension=data.dim.name,
|
|
631
|
-
metadata=
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
distribution='spread',
|
|
635
|
-
origin=data.origin,
|
|
636
|
-
nav_indexes=tuple(nav_indexes)))
|
|
649
|
+
metadata=self._get_data_metadata(data, ind_data,
|
|
650
|
+
distribution='spread',
|
|
651
|
+
nav_indexes=tuple(nav_indexes)))
|
|
637
652
|
if data.errors is not None:
|
|
638
653
|
error_dwa = data.errors_as_dwa()
|
|
639
654
|
self._h5saver.add_array(where,
|
|
@@ -645,12 +660,9 @@ class DataEnlargeableSaver(DataSaverLoader):
|
|
|
645
660
|
array_type=error_dwa[ind_data].dtype,
|
|
646
661
|
enlargeable=True,
|
|
647
662
|
data_dimension=error_dwa.dim.name,
|
|
648
|
-
metadata=
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
distribution='spread',
|
|
652
|
-
origin=error_dwa.origin,
|
|
653
|
-
nav_indexes=tuple(nav_indexes)))
|
|
663
|
+
metadata=self._get_data_metadata(error_dwa, ind_data,
|
|
664
|
+
distribution='spread',
|
|
665
|
+
nav_indexes=tuple(nav_indexes)))
|
|
654
666
|
|
|
655
667
|
if add_enl_axes:
|
|
656
668
|
for ind_enl_axis in range(self._n_enl_axes):
|
|
@@ -781,10 +793,9 @@ class DataExtendedSaver(DataSaverLoader):
|
|
|
781
793
|
scan_shape=self.extended_shape,
|
|
782
794
|
add_scan_dim=True,
|
|
783
795
|
data_dimension=data.dim.name,
|
|
784
|
-
metadata=
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
nav_indexes=tuple(nav_indexes)))
|
|
796
|
+
metadata=self._get_data_metadata(data, ind_data,
|
|
797
|
+
distribution=distribution.name,
|
|
798
|
+
nav_indexes=tuple(nav_indexes)))
|
|
788
799
|
|
|
789
800
|
if data.errors is not None:
|
|
790
801
|
error_dwa = data.errors_as_dwa()
|
|
@@ -798,10 +809,9 @@ class DataExtendedSaver(DataSaverLoader):
|
|
|
798
809
|
scan_shape=self.extended_shape,
|
|
799
810
|
add_scan_dim=True,
|
|
800
811
|
data_dimension=error_dwa.dim.name,
|
|
801
|
-
metadata=
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
nav_indexes=tuple(nav_indexes)))
|
|
812
|
+
metadata=self._get_data_metadata(error_dwa, ind_data,
|
|
813
|
+
distribution=distribution.name,
|
|
814
|
+
nav_indexes=tuple(nav_indexes)))
|
|
805
815
|
|
|
806
816
|
|
|
807
817
|
if save_axes:
|
|
@@ -908,6 +918,21 @@ class DataToExportSaver:
|
|
|
908
918
|
def add_data_bundle(self, data: DataBundle):
|
|
909
919
|
self.add_data(data.node, data.dte, data.settings)
|
|
910
920
|
|
|
921
|
+
@staticmethod
|
|
922
|
+
def _warn_settings_deprecated(settings_as_xml=''):
|
|
923
|
+
"""Warn if settings are given to ``add_data``: they are ignored
|
|
924
|
+
|
|
925
|
+
The settings of a module, a scan or an extension are written as the ``settings`` attribute
|
|
926
|
+
of its group when this group is created (``add_det_group``, ``add_act_group``,
|
|
927
|
+
``add_scan_group``, ``add_incremental_group``...), not when data is added.
|
|
928
|
+
"""
|
|
929
|
+
if settings_as_xml not in ('', b'', None):
|
|
930
|
+
deprecation_msg('The settings_as_xml argument of DataToExportSaver.add_data (and '
|
|
931
|
+
'DataBundle.settings) is deprecated and ignored: give the settings '
|
|
932
|
+
'when creating the group, with the settings_as_xml argument of '
|
|
933
|
+
'H5SaverLowLevel.add_incremental_group (add_det_group, add_act_group, '
|
|
934
|
+
'add_scan_group...)', stacklevel=4)
|
|
935
|
+
|
|
911
936
|
def add_data(self, where: Union[Node, str], data: DataToExport, settings_as_xml='', **kwargs):
|
|
912
937
|
"""
|
|
913
938
|
|
|
@@ -917,15 +942,14 @@ class DataToExportSaver:
|
|
|
917
942
|
the path of a given node or the node itself
|
|
918
943
|
data: DataToExport
|
|
919
944
|
settings_as_xml: str
|
|
920
|
-
|
|
945
|
+
deprecated and ignored, settings are written when the group is created
|
|
921
946
|
Keyword Arguments:
|
|
922
|
-
all extra metadata to be saved
|
|
947
|
+
all extra metadata to be saved as attributes of each data array
|
|
923
948
|
|
|
924
949
|
"""
|
|
925
950
|
|
|
951
|
+
self._warn_settings_deprecated(settings_as_xml)
|
|
926
952
|
dims = data.get_dim_presents()
|
|
927
|
-
if settings_as_xml != '' and 'settings' not in kwargs:
|
|
928
|
-
kwargs['settings]'] = settings_as_xml
|
|
929
953
|
for dim in dims:
|
|
930
954
|
dim_group = self._h5saver.get_set_group(where, dim)
|
|
931
955
|
for ind, dwa in enumerate(data.get_data_from_dim(dim)):
|
|
@@ -1041,9 +1065,9 @@ class DataToExportEnlargeableSaver(DataToExportSaver):
|
|
|
1041
1065
|
axis_value: float or np.ndarray #deprecated in 4.2.0, use axis_values
|
|
1042
1066
|
The next value (or values) of the enlarged axis
|
|
1043
1067
|
settings_as_xml: str
|
|
1044
|
-
|
|
1068
|
+
deprecated and ignored, settings are written when the group is created
|
|
1045
1069
|
Keyword Arguments:
|
|
1046
|
-
all extra metadata to be saved
|
|
1070
|
+
all extra metadata to be saved as attributes of each data array
|
|
1047
1071
|
"""
|
|
1048
1072
|
|
|
1049
1073
|
if axis_values is None and axis_value is not None:
|
|
@@ -1145,11 +1169,12 @@ class DataToExportExtendedSaver(DataToExportSaver):
|
|
|
1145
1169
|
indexes where to save data in the init h5array (should have the same length as
|
|
1146
1170
|
extended_shape and with values coherent with this shape
|
|
1147
1171
|
settings_as_xml: str
|
|
1148
|
-
|
|
1172
|
+
deprecated and ignored, settings are written when the group is created
|
|
1149
1173
|
Keyword Arguments:
|
|
1150
|
-
|
|
1174
|
+
not used by this saver
|
|
1151
1175
|
|
|
1152
1176
|
"""
|
|
1177
|
+
self._warn_settings_deprecated(settings_as_xml)
|
|
1153
1178
|
dims = data.get_dim_presents()
|
|
1154
1179
|
for dim in dims:
|
|
1155
1180
|
dim_group = self._h5saver.get_set_group(where, dim)
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Migration of the h5 files written by older versions of PyMoDAQ
|
|
2
|
+
|
|
3
|
+
Older versions stored the strings of the string nodes (such as the logger) as pickles. They are still read, but
|
|
4
|
+
the migration rewrites them as plain UTF-8, as the current version does, in a copy of the file.
|
|
5
|
+
"""
|
|
6
|
+
import os
|
|
7
|
+
import shutil
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Optional, Tuple, Union
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
|
|
13
|
+
from pymodaq_utils.safe_pickle import restricted_loads, UnsafePickleError
|
|
14
|
+
|
|
15
|
+
from pymodaq_data.h5modules.backends import get_attr
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def default_destination(source: Path) -> Path:
|
|
19
|
+
return source.with_name(f'{source.stem}_migrated{source.suffix}')
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def migrate_h5_file(source: Union[str, Path], destination: Optional[Union[str, Path]] = None) -> Tuple[Path, int]:
|
|
23
|
+
"""Write a copy of a h5 file in which the pickled strings are rewritten as UTF-8
|
|
24
|
+
|
|
25
|
+
The source file is never modified, and an existing destination file is never overwritten.
|
|
26
|
+
|
|
27
|
+
Parameters
|
|
28
|
+
----------
|
|
29
|
+
source: str or Path
|
|
30
|
+
The file to migrate
|
|
31
|
+
destination: str or Path
|
|
32
|
+
The file to create, by default the name of the source followed by _migrated
|
|
33
|
+
|
|
34
|
+
Returns
|
|
35
|
+
-------
|
|
36
|
+
Path: the migrated file
|
|
37
|
+
int: the number of string nodes that were migrated
|
|
38
|
+
|
|
39
|
+
Raises
|
|
40
|
+
------
|
|
41
|
+
FileExistsError
|
|
42
|
+
if the destination exists
|
|
43
|
+
UnsafePickleError
|
|
44
|
+
if a string node contains anything else than a string, in this case nothing is written
|
|
45
|
+
"""
|
|
46
|
+
import h5py
|
|
47
|
+
|
|
48
|
+
source = Path(source)
|
|
49
|
+
destination = default_destination(source) if destination is None else Path(destination)
|
|
50
|
+
if destination.exists():
|
|
51
|
+
raise FileExistsError(f'{destination} already exists, it will not be overwritten')
|
|
52
|
+
part = destination.with_name(destination.name + '.part')
|
|
53
|
+
shutil.copyfile(source, part)
|
|
54
|
+
try:
|
|
55
|
+
migrated = 0
|
|
56
|
+
with h5py.File(part, 'r+') as h5file:
|
|
57
|
+
to_migrate = []
|
|
58
|
+
h5file.visititems(lambda name, obj: to_migrate.append(obj) if _is_string_node(obj) else None)
|
|
59
|
+
for dataset in to_migrate:
|
|
60
|
+
migrated += _migrate_string_node(dataset)
|
|
61
|
+
os.replace(part, destination)
|
|
62
|
+
finally:
|
|
63
|
+
if part.exists():
|
|
64
|
+
part.unlink()
|
|
65
|
+
return destination, migrated
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _is_string_node(obj) -> bool:
|
|
69
|
+
import h5py
|
|
70
|
+
if not isinstance(obj, h5py.Dataset):
|
|
71
|
+
return False
|
|
72
|
+
try:
|
|
73
|
+
return get_attr(obj, 'CLASS', 'h5py') == 'VLARRAY' and get_attr(obj, 'subdtype', 'h5py') == 'string'
|
|
74
|
+
except KeyError:
|
|
75
|
+
return False
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _migrate_string_node(dataset) -> int:
|
|
79
|
+
"""Rewrite the pickled strings of a node, returns 1 if the node had some, 0 otherwise"""
|
|
80
|
+
new_values = {}
|
|
81
|
+
for index in range(len(dataset)):
|
|
82
|
+
raw = bytes(dataset[index])
|
|
83
|
+
if raw[:1] == b'\x80':
|
|
84
|
+
text = restricted_loads(raw)
|
|
85
|
+
if not isinstance(text, str):
|
|
86
|
+
raise UnsafePickleError(f'Refused to migrate {dataset.name}: it does not contain only strings')
|
|
87
|
+
new_values[index] = text
|
|
88
|
+
for index, text in new_values.items(): # nothing is written if a value above was refused
|
|
89
|
+
dataset[index] = np.frombuffer(text.encode('utf-8'), np.uint8)
|
|
90
|
+
return 1 if new_values else 0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/plotting/plotter/plotters/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/post_treatment/process_to_scalar.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|