pymodaq_data 5.3.9__tar.gz → 5.3.11__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/PKG-INFO +12 -12
  2. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/README.rst +11 -11
  3. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/backends.py +21 -3
  4. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/data_saving.py +63 -38
  5. pymodaq_data-5.3.11/src/pymodaq_data/h5modules/migration.py +90 -0
  6. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/.gitignore +0 -0
  7. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/LICENSE +0 -0
  8. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/pyproject.toml +0 -0
  9. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/__init__.py +0 -0
  10. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/config.py +0 -0
  11. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/data.py +0 -0
  12. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/__init__.py +0 -0
  13. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/browsing.py +0 -0
  14. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/exporter.py +0 -0
  15. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/exporters/__init__.py +0 -0
  16. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/exporters/base.py +0 -0
  17. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/exporters/flimj.py +0 -0
  18. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/exporters/hyperspy.py +0 -0
  19. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/saving.py +0 -0
  20. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/swmr.py +0 -0
  21. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/h5modules/utils.py +0 -0
  22. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/icon.ico +0 -0
  23. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/numpy_func.py +0 -0
  24. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/plotting/__init__.py +0 -0
  25. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/plotting/plotter/plotter.py +0 -0
  26. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/plotting/plotter/plotters/__init__.py +0 -0
  27. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/plotting/plotter/plotters/matplotlib_plotters.py +0 -0
  28. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/plotting/utils.py +0 -0
  29. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/post_treatment/__init__.py +0 -0
  30. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/post_treatment/process_to_scalar.py +0 -0
  31. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/py.typed +0 -0
  32. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/resources/__init__.py +0 -0
  33. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/resources/config_template.toml +0 -0
  34. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/slicing.py +0 -0
  35. {pymodaq_data-5.3.9 → pymodaq_data-5.3.11}/src/pymodaq_data/splash.png +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: pymodaq_data
3
- Version: 5.3.9
3
+ Version: 5.3.11
4
4
  Summary: Modular Data Acquisition with Python
5
5
  Project-URL: Homepage, http://pymodaq.cnrs.fr
6
6
  Project-URL: Source, https://github.com/PyMoDAQ/PyMoDAQ
@@ -71,11 +71,11 @@ PyMoDAQ Data
71
71
  :target: https://pypi.org/project/pymodaq_data/
72
72
  :alt: Latest Version
73
73
 
74
- .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=patch-3
75
- :target: https://pymodaq.readthedocs.io/en/stable/?badge=patch-3
74
+ .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=5.3.x
75
+ :target: https://pymodaq.readthedocs.io/en/stable/?badge=5.3.x
76
76
  :alt: Documentation Status
77
77
 
78
- .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/patch-3/graph/badge.svg?token=IQNJRCQDM2
78
+ .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/5.3.x/graph/badge.svg?token=IQNJRCQDM2
79
79
  :target: https://codecov.io/gh/PyMoDAQ/PyMoDAQ
80
80
 
81
81
  +-------------+-------------+---------------+
@@ -94,28 +94,28 @@ PyMoDAQ Data
94
94
 
95
95
 
96
96
 
97
- .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Linux_3.10.svg
97
+ .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.10.svg
98
98
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
99
99
 
100
- .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Linux_3.11.svg
100
+ .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.11.svg
101
101
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
102
102
 
103
- .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Linux_3.12.svg
103
+ .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.12.svg
104
104
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
105
105
 
106
- .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Linux_3.13.svg
106
+ .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.13.svg
107
107
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
108
108
 
109
- .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Windows_3.10.svg
109
+ .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.10.svg
110
110
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
111
111
 
112
- .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Windows_3.11.svg
112
+ .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.11.svg
113
113
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
114
114
 
115
- .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Windows_3.12.svg
115
+ .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.12.svg
116
116
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
117
117
 
118
- .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Windows_3.13.svg
118
+ .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.13.svg
119
119
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
120
120
 
121
121
 
@@ -11,11 +11,11 @@ PyMoDAQ Data
11
11
  :target: https://pypi.org/project/pymodaq_data/
12
12
  :alt: Latest Version
13
13
 
14
- .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=patch-3
15
- :target: https://pymodaq.readthedocs.io/en/stable/?badge=patch-3
14
+ .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=5.3.x
15
+ :target: https://pymodaq.readthedocs.io/en/stable/?badge=5.3.x
16
16
  :alt: Documentation Status
17
17
 
18
- .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/patch-3/graph/badge.svg?token=IQNJRCQDM2
18
+ .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/5.3.x/graph/badge.svg?token=IQNJRCQDM2
19
19
  :target: https://codecov.io/gh/PyMoDAQ/PyMoDAQ
20
20
 
21
21
  +-------------+-------------+---------------+
@@ -34,28 +34,28 @@ PyMoDAQ Data
34
34
 
35
35
 
36
36
 
37
- .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Linux_3.10.svg
37
+ .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.10.svg
38
38
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
39
39
 
40
- .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Linux_3.11.svg
40
+ .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.11.svg
41
41
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
42
42
 
43
- .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Linux_3.12.svg
43
+ .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.12.svg
44
44
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
45
45
 
46
- .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Linux_3.13.svg
46
+ .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.13.svg
47
47
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
48
48
 
49
- .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Windows_3.10.svg
49
+ .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.10.svg
50
50
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
51
51
 
52
- .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Windows_3.11.svg
52
+ .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.11.svg
53
53
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
54
54
 
55
- .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Windows_3.12.svg
55
+ .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.12.svg
56
56
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
57
57
 
58
- .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Windows_3.13.svg
58
+ .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.13.svg
59
59
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
60
60
 
61
61
 
@@ -10,11 +10,11 @@ from pathlib import Path
10
10
  import numpy as np
11
11
  import importlib
12
12
  from importlib import metadata
13
- import pickle
14
13
 
15
14
  from pymodaq_utils.logger import set_logger, get_module_name
16
15
  from pymodaq_utils.config import GlobalConfig as Config
17
16
  from pymodaq_utils.utils import capitalize, JsonConverter
17
+ from pymodaq_utils.safe_pickle import restricted_loads, warn_legacy_format
18
18
  from pymodaq_utils import utils
19
19
  from pymodaq_utils.enums import BaseEnum, enum_checker
20
20
 
@@ -475,10 +475,28 @@ class StringARRAY(VLARRAY):
475
475
  super().append(data)
476
476
 
477
477
  def array_to_string(self, array):
478
- return pickle.loads(array)
478
+ """Decode a stored string
479
+
480
+ New files hold the plain UTF-8 bytes of the string. Files written by older versions hold a pickle
481
+ (always starting with the 0x80 protocol byte, which can never start a valid UTF-8 text): these are
482
+ read by a restricted loader that only rebuilds plain data and never runs code.
483
+ """
484
+ raw = bytes(array)
485
+ if raw[:1] == b'\x80':
486
+ warn_legacy_format(f'The string node {self.path} of the file {self._file_name()}')
487
+ return restricted_loads(raw)
488
+ return raw.decode('utf-8')
479
489
 
480
490
  def string_to_array(self, string):
481
- return np.frombuffer(pickle.dumps(string), np.uint8)
491
+ return np.frombuffer(string.encode('utf-8'), np.uint8)
492
+
493
+ def _file_name(self) -> str:
494
+ try:
495
+ if self.backend == 'tables':
496
+ return str(self.node._v_file.filename)
497
+ return str(self.node.file.filename)
498
+ except Exception:
499
+ return '(unknown)'
482
500
 
483
501
 
484
502
  class Attributes(object):
@@ -16,6 +16,7 @@ import numpy as np
16
16
 
17
17
  from pymodaq_utils.abstract import ABCMeta, abstract_attribute
18
18
  from pymodaq_utils.enums import enum_checker
19
+ from pymodaq_utils.warnings import deprecation_msg
19
20
  from pymodaq_data.data import (Axis, DataDim, DataWithAxes, DataToExport, DataDistribution,
20
21
  DataDimError, squeeze)
21
22
  from .saving import DataType, H5SaverLowLevel
@@ -36,7 +37,7 @@ class DataBundle:
36
37
  """Convenience class to hold data to be saved or plotted"""
37
38
  dte: DataToExport
38
39
  node: str = None
39
- settings: str = ''
40
+ settings: str = '' # deprecated and ignored: settings are written when the group is created
40
41
  indexes: list[int] = None # indexes within an eventual Extended array (see DAQ_Scan)
41
42
  axis_values: list[float | np.ndarray] = None # axis values within an eventual Enlargeable array (see Optimizers)
42
43
  distribution: DataDistribution = field(
@@ -337,6 +338,31 @@ class DataSaverLoader(DataManagement):
337
338
  """ Get the opened status of the underlying hdf5 file"""
338
339
  return self._h5saver.isopen()
339
340
 
341
+ @staticmethod
342
+ def _get_data_metadata(data: DataWithAxes, ind_data: int, **metadata) -> dict:
343
+ """Metadata saved as attributes of the h5 array holding the ind_data-th array of a DataWithAxes
344
+
345
+ Common to all the savers so that they all write the same attributes: the ones listed below, the
346
+ ``units`` of the data and the values of its ``extra_attributes`` (see
347
+ :meth:`DataWithAxes.add_extra_attribute`).
348
+
349
+ Parameters
350
+ ----------
351
+ data: DataWithAxes
352
+ ind_data: int
353
+ index of the array within the data
354
+ metadata: dict
355
+ specific entries (``nav_indexes``, ``distribution``...), they take precedence over the defaults
356
+ """
357
+ meta = dict(timestamp=data.timestamp, label=data.labels[ind_data],
358
+ source=data.source.name, distribution=data.distribution.name,
359
+ origin=data.origin, units=data.units,
360
+ nav_indexes=tuple(data.nav_indexes) if data.nav_indexes is not None else None)
361
+ meta.update(metadata)
362
+ for name in data.extra_attributes:
363
+ meta[name] = getattr(data, name)
364
+ return meta
365
+
340
366
  def add_data(self, where: Union[Node, str], data: DataWithAxes, save_axes=True, **kwargs):
341
367
  """Adds Array nodes to a given location adding eventually axes as others nodes and metadata
342
368
 
@@ -349,15 +375,7 @@ class DataSaverLoader(DataManagement):
349
375
  """
350
376
 
351
377
  for ind_data in range(len(data)):
352
- metadata = dict(timestamp=data.timestamp, label=data.labels[ind_data],
353
- source=data.source.name, distribution=data.distribution.name,
354
- origin=data.origin,
355
- units=data.units,
356
- nav_indexes=tuple(data.nav_indexes)
357
- if data.nav_indexes is not None else None)
358
- metadata.update(kwargs)
359
- for name in data.extra_attributes:
360
- metadata[name] = getattr(data, name)
378
+ metadata = self._get_data_metadata(data, ind_data, **kwargs)
361
379
  self._h5saver.add_array(where, self._get_next_node_name(where), self.data_type,
362
380
  title=data.name, array_to_save=data[ind_data],
363
381
  data_dimension=data.dim.name, metadata=metadata)
@@ -628,12 +646,9 @@ class DataEnlargeableSaver(DataSaverLoader):
628
646
  array_type=data[ind_data].dtype,
629
647
  enlargeable=True,
630
648
  data_dimension=data.dim.name,
631
- metadata=dict(timestamp=data.timestamp,
632
- label=data.labels[ind_data],
633
- source=data.source.name,
634
- distribution='spread',
635
- origin=data.origin,
636
- nav_indexes=tuple(nav_indexes)))
649
+ metadata=self._get_data_metadata(data, ind_data,
650
+ distribution='spread',
651
+ nav_indexes=tuple(nav_indexes)))
637
652
  if data.errors is not None:
638
653
  error_dwa = data.errors_as_dwa()
639
654
  self._h5saver.add_array(where,
@@ -645,12 +660,9 @@ class DataEnlargeableSaver(DataSaverLoader):
645
660
  array_type=error_dwa[ind_data].dtype,
646
661
  enlargeable=True,
647
662
  data_dimension=error_dwa.dim.name,
648
- metadata=dict(timestamp=error_dwa.timestamp,
649
- label=error_dwa.labels[ind_data],
650
- source=error_dwa.source.name,
651
- distribution='spread',
652
- origin=error_dwa.origin,
653
- nav_indexes=tuple(nav_indexes)))
663
+ metadata=self._get_data_metadata(error_dwa, ind_data,
664
+ distribution='spread',
665
+ nav_indexes=tuple(nav_indexes)))
654
666
 
655
667
  if add_enl_axes:
656
668
  for ind_enl_axis in range(self._n_enl_axes):
@@ -781,10 +793,9 @@ class DataExtendedSaver(DataSaverLoader):
781
793
  scan_shape=self.extended_shape,
782
794
  add_scan_dim=True,
783
795
  data_dimension=data.dim.name,
784
- metadata=dict(timestamp=data.timestamp, label=data.labels[ind_data],
785
- source=data.source.name, distribution=distribution.name,
786
- origin=data.origin,
787
- nav_indexes=tuple(nav_indexes)))
796
+ metadata=self._get_data_metadata(data, ind_data,
797
+ distribution=distribution.name,
798
+ nav_indexes=tuple(nav_indexes)))
788
799
 
789
800
  if data.errors is not None:
790
801
  error_dwa = data.errors_as_dwa()
@@ -798,10 +809,9 @@ class DataExtendedSaver(DataSaverLoader):
798
809
  scan_shape=self.extended_shape,
799
810
  add_scan_dim=True,
800
811
  data_dimension=error_dwa.dim.name,
801
- metadata=dict(timestamp=error_dwa.timestamp, label=error_dwa.labels[ind_data],
802
- source=error_dwa.source.name, distribution=distribution.name,
803
- origin=error_dwa.origin,
804
- nav_indexes=tuple(nav_indexes)))
812
+ metadata=self._get_data_metadata(error_dwa, ind_data,
813
+ distribution=distribution.name,
814
+ nav_indexes=tuple(nav_indexes)))
805
815
 
806
816
 
807
817
  if save_axes:
@@ -908,6 +918,21 @@ class DataToExportSaver:
908
918
  def add_data_bundle(self, data: DataBundle):
909
919
  self.add_data(data.node, data.dte, data.settings)
910
920
 
921
+ @staticmethod
922
+ def _warn_settings_deprecated(settings_as_xml=''):
923
+ """Warn if settings are given to ``add_data``: they are ignored
924
+
925
+ The settings of a module, a scan or an extension are written as the ``settings`` attribute
926
+ of its group when this group is created (``add_det_group``, ``add_act_group``,
927
+ ``add_scan_group``, ``add_incremental_group``...), not when data is added.
928
+ """
929
+ if settings_as_xml not in ('', b'', None):
930
+ deprecation_msg('The settings_as_xml argument of DataToExportSaver.add_data (and '
931
+ 'DataBundle.settings) is deprecated and ignored: give the settings '
932
+ 'when creating the group, with the settings_as_xml argument of '
933
+ 'H5SaverLowLevel.add_incremental_group (add_det_group, add_act_group, '
934
+ 'add_scan_group...)', stacklevel=4)
935
+
911
936
  def add_data(self, where: Union[Node, str], data: DataToExport, settings_as_xml='', **kwargs):
912
937
  """
913
938
 
@@ -917,15 +942,14 @@ class DataToExportSaver:
917
942
  the path of a given node or the node itself
918
943
  data: DataToExport
919
944
  settings_as_xml: str
920
- The settings parameter as an XML string
945
+ deprecated and ignored, settings are written when the group is created
921
946
  Keyword Arguments:
922
- all extra metadata to be saved in the group node where data will be saved
947
+ all extra metadata to be saved as attributes of each data array
923
948
 
924
949
  """
925
950
 
951
+ self._warn_settings_deprecated(settings_as_xml)
926
952
  dims = data.get_dim_presents()
927
- if settings_as_xml != '' and 'settings' not in kwargs:
928
- kwargs['settings]'] = settings_as_xml
929
953
  for dim in dims:
930
954
  dim_group = self._h5saver.get_set_group(where, dim)
931
955
  for ind, dwa in enumerate(data.get_data_from_dim(dim)):
@@ -1041,9 +1065,9 @@ class DataToExportEnlargeableSaver(DataToExportSaver):
1041
1065
  axis_value: float or np.ndarray #deprecated in 4.2.0, use axis_values
1042
1066
  The next value (or values) of the enlarged axis
1043
1067
  settings_as_xml: str
1044
- The settings parameter as an XML string
1068
+ deprecated and ignored, settings are written when the group is created
1045
1069
  Keyword Arguments:
1046
- all extra metadata to be saved in the group node where data will be saved
1070
+ all extra metadata to be saved as attributes of each data array
1047
1071
  """
1048
1072
 
1049
1073
  if axis_values is None and axis_value is not None:
@@ -1145,11 +1169,12 @@ class DataToExportExtendedSaver(DataToExportSaver):
1145
1169
  indexes where to save data in the init h5array (should have the same length as
1146
1170
  extended_shape and with values coherent with this shape
1147
1171
  settings_as_xml: str
1148
- The settings parameter as an XML string
1172
+ deprecated and ignored, settings are written when the group is created
1149
1173
  Keyword Arguments:
1150
- all extra metadata to be saved in the group node where data will be saved
1174
+ not used by this saver
1151
1175
 
1152
1176
  """
1177
+ self._warn_settings_deprecated(settings_as_xml)
1153
1178
  dims = data.get_dim_presents()
1154
1179
  for dim in dims:
1155
1180
  dim_group = self._h5saver.get_set_group(where, dim)
@@ -0,0 +1,90 @@
1
+ """Migration of the h5 files written by older versions of PyMoDAQ
2
+
3
+ Older versions stored the strings of the string nodes (such as the logger) as pickles. They are still read, but
4
+ the migration rewrites them as plain UTF-8, as the current version does, in a copy of the file.
5
+ """
6
+ import os
7
+ import shutil
8
+ from pathlib import Path
9
+ from typing import Optional, Tuple, Union
10
+
11
+ import numpy as np
12
+
13
+ from pymodaq_utils.safe_pickle import restricted_loads, UnsafePickleError
14
+
15
+ from pymodaq_data.h5modules.backends import get_attr
16
+
17
+
18
+ def default_destination(source: Path) -> Path:
19
+ return source.with_name(f'{source.stem}_migrated{source.suffix}')
20
+
21
+
22
+ def migrate_h5_file(source: Union[str, Path], destination: Optional[Union[str, Path]] = None) -> Tuple[Path, int]:
23
+ """Write a copy of a h5 file in which the pickled strings are rewritten as UTF-8
24
+
25
+ The source file is never modified, and an existing destination file is never overwritten.
26
+
27
+ Parameters
28
+ ----------
29
+ source: str or Path
30
+ The file to migrate
31
+ destination: str or Path
32
+ The file to create, by default the name of the source followed by _migrated
33
+
34
+ Returns
35
+ -------
36
+ Path: the migrated file
37
+ int: the number of string nodes that were migrated
38
+
39
+ Raises
40
+ ------
41
+ FileExistsError
42
+ if the destination exists
43
+ UnsafePickleError
44
+ if a string node contains anything else than a string, in this case nothing is written
45
+ """
46
+ import h5py
47
+
48
+ source = Path(source)
49
+ destination = default_destination(source) if destination is None else Path(destination)
50
+ if destination.exists():
51
+ raise FileExistsError(f'{destination} already exists, it will not be overwritten')
52
+ part = destination.with_name(destination.name + '.part')
53
+ shutil.copyfile(source, part)
54
+ try:
55
+ migrated = 0
56
+ with h5py.File(part, 'r+') as h5file:
57
+ to_migrate = []
58
+ h5file.visititems(lambda name, obj: to_migrate.append(obj) if _is_string_node(obj) else None)
59
+ for dataset in to_migrate:
60
+ migrated += _migrate_string_node(dataset)
61
+ os.replace(part, destination)
62
+ finally:
63
+ if part.exists():
64
+ part.unlink()
65
+ return destination, migrated
66
+
67
+
68
+ def _is_string_node(obj) -> bool:
69
+ import h5py
70
+ if not isinstance(obj, h5py.Dataset):
71
+ return False
72
+ try:
73
+ return get_attr(obj, 'CLASS', 'h5py') == 'VLARRAY' and get_attr(obj, 'subdtype', 'h5py') == 'string'
74
+ except KeyError:
75
+ return False
76
+
77
+
78
+ def _migrate_string_node(dataset) -> int:
79
+ """Rewrite the pickled strings of a node, returns 1 if the node had some, 0 otherwise"""
80
+ new_values = {}
81
+ for index in range(len(dataset)):
82
+ raw = bytes(dataset[index])
83
+ if raw[:1] == b'\x80':
84
+ text = restricted_loads(raw)
85
+ if not isinstance(text, str):
86
+ raise UnsafePickleError(f'Refused to migrate {dataset.name}: it does not contain only strings')
87
+ new_values[index] = text
88
+ for index, text in new_values.items(): # nothing is written if a value above was refused
89
+ dataset[index] = np.frombuffer(text.encode('utf-8'), np.uint8)
90
+ return 1 if new_values else 0
File without changes
File without changes