pymodaq_data 5.2.12__tar.gz → 5.2.14__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/PKG-INFO +12 -12
  2. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/README.rst +11 -11
  3. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/data.py +5 -5
  4. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/backends.py +21 -3
  5. pymodaq_data-5.2.14/src/pymodaq_data/h5modules/migration.py +90 -0
  6. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/.gitignore +0 -0
  7. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/LICENSE +0 -0
  8. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/pyproject.toml +0 -0
  9. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/__init__.py +0 -0
  10. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/config.py +0 -0
  11. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/__init__.py +0 -0
  12. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/browsing.py +0 -0
  13. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/data_saving.py +0 -0
  14. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/exporter.py +0 -0
  15. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/exporters/__init__.py +0 -0
  16. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/exporters/base.py +0 -0
  17. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/exporters/flimj.py +0 -0
  18. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/exporters/hyperspy.py +0 -0
  19. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/saving.py +0 -0
  20. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/swmr.py +0 -0
  21. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/utils.py +0 -0
  22. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/icon.ico +0 -0
  23. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/numpy_func.py +0 -0
  24. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/plotting/__init__.py +0 -0
  25. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/plotting/plotter/plotter.py +0 -0
  26. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/plotting/plotter/plotters/__init__.py +0 -0
  27. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/plotting/plotter/plotters/matplotlib_plotters.py +0 -0
  28. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/plotting/utils.py +0 -0
  29. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/post_treatment/__init__.py +0 -0
  30. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/post_treatment/process_to_scalar.py +0 -0
  31. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/resources/__init__.py +0 -0
  32. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/resources/config_template.toml +0 -0
  33. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/slicing.py +0 -0
  34. {pymodaq_data-5.2.12 → pymodaq_data-5.2.14}/src/pymodaq_data/splash.png +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: pymodaq_data
3
- Version: 5.2.12
3
+ Version: 5.2.14
4
4
  Summary: Modular Data Acquisition with Python
5
5
  Project-URL: Homepage, http://pymodaq.cnrs.fr
6
6
  Project-URL: Source, https://github.com/PyMoDAQ/PyMoDAQ
@@ -71,11 +71,11 @@ PyMoDAQ Data
71
71
  :target: https://pypi.org/project/pymodaq_data/
72
72
  :alt: Latest Version
73
73
 
74
- .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=dev
75
- :target: https://pymodaq.readthedocs.io/en/stable/?badge=dev
74
+ .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=5.2.x
75
+ :target: https://pymodaq.readthedocs.io/en/stable/?badge=5.2.x
76
76
  :alt: Documentation Status
77
77
 
78
- .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/dev/graph/badge.svg?token=IQNJRCQDM2
78
+ .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/5.2.x/graph/badge.svg?token=IQNJRCQDM2
79
79
  :target: https://codecov.io/gh/PyMoDAQ/PyMoDAQ
80
80
 
81
81
  +-------------+-------------+---------------+
@@ -94,28 +94,28 @@ PyMoDAQ Data
94
94
 
95
95
 
96
96
 
97
- .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/dev/tests_Linux_3.10.svg
97
+ .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.2.x/tests_Linux_3.10.svg
98
98
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
99
99
 
100
- .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/dev/tests_Linux_3.11.svg
100
+ .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.2.x/tests_Linux_3.11.svg
101
101
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
102
102
 
103
- .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/dev/tests_Linux_3.12.svg
103
+ .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.2.x/tests_Linux_3.12.svg
104
104
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
105
105
 
106
- .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/dev/tests_Linux_3.13.svg
106
+ .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.2.x/tests_Linux_3.13.svg
107
107
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
108
108
 
109
- .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/dev/tests_Windows_3.10.svg
109
+ .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.2.x/tests_Windows_3.10.svg
110
110
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
111
111
 
112
- .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/dev/tests_Windows_3.11.svg
112
+ .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.2.x/tests_Windows_3.11.svg
113
113
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
114
114
 
115
- .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/dev/tests_Windows_3.12.svg
115
+ .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.2.x/tests_Windows_3.12.svg
116
116
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
117
117
 
118
- .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/dev/tests_Windows_3.13.svg
118
+ .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.2.x/tests_Windows_3.13.svg
119
119
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
120
120
 
121
121
 
@@ -11,11 +11,11 @@ PyMoDAQ Data
11
11
  :target: https://pypi.org/project/pymodaq_data/
12
12
  :alt: Latest Version
13
13
 
14
- .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=dev
15
- :target: https://pymodaq.readthedocs.io/en/stable/?badge=dev
14
+ .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=5.2.x
15
+ :target: https://pymodaq.readthedocs.io/en/stable/?badge=5.2.x
16
16
  :alt: Documentation Status
17
17
 
18
- .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/dev/graph/badge.svg?token=IQNJRCQDM2
18
+ .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/5.2.x/graph/badge.svg?token=IQNJRCQDM2
19
19
  :target: https://codecov.io/gh/PyMoDAQ/PyMoDAQ
20
20
 
21
21
  +-------------+-------------+---------------+
@@ -34,28 +34,28 @@ PyMoDAQ Data
34
34
 
35
35
 
36
36
 
37
- .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/dev/tests_Linux_3.10.svg
37
+ .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.2.x/tests_Linux_3.10.svg
38
38
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
39
39
 
40
- .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/dev/tests_Linux_3.11.svg
40
+ .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.2.x/tests_Linux_3.11.svg
41
41
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
42
42
 
43
- .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/dev/tests_Linux_3.12.svg
43
+ .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.2.x/tests_Linux_3.12.svg
44
44
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
45
45
 
46
- .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/dev/tests_Linux_3.13.svg
46
+ .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.2.x/tests_Linux_3.13.svg
47
47
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
48
48
 
49
- .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/dev/tests_Windows_3.10.svg
49
+ .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.2.x/tests_Windows_3.10.svg
50
50
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
51
51
 
52
- .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/dev/tests_Windows_3.11.svg
52
+ .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.2.x/tests_Windows_3.11.svg
53
53
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
54
54
 
55
- .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/dev/tests_Windows_3.12.svg
55
+ .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.2.x/tests_Windows_3.12.svg
56
56
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
57
57
 
58
- .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/dev/tests_Windows_3.13.svg
58
+ .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.2.x/tests_Windows_3.13.svg
59
59
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
60
60
 
61
61
 
@@ -1661,12 +1661,12 @@ class AxesManagerBase:
1661
1661
  return axes
1662
1662
 
1663
1663
  def is_axis_signal(self, axis: Axis) -> bool:
1664
- """Check if an axis is considered signal or navigation"""
1665
- return axis.index in self._nav_indexes
1664
+ """Check if an axis is a signal one (its index is not a navigation index)"""
1665
+ return axis.index not in self._nav_indexes
1666
1666
 
1667
1667
  def is_axis_navigation(self, axis: Axis) -> bool:
1668
- """Check if an axis is considered signal or navigation"""
1669
- return axis.index not in self._nav_indexes
1668
+ """Check if an axis is a navigation one (its index is a navigation index)"""
1669
+ return axis.index in self._nav_indexes
1670
1670
 
1671
1671
  def __repr__(self):
1672
1672
  return self._get_dimension_str()
@@ -2894,7 +2894,7 @@ class DataWithAxes(DataBase, SerializableBase):
2894
2894
  return False
2895
2895
  return True
2896
2896
  try:
2897
- if errors is not None and check_errors(data, errors):
2897
+ if errors is not None and not check_errors(data, errors):
2898
2898
  errors = None
2899
2899
  old_data = self.data
2900
2900
  self._data = None
@@ -10,11 +10,11 @@ from pathlib import Path
10
10
  import numpy as np
11
11
  import importlib
12
12
  from importlib import metadata
13
- import pickle
14
13
 
15
14
  from pymodaq_utils.logger import set_logger, get_module_name
16
15
  from pymodaq_utils.config import GlobalConfig as Config
17
16
  from pymodaq_utils.utils import capitalize, JsonConverter
17
+ from pymodaq_utils.safe_pickle import restricted_loads, warn_legacy_format
18
18
  from pymodaq_utils import utils
19
19
  from pymodaq_utils.enums import BaseEnum, enum_checker
20
20
 
@@ -476,10 +476,28 @@ class StringARRAY(VLARRAY):
476
476
  super().append(data)
477
477
 
478
478
  def array_to_string(self, array):
479
- return pickle.loads(array)
479
+ """Decode a stored string
480
+
481
+ New files hold the plain UTF-8 bytes of the string. Files written by older versions hold a pickle
482
+ (always starting with the 0x80 protocol byte, which can never start a valid UTF-8 text): these are
483
+ read by a restricted loader that only rebuilds plain data and never runs code.
484
+ """
485
+ raw = bytes(array)
486
+ if raw[:1] == b'\x80':
487
+ warn_legacy_format(f'The string node {self.path} of the file {self._file_name()}')
488
+ return restricted_loads(raw)
489
+ return raw.decode('utf-8')
480
490
 
481
491
  def string_to_array(self, string):
482
- return np.frombuffer(pickle.dumps(string), np.uint8)
492
+ return np.frombuffer(string.encode('utf-8'), np.uint8)
493
+
494
+ def _file_name(self) -> str:
495
+ try:
496
+ if self.backend == 'tables':
497
+ return str(self.node._v_file.filename)
498
+ return str(self.node.file.filename)
499
+ except Exception:
500
+ return '(unknown)'
483
501
 
484
502
 
485
503
  class Attributes(object):
@@ -0,0 +1,90 @@
1
+ """Migration of the h5 files written by older versions of PyMoDAQ
2
+
3
+ Older versions stored the strings of the string nodes (such as the logger) as pickles. They are still read, but
4
+ the migration rewrites them as plain UTF-8, as the current version does, in a copy of the file.
5
+ """
6
+ import os
7
+ import shutil
8
+ from pathlib import Path
9
+ from typing import Optional, Tuple, Union
10
+
11
+ import numpy as np
12
+
13
+ from pymodaq_utils.safe_pickle import restricted_loads, UnsafePickleError
14
+
15
+ from pymodaq_data.h5modules.backends import get_attr
16
+
17
+
18
+ def default_destination(source: Path) -> Path:
19
+ return source.with_name(f'{source.stem}_migrated{source.suffix}')
20
+
21
+
22
+ def migrate_h5_file(source: Union[str, Path], destination: Optional[Union[str, Path]] = None) -> Tuple[Path, int]:
23
+ """Write a copy of a h5 file in which the pickled strings are rewritten as UTF-8
24
+
25
+ The source file is never modified, and an existing destination file is never overwritten.
26
+
27
+ Parameters
28
+ ----------
29
+ source: str or Path
30
+ The file to migrate
31
+ destination: str or Path
32
+ The file to create, by default the name of the source followed by _migrated
33
+
34
+ Returns
35
+ -------
36
+ Path: the migrated file
37
+ int: the number of string nodes that were migrated
38
+
39
+ Raises
40
+ ------
41
+ FileExistsError
42
+ if the destination exists
43
+ UnsafePickleError
44
+ if a string node contains anything else than a string, in this case nothing is written
45
+ """
46
+ import h5py
47
+
48
+ source = Path(source)
49
+ destination = default_destination(source) if destination is None else Path(destination)
50
+ if destination.exists():
51
+ raise FileExistsError(f'{destination} already exists, it will not be overwritten')
52
+ part = destination.with_name(destination.name + '.part')
53
+ shutil.copyfile(source, part)
54
+ try:
55
+ migrated = 0
56
+ with h5py.File(part, 'r+') as h5file:
57
+ to_migrate = []
58
+ h5file.visititems(lambda name, obj: to_migrate.append(obj) if _is_string_node(obj) else None)
59
+ for dataset in to_migrate:
60
+ migrated += _migrate_string_node(dataset)
61
+ os.replace(part, destination)
62
+ finally:
63
+ if part.exists():
64
+ part.unlink()
65
+ return destination, migrated
66
+
67
+
68
+ def _is_string_node(obj) -> bool:
69
+ import h5py
70
+ if not isinstance(obj, h5py.Dataset):
71
+ return False
72
+ try:
73
+ return get_attr(obj, 'CLASS', 'h5py') == 'VLARRAY' and get_attr(obj, 'subdtype', 'h5py') == 'string'
74
+ except KeyError:
75
+ return False
76
+
77
+
78
+ def _migrate_string_node(dataset) -> int:
79
+ """Rewrite the pickled strings of a node, returns 1 if the node had some, 0 otherwise"""
80
+ new_values = {}
81
+ for index in range(len(dataset)):
82
+ raw = bytes(dataset[index])
83
+ if raw[:1] == b'\x80':
84
+ text = restricted_loads(raw)
85
+ if not isinstance(text, str):
86
+ raise UnsafePickleError(f'Refused to migrate {dataset.name}: it does not contain only strings')
87
+ new_values[index] = text
88
+ for index, text in new_values.items(): # nothing is written if a value above was refused
89
+ dataset[index] = np.frombuffer(text.encode('utf-8'), np.uint8)
90
+ return 1 if new_values else 0
File without changes
File without changes