pymodaq_data 5.1.9__tar.gz → 5.1.15__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/PKG-INFO +13 -13
  2. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/README.rst +11 -11
  3. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/data.py +4 -4
  4. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/backends.py +21 -3
  5. pymodaq_data-5.1.15/src/pymodaq_data/h5modules/migration.py +90 -0
  6. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/.gitignore +0 -0
  7. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/LICENSE +0 -0
  8. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/pyproject.toml +0 -0
  9. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/__init__.py +0 -0
  10. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/__init__.py +0 -0
  11. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/browsing.py +0 -0
  12. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/data_saving.py +0 -0
  13. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/exporter.py +0 -0
  14. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/exporters/__init__.py +0 -0
  15. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/exporters/base.py +0 -0
  16. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/exporters/flimj.py +0 -0
  17. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/exporters/hyperspy.py +0 -0
  18. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/saving.py +0 -0
  19. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/utils.py +0 -0
  20. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/icon.ico +0 -0
  21. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/numpy_func.py +0 -0
  22. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/plotting/__init__.py +0 -0
  23. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/plotting/plotter/plotter.py +0 -0
  24. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/plotting/plotter/plotters/__init__.py +0 -0
  25. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/plotting/plotter/plotters/matplotlib_plotters.py +0 -0
  26. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/post_treatment/__init__.py +0 -0
  27. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/post_treatment/process_to_scalar.py +0 -0
  28. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/slicing.py +0 -0
  29. {pymodaq_data-5.1.9 → pymodaq_data-5.1.15}/src/pymodaq_data/splash.png +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: pymodaq_data
3
- Version: 5.1.9
3
+ Version: 5.1.15
4
4
  Summary: Modular Data Acquisition with Python
5
5
  Project-URL: Homepage, http://pymodaq.cnrs.fr
6
6
  Project-URL: Source, https://github.com/PyMoDAQ/PyMoDAQ
@@ -68,11 +68,11 @@ PyMoDAQ Data
68
68
  :target: https://pypi.org/project/pymodaq_data/
69
69
  :alt: Latest Version
70
70
 
71
- .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=malik-irain-patch-1
72
- :target: https://pymodaq.readthedocs.io/en/stable/?badge=malik-irain-patch-1
71
+ .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=5.1.x
72
+ :target: https://pymodaq.readthedocs.io/en/stable/?badge=5.1.x
73
73
  :alt: Documentation Status
74
74
 
75
- .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/malik-irain-patch-1/graph/badge.svg?token=IQNJRCQDM2
75
+ .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/5.1.x/graph/badge.svg?token=IQNJRCQDM2
76
76
  :target: https://codecov.io/gh/PyMoDAQ/PyMoDAQ
77
77
 
78
78
  +-------------+-------------+---------------+
@@ -91,28 +91,28 @@ PyMoDAQ Data
91
91
 
92
92
 
93
93
 
94
- .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/malik-irain-patch-1/tests_Linux_3.10.svg
94
+ .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.10.svg
95
95
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
96
96
 
97
- .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/malik-irain-patch-1/tests_Linux_3.11.svg
97
+ .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.11.svg
98
98
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
99
99
 
100
- .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/malik-irain-patch-1/tests_Linux_3.12.svg
100
+ .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.12.svg
101
101
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
102
102
 
103
- .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/malik-irain-patch-1/tests_Linux_3.13.svg
103
+ .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.13.svg
104
104
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
105
105
 
106
- .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/malik-irain-patch-1/tests_Windows_3.10.svg
106
+ .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.10.svg
107
107
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
108
108
 
109
- .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/malik-irain-patch-1/tests_Windows_3.11.svg
109
+ .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.11.svg
110
110
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
111
111
 
112
- .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/malik-irain-patch-1/tests_Windows_3.12.svg
112
+ .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.12.svg
113
113
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
114
114
 
115
- .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/malik-irain-patch-1/tests_Windows_3.13.svg
115
+ .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.13.svg
116
116
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
117
117
 
118
118
 
@@ -11,11 +11,11 @@ PyMoDAQ Data
11
11
  :target: https://pypi.org/project/pymodaq_data/
12
12
  :alt: Latest Version
13
13
 
14
- .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=malik-irain-patch-1
15
- :target: https://pymodaq.readthedocs.io/en/stable/?badge=malik-irain-patch-1
14
+ .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=5.1.x
15
+ :target: https://pymodaq.readthedocs.io/en/stable/?badge=5.1.x
16
16
  :alt: Documentation Status
17
17
 
18
- .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/malik-irain-patch-1/graph/badge.svg?token=IQNJRCQDM2
18
+ .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/5.1.x/graph/badge.svg?token=IQNJRCQDM2
19
19
  :target: https://codecov.io/gh/PyMoDAQ/PyMoDAQ
20
20
 
21
21
  +-------------+-------------+---------------+
@@ -34,28 +34,28 @@ PyMoDAQ Data
34
34
 
35
35
 
36
36
 
37
- .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/malik-irain-patch-1/tests_Linux_3.10.svg
37
+ .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.10.svg
38
38
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
39
39
 
40
- .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/malik-irain-patch-1/tests_Linux_3.11.svg
40
+ .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.11.svg
41
41
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
42
42
 
43
- .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/malik-irain-patch-1/tests_Linux_3.12.svg
43
+ .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.12.svg
44
44
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
45
45
 
46
- .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/malik-irain-patch-1/tests_Linux_3.13.svg
46
+ .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.13.svg
47
47
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
48
48
 
49
- .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/malik-irain-patch-1/tests_Windows_3.10.svg
49
+ .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.10.svg
50
50
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
51
51
 
52
- .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/malik-irain-patch-1/tests_Windows_3.11.svg
52
+ .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.11.svg
53
53
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
54
54
 
55
- .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/malik-irain-patch-1/tests_Windows_3.12.svg
55
+ .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.12.svg
56
56
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
57
57
 
58
- .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/malik-irain-patch-1/tests_Windows_3.13.svg
58
+ .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.13.svg
59
59
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
60
60
 
61
61
 
@@ -1536,12 +1536,12 @@ class AxesManagerBase:
1536
1536
  return axes
1537
1537
 
1538
1538
  def is_axis_signal(self, axis: Axis) -> bool:
1539
- """Check if an axis is considered signal or navigation"""
1540
- return axis.index in self._nav_indexes
1539
+ """Check if an axis is a signal one (its index is not a navigation index)"""
1540
+ return axis.index not in self._nav_indexes
1541
1541
 
1542
1542
  def is_axis_navigation(self, axis: Axis) -> bool:
1543
- """Check if an axis is considered signal or navigation"""
1544
- return axis.index not in self._nav_indexes
1543
+ """Check if an axis is a navigation one (its index is a navigation index)"""
1544
+ return axis.index in self._nav_indexes
1545
1545
 
1546
1546
  def __repr__(self):
1547
1547
  return self._get_dimension_str()
@@ -9,12 +9,12 @@ from enum import Enum
9
9
  import numpy as np
10
10
  import importlib
11
11
  from importlib import metadata
12
- import pickle
13
12
  from typing import Dict
14
13
 
15
14
  from pymodaq_utils.logger import set_logger, get_module_name
16
15
  from pymodaq_utils.config import Config
17
16
  from pymodaq_utils.utils import capitalize, JsonConverter
17
+ from pymodaq_utils.safe_pickle import restricted_loads, warn_legacy_format
18
18
  from pymodaq_utils import utils
19
19
  from pymodaq_utils.enums import BaseEnum, enum_checker
20
20
 
@@ -450,10 +450,28 @@ class StringARRAY(VLARRAY):
450
450
  super().append(data)
451
451
 
452
452
  def array_to_string(self, array):
453
- return pickle.loads(array)
453
+ """Decode a stored string
454
+
455
+ New files hold the plain UTF-8 bytes of the string. Files written by older versions hold a pickle
456
+ (always starting with the 0x80 protocol byte, which can never start a valid UTF-8 text): these are
457
+ read by a restricted loader that only rebuilds plain data and never runs code.
458
+ """
459
+ raw = bytes(array)
460
+ if raw[:1] == b'\x80':
461
+ warn_legacy_format(f'The string node {self.path} of the file {self._file_name()}')
462
+ return restricted_loads(raw)
463
+ return raw.decode('utf-8')
454
464
 
455
465
  def string_to_array(self, string):
456
- return np.frombuffer(pickle.dumps(string), np.uint8)
466
+ return np.frombuffer(string.encode('utf-8'), np.uint8)
467
+
468
+ def _file_name(self) -> str:
469
+ try:
470
+ if self.backend == 'tables':
471
+ return str(self.node._v_file.filename)
472
+ return str(self.node.file.filename)
473
+ except Exception:
474
+ return '(unknown)'
457
475
 
458
476
 
459
477
  class Attributes(object):
@@ -0,0 +1,90 @@
1
+ """Migration of the h5 files written by older versions of PyMoDAQ
2
+
3
+ Older versions stored the strings of the string nodes (such as the logger) as pickles. They are still read, but
4
+ the migration rewrites them as plain UTF-8, as the current version does, in a copy of the file.
5
+ """
6
+ import os
7
+ import shutil
8
+ from pathlib import Path
9
+ from typing import Optional, Tuple, Union
10
+
11
+ import numpy as np
12
+
13
+ from pymodaq_utils.safe_pickle import restricted_loads, UnsafePickleError
14
+
15
+ from pymodaq_data.h5modules.backends import get_attr
16
+
17
+
18
+ def default_destination(source: Path) -> Path:
19
+ return source.with_name(f'{source.stem}_migrated{source.suffix}')
20
+
21
+
22
+ def migrate_h5_file(source: Union[str, Path], destination: Optional[Union[str, Path]] = None) -> Tuple[Path, int]:
23
+ """Write a copy of a h5 file in which the pickled strings are rewritten as UTF-8
24
+
25
+ The source file is never modified, and an existing destination file is never overwritten.
26
+
27
+ Parameters
28
+ ----------
29
+ source: str or Path
30
+ The file to migrate
31
+ destination: str or Path
32
+ The file to create, by default the name of the source followed by _migrated
33
+
34
+ Returns
35
+ -------
36
+ Path: the migrated file
37
+ int: the number of string nodes that were migrated
38
+
39
+ Raises
40
+ ------
41
+ FileExistsError
42
+ if the destination exists
43
+ UnsafePickleError
44
+ if a string node contains anything else than a string, in this case nothing is written
45
+ """
46
+ import h5py
47
+
48
+ source = Path(source)
49
+ destination = default_destination(source) if destination is None else Path(destination)
50
+ if destination.exists():
51
+ raise FileExistsError(f'{destination} already exists, it will not be overwritten')
52
+ part = destination.with_name(destination.name + '.part')
53
+ shutil.copyfile(source, part)
54
+ try:
55
+ migrated = 0
56
+ with h5py.File(part, 'r+') as h5file:
57
+ to_migrate = []
58
+ h5file.visititems(lambda name, obj: to_migrate.append(obj) if _is_string_node(obj) else None)
59
+ for dataset in to_migrate:
60
+ migrated += _migrate_string_node(dataset)
61
+ os.replace(part, destination)
62
+ finally:
63
+ if part.exists():
64
+ part.unlink()
65
+ return destination, migrated
66
+
67
+
68
+ def _is_string_node(obj) -> bool:
69
+ import h5py
70
+ if not isinstance(obj, h5py.Dataset):
71
+ return False
72
+ try:
73
+ return get_attr(obj, 'CLASS', 'h5py') == 'VLARRAY' and get_attr(obj, 'subdtype', 'h5py') == 'string'
74
+ except KeyError:
75
+ return False
76
+
77
+
78
+ def _migrate_string_node(dataset) -> int:
79
+ """Rewrite the pickled strings of a node, returns 1 if the node had some, 0 otherwise"""
80
+ new_values = {}
81
+ for index in range(len(dataset)):
82
+ raw = bytes(dataset[index])
83
+ if raw[:1] == b'\x80':
84
+ text = restricted_loads(raw)
85
+ if not isinstance(text, str):
86
+ raise UnsafePickleError(f'Refused to migrate {dataset.name}: it does not contain only strings')
87
+ new_values[index] = text
88
+ for index, text in new_values.items(): # nothing is written if a value above was refused
89
+ dataset[index] = np.frombuffer(text.encode('utf-8'), np.uint8)
90
+ return 1 if new_values else 0
File without changes
File without changes