pymodaq_data 5.3.9__tar.gz → 5.3.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/PKG-INFO +12 -12
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/README.rst +11 -11
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/backends.py +21 -3
- pymodaq_data-5.3.10/src/pymodaq_data/h5modules/migration.py +90 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/.gitignore +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/LICENSE +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/pyproject.toml +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/__init__.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/config.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/data.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/__init__.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/browsing.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/data_saving.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/exporter.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/exporters/__init__.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/exporters/base.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/exporters/flimj.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/exporters/hyperspy.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/saving.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/swmr.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/utils.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/icon.ico +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/numpy_func.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/plotting/__init__.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/plotting/plotter/plotter.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/plotting/plotter/plotters/__init__.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/plotting/plotter/plotters/matplotlib_plotters.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/plotting/utils.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/post_treatment/__init__.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/post_treatment/process_to_scalar.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/py.typed +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/resources/__init__.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/resources/config_template.toml +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/slicing.py +0 -0
- {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/splash.png +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: pymodaq_data
|
|
3
|
-
Version: 5.3.
|
|
3
|
+
Version: 5.3.10
|
|
4
4
|
Summary: Modular Data Acquisition with Python
|
|
5
5
|
Project-URL: Homepage, http://pymodaq.cnrs.fr
|
|
6
6
|
Project-URL: Source, https://github.com/PyMoDAQ/PyMoDAQ
|
|
@@ -71,11 +71,11 @@ PyMoDAQ Data
|
|
|
71
71
|
:target: https://pypi.org/project/pymodaq_data/
|
|
72
72
|
:alt: Latest Version
|
|
73
73
|
|
|
74
|
-
.. image:: https://readthedocs.org/projects/pymodaq/badge/?version=
|
|
75
|
-
:target: https://pymodaq.readthedocs.io/en/stable/?badge=
|
|
74
|
+
.. image:: https://readthedocs.org/projects/pymodaq/badge/?version=5.3.x
|
|
75
|
+
:target: https://pymodaq.readthedocs.io/en/stable/?badge=5.3.x
|
|
76
76
|
:alt: Documentation Status
|
|
77
77
|
|
|
78
|
-
.. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/
|
|
78
|
+
.. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/5.3.x/graph/badge.svg?token=IQNJRCQDM2
|
|
79
79
|
:target: https://codecov.io/gh/PyMoDAQ/PyMoDAQ
|
|
80
80
|
|
|
81
81
|
+-------------+-------------+---------------+
|
|
@@ -94,28 +94,28 @@ PyMoDAQ Data
|
|
|
94
94
|
|
|
95
95
|
|
|
96
96
|
|
|
97
|
-
.. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
97
|
+
.. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.10.svg
|
|
98
98
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
99
99
|
|
|
100
|
-
.. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
100
|
+
.. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.11.svg
|
|
101
101
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
102
102
|
|
|
103
|
-
.. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
103
|
+
.. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.12.svg
|
|
104
104
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
105
105
|
|
|
106
|
-
.. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
106
|
+
.. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.13.svg
|
|
107
107
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
108
108
|
|
|
109
|
-
.. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
109
|
+
.. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.10.svg
|
|
110
110
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
111
111
|
|
|
112
|
-
.. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
112
|
+
.. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.11.svg
|
|
113
113
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
114
114
|
|
|
115
|
-
.. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
115
|
+
.. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.12.svg
|
|
116
116
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
117
117
|
|
|
118
|
-
.. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
118
|
+
.. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.13.svg
|
|
119
119
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
120
120
|
|
|
121
121
|
|
|
@@ -11,11 +11,11 @@ PyMoDAQ Data
|
|
|
11
11
|
:target: https://pypi.org/project/pymodaq_data/
|
|
12
12
|
:alt: Latest Version
|
|
13
13
|
|
|
14
|
-
.. image:: https://readthedocs.org/projects/pymodaq/badge/?version=
|
|
15
|
-
:target: https://pymodaq.readthedocs.io/en/stable/?badge=
|
|
14
|
+
.. image:: https://readthedocs.org/projects/pymodaq/badge/?version=5.3.x
|
|
15
|
+
:target: https://pymodaq.readthedocs.io/en/stable/?badge=5.3.x
|
|
16
16
|
:alt: Documentation Status
|
|
17
17
|
|
|
18
|
-
.. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/
|
|
18
|
+
.. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/5.3.x/graph/badge.svg?token=IQNJRCQDM2
|
|
19
19
|
:target: https://codecov.io/gh/PyMoDAQ/PyMoDAQ
|
|
20
20
|
|
|
21
21
|
+-------------+-------------+---------------+
|
|
@@ -34,28 +34,28 @@ PyMoDAQ Data
|
|
|
34
34
|
|
|
35
35
|
|
|
36
36
|
|
|
37
|
-
.. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
37
|
+
.. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.10.svg
|
|
38
38
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
39
39
|
|
|
40
|
-
.. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
40
|
+
.. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.11.svg
|
|
41
41
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
42
42
|
|
|
43
|
-
.. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
43
|
+
.. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.12.svg
|
|
44
44
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
45
45
|
|
|
46
|
-
.. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
46
|
+
.. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.13.svg
|
|
47
47
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
48
48
|
|
|
49
|
-
.. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
49
|
+
.. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.10.svg
|
|
50
50
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
51
51
|
|
|
52
|
-
.. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
52
|
+
.. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.11.svg
|
|
53
53
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
54
54
|
|
|
55
|
-
.. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
55
|
+
.. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.12.svg
|
|
56
56
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
57
57
|
|
|
58
|
-
.. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/
|
|
58
|
+
.. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.13.svg
|
|
59
59
|
:target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
|
|
60
60
|
|
|
61
61
|
|
|
@@ -10,11 +10,11 @@ from pathlib import Path
|
|
|
10
10
|
import numpy as np
|
|
11
11
|
import importlib
|
|
12
12
|
from importlib import metadata
|
|
13
|
-
import pickle
|
|
14
13
|
|
|
15
14
|
from pymodaq_utils.logger import set_logger, get_module_name
|
|
16
15
|
from pymodaq_utils.config import GlobalConfig as Config
|
|
17
16
|
from pymodaq_utils.utils import capitalize, JsonConverter
|
|
17
|
+
from pymodaq_utils.safe_pickle import restricted_loads, warn_legacy_format
|
|
18
18
|
from pymodaq_utils import utils
|
|
19
19
|
from pymodaq_utils.enums import BaseEnum, enum_checker
|
|
20
20
|
|
|
@@ -475,10 +475,28 @@ class StringARRAY(VLARRAY):
|
|
|
475
475
|
super().append(data)
|
|
476
476
|
|
|
477
477
|
def array_to_string(self, array):
|
|
478
|
-
|
|
478
|
+
"""Decode a stored string
|
|
479
|
+
|
|
480
|
+
New files hold the plain UTF-8 bytes of the string. Files written by older versions hold a pickle
|
|
481
|
+
(always starting with the 0x80 protocol byte, which can never start a valid UTF-8 text): these are
|
|
482
|
+
read by a restricted loader that only rebuilds plain data and never runs code.
|
|
483
|
+
"""
|
|
484
|
+
raw = bytes(array)
|
|
485
|
+
if raw[:1] == b'\x80':
|
|
486
|
+
warn_legacy_format(f'The string node {self.path} of the file {self._file_name()}')
|
|
487
|
+
return restricted_loads(raw)
|
|
488
|
+
return raw.decode('utf-8')
|
|
479
489
|
|
|
480
490
|
def string_to_array(self, string):
|
|
481
|
-
return np.frombuffer(
|
|
491
|
+
return np.frombuffer(string.encode('utf-8'), np.uint8)
|
|
492
|
+
|
|
493
|
+
def _file_name(self) -> str:
|
|
494
|
+
try:
|
|
495
|
+
if self.backend == 'tables':
|
|
496
|
+
return str(self.node._v_file.filename)
|
|
497
|
+
return str(self.node.file.filename)
|
|
498
|
+
except Exception:
|
|
499
|
+
return '(unknown)'
|
|
482
500
|
|
|
483
501
|
|
|
484
502
|
class Attributes(object):
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Migration of the h5 files written by older versions of PyMoDAQ
|
|
2
|
+
|
|
3
|
+
Older versions stored the strings of the string nodes (such as the logger) as pickles. They are still read, but
|
|
4
|
+
the migration rewrites them as plain UTF-8, as the current version does, in a copy of the file.
|
|
5
|
+
"""
|
|
6
|
+
import os
|
|
7
|
+
import shutil
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Optional, Tuple, Union
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
|
|
13
|
+
from pymodaq_utils.safe_pickle import restricted_loads, UnsafePickleError
|
|
14
|
+
|
|
15
|
+
from pymodaq_data.h5modules.backends import get_attr
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def default_destination(source: Path) -> Path:
|
|
19
|
+
return source.with_name(f'{source.stem}_migrated{source.suffix}')
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def migrate_h5_file(source: Union[str, Path], destination: Optional[Union[str, Path]] = None) -> Tuple[Path, int]:
|
|
23
|
+
"""Write a copy of a h5 file in which the pickled strings are rewritten as UTF-8
|
|
24
|
+
|
|
25
|
+
The source file is never modified, and an existing destination file is never overwritten.
|
|
26
|
+
|
|
27
|
+
Parameters
|
|
28
|
+
----------
|
|
29
|
+
source: str or Path
|
|
30
|
+
The file to migrate
|
|
31
|
+
destination: str or Path
|
|
32
|
+
The file to create, by default the name of the source followed by _migrated
|
|
33
|
+
|
|
34
|
+
Returns
|
|
35
|
+
-------
|
|
36
|
+
Path: the migrated file
|
|
37
|
+
int: the number of string nodes that were migrated
|
|
38
|
+
|
|
39
|
+
Raises
|
|
40
|
+
------
|
|
41
|
+
FileExistsError
|
|
42
|
+
if the destination exists
|
|
43
|
+
UnsafePickleError
|
|
44
|
+
if a string node contains anything else than a string, in this case nothing is written
|
|
45
|
+
"""
|
|
46
|
+
import h5py
|
|
47
|
+
|
|
48
|
+
source = Path(source)
|
|
49
|
+
destination = default_destination(source) if destination is None else Path(destination)
|
|
50
|
+
if destination.exists():
|
|
51
|
+
raise FileExistsError(f'{destination} already exists, it will not be overwritten')
|
|
52
|
+
part = destination.with_name(destination.name + '.part')
|
|
53
|
+
shutil.copyfile(source, part)
|
|
54
|
+
try:
|
|
55
|
+
migrated = 0
|
|
56
|
+
with h5py.File(part, 'r+') as h5file:
|
|
57
|
+
to_migrate = []
|
|
58
|
+
h5file.visititems(lambda name, obj: to_migrate.append(obj) if _is_string_node(obj) else None)
|
|
59
|
+
for dataset in to_migrate:
|
|
60
|
+
migrated += _migrate_string_node(dataset)
|
|
61
|
+
os.replace(part, destination)
|
|
62
|
+
finally:
|
|
63
|
+
if part.exists():
|
|
64
|
+
part.unlink()
|
|
65
|
+
return destination, migrated
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _is_string_node(obj) -> bool:
|
|
69
|
+
import h5py
|
|
70
|
+
if not isinstance(obj, h5py.Dataset):
|
|
71
|
+
return False
|
|
72
|
+
try:
|
|
73
|
+
return get_attr(obj, 'CLASS', 'h5py') == 'VLARRAY' and get_attr(obj, 'subdtype', 'h5py') == 'string'
|
|
74
|
+
except KeyError:
|
|
75
|
+
return False
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _migrate_string_node(dataset) -> int:
|
|
79
|
+
"""Rewrite the pickled strings of a node, returns 1 if the node had some, 0 otherwise"""
|
|
80
|
+
new_values = {}
|
|
81
|
+
for index in range(len(dataset)):
|
|
82
|
+
raw = bytes(dataset[index])
|
|
83
|
+
if raw[:1] == b'\x80':
|
|
84
|
+
text = restricted_loads(raw)
|
|
85
|
+
if not isinstance(text, str):
|
|
86
|
+
raise UnsafePickleError(f'Refused to migrate {dataset.name}: it does not contain only strings')
|
|
87
|
+
new_values[index] = text
|
|
88
|
+
for index, text in new_values.items(): # nothing is written if a value above was refused
|
|
89
|
+
dataset[index] = np.frombuffer(text.encode('utf-8'), np.uint8)
|
|
90
|
+
return 1 if new_values else 0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/plotting/plotter/plotters/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/post_treatment/process_to_scalar.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|