junifer 0.0.6.dev248__py3-none-any.whl → 0.0.6.dev252__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- junifer/_version.py +2 -2
- junifer/api/decorators.py +1 -2
- junifer/api/functions.py +18 -18
- junifer/api/queue_context/gnu_parallel_local_adapter.py +4 -4
- junifer/api/queue_context/htcondor_adapter.py +4 -4
- junifer/api/queue_context/tests/test_gnu_parallel_local_adapter.py +3 -3
- junifer/api/queue_context/tests/test_htcondor_adapter.py +3 -3
- junifer/api/tests/test_functions.py +32 -32
- junifer/cli/cli.py +3 -3
- junifer/cli/parser.py +4 -4
- junifer/cli/tests/test_cli.py +5 -5
- junifer/cli/utils.py +5 -6
- junifer/configs/juseless/datagrabbers/ixi_vbm.py +2 -2
- junifer/configs/juseless/datagrabbers/tests/test_ucla.py +2 -2
- junifer/configs/juseless/datagrabbers/ucla.py +4 -4
- junifer/data/_dispatch.py +11 -14
- junifer/data/coordinates/_ants_coordinates_warper.py +3 -3
- junifer/data/coordinates/_coordinates.py +6 -6
- junifer/data/coordinates/_fsl_coordinates_warper.py +3 -3
- junifer/data/masks/_ants_mask_warper.py +3 -3
- junifer/data/masks/_fsl_mask_warper.py +3 -3
- junifer/data/masks/_masks.py +6 -9
- junifer/data/masks/tests/test_masks.py +4 -4
- junifer/data/parcellations/_ants_parcellation_warper.py +3 -3
- junifer/data/parcellations/_fsl_parcellation_warper.py +3 -3
- junifer/data/parcellations/_parcellations.py +19 -19
- junifer/data/parcellations/tests/test_parcellations.py +1 -2
- junifer/data/pipeline_data_registry_base.py +3 -2
- junifer/data/template_spaces.py +3 -3
- junifer/data/tests/test_data_utils.py +1 -2
- junifer/data/utils.py +4 -3
- junifer/datagrabber/aomic/id1000.py +2 -2
- junifer/datagrabber/aomic/piop1.py +5 -5
- junifer/datagrabber/aomic/piop2.py +5 -5
- junifer/datagrabber/aomic/tests/test_id1000.py +3 -3
- junifer/datagrabber/aomic/tests/test_piop1.py +4 -4
- junifer/datagrabber/aomic/tests/test_piop2.py +4 -4
- junifer/datagrabber/base.py +12 -11
- junifer/datagrabber/datalad_base.py +3 -3
- junifer/datagrabber/dmcc13_benchmark.py +8 -8
- junifer/datagrabber/hcp1200/datalad_hcp1200.py +3 -3
- junifer/datagrabber/hcp1200/hcp1200.py +8 -8
- junifer/datagrabber/hcp1200/tests/test_hcp1200.py +2 -1
- junifer/datagrabber/multiple.py +7 -7
- junifer/datagrabber/pattern.py +10 -10
- junifer/datagrabber/pattern_validation_mixin.py +10 -10
- junifer/datagrabber/tests/test_datalad_base.py +7 -8
- junifer/datagrabber/tests/test_dmcc13_benchmark.py +2 -2
- junifer/datagrabber/tests/test_pattern_validation_mixin.py +6 -6
- junifer/datareader/default.py +6 -6
- junifer/external/nilearn/junifer_connectivity_measure.py +2 -2
- junifer/external/nilearn/junifer_nifti_spheres_masker.py +4 -4
- junifer/external/nilearn/tests/test_junifer_connectivity_measure.py +15 -15
- junifer/external/nilearn/tests/test_junifer_nifti_spheres_masker.py +2 -3
- junifer/markers/base.py +8 -8
- junifer/markers/brainprint.py +7 -9
- junifer/markers/complexity/complexity_base.py +6 -8
- junifer/markers/complexity/hurst_exponent.py +5 -5
- junifer/markers/complexity/multiscale_entropy_auc.py +5 -5
- junifer/markers/complexity/perm_entropy.py +5 -5
- junifer/markers/complexity/range_entropy.py +5 -5
- junifer/markers/complexity/range_entropy_auc.py +5 -5
- junifer/markers/complexity/sample_entropy.py +5 -5
- junifer/markers/complexity/weighted_perm_entropy.py +5 -5
- junifer/markers/ets_rss.py +7 -7
- junifer/markers/falff/_afni_falff.py +1 -2
- junifer/markers/falff/_junifer_falff.py +1 -2
- junifer/markers/falff/falff_base.py +2 -4
- junifer/markers/falff/falff_parcels.py +7 -7
- junifer/markers/falff/falff_spheres.py +6 -6
- junifer/markers/functional_connectivity/crossparcellation_functional_connectivity.py +6 -6
- junifer/markers/functional_connectivity/edge_functional_connectivity_parcels.py +7 -7
- junifer/markers/functional_connectivity/edge_functional_connectivity_spheres.py +6 -6
- junifer/markers/functional_connectivity/functional_connectivity_base.py +10 -10
- junifer/markers/functional_connectivity/functional_connectivity_parcels.py +7 -7
- junifer/markers/functional_connectivity/functional_connectivity_spheres.py +6 -6
- junifer/markers/functional_connectivity/tests/test_edge_functional_connectivity_parcels.py +1 -2
- junifer/markers/functional_connectivity/tests/test_edge_functional_connectivity_spheres.py +1 -2
- junifer/markers/functional_connectivity/tests/test_functional_connectivity_parcels.py +3 -3
- junifer/markers/functional_connectivity/tests/test_functional_connectivity_spheres.py +3 -3
- junifer/markers/parcel_aggregation.py +8 -8
- junifer/markers/reho/_afni_reho.py +1 -2
- junifer/markers/reho/_junifer_reho.py +1 -2
- junifer/markers/reho/reho_base.py +2 -4
- junifer/markers/reho/reho_parcels.py +8 -8
- junifer/markers/reho/reho_spheres.py +7 -7
- junifer/markers/sphere_aggregation.py +8 -8
- junifer/markers/temporal_snr/temporal_snr_base.py +8 -8
- junifer/markers/temporal_snr/temporal_snr_parcels.py +6 -6
- junifer/markers/temporal_snr/temporal_snr_spheres.py +5 -5
- junifer/markers/utils.py +3 -3
- junifer/onthefly/_brainprint.py +2 -2
- junifer/onthefly/read_transform.py +3 -3
- junifer/pipeline/marker_collection.py +4 -4
- junifer/pipeline/pipeline_component_registry.py +5 -4
- junifer/pipeline/pipeline_step_mixin.py +7 -7
- junifer/pipeline/tests/test_pipeline_component_registry.py +2 -3
- junifer/pipeline/tests/test_pipeline_step_mixin.py +19 -19
- junifer/pipeline/tests/test_update_meta_mixin.py +4 -4
- junifer/pipeline/update_meta_mixin.py +2 -2
- junifer/pipeline/utils.py +5 -5
- junifer/preprocess/base.py +10 -10
- junifer/preprocess/confounds/fmriprep_confound_remover.py +11 -14
- junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py +1 -2
- junifer/preprocess/smoothing/smoothing.py +7 -7
- junifer/preprocess/warping/_ants_warper.py +3 -4
- junifer/preprocess/warping/_fsl_warper.py +3 -4
- junifer/preprocess/warping/space_warper.py +6 -6
- junifer/preprocess/warping/tests/test_space_warper.py +3 -4
- junifer/stats.py +4 -4
- junifer/storage/base.py +14 -13
- junifer/storage/hdf5.py +21 -20
- junifer/storage/pandas_base.py +12 -11
- junifer/storage/sqlite.py +11 -11
- junifer/storage/tests/test_hdf5.py +1 -2
- junifer/storage/tests/test_sqlite.py +2 -2
- junifer/storage/tests/test_utils.py +8 -7
- junifer/storage/utils.py +7 -7
- junifer/testing/datagrabbers.py +9 -10
- junifer/tests/test_stats.py +2 -2
- junifer/typing/_typing.py +6 -9
- junifer/utils/helpers.py +2 -3
- junifer/utils/logging.py +5 -5
- junifer/utils/singleton.py +3 -3
- {junifer-0.0.6.dev248.dist-info → junifer-0.0.6.dev252.dist-info}/METADATA +2 -2
- {junifer-0.0.6.dev248.dist-info → junifer-0.0.6.dev252.dist-info}/RECORD +131 -131
- {junifer-0.0.6.dev248.dist-info → junifer-0.0.6.dev252.dist-info}/AUTHORS.rst +0 -0
- {junifer-0.0.6.dev248.dist-info → junifer-0.0.6.dev252.dist-info}/LICENSE.md +0 -0
- {junifer-0.0.6.dev248.dist-info → junifer-0.0.6.dev252.dist-info}/WHEEL +0 -0
- {junifer-0.0.6.dev248.dist-info → junifer-0.0.6.dev252.dist-info}/entry_points.txt +0 -0
- {junifer-0.0.6.dev248.dist-info → junifer-0.0.6.dev252.dist-info}/top_level.txt +0 -0
@@ -3,7 +3,7 @@
|
|
3
3
|
# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
|
4
4
|
# License: AGPL
|
5
5
|
|
6
|
-
from typing import Any, ClassVar,
|
6
|
+
from typing import Any, ClassVar, Optional, Union
|
7
7
|
|
8
8
|
from templateflow import api as tflow
|
9
9
|
|
@@ -64,7 +64,7 @@ class SpaceWarper(BasePreprocessor):
|
|
64
64
|
]
|
65
65
|
|
66
66
|
def __init__(
|
67
|
-
self, using: str, reference: str, on: Union[
|
67
|
+
self, using: str, reference: str, on: Union[list[str], str]
|
68
68
|
) -> None:
|
69
69
|
"""Initialize the class."""
|
70
70
|
# Validate `using` parameter
|
@@ -94,7 +94,7 @@ class SpaceWarper(BasePreprocessor):
|
|
94
94
|
else:
|
95
95
|
raise_error(f"Unknown reference: {self.reference}")
|
96
96
|
|
97
|
-
def get_valid_inputs(self) ->
|
97
|
+
def get_valid_inputs(self) -> list[str]:
|
98
98
|
"""Get valid data types for input.
|
99
99
|
|
100
100
|
Returns
|
@@ -135,9 +135,9 @@ class SpaceWarper(BasePreprocessor):
|
|
135
135
|
|
136
136
|
def preprocess(
|
137
137
|
self,
|
138
|
-
input:
|
139
|
-
extra_input: Optional[
|
140
|
-
) ->
|
138
|
+
input: dict[str, Any],
|
139
|
+
extra_input: Optional[dict[str, Any]] = None,
|
140
|
+
) -> tuple[dict[str, Any], Optional[dict[str, dict[str, Any]]]]:
|
141
141
|
"""Preprocess.
|
142
142
|
|
143
143
|
Parameters
|
@@ -4,7 +4,6 @@
|
|
4
4
|
# License: AGPL
|
5
5
|
|
6
6
|
import socket
|
7
|
-
from typing import Tuple, Type
|
8
7
|
|
9
8
|
import pytest
|
10
9
|
from numpy.testing import assert_array_equal, assert_raises
|
@@ -29,7 +28,7 @@ from junifer.typing import DataGrabberLike
|
|
29
28
|
def test_SpaceWarper_errors(
|
30
29
|
using: str,
|
31
30
|
reference: str,
|
32
|
-
error_type:
|
31
|
+
error_type: type[Exception],
|
33
32
|
error_msg: str,
|
34
33
|
) -> None:
|
35
34
|
"""Test SpaceWarper errors.
|
@@ -96,7 +95,7 @@ def test_SpaceWarper_errors(
|
|
96
95
|
reason="only for juseless",
|
97
96
|
)
|
98
97
|
def test_SpaceWarper_native(
|
99
|
-
datagrabber: DataGrabberLike, element:
|
98
|
+
datagrabber: DataGrabberLike, element: tuple[str, ...], using: str
|
100
99
|
) -> None:
|
101
100
|
"""Test SpaceWarper for native space warping.
|
102
101
|
|
@@ -160,7 +159,7 @@ def test_SpaceWarper_native(
|
|
160
159
|
)
|
161
160
|
def test_SpaceWarper_multi_mni(
|
162
161
|
datagrabber: DataGrabberLike,
|
163
|
-
element:
|
162
|
+
element: tuple[str, ...],
|
164
163
|
space: str,
|
165
164
|
) -> None:
|
166
165
|
"""Test SpaceWarper for MNI space warping.
|
junifer/stats.py
CHANGED
@@ -4,7 +4,7 @@
|
|
4
4
|
# Synchon Mandal <s.mandal@fz-juelich.de>
|
5
5
|
# License: AGPL
|
6
6
|
|
7
|
-
from typing import Any, Callable,
|
7
|
+
from typing import Any, Callable, Optional
|
8
8
|
|
9
9
|
import numpy as np
|
10
10
|
from scipy.stats import mode, trim_mean
|
@@ -17,7 +17,7 @@ __all__ = ["get_aggfunc_by_name", "count", "winsorized_mean", "select"]
|
|
17
17
|
|
18
18
|
|
19
19
|
def get_aggfunc_by_name(
|
20
|
-
name: str, func_params: Optional[
|
20
|
+
name: str, func_params: Optional[dict[str, Any]] = None
|
21
21
|
) -> Callable:
|
22
22
|
"""Get an aggregation function by its name.
|
23
23
|
|
@@ -169,8 +169,8 @@ def winsorized_mean(
|
|
169
169
|
def select(
|
170
170
|
data: np.ndarray,
|
171
171
|
axis: int = 0,
|
172
|
-
pick: Optional[
|
173
|
-
drop: Optional[
|
172
|
+
pick: Optional[list[int]] = None,
|
173
|
+
drop: Optional[list[int]] = None,
|
174
174
|
) -> np.ndarray:
|
175
175
|
"""Select a subset of the data.
|
176
176
|
|
junifer/storage/base.py
CHANGED
@@ -5,8 +5,9 @@
|
|
5
5
|
# License: AGPL
|
6
6
|
|
7
7
|
from abc import ABC, abstractmethod
|
8
|
+
from collections.abc import Iterable
|
8
9
|
from pathlib import Path
|
9
|
-
from typing import Any,
|
10
|
+
from typing import Any, Optional, Union
|
10
11
|
|
11
12
|
import numpy as np
|
12
13
|
import pandas as pd
|
@@ -43,7 +44,7 @@ class BaseFeatureStorage(ABC):
|
|
43
44
|
def __init__(
|
44
45
|
self,
|
45
46
|
uri: Union[str, Path],
|
46
|
-
storage_types: Union[
|
47
|
+
storage_types: Union[list[str], str],
|
47
48
|
single_output: bool = True,
|
48
49
|
) -> None:
|
49
50
|
self.uri = uri
|
@@ -61,7 +62,7 @@ class BaseFeatureStorage(ABC):
|
|
61
62
|
self._valid_inputs = storage_types
|
62
63
|
self.single_output = single_output
|
63
64
|
|
64
|
-
def get_valid_inputs(self) ->
|
65
|
+
def get_valid_inputs(self) -> list[str]:
|
65
66
|
"""Get valid storage types for input.
|
66
67
|
|
67
68
|
Returns
|
@@ -76,7 +77,7 @@ class BaseFeatureStorage(ABC):
|
|
76
77
|
klass=NotImplementedError,
|
77
78
|
)
|
78
79
|
|
79
|
-
def validate(self, input_:
|
80
|
+
def validate(self, input_: list[str]) -> None:
|
80
81
|
"""Validate the input to the pipeline step.
|
81
82
|
|
82
83
|
Parameters
|
@@ -98,7 +99,7 @@ class BaseFeatureStorage(ABC):
|
|
98
99
|
)
|
99
100
|
|
100
101
|
@abstractmethod
|
101
|
-
def list_features(self) ->
|
102
|
+
def list_features(self) -> dict[str, dict[str, Any]]:
|
102
103
|
"""List the features in the storage.
|
103
104
|
|
104
105
|
Returns
|
@@ -119,8 +120,8 @@ class BaseFeatureStorage(ABC):
|
|
119
120
|
self,
|
120
121
|
feature_name: Optional[str] = None,
|
121
122
|
feature_md5: Optional[str] = None,
|
122
|
-
) ->
|
123
|
-
str, Union[str,
|
123
|
+
) -> dict[
|
124
|
+
str, Union[str, list[Union[int, str, dict[str, str]]], np.ndarray]
|
124
125
|
]:
|
125
126
|
"""Read stored feature.
|
126
127
|
|
@@ -169,7 +170,7 @@ class BaseFeatureStorage(ABC):
|
|
169
170
|
)
|
170
171
|
|
171
172
|
@abstractmethod
|
172
|
-
def store_metadata(self, meta_md5: str, element:
|
173
|
+
def store_metadata(self, meta_md5: str, element: dict, meta: dict) -> None:
|
173
174
|
"""Store metadata.
|
174
175
|
|
175
176
|
Parameters
|
@@ -229,7 +230,7 @@ class BaseFeatureStorage(ABC):
|
|
229
230
|
def store_matrix(
|
230
231
|
self,
|
231
232
|
meta_md5: str,
|
232
|
-
element:
|
233
|
+
element: dict,
|
233
234
|
data: np.ndarray,
|
234
235
|
col_names: Optional[Iterable[str]] = None,
|
235
236
|
row_names: Optional[Iterable[str]] = None,
|
@@ -271,8 +272,8 @@ class BaseFeatureStorage(ABC):
|
|
271
272
|
def store_vector(
|
272
273
|
self,
|
273
274
|
meta_md5: str,
|
274
|
-
element:
|
275
|
-
data: Union[np.ndarray,
|
275
|
+
element: dict,
|
276
|
+
data: Union[np.ndarray, list],
|
276
277
|
col_names: Optional[Iterable[str]] = None,
|
277
278
|
) -> None:
|
278
279
|
"""Store vector.
|
@@ -297,7 +298,7 @@ class BaseFeatureStorage(ABC):
|
|
297
298
|
def store_timeseries(
|
298
299
|
self,
|
299
300
|
meta_md5: str,
|
300
|
-
element:
|
301
|
+
element: dict,
|
301
302
|
data: np.ndarray,
|
302
303
|
col_names: Optional[Iterable[str]] = None,
|
303
304
|
) -> None:
|
@@ -323,7 +324,7 @@ class BaseFeatureStorage(ABC):
|
|
323
324
|
def store_scalar_table(
|
324
325
|
self,
|
325
326
|
meta_md5: str,
|
326
|
-
element:
|
327
|
+
element: dict,
|
327
328
|
data: np.ndarray,
|
328
329
|
col_names: Optional[Iterable[str]] = None,
|
329
330
|
row_names: Optional[Iterable[str]] = None,
|
junifer/storage/hdf5.py
CHANGED
@@ -6,8 +6,9 @@
|
|
6
6
|
|
7
7
|
|
8
8
|
from collections import defaultdict
|
9
|
+
from collections.abc import Iterable
|
9
10
|
from pathlib import Path
|
10
|
-
from typing import Any,
|
11
|
+
from typing import Any, Optional, Union
|
11
12
|
|
12
13
|
import numpy as np
|
13
14
|
import pandas as pd
|
@@ -30,7 +31,7 @@ __all__ = ["HDF5FeatureStorage"]
|
|
30
31
|
|
31
32
|
|
32
33
|
def _create_chunk(
|
33
|
-
chunk_data:
|
34
|
+
chunk_data: list[np.ndarray],
|
34
35
|
kind: str,
|
35
36
|
element_count: int,
|
36
37
|
chunk_size: int,
|
@@ -164,7 +165,7 @@ class HDF5FeatureStorage(BaseFeatureStorage):
|
|
164
165
|
self.force_float32 = force_float32
|
165
166
|
self.chunk_size = chunk_size
|
166
167
|
|
167
|
-
def get_valid_inputs(self) ->
|
168
|
+
def get_valid_inputs(self) -> list[str]:
|
168
169
|
"""Get valid storage types for input.
|
169
170
|
|
170
171
|
Returns
|
@@ -176,7 +177,7 @@ class HDF5FeatureStorage(BaseFeatureStorage):
|
|
176
177
|
"""
|
177
178
|
return ["matrix", "vector", "timeseries", "scalar_table"]
|
178
179
|
|
179
|
-
def _fetch_correct_uri_for_io(self, element: Optional[
|
180
|
+
def _fetch_correct_uri_for_io(self, element: Optional[dict]) -> str:
|
180
181
|
"""Return proper URI for I/O based on `element`.
|
181
182
|
|
182
183
|
If `element` is None, will return `self.uri`.
|
@@ -210,8 +211,8 @@ class HDF5FeatureStorage(BaseFeatureStorage):
|
|
210
211
|
return f"{self.uri.parent}/{prefix}{self.uri.name}" # type: ignore
|
211
212
|
|
212
213
|
def _read_metadata(
|
213
|
-
self, element: Optional[
|
214
|
-
) ->
|
214
|
+
self, element: Optional[dict[str, str]] = None
|
215
|
+
) -> dict[str, dict[str, Any]]:
|
215
216
|
"""Read metadata (should not be called directly).
|
216
217
|
|
217
218
|
Parameters
|
@@ -261,7 +262,7 @@ class HDF5FeatureStorage(BaseFeatureStorage):
|
|
261
262
|
|
262
263
|
return metadata
|
263
264
|
|
264
|
-
def list_features(self) ->
|
265
|
+
def list_features(self) -> dict[str, dict[str, Any]]:
|
265
266
|
"""List the features in the storage.
|
266
267
|
|
267
268
|
Returns
|
@@ -281,8 +282,8 @@ class HDF5FeatureStorage(BaseFeatureStorage):
|
|
281
282
|
return metadata
|
282
283
|
|
283
284
|
def _read_data(
|
284
|
-
self, md5: str, element: Optional[
|
285
|
-
) ->
|
285
|
+
self, md5: str, element: Optional[dict[str, str]] = None
|
286
|
+
) -> dict[str, Any]:
|
286
287
|
"""Read data (should not be called directly).
|
287
288
|
|
288
289
|
Parameters
|
@@ -338,8 +339,8 @@ class HDF5FeatureStorage(BaseFeatureStorage):
|
|
338
339
|
self,
|
339
340
|
feature_name: Optional[str] = None,
|
340
341
|
feature_md5: Optional[str] = None,
|
341
|
-
) ->
|
342
|
-
str, Union[str,
|
342
|
+
) -> dict[
|
343
|
+
str, Union[str, list[Union[int, str, dict[str, str]]], np.ndarray]
|
343
344
|
]:
|
344
345
|
"""Read stored feature.
|
345
346
|
|
@@ -562,7 +563,7 @@ class HDF5FeatureStorage(BaseFeatureStorage):
|
|
562
563
|
return df
|
563
564
|
|
564
565
|
def _write_processed_data(
|
565
|
-
self, fname: str, processed_data:
|
566
|
+
self, fname: str, processed_data: dict[str, Any], title: str
|
566
567
|
) -> None:
|
567
568
|
"""Write processed data to HDF5 (should not be called directly).
|
568
569
|
|
@@ -594,8 +595,8 @@ class HDF5FeatureStorage(BaseFeatureStorage):
|
|
594
595
|
def store_metadata(
|
595
596
|
self,
|
596
597
|
meta_md5: str,
|
597
|
-
element:
|
598
|
-
meta:
|
598
|
+
element: dict[str, str],
|
599
|
+
meta: dict[str, Any],
|
599
600
|
) -> None:
|
600
601
|
"""Store metadata.
|
601
602
|
|
@@ -655,7 +656,7 @@ class HDF5FeatureStorage(BaseFeatureStorage):
|
|
655
656
|
self,
|
656
657
|
kind: str,
|
657
658
|
meta_md5: str,
|
658
|
-
element:
|
659
|
+
element: list[dict[str, str]],
|
659
660
|
data: np.ndarray,
|
660
661
|
**kwargs: Any,
|
661
662
|
) -> None:
|
@@ -797,7 +798,7 @@ class HDF5FeatureStorage(BaseFeatureStorage):
|
|
797
798
|
def store_matrix(
|
798
799
|
self,
|
799
800
|
meta_md5: str,
|
800
|
-
element:
|
801
|
+
element: dict[str, str],
|
801
802
|
data: np.ndarray,
|
802
803
|
col_names: Optional[Iterable[str]] = None,
|
803
804
|
row_names: Optional[Iterable[str]] = None,
|
@@ -876,8 +877,8 @@ class HDF5FeatureStorage(BaseFeatureStorage):
|
|
876
877
|
def store_vector(
|
877
878
|
self,
|
878
879
|
meta_md5: str,
|
879
|
-
element:
|
880
|
-
data: Union[np.ndarray,
|
880
|
+
element: dict[str, str],
|
881
|
+
data: Union[np.ndarray, list],
|
881
882
|
col_names: Optional[Iterable[str]] = None,
|
882
883
|
) -> None:
|
883
884
|
"""Store vector.
|
@@ -919,7 +920,7 @@ class HDF5FeatureStorage(BaseFeatureStorage):
|
|
919
920
|
def store_timeseries(
|
920
921
|
self,
|
921
922
|
meta_md5: str,
|
922
|
-
element:
|
923
|
+
element: dict[str, str],
|
923
924
|
data: np.ndarray,
|
924
925
|
col_names: Optional[Iterable[str]] = None,
|
925
926
|
) -> None:
|
@@ -949,7 +950,7 @@ class HDF5FeatureStorage(BaseFeatureStorage):
|
|
949
950
|
def store_scalar_table(
|
950
951
|
self,
|
951
952
|
meta_md5: str,
|
952
|
-
element:
|
953
|
+
element: dict,
|
953
954
|
data: np.ndarray,
|
954
955
|
col_names: Optional[Iterable[str]] = None,
|
955
956
|
row_names: Optional[Iterable[str]] = None,
|
junifer/storage/pandas_base.py
CHANGED
@@ -5,8 +5,9 @@
|
|
5
5
|
# License: AGPL
|
6
6
|
|
7
7
|
import json
|
8
|
+
from collections.abc import Iterable
|
8
9
|
from pathlib import Path
|
9
|
-
from typing import
|
10
|
+
from typing import Optional, Union
|
10
11
|
|
11
12
|
import numpy as np
|
12
13
|
import pandas as pd
|
@@ -44,7 +45,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
|
|
44
45
|
) -> None:
|
45
46
|
super().__init__(uri=uri, single_output=single_output, **kwargs)
|
46
47
|
|
47
|
-
def get_valid_inputs(self) ->
|
48
|
+
def get_valid_inputs(self) -> list[str]:
|
48
49
|
"""Get valid storage types for input.
|
49
50
|
|
50
51
|
Returns
|
@@ -56,7 +57,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
|
|
56
57
|
"""
|
57
58
|
return ["matrix", "vector", "timeseries"]
|
58
59
|
|
59
|
-
def _meta_row(self, meta:
|
60
|
+
def _meta_row(self, meta: dict, meta_md5: str) -> pd.DataFrame:
|
60
61
|
"""Convert the metadata to a pandas DataFrame.
|
61
62
|
|
62
63
|
Parameters
|
@@ -80,7 +81,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
|
|
80
81
|
|
81
82
|
@staticmethod
|
82
83
|
def element_to_index(
|
83
|
-
element:
|
84
|
+
element: dict, n_rows: int = 1, rows_col_name: Optional[str] = None
|
84
85
|
) -> Union[pd.Index, pd.MultiIndex]:
|
85
86
|
"""Convert the element metadata to index.
|
86
87
|
|
@@ -101,7 +102,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
|
|
101
102
|
|
102
103
|
"""
|
103
104
|
# Make mapping between element access keys and values
|
104
|
-
elem_idx:
|
105
|
+
elem_idx: dict[str, Iterable[str]] = {
|
105
106
|
k: [v] * n_rows for k, v in element.items()
|
106
107
|
}
|
107
108
|
|
@@ -129,7 +130,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
|
|
129
130
|
return index
|
130
131
|
|
131
132
|
def store_df(
|
132
|
-
self, meta_md5: str, element:
|
133
|
+
self, meta_md5: str, element: dict, df: Union[pd.DataFrame, pd.Series]
|
133
134
|
) -> None:
|
134
135
|
"""Implement pandas DataFrame storing.
|
135
136
|
|
@@ -157,8 +158,8 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
|
|
157
158
|
def _store_2d(
|
158
159
|
self,
|
159
160
|
meta_md5: str,
|
160
|
-
element:
|
161
|
-
data: Union[np.ndarray,
|
161
|
+
element: dict,
|
162
|
+
data: Union[np.ndarray, list],
|
162
163
|
col_names: Optional[Iterable[str]] = None,
|
163
164
|
rows_col_name: Optional[str] = None,
|
164
165
|
) -> None:
|
@@ -194,8 +195,8 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
|
|
194
195
|
def store_vector(
|
195
196
|
self,
|
196
197
|
meta_md5: str,
|
197
|
-
element:
|
198
|
-
data: Union[np.ndarray,
|
198
|
+
element: dict,
|
199
|
+
data: Union[np.ndarray, list],
|
199
200
|
col_names: Optional[Iterable[str]] = None,
|
200
201
|
) -> None:
|
201
202
|
"""Store vector.
|
@@ -232,7 +233,7 @@ class PandasBaseFeatureStorage(BaseFeatureStorage):
|
|
232
233
|
def store_timeseries(
|
233
234
|
self,
|
234
235
|
meta_md5: str,
|
235
|
-
element:
|
236
|
+
element: dict,
|
236
237
|
data: np.ndarray,
|
237
238
|
col_names: Optional[Iterable[str]] = None,
|
238
239
|
) -> None:
|
junifer/storage/sqlite.py
CHANGED
@@ -6,7 +6,7 @@
|
|
6
6
|
|
7
7
|
import json
|
8
8
|
from pathlib import Path
|
9
|
-
from typing import TYPE_CHECKING, Any,
|
9
|
+
from typing import TYPE_CHECKING, Any, Optional, Union
|
10
10
|
|
11
11
|
import numpy as np
|
12
12
|
import pandas as pd
|
@@ -92,7 +92,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
|
|
92
92
|
# Set upsert
|
93
93
|
self._upsert = upsert
|
94
94
|
|
95
|
-
def get_engine(self, element: Optional[
|
95
|
+
def get_engine(self, element: Optional[dict] = None) -> "Engine":
|
96
96
|
"""Get engine.
|
97
97
|
|
98
98
|
Parameters
|
@@ -209,7 +209,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
|
|
209
209
|
msg=f"Invalid option {if_exists} for if_exists."
|
210
210
|
)
|
211
211
|
|
212
|
-
def list_features(self) ->
|
212
|
+
def list_features(self) -> dict[str, dict[str, Any]]:
|
213
213
|
"""List the features in the storage.
|
214
214
|
|
215
215
|
Returns
|
@@ -229,7 +229,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
|
|
229
229
|
# Format index names for retrieved data
|
230
230
|
meta_df.index = meta_df.index.str.replace(r"meta_", "")
|
231
231
|
# Convert dataframe to dictionary
|
232
|
-
out:
|
232
|
+
out: dict[str, dict[str, str]] = meta_df.to_dict(
|
233
233
|
orient="index"
|
234
234
|
) # type: ignore
|
235
235
|
# Format output
|
@@ -242,8 +242,8 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
|
|
242
242
|
self,
|
243
243
|
feature_name: Optional[str] = None,
|
244
244
|
feature_md5: Optional[str] = None,
|
245
|
-
) ->
|
246
|
-
str, Union[str,
|
245
|
+
) -> dict[
|
246
|
+
str, Union[str, list[Union[int, str, dict[str, str]]], np.ndarray]
|
247
247
|
]:
|
248
248
|
"""Read stored feature.
|
249
249
|
|
@@ -358,7 +358,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
|
|
358
358
|
df = df.set_index(index_names)
|
359
359
|
return df
|
360
360
|
|
361
|
-
def store_metadata(self, meta_md5: str, element:
|
361
|
+
def store_metadata(self, meta_md5: str, element: dict, meta: dict) -> None:
|
362
362
|
"""Implement metadata storing in the storage.
|
363
363
|
|
364
364
|
Parameters
|
@@ -381,7 +381,7 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
|
|
381
381
|
self._save_upsert(meta_df, "meta", engine)
|
382
382
|
|
383
383
|
def store_df(
|
384
|
-
self, meta_md5: str, element:
|
384
|
+
self, meta_md5: str, element: dict, df: Union[pd.DataFrame, pd.Series]
|
385
385
|
) -> None:
|
386
386
|
"""Implement pandas DataFrame storing.
|
387
387
|
|
@@ -434,10 +434,10 @@ class SQLiteFeatureStorage(PandasBaseFeatureStorage):
|
|
434
434
|
def store_matrix(
|
435
435
|
self,
|
436
436
|
meta_md5: str,
|
437
|
-
element:
|
437
|
+
element: dict,
|
438
438
|
data: np.ndarray,
|
439
|
-
col_names: Optional[
|
440
|
-
row_names: Optional[
|
439
|
+
col_names: Optional[list[str]] = None,
|
440
|
+
row_names: Optional[list[str]] = None,
|
441
441
|
matrix_kind: str = "full",
|
442
442
|
diagonal: bool = True,
|
443
443
|
) -> None:
|
@@ -6,7 +6,6 @@
|
|
6
6
|
|
7
7
|
from copy import deepcopy
|
8
8
|
from pathlib import Path
|
9
|
-
from typing import Dict, Tuple
|
10
9
|
|
11
10
|
import h5py
|
12
11
|
import numpy as np
|
@@ -874,7 +873,7 @@ def test_store_scalar_table(tmp_path: Path) -> None:
|
|
874
873
|
assert_array_equal(read_df.values, data)
|
875
874
|
|
876
875
|
|
877
|
-
def _create_data_to_store(n_elements: int, kind: str) ->
|
876
|
+
def _create_data_to_store(n_elements: int, kind: str) -> tuple[str, dict]:
|
878
877
|
"""Create data to store.
|
879
878
|
|
880
879
|
Parameters
|
@@ -5,7 +5,7 @@
|
|
5
5
|
# License: AGPL
|
6
6
|
|
7
7
|
from pathlib import Path
|
8
|
-
from typing import
|
8
|
+
from typing import Union
|
9
9
|
|
10
10
|
import numpy as np
|
11
11
|
import pandas as pd
|
@@ -56,7 +56,7 @@ df_ignore = pd.DataFrame(
|
|
56
56
|
|
57
57
|
|
58
58
|
def _read_sql(
|
59
|
-
table_name: str, uri: str, index_col: Union[str,
|
59
|
+
table_name: str, uri: str, index_col: Union[str, list[str]]
|
60
60
|
) -> pd.DataFrame:
|
61
61
|
"""Read database table into a pandas DataFrame.
|
62
62
|
|
@@ -4,7 +4,8 @@
|
|
4
4
|
# Synchon Mandal <s.mandal@fz-juelich.de>
|
5
5
|
# License: AGPL
|
6
6
|
|
7
|
-
from
|
7
|
+
from collections.abc import Iterable
|
8
|
+
from typing import Union
|
8
9
|
|
9
10
|
import numpy as np
|
10
11
|
import pytest
|
@@ -198,7 +199,7 @@ def test_process_meta_invalid_metadata_key() -> None:
|
|
198
199
|
),
|
199
200
|
],
|
200
201
|
)
|
201
|
-
def test_process_meta_element(meta:
|
202
|
+
def test_process_meta_element(meta: dict, elements: list[str]) -> None:
|
202
203
|
"""Test metadata element after processing.
|
203
204
|
|
204
205
|
Parameters
|
@@ -215,7 +216,7 @@ def test_process_meta_element(meta: Dict, elements: List[str]) -> None:
|
|
215
216
|
assert "A" in processed_meta
|
216
217
|
assert "B" in processed_meta
|
217
218
|
assert "element" not in processed_meta
|
218
|
-
assert isinstance(processed_meta["dependencies"],
|
219
|
+
assert isinstance(processed_meta["dependencies"], dict)
|
219
220
|
assert all(
|
220
221
|
x in processed_meta["dependencies"] for x in meta["dependencies"]
|
221
222
|
)
|
@@ -232,7 +233,7 @@ def test_process_meta_element(meta: Dict, elements: List[str]) -> None:
|
|
232
233
|
({"subject": 1, "session": 2}, "element_1_2_"),
|
233
234
|
],
|
234
235
|
)
|
235
|
-
def test_element_to_prefix(element:
|
236
|
+
def test_element_to_prefix(element: dict, prefix: str) -> None:
|
236
237
|
"""Test converting element to prefix (for file naming).
|
237
238
|
|
238
239
|
Parameters
|
@@ -320,7 +321,7 @@ def test_element_to_prefix_invalid_type() -> None:
|
|
320
321
|
],
|
321
322
|
)
|
322
323
|
def test_store_matrix_checks(
|
323
|
-
params:
|
324
|
+
params: dict[str, Union[str, bool, tuple[int, int], int]], err_msg: str
|
324
325
|
) -> None:
|
325
326
|
"""Test matrix storing parameter checks.
|
326
327
|
|
@@ -401,9 +402,9 @@ def test_store_matrix_checks(
|
|
401
402
|
],
|
402
403
|
)
|
403
404
|
def test_matrix_to_vector(
|
404
|
-
params:
|
405
|
+
params: dict[str, Union[np.ndarray, Iterable[str], str, bool]],
|
405
406
|
expected_data: np.ndarray,
|
406
|
-
expected_columns:
|
407
|
+
expected_columns: list[str],
|
407
408
|
) -> None:
|
408
409
|
"""Test matrix to vector.
|
409
410
|
|
junifer/storage/utils.py
CHANGED
@@ -6,8 +6,8 @@
|
|
6
6
|
|
7
7
|
import hashlib
|
8
8
|
import json
|
9
|
+
from collections.abc import Iterable
|
9
10
|
from importlib.metadata import PackageNotFoundError, version
|
10
|
-
from typing import Dict, Iterable, List, Tuple
|
11
11
|
|
12
12
|
import numpy as np
|
13
13
|
|
@@ -50,7 +50,7 @@ def get_dependency_version(dependency: str) -> str:
|
|
50
50
|
return dep_version
|
51
51
|
|
52
52
|
|
53
|
-
def _meta_hash(meta:
|
53
|
+
def _meta_hash(meta: dict) -> str:
|
54
54
|
"""Compute the MD5 hash of the metadata.
|
55
55
|
|
56
56
|
Parameters
|
@@ -78,7 +78,7 @@ def _meta_hash(meta: Dict) -> str:
|
|
78
78
|
return meta_md5
|
79
79
|
|
80
80
|
|
81
|
-
def process_meta(meta:
|
81
|
+
def process_meta(meta: dict) -> tuple[str, dict, dict]:
|
82
82
|
"""Process the metadata for storage.
|
83
83
|
|
84
84
|
It removes the key "element" and adds the "_element_keys" with the keys
|
@@ -109,7 +109,7 @@ def process_meta(meta: Dict) -> Tuple[str, Dict, Dict]:
|
|
109
109
|
# Copy the metadata
|
110
110
|
t_meta = meta.copy()
|
111
111
|
# Remove key "element"
|
112
|
-
element:
|
112
|
+
element: dict = t_meta.pop("element", None)
|
113
113
|
if element is None:
|
114
114
|
raise_error(msg="`meta` must contain the key 'element'")
|
115
115
|
if "marker" not in t_meta:
|
@@ -128,7 +128,7 @@ def process_meta(meta: Dict) -> Tuple[str, Dict, Dict]:
|
|
128
128
|
return md5_hash, t_meta, element
|
129
129
|
|
130
130
|
|
131
|
-
def element_to_prefix(element:
|
131
|
+
def element_to_prefix(element: dict) -> str:
|
132
132
|
"""Convert the element metadata to prefix.
|
133
133
|
|
134
134
|
Parameters
|
@@ -156,7 +156,7 @@ def element_to_prefix(element: Dict) -> str:
|
|
156
156
|
def store_matrix_checks(
|
157
157
|
matrix_kind: str,
|
158
158
|
diagonal: bool,
|
159
|
-
data_shape:
|
159
|
+
data_shape: tuple[int, int],
|
160
160
|
row_names_len: int,
|
161
161
|
col_names_len: int,
|
162
162
|
) -> None:
|
@@ -218,7 +218,7 @@ def matrix_to_vector(
|
|
218
218
|
row_names: Iterable[str],
|
219
219
|
matrix_kind: str,
|
220
220
|
diagonal: bool,
|
221
|
-
) ->
|
221
|
+
) -> tuple[np.ndarray, list[str]]:
|
222
222
|
"""Convert matrix to vector based on parameters.
|
223
223
|
|
224
224
|
Parameters
|