piblin 0.0.0a0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- piblin/__init__.py +27 -0
- piblin/data/__init__.py +20 -0
- piblin/data/data_collections/__init__.py +66 -0
- piblin/data/data_collections/consistent_measurement_set.py +457 -0
- piblin/data/data_collections/data_collection_factory.py +100 -0
- piblin/data/data_collections/experiment.py +825 -0
- piblin/data/data_collections/experiment_set.py +1347 -0
- piblin/data/data_collections/measurement.py +2712 -0
- piblin/data/data_collections/measurement_set.py +2596 -0
- piblin/data/data_collections/summary.py +170 -0
- piblin/data/data_collections/tabular_measurement_set.py +349 -0
- piblin/data/data_collections/tidy_measurement_set.py +186 -0
- piblin/data/datasets/__init__.py +3 -0
- piblin/data/datasets/abc/__init__.py +0 -0
- piblin/data/datasets/abc/dataset.py +854 -0
- piblin/data/datasets/abc/dataset_factory.py +171 -0
- piblin/data/datasets/abc/split_datasets/__init__.py +891 -0
- piblin/data/datasets/abc/split_datasets/distribution.py +103 -0
- piblin/data/datasets/abc/split_datasets/histogram.py +46 -0
- piblin/data/datasets/abc/split_datasets/one_dimensional_composite_dataset.py +320 -0
- piblin/data/datasets/abc/split_datasets/one_dimensional_dataset.py +966 -0
- piblin/data/datasets/abc/split_datasets/three_dimensional_dataset.py +252 -0
- piblin/data/datasets/abc/split_datasets/two_dimensional_dataset.py +468 -0
- piblin/data/datasets/abc/split_datasets/zero_dimensional_dataset.py +540 -0
- piblin/data/datasets/abc/unambiguous_datasets/__init__.py +0 -0
- piblin/data/datasets/abc/unambiguous_datasets/one_dimensional_unambiguous_dataset.py +13 -0
- piblin/data/datasets/abc/unambiguous_datasets/unambiguous_dataset.py +224 -0
- piblin/data/datasets/linear_regions.py +176 -0
- piblin/data/datasets/roi.py +276 -0
- piblin/dataio/__init__.py +299 -0
- piblin/dataio/data_collection_reader.py +512 -0
- piblin/dataio/fileio/__init__.py +202 -0
- piblin/dataio/fileio/autodetect.py +156 -0
- piblin/dataio/fileio/batch_conversion.py +417 -0
- piblin/dataio/fileio/binary.py +317 -0
- piblin/dataio/fileio/read/__init__.py +8 -0
- piblin/dataio/fileio/read/file_reader.py +188 -0
- piblin/dataio/fileio/read/file_reader_template.py +73 -0
- piblin/dataio/fileio/read/generic/__init__.py +0 -0
- piblin/dataio/fileio/read/generic/generic_csv.py +116 -0
- piblin/dataio/fileio/read/generic/generic_file_reader.py +7 -0
- piblin/dataio/fileio/read/generic/generic_txt.py +97 -0
- piblin/dataio/fileio/read/multifile/__init__.py +0 -0
- piblin/dataio/fileio/read/multifile/multifile_reader.py +35 -0
- piblin/dataio/fileio/read/specific/__init__.py +0 -0
- piblin/dataio/fileio/read/summary/__init__.py +0 -0
- piblin/dataio/fileio/read/summary/summary_file_reader.py +68 -0
- piblin/dataio/fileio/write/__init__.py +0 -0
- piblin/dataio/fileio/write/file_writer.py +17 -0
- piblin/dataio/metadata_converter.py +121 -0
- piblin/dataio/pandas.py +414 -0
- piblin/dataio/read_summary.py +149 -0
- piblin/transform/__init__.py +31 -0
- piblin/transform/abc/__init__.py +5 -0
- piblin/transform/abc/dataset_transform.py +335 -0
- piblin/transform/abc/dynamic_transform.py +58 -0
- piblin/transform/abc/experiment_set_transform.py +32 -0
- piblin/transform/abc/experiment_transform.py +88 -0
- piblin/transform/abc/lambda_transform/__init__.py +0 -0
- piblin/transform/abc/lambda_transform/dataset_lambda_transform.py +27 -0
- piblin/transform/abc/lambda_transform/experiment_set_lambda_transform.py +25 -0
- piblin/transform/abc/lambda_transform/lambda_transform.py +41 -0
- piblin/transform/abc/lambda_transform/measurement_lambda_transform.py +27 -0
- piblin/transform/abc/lambda_transform/measurement_set_lambda_transform.py +26 -0
- piblin/transform/abc/measurement_set_transform.py +100 -0
- piblin/transform/abc/measurement_transform.py +109 -0
- piblin/transform/abc/region_transform/__init__.py +0 -0
- piblin/transform/abc/region_transform/dataset_region_transform.py +9 -0
- piblin/transform/abc/region_transform/measurement_region_transform.py +11 -0
- piblin/transform/abc/region_transform/measurement_set_region_transform.py +11 -0
- piblin/transform/abc/region_transform/region_transform.py +162 -0
- piblin/transform/abc/transform.py +591 -0
- piblin/transform/combine/__init__.py +18 -0
- piblin/transform/modify/__init__.py +0 -0
- piblin/transform/modify/interpolate.py +654 -0
- piblin/transform/pipeline.py +191 -0
- piblin/transform/split/__init__.py +0 -0
- piblin/transform/split/filter_datasets.py +41 -0
- piblin/transform/split/filter_measurements.py +47 -0
- piblin/transform/split/split_by_independent_variable_ranges.py +36 -0
- piblin/transform/split/split_transform.py +53 -0
- piblin-0.0.0a0.dist-info/LICENSE +21 -0
- piblin-0.0.0a0.dist-info/METADATA +52 -0
- piblin-0.0.0a0.dist-info/RECORD +86 -0
- piblin-0.0.0a0.dist-info/WHEEL +5 -0
- piblin-0.0.0a0.dist-info/top_level.txt +1 -0
piblin/__init__.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""(C)orporate (R)esearch (A)nalytical (L)aboratory (D)ata (S)cience
|
|
2
|
+
|
|
3
|
+
Data science tools produced as part of the R&D Data Analytics (gALAxy)
|
|
4
|
+
project of the 3M Corporate Research Analytical Laboratory (CRAL).
|
|
5
|
+
|
|
6
|
+
This code forms the basis for a number of notebook and UI applications and
|
|
7
|
+
can be used in the IPython, Jupyter or Databricks environments for
|
|
8
|
+
interactive problem-solving. The package functionality which is intended to
|
|
9
|
+
be public is available by importing from the following subpackages of
|
|
10
|
+
cralds.
|
|
11
|
+
|
|
12
|
+
Packages
|
|
13
|
+
--------
|
|
14
|
+
|
|
15
|
+
piblin.data - Represent analytical datasets and hierarchical collections thereof.
|
|
16
|
+
piblin.dataio - Read/write hierarchically structured collections of datasets to/from files.
|
|
17
|
+
piblin.transform - Transform collections of datasets.
|
|
18
|
+
"""
|
|
19
|
+
import numpy as np
|
|
20
|
+
|
|
21
|
+
np.set_printoptions(threshold=0,
|
|
22
|
+
precision=4,
|
|
23
|
+
edgeitems=2,
|
|
24
|
+
suppress=True,
|
|
25
|
+
linewidth=10000,
|
|
26
|
+
sign="+",
|
|
27
|
+
floatmode="fixed")
|
piblin/data/__init__.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from .data_collections.measurement import Measurement
|
|
2
|
+
from .data_collections.measurement_set import MeasurementSet
|
|
3
|
+
from .data_collections.consistent_measurement_set import ConsistentMeasurementSet
|
|
4
|
+
from .data_collections.tidy_measurement_set import TidyMeasurementSet
|
|
5
|
+
from .data_collections.experiment import Experiment
|
|
6
|
+
from .data_collections.experiment_set import ExperimentSet
|
|
7
|
+
from .data_collections.tabular_measurement_set import TabularMeasurementSet
|
|
8
|
+
|
|
9
|
+
from .datasets.roi import LinearRegion, CompoundRegion
|
|
10
|
+
from .datasets.abc.dataset import Dataset
|
|
11
|
+
from .datasets.abc.unambiguous_datasets.unambiguous_dataset import UnambiguousDataset
|
|
12
|
+
from .datasets.abc.unambiguous_datasets.one_dimensional_unambiguous_dataset import OneDimensionalUnambiguousDataset
|
|
13
|
+
from .datasets.abc.split_datasets import SplitDataset
|
|
14
|
+
from .datasets.abc.split_datasets.zero_dimensional_dataset import ZeroDimensionalDataset
|
|
15
|
+
from .datasets.abc.split_datasets.one_dimensional_dataset import OneDimensionalDataset
|
|
16
|
+
from .datasets.abc.split_datasets.two_dimensional_dataset import TwoDimensionalDataset
|
|
17
|
+
from .datasets.abc.split_datasets.three_dimensional_dataset import ThreeDimensionalDataset
|
|
18
|
+
from .datasets.abc.split_datasets.one_dimensional_composite_dataset import OneDimensionalCompositeDataset
|
|
19
|
+
from .datasets.abc.split_datasets.histogram import Histogram
|
|
20
|
+
from .datasets.abc.split_datasets.distribution import Distribution
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Package: piblin.data.data_collections - Hierarchical organization of measurements (data and metadata).
|
|
3
|
+
|
|
4
|
+
Modules
|
|
5
|
+
-------
|
|
6
|
+
experiment - Classes abstracting the concept of a scientific experiment.
|
|
7
|
+
measurement - Classes abstracting the concept of a scientific measurement.
|
|
8
|
+
measurement_set - Collections of scientific measurements and experiments.
|
|
9
|
+
|
|
10
|
+
Functions
|
|
11
|
+
---------
|
|
12
|
+
dict_repr - Return eval()-able string representation of a dict.
|
|
13
|
+
dict_str - Return human-readable representation of a dict.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def dict_repr(dict_):
|
|
18
|
+
"""Return eval()-able string representation of a dict.
|
|
19
|
+
|
|
20
|
+
Parameters
|
|
21
|
+
----------
|
|
22
|
+
dict_ : dict
|
|
23
|
+
The dictionary to convert to an eval()-able string.
|
|
24
|
+
|
|
25
|
+
Notes
|
|
26
|
+
-----
|
|
27
|
+
Expects dictionary keys to be strings.
|
|
28
|
+
"""
|
|
29
|
+
output = "{"
|
|
30
|
+
for key, value in dict_.items():
|
|
31
|
+
|
|
32
|
+
key_str = "\"" + key + "\""
|
|
33
|
+
|
|
34
|
+
if isinstance(value, str):
|
|
35
|
+
value_str = "\"" + value + "\""
|
|
36
|
+
else:
|
|
37
|
+
value_str = str(value)
|
|
38
|
+
|
|
39
|
+
output += key_str + ": " + value_str + ", "
|
|
40
|
+
return output[:-2] + "}"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def dict_str(dict_, title="", whitespace=None):
|
|
44
|
+
"""Return human-readable representation of a dict.
|
|
45
|
+
|
|
46
|
+
Parameters
|
|
47
|
+
----------
|
|
48
|
+
dict_ : dict
|
|
49
|
+
The dictionary to convert to a human-readable representation.
|
|
50
|
+
title : str
|
|
51
|
+
A string to use for the table title.
|
|
52
|
+
whitespace : str, None
|
|
53
|
+
Whether to add a tab in front of each key-value pair string.
|
|
54
|
+
|
|
55
|
+
Returns
|
|
56
|
+
-------
|
|
57
|
+
output : str
|
|
58
|
+
A human-readable representation of the dict.
|
|
59
|
+
"""
|
|
60
|
+
# output = title + "\n" + "-"*len(title) + "\n"
|
|
61
|
+
output = f"{title}:\n\n"
|
|
62
|
+
for key, value in dict_.items():
|
|
63
|
+
if whitespace:
|
|
64
|
+
output += whitespace
|
|
65
|
+
output += str(key) + " = " + str(value) + "\n"
|
|
66
|
+
return output
|
|
@@ -0,0 +1,457 @@
|
|
|
1
|
+
from typing import List, Tuple, Dict
|
|
2
|
+
import matplotlib.pyplot as plt
|
|
3
|
+
import matplotlib.axes
|
|
4
|
+
import matplotlib.figure
|
|
5
|
+
import matplotlib.colors
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
import piblin.data.data_collections.measurement as measurement_
|
|
9
|
+
import piblin.data.data_collections.measurement_set as measurement_set
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class InconsistentMeasurementsError(Exception):
|
|
13
|
+
...
|
|
14
|
+
"""Raised when inconsistent measurements are present."""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ConsistentMeasurementSet(measurement_set.MeasurementSet):
|
|
18
|
+
"""A set of measurements with consistent datasets.
|
|
19
|
+
|
|
20
|
+
To be consistent, a set of measurements must have the same
|
|
21
|
+
number and types of dataset. This simplifies the implementation
|
|
22
|
+
of several properties inherited from the superclass but restricts the
|
|
23
|
+
manipulation of the measurements to changes that do not remove the
|
|
24
|
+
consistency.
|
|
25
|
+
All metadata properties are not affected by the notion of consistency so
|
|
26
|
+
their implementations are all directly inherited from the base class.
|
|
27
|
+
Any method that removes measurements from an already consistent measurement
|
|
28
|
+
set by using conditions will not affect the consistency.
|
|
29
|
+
|
|
30
|
+
Parameters
|
|
31
|
+
----------
|
|
32
|
+
measurements : list of piblin.data.measurement.Measurement
|
|
33
|
+
The measurements to be collected.
|
|
34
|
+
|
|
35
|
+
Attributes
|
|
36
|
+
----------
|
|
37
|
+
measurements -> list of piblin.data.measurement.Measurement
|
|
38
|
+
The collected measurements. Inherited directly.
|
|
39
|
+
num_measurements -> int
|
|
40
|
+
The number of collected measurements. Inherited directly.
|
|
41
|
+
datasets -> list of list of dataset
|
|
42
|
+
The datasets of the set of measurements. Inherited directly.
|
|
43
|
+
num_datasets -> int
|
|
44
|
+
The number of datasets in each measurement of the set.
|
|
45
|
+
A consistent measurement set has a constant number of datasets across all measurements.
|
|
46
|
+
This property is therefore an int instead of a list of ints.
|
|
47
|
+
dataset_types -> list of type
|
|
48
|
+
The types of datasets in each measurement of the set.
|
|
49
|
+
A consistent measurement set has a shared set of types across all measurements.
|
|
50
|
+
This property is therefore simply a list of types.
|
|
51
|
+
dataset_lengths -> list of list of int
|
|
52
|
+
The lengths of datasets in each measurement of the set. Not directly inherited.
|
|
53
|
+
dataset_independent_variables() -> list of np.ndarray
|
|
54
|
+
The independent variables of each measurement of the set.
|
|
55
|
+
is_tidy -> bool
|
|
56
|
+
Determine if the measurement set is tidy. Directly inherited.
|
|
57
|
+
is_consistent -> bool
|
|
58
|
+
Determine if this measurement set is consistent. Directly inherited.
|
|
59
|
+
all_condition_names -> Set of str
|
|
60
|
+
All condition names present in at least one measurement.
|
|
61
|
+
shared_condition_names -> set of str
|
|
62
|
+
The condition names shared by all measurements.
|
|
63
|
+
unshared_condition_names -> set of str
|
|
64
|
+
The condition names not shared by all measurements.
|
|
65
|
+
equal_shared_condition_names -> set of str
|
|
66
|
+
The shared condition names with the same value in all measurements.
|
|
67
|
+
varying_shared_condition_names -> set of str
|
|
68
|
+
The condition names shared by all measurements with varying values.
|
|
69
|
+
equal_shared_conditions -> dict
|
|
70
|
+
The conditions shared by all measurements with equal values.
|
|
71
|
+
varying_shared_conditions -> dict
|
|
72
|
+
The conditions shared by all measurements with varying values.
|
|
73
|
+
all_detail_names -> set of str
|
|
74
|
+
All detail names present in at least one measurement.
|
|
75
|
+
shared_detail_names -> set of str
|
|
76
|
+
The detail names shared by all measurements.
|
|
77
|
+
unshared_detail_names -> set of str
|
|
78
|
+
The detail names not shared by all measurements.
|
|
79
|
+
equal_shared_detail_names -> set of str
|
|
80
|
+
The detail names with the same value in all measurements.
|
|
81
|
+
varying_shared_detail_names -> set of str
|
|
82
|
+
The detail names shared by all measurements without the same value in all measurements.
|
|
83
|
+
equal_shared_details -> dict
|
|
84
|
+
The details shared by all measurements with the same value.
|
|
85
|
+
|
|
86
|
+
Methods
|
|
87
|
+
-------
|
|
88
|
+
from_single_measurement(Measurement) -> ConsistentMeasurementSet
|
|
89
|
+
Create a measurement set from a single measurement. Directly inherited.
|
|
90
|
+
from_measurement_set(MeasurementSet) -> ConsistentMeasurementSet
|
|
91
|
+
from_measurement_sets([MeasurementSet]) -> ConsistentMeasurementSet
|
|
92
|
+
Create a single measurement set by combining multiple measurement sets. Directly inherited.
|
|
93
|
+
from_flat_data(np.ndarray, [[type]], [[int]], [[[object]]]) -> MeasurementSet
|
|
94
|
+
Create a measurement set from flat data. TODO - may have different signature
|
|
95
|
+
to_tidy_measurement_set([Interpolate]) -> TidyMeasurementSet
|
|
96
|
+
Convert this consistent measurement set to a tidy measurement set. Currently directly inherited.
|
|
97
|
+
to_tabular_dataset() -> TabularMeasurementSet
|
|
98
|
+
Convert this measurement set to a tabular dataset. Currently directly inherited.
|
|
99
|
+
are_repetitions() -> bool
|
|
100
|
+
Determine whether the list of measurements are repetitions. Directly inherited.
|
|
101
|
+
are_not_repetitions -> bool
|
|
102
|
+
Determine whether the list of measurements are not repetitions. Directly inherited.
|
|
103
|
+
condition_to_detail(str) -> None
|
|
104
|
+
Convert a condition to a detail.
|
|
105
|
+
conditions_to_details([str]) -> None
|
|
106
|
+
Convert a set of conditions to details for all measurements.
|
|
107
|
+
detail_to_condition(str) -> None
|
|
108
|
+
Convert a detail to a condition.
|
|
109
|
+
details_to_conditions(list of str) -> None
|
|
110
|
+
Convert a set of details to conditions for all measurements.
|
|
111
|
+
add_equal_shared_condition(str, object) -> None
|
|
112
|
+
Add a condition with a specified value to all measurements.
|
|
113
|
+
add_equal_shared_conditions([str], [object]) -> None
|
|
114
|
+
Add conditions with specified values to all measurements.
|
|
115
|
+
add_varying_shared_condition(str, [object]) -> None
|
|
116
|
+
Add a condition with different values for each measurement.
|
|
117
|
+
add_varying_shared_conditions([str], [[object]]) -> None
|
|
118
|
+
Add conditions with different values for each measurement.
|
|
119
|
+
add_condition(str, object) -> None
|
|
120
|
+
Add a condition to all measurements in the collection.
|
|
121
|
+
add_detail(str, object) -> None
|
|
122
|
+
Add a detail to all measurements in the collection.
|
|
123
|
+
add_equal_shared_detail(str, object) -> None
|
|
124
|
+
Add a detail with a specified value to all measurements.
|
|
125
|
+
add_equal_shared_details([str], [object]) -> None
|
|
126
|
+
Add details with specified values to all measurements.
|
|
127
|
+
add_varying_shared_detail(str, [object]) -> None
|
|
128
|
+
Add a detail with a specific value for each measurement.
|
|
129
|
+
add_varying_shared_details([str], [[object]]) -> None
|
|
130
|
+
Add details with varying values for each measurement.
|
|
131
|
+
remove_condition(str) -> None
|
|
132
|
+
Remove the condition with the specified name from all measurements.
|
|
133
|
+
remove_conditions([str])
|
|
134
|
+
Remove conditions with the specified name from all measurements.
|
|
135
|
+
remove_detail(str) -> None
|
|
136
|
+
Remove the detail with the specified name from all measurements.
|
|
137
|
+
remove_details([str])
|
|
138
|
+
Remove details with the specified name from all measurements.
|
|
139
|
+
flatten()
|
|
140
|
+
Convert the set of measurements to a flat dataset.
|
|
141
|
+
flatten_datasets()
|
|
142
|
+
Convert the measurement data to a set of rows.
|
|
143
|
+
flatten_metadata()
|
|
144
|
+
Convert the measurement metadata to a set of rows.
|
|
145
|
+
datasets_at_index() -> list of dataset
|
|
146
|
+
The dataset at a given index from each measurement.
|
|
147
|
+
delete_dataset_of_measurement_at_index(int, int) -> None # TODO
|
|
148
|
+
Delete the dataset at the specified index in the specified measurement.
|
|
149
|
+
This operation can remove consistency of the measurement set.
|
|
150
|
+
remove_measurements_with_condition_name() -> None
|
|
151
|
+
Remove measurements with the specified condition name.
|
|
152
|
+
remove_measurements_without_condition_name() -> None
|
|
153
|
+
Remove measurements without the specified condition name.
|
|
154
|
+
remove_measurements_with_condition() -> None
|
|
155
|
+
Remove measurements with the specified condition.
|
|
156
|
+
remove_measurements_without_condition() -> None
|
|
157
|
+
Remove measurements without the specified condition.
|
|
158
|
+
remove_measurements_by_condition_test() -> None
|
|
159
|
+
Remove measurements by testing a condition value.
|
|
160
|
+
remove_measurements_by_conditions_test() -> None
|
|
161
|
+
Remove measurements by testing multiple condition values.
|
|
162
|
+
visualize() -> Figure, Axes
|
|
163
|
+
Create a visual representation of this set of measurements.
|
|
164
|
+
produce_color_map()
|
|
165
|
+
Create a color map varying across measurements.
|
|
166
|
+
compute_figure_size()
|
|
167
|
+
Compute an appropriate size for the complete figure.
|
|
168
|
+
validate_title()
|
|
169
|
+
Assess and potentially create a title for the complete figure.
|
|
170
|
+
validate_figsize()
|
|
171
|
+
Assess and potentially create a size for the complete figure.
|
|
172
|
+
produce_color_maps()
|
|
173
|
+
Produce a color map for each measurement.
|
|
174
|
+
visualize_on_single_plot()
|
|
175
|
+
Visualize all measurements on a single plot per dataset.
|
|
176
|
+
visualize_on_multiple_plots()
|
|
177
|
+
Visualize the measurement datasets on individual plots.
|
|
178
|
+
"""
|
|
179
|
+
def __init__(self,
|
|
180
|
+
measurements: List[measurement_.Measurement] = None,
|
|
181
|
+
merge_redundant: bool = True):
|
|
182
|
+
|
|
183
|
+
super().__init__(measurements=measurements,
|
|
184
|
+
merge_redundant=merge_redundant)
|
|
185
|
+
|
|
186
|
+
if not self.is_consistent:
|
|
187
|
+
try:
|
|
188
|
+
self.measurements = self.enforce_consistency(measurements)
|
|
189
|
+
except ValueError:
|
|
190
|
+
raise InconsistentMeasurementsError("Measurements are not consistent." + self.consistency_str)
|
|
191
|
+
|
|
192
|
+
def _determine_unique_condition_combinations(self) -> List[Dict[str, object]]:
|
|
193
|
+
"""Determine the unique combinations of conditions among the measurements in this set.
|
|
194
|
+
|
|
195
|
+
When determining replicate relationships among measurements, the set of unique combinations
|
|
196
|
+
of conditions is required in order to organize the measurements more highly than they are
|
|
197
|
+
organized in the measurement set. The two examples in current use are determining sets of replicates
|
|
198
|
+
prior to creating experiment objects out of the measurements, and finding repetitions with different
|
|
199
|
+
datasets that can be combined into consistent measurements.
|
|
200
|
+
|
|
201
|
+
Returns
|
|
202
|
+
-------
|
|
203
|
+
List of Dict
|
|
204
|
+
The unique combinations of condition values in this measurement set.
|
|
205
|
+
"""
|
|
206
|
+
# which conditions are present in all measurements?
|
|
207
|
+
shared_condition_names = \
|
|
208
|
+
self.shared_condition_names
|
|
209
|
+
|
|
210
|
+
# cache these for performance reasons
|
|
211
|
+
measurement_conditions = [measurement.conditions for measurement in self.measurements]
|
|
212
|
+
|
|
213
|
+
# what are the unique combinations of values for these conditions?
|
|
214
|
+
unique_combinations: List[Dict] = []
|
|
215
|
+
for i, measurement in enumerate(self.measurements):
|
|
216
|
+
shared_conditions = {}
|
|
217
|
+
# which measurement conditions are shared across the measurement set
|
|
218
|
+
for condition_name in measurement.condition_names:
|
|
219
|
+
if condition_name in shared_condition_names:
|
|
220
|
+
shared_conditions[condition_name] = \
|
|
221
|
+
measurement_conditions[i][condition_name]
|
|
222
|
+
|
|
223
|
+
if unique_combinations:
|
|
224
|
+
unique = True
|
|
225
|
+
for unique_combination in unique_combinations:
|
|
226
|
+
if unique_combination == shared_conditions:
|
|
227
|
+
unique = False
|
|
228
|
+
break
|
|
229
|
+
|
|
230
|
+
if unique:
|
|
231
|
+
unique_combinations.append(shared_conditions)
|
|
232
|
+
else:
|
|
233
|
+
unique_combinations.append(shared_conditions)
|
|
234
|
+
|
|
235
|
+
return unique_combinations
|
|
236
|
+
|
|
237
|
+
def enforce_consistency(self, measurements: List[measurement_.Measurement]):
|
|
238
|
+
"""Can potentially use the replicate relationship to force consistency onto a measurement set.
|
|
239
|
+
the basic idea is that any set of measurements that shares the same condition metadata but has
|
|
240
|
+
different dataset number, type, names or units can have those datasets placed into a single
|
|
241
|
+
measurement with a single list of those datasets and the equal condition metadata.
|
|
242
|
+
To do this: 1) determine all sets of replicates (c.f. experiment set), concatenate their datasets
|
|
243
|
+
and create a new measurement for each set. Then create a set of all new measurements and check it for
|
|
244
|
+
consistency. If it's ok, set this class' measurement set to one made from those measurements.
|
|
245
|
+
Look at _update_experiments!
|
|
246
|
+
"""
|
|
247
|
+
shared_condition_names = self.shared_condition_names
|
|
248
|
+
|
|
249
|
+
unique_combinations = self._determine_unique_condition_combinations()
|
|
250
|
+
|
|
251
|
+
# cache these for performance reasons
|
|
252
|
+
measurement_conditions = [measurement.conditions for measurement in self.measurements]
|
|
253
|
+
|
|
254
|
+
new_measurements = []
|
|
255
|
+
for unique_combination in unique_combinations:
|
|
256
|
+
|
|
257
|
+
combine_measurements = []
|
|
258
|
+
|
|
259
|
+
# create a list of measurements with the given conditions
|
|
260
|
+
for i, measurement in enumerate(self.measurements):
|
|
261
|
+
|
|
262
|
+
shared_conditions = {}
|
|
263
|
+
|
|
264
|
+
for condition_name in measurement.condition_names:
|
|
265
|
+
if condition_name in shared_condition_names:
|
|
266
|
+
shared_conditions[condition_name] = \
|
|
267
|
+
measurement_conditions[i][condition_name]
|
|
268
|
+
|
|
269
|
+
if shared_conditions == unique_combination:
|
|
270
|
+
combine_measurements.append(measurement)
|
|
271
|
+
|
|
272
|
+
all_datasets = []
|
|
273
|
+
for measurement in combine_measurements:
|
|
274
|
+
all_datasets.extend(measurement.datasets)
|
|
275
|
+
|
|
276
|
+
combined_measurement = measurement_.Measurement(datasets=all_datasets,
|
|
277
|
+
conditions=unique_combination)
|
|
278
|
+
|
|
279
|
+
new_measurements.append(combined_measurement)
|
|
280
|
+
|
|
281
|
+
new_measurement_set = measurement_set.MeasurementSet(new_measurements)
|
|
282
|
+
if not new_measurement_set.is_consistent:
|
|
283
|
+
raise ValueError
|
|
284
|
+
else:
|
|
285
|
+
return new_measurements
|
|
286
|
+
|
|
287
|
+
@property
|
|
288
|
+
def num_datasets(self) -> int:
|
|
289
|
+
"""The number of datasets in each consistent measurement.
|
|
290
|
+
|
|
291
|
+
In a consistent measurement set this is guaranteed to be a single integer.
|
|
292
|
+
"""
|
|
293
|
+
if self.num_measurements == 0:
|
|
294
|
+
return 0
|
|
295
|
+
|
|
296
|
+
return self.measurements[0].num_datasets
|
|
297
|
+
|
|
298
|
+
@property
|
|
299
|
+
def dataset_types(self) -> List[type]:
|
|
300
|
+
"""The types of dataset in each consistent measurement.
|
|
301
|
+
|
|
302
|
+
In a consistent measurement set this is guaranteed to be a list of types.
|
|
303
|
+
"""
|
|
304
|
+
if self.num_measurements == 0:
|
|
305
|
+
return []
|
|
306
|
+
|
|
307
|
+
return self.measurements[0].dataset_types
|
|
308
|
+
|
|
309
|
+
@property
|
|
310
|
+
def dataset_dimensionalities(self) -> List[int]:
|
|
311
|
+
"""The number of independent variables of each dataset in each consistent measurement."""
|
|
312
|
+
if self.num_measurements == 0:
|
|
313
|
+
return []
|
|
314
|
+
|
|
315
|
+
return self.measurements[0].dataset_dimensionalities
|
|
316
|
+
|
|
317
|
+
@property
|
|
318
|
+
def dataset_dependent_variable_names(self) -> List[List[str]]:
|
|
319
|
+
"""The dependent variable names of each dataset in each consistent measurement."""
|
|
320
|
+
if self.num_measurements == 0:
|
|
321
|
+
return []
|
|
322
|
+
|
|
323
|
+
return self.measurements[0].dataset_dependent_variable_names
|
|
324
|
+
|
|
325
|
+
@property
|
|
326
|
+
def dataset_dependent_variable_units(self) -> List[List[str]]:
|
|
327
|
+
"""The dependent variable units of each dataset in each consistent measurement."""
|
|
328
|
+
if self.num_measurements == 0:
|
|
329
|
+
return []
|
|
330
|
+
|
|
331
|
+
return self.measurements[0].dataset_dependent_variable_units
|
|
332
|
+
|
|
333
|
+
@property
|
|
334
|
+
def dataset_independent_variable_names(self) -> List[List[str]]:
|
|
335
|
+
"""The independent variable names of each dataset in each consistent measurement."""
|
|
336
|
+
if self.num_measurements == 0:
|
|
337
|
+
return []
|
|
338
|
+
|
|
339
|
+
return self.measurements[0].dataset_independent_variable_names
|
|
340
|
+
|
|
341
|
+
@property
|
|
342
|
+
def dataset_independent_variable_units(self) -> List[List[str]]:
|
|
343
|
+
"""The independent variable units of each dataset in each consistent measurement."""
|
|
344
|
+
if self.num_measurements == 0:
|
|
345
|
+
return []
|
|
346
|
+
|
|
347
|
+
return self.measurements[0].dataset_independent_variable_names
|
|
348
|
+
|
|
349
|
+
def datasets_at_index(self, index: int):
|
|
350
|
+
"""The datasets of each measurement in this measurement set at the given index."""
|
|
351
|
+
return [measurement.datasets[index] for measurement in self.measurements]
|
|
352
|
+
|
|
353
|
+
@classmethod
|
|
354
|
+
def from_measurement_set(cls,
|
|
355
|
+
measurement_set_: measurement_set.MeasurementSet,
|
|
356
|
+
dataset_indices_to_retain: List[int] = None,
|
|
357
|
+
make_copy=True
|
|
358
|
+
) -> "ConsistentMeasurementSet":
|
|
359
|
+
"""Create a consistent measurement set from a measurement set.
|
|
360
|
+
|
|
361
|
+
A measurement set may be partially consistent, allowing it to be stripped of
|
|
362
|
+
inconsistent datasets to become consistent.
|
|
363
|
+
This method wraps the corresponding method from MeasurementSet.
|
|
364
|
+
|
|
365
|
+
Parameters
|
|
366
|
+
----------
|
|
367
|
+
measurement_set_ : MeasurementSet
|
|
368
|
+
The measurement set from which to create a consistent measurement set.
|
|
369
|
+
dataset_indices_to_retain : list of int
|
|
370
|
+
The indices of the datasets to retain if consistent.
|
|
371
|
+
make_copy : bool
|
|
372
|
+
Whether to alter the measurement set in place or create a duplicate.
|
|
373
|
+
|
|
374
|
+
"""
|
|
375
|
+
ms = measurement_set_.to_consistent_measurement_set(dataset_indices_to_retain=dataset_indices_to_retain,
|
|
376
|
+
make_copy=make_copy)
|
|
377
|
+
|
|
378
|
+
return cls(ms.measurements)
|
|
379
|
+
|
|
380
|
+
def _has_consistent_dataset_type_at_index(self, dataset_index):
|
|
381
|
+
"""Check whether this set of measurements have consistent dataset types at a given index.
|
|
382
|
+
|
|
383
|
+
This implementation is simpler than for the superclass due to the
|
|
384
|
+
guaranteed constant number of datasets across all measurements.
|
|
385
|
+
|
|
386
|
+
Parameters
|
|
387
|
+
----------
|
|
388
|
+
dataset_index : int
|
|
389
|
+
The index of the dataset to check for consistency.
|
|
390
|
+
|
|
391
|
+
Returns
|
|
392
|
+
-------
|
|
393
|
+
bool
|
|
394
|
+
Whether the type of datasets are the given index are consistent.
|
|
395
|
+
"""
|
|
396
|
+
if dataset_index >= self.num_datasets:
|
|
397
|
+
return False
|
|
398
|
+
|
|
399
|
+
dataset_type = type(self.measurements[0].datasets[dataset_index])
|
|
400
|
+
for measurement in self.measurements[1:]:
|
|
401
|
+
if type(measurement.datasets[dataset_index]) != dataset_type:
|
|
402
|
+
return False
|
|
403
|
+
|
|
404
|
+
return True
|
|
405
|
+
|
|
406
|
+
def remove_dataset_of_measurement_at_index(self, measurement_index, dataset_index) -> None:
|
|
407
|
+
"""Delete the dataset at the specified index in the specified measurement.
|
|
408
|
+
|
|
409
|
+
Applying this method will remove consistency for almost all measurement sets that
|
|
410
|
+
will exist (exceptions being single-measurement measurement sets).
|
|
411
|
+
|
|
412
|
+
Parameters
|
|
413
|
+
----------
|
|
414
|
+
measurement_index : int
|
|
415
|
+
The index of the measurement.
|
|
416
|
+
dataset_index : int
|
|
417
|
+
The index of the dataset.
|
|
418
|
+
"""
|
|
419
|
+
if self.num_measurements == 1:
|
|
420
|
+
super().remove_dataset_of_measurement_at_index(measurement_index, dataset_index)
|
|
421
|
+
else:
|
|
422
|
+
raise NotImplementedError("Cannot currently delete single measurement from consistent measurement set.")
|
|
423
|
+
|
|
424
|
+
@property
|
|
425
|
+
def max_num_datasets(self) -> int:
|
|
426
|
+
return self.num_datasets
|
|
427
|
+
|
|
428
|
+
def _has_collapsible_measurements(self) -> bool:
|
|
429
|
+
return True
|
|
430
|
+
|
|
431
|
+
@property
|
|
432
|
+
def dataset_colormaps(self) -> List[matplotlib.colors.Colormap]:
|
|
433
|
+
"""Create color maps for plotting this measurement set's datasets.
|
|
434
|
+
|
|
435
|
+
Each dataset column in the consistent measurement set will share a
|
|
436
|
+
dataset type due to the guarantee of consistency in this measurement
|
|
437
|
+
set. Each dataset class provides a default color map that can be used
|
|
438
|
+
to generate distinct colors for multiple datasets of the same type.
|
|
439
|
+
"""
|
|
440
|
+
return [dataset.DEFAULT_COLOR_MAP for dataset in self.measurements[0].datasets]
|
|
441
|
+
|
|
442
|
+
@property
|
|
443
|
+
def dataset_colors(self,
|
|
444
|
+
dataset_colormaps: List[matplotlib.colors.Colormap] = None) -> List[List[np.ndarray]]:
|
|
445
|
+
"""Get a set of per-dataset colors for this consistent measurement set.
|
|
446
|
+
|
|
447
|
+
Returns
|
|
448
|
+
-------
|
|
449
|
+
A per-dataset list of per-measurement colors drawn from default color maps.
|
|
450
|
+
"""
|
|
451
|
+
if dataset_colormaps is None:
|
|
452
|
+
dataset_colormaps = self.dataset_colormaps
|
|
453
|
+
|
|
454
|
+
return [dataset_colormaps[i](np.linspace(0,
|
|
455
|
+
1,
|
|
456
|
+
self.num_measurements))
|
|
457
|
+
for i in range(self.num_datasets)]
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
from typing import List, Union
|
|
2
|
+
import piblin.data.datasets.abc.dataset as dataset_
|
|
3
|
+
import piblin.data.data_collections.measurement as measurement_
|
|
4
|
+
import piblin.data.data_collections.measurement_set as measurement_set_
|
|
5
|
+
import piblin.data.data_collections.consistent_measurement_set as consistent_measurement_set_
|
|
6
|
+
import piblin.data.data_collections.tidy_measurement_set as tidy_measurement_set_
|
|
7
|
+
import piblin.data.data_collections.experiment_set as experiment_set_
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class DataCollectionFactory(object):
|
|
11
|
+
"""A class to create instances of measurement set subclasses."""
|
|
12
|
+
@staticmethod
|
|
13
|
+
def from_measurements(measurements: List[measurement_.Measurement],
|
|
14
|
+
to_experiments: bool = True,
|
|
15
|
+
minimize_hierarchy: bool = False,
|
|
16
|
+
merge_redundant: bool = True) -> Union[measurement_set_.MeasurementSet,
|
|
17
|
+
consistent_measurement_set_.ConsistentMeasurementSet,
|
|
18
|
+
tidy_measurement_set_.TidyMeasurementSet,
|
|
19
|
+
"experiment_set_.ExperimentSet"]:
|
|
20
|
+
"""Create an appropriate measurement set subclass from given measurements.
|
|
21
|
+
|
|
22
|
+
Given a set of measurements, creates an instance of the least structured data collection class (the measurement
|
|
23
|
+
set), then based on the properties of that measurement set (tidiness and consistency) creates the most
|
|
24
|
+
structured data collection class possible. By default, a measurement set that is tidy or consistent is converted
|
|
25
|
+
to the corresponding experiment set unless the caller specifies that conversion to experiments should not be
|
|
26
|
+
carried out. In the latter case a tidy or consistent measurement set will be returned in lieu of an experiment
|
|
27
|
+
set.
|
|
28
|
+
|
|
29
|
+
Parameters
|
|
30
|
+
----------
|
|
31
|
+
measurements : List of Measurement
|
|
32
|
+
The measurements to convert to the appropriate measurement set subclass.
|
|
33
|
+
to_experiments : bool
|
|
34
|
+
Whether to convert to an experiment set if measurements are consistent.
|
|
35
|
+
minimize_hierarchy : bool
|
|
36
|
+
Whether to remove as much hierarchy as possible from the data collection.
|
|
37
|
+
merge_redundant : bool
|
|
38
|
+
Whether to merge measurements that are redundant based on conditions.
|
|
39
|
+
|
|
40
|
+
Returns
|
|
41
|
+
-------
|
|
42
|
+
MeasurementSet or ConsistentMeasurementSet or TidyMeasurementSet or ExperimentSet
|
|
43
|
+
The appropriate measurement set subclass.
|
|
44
|
+
"""
|
|
45
|
+
measurement_set = measurement_set_.MeasurementSet(measurements,
|
|
46
|
+
merge_redundant=merge_redundant)
|
|
47
|
+
|
|
48
|
+
if measurement_set.is_tidy and measurement_set.is_consistent:
|
|
49
|
+
if not to_experiments:
|
|
50
|
+
measurement_set = tidy_measurement_set_.TidyMeasurementSet(measurements=measurements,
|
|
51
|
+
merge_redundant=merge_redundant)
|
|
52
|
+
else:
|
|
53
|
+
measurement_set = experiment_set_.ExperimentSet(measurements=measurements,
|
|
54
|
+
merge_redundant=merge_redundant)
|
|
55
|
+
|
|
56
|
+
elif measurement_set.is_consistent and not measurement_set.is_tidy:
|
|
57
|
+
if not to_experiments:
|
|
58
|
+
measurement_set = consistent_measurement_set_.ConsistentMeasurementSet(measurements=measurements,
|
|
59
|
+
merge_redundant=merge_redundant)
|
|
60
|
+
else:
|
|
61
|
+
measurement_set = experiment_set_.ExperimentSet(measurements=measurements,
|
|
62
|
+
merge_redundant=merge_redundant)
|
|
63
|
+
|
|
64
|
+
else:
|
|
65
|
+
if to_experiments:
|
|
66
|
+
print(f"Warning: Measurements are not consistent so could not be converted to experiments. "
|
|
67
|
+
f"A measurement set has been created.")
|
|
68
|
+
print(measurement_set.consistency_str)
|
|
69
|
+
if minimize_hierarchy:
|
|
70
|
+
return DataCollectionFactory.__minimize_hierarchy(measurement_set)
|
|
71
|
+
else:
|
|
72
|
+
return measurement_set
|
|
73
|
+
|
|
74
|
+
@staticmethod
|
|
75
|
+
def __minimize_hierarchy(data_collection: Union[measurement_set_.MeasurementSet,
|
|
76
|
+
consistent_measurement_set_.ConsistentMeasurementSet,
|
|
77
|
+
tidy_measurement_set_.TidyMeasurementSet,
|
|
78
|
+
"experiment_set_.ExperimentSet"]) -> \
|
|
79
|
+
Union[dataset_.Dataset,
|
|
80
|
+
measurement_.Measurement,
|
|
81
|
+
measurement_set_.MeasurementSet,
|
|
82
|
+
consistent_measurement_set_.ConsistentMeasurementSet,
|
|
83
|
+
tidy_measurement_set_.TidyMeasurementSet,
|
|
84
|
+
"experiment_set_.ExperimentSet"]:
|
|
85
|
+
|
|
86
|
+
if len(data_collection) == 1:
|
|
87
|
+
experiment = data_collection[0]
|
|
88
|
+
if len(experiment) == 1:
|
|
89
|
+
repetitions = experiment.datasets
|
|
90
|
+
if len(repetitions) == 1: # one repetition
|
|
91
|
+
if len(repetitions[0]) == 1:
|
|
92
|
+
return repetitions[0][0]
|
|
93
|
+
else:
|
|
94
|
+
return experiment # should be a measurement
|
|
95
|
+
else:
|
|
96
|
+
return experiment
|
|
97
|
+
else:
|
|
98
|
+
return experiment
|
|
99
|
+
else:
|
|
100
|
+
return data_collection
|