PyOPIA 2.4.0__tar.gz → 2.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyopia-2.4.0 → pyopia-2.4.2}/PKG-INFO +1 -1
- pyopia-2.4.2/pyopia/__init__.py +1 -0
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/background.py +86 -49
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/io.py +52 -36
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/pipeline.py +16 -8
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/plotting.py +14 -10
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/process.py +141 -100
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/simulator/silcam.py +30 -3
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/statistics.py +413 -280
- pyopia-2.4.2/pyopia/tests/__init__.py +1 -0
- pyopia-2.4.0/pyopia/__init__.py +0 -1
- {pyopia-2.4.0 → pyopia-2.4.2}/LICENSE +0 -0
- {pyopia-2.4.0 → pyopia-2.4.2}/README.md +0 -0
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/classify.py +0 -0
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/cli.py +0 -0
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/exampledata.py +0 -0
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/instrument/__init__.py +0 -0
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/instrument/common.py +0 -0
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/instrument/holo.py +0 -0
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/instrument/silcam.py +0 -0
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/instrument/uvp.py +0 -0
- {pyopia-2.4.0/pyopia/tests → pyopia-2.4.2/pyopia/simulator}/__init__.py +0 -0
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/tests/test_classify.py +0 -0
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/tests/test_notebooks.py +0 -0
- {pyopia-2.4.0 → pyopia-2.4.2}/pyopia/tests/test_pipeline.py +0 -0
- {pyopia-2.4.0 → pyopia-2.4.2}/pyproject.toml +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '2.4.2'
|
|
@@ -8,13 +8,19 @@ def ini_background(bgfiles, load_function):
|
|
|
8
8
|
'''
|
|
9
9
|
Create and initial background stack and average image
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
11
|
+
Parameters:
|
|
12
|
+
-----------
|
|
13
|
+
bgfiles : list
|
|
14
|
+
List of strings of filenames to be used in background creation
|
|
15
|
+
load_function : object
|
|
16
|
+
This function should take a filename and return an image, for example: :func:`pyopia.instrument.silcam.load_image`
|
|
17
|
+
|
|
18
|
+
Returns
|
|
19
|
+
-------
|
|
20
|
+
bgstack : list
|
|
21
|
+
list of all images in the background stack
|
|
22
|
+
imbg : array
|
|
23
|
+
background image
|
|
18
24
|
'''
|
|
19
25
|
bgstack = []
|
|
20
26
|
for f in bgfiles:
|
|
@@ -33,14 +39,21 @@ def shift_bgstack_accurate(bgstack, imbg, imnew):
|
|
|
33
39
|
The new background is calculated slowly by computing the mean of all images
|
|
34
40
|
in the background stack.
|
|
35
41
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
Parameters:
|
|
43
|
+
-----------
|
|
44
|
+
bgstack : list
|
|
45
|
+
list of all images in the background stack
|
|
46
|
+
imbg : array
|
|
47
|
+
background image
|
|
48
|
+
imnew : array
|
|
49
|
+
new image to be added to stack
|
|
50
|
+
|
|
51
|
+
Returns
|
|
52
|
+
-------
|
|
53
|
+
bgstack : list
|
|
54
|
+
updated list of all background images
|
|
55
|
+
imbg : array
|
|
56
|
+
updated actual background image
|
|
44
57
|
'''
|
|
45
58
|
bgstack.pop(0) # pop the oldest image from the stack,
|
|
46
59
|
bgstack.append(imnew) # append the new image to the stack
|
|
@@ -56,14 +69,21 @@ def shift_bgstack_fast(bgstack, imbg, imnew):
|
|
|
56
69
|
adding the new image (both scaled by the stacklength).
|
|
57
70
|
This is close to a running mean, but not quite.
|
|
58
71
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
72
|
+
Parameters:
|
|
73
|
+
-----------
|
|
74
|
+
bgstac : list
|
|
75
|
+
list of all images in the background stack
|
|
76
|
+
imbg : uint8
|
|
77
|
+
background image
|
|
78
|
+
imnew : unit8
|
|
79
|
+
new image to be added to stack
|
|
80
|
+
|
|
81
|
+
Returns
|
|
82
|
+
-------
|
|
83
|
+
bgstack : list
|
|
84
|
+
updated list of all background images
|
|
85
|
+
imbg : array
|
|
86
|
+
updated actual background image
|
|
67
87
|
'''
|
|
68
88
|
stacklength = len(bgstack)
|
|
69
89
|
imold = bgstack.pop(0) # pop the oldest image from the stack,
|
|
@@ -82,14 +102,17 @@ def correct_im_accurate(imbg, imraw):
|
|
|
82
102
|
There is a small chance of clipping of imc in both crushed blacks and blown
|
|
83
103
|
highlights if the background or raw images are very poorly obtained
|
|
84
104
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
105
|
+
Parameters:
|
|
106
|
+
-----------
|
|
107
|
+
imbg : float64
|
|
108
|
+
background averaged image
|
|
109
|
+
imraw : float64
|
|
110
|
+
raw image
|
|
111
|
+
|
|
112
|
+
Returns
|
|
113
|
+
-------
|
|
114
|
+
im_corrected : float64
|
|
115
|
+
corrected image, same type as input
|
|
93
116
|
'''
|
|
94
117
|
|
|
95
118
|
im_corrected = imraw - imbg
|
|
@@ -108,12 +131,17 @@ def correct_im_fast(imbg, imraw):
|
|
|
108
131
|
There is high potential for clipping of imc in both crushed blacks an blown
|
|
109
132
|
highlights, especially if the background or raw images are not properly obtained
|
|
110
133
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
134
|
+
Parameters:
|
|
135
|
+
-----------
|
|
136
|
+
imraw : array
|
|
137
|
+
raw image
|
|
138
|
+
imbg : array
|
|
139
|
+
background averaged image
|
|
140
|
+
|
|
141
|
+
Returns
|
|
142
|
+
-------
|
|
143
|
+
im_corrected : array
|
|
144
|
+
corrected image
|
|
117
145
|
'''
|
|
118
146
|
im_corrected = imraw - imbg
|
|
119
147
|
|
|
@@ -130,18 +158,27 @@ def shift_and_correct(bgstack, imbg, imraw, stacklength, real_time_stats=False):
|
|
|
130
158
|
|
|
131
159
|
This is a wrapper for shift_bgstack and correct_im
|
|
132
160
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
161
|
+
Parameters:
|
|
162
|
+
-----------
|
|
163
|
+
bgstack : list
|
|
164
|
+
list of all images in the background stack
|
|
165
|
+
imbg : float64
|
|
166
|
+
background image
|
|
167
|
+
imraw : float64
|
|
168
|
+
raw image
|
|
169
|
+
stacklength : int
|
|
170
|
+
unused int here - just there to maintain the same behaviour as shift_bgstack_fast()
|
|
171
|
+
real_time_stats : Bool, optional
|
|
172
|
+
True use fast functions, if False use accurate functions., by default False
|
|
173
|
+
|
|
174
|
+
Returns
|
|
175
|
+
-------
|
|
176
|
+
bgstack : list
|
|
177
|
+
list of all images in the background stack
|
|
178
|
+
imbg : float64
|
|
179
|
+
background averaged image
|
|
180
|
+
im_corrected : float64
|
|
181
|
+
corrected image
|
|
145
182
|
'''
|
|
146
183
|
|
|
147
184
|
if real_time_stats:
|
|
@@ -32,18 +32,24 @@ def write_stats(stats,
|
|
|
32
32
|
Writes particle stats into the ouput file.
|
|
33
33
|
Appends if file already exists.
|
|
34
34
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
35
|
+
Parameters
|
|
36
|
+
----------
|
|
37
|
+
datafilename : str
|
|
38
|
+
Filame prefix for -STATS.h5 file that may or may not include a path
|
|
39
|
+
stats : DataFrame or xr.Dataset
|
|
40
|
+
particle statistics
|
|
41
|
+
export_name_len : int
|
|
42
|
+
Max number of chars allowed for col 'export name'
|
|
43
|
+
append : bool
|
|
44
|
+
Append all processed data into one nc file.
|
|
45
|
+
Defaults to True.
|
|
46
|
+
If False, then one nc file will be generated per raw image,
|
|
47
|
+
which can be loaded using :func:`pyopia.io.combine_stats_netcdf_files`
|
|
48
|
+
This is useful for larger datasets,
|
|
49
|
+
where appending causes substantial slowdown
|
|
50
|
+
as the dataset gets larger.
|
|
51
|
+
image_stats : xr.Dataset
|
|
52
|
+
summary statistics of each raw image (including those with no particles)
|
|
47
53
|
'''
|
|
48
54
|
|
|
49
55
|
if len(stats) == 0: # to avoid issue with wrong time datatypes in xarray
|
|
@@ -99,7 +105,7 @@ def make_xstats(stats, toml_steps):
|
|
|
99
105
|
|
|
100
106
|
Returns
|
|
101
107
|
-------
|
|
102
|
-
xarray.
|
|
108
|
+
xstats : xarray.Dataset
|
|
103
109
|
Xarray version of stats dataframe, including metadata
|
|
104
110
|
'''
|
|
105
111
|
xstats = stats.to_xarray()
|
|
@@ -120,8 +126,8 @@ def load_image_stats(datafilename):
|
|
|
120
126
|
|
|
121
127
|
Returns
|
|
122
128
|
-------
|
|
123
|
-
xarray.
|
|
124
|
-
|
|
129
|
+
image_stats : xarray.Dataset
|
|
130
|
+
summary statistics of each raw image (including those with no particles)
|
|
125
131
|
'''
|
|
126
132
|
with xarray.open_dataset(datafilename, engine=NETCDF_ENGINE, group='image_stats') as image_stats:
|
|
127
133
|
image_stats.load()
|
|
@@ -138,7 +144,7 @@ def load_stats(datafilename):
|
|
|
138
144
|
|
|
139
145
|
Returns
|
|
140
146
|
-------
|
|
141
|
-
DataFrame
|
|
147
|
+
stats : DataFrame
|
|
142
148
|
STATS DataFrame / xarray dataset
|
|
143
149
|
'''
|
|
144
150
|
|
|
@@ -170,8 +176,10 @@ def combine_stats_netcdf_files(path_to_data, prefix='*'):
|
|
|
170
176
|
|
|
171
177
|
Returns
|
|
172
178
|
-------
|
|
173
|
-
|
|
174
|
-
|
|
179
|
+
xstats : xarray.Dataset
|
|
180
|
+
Particle statistics and metatdata from processing steps
|
|
181
|
+
image_stats : xarray.Dataset
|
|
182
|
+
summary statistics of each raw image (including those with no particles)
|
|
175
183
|
'''
|
|
176
184
|
|
|
177
185
|
sorted_filelist = sorted(glob(os.path.join(path_to_data, prefix + 'Image-D*-STATS.nc')))
|
|
@@ -234,7 +242,7 @@ def steps_from_xstats(xstats):
|
|
|
234
242
|
|
|
235
243
|
Returns
|
|
236
244
|
-------
|
|
237
|
-
dict
|
|
245
|
+
steps : dict
|
|
238
246
|
TOML-formatted dictionary of pipeline steps
|
|
239
247
|
'''
|
|
240
248
|
steps = toml.loads(xstats.__getattr__('steps'))
|
|
@@ -251,7 +259,7 @@ def load_stats_as_dataframe(stats_file):
|
|
|
251
259
|
|
|
252
260
|
Returns
|
|
253
261
|
-------
|
|
254
|
-
DataFrame
|
|
262
|
+
stats : DataFrame
|
|
255
263
|
stats pandas dataframe
|
|
256
264
|
'''
|
|
257
265
|
# obtain particle statistics from the stats file
|
|
@@ -269,8 +277,10 @@ def show_h5_meta(h5file):
|
|
|
269
277
|
'''
|
|
270
278
|
prints metadata from an exported hdf5 file created from pyopia.process
|
|
271
279
|
|
|
272
|
-
|
|
273
|
-
|
|
280
|
+
Parameters
|
|
281
|
+
----------
|
|
282
|
+
h5file : str
|
|
283
|
+
h5 filename from exported data from pyopia.process
|
|
274
284
|
'''
|
|
275
285
|
|
|
276
286
|
with h5py.File(h5file, 'r') as f:
|
|
@@ -286,20 +296,26 @@ class StatsToDisc():
|
|
|
286
296
|
|
|
287
297
|
Replaces the old StatsH5 class
|
|
288
298
|
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
299
|
+
Parameters
|
|
300
|
+
----------
|
|
301
|
+
output_datafile : str
|
|
302
|
+
prefix path for output nc file
|
|
303
|
+
dataformat : str
|
|
304
|
+
either 'nc' or 'h5
|
|
305
|
+
export_name_len : int
|
|
306
|
+
max number of chars allowed for col 'export name'. Defaults to 40
|
|
307
|
+
append : bool
|
|
308
|
+
Append all processed data into one nc file.
|
|
309
|
+
Defaults to True.
|
|
310
|
+
If False, then one nc file will be generated per raw image,
|
|
311
|
+
which can be loaded using :func:`pyopia.io.combine_stats_netcdf_files`
|
|
312
|
+
This is useful for larger datasets, where appending causes substantial slowdown
|
|
313
|
+
as the dataset gets larger.
|
|
314
|
+
|
|
315
|
+
Returns
|
|
316
|
+
-------
|
|
317
|
+
data : dict
|
|
318
|
+
data from pipeline
|
|
303
319
|
|
|
304
320
|
Example config for pipeline useage:
|
|
305
321
|
|
|
@@ -84,11 +84,15 @@ class Pipeline():
|
|
|
84
84
|
def run(self, filename):
|
|
85
85
|
'''Method for executing the processing pipeline.
|
|
86
86
|
|
|
87
|
-
|
|
88
|
-
|
|
87
|
+
Parameters
|
|
88
|
+
----------
|
|
89
|
+
filename : str
|
|
90
|
+
file to be processed
|
|
89
91
|
|
|
90
|
-
Returns
|
|
91
|
-
|
|
92
|
+
Returns
|
|
93
|
+
-------
|
|
94
|
+
stats : DataFrame
|
|
95
|
+
particle statistics associated with 'filename'
|
|
92
96
|
|
|
93
97
|
Note: the returned stats from this function are single-image only and not appended
|
|
94
98
|
if you loop through several filenames! It is recommended to use this step in the pipeline
|
|
@@ -259,11 +263,15 @@ class Data(TypedDict):
|
|
|
259
263
|
def steps_to_string(steps):
|
|
260
264
|
'''Deprecated. Convert pipeline steps dictionary to a human-readable string
|
|
261
265
|
|
|
262
|
-
|
|
263
|
-
|
|
266
|
+
Parameters
|
|
267
|
+
----------
|
|
268
|
+
steps : dict
|
|
269
|
+
pipeline steps dictionary
|
|
264
270
|
|
|
265
|
-
Returns
|
|
266
|
-
|
|
271
|
+
Returns
|
|
272
|
+
-------
|
|
273
|
+
steps_str : str
|
|
274
|
+
human-readable string of the types and variables
|
|
267
275
|
'''
|
|
268
276
|
|
|
269
277
|
steps_str = '\n'
|
|
@@ -9,12 +9,14 @@ import numpy as np
|
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
def show_image(image, pixel_size):
|
|
12
|
-
'''
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
12
|
+
'''Plots a scaled figure (in mm) of an image
|
|
13
|
+
|
|
14
|
+
Parameters
|
|
15
|
+
----------
|
|
16
|
+
image : float
|
|
17
|
+
Image (usually a corrected image, such as im_corrected)
|
|
18
|
+
pixel_size : float
|
|
19
|
+
the pixel size (um) of the imaging system used
|
|
18
20
|
'''
|
|
19
21
|
r, c = np.shape(image[:, :, 0])
|
|
20
22
|
|
|
@@ -31,10 +33,12 @@ def montage_plot(montage, pixel_size):
|
|
|
31
33
|
'''
|
|
32
34
|
Plots a SilCam particle montage with a 1mm scale reference
|
|
33
35
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
36
|
+
Parameters
|
|
37
|
+
----------
|
|
38
|
+
montage : uint8
|
|
39
|
+
a montage created with scpp.make_montage
|
|
40
|
+
pixel_size : float
|
|
41
|
+
the pixel size (um) of the imaging system used
|
|
38
42
|
'''
|
|
39
43
|
msize = np.shape(montage)[0]
|
|
40
44
|
ex = pixel_size * np.float64(msize) / 1000.
|