PYME-extra 1.0.4.post0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. PYMEcs/Acquire/Actions/__init__.py +0 -0
  2. PYMEcs/Acquire/Actions/custom.py +167 -0
  3. PYMEcs/Acquire/Hardware/LPthreadedSimple.py +248 -0
  4. PYMEcs/Acquire/Hardware/LPthreadedSimpleSim.py +246 -0
  5. PYMEcs/Acquire/Hardware/NikonTiFlaskServer.py +45 -0
  6. PYMEcs/Acquire/Hardware/NikonTiFlaskServerT.py +59 -0
  7. PYMEcs/Acquire/Hardware/NikonTiRESTClient.py +73 -0
  8. PYMEcs/Acquire/Hardware/NikonTiSim.py +35 -0
  9. PYMEcs/Acquire/Hardware/__init__.py +0 -0
  10. PYMEcs/Acquire/Hardware/driftTrackGUI.py +329 -0
  11. PYMEcs/Acquire/Hardware/driftTrackGUI_n.py +472 -0
  12. PYMEcs/Acquire/Hardware/driftTracking.py +424 -0
  13. PYMEcs/Acquire/Hardware/driftTracking_n.py +433 -0
  14. PYMEcs/Acquire/Hardware/fakeCamX.py +15 -0
  15. PYMEcs/Acquire/Hardware/offsetPiezoRESTCorrelLog.py +38 -0
  16. PYMEcs/Acquire/__init__.py +0 -0
  17. PYMEcs/Analysis/MBMcollection.py +552 -0
  18. PYMEcs/Analysis/MINFLUX.py +280 -0
  19. PYMEcs/Analysis/MapUtils.py +77 -0
  20. PYMEcs/Analysis/NPC.py +1176 -0
  21. PYMEcs/Analysis/Paraflux.py +218 -0
  22. PYMEcs/Analysis/Simpler.py +81 -0
  23. PYMEcs/Analysis/Sofi.py +140 -0
  24. PYMEcs/Analysis/__init__.py +0 -0
  25. PYMEcs/Analysis/decSofi.py +211 -0
  26. PYMEcs/Analysis/eventProperties.py +50 -0
  27. PYMEcs/Analysis/fitDarkTimes.py +569 -0
  28. PYMEcs/Analysis/objectVolumes.py +20 -0
  29. PYMEcs/Analysis/offlineTracker.py +130 -0
  30. PYMEcs/Analysis/stackTracker.py +180 -0
  31. PYMEcs/Analysis/timeSeries.py +63 -0
  32. PYMEcs/Analysis/trackFiducials.py +186 -0
  33. PYMEcs/Analysis/zerocross.py +91 -0
  34. PYMEcs/IO/MINFLUX.py +851 -0
  35. PYMEcs/IO/NPC.py +117 -0
  36. PYMEcs/IO/__init__.py +0 -0
  37. PYMEcs/IO/darkTimes.py +19 -0
  38. PYMEcs/IO/picasso.py +219 -0
  39. PYMEcs/IO/tabular.py +11 -0
  40. PYMEcs/__init__.py +0 -0
  41. PYMEcs/experimental/CalcZfactor.py +51 -0
  42. PYMEcs/experimental/FRC.py +338 -0
  43. PYMEcs/experimental/ImageJROItools.py +49 -0
  44. PYMEcs/experimental/MINFLUX.py +1537 -0
  45. PYMEcs/experimental/NPCcalcLM.py +560 -0
  46. PYMEcs/experimental/Simpler.py +369 -0
  47. PYMEcs/experimental/Sofi.py +78 -0
  48. PYMEcs/experimental/__init__.py +0 -0
  49. PYMEcs/experimental/binEventProperty.py +187 -0
  50. PYMEcs/experimental/chaining.py +23 -0
  51. PYMEcs/experimental/clusterTrack.py +179 -0
  52. PYMEcs/experimental/combine_maps.py +104 -0
  53. PYMEcs/experimental/eventProcessing.py +93 -0
  54. PYMEcs/experimental/fiducials.py +323 -0
  55. PYMEcs/experimental/fiducialsNew.py +402 -0
  56. PYMEcs/experimental/mapTools.py +271 -0
  57. PYMEcs/experimental/meas2DplotDh5view.py +107 -0
  58. PYMEcs/experimental/mortensen.py +131 -0
  59. PYMEcs/experimental/ncsDenoise.py +158 -0
  60. PYMEcs/experimental/onTimes.py +295 -0
  61. PYMEcs/experimental/procPoints.py +77 -0
  62. PYMEcs/experimental/pyme2caml.py +73 -0
  63. PYMEcs/experimental/qPAINT.py +965 -0
  64. PYMEcs/experimental/randMap.py +188 -0
  65. PYMEcs/experimental/regExtraCmaps.py +11 -0
  66. PYMEcs/experimental/selectROIfilterTable.py +72 -0
  67. PYMEcs/experimental/showErrs.py +51 -0
  68. PYMEcs/experimental/showErrsDh5view.py +58 -0
  69. PYMEcs/experimental/showShiftMap.py +56 -0
  70. PYMEcs/experimental/snrEvents.py +188 -0
  71. PYMEcs/experimental/specLabeling.py +51 -0
  72. PYMEcs/experimental/splitRender.py +246 -0
  73. PYMEcs/experimental/testChannelByName.py +36 -0
  74. PYMEcs/experimental/timedSpecies.py +28 -0
  75. PYMEcs/experimental/utils.py +31 -0
  76. PYMEcs/misc/ExtraCmaps.py +177 -0
  77. PYMEcs/misc/__init__.py +0 -0
  78. PYMEcs/misc/configUtils.py +169 -0
  79. PYMEcs/misc/guiMsgBoxes.py +27 -0
  80. PYMEcs/misc/mapUtils.py +230 -0
  81. PYMEcs/misc/matplotlib.py +136 -0
  82. PYMEcs/misc/rectsFromSVG.py +182 -0
  83. PYMEcs/misc/shellutils.py +1110 -0
  84. PYMEcs/misc/utils.py +205 -0
  85. PYMEcs/misc/versionCheck.py +20 -0
  86. PYMEcs/misc/zcInfo.py +90 -0
  87. PYMEcs/pyme_warnings.py +4 -0
  88. PYMEcs/recipes/__init__.py +0 -0
  89. PYMEcs/recipes/base.py +75 -0
  90. PYMEcs/recipes/localisations.py +2380 -0
  91. PYMEcs/recipes/manipulate_yaml.py +83 -0
  92. PYMEcs/recipes/output.py +177 -0
  93. PYMEcs/recipes/processing.py +247 -0
  94. PYMEcs/recipes/simpler.py +290 -0
  95. PYMEcs/version.py +2 -0
  96. pyme_extra-1.0.4.post0.dist-info/METADATA +114 -0
  97. pyme_extra-1.0.4.post0.dist-info/RECORD +101 -0
  98. pyme_extra-1.0.4.post0.dist-info/WHEEL +5 -0
  99. pyme_extra-1.0.4.post0.dist-info/entry_points.txt +3 -0
  100. pyme_extra-1.0.4.post0.dist-info/licenses/LICENSE +674 -0
  101. pyme_extra-1.0.4.post0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1537 @@
1
+ import matplotlib.pyplot as plt
2
+ import numpy as np
3
+ import wx
4
+
5
+ import logging
6
+ logger = logging.getLogger(__file__)
7
+
8
+ from PYMEcs.pyme_warnings import warn
9
+ import PYMEcs.misc.utils as mu
10
+
11
+ def plot_errors(pipeline,ds='coalesced_nz',dsclumps='with_clumps'):
12
+ if not ds in pipeline.dataSources:
13
+ warn('no data source named "%s" - check recipe and ensure this is MINFLUX data' % ds)
14
+ return
15
+ curds = pipeline.selectedDataSourceKey
16
+ pipeline.selectDataSource(ds)
17
+ p = pipeline
18
+ clumpSize = p['clumpSize']
19
+ plt.figure()
20
+ plt.subplot(221)
21
+ if 'error_z' in pipeline.keys():
22
+ plt.boxplot([p['error_x'],p['error_y'],p['error_z']],labels=['error_x','error_y','error_z'])
23
+ else:
24
+ plt.boxplot([p['error_x'],p['error_y']],labels=['error_x','error_y'])
25
+ plt.ylabel('loc error - coalesced (nm)')
26
+ pipeline.selectDataSource(dsclumps)
27
+ plt.subplot(222)
28
+ if 'nPhotons' in p.keys() and 'fbg' in p.keys():
29
+ bp_dict = plt.boxplot([p['nPhotons'],p['fbg']],labels=['photons','background rate'])
30
+ for line in bp_dict['medians']:
31
+ # get position data for median line
32
+ x, y = line.get_xydata()[0] # top of median line
33
+ # overlay median value
34
+ plt.text(x, y, '%.0f' % y,
35
+ horizontalalignment='right') # draw above, centered
36
+
37
+ uids, idx = np.unique(p['clumpIndex'],return_index=True)
38
+ plt.subplot(223)
39
+ if 'error_z' in pipeline.keys():
40
+ plt.boxplot([p['error_x'][idx],p['error_y'][idx],p['error_z'][idx]],
41
+ labels=['error_x','error_y','error_z'])
42
+ else:
43
+ plt.boxplot([p['error_x'][idx],p['error_y'][idx]],labels=['error_x','error_y'])
44
+ plt.ylabel('loc error - raw (nm)')
45
+ plt.subplot(224)
46
+ bp_dict = plt.boxplot([clumpSize],labels=['clump size'])
47
+ for line in bp_dict['medians']:
48
+ # get position data for median line
49
+ x, y = line.get_xydata()[0] # top of median line
50
+ # overlay median value
51
+ plt.text(x, y, '%.0f' % y,
52
+ horizontalalignment='right') # draw above, centered
53
+ plt.tight_layout()
54
+ if mu.autosave_check():
55
+ fpath = mu.get_ds_path(p)
56
+ plt.savefig(mu.fname_from_timestamp(fpath,p.mdh,'_locError',ext='.png'),
57
+ dpi=300, bbox_inches='tight')
58
+ pipeline.selectDataSource(ds)
59
+ df = pd.DataFrame({
60
+ "Metric": ["Photons", "Background", "Clump Size", "Error X", "Error Y"],
61
+ "Median": [np.median(p[key]) for key in ['nPhotons','fbg','clumpSize','error_x','error_y']],
62
+ "Unit": ["","","","nm","nm",]
63
+ })
64
+ if p.mdh['MINFLUX.Is3D']: # any code needs to check for 2D vs 3D
65
+ df.loc[df.index.max() + 1] = ["Error Z",np.median(p['error_z']),'nm']
66
+ mu.autosave_csv(df,fpath,p.mdh,'_locError')
67
+
68
+ pipeline.selectDataSource(curds)
69
+
70
+ from PYMEcs.misc.matplotlib import boxswarmplot
71
+ import pandas as pd
72
+ def _plot_clustersize_counts(cts, ctsgt1, xlabel='Cluster Size', wintitle=None, bigCfraction=None,bigcf_percluster=None, plotints=True, **kwargs):
73
+ if 'range' in kwargs:
74
+ enforce_xlims = True
75
+ xlims0=kwargs['range']
76
+ extent = xlims0[1] - xlims0[0]
77
+ frac = 0.05
78
+ xlims = [xlims0[0] - frac*extent, xlims0[1] + frac*extent]
79
+ else:
80
+ enforce_xlims = False
81
+ fig = plt.figure()
82
+ if (plotints):
83
+ plotn=300
84
+ else:
85
+ plotn=200
86
+ plt.subplot(plotn+21)
87
+ h = plt.hist(cts,**kwargs,log=True)
88
+ plt.xlabel(xlabel)
89
+ plt.plot([np.mean(cts),np.mean(cts)],[0,h[0].max()])
90
+ plt.plot([np.median(cts),np.median(cts)],[0,h[0].max()],'--')
91
+ if enforce_xlims:
92
+ plt.xlim(*xlims)
93
+ plt.subplot(plotn+22)
94
+ h = plt.hist(ctsgt1,**kwargs,log=True)
95
+ plt.xlabel('%s ( > 1)' % xlabel)
96
+ plt.plot([np.mean(ctsgt1),np.mean(ctsgt1)],[0,h[0].max()])
97
+ plt.plot([np.median(ctsgt1),np.median(ctsgt1)],[0,h[0].max()],'--')
98
+ if enforce_xlims:
99
+ plt.xlim(*xlims)
100
+ plt.subplot(plotn+23)
101
+ dfcs = pd.DataFrame.from_dict(dict(SUclusterSize=cts))
102
+ boxswarmplot(dfcs,format="%.1f",swarmsize=5,width=0.2,annotate_means=True,annotate_medians=True,swarmalpha=0.15,strip=True)
103
+ plt.subplot(plotn+24)
104
+ dfcsgt1 = pd.DataFrame.from_dict(dict(SUclusterSizeGT1=ctsgt1))
105
+ boxswarmplot(dfcsgt1,format="%.1f",swarmsize=5,width=0.2,annotate_means=True,annotate_medians=True,swarmalpha=0.15,strip=True)
106
+ if (plotints):
107
+ plt.subplot(plotn+25)
108
+ dfcs = pd.DataFrame.from_dict(dict(RyRclusterSizeInts=np.ceil(0.25*cts)))
109
+ boxswarmplot(dfcs,format="%.1f",swarmsize=5,width=0.2,annotate_means=True,annotate_medians=True,swarmalpha=0.15,strip=True)
110
+ plt.subplot(plotn+26)
111
+ dfcsgt1 = pd.DataFrame.from_dict(dict(RyRclusterSizeIntsGT1=np.ceil(0.25*ctsgt1)))
112
+ boxswarmplot(dfcsgt1,format="%.1f",swarmsize=5,width=0.2,annotate_means=True,annotate_medians=True,swarmalpha=0.15,strip=True)
113
+
114
+ largest = cts[np.argsort(cts)][-3:]
115
+ fraction = largest.sum(dtype=float) / cts.sum()
116
+ msg = "3 largest Cs (%s) make up %.1f %% of SUs" %(largest,100.0*fraction)
117
+ fig.suptitle(msg)
118
+
119
+ # bp_dict = plt.boxplot([cts,ctsgt1],labels=['cluster size','clusters > 1'], showmeans=True)
120
+ # for line in bp_dict['means']:
121
+ # # get position data for median line
122
+ # x, y = line.get_xydata()[0] # top of median line
123
+ # # overlay median value
124
+ # plt.text(x-0.25, y, '%.1f' % y,
125
+ # horizontalalignment='center') # draw above, centered
126
+ plt.tight_layout()
127
+
128
+ if wintitle is not None:
129
+ figtitle = "%s" % wintitle
130
+ else:
131
+ figtitle = ''
132
+ if bigCfraction is not None:
133
+ figtitle = figtitle + " bigC fraction %.1f %%" % (100*bigCfraction)
134
+ if bigcf_percluster is not None:
135
+ figtitle = figtitle + " per cluster %.1f %%" % (100*bigcf_percluster)
136
+ else:
137
+ if bigcf_percluster is not None:
138
+ figtitle = figtitle + " bigC frac per cluster %.1f %%" % (100*bigcf_percluster)
139
+
140
+ fig.canvas.manager.set_window_title(figtitle)
141
+
142
+ def plot_cluster_analysis(pipeline, ds='dbscanClustered',showPlot=True, return_means=False,
143
+ return_data=False, psu=None, bins=15, bigc_thresh=50, **kwargs):
144
+ if not ds in pipeline.dataSources:
145
+ warn('no data source named "%s" - check recipe and ensure this is MINFLUX data' % ds)
146
+ return
147
+ curds = pipeline.selectedDataSourceKey
148
+ pipeline.selectDataSource(ds)
149
+ p = pipeline
150
+ uids, idx, cts = np.unique(p['dbscanClumpID'], return_index=True, return_counts=True)
151
+ nall = p['x'].size
152
+ if 'bigCs' in p.dataSources:
153
+ fraction = p.dataSources['bigCs']['x'].size/float(nall)
154
+ else:
155
+ fraction=None
156
+ clustersizes = p['dbscanClumpSize'][idx]
157
+ bigc_fracpercluster = float(np.sum(clustersizes>bigc_thresh))/clustersizes.size
158
+ ctsgt1 = cts[cts > 1.1]
159
+ pipeline.selectDataSource(curds)
160
+ timestamp = pipeline.mdh.get('MINFLUX.TimeStamp')
161
+ if showPlot:
162
+ if psu is not None:
163
+ _plot_clustersize_counts(cts, ctsgt1,bins=bins,xlabel='# subunits',wintitle=timestamp,bigCfraction=fraction,bigcf_percluster=bigc_fracpercluster,**kwargs)
164
+ else:
165
+ _plot_clustersize_counts(cts, ctsgt1,bins=bins,wintitle=timestamp,bigCfraction=fraction,bigcf_percluster=bigc_fracpercluster,**kwargs)
166
+ if psu is not None:
167
+ _plot_clustersize_counts(cts/4.0/psu, ctsgt1/4.0/psu, xlabel='# RyRs, corrected', bins=bins,wintitle=timestamp,
168
+ bigCfraction=fraction,bigcf_percluster=bigc_fracpercluster,**kwargs)
169
+
170
+ csm = cts.mean()
171
+ csgt1m = ctsgt1.mean()
172
+ csmd = np.median(cts)
173
+ csgt1md = np.median(ctsgt1)
174
+
175
+ print("Mean cluster size: %.2f" % csm)
176
+ print("Mean cluster size > 1: %.2f" % csgt1m)
177
+ print("Median cluster size: %.2f" % csmd)
178
+ print("Median cluster size > 1: %.2f" % csgt1md)
179
+
180
+ if return_means:
181
+ return (csm,csgt1m)
182
+
183
+ if return_data:
184
+ return (cts,ctsgt1)
185
+
186
+ def cluster_analysis(pipeline):
187
+ return plot_cluster_analysis(pipeline, ds='dbscanClustered',showPlot=False,return_means=True)
188
+
189
+ def plot_intra_clusters_dists(pipeline, ds='dbscanClustered',bins=15,NNs=1,**kwargs):
190
+ if not ds in pipeline.dataSources:
191
+ warn('no data source named "%s" - check recipe and ensure this is MINFLUX data' % ds)
192
+ return
193
+ from scipy.spatial import KDTree
194
+ curds = pipeline.selectedDataSourceKey
195
+ pipeline.selectDataSource(ds)
196
+ p = pipeline
197
+ uids, cts = np.unique(p['dbscanClumpID'], return_counts=True)
198
+ checkids = uids[cts > 5.0]
199
+ dists = []
200
+ for cid in checkids:
201
+ coords = np.vstack([p[k][p['dbscanClumpID'] == cid] for k in ['x','y','z']]).T
202
+ tree = KDTree(coords)
203
+ dd, ii = tree.query(coords,k=NNs+1)
204
+ dists.extend(list(dd[:,1:].flatten()))
205
+ pipeline.selectDataSource(curds)
206
+ plt.figure()
207
+ h=plt.hist(dists,bins=bins,**kwargs)
208
+
209
+ def cornercounts(pipeline,backgroundFraction=0.0):
210
+ curds = pipeline.selectedDataSourceKey
211
+ allds = 'withNNdist'
212
+ ds2 = 'group2'
213
+ ds3 = 'group3'
214
+ ds4 = 'group4'
215
+ p = pipeline
216
+ pipeline.selectDataSource(allds)
217
+ n_all = p['x'].size
218
+ pipeline.selectDataSource(ds2)
219
+ n_2 = p['x'].size
220
+ pipeline.selectDataSource(ds3)
221
+ n_3 = p['x'].size
222
+ pipeline.selectDataSource(ds4)
223
+ n_4 = p['x'].size
224
+ pipeline.selectDataSource(curds)
225
+ # note: we count RyRs, not corners
226
+ # double check these ideas
227
+ ccs = np.array([(1.0-backgroundFraction)*(n_all-(n_2+n_3+n_4)), n_2/2.0, n_3/3.0, n_4/4.0])
228
+ return ccs
229
+
230
+ def print_basiccornerstats(pipeline, ds='filtered_localizations'):
231
+ curds = pipeline.selectedDataSourceKey
232
+ pipeline.selectDataSource(ds)
233
+ data = pipeline
234
+ dx_um = (data['x'].max() - data['x'].min())/1e3
235
+ dy_um = (data['y'].max() - data['y'].min())/1e3
236
+ area_um2 = dx_um * dy_um
237
+ print("x extent: %.2f um, y extent: %.2f um, area: %.1f um^2" % (dx_um,dy_um,area_um2))
238
+ pipeline.selectDataSource('coalesced_nz')
239
+ from scipy.stats import iqr
240
+ z_iqr_nm = iqr(data['z'])
241
+ z_fwhm_nm = z_iqr_nm * 2.35/(2*0.675)
242
+ z_full_nm = 4.0 * z_iqr_nm
243
+ print("z extent (IQR): %.1f nm, (FWHM): %.1f nm, (Full extent): %.1f nm" % (z_iqr_nm, z_fwhm_nm, z_full_nm))
244
+ n1 = data['x'].size
245
+ pipeline.selectDataSource('closemerged')
246
+ n2 = data['x'].size
247
+ print("number corners: %d, (%d closemerged)" % (n1,n2))
248
+ print("cornerdensity %.1f corners/um^2" % (n1/area_um2))
249
+ print("cornerdensity %.1f corners/um^2 (closemerged)" % (n2/area_um2))
250
+ z_fwhm_ratio = z_fwhm_nm / 100.0
251
+ print("corner volume density: %.1f corners/um^2/100nm" % (n1/area_um2/z_fwhm_ratio))
252
+ print("corner volume density: %.1f corners/um^2/100nm (closemerged)" % (n2/area_um2/z_fwhm_ratio))
253
+ pipeline.selectDataSource(curds)
254
+
255
+ def plot_zextent(pipeline, ds='closemerged', series_name='This series'):
256
+
257
+ def set_axis_style(ax, labels):
258
+ ax.xaxis.set_tick_params(direction='out')
259
+ ax.xaxis.set_ticks_position('bottom')
260
+ ax.set_xticks(np.arange(1, len(labels) + 1), labels=labels)
261
+ ax.set_xlim(0.25, len(labels) + 0.75)
262
+ ax.set_xlabel('Sample name')
263
+
264
+ fig, ax2 = plt.subplots(1, 1, figsize=(9, 4))
265
+
266
+ curds = pipeline.selectedDataSourceKey
267
+ pipeline.selectDataSource(ds)
268
+ zdata = pipeline['z']
269
+ pipeline.selectDataSource(curds)
270
+ q005,quartile1, medians, quartile3 = np.percentile(zdata, [0.5,25, 50, 75])
271
+ zdatac = zdata - q005
272
+
273
+ ax2.set_title('z - axis value distribution')
274
+ parts = ax2.violinplot(
275
+ zdatac, showmeans=True, showmedians=False,
276
+ showextrema=False, quantiles = [0.25,0.75])
277
+
278
+ for pc in parts['bodies']:
279
+ pc.set_facecolor('#D43F3A')
280
+ pc.set_edgecolor('black')
281
+ pc.set_alpha(1)
282
+
283
+ # set style for the axes
284
+ labels = [series_name]
285
+ set_axis_style(ax2, labels)
286
+
287
+ plt.subplots_adjust(bottom=0.15, wspace=0.05)
288
+
289
+
290
+ from scipy.special import binom
291
+ from scipy.optimize import curve_fit
292
+
293
+ def sigpn(p):
294
+ return pn(1,p)+pn(2,p)+pn(3,p)+pn(4,p)
295
+
296
+ def sigptot(p):
297
+ return pn(0,p) + sigpn(p)
298
+
299
+ def pn(k,p):
300
+ return (binom(4,k)*(np.power(p,k)*np.power((1-p),(4-k))))
301
+
302
+ def pnn(k,p):
303
+ return (pn(k,p)/(1-pn(0,p)))
304
+
305
+ def fourcornerplot(pipeline,sigma=None,backgroundFraction=0.0,showplot=True,quiet=False):
306
+ ccs = cornercounts(pipeline,backgroundFraction=backgroundFraction)
307
+ ccsn = ccs/ccs.sum()
308
+ ks = np.arange(4)+1
309
+ popt, pcov = curve_fit(pnn, ks, ccsn,sigma=sigma)
310
+ perr = np.sqrt(np.diag(pcov))
311
+ p_missed = pn(0,popt[0])
312
+ p_m_min = pn(0,popt[0]+perr[0])
313
+ p_m_max = pn(0,popt[0]-perr[0])
314
+ if showplot:
315
+ plt.figure()
316
+ ax = plt.subplot(111)
317
+ ax.bar(ks-0.4, ccsn, width=0.4, color='b', align='center')
318
+ ax.bar(ks, pnn(ks,popt[0]), width=0.4, color='g', align='center')
319
+ ax.legend(['Experimental data', 'Fit'])
320
+ plt.title("Best fit p_lab=%.3f +- %.3f" % (popt[0],perr[0]))
321
+ if not quiet:
322
+ print('optimal p: %.3f +- %.3f' % (popt[0],perr[0]))
323
+ print('missed fraction: %.2f (%.2f...%.2f)' % (p_missed,p_m_min,p_m_max))
324
+ return (popt[0],perr[0],pnn(ks,popt[0]),ccsn)
325
+
326
+
327
+ sigmaDefault = [0.15,0.05,0.03,0.03]
328
+ backgroundDefault = 0.15
329
+
330
+ def fourcornerplot_default(pipeline,sigma=sigmaDefault,backgroundFraction=backgroundDefault,showplot=True,quiet=False):
331
+ return fourcornerplot(pipeline,sigma=sigma,backgroundFraction=backgroundFraction,showplot=showplot,quiet=False)
332
+
333
+ def subunitfit(pipeline):
334
+ return fourcornerplot_default(pipeline,showplot=False)
335
+
336
+ def plot_tracking(pipeline,is_coalesced=False,lowess_fraction=0.05):
337
+ p = pipeline
338
+ has_z = 'z_nc' in pipeline.keys()
339
+ if has_z:
340
+ nrows = 3
341
+ else:
342
+ nrows = 2
343
+
344
+ t_s = 1e-3*p['t']
345
+ xmbm = -(p['x']-p['x_nc']) # we flip the sign from now on
346
+ ymbm = -(p['y']-p['y_nc'])
347
+ if has_z:
348
+ zmbm = -(p['z']-p['z_nc'])
349
+
350
+ if is_coalesced:
351
+ from statsmodels.nonparametric.smoothers_lowess import lowess
352
+ xmbms = lowess(xmbm, t_s, frac=lowess_fraction, return_sorted=False)
353
+ ymbms = lowess(ymbm, t_s, frac=lowess_fraction, return_sorted=False)
354
+ if has_z:
355
+ zmbms = lowess(zmbm, t_s, frac=lowess_fraction, return_sorted=False)
356
+
357
+ plt.figure(num='beamline monitoring corrections')
358
+ plt.subplot(nrows,1,1)
359
+ plt.plot(t_s,xmbm)
360
+ if is_coalesced:
361
+ plt.plot(t_s,xmbms)
362
+ plt.xlabel('Time (s)')
363
+ plt.ylabel('x-difference (nm)')
364
+ plt.subplot(nrows,1,2)
365
+ plt.plot(t_s,ymbm)
366
+ if is_coalesced:
367
+ plt.plot(t_s,ymbms)
368
+ plt.xlabel('Time (s)')
369
+ plt.ylabel('y-difference (nm)')
370
+ if 'z_nc' in pipeline.keys():
371
+ plt.subplot(nrows,1,3)
372
+ plt.plot(t_s,zmbm)
373
+ if is_coalesced:
374
+ plt.plot(t_s,zmbms)
375
+ plt.xlabel('Time (s)')
376
+ plt.ylabel('z-difference (nm)')
377
+ plt.tight_layout()
378
+
379
+ if not is_coalesced:
380
+ return # skip HF plot
381
+
382
+ plt.figure(num='MBM corrections HF component')
383
+ plt.subplot(nrows,1,1)
384
+ plt.plot(t_s,xmbm-xmbms)
385
+ plt.xlabel('Time (s)')
386
+ plt.ylabel('x-difference (nm)')
387
+ plt.grid(axis='y')
388
+ plt.subplot(nrows,1,2)
389
+ plt.plot(t_s,ymbm-ymbms)
390
+ plt.grid(axis='y')
391
+ plt.xlabel('Time (s)')
392
+ plt.ylabel('y-difference (nm)')
393
+ if 'z_nc' in pipeline.keys():
394
+ plt.subplot(nrows,1,3)
395
+ plt.plot(t_s,zmbm-zmbms)
396
+ plt.grid(axis='y')
397
+ plt.xlabel('Time (s)')
398
+ plt.ylabel('z-difference (nm)')
399
+ plt.tight_layout()
400
+
401
+
402
+ def plot_site_tracking(pipeline,fignum=None,plotSmoothingCurve=True):
403
+ p=pipeline
404
+ t_s = 1e-3*p['t']
405
+ if fignum is not None:
406
+ fig, axs = plt.subplots(2, 2,num='origami site tracks %d' % fignum)
407
+ else:
408
+ fig, axs = plt.subplots(2, 2)
409
+
410
+ axs[0, 0].scatter(t_s,p['x_site_nc'],s=0.3,c='black',alpha=0.5)
411
+ if plotSmoothingCurve:
412
+ axs[0, 0].plot(t_s,p['x_ori']-p['x'],'r',alpha=0.4)
413
+ axs[0, 0].set_ylim(-15,15)
414
+ axs[0, 0].set_xlabel('t (s)')
415
+ axs[0, 0].set_ylabel('x (nm)')
416
+
417
+ axs[0, 1].scatter(t_s,p['y_site_nc'],s=0.3,c='black',alpha=0.5)
418
+ if plotSmoothingCurve:
419
+ axs[0, 1].plot(t_s,p['y_ori']-p['y'],'r',alpha=0.4)
420
+ axs[0, 1].set_ylim(-15,15)
421
+ axs[0, 1].set_xlabel('t (s)')
422
+ axs[0, 1].set_ylabel('y (nm)')
423
+
424
+ if p.mdh.get('MINFLUX.Is3D',False):
425
+ axs[1, 0].scatter(t_s,p['z_site_nc'],s=0.3,c='black',alpha=0.5)
426
+ if plotSmoothingCurve:
427
+ axs[1, 0].plot(t_s,p['z_ori']-p['z'],'r',alpha=0.4)
428
+ axs[1, 0].set_ylim(-15,15)
429
+ axs[1, 0].set_xlabel('t (s)')
430
+ axs[1, 0].set_ylabel('z (nm)')
431
+
432
+ ax = axs[1,1]
433
+ if plotSmoothingCurve and 'x_nc' in p.keys():
434
+ # plot the MBM track
435
+ ax.plot(t_s,-(p['x_ori']-p['x_nc']),alpha=0.5,label='x')
436
+ plt.plot(t_s,-(p['y_ori']-p['y_nc']),alpha=0.5,label='y')
437
+ if p.mdh.get('MINFLUX.Is3D',False) and 'z_nc' in p.keys():
438
+ ax.plot(t_s,-(p['z_ori']-p['z_nc']),alpha=0.5,label='z')
439
+ ax.set_xlabel('t (s)')
440
+ ax.set_ylabel('MBM corr (nm)')
441
+ ax.legend()
442
+ else:
443
+ axs[1, 1].plot(t_s,p['x_ori']-p['x'])
444
+ axs[1, 1].plot(t_s,p['y_ori']-p['y'])
445
+ if p.mdh.get('MINFLUX.Is3D',False):
446
+ axs[1, 1].plot(t_s,p['z_ori']-p['z'])
447
+ axs[1, 1].set_xlabel('t [s]')
448
+ axs[1, 1].set_ylabel('orig. corr (nm)')
449
+ plt.tight_layout()
450
+
451
+ from PYMEcs.Analysis.MINFLUX import analyse_locrate
452
+ from PYMEcs.misc.guiMsgBoxes import Error
453
+ from PYMEcs.misc.utils import unique_name
454
+ from PYMEcs.IO.MINFLUX import findmbm
455
+
456
+ from PYME.recipes.traits import HasTraits, Float, Enum, CStr, Bool, Int, List
457
+ import PYME.config
458
+
459
+ class MINFLUXSettings(HasTraits):
460
+ withOrigamiSmoothingCurves = Bool(True,label='Plot smoothing curves',desc="if overplotting smoothing curves " +
461
+ "in origami site correction analysis")
462
+ defaultDatasourceForAnalysis = CStr('Localizations',label='default datasource for analysis',
463
+ desc="the datasource key that will be used by default in the MINFLUX " +
464
+ "properties functions (EFO, localisation rate, etc)") # default datasource for acquisition analysis
465
+ defaultDatasourceCoalesced = CStr('coalesced_nz',label='default datasource for coalesced analysis',
466
+ desc="the datasource key that will be used by default when a " +
467
+ "coalesced data source is required")
468
+ defaultDatasourceWithClumps = CStr('with_clumps',label='default datasource for clump analysis',
469
+ desc="the datasource key that will be used by default when a " +
470
+ "data source with clump info is required")
471
+ defaultDatasourceForMBM = CStr('coalesced_nz',label='default datasource for MBM analysis and plotting',
472
+ desc="the datasource key that will be used by default in the MINFLUX " +
473
+ "MBM analysis") # default datasource for MBM analysis
474
+ datasourceForClusterAnalysis = CStr(PYME.config.get('MINFLUX-clusterDS','dbscan_clustered'),label='datasource for 3D cluster analysis',
475
+ desc="the datasource key that will be used to generate the 3D cluster size analysis")
476
+
477
+ largeClusterThreshold = Float(50,label='Threshold for large clusters',
478
+ desc='minimum number of events to classify as large cluster')
479
+ clustercountsPlotWithInts = Bool(False,label='Include "integer" plots in cluster stats',
480
+ desc='plot integer quantized cluster stats that avoid counting fractional RyR numbers')
481
+ origamiWith_nc = Bool(False,label='add 2nd moduleset (no MBM corr)',
482
+ desc="if a full second module set is inserted to also analyse the origami data without any MBM corrections")
483
+ origamiErrorLimit = Float(10.0,label='xLimit when plotting origami errors',
484
+ desc="sets the upper limit in x (in nm) when plotting origami site errors")
485
+
486
+
487
+ class MINFLUXSiteSettings(HasTraits):
488
+ showPoints = Bool(True)
489
+ plotMode = Enum(['box','violin'])
490
+ pointsMode = Enum(['swarm','strip'])
491
+ siteMaxNum = Int(100,label='Max number of sites for box plot',
492
+ desc="the maximum number of sites for which site stats boxswarmplot is generated, violinplot otherwise")
493
+ precisionRange_nm = Float(10)
494
+
495
+ class DateString(HasTraits):
496
+ TimeStampString = CStr('',label="Time stamp",desc='the time stamp string in format yymmdd-HHMMSS')
497
+
498
+
499
+ class MBMaxisSelection(HasTraits):
500
+ SelectAxis = Enum(['x-y-z','x','y','z','std_x','std_y','std_z'])
501
+
502
+ class MINFLUXplottingDefaults(HasTraits):
503
+ FontSize = Float(12)
504
+ LineWidth = Float(1.5)
505
+
506
+ class MINFLUXanalyser():
507
+ def __init__(self, visFr):
508
+ self.visFr = visFr
509
+ self.minfluxRIDs = {}
510
+ self.origamiErrorFignum = 0
511
+ self.origamiTrackFignum = 0
512
+ self.analysisSettings = MINFLUXSettings()
513
+ self.dstring = DateString()
514
+ self.mbmAxisSelection = MBMaxisSelection()
515
+ self.plottingDefaults = MINFLUXplottingDefaults()
516
+ self.siteSettings = MINFLUXSiteSettings()
517
+
518
+ visFr.AddMenuItem('MINFLUX', "Localisation Error analysis", self.OnErrorAnalysis)
519
+ visFr.AddMenuItem('MINFLUX', "Cluster sizes - 3D", self.OnCluster3D)
520
+ visFr.AddMenuItem('MINFLUX', "Cluster sizes - 2D", self.OnCluster2D)
521
+ visFr.AddMenuItem('MINFLUX', "Analyse Localization Rate", self.OnLocalisationRate)
522
+ visFr.AddMenuItem('MINFLUX', "EFO histogram (photon rates)", self.OnEfoAnalysis)
523
+ visFr.AddMenuItem('MINFLUX', "plot tracking correction (if available)", self.OnTrackPlot)
524
+ visFr.AddMenuItem('MINFLUX', "Analysis settings", self.OnMINFLUXSettings)
525
+ visFr.AddMenuItem('MINFLUX', "Toggle MINFLUX analysis autosaving", self.OnToggleMINFLUXautosave)
526
+ visFr.AddMenuItem('MINFLUX', "Manually create Colour panel", self.OnMINFLUXColour)
527
+
528
+ visFr.AddMenuItem('MINFLUX>Origami', "group and analyse origami sites", self.OnOrigamiSiteRecipe)
529
+ visFr.AddMenuItem('MINFLUX>Origami', "plot origami site correction", self.OnOrigamiSiteTrackPlot)
530
+ visFr.AddMenuItem('MINFLUX>Origami', "plot origami error estimates", self.OnOrigamiErrorPlot)
531
+ visFr.AddMenuItem('MINFLUX>Origami', "plot origami site stats", self.OnOrigamiSiteStats)
532
+ visFr.AddMenuItem('MINFLUX>Origami', "add final filter for site-based corrected data", self.OnOrigamiFinalFilter)
533
+ visFr.AddMenuItem('MINFLUX>Origami', "site settings", self.OnMINFLUXSiteSettings)
534
+
535
+ visFr.AddMenuItem('MINFLUX>Util', "Plot temperature record matching current data series",self.OnMINFLUXplotTemperatureData)
536
+ visFr.AddMenuItem('MINFLUX>Util', "Set MINFLUX temperature folder location", self.OnMINFLUXsetTempDataFolder)
537
+ visFr.AddMenuItem('MINFLUX>Util', "Check if clumpIndex contiguous", self.OnClumpIndexContig)
538
+ visFr.AddMenuItem('MINFLUX>Util', "Plot event scatter as function of position in clump", self.OnClumpScatterPosPlot)
539
+ visFr.AddMenuItem('MINFLUX>Util', "Set plotting defaults (inc font size)", self.OnSetMINFLUXPlottingdefaults)
540
+ visFr.AddMenuItem('MINFLUX>Util', "Estimate region size (with output filter)", self.OnEstimateMINFLUXRegionSize)
541
+
542
+ visFr.AddMenuItem('MINFLUX>MBM', "Plot mean MBM info (and if present origami info)", self.OnMBMplot)
543
+ visFr.AddMenuItem('MINFLUX>MBM', "Show MBM tracks", self.OnMBMtracks)
544
+ visFr.AddMenuItem('MINFLUX>MBM', "Add MBM track labels to view", self.OnMBMaddTrackLabels)
545
+ visFr.AddMenuItem('MINFLUX>MBM', "Save MBM bead trajectories to npz file", self.OnMBMSave)
546
+ visFr.AddMenuItem('MINFLUX>MBM', "Save MBM bead settings to json file", self.OnMBMSettingsSave)
547
+ visFr.AddMenuItem('MINFLUX>MBM', "Save MBM lowess cache", self.OnMBMLowessCacheSave)
548
+
549
+ visFr.AddMenuItem('MINFLUX>RyRs', "Plot corner info", self.OnCornerplot)
550
+ visFr.AddMenuItem('MINFLUX>RyRs', "Plot density stats", self.OnDensityStats)
551
+ visFr.AddMenuItem('MINFLUX>RyRs', "Show cluster alpha shapes", self.OnAlphaShapes)
552
+
553
+ visFr.AddMenuItem('MINFLUX>Zarr', "Show MBM attributes", self.OnMBMAttributes)
554
+ visFr.AddMenuItem('MINFLUX>Zarr', "Show MFX attributes", self.OnMFXAttributes)
555
+ visFr.AddMenuItem('MINFLUX>Zarr', "Show MFX metadata info (now in PYME metadata)", self.OnMFXInfo)
556
+ visFr.AddMenuItem('MINFLUX>Zarr', "Convert zarr file store to zarr zip store", self.OnZarrToZipStore)
557
+ visFr.AddMenuItem('MINFLUX>Zarr', "Run Paraflux Analysis", self.OnRunParafluxAnalysis)
558
+
559
+ visFr.AddMenuItem('MINFLUX>Tracking', "Add traces as tracks (from clumpIndex)", self.OnAddMINFLUXTracksCI)
560
+ visFr.AddMenuItem('MINFLUX>Tracking', "Add traces as tracks (from tid)", self.OnAddMINFLUXTracksTid)
561
+ visFr.AddMenuItem('MINFLUX>Colour', "Plot colour stats", self.OnPlotColourStats)
562
+
563
+ # this section establishes Menu entries for loading MINFLUX recipes in one click
564
+ # these recipes should be MINFLUX processing recipes of general interest
565
+ # and are populated from the customrecipes folder in the PYME config directories
566
+ # code adapted from PYME.DSView.modules.recipes
567
+ import PYME.config
568
+ customRecipes = PYME.config.get_custom_recipes()
569
+ minfluxRecipes = dict((k, v) for k, v in customRecipes.items() if k.startswith('MINFLUX'))
570
+ if len(minfluxRecipes) > 0:
571
+ for r in minfluxRecipes:
572
+ ID = visFr.AddMenuItem('MINFLUX>Recipes', r, self.OnLoadCustom).GetId()
573
+ self.minfluxRIDs[ID] = minfluxRecipes[r]
574
+
575
+ def OnEstimateMINFLUXRegionSize(self, event):
576
+ p = self.visFr.pipeline # Get the pipeline from the GUI
577
+ xsize = p['x'].max() - p['x'].min()
578
+ ysize = p['y'].max() - p['y'].min()
579
+ warn("region size is %d x % d nm (%.1f x %.1f um" % (xsize,ysize,1e-3*xsize,1e-3*ysize))
580
+
581
+ def OnSetMINFLUXPlottingdefaults(self, event):
582
+ if not self.plottingDefaults.configure_traits(kind='modal'):
583
+ return
584
+ from PYMEcs.misc.matplotlib import figuredefaults
585
+ figuredefaults(fontsize=self.plottingDefaults.FontSize,linewidth=self.plottingDefaults.LineWidth)
586
+
587
+ # --- Alex B provided function (to save - not yet) and plot ITR stats (Paraflux like) ---
588
+ def OnRunParafluxAnalysis(self, event):
589
+ from pathlib import Path
590
+
591
+ # ======================================================================================
592
+ # --- Select, load Zarr.zip file, convert into DataFrame, Run the analysis functions ---
593
+ # ======================================================================================
594
+ pipeline = self.visFr.pipeline # Get the pipeline from the GUI
595
+
596
+ if pipeline is None:
597
+ Error(self.visFr, "No data found. Please load a MINFLUX dataset first.")
598
+ return
599
+ if not pipeline.mdh['MINFLUX.Is3D']: # TODO: make paraflux analysis code 2D aware
600
+ warn('paraflux analysis currently only implemented for 3D data, this is apparently 2D data; giving up...')
601
+ return
602
+ try:
603
+ # if this is a zarr archive we should have a zarr attribute in the FitResults datasource
604
+ zarr_archive = pipeline.dataSources['FitResults'].zarr
605
+ except:
606
+ warn("data is not from a zarr archive, giving up...")
607
+ return
608
+ try:
609
+ zarr_path = zarr_archive.store.path
610
+ except AttributeError:
611
+ warn("cannot get zarr store path from zarr object, not saving analysis data...")
612
+ zarr_path = None
613
+
614
+ # possible storage code, not yet used/implemented
615
+ # datasources = pipeline._get_session_datasources()
616
+ # store_path = datasources.get('FitResults')
617
+ # store_path = Path(store_path)
618
+
619
+ # paraflux analysis with progress dialog follows
620
+ import PYMEcs.Analysis.Paraflux as pf
621
+ mfx_zarrsource = pipeline.dataSources['FitResults'] # this should be a MinfluxZarrSource instance
622
+
623
+ # check if we have a cached result
624
+ if mfx_zarrsource._paraflux_analysis is None:
625
+ # for example use of ProgressDialog see also
626
+ # https://github.com/Metallicow/wxPython-Sample-Apps-and-Demos/blob/master/101_Common_Dialogs/ProgressDialog/ProgressDialog_extended.py
627
+ progress = wx.ProgressDialog("Paraflux analysis in progress", "please wait", maximum=4,
628
+ parent=self.visFr,
629
+ style=wx.PD_SMOOTH
630
+ | wx.PD_AUTO_HIDE)
631
+ def upd(n):
632
+ progress.Update(n)
633
+ wx.Yield()
634
+
635
+ # read all data from the zarr archive
636
+ mfxdata = zarr_archive['mfx'][:]; upd(1)
637
+ # processing 1st step, move data into pandas dataframe
638
+ df_mfx, failure_map = pf.paraflux_mk_df_fm(mfxdata); upd(2)
639
+ # Run the analysis steps
640
+ vld_itr = pf.build_valid_df(df_mfx); upd(3)
641
+ vld_itr = pf.compute_percentages(vld_itr)
642
+ vld_itr = pf.analyze_failures(vld_itr, df_mfx, failure_map)
643
+ initial_count = vld_itr['vld loc count'].iloc[0]
644
+ vld_itr = pf.add_failure_metrics(vld_itr, initial_count); upd(4)
645
+ mfx_zarrsource._paraflux_analysis = vld_itr
646
+ else:
647
+ vld_itr = mfx_zarrsource._paraflux_analysis
648
+
649
+ vld_paraflux = pf.paraflux_itr_plot(vld_itr[['itr', 'passed itr %', 'CFR failure %', 'No signal % per itr pairs']])
650
+
651
+ # here possible storage command, only if autosaving is enabled in config
652
+ if mu.autosave_check() and zarr_path is not None:
653
+ mu.autosave_csv(vld_itr.drop(columns='tid', errors='ignore'),
654
+ zarr_path,pipeline.mdh,'_iteration_stats_full')
655
+ ### --- End of Alex B added functionality ---
656
+
657
+ def OnClumpScatterPosPlot(self,event):
658
+ from scipy.stats import binned_statistic
659
+ from PYMEcs.IO.MINFLUX import get_stddev_property
660
+ def detect_coalesced(pipeline):
661
+ # placeholder, to be implemented
662
+ return False
663
+
664
+ bigClumpThreshold = 10
665
+ pipeline = self.visFr.pipeline
666
+ if "posInClump" not in pipeline.keys():
667
+ warn("No posInClump info, is this MINFLUX data imported with recent PAME-extra; aborting...")
668
+ if detect_coalesced(pipeline):
669
+ warn("This is coalesced data, need a datasource with non-coalesced clump info, aborting...")
670
+
671
+ bigclumps = pipeline['clumpSize'] >= bigClumpThreshold
672
+ ids = pipeline['clumpIndex'][bigclumps]
673
+ posIC = pipeline['posInClump'][bigclumps]
674
+ # uids,revids = np.unique(ids,return_inverse=True)
675
+ xIC = pipeline['x'][bigclumps]
676
+ yIC = pipeline['y'][bigclumps]
677
+ xctrd = get_stddev_property(ids,xIC,statistic='mean')
678
+ yctrd = get_stddev_property(ids,yIC,statistic='mean')
679
+ dists = np.sqrt((xIC - xctrd)**2 + (yIC - yctrd)**2)
680
+ maxpIC = min(int(posIC.max()),500) # we do not use clumps longer than 500
681
+ edges = -0.5+np.arange(maxpIC+2)
682
+ idrange = (0,maxpIC)
683
+
684
+ picDists, bin_edge, binno = binned_statistic(posIC, dists, statistic='mean',
685
+ bins=edges, range=idrange)
686
+ binctrs = 0.5*(bin_edge[:-1]+bin_edge[1:])
687
+ plt.figure()
688
+ plt.plot(binctrs, picDists)
689
+ plt.xlabel("position in clump")
690
+ plt.ylabel("lateral distance from ctrd (nm)")
691
+ plt.xlim(-5,100)
692
+ plt.ylim(0,None)
693
+
694
+ if pipeline.mdh.get('MINFLUX.Is3D'):
695
+ zIC = pipeline['z'][bigclumps]
696
+ zctrd = get_stddev_property(ids,zIC,statistic='mean')
697
+ zdists = np.abs(zIC - zctrd)
698
+ zpicDists, bin_edge, binno = binned_statistic(posIC, zdists, statistic='mean',
699
+ bins=edges, range=idrange)
700
+ binctrs = 0.5*(bin_edge[:-1]+bin_edge[1:])
701
+ plt.figure()
702
+ plt.plot(binctrs, zpicDists)
703
+ plt.xlabel("position in clump")
704
+ plt.ylabel("z distance from ctrd (nm)")
705
+ plt.xlim(-5,100)
706
+ plt.ylim(0,None)
707
+
708
+ def OnPlotColourStats(self,event):
709
+ pipeline = self.visFr.pipeline
710
+ chans = pipeline.colourFilter.getColourChans()
711
+ if len(chans) < 1:
712
+ warn("No colour channel info, check if colour filtering module is active; aborting plotting")
713
+ return
714
+ cols = ['m','c','y']
715
+ if len(chans) > 3:
716
+ raise RuntimeError("too many channels, can only deal with max 3, got %d" % len(chans))
717
+ bins = np.linspace(0.45,0.95,int(0.5/0.005))
718
+ fig, axs = plt.subplots(nrows=2)
719
+ axs[0].hist(pipeline['dcr_trace'],bins=bins,color='gray',alpha=0.5,label='other')
720
+ for chan,col in zip(chans,cols[0:len(chans)]):
721
+ axs[0].hist(pipeline.colourFilter.get_channel_column(chan,'dcr_trace'),bins=bins,color=col,alpha=0.5,label=chan)
722
+ axs[0].set_xlabel('dcr_trace')
723
+ axs[0].set_ylabel('#')
724
+ axs[0].legend(loc="upper right")
725
+ axs[1].hist(pipeline['dcr'],bins=bins,color='gray',alpha=0.5,label='other')
726
+ for chan,col in zip(chans,cols[0:len(chans)]):
727
+ axs[1].hist(pipeline.colourFilter.get_channel_column(chan,'dcr'),bins=bins,color=col,alpha=0.5,label=chan)
728
+ axs[1].set_xlabel('dcr')
729
+ axs[1].set_ylabel('#')
730
+ axs[1].legend(loc="upper right")
731
+ plt.tight_layout()
732
+
733
+ def OnMBMLowessCacheSave(self,event):
734
+ pipeline = self.visFr.pipeline
735
+ mod = findmbm(pipeline,return_mod=True)
736
+ if mod is None:
737
+ return
738
+ mod.lowess_cachesave()
739
+
740
+ def OnMBMAttributes(self, event):
741
+ from wx.lib.dialogs import ScrolledMessageDialog
742
+ fres = self.visFr.pipeline.dataSources['FitResults']
743
+ if 'zarr' in dir(fres):
744
+ try:
745
+ mbm_attrs = fres.zarr['grd']['mbm'].points.attrs['points_by_gri']
746
+ except AttributeError:
747
+ warn("could not access MBM attributes - do we have MBM data in zarr?")
748
+ return
749
+ import pprint
750
+ mbm_attr_str = pprint.pformat(mbm_attrs,indent=4,width=120)
751
+ with ScrolledMessageDialog(self.visFr, mbm_attr_str, "MBM attributes", size=(900,400),
752
+ style=wx.RESIZE_BORDER | wx.DEFAULT_DIALOG_STYLE ) as dlg:
753
+ dlg.ShowModal()
754
+ else:
755
+ warn("could not find zarr attribute - is this a MFX zarr file?")
756
+
757
+ def OnMFXAttributes(self, event):
758
+ from wx.lib.dialogs import ScrolledMessageDialog
759
+ fres = self.visFr.pipeline.dataSources['FitResults']
760
+ if 'zarr' in dir(fres):
761
+ try:
762
+ mfx_attrs = fres.zarr['mfx'].attrs.asdict()
763
+ except AttributeError:
764
+ warn("could not access MFX attributes - do we have MFX data in zarr?")
765
+ return
766
+ import pprint
767
+ mfx_attr_str = pprint.pformat(mfx_attrs,indent=4,width=120)
768
+ with ScrolledMessageDialog(self.visFr, mfx_attr_str, "MFX attributes", size=(900,400),
769
+ style=wx.RESIZE_BORDER | wx.DEFAULT_DIALOG_STYLE ) as dlg:
770
+ dlg.ShowModal()
771
+ else:
772
+ warn("could not find zarr attribute - is this a MFX zarr file?")
773
+
774
+ def OnMFXInfo(self, event):
775
+ import wx.html
776
+
777
+ fres = self.visFr.pipeline.dataSources['FitResults']
778
+ if 'zarr' not in dir(fres):
779
+ warn("could not find zarr attribute - is this a MFX zarr file?")
780
+ return
781
+ try:
782
+ mfx_attrs = fres.zarr['mfx'].attrs.asdict()
783
+ except AttributeError:
784
+ warn("could not access MFX attributes - do we have MFX data in zarr?")
785
+ return
786
+ if '_legacy' in mfx_attrs:
787
+ warn("legacy data detected - no useful MFX metadata in legacy data")
788
+ return
789
+
790
+ # if we make it to this part of the code there is some useful metadata to be looked at
791
+ from PYMEcs.IO.MINFLUX import get_metadata_from_mfx_attrs
792
+ md_itr_info, md_globals = get_metadata_from_mfx_attrs(mfx_attrs)
793
+
794
+ # Create an HTML dialog
795
+ dlg = wx.Dialog(self.visFr, title="MINFLUX Metadata Information",
796
+ size=(950, 600), style=wx.DEFAULT_DIALOG_STYLE | wx.RESIZE_BORDER)
797
+
798
+ # Create HTML content
799
+ html_window = wx.html.HtmlWindow(dlg, style=wx.html.HW_SCROLLBAR_AUTO)
800
+
801
+ # Format the DataFrame as an HTML table
802
+ html_content = "<html><body>"
803
+ html_content += "<p><b>Note:</b> This info should now be available via the PYME metadata, please inspect your metadata tab if using the GUI.</p>"
804
+
805
+ html_content += "<h2>MINFLUX Iteration Parameters</h2>"
806
+ html_content += md_itr_info.to_html(
807
+ classes='table table-striped',
808
+ float_format=lambda x: f"{x:.2f}" if isinstance(x, float) else x
809
+ )
810
+
811
+ # Format global parameters
812
+ html_content += "<h2>MINFLUX Global Parameters</h2>"
813
+ html_content += "<table border=1 class='table table-striped'>"
814
+ for key, value in sorted(md_globals.items()):
815
+ html_content += f"<tr><td><b>{key}</b></td><td>{value}</td></tr>"
816
+ html_content += "</table>"
817
+ html_content += "</body></html>"
818
+
819
+ html_window.SetPage(html_content)
820
+
821
+ # Add OK button
822
+ btn_sizer = wx.StdDialogButtonSizer()
823
+ btn = wx.Button(dlg, wx.ID_OK)
824
+ btn.SetDefault()
825
+ btn_sizer.AddButton(btn)
826
+ btn_sizer.Realize()
827
+
828
+ # Layout
829
+ sizer = wx.BoxSizer(wx.VERTICAL)
830
+ sizer.Add(html_window, 1, wx.EXPAND | wx.ALL, 5)
831
+ sizer.Add(btn_sizer, 0, wx.ALIGN_CENTER | wx.ALL, 5)
832
+ dlg.SetSizer(sizer)
833
+
834
+ dlg.ShowModal()
835
+ dlg.Destroy()
836
+
837
+ def OnZarrToZipStore(self, event):
838
+ with wx.DirDialog(self.visFr, 'Zarr to convert ...',
839
+ style=wx.DD_DEFAULT_STYLE | wx.DD_DIR_MUST_EXIST) as ddialog:
840
+ if ddialog.ShowModal() != wx.ID_OK:
841
+ return
842
+ fpath = ddialog.GetPath()
843
+ from PYMEcs.misc.utils import zarrtozipstore
844
+ from pathlib import Path
845
+ zarr_root = Path(fpath)
846
+ dest_dir = zarr_root.parent
847
+ archive_name = dest_dir / zarr_root.with_suffix('.zarr').name # we make archive_name here in the calling routine so that we can check for existence etc
848
+
849
+ if archive_name.with_suffix('.zarr.zip').exists():
850
+ with wx.FileDialog(self.visFr, 'Select archive name ...',
851
+ wildcard='ZIP (*.zip)|*.zip',
852
+ defaultFile=str(archive_name.with_suffix('.zarr.zip').name),
853
+ defaultDir=str(archive_name.with_suffix('.zarr.zip').parent),
854
+ style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT) as fdialog:
855
+ if fdialog.ShowModal() != wx.ID_OK:
856
+ return
857
+ archive_name = Path(fdialog.GetPath())
858
+ while archive_name.suffix in {'.zarr', '.zip'}: # this loop progressively removes the .zarr.zip suffix
859
+ archive_name = archive_name.with_suffix('')
860
+ archive_name = archive_name.with_suffix('.zarr')
861
+ # warn("got back name %s, using archive name %s" % (Path(fdialog.GetPath()).name,archive_name.name))
862
+
863
+ progress = wx.ProgressDialog("converting to zarr zip store", "please wait", maximum=2,
864
+ parent=self.visFr,
865
+ style=wx.PD_SMOOTH | wx.PD_AUTO_HIDE
866
+ )
867
+ progress.Update(1)
868
+ created = Path(zarrtozipstore(zarr_root,archive_name))
869
+ progress.Update(2)
870
+ progress.Destroy()
871
+
872
+ from PYMEcs.misc.guiMsgBoxes import YesNo
873
+ do_open = YesNo(self.visFr,("created new zip store\n\n'%s'\n\nin directory\n\n'%s'"
874
+ + "\n\nOpen newly created zipstore data?")
875
+ % (created.name,created.parent),
876
+ caption="Open new zarr zipstore?")
877
+
878
+ if do_open:
879
+ self.visFr.OpenFile(str(created))
880
+
881
+ def OnDensityStats(self, event):
882
+ from PYMEcs.Analysis.MINFLUX import plot_density_stats_sns
883
+ plot_density_stats_sns(self.visFr.pipeline)
884
+
885
+ def OnAlphaShapes(self, event):
886
+ if 'cluster_shapes' not in self.visFr.pipeline.dataSources.keys():
887
+ warn("missing data source 'cluster_shapes', will not display alpha shapes")
888
+ return
889
+
890
+ # now we add a layer to render our alpha shape polygons
891
+ from PYME.LMVis.layers.tracks import TrackRenderLayer # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar
892
+ layer = TrackRenderLayer(self.visFr.pipeline, dsname='cluster_shapes', method='tracks', clump_key='polyIndex', line_width=2.0, alpha=0.5)
893
+ self.visFr.add_layer(layer)
894
+
895
+ def OnAddMINFLUXTracksCI(self, event):
896
+ # now we add a track layer to render our traces
897
+ from PYME.LMVis.layers.tracks import TrackRenderLayer # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar
898
+ layer = TrackRenderLayer(self.visFr.pipeline, dsname='output', method='tracks', clump_key='clumpIndex', line_width=2.0, alpha=0.5)
899
+ self.visFr.add_layer(layer)
900
+
901
+ def OnAddMINFLUXTracksTid(self, event):
902
+ # now we add a track layer to render our traces
903
+ from PYME.LMVis.layers.tracks import TrackRenderLayer # NOTE: we may rename the clumpIndex variable in this layer to polyIndex or similar
904
+ layer = TrackRenderLayer(self.visFr.pipeline, dsname='output', method='tracks', clump_key='tid', line_width=2.0, alpha=0.5)
905
+ self.visFr.add_layer(layer)
906
+
907
+ def OnLoadCustom(self, event):
908
+ self.visFr._recipe_manager.LoadRecipe(self.minfluxRIDs[event.GetId()])
909
+
910
+
911
+ def OnMBMSave(self,event):
912
+ from pathlib import Path
913
+ pipeline = self.visFr.pipeline
914
+ mbm = findmbm(pipeline)
915
+ if mbm is None:
916
+ return
917
+ defaultFile = None
918
+ MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp')
919
+ if MINFLUXts is not None:
920
+ defaultFile = "%s__MBM-beads.npz" % MINFLUXts
921
+ fdialog = wx.FileDialog(self.visFr, 'Save MBM beads as ...',
922
+ wildcard='NPZ (*.npz)|*.npz',
923
+ defaultFile=defaultFile,
924
+ style=wx.FD_SAVE)
925
+ if fdialog.ShowModal() != wx.ID_OK:
926
+ return
927
+
928
+ fpath = fdialog.GetPath()
929
+ np.savez(fpath,**mbm._raw_beads)
930
+
931
+ def OnMBMSettingsSave(self,event):
932
+ import json
933
+ pipeline = self.visFr.pipeline
934
+ mbm = findmbm(pipeline)
935
+ if mbm is None:
936
+ return
937
+ mod = findmbm(pipeline,return_mod=True)
938
+ settings = {}
939
+ beadisgood = {}
940
+ for bead in mbm.beadisgood:
941
+ beadisgood[bead] = mbm.beadisgood[bead] and bead in mod._mbm_allbeads
942
+ settings['beads'] = beadisgood
943
+ settings['Median_window'] = mod.Median_window
944
+ settings['Lowess_fraction'] = mod.MBM_lowess_fraction
945
+ settings['Filename'] = mbm.name
946
+ defaultFile = None
947
+ MINFLUXts = pipeline.mdh.get('MINFLUX.TimeStamp')
948
+ if MINFLUXts is not None:
949
+ defaultFile = "%s__MBM-beads.npz-settings.json" % MINFLUXts
950
+ fdialog = wx.FileDialog(self.visFr, 'Save MBM Settings as ...',
951
+ wildcard='JSON (*.json)|*.json',
952
+ defaultFile=defaultFile,
953
+ style=wx.FD_SAVE)
954
+ if fdialog.ShowModal() != wx.ID_OK:
955
+ return
956
+
957
+ fpath = fdialog.GetPath()
958
+
959
+ with open(fpath, 'w') as f:
960
+ json.dump(settings, f, indent=4)
961
+
962
+ def OnMBMplot(self,event):
963
+ def drift_total(p,caxis):
964
+ has_drift = 'driftx' in p.keys()
965
+ has_drift_ori = 'driftx_ori' in p.keys()
966
+ caxis_nc = "%s_nc" % caxis
967
+ caxis_ori = "%s_ori" % caxis
968
+ if has_drift:
969
+ if has_drift_ori:
970
+ # note we have a driftaxis_ori hiding in -p[caxis_ori], so no need to explicitly add
971
+ drift_2ndpass = p['drift%s' % caxis] - (p[caxis_ori]-p[caxis_nc])
972
+ drift_1stpass = - (p[caxis_ori]-p[caxis_nc]) # again we have an implicit driftaxis_ori hiding in -p[caxis_ori]
973
+ else:
974
+ drift_1stpass = p['drift%s' % caxis] - (p[caxis_ori]-p[caxis_nc])
975
+ drift_2ndpass = None
976
+ return drift_1stpass, drift_2ndpass
977
+
978
+ def plot_drift(p,ax,drift_1stpass, drift_2ndpass):
979
+ has_drift = 'driftx' in p.keys()
980
+ has_drift_ori = 'driftx_ori' in p.keys()
981
+ if has_drift:
982
+ if has_drift_ori:
983
+ ax.plot(t_s,drift_2ndpass, label='site-based 2nd pass')
984
+ ax.plot(t_s,drift_1stpass,'--', label='site-based 1st pass')
985
+ else:
986
+ ax.plot(t_s,drift_1stpass, label='site-based 1st pass')
987
+
988
+ p = self.visFr.pipeline
989
+ if p.mdh['MINFLUX.Is3D']:
990
+ axes = ['x','y','z']
991
+ naxes = 3
992
+ else:
993
+ axes = ['x','y']
994
+ naxes = 2
995
+ has_drift = 'driftx' in p.keys()
996
+ has_drift_ori = 'driftx_ori' in p.keys()
997
+ mbm = findmbm(p,warnings=False)
998
+ has_mbm = mbm is not None
999
+ has_mbm2 = 'mbmx' in p.keys()
1000
+
1001
+ if not has_drift and not (has_mbm or has_mbm2):
1002
+ warn("pipeline has neither drift info nor MBM info, aborting...")
1003
+
1004
+ t_s = 1e-3*p['t']
1005
+ mbm_mean = {} # for caching
1006
+ mbm_meansm = {} # for caching
1007
+ t_sm = {}
1008
+
1009
+ ### Fig 1 ####
1010
+ fig, axs = plt.subplots(nrows=naxes)
1011
+ for caxis, ax in zip(axes,axs):
1012
+ if has_drift:
1013
+ drift_1stpass, drift_2ndpass = drift_total(p,caxis)
1014
+ plot_drift(p,ax,drift_1stpass, drift_2ndpass)
1015
+ if has_mbm:
1016
+ mod = findmbm(p,warnings=False,return_mod=True)
1017
+ MBM_lowess_fraction = mod.MBM_lowess_fraction
1018
+ mbm_mean[caxis] = mbm.mean(caxis)
1019
+ ax.plot(mbm.t,mbm_mean[caxis],':',label='MBM mean')
1020
+ t_sm[caxis],mbm_meansm[caxis] = mod.lowess_calc(caxis) # this is now a cached version of the lowess calc!
1021
+ ax.plot(t_sm[caxis],mbm_meansm[caxis],'-.',label='MBM lowess (lf=%.2f)' % MBM_lowess_fraction)
1022
+ if has_mbm2:
1023
+ ax.plot(t_s,p['mbm%s' % caxis], label='MBM from module')
1024
+ ax.set_xlabel('time (s)')
1025
+ ax.set_ylabel('drift in %s (nm)' % caxis)
1026
+ ax.legend(loc="upper right")
1027
+ ax.set_title("Total drift (Site based plus any previous corrections)")
1028
+ fig.tight_layout()
1029
+ ### Fig 2 ####
1030
+ if has_mbm: # also plot a second figure without the non-smoothed MBM track
1031
+ fig, axs = plt.subplots(nrows=naxes)
1032
+ for caxis, ax in zip(axes,axs):
1033
+ if has_drift:
1034
+ drift_1stpass, drift_2ndpass = drift_total(p,caxis)
1035
+ plot_drift(p,ax,drift_1stpass, drift_2ndpass)
1036
+ if has_mbm:
1037
+ #ax.plot(mbm.t,mbm_mean[caxis],':',label='MBM mean')
1038
+ ax.plot(t_sm[caxis],mbm_meansm[caxis],'r-.',label='MBM lowess (lf=%.2f)' % MBM_lowess_fraction)
1039
+ ax.set_xlabel('time (s)')
1040
+ ax.set_ylabel('drift in %s (nm)' % caxis)
1041
+ ax.legend(loc="upper left")
1042
+ fig.tight_layout()
1043
+ ### Fig 3 ####
1044
+ if has_mbm: # also plot a third figure with all MBM tracks
1045
+ fig, axs = plt.subplots(nrows=naxes)
1046
+ for caxis, ax in zip(axes,axs):
1047
+ if has_drift:
1048
+ drift_1stpass, drift_2ndpass = drift_total(p,caxis)
1049
+ plot_drift(p,ax,drift_1stpass, drift_2ndpass)
1050
+ if has_mbm:
1051
+ #ax.plot(mbm.t,mbm_mean[caxis],':',label='MBM mean')
1052
+ ax.plot(t_sm[caxis],mbm_meansm[caxis],'r-.',label='MBM lowess (lf=%.2f)' % MBM_lowess_fraction)
1053
+ mbm.plot_tracks_matplotlib(caxis,ax=ax,goodalpha=0.4)
1054
+ ax.set_xlabel('time (s)')
1055
+ ax.set_ylabel('drift in %s (nm)' % caxis)
1056
+ ax.legend(loc="upper left")
1057
+ fig.tight_layout()
1058
+ ### Fig 4 ####
1059
+ if has_mbm and has_drift: # also plot a fourth figure with a difference track for all axes
1060
+ tnew = 1e-3*p['t']
1061
+ mbmcorr = {}
1062
+ for axis in axes:
1063
+ axis_interp_msm = np.interp(tnew,t_sm[caxis],mbm_meansm[axis])
1064
+ mbmcorr[axis] = axis_interp_msm
1065
+ fig, axs = plt.subplots(nrows=naxes)
1066
+ for caxis, ax in zip(axes,axs):
1067
+ drift_1stpass, drift_2ndpass = drift_total(p,caxis)
1068
+ if has_drift:
1069
+ if has_drift_ori:
1070
+ ax.plot(t_s,drift_2ndpass-mbmcorr[caxis], label='diff to site-based 2nd pass')
1071
+ else:
1072
+ ax.plot(t_s,drift_1stpass-mbmcorr[caxis], label='diff to site-based 1st pass')
1073
+ ax.plot([t_s.min(),t_s.max()],[0,0],'r-.')
1074
+ ax.set_xlabel('time (s)')
1075
+ ax.set_ylabel('differential drift in %s (nm)' % caxis)
1076
+ ax.legend(loc="upper left")
1077
+ fig.tight_layout()
1078
+
1079
+
1080
+ def OnMBMtracks(self, event):
1081
+ pipeline = self.visFr.pipeline
1082
+ mbm = findmbm(pipeline)
1083
+ if mbm is None:
1084
+ return # note that findmbm has already warned in this case
1085
+ if not self.mbmAxisSelection.configure_traits(kind='modal'):
1086
+ return
1087
+ ori_win = mbm.median_window
1088
+ mbm.median_window = 21 # go for pretty agressive smoothing
1089
+ if self.mbmAxisSelection.SelectAxis == 'x-y-z':
1090
+ fig, axes = plt.subplots(nrows=3)
1091
+ for axis,plotax in zip(['x','y','z'],axes):
1092
+ mbm.plot_tracks_matplotlib(axis,ax=plotax)
1093
+ fig.tight_layout()
1094
+ else:
1095
+ mbm.plot_tracks_matplotlib(self.mbmAxisSelection.SelectAxis)
1096
+ mbm.median_window = ori_win
1097
+
1098
+ def OnMBMaddTrackLabels(self, event):
1099
+ pipeline = self.visFr.pipeline
1100
+ try:
1101
+ from PYME.LMVis.layers.labels import LabelLayer
1102
+ except:
1103
+ hasLL = False
1104
+ else:
1105
+ hasLL = True
1106
+
1107
+ # note: should also add the merge module?
1108
+ # note: should also add the layer for mbm_tracks?
1109
+
1110
+ if 'mbm_pos' not in pipeline.dataSources.keys():
1111
+ #warn("no datasource 'mbm_pos' which is needed for label display")
1112
+ #return
1113
+ mod = findmbm(pipeline, warnings=True, return_mod=True)
1114
+ if mod is None:
1115
+ return
1116
+ from PYME.recipes.localisations import MergeClumps
1117
+ mc = MergeClumps(pipeline.recipe,
1118
+ inputName=mod.outputTracksCorr,
1119
+ outputName='mbm_pos',
1120
+ labelKey='objectID',
1121
+ # important, otherwise we get a spurious bead labele R0
1122
+ discardTrivial=True)
1123
+ pipeline.recipe.add_module(mc)
1124
+ pipeline.recipe.execute()
1125
+ if not hasLL:
1126
+ warn("could not load new experimental feature label layer, aborting...")
1127
+ return
1128
+
1129
+ if hasLL:
1130
+ ll = LabelLayer(pipeline, dsname='mbm_pos', format_string='R{beadID:.0f}', cmap='grey_overflow', font_size=13, textColour='good')
1131
+ self.visFr.add_layer(ll)
1132
+ ll.update()
1133
+
1134
+ def OnMINFLUXsetTempDataFolder(self, event):
1135
+ import PYME.config as config
1136
+ config_var = 'MINFLUX-temperature_folder'
1137
+ curfolder = config.get(config_var)
1138
+ if curfolder is None:
1139
+ warn("currently the MINFLUX temperature file folder is not set. Set the path in the following file dialog")
1140
+ else:
1141
+ warn("MINFLUX temperature file folder currently set to '%s'.\n\nAlter in the following dialog if needed or cancel that dialog if want to leave as is" % curfolder)
1142
+ with wx.DirDialog(self.visFr, "Choose folder containing temperature CSV files") as dialog:
1143
+ if dialog.ShowModal() == wx.ID_CANCEL:
1144
+ return
1145
+ folder = dialog.GetPath()
1146
+ if config.get(config_var) == folder:
1147
+ warn("config option '%s' already set to %s, leaving as is" % (config_var,folder))
1148
+ return # already set to this value, return
1149
+
1150
+ config.update_config({config_var: folder},
1151
+ config='user', create_backup=True)
1152
+
1153
+ def OnToggleMINFLUXautosave(self, event):
1154
+ import PYME.config as config
1155
+ config_var = 'MINFLUX-autosave'
1156
+
1157
+ if config.get(config_var,False):
1158
+ newval = False
1159
+ else:
1160
+ newval = True
1161
+
1162
+ config.update_config({config_var: newval},
1163
+ config='user', create_backup=False)
1164
+
1165
+ warn("MINFLUX analysis autosave was set to %s" % config.get(config_var))
1166
+
1167
+
1168
+ def OnMINFLUXplotTemperatureData(self, event):
1169
+ import PYME.config as config
1170
+ import os
1171
+ from os.path import basename
1172
+ from glob import glob
1173
+
1174
+ configvar = 'MINFLUX-temperature_folder'
1175
+ folder = config.get(configvar)
1176
+ if folder is None:
1177
+ warn("Need to set Temperature file location first by setting config variable %s" % configvar)
1178
+ return
1179
+ elif not os.path.isdir(folder):
1180
+ warn(("Config variable %s is not set to a folder;\n" % (configvar)) +
1181
+ ("needs to be a **folder** location, currently set to %s" % (folder)))
1182
+ return
1183
+
1184
+ from PYMEcs.misc.utils import read_temperature_csv, set_diff, timestamp_to_datetime
1185
+
1186
+ if len(self.visFr.pipeline.dataSources) == 0:
1187
+ warn("no datasources, this is probably an empty pipeline, have you loaded any data?")
1188
+ return
1189
+
1190
+ t0 = self.visFr.pipeline.mdh.get('MINFLUX.TimeStamp')
1191
+ if t0 is None:
1192
+ warn("no MINFLUX TimeStamp in metadata, giving up")
1193
+ return
1194
+ # Convert t0 from timestamp for comparing it with timedates from csv file
1195
+ t0_dt = timestamp_to_datetime(t0)
1196
+
1197
+ # Identify the correct temperature CSV files in the folder
1198
+ # Loop over CSVs to find matching file
1199
+ timeformat = config.get('MINFLUX-temperature_time_format', ['%d.%m.%Y %H:%M:%S',
1200
+ '%d/%m/%Y %H:%M:%S'])
1201
+ candidate_files = sorted(glob(os.path.join(folder, '*.csv')))
1202
+
1203
+ for f in candidate_files:
1204
+ try:
1205
+ # print(f"\nChecking file: {basename(f)}\n") # Debugging, Show which file is being checked
1206
+
1207
+ df = read_temperature_csv(f, timeformat=timeformat)
1208
+
1209
+ if 'datetime' not in df.columns:
1210
+ print(f"File {f} has no 'datetime' column after parsing, skipping.")
1211
+ continue
1212
+
1213
+ logger.debug(f"successfully read and parsed temperature file {f}")
1214
+
1215
+ if df['datetime'].min() <= t0_dt <= df['datetime'].max():
1216
+ selected_file = f
1217
+ logger.debug(f"found relevant time period in file {f}")
1218
+ break
1219
+ except Exception as e:
1220
+ logger.debug(f"Error reading {f}: {e}")
1221
+ continue
1222
+ else:
1223
+ warn("No temperature file found that includes the MINFLUX TimeStamp")
1224
+ return
1225
+
1226
+ # Read temperature data from the correct CSV file
1227
+ mtemps = read_temperature_csv(selected_file, timeformat=timeformat)
1228
+
1229
+ set_diff(mtemps,timestamp_to_datetime(t0))
1230
+ p = self.visFr.pipeline
1231
+ range = (1e-3*p['t'].min(),1e-3*p['t'].max())
1232
+ sertemps = mtemps[mtemps['tdiff_s'].between(range[0],range[1])]
1233
+ if sertemps.empty:
1234
+ warn("no records in requested time window, is series time before or after start/end of available temperature records?\n" +
1235
+ ("current records cover %s to %s" % (mtemps['Time'].iloc[0],mtemps['Time'].iloc[-1])) +
1236
+ ("\nseries starts at %s" % (t0)))
1237
+ return
1238
+ else:
1239
+ # for now we make 2 subplots so that we can provide both s units and actual time
1240
+ fig, axes = plt.subplots(nrows=2, ncols=1)
1241
+ sertemps.plot('datetime','Stand',style='.-',
1242
+ title="temperature record for series starting at %s" % t0, ax=axes[0])
1243
+ sertemps.plot('tdiff_s','Stand',style='.-', ax=axes[1])
1244
+ plt.tight_layout()
1245
+
1246
+ fig, axes = plt.subplots(nrows=2, ncols=1)
1247
+ sertemps.plot('datetime','Box',style='.-',
1248
+ title="temperature record for series starting at %s" % t0, ax=axes[0])
1249
+ sertemps.plot('tdiff_s','Box',style='.-', ax=axes[1])
1250
+ plt.tight_layout()
1251
+
1252
+ def OnErrorAnalysis(self, event):
1253
+ plot_errors(self.visFr.pipeline,ds=self.analysisSettings.defaultDatasourceCoalesced,dsclumps=self.analysisSettings.defaultDatasourceWithClumps)
1254
+
1255
+ def OnCluster3D(self, event):
1256
+ plot_cluster_analysis(self.visFr.pipeline, ds=self.analysisSettings.datasourceForClusterAnalysis,
1257
+ bigc_thresh=self.analysisSettings.largeClusterThreshold,
1258
+ plotints=self.analysisSettings.clustercountsPlotWithInts)
1259
+
1260
+ def OnCluster2D(self, event):
1261
+ plot_cluster_analysis(self.visFr.pipeline, ds='dbscan2D')
1262
+
1263
+ def OnClumpIndexContig(self, event):
1264
+ pipeline = self.visFr.pipeline
1265
+ curds = pipeline.selectedDataSourceKey
1266
+ pipeline.selectDataSource(self.analysisSettings.defaultDatasourceForAnalysis)
1267
+ if not 'clumpIndex' in pipeline.keys():
1268
+ Error(self.visFr,'no property called "clumpIndex", cannot check')
1269
+ pipeline.selectDataSource(curds)
1270
+ return
1271
+ uids = np.unique(pipeline['clumpIndex'])
1272
+ maxgap = np.max(uids[1:]-uids[:-1])
1273
+ pipeline.selectDataSource(curds)
1274
+
1275
+ if maxgap > 1:
1276
+ msg = "clumpIndex not contiguous, maximal gap is %d\nCI 0..9 %s" % (maxgap,uids[0:10])
1277
+ else:
1278
+ msg = "clumpIndex is contiguous\nCI 0..9 %s" % uids[0:10]
1279
+
1280
+ warn(msg)
1281
+
1282
+ def OnLocalisationRate(self, event):
1283
+ pipeline = self.visFr.pipeline
1284
+ curds = pipeline.selectedDataSourceKey
1285
+ pipeline.selectDataSource(self.analysisSettings.defaultDatasourceForAnalysis)
1286
+ if not 'cfr' in pipeline.keys():
1287
+ Error(self.visFr,'no property called "cfr", likely no MINFLUX data - aborting')
1288
+ pipeline.selectDataSource(curds)
1289
+ return
1290
+ if not 'tim' in pipeline.keys():
1291
+ Error(self.visFr,'no property called "tim", you need to convert to CSV with a more recent version of PYME-Extra - aborting')
1292
+ pipeline.selectDataSource(curds)
1293
+ return
1294
+ pipeline.selectDataSource(curds)
1295
+
1296
+ analyse_locrate(pipeline,datasource=self.analysisSettings.defaultDatasourceForAnalysis,showTimeAverages=True)
1297
+
1298
+ def OnEfoAnalysis(self, event):
1299
+ pipeline = self.visFr.pipeline
1300
+ curds = pipeline.selectedDataSourceKey
1301
+ pipeline.selectDataSource(self.analysisSettings.defaultDatasourceForAnalysis)
1302
+ if not 'efo' in pipeline.keys():
1303
+ Error(self.visFr,'no property called "efo", likely no MINFLUX data or wrong datasource (CHECK) - aborting')
1304
+ return
1305
+ plt.figure()
1306
+ h = plt.hist(1e-3*pipeline['efo'],bins='auto',range=(0,200))
1307
+ dskey = pipeline.selectedDataSourceKey
1308
+ plt.xlabel('efo (photon rate in kHz)')
1309
+ plt.title("EFO stats, using datasource '%s'" % dskey)
1310
+
1311
+ pipeline.selectDataSource(curds)
1312
+
1313
+ def OnTrackPlot(self, event):
1314
+ p = self.visFr.pipeline
1315
+ curds = p.selectedDataSourceKey
1316
+ if self.analysisSettings.defaultDatasourceForMBM in p.dataSources.keys():
1317
+ # should be coalesced datasource
1318
+ p.selectDataSource(self.analysisSettings.defaultDatasourceForMBM)
1319
+ is_coalesced = 'coalesced' in self.analysisSettings.defaultDatasourceForMBM.lower()
1320
+ else:
1321
+ # try instead something that should exist
1322
+ p.selectDataSource(self.analysisSettings.defaultDatasourceForAnalysis)
1323
+ is_coalesced = 'coalesced' in self.analysisSettings.defaultDatasourceForAnalysis.lower()
1324
+ plot_tracking(p,is_coalesced,lowess_fraction=0.03)
1325
+ p.selectDataSource(curds)
1326
+
1327
+ def OnOrigamiFinalFilter(self, event=None):
1328
+ from PYME.recipes.tablefilters import FilterTable
1329
+ pipeline = self.visFr.pipeline
1330
+ recipe = pipeline.recipe
1331
+ curds = pipeline.selectedDataSourceKey
1332
+
1333
+ finalFiltered = unique_name('filtered_final',pipeline.dataSources.keys())
1334
+
1335
+ modules = [FilterTable(recipe,inputName=curds,outputName=finalFiltered,
1336
+ filters={'error_x' : [0,3.5],
1337
+ 'error_x' : [0,3.5],
1338
+ 'error_x' : [0,3.5],
1339
+ 'efo' : [10e3,1e5],
1340
+ })]
1341
+ recipe.add_modules_and_execute(modules)
1342
+ pipeline.selectDataSource(finalFiltered)
1343
+
1344
+ def OnOrigamiSiteRecipe(self, event=None):
1345
+ from PYMEcs.recipes.localisations import OrigamiSiteTrack, DBSCANClustering2
1346
+ from PYME.recipes.localisations import MergeClumps
1347
+ from PYME.recipes.tablefilters import FilterTable, Mapping
1348
+
1349
+ pipeline = self.visFr.pipeline
1350
+ recipe = pipeline.recipe
1351
+
1352
+ filters={'error_x' : [0,3.5],
1353
+ 'error_y' : [0,3.5]}
1354
+ if 'error_z' in pipeline.keys():
1355
+ filters['error_z'] = [0,3.5]
1356
+
1357
+ preFiltered = unique_name('prefiltered',pipeline.dataSources.keys())
1358
+ corrSiteClumps = unique_name('corrected_siteclumps',pipeline.dataSources.keys())
1359
+ corrAll = unique_name('corrected_allpoints',pipeline.dataSources.keys())
1360
+ siteClumps = unique_name('siteclumps',pipeline.dataSources.keys())
1361
+ dbscanClusteredSites = unique_name('dbscanClusteredSites',pipeline.dataSources.keys())
1362
+ sites = unique_name('sites',pipeline.dataSources.keys())
1363
+ sites_c = unique_name('sites_c',pipeline.dataSources.keys())
1364
+
1365
+ curds = pipeline.selectedDataSourceKey
1366
+ modules = [FilterTable(recipe,inputName=curds,outputName=preFiltered,
1367
+ filters=filters),
1368
+ DBSCANClustering2(recipe,inputName=preFiltered,outputName=dbscanClusteredSites,
1369
+ searchRadius = 15.0,
1370
+ clumpColumnName = 'siteID',
1371
+ sizeColumnName='siteClumpSize'),
1372
+ FilterTable(recipe,inputName=dbscanClusteredSites,outputName=siteClumps,
1373
+ filters={'siteClumpSize' : [3,50]}), # need a minimum clumpsize and also maximal to avoid "fused" sites
1374
+ MergeClumps(recipe,inputName=siteClumps,outputName=sites,
1375
+ labelKey='siteID',discardTrivial=True),
1376
+ OrigamiSiteTrack(recipe,inputClusters=siteClumps,inputSites=sites,outputName=corrSiteClumps,smoothingBinWidthsSeconds=400,
1377
+ outputAllPoints=corrAll,inputAllPoints=curds,labelKey='siteID',binnedStatistic='median'), # median to play it safe
1378
+ MergeClumps(recipe,inputName=corrSiteClumps,outputName=sites_c,
1379
+ labelKey='siteID',discardTrivial=True)]
1380
+ recipe.add_modules_and_execute(modules)
1381
+
1382
+ if self.analysisSettings.origamiWith_nc:
1383
+ preFiltered = unique_name('prefiltered_nc',pipeline.dataSources.keys())
1384
+ corrSiteClumps = unique_name('corrected_siteclumps_nc',pipeline.dataSources.keys())
1385
+ siteClumps = unique_name('siteclumps_nc',pipeline.dataSources.keys())
1386
+ dbscanClusteredSites = unique_name('dbscanClusteredSites_nc',pipeline.dataSources.keys())
1387
+ sites = unique_name('sites_nc',pipeline.dataSources.keys())
1388
+ sites_c = unique_name('sites_c_nc',pipeline.dataSources.keys())
1389
+ dbsnc = unique_name('dbs_nc',pipeline.dataSources.keys())
1390
+
1391
+ modules = [FilterTable(recipe,inputName=curds,outputName=preFiltered,
1392
+ filters=filters),
1393
+ DBSCANClustering2(recipe,inputName=preFiltered,outputName=dbscanClusteredSites,
1394
+ searchRadius = 15.0,
1395
+ clumpColumnName = 'siteID',
1396
+ sizeColumnName='siteClumpSize'),
1397
+ Mapping(recipe,inputName=dbscanClusteredSites,outputName=dbsnc,
1398
+ mappings={'x': 'x_nc', 'y': 'y_nc', 'z': 'z_nc'}),
1399
+ FilterTable(recipe,inputName=dbsnc,outputName=siteClumps,
1400
+ filters={'siteClumpSize' : [3,50]}), # need a minimum clumpsize and also maximal to avoid "fused" sites
1401
+ MergeClumps(recipe,inputName=siteClumps,outputName=sites,
1402
+ labelKey='siteID',discardTrivial=True),
1403
+ OrigamiSiteTrack(recipe,inputClusters=siteClumps,inputSites=sites,outputName=corrSiteClumps,
1404
+ labelKey='siteID',binnedStatistic='median'),
1405
+ MergeClumps(recipe,inputName=corrSiteClumps,outputName=sites_c,
1406
+ labelKey='siteID',discardTrivial=True)]
1407
+
1408
+ recipe.add_modules_and_execute(modules)
1409
+
1410
+ pipeline.selectDataSource(corrSiteClumps)
1411
+
1412
+ def OnOrigamiSiteTrackPlot(self, event):
1413
+ p = self.visFr.pipeline
1414
+ # need to add checks if the required properties are present in the datasource!!
1415
+ plot_site_tracking(p,fignum=self.origamiTrackFignum,
1416
+ plotSmoothingCurve=self.analysisSettings.withOrigamiSmoothingCurves)
1417
+ self.origamiTrackFignum += 1
1418
+
1419
+ def OnMINFLUXSettings(self, event):
1420
+ if self.analysisSettings.configure_traits(kind='modal'):
1421
+ pass
1422
+
1423
+ def OnMINFLUXSiteSettings(self, event):
1424
+ if self.siteSettings.configure_traits(kind='modal'):
1425
+ pass
1426
+
1427
+ def OnOrigamiErrorPlot(self, event):
1428
+ p = self.visFr.pipeline
1429
+ # need to check if the required properties are present in the datasource
1430
+ if 'error_x_ori' not in p.keys():
1431
+ warn("property 'error_x_ori' not present, possibly not the right datasource for origami site info. Aborting...")
1432
+ return
1433
+
1434
+ def plot_errs(ax,axisname,errkeys):
1435
+ ax.hist(p[errkeys[0]],bins='auto',alpha=0.5,density=True,label='Trace est')
1436
+ ax.hist(p[errkeys[1]],bins='auto',alpha=0.5,density=True,label='Site est')
1437
+ ax.hist(p[errkeys[2]],bins='auto',alpha=0.5,density=True,label='Site est corr')
1438
+ ax.legend()
1439
+ ax.set_xlabel('error %s (nm)' % axisname)
1440
+ ax.set_ylabel('#')
1441
+
1442
+ fig, axs = plt.subplots(2, 2,num='origami error estimates %d' % self.origamiErrorFignum)
1443
+ plot_errs(axs[0, 0], 'x', ['error_x_ori','error_x_nc','error_x'])
1444
+ axs[0, 0].set_xlim(0,self.analysisSettings.origamiErrorLimit)
1445
+ plot_errs(axs[0, 1], 'y', ['error_y_ori','error_y_nc','error_y'])
1446
+ axs[0, 1].set_xlim(0,self.analysisSettings.origamiErrorLimit)
1447
+ if p.mdh.get('MINFLUX.Is3D'):
1448
+ plot_errs(axs[1, 0], 'z', ['error_z_ori','error_z_nc','error_z'])
1449
+ axs[1, 0].set_xlim(0,self.analysisSettings.origamiErrorLimit)
1450
+ ax = axs[1,1]
1451
+ # plot the MBM track, this way we know if we are using the _nc data or the MBM corrected data for analysis
1452
+ t_s = 1e-3*p['t']
1453
+ ax.plot(t_s,p['x_ori']-p['x_nc'],alpha=0.5,label='x')
1454
+ plt.plot(t_s,p['y_ori']-p['y_nc'],alpha=0.5,label='y')
1455
+ if 'z_nc' in p.keys():
1456
+ ax.plot(t_s,p['z_ori']-p['z_nc'],alpha=0.5,label='z')
1457
+ ax.set_xlabel('t (s)')
1458
+ ax.set_ylabel('MBM corr [nm]')
1459
+ ax.legend()
1460
+ plt.tight_layout()
1461
+
1462
+ uids = np.unique(p['siteID']) # currently siteID is hard coded - possibly make config option
1463
+ from PYMEcs.Analysis.MINFLUX import plotsitestats
1464
+ if uids.size < self.siteSettings.siteMaxNum:
1465
+ plotsitestats(p,fignum=('origami site stats %d' % self.origamiErrorFignum))
1466
+ self.origamiErrorFignum += 1
1467
+
1468
+ def OnOrigamiSiteStats(self, event):
1469
+ from PYMEcs.IO.MINFLUX import get_stddev_property
1470
+ p = self.visFr.pipeline
1471
+ plotted = False
1472
+ # need to check if the required properties are present in the datasource
1473
+ if 'error_x_ori' not in p.keys():
1474
+ warn("property 'error_x_ori' not present, possibly not the right datasource for origami site info. Aborting...")
1475
+ return
1476
+ plotmode = self.siteSettings.plotMode
1477
+ from PYMEcs.Analysis.MINFLUX import plotsitestats
1478
+ uids,idx = np.unique(p['siteID'],return_index=True) # currently siteID is hard coded - possibly make config option
1479
+ if uids.size < self.siteSettings.siteMaxNum:
1480
+ swarmsize = 3
1481
+ else:
1482
+ swarmsize = 1.5
1483
+
1484
+ plotsitestats(p,fignum=('origami site stats %d' % self.origamiErrorFignum),
1485
+ swarmsize=swarmsize,mode=plotmode,showpoints=self.siteSettings.showPoints,
1486
+ origamiErrorLimit=self.siteSettings.precisionRange_nm,
1487
+ strip=(self.siteSettings.pointsMode == 'strip'))
1488
+ plotted = True
1489
+ #else:
1490
+ # warn("Number of sites (%d) > max number for plotting (%d); check settings"
1491
+ # % (uids.size,self.analysisSettings.origamiSiteMaxNum))
1492
+
1493
+ counts = get_stddev_property(p['siteID'],p['siteID'],statistic='count')
1494
+ plt.figure(num=('site visits %d' % self.origamiErrorFignum))
1495
+ ax = plt.gca()
1496
+ ctmedian = np.median(counts[idx])
1497
+ ctmean = np.mean(counts[idx])
1498
+
1499
+ h = plt.hist(counts[idx],bins='auto')
1500
+ ax.plot([ctmedian,ctmedian],[0,h[0].max()])
1501
+ plt.xlabel('Number of site visits')
1502
+ plt.text(0.85, 0.8, 'median %d' % ctmedian, horizontalalignment='right',
1503
+ verticalalignment='bottom', transform=ax.transAxes)
1504
+ plt.text(0.85, 0.7, ' mean %.1f' % ctmean, horizontalalignment='right',
1505
+ verticalalignment='bottom', transform=ax.transAxes)
1506
+ plotted = True
1507
+ if plotted:
1508
+ self.origamiErrorFignum += 1
1509
+
1510
+ def OnMINFLUXColour(self,event):
1511
+ from PYME.LMVis import colourPanel
1512
+
1513
+ mw = self.visFr
1514
+ if mw.colp is None: # no colourPanel yet
1515
+ self.visFr.pipeline.selectDataSource('colour_mapped')
1516
+ mw.adding_panes=True
1517
+ mw.colp = colourPanel.colourPanel(mw, mw.pipeline, mw)
1518
+ mw.AddPage(mw.colp, caption='Colour', select=False, update=False)
1519
+ mw.adding_panes=False
1520
+ else:
1521
+ warn('Colour panel appears to already exist - not creating new colour panel')
1522
+
1523
+ def OnCornerplot(self,event):
1524
+ for ds in ['withNNdist','group2','group3','group4']:
1525
+ if ds not in self.visFr.pipeline.dataSources.keys():
1526
+ warn("need datasource %s which is not present, giving up..." % ds)
1527
+ return
1528
+ fourcornerplot_default(self.visFr.pipeline)
1529
+
1530
+ def Plug(visFr):
1531
+ # we are trying to monkeypatch pipeline and VisGUIFrame methods to sneak MINFLUX npy IO in;
1532
+ # in future we will ask for a way to get this considered by David B for a proper hook
1533
+ # in the IO code
1534
+ from PYMEcs.IO.MINFLUX import monkeypatch_npyorzarr_io
1535
+ monkeypatch_npyorzarr_io(visFr)
1536
+
1537
+ return MINFLUXanalyser(visFr)