mediml 0.9.9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. MEDiml/MEDscan.py +1696 -0
  2. MEDiml/__init__.py +21 -0
  3. MEDiml/biomarkers/BatchExtractor.py +806 -0
  4. MEDiml/biomarkers/BatchExtractorTexturalFilters.py +840 -0
  5. MEDiml/biomarkers/__init__.py +16 -0
  6. MEDiml/biomarkers/diagnostics.py +125 -0
  7. MEDiml/biomarkers/get_oriented_bound_box.py +158 -0
  8. MEDiml/biomarkers/glcm.py +1602 -0
  9. MEDiml/biomarkers/gldzm.py +523 -0
  10. MEDiml/biomarkers/glrlm.py +1315 -0
  11. MEDiml/biomarkers/glszm.py +555 -0
  12. MEDiml/biomarkers/int_vol_hist.py +527 -0
  13. MEDiml/biomarkers/intensity_histogram.py +615 -0
  14. MEDiml/biomarkers/local_intensity.py +89 -0
  15. MEDiml/biomarkers/morph.py +1756 -0
  16. MEDiml/biomarkers/ngldm.py +780 -0
  17. MEDiml/biomarkers/ngtdm.py +414 -0
  18. MEDiml/biomarkers/stats.py +373 -0
  19. MEDiml/biomarkers/utils.py +389 -0
  20. MEDiml/filters/TexturalFilter.py +299 -0
  21. MEDiml/filters/__init__.py +9 -0
  22. MEDiml/filters/apply_filter.py +134 -0
  23. MEDiml/filters/gabor.py +215 -0
  24. MEDiml/filters/laws.py +283 -0
  25. MEDiml/filters/log.py +147 -0
  26. MEDiml/filters/mean.py +121 -0
  27. MEDiml/filters/textural_filters_kernels.py +1738 -0
  28. MEDiml/filters/utils.py +107 -0
  29. MEDiml/filters/wavelet.py +237 -0
  30. MEDiml/learning/DataCleaner.py +198 -0
  31. MEDiml/learning/DesignExperiment.py +480 -0
  32. MEDiml/learning/FSR.py +667 -0
  33. MEDiml/learning/Normalization.py +112 -0
  34. MEDiml/learning/RadiomicsLearner.py +714 -0
  35. MEDiml/learning/Results.py +2237 -0
  36. MEDiml/learning/Stats.py +694 -0
  37. MEDiml/learning/__init__.py +10 -0
  38. MEDiml/learning/cleaning_utils.py +107 -0
  39. MEDiml/learning/ml_utils.py +1015 -0
  40. MEDiml/processing/__init__.py +6 -0
  41. MEDiml/processing/compute_suv_map.py +121 -0
  42. MEDiml/processing/discretisation.py +149 -0
  43. MEDiml/processing/interpolation.py +275 -0
  44. MEDiml/processing/resegmentation.py +66 -0
  45. MEDiml/processing/segmentation.py +912 -0
  46. MEDiml/utils/__init__.py +25 -0
  47. MEDiml/utils/batch_patients.py +45 -0
  48. MEDiml/utils/create_radiomics_table.py +131 -0
  49. MEDiml/utils/data_frame_export.py +42 -0
  50. MEDiml/utils/find_process_names.py +16 -0
  51. MEDiml/utils/get_file_paths.py +34 -0
  52. MEDiml/utils/get_full_rad_names.py +21 -0
  53. MEDiml/utils/get_institutions_from_ids.py +16 -0
  54. MEDiml/utils/get_patient_id_from_scan_name.py +22 -0
  55. MEDiml/utils/get_patient_names.py +26 -0
  56. MEDiml/utils/get_radiomic_names.py +27 -0
  57. MEDiml/utils/get_scan_name_from_rad_name.py +22 -0
  58. MEDiml/utils/image_reader_SITK.py +37 -0
  59. MEDiml/utils/image_volume_obj.py +22 -0
  60. MEDiml/utils/imref.py +340 -0
  61. MEDiml/utils/initialize_features_names.py +62 -0
  62. MEDiml/utils/inpolygon.py +159 -0
  63. MEDiml/utils/interp3.py +43 -0
  64. MEDiml/utils/json_utils.py +78 -0
  65. MEDiml/utils/mode.py +31 -0
  66. MEDiml/utils/parse_contour_string.py +58 -0
  67. MEDiml/utils/save_MEDscan.py +30 -0
  68. MEDiml/utils/strfind.py +32 -0
  69. MEDiml/utils/textureTools.py +188 -0
  70. MEDiml/utils/texture_features_names.py +115 -0
  71. MEDiml/utils/write_radiomics_csv.py +47 -0
  72. MEDiml/wrangling/DataManager.py +1724 -0
  73. MEDiml/wrangling/ProcessDICOM.py +512 -0
  74. MEDiml/wrangling/__init__.py +3 -0
  75. mediml-0.9.9.dist-info/LICENSE.md +674 -0
  76. mediml-0.9.9.dist-info/METADATA +232 -0
  77. mediml-0.9.9.dist-info/RECORD +78 -0
  78. mediml-0.9.9.dist-info/WHEEL +4 -0
@@ -0,0 +1,840 @@
1
+ import logging
2
+ import math
3
+ import os
4
+ import pickle
5
+ import sys
6
+ from copy import deepcopy
7
+ from datetime import datetime
8
+ from itertools import product
9
+ from pathlib import Path
10
+ from time import time
11
+ from typing import Dict, List, Union
12
+
13
+ import numpy as np
14
+ import pandas as pd
15
+ import ray
16
+ from tqdm import trange
17
+
18
+ import MEDiml
19
+
20
+
21
+ class BatchExtractorTexturalFilters(object):
22
+ """
23
+ Organizes all the patients/scans in batches to extract all the radiomic features
24
+ """
25
+
26
+ def __init__(
27
+ self,
28
+ path_read: Union[str, Path],
29
+ path_csv: Union[str, Path],
30
+ path_params: Union[str, Path],
31
+ path_save: Union[str, Path],
32
+ n_batch: int = 4
33
+ ) -> None:
34
+ """
35
+ constructor of the BatchExtractor class
36
+ """
37
+ self._path_csv = Path(path_csv)
38
+ self._path_params = Path(path_params)
39
+ self._path_read = Path(path_read)
40
+ self._path_save = Path(path_save)
41
+ self.roi_types = []
42
+ self.roi_type_labels = []
43
+ self.n_bacth = n_batch
44
+ self.glcm_features = [
45
+ "Fcm_joint_max",
46
+ "Fcm_joint_avg",
47
+ "Fcm_joint_var",
48
+ "Fcm_joint_entr",
49
+ "Fcm_diff_avg",
50
+ "Fcm_diff_var",
51
+ "Fcm_diff_entr",
52
+ "Fcm_sum_avg",
53
+ "Fcm_sum_var",
54
+ "Fcm_sum_entr",
55
+ "Fcm_energy",
56
+ "Fcm_contrast",
57
+ "Fcm_dissimilarity",
58
+ "Fcm_inv_diff",
59
+ "Fcm_inv_diff_norm",
60
+ "Fcm_inv_diff_mom",
61
+ "Fcm_inv_diff_mom_norm",
62
+ "Fcm_inv_var",
63
+ "Fcm_corr",
64
+ "Fcm_auto_corr",
65
+ "Fcm_clust_tend",
66
+ "Fcm_clust_shade",
67
+ "Fcm_clust_prom",
68
+ "Fcm_info_corr1",
69
+ "Fcm_info_corr2"
70
+ ]
71
+
72
+ def __load_and_process_params(self) -> Dict:
73
+ """Load and process the computing & batch parameters from JSON file"""
74
+ # Load json parameters
75
+ im_params = MEDiml.utils.json_utils.load_json(self._path_params)
76
+
77
+ # Update class attributes
78
+ self.roi_types.extend(im_params['roi_types'])
79
+ self.roi_type_labels.extend(im_params['roi_type_labels'])
80
+ self.n_bacth = im_params['n_batch'] if 'n_batch' in im_params else self.n_bacth
81
+
82
+ return im_params
83
+
84
+ def __compute_radiomics_one_patient(
85
+ self,
86
+ name_patient: str,
87
+ roi_name: str,
88
+ im_params: Dict,
89
+ roi_type: str,
90
+ roi_type_label: str,
91
+ log_file: Union[Path, str],
92
+ skip_existing: bool
93
+ ) -> str:
94
+ """
95
+ Computes all radiomics features (Texture & Non-texture) for one patient/scan
96
+
97
+ Args:
98
+ name_patient(str): scan or patient full name. It has to respect the MEDiml naming convention:
99
+ PatientID__ImagingScanName.ImagingModality.npy
100
+ roi_name(str): name of the ROI that will be used in computation.
101
+ im_params(Dict): Dict of parameters/settings that will be used in the processing and computation.
102
+ roi_type(str): Type of ROI used in the processing and computation (for identification purposes)
103
+ roi_type_label(str): Label of the ROI used, to make it identifiable from other ROIs.
104
+ log_file(Union[Path, str]): Path to the logging file.
105
+ skip_existing(bool): True to skip the computation of the features for the scans that already have been computed.
106
+
107
+ Returns:
108
+ Union[Path, str]: Path to the updated logging file.
109
+ """
110
+ # Check if the features for the current filter have already been computed
111
+ if skip_existing:
112
+ list_feature = []
113
+ # Find the glcm filters that have not been computed yet
114
+ for i in range(len(self.glcm_features)):
115
+ index_dot = name_patient.find('.')
116
+ ext = name_patient.find('.npy')
117
+ name_save = name_patient[:index_dot] + '(' + roi_type_label + ')' + name_patient[index_dot : ext] + ".json"
118
+ name_roi_type = roi_type + '_' + self.glcm_features[i]
119
+ path_to_check = Path(self._path_save / f'features({name_roi_type})')
120
+ if not (path_to_check / name_save).exists():
121
+ list_feature.append(i)
122
+ # If all the features have already been computed, skip the computation
123
+ if len(list_feature) == 0:
124
+ return log_file
125
+
126
+ # Setting up logging settings
127
+ logging.basicConfig(filename=log_file, level=logging.DEBUG, force=True)
128
+
129
+ # start timer
130
+ t_start = time()
131
+
132
+ # Initialization
133
+ message = f"\n***************** COMPUTING FEATURES: {name_patient} *****************"
134
+ logging.info(message)
135
+
136
+ # Load MEDscan instance
137
+ try:
138
+ with open(self._path_read / name_patient, 'rb') as f: medscan = pickle.load(f)
139
+ medscan = MEDiml.MEDscan(medscan)
140
+ except Exception as e:
141
+ logging.error(f"\n ERROR LOADING PATIENT {name_patient}:\n {e}")
142
+ return None
143
+
144
+ # Init processing & computation parameters
145
+ medscan.init_params(im_params)
146
+ logging.debug('Parameters parsed, json file is valid.')
147
+
148
+ # Get ROI (region of interest)
149
+ logging.info("\n--> Extraction of ROI mask:")
150
+ try:
151
+ vol_obj_init, roi_obj_init = MEDiml.processing.get_roi_from_indexes(
152
+ medscan,
153
+ name_roi=roi_name,
154
+ box_string=medscan.params.process.box_string
155
+ )
156
+ except:
157
+ # if for the current scan ROI is not found, computation is aborted.
158
+ return log_file
159
+
160
+ start = time()
161
+ message = '--> Non-texture features pre-processing (interp + re-seg) for "Scale={}"'.\
162
+ format(str(medscan.params.process.scale_non_text[0]))
163
+ logging.info(message)
164
+
165
+ # Interpolation
166
+ # Intensity Mask
167
+ vol_obj = MEDiml.processing.interp_volume(
168
+ medscan=medscan,
169
+ vol_obj_s=vol_obj_init,
170
+ vox_dim=medscan.params.process.scale_non_text,
171
+ interp_met=medscan.params.process.vol_interp,
172
+ round_val=medscan.params.process.gl_round,
173
+ image_type='image',
174
+ roi_obj_s=roi_obj_init,
175
+ box_string=medscan.params.process.box_string
176
+ )
177
+ # Morphological Mask
178
+ roi_obj_morph = MEDiml.processing.interp_volume(
179
+ medscan=medscan,
180
+ vol_obj_s=roi_obj_init,
181
+ vox_dim=medscan.params.process.scale_non_text,
182
+ interp_met=medscan.params.process.roi_interp,
183
+ round_val=medscan.params.process.roi_pv,
184
+ image_type='roi',
185
+ roi_obj_s=roi_obj_init,
186
+ box_string=medscan.params.process.box_string
187
+ )
188
+
189
+ # Re-segmentation
190
+ # Intensity mask range re-segmentation
191
+ roi_obj_int = deepcopy(roi_obj_morph)
192
+ roi_obj_int.data = MEDiml.processing.range_re_seg(
193
+ vol=vol_obj.data,
194
+ roi=roi_obj_int.data,
195
+ im_range=medscan.params.process.im_range
196
+ )
197
+ # Intensity mask outlier re-segmentation
198
+ roi_obj_int.data = np.logical_and(
199
+ MEDiml.processing.outlier_re_seg(
200
+ vol=vol_obj.data,
201
+ roi=roi_obj_int.data,
202
+ outliers=medscan.params.process.outliers
203
+ ),
204
+ roi_obj_int.data
205
+ ).astype(int)
206
+ logging.info(f"{time() - start}\n")
207
+
208
+ # Reset timer
209
+ start = time()
210
+
211
+ # Image textural filtering
212
+ logging.info("--> Image textural filtering:")
213
+
214
+ # Preparation of computation :
215
+ medscan.init_ntf_calculation(vol_obj)
216
+
217
+ # ROI Extraction :
218
+ try:
219
+ vol_int_re = MEDiml.processing.roi_extract(
220
+ vol=vol_obj.data,
221
+ roi=roi_obj_int.data
222
+ )
223
+ except Exception as e:
224
+ print(name_patient, e)
225
+ return log_file
226
+
227
+ # Apply textural filter
228
+ try:
229
+ if medscan.params.process.user_set_min_value is None:
230
+ medscan.params.process.user_set_min_value = np.nanmin(vol_int_re)
231
+ vol_obj_all_features = MEDiml.filters.apply_filter(
232
+ medscan,
233
+ vol_int_re,
234
+ user_set_min_val=medscan.params.process.user_set_min_value
235
+ )
236
+ except Exception as e:
237
+ print(e)
238
+ logging.error(f'PROBLEM WITH TEXTURAL FILTERING: {e}')
239
+ return log_file
240
+
241
+ # Initialize ray
242
+ if ray.is_initialized():
243
+ ray.shutdown()
244
+
245
+ ray.init(local_mode=True, include_dashboard=True, num_cpus=self.n_bacth)
246
+
247
+ # Loop through all the filters and extract the features for each filter
248
+ ids = []
249
+ nb_filters = len(list_feature)
250
+ if nb_filters < self.n_bacth:
251
+ self.n_bacth = nb_filters
252
+ for i in range(self.n_bacth):
253
+ # Extract the filtered volume
254
+ filter_idx = list_feature[i]
255
+ vol_obj.data = deepcopy(vol_obj_all_features[...,filter_idx])
256
+
257
+ # Compute radiomics features
258
+ logging.info(f"--> Computation of radiomics features for filter {filter_idx}:")
259
+
260
+ ids.append(
261
+ self.__compute_radiomics_filtered_volume.remote(
262
+ self,
263
+ medscan=medscan,
264
+ vol_obj=vol_obj,
265
+ roi_obj_int=roi_obj_int,
266
+ roi_obj_morph=roi_obj_morph,
267
+ name_patient=name_patient,
268
+ roi_name=roi_name,
269
+ roi_type=roi_type + '_' + self.glcm_features[filter_idx],
270
+ roi_type_label=roi_type_label,
271
+ log_file=log_file
272
+ )
273
+ )
274
+ # Distribute the remaining tasks
275
+ nb_job_left = nb_filters - self.n_bacth
276
+ if nb_job_left > 0:
277
+ for i in range(nb_filters - nb_job_left, nb_filters):
278
+ ready, not_ready = ray.wait(ids, num_returns=1)
279
+ ids = not_ready
280
+ try:
281
+ log_file = ray.get(ready)[0]
282
+ except:
283
+ pass
284
+ # Extract the filtered volume
285
+ filter_idx = list_feature[i]
286
+ vol_obj.data = deepcopy(vol_obj_all_features[...,filter_idx])
287
+
288
+ # Compute radiomics features
289
+ logging.info(f"--> Computation of radiomics features for filter {filter_idx}:")
290
+
291
+ ids.append(
292
+ self.__compute_radiomics_filtered_volume.remote(
293
+ self,
294
+ medscan=medscan,
295
+ vol_obj=vol_obj,
296
+ roi_obj_int=roi_obj_int,
297
+ roi_obj_morph=roi_obj_morph,
298
+ name_patient=name_patient,
299
+ roi_name=roi_name,
300
+ roi_type=roi_type + '_' + self.glcm_features[filter_idx],
301
+ roi_type_label=roi_type_label,
302
+ log_file=log_file
303
+ )
304
+ )
305
+
306
+ logging.info(f"TOTAL TIME:{time() - t_start} seconds\n\n")
307
+
308
+ # Empty memory
309
+ del medscan
310
+
311
+ @ray.remote
312
+ def __compute_radiomics_filtered_volume(
313
+ self,
314
+ medscan: MEDiml.MEDscan,
315
+ vol_obj,
316
+ roi_obj_int,
317
+ roi_obj_morph,
318
+ name_patient,
319
+ roi_name,
320
+ roi_type,
321
+ roi_type_label,
322
+ log_file
323
+ ) -> Union[Path, str]:
324
+
325
+ # time
326
+ t_start = time()
327
+
328
+ # ROI Extraction :
329
+ vol_int_re = deepcopy(vol_obj.data)
330
+
331
+ # check if ROI is empty
332
+ if math.isnan(np.nanmax(vol_int_re)) and math.isnan(np.nanmin(vol_int_re)):
333
+ logging.error(f'PROBLEM WITH INTENSITY MASK. ROI {roi_name} IS EMPTY.')
334
+ return log_file
335
+
336
+ # Computation of non-texture features
337
+ logging.info("--> Computation of non-texture features:")
338
+
339
+ # Morphological features extraction
340
+ try:
341
+ morph = MEDiml.biomarkers.morph.extract_all(
342
+ vol=vol_obj.data,
343
+ mask_int=roi_obj_int.data,
344
+ mask_morph=roi_obj_morph.data,
345
+ res=medscan.params.process.scale_non_text,
346
+ intensity_type=medscan.params.process.intensity_type
347
+ )
348
+ except Exception as e:
349
+ logging.error(f'PROBLEM WITH COMPUTATION OF MORPHOLOGICAL FEATURES {e}')
350
+ morph = None
351
+
352
+ # Local intensity features extraction
353
+ try:
354
+ local_intensity = MEDiml.biomarkers.local_intensity.extract_all(
355
+ img_obj=vol_obj.data,
356
+ roi_obj=roi_obj_int.data,
357
+ res=medscan.params.process.scale_non_text,
358
+ intensity_type=medscan.params.process.intensity_type
359
+ )
360
+ except Exception as e:
361
+ logging.error(f'PROBLEM WITH COMPUTATION OF LOCAL INTENSITY FEATURES {e}')
362
+ local_intensity = None
363
+
364
+ # statistical features extraction
365
+ try:
366
+ stats = MEDiml.biomarkers.stats.extract_all(
367
+ vol=vol_int_re,
368
+ intensity_type=medscan.params.process.intensity_type
369
+ )
370
+ except Exception as e:
371
+ logging.error(f'PROBLEM WITH COMPUTATION OF STATISTICAL FEATURES {e}')
372
+ stats = None
373
+
374
+ # Intensity histogram equalization of the imaging volume
375
+ vol_quant_re, _ = MEDiml.processing.discretize(
376
+ vol_re=vol_int_re,
377
+ discr_type=medscan.params.process.ih['type'],
378
+ n_q=medscan.params.process.ih['val'],
379
+ user_set_min_val=medscan.params.process.user_set_min_value
380
+ )
381
+
382
+ # Intensity histogram features extraction
383
+ try:
384
+ int_hist = MEDiml.biomarkers.intensity_histogram.extract_all(
385
+ vol=vol_quant_re
386
+ )
387
+ except Exception as e:
388
+ logging.error(f'PROBLEM WITH COMPUTATION OF INTENSITY HISTOGRAM FEATURES {e}')
389
+ int_hist = None
390
+
391
+ # Intensity histogram equalization of the imaging volume
392
+ if medscan.params.process.ivh and 'type' in medscan.params.process.ivh and 'val' in medscan.params.process.ivh:
393
+ if medscan.params.process.ivh['type'] and medscan.params.process.ivh['val']:
394
+ vol_quant_re, wd = MEDiml.processing.discretize(
395
+ vol_re=vol_int_re,
396
+ discr_type=medscan.params.process.ivh['type'],
397
+ n_q=medscan.params.process.ivh['val'],
398
+ user_set_min_val=medscan.params.process.user_set_min_value,
399
+ ivh=True
400
+ )
401
+ else:
402
+ vol_quant_re = vol_int_re
403
+ wd = 1
404
+
405
+ # Intensity volume histogram features extraction
406
+ try:
407
+ int_vol_hist = MEDiml.biomarkers.int_vol_hist.extract_all(
408
+ medscan=medscan,
409
+ vol=vol_quant_re,
410
+ vol_int_re=vol_int_re,
411
+ wd=wd
412
+ )
413
+ except:
414
+ print("Error ivh:",name_patient)
415
+ int_vol_hist = {'Fivh_V10': [],
416
+ 'Fivh_V90': [],
417
+ 'Fivh_I10': [],
418
+ 'Fivh_I90': [],
419
+ 'Fivh_V10minusV90': [],
420
+ 'Fivh_I10minusI90': [],
421
+ 'Fivh_auc': []
422
+ }
423
+
424
+ # End of Non-Texture features extraction
425
+ logging.info(f"End of non-texture features extraction: {time() - t_start}\n")
426
+
427
+ # Computation of texture features
428
+ logging.info("--> Computation of texture features:")
429
+
430
+ # Compute radiomics features for each scale text
431
+ count = 0
432
+ logging.info(f"{time() - t_start}\n")
433
+
434
+ # Compute features for each discretisation algorithm and for each grey-level
435
+ for a, n in product(range(medscan.params.process.n_algo), range(medscan.params.process.n_gl)):
436
+ count += 1
437
+ start = time()
438
+ message = '--> Computation of texture features in image ' \
439
+ 'space for "Scale= {}", "Algo={}", "GL={}" ({}):'.format(
440
+ str(medscan.params.process.scale_text[0][1]),
441
+ medscan.params.process.algo[a],
442
+ str(medscan.params.process.gray_levels[a][n]),
443
+ str(count) + '/' + str(medscan.params.process.n_exp)
444
+ )
445
+ logging.info(message)
446
+
447
+ # Preparation of computation :
448
+ medscan.init_tf_calculation(algo=a, gl=n, scale=0)
449
+
450
+ # Discretisation :
451
+ try:
452
+ vol_quant_re, _ = MEDiml.processing.discretize(
453
+ vol_re=vol_int_re,
454
+ discr_type=medscan.params.process.algo[a],
455
+ n_q=medscan.params.process.gray_levels[a][n],
456
+ user_set_min_val=medscan.params.process.user_set_min_value
457
+ )
458
+ except Exception as e:
459
+ logging.error(f'PROBLEM WITH DISCRETIZATION: {e}')
460
+ vol_quant_re = None
461
+
462
+ # GLCM features extraction
463
+ try:
464
+ glcm = MEDiml.biomarkers.glcm.extract_all(
465
+ vol=vol_quant_re,
466
+ dist_correction=medscan.params.radiomics.glcm.dist_correction
467
+ )
468
+ except Exception as e:
469
+ logging.error(f'PROBLEM WITH COMPUTATION OF GLCM FEATURES {e}')
470
+ glcm = None
471
+
472
+ # GLRLM features extraction
473
+ try:
474
+ glrlm = MEDiml.biomarkers.glrlm.extract_all(
475
+ vol=vol_quant_re,
476
+ dist_correction=medscan.params.radiomics.glrlm.dist_correction
477
+ )
478
+ except Exception as e:
479
+ logging.error(f'PROBLEM WITH COMPUTATION OF GLRLM FEATURES {e}')
480
+ glrlm = None
481
+
482
+ # GLSZM features extraction
483
+ try:
484
+ glszm = MEDiml.biomarkers.glszm.extract_all(vol=vol_quant_re)
485
+ except Exception as e:
486
+ logging.error(f'PROBLEM WITH COMPUTATION OF GLSZM FEATURES {e}')
487
+ glszm = None
488
+
489
+ # GLDZM features extraction
490
+ try:
491
+ gldzm = MEDiml.biomarkers.gldzm.extract_all(
492
+ vol_int=vol_quant_re,
493
+ mask_morph=roi_obj_morph.data
494
+ )
495
+ except Exception as e:
496
+ logging.error(f'PROBLEM WITH COMPUTATION OF GLDZM FEATURES {e}')
497
+ gldzm = None
498
+
499
+ # NGTDM features extraction
500
+ try:
501
+ ngtdm = MEDiml.biomarkers.ngtdm.extract_all(
502
+ vol=vol_quant_re,
503
+ dist_correction=medscan.params.radiomics.ngtdm.dist_correction
504
+ )
505
+ except Exception as e:
506
+ logging.error(f'PROBLEM WITH COMPUTATION OF NGTDM FEATURES {e}')
507
+ ngtdm = None
508
+
509
+ # NGLDM features extraction
510
+ try:
511
+ ngldm = MEDiml.biomarkers.ngldm.extract_all(vol=vol_quant_re)
512
+ except Exception as e:
513
+ logging.error(f'PROBLEM WITH COMPUTATION OF NGLDM FEATURES {e}')
514
+ ngldm = None
515
+
516
+ # Update radiomics results class
517
+ medscan.update_radiomics(
518
+ int_vol_hist_features=int_vol_hist,
519
+ morph_features=morph,
520
+ loc_int_features=local_intensity,
521
+ stats_features=stats,
522
+ int_hist_features=int_hist,
523
+ glcm_features=glcm,
524
+ glrlm_features=glrlm,
525
+ glszm_features=glszm,
526
+ gldzm_features=gldzm,
527
+ ngtdm_features=ngtdm,
528
+ ngldm_features=ngldm
529
+ )
530
+
531
+ # End of texture features extraction
532
+ logging.info(f"End of texture features extraction: {time() - start}\n")
533
+
534
+ # Saving radiomics results
535
+ medscan.save_radiomics(
536
+ scan_file_name=name_patient,
537
+ path_save=self._path_save,
538
+ roi_type=roi_type,
539
+ roi_type_label=roi_type_label,
540
+ )
541
+
542
+ logging.info(f"TOTAL TIME 1 FILTER:{time() - t_start} seconds\n\n")
543
+
544
+ return log_file
545
+
546
+ @ray.remote
547
+ def __compute_radiomics_tables(
548
+ self,
549
+ table_tags: List,
550
+ log_file: Union[str, Path],
551
+ im_params: Dict,
552
+ feature_name: str
553
+ ) -> None:
554
+ """
555
+ Creates radiomic tables off of the saved dicts with the computed features and save it as CSV files
556
+
557
+ Args:
558
+ table_tags(List): Lists of information about scans, roi type and imaging space (or filter space)
559
+ log_file(Union[str, Path]): Path to logging file.
560
+ im_params(Dict): Dictionary of parameters.
561
+
562
+ Returns:
563
+ None.
564
+ """
565
+ n_tables = len(table_tags)
566
+
567
+ for t in range(0, n_tables):
568
+ scan = table_tags[t][0]
569
+ roi_type = table_tags[t][1]
570
+ roi_label = table_tags[t][2]
571
+ im_space = table_tags[t][3]
572
+ modality = table_tags[t][4]
573
+
574
+ # extract parameters for the current modality
575
+ if modality == 'CTscan' and 'imParamCT' in im_params:
576
+ im_params_mod = im_params['imParamCT']
577
+ elif modality== 'MRscan' and 'imParamMR' in im_params:
578
+ im_params_mod = im_params['imParamMR']
579
+ elif modality == 'PTscan' and 'imParamPET' in im_params:
580
+ im_params_mod = im_params['imParamPET']
581
+ # extract name save of the used filter
582
+ if 'filter_type' in im_params_mod:
583
+ filter_type = im_params_mod['filter_type']
584
+ if filter_type in im_params['imParamFilter'] and 'name_save' in im_params['imParamFilter'][filter_type]:
585
+ name_save = im_params['imParamFilter'][filter_type]['name_save'] + '_' + feature_name
586
+ else:
587
+ name_save= feature_name
588
+ else:
589
+ name_save= feature_name
590
+
591
+ # set up table name
592
+ if name_save:
593
+ name_table = 'radiomics__' + scan + \
594
+ '(' + roi_type + ')__' + name_save + '.npy'
595
+ else:
596
+ name_table = 'radiomics__' + scan + \
597
+ '(' + roi_type + ')__' + im_space + '.npy'
598
+
599
+ # Start timer
600
+ start = time()
601
+ logging.info("\n --> Computing radiomics table: {name_table}...")
602
+
603
+ # Wildcard used to look only in the parent folder (save path),
604
+ # no need to recursively look into sub-folders using '**/'.
605
+ wildcard = '*_' + scan + '(' + roi_type + ')*.json'
606
+
607
+ # Create radiomics table
608
+ radiomics_table_dict = MEDiml.utils.create_radiomics_table(
609
+ MEDiml.utils.get_file_paths(self._path_save / f'features({roi_label})', wildcard),
610
+ im_space,
611
+ log_file
612
+ )
613
+ radiomics_table_dict['Properties']['Description'] = name_table
614
+
615
+ # Save radiomics table
616
+ save_path = self._path_save / f'features({roi_label})' / name_table
617
+ np.save(save_path, [radiomics_table_dict])
618
+
619
+ # Create CSV table and Definitions
620
+ MEDiml.utils.write_radiomics_csv(save_path)
621
+
622
+ logging.info(f"DONE\n {time() - start}\n")
623
+
624
+ return log_file
625
+
626
+ def __batch_all_patients(self, im_params: Dict, skip_existing) -> None:
627
+ """
628
+ Create batches of scans to process and compute radiomics features for every single scan.
629
+
630
+ Args:
631
+ im_params(Dict): Dict of the processing & computation parameters.
632
+ skip_existing(bool) : True to skip the computation of the features for the scans that already have been computed.
633
+
634
+ Returns:
635
+ None
636
+ """
637
+ # create a batch for each roi type
638
+ n_roi_types = len(self.roi_type_labels)
639
+ for r in range(0, n_roi_types):
640
+ roi_type = self.roi_types[r]
641
+ roi_type_label = self.roi_type_labels[r]
642
+ print(f'\n --> Computing features for the "{roi_type_label}" roi type ...', end = '')
643
+
644
+ # READING CSV EXPERIMENT TABLE
645
+ tabel_roi = pd.read_csv(self._path_csv / ('roiNames_' + roi_type_label + '.csv'))
646
+ tabel_roi['under'] = '_'
647
+ tabel_roi['dot'] = '.'
648
+ tabel_roi['npy'] = '.npy'
649
+ name_patients = (pd.Series(
650
+ tabel_roi[['PatientID', 'under', 'under',
651
+ 'ImagingScanName',
652
+ 'dot',
653
+ 'ImagingModality',
654
+ 'npy']].fillna('').values.tolist()).str.join('')).tolist()
655
+ tabel_roi = tabel_roi.drop(columns=['under', 'under', 'dot', 'npy'])
656
+ roi_names = tabel_roi.ROIname.tolist()
657
+
658
+ # INITIALIZATION
659
+ os.chdir(self._path_save)
660
+ name_bacth_log = 'batchLog_' + roi_type_label
661
+ p = Path.cwd().glob('*')
662
+ files = [x for x in p if x.is_dir()]
663
+ n_files = len(files)
664
+ exist_file = name_bacth_log in [x.name for x in files]
665
+ if exist_file and (n_files > 0):
666
+ for i in range(0, n_files):
667
+ if (files[i].name == name_bacth_log):
668
+ mod_timestamp = datetime.fromtimestamp(
669
+ Path(files[i]).stat().st_mtime)
670
+ date = mod_timestamp.strftime("%d-%b-%Y_%HH%MM%SS")
671
+ new_name = name_bacth_log+'_'+date
672
+ if sys.platform == 'win32':
673
+ os.system('move ' + name_bacth_log + ' ' + new_name)
674
+ else:
675
+ os.system('mv ' + name_bacth_log + ' ' + new_name)
676
+
677
+ os.makedirs(name_bacth_log, 0o777, True)
678
+ path_batch = Path.cwd() / name_bacth_log
679
+
680
+ # PRODUCE BATCH COMPUTATIONS
681
+ n_patients = len(name_patients)
682
+
683
+ # Produce a list log_file path.
684
+ log_files = [path_batch / ('log_file_' + str(i) + '.log') for i in range(n_patients)]
685
+
686
+ # Features computation for each patient (patients loop)
687
+ for i in trange(n_patients):
688
+ self.__compute_radiomics_one_patient(
689
+ name_patients[i],
690
+ roi_names[i],
691
+ im_params,
692
+ roi_type,
693
+ roi_type_label,
694
+ log_files[i],
695
+ skip_existing
696
+ )
697
+
698
+ print('DONE')
699
+
700
+ def __batch_all_tables(self, im_params: Dict):
701
+ """
702
+ Create batches of tables of the extracted features for every imaging scan type (CT, PET...).
703
+
704
+ Args:
705
+ im_params(Dict): Dictionary of parameters.
706
+
707
+ Returns:
708
+ None
709
+ """
710
+ # INITIALIZATION
711
+ os.chdir(self._path_save)
712
+ name_batch_log = 'batchLog_tables'
713
+ p = Path.cwd().glob('*')
714
+ files = [x for x in p if x.is_dir()]
715
+ n_files = len(files)
716
+ exist_file = name_batch_log in [x.name for x in files]
717
+ if exist_file and (n_files > 0):
718
+ for i in range(0, n_files):
719
+ if files[i].name == name_batch_log:
720
+ mod_timestamp = datetime.fromtimestamp(
721
+ Path(files[i]).stat().st_mtime)
722
+ date = mod_timestamp.strftime("%d-%b-%Y_%H:%M:%S")
723
+ new_name = name_batch_log+'_'+date
724
+ if sys.platform == 'win32':
725
+ os.system('move ' + name_batch_log + ' ' + new_name)
726
+ else:
727
+ os.system('mv ' + name_batch_log + ' ' + new_name)
728
+
729
+ os.makedirs(name_batch_log, 0o777, True)
730
+ path_batch = Path.cwd()
731
+
732
+ # GETTING COMBINATIONS OF scan, roi_type and imageSpaces
733
+ n_roi_types = len(self.roi_type_labels)
734
+
735
+ # Get all scan names present for the given roi_type_label
736
+ for f_idx in range(0, len(self.glcm_features)):
737
+ # RE-INITIALIZATION
738
+ table_tags = []
739
+ for r in range(0, n_roi_types):
740
+ label = self.roi_type_labels[r]
741
+ wildcard = '*' + label + '*.json'
742
+ roi_type = self.roi_types[r] + '_' + self.glcm_features[f_idx]
743
+ file_paths = MEDiml.utils.get_file_paths(self._path_save / f'features({roi_type})', wildcard)
744
+ n_files = len(file_paths)
745
+ scans = [0] * n_files
746
+ modalities = [0] * n_files
747
+ for f in range(0, n_files):
748
+ rad_file_name = file_paths[f].stem
749
+ scans[f] = MEDiml.utils.get_scan_name_from_rad_name(rad_file_name)
750
+ modalities[f] = rad_file_name.split('.')[1]
751
+ scans = s = (np.unique(np.array(scans))).tolist()
752
+ n_scans = len(scans)
753
+ # Get all scan names present for the given roi_type_label and scans
754
+ for s in range(0, n_scans):
755
+ scan = scans[s]
756
+ modality = modalities[s]
757
+ wildcard = '*' + scan + '(' + label + ')*.json'
758
+ file_paths = MEDiml.utils.get_file_paths(self._path_save / f'features({roi_type})', wildcard)
759
+ n_files = len(file_paths)
760
+
761
+ # Finding the images spaces for a test file (assuming that all
762
+ # files for a given scan and roi_type_label have the same image spaces
763
+ radiomics = MEDiml.utils.json_utils.load_json(file_paths[0])
764
+ im_spaces = [key for key in radiomics.keys()]
765
+ im_spaces = im_spaces[:-1]
766
+ n_im_spaces = len(im_spaces)
767
+ # Constructing the table_tags variable
768
+ for i in range(0, n_im_spaces):
769
+ im_space = im_spaces[i]
770
+ table_tags = table_tags + [[scan, label, roi_type, im_space, modality]]
771
+
772
+ # PRODUCE BATCH COMPUTATIONS
773
+ n_tables = len(table_tags)
774
+ self.n_bacth = self.n_bacth
775
+ if self.n_bacth is None or self.n_bacth < 0:
776
+ self.n_bacth = 1
777
+ elif n_tables < self.n_bacth:
778
+ self.n_bacth = n_tables
779
+
780
+ # Produce a list log_file path.
781
+ log_files = [path_batch / ('log_file_' + str(i) + '.txt') for i in range(self.n_bacth)]
782
+
783
+ # Initialize ray
784
+ if ray.is_initialized():
785
+ ray.shutdown()
786
+
787
+ ray.init(local_mode=True, include_dashboard=True, num_cpus=self.n_bacth)
788
+
789
+ # Distribute the first tasks to all workers
790
+ ids = [self.__compute_radiomics_tables.remote(
791
+ self,
792
+ [table_tags[i]],
793
+ log_files[i],
794
+ im_params,
795
+ self.glcm_features[f_idx])
796
+ for i in range(self.n_bacth)]
797
+
798
+ nb_job_left = n_tables - self.n_bacth
799
+
800
+ for _ in trange(n_tables):
801
+ ready, not_ready = ray.wait(ids, num_returns=1)
802
+ ids = not_ready
803
+
804
+ # We verify if error has occur during the process
805
+ log_file = ray.get(ready)[0]
806
+
807
+ # Distribute the remaining tasks
808
+ if nb_job_left > 0:
809
+ idx = n_tables - nb_job_left
810
+ ids.extend([self.__compute_radiomics_tables.remote(
811
+ self,
812
+ [table_tags[idx]],
813
+ log_file,
814
+ im_params,
815
+ self.glcm_features[f_idx])])
816
+ nb_job_left -= 1
817
+
818
+ print('DONE')
819
+
820
+ def compute_radiomics(self, create_tables: bool = True, skip_existing: bool = False) -> None:
821
+ """Compute all radiomic features for all scans in the CSV file (set in initialization) and organize it
822
+ in JSON and CSV files
823
+
824
+ Args:
825
+ create_tables(bool) : True to create CSV tables for the extracted features and not save it in JSON only.
826
+ skip_existing(bool) : True to skip the computation of the features for the scans that already have been computed.
827
+
828
+ Returns:
829
+ None.
830
+ """
831
+
832
+ # Load and process computing parameters
833
+ im_params = self.__load_and_process_params()
834
+
835
+ # Batch all scans from CSV file and compute radiomics for each scan
836
+ self.__batch_all_patients(im_params, skip_existing)
837
+
838
+ # Create a CSV file off of the computed features for all the scans
839
+ if create_tables:
840
+ self.__batch_all_tables(im_params)