qmenta-client 1.1.dev1507__py3-none-any.whl → 2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qmenta/client/Account.py +43 -11
- qmenta/client/File.py +44 -12
- qmenta/client/Project.py +908 -441
- qmenta/client/Subject.py +10 -3
- qmenta/client/utils.py +6 -2
- {qmenta_client-1.1.dev1507.dist-info → qmenta_client-2.1.dist-info}/METADATA +3 -2
- qmenta_client-2.1.dist-info/RECORD +10 -0
- {qmenta_client-1.1.dev1507.dist-info → qmenta_client-2.1.dist-info}/WHEEL +1 -1
- qmenta_client-1.1.dev1507.dist-info/RECORD +0 -10
qmenta/client/Project.py
CHANGED
|
@@ -9,9 +9,10 @@ import sys
|
|
|
9
9
|
import time
|
|
10
10
|
from collections import defaultdict
|
|
11
11
|
from enum import Enum
|
|
12
|
-
|
|
12
|
+
from typing import Any, Dict, Union
|
|
13
13
|
from qmenta.core import errors
|
|
14
14
|
from qmenta.core import platform
|
|
15
|
+
from qmenta.core.upload.single import SingleUpload, FileInfo, UploadStatus
|
|
15
16
|
|
|
16
17
|
from qmenta.client import Account
|
|
17
18
|
|
|
@@ -21,6 +22,17 @@ if sys.version_info[0] == 3:
|
|
|
21
22
|
|
|
22
23
|
logger_name = "qmenta.client"
|
|
23
24
|
OPERATOR_LIST = ["eq", "ne", "gt", "gte", "lt", "lte"]
|
|
25
|
+
ANALYSIS_NAME_EXCLUDED_CHARACTERS = [
|
|
26
|
+
"\\",
|
|
27
|
+
"[",
|
|
28
|
+
"]",
|
|
29
|
+
"(",
|
|
30
|
+
")",
|
|
31
|
+
"{",
|
|
32
|
+
"}",
|
|
33
|
+
"+",
|
|
34
|
+
"*",
|
|
35
|
+
]
|
|
24
36
|
|
|
25
37
|
|
|
26
38
|
def convert_qc_value_to_qcstatus(value):
|
|
@@ -46,7 +58,9 @@ def convert_qc_value_to_qcstatus(value):
|
|
|
46
58
|
elif value == "":
|
|
47
59
|
return QCStatus.UNDERTERMINED
|
|
48
60
|
else:
|
|
49
|
-
logger.error(
|
|
61
|
+
logger.error(
|
|
62
|
+
f"The input value '{value}' cannot be converted to class QCStatus."
|
|
63
|
+
)
|
|
50
64
|
return False
|
|
51
65
|
|
|
52
66
|
|
|
@@ -84,11 +98,24 @@ class Project:
|
|
|
84
98
|
# project id (int)
|
|
85
99
|
if isinstance(project_id, str):
|
|
86
100
|
project_name = project_id
|
|
87
|
-
project_id = next(
|
|
101
|
+
project_id = next(
|
|
102
|
+
iter(
|
|
103
|
+
filter(
|
|
104
|
+
lambda proj: proj["name"] == project_id,
|
|
105
|
+
account.projects,
|
|
106
|
+
)
|
|
107
|
+
)
|
|
108
|
+
)["id"]
|
|
88
109
|
else:
|
|
89
110
|
if isinstance(project_id, float):
|
|
90
111
|
project_id = int(project_id)
|
|
91
|
-
project_name = next(
|
|
112
|
+
project_name = next(
|
|
113
|
+
iter(
|
|
114
|
+
filter(
|
|
115
|
+
lambda proj: proj["id"] == project_id, account.projects
|
|
116
|
+
)
|
|
117
|
+
)
|
|
118
|
+
)["name"]
|
|
92
119
|
|
|
93
120
|
self._account = account
|
|
94
121
|
self._project_id = project_id
|
|
@@ -121,7 +148,9 @@ class Project:
|
|
|
121
148
|
try:
|
|
122
149
|
platform.parse_response(
|
|
123
150
|
platform.post(
|
|
124
|
-
self._account.auth,
|
|
151
|
+
self._account.auth,
|
|
152
|
+
"projectset_manager/activate_project",
|
|
153
|
+
data={"project_id": int(project_id)},
|
|
125
154
|
)
|
|
126
155
|
)
|
|
127
156
|
except errors.PlatformError:
|
|
@@ -156,6 +185,7 @@ class Project:
|
|
|
156
185
|
result=False,
|
|
157
186
|
add_to_container_id=0,
|
|
158
187
|
split_data=False,
|
|
188
|
+
mock_response=False,
|
|
159
189
|
):
|
|
160
190
|
"""
|
|
161
191
|
Upload a chunk of a file to the platform.
|
|
@@ -189,6 +219,8 @@ class Project:
|
|
|
189
219
|
"Content-Length": str(length),
|
|
190
220
|
"Content-Disposition": disposition,
|
|
191
221
|
}
|
|
222
|
+
if mock_response:
|
|
223
|
+
request_headers["mock_case"] = mock_response
|
|
192
224
|
|
|
193
225
|
if last_chunk:
|
|
194
226
|
request_headers["X-Mint-Name"] = name
|
|
@@ -215,9 +247,12 @@ class Project:
|
|
|
215
247
|
|
|
216
248
|
response_time = 900.0 if last_chunk else 120.0
|
|
217
249
|
response = platform.post(
|
|
218
|
-
auth=self._account.auth,
|
|
250
|
+
auth=self._account.auth,
|
|
251
|
+
endpoint="upload",
|
|
252
|
+
data=data,
|
|
253
|
+
headers=request_headers,
|
|
254
|
+
timeout=response_time,
|
|
219
255
|
)
|
|
220
|
-
|
|
221
256
|
return response
|
|
222
257
|
|
|
223
258
|
def upload_file(
|
|
@@ -229,10 +264,12 @@ class Project:
|
|
|
229
264
|
description="",
|
|
230
265
|
result=False,
|
|
231
266
|
name="",
|
|
232
|
-
input_data_type="qmenta_medical_image_data:3.
|
|
267
|
+
input_data_type="qmenta_medical_image_data:3.11.3",
|
|
233
268
|
add_to_container_id=0,
|
|
234
|
-
chunk_size=2**
|
|
269
|
+
chunk_size=2**24, # Optimized for GCS Bucket Storage
|
|
270
|
+
max_retries=5,
|
|
235
271
|
split_data=False,
|
|
272
|
+
mock_response=None,
|
|
236
273
|
):
|
|
237
274
|
"""
|
|
238
275
|
Upload a ZIP file to the platform.
|
|
@@ -260,121 +297,81 @@ class Project:
|
|
|
260
297
|
chunk_size : int
|
|
261
298
|
Size in kB of each chunk. Should be expressed as
|
|
262
299
|
a power of 2: 2**x. Default value of x is 9 (chunk_size = 512 kB)
|
|
300
|
+
max_retries: int
|
|
301
|
+
Maximum number of retries when uploading a file before raising
|
|
302
|
+
an error. Default value: 5
|
|
263
303
|
split_data : bool
|
|
264
304
|
If True, the platform will try to split the uploaded file into
|
|
265
305
|
different sessions. It will be ignored when the ssid or a
|
|
266
306
|
add_to_container_id are given.
|
|
307
|
+
mock_response: None
|
|
308
|
+
ONLY USED IN UNITTESTING
|
|
267
309
|
|
|
268
310
|
Returns
|
|
269
311
|
-------
|
|
270
312
|
bool
|
|
271
313
|
True if correctly uploaded, False otherwise.
|
|
272
314
|
"""
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
chunk_size *= 1024
|
|
277
|
-
max_retries = 10
|
|
278
|
-
|
|
279
|
-
name = name or os.path.split(file_path)[1]
|
|
315
|
+
input_data_type = (
|
|
316
|
+
"qmenta_upload_offline_analysis:1.0" if result else input_data_type
|
|
317
|
+
)
|
|
280
318
|
|
|
281
|
-
|
|
319
|
+
single_upload = SingleUpload(
|
|
320
|
+
self._account.auth,
|
|
321
|
+
file_path,
|
|
322
|
+
file_info=FileInfo(
|
|
323
|
+
project_id=self._project_id,
|
|
324
|
+
subject_name=subject_name,
|
|
325
|
+
session_id=str(ssid),
|
|
326
|
+
input_data_type=input_data_type,
|
|
327
|
+
split_data=split_data,
|
|
328
|
+
add_to_container_id=add_to_container_id,
|
|
329
|
+
date_of_scan=date_of_scan,
|
|
330
|
+
description=description,
|
|
331
|
+
name=name,
|
|
332
|
+
),
|
|
333
|
+
anonymise=False, # will be anonymised in the upload tool.
|
|
334
|
+
chunk_size=chunk_size,
|
|
335
|
+
max_retries=max_retries,
|
|
336
|
+
)
|
|
282
337
|
|
|
283
|
-
|
|
338
|
+
single_upload.start()
|
|
284
339
|
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
340
|
+
if single_upload.status == UploadStatus.FAILED: # FAILED
|
|
341
|
+
print("Upload Failed!")
|
|
342
|
+
return False
|
|
288
343
|
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
chunk_num = 0
|
|
296
|
-
retries_count = 0
|
|
297
|
-
uploaded_bytes = 0
|
|
298
|
-
response = None
|
|
299
|
-
last_chunk = False
|
|
300
|
-
|
|
301
|
-
while True:
|
|
302
|
-
data = file_object.read(chunk_size)
|
|
303
|
-
if not data:
|
|
304
|
-
break
|
|
305
|
-
|
|
306
|
-
start_position = chunk_num * chunk_size
|
|
307
|
-
end_position = start_position + chunk_size - 1
|
|
308
|
-
bytes_to_send = chunk_size
|
|
309
|
-
|
|
310
|
-
if end_position >= total_bytes:
|
|
311
|
-
last_chunk = True
|
|
312
|
-
end_position = total_bytes - 1
|
|
313
|
-
bytes_to_send = total_bytes - uploaded_bytes
|
|
314
|
-
|
|
315
|
-
bytes_range = "bytes " + str(start_position) + "-" + str(end_position) + "/" + str(total_bytes)
|
|
316
|
-
|
|
317
|
-
dispstr = f"attachment; filename={filename}"
|
|
318
|
-
response = self._upload_chunk(
|
|
319
|
-
data,
|
|
320
|
-
bytes_range,
|
|
321
|
-
bytes_to_send,
|
|
322
|
-
session_id,
|
|
323
|
-
dispstr,
|
|
324
|
-
last_chunk,
|
|
325
|
-
name,
|
|
326
|
-
date_of_scan,
|
|
327
|
-
description,
|
|
328
|
-
subject_name,
|
|
329
|
-
ssid,
|
|
330
|
-
filename,
|
|
331
|
-
input_data_type,
|
|
332
|
-
result,
|
|
333
|
-
add_to_container_id,
|
|
334
|
-
split_data,
|
|
335
|
-
)
|
|
344
|
+
message = (
|
|
345
|
+
"Your data was successfully uploaded. "
|
|
346
|
+
"The uploaded file will be soon processed !"
|
|
347
|
+
)
|
|
348
|
+
print(message)
|
|
349
|
+
return True
|
|
336
350
|
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
error_message = "HTTP Connection Problem"
|
|
342
|
-
logger.error(error_message)
|
|
343
|
-
break
|
|
344
|
-
elif int(response.status_code) == 201:
|
|
345
|
-
chunk_num += 1
|
|
346
|
-
retries_count = 0
|
|
347
|
-
uploaded_bytes += chunk_size
|
|
348
|
-
elif int(response.status_code) == 200:
|
|
349
|
-
self.__show_progress(file_size, file_size, finish=True)
|
|
350
|
-
break
|
|
351
|
-
elif int(response.status_code) == 416:
|
|
352
|
-
retries_count += 1
|
|
353
|
-
time.sleep(retries_count * 5)
|
|
354
|
-
if retries_count > self.max_retries:
|
|
355
|
-
error_message = "Error Code: 416; Requested Range Not Satisfiable (NGINX)"
|
|
356
|
-
logger.error(error_message)
|
|
357
|
-
break
|
|
358
|
-
else:
|
|
359
|
-
retries_count += 1
|
|
360
|
-
time.sleep(retries_count * 5)
|
|
361
|
-
if retries_count > max_retries:
|
|
362
|
-
error_message = "Number of retries has been reached. Upload process stops here !"
|
|
363
|
-
logger.error(error_message)
|
|
364
|
-
break
|
|
351
|
+
def delete_file(self, container_id, filenames):
|
|
352
|
+
"""
|
|
353
|
+
Delete a file or files from a container.
|
|
354
|
+
Can be an input or an output container
|
|
365
355
|
|
|
366
|
-
|
|
367
|
-
|
|
356
|
+
Parameters
|
|
357
|
+
----------
|
|
358
|
+
container_id : int
|
|
359
|
+
filenames : str or list of str
|
|
368
360
|
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
361
|
+
"""
|
|
362
|
+
if not isinstance(filenames, str):
|
|
363
|
+
if isinstance(filenames, list):
|
|
364
|
+
if not all([isinstance(f, str) for f in filenames]):
|
|
365
|
+
raise TypeError("Elements of `filenames` must be str")
|
|
366
|
+
filenames = ";".join(filenames)
|
|
367
|
+
else:
|
|
368
|
+
raise TypeError("`filenames` must be str or list of str")
|
|
374
369
|
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
370
|
+
platform.post(
|
|
371
|
+
self._account.auth,
|
|
372
|
+
"file_manager/delete_files",
|
|
373
|
+
data={"container_id": container_id, "files": filenames},
|
|
374
|
+
)
|
|
378
375
|
|
|
379
376
|
def upload_mri(self, file_path, subject_name):
|
|
380
377
|
"""
|
|
@@ -412,7 +409,11 @@ class Project:
|
|
|
412
409
|
"""
|
|
413
410
|
|
|
414
411
|
if self.__check_upload_file(file_path):
|
|
415
|
-
return self.upload_file(
|
|
412
|
+
return self.upload_file(
|
|
413
|
+
file_path,
|
|
414
|
+
subject_name,
|
|
415
|
+
input_data_type="parkinson_gametection",
|
|
416
|
+
)
|
|
416
417
|
return False
|
|
417
418
|
|
|
418
419
|
def upload_result(self, file_path, subject_name):
|
|
@@ -435,7 +436,9 @@ class Project:
|
|
|
435
436
|
return self.upload_file(file_path, subject_name, result=True)
|
|
436
437
|
return False
|
|
437
438
|
|
|
438
|
-
def download_file(
|
|
439
|
+
def download_file(
|
|
440
|
+
self, container_id, file_name, local_filename=None, overwrite=False
|
|
441
|
+
):
|
|
439
442
|
"""
|
|
440
443
|
Download a single file from a specific container.
|
|
441
444
|
|
|
@@ -445,43 +448,57 @@ class Project:
|
|
|
445
448
|
ID of the container inside which the file is.
|
|
446
449
|
file_name : str
|
|
447
450
|
Name of the file in the container.
|
|
448
|
-
local_filename : str
|
|
451
|
+
local_filename : str, optional
|
|
449
452
|
Name of the file to be created. By default, the same as file_name.
|
|
450
453
|
overwrite : bool
|
|
451
454
|
Whether to overwrite the file if existing.
|
|
452
455
|
"""
|
|
453
456
|
logger = logging.getLogger(logger_name)
|
|
454
457
|
if not isinstance(file_name, str):
|
|
455
|
-
raise ValueError(
|
|
456
|
-
|
|
457
|
-
|
|
458
|
+
raise ValueError(
|
|
459
|
+
"The name of the file to download (file_name) should be of "
|
|
460
|
+
"type string."
|
|
461
|
+
)
|
|
462
|
+
if not isinstance(local_filename, str):
|
|
463
|
+
raise ValueError(
|
|
464
|
+
"The name of the output file (local_filename) should be of "
|
|
465
|
+
"type string."
|
|
466
|
+
)
|
|
458
467
|
|
|
459
468
|
if file_name not in self.list_container_files(container_id):
|
|
460
|
-
msg =
|
|
461
|
-
|
|
462
|
-
|
|
469
|
+
msg = (
|
|
470
|
+
f'File "{file_name}" does not exist in container '
|
|
471
|
+
f"{container_id}"
|
|
472
|
+
)
|
|
473
|
+
raise Exception(msg)
|
|
463
474
|
|
|
464
475
|
local_filename = local_filename or file_name
|
|
465
476
|
|
|
466
477
|
if os.path.exists(local_filename) and not overwrite:
|
|
467
478
|
msg = f"File {local_filename} already exists"
|
|
468
|
-
|
|
469
|
-
return False
|
|
479
|
+
raise Exception(msg)
|
|
470
480
|
|
|
471
481
|
params = {"container_id": container_id, "files": file_name}
|
|
472
|
-
|
|
473
482
|
with platform.post(
|
|
474
|
-
self._account.auth,
|
|
483
|
+
self._account.auth,
|
|
484
|
+
"file_manager/download_file",
|
|
485
|
+
data=params,
|
|
486
|
+
stream=True,
|
|
475
487
|
) as response, open(local_filename, "wb") as f:
|
|
476
488
|
|
|
477
489
|
for chunk in response.iter_content(chunk_size=2**9 * 1024):
|
|
478
490
|
f.write(chunk)
|
|
479
491
|
f.flush()
|
|
480
492
|
|
|
481
|
-
logger.info(
|
|
493
|
+
logger.info(
|
|
494
|
+
f"File {file_name} from container {container_id} saved "
|
|
495
|
+
f"to {local_filename}"
|
|
496
|
+
)
|
|
482
497
|
return True
|
|
483
498
|
|
|
484
|
-
def download_files(
|
|
499
|
+
def download_files(
|
|
500
|
+
self, container_id, filenames, zip_name="files.zip", overwrite=False
|
|
501
|
+
):
|
|
485
502
|
"""
|
|
486
503
|
Download a set of files from a given container.
|
|
487
504
|
|
|
@@ -499,32 +516,51 @@ class Project:
|
|
|
499
516
|
logger = logging.getLogger(logger_name)
|
|
500
517
|
|
|
501
518
|
if not all([isinstance(file_name, str) for file_name in filenames]):
|
|
502
|
-
raise ValueError(
|
|
519
|
+
raise ValueError(
|
|
520
|
+
"The name of the files to download (filenames) should be "
|
|
521
|
+
"of type string."
|
|
522
|
+
)
|
|
503
523
|
if not isinstance(zip_name, str):
|
|
504
|
-
raise ValueError(
|
|
524
|
+
raise ValueError(
|
|
525
|
+
"The name of the output ZIP file (zip_name) should be "
|
|
526
|
+
"of type string."
|
|
527
|
+
)
|
|
505
528
|
|
|
506
|
-
files_not_in_container = list(
|
|
529
|
+
files_not_in_container = list(
|
|
530
|
+
filter(
|
|
531
|
+
lambda f: f not in self.list_container_files(container_id),
|
|
532
|
+
filenames,
|
|
533
|
+
)
|
|
534
|
+
)
|
|
507
535
|
|
|
508
536
|
if files_not_in_container:
|
|
509
|
-
msg =
|
|
510
|
-
|
|
511
|
-
|
|
537
|
+
msg = (
|
|
538
|
+
f"The following files are missing in container "
|
|
539
|
+
f"{container_id}: {', '.join(files_not_in_container)}"
|
|
540
|
+
)
|
|
541
|
+
raise Exception(msg)
|
|
512
542
|
|
|
513
543
|
if os.path.exists(zip_name) and not overwrite:
|
|
514
544
|
msg = f'File "{zip_name}" already exists'
|
|
515
|
-
|
|
516
|
-
return False
|
|
545
|
+
raise Exception(msg)
|
|
517
546
|
|
|
518
547
|
params = {"container_id": container_id, "files": ";".join(filenames)}
|
|
519
548
|
with platform.post(
|
|
520
|
-
self._account.auth,
|
|
549
|
+
self._account.auth,
|
|
550
|
+
"file_manager/download_file",
|
|
551
|
+
data=params,
|
|
552
|
+
stream=True,
|
|
521
553
|
) as response, open(zip_name, "wb") as f:
|
|
522
554
|
|
|
523
555
|
for chunk in response.iter_content(chunk_size=2**9 * 1024):
|
|
524
556
|
f.write(chunk)
|
|
525
557
|
f.flush()
|
|
526
558
|
|
|
527
|
-
logger.info(
|
|
559
|
+
logger.info(
|
|
560
|
+
"Files from container {} saved to {}".format(
|
|
561
|
+
container_id, zip_name
|
|
562
|
+
)
|
|
563
|
+
)
|
|
528
564
|
return True
|
|
529
565
|
|
|
530
566
|
def copy_container_to_project(self, container_id, project_id):
|
|
@@ -548,9 +584,14 @@ class Project:
|
|
|
548
584
|
p_id = int(project_id)
|
|
549
585
|
elif type(project_id) is str:
|
|
550
586
|
projects = self._account.projects
|
|
551
|
-
projects_match = [
|
|
587
|
+
projects_match = [
|
|
588
|
+
proj for proj in projects if proj["name"] == project_id
|
|
589
|
+
]
|
|
552
590
|
if not projects_match:
|
|
553
|
-
raise Exception(
|
|
591
|
+
raise Exception(
|
|
592
|
+
f"Project {project_id}"
|
|
593
|
+
+ " does not exist or is not available for this user."
|
|
594
|
+
)
|
|
554
595
|
p_id = int(projects_match[0]["id"])
|
|
555
596
|
else:
|
|
556
597
|
raise TypeError("project_id")
|
|
@@ -561,10 +602,16 @@ class Project:
|
|
|
561
602
|
|
|
562
603
|
try:
|
|
563
604
|
platform.parse_response(
|
|
564
|
-
platform.post(
|
|
605
|
+
platform.post(
|
|
606
|
+
self._account.auth,
|
|
607
|
+
"file_manager/copy_container_to_another_project",
|
|
608
|
+
data=data,
|
|
609
|
+
)
|
|
565
610
|
)
|
|
566
611
|
except errors.PlatformError as e:
|
|
567
|
-
logging.getLogger(logger_name).error(
|
|
612
|
+
logging.getLogger(logger_name).error(
|
|
613
|
+
"Couldn not copy container: {}".format(e)
|
|
614
|
+
)
|
|
568
615
|
return False
|
|
569
616
|
|
|
570
617
|
return True
|
|
@@ -598,7 +645,7 @@ class Project:
|
|
|
598
645
|
return self.get_subjects_metadata()
|
|
599
646
|
|
|
600
647
|
@property
|
|
601
|
-
def metadata_parameters(self):
|
|
648
|
+
def metadata_parameters(self) -> Union[Dict, None]:
|
|
602
649
|
"""
|
|
603
650
|
List all the parameters in the subject-level metadata.
|
|
604
651
|
|
|
@@ -609,20 +656,28 @@ class Project:
|
|
|
609
656
|
modification of these subject-level metadata parameters via the
|
|
610
657
|
'change_subject_metadata()' method.
|
|
611
658
|
|
|
659
|
+
Example:
|
|
660
|
+
dictionary {"param_name":
|
|
661
|
+
{ "order": Int,
|
|
662
|
+
"tags": [tag1, tag2, ..., ],
|
|
663
|
+
"title: "Title",
|
|
664
|
+
"type": "integer|string|date|list|decimal",
|
|
665
|
+
"visible": 0|1
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
|
|
612
669
|
Returns
|
|
613
670
|
-------
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
{ "order": Int,
|
|
617
|
-
"tags": [tag1, tag2, ..., ],
|
|
618
|
-
"title: "Title",
|
|
619
|
-
"type": "integer|string|date|list|decimal",
|
|
620
|
-
"visible": 0|1
|
|
621
|
-
}}
|
|
671
|
+
metadata_parameters : dict[str] or None
|
|
672
|
+
|
|
622
673
|
"""
|
|
623
674
|
logger = logging.getLogger(logger_name)
|
|
624
675
|
try:
|
|
625
|
-
data = platform.parse_response(
|
|
676
|
+
data = platform.parse_response(
|
|
677
|
+
platform.post(
|
|
678
|
+
self._account.auth, "patient_manager/module_config"
|
|
679
|
+
)
|
|
680
|
+
)
|
|
626
681
|
except errors.PlatformError:
|
|
627
682
|
logger.error("Could not retrieve metadata parameters.")
|
|
628
683
|
return None
|
|
@@ -668,7 +723,10 @@ class Project:
|
|
|
668
723
|
response = self.list_input_containers(search_criteria=search_criteria)
|
|
669
724
|
|
|
670
725
|
for subject in response:
|
|
671
|
-
if
|
|
726
|
+
if (
|
|
727
|
+
subject["patient_secret_name"] == subject_name
|
|
728
|
+
and subject["ssid"] == ssid
|
|
729
|
+
):
|
|
672
730
|
return subject["container_id"]
|
|
673
731
|
return False
|
|
674
732
|
|
|
@@ -692,20 +750,25 @@ class Project:
|
|
|
692
750
|
"""
|
|
693
751
|
|
|
694
752
|
for user in self.get_subjects_metadata():
|
|
695
|
-
if user["patient_secret_name"] == str(subject_name) and user[
|
|
753
|
+
if user["patient_secret_name"] == str(subject_name) and user[
|
|
754
|
+
"ssid"
|
|
755
|
+
] == str(ssid):
|
|
696
756
|
return int(user["_id"])
|
|
697
757
|
return False
|
|
698
758
|
|
|
699
|
-
def get_subjects_metadata(self, search_criteria=
|
|
759
|
+
def get_subjects_metadata(self, search_criteria=None, items=(0, 9999)):
|
|
700
760
|
"""
|
|
701
761
|
List all Subject ID/Session ID from the selected project that meet the
|
|
702
|
-
|
|
762
|
+
defined search criteria at a session level.
|
|
703
763
|
|
|
704
764
|
Parameters
|
|
705
765
|
----------
|
|
706
766
|
search_criteria: dict
|
|
707
767
|
Each element is a string and is built using the formatting
|
|
708
768
|
"type;value", or "type;operation|value"
|
|
769
|
+
items : List[int]
|
|
770
|
+
list containing two elements [min, max] that correspond to the
|
|
771
|
+
mininum and maximum range of analysis listed
|
|
709
772
|
|
|
710
773
|
Complete search_criteria Dictionary Explanation:
|
|
711
774
|
|
|
@@ -719,8 +782,8 @@ class Project:
|
|
|
719
782
|
"pars_PROJECTMETADATA": "METADATATYPE;METADATAVALUE",
|
|
720
783
|
}
|
|
721
784
|
|
|
722
|
-
|
|
723
|
-
|
|
785
|
+
where "pars_patient_secret_name": Applies the search to the
|
|
786
|
+
'Subject ID'.
|
|
724
787
|
SUBJECTID is a comma separated list of strings.
|
|
725
788
|
"pars_ssid": Applies the search to the 'Session ID'.
|
|
726
789
|
SSID is an integer.
|
|
@@ -806,12 +869,26 @@ class Project:
|
|
|
806
869
|
|
|
807
870
|
"""
|
|
808
871
|
|
|
809
|
-
|
|
810
|
-
|
|
872
|
+
if search_criteria is None:
|
|
873
|
+
search_criteria = {}
|
|
874
|
+
if len(items) != 2:
|
|
875
|
+
raise ValueError(
|
|
876
|
+
f"The number of elements in items '{len(items)}' "
|
|
877
|
+
f"should be equal to two."
|
|
878
|
+
)
|
|
811
879
|
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
880
|
+
if not all([isinstance(item, int) for item in items]):
|
|
881
|
+
raise ValueError(
|
|
882
|
+
f"All values in items " f"'{items}' must be integers"
|
|
883
|
+
)
|
|
884
|
+
|
|
885
|
+
if search_criteria != {} and not all(
|
|
886
|
+
[item.startswith("pars_") for item in search_criteria.keys()]
|
|
887
|
+
):
|
|
888
|
+
raise ValueError(
|
|
889
|
+
f"All keys of the search_criteria dictionary "
|
|
890
|
+
f"'{search_criteria.keys()}' must start with 'pars_'."
|
|
891
|
+
)
|
|
815
892
|
|
|
816
893
|
for key, value in search_criteria.items():
|
|
817
894
|
if value.split(";")[0] in ["integer", "decimal"]:
|
|
@@ -830,7 +907,9 @@ class Project:
|
|
|
830
907
|
)
|
|
831
908
|
return content
|
|
832
909
|
|
|
833
|
-
def change_subject_metadata(
|
|
910
|
+
def change_subject_metadata(
|
|
911
|
+
self, patient_id, subject_name, ssid, tags, age_at_scan, metadata
|
|
912
|
+
):
|
|
834
913
|
"""
|
|
835
914
|
Change the Subject ID, Session ID, Tags, Age at Scan and Metadata of
|
|
836
915
|
the session with Patient ID
|
|
@@ -865,36 +944,60 @@ class Project:
|
|
|
865
944
|
try:
|
|
866
945
|
patient_id = str(int(patient_id))
|
|
867
946
|
except ValueError:
|
|
868
|
-
raise ValueError(
|
|
947
|
+
raise ValueError(
|
|
948
|
+
f"'patient_id': '{patient_id}' not valid. Must be convertible "
|
|
949
|
+
f"to int."
|
|
950
|
+
)
|
|
869
951
|
|
|
870
|
-
|
|
952
|
+
if not isinstance(tags, list) or not all(
|
|
871
953
|
isinstance(item, str) for item in tags
|
|
872
|
-
)
|
|
954
|
+
):
|
|
955
|
+
raise ValueError(f"tags: '{tags}' should be a list of strings.")
|
|
873
956
|
tags = [tag.lower() for tag in tags]
|
|
874
957
|
|
|
875
|
-
|
|
876
|
-
|
|
958
|
+
if not isinstance(subject_name, str) or (
|
|
959
|
+
subject_name is None or subject_name == ""
|
|
960
|
+
):
|
|
961
|
+
raise ValueError("subject_name must be a non empty string.")
|
|
962
|
+
|
|
963
|
+
if not isinstance(ssid, str) or (ssid is None or ssid == ""):
|
|
964
|
+
raise ValueError("ssid must be a non empty string.")
|
|
877
965
|
|
|
878
966
|
try:
|
|
879
967
|
age_at_scan = str(int(age_at_scan)) if age_at_scan else None
|
|
880
968
|
except ValueError:
|
|
881
|
-
raise ValueError(
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
"
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
969
|
+
raise ValueError(
|
|
970
|
+
f"age_at_scan: '{age_at_scan}' not valid. Must be an integer."
|
|
971
|
+
)
|
|
972
|
+
|
|
973
|
+
if not isinstance(metadata, dict):
|
|
974
|
+
raise ValueError(f"metadata: '{metadata}' should be a dictionary.")
|
|
975
|
+
|
|
976
|
+
has_md_prefix = ["md_" == key[:3] for key in metadata.keys()]
|
|
977
|
+
if not (all(has_md_prefix) or not any(has_md_prefix)):
|
|
978
|
+
raise ValueError(
|
|
979
|
+
f"metadata: '{metadata}' must be a dictionary whose keys are "
|
|
980
|
+
f"either all starting with 'md_' or none."
|
|
981
|
+
)
|
|
982
|
+
|
|
983
|
+
metadata_keys = []
|
|
984
|
+
if self.metadata_parameters:
|
|
985
|
+
metadata_keys = self.metadata_parameters.keys()
|
|
986
|
+
|
|
987
|
+
if not all(
|
|
988
|
+
(
|
|
989
|
+
key[3:] in metadata_keys
|
|
990
|
+
if "md_" == key[:3]
|
|
991
|
+
else key in metadata_keys
|
|
992
|
+
)
|
|
993
|
+
for key in metadata.keys()
|
|
994
|
+
):
|
|
995
|
+
raise ValueError(
|
|
996
|
+
f"Some metadata keys provided ({', '.join(metadata.keys())}) "
|
|
997
|
+
"are not available in the project. They can be added via the "
|
|
998
|
+
"Metadata Manager via the QMENTA Platform graphical user "
|
|
999
|
+
"interface (GUI)."
|
|
1000
|
+
)
|
|
898
1001
|
|
|
899
1002
|
post_data = {
|
|
900
1003
|
"patient_id": patient_id,
|
|
@@ -904,11 +1007,17 @@ class Project:
|
|
|
904
1007
|
"age_at_scan": age_at_scan,
|
|
905
1008
|
}
|
|
906
1009
|
for key, value in metadata.items():
|
|
907
|
-
|
|
908
|
-
post_data[f"last_vals.{
|
|
1010
|
+
id_ = key[3:] if "md_" == key[:3] else key
|
|
1011
|
+
post_data[f"last_vals.{id_}"] = value
|
|
909
1012
|
|
|
910
1013
|
try:
|
|
911
|
-
platform.parse_response(
|
|
1014
|
+
platform.parse_response(
|
|
1015
|
+
platform.post(
|
|
1016
|
+
self._account.auth,
|
|
1017
|
+
"patient_manager/upsert_patient",
|
|
1018
|
+
data=post_data,
|
|
1019
|
+
)
|
|
1020
|
+
)
|
|
912
1021
|
except errors.PlatformError:
|
|
913
1022
|
logger.error(f"Patient ID '{patient_id}' could not be modified.")
|
|
914
1023
|
return False
|
|
@@ -916,7 +1025,9 @@ class Project:
|
|
|
916
1025
|
logger.info(f"Patient ID '{patient_id}' successfully modified.")
|
|
917
1026
|
return True
|
|
918
1027
|
|
|
919
|
-
def get_subjects_files_metadata(
|
|
1028
|
+
def get_subjects_files_metadata(
|
|
1029
|
+
self, search_criteria=None, items=(0, 9999)
|
|
1030
|
+
):
|
|
920
1031
|
"""
|
|
921
1032
|
List all Subject ID/Session ID from the selected project that meet the
|
|
922
1033
|
defined search criteria at a file level.
|
|
@@ -932,6 +1043,9 @@ class Project:
|
|
|
932
1043
|
search_criteria: dict
|
|
933
1044
|
Each element is a string and is built using the formatting
|
|
934
1045
|
"type;value", or "type;operation|value"
|
|
1046
|
+
items : List[int]
|
|
1047
|
+
list containing two elements [min, max] that correspond to the
|
|
1048
|
+
mininum and maximum range of analysis listed
|
|
935
1049
|
|
|
936
1050
|
Complete search_criteria Dictionary Explanation:
|
|
937
1051
|
|
|
@@ -1034,10 +1148,14 @@ class Project:
|
|
|
1034
1148
|
|
|
1035
1149
|
"""
|
|
1036
1150
|
|
|
1037
|
-
|
|
1151
|
+
if search_criteria is None:
|
|
1152
|
+
search_criteria = {}
|
|
1153
|
+
content = self.get_subjects_metadata(search_criteria, items=items)
|
|
1038
1154
|
|
|
1039
1155
|
# Wrap search criteria.
|
|
1040
|
-
modality, tags,
|
|
1156
|
+
modality, tags, dicom_metadata = self.__wrap_search_criteria(
|
|
1157
|
+
search_criteria
|
|
1158
|
+
)
|
|
1041
1159
|
|
|
1042
1160
|
# Iterate over the files of each subject selected to include/exclude
|
|
1043
1161
|
# them from the results.
|
|
@@ -1052,17 +1170,23 @@ class Project:
|
|
|
1052
1170
|
)
|
|
1053
1171
|
|
|
1054
1172
|
for file in files["meta"]:
|
|
1055
|
-
if modality and modality != (file.get("metadata") or {}).get(
|
|
1173
|
+
if modality and modality != (file.get("metadata") or {}).get(
|
|
1174
|
+
"modality"
|
|
1175
|
+
):
|
|
1056
1176
|
continue
|
|
1057
1177
|
if tags and not all([tag in file.get("tags") for tag in tags]):
|
|
1058
1178
|
continue
|
|
1059
|
-
if
|
|
1179
|
+
if dicom_metadata:
|
|
1060
1180
|
result_values = list()
|
|
1061
|
-
for key, dict_value in
|
|
1062
|
-
f_value = (
|
|
1181
|
+
for key, dict_value in dicom_metadata.items():
|
|
1182
|
+
f_value = (
|
|
1183
|
+
(file.get("metadata") or {}).get("info") or {}
|
|
1184
|
+
).get(key)
|
|
1063
1185
|
d_operator = dict_value["operation"]
|
|
1064
1186
|
d_value = dict_value["value"]
|
|
1065
|
-
result_values.append(
|
|
1187
|
+
result_values.append(
|
|
1188
|
+
self.__operation(d_value, d_operator, f_value)
|
|
1189
|
+
)
|
|
1066
1190
|
|
|
1067
1191
|
if not all(result_values):
|
|
1068
1192
|
continue
|
|
@@ -1083,7 +1207,7 @@ class Project:
|
|
|
1083
1207
|
|
|
1084
1208
|
Returns
|
|
1085
1209
|
-------
|
|
1086
|
-
dict
|
|
1210
|
+
dict or bool
|
|
1087
1211
|
Dictionary with the metadata. False otherwise.
|
|
1088
1212
|
"""
|
|
1089
1213
|
all_metadata = self.list_container_files_metadata(container_id)
|
|
@@ -1116,7 +1240,12 @@ class Project:
|
|
|
1116
1240
|
platform.post(
|
|
1117
1241
|
self._account.auth,
|
|
1118
1242
|
"file_manager/edit_file",
|
|
1119
|
-
data={
|
|
1243
|
+
data={
|
|
1244
|
+
"container_id": container_id,
|
|
1245
|
+
"filename": filename,
|
|
1246
|
+
"tags": tags_str,
|
|
1247
|
+
"modality": modality,
|
|
1248
|
+
},
|
|
1120
1249
|
)
|
|
1121
1250
|
)
|
|
1122
1251
|
|
|
@@ -1129,7 +1258,7 @@ class Project:
|
|
|
1129
1258
|
----------
|
|
1130
1259
|
subject_name : str
|
|
1131
1260
|
Subject ID of the subject
|
|
1132
|
-
session_id :
|
|
1261
|
+
session_id : str
|
|
1133
1262
|
The Session ID of the session that will be deleted
|
|
1134
1263
|
|
|
1135
1264
|
Returns
|
|
@@ -1141,16 +1270,29 @@ class Project:
|
|
|
1141
1270
|
all_sessions = self.get_subjects_metadata()
|
|
1142
1271
|
|
|
1143
1272
|
session_to_del = [
|
|
1144
|
-
s
|
|
1273
|
+
s
|
|
1274
|
+
for s in all_sessions
|
|
1275
|
+
if s["patient_secret_name"] == subject_name
|
|
1276
|
+
and s["ssid"] == session_id
|
|
1145
1277
|
]
|
|
1146
1278
|
|
|
1147
1279
|
if not session_to_del:
|
|
1148
|
-
logger.error(
|
|
1280
|
+
logger.error(
|
|
1281
|
+
f"Session {subject_name}/{session_id} could not be found in "
|
|
1282
|
+
f"this project."
|
|
1283
|
+
)
|
|
1149
1284
|
return False
|
|
1150
1285
|
elif len(session_to_del) > 1:
|
|
1151
|
-
raise RuntimeError(
|
|
1286
|
+
raise RuntimeError(
|
|
1287
|
+
"Multiple sessions with same Subject ID and Session ID. "
|
|
1288
|
+
"Contact support."
|
|
1289
|
+
)
|
|
1152
1290
|
else:
|
|
1153
|
-
logger.info(
|
|
1291
|
+
logger.info(
|
|
1292
|
+
"{}/{} found (id {})".format(
|
|
1293
|
+
subject_name, session_id, session_to_del[0]["_id"]
|
|
1294
|
+
)
|
|
1295
|
+
)
|
|
1154
1296
|
|
|
1155
1297
|
session = session_to_del[0]
|
|
1156
1298
|
|
|
@@ -1159,14 +1301,23 @@ class Project:
|
|
|
1159
1301
|
platform.post(
|
|
1160
1302
|
self._account.auth,
|
|
1161
1303
|
"patient_manager/delete_patient",
|
|
1162
|
-
data={
|
|
1304
|
+
data={
|
|
1305
|
+
"patient_id": str(int(session["_id"])),
|
|
1306
|
+
"delete_files": 1,
|
|
1307
|
+
},
|
|
1163
1308
|
)
|
|
1164
1309
|
)
|
|
1165
1310
|
except errors.PlatformError:
|
|
1166
|
-
logger.error(
|
|
1311
|
+
logger.error(
|
|
1312
|
+
f"Session \"{subject_name}/{session['ssid']}\" could "
|
|
1313
|
+
f"not be deleted."
|
|
1314
|
+
)
|
|
1167
1315
|
return False
|
|
1168
1316
|
|
|
1169
|
-
logger.info(
|
|
1317
|
+
logger.info(
|
|
1318
|
+
f"Session \"{subject_name}/{session['ssid']}\" successfully "
|
|
1319
|
+
f"deleted."
|
|
1320
|
+
)
|
|
1170
1321
|
return True
|
|
1171
1322
|
|
|
1172
1323
|
def delete_session_by_patientid(self, patient_id):
|
|
@@ -1191,7 +1342,10 @@ class Project:
|
|
|
1191
1342
|
platform.post(
|
|
1192
1343
|
self._account.auth,
|
|
1193
1344
|
"patient_manager/delete_patient",
|
|
1194
|
-
data={
|
|
1345
|
+
data={
|
|
1346
|
+
"patient_id": str(int(patient_id)),
|
|
1347
|
+
"delete_files": 1,
|
|
1348
|
+
},
|
|
1195
1349
|
)
|
|
1196
1350
|
)
|
|
1197
1351
|
except errors.PlatformError:
|
|
@@ -1221,10 +1375,16 @@ class Project:
|
|
|
1221
1375
|
# Always fetch the session IDs from the platform before deleting them
|
|
1222
1376
|
all_sessions = self.get_subjects_metadata()
|
|
1223
1377
|
|
|
1224
|
-
sessions_to_del = [
|
|
1378
|
+
sessions_to_del = [
|
|
1379
|
+
s for s in all_sessions if s["patient_secret_name"] == subject_name
|
|
1380
|
+
]
|
|
1225
1381
|
|
|
1226
1382
|
if not sessions_to_del:
|
|
1227
|
-
logger.error(
|
|
1383
|
+
logger.error(
|
|
1384
|
+
"Subject {} cannot be found in this project.".format(
|
|
1385
|
+
subject_name
|
|
1386
|
+
)
|
|
1387
|
+
)
|
|
1228
1388
|
return False
|
|
1229
1389
|
|
|
1230
1390
|
for ssid in [s["ssid"] for s in sessions_to_del]:
|
|
@@ -1234,7 +1394,7 @@ class Project:
|
|
|
1234
1394
|
|
|
1235
1395
|
""" Container Related Methods """
|
|
1236
1396
|
|
|
1237
|
-
def list_input_containers(self, search_criteria=
|
|
1397
|
+
def list_input_containers(self, search_criteria=None, items=(0, 9999)):
|
|
1238
1398
|
"""
|
|
1239
1399
|
Retrieve the list of input containers available to the user under a
|
|
1240
1400
|
certain search criteria.
|
|
@@ -1268,8 +1428,17 @@ class Project:
|
|
|
1268
1428
|
{"container_name", "container_id", "patient_secret_name", "ssid"}
|
|
1269
1429
|
"""
|
|
1270
1430
|
|
|
1271
|
-
|
|
1272
|
-
|
|
1431
|
+
if search_criteria is None:
|
|
1432
|
+
search_criteria = {}
|
|
1433
|
+
if len(items) != 2:
|
|
1434
|
+
raise ValueError(
|
|
1435
|
+
f"The number of elements in items '{len(items)}' "
|
|
1436
|
+
f"should be equal to two."
|
|
1437
|
+
)
|
|
1438
|
+
if not all(isinstance(item, int) for item in items):
|
|
1439
|
+
raise ValueError(
|
|
1440
|
+
f"All items elements '{items}' should be integers."
|
|
1441
|
+
)
|
|
1273
1442
|
|
|
1274
1443
|
response = platform.parse_response(
|
|
1275
1444
|
platform.post(
|
|
@@ -1282,7 +1451,7 @@ class Project:
|
|
|
1282
1451
|
containers = [
|
|
1283
1452
|
{
|
|
1284
1453
|
"patient_secret_name": container_item["patient_secret_name"],
|
|
1285
|
-
"container_name": container_item["name"],
|
|
1454
|
+
"container_name": container_item["name"], # ???
|
|
1286
1455
|
"container_id": container_item["_id"],
|
|
1287
1456
|
"ssid": container_item["ssid"],
|
|
1288
1457
|
}
|
|
@@ -1290,7 +1459,7 @@ class Project:
|
|
|
1290
1459
|
]
|
|
1291
1460
|
return containers
|
|
1292
1461
|
|
|
1293
|
-
def list_result_containers(self, search_condition=
|
|
1462
|
+
def list_result_containers(self, search_condition=None, items=(0, 9999)):
|
|
1294
1463
|
"""
|
|
1295
1464
|
List the result containers available to the user.
|
|
1296
1465
|
Examples
|
|
@@ -1318,7 +1487,8 @@ class Project:
|
|
|
1318
1487
|
- qa_status: str or None pass/fail/nd QC status
|
|
1319
1488
|
- secret_name: str or None Subject ID
|
|
1320
1489
|
- tags: str or None
|
|
1321
|
-
- with_child_analysis: 1 or None if 1, child analysis of workflows
|
|
1490
|
+
- with_child_analysis: 1 or None if 1, child analysis of workflows
|
|
1491
|
+
will appear
|
|
1322
1492
|
- id: str or None ID
|
|
1323
1493
|
- state: running, completed, pending, exception or None
|
|
1324
1494
|
- username: str or None
|
|
@@ -1335,13 +1505,21 @@ class Project:
|
|
|
1335
1505
|
if "id": None, that analysis did not had an output container,
|
|
1336
1506
|
probably it is a workflow
|
|
1337
1507
|
"""
|
|
1508
|
+
if search_condition is None:
|
|
1509
|
+
search_condition = {}
|
|
1338
1510
|
analyses = self.list_analysis(search_condition, items)
|
|
1339
|
-
return [
|
|
1511
|
+
return [
|
|
1512
|
+
{
|
|
1513
|
+
"name": analysis["name"],
|
|
1514
|
+
"id": (analysis.get("out_container_id") or None),
|
|
1515
|
+
}
|
|
1516
|
+
for analysis in analyses
|
|
1517
|
+
]
|
|
1340
1518
|
|
|
1341
1519
|
def list_container_files(
|
|
1342
1520
|
self,
|
|
1343
1521
|
container_id,
|
|
1344
|
-
):
|
|
1522
|
+
) -> Any:
|
|
1345
1523
|
"""
|
|
1346
1524
|
List the name of the files available inside a given container.
|
|
1347
1525
|
Parameters
|
|
@@ -1357,7 +1535,9 @@ class Project:
|
|
|
1357
1535
|
try:
|
|
1358
1536
|
content = platform.parse_response(
|
|
1359
1537
|
platform.post(
|
|
1360
|
-
self._account.auth,
|
|
1538
|
+
self._account.auth,
|
|
1539
|
+
"file_manager/get_container_files",
|
|
1540
|
+
data={"container_id": container_id},
|
|
1361
1541
|
)
|
|
1362
1542
|
)
|
|
1363
1543
|
except errors.PlatformError as e:
|
|
@@ -1368,7 +1548,9 @@ class Project:
|
|
|
1368
1548
|
return False
|
|
1369
1549
|
return content["files"]
|
|
1370
1550
|
|
|
1371
|
-
def list_container_filter_files(
|
|
1551
|
+
def list_container_filter_files(
|
|
1552
|
+
self, container_id, modality="", metadata_info={}, tags=[]
|
|
1553
|
+
):
|
|
1372
1554
|
"""
|
|
1373
1555
|
List the name of the files available inside a given container.
|
|
1374
1556
|
search condition example:
|
|
@@ -1404,17 +1586,23 @@ class Project:
|
|
|
1404
1586
|
if modality == "":
|
|
1405
1587
|
modality_bool = True
|
|
1406
1588
|
else:
|
|
1407
|
-
modality_bool = modality == metadata_file["metadata"].get(
|
|
1589
|
+
modality_bool = modality == metadata_file["metadata"].get(
|
|
1590
|
+
"modality"
|
|
1591
|
+
)
|
|
1408
1592
|
for key in metadata_info.keys():
|
|
1409
|
-
meta_key = (
|
|
1593
|
+
meta_key = (
|
|
1594
|
+
(metadata_file.get("metadata") or {}).get("info") or {}
|
|
1595
|
+
).get(key)
|
|
1410
1596
|
if meta_key is None:
|
|
1411
|
-
logging.getLogger(logger_name).warning(
|
|
1597
|
+
logging.getLogger(logger_name).warning(
|
|
1598
|
+
f"{key} is not in file_info from file {file}"
|
|
1599
|
+
)
|
|
1412
1600
|
info_bool.append(metadata_info[key] == meta_key)
|
|
1413
1601
|
if all(tags_bool) and all(info_bool) and modality_bool:
|
|
1414
1602
|
selected_files.append(file)
|
|
1415
1603
|
return selected_files
|
|
1416
1604
|
|
|
1417
|
-
def list_container_files_metadata(self, container_id):
|
|
1605
|
+
def list_container_files_metadata(self, container_id) -> dict:
|
|
1418
1606
|
"""
|
|
1419
1607
|
List all the metadata of the files available inside a given container.
|
|
1420
1608
|
|
|
@@ -1432,7 +1620,9 @@ class Project:
|
|
|
1432
1620
|
try:
|
|
1433
1621
|
data = platform.parse_response(
|
|
1434
1622
|
platform.post(
|
|
1435
|
-
self._account.auth,
|
|
1623
|
+
self._account.auth,
|
|
1624
|
+
"file_manager/get_container_files",
|
|
1625
|
+
data={"container_id": container_id},
|
|
1436
1626
|
)
|
|
1437
1627
|
)
|
|
1438
1628
|
except errors.PlatformError as e:
|
|
@@ -1443,9 +1633,10 @@ class Project:
|
|
|
1443
1633
|
|
|
1444
1634
|
""" Analysis Related Methods """
|
|
1445
1635
|
|
|
1446
|
-
def get_analysis(self, analysis_name_or_id):
|
|
1636
|
+
def get_analysis(self, analysis_name_or_id) -> dict:
|
|
1447
1637
|
"""
|
|
1448
|
-
Returns the analysis corresponding with the analysis id or analysis
|
|
1638
|
+
Returns the analysis corresponding with the analysis id or analysis
|
|
1639
|
+
name
|
|
1449
1640
|
|
|
1450
1641
|
Parameters
|
|
1451
1642
|
----------
|
|
@@ -1465,28 +1656,41 @@ class Project:
|
|
|
1465
1656
|
analysis_name_or_id = int(analysis_name_or_id)
|
|
1466
1657
|
else:
|
|
1467
1658
|
search_tag = "p_n"
|
|
1468
|
-
|
|
1469
|
-
|
|
1659
|
+
excluded_bool = [
|
|
1660
|
+
character in analysis_name_or_id
|
|
1661
|
+
for character in ANALYSIS_NAME_EXCLUDED_CHARACTERS
|
|
1662
|
+
]
|
|
1470
1663
|
if any(excluded_bool):
|
|
1471
|
-
raise Exception(
|
|
1664
|
+
raise Exception(
|
|
1665
|
+
f"p_n does not allow "
|
|
1666
|
+
f"characters {ANALYSIS_NAME_EXCLUDED_CHARACTERS}"
|
|
1667
|
+
)
|
|
1472
1668
|
else:
|
|
1473
|
-
raise Exception(
|
|
1669
|
+
raise Exception(
|
|
1670
|
+
"The analysis identifier must be its name or an integer"
|
|
1671
|
+
)
|
|
1474
1672
|
|
|
1475
1673
|
search_condition = {
|
|
1476
1674
|
search_tag: analysis_name_or_id,
|
|
1477
1675
|
}
|
|
1478
1676
|
response = platform.parse_response(
|
|
1479
|
-
platform.post(
|
|
1677
|
+
platform.post(
|
|
1678
|
+
self._account.auth,
|
|
1679
|
+
"analysis_manager/get_analysis_list",
|
|
1680
|
+
data=search_condition,
|
|
1681
|
+
)
|
|
1480
1682
|
)
|
|
1481
1683
|
|
|
1482
1684
|
if len(response) > 1:
|
|
1483
|
-
raise Exception(
|
|
1685
|
+
raise Exception(
|
|
1686
|
+
f"multiple analyses with name {analysis_name_or_id} found"
|
|
1687
|
+
)
|
|
1484
1688
|
elif len(response) == 1:
|
|
1485
1689
|
return response[0]
|
|
1486
1690
|
else:
|
|
1487
1691
|
return None
|
|
1488
1692
|
|
|
1489
|
-
def list_analysis(self, search_condition=
|
|
1693
|
+
def list_analysis(self, search_condition=None, items=(0, 9999)):
|
|
1490
1694
|
"""
|
|
1491
1695
|
List the analysis available to the user.
|
|
1492
1696
|
|
|
@@ -1515,10 +1719,12 @@ class Project:
|
|
|
1515
1719
|
- qa_status: str or None pass/fail/nd QC status
|
|
1516
1720
|
- secret_name: str or None Subject ID
|
|
1517
1721
|
- tags: str or None
|
|
1518
|
-
- with_child_analysis: 1 or None if 1, child analysis of workflows
|
|
1722
|
+
- with_child_analysis: 1 or None if 1, child analysis of workflows
|
|
1723
|
+
will appear
|
|
1519
1724
|
- id: int or None ID
|
|
1520
1725
|
- state: running, completed, pending, exception or None
|
|
1521
1726
|
- username: str or None
|
|
1727
|
+
- only_data: int or None
|
|
1522
1728
|
|
|
1523
1729
|
items : List[int]
|
|
1524
1730
|
list containing two elements [min, max] that correspond to the
|
|
@@ -1529,8 +1735,17 @@ class Project:
|
|
|
1529
1735
|
dict
|
|
1530
1736
|
List of analysis, each a dictionary
|
|
1531
1737
|
"""
|
|
1532
|
-
|
|
1533
|
-
|
|
1738
|
+
if search_condition is None:
|
|
1739
|
+
search_condition = {}
|
|
1740
|
+
if len(items) != 2:
|
|
1741
|
+
raise ValueError(
|
|
1742
|
+
f"The number of elements in items '{len(items)}' "
|
|
1743
|
+
f"should be equal to two."
|
|
1744
|
+
)
|
|
1745
|
+
if not all(isinstance(item, int) for item in items):
|
|
1746
|
+
raise ValueError(
|
|
1747
|
+
f"All items elements '{items}' should be integers."
|
|
1748
|
+
)
|
|
1534
1749
|
search_keys = {
|
|
1535
1750
|
"p_n": str,
|
|
1536
1751
|
"type": str,
|
|
@@ -1543,19 +1758,37 @@ class Project:
|
|
|
1543
1758
|
"with_child_analysis": int,
|
|
1544
1759
|
"id": int,
|
|
1545
1760
|
"state": str,
|
|
1761
|
+
"only_data": int,
|
|
1546
1762
|
"username": str,
|
|
1547
1763
|
}
|
|
1548
1764
|
for key in search_condition.keys():
|
|
1549
1765
|
if key not in search_keys.keys():
|
|
1550
|
-
raise Exception(
|
|
1551
|
-
|
|
1552
|
-
|
|
1766
|
+
raise Exception(
|
|
1767
|
+
(
|
|
1768
|
+
f"This key '{key}' is not accepted by this search "
|
|
1769
|
+
f"condition"
|
|
1770
|
+
)
|
|
1771
|
+
)
|
|
1772
|
+
if (
|
|
1773
|
+
not isinstance(search_condition[key], search_keys[key])
|
|
1774
|
+
and search_condition[key] is not None
|
|
1775
|
+
):
|
|
1776
|
+
raise Exception(
|
|
1777
|
+
(
|
|
1778
|
+
f"The key {key} in the search condition is not type "
|
|
1779
|
+
f"{search_keys[key]}"
|
|
1780
|
+
)
|
|
1781
|
+
)
|
|
1553
1782
|
if "p_n" == key:
|
|
1554
|
-
|
|
1555
|
-
|
|
1783
|
+
excluded_bool = [
|
|
1784
|
+
character in search_condition["p_n"]
|
|
1785
|
+
for character in ANALYSIS_NAME_EXCLUDED_CHARACTERS
|
|
1786
|
+
]
|
|
1556
1787
|
if any(excluded_bool):
|
|
1557
|
-
raise Exception(
|
|
1558
|
-
|
|
1788
|
+
raise Exception(
|
|
1789
|
+
"p_n does not allow "
|
|
1790
|
+
f"characters {ANALYSIS_NAME_EXCLUDED_CHARACTERS}"
|
|
1791
|
+
)
|
|
1559
1792
|
req_headers = {"X-Range": f"items={items[0]}-{items[1] - 1}"}
|
|
1560
1793
|
return platform.parse_response(
|
|
1561
1794
|
platform.post(
|
|
@@ -1620,7 +1853,9 @@ class Project:
|
|
|
1620
1853
|
logger = logging.getLogger(logger_name)
|
|
1621
1854
|
|
|
1622
1855
|
if in_container_id is None and settings is None:
|
|
1623
|
-
raise ValueError(
|
|
1856
|
+
raise ValueError(
|
|
1857
|
+
"Pass a value for either in_container_id or settings."
|
|
1858
|
+
)
|
|
1624
1859
|
|
|
1625
1860
|
post_data = {"script_name": script_name, "version": version}
|
|
1626
1861
|
|
|
@@ -1653,15 +1888,19 @@ class Project:
|
|
|
1653
1888
|
|
|
1654
1889
|
logger.debug(f"post_data = {post_data}")
|
|
1655
1890
|
return self.__handle_start_analysis(
|
|
1656
|
-
post_data,
|
|
1891
|
+
post_data,
|
|
1892
|
+
ignore_warnings=ignore_warnings,
|
|
1893
|
+
ignore_file_selection=ignore_file_selection,
|
|
1657
1894
|
)
|
|
1658
1895
|
|
|
1659
1896
|
def delete_analysis(self, analysis_id):
|
|
1660
1897
|
"""
|
|
1661
1898
|
Delete an analysis
|
|
1662
1899
|
|
|
1663
|
-
|
|
1664
|
-
|
|
1900
|
+
Parameters
|
|
1901
|
+
----------
|
|
1902
|
+
analysis_id : int
|
|
1903
|
+
ID of the analysis to be deleted
|
|
1665
1904
|
"""
|
|
1666
1905
|
logger = logging.getLogger(logger_name)
|
|
1667
1906
|
|
|
@@ -1689,18 +1928,23 @@ class Project:
|
|
|
1689
1928
|
Tools can not be restarted given that they are considered as single
|
|
1690
1929
|
processing units. You can start execution of another analysis instead.
|
|
1691
1930
|
|
|
1692
|
-
For the workflow to restart, all its failed child must be removed
|
|
1693
|
-
You can only restart your own analysis.
|
|
1931
|
+
For the workflow to restart, all its failed child must be removed
|
|
1932
|
+
first. You can only restart your own analysis.
|
|
1694
1933
|
|
|
1695
|
-
|
|
1696
|
-
|
|
1934
|
+
Parameters
|
|
1935
|
+
----------
|
|
1936
|
+
analysis_id : int
|
|
1937
|
+
ID of the analysis to be restarted
|
|
1697
1938
|
"""
|
|
1698
1939
|
logger = logging.getLogger(logger_name)
|
|
1699
1940
|
|
|
1700
1941
|
analysis = self.list_analysis({"id": analysis_id})[0]
|
|
1701
1942
|
|
|
1702
1943
|
if analysis.get("super_analysis_type") != 1:
|
|
1703
|
-
raise ValueError(
|
|
1944
|
+
raise ValueError(
|
|
1945
|
+
"The analysis indicated is not a workflow and hence, it "
|
|
1946
|
+
"cannot be restarted."
|
|
1947
|
+
)
|
|
1704
1948
|
|
|
1705
1949
|
try:
|
|
1706
1950
|
platform.parse_response(
|
|
@@ -1722,7 +1966,8 @@ class Project:
|
|
|
1722
1966
|
Get the log of an analysis and save it in the provided file.
|
|
1723
1967
|
The logs of analysis can only be obtained for the tools you created.
|
|
1724
1968
|
|
|
1725
|
-
Note workflows do not have a log so the printed message will only be
|
|
1969
|
+
Note workflows do not have a log so the printed message will only be
|
|
1970
|
+
ERROR.
|
|
1726
1971
|
You can only download the anlaysis log of the tools that you own.
|
|
1727
1972
|
|
|
1728
1973
|
Note this method is very time consuming.
|
|
@@ -1745,22 +1990,32 @@ class Project:
|
|
|
1745
1990
|
try:
|
|
1746
1991
|
analysis_id = str(int(analysis_id))
|
|
1747
1992
|
except ValueError:
|
|
1748
|
-
raise ValueError(
|
|
1993
|
+
raise ValueError(
|
|
1994
|
+
f"'analysis_id' has to be an integer not '{analysis_id}'."
|
|
1995
|
+
)
|
|
1749
1996
|
|
|
1750
1997
|
file_name = file_name if file_name else f"logs_{analysis_id}.txt"
|
|
1751
1998
|
try:
|
|
1752
1999
|
res = platform.post(
|
|
1753
2000
|
auth=self._account.auth,
|
|
1754
2001
|
endpoint="analysis_manager/download_execution_file",
|
|
1755
|
-
data={
|
|
2002
|
+
data={
|
|
2003
|
+
"project_id": analysis_id,
|
|
2004
|
+
"file": f"logs_{analysis_id}",
|
|
2005
|
+
},
|
|
1756
2006
|
timeout=1000,
|
|
1757
2007
|
)
|
|
1758
2008
|
except Exception:
|
|
1759
|
-
logger.error(
|
|
2009
|
+
logger.error(
|
|
2010
|
+
f"Could not export the analysis log of '{analysis_id}'"
|
|
2011
|
+
)
|
|
1760
2012
|
return False
|
|
1761
2013
|
|
|
1762
2014
|
if not res.ok:
|
|
1763
|
-
logger.error(
|
|
2015
|
+
logger.error(
|
|
2016
|
+
f"The log file could not be extracted for Analysis ID:"
|
|
2017
|
+
f" {analysis_id}."
|
|
2018
|
+
)
|
|
1764
2019
|
return False
|
|
1765
2020
|
|
|
1766
2021
|
with open(file_name, "w") as f:
|
|
@@ -1769,7 +2024,9 @@ class Project:
|
|
|
1769
2024
|
|
|
1770
2025
|
""" QC Status Related Methods """
|
|
1771
2026
|
|
|
1772
|
-
def set_qc_status_analysis(
|
|
2027
|
+
def set_qc_status_analysis(
|
|
2028
|
+
self, analysis_id, status=QCStatus.UNDERTERMINED, comments=""
|
|
2029
|
+
):
|
|
1773
2030
|
"""
|
|
1774
2031
|
Changes the analysis QC status.
|
|
1775
2032
|
|
|
@@ -1798,7 +2055,10 @@ class Project:
|
|
|
1798
2055
|
try:
|
|
1799
2056
|
analysis_id = str(int(analysis_id))
|
|
1800
2057
|
except ValueError:
|
|
1801
|
-
raise ValueError(
|
|
2058
|
+
raise ValueError(
|
|
2059
|
+
f"analysis_id: '{analysis_id}' not valid. Must be convertible "
|
|
2060
|
+
f"to int."
|
|
2061
|
+
)
|
|
1802
2062
|
|
|
1803
2063
|
try:
|
|
1804
2064
|
platform.parse_response(
|
|
@@ -1814,11 +2074,16 @@ class Project:
|
|
|
1814
2074
|
)
|
|
1815
2075
|
)
|
|
1816
2076
|
except Exception:
|
|
1817
|
-
logger.error(
|
|
2077
|
+
logger.error(
|
|
2078
|
+
f"It was not possible to change the QC status of Analysis ID:"
|
|
2079
|
+
f" {analysis_id}"
|
|
2080
|
+
)
|
|
1818
2081
|
return False
|
|
1819
2082
|
return True
|
|
1820
2083
|
|
|
1821
|
-
def set_qc_status_subject(
|
|
2084
|
+
def set_qc_status_subject(
|
|
2085
|
+
self, patient_id, status=QCStatus.UNDERTERMINED, comments=""
|
|
2086
|
+
):
|
|
1822
2087
|
"""
|
|
1823
2088
|
Changes the QC status of a Patient ID (equivalent to a
|
|
1824
2089
|
Subject ID/Session ID).
|
|
@@ -1847,7 +2112,10 @@ class Project:
|
|
|
1847
2112
|
try:
|
|
1848
2113
|
patient_id = str(int(patient_id))
|
|
1849
2114
|
except ValueError:
|
|
1850
|
-
raise ValueError(
|
|
2115
|
+
raise ValueError(
|
|
2116
|
+
f"'patient_id': '{patient_id}' not valid. Must be convertible"
|
|
2117
|
+
f" to int."
|
|
2118
|
+
)
|
|
1851
2119
|
|
|
1852
2120
|
try:
|
|
1853
2121
|
platform.parse_response(
|
|
@@ -1863,7 +2131,10 @@ class Project:
|
|
|
1863
2131
|
)
|
|
1864
2132
|
)
|
|
1865
2133
|
except Exception:
|
|
1866
|
-
logger.error(
|
|
2134
|
+
logger.error(
|
|
2135
|
+
f"It was not possible to change the QC status of Patient ID:"
|
|
2136
|
+
f" {patient_id}"
|
|
2137
|
+
)
|
|
1867
2138
|
return False
|
|
1868
2139
|
return True
|
|
1869
2140
|
|
|
@@ -1888,17 +2159,28 @@ class Project:
|
|
|
1888
2159
|
try:
|
|
1889
2160
|
search_criteria = {"id": analysis_id}
|
|
1890
2161
|
to_return = self.list_analysis(search_criteria)
|
|
1891
|
-
return
|
|
2162
|
+
return (
|
|
2163
|
+
convert_qc_value_to_qcstatus(to_return[0]["qa_status"]),
|
|
2164
|
+
to_return[0]["qa_comments"],
|
|
2165
|
+
)
|
|
1892
2166
|
except IndexError:
|
|
1893
2167
|
# Handle the case where no matching analysis is found
|
|
1894
|
-
logging.error(
|
|
2168
|
+
logging.error(
|
|
2169
|
+
f"No analysis was found with such Analysis ID: "
|
|
2170
|
+
f"'{analysis_id}'."
|
|
2171
|
+
)
|
|
1895
2172
|
return False, False
|
|
1896
2173
|
except Exception:
|
|
1897
2174
|
# Handle other potential exceptions
|
|
1898
|
-
logging.error(
|
|
2175
|
+
logging.error(
|
|
2176
|
+
f"It was not possible to extract the QC status from Analysis "
|
|
2177
|
+
f"ID: {analysis_id}"
|
|
2178
|
+
)
|
|
1899
2179
|
return False, False
|
|
1900
2180
|
|
|
1901
|
-
def get_qc_status_subject(
|
|
2181
|
+
def get_qc_status_subject(
|
|
2182
|
+
self, patient_id=None, subject_name=None, ssid=None
|
|
2183
|
+
):
|
|
1902
2184
|
"""
|
|
1903
2185
|
Gets the session QC status via the patient ID or the Subject ID
|
|
1904
2186
|
and the Session ID.
|
|
@@ -1926,26 +2208,50 @@ class Project:
|
|
|
1926
2208
|
try:
|
|
1927
2209
|
patient_id = int(patient_id)
|
|
1928
2210
|
except ValueError:
|
|
1929
|
-
raise ValueError(
|
|
2211
|
+
raise ValueError(
|
|
2212
|
+
f"patient_id '{patient_id}' should be an integer."
|
|
2213
|
+
)
|
|
1930
2214
|
sessions = self.get_subjects_metadata(search_criteria={})
|
|
1931
|
-
session = [
|
|
2215
|
+
session = [
|
|
2216
|
+
session
|
|
2217
|
+
for session in sessions
|
|
2218
|
+
if int(session["_id"]) == patient_id
|
|
2219
|
+
]
|
|
1932
2220
|
if len(session) < 1:
|
|
1933
|
-
logging.error(
|
|
2221
|
+
logging.error(
|
|
2222
|
+
f"No session was found with Patient ID: '{patient_id}'."
|
|
2223
|
+
)
|
|
1934
2224
|
return False, False
|
|
1935
|
-
return
|
|
2225
|
+
return (
|
|
2226
|
+
convert_qc_value_to_qcstatus(session[0]["qa_status"]),
|
|
2227
|
+
session[0]["qa_comments"],
|
|
2228
|
+
)
|
|
1936
2229
|
elif subject_name and ssid:
|
|
1937
2230
|
session = self.get_subjects_metadata(
|
|
1938
2231
|
search_criteria={
|
|
1939
2232
|
"pars_patient_secret_name": f"string;{subject_name}",
|
|
1940
|
-
"pars_ssid":
|
|
2233
|
+
"pars_ssid": (
|
|
2234
|
+
f"integer;eq|{ssid}"
|
|
2235
|
+
if str(ssid).isdigit()
|
|
2236
|
+
else f"string;{ssid}"
|
|
2237
|
+
),
|
|
1941
2238
|
}
|
|
1942
2239
|
)
|
|
1943
2240
|
if len(session) < 1:
|
|
1944
|
-
logging.error(
|
|
2241
|
+
logging.error(
|
|
2242
|
+
f"No session was found with Subject ID: '{subject_name}'"
|
|
2243
|
+
f" and Session ID: '{ssid}'."
|
|
2244
|
+
)
|
|
1945
2245
|
return False, False
|
|
1946
|
-
return
|
|
2246
|
+
return (
|
|
2247
|
+
convert_qc_value_to_qcstatus(session[0]["qa_status"]),
|
|
2248
|
+
session[0]["qa_comments"],
|
|
2249
|
+
)
|
|
1947
2250
|
else:
|
|
1948
|
-
raise ValueError(
|
|
2251
|
+
raise ValueError(
|
|
2252
|
+
"Either 'patient_id' or 'subject_name' and 'ssid' must "
|
|
2253
|
+
"not be empty."
|
|
2254
|
+
)
|
|
1949
2255
|
|
|
1950
2256
|
""" Protocol Adherence Related Methods """
|
|
1951
2257
|
|
|
@@ -1973,7 +2279,9 @@ class Project:
|
|
|
1973
2279
|
with open(rules_file_path, "r") as fr:
|
|
1974
2280
|
rules = json.load(fr)
|
|
1975
2281
|
except FileNotFoundError:
|
|
1976
|
-
logger.error(
|
|
2282
|
+
logger.error(
|
|
2283
|
+
f"Protocol adherence rule file '{rules_file_path}' not found."
|
|
2284
|
+
)
|
|
1977
2285
|
return False
|
|
1978
2286
|
|
|
1979
2287
|
# Update the project's QA rules
|
|
@@ -1981,18 +2289,26 @@ class Project:
|
|
|
1981
2289
|
platform.post(
|
|
1982
2290
|
auth=self._account.auth,
|
|
1983
2291
|
endpoint="projectset_manager/set_session_qa_requirements",
|
|
1984
|
-
data={
|
|
2292
|
+
data={
|
|
2293
|
+
"project_id": self._project_id,
|
|
2294
|
+
"rules": json.dumps(rules),
|
|
2295
|
+
"guidance_text": guidance_text,
|
|
2296
|
+
},
|
|
1985
2297
|
)
|
|
1986
2298
|
)
|
|
1987
2299
|
|
|
1988
2300
|
if not res.get("success") == 1:
|
|
1989
|
-
logger.error(
|
|
2301
|
+
logger.error(
|
|
2302
|
+
"There was an error setting up the protocol adherence rules."
|
|
2303
|
+
)
|
|
1990
2304
|
logger.error(platform.parse_response(res))
|
|
1991
2305
|
return False
|
|
1992
2306
|
|
|
1993
2307
|
return True
|
|
1994
2308
|
|
|
1995
|
-
def get_project_pa_rules(
|
|
2309
|
+
def get_project_pa_rules(
|
|
2310
|
+
self, rules_file_path, project_has_no_rules=False
|
|
2311
|
+
):
|
|
1996
2312
|
"""
|
|
1997
2313
|
Retrive the active project's protocol adherence rules
|
|
1998
2314
|
|
|
@@ -2001,6 +2317,8 @@ class Project:
|
|
|
2001
2317
|
rules_file_path : str
|
|
2002
2318
|
The file path to the JSON file to store the protocol adherence
|
|
2003
2319
|
rules.
|
|
2320
|
+
project_has_no_rules: bool
|
|
2321
|
+
for testing purposes
|
|
2004
2322
|
|
|
2005
2323
|
Returns
|
|
2006
2324
|
-------
|
|
@@ -2020,47 +2338,58 @@ class Project:
|
|
|
2020
2338
|
)
|
|
2021
2339
|
)
|
|
2022
2340
|
|
|
2023
|
-
if "rules" not in res:
|
|
2024
|
-
logger.error(
|
|
2341
|
+
if "rules" not in res or project_has_no_rules:
|
|
2342
|
+
logger.error(
|
|
2343
|
+
f"There was an error extracting the protocol adherence rules"
|
|
2344
|
+
f" from {self._project_name}."
|
|
2345
|
+
)
|
|
2025
2346
|
logger.error(platform.parse_response(res))
|
|
2026
2347
|
return False
|
|
2027
2348
|
|
|
2028
2349
|
try:
|
|
2029
2350
|
for rule in res["rules"]:
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2351
|
+
for key in ["_id", "order", "time_modified"]:
|
|
2352
|
+
if rule.get(key, False):
|
|
2353
|
+
del rule[key]
|
|
2033
2354
|
with open(rules_file_path, "w") as fr:
|
|
2034
2355
|
json.dump(res["rules"], fr, indent=4)
|
|
2035
2356
|
except FileNotFoundError:
|
|
2036
|
-
logger.error(
|
|
2357
|
+
logger.error(
|
|
2358
|
+
f"Protocol adherence rules could not be exported to file: "
|
|
2359
|
+
f"'{rules_file_path}'."
|
|
2360
|
+
)
|
|
2037
2361
|
return False
|
|
2038
2362
|
|
|
2039
2363
|
return res["guidance_text"]
|
|
2040
2364
|
|
|
2041
2365
|
def parse_qc_text(self, patient_id=None, subject_name=None, ssid=None):
|
|
2042
2366
|
"""
|
|
2043
|
-
Parse QC (Quality Control) text output into a structured dictionary
|
|
2367
|
+
Parse QC (Quality Control) text output into a structured dictionary
|
|
2368
|
+
format.
|
|
2044
2369
|
|
|
2045
|
-
This function takes raw QC text output (from the Protocol Adherence
|
|
2046
|
-
and parses it into a structured format that
|
|
2047
|
-
along with their associated files
|
|
2370
|
+
This function takes raw QC text output (from the Protocol Adherence
|
|
2371
|
+
analysis) and parses it into a structured format that
|
|
2372
|
+
separates passed and failed rules, along with their associated files
|
|
2373
|
+
and conditions.
|
|
2048
2374
|
|
|
2049
2375
|
Args:
|
|
2050
2376
|
patient_id (str, optional):
|
|
2051
2377
|
Patient identifier. Defaults to None.
|
|
2052
2378
|
subject_name (str, optional):
|
|
2053
|
-
Subject/patient name. Defaults to None. Mandatory if no
|
|
2379
|
+
Subject/patient name. Defaults to None. Mandatory if no
|
|
2380
|
+
patient_id is provided.
|
|
2054
2381
|
ssid (str, optional):
|
|
2055
|
-
Session ID. Defaults to None. Mandatory if subject_name is
|
|
2382
|
+
Session ID. Defaults to None. Mandatory if subject_name is
|
|
2383
|
+
provided.
|
|
2056
2384
|
|
|
2057
2385
|
Returns:
|
|
2058
|
-
dict: A structured dictionary containing a list of dictionaries
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2386
|
+
dict: A structured dictionary containing a list of dictionaries
|
|
2387
|
+
with passed rules and their details and failed rules and their
|
|
2388
|
+
details. Details of passed rules are:
|
|
2389
|
+
per each rule: Files that have passed the rule. Per each file name
|
|
2390
|
+
of the file and number of conditions of the rule. Details of
|
|
2391
|
+
failed rules are: per each rule failed conditions: Number of
|
|
2392
|
+
times it failed. Each condition status.
|
|
2064
2393
|
|
|
2065
2394
|
Example:
|
|
2066
2395
|
>>> parse_qc_text(subject_name="patient_123", ssid=1)
|
|
@@ -2086,7 +2415,7 @@ class Project:
|
|
|
2086
2415
|
"conditions": [
|
|
2087
2416
|
{
|
|
2088
2417
|
"status": "failed",
|
|
2089
|
-
"condition": "SliceThickness between
|
|
2418
|
+
"condition": "SliceThickness between.."
|
|
2090
2419
|
}
|
|
2091
2420
|
]
|
|
2092
2421
|
}
|
|
@@ -2099,7 +2428,9 @@ class Project:
|
|
|
2099
2428
|
}
|
|
2100
2429
|
"""
|
|
2101
2430
|
|
|
2102
|
-
_, text = self.get_qc_status_subject(
|
|
2431
|
+
_, text = self.get_qc_status_subject(
|
|
2432
|
+
patient_id=patient_id, subject_name=subject_name, ssid=ssid
|
|
2433
|
+
)
|
|
2103
2434
|
|
|
2104
2435
|
result = {"passed": [], "failed": []}
|
|
2105
2436
|
|
|
@@ -2129,22 +2460,27 @@ class Project:
|
|
|
2129
2460
|
|
|
2130
2461
|
def calculate_qc_statistics(self):
|
|
2131
2462
|
"""
|
|
2132
|
-
Calculate comprehensive statistics from multiple QC results across
|
|
2133
|
-
platform.
|
|
2463
|
+
Calculate comprehensive statistics from multiple QC results across
|
|
2464
|
+
subjects from a project in the QMENTA platform.
|
|
2134
2465
|
|
|
2135
|
-
This function aggregates and analyzes QC results from
|
|
2136
|
-
providing statistical insights about
|
|
2137
|
-
and condition failure patterns.
|
|
2466
|
+
This function aggregates and analyzes QC results from
|
|
2467
|
+
multiple subjects/containers, providing statistical insights about
|
|
2468
|
+
rule pass/fail rates, file statistics, and condition failure patterns.
|
|
2138
2469
|
|
|
2139
2470
|
Returns:
|
|
2140
|
-
dict: A dictionary containing comprehensive QC statistics
|
|
2471
|
+
dict: A dictionary containing comprehensive QC statistics
|
|
2472
|
+
including:
|
|
2141
2473
|
- passed_rules: Total count of passed rules across all subjects
|
|
2142
2474
|
- failed_rules: Total count of failed rules across all subjects
|
|
2143
2475
|
- subjects_passed: Count of subjects with no failed rules
|
|
2144
|
-
- subjects_with_failed: Count of subjects with at least one
|
|
2145
|
-
|
|
2146
|
-
-
|
|
2147
|
-
|
|
2476
|
+
- subjects_with_failed: Count of subjects with at least one
|
|
2477
|
+
failed rule
|
|
2478
|
+
- num_passed_files_distribution: Distribution of how many
|
|
2479
|
+
rules have N passed files
|
|
2480
|
+
- file_stats: File-level statistics (total, passed, failed,
|
|
2481
|
+
pass percentage)
|
|
2482
|
+
- condition_failure_rates: Frequency and percentage of each
|
|
2483
|
+
failed condition
|
|
2148
2484
|
- rule_success_rates: Success rates for each rule type
|
|
2149
2485
|
|
|
2150
2486
|
The statistics help identify:
|
|
@@ -2190,7 +2526,11 @@ class Project:
|
|
|
2190
2526
|
containers = self.list_input_containers()
|
|
2191
2527
|
|
|
2192
2528
|
for c in containers:
|
|
2193
|
-
qc_results_list.append(
|
|
2529
|
+
qc_results_list.append(
|
|
2530
|
+
self.parse_qc_text(
|
|
2531
|
+
subject_name=c["patient_secret_name"], ssid=c["ssid"]
|
|
2532
|
+
)
|
|
2533
|
+
)
|
|
2194
2534
|
|
|
2195
2535
|
# Initialize statistics
|
|
2196
2536
|
stats = {
|
|
@@ -2198,22 +2538,49 @@ class Project:
|
|
|
2198
2538
|
"failed_rules": 0,
|
|
2199
2539
|
"subjects_passed": 0,
|
|
2200
2540
|
"subjects_with_failed": 0,
|
|
2201
|
-
"num_passed_files_distribution": defaultdict(
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
"
|
|
2541
|
+
"num_passed_files_distribution": defaultdict(
|
|
2542
|
+
int
|
|
2543
|
+
), # How many rules have N passed files
|
|
2544
|
+
"file_stats": {
|
|
2545
|
+
"total": 0,
|
|
2546
|
+
"passed": 0,
|
|
2547
|
+
"failed": 0,
|
|
2548
|
+
"pass_percentage": 0.0,
|
|
2549
|
+
},
|
|
2550
|
+
"condition_failure_rates": defaultdict(
|
|
2551
|
+
lambda: {"count": 0, "percentage": 0.0}
|
|
2552
|
+
),
|
|
2553
|
+
"rule_success_rates": defaultdict(
|
|
2554
|
+
lambda: {"passed": 0, "failed": 0, "success_rate": 0.0}
|
|
2555
|
+
),
|
|
2205
2556
|
}
|
|
2206
2557
|
|
|
2207
2558
|
total_failures = 0
|
|
2208
2559
|
|
|
2209
2560
|
# sum subjects with not failed qc message
|
|
2210
|
-
stats["subjects_passed"] = sum(
|
|
2561
|
+
stats["subjects_passed"] = sum(
|
|
2562
|
+
[1 for rules in qc_results_list if not rules["failed"]]
|
|
2563
|
+
)
|
|
2211
2564
|
# sum subjects with some failed qc message
|
|
2212
|
-
stats["subjects_with_failed"] = sum(
|
|
2565
|
+
stats["subjects_with_failed"] = sum(
|
|
2566
|
+
[1 for rules in qc_results_list if rules["failed"]]
|
|
2567
|
+
)
|
|
2213
2568
|
# sum rules that have passed
|
|
2214
|
-
stats["passed_rules"] = sum(
|
|
2569
|
+
stats["passed_rules"] = sum(
|
|
2570
|
+
[
|
|
2571
|
+
len(rules["passed"])
|
|
2572
|
+
for rules in qc_results_list
|
|
2573
|
+
if rules["failed"]
|
|
2574
|
+
]
|
|
2575
|
+
)
|
|
2215
2576
|
# sum rules that have failed
|
|
2216
|
-
stats["failed_rules"] = sum(
|
|
2577
|
+
stats["failed_rules"] = sum(
|
|
2578
|
+
[
|
|
2579
|
+
len(rules["failed"])
|
|
2580
|
+
for rules in qc_results_list
|
|
2581
|
+
if rules["failed"]
|
|
2582
|
+
]
|
|
2583
|
+
)
|
|
2217
2584
|
|
|
2218
2585
|
for qc_results in qc_results_list:
|
|
2219
2586
|
|
|
@@ -2231,42 +2598,72 @@ class Project:
|
|
|
2231
2598
|
stats["file_stats"]["failed"] += len(rule["files"])
|
|
2232
2599
|
for condition, count in rule["failed_conditions"].items():
|
|
2233
2600
|
# Extract just the condition text without actual value
|
|
2234
|
-
clean_condition = re.sub(
|
|
2235
|
-
|
|
2601
|
+
clean_condition = re.sub(
|
|
2602
|
+
r"\.\s*Actual value:.*$", "", condition
|
|
2603
|
+
)
|
|
2604
|
+
stats["condition_failure_rates"][clean_condition][
|
|
2605
|
+
"count"
|
|
2606
|
+
] += count
|
|
2236
2607
|
total_failures += count
|
|
2237
2608
|
rule_name = rule["rule"]
|
|
2238
2609
|
stats["rule_success_rates"][rule_name]["failed"] += 1
|
|
2239
2610
|
|
|
2240
2611
|
if stats["file_stats"]["total"] > 0:
|
|
2241
2612
|
stats["file_stats"]["pass_percentage"] = round(
|
|
2242
|
-
(stats["file_stats"]["passed"] / stats["file_stats"]["total"])
|
|
2613
|
+
(stats["file_stats"]["passed"] / stats["file_stats"]["total"])
|
|
2614
|
+
* 100,
|
|
2615
|
+
2,
|
|
2243
2616
|
)
|
|
2244
2617
|
|
|
2245
2618
|
# Calculate condition failure percentages
|
|
2246
2619
|
for condition in stats["condition_failure_rates"]:
|
|
2247
2620
|
if total_failures > 0:
|
|
2248
|
-
stats["condition_failure_rates"][condition]["percentage"] =
|
|
2249
|
-
(
|
|
2621
|
+
stats["condition_failure_rates"][condition]["percentage"] = (
|
|
2622
|
+
round(
|
|
2623
|
+
(
|
|
2624
|
+
stats["condition_failure_rates"][condition][
|
|
2625
|
+
"count"
|
|
2626
|
+
]
|
|
2627
|
+
/ total_failures
|
|
2628
|
+
)
|
|
2629
|
+
* 100,
|
|
2630
|
+
2,
|
|
2631
|
+
)
|
|
2250
2632
|
)
|
|
2251
2633
|
|
|
2252
2634
|
# Calculate rule success rates
|
|
2253
2635
|
for rule in stats["rule_success_rates"]:
|
|
2254
|
-
total =
|
|
2636
|
+
total = (
|
|
2637
|
+
stats["rule_success_rates"][rule]["passed"]
|
|
2638
|
+
+ stats["rule_success_rates"][rule]["failed"]
|
|
2639
|
+
)
|
|
2255
2640
|
if total > 0:
|
|
2256
2641
|
stats["rule_success_rates"][rule]["success_rate"] = round(
|
|
2257
|
-
(stats["rule_success_rates"][rule]["passed"] / total)
|
|
2642
|
+
(stats["rule_success_rates"][rule]["passed"] / total)
|
|
2643
|
+
* 100,
|
|
2644
|
+
2,
|
|
2258
2645
|
)
|
|
2259
2646
|
|
|
2260
2647
|
# Convert defaultdict to regular dict for cleaner JSON output
|
|
2261
|
-
stats["num_passed_files_distribution"] = dict(
|
|
2262
|
-
|
|
2648
|
+
stats["num_passed_files_distribution"] = dict(
|
|
2649
|
+
stats["num_passed_files_distribution"]
|
|
2650
|
+
)
|
|
2651
|
+
stats["condition_failure_rates"] = dict(
|
|
2652
|
+
stats["condition_failure_rates"]
|
|
2653
|
+
)
|
|
2263
2654
|
stats["rule_success_rates"] = dict(stats["rule_success_rates"])
|
|
2264
2655
|
|
|
2265
2656
|
return stats
|
|
2266
2657
|
|
|
2267
2658
|
""" Helper Methods """
|
|
2268
2659
|
|
|
2269
|
-
def __handle_start_analysis(
|
|
2660
|
+
def __handle_start_analysis(
|
|
2661
|
+
self,
|
|
2662
|
+
post_data,
|
|
2663
|
+
ignore_warnings=False,
|
|
2664
|
+
ignore_file_selection=True,
|
|
2665
|
+
n_calls=0,
|
|
2666
|
+
):
|
|
2270
2667
|
"""
|
|
2271
2668
|
Handle the possible responses from the server after start_analysis.
|
|
2272
2669
|
Sometimes we have to send a request again, and then check again the
|
|
@@ -2286,13 +2683,21 @@ class Project:
|
|
|
2286
2683
|
than {n_calls} times: aborting."
|
|
2287
2684
|
)
|
|
2288
2685
|
return None
|
|
2289
|
-
|
|
2686
|
+
response = None
|
|
2290
2687
|
try:
|
|
2291
2688
|
response = platform.parse_response(
|
|
2292
|
-
platform.post(
|
|
2689
|
+
platform.post(
|
|
2690
|
+
self._account.auth,
|
|
2691
|
+
"analysis_manager/analysis_registration",
|
|
2692
|
+
data=post_data,
|
|
2693
|
+
)
|
|
2293
2694
|
)
|
|
2294
2695
|
logger.info(response["message"])
|
|
2295
|
-
return
|
|
2696
|
+
return (
|
|
2697
|
+
int(response["analysis_id"])
|
|
2698
|
+
if "analysis_id" in response
|
|
2699
|
+
else None
|
|
2700
|
+
)
|
|
2296
2701
|
|
|
2297
2702
|
except platform.ChooseDataError as choose_data:
|
|
2298
2703
|
if ignore_file_selection:
|
|
@@ -2312,31 +2717,39 @@ class Project:
|
|
|
2312
2717
|
# logging any warning that we have
|
|
2313
2718
|
if choose_data.warning:
|
|
2314
2719
|
has_warning = True
|
|
2315
|
-
logger.warning(
|
|
2720
|
+
logger.warning(choose_data.warning)
|
|
2316
2721
|
|
|
2317
2722
|
new_post = {
|
|
2318
2723
|
"analysis_id": choose_data.analysis_id,
|
|
2319
2724
|
"script_name": post_data["script_name"],
|
|
2320
2725
|
"version": post_data["version"],
|
|
2321
2726
|
}
|
|
2727
|
+
if "tags" in post_data.keys():
|
|
2728
|
+
new_post["tags"] = post_data["tags"]
|
|
2322
2729
|
|
|
2323
2730
|
if choose_data.data_to_choose:
|
|
2324
2731
|
self.__handle_manual_choose_data(new_post, choose_data)
|
|
2325
2732
|
else:
|
|
2326
2733
|
if has_warning and not ignore_warnings:
|
|
2327
|
-
logger.error(
|
|
2734
|
+
logger.error(
|
|
2735
|
+
"Cancelling analysis due to warnings, set "
|
|
2736
|
+
"'ignore_warnings' to True to override."
|
|
2737
|
+
)
|
|
2328
2738
|
new_post["cancel"] = "1"
|
|
2329
2739
|
else:
|
|
2330
2740
|
logger.info("suppressing warnings")
|
|
2331
2741
|
new_post["user_preference"] = "{}"
|
|
2332
2742
|
new_post["_mint_only_warning"] = "1"
|
|
2333
2743
|
|
|
2334
|
-
return self.__handle_start_analysis(
|
|
2744
|
+
return self.__handle_start_analysis(
|
|
2745
|
+
new_post, ignore_warnings, ignore_file_selection, n_calls
|
|
2746
|
+
)
|
|
2335
2747
|
except platform.ActionFailedError as e:
|
|
2336
2748
|
logger.error(f"Unable to start the analysis: {e}.")
|
|
2337
2749
|
return None
|
|
2338
2750
|
|
|
2339
|
-
|
|
2751
|
+
@staticmethod
|
|
2752
|
+
def __handle_manual_choose_data(post_data, choose_data):
|
|
2340
2753
|
"""
|
|
2341
2754
|
Handle the responses of the user when there is need to select a file
|
|
2342
2755
|
to start the analysis.
|
|
@@ -2349,15 +2762,22 @@ class Project:
|
|
|
2349
2762
|
post_data : dict
|
|
2350
2763
|
Current post_data dictionary. To be mofidied in-place.
|
|
2351
2764
|
choose_data : platform.ChooseDataError
|
|
2352
|
-
Error raised when trying to start an analysis, but data has to
|
|
2765
|
+
Error raised when trying to start an analysis, but data has to
|
|
2766
|
+
be chosen.
|
|
2353
2767
|
"""
|
|
2354
2768
|
|
|
2355
2769
|
logger = logging.getLogger(logger_name)
|
|
2356
|
-
logger.warning(
|
|
2770
|
+
logger.warning(
|
|
2771
|
+
"Multiple inputs available. You have to select the desired file/s "
|
|
2772
|
+
"to continue."
|
|
2773
|
+
)
|
|
2357
2774
|
# in case we have data to choose
|
|
2358
2775
|
chosen_files = {}
|
|
2359
2776
|
for settings_key in choose_data.data_to_choose:
|
|
2360
|
-
logger.warning(
|
|
2777
|
+
logger.warning(
|
|
2778
|
+
f"Type next the file/s for the input with ID: "
|
|
2779
|
+
f"'{settings_key}'."
|
|
2780
|
+
)
|
|
2361
2781
|
chosen_files[settings_key] = {}
|
|
2362
2782
|
filters = choose_data.data_to_choose[settings_key]["filters"]
|
|
2363
2783
|
for filter_key in filters:
|
|
@@ -2372,7 +2792,9 @@ class Project:
|
|
|
2372
2792
|
if filter_data["range"][0] != 0:
|
|
2373
2793
|
number_of_files_to_select = filter_data["range"][0]
|
|
2374
2794
|
elif filter_data["range"][1] != 0:
|
|
2375
|
-
number_of_files_to_select = min(
|
|
2795
|
+
number_of_files_to_select = min(
|
|
2796
|
+
filter_data["range"][1], len(filter_data["files"])
|
|
2797
|
+
)
|
|
2376
2798
|
else:
|
|
2377
2799
|
number_of_files_to_select = len(filter_data["files"])
|
|
2378
2800
|
|
|
@@ -2384,19 +2806,29 @@ class Project:
|
|
|
2384
2806
|
# list_container_filter_files()
|
|
2385
2807
|
|
|
2386
2808
|
if number_of_files_to_select != len(filter_data["files"]):
|
|
2809
|
+
substring = ""
|
|
2810
|
+
if number_of_files_to_select > 1:
|
|
2811
|
+
substring = "s (i.e., file1.zip, file2.zip, file3.zip)"
|
|
2387
2812
|
logger.warning(
|
|
2388
2813
|
f" · File filter name: '{filter_key}'. Type "
|
|
2389
|
-
f"{number_of_files_to_select} file"
|
|
2390
|
-
f"{'s (i.e., file1.zip, file2.zip, file3.zip)' if number_of_files_to_select > 1 else ''}."
|
|
2814
|
+
f"{number_of_files_to_select} file{substring}."
|
|
2391
2815
|
)
|
|
2392
2816
|
save_file_ids, select_file_filter = {}, ""
|
|
2393
2817
|
for file_ in filter_data["files"]:
|
|
2394
|
-
select_file_filter +=
|
|
2818
|
+
select_file_filter += (
|
|
2819
|
+
f" · File name: {file_['name']}\n"
|
|
2820
|
+
)
|
|
2395
2821
|
save_file_ids[file_["name"]] = file_["_id"]
|
|
2396
|
-
names = [
|
|
2822
|
+
names = [
|
|
2823
|
+
el.strip()
|
|
2824
|
+
for el in input(select_file_filter).strip().split(",")
|
|
2825
|
+
]
|
|
2397
2826
|
|
|
2398
2827
|
if len(names) != number_of_files_to_select:
|
|
2399
|
-
logger.error(
|
|
2828
|
+
logger.error(
|
|
2829
|
+
"The number of files selected does not correspond "
|
|
2830
|
+
"to the number of needed files."
|
|
2831
|
+
)
|
|
2400
2832
|
logger.error(
|
|
2401
2833
|
f"Selected: {len(names)} vs. "
|
|
2402
2834
|
f"Number of files to select: "
|
|
@@ -2406,14 +2838,27 @@ class Project:
|
|
|
2406
2838
|
post_data["cancel"] = "1"
|
|
2407
2839
|
|
|
2408
2840
|
elif any([name not in save_file_ids for name in names]):
|
|
2409
|
-
logger.error(
|
|
2841
|
+
logger.error(
|
|
2842
|
+
f"Some selected file/s '{', '.join(names)}' "
|
|
2843
|
+
f"do not exist. Cancelling analysis..."
|
|
2844
|
+
)
|
|
2410
2845
|
post_data["cancel"] = "1"
|
|
2411
2846
|
else:
|
|
2412
|
-
chosen_files[settings_key][filter_key] = [
|
|
2847
|
+
chosen_files[settings_key][filter_key] = [
|
|
2848
|
+
save_file_ids[name] for name in names
|
|
2849
|
+
]
|
|
2413
2850
|
|
|
2414
2851
|
else:
|
|
2415
|
-
logger.warning(
|
|
2416
|
-
|
|
2852
|
+
logger.warning(
|
|
2853
|
+
"Setting all available files to be input to the "
|
|
2854
|
+
"analysis."
|
|
2855
|
+
)
|
|
2856
|
+
files_selection = [
|
|
2857
|
+
ff["_id"]
|
|
2858
|
+
for ff in filter_data["files"][
|
|
2859
|
+
:number_of_files_to_select
|
|
2860
|
+
]
|
|
2861
|
+
]
|
|
2417
2862
|
chosen_files[settings_key][filter_key] = files_selection
|
|
2418
2863
|
|
|
2419
2864
|
post_data["user_preference"] = json.dumps(chosen_files)
|
|
@@ -2427,20 +2872,6 @@ class Project:
|
|
|
2427
2872
|
modalities.append(modality)
|
|
2428
2873
|
return modalities
|
|
2429
2874
|
|
|
2430
|
-
def __show_progress(self, done, total, finish=False):
|
|
2431
|
-
bytes_in_mb = 1024 * 1024
|
|
2432
|
-
progress_message = "\r[{:.2f} %] Uploaded {:.2f} of {:.2f} Mb".format(
|
|
2433
|
-
done / float(total) * 100, done / bytes_in_mb, total / bytes_in_mb
|
|
2434
|
-
)
|
|
2435
|
-
sys.stdout.write(progress_message)
|
|
2436
|
-
sys.stdout.flush()
|
|
2437
|
-
if not finish:
|
|
2438
|
-
pass
|
|
2439
|
-
# sys.stdout.write("")
|
|
2440
|
-
# sys.stdout.flush()
|
|
2441
|
-
else:
|
|
2442
|
-
sys.stdout.write("\n")
|
|
2443
|
-
|
|
2444
2875
|
def __get_session_id(self, file_path):
|
|
2445
2876
|
m = hashlib.md5()
|
|
2446
2877
|
m.update(file_path.encode("utf-8"))
|
|
@@ -2472,11 +2903,12 @@ class Project:
|
|
|
2472
2903
|
else:
|
|
2473
2904
|
return True
|
|
2474
2905
|
|
|
2475
|
-
|
|
2906
|
+
@staticmethod
|
|
2907
|
+
def __operation(reference_value, operator, input_value):
|
|
2476
2908
|
"""
|
|
2477
2909
|
The method performs an operation by comparing the two input values.
|
|
2478
|
-
The Operation is applied to the Input Value in comparison to the
|
|
2479
|
-
Value.
|
|
2910
|
+
The Operation is applied to the Input Value in comparison to the
|
|
2911
|
+
Reference Value.
|
|
2480
2912
|
|
|
2481
2913
|
Parameters
|
|
2482
2914
|
----------
|
|
@@ -2492,39 +2924,32 @@ class Project:
|
|
|
2492
2924
|
bool
|
|
2493
2925
|
True if the operation is satisfied, False otherwise.
|
|
2494
2926
|
"""
|
|
2495
|
-
if input_value
|
|
2927
|
+
if not input_value: # Handles None, "", and other falsy values
|
|
2496
2928
|
return False
|
|
2497
2929
|
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
elif operator == "gte":
|
|
2511
|
-
return input_value >= reference_value
|
|
2512
|
-
|
|
2513
|
-
elif operator == "lt":
|
|
2514
|
-
return input_value < reference_value
|
|
2930
|
+
operator_actions = {
|
|
2931
|
+
"in": lambda: reference_value in input_value,
|
|
2932
|
+
"in-list": lambda: all(
|
|
2933
|
+
el in input_value for el in reference_value
|
|
2934
|
+
),
|
|
2935
|
+
"eq": lambda: input_value == reference_value,
|
|
2936
|
+
"gt": lambda: input_value > reference_value,
|
|
2937
|
+
"gte": lambda: input_value >= reference_value,
|
|
2938
|
+
"lt": lambda: input_value < reference_value,
|
|
2939
|
+
"lte": lambda: input_value <= reference_value,
|
|
2940
|
+
}
|
|
2515
2941
|
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
else:
|
|
2519
|
-
return False
|
|
2942
|
+
action = operator_actions.get(operator, lambda: False)
|
|
2943
|
+
return action()
|
|
2520
2944
|
|
|
2521
|
-
|
|
2945
|
+
@staticmethod
|
|
2946
|
+
def __wrap_search_criteria(search_criteria=None):
|
|
2522
2947
|
"""
|
|
2523
2948
|
Wraps the conditions specified within the Search Criteria in order for
|
|
2524
2949
|
other methods to handle it easily. The conditions are grouped only into
|
|
2525
|
-
three groups: Modality, Tags and the File Metadata (if DICOM it
|
|
2526
|
-
to the DICOM information), and each of them is output
|
|
2527
|
-
variable.
|
|
2950
|
+
three groups: Modality, Tags and the File Metadata (if DICOM it
|
|
2951
|
+
corresponds to the DICOM information), and each of them is output
|
|
2952
|
+
in a different variable.
|
|
2528
2953
|
|
|
2529
2954
|
Parameters
|
|
2530
2955
|
----------
|
|
@@ -2548,27 +2973,27 @@ class Project:
|
|
|
2548
2973
|
|
|
2549
2974
|
Returns
|
|
2550
2975
|
-------
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
file_metadata : Dict
|
|
2560
|
-
Dictionary containing the file metadata of the search criteria
|
|
2976
|
+
tuple
|
|
2977
|
+
A tuple containing:
|
|
2978
|
+
- str: modality is a string containing the modality of the search
|
|
2979
|
+
criteria extracted from 'pars_modalities';
|
|
2980
|
+
- list: tags is a list of strings containing the tags of the search
|
|
2981
|
+
criteria extracted 'from pars_tags',
|
|
2982
|
+
- dict: containing the file metadata of the search criteria
|
|
2561
2983
|
extracted from 'pars_[dicom]_KEY'
|
|
2562
2984
|
"""
|
|
2563
2985
|
|
|
2564
2986
|
# The keys not included bellow apply to the whole session.
|
|
2987
|
+
if search_criteria is None:
|
|
2988
|
+
search_criteria = {}
|
|
2565
2989
|
modality, tags, file_metadata = "", list(), dict()
|
|
2566
2990
|
for key, value in search_criteria.items():
|
|
2567
2991
|
if key == "pars_modalities":
|
|
2568
2992
|
modalities = value.split(";")[1].split(",")
|
|
2569
2993
|
if len(modalities) != 1:
|
|
2570
2994
|
raise ValueError(
|
|
2571
|
-
f"A file can only have one modality.
|
|
2995
|
+
f"A file can only have one modality. "
|
|
2996
|
+
f"Provided Modalities: {', '.join(modalities)}."
|
|
2572
2997
|
)
|
|
2573
2998
|
modality = modalities[0]
|
|
2574
2999
|
elif key == "pars_tags":
|
|
@@ -2577,21 +3002,34 @@ class Project:
|
|
|
2577
3002
|
d_tag = key.split("pars_[dicom]_")[1]
|
|
2578
3003
|
d_type = value.split(";")[0]
|
|
2579
3004
|
if d_type == "string":
|
|
2580
|
-
file_metadata[d_tag] = {
|
|
3005
|
+
file_metadata[d_tag] = {
|
|
3006
|
+
"operation": "in",
|
|
3007
|
+
"value": value.replace(d_type + ";", ""),
|
|
3008
|
+
}
|
|
2581
3009
|
elif d_type == "integer":
|
|
2582
3010
|
d_operator = value.split(";")[1].split("|")[0]
|
|
2583
3011
|
d_value = value.split(";")[1].split("|")[1]
|
|
2584
|
-
file_metadata[d_tag] = {
|
|
3012
|
+
file_metadata[d_tag] = {
|
|
3013
|
+
"operation": d_operator,
|
|
3014
|
+
"value": int(d_value),
|
|
3015
|
+
}
|
|
2585
3016
|
elif d_type == "decimal":
|
|
2586
3017
|
d_operator = value.split(";")[1].split("|")[0]
|
|
2587
3018
|
d_value = value.split(";")[1].split("|")[1]
|
|
2588
|
-
file_metadata[d_tag] = {
|
|
3019
|
+
file_metadata[d_tag] = {
|
|
3020
|
+
"operation": d_operator,
|
|
3021
|
+
"value": float(d_value),
|
|
3022
|
+
}
|
|
2589
3023
|
elif d_type == "list":
|
|
2590
3024
|
value.replace(d_type + ";", "")
|
|
2591
|
-
file_metadata[d_tag] = {
|
|
3025
|
+
file_metadata[d_tag] = {
|
|
3026
|
+
"operation": "in-list",
|
|
3027
|
+
"value": value.replace(d_type + ";", "").split(";"),
|
|
3028
|
+
}
|
|
2592
3029
|
return modality, tags, file_metadata
|
|
2593
3030
|
|
|
2594
|
-
|
|
3031
|
+
@staticmethod
|
|
3032
|
+
def __assert_split_data(split_data, ssid, add_to_container_id):
|
|
2595
3033
|
"""
|
|
2596
3034
|
Assert if the split_data parameter is possible to use in regards
|
|
2597
3035
|
to the ssid and add_to_container_id parameters during upload.
|
|
@@ -2614,29 +3052,81 @@ class Project:
|
|
|
2614
3052
|
|
|
2615
3053
|
logger = logging.getLogger(logger_name)
|
|
2616
3054
|
if ssid and split_data:
|
|
2617
|
-
logger.warning(
|
|
3055
|
+
logger.warning(
|
|
3056
|
+
"split-data argument will be ignored because ssid has been "
|
|
3057
|
+
"specified"
|
|
3058
|
+
)
|
|
2618
3059
|
split_data = False
|
|
2619
3060
|
|
|
2620
3061
|
if add_to_container_id and split_data:
|
|
2621
|
-
logger.warning(
|
|
3062
|
+
logger.warning(
|
|
3063
|
+
"split-data argument will be ignored because "
|
|
3064
|
+
"add_to_container_id has been specified"
|
|
3065
|
+
)
|
|
2622
3066
|
split_data = False
|
|
2623
3067
|
|
|
2624
3068
|
return split_data
|
|
2625
3069
|
|
|
2626
|
-
|
|
3070
|
+
@staticmethod
|
|
3071
|
+
def __parse_pass_rules(passed_rules, result):
|
|
3072
|
+
"""
|
|
3073
|
+
Parse pass rules.
|
|
3074
|
+
"""
|
|
3075
|
+
|
|
3076
|
+
for rule_text in passed_rules[1:]: # Skip first empty part
|
|
3077
|
+
rule_name = rule_text.split(" ✅")[0].strip()
|
|
3078
|
+
rule_data = {"rule": rule_name, "sub_rule": None, "files": []}
|
|
3079
|
+
|
|
3080
|
+
# Get sub-rule
|
|
3081
|
+
sub_rule_match = re.search(r"Sub-rule: (.*?)\n", rule_text)
|
|
3082
|
+
if sub_rule_match:
|
|
3083
|
+
rule_data["sub_rule"] = sub_rule_match.group(1).strip()
|
|
3084
|
+
|
|
3085
|
+
# Get files passed
|
|
3086
|
+
files_passed = re.search(
|
|
3087
|
+
r"List of files passed:(.*?)(?=\n\n|\Z)", rule_text, re.DOTALL
|
|
3088
|
+
)
|
|
3089
|
+
if files_passed:
|
|
3090
|
+
for line in files_passed.group(1).split("\n"):
|
|
3091
|
+
line = line.strip()
|
|
3092
|
+
if line.startswith("·"):
|
|
3093
|
+
file_match = re.match(r"· (.*?) \((\d+)/(\d+)\)", line)
|
|
3094
|
+
if file_match:
|
|
3095
|
+
rule_data["files"].append(
|
|
3096
|
+
{
|
|
3097
|
+
"file": file_match.group(1).strip(),
|
|
3098
|
+
"passed_conditions": int(
|
|
3099
|
+
file_match.group(2)
|
|
3100
|
+
),
|
|
3101
|
+
}
|
|
3102
|
+
)
|
|
3103
|
+
|
|
3104
|
+
result["passed"].append(rule_data)
|
|
3105
|
+
return result
|
|
3106
|
+
|
|
3107
|
+
@staticmethod
|
|
3108
|
+
def __parse_fail_rules(failed_rules, result):
|
|
2627
3109
|
"""
|
|
2628
3110
|
Parse fail rules.
|
|
2629
3111
|
"""
|
|
2630
3112
|
|
|
2631
3113
|
for rule_text in failed_rules[1:]: # Skip first empty part
|
|
2632
3114
|
rule_name = rule_text.split(" ❌")[0].strip()
|
|
2633
|
-
rule_data = {
|
|
3115
|
+
rule_data = {
|
|
3116
|
+
"rule": rule_name,
|
|
3117
|
+
"files": [],
|
|
3118
|
+
"failed_conditions": {},
|
|
3119
|
+
}
|
|
2634
3120
|
|
|
2635
3121
|
# Extract all file comparisons for this rule
|
|
2636
|
-
file_comparisons = re.split(r"
|
|
3122
|
+
file_comparisons = re.split(r"- Comparison with file:", rule_text)
|
|
2637
3123
|
for comp in file_comparisons[1:]: # Skip first part
|
|
2638
3124
|
file_name = comp.split("\n")[0].strip()
|
|
2639
|
-
conditions_match = re.search(
|
|
3125
|
+
conditions_match = re.search(
|
|
3126
|
+
r"Conditions:(.*?)(?=\n\t- Comparison|\n\n|$)",
|
|
3127
|
+
comp,
|
|
3128
|
+
re.DOTALL,
|
|
3129
|
+
)
|
|
2640
3130
|
if not conditions_match:
|
|
2641
3131
|
continue
|
|
2642
3132
|
|
|
@@ -2648,7 +3138,14 @@ class Project:
|
|
|
2648
3138
|
if line.startswith("·"):
|
|
2649
3139
|
status = "✔" if "✔" in line else "🚫"
|
|
2650
3140
|
condition = re.sub(r"^· [✔🚫]\s*", "", line)
|
|
2651
|
-
conditions.append(
|
|
3141
|
+
conditions.append(
|
|
3142
|
+
{
|
|
3143
|
+
"status": (
|
|
3144
|
+
"passed" if status == "✔" else "failed"
|
|
3145
|
+
),
|
|
3146
|
+
"condition": condition,
|
|
3147
|
+
}
|
|
3148
|
+
)
|
|
2652
3149
|
|
|
2653
3150
|
# Add to failed conditions summary
|
|
2654
3151
|
for cond in conditions:
|
|
@@ -2658,39 +3155,9 @@ class Project:
|
|
|
2658
3155
|
rule_data["failed_conditions"][cond_text] = 0
|
|
2659
3156
|
rule_data["failed_conditions"][cond_text] += 1
|
|
2660
3157
|
|
|
2661
|
-
rule_data["files"].append(
|
|
3158
|
+
rule_data["files"].append(
|
|
3159
|
+
{"file": file_name, "conditions": conditions}
|
|
3160
|
+
)
|
|
2662
3161
|
|
|
2663
3162
|
result["failed"].append(rule_data)
|
|
2664
3163
|
return result
|
|
2665
|
-
|
|
2666
|
-
def __parse_pass_rules(self, passed_rules, result):
|
|
2667
|
-
"""
|
|
2668
|
-
Parse pass rules.
|
|
2669
|
-
"""
|
|
2670
|
-
|
|
2671
|
-
for rule_text in passed_rules[1:]: # Skip first empty part
|
|
2672
|
-
rule_name = rule_text.split(" ✅")[0].strip()
|
|
2673
|
-
rule_data = {"rule": rule_name, "sub_rule": None, "files": []}
|
|
2674
|
-
|
|
2675
|
-
# Get sub-rule
|
|
2676
|
-
sub_rule_match = re.search(r"Sub-rule: (.*?)\n", rule_text)
|
|
2677
|
-
if sub_rule_match:
|
|
2678
|
-
rule_data["sub_rule"] = sub_rule_match.group(1).strip()
|
|
2679
|
-
|
|
2680
|
-
# Get files passed
|
|
2681
|
-
files_passed = re.search(r"List of files passed:(.*?)(?=\n\n|\Z)", rule_text, re.DOTALL)
|
|
2682
|
-
if files_passed:
|
|
2683
|
-
for line in files_passed.group(1).split("\n"):
|
|
2684
|
-
line = line.strip()
|
|
2685
|
-
if line.startswith("·"):
|
|
2686
|
-
file_match = re.match(r"· (.*?) \((\d+)/(\d+)\)", line)
|
|
2687
|
-
if file_match:
|
|
2688
|
-
rule_data["files"].append(
|
|
2689
|
-
{
|
|
2690
|
-
"file": file_match.group(1).strip(),
|
|
2691
|
-
"passed_conditions": int(file_match.group(2)),
|
|
2692
|
-
}
|
|
2693
|
-
)
|
|
2694
|
-
|
|
2695
|
-
result["passed"].append(rule_data)
|
|
2696
|
-
return result
|