qmenta-client 2.0__py3-none-any.whl → 2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qmenta/client/Account.py +43 -11
- qmenta/client/File.py +44 -12
- qmenta/client/Project.py +1006 -518
- qmenta/client/Subject.py +10 -3
- qmenta/client/utils.py +6 -2
- {qmenta_client-2.0.dist-info → qmenta_client-2.1.dist-info}/METADATA +3 -2
- qmenta_client-2.1.dist-info/RECORD +10 -0
- qmenta_client-2.0.dist-info/RECORD +0 -10
- {qmenta_client-2.0.dist-info → qmenta_client-2.1.dist-info}/WHEEL +0 -0
qmenta/client/Project.py
CHANGED
|
@@ -9,9 +9,10 @@ import sys
|
|
|
9
9
|
import time
|
|
10
10
|
from collections import defaultdict
|
|
11
11
|
from enum import Enum
|
|
12
|
-
|
|
12
|
+
from typing import Any, Dict, Union
|
|
13
13
|
from qmenta.core import errors
|
|
14
14
|
from qmenta.core import platform
|
|
15
|
+
from qmenta.core.upload.single import SingleUpload, FileInfo, UploadStatus
|
|
15
16
|
|
|
16
17
|
from qmenta.client import Account
|
|
17
18
|
|
|
@@ -21,6 +22,17 @@ if sys.version_info[0] == 3:
|
|
|
21
22
|
|
|
22
23
|
logger_name = "qmenta.client"
|
|
23
24
|
OPERATOR_LIST = ["eq", "ne", "gt", "gte", "lt", "lte"]
|
|
25
|
+
ANALYSIS_NAME_EXCLUDED_CHARACTERS = [
|
|
26
|
+
"\\",
|
|
27
|
+
"[",
|
|
28
|
+
"]",
|
|
29
|
+
"(",
|
|
30
|
+
")",
|
|
31
|
+
"{",
|
|
32
|
+
"}",
|
|
33
|
+
"+",
|
|
34
|
+
"*",
|
|
35
|
+
]
|
|
24
36
|
|
|
25
37
|
|
|
26
38
|
def convert_qc_value_to_qcstatus(value):
|
|
@@ -46,7 +58,9 @@ def convert_qc_value_to_qcstatus(value):
|
|
|
46
58
|
elif value == "":
|
|
47
59
|
return QCStatus.UNDERTERMINED
|
|
48
60
|
else:
|
|
49
|
-
logger.error(
|
|
61
|
+
logger.error(
|
|
62
|
+
f"The input value '{value}' cannot be converted to class QCStatus."
|
|
63
|
+
)
|
|
50
64
|
return False
|
|
51
65
|
|
|
52
66
|
|
|
@@ -84,11 +98,24 @@ class Project:
|
|
|
84
98
|
# project id (int)
|
|
85
99
|
if isinstance(project_id, str):
|
|
86
100
|
project_name = project_id
|
|
87
|
-
project_id = next(
|
|
101
|
+
project_id = next(
|
|
102
|
+
iter(
|
|
103
|
+
filter(
|
|
104
|
+
lambda proj: proj["name"] == project_id,
|
|
105
|
+
account.projects,
|
|
106
|
+
)
|
|
107
|
+
)
|
|
108
|
+
)["id"]
|
|
88
109
|
else:
|
|
89
110
|
if isinstance(project_id, float):
|
|
90
111
|
project_id = int(project_id)
|
|
91
|
-
project_name = next(
|
|
112
|
+
project_name = next(
|
|
113
|
+
iter(
|
|
114
|
+
filter(
|
|
115
|
+
lambda proj: proj["id"] == project_id, account.projects
|
|
116
|
+
)
|
|
117
|
+
)
|
|
118
|
+
)["name"]
|
|
92
119
|
|
|
93
120
|
self._account = account
|
|
94
121
|
self._project_id = project_id
|
|
@@ -121,7 +148,9 @@ class Project:
|
|
|
121
148
|
try:
|
|
122
149
|
platform.parse_response(
|
|
123
150
|
platform.post(
|
|
124
|
-
self._account.auth,
|
|
151
|
+
self._account.auth,
|
|
152
|
+
"projectset_manager/activate_project",
|
|
153
|
+
data={"project_id": int(project_id)},
|
|
125
154
|
)
|
|
126
155
|
)
|
|
127
156
|
except errors.PlatformError:
|
|
@@ -156,6 +185,7 @@ class Project:
|
|
|
156
185
|
result=False,
|
|
157
186
|
add_to_container_id=0,
|
|
158
187
|
split_data=False,
|
|
188
|
+
mock_response=False,
|
|
159
189
|
):
|
|
160
190
|
"""
|
|
161
191
|
Upload a chunk of a file to the platform.
|
|
@@ -189,6 +219,8 @@ class Project:
|
|
|
189
219
|
"Content-Length": str(length),
|
|
190
220
|
"Content-Disposition": disposition,
|
|
191
221
|
}
|
|
222
|
+
if mock_response:
|
|
223
|
+
request_headers["mock_case"] = mock_response
|
|
192
224
|
|
|
193
225
|
if last_chunk:
|
|
194
226
|
request_headers["X-Mint-Name"] = name
|
|
@@ -215,9 +247,12 @@ class Project:
|
|
|
215
247
|
|
|
216
248
|
response_time = 900.0 if last_chunk else 120.0
|
|
217
249
|
response = platform.post(
|
|
218
|
-
auth=self._account.auth,
|
|
250
|
+
auth=self._account.auth,
|
|
251
|
+
endpoint="upload",
|
|
252
|
+
data=data,
|
|
253
|
+
headers=request_headers,
|
|
254
|
+
timeout=response_time,
|
|
219
255
|
)
|
|
220
|
-
|
|
221
256
|
return response
|
|
222
257
|
|
|
223
258
|
def upload_file(
|
|
@@ -229,10 +264,12 @@ class Project:
|
|
|
229
264
|
description="",
|
|
230
265
|
result=False,
|
|
231
266
|
name="",
|
|
232
|
-
input_data_type="qmenta_medical_image_data:3.
|
|
267
|
+
input_data_type="qmenta_medical_image_data:3.11.3",
|
|
233
268
|
add_to_container_id=0,
|
|
234
|
-
chunk_size=2
|
|
269
|
+
chunk_size=2**24, # Optimized for GCS Bucket Storage
|
|
270
|
+
max_retries=5,
|
|
235
271
|
split_data=False,
|
|
272
|
+
mock_response=None,
|
|
236
273
|
):
|
|
237
274
|
"""
|
|
238
275
|
Upload a ZIP file to the platform.
|
|
@@ -260,124 +297,81 @@ class Project:
|
|
|
260
297
|
chunk_size : int
|
|
261
298
|
Size in kB of each chunk. Should be expressed as
|
|
262
299
|
a power of 2: 2**x. Default value of x is 9 (chunk_size = 512 kB)
|
|
300
|
+
max_retries: int
|
|
301
|
+
Maximum number of retries when uploading a file before raising
|
|
302
|
+
an error. Default value: 5
|
|
263
303
|
split_data : bool
|
|
264
304
|
If True, the platform will try to split the uploaded file into
|
|
265
|
-
different sessions. It will be ignored when the ssid
|
|
305
|
+
different sessions. It will be ignored when the ssid or a
|
|
306
|
+
add_to_container_id are given.
|
|
307
|
+
mock_response: None
|
|
308
|
+
ONLY USED IN UNITTESTING
|
|
266
309
|
|
|
267
310
|
Returns
|
|
268
311
|
-------
|
|
269
312
|
bool
|
|
270
313
|
True if correctly uploaded, False otherwise.
|
|
271
314
|
"""
|
|
315
|
+
input_data_type = (
|
|
316
|
+
"qmenta_upload_offline_analysis:1.0" if result else input_data_type
|
|
317
|
+
)
|
|
272
318
|
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
319
|
+
single_upload = SingleUpload(
|
|
320
|
+
self._account.auth,
|
|
321
|
+
file_path,
|
|
322
|
+
file_info=FileInfo(
|
|
323
|
+
project_id=self._project_id,
|
|
324
|
+
subject_name=subject_name,
|
|
325
|
+
session_id=str(ssid),
|
|
326
|
+
input_data_type=input_data_type,
|
|
327
|
+
split_data=split_data,
|
|
328
|
+
add_to_container_id=add_to_container_id,
|
|
329
|
+
date_of_scan=date_of_scan,
|
|
330
|
+
description=description,
|
|
331
|
+
name=name,
|
|
332
|
+
),
|
|
333
|
+
anonymise=False, # will be anonymised in the upload tool.
|
|
334
|
+
chunk_size=chunk_size,
|
|
335
|
+
max_retries=max_retries,
|
|
336
|
+
)
|
|
280
337
|
|
|
281
|
-
|
|
338
|
+
single_upload.start()
|
|
282
339
|
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
340
|
+
if single_upload.status == UploadStatus.FAILED: # FAILED
|
|
341
|
+
print("Upload Failed!")
|
|
342
|
+
return False
|
|
286
343
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
chunk_num = 0
|
|
294
|
-
retries_count = 0
|
|
295
|
-
uploaded_bytes = 0
|
|
296
|
-
response = None
|
|
297
|
-
last_chunk = False
|
|
298
|
-
|
|
299
|
-
if ssid and split_data:
|
|
300
|
-
logger.warning("split-data argument will be ignored because" + " ssid has been specified")
|
|
301
|
-
split_data = False
|
|
302
|
-
|
|
303
|
-
while True:
|
|
304
|
-
data = file_object.read(chunk_size)
|
|
305
|
-
if not data:
|
|
306
|
-
break
|
|
307
|
-
|
|
308
|
-
start_position = chunk_num * chunk_size
|
|
309
|
-
end_position = start_position + chunk_size - 1
|
|
310
|
-
bytes_to_send = chunk_size
|
|
311
|
-
|
|
312
|
-
if end_position >= total_bytes:
|
|
313
|
-
last_chunk = True
|
|
314
|
-
end_position = total_bytes - 1
|
|
315
|
-
bytes_to_send = total_bytes - uploaded_bytes
|
|
316
|
-
|
|
317
|
-
bytes_range = "bytes " + str(start_position) + "-" + str(end_position) + "/" + str(total_bytes)
|
|
318
|
-
|
|
319
|
-
dispstr = f"attachment; filename={filename}"
|
|
320
|
-
response = self._upload_chunk(
|
|
321
|
-
data,
|
|
322
|
-
bytes_range,
|
|
323
|
-
bytes_to_send,
|
|
324
|
-
session_id,
|
|
325
|
-
dispstr,
|
|
326
|
-
last_chunk,
|
|
327
|
-
name,
|
|
328
|
-
date_of_scan,
|
|
329
|
-
description,
|
|
330
|
-
subject_name,
|
|
331
|
-
ssid,
|
|
332
|
-
filename,
|
|
333
|
-
input_data_type,
|
|
334
|
-
result,
|
|
335
|
-
add_to_container_id,
|
|
336
|
-
split_data,
|
|
337
|
-
)
|
|
344
|
+
message = (
|
|
345
|
+
"Your data was successfully uploaded. "
|
|
346
|
+
"The uploaded file will be soon processed !"
|
|
347
|
+
)
|
|
348
|
+
print(message)
|
|
349
|
+
return True
|
|
338
350
|
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
error_message = "HTTP Connection Problem"
|
|
344
|
-
logger.error(error_message)
|
|
345
|
-
break
|
|
346
|
-
elif int(response.status_code) == 201:
|
|
347
|
-
chunk_num += 1
|
|
348
|
-
retries_count = 0
|
|
349
|
-
uploaded_bytes += chunk_size
|
|
350
|
-
elif int(response.status_code) == 200:
|
|
351
|
-
self.__show_progress(file_size, file_size, finish=True)
|
|
352
|
-
break
|
|
353
|
-
elif int(response.status_code) == 416:
|
|
354
|
-
retries_count += 1
|
|
355
|
-
time.sleep(retries_count * 5)
|
|
356
|
-
if retries_count > self.max_retries:
|
|
357
|
-
error_message = "Error Code: 416; Requested Range Not Satisfiable (NGINX)"
|
|
358
|
-
logger.error(error_message)
|
|
359
|
-
break
|
|
360
|
-
else:
|
|
361
|
-
retries_count += 1
|
|
362
|
-
time.sleep(retries_count * 5)
|
|
363
|
-
if retries_count > max_retries:
|
|
364
|
-
error_message = "Number of retries has been reached. Upload process stops here !"
|
|
365
|
-
logger.error(error_message)
|
|
366
|
-
break
|
|
351
|
+
def delete_file(self, container_id, filenames):
|
|
352
|
+
"""
|
|
353
|
+
Delete a file or files from a container.
|
|
354
|
+
Can be an input or an output container
|
|
367
355
|
|
|
368
|
-
|
|
369
|
-
|
|
356
|
+
Parameters
|
|
357
|
+
----------
|
|
358
|
+
container_id : int
|
|
359
|
+
filenames : str or list of str
|
|
370
360
|
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
361
|
+
"""
|
|
362
|
+
if not isinstance(filenames, str):
|
|
363
|
+
if isinstance(filenames, list):
|
|
364
|
+
if not all([isinstance(f, str) for f in filenames]):
|
|
365
|
+
raise TypeError("Elements of `filenames` must be str")
|
|
366
|
+
filenames = ";".join(filenames)
|
|
367
|
+
else:
|
|
368
|
+
raise TypeError("`filenames` must be str or list of str")
|
|
376
369
|
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
370
|
+
platform.post(
|
|
371
|
+
self._account.auth,
|
|
372
|
+
"file_manager/delete_files",
|
|
373
|
+
data={"container_id": container_id, "files": filenames},
|
|
374
|
+
)
|
|
381
375
|
|
|
382
376
|
def upload_mri(self, file_path, subject_name):
|
|
383
377
|
"""
|
|
@@ -415,7 +409,11 @@ class Project:
|
|
|
415
409
|
"""
|
|
416
410
|
|
|
417
411
|
if self.__check_upload_file(file_path):
|
|
418
|
-
return self.upload_file(
|
|
412
|
+
return self.upload_file(
|
|
413
|
+
file_path,
|
|
414
|
+
subject_name,
|
|
415
|
+
input_data_type="parkinson_gametection",
|
|
416
|
+
)
|
|
419
417
|
return False
|
|
420
418
|
|
|
421
419
|
def upload_result(self, file_path, subject_name):
|
|
@@ -438,7 +436,9 @@ class Project:
|
|
|
438
436
|
return self.upload_file(file_path, subject_name, result=True)
|
|
439
437
|
return False
|
|
440
438
|
|
|
441
|
-
def download_file(
|
|
439
|
+
def download_file(
|
|
440
|
+
self, container_id, file_name, local_filename=None, overwrite=False
|
|
441
|
+
):
|
|
442
442
|
"""
|
|
443
443
|
Download a single file from a specific container.
|
|
444
444
|
|
|
@@ -448,43 +448,57 @@ class Project:
|
|
|
448
448
|
ID of the container inside which the file is.
|
|
449
449
|
file_name : str
|
|
450
450
|
Name of the file in the container.
|
|
451
|
-
local_filename : str
|
|
451
|
+
local_filename : str, optional
|
|
452
452
|
Name of the file to be created. By default, the same as file_name.
|
|
453
453
|
overwrite : bool
|
|
454
454
|
Whether to overwrite the file if existing.
|
|
455
455
|
"""
|
|
456
456
|
logger = logging.getLogger(logger_name)
|
|
457
457
|
if not isinstance(file_name, str):
|
|
458
|
-
raise ValueError(
|
|
459
|
-
|
|
460
|
-
|
|
458
|
+
raise ValueError(
|
|
459
|
+
"The name of the file to download (file_name) should be of "
|
|
460
|
+
"type string."
|
|
461
|
+
)
|
|
462
|
+
if not isinstance(local_filename, str):
|
|
463
|
+
raise ValueError(
|
|
464
|
+
"The name of the output file (local_filename) should be of "
|
|
465
|
+
"type string."
|
|
466
|
+
)
|
|
461
467
|
|
|
462
468
|
if file_name not in self.list_container_files(container_id):
|
|
463
|
-
msg =
|
|
464
|
-
|
|
465
|
-
|
|
469
|
+
msg = (
|
|
470
|
+
f'File "{file_name}" does not exist in container '
|
|
471
|
+
f"{container_id}"
|
|
472
|
+
)
|
|
473
|
+
raise Exception(msg)
|
|
466
474
|
|
|
467
475
|
local_filename = local_filename or file_name
|
|
468
476
|
|
|
469
477
|
if os.path.exists(local_filename) and not overwrite:
|
|
470
478
|
msg = f"File {local_filename} already exists"
|
|
471
|
-
|
|
472
|
-
return False
|
|
479
|
+
raise Exception(msg)
|
|
473
480
|
|
|
474
481
|
params = {"container_id": container_id, "files": file_name}
|
|
475
|
-
|
|
476
482
|
with platform.post(
|
|
477
|
-
self._account.auth,
|
|
483
|
+
self._account.auth,
|
|
484
|
+
"file_manager/download_file",
|
|
485
|
+
data=params,
|
|
486
|
+
stream=True,
|
|
478
487
|
) as response, open(local_filename, "wb") as f:
|
|
479
488
|
|
|
480
|
-
for chunk in response.iter_content(chunk_size=2
|
|
489
|
+
for chunk in response.iter_content(chunk_size=2**9 * 1024):
|
|
481
490
|
f.write(chunk)
|
|
482
491
|
f.flush()
|
|
483
492
|
|
|
484
|
-
logger.info(
|
|
493
|
+
logger.info(
|
|
494
|
+
f"File {file_name} from container {container_id} saved "
|
|
495
|
+
f"to {local_filename}"
|
|
496
|
+
)
|
|
485
497
|
return True
|
|
486
498
|
|
|
487
|
-
def download_files(
|
|
499
|
+
def download_files(
|
|
500
|
+
self, container_id, filenames, zip_name="files.zip", overwrite=False
|
|
501
|
+
):
|
|
488
502
|
"""
|
|
489
503
|
Download a set of files from a given container.
|
|
490
504
|
|
|
@@ -502,32 +516,51 @@ class Project:
|
|
|
502
516
|
logger = logging.getLogger(logger_name)
|
|
503
517
|
|
|
504
518
|
if not all([isinstance(file_name, str) for file_name in filenames]):
|
|
505
|
-
raise ValueError(
|
|
519
|
+
raise ValueError(
|
|
520
|
+
"The name of the files to download (filenames) should be "
|
|
521
|
+
"of type string."
|
|
522
|
+
)
|
|
506
523
|
if not isinstance(zip_name, str):
|
|
507
|
-
raise ValueError(
|
|
524
|
+
raise ValueError(
|
|
525
|
+
"The name of the output ZIP file (zip_name) should be "
|
|
526
|
+
"of type string."
|
|
527
|
+
)
|
|
508
528
|
|
|
509
|
-
files_not_in_container = list(
|
|
529
|
+
files_not_in_container = list(
|
|
530
|
+
filter(
|
|
531
|
+
lambda f: f not in self.list_container_files(container_id),
|
|
532
|
+
filenames,
|
|
533
|
+
)
|
|
534
|
+
)
|
|
510
535
|
|
|
511
536
|
if files_not_in_container:
|
|
512
|
-
msg =
|
|
513
|
-
|
|
514
|
-
|
|
537
|
+
msg = (
|
|
538
|
+
f"The following files are missing in container "
|
|
539
|
+
f"{container_id}: {', '.join(files_not_in_container)}"
|
|
540
|
+
)
|
|
541
|
+
raise Exception(msg)
|
|
515
542
|
|
|
516
543
|
if os.path.exists(zip_name) and not overwrite:
|
|
517
544
|
msg = f'File "{zip_name}" already exists'
|
|
518
|
-
|
|
519
|
-
return False
|
|
545
|
+
raise Exception(msg)
|
|
520
546
|
|
|
521
547
|
params = {"container_id": container_id, "files": ";".join(filenames)}
|
|
522
548
|
with platform.post(
|
|
523
|
-
self._account.auth,
|
|
549
|
+
self._account.auth,
|
|
550
|
+
"file_manager/download_file",
|
|
551
|
+
data=params,
|
|
552
|
+
stream=True,
|
|
524
553
|
) as response, open(zip_name, "wb") as f:
|
|
525
554
|
|
|
526
|
-
for chunk in response.iter_content(chunk_size=2
|
|
555
|
+
for chunk in response.iter_content(chunk_size=2**9 * 1024):
|
|
527
556
|
f.write(chunk)
|
|
528
557
|
f.flush()
|
|
529
558
|
|
|
530
|
-
logger.info(
|
|
559
|
+
logger.info(
|
|
560
|
+
"Files from container {} saved to {}".format(
|
|
561
|
+
container_id, zip_name
|
|
562
|
+
)
|
|
563
|
+
)
|
|
531
564
|
return True
|
|
532
565
|
|
|
533
566
|
def copy_container_to_project(self, container_id, project_id):
|
|
@@ -551,9 +584,14 @@ class Project:
|
|
|
551
584
|
p_id = int(project_id)
|
|
552
585
|
elif type(project_id) is str:
|
|
553
586
|
projects = self._account.projects
|
|
554
|
-
projects_match = [
|
|
587
|
+
projects_match = [
|
|
588
|
+
proj for proj in projects if proj["name"] == project_id
|
|
589
|
+
]
|
|
555
590
|
if not projects_match:
|
|
556
|
-
raise Exception(
|
|
591
|
+
raise Exception(
|
|
592
|
+
f"Project {project_id}"
|
|
593
|
+
+ " does not exist or is not available for this user."
|
|
594
|
+
)
|
|
557
595
|
p_id = int(projects_match[0]["id"])
|
|
558
596
|
else:
|
|
559
597
|
raise TypeError("project_id")
|
|
@@ -564,10 +602,16 @@ class Project:
|
|
|
564
602
|
|
|
565
603
|
try:
|
|
566
604
|
platform.parse_response(
|
|
567
|
-
platform.post(
|
|
605
|
+
platform.post(
|
|
606
|
+
self._account.auth,
|
|
607
|
+
"file_manager/copy_container_to_another_project",
|
|
608
|
+
data=data,
|
|
609
|
+
)
|
|
568
610
|
)
|
|
569
611
|
except errors.PlatformError as e:
|
|
570
|
-
logging.getLogger(logger_name).error(
|
|
612
|
+
logging.getLogger(logger_name).error(
|
|
613
|
+
"Couldn not copy container: {}".format(e)
|
|
614
|
+
)
|
|
571
615
|
return False
|
|
572
616
|
|
|
573
617
|
return True
|
|
@@ -601,7 +645,7 @@ class Project:
|
|
|
601
645
|
return self.get_subjects_metadata()
|
|
602
646
|
|
|
603
647
|
@property
|
|
604
|
-
def metadata_parameters(self):
|
|
648
|
+
def metadata_parameters(self) -> Union[Dict, None]:
|
|
605
649
|
"""
|
|
606
650
|
List all the parameters in the subject-level metadata.
|
|
607
651
|
|
|
@@ -612,20 +656,28 @@ class Project:
|
|
|
612
656
|
modification of these subject-level metadata parameters via the
|
|
613
657
|
'change_subject_metadata()' method.
|
|
614
658
|
|
|
659
|
+
Example:
|
|
660
|
+
dictionary {"param_name":
|
|
661
|
+
{ "order": Int,
|
|
662
|
+
"tags": [tag1, tag2, ..., ],
|
|
663
|
+
"title: "Title",
|
|
664
|
+
"type": "integer|string|date|list|decimal",
|
|
665
|
+
"visible": 0|1
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
|
|
615
669
|
Returns
|
|
616
670
|
-------
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
{ "order": Int,
|
|
620
|
-
"tags": [tag1, tag2, ..., ],
|
|
621
|
-
"title: "Title",
|
|
622
|
-
"type": "integer|string|date|list|decimal",
|
|
623
|
-
"visible": 0|1
|
|
624
|
-
}}
|
|
671
|
+
metadata_parameters : dict[str] or None
|
|
672
|
+
|
|
625
673
|
"""
|
|
626
674
|
logger = logging.getLogger(logger_name)
|
|
627
675
|
try:
|
|
628
|
-
data = platform.parse_response(
|
|
676
|
+
data = platform.parse_response(
|
|
677
|
+
platform.post(
|
|
678
|
+
self._account.auth, "patient_manager/module_config"
|
|
679
|
+
)
|
|
680
|
+
)
|
|
629
681
|
except errors.PlatformError:
|
|
630
682
|
logger.error("Could not retrieve metadata parameters.")
|
|
631
683
|
return None
|
|
@@ -671,7 +723,10 @@ class Project:
|
|
|
671
723
|
response = self.list_input_containers(search_criteria=search_criteria)
|
|
672
724
|
|
|
673
725
|
for subject in response:
|
|
674
|
-
if
|
|
726
|
+
if (
|
|
727
|
+
subject["patient_secret_name"] == subject_name
|
|
728
|
+
and subject["ssid"] == ssid
|
|
729
|
+
):
|
|
675
730
|
return subject["container_id"]
|
|
676
731
|
return False
|
|
677
732
|
|
|
@@ -695,20 +750,25 @@ class Project:
|
|
|
695
750
|
"""
|
|
696
751
|
|
|
697
752
|
for user in self.get_subjects_metadata():
|
|
698
|
-
if user["patient_secret_name"] == str(subject_name) and user[
|
|
753
|
+
if user["patient_secret_name"] == str(subject_name) and user[
|
|
754
|
+
"ssid"
|
|
755
|
+
] == str(ssid):
|
|
699
756
|
return int(user["_id"])
|
|
700
757
|
return False
|
|
701
758
|
|
|
702
|
-
def get_subjects_metadata(self, search_criteria=
|
|
759
|
+
def get_subjects_metadata(self, search_criteria=None, items=(0, 9999)):
|
|
703
760
|
"""
|
|
704
761
|
List all Subject ID/Session ID from the selected project that meet the
|
|
705
|
-
|
|
762
|
+
defined search criteria at a session level.
|
|
706
763
|
|
|
707
764
|
Parameters
|
|
708
765
|
----------
|
|
709
766
|
search_criteria: dict
|
|
710
767
|
Each element is a string and is built using the formatting
|
|
711
768
|
"type;value", or "type;operation|value"
|
|
769
|
+
items : List[int]
|
|
770
|
+
list containing two elements [min, max] that correspond to the
|
|
771
|
+
mininum and maximum range of analysis listed
|
|
712
772
|
|
|
713
773
|
Complete search_criteria Dictionary Explanation:
|
|
714
774
|
|
|
@@ -722,8 +782,8 @@ class Project:
|
|
|
722
782
|
"pars_PROJECTMETADATA": "METADATATYPE;METADATAVALUE",
|
|
723
783
|
}
|
|
724
784
|
|
|
725
|
-
|
|
726
|
-
|
|
785
|
+
where "pars_patient_secret_name": Applies the search to the
|
|
786
|
+
'Subject ID'.
|
|
727
787
|
SUBJECTID is a comma separated list of strings.
|
|
728
788
|
"pars_ssid": Applies the search to the 'Session ID'.
|
|
729
789
|
SSID is an integer.
|
|
@@ -809,12 +869,26 @@ class Project:
|
|
|
809
869
|
|
|
810
870
|
"""
|
|
811
871
|
|
|
812
|
-
|
|
813
|
-
|
|
872
|
+
if search_criteria is None:
|
|
873
|
+
search_criteria = {}
|
|
874
|
+
if len(items) != 2:
|
|
875
|
+
raise ValueError(
|
|
876
|
+
f"The number of elements in items '{len(items)}' "
|
|
877
|
+
f"should be equal to two."
|
|
878
|
+
)
|
|
814
879
|
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
880
|
+
if not all([isinstance(item, int) for item in items]):
|
|
881
|
+
raise ValueError(
|
|
882
|
+
f"All values in items " f"'{items}' must be integers"
|
|
883
|
+
)
|
|
884
|
+
|
|
885
|
+
if search_criteria != {} and not all(
|
|
886
|
+
[item.startswith("pars_") for item in search_criteria.keys()]
|
|
887
|
+
):
|
|
888
|
+
raise ValueError(
|
|
889
|
+
f"All keys of the search_criteria dictionary "
|
|
890
|
+
f"'{search_criteria.keys()}' must start with 'pars_'."
|
|
891
|
+
)
|
|
818
892
|
|
|
819
893
|
for key, value in search_criteria.items():
|
|
820
894
|
if value.split(";")[0] in ["integer", "decimal"]:
|
|
@@ -833,7 +907,9 @@ class Project:
|
|
|
833
907
|
)
|
|
834
908
|
return content
|
|
835
909
|
|
|
836
|
-
def change_subject_metadata(
|
|
910
|
+
def change_subject_metadata(
|
|
911
|
+
self, patient_id, subject_name, ssid, tags, age_at_scan, metadata
|
|
912
|
+
):
|
|
837
913
|
"""
|
|
838
914
|
Change the Subject ID, Session ID, Tags, Age at Scan and Metadata of
|
|
839
915
|
the session with Patient ID
|
|
@@ -868,36 +944,60 @@ class Project:
|
|
|
868
944
|
try:
|
|
869
945
|
patient_id = str(int(patient_id))
|
|
870
946
|
except ValueError:
|
|
871
|
-
raise ValueError(
|
|
947
|
+
raise ValueError(
|
|
948
|
+
f"'patient_id': '{patient_id}' not valid. Must be convertible "
|
|
949
|
+
f"to int."
|
|
950
|
+
)
|
|
872
951
|
|
|
873
|
-
|
|
952
|
+
if not isinstance(tags, list) or not all(
|
|
874
953
|
isinstance(item, str) for item in tags
|
|
875
|
-
)
|
|
954
|
+
):
|
|
955
|
+
raise ValueError(f"tags: '{tags}' should be a list of strings.")
|
|
876
956
|
tags = [tag.lower() for tag in tags]
|
|
877
957
|
|
|
878
|
-
|
|
879
|
-
|
|
958
|
+
if not isinstance(subject_name, str) or (
|
|
959
|
+
subject_name is None or subject_name == ""
|
|
960
|
+
):
|
|
961
|
+
raise ValueError("subject_name must be a non empty string.")
|
|
962
|
+
|
|
963
|
+
if not isinstance(ssid, str) or (ssid is None or ssid == ""):
|
|
964
|
+
raise ValueError("ssid must be a non empty string.")
|
|
880
965
|
|
|
881
966
|
try:
|
|
882
967
|
age_at_scan = str(int(age_at_scan)) if age_at_scan else None
|
|
883
968
|
except ValueError:
|
|
884
|
-
raise ValueError(
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
"
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
969
|
+
raise ValueError(
|
|
970
|
+
f"age_at_scan: '{age_at_scan}' not valid. Must be an integer."
|
|
971
|
+
)
|
|
972
|
+
|
|
973
|
+
if not isinstance(metadata, dict):
|
|
974
|
+
raise ValueError(f"metadata: '{metadata}' should be a dictionary.")
|
|
975
|
+
|
|
976
|
+
has_md_prefix = ["md_" == key[:3] for key in metadata.keys()]
|
|
977
|
+
if not (all(has_md_prefix) or not any(has_md_prefix)):
|
|
978
|
+
raise ValueError(
|
|
979
|
+
f"metadata: '{metadata}' must be a dictionary whose keys are "
|
|
980
|
+
f"either all starting with 'md_' or none."
|
|
981
|
+
)
|
|
982
|
+
|
|
983
|
+
metadata_keys = []
|
|
984
|
+
if self.metadata_parameters:
|
|
985
|
+
metadata_keys = self.metadata_parameters.keys()
|
|
986
|
+
|
|
987
|
+
if not all(
|
|
988
|
+
(
|
|
989
|
+
key[3:] in metadata_keys
|
|
990
|
+
if "md_" == key[:3]
|
|
991
|
+
else key in metadata_keys
|
|
992
|
+
)
|
|
993
|
+
for key in metadata.keys()
|
|
994
|
+
):
|
|
995
|
+
raise ValueError(
|
|
996
|
+
f"Some metadata keys provided ({', '.join(metadata.keys())}) "
|
|
997
|
+
"are not available in the project. They can be added via the "
|
|
998
|
+
"Metadata Manager via the QMENTA Platform graphical user "
|
|
999
|
+
"interface (GUI)."
|
|
1000
|
+
)
|
|
901
1001
|
|
|
902
1002
|
post_data = {
|
|
903
1003
|
"patient_id": patient_id,
|
|
@@ -907,11 +1007,17 @@ class Project:
|
|
|
907
1007
|
"age_at_scan": age_at_scan,
|
|
908
1008
|
}
|
|
909
1009
|
for key, value in metadata.items():
|
|
910
|
-
|
|
911
|
-
post_data[f"last_vals.{
|
|
1010
|
+
id_ = key[3:] if "md_" == key[:3] else key
|
|
1011
|
+
post_data[f"last_vals.{id_}"] = value
|
|
912
1012
|
|
|
913
1013
|
try:
|
|
914
|
-
platform.parse_response(
|
|
1014
|
+
platform.parse_response(
|
|
1015
|
+
platform.post(
|
|
1016
|
+
self._account.auth,
|
|
1017
|
+
"patient_manager/upsert_patient",
|
|
1018
|
+
data=post_data,
|
|
1019
|
+
)
|
|
1020
|
+
)
|
|
915
1021
|
except errors.PlatformError:
|
|
916
1022
|
logger.error(f"Patient ID '{patient_id}' could not be modified.")
|
|
917
1023
|
return False
|
|
@@ -919,7 +1025,9 @@ class Project:
|
|
|
919
1025
|
logger.info(f"Patient ID '{patient_id}' successfully modified.")
|
|
920
1026
|
return True
|
|
921
1027
|
|
|
922
|
-
def get_subjects_files_metadata(
|
|
1028
|
+
def get_subjects_files_metadata(
|
|
1029
|
+
self, search_criteria=None, items=(0, 9999)
|
|
1030
|
+
):
|
|
923
1031
|
"""
|
|
924
1032
|
List all Subject ID/Session ID from the selected project that meet the
|
|
925
1033
|
defined search criteria at a file level.
|
|
@@ -935,6 +1043,9 @@ class Project:
|
|
|
935
1043
|
search_criteria: dict
|
|
936
1044
|
Each element is a string and is built using the formatting
|
|
937
1045
|
"type;value", or "type;operation|value"
|
|
1046
|
+
items : List[int]
|
|
1047
|
+
list containing two elements [min, max] that correspond to the
|
|
1048
|
+
mininum and maximum range of analysis listed
|
|
938
1049
|
|
|
939
1050
|
Complete search_criteria Dictionary Explanation:
|
|
940
1051
|
|
|
@@ -1037,10 +1148,14 @@ class Project:
|
|
|
1037
1148
|
|
|
1038
1149
|
"""
|
|
1039
1150
|
|
|
1040
|
-
|
|
1151
|
+
if search_criteria is None:
|
|
1152
|
+
search_criteria = {}
|
|
1153
|
+
content = self.get_subjects_metadata(search_criteria, items=items)
|
|
1041
1154
|
|
|
1042
1155
|
# Wrap search criteria.
|
|
1043
|
-
modality, tags,
|
|
1156
|
+
modality, tags, dicom_metadata = self.__wrap_search_criteria(
|
|
1157
|
+
search_criteria
|
|
1158
|
+
)
|
|
1044
1159
|
|
|
1045
1160
|
# Iterate over the files of each subject selected to include/exclude
|
|
1046
1161
|
# them from the results.
|
|
@@ -1055,17 +1170,23 @@ class Project:
|
|
|
1055
1170
|
)
|
|
1056
1171
|
|
|
1057
1172
|
for file in files["meta"]:
|
|
1058
|
-
if modality and modality != (file.get("metadata") or {}).get(
|
|
1173
|
+
if modality and modality != (file.get("metadata") or {}).get(
|
|
1174
|
+
"modality"
|
|
1175
|
+
):
|
|
1059
1176
|
continue
|
|
1060
1177
|
if tags and not all([tag in file.get("tags") for tag in tags]):
|
|
1061
1178
|
continue
|
|
1062
|
-
if
|
|
1179
|
+
if dicom_metadata:
|
|
1063
1180
|
result_values = list()
|
|
1064
|
-
for key, dict_value in
|
|
1065
|
-
f_value = (
|
|
1181
|
+
for key, dict_value in dicom_metadata.items():
|
|
1182
|
+
f_value = (
|
|
1183
|
+
(file.get("metadata") or {}).get("info") or {}
|
|
1184
|
+
).get(key)
|
|
1066
1185
|
d_operator = dict_value["operation"]
|
|
1067
1186
|
d_value = dict_value["value"]
|
|
1068
|
-
result_values.append(
|
|
1187
|
+
result_values.append(
|
|
1188
|
+
self.__operation(d_value, d_operator, f_value)
|
|
1189
|
+
)
|
|
1069
1190
|
|
|
1070
1191
|
if not all(result_values):
|
|
1071
1192
|
continue
|
|
@@ -1086,7 +1207,7 @@ class Project:
|
|
|
1086
1207
|
|
|
1087
1208
|
Returns
|
|
1088
1209
|
-------
|
|
1089
|
-
dict
|
|
1210
|
+
dict or bool
|
|
1090
1211
|
Dictionary with the metadata. False otherwise.
|
|
1091
1212
|
"""
|
|
1092
1213
|
all_metadata = self.list_container_files_metadata(container_id)
|
|
@@ -1119,7 +1240,12 @@ class Project:
|
|
|
1119
1240
|
platform.post(
|
|
1120
1241
|
self._account.auth,
|
|
1121
1242
|
"file_manager/edit_file",
|
|
1122
|
-
data={
|
|
1243
|
+
data={
|
|
1244
|
+
"container_id": container_id,
|
|
1245
|
+
"filename": filename,
|
|
1246
|
+
"tags": tags_str,
|
|
1247
|
+
"modality": modality,
|
|
1248
|
+
},
|
|
1123
1249
|
)
|
|
1124
1250
|
)
|
|
1125
1251
|
|
|
@@ -1132,7 +1258,7 @@ class Project:
|
|
|
1132
1258
|
----------
|
|
1133
1259
|
subject_name : str
|
|
1134
1260
|
Subject ID of the subject
|
|
1135
|
-
session_id :
|
|
1261
|
+
session_id : str
|
|
1136
1262
|
The Session ID of the session that will be deleted
|
|
1137
1263
|
|
|
1138
1264
|
Returns
|
|
@@ -1144,16 +1270,29 @@ class Project:
|
|
|
1144
1270
|
all_sessions = self.get_subjects_metadata()
|
|
1145
1271
|
|
|
1146
1272
|
session_to_del = [
|
|
1147
|
-
s
|
|
1273
|
+
s
|
|
1274
|
+
for s in all_sessions
|
|
1275
|
+
if s["patient_secret_name"] == subject_name
|
|
1276
|
+
and s["ssid"] == session_id
|
|
1148
1277
|
]
|
|
1149
1278
|
|
|
1150
1279
|
if not session_to_del:
|
|
1151
|
-
logger.error(
|
|
1280
|
+
logger.error(
|
|
1281
|
+
f"Session {subject_name}/{session_id} could not be found in "
|
|
1282
|
+
f"this project."
|
|
1283
|
+
)
|
|
1152
1284
|
return False
|
|
1153
1285
|
elif len(session_to_del) > 1:
|
|
1154
|
-
raise RuntimeError(
|
|
1286
|
+
raise RuntimeError(
|
|
1287
|
+
"Multiple sessions with same Subject ID and Session ID. "
|
|
1288
|
+
"Contact support."
|
|
1289
|
+
)
|
|
1155
1290
|
else:
|
|
1156
|
-
logger.info(
|
|
1291
|
+
logger.info(
|
|
1292
|
+
"{}/{} found (id {})".format(
|
|
1293
|
+
subject_name, session_id, session_to_del[0]["_id"]
|
|
1294
|
+
)
|
|
1295
|
+
)
|
|
1157
1296
|
|
|
1158
1297
|
session = session_to_del[0]
|
|
1159
1298
|
|
|
@@ -1162,14 +1301,23 @@ class Project:
|
|
|
1162
1301
|
platform.post(
|
|
1163
1302
|
self._account.auth,
|
|
1164
1303
|
"patient_manager/delete_patient",
|
|
1165
|
-
data={
|
|
1304
|
+
data={
|
|
1305
|
+
"patient_id": str(int(session["_id"])),
|
|
1306
|
+
"delete_files": 1,
|
|
1307
|
+
},
|
|
1166
1308
|
)
|
|
1167
1309
|
)
|
|
1168
1310
|
except errors.PlatformError:
|
|
1169
|
-
logger.error(
|
|
1311
|
+
logger.error(
|
|
1312
|
+
f"Session \"{subject_name}/{session['ssid']}\" could "
|
|
1313
|
+
f"not be deleted."
|
|
1314
|
+
)
|
|
1170
1315
|
return False
|
|
1171
1316
|
|
|
1172
|
-
logger.info(
|
|
1317
|
+
logger.info(
|
|
1318
|
+
f"Session \"{subject_name}/{session['ssid']}\" successfully "
|
|
1319
|
+
f"deleted."
|
|
1320
|
+
)
|
|
1173
1321
|
return True
|
|
1174
1322
|
|
|
1175
1323
|
def delete_session_by_patientid(self, patient_id):
|
|
@@ -1194,7 +1342,10 @@ class Project:
|
|
|
1194
1342
|
platform.post(
|
|
1195
1343
|
self._account.auth,
|
|
1196
1344
|
"patient_manager/delete_patient",
|
|
1197
|
-
data={
|
|
1345
|
+
data={
|
|
1346
|
+
"patient_id": str(int(patient_id)),
|
|
1347
|
+
"delete_files": 1,
|
|
1348
|
+
},
|
|
1198
1349
|
)
|
|
1199
1350
|
)
|
|
1200
1351
|
except errors.PlatformError:
|
|
@@ -1224,10 +1375,16 @@ class Project:
|
|
|
1224
1375
|
# Always fetch the session IDs from the platform before deleting them
|
|
1225
1376
|
all_sessions = self.get_subjects_metadata()
|
|
1226
1377
|
|
|
1227
|
-
sessions_to_del = [
|
|
1378
|
+
sessions_to_del = [
|
|
1379
|
+
s for s in all_sessions if s["patient_secret_name"] == subject_name
|
|
1380
|
+
]
|
|
1228
1381
|
|
|
1229
1382
|
if not sessions_to_del:
|
|
1230
|
-
logger.error(
|
|
1383
|
+
logger.error(
|
|
1384
|
+
"Subject {} cannot be found in this project.".format(
|
|
1385
|
+
subject_name
|
|
1386
|
+
)
|
|
1387
|
+
)
|
|
1231
1388
|
return False
|
|
1232
1389
|
|
|
1233
1390
|
for ssid in [s["ssid"] for s in sessions_to_del]:
|
|
@@ -1237,7 +1394,7 @@ class Project:
|
|
|
1237
1394
|
|
|
1238
1395
|
""" Container Related Methods """
|
|
1239
1396
|
|
|
1240
|
-
def list_input_containers(self, search_criteria=
|
|
1397
|
+
def list_input_containers(self, search_criteria=None, items=(0, 9999)):
|
|
1241
1398
|
"""
|
|
1242
1399
|
Retrieve the list of input containers available to the user under a
|
|
1243
1400
|
certain search criteria.
|
|
@@ -1271,8 +1428,17 @@ class Project:
|
|
|
1271
1428
|
{"container_name", "container_id", "patient_secret_name", "ssid"}
|
|
1272
1429
|
"""
|
|
1273
1430
|
|
|
1274
|
-
|
|
1275
|
-
|
|
1431
|
+
if search_criteria is None:
|
|
1432
|
+
search_criteria = {}
|
|
1433
|
+
if len(items) != 2:
|
|
1434
|
+
raise ValueError(
|
|
1435
|
+
f"The number of elements in items '{len(items)}' "
|
|
1436
|
+
f"should be equal to two."
|
|
1437
|
+
)
|
|
1438
|
+
if not all(isinstance(item, int) for item in items):
|
|
1439
|
+
raise ValueError(
|
|
1440
|
+
f"All items elements '{items}' should be integers."
|
|
1441
|
+
)
|
|
1276
1442
|
|
|
1277
1443
|
response = platform.parse_response(
|
|
1278
1444
|
platform.post(
|
|
@@ -1285,7 +1451,7 @@ class Project:
|
|
|
1285
1451
|
containers = [
|
|
1286
1452
|
{
|
|
1287
1453
|
"patient_secret_name": container_item["patient_secret_name"],
|
|
1288
|
-
"container_name": container_item["name"],
|
|
1454
|
+
"container_name": container_item["name"], # ???
|
|
1289
1455
|
"container_id": container_item["_id"],
|
|
1290
1456
|
"ssid": container_item["ssid"],
|
|
1291
1457
|
}
|
|
@@ -1293,7 +1459,7 @@ class Project:
|
|
|
1293
1459
|
]
|
|
1294
1460
|
return containers
|
|
1295
1461
|
|
|
1296
|
-
def list_result_containers(self, search_condition=
|
|
1462
|
+
def list_result_containers(self, search_condition=None, items=(0, 9999)):
|
|
1297
1463
|
"""
|
|
1298
1464
|
List the result containers available to the user.
|
|
1299
1465
|
Examples
|
|
@@ -1321,7 +1487,8 @@ class Project:
|
|
|
1321
1487
|
- qa_status: str or None pass/fail/nd QC status
|
|
1322
1488
|
- secret_name: str or None Subject ID
|
|
1323
1489
|
- tags: str or None
|
|
1324
|
-
- with_child_analysis: 1 or None if 1, child analysis of workflows
|
|
1490
|
+
- with_child_analysis: 1 or None if 1, child analysis of workflows
|
|
1491
|
+
will appear
|
|
1325
1492
|
- id: str or None ID
|
|
1326
1493
|
- state: running, completed, pending, exception or None
|
|
1327
1494
|
- username: str or None
|
|
@@ -1338,13 +1505,21 @@ class Project:
|
|
|
1338
1505
|
if "id": None, that analysis did not had an output container,
|
|
1339
1506
|
probably it is a workflow
|
|
1340
1507
|
"""
|
|
1508
|
+
if search_condition is None:
|
|
1509
|
+
search_condition = {}
|
|
1341
1510
|
analyses = self.list_analysis(search_condition, items)
|
|
1342
|
-
return [
|
|
1511
|
+
return [
|
|
1512
|
+
{
|
|
1513
|
+
"name": analysis["name"],
|
|
1514
|
+
"id": (analysis.get("out_container_id") or None),
|
|
1515
|
+
}
|
|
1516
|
+
for analysis in analyses
|
|
1517
|
+
]
|
|
1343
1518
|
|
|
1344
1519
|
def list_container_files(
|
|
1345
1520
|
self,
|
|
1346
1521
|
container_id,
|
|
1347
|
-
):
|
|
1522
|
+
) -> Any:
|
|
1348
1523
|
"""
|
|
1349
1524
|
List the name of the files available inside a given container.
|
|
1350
1525
|
Parameters
|
|
@@ -1360,7 +1535,9 @@ class Project:
|
|
|
1360
1535
|
try:
|
|
1361
1536
|
content = platform.parse_response(
|
|
1362
1537
|
platform.post(
|
|
1363
|
-
self._account.auth,
|
|
1538
|
+
self._account.auth,
|
|
1539
|
+
"file_manager/get_container_files",
|
|
1540
|
+
data={"container_id": container_id},
|
|
1364
1541
|
)
|
|
1365
1542
|
)
|
|
1366
1543
|
except errors.PlatformError as e:
|
|
@@ -1371,7 +1548,9 @@ class Project:
|
|
|
1371
1548
|
return False
|
|
1372
1549
|
return content["files"]
|
|
1373
1550
|
|
|
1374
|
-
def list_container_filter_files(
|
|
1551
|
+
def list_container_filter_files(
|
|
1552
|
+
self, container_id, modality="", metadata_info={}, tags=[]
|
|
1553
|
+
):
|
|
1375
1554
|
"""
|
|
1376
1555
|
List the name of the files available inside a given container.
|
|
1377
1556
|
search condition example:
|
|
@@ -1407,17 +1586,23 @@ class Project:
|
|
|
1407
1586
|
if modality == "":
|
|
1408
1587
|
modality_bool = True
|
|
1409
1588
|
else:
|
|
1410
|
-
modality_bool = modality == metadata_file["metadata"].get(
|
|
1589
|
+
modality_bool = modality == metadata_file["metadata"].get(
|
|
1590
|
+
"modality"
|
|
1591
|
+
)
|
|
1411
1592
|
for key in metadata_info.keys():
|
|
1412
|
-
meta_key = (
|
|
1593
|
+
meta_key = (
|
|
1594
|
+
(metadata_file.get("metadata") or {}).get("info") or {}
|
|
1595
|
+
).get(key)
|
|
1413
1596
|
if meta_key is None:
|
|
1414
|
-
logging.getLogger(logger_name).warning(
|
|
1597
|
+
logging.getLogger(logger_name).warning(
|
|
1598
|
+
f"{key} is not in file_info from file {file}"
|
|
1599
|
+
)
|
|
1415
1600
|
info_bool.append(metadata_info[key] == meta_key)
|
|
1416
1601
|
if all(tags_bool) and all(info_bool) and modality_bool:
|
|
1417
1602
|
selected_files.append(file)
|
|
1418
1603
|
return selected_files
|
|
1419
1604
|
|
|
1420
|
-
def list_container_files_metadata(self, container_id):
|
|
1605
|
+
def list_container_files_metadata(self, container_id) -> dict:
|
|
1421
1606
|
"""
|
|
1422
1607
|
List all the metadata of the files available inside a given container.
|
|
1423
1608
|
|
|
@@ -1435,7 +1620,9 @@ class Project:
|
|
|
1435
1620
|
try:
|
|
1436
1621
|
data = platform.parse_response(
|
|
1437
1622
|
platform.post(
|
|
1438
|
-
self._account.auth,
|
|
1623
|
+
self._account.auth,
|
|
1624
|
+
"file_manager/get_container_files",
|
|
1625
|
+
data={"container_id": container_id},
|
|
1439
1626
|
)
|
|
1440
1627
|
)
|
|
1441
1628
|
except errors.PlatformError as e:
|
|
@@ -1446,9 +1633,10 @@ class Project:
|
|
|
1446
1633
|
|
|
1447
1634
|
""" Analysis Related Methods """
|
|
1448
1635
|
|
|
1449
|
-
def get_analysis(self, analysis_name_or_id):
|
|
1636
|
+
def get_analysis(self, analysis_name_or_id) -> dict:
|
|
1450
1637
|
"""
|
|
1451
|
-
Returns the analysis corresponding with the analysis id or analysis
|
|
1638
|
+
Returns the analysis corresponding with the analysis id or analysis
|
|
1639
|
+
name
|
|
1452
1640
|
|
|
1453
1641
|
Parameters
|
|
1454
1642
|
----------
|
|
@@ -1468,28 +1656,41 @@ class Project:
|
|
|
1468
1656
|
analysis_name_or_id = int(analysis_name_or_id)
|
|
1469
1657
|
else:
|
|
1470
1658
|
search_tag = "p_n"
|
|
1471
|
-
|
|
1472
|
-
|
|
1659
|
+
excluded_bool = [
|
|
1660
|
+
character in analysis_name_or_id
|
|
1661
|
+
for character in ANALYSIS_NAME_EXCLUDED_CHARACTERS
|
|
1662
|
+
]
|
|
1473
1663
|
if any(excluded_bool):
|
|
1474
|
-
raise Exception(
|
|
1664
|
+
raise Exception(
|
|
1665
|
+
f"p_n does not allow "
|
|
1666
|
+
f"characters {ANALYSIS_NAME_EXCLUDED_CHARACTERS}"
|
|
1667
|
+
)
|
|
1475
1668
|
else:
|
|
1476
|
-
raise Exception(
|
|
1669
|
+
raise Exception(
|
|
1670
|
+
"The analysis identifier must be its name or an integer"
|
|
1671
|
+
)
|
|
1477
1672
|
|
|
1478
1673
|
search_condition = {
|
|
1479
1674
|
search_tag: analysis_name_or_id,
|
|
1480
1675
|
}
|
|
1481
1676
|
response = platform.parse_response(
|
|
1482
|
-
platform.post(
|
|
1677
|
+
platform.post(
|
|
1678
|
+
self._account.auth,
|
|
1679
|
+
"analysis_manager/get_analysis_list",
|
|
1680
|
+
data=search_condition,
|
|
1681
|
+
)
|
|
1483
1682
|
)
|
|
1484
1683
|
|
|
1485
1684
|
if len(response) > 1:
|
|
1486
|
-
raise Exception(
|
|
1685
|
+
raise Exception(
|
|
1686
|
+
f"multiple analyses with name {analysis_name_or_id} found"
|
|
1687
|
+
)
|
|
1487
1688
|
elif len(response) == 1:
|
|
1488
1689
|
return response[0]
|
|
1489
1690
|
else:
|
|
1490
1691
|
return None
|
|
1491
1692
|
|
|
1492
|
-
def list_analysis(self, search_condition=
|
|
1693
|
+
def list_analysis(self, search_condition=None, items=(0, 9999)):
|
|
1493
1694
|
"""
|
|
1494
1695
|
List the analysis available to the user.
|
|
1495
1696
|
|
|
@@ -1518,10 +1719,12 @@ class Project:
|
|
|
1518
1719
|
- qa_status: str or None pass/fail/nd QC status
|
|
1519
1720
|
- secret_name: str or None Subject ID
|
|
1520
1721
|
- tags: str or None
|
|
1521
|
-
- with_child_analysis: 1 or None if 1, child analysis of workflows
|
|
1722
|
+
- with_child_analysis: 1 or None if 1, child analysis of workflows
|
|
1723
|
+
will appear
|
|
1522
1724
|
- id: int or None ID
|
|
1523
1725
|
- state: running, completed, pending, exception or None
|
|
1524
1726
|
- username: str or None
|
|
1727
|
+
- only_data: int or None
|
|
1525
1728
|
|
|
1526
1729
|
items : List[int]
|
|
1527
1730
|
list containing two elements [min, max] that correspond to the
|
|
@@ -1532,8 +1735,17 @@ class Project:
|
|
|
1532
1735
|
dict
|
|
1533
1736
|
List of analysis, each a dictionary
|
|
1534
1737
|
"""
|
|
1535
|
-
|
|
1536
|
-
|
|
1738
|
+
if search_condition is None:
|
|
1739
|
+
search_condition = {}
|
|
1740
|
+
if len(items) != 2:
|
|
1741
|
+
raise ValueError(
|
|
1742
|
+
f"The number of elements in items '{len(items)}' "
|
|
1743
|
+
f"should be equal to two."
|
|
1744
|
+
)
|
|
1745
|
+
if not all(isinstance(item, int) for item in items):
|
|
1746
|
+
raise ValueError(
|
|
1747
|
+
f"All items elements '{items}' should be integers."
|
|
1748
|
+
)
|
|
1537
1749
|
search_keys = {
|
|
1538
1750
|
"p_n": str,
|
|
1539
1751
|
"type": str,
|
|
@@ -1546,19 +1758,37 @@ class Project:
|
|
|
1546
1758
|
"with_child_analysis": int,
|
|
1547
1759
|
"id": int,
|
|
1548
1760
|
"state": str,
|
|
1761
|
+
"only_data": int,
|
|
1549
1762
|
"username": str,
|
|
1550
1763
|
}
|
|
1551
1764
|
for key in search_condition.keys():
|
|
1552
1765
|
if key not in search_keys.keys():
|
|
1553
|
-
raise Exception(
|
|
1554
|
-
|
|
1555
|
-
|
|
1766
|
+
raise Exception(
|
|
1767
|
+
(
|
|
1768
|
+
f"This key '{key}' is not accepted by this search "
|
|
1769
|
+
f"condition"
|
|
1770
|
+
)
|
|
1771
|
+
)
|
|
1772
|
+
if (
|
|
1773
|
+
not isinstance(search_condition[key], search_keys[key])
|
|
1774
|
+
and search_condition[key] is not None
|
|
1775
|
+
):
|
|
1776
|
+
raise Exception(
|
|
1777
|
+
(
|
|
1778
|
+
f"The key {key} in the search condition is not type "
|
|
1779
|
+
f"{search_keys[key]}"
|
|
1780
|
+
)
|
|
1781
|
+
)
|
|
1556
1782
|
if "p_n" == key:
|
|
1557
|
-
|
|
1558
|
-
|
|
1783
|
+
excluded_bool = [
|
|
1784
|
+
character in search_condition["p_n"]
|
|
1785
|
+
for character in ANALYSIS_NAME_EXCLUDED_CHARACTERS
|
|
1786
|
+
]
|
|
1559
1787
|
if any(excluded_bool):
|
|
1560
|
-
raise Exception(
|
|
1561
|
-
|
|
1788
|
+
raise Exception(
|
|
1789
|
+
"p_n does not allow "
|
|
1790
|
+
f"characters {ANALYSIS_NAME_EXCLUDED_CHARACTERS}"
|
|
1791
|
+
)
|
|
1562
1792
|
req_headers = {"X-Range": f"items={items[0]}-{items[1] - 1}"}
|
|
1563
1793
|
return platform.parse_response(
|
|
1564
1794
|
platform.post(
|
|
@@ -1623,7 +1853,9 @@ class Project:
|
|
|
1623
1853
|
logger = logging.getLogger(logger_name)
|
|
1624
1854
|
|
|
1625
1855
|
if in_container_id is None and settings is None:
|
|
1626
|
-
raise ValueError(
|
|
1856
|
+
raise ValueError(
|
|
1857
|
+
"Pass a value for either in_container_id or settings."
|
|
1858
|
+
)
|
|
1627
1859
|
|
|
1628
1860
|
post_data = {"script_name": script_name, "version": version}
|
|
1629
1861
|
|
|
@@ -1656,15 +1888,19 @@ class Project:
|
|
|
1656
1888
|
|
|
1657
1889
|
logger.debug(f"post_data = {post_data}")
|
|
1658
1890
|
return self.__handle_start_analysis(
|
|
1659
|
-
post_data,
|
|
1891
|
+
post_data,
|
|
1892
|
+
ignore_warnings=ignore_warnings,
|
|
1893
|
+
ignore_file_selection=ignore_file_selection,
|
|
1660
1894
|
)
|
|
1661
1895
|
|
|
1662
1896
|
def delete_analysis(self, analysis_id):
|
|
1663
1897
|
"""
|
|
1664
1898
|
Delete an analysis
|
|
1665
1899
|
|
|
1666
|
-
|
|
1667
|
-
|
|
1900
|
+
Parameters
|
|
1901
|
+
----------
|
|
1902
|
+
analysis_id : int
|
|
1903
|
+
ID of the analysis to be deleted
|
|
1668
1904
|
"""
|
|
1669
1905
|
logger = logging.getLogger(logger_name)
|
|
1670
1906
|
|
|
@@ -1692,18 +1928,23 @@ class Project:
|
|
|
1692
1928
|
Tools can not be restarted given that they are considered as single
|
|
1693
1929
|
processing units. You can start execution of another analysis instead.
|
|
1694
1930
|
|
|
1695
|
-
For the workflow to restart, all its failed child must be removed
|
|
1696
|
-
You can only restart your own analysis.
|
|
1931
|
+
For the workflow to restart, all its failed child must be removed
|
|
1932
|
+
first. You can only restart your own analysis.
|
|
1697
1933
|
|
|
1698
|
-
|
|
1699
|
-
|
|
1934
|
+
Parameters
|
|
1935
|
+
----------
|
|
1936
|
+
analysis_id : int
|
|
1937
|
+
ID of the analysis to be restarted
|
|
1700
1938
|
"""
|
|
1701
1939
|
logger = logging.getLogger(logger_name)
|
|
1702
1940
|
|
|
1703
1941
|
analysis = self.list_analysis({"id": analysis_id})[0]
|
|
1704
1942
|
|
|
1705
1943
|
if analysis.get("super_analysis_type") != 1:
|
|
1706
|
-
raise ValueError(
|
|
1944
|
+
raise ValueError(
|
|
1945
|
+
"The analysis indicated is not a workflow and hence, it "
|
|
1946
|
+
"cannot be restarted."
|
|
1947
|
+
)
|
|
1707
1948
|
|
|
1708
1949
|
try:
|
|
1709
1950
|
platform.parse_response(
|
|
@@ -1725,7 +1966,8 @@ class Project:
|
|
|
1725
1966
|
Get the log of an analysis and save it in the provided file.
|
|
1726
1967
|
The logs of analysis can only be obtained for the tools you created.
|
|
1727
1968
|
|
|
1728
|
-
Note workflows do not have a log so the printed message will only be
|
|
1969
|
+
Note workflows do not have a log so the printed message will only be
|
|
1970
|
+
ERROR.
|
|
1729
1971
|
You can only download the anlaysis log of the tools that you own.
|
|
1730
1972
|
|
|
1731
1973
|
Note this method is very time consuming.
|
|
@@ -1748,22 +1990,32 @@ class Project:
|
|
|
1748
1990
|
try:
|
|
1749
1991
|
analysis_id = str(int(analysis_id))
|
|
1750
1992
|
except ValueError:
|
|
1751
|
-
raise ValueError(
|
|
1993
|
+
raise ValueError(
|
|
1994
|
+
f"'analysis_id' has to be an integer not '{analysis_id}'."
|
|
1995
|
+
)
|
|
1752
1996
|
|
|
1753
1997
|
file_name = file_name if file_name else f"logs_{analysis_id}.txt"
|
|
1754
1998
|
try:
|
|
1755
1999
|
res = platform.post(
|
|
1756
2000
|
auth=self._account.auth,
|
|
1757
2001
|
endpoint="analysis_manager/download_execution_file",
|
|
1758
|
-
data={
|
|
2002
|
+
data={
|
|
2003
|
+
"project_id": analysis_id,
|
|
2004
|
+
"file": f"logs_{analysis_id}",
|
|
2005
|
+
},
|
|
1759
2006
|
timeout=1000,
|
|
1760
2007
|
)
|
|
1761
2008
|
except Exception:
|
|
1762
|
-
logger.error(
|
|
2009
|
+
logger.error(
|
|
2010
|
+
f"Could not export the analysis log of '{analysis_id}'"
|
|
2011
|
+
)
|
|
1763
2012
|
return False
|
|
1764
2013
|
|
|
1765
2014
|
if not res.ok:
|
|
1766
|
-
logger.error(
|
|
2015
|
+
logger.error(
|
|
2016
|
+
f"The log file could not be extracted for Analysis ID:"
|
|
2017
|
+
f" {analysis_id}."
|
|
2018
|
+
)
|
|
1767
2019
|
return False
|
|
1768
2020
|
|
|
1769
2021
|
with open(file_name, "w") as f:
|
|
@@ -1772,7 +2024,9 @@ class Project:
|
|
|
1772
2024
|
|
|
1773
2025
|
""" QC Status Related Methods """
|
|
1774
2026
|
|
|
1775
|
-
def set_qc_status_analysis(
|
|
2027
|
+
def set_qc_status_analysis(
|
|
2028
|
+
self, analysis_id, status=QCStatus.UNDERTERMINED, comments=""
|
|
2029
|
+
):
|
|
1776
2030
|
"""
|
|
1777
2031
|
Changes the analysis QC status.
|
|
1778
2032
|
|
|
@@ -1801,7 +2055,10 @@ class Project:
|
|
|
1801
2055
|
try:
|
|
1802
2056
|
analysis_id = str(int(analysis_id))
|
|
1803
2057
|
except ValueError:
|
|
1804
|
-
raise ValueError(
|
|
2058
|
+
raise ValueError(
|
|
2059
|
+
f"analysis_id: '{analysis_id}' not valid. Must be convertible "
|
|
2060
|
+
f"to int."
|
|
2061
|
+
)
|
|
1805
2062
|
|
|
1806
2063
|
try:
|
|
1807
2064
|
platform.parse_response(
|
|
@@ -1817,11 +2074,16 @@ class Project:
|
|
|
1817
2074
|
)
|
|
1818
2075
|
)
|
|
1819
2076
|
except Exception:
|
|
1820
|
-
logger.error(
|
|
2077
|
+
logger.error(
|
|
2078
|
+
f"It was not possible to change the QC status of Analysis ID:"
|
|
2079
|
+
f" {analysis_id}"
|
|
2080
|
+
)
|
|
1821
2081
|
return False
|
|
1822
2082
|
return True
|
|
1823
2083
|
|
|
1824
|
-
def set_qc_status_subject(
|
|
2084
|
+
def set_qc_status_subject(
|
|
2085
|
+
self, patient_id, status=QCStatus.UNDERTERMINED, comments=""
|
|
2086
|
+
):
|
|
1825
2087
|
"""
|
|
1826
2088
|
Changes the QC status of a Patient ID (equivalent to a
|
|
1827
2089
|
Subject ID/Session ID).
|
|
@@ -1850,7 +2112,10 @@ class Project:
|
|
|
1850
2112
|
try:
|
|
1851
2113
|
patient_id = str(int(patient_id))
|
|
1852
2114
|
except ValueError:
|
|
1853
|
-
raise ValueError(
|
|
2115
|
+
raise ValueError(
|
|
2116
|
+
f"'patient_id': '{patient_id}' not valid. Must be convertible"
|
|
2117
|
+
f" to int."
|
|
2118
|
+
)
|
|
1854
2119
|
|
|
1855
2120
|
try:
|
|
1856
2121
|
platform.parse_response(
|
|
@@ -1866,7 +2131,10 @@ class Project:
|
|
|
1866
2131
|
)
|
|
1867
2132
|
)
|
|
1868
2133
|
except Exception:
|
|
1869
|
-
logger.error(
|
|
2134
|
+
logger.error(
|
|
2135
|
+
f"It was not possible to change the QC status of Patient ID:"
|
|
2136
|
+
f" {patient_id}"
|
|
2137
|
+
)
|
|
1870
2138
|
return False
|
|
1871
2139
|
return True
|
|
1872
2140
|
|
|
@@ -1891,17 +2159,28 @@ class Project:
|
|
|
1891
2159
|
try:
|
|
1892
2160
|
search_criteria = {"id": analysis_id}
|
|
1893
2161
|
to_return = self.list_analysis(search_criteria)
|
|
1894
|
-
return
|
|
2162
|
+
return (
|
|
2163
|
+
convert_qc_value_to_qcstatus(to_return[0]["qa_status"]),
|
|
2164
|
+
to_return[0]["qa_comments"],
|
|
2165
|
+
)
|
|
1895
2166
|
except IndexError:
|
|
1896
2167
|
# Handle the case where no matching analysis is found
|
|
1897
|
-
logging.error(
|
|
2168
|
+
logging.error(
|
|
2169
|
+
f"No analysis was found with such Analysis ID: "
|
|
2170
|
+
f"'{analysis_id}'."
|
|
2171
|
+
)
|
|
1898
2172
|
return False, False
|
|
1899
2173
|
except Exception:
|
|
1900
2174
|
# Handle other potential exceptions
|
|
1901
|
-
logging.error(
|
|
2175
|
+
logging.error(
|
|
2176
|
+
f"It was not possible to extract the QC status from Analysis "
|
|
2177
|
+
f"ID: {analysis_id}"
|
|
2178
|
+
)
|
|
1902
2179
|
return False, False
|
|
1903
2180
|
|
|
1904
|
-
def get_qc_status_subject(
|
|
2181
|
+
def get_qc_status_subject(
|
|
2182
|
+
self, patient_id=None, subject_name=None, ssid=None
|
|
2183
|
+
):
|
|
1905
2184
|
"""
|
|
1906
2185
|
Gets the session QC status via the patient ID or the Subject ID
|
|
1907
2186
|
and the Session ID.
|
|
@@ -1929,26 +2208,50 @@ class Project:
|
|
|
1929
2208
|
try:
|
|
1930
2209
|
patient_id = int(patient_id)
|
|
1931
2210
|
except ValueError:
|
|
1932
|
-
raise ValueError(
|
|
2211
|
+
raise ValueError(
|
|
2212
|
+
f"patient_id '{patient_id}' should be an integer."
|
|
2213
|
+
)
|
|
1933
2214
|
sessions = self.get_subjects_metadata(search_criteria={})
|
|
1934
|
-
session = [
|
|
2215
|
+
session = [
|
|
2216
|
+
session
|
|
2217
|
+
for session in sessions
|
|
2218
|
+
if int(session["_id"]) == patient_id
|
|
2219
|
+
]
|
|
1935
2220
|
if len(session) < 1:
|
|
1936
|
-
logging.error(
|
|
2221
|
+
logging.error(
|
|
2222
|
+
f"No session was found with Patient ID: '{patient_id}'."
|
|
2223
|
+
)
|
|
1937
2224
|
return False, False
|
|
1938
|
-
return
|
|
2225
|
+
return (
|
|
2226
|
+
convert_qc_value_to_qcstatus(session[0]["qa_status"]),
|
|
2227
|
+
session[0]["qa_comments"],
|
|
2228
|
+
)
|
|
1939
2229
|
elif subject_name and ssid:
|
|
1940
2230
|
session = self.get_subjects_metadata(
|
|
1941
2231
|
search_criteria={
|
|
1942
2232
|
"pars_patient_secret_name": f"string;{subject_name}",
|
|
1943
|
-
"pars_ssid":
|
|
2233
|
+
"pars_ssid": (
|
|
2234
|
+
f"integer;eq|{ssid}"
|
|
2235
|
+
if str(ssid).isdigit()
|
|
2236
|
+
else f"string;{ssid}"
|
|
2237
|
+
),
|
|
1944
2238
|
}
|
|
1945
2239
|
)
|
|
1946
2240
|
if len(session) < 1:
|
|
1947
|
-
logging.error(
|
|
2241
|
+
logging.error(
|
|
2242
|
+
f"No session was found with Subject ID: '{subject_name}'"
|
|
2243
|
+
f" and Session ID: '{ssid}'."
|
|
2244
|
+
)
|
|
1948
2245
|
return False, False
|
|
1949
|
-
return
|
|
2246
|
+
return (
|
|
2247
|
+
convert_qc_value_to_qcstatus(session[0]["qa_status"]),
|
|
2248
|
+
session[0]["qa_comments"],
|
|
2249
|
+
)
|
|
1950
2250
|
else:
|
|
1951
|
-
raise ValueError(
|
|
2251
|
+
raise ValueError(
|
|
2252
|
+
"Either 'patient_id' or 'subject_name' and 'ssid' must "
|
|
2253
|
+
"not be empty."
|
|
2254
|
+
)
|
|
1952
2255
|
|
|
1953
2256
|
""" Protocol Adherence Related Methods """
|
|
1954
2257
|
|
|
@@ -1976,7 +2279,9 @@ class Project:
|
|
|
1976
2279
|
with open(rules_file_path, "r") as fr:
|
|
1977
2280
|
rules = json.load(fr)
|
|
1978
2281
|
except FileNotFoundError:
|
|
1979
|
-
logger.error(
|
|
2282
|
+
logger.error(
|
|
2283
|
+
f"Protocol adherence rule file '{rules_file_path}' not found."
|
|
2284
|
+
)
|
|
1980
2285
|
return False
|
|
1981
2286
|
|
|
1982
2287
|
# Update the project's QA rules
|
|
@@ -1984,18 +2289,26 @@ class Project:
|
|
|
1984
2289
|
platform.post(
|
|
1985
2290
|
auth=self._account.auth,
|
|
1986
2291
|
endpoint="projectset_manager/set_session_qa_requirements",
|
|
1987
|
-
data={
|
|
2292
|
+
data={
|
|
2293
|
+
"project_id": self._project_id,
|
|
2294
|
+
"rules": json.dumps(rules),
|
|
2295
|
+
"guidance_text": guidance_text,
|
|
2296
|
+
},
|
|
1988
2297
|
)
|
|
1989
2298
|
)
|
|
1990
2299
|
|
|
1991
2300
|
if not res.get("success") == 1:
|
|
1992
|
-
logger.error(
|
|
2301
|
+
logger.error(
|
|
2302
|
+
"There was an error setting up the protocol adherence rules."
|
|
2303
|
+
)
|
|
1993
2304
|
logger.error(platform.parse_response(res))
|
|
1994
2305
|
return False
|
|
1995
2306
|
|
|
1996
2307
|
return True
|
|
1997
2308
|
|
|
1998
|
-
def get_project_pa_rules(
|
|
2309
|
+
def get_project_pa_rules(
|
|
2310
|
+
self, rules_file_path, project_has_no_rules=False
|
|
2311
|
+
):
|
|
1999
2312
|
"""
|
|
2000
2313
|
Retrive the active project's protocol adherence rules
|
|
2001
2314
|
|
|
@@ -2004,6 +2317,8 @@ class Project:
|
|
|
2004
2317
|
rules_file_path : str
|
|
2005
2318
|
The file path to the JSON file to store the protocol adherence
|
|
2006
2319
|
rules.
|
|
2320
|
+
project_has_no_rules: bool
|
|
2321
|
+
for testing purposes
|
|
2007
2322
|
|
|
2008
2323
|
Returns
|
|
2009
2324
|
-------
|
|
@@ -2023,47 +2338,58 @@ class Project:
|
|
|
2023
2338
|
)
|
|
2024
2339
|
)
|
|
2025
2340
|
|
|
2026
|
-
if "rules" not in res:
|
|
2027
|
-
logger.error(
|
|
2341
|
+
if "rules" not in res or project_has_no_rules:
|
|
2342
|
+
logger.error(
|
|
2343
|
+
f"There was an error extracting the protocol adherence rules"
|
|
2344
|
+
f" from {self._project_name}."
|
|
2345
|
+
)
|
|
2028
2346
|
logger.error(platform.parse_response(res))
|
|
2029
2347
|
return False
|
|
2030
2348
|
|
|
2031
2349
|
try:
|
|
2032
2350
|
for rule in res["rules"]:
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2351
|
+
for key in ["_id", "order", "time_modified"]:
|
|
2352
|
+
if rule.get(key, False):
|
|
2353
|
+
del rule[key]
|
|
2036
2354
|
with open(rules_file_path, "w") as fr:
|
|
2037
2355
|
json.dump(res["rules"], fr, indent=4)
|
|
2038
2356
|
except FileNotFoundError:
|
|
2039
|
-
logger.error(
|
|
2357
|
+
logger.error(
|
|
2358
|
+
f"Protocol adherence rules could not be exported to file: "
|
|
2359
|
+
f"'{rules_file_path}'."
|
|
2360
|
+
)
|
|
2040
2361
|
return False
|
|
2041
2362
|
|
|
2042
2363
|
return res["guidance_text"]
|
|
2043
2364
|
|
|
2044
2365
|
def parse_qc_text(self, patient_id=None, subject_name=None, ssid=None):
|
|
2045
2366
|
"""
|
|
2046
|
-
Parse QC (Quality Control) text output into a structured dictionary
|
|
2367
|
+
Parse QC (Quality Control) text output into a structured dictionary
|
|
2368
|
+
format.
|
|
2047
2369
|
|
|
2048
|
-
This function takes raw QC text output (
|
|
2049
|
-
and parses it into a structured format that
|
|
2050
|
-
along with their associated files
|
|
2370
|
+
This function takes raw QC text output (from the Protocol Adherence
|
|
2371
|
+
analysis) and parses it into a structured format that
|
|
2372
|
+
separates passed and failed rules, along with their associated files
|
|
2373
|
+
and conditions.
|
|
2051
2374
|
|
|
2052
2375
|
Args:
|
|
2053
2376
|
patient_id (str, optional):
|
|
2054
2377
|
Patient identifier. Defaults to None.
|
|
2055
2378
|
subject_name (str, optional):
|
|
2056
|
-
Subject/patient name. Defaults to None. Mandatory if no
|
|
2379
|
+
Subject/patient name. Defaults to None. Mandatory if no
|
|
2380
|
+
patient_id is provided.
|
|
2057
2381
|
ssid (str, optional):
|
|
2058
|
-
Session ID. Defaults to None. Mandatory if subject_name is
|
|
2382
|
+
Session ID. Defaults to None. Mandatory if subject_name is
|
|
2383
|
+
provided.
|
|
2059
2384
|
|
|
2060
2385
|
Returns:
|
|
2061
|
-
dict: A structured dictionary containing a list of dictionaries
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2386
|
+
dict: A structured dictionary containing a list of dictionaries
|
|
2387
|
+
with passed rules and their details and failed rules and their
|
|
2388
|
+
details. Details of passed rules are:
|
|
2389
|
+
per each rule: Files that have passed the rule. Per each file name
|
|
2390
|
+
of the file and number of conditions of the rule. Details of
|
|
2391
|
+
failed rules are: per each rule failed conditions: Number of
|
|
2392
|
+
times it failed. Each condition status.
|
|
2067
2393
|
|
|
2068
2394
|
Example:
|
|
2069
2395
|
>>> parse_qc_text(subject_name="patient_123", ssid=1)
|
|
@@ -2089,7 +2415,7 @@ class Project:
|
|
|
2089
2415
|
"conditions": [
|
|
2090
2416
|
{
|
|
2091
2417
|
"status": "failed",
|
|
2092
|
-
"condition": "SliceThickness between
|
|
2418
|
+
"condition": "SliceThickness between.."
|
|
2093
2419
|
}
|
|
2094
2420
|
]
|
|
2095
2421
|
}
|
|
@@ -2102,20 +2428,19 @@ class Project:
|
|
|
2102
2428
|
}
|
|
2103
2429
|
"""
|
|
2104
2430
|
|
|
2105
|
-
_, text = self.get_qc_status_subject(
|
|
2431
|
+
_, text = self.get_qc_status_subject(
|
|
2432
|
+
patient_id=patient_id, subject_name=subject_name, ssid=ssid
|
|
2433
|
+
)
|
|
2106
2434
|
|
|
2107
|
-
result = {
|
|
2108
|
-
"passed": [],
|
|
2109
|
-
"failed": []
|
|
2110
|
-
}
|
|
2435
|
+
result = {"passed": [], "failed": []}
|
|
2111
2436
|
|
|
2112
2437
|
# Split into failed and passed sections
|
|
2113
|
-
sections = re.split(r
|
|
2438
|
+
sections = re.split(r"={10,}\n\n", text)
|
|
2114
2439
|
if len(sections) == 3:
|
|
2115
|
-
failed_section = sections[1].split(
|
|
2440
|
+
failed_section = sections[1].split("=" * 10)[0].strip()
|
|
2116
2441
|
passed_section = sections[2].strip()
|
|
2117
2442
|
else:
|
|
2118
|
-
section = sections[1].split(
|
|
2443
|
+
section = sections[1].split("=" * 10)[0].strip()
|
|
2119
2444
|
if "PASSED QC MESSAGES" in section:
|
|
2120
2445
|
passed_section = section
|
|
2121
2446
|
failed_section = ""
|
|
@@ -2123,106 +2448,39 @@ class Project:
|
|
|
2123
2448
|
failed_section = section
|
|
2124
2449
|
passed_section = ""
|
|
2125
2450
|
|
|
2126
|
-
|
|
2127
|
-
failed_rules = re.split(r
|
|
2128
|
-
|
|
2129
|
-
rule_name = rule_text.split(' ❌')[0].strip()
|
|
2130
|
-
rule_data = {
|
|
2131
|
-
"rule": rule_name,
|
|
2132
|
-
"files": [],
|
|
2133
|
-
"failed_conditions": {}
|
|
2134
|
-
}
|
|
2135
|
-
|
|
2136
|
-
# Extract all file comparisons for this rule
|
|
2137
|
-
file_comparisons = re.split(r'\t- Comparison with file:', rule_text)
|
|
2138
|
-
for comp in file_comparisons[1:]: # Skip first part
|
|
2139
|
-
file_name = comp.split('\n')[0].strip()
|
|
2140
|
-
conditions_match = re.search(
|
|
2141
|
-
r'Conditions:(.*?)(?=\n\t- Comparison|\n\n|$)',
|
|
2142
|
-
comp,
|
|
2143
|
-
re.DOTALL
|
|
2144
|
-
)
|
|
2145
|
-
if not conditions_match:
|
|
2146
|
-
continue
|
|
2147
|
-
|
|
2148
|
-
conditions_text = conditions_match.group(1).strip()
|
|
2149
|
-
# Parse conditions
|
|
2150
|
-
conditions = []
|
|
2151
|
-
for line in conditions_text.split('\n'):
|
|
2152
|
-
line = line.strip()
|
|
2153
|
-
if line.startswith('·'):
|
|
2154
|
-
status = '✔' if '✔' in line else '🚫'
|
|
2155
|
-
condition = re.sub(r'^· [✔🚫]\s*', '', line)
|
|
2156
|
-
conditions.append({
|
|
2157
|
-
"status": "passed" if status == '✔' else "failed",
|
|
2158
|
-
"condition": condition
|
|
2159
|
-
})
|
|
2160
|
-
|
|
2161
|
-
# Add to failed conditions summary
|
|
2162
|
-
for cond in conditions:
|
|
2163
|
-
if cond['status'] == 'failed':
|
|
2164
|
-
cond_text = cond['condition']
|
|
2165
|
-
if cond_text not in rule_data['failed_conditions']:
|
|
2166
|
-
rule_data['failed_conditions'][cond_text] = 0
|
|
2167
|
-
rule_data['failed_conditions'][cond_text] += 1
|
|
2168
|
-
|
|
2169
|
-
rule_data['files'].append({
|
|
2170
|
-
"file": file_name,
|
|
2171
|
-
"conditions": conditions
|
|
2172
|
-
})
|
|
2173
|
-
|
|
2174
|
-
result['failed'].append(rule_data)
|
|
2451
|
+
# Parse failed rules
|
|
2452
|
+
failed_rules = re.split(r"\n ❌ ", failed_section)
|
|
2453
|
+
result = self.__parse_fail_rules(failed_rules, result)
|
|
2175
2454
|
|
|
2176
2455
|
# Parse passed rules
|
|
2177
|
-
passed_rules = re.split(r
|
|
2178
|
-
|
|
2179
|
-
rule_name = rule_text.split(' ✅')[0].strip()
|
|
2180
|
-
rule_data = {
|
|
2181
|
-
"rule": rule_name,
|
|
2182
|
-
"sub_rule": None,
|
|
2183
|
-
"files": []
|
|
2184
|
-
}
|
|
2185
|
-
|
|
2186
|
-
# Get sub-rule
|
|
2187
|
-
sub_rule_match = re.search(r'Sub-rule: (.*?)\n', rule_text)
|
|
2188
|
-
if sub_rule_match:
|
|
2189
|
-
rule_data['sub_rule'] = sub_rule_match.group(1).strip()
|
|
2190
|
-
|
|
2191
|
-
# Get files passed
|
|
2192
|
-
files_passed = re.search(r'List of files passed:(.*?)(?=\n\n|\Z)', rule_text, re.DOTALL)
|
|
2193
|
-
if files_passed:
|
|
2194
|
-
for line in files_passed.group(1).split('\n'):
|
|
2195
|
-
line = line.strip()
|
|
2196
|
-
if line.startswith('·'):
|
|
2197
|
-
file_match = re.match(r'· (.*?) \((\d+)/(\d+)\)', line)
|
|
2198
|
-
if file_match:
|
|
2199
|
-
rule_data['files'].append({
|
|
2200
|
-
"file": file_match.group(1).strip(),
|
|
2201
|
-
"passed_conditions": int(file_match.group(2)),
|
|
2202
|
-
})
|
|
2203
|
-
|
|
2204
|
-
result['passed'].append(rule_data)
|
|
2456
|
+
passed_rules = re.split(r"\n ✅ ", passed_section)
|
|
2457
|
+
result = self.__parse_pass_rules(passed_rules, result)
|
|
2205
2458
|
|
|
2206
2459
|
return result
|
|
2207
2460
|
|
|
2208
2461
|
def calculate_qc_statistics(self):
|
|
2209
2462
|
"""
|
|
2210
|
-
Calculate comprehensive statistics from multiple QC results across
|
|
2211
|
-
platform.
|
|
2463
|
+
Calculate comprehensive statistics from multiple QC results across
|
|
2464
|
+
subjects from a project in the QMENTA platform.
|
|
2212
2465
|
|
|
2213
|
-
This function aggregates and analyzes QC results from
|
|
2214
|
-
providing statistical insights about
|
|
2215
|
-
and condition failure patterns.
|
|
2466
|
+
This function aggregates and analyzes QC results from
|
|
2467
|
+
multiple subjects/containers, providing statistical insights about
|
|
2468
|
+
rule pass/fail rates, file statistics, and condition failure patterns.
|
|
2216
2469
|
|
|
2217
2470
|
Returns:
|
|
2218
|
-
dict: A dictionary containing comprehensive QC statistics
|
|
2471
|
+
dict: A dictionary containing comprehensive QC statistics
|
|
2472
|
+
including:
|
|
2219
2473
|
- passed_rules: Total count of passed rules across all subjects
|
|
2220
2474
|
- failed_rules: Total count of failed rules across all subjects
|
|
2221
2475
|
- subjects_passed: Count of subjects with no failed rules
|
|
2222
|
-
- subjects_with_failed: Count of subjects with at least one
|
|
2223
|
-
|
|
2224
|
-
-
|
|
2225
|
-
|
|
2476
|
+
- subjects_with_failed: Count of subjects with at least one
|
|
2477
|
+
failed rule
|
|
2478
|
+
- num_passed_files_distribution: Distribution of how many
|
|
2479
|
+
rules have N passed files
|
|
2480
|
+
- file_stats: File-level statistics (total, passed, failed,
|
|
2481
|
+
pass percentage)
|
|
2482
|
+
- condition_failure_rates: Frequency and percentage of each
|
|
2483
|
+
failed condition
|
|
2226
2484
|
- rule_success_rates: Success rates for each rule type
|
|
2227
2485
|
|
|
2228
2486
|
The statistics help identify:
|
|
@@ -2268,88 +2526,144 @@ class Project:
|
|
|
2268
2526
|
containers = self.list_input_containers()
|
|
2269
2527
|
|
|
2270
2528
|
for c in containers:
|
|
2271
|
-
qc_results_list.append(
|
|
2529
|
+
qc_results_list.append(
|
|
2530
|
+
self.parse_qc_text(
|
|
2531
|
+
subject_name=c["patient_secret_name"], ssid=c["ssid"]
|
|
2532
|
+
)
|
|
2533
|
+
)
|
|
2272
2534
|
|
|
2273
2535
|
# Initialize statistics
|
|
2274
2536
|
stats = {
|
|
2275
|
-
|
|
2276
|
-
|
|
2537
|
+
"passed_rules": 0,
|
|
2538
|
+
"failed_rules": 0,
|
|
2277
2539
|
"subjects_passed": 0,
|
|
2278
2540
|
"subjects_with_failed": 0,
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2541
|
+
"num_passed_files_distribution": defaultdict(
|
|
2542
|
+
int
|
|
2543
|
+
), # How many rules have N passed files
|
|
2544
|
+
"file_stats": {
|
|
2545
|
+
"total": 0,
|
|
2546
|
+
"passed": 0,
|
|
2547
|
+
"failed": 0,
|
|
2548
|
+
"pass_percentage": 0.0,
|
|
2285
2549
|
},
|
|
2286
|
-
|
|
2287
|
-
|
|
2550
|
+
"condition_failure_rates": defaultdict(
|
|
2551
|
+
lambda: {"count": 0, "percentage": 0.0}
|
|
2552
|
+
),
|
|
2553
|
+
"rule_success_rates": defaultdict(
|
|
2554
|
+
lambda: {"passed": 0, "failed": 0, "success_rate": 0.0}
|
|
2555
|
+
),
|
|
2288
2556
|
}
|
|
2289
2557
|
|
|
2290
2558
|
total_failures = 0
|
|
2291
2559
|
|
|
2292
2560
|
# sum subjects with not failed qc message
|
|
2293
|
-
stats["subjects_passed"] = sum(
|
|
2561
|
+
stats["subjects_passed"] = sum(
|
|
2562
|
+
[1 for rules in qc_results_list if not rules["failed"]]
|
|
2563
|
+
)
|
|
2294
2564
|
# sum subjects with some failed qc message
|
|
2295
|
-
stats["subjects_with_failed"] = sum(
|
|
2565
|
+
stats["subjects_with_failed"] = sum(
|
|
2566
|
+
[1 for rules in qc_results_list if rules["failed"]]
|
|
2567
|
+
)
|
|
2296
2568
|
# sum rules that have passed
|
|
2297
|
-
stats["passed_rules"] = sum(
|
|
2569
|
+
stats["passed_rules"] = sum(
|
|
2570
|
+
[
|
|
2571
|
+
len(rules["passed"])
|
|
2572
|
+
for rules in qc_results_list
|
|
2573
|
+
if rules["failed"]
|
|
2574
|
+
]
|
|
2575
|
+
)
|
|
2298
2576
|
# sum rules that have failed
|
|
2299
|
-
stats["failed_rules"] = sum(
|
|
2577
|
+
stats["failed_rules"] = sum(
|
|
2578
|
+
[
|
|
2579
|
+
len(rules["failed"])
|
|
2580
|
+
for rules in qc_results_list
|
|
2581
|
+
if rules["failed"]
|
|
2582
|
+
]
|
|
2583
|
+
)
|
|
2300
2584
|
|
|
2301
2585
|
for qc_results in qc_results_list:
|
|
2302
2586
|
|
|
2303
2587
|
# Count passed files distribution
|
|
2304
|
-
for rule in qc_results[
|
|
2305
|
-
num_files = len(rule[
|
|
2306
|
-
stats[
|
|
2307
|
-
stats[
|
|
2308
|
-
stats[
|
|
2309
|
-
rule_name = rule[
|
|
2310
|
-
stats[
|
|
2311
|
-
|
|
2312
|
-
for rule in qc_results[
|
|
2313
|
-
stats[
|
|
2314
|
-
stats[
|
|
2315
|
-
for condition, count in rule[
|
|
2588
|
+
for rule in qc_results["passed"]:
|
|
2589
|
+
num_files = len(rule["files"])
|
|
2590
|
+
stats["num_passed_files_distribution"][num_files] += 1
|
|
2591
|
+
stats["file_stats"]["passed"] += len(rule["files"])
|
|
2592
|
+
stats["file_stats"]["total"] += len(rule["files"])
|
|
2593
|
+
rule_name = rule["rule"]
|
|
2594
|
+
stats["rule_success_rates"][rule_name]["passed"] += 1
|
|
2595
|
+
|
|
2596
|
+
for rule in qc_results["failed"]:
|
|
2597
|
+
stats["file_stats"]["total"] += len(rule["files"])
|
|
2598
|
+
stats["file_stats"]["failed"] += len(rule["files"])
|
|
2599
|
+
for condition, count in rule["failed_conditions"].items():
|
|
2316
2600
|
# Extract just the condition text without actual value
|
|
2317
|
-
clean_condition = re.sub(
|
|
2318
|
-
|
|
2601
|
+
clean_condition = re.sub(
|
|
2602
|
+
r"\.\s*Actual value:.*$", "", condition
|
|
2603
|
+
)
|
|
2604
|
+
stats["condition_failure_rates"][clean_condition][
|
|
2605
|
+
"count"
|
|
2606
|
+
] += count
|
|
2319
2607
|
total_failures += count
|
|
2320
|
-
rule_name = rule[
|
|
2321
|
-
stats[
|
|
2322
|
-
|
|
2323
|
-
if stats[
|
|
2324
|
-
stats[
|
|
2325
|
-
(stats[
|
|
2608
|
+
rule_name = rule["rule"]
|
|
2609
|
+
stats["rule_success_rates"][rule_name]["failed"] += 1
|
|
2610
|
+
|
|
2611
|
+
if stats["file_stats"]["total"] > 0:
|
|
2612
|
+
stats["file_stats"]["pass_percentage"] = round(
|
|
2613
|
+
(stats["file_stats"]["passed"] / stats["file_stats"]["total"])
|
|
2614
|
+
* 100,
|
|
2615
|
+
2,
|
|
2326
2616
|
)
|
|
2327
2617
|
|
|
2328
2618
|
# Calculate condition failure percentages
|
|
2329
|
-
for condition in stats[
|
|
2619
|
+
for condition in stats["condition_failure_rates"]:
|
|
2330
2620
|
if total_failures > 0:
|
|
2331
|
-
stats[
|
|
2332
|
-
(
|
|
2621
|
+
stats["condition_failure_rates"][condition]["percentage"] = (
|
|
2622
|
+
round(
|
|
2623
|
+
(
|
|
2624
|
+
stats["condition_failure_rates"][condition][
|
|
2625
|
+
"count"
|
|
2626
|
+
]
|
|
2627
|
+
/ total_failures
|
|
2628
|
+
)
|
|
2629
|
+
* 100,
|
|
2630
|
+
2,
|
|
2631
|
+
)
|
|
2333
2632
|
)
|
|
2334
2633
|
|
|
2335
2634
|
# Calculate rule success rates
|
|
2336
|
-
for rule in stats[
|
|
2337
|
-
total =
|
|
2635
|
+
for rule in stats["rule_success_rates"]:
|
|
2636
|
+
total = (
|
|
2637
|
+
stats["rule_success_rates"][rule]["passed"]
|
|
2638
|
+
+ stats["rule_success_rates"][rule]["failed"]
|
|
2639
|
+
)
|
|
2338
2640
|
if total > 0:
|
|
2339
|
-
stats[
|
|
2340
|
-
(stats[
|
|
2641
|
+
stats["rule_success_rates"][rule]["success_rate"] = round(
|
|
2642
|
+
(stats["rule_success_rates"][rule]["passed"] / total)
|
|
2643
|
+
* 100,
|
|
2644
|
+
2,
|
|
2341
2645
|
)
|
|
2342
2646
|
|
|
2343
2647
|
# Convert defaultdict to regular dict for cleaner JSON output
|
|
2344
|
-
stats[
|
|
2345
|
-
|
|
2346
|
-
|
|
2648
|
+
stats["num_passed_files_distribution"] = dict(
|
|
2649
|
+
stats["num_passed_files_distribution"]
|
|
2650
|
+
)
|
|
2651
|
+
stats["condition_failure_rates"] = dict(
|
|
2652
|
+
stats["condition_failure_rates"]
|
|
2653
|
+
)
|
|
2654
|
+
stats["rule_success_rates"] = dict(stats["rule_success_rates"])
|
|
2347
2655
|
|
|
2348
2656
|
return stats
|
|
2349
2657
|
|
|
2350
2658
|
""" Helper Methods """
|
|
2351
2659
|
|
|
2352
|
-
def __handle_start_analysis(
|
|
2660
|
+
def __handle_start_analysis(
|
|
2661
|
+
self,
|
|
2662
|
+
post_data,
|
|
2663
|
+
ignore_warnings=False,
|
|
2664
|
+
ignore_file_selection=True,
|
|
2665
|
+
n_calls=0,
|
|
2666
|
+
):
|
|
2353
2667
|
"""
|
|
2354
2668
|
Handle the possible responses from the server after start_analysis.
|
|
2355
2669
|
Sometimes we have to send a request again, and then check again the
|
|
@@ -2369,13 +2683,21 @@ class Project:
|
|
|
2369
2683
|
than {n_calls} times: aborting."
|
|
2370
2684
|
)
|
|
2371
2685
|
return None
|
|
2372
|
-
|
|
2686
|
+
response = None
|
|
2373
2687
|
try:
|
|
2374
2688
|
response = platform.parse_response(
|
|
2375
|
-
platform.post(
|
|
2689
|
+
platform.post(
|
|
2690
|
+
self._account.auth,
|
|
2691
|
+
"analysis_manager/analysis_registration",
|
|
2692
|
+
data=post_data,
|
|
2693
|
+
)
|
|
2376
2694
|
)
|
|
2377
2695
|
logger.info(response["message"])
|
|
2378
|
-
return
|
|
2696
|
+
return (
|
|
2697
|
+
int(response["analysis_id"])
|
|
2698
|
+
if "analysis_id" in response
|
|
2699
|
+
else None
|
|
2700
|
+
)
|
|
2379
2701
|
|
|
2380
2702
|
except platform.ChooseDataError as choose_data:
|
|
2381
2703
|
if ignore_file_selection:
|
|
@@ -2395,31 +2717,39 @@ class Project:
|
|
|
2395
2717
|
# logging any warning that we have
|
|
2396
2718
|
if choose_data.warning:
|
|
2397
2719
|
has_warning = True
|
|
2398
|
-
logger.warning(
|
|
2720
|
+
logger.warning(choose_data.warning)
|
|
2399
2721
|
|
|
2400
2722
|
new_post = {
|
|
2401
2723
|
"analysis_id": choose_data.analysis_id,
|
|
2402
2724
|
"script_name": post_data["script_name"],
|
|
2403
2725
|
"version": post_data["version"],
|
|
2404
2726
|
}
|
|
2727
|
+
if "tags" in post_data.keys():
|
|
2728
|
+
new_post["tags"] = post_data["tags"]
|
|
2405
2729
|
|
|
2406
2730
|
if choose_data.data_to_choose:
|
|
2407
2731
|
self.__handle_manual_choose_data(new_post, choose_data)
|
|
2408
2732
|
else:
|
|
2409
2733
|
if has_warning and not ignore_warnings:
|
|
2410
|
-
logger.error(
|
|
2734
|
+
logger.error(
|
|
2735
|
+
"Cancelling analysis due to warnings, set "
|
|
2736
|
+
"'ignore_warnings' to True to override."
|
|
2737
|
+
)
|
|
2411
2738
|
new_post["cancel"] = "1"
|
|
2412
2739
|
else:
|
|
2413
2740
|
logger.info("suppressing warnings")
|
|
2414
2741
|
new_post["user_preference"] = "{}"
|
|
2415
2742
|
new_post["_mint_only_warning"] = "1"
|
|
2416
2743
|
|
|
2417
|
-
return self.__handle_start_analysis(
|
|
2744
|
+
return self.__handle_start_analysis(
|
|
2745
|
+
new_post, ignore_warnings, ignore_file_selection, n_calls
|
|
2746
|
+
)
|
|
2418
2747
|
except platform.ActionFailedError as e:
|
|
2419
2748
|
logger.error(f"Unable to start the analysis: {e}.")
|
|
2420
2749
|
return None
|
|
2421
2750
|
|
|
2422
|
-
|
|
2751
|
+
@staticmethod
|
|
2752
|
+
def __handle_manual_choose_data(post_data, choose_data):
|
|
2423
2753
|
"""
|
|
2424
2754
|
Handle the responses of the user when there is need to select a file
|
|
2425
2755
|
to start the analysis.
|
|
@@ -2432,15 +2762,22 @@ class Project:
|
|
|
2432
2762
|
post_data : dict
|
|
2433
2763
|
Current post_data dictionary. To be mofidied in-place.
|
|
2434
2764
|
choose_data : platform.ChooseDataError
|
|
2435
|
-
Error raised when trying to start an analysis, but data has to
|
|
2765
|
+
Error raised when trying to start an analysis, but data has to
|
|
2766
|
+
be chosen.
|
|
2436
2767
|
"""
|
|
2437
2768
|
|
|
2438
2769
|
logger = logging.getLogger(logger_name)
|
|
2439
|
-
logger.warning(
|
|
2770
|
+
logger.warning(
|
|
2771
|
+
"Multiple inputs available. You have to select the desired file/s "
|
|
2772
|
+
"to continue."
|
|
2773
|
+
)
|
|
2440
2774
|
# in case we have data to choose
|
|
2441
2775
|
chosen_files = {}
|
|
2442
2776
|
for settings_key in choose_data.data_to_choose:
|
|
2443
|
-
logger.warning(
|
|
2777
|
+
logger.warning(
|
|
2778
|
+
f"Type next the file/s for the input with ID: "
|
|
2779
|
+
f"'{settings_key}'."
|
|
2780
|
+
)
|
|
2444
2781
|
chosen_files[settings_key] = {}
|
|
2445
2782
|
filters = choose_data.data_to_choose[settings_key]["filters"]
|
|
2446
2783
|
for filter_key in filters:
|
|
@@ -2455,7 +2792,9 @@ class Project:
|
|
|
2455
2792
|
if filter_data["range"][0] != 0:
|
|
2456
2793
|
number_of_files_to_select = filter_data["range"][0]
|
|
2457
2794
|
elif filter_data["range"][1] != 0:
|
|
2458
|
-
number_of_files_to_select = min(
|
|
2795
|
+
number_of_files_to_select = min(
|
|
2796
|
+
filter_data["range"][1], len(filter_data["files"])
|
|
2797
|
+
)
|
|
2459
2798
|
else:
|
|
2460
2799
|
number_of_files_to_select = len(filter_data["files"])
|
|
2461
2800
|
|
|
@@ -2467,19 +2806,29 @@ class Project:
|
|
|
2467
2806
|
# list_container_filter_files()
|
|
2468
2807
|
|
|
2469
2808
|
if number_of_files_to_select != len(filter_data["files"]):
|
|
2809
|
+
substring = ""
|
|
2810
|
+
if number_of_files_to_select > 1:
|
|
2811
|
+
substring = "s (i.e., file1.zip, file2.zip, file3.zip)"
|
|
2470
2812
|
logger.warning(
|
|
2471
2813
|
f" · File filter name: '{filter_key}'. Type "
|
|
2472
|
-
f"{number_of_files_to_select} file"
|
|
2473
|
-
f"{'s (i.e., file1.zip, file2.zip, file3.zip)' if number_of_files_to_select > 1 else ''}."
|
|
2814
|
+
f"{number_of_files_to_select} file{substring}."
|
|
2474
2815
|
)
|
|
2475
2816
|
save_file_ids, select_file_filter = {}, ""
|
|
2476
2817
|
for file_ in filter_data["files"]:
|
|
2477
|
-
select_file_filter +=
|
|
2818
|
+
select_file_filter += (
|
|
2819
|
+
f" · File name: {file_['name']}\n"
|
|
2820
|
+
)
|
|
2478
2821
|
save_file_ids[file_["name"]] = file_["_id"]
|
|
2479
|
-
names = [
|
|
2822
|
+
names = [
|
|
2823
|
+
el.strip()
|
|
2824
|
+
for el in input(select_file_filter).strip().split(",")
|
|
2825
|
+
]
|
|
2480
2826
|
|
|
2481
2827
|
if len(names) != number_of_files_to_select:
|
|
2482
|
-
logger.error(
|
|
2828
|
+
logger.error(
|
|
2829
|
+
"The number of files selected does not correspond "
|
|
2830
|
+
"to the number of needed files."
|
|
2831
|
+
)
|
|
2483
2832
|
logger.error(
|
|
2484
2833
|
f"Selected: {len(names)} vs. "
|
|
2485
2834
|
f"Number of files to select: "
|
|
@@ -2489,14 +2838,27 @@ class Project:
|
|
|
2489
2838
|
post_data["cancel"] = "1"
|
|
2490
2839
|
|
|
2491
2840
|
elif any([name not in save_file_ids for name in names]):
|
|
2492
|
-
logger.error(
|
|
2841
|
+
logger.error(
|
|
2842
|
+
f"Some selected file/s '{', '.join(names)}' "
|
|
2843
|
+
f"do not exist. Cancelling analysis..."
|
|
2844
|
+
)
|
|
2493
2845
|
post_data["cancel"] = "1"
|
|
2494
2846
|
else:
|
|
2495
|
-
chosen_files[settings_key][filter_key] = [
|
|
2847
|
+
chosen_files[settings_key][filter_key] = [
|
|
2848
|
+
save_file_ids[name] for name in names
|
|
2849
|
+
]
|
|
2496
2850
|
|
|
2497
2851
|
else:
|
|
2498
|
-
logger.warning(
|
|
2499
|
-
|
|
2852
|
+
logger.warning(
|
|
2853
|
+
"Setting all available files to be input to the "
|
|
2854
|
+
"analysis."
|
|
2855
|
+
)
|
|
2856
|
+
files_selection = [
|
|
2857
|
+
ff["_id"]
|
|
2858
|
+
for ff in filter_data["files"][
|
|
2859
|
+
:number_of_files_to_select
|
|
2860
|
+
]
|
|
2861
|
+
]
|
|
2500
2862
|
chosen_files[settings_key][filter_key] = files_selection
|
|
2501
2863
|
|
|
2502
2864
|
post_data["user_preference"] = json.dumps(chosen_files)
|
|
@@ -2510,20 +2872,6 @@ class Project:
|
|
|
2510
2872
|
modalities.append(modality)
|
|
2511
2873
|
return modalities
|
|
2512
2874
|
|
|
2513
|
-
def __show_progress(self, done, total, finish=False):
|
|
2514
|
-
bytes_in_mb = 1024 * 1024
|
|
2515
|
-
progress_message = "\r[{:.2f} %] Uploaded {:.2f} of {:.2f} Mb".format(
|
|
2516
|
-
done / float(total) * 100, done / bytes_in_mb, total / bytes_in_mb
|
|
2517
|
-
)
|
|
2518
|
-
sys.stdout.write(progress_message)
|
|
2519
|
-
sys.stdout.flush()
|
|
2520
|
-
if not finish:
|
|
2521
|
-
pass
|
|
2522
|
-
# sys.stdout.write("")
|
|
2523
|
-
# sys.stdout.flush()
|
|
2524
|
-
else:
|
|
2525
|
-
sys.stdout.write("\n")
|
|
2526
|
-
|
|
2527
2875
|
def __get_session_id(self, file_path):
|
|
2528
2876
|
m = hashlib.md5()
|
|
2529
2877
|
m.update(file_path.encode("utf-8"))
|
|
@@ -2555,11 +2903,12 @@ class Project:
|
|
|
2555
2903
|
else:
|
|
2556
2904
|
return True
|
|
2557
2905
|
|
|
2558
|
-
|
|
2906
|
+
@staticmethod
|
|
2907
|
+
def __operation(reference_value, operator, input_value):
|
|
2559
2908
|
"""
|
|
2560
2909
|
The method performs an operation by comparing the two input values.
|
|
2561
|
-
The Operation is applied to the Input Value in comparison to the
|
|
2562
|
-
Value.
|
|
2910
|
+
The Operation is applied to the Input Value in comparison to the
|
|
2911
|
+
Reference Value.
|
|
2563
2912
|
|
|
2564
2913
|
Parameters
|
|
2565
2914
|
----------
|
|
@@ -2575,39 +2924,32 @@ class Project:
|
|
|
2575
2924
|
bool
|
|
2576
2925
|
True if the operation is satisfied, False otherwise.
|
|
2577
2926
|
"""
|
|
2578
|
-
if input_value
|
|
2927
|
+
if not input_value: # Handles None, "", and other falsy values
|
|
2579
2928
|
return False
|
|
2580
2929
|
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
elif operator == "gte":
|
|
2594
|
-
return input_value >= reference_value
|
|
2595
|
-
|
|
2596
|
-
elif operator == "lt":
|
|
2597
|
-
return input_value < reference_value
|
|
2930
|
+
operator_actions = {
|
|
2931
|
+
"in": lambda: reference_value in input_value,
|
|
2932
|
+
"in-list": lambda: all(
|
|
2933
|
+
el in input_value for el in reference_value
|
|
2934
|
+
),
|
|
2935
|
+
"eq": lambda: input_value == reference_value,
|
|
2936
|
+
"gt": lambda: input_value > reference_value,
|
|
2937
|
+
"gte": lambda: input_value >= reference_value,
|
|
2938
|
+
"lt": lambda: input_value < reference_value,
|
|
2939
|
+
"lte": lambda: input_value <= reference_value,
|
|
2940
|
+
}
|
|
2598
2941
|
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
else:
|
|
2602
|
-
return False
|
|
2942
|
+
action = operator_actions.get(operator, lambda: False)
|
|
2943
|
+
return action()
|
|
2603
2944
|
|
|
2604
|
-
|
|
2945
|
+
@staticmethod
|
|
2946
|
+
def __wrap_search_criteria(search_criteria=None):
|
|
2605
2947
|
"""
|
|
2606
2948
|
Wraps the conditions specified within the Search Criteria in order for
|
|
2607
2949
|
other methods to handle it easily. The conditions are grouped only into
|
|
2608
|
-
three groups: Modality, Tags and the File Metadata (if DICOM it
|
|
2609
|
-
to the DICOM information), and each of them is output
|
|
2610
|
-
variable.
|
|
2950
|
+
three groups: Modality, Tags and the File Metadata (if DICOM it
|
|
2951
|
+
corresponds to the DICOM information), and each of them is output
|
|
2952
|
+
in a different variable.
|
|
2611
2953
|
|
|
2612
2954
|
Parameters
|
|
2613
2955
|
----------
|
|
@@ -2631,27 +2973,27 @@ class Project:
|
|
|
2631
2973
|
|
|
2632
2974
|
Returns
|
|
2633
2975
|
-------
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
file_metadata : Dict
|
|
2643
|
-
Dictionary containing the file metadata of the search criteria
|
|
2976
|
+
tuple
|
|
2977
|
+
A tuple containing:
|
|
2978
|
+
- str: modality is a string containing the modality of the search
|
|
2979
|
+
criteria extracted from 'pars_modalities';
|
|
2980
|
+
- list: tags is a list of strings containing the tags of the search
|
|
2981
|
+
criteria extracted 'from pars_tags',
|
|
2982
|
+
- dict: containing the file metadata of the search criteria
|
|
2644
2983
|
extracted from 'pars_[dicom]_KEY'
|
|
2645
2984
|
"""
|
|
2646
2985
|
|
|
2647
2986
|
# The keys not included bellow apply to the whole session.
|
|
2987
|
+
if search_criteria is None:
|
|
2988
|
+
search_criteria = {}
|
|
2648
2989
|
modality, tags, file_metadata = "", list(), dict()
|
|
2649
2990
|
for key, value in search_criteria.items():
|
|
2650
2991
|
if key == "pars_modalities":
|
|
2651
2992
|
modalities = value.split(";")[1].split(",")
|
|
2652
2993
|
if len(modalities) != 1:
|
|
2653
2994
|
raise ValueError(
|
|
2654
|
-
f"A file can only have one modality.
|
|
2995
|
+
f"A file can only have one modality. "
|
|
2996
|
+
f"Provided Modalities: {', '.join(modalities)}."
|
|
2655
2997
|
)
|
|
2656
2998
|
modality = modalities[0]
|
|
2657
2999
|
elif key == "pars_tags":
|
|
@@ -2660,16 +3002,162 @@ class Project:
|
|
|
2660
3002
|
d_tag = key.split("pars_[dicom]_")[1]
|
|
2661
3003
|
d_type = value.split(";")[0]
|
|
2662
3004
|
if d_type == "string":
|
|
2663
|
-
file_metadata[d_tag] = {
|
|
3005
|
+
file_metadata[d_tag] = {
|
|
3006
|
+
"operation": "in",
|
|
3007
|
+
"value": value.replace(d_type + ";", ""),
|
|
3008
|
+
}
|
|
2664
3009
|
elif d_type == "integer":
|
|
2665
3010
|
d_operator = value.split(";")[1].split("|")[0]
|
|
2666
3011
|
d_value = value.split(";")[1].split("|")[1]
|
|
2667
|
-
file_metadata[d_tag] = {
|
|
3012
|
+
file_metadata[d_tag] = {
|
|
3013
|
+
"operation": d_operator,
|
|
3014
|
+
"value": int(d_value),
|
|
3015
|
+
}
|
|
2668
3016
|
elif d_type == "decimal":
|
|
2669
3017
|
d_operator = value.split(";")[1].split("|")[0]
|
|
2670
3018
|
d_value = value.split(";")[1].split("|")[1]
|
|
2671
|
-
file_metadata[d_tag] = {
|
|
3019
|
+
file_metadata[d_tag] = {
|
|
3020
|
+
"operation": d_operator,
|
|
3021
|
+
"value": float(d_value),
|
|
3022
|
+
}
|
|
2672
3023
|
elif d_type == "list":
|
|
2673
3024
|
value.replace(d_type + ";", "")
|
|
2674
|
-
file_metadata[d_tag] = {
|
|
3025
|
+
file_metadata[d_tag] = {
|
|
3026
|
+
"operation": "in-list",
|
|
3027
|
+
"value": value.replace(d_type + ";", "").split(";"),
|
|
3028
|
+
}
|
|
2675
3029
|
return modality, tags, file_metadata
|
|
3030
|
+
|
|
3031
|
+
@staticmethod
|
|
3032
|
+
def __assert_split_data(split_data, ssid, add_to_container_id):
|
|
3033
|
+
"""
|
|
3034
|
+
Assert if the split_data parameter is possible to use in regards
|
|
3035
|
+
to the ssid and add_to_container_id parameters during upload.
|
|
3036
|
+
Changes its status to False if needed.
|
|
3037
|
+
|
|
3038
|
+
Parameters
|
|
3039
|
+
----------
|
|
3040
|
+
split_data : Bool
|
|
3041
|
+
split_data parameter from method 'upload_file'.
|
|
3042
|
+
ssid : str
|
|
3043
|
+
Session ID.
|
|
3044
|
+
add_to_container_id : int or bool
|
|
3045
|
+
Container ID or False
|
|
3046
|
+
|
|
3047
|
+
Returns
|
|
3048
|
+
-------
|
|
3049
|
+
split_data : Bool
|
|
3050
|
+
|
|
3051
|
+
"""
|
|
3052
|
+
|
|
3053
|
+
logger = logging.getLogger(logger_name)
|
|
3054
|
+
if ssid and split_data:
|
|
3055
|
+
logger.warning(
|
|
3056
|
+
"split-data argument will be ignored because ssid has been "
|
|
3057
|
+
"specified"
|
|
3058
|
+
)
|
|
3059
|
+
split_data = False
|
|
3060
|
+
|
|
3061
|
+
if add_to_container_id and split_data:
|
|
3062
|
+
logger.warning(
|
|
3063
|
+
"split-data argument will be ignored because "
|
|
3064
|
+
"add_to_container_id has been specified"
|
|
3065
|
+
)
|
|
3066
|
+
split_data = False
|
|
3067
|
+
|
|
3068
|
+
return split_data
|
|
3069
|
+
|
|
3070
|
+
@staticmethod
|
|
3071
|
+
def __parse_pass_rules(passed_rules, result):
|
|
3072
|
+
"""
|
|
3073
|
+
Parse pass rules.
|
|
3074
|
+
"""
|
|
3075
|
+
|
|
3076
|
+
for rule_text in passed_rules[1:]: # Skip first empty part
|
|
3077
|
+
rule_name = rule_text.split(" ✅")[0].strip()
|
|
3078
|
+
rule_data = {"rule": rule_name, "sub_rule": None, "files": []}
|
|
3079
|
+
|
|
3080
|
+
# Get sub-rule
|
|
3081
|
+
sub_rule_match = re.search(r"Sub-rule: (.*?)\n", rule_text)
|
|
3082
|
+
if sub_rule_match:
|
|
3083
|
+
rule_data["sub_rule"] = sub_rule_match.group(1).strip()
|
|
3084
|
+
|
|
3085
|
+
# Get files passed
|
|
3086
|
+
files_passed = re.search(
|
|
3087
|
+
r"List of files passed:(.*?)(?=\n\n|\Z)", rule_text, re.DOTALL
|
|
3088
|
+
)
|
|
3089
|
+
if files_passed:
|
|
3090
|
+
for line in files_passed.group(1).split("\n"):
|
|
3091
|
+
line = line.strip()
|
|
3092
|
+
if line.startswith("·"):
|
|
3093
|
+
file_match = re.match(r"· (.*?) \((\d+)/(\d+)\)", line)
|
|
3094
|
+
if file_match:
|
|
3095
|
+
rule_data["files"].append(
|
|
3096
|
+
{
|
|
3097
|
+
"file": file_match.group(1).strip(),
|
|
3098
|
+
"passed_conditions": int(
|
|
3099
|
+
file_match.group(2)
|
|
3100
|
+
),
|
|
3101
|
+
}
|
|
3102
|
+
)
|
|
3103
|
+
|
|
3104
|
+
result["passed"].append(rule_data)
|
|
3105
|
+
return result
|
|
3106
|
+
|
|
3107
|
+
@staticmethod
|
|
3108
|
+
def __parse_fail_rules(failed_rules, result):
|
|
3109
|
+
"""
|
|
3110
|
+
Parse fail rules.
|
|
3111
|
+
"""
|
|
3112
|
+
|
|
3113
|
+
for rule_text in failed_rules[1:]: # Skip first empty part
|
|
3114
|
+
rule_name = rule_text.split(" ❌")[0].strip()
|
|
3115
|
+
rule_data = {
|
|
3116
|
+
"rule": rule_name,
|
|
3117
|
+
"files": [],
|
|
3118
|
+
"failed_conditions": {},
|
|
3119
|
+
}
|
|
3120
|
+
|
|
3121
|
+
# Extract all file comparisons for this rule
|
|
3122
|
+
file_comparisons = re.split(r"- Comparison with file:", rule_text)
|
|
3123
|
+
for comp in file_comparisons[1:]: # Skip first part
|
|
3124
|
+
file_name = comp.split("\n")[0].strip()
|
|
3125
|
+
conditions_match = re.search(
|
|
3126
|
+
r"Conditions:(.*?)(?=\n\t- Comparison|\n\n|$)",
|
|
3127
|
+
comp,
|
|
3128
|
+
re.DOTALL,
|
|
3129
|
+
)
|
|
3130
|
+
if not conditions_match:
|
|
3131
|
+
continue
|
|
3132
|
+
|
|
3133
|
+
conditions_text = conditions_match.group(1).strip()
|
|
3134
|
+
# Parse conditions
|
|
3135
|
+
conditions = []
|
|
3136
|
+
for line in conditions_text.split("\n"):
|
|
3137
|
+
line = line.strip()
|
|
3138
|
+
if line.startswith("·"):
|
|
3139
|
+
status = "✔" if "✔" in line else "🚫"
|
|
3140
|
+
condition = re.sub(r"^· [✔🚫]\s*", "", line)
|
|
3141
|
+
conditions.append(
|
|
3142
|
+
{
|
|
3143
|
+
"status": (
|
|
3144
|
+
"passed" if status == "✔" else "failed"
|
|
3145
|
+
),
|
|
3146
|
+
"condition": condition,
|
|
3147
|
+
}
|
|
3148
|
+
)
|
|
3149
|
+
|
|
3150
|
+
# Add to failed conditions summary
|
|
3151
|
+
for cond in conditions:
|
|
3152
|
+
if cond["status"] == "failed":
|
|
3153
|
+
cond_text = cond["condition"]
|
|
3154
|
+
if cond_text not in rule_data["failed_conditions"]:
|
|
3155
|
+
rule_data["failed_conditions"][cond_text] = 0
|
|
3156
|
+
rule_data["failed_conditions"][cond_text] += 1
|
|
3157
|
+
|
|
3158
|
+
rule_data["files"].append(
|
|
3159
|
+
{"file": file_name, "conditions": conditions}
|
|
3160
|
+
)
|
|
3161
|
+
|
|
3162
|
+
result["failed"].append(rule_data)
|
|
3163
|
+
return result
|