qmenta-client 1.1.dev1382__py3-none-any.whl → 1.1.dev1394__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qmenta/client/Project.py +1150 -1105
- {qmenta_client-1.1.dev1382.dist-info → qmenta_client-1.1.dev1394.dist-info}/METADATA +1 -1
- {qmenta_client-1.1.dev1382.dist-info → qmenta_client-1.1.dev1394.dist-info}/RECORD +4 -4
- {qmenta_client-1.1.dev1382.dist-info → qmenta_client-1.1.dev1394.dist-info}/WHEEL +0 -0
qmenta/client/Project.py
CHANGED
|
@@ -75,6 +75,7 @@ class Project:
|
|
|
75
75
|
|
|
76
76
|
"""
|
|
77
77
|
|
|
78
|
+
""" Project Related Methods """
|
|
78
79
|
def __init__(self, account: Account, project_id, max_upload_retries=5):
|
|
79
80
|
# if project_id is a string (the name of the project), get the
|
|
80
81
|
# project id (int)
|
|
@@ -132,770 +133,306 @@ class Project:
|
|
|
132
133
|
rep = "<Project {}>".format(self._project_name)
|
|
133
134
|
return rep
|
|
134
135
|
|
|
135
|
-
|
|
136
|
-
def
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
""
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
""
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
Each element is a string and is built using the formatting
|
|
156
|
-
"type;value", or "type;operation|value"
|
|
157
|
-
|
|
158
|
-
Complete search_criteria Dictionary Explanation:
|
|
159
|
-
|
|
160
|
-
search_criteria = {
|
|
161
|
-
"pars_patient_secret_name": "string;SUBJECTID",
|
|
162
|
-
"pars_ssid": "integer;OPERATOR|SSID",
|
|
163
|
-
"pars_modalities": "string;MODALITY",
|
|
164
|
-
"pars_tags": "tags;TAGS",
|
|
165
|
-
"pars_age_at_scan": "integer;OPERATOR|AGE_AT_SCAN",
|
|
166
|
-
"pars_[dicom]_KEY": "KEYTYPE;KEYVALUE",
|
|
167
|
-
"pars_PROJECTMETADATA": "METADATATYPE;METADATAVALUE",
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
Where:
|
|
171
|
-
"pars_patient_secret_name": Applies the search to the 'Subject ID'.
|
|
172
|
-
SUBJECTID is a comma separated list of strings.
|
|
173
|
-
"pars_ssid": Applies the search to the 'Session ID'.
|
|
174
|
-
SSID is an integer.
|
|
175
|
-
OPERATOR is the operator to apply. One of:
|
|
176
|
-
- Equal: eq
|
|
177
|
-
- Different Than: ne
|
|
178
|
-
- Greater Than: gt
|
|
179
|
-
- Greater/Equal To: gte
|
|
180
|
-
- Lower Than: lt
|
|
181
|
-
- Lower/Equal To: lte
|
|
182
|
-
|
|
183
|
-
"pars_modalities": Applies the search to the file 'Modalities'
|
|
184
|
-
available within each Subject ID.
|
|
185
|
-
MODALITY is a comma separated list of string. A session is provided as
|
|
186
|
-
long as one MODALITY is available.
|
|
187
|
-
"pars_tags": Applies the search to the file 'Tags' available within
|
|
188
|
-
each Subject ID and to the subject-level 'Tags'.
|
|
189
|
-
TAGS is a comma separated list of strings. A session is provided as
|
|
190
|
-
long as one tag is available.
|
|
191
|
-
"pars_age_at_scan": Applies the search to the 'age_at_scan' metadata
|
|
192
|
-
field.
|
|
193
|
-
AGE_AT_SCAN is an integer.
|
|
194
|
-
"pars_[dicom]_KEY": Applies the search to the metadata fields
|
|
195
|
-
available within each file. KEY must be one of the
|
|
196
|
-
metadata keys of the files. The full list of KEYS is shown above via
|
|
197
|
-
'file_m["metadata"]["info"].keys()'.
|
|
198
|
-
KEYTYPE is the type of the KEY. One of:
|
|
199
|
-
- integer
|
|
200
|
-
- decimal
|
|
201
|
-
- string
|
|
202
|
-
- list
|
|
203
|
-
|
|
204
|
-
if 'integer' or 'decimal' you must also include an OPERATOR
|
|
205
|
-
(i.e., "integer;OPERATOR|KEYVALUE").
|
|
206
|
-
if 'list' the KEYVALUE should be a semicolon separated list of
|
|
207
|
-
values. (i.e., "list;KEYVALUE1;KEYVALUE2;KEYVALUE3)
|
|
208
|
-
KEYVALUEs must be strings.
|
|
209
|
-
KEYVALUE is the expected value of the KEY.
|
|
210
|
-
"pars_[dicom]_PROJECTMETADATA": Applies to the metadata defined
|
|
211
|
-
within the 'Metadata Manager' of the project.
|
|
212
|
-
PROJECTMETADATA is the ID of the metadata field.
|
|
213
|
-
METADATATYPE is the type of the metadata field. One of:
|
|
214
|
-
- string
|
|
215
|
-
- integer
|
|
216
|
-
- list
|
|
217
|
-
- decimal
|
|
218
|
-
- single_option
|
|
219
|
-
- multiple_option
|
|
220
|
-
|
|
221
|
-
if 'integer' or 'decimal' you must also include an OPERATOR
|
|
222
|
-
(i.e., "integer;OPERATOR|METADATAVALUE").
|
|
223
|
-
KEYVALUE is the expected value of the metadata.
|
|
224
|
-
|
|
225
|
-
1) Example:
|
|
226
|
-
search_criteria = {
|
|
227
|
-
"pars_patient_secret_name": "string;abide",
|
|
228
|
-
"pars_ssid": "integer;eq|2"
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
2) Example:
|
|
232
|
-
search_criteria = {
|
|
233
|
-
"pars_modalities": "string;T1",
|
|
234
|
-
"pars_tags": "tags;flair",
|
|
235
|
-
"pars_[dicom]_Manufacturer": "string;ge",
|
|
236
|
-
"pars_[dicom]_FlipAngle": "integer;gt|5",
|
|
237
|
-
"pars_[dicom]_ImageType": "list;PRIMARY;SECONDARY",
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
Note the search criteria applies to all the files included within a
|
|
241
|
-
session. Hence, it does not imply that all the criteria conditions
|
|
242
|
-
are applied to the same files. In example 2) above, it means that
|
|
243
|
-
any Subject ID/Session ID that has a file classified with a 'T1'
|
|
244
|
-
modality, a file with a 'flair' tag, a file whose Manufacturer
|
|
245
|
-
contains 'ge', a file whose FlipAngle is greater than '5º', and a
|
|
246
|
-
file with ImageType with any of the values: PRIMARY or SECONDARY
|
|
247
|
-
will be selected.
|
|
248
|
-
|
|
249
|
-
Returns
|
|
250
|
-
-------
|
|
251
|
-
dict
|
|
252
|
-
A list of dictionary of {"metadata_name": "metadata_value"}
|
|
253
|
-
|
|
254
|
-
"""
|
|
255
|
-
|
|
256
|
-
assert len(items) == 2, f"The number of elements in items " f"'{len(items)}' should be equal to two."
|
|
257
|
-
assert all([isinstance(item, int) for item in items]), f"All items elements '{items}' should be integers."
|
|
258
|
-
|
|
259
|
-
assert all([key[:5] == "pars_" for key in search_criteria.keys()]), (
|
|
260
|
-
f"All keys of the search_criteria dictionary " f"'{search_criteria.keys()}' must start with 'pars_'."
|
|
261
|
-
)
|
|
262
|
-
|
|
263
|
-
for key, value in search_criteria.items():
|
|
264
|
-
if value.split(";")[0] in ["integer", "decimal"]:
|
|
265
|
-
assert value.split(";")[1].split("|")[0] in OPERATOR_LIST, (
|
|
266
|
-
f"Search criteria of type '{value.split(';')[0]}' must "
|
|
267
|
-
f"include an operator ({', '.join(OPERATOR_LIST)})."
|
|
268
|
-
)
|
|
269
|
-
|
|
270
|
-
content = platform.parse_response(
|
|
271
|
-
platform.post(
|
|
272
|
-
self._account.auth,
|
|
273
|
-
"patient_manager/get_patient_list",
|
|
274
|
-
data=search_criteria,
|
|
275
|
-
headers={"X-Range": f"items={items[0]}-{items[1]}"},
|
|
276
|
-
)
|
|
277
|
-
)
|
|
278
|
-
return content
|
|
279
|
-
|
|
280
|
-
def get_subjects_files_metadata(self, search_criteria={}, items=(0, 9999)):
|
|
281
|
-
"""
|
|
282
|
-
List all Subject ID/Session ID from the selected project that meet the
|
|
283
|
-
defined search criteria at a file level.
|
|
284
|
-
|
|
285
|
-
Note, albeit the search criteria is similar to the one defined in
|
|
286
|
-
method 'get_subjects_metadata()' (see differences below), the
|
|
287
|
-
output is different as this method provides the sessions which
|
|
288
|
-
have a file that satisfy all the conditions of the search criteria.
|
|
289
|
-
This method is slow.
|
|
290
|
-
|
|
291
|
-
Parameters
|
|
292
|
-
----------
|
|
293
|
-
search_criteria: dict
|
|
294
|
-
Each element is a string and is built using the formatting
|
|
295
|
-
"type;value", or "type;operation|value"
|
|
296
|
-
|
|
297
|
-
Complete search_criteria Dictionary Explanation:
|
|
298
|
-
|
|
299
|
-
search_criteria = {
|
|
300
|
-
"pars_patient_secret_name": "string;SUBJECTID",
|
|
301
|
-
"pars_ssid": "integer;OPERATOR|SSID",
|
|
302
|
-
"pars_modalities": "string;MODALITY",
|
|
303
|
-
"pars_tags": "tags;TAGS",
|
|
304
|
-
"pars_age_at_scan": "integer;OPERATOR|AGE_AT_SCAN",
|
|
305
|
-
"pars_[dicom]_KEY": "KEYTYPE;KEYVALUE",
|
|
306
|
-
"pars_PROJECTMETADATA": "METADATATYPE;METADATAVALUE",
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
Where:
|
|
310
|
-
"pars_patient_secret_name": Applies the search to the 'Subject ID'.
|
|
311
|
-
SUBJECTID is a comma separated list of strings.
|
|
312
|
-
"pars_ssid": Applies the search to the 'Session ID'.
|
|
313
|
-
SSID is an integer.
|
|
314
|
-
OPERATOR is the operator to apply. One of:
|
|
315
|
-
- Equal: eq
|
|
316
|
-
- Different Than: ne
|
|
317
|
-
- Greater Than: gt
|
|
318
|
-
- Greater/Equal To: gte
|
|
319
|
-
- Lower Than: lt
|
|
320
|
-
- Lower/Equal To: lte
|
|
321
|
-
|
|
322
|
-
"pars_modalities": Applies the search to the file 'Modalities'
|
|
323
|
-
available within each Subject ID.
|
|
324
|
-
MODALITY is a string.
|
|
325
|
-
"pars_tags": Applies only the search to the file 'Tags' available
|
|
326
|
-
within each Subject ID.
|
|
327
|
-
TAGS is a comma separated list of strings. All tags must be present in
|
|
328
|
-
the same file.
|
|
329
|
-
"pars_age_at_scan": Applies the search to the 'age_at_scan' metadata
|
|
330
|
-
field.
|
|
331
|
-
AGE_AT_SCAN is an integer.
|
|
332
|
-
"pars_[dicom]_KEY": Applies the search to the metadata fields
|
|
333
|
-
available within each file. KEY must be one of the
|
|
334
|
-
metadata keys of the files. The full list of KEYS is shown above via
|
|
335
|
-
'file_m["metadata"]["info"].keys()'. # TODO: SEE HOW TO WRITE THIS
|
|
336
|
-
KEYTYPE is the type of the KEY. One of:
|
|
337
|
-
- integer
|
|
338
|
-
- decimal
|
|
339
|
-
- string
|
|
340
|
-
- list
|
|
341
|
-
|
|
342
|
-
if 'integer' or 'decimal' you must also include an OPERATOR
|
|
343
|
-
(i.e., "integer;OPERATOR|KEYVALUE").
|
|
344
|
-
if 'list' the KEYVALUE should be a semicolon separated list of
|
|
345
|
-
values. (i.e., "list;KEYVALUE1;KEYVALUE2;KEYVALUE3)
|
|
346
|
-
KEYVALUEs must be strings.
|
|
347
|
-
KEYVALUE is the expected value of the KEY.
|
|
348
|
-
"pars_[dicom]_PROJECTMETADATA": Applies to the metadata defined
|
|
349
|
-
within the 'Metadata Manager' of the project.
|
|
350
|
-
PROJECTMETADATA is the ID of the metadata field.
|
|
351
|
-
METADATATYPE is the type of the metadata field. One of:
|
|
352
|
-
- string
|
|
353
|
-
- integer
|
|
354
|
-
- list
|
|
355
|
-
- decimal
|
|
356
|
-
- single_option
|
|
357
|
-
- multiple_option
|
|
358
|
-
|
|
359
|
-
if 'integer' or 'decimal' you must also include an OPERATOR
|
|
360
|
-
(i.e., "integer;OPERATOR|METADATAVALUE").
|
|
361
|
-
KEYVALUE is the expected value of the metadata.
|
|
362
|
-
|
|
363
|
-
1) Example:
|
|
364
|
-
search_criteria = {
|
|
365
|
-
"pars_patient_secret_name": "string;abide",
|
|
366
|
-
"pars_ssid": "integer;eq|2"
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
2) Example:
|
|
370
|
-
search_criteria = {
|
|
371
|
-
"pars_patient_secret_name": "string;001"
|
|
372
|
-
"pars_modalities": "string;T2",
|
|
373
|
-
"pars_tags": "tags;flair",
|
|
374
|
-
"pars_[dicom]_Manufacturer": "string;ge",
|
|
375
|
-
"pars_[dicom]_FlipAngle": "integer;gt|5",
|
|
376
|
-
"pars_[dicom]_ImageType": "list;PRIMARY;SECONDARY",
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
Note the search criteria might apply to both the files metadata
|
|
380
|
-
information available within a session and the metadata of the
|
|
381
|
-
session. And the method provides a session only if all the file
|
|
382
|
-
related conditions are satisfied within the same file.
|
|
383
|
-
In example 2) above, it means that the output will contain any
|
|
384
|
-
session whose Subject ID contains '001', and there is a file with
|
|
385
|
-
modality 'T2', tag 'flair', FlipAngle greater than 5º, and
|
|
386
|
-
ImageType with both values PRIMARY and SECONDARY.
|
|
387
|
-
Further, the acquisition had to be performed in a Manufacturer
|
|
388
|
-
containing 'ge'.
|
|
389
|
-
|
|
390
|
-
Returns
|
|
391
|
-
-------
|
|
392
|
-
dict
|
|
393
|
-
A list of dictionary of {"metadata_name": "metadata_value"}
|
|
394
|
-
|
|
395
|
-
"""
|
|
396
|
-
|
|
397
|
-
content = self.get_subjects_metadata(search_criteria, items=(0, 9999))
|
|
398
|
-
|
|
399
|
-
# Wrap search criteria.
|
|
400
|
-
modality, tags, dicoms = self.__wrap_search_criteria(search_criteria)
|
|
401
|
-
|
|
402
|
-
# Iterate over the files of each subject selected to include/exclude
|
|
403
|
-
# them from the results.
|
|
404
|
-
subjects = list()
|
|
405
|
-
for subject in content:
|
|
406
|
-
files = platform.parse_response(platform.post(
|
|
407
|
-
self._account.auth, "file_manager/get_container_files",
|
|
408
|
-
data={"container_id": str(int(subject["container_id"]))}
|
|
409
|
-
))
|
|
410
|
-
|
|
411
|
-
for file in files["meta"]:
|
|
412
|
-
if modality and \
|
|
413
|
-
modality != (file.get("metadata") or {}).get("modality"):
|
|
414
|
-
continue
|
|
415
|
-
if tags and not all([tag in file.get("tags") for tag in tags]):
|
|
416
|
-
continue
|
|
417
|
-
if dicoms:
|
|
418
|
-
result_values = list()
|
|
419
|
-
for key, dict_value in dicoms.items():
|
|
420
|
-
f_value = ((file.get("metadata") or {})
|
|
421
|
-
.get("info") or {}).get(key)
|
|
422
|
-
d_operator = dict_value["operation"]
|
|
423
|
-
d_value = dict_value["value"]
|
|
424
|
-
result_values.append(
|
|
425
|
-
self.__operation(d_value, d_operator, f_value)
|
|
426
|
-
)
|
|
427
|
-
|
|
428
|
-
if not all(result_values):
|
|
429
|
-
continue
|
|
430
|
-
subjects.append(subject)
|
|
431
|
-
break
|
|
432
|
-
return subjects
|
|
433
|
-
|
|
434
|
-
@property
|
|
435
|
-
def subjects(self):
|
|
436
|
-
"""
|
|
437
|
-
Return the list of subject names (Subject ID) from the selected
|
|
438
|
-
project.
|
|
439
|
-
|
|
440
|
-
:return: a list of subject names
|
|
441
|
-
:rtype: List(Strings)
|
|
442
|
-
"""
|
|
443
|
-
|
|
444
|
-
subjects = self.subjects_metadata
|
|
445
|
-
names = [s["patient_secret_name"] for s in subjects]
|
|
446
|
-
return list(set(names))
|
|
447
|
-
|
|
448
|
-
def check_subject_name(self, subject_name):
|
|
449
|
-
"""
|
|
450
|
-
Check if a given subject name (Subject ID) exists in the selected
|
|
451
|
-
project.
|
|
452
|
-
|
|
453
|
-
Parameters
|
|
454
|
-
----------
|
|
455
|
-
subject_name : str
|
|
456
|
-
Subject ID of the subject to check
|
|
457
|
-
|
|
458
|
-
Returns
|
|
459
|
-
-------
|
|
460
|
-
bool
|
|
461
|
-
True if subject name exists in project, False otherwise
|
|
462
|
-
"""
|
|
463
|
-
|
|
464
|
-
return subject_name in self.subjects
|
|
465
|
-
|
|
466
|
-
@property
|
|
467
|
-
def metadata_parameters(self):
|
|
468
|
-
"""
|
|
469
|
-
List all the parameters in the subject-level metadata.
|
|
470
|
-
|
|
471
|
-
Each project has a set of parameters that define the subjects-level
|
|
472
|
-
metadata. This function returns all these parameters and its
|
|
473
|
-
properties. New subject-level metadata parameters can be creted in the
|
|
474
|
-
QMENTA Platform via the Metadata Manager. The API only allow
|
|
475
|
-
modification of these subject-level metadata parameters via the
|
|
476
|
-
'change_subject_metadata()' method.
|
|
477
|
-
|
|
478
|
-
Returns
|
|
479
|
-
-------
|
|
480
|
-
dict[str] -> dict[str] -> x
|
|
481
|
-
dictionary {"param_name":
|
|
482
|
-
{ "order": Int,
|
|
483
|
-
"tags": [tag1, tag2, ..., ],
|
|
484
|
-
"title: "Title",
|
|
485
|
-
"type": "integer|string|date|list|decimal",
|
|
486
|
-
"visible": 0|1
|
|
487
|
-
}}
|
|
488
|
-
"""
|
|
489
|
-
logger = logging.getLogger(logger_name)
|
|
490
|
-
try:
|
|
491
|
-
data = platform.parse_response(platform.post(self._account.auth, "patient_manager/module_config"))
|
|
492
|
-
except errors.PlatformError:
|
|
493
|
-
logger.error("Could not retrieve metadata parameters.")
|
|
494
|
-
return None
|
|
495
|
-
return data["fields"]
|
|
496
|
-
|
|
497
|
-
def get_analysis(self, analysis_name_or_id):
|
|
498
|
-
"""
|
|
499
|
-
Returns the analysis corresponding with the analysis id or analysis name
|
|
500
|
-
|
|
501
|
-
Parameters
|
|
502
|
-
----------
|
|
503
|
-
analysis_name_or_id : int, str
|
|
504
|
-
analysis_id or analysis name to search for
|
|
505
|
-
|
|
506
|
-
Returns
|
|
507
|
-
-------
|
|
508
|
-
analysis: dict
|
|
509
|
-
A dictionary containing the analysis information
|
|
510
|
-
"""
|
|
511
|
-
if isinstance(analysis_name_or_id, int):
|
|
512
|
-
search_tag = "id"
|
|
513
|
-
elif isinstance(analysis_name_or_id, str):
|
|
514
|
-
if analysis_name_or_id.isdigit():
|
|
515
|
-
search_tag = "id"
|
|
516
|
-
analysis_name_or_id = int(analysis_name_or_id)
|
|
517
|
-
else:
|
|
518
|
-
search_tag = "p_n"
|
|
519
|
-
else:
|
|
520
|
-
raise Exception("The analysis identifier must be its name or an " "integer")
|
|
521
|
-
|
|
522
|
-
search_condition = {
|
|
523
|
-
search_tag: analysis_name_or_id,
|
|
524
|
-
}
|
|
525
|
-
response = platform.parse_response(
|
|
526
|
-
platform.post(self._account.auth, "analysis_manager/get_analysis_list", data=search_condition)
|
|
527
|
-
)
|
|
528
|
-
|
|
529
|
-
if len(response) > 1:
|
|
530
|
-
raise Exception(f"multiple analyses with name " f"{analysis_name_or_id} found")
|
|
531
|
-
elif len(response) == 1:
|
|
532
|
-
return response[0]
|
|
533
|
-
else:
|
|
534
|
-
return None
|
|
535
|
-
|
|
536
|
-
def list_analysis(self, search_condition={}, items=(0, 9999)):
|
|
537
|
-
"""
|
|
538
|
-
List the analysis available to the user.\n
|
|
539
|
-
|
|
540
|
-
Examples
|
|
541
|
-
--------
|
|
542
|
-
|
|
543
|
-
>>> search_condition = {
|
|
544
|
-
"secret_name":"014_S_6920",
|
|
545
|
-
"from_d": "06.02.2025",
|
|
546
|
-
"with_child_analysis": 1,
|
|
547
|
-
"state": "completed"
|
|
548
|
-
}
|
|
549
|
-
list_analysis(search_condition=search_condition)
|
|
550
|
-
|
|
551
|
-
Parameters
|
|
552
|
-
----------
|
|
553
|
-
search_condition : dict
|
|
554
|
-
- p_n: str or None Analysis name
|
|
555
|
-
- type: str or None Type
|
|
556
|
-
- from_d: str or None dd.mm.yyyy Date from
|
|
557
|
-
- to_d: str or None dd.mm.yyyy Date to
|
|
558
|
-
- qa_status: str or None pass/fail/nd QC status
|
|
559
|
-
- secret_name: str or None Subject ID
|
|
560
|
-
- tags: str or None
|
|
561
|
-
- with_child_analysis: 1 or None if 1, child analysis of workflows will appear
|
|
562
|
-
- id: str or None ID
|
|
563
|
-
- state: running, completed, pending, exception or None
|
|
564
|
-
- username: str or None
|
|
565
|
-
|
|
566
|
-
items : List[int]
|
|
567
|
-
list containing two elements [min, max] that correspond to the
|
|
568
|
-
mininum and maximum range of analysis listed
|
|
569
|
-
|
|
570
|
-
Returns
|
|
571
|
-
-------
|
|
572
|
-
dict
|
|
573
|
-
List of analysis, each a dictionary
|
|
574
|
-
"""
|
|
575
|
-
assert len(items) == 2, f"The number of elements in items " f"'{len(items)}' should be equal to two."
|
|
576
|
-
assert all([isinstance(item, int) for item in items]), f"All items elements '{items}' should be integers."
|
|
577
|
-
search_keys = {
|
|
578
|
-
"p_n": str,
|
|
579
|
-
"type": str,
|
|
580
|
-
"from_d": str,
|
|
581
|
-
"to_d": str,
|
|
582
|
-
"qa_status": str,
|
|
583
|
-
"secret_name": str,
|
|
584
|
-
"tags": str,
|
|
585
|
-
"with_child_analysis": int,
|
|
586
|
-
"id": int,
|
|
587
|
-
"state": str,
|
|
588
|
-
"username": str,
|
|
589
|
-
}
|
|
590
|
-
for key in search_condition.keys():
|
|
591
|
-
if key not in search_keys.keys():
|
|
592
|
-
raise Exception((f"This key '{key}' is not accepted by this" "search condition"))
|
|
593
|
-
if not isinstance(search_condition[key], search_keys[key]) and search_condition[key] is not None:
|
|
594
|
-
raise Exception((f"The key {key} in the search condition" f"is not type {search_keys[key]}"))
|
|
595
|
-
req_headers = {"X-Range": f"items={items[0]}-{items[1] - 1}"}
|
|
596
|
-
return platform.parse_response(
|
|
597
|
-
platform.post(
|
|
598
|
-
auth=self._account.auth,
|
|
599
|
-
endpoint="analysis_manager/get_analysis_list",
|
|
600
|
-
headers=req_headers,
|
|
601
|
-
data=search_condition,
|
|
602
|
-
)
|
|
603
|
-
)
|
|
604
|
-
|
|
605
|
-
def get_subject_container_id(self, subject_name, ssid):
|
|
136
|
+
""" Upload / Download Data Related Methods """
|
|
137
|
+
def _upload_chunk(
|
|
138
|
+
self,
|
|
139
|
+
data,
|
|
140
|
+
range_str,
|
|
141
|
+
length,
|
|
142
|
+
session_id,
|
|
143
|
+
disposition,
|
|
144
|
+
last_chunk,
|
|
145
|
+
name="",
|
|
146
|
+
date_of_scan="",
|
|
147
|
+
description="",
|
|
148
|
+
subject_name="",
|
|
149
|
+
ssid="",
|
|
150
|
+
filename="DATA.zip",
|
|
151
|
+
input_data_type="mri_brain_data:1.0",
|
|
152
|
+
result=False,
|
|
153
|
+
add_to_container_id=0,
|
|
154
|
+
split_data=False,
|
|
155
|
+
):
|
|
606
156
|
"""
|
|
607
|
-
|
|
157
|
+
Upload a chunk of a file to the platform.
|
|
608
158
|
|
|
609
159
|
Parameters
|
|
610
160
|
----------
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
the
|
|
161
|
+
data
|
|
162
|
+
The file chunk to upload
|
|
163
|
+
range_str
|
|
164
|
+
The string to send that describes the content range
|
|
165
|
+
length
|
|
166
|
+
The content length of the chunk to send
|
|
167
|
+
session_id
|
|
168
|
+
The session ID from the file path
|
|
169
|
+
filename
|
|
170
|
+
The name of the file to be sent
|
|
171
|
+
disposition
|
|
172
|
+
The disposition of the content
|
|
173
|
+
last_chunk
|
|
174
|
+
Set this only for the last chunk to be uploaded.
|
|
175
|
+
All following parameters are ignored when False.
|
|
176
|
+
split_data
|
|
177
|
+
Sets the header that informs the platform to split
|
|
178
|
+
the uploaded file into multiple sessions.
|
|
621
179
|
"""
|
|
622
180
|
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
def list_input_containers(self, search_criteria={}, items=(0, 9999)):
|
|
632
|
-
"""
|
|
633
|
-
Retrieve the list of input containers available to the user under a
|
|
634
|
-
certain search criteria.
|
|
181
|
+
request_headers = {
|
|
182
|
+
"Content-Type": "application/zip",
|
|
183
|
+
"Content-Range": range_str,
|
|
184
|
+
"Session-ID": str(session_id),
|
|
185
|
+
"Content-Length": str(length),
|
|
186
|
+
"Content-Disposition": disposition,
|
|
187
|
+
}
|
|
635
188
|
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
189
|
+
if last_chunk:
|
|
190
|
+
request_headers["X-Mint-Name"] = name
|
|
191
|
+
request_headers["X-Mint-Date"] = date_of_scan
|
|
192
|
+
request_headers["X-Mint-Description"] = description
|
|
193
|
+
request_headers["X-Mint-Patient-Secret"] = subject_name
|
|
194
|
+
request_headers["X-Mint-SSID"] = ssid
|
|
195
|
+
request_headers["X-Mint-Filename"] = filename
|
|
196
|
+
request_headers["X-Mint-Project-Id"] = str(self._project_id)
|
|
197
|
+
request_headers["X-Mint-Split-Data"] = str(int(split_data))
|
|
641
198
|
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
s_n: subject_id
|
|
645
|
-
Subject ID of the subject in the platform.
|
|
646
|
-
ssid: session_id
|
|
647
|
-
Session ID of the subejct in the platform.
|
|
648
|
-
from_d: from date
|
|
649
|
-
Starting date in which perform the search. Format: DD.MM.YYYY
|
|
650
|
-
to_d: to date
|
|
651
|
-
End date in which perform the search. Format: DD.MM.YYYY
|
|
652
|
-
sets: data sets (modalities) # TODO: WHAT IS THIS???
|
|
199
|
+
if input_data_type:
|
|
200
|
+
request_headers["X-Mint-Type"] = input_data_type
|
|
653
201
|
|
|
654
|
-
|
|
655
|
-
|
|
202
|
+
if result:
|
|
203
|
+
request_headers["X-Mint-In-Out"] = "out"
|
|
204
|
+
else:
|
|
205
|
+
request_headers["X-Mint-In-Out"] = "in"
|
|
656
206
|
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
dict
|
|
660
|
-
List of containers, each a dictionary containing the following
|
|
661
|
-
information:
|
|
662
|
-
{"container_name", "container_id", "patient_secret_name", "ssid"}
|
|
663
|
-
"""
|
|
207
|
+
if add_to_container_id > 0:
|
|
208
|
+
request_headers["X-Mint-Add-To"] = str(add_to_container_id)
|
|
664
209
|
|
|
665
|
-
|
|
666
|
-
assert all([isinstance(item, int) for item in items]), f"All items elements '{items}' should be integers."
|
|
210
|
+
request_headers["X-Requested-With"] = "XMLHttpRequest"
|
|
667
211
|
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
"file_manager/get_container_list",
|
|
672
|
-
data=search_criteria,
|
|
673
|
-
headers={"X-Range": f"items={items[0]}-{items[1]}"},
|
|
674
|
-
)
|
|
212
|
+
response_time = 900.0 if last_chunk else 120.0
|
|
213
|
+
response = platform.post(
|
|
214
|
+
auth=self._account.auth, endpoint="upload", data=data, headers=request_headers, timeout=response_time
|
|
675
215
|
)
|
|
676
|
-
containers = [
|
|
677
|
-
{
|
|
678
|
-
"patient_secret_name": container_item["patient_secret_name"],
|
|
679
|
-
"container_name": container_item["name"],
|
|
680
|
-
"container_id": container_item["_id"],
|
|
681
|
-
"ssid": container_item["ssid"],
|
|
682
|
-
}
|
|
683
|
-
for container_item in response
|
|
684
|
-
]
|
|
685
|
-
return containers
|
|
686
|
-
|
|
687
|
-
def list_result_containers(self, search_condition={}, items=(0, 9999)):
|
|
688
|
-
"""
|
|
689
|
-
List the result containers available to the user.
|
|
690
|
-
Examples
|
|
691
|
-
--------
|
|
692
|
-
|
|
693
|
-
>>> search_condition = {
|
|
694
|
-
"secret_name":"014_S_6920",
|
|
695
|
-
"from_d": "06.02.2025",
|
|
696
|
-
"with_child_analysis": 1,
|
|
697
|
-
"state": "completed"
|
|
698
|
-
}
|
|
699
|
-
list_result_containers(search_condition=search_condition)
|
|
700
|
-
|
|
701
|
-
Parameters
|
|
702
|
-
----------
|
|
703
|
-
search_condition : dict
|
|
704
|
-
- p_n: str or None Analysis name
|
|
705
|
-
- type: str or None Type
|
|
706
|
-
- from_d: str or None dd.mm.yyyy Date from
|
|
707
|
-
- to_d: str or None dd.mm.yyyy Date to
|
|
708
|
-
- qa_status: str or None pass/fail/nd QC status
|
|
709
|
-
- secret_name: str or None Subject ID
|
|
710
|
-
- tags: str or None
|
|
711
|
-
- with_child_analysis: 1 or None if 1, child analysis of workflows will appear
|
|
712
|
-
- id: str or None ID
|
|
713
|
-
- state: running, completed, pending, exception or None
|
|
714
|
-
- username: str or None
|
|
715
|
-
items : List[int]
|
|
716
|
-
list containing two elements [min, max] that correspond to the
|
|
717
|
-
mininum and maximum range of analysis listed
|
|
718
216
|
|
|
719
|
-
|
|
720
|
-
-------
|
|
721
|
-
dict
|
|
722
|
-
List of containers, each a dictionary
|
|
723
|
-
{"name": "container-name", "id": "container_id"}
|
|
724
|
-
if "id": None, that analysis did not had an output container,
|
|
725
|
-
probably it is a workflow
|
|
726
|
-
"""
|
|
727
|
-
analyses = self.list_analysis(search_condition, items)
|
|
728
|
-
return [{"name": analysis["name"], "id": (analysis.get("out_container_id") or None)} for analysis in analyses]
|
|
217
|
+
return response
|
|
729
218
|
|
|
730
|
-
def
|
|
219
|
+
def upload_file(
|
|
731
220
|
self,
|
|
732
|
-
|
|
221
|
+
file_path,
|
|
222
|
+
subject_name,
|
|
223
|
+
ssid="",
|
|
224
|
+
date_of_scan="",
|
|
225
|
+
description="",
|
|
226
|
+
result=False,
|
|
227
|
+
name="",
|
|
228
|
+
input_data_type="qmenta_mri_brain_data:1.0",
|
|
229
|
+
add_to_container_id=0,
|
|
230
|
+
chunk_size=2**9,
|
|
231
|
+
split_data=False,
|
|
733
232
|
):
|
|
734
233
|
"""
|
|
735
|
-
|
|
234
|
+
Upload a ZIP file to the platform.
|
|
235
|
+
|
|
736
236
|
Parameters
|
|
737
237
|
----------
|
|
738
|
-
|
|
739
|
-
|
|
238
|
+
file_path : str
|
|
239
|
+
Path to the ZIP file to upload.
|
|
240
|
+
subject_name : str
|
|
241
|
+
Subject ID of the data to upload in the project in QMENTA Platform.
|
|
242
|
+
ssid : str
|
|
243
|
+
Session ID of the Subject ID (i.e., ID of the timepoint).
|
|
244
|
+
date_of_scan : str
|
|
245
|
+
Date of scan/creation of the file
|
|
246
|
+
description : str
|
|
247
|
+
Description of the file
|
|
248
|
+
result : bool
|
|
249
|
+
If result=True then the upload will be taken as an offline analysis
|
|
250
|
+
name : str
|
|
251
|
+
Name of the file in the platform
|
|
252
|
+
input_data_type : str
|
|
253
|
+
qmenta_medical_image_data:3.11
|
|
254
|
+
add_to_container_id : int
|
|
255
|
+
ID of the container to which this file should be added (if id > 0)
|
|
256
|
+
chunk_size : int
|
|
257
|
+
Size in kB of each chunk. Should be expressed as
|
|
258
|
+
a power of 2: 2**x. Default value of x is 9 (chunk_size = 512 kB)
|
|
259
|
+
split_data : bool
|
|
260
|
+
If True, the platform will try to split the uploaded file into
|
|
261
|
+
different sessions. It will be ignored when the ssid is given.
|
|
740
262
|
|
|
741
263
|
Returns
|
|
742
264
|
-------
|
|
743
|
-
|
|
744
|
-
|
|
265
|
+
bool
|
|
266
|
+
True if correctly uploaded, False otherwise.
|
|
745
267
|
"""
|
|
746
|
-
try:
|
|
747
|
-
content = platform.parse_response(
|
|
748
|
-
platform.post(
|
|
749
|
-
self._account.auth, "file_manager/get_container_files", data={"container_id": container_id}
|
|
750
|
-
)
|
|
751
|
-
)
|
|
752
|
-
except errors.PlatformError as e:
|
|
753
|
-
logging.getLogger(logger_name).error(e)
|
|
754
|
-
return False
|
|
755
|
-
if "files" not in content.keys():
|
|
756
|
-
logging.getLogger(logger_name).error("Could not get files")
|
|
757
|
-
return False
|
|
758
|
-
return content["files"]
|
|
759
268
|
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
container_id,
|
|
763
|
-
modality="",
|
|
764
|
-
metadata_info={},
|
|
765
|
-
tags=[]
|
|
766
|
-
):
|
|
767
|
-
"""
|
|
768
|
-
List the name of the files available inside a given container.
|
|
769
|
-
search condition example:
|
|
770
|
-
"metadata": {"SliceThickness":1},
|
|
771
|
-
}
|
|
772
|
-
Parameters
|
|
773
|
-
----------
|
|
774
|
-
container_id : str or int
|
|
775
|
-
Container identifier.
|
|
269
|
+
filename = os.path.split(file_path)[1]
|
|
270
|
+
input_data_type = "offline_analysis:1.0" if result else input_data_type
|
|
776
271
|
|
|
777
|
-
|
|
778
|
-
|
|
272
|
+
chunk_size *= 1024
|
|
273
|
+
max_retries = 10
|
|
779
274
|
|
|
780
|
-
|
|
781
|
-
Dictionary containing the file metadata of the files being filtered
|
|
275
|
+
name = name or os.path.split(file_path)[1]
|
|
782
276
|
|
|
783
|
-
|
|
784
|
-
List of strings containing the tags of the files being filtered
|
|
277
|
+
total_bytes = os.path.getsize(file_path)
|
|
785
278
|
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
279
|
+
# making chunks of the file and sending one by one
|
|
280
|
+
logger = logging.getLogger(logger_name)
|
|
281
|
+
with open(file_path, "rb") as file_object:
|
|
282
|
+
|
|
283
|
+
file_size = os.path.getsize(file_path)
|
|
284
|
+
if file_size == 0:
|
|
285
|
+
logger.error("Cannot upload empty file {}".format(file_path))
|
|
286
|
+
return False
|
|
287
|
+
uploaded = 0
|
|
288
|
+
session_id = self.__get_session_id(file_path)
|
|
289
|
+
chunk_num = 0
|
|
290
|
+
retries_count = 0
|
|
291
|
+
uploaded_bytes = 0
|
|
292
|
+
response = None
|
|
293
|
+
last_chunk = False
|
|
294
|
+
|
|
295
|
+
if ssid and split_data:
|
|
296
|
+
logger.warning("split-data argument will be ignored because" + " ssid has been specified")
|
|
297
|
+
split_data = False
|
|
298
|
+
|
|
299
|
+
while True:
|
|
300
|
+
data = file_object.read(chunk_size)
|
|
301
|
+
if not data:
|
|
302
|
+
break
|
|
303
|
+
|
|
304
|
+
start_position = chunk_num * chunk_size
|
|
305
|
+
end_position = start_position + chunk_size - 1
|
|
306
|
+
bytes_to_send = chunk_size
|
|
307
|
+
|
|
308
|
+
if end_position >= total_bytes:
|
|
309
|
+
last_chunk = True
|
|
310
|
+
end_position = total_bytes - 1
|
|
311
|
+
bytes_to_send = total_bytes - uploaded_bytes
|
|
312
|
+
|
|
313
|
+
bytes_range = "bytes " + str(start_position) + "-" + str(end_position) + "/" + str(total_bytes)
|
|
314
|
+
|
|
315
|
+
dispstr = f"attachment; filename={filename}"
|
|
316
|
+
response = self._upload_chunk(
|
|
317
|
+
data,
|
|
318
|
+
bytes_range,
|
|
319
|
+
bytes_to_send,
|
|
320
|
+
session_id,
|
|
321
|
+
dispstr,
|
|
322
|
+
last_chunk,
|
|
323
|
+
name,
|
|
324
|
+
date_of_scan,
|
|
325
|
+
description,
|
|
326
|
+
subject_name,
|
|
327
|
+
ssid,
|
|
328
|
+
filename,
|
|
329
|
+
input_data_type,
|
|
330
|
+
result,
|
|
331
|
+
add_to_container_id,
|
|
332
|
+
split_data,
|
|
804
333
|
)
|
|
805
|
-
for key in metadata_info.keys():
|
|
806
|
-
meta_key = (
|
|
807
|
-
(
|
|
808
|
-
metadata_file.get("metadata") or {}
|
|
809
|
-
).get("info") or {}).get(
|
|
810
|
-
key
|
|
811
|
-
)
|
|
812
|
-
if meta_key is None:
|
|
813
|
-
logging.getLogger(logger_name).warning(
|
|
814
|
-
f"{key} is not in file_info from file {file}"
|
|
815
|
-
)
|
|
816
|
-
info_bool.append(
|
|
817
|
-
metadata_info[key] == meta_key
|
|
818
|
-
)
|
|
819
|
-
if all(tags_bool) and all(info_bool) and modality_bool:
|
|
820
|
-
selected_files.append(file)
|
|
821
|
-
return selected_files
|
|
822
334
|
|
|
823
|
-
|
|
335
|
+
if response is None:
|
|
336
|
+
retries_count += 1
|
|
337
|
+
time.sleep(retries_count * 5)
|
|
338
|
+
if retries_count > max_retries:
|
|
339
|
+
error_message = "HTTP Connection Problem"
|
|
340
|
+
logger.error(error_message)
|
|
341
|
+
break
|
|
342
|
+
elif int(response.status_code) == 201:
|
|
343
|
+
chunk_num += 1
|
|
344
|
+
retries_count = 0
|
|
345
|
+
uploaded_bytes += chunk_size
|
|
346
|
+
elif int(response.status_code) == 200:
|
|
347
|
+
self.__show_progress(file_size, file_size, finish=True)
|
|
348
|
+
break
|
|
349
|
+
elif int(response.status_code) == 416:
|
|
350
|
+
retries_count += 1
|
|
351
|
+
time.sleep(retries_count * 5)
|
|
352
|
+
if retries_count > self.max_retries:
|
|
353
|
+
error_message = "Error Code: 416; " "Requested Range Not Satisfiable (NGINX)"
|
|
354
|
+
logger.error(error_message)
|
|
355
|
+
break
|
|
356
|
+
else:
|
|
357
|
+
retries_count += 1
|
|
358
|
+
time.sleep(retries_count * 5)
|
|
359
|
+
if retries_count > max_retries:
|
|
360
|
+
error_message = "Number of retries has been reached. " "Upload process stops here !"
|
|
361
|
+
logger.error(error_message)
|
|
362
|
+
break
|
|
363
|
+
|
|
364
|
+
uploaded += chunk_size
|
|
365
|
+
self.__show_progress(uploaded, file_size)
|
|
366
|
+
|
|
367
|
+
try:
|
|
368
|
+
platform.parse_response(response)
|
|
369
|
+
except errors.PlatformError as error:
|
|
370
|
+
logger.error(error)
|
|
371
|
+
return False
|
|
372
|
+
|
|
373
|
+
message = "Your data was successfully uploaded."
|
|
374
|
+
message += "The uploaded file will be soon processed !"
|
|
375
|
+
logger.info(message)
|
|
376
|
+
return True
|
|
377
|
+
|
|
378
|
+
def upload_mri(self, file_path, subject_name):
|
|
824
379
|
"""
|
|
825
|
-
|
|
380
|
+
Upload new MRI data to the subject.
|
|
826
381
|
|
|
827
382
|
Parameters
|
|
828
383
|
----------
|
|
829
|
-
|
|
830
|
-
|
|
384
|
+
file_path : str
|
|
385
|
+
Path to the file to upload
|
|
386
|
+
subject_name: str
|
|
831
387
|
|
|
832
388
|
Returns
|
|
833
389
|
-------
|
|
834
|
-
|
|
835
|
-
|
|
390
|
+
bool
|
|
391
|
+
True if upload was correctly done, False otherwise.
|
|
836
392
|
"""
|
|
837
393
|
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
platform.post(
|
|
841
|
-
self._account.auth, "file_manager/get_container_files", data={"container_id": container_id}
|
|
842
|
-
)
|
|
843
|
-
)
|
|
844
|
-
except errors.PlatformError as e:
|
|
845
|
-
logging.getLogger(logger_name).error(e)
|
|
846
|
-
return False
|
|
847
|
-
|
|
848
|
-
return data["meta"]
|
|
394
|
+
if self.__check_upload_file(file_path):
|
|
395
|
+
return self.upload_file(file_path, subject_name)
|
|
849
396
|
|
|
850
|
-
def
|
|
397
|
+
def upload_gametection(self, file_path, subject_name):
|
|
851
398
|
"""
|
|
852
|
-
|
|
399
|
+
Upload new Gametection data to the subject.
|
|
853
400
|
|
|
854
401
|
Parameters
|
|
855
402
|
----------
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
Name of the file.
|
|
403
|
+
file_path : str
|
|
404
|
+
Path to the file to upload
|
|
405
|
+
subject_name: str
|
|
860
406
|
|
|
861
407
|
Returns
|
|
862
408
|
-------
|
|
863
|
-
|
|
864
|
-
|
|
409
|
+
bool
|
|
410
|
+
True if upload was correctly done, False otherwise.
|
|
865
411
|
"""
|
|
866
|
-
all_metadata = self.list_container_files_metadata(container_id)
|
|
867
|
-
if all_metadata:
|
|
868
|
-
for file_meta in all_metadata:
|
|
869
|
-
if file_meta["name"] == filename:
|
|
870
|
-
return file_meta
|
|
871
|
-
else:
|
|
872
|
-
return False
|
|
873
412
|
|
|
874
|
-
|
|
413
|
+
if self.__check_upload_file(file_path):
|
|
414
|
+
return self.upload_file(file_path, subject_name, input_data_type="parkinson_gametection")
|
|
415
|
+
return False
|
|
416
|
+
|
|
417
|
+
def upload_result(self, file_path, subject_name):
|
|
875
418
|
"""
|
|
876
|
-
|
|
419
|
+
Upload new result data to the subject.
|
|
877
420
|
|
|
878
421
|
Parameters
|
|
879
422
|
----------
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
List of tags, or None if the filename shouldn't have any tags
|
|
423
|
+
file_path : str
|
|
424
|
+
Path to the file to upload
|
|
425
|
+
subject_name: str
|
|
426
|
+
|
|
427
|
+
Returns
|
|
428
|
+
-------
|
|
429
|
+
bool
|
|
430
|
+
True if upload was correctly done, False otherwise.
|
|
889
431
|
"""
|
|
890
432
|
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
self._account.auth,
|
|
895
|
-
"file_manager/edit_file",
|
|
896
|
-
data={"container_id": container_id, "filename": filename, "tags": tags_str, "modality": modality},
|
|
897
|
-
)
|
|
898
|
-
)
|
|
433
|
+
if self.__check_upload_file(file_path):
|
|
434
|
+
return self.upload_file(file_path, subject_name, result=True)
|
|
435
|
+
return False
|
|
899
436
|
|
|
900
437
|
def download_file(self, container_id, file_name, local_filename=False, overwrite=False):
|
|
901
438
|
"""
|
|
@@ -913,6 +450,13 @@ class Project:
|
|
|
913
450
|
Whether to overwrite the file if existing.
|
|
914
451
|
"""
|
|
915
452
|
logger = logging.getLogger(logger_name)
|
|
453
|
+
if not isinstance(file_name, str):
|
|
454
|
+
raise ValueError("The name of the file to download (file_name) "
|
|
455
|
+
"should be of type string.")
|
|
456
|
+
if not isinstance(file_name, str):
|
|
457
|
+
raise ValueError("The name of the output file (local_filename) "
|
|
458
|
+
"should be of type string.")
|
|
459
|
+
|
|
916
460
|
if file_name not in self.list_container_files(container_id):
|
|
917
461
|
msg = f'File "{file_name}" does not exist in container ' f"{container_id}"
|
|
918
462
|
logger.error(msg)
|
|
@@ -954,6 +498,14 @@ class Project:
|
|
|
954
498
|
Name of the zip where the downloaded files are stored.
|
|
955
499
|
"""
|
|
956
500
|
logger = logging.getLogger(logger_name)
|
|
501
|
+
|
|
502
|
+
if not all([isinstance(file_name, str) for file_name in filenames]):
|
|
503
|
+
raise ValueError("The name of the files to download (filenames) "
|
|
504
|
+
"should be of type string.")
|
|
505
|
+
if not isinstance(zip_name, str):
|
|
506
|
+
raise ValueError("The name of the output ZIP file (zip_name) "
|
|
507
|
+
"should be of type string.")
|
|
508
|
+
|
|
957
509
|
files_not_in_container = list(filter(lambda f: f not in self.list_container_files(container_id), filenames))
|
|
958
510
|
|
|
959
511
|
if files_not_in_container:
|
|
@@ -973,36 +525,314 @@ class Project:
|
|
|
973
525
|
self._account.auth, "file_manager/download_file", data=params, stream=True
|
|
974
526
|
) as response, open(zip_name, "wb") as f:
|
|
975
527
|
|
|
976
|
-
for chunk in response.iter_content(chunk_size=2**9 * 1024):
|
|
977
|
-
f.write(chunk)
|
|
978
|
-
f.flush()
|
|
528
|
+
for chunk in response.iter_content(chunk_size=2**9 * 1024):
|
|
529
|
+
f.write(chunk)
|
|
530
|
+
f.flush()
|
|
531
|
+
|
|
532
|
+
logger.info("Files from container {} saved to {}".format(container_id, zip_name))
|
|
533
|
+
return True
|
|
534
|
+
|
|
535
|
+
def copy_container_to_project(self, container_id, project_id):
|
|
536
|
+
"""
|
|
537
|
+
Copy a container to another project.
|
|
538
|
+
|
|
539
|
+
Parameters
|
|
540
|
+
----------
|
|
541
|
+
container_id : int
|
|
542
|
+
ID of the container to copy.
|
|
543
|
+
project_id : int or str
|
|
544
|
+
ID of the project to retrieve, either the numeric ID or the name
|
|
545
|
+
|
|
546
|
+
Returns
|
|
547
|
+
-------
|
|
548
|
+
bool
|
|
549
|
+
True on success, False on fail
|
|
550
|
+
"""
|
|
551
|
+
|
|
552
|
+
if type(project_id) == int or type(project_id) == float:
|
|
553
|
+
p_id = int(project_id)
|
|
554
|
+
elif type(project_id) == str:
|
|
555
|
+
projects = self._account.projects
|
|
556
|
+
projects_match = [proj for proj in projects if proj["name"] == project_id]
|
|
557
|
+
if not projects_match:
|
|
558
|
+
raise Exception(f"Project {project_id}" + " does not exist or is not available for this user.")
|
|
559
|
+
p_id = int(projects_match[0]["id"])
|
|
560
|
+
else:
|
|
561
|
+
raise TypeError("project_id")
|
|
562
|
+
data = {
|
|
563
|
+
"container_id": container_id,
|
|
564
|
+
"project_id": p_id,
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
try:
|
|
568
|
+
platform.parse_response(
|
|
569
|
+
platform.post(self._account.auth, "file_manager/copy_container_to_another_project", data=data)
|
|
570
|
+
)
|
|
571
|
+
except errors.PlatformError as e:
|
|
572
|
+
logging.getLogger(logger_name).error("Couldn not copy container: {}".format(e))
|
|
573
|
+
return False
|
|
574
|
+
|
|
575
|
+
return True
|
|
576
|
+
|
|
577
|
+
""" Subject/Session Related Methods """
|
|
578
|
+
@property
|
|
579
|
+
def subjects(self):
|
|
580
|
+
"""
|
|
581
|
+
Return the list of subject names (Subject ID) from the selected
|
|
582
|
+
project.
|
|
583
|
+
|
|
584
|
+
:return: a list of subject names
|
|
585
|
+
:rtype: List(Strings)
|
|
586
|
+
"""
|
|
587
|
+
|
|
588
|
+
subjects = self.subjects_metadata
|
|
589
|
+
names = [s["patient_secret_name"] for s in subjects]
|
|
590
|
+
return list(set(names))
|
|
591
|
+
|
|
592
|
+
@property
|
|
593
|
+
def subjects_metadata(self):
|
|
594
|
+
"""
|
|
595
|
+
List all subject data from the selected project.
|
|
596
|
+
|
|
597
|
+
Returns
|
|
598
|
+
-------
|
|
599
|
+
dict
|
|
600
|
+
A list of dictionary of {"metadata_name": "metadata_value"}
|
|
601
|
+
"""
|
|
602
|
+
return self.get_subjects_metadata()
|
|
603
|
+
|
|
604
|
+
@property
|
|
605
|
+
def metadata_parameters(self):
|
|
606
|
+
"""
|
|
607
|
+
List all the parameters in the subject-level metadata.
|
|
608
|
+
|
|
609
|
+
Each project has a set of parameters that define the subjects-level
|
|
610
|
+
metadata. This function returns all these parameters and its
|
|
611
|
+
properties. New subject-level metadata parameters can be creted in the
|
|
612
|
+
QMENTA Platform via the Metadata Manager. The API only allow
|
|
613
|
+
modification of these subject-level metadata parameters via the
|
|
614
|
+
'change_subject_metadata()' method.
|
|
615
|
+
|
|
616
|
+
Returns
|
|
617
|
+
-------
|
|
618
|
+
dict[str] -> dict[str] -> x
|
|
619
|
+
dictionary {"param_name":
|
|
620
|
+
{ "order": Int,
|
|
621
|
+
"tags": [tag1, tag2, ..., ],
|
|
622
|
+
"title: "Title",
|
|
623
|
+
"type": "integer|string|date|list|decimal",
|
|
624
|
+
"visible": 0|1
|
|
625
|
+
}}
|
|
626
|
+
"""
|
|
627
|
+
logger = logging.getLogger(logger_name)
|
|
628
|
+
try:
|
|
629
|
+
data = platform.parse_response(platform.post(self._account.auth, "patient_manager/module_config"))
|
|
630
|
+
except errors.PlatformError:
|
|
631
|
+
logger.error("Could not retrieve metadata parameters.")
|
|
632
|
+
return None
|
|
633
|
+
return data["fields"]
|
|
634
|
+
|
|
635
|
+
def check_subject_name(self, subject_name):
|
|
636
|
+
"""
|
|
637
|
+
Check if a given subject name (Subject ID) exists in the selected
|
|
638
|
+
project.
|
|
639
|
+
|
|
640
|
+
Parameters
|
|
641
|
+
----------
|
|
642
|
+
subject_name : str
|
|
643
|
+
Subject ID of the subject to check
|
|
644
|
+
|
|
645
|
+
Returns
|
|
646
|
+
-------
|
|
647
|
+
bool
|
|
648
|
+
True if subject name exists in project, False otherwise
|
|
649
|
+
"""
|
|
650
|
+
|
|
651
|
+
return subject_name in self.subjects
|
|
652
|
+
|
|
653
|
+
def get_subject_container_id(self, subject_name, ssid):
|
|
654
|
+
"""
|
|
655
|
+
Given a Subject ID and Session ID, return its Container ID.
|
|
656
|
+
|
|
657
|
+
Parameters
|
|
658
|
+
----------
|
|
659
|
+
subject_name : str
|
|
660
|
+
Subject ID of the subject in the project.
|
|
661
|
+
ssid : str
|
|
662
|
+
Session ID of the subject in the project.
|
|
663
|
+
|
|
664
|
+
Returns
|
|
665
|
+
-------
|
|
666
|
+
int or bool
|
|
667
|
+
The Container ID of the subject in the project, or False if
|
|
668
|
+
the subject is not found.
|
|
669
|
+
"""
|
|
670
|
+
|
|
671
|
+
search_criteria = {"s_n": subject_name, "ssid": ssid}
|
|
672
|
+
response = self.list_input_containers(search_criteria=search_criteria)
|
|
673
|
+
|
|
674
|
+
for subject in response:
|
|
675
|
+
if subject["patient_secret_name"] == subject_name and subject["ssid"] == ssid:
|
|
676
|
+
return subject["container_id"]
|
|
677
|
+
return False
|
|
678
|
+
|
|
679
|
+
def get_subject_id(self, subject_name, ssid):
|
|
680
|
+
"""
|
|
681
|
+
Given a Subject ID and Session ID, return its Patient ID in the
|
|
682
|
+
project.
|
|
683
|
+
|
|
684
|
+
Parameters
|
|
685
|
+
----------
|
|
686
|
+
subject_name : str
|
|
687
|
+
Subject ID of the subject in the project.
|
|
688
|
+
ssid : str
|
|
689
|
+
Session ID of the subject in the project.
|
|
690
|
+
|
|
691
|
+
Returns
|
|
692
|
+
-------
|
|
693
|
+
int or bool
|
|
694
|
+
The ID of the subject in the project, or False if
|
|
695
|
+
the subject is not found.
|
|
696
|
+
"""
|
|
697
|
+
|
|
698
|
+
for user in self.get_subjects_metadata():
|
|
699
|
+
if user["patient_secret_name"] == str(subject_name) and user["ssid"] == str(ssid):
|
|
700
|
+
return int(user["_id"])
|
|
701
|
+
return False
|
|
702
|
+
|
|
703
|
+
def get_subjects_metadata(self, search_criteria={}, items=(0, 9999)):
|
|
704
|
+
"""
|
|
705
|
+
List all Subject ID/Session ID from the selected project that meet the
|
|
706
|
+
defined search criteria at a session level.
|
|
707
|
+
|
|
708
|
+
Parameters
|
|
709
|
+
----------
|
|
710
|
+
search_criteria: dict
|
|
711
|
+
Each element is a string and is built using the formatting
|
|
712
|
+
"type;value", or "type;operation|value"
|
|
713
|
+
|
|
714
|
+
Complete search_criteria Dictionary Explanation:
|
|
715
|
+
|
|
716
|
+
search_criteria = {
|
|
717
|
+
"pars_patient_secret_name": "string;SUBJECTID",
|
|
718
|
+
"pars_ssid": "integer;OPERATOR|SSID",
|
|
719
|
+
"pars_modalities": "string;MODALITY",
|
|
720
|
+
"pars_tags": "tags;TAGS",
|
|
721
|
+
"pars_age_at_scan": "integer;OPERATOR|AGE_AT_SCAN",
|
|
722
|
+
"pars_[dicom]_KEY": "KEYTYPE;KEYVALUE",
|
|
723
|
+
"pars_PROJECTMETADATA": "METADATATYPE;METADATAVALUE",
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
Where:
|
|
727
|
+
"pars_patient_secret_name": Applies the search to the 'Subject ID'.
|
|
728
|
+
SUBJECTID is a comma separated list of strings.
|
|
729
|
+
"pars_ssid": Applies the search to the 'Session ID'.
|
|
730
|
+
SSID is an integer.
|
|
731
|
+
OPERATOR is the operator to apply. One of:
|
|
732
|
+
- Equal: eq
|
|
733
|
+
- Different Than: ne
|
|
734
|
+
- Greater Than: gt
|
|
735
|
+
- Greater/Equal To: gte
|
|
736
|
+
- Lower Than: lt
|
|
737
|
+
- Lower/Equal To: lte
|
|
738
|
+
|
|
739
|
+
"pars_modalities": Applies the search to the file 'Modalities'
|
|
740
|
+
available within each Subject ID.
|
|
741
|
+
MODALITY is a comma separated list of string. A session is provided as
|
|
742
|
+
long as one MODALITY is available.
|
|
743
|
+
"pars_tags": Applies the search to the file 'Tags' available within
|
|
744
|
+
each Subject ID and to the subject-level 'Tags'.
|
|
745
|
+
TAGS is a comma separated list of strings. A session is provided as
|
|
746
|
+
long as one tag is available.
|
|
747
|
+
"pars_age_at_scan": Applies the search to the 'age_at_scan' metadata
|
|
748
|
+
field.
|
|
749
|
+
AGE_AT_SCAN is an integer.
|
|
750
|
+
"pars_[dicom]_KEY": Applies the search to the metadata fields
|
|
751
|
+
available within each file. KEY must be one of the
|
|
752
|
+
metadata keys of the files. The full list of KEYS for a given
|
|
753
|
+
file is shown in the QMENTA Platform within the File Information
|
|
754
|
+
of such File.
|
|
755
|
+
KEYTYPE is the type of the KEY. One of:
|
|
756
|
+
- integer
|
|
757
|
+
- decimal
|
|
758
|
+
- string
|
|
759
|
+
- list
|
|
760
|
+
|
|
761
|
+
if 'integer' or 'decimal' you must also include an OPERATOR
|
|
762
|
+
(i.e., "integer;OPERATOR|KEYVALUE").
|
|
763
|
+
if 'list' the KEYVALUE should be a semicolon separated list of
|
|
764
|
+
values. (i.e., "list;KEYVALUE1;KEYVALUE2;KEYVALUE3)
|
|
765
|
+
KEYVALUEs must be strings.
|
|
766
|
+
KEYVALUE is the expected value of the KEY.
|
|
767
|
+
"pars_[dicom]_PROJECTMETADATA": Applies to the metadata defined
|
|
768
|
+
within the 'Metadata Manager' of the project.
|
|
769
|
+
PROJECTMETADATA is the ID of the metadata field.
|
|
770
|
+
METADATATYPE is the type of the metadata field. One of:
|
|
771
|
+
- string
|
|
772
|
+
- integer
|
|
773
|
+
- list
|
|
774
|
+
- decimal
|
|
775
|
+
- single_option
|
|
776
|
+
- multiple_option
|
|
777
|
+
|
|
778
|
+
if 'integer' or 'decimal' you must also include an OPERATOR
|
|
779
|
+
(i.e., "integer;OPERATOR|METADATAVALUE").
|
|
780
|
+
KEYVALUE is the expected value of the metadata.
|
|
979
781
|
|
|
980
|
-
|
|
981
|
-
|
|
782
|
+
1) Example:
|
|
783
|
+
search_criteria = {
|
|
784
|
+
"pars_patient_secret_name": "string;abide",
|
|
785
|
+
"pars_ssid": "integer;eq|2"
|
|
786
|
+
}
|
|
982
787
|
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
788
|
+
2) Example:
|
|
789
|
+
search_criteria = {
|
|
790
|
+
"pars_modalities": "string;T1",
|
|
791
|
+
"pars_tags": "tags;flair",
|
|
792
|
+
"pars_[dicom]_Manufacturer": "string;ge",
|
|
793
|
+
"pars_[dicom]_FlipAngle": "integer;gt|5",
|
|
794
|
+
"pars_[dicom]_ImageType": "list;PRIMARY;SECONDARY",
|
|
795
|
+
}
|
|
987
796
|
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
Subject ID
|
|
992
|
-
|
|
993
|
-
|
|
797
|
+
Note the search criteria applies to all the files included within a
|
|
798
|
+
session. Hence, it does not imply that all the criteria conditions
|
|
799
|
+
are applied to the same files. In example 2) above, it means that
|
|
800
|
+
any Subject ID/Session ID that has a file classified with a 'T1'
|
|
801
|
+
modality, a file with a 'flair' tag, a file whose Manufacturer
|
|
802
|
+
contains 'ge', a file whose FlipAngle is greater than '5º', and a
|
|
803
|
+
file with ImageType with any of the values: PRIMARY or SECONDARY
|
|
804
|
+
will be selected.
|
|
994
805
|
|
|
995
806
|
Returns
|
|
996
807
|
-------
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
808
|
+
dict
|
|
809
|
+
A list of dictionary of {"metadata_name": "metadata_value"}
|
|
810
|
+
|
|
1000
811
|
"""
|
|
1001
812
|
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
813
|
+
assert len(items) == 2, f"The number of elements in items " f"'{len(items)}' should be equal to two."
|
|
814
|
+
assert all([isinstance(item, int) for item in items]), f"All items elements '{items}' should be integers."
|
|
815
|
+
|
|
816
|
+
assert all([key[:5] == "pars_" for key in search_criteria.keys()]), (
|
|
817
|
+
f"All keys of the search_criteria dictionary " f"'{search_criteria.keys()}' must start with 'pars_'."
|
|
818
|
+
)
|
|
819
|
+
|
|
820
|
+
for key, value in search_criteria.items():
|
|
821
|
+
if value.split(";")[0] in ["integer", "decimal"]:
|
|
822
|
+
assert value.split(";")[1].split("|")[0] in OPERATOR_LIST, (
|
|
823
|
+
f"Search criteria of type '{value.split(';')[0]}' must "
|
|
824
|
+
f"include an operator ({', '.join(OPERATOR_LIST)})."
|
|
825
|
+
)
|
|
826
|
+
|
|
827
|
+
content = platform.parse_response(
|
|
828
|
+
platform.post(
|
|
829
|
+
self._account.auth,
|
|
830
|
+
"patient_manager/get_patient_list",
|
|
831
|
+
data=search_criteria,
|
|
832
|
+
headers={"X-Range": f"items={items[0]}-{items[1]}"},
|
|
833
|
+
)
|
|
834
|
+
)
|
|
835
|
+
return content
|
|
1006
836
|
|
|
1007
837
|
def change_subject_metadata(self, patient_id, subject_name, ssid, tags, age_at_scan, metadata):
|
|
1008
838
|
"""
|
|
@@ -1087,8 +917,213 @@ class Project:
|
|
|
1087
917
|
logger.error(f"Patient ID '{patient_id}' could not be modified.")
|
|
1088
918
|
return False
|
|
1089
919
|
|
|
1090
|
-
logger.info(f"Patient ID '{patient_id}' successfully modified.")
|
|
1091
|
-
return True
|
|
920
|
+
logger.info(f"Patient ID '{patient_id}' successfully modified.")
|
|
921
|
+
return True
|
|
922
|
+
|
|
923
|
+
def get_subjects_files_metadata(self, search_criteria={}, items=(0, 9999)):
|
|
924
|
+
"""
|
|
925
|
+
List all Subject ID/Session ID from the selected project that meet the
|
|
926
|
+
defined search criteria at a file level.
|
|
927
|
+
|
|
928
|
+
Note, albeit the search criteria is similar to the one defined in
|
|
929
|
+
method 'get_subjects_metadata()' (see differences below), the
|
|
930
|
+
output is different as this method provides the sessions which
|
|
931
|
+
have a file that satisfy all the conditions of the search criteria.
|
|
932
|
+
This method is slow.
|
|
933
|
+
|
|
934
|
+
Parameters
|
|
935
|
+
----------
|
|
936
|
+
search_criteria: dict
|
|
937
|
+
Each element is a string and is built using the formatting
|
|
938
|
+
"type;value", or "type;operation|value"
|
|
939
|
+
|
|
940
|
+
Complete search_criteria Dictionary Explanation:
|
|
941
|
+
|
|
942
|
+
search_criteria = {
|
|
943
|
+
"pars_patient_secret_name": "string;SUBJECTID",
|
|
944
|
+
"pars_ssid": "integer;OPERATOR|SSID",
|
|
945
|
+
"pars_modalities": "string;MODALITY",
|
|
946
|
+
"pars_tags": "tags;TAGS",
|
|
947
|
+
"pars_age_at_scan": "integer;OPERATOR|AGE_AT_SCAN",
|
|
948
|
+
"pars_[dicom]_KEY": "KEYTYPE;KEYVALUE",
|
|
949
|
+
"pars_PROJECTMETADATA": "METADATATYPE;METADATAVALUE",
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
Where:
|
|
953
|
+
"pars_patient_secret_name": Applies the search to the 'Subject ID'.
|
|
954
|
+
SUBJECTID is a comma separated list of strings.
|
|
955
|
+
"pars_ssid": Applies the search to the 'Session ID'.
|
|
956
|
+
SSID is an integer.
|
|
957
|
+
OPERATOR is the operator to apply. One of:
|
|
958
|
+
- Equal: eq
|
|
959
|
+
- Different Than: ne
|
|
960
|
+
- Greater Than: gt
|
|
961
|
+
- Greater/Equal To: gte
|
|
962
|
+
- Lower Than: lt
|
|
963
|
+
- Lower/Equal To: lte
|
|
964
|
+
|
|
965
|
+
"pars_modalities": Applies the search to the file 'Modalities'
|
|
966
|
+
available within each Subject ID.
|
|
967
|
+
MODALITY is a string.
|
|
968
|
+
"pars_tags": Applies only the search to the file 'Tags' available
|
|
969
|
+
within each Subject ID.
|
|
970
|
+
TAGS is a comma separated list of strings. All tags must be present in
|
|
971
|
+
the same file.
|
|
972
|
+
"pars_age_at_scan": Applies the search to the 'age_at_scan' metadata
|
|
973
|
+
field.
|
|
974
|
+
AGE_AT_SCAN is an integer.
|
|
975
|
+
"pars_[dicom]_KEY": Applies the search to the metadata fields
|
|
976
|
+
available within each file. KEY must be one of the
|
|
977
|
+
metadata keys of the files. The full list of KEYS for a given
|
|
978
|
+
file is shown in the QMENTA Platform within the File Information
|
|
979
|
+
of such File.
|
|
980
|
+
KEYTYPE is the type of the KEY. One of:
|
|
981
|
+
- integer
|
|
982
|
+
- decimal
|
|
983
|
+
- string
|
|
984
|
+
- list
|
|
985
|
+
|
|
986
|
+
if 'integer' or 'decimal' you must also include an OPERATOR
|
|
987
|
+
(i.e., "integer;OPERATOR|KEYVALUE").
|
|
988
|
+
if 'list' the KEYVALUE should be a semicolon separated list of
|
|
989
|
+
values. (i.e., "list;KEYVALUE1;KEYVALUE2;KEYVALUE3)
|
|
990
|
+
KEYVALUEs must be strings.
|
|
991
|
+
KEYVALUE is the expected value of the KEY.
|
|
992
|
+
"pars_[dicom]_PROJECTMETADATA": Applies to the metadata defined
|
|
993
|
+
within the 'Metadata Manager' of the project.
|
|
994
|
+
PROJECTMETADATA is the ID of the metadata field.
|
|
995
|
+
METADATATYPE is the type of the metadata field. One of:
|
|
996
|
+
- string
|
|
997
|
+
- integer
|
|
998
|
+
- list
|
|
999
|
+
- decimal
|
|
1000
|
+
- single_option
|
|
1001
|
+
- multiple_option
|
|
1002
|
+
|
|
1003
|
+
if 'integer' or 'decimal' you must also include an OPERATOR
|
|
1004
|
+
(i.e., "integer;OPERATOR|METADATAVALUE").
|
|
1005
|
+
KEYVALUE is the expected value of the metadata.
|
|
1006
|
+
|
|
1007
|
+
1) Example:
|
|
1008
|
+
search_criteria = {
|
|
1009
|
+
"pars_patient_secret_name": "string;abide",
|
|
1010
|
+
"pars_ssid": "integer;eq|2"
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
2) Example:
|
|
1014
|
+
search_criteria = {
|
|
1015
|
+
"pars_patient_secret_name": "string;001"
|
|
1016
|
+
"pars_modalities": "string;T2",
|
|
1017
|
+
"pars_tags": "tags;flair",
|
|
1018
|
+
"pars_[dicom]_Manufacturer": "string;ge",
|
|
1019
|
+
"pars_[dicom]_FlipAngle": "integer;gt|5",
|
|
1020
|
+
"pars_[dicom]_ImageType": "list;PRIMARY;SECONDARY",
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
Note the search criteria might apply to both the files metadata
|
|
1024
|
+
information available within a session and the metadata of the
|
|
1025
|
+
session. And the method provides a session only if all the file
|
|
1026
|
+
related conditions are satisfied within the same file.
|
|
1027
|
+
In example 2) above, it means that the output will contain any
|
|
1028
|
+
session whose Subject ID contains '001', and there is a file with
|
|
1029
|
+
modality 'T2', tag 'flair', FlipAngle greater than 5º, and
|
|
1030
|
+
ImageType with both values PRIMARY and SECONDARY.
|
|
1031
|
+
Further, the acquisition had to be performed in a Manufacturer
|
|
1032
|
+
containing 'ge'.
|
|
1033
|
+
|
|
1034
|
+
Returns
|
|
1035
|
+
-------
|
|
1036
|
+
dict
|
|
1037
|
+
A list of dictionary of {"metadata_name": "metadata_value"}
|
|
1038
|
+
|
|
1039
|
+
"""
|
|
1040
|
+
|
|
1041
|
+
content = self.get_subjects_metadata(search_criteria, items=(0, 9999))
|
|
1042
|
+
|
|
1043
|
+
# Wrap search criteria.
|
|
1044
|
+
modality, tags, dicoms = self.__wrap_search_criteria(search_criteria)
|
|
1045
|
+
|
|
1046
|
+
# Iterate over the files of each subject selected to include/exclude
|
|
1047
|
+
# them from the results.
|
|
1048
|
+
subjects = list()
|
|
1049
|
+
for subject in content:
|
|
1050
|
+
files = platform.parse_response(platform.post(
|
|
1051
|
+
self._account.auth, "file_manager/get_container_files",
|
|
1052
|
+
data={"container_id": str(int(subject["container_id"]))}
|
|
1053
|
+
))
|
|
1054
|
+
|
|
1055
|
+
for file in files["meta"]:
|
|
1056
|
+
if modality and \
|
|
1057
|
+
modality != (file.get("metadata") or {}).get("modality"):
|
|
1058
|
+
continue
|
|
1059
|
+
if tags and not all([tag in file.get("tags") for tag in tags]):
|
|
1060
|
+
continue
|
|
1061
|
+
if dicoms:
|
|
1062
|
+
result_values = list()
|
|
1063
|
+
for key, dict_value in dicoms.items():
|
|
1064
|
+
f_value = ((file.get("metadata") or {})
|
|
1065
|
+
.get("info") or {}).get(key)
|
|
1066
|
+
d_operator = dict_value["operation"]
|
|
1067
|
+
d_value = dict_value["value"]
|
|
1068
|
+
result_values.append(
|
|
1069
|
+
self.__operation(d_value, d_operator, f_value)
|
|
1070
|
+
)
|
|
1071
|
+
|
|
1072
|
+
if not all(result_values):
|
|
1073
|
+
continue
|
|
1074
|
+
subjects.append(subject)
|
|
1075
|
+
break
|
|
1076
|
+
return subjects
|
|
1077
|
+
|
|
1078
|
+
def get_file_metadata(self, container_id, filename):
|
|
1079
|
+
"""
|
|
1080
|
+
Retrieve the metadata from a particular file in a particular container.
|
|
1081
|
+
|
|
1082
|
+
Parameters
|
|
1083
|
+
----------
|
|
1084
|
+
container_id : str
|
|
1085
|
+
Container identifier.
|
|
1086
|
+
filename : str
|
|
1087
|
+
Name of the file.
|
|
1088
|
+
|
|
1089
|
+
Returns
|
|
1090
|
+
-------
|
|
1091
|
+
dict
|
|
1092
|
+
Dictionary with the metadata. False otherwise.
|
|
1093
|
+
"""
|
|
1094
|
+
all_metadata = self.list_container_files_metadata(container_id)
|
|
1095
|
+
if all_metadata:
|
|
1096
|
+
for file_meta in all_metadata:
|
|
1097
|
+
if file_meta["name"] == filename:
|
|
1098
|
+
return file_meta
|
|
1099
|
+
else:
|
|
1100
|
+
return False
|
|
1101
|
+
|
|
1102
|
+
def change_file_metadata(self, container_id, filename, modality, tags):
|
|
1103
|
+
"""
|
|
1104
|
+
Change modality and tags of `filename` in `container_id`
|
|
1105
|
+
|
|
1106
|
+
Parameters
|
|
1107
|
+
----------
|
|
1108
|
+
container_id : int
|
|
1109
|
+
Container identifier.
|
|
1110
|
+
filename : str
|
|
1111
|
+
Name of the file to be edited.
|
|
1112
|
+
modality : str or None
|
|
1113
|
+
Modality identifier, or None if the file shouldn't have
|
|
1114
|
+
any modality
|
|
1115
|
+
tags : list[str] or None
|
|
1116
|
+
List of tags, or None if the filename shouldn't have any tags
|
|
1117
|
+
"""
|
|
1118
|
+
|
|
1119
|
+
tags_str = "" if tags is None else ";".join(tags)
|
|
1120
|
+
platform.parse_response(
|
|
1121
|
+
platform.post(
|
|
1122
|
+
self._account.auth,
|
|
1123
|
+
"file_manager/edit_file",
|
|
1124
|
+
data={"container_id": container_id, "filename": filename, "tags": tags_str, "modality": modality},
|
|
1125
|
+
)
|
|
1126
|
+
)
|
|
1092
1127
|
|
|
1093
1128
|
def delete_session(self, subject_name, session_id):
|
|
1094
1129
|
"""
|
|
@@ -1202,347 +1237,353 @@ class Project:
|
|
|
1202
1237
|
return False
|
|
1203
1238
|
return True
|
|
1204
1239
|
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
range_str,
|
|
1209
|
-
length,
|
|
1210
|
-
session_id,
|
|
1211
|
-
disposition,
|
|
1212
|
-
last_chunk,
|
|
1213
|
-
name="",
|
|
1214
|
-
date_of_scan="",
|
|
1215
|
-
description="",
|
|
1216
|
-
subject_name="",
|
|
1217
|
-
ssid="",
|
|
1218
|
-
filename="DATA.zip",
|
|
1219
|
-
input_data_type="mri_brain_data:1.0",
|
|
1220
|
-
result=False,
|
|
1221
|
-
add_to_container_id=0,
|
|
1222
|
-
split_data=False,
|
|
1223
|
-
):
|
|
1240
|
+
""" Container Related Methods """
|
|
1241
|
+
|
|
1242
|
+
def list_input_containers(self, search_criteria={}, items=(0, 9999)):
|
|
1224
1243
|
"""
|
|
1225
|
-
|
|
1244
|
+
Retrieve the list of input containers available to the user under a
|
|
1245
|
+
certain search criteria.
|
|
1226
1246
|
|
|
1227
1247
|
Parameters
|
|
1228
1248
|
----------
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
The string to send that describes the content range
|
|
1233
|
-
length
|
|
1234
|
-
The content length of the chunk to send
|
|
1235
|
-
session_id
|
|
1236
|
-
The session ID from the file path
|
|
1237
|
-
filename
|
|
1238
|
-
The name of the file to be sent
|
|
1239
|
-
disposition
|
|
1240
|
-
The disposition of the content
|
|
1241
|
-
last_chunk
|
|
1242
|
-
Set this only for the last chunk to be uploaded.
|
|
1243
|
-
All following parameters are ignored when False.
|
|
1244
|
-
split_data
|
|
1245
|
-
Sets the header that informs the platform to split
|
|
1246
|
-
the uploaded file into multiple sessions.
|
|
1247
|
-
"""
|
|
1248
|
-
|
|
1249
|
-
request_headers = {
|
|
1250
|
-
"Content-Type": "application/zip",
|
|
1251
|
-
"Content-Range": range_str,
|
|
1252
|
-
"Session-ID": str(session_id),
|
|
1253
|
-
"Content-Length": str(length),
|
|
1254
|
-
"Content-Disposition": disposition,
|
|
1255
|
-
}
|
|
1256
|
-
|
|
1257
|
-
if last_chunk:
|
|
1258
|
-
request_headers["X-Mint-Name"] = name
|
|
1259
|
-
request_headers["X-Mint-Date"] = date_of_scan
|
|
1260
|
-
request_headers["X-Mint-Description"] = description
|
|
1261
|
-
request_headers["X-Mint-Patient-Secret"] = subject_name
|
|
1262
|
-
request_headers["X-Mint-SSID"] = ssid
|
|
1263
|
-
request_headers["X-Mint-Filename"] = filename
|
|
1264
|
-
request_headers["X-Mint-Project-Id"] = str(self._project_id)
|
|
1265
|
-
request_headers["X-Mint-Split-Data"] = str(int(split_data))
|
|
1266
|
-
|
|
1267
|
-
if input_data_type:
|
|
1268
|
-
request_headers["X-Mint-Type"] = input_data_type
|
|
1269
|
-
|
|
1270
|
-
if result:
|
|
1271
|
-
request_headers["X-Mint-In-Out"] = "out"
|
|
1272
|
-
else:
|
|
1273
|
-
request_headers["X-Mint-In-Out"] = "in"
|
|
1274
|
-
|
|
1275
|
-
if add_to_container_id > 0:
|
|
1276
|
-
request_headers["X-Mint-Add-To"] = str(add_to_container_id)
|
|
1277
|
-
|
|
1278
|
-
request_headers["X-Requested-With"] = "XMLHttpRequest"
|
|
1279
|
-
|
|
1280
|
-
response_time = 900.0 if last_chunk else 120.0
|
|
1281
|
-
response = platform.post(
|
|
1282
|
-
auth=self._account.auth, endpoint="upload", data=data, headers=request_headers, timeout=response_time
|
|
1283
|
-
)
|
|
1284
|
-
|
|
1285
|
-
return response
|
|
1249
|
+
search_criteria : dict
|
|
1250
|
+
Each element is a string and is built using the formatting
|
|
1251
|
+
"type;value".
|
|
1286
1252
|
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
ssid
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
chunk_size=2**9,
|
|
1299
|
-
split_data=False,
|
|
1300
|
-
):
|
|
1301
|
-
"""
|
|
1302
|
-
Upload a ZIP file to the platform.
|
|
1253
|
+
List of possible keys:
|
|
1254
|
+
d_n: container_name # TODO: WHAT IS THIS???
|
|
1255
|
+
s_n: subject_id
|
|
1256
|
+
Subject ID of the subject in the platform.
|
|
1257
|
+
ssid: session_id
|
|
1258
|
+
Session ID of the subejct in the platform.
|
|
1259
|
+
from_d: from date
|
|
1260
|
+
Starting date in which perform the search. Format: DD.MM.YYYY
|
|
1261
|
+
to_d: to date
|
|
1262
|
+
End date in which perform the search. Format: DD.MM.YYYY
|
|
1263
|
+
sets: data sets (modalities) # TODO: WHAT IS THIS???
|
|
1303
1264
|
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
file_path : str
|
|
1307
|
-
Path to the ZIP file to upload.
|
|
1308
|
-
subject_name : str
|
|
1309
|
-
Subject ID of the data to upload in the project in QMENTA Platform.
|
|
1310
|
-
ssid : str
|
|
1311
|
-
Session ID of the Subject ID (i.e., ID of the timepoint).
|
|
1312
|
-
date_of_scan : str
|
|
1313
|
-
Date of scan/creation of the file
|
|
1314
|
-
description : str
|
|
1315
|
-
Description of the file
|
|
1316
|
-
result : bool
|
|
1317
|
-
If result=True then the upload will be taken as an offline analysis
|
|
1318
|
-
name : str
|
|
1319
|
-
Name of the file in the platform
|
|
1320
|
-
input_data_type : str
|
|
1321
|
-
qmenta_medical_image_data:3.11
|
|
1322
|
-
add_to_container_id : int
|
|
1323
|
-
ID of the container to which this file should be added (if id > 0)
|
|
1324
|
-
chunk_size : int
|
|
1325
|
-
Size in kB of each chunk. Should be expressed as
|
|
1326
|
-
a power of 2: 2**x. Default value of x is 9 (chunk_size = 512 kB)
|
|
1327
|
-
split_data : bool
|
|
1328
|
-
If True, the platform will try to split the uploaded file into
|
|
1329
|
-
different sessions. It will be ignored when the ssid is given.
|
|
1265
|
+
items: Tuple(int, int)
|
|
1266
|
+
Starting and ending element of the search.
|
|
1330
1267
|
|
|
1331
1268
|
Returns
|
|
1332
1269
|
-------
|
|
1333
|
-
|
|
1334
|
-
|
|
1270
|
+
dict
|
|
1271
|
+
List of containers, each a dictionary containing the following
|
|
1272
|
+
information:
|
|
1273
|
+
{"container_name", "container_id", "patient_secret_name", "ssid"}
|
|
1335
1274
|
"""
|
|
1336
1275
|
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
chunk_size *= 1024
|
|
1341
|
-
max_retries = 10
|
|
1342
|
-
|
|
1343
|
-
name = name or os.path.split(file_path)[1]
|
|
1344
|
-
|
|
1345
|
-
total_bytes = os.path.getsize(file_path)
|
|
1346
|
-
|
|
1347
|
-
# making chunks of the file and sending one by one
|
|
1348
|
-
logger = logging.getLogger(logger_name)
|
|
1349
|
-
with open(file_path, "rb") as file_object:
|
|
1350
|
-
|
|
1351
|
-
file_size = os.path.getsize(file_path)
|
|
1352
|
-
if file_size == 0:
|
|
1353
|
-
logger.error("Cannot upload empty file {}".format(file_path))
|
|
1354
|
-
return False
|
|
1355
|
-
uploaded = 0
|
|
1356
|
-
session_id = self.__get_session_id(file_path)
|
|
1357
|
-
chunk_num = 0
|
|
1358
|
-
retries_count = 0
|
|
1359
|
-
uploaded_bytes = 0
|
|
1360
|
-
response = None
|
|
1361
|
-
last_chunk = False
|
|
1362
|
-
|
|
1363
|
-
if ssid and split_data:
|
|
1364
|
-
logger.warning("split-data argument will be ignored because" + " ssid has been specified")
|
|
1365
|
-
split_data = False
|
|
1366
|
-
|
|
1367
|
-
while True:
|
|
1368
|
-
data = file_object.read(chunk_size)
|
|
1369
|
-
if not data:
|
|
1370
|
-
break
|
|
1371
|
-
|
|
1372
|
-
start_position = chunk_num * chunk_size
|
|
1373
|
-
end_position = start_position + chunk_size - 1
|
|
1374
|
-
bytes_to_send = chunk_size
|
|
1375
|
-
|
|
1376
|
-
if end_position >= total_bytes:
|
|
1377
|
-
last_chunk = True
|
|
1378
|
-
end_position = total_bytes - 1
|
|
1379
|
-
bytes_to_send = total_bytes - uploaded_bytes
|
|
1380
|
-
|
|
1381
|
-
bytes_range = "bytes " + str(start_position) + "-" + str(end_position) + "/" + str(total_bytes)
|
|
1276
|
+
assert len(items) == 2, f"The number of elements in items " f"'{len(items)}' should be equal to two."
|
|
1277
|
+
assert all([isinstance(item, int) for item in items]), f"All items elements '{items}' should be integers."
|
|
1382
1278
|
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
)
|
|
1279
|
+
response = platform.parse_response(
|
|
1280
|
+
platform.post(
|
|
1281
|
+
self._account.auth,
|
|
1282
|
+
"file_manager/get_container_list",
|
|
1283
|
+
data=search_criteria,
|
|
1284
|
+
headers={"X-Range": f"items={items[0]}-{items[1]}"},
|
|
1285
|
+
)
|
|
1286
|
+
)
|
|
1287
|
+
containers = [
|
|
1288
|
+
{
|
|
1289
|
+
"patient_secret_name": container_item["patient_secret_name"],
|
|
1290
|
+
"container_name": container_item["name"],
|
|
1291
|
+
"container_id": container_item["_id"],
|
|
1292
|
+
"ssid": container_item["ssid"],
|
|
1293
|
+
}
|
|
1294
|
+
for container_item in response
|
|
1295
|
+
]
|
|
1296
|
+
return containers
|
|
1402
1297
|
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
logger.error(error_message)
|
|
1409
|
-
break
|
|
1410
|
-
elif int(response.status_code) == 201:
|
|
1411
|
-
chunk_num += 1
|
|
1412
|
-
retries_count = 0
|
|
1413
|
-
uploaded_bytes += chunk_size
|
|
1414
|
-
elif int(response.status_code) == 200:
|
|
1415
|
-
self.__show_progress(file_size, file_size, finish=True)
|
|
1416
|
-
break
|
|
1417
|
-
elif int(response.status_code) == 416:
|
|
1418
|
-
retries_count += 1
|
|
1419
|
-
time.sleep(retries_count * 5)
|
|
1420
|
-
if retries_count > self.max_retries:
|
|
1421
|
-
error_message = "Error Code: 416; " "Requested Range Not Satisfiable (NGINX)"
|
|
1422
|
-
logger.error(error_message)
|
|
1423
|
-
break
|
|
1424
|
-
else:
|
|
1425
|
-
retries_count += 1
|
|
1426
|
-
time.sleep(retries_count * 5)
|
|
1427
|
-
if retries_count > max_retries:
|
|
1428
|
-
error_message = "Number of retries has been reached. " "Upload process stops here !"
|
|
1429
|
-
logger.error(error_message)
|
|
1430
|
-
break
|
|
1298
|
+
def list_result_containers(self, search_condition={}, items=(0, 9999)):
|
|
1299
|
+
"""
|
|
1300
|
+
List the result containers available to the user.
|
|
1301
|
+
Examples
|
|
1302
|
+
--------
|
|
1431
1303
|
|
|
1432
|
-
|
|
1433
|
-
|
|
1304
|
+
>>> search_condition = {
|
|
1305
|
+
"secret_name":"014_S_6920",
|
|
1306
|
+
"from_d": "06.02.2025",
|
|
1307
|
+
"with_child_analysis": 1,
|
|
1308
|
+
"state": "completed"
|
|
1309
|
+
}
|
|
1310
|
+
list_result_containers(search_condition=search_condition)
|
|
1434
1311
|
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
except errors.PlatformError as error:
|
|
1438
|
-
logger.error(error)
|
|
1439
|
-
return False
|
|
1312
|
+
Note the keys not needed for the search do not have to be included in
|
|
1313
|
+
the search condition.
|
|
1440
1314
|
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1315
|
+
Parameters
|
|
1316
|
+
----------
|
|
1317
|
+
search_condition : dict
|
|
1318
|
+
- p_n: str or None Analysis name
|
|
1319
|
+
- type: str or None Type
|
|
1320
|
+
- from_d: str or None dd.mm.yyyy Date from
|
|
1321
|
+
- to_d: str or None dd.mm.yyyy Date to
|
|
1322
|
+
- qa_status: str or None pass/fail/nd QC status
|
|
1323
|
+
- secret_name: str or None Subject ID
|
|
1324
|
+
- tags: str or None
|
|
1325
|
+
- with_child_analysis: 1 or None if 1, child analysis of workflows will appear
|
|
1326
|
+
- id: str or None ID
|
|
1327
|
+
- state: running, completed, pending, exception or None
|
|
1328
|
+
- username: str or None
|
|
1445
1329
|
|
|
1446
|
-
|
|
1330
|
+
items : List[int]
|
|
1331
|
+
list containing two elements [min, max] that correspond to the
|
|
1332
|
+
mininum and maximum range of analysis listed
|
|
1333
|
+
|
|
1334
|
+
Returns
|
|
1335
|
+
-------
|
|
1336
|
+
dict
|
|
1337
|
+
List of containers, each a dictionary
|
|
1338
|
+
{"name": "container-name", "id": "container_id"}
|
|
1339
|
+
if "id": None, that analysis did not had an output container,
|
|
1340
|
+
probably it is a workflow
|
|
1447
1341
|
"""
|
|
1448
|
-
|
|
1342
|
+
analyses = self.list_analysis(search_condition, items)
|
|
1343
|
+
return [{"name": analysis["name"], "id": (analysis.get("out_container_id") or None)} for analysis in analyses]
|
|
1449
1344
|
|
|
1345
|
+
def list_container_files(
|
|
1346
|
+
self,
|
|
1347
|
+
container_id,
|
|
1348
|
+
):
|
|
1349
|
+
"""
|
|
1350
|
+
List the name of the files available inside a given container.
|
|
1450
1351
|
Parameters
|
|
1451
1352
|
----------
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
subject_name: str
|
|
1353
|
+
container_id : str or int
|
|
1354
|
+
Container identifier.
|
|
1455
1355
|
|
|
1456
1356
|
Returns
|
|
1457
1357
|
-------
|
|
1458
|
-
|
|
1459
|
-
|
|
1358
|
+
list[str]
|
|
1359
|
+
List of file names (strings)
|
|
1460
1360
|
"""
|
|
1361
|
+
try:
|
|
1362
|
+
content = platform.parse_response(
|
|
1363
|
+
platform.post(
|
|
1364
|
+
self._account.auth, "file_manager/get_container_files", data={"container_id": container_id}
|
|
1365
|
+
)
|
|
1366
|
+
)
|
|
1367
|
+
except errors.PlatformError as e:
|
|
1368
|
+
logging.getLogger(logger_name).error(e)
|
|
1369
|
+
return False
|
|
1370
|
+
if "files" not in content.keys():
|
|
1371
|
+
logging.getLogger(logger_name).error("Could not get files")
|
|
1372
|
+
return False
|
|
1373
|
+
return content["files"]
|
|
1461
1374
|
|
|
1462
|
-
|
|
1463
|
-
|
|
1375
|
+
def list_container_filter_files(
|
|
1376
|
+
self,
|
|
1377
|
+
container_id,
|
|
1378
|
+
modality="",
|
|
1379
|
+
metadata_info={},
|
|
1380
|
+
tags=[]
|
|
1381
|
+
):
|
|
1382
|
+
"""
|
|
1383
|
+
List the name of the files available inside a given container.
|
|
1384
|
+
search condition example:
|
|
1385
|
+
"metadata": {"SliceThickness":1},
|
|
1386
|
+
}
|
|
1387
|
+
Parameters
|
|
1388
|
+
----------
|
|
1389
|
+
container_id : str or int
|
|
1390
|
+
Container identifier.
|
|
1464
1391
|
|
|
1465
|
-
|
|
1392
|
+
modality: str
|
|
1393
|
+
String containing the modality of the files being filtered
|
|
1394
|
+
|
|
1395
|
+
metadata_info: dict
|
|
1396
|
+
Dictionary containing the file metadata of the files being filtered
|
|
1397
|
+
|
|
1398
|
+
tags: list[str]
|
|
1399
|
+
List of strings containing the tags of the files being filtered
|
|
1400
|
+
|
|
1401
|
+
Returns
|
|
1402
|
+
-------
|
|
1403
|
+
selected_files: list[str]
|
|
1404
|
+
List of file names (strings)
|
|
1466
1405
|
"""
|
|
1467
|
-
|
|
1406
|
+
content_files = self.list_container_files(container_id)
|
|
1407
|
+
content_meta = self.list_container_files_metadata(container_id)
|
|
1408
|
+
selected_files = []
|
|
1409
|
+
for index, file in enumerate(content_files):
|
|
1410
|
+
metadata_file = content_meta[index]
|
|
1411
|
+
tags_file = metadata_file.get("tags")
|
|
1412
|
+
tags_bool = [tag in tags_file for tag in tags]
|
|
1413
|
+
info_bool = []
|
|
1414
|
+
if modality == "":
|
|
1415
|
+
modality_bool = True
|
|
1416
|
+
else:
|
|
1417
|
+
modality_bool = modality == metadata_file["metadata"].get(
|
|
1418
|
+
"modality"
|
|
1419
|
+
)
|
|
1420
|
+
for key in metadata_info.keys():
|
|
1421
|
+
meta_key = (
|
|
1422
|
+
(
|
|
1423
|
+
metadata_file.get("metadata") or {}
|
|
1424
|
+
).get("info") or {}).get(
|
|
1425
|
+
key
|
|
1426
|
+
)
|
|
1427
|
+
if meta_key is None:
|
|
1428
|
+
logging.getLogger(logger_name).warning(
|
|
1429
|
+
f"{key} is not in file_info from file {file}"
|
|
1430
|
+
)
|
|
1431
|
+
info_bool.append(
|
|
1432
|
+
metadata_info[key] == meta_key
|
|
1433
|
+
)
|
|
1434
|
+
if all(tags_bool) and all(info_bool) and modality_bool:
|
|
1435
|
+
selected_files.append(file)
|
|
1436
|
+
return selected_files
|
|
1437
|
+
|
|
1438
|
+
def list_container_files_metadata(self, container_id):
|
|
1439
|
+
"""
|
|
1440
|
+
List all the metadata of the files available inside a given container.
|
|
1468
1441
|
|
|
1469
1442
|
Parameters
|
|
1470
1443
|
----------
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
subject_name: str
|
|
1444
|
+
container_id : str
|
|
1445
|
+
Container identifier.
|
|
1474
1446
|
|
|
1475
1447
|
Returns
|
|
1476
1448
|
-------
|
|
1477
|
-
|
|
1478
|
-
|
|
1449
|
+
dict
|
|
1450
|
+
Dictionary of {"metadata_name": "metadata_value"}
|
|
1479
1451
|
"""
|
|
1480
1452
|
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1453
|
+
try:
|
|
1454
|
+
data = platform.parse_response(
|
|
1455
|
+
platform.post(
|
|
1456
|
+
self._account.auth, "file_manager/get_container_files", data={"container_id": container_id}
|
|
1457
|
+
)
|
|
1458
|
+
)
|
|
1459
|
+
except errors.PlatformError as e:
|
|
1460
|
+
logging.getLogger(logger_name).error(e)
|
|
1461
|
+
return False
|
|
1484
1462
|
|
|
1485
|
-
|
|
1463
|
+
return data["meta"]
|
|
1464
|
+
|
|
1465
|
+
""" Analysis Related Methods """
|
|
1466
|
+
|
|
1467
|
+
def get_analysis(self, analysis_name_or_id):
|
|
1486
1468
|
"""
|
|
1487
|
-
|
|
1469
|
+
Returns the analysis corresponding with the analysis id or analysis name
|
|
1488
1470
|
|
|
1489
1471
|
Parameters
|
|
1490
1472
|
----------
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
subject_name: str
|
|
1473
|
+
analysis_name_or_id : int, str
|
|
1474
|
+
analysis_id or analysis name to search for
|
|
1494
1475
|
|
|
1495
1476
|
Returns
|
|
1496
1477
|
-------
|
|
1497
|
-
|
|
1498
|
-
|
|
1478
|
+
analysis: dict
|
|
1479
|
+
A dictionary containing the analysis information
|
|
1480
|
+
"""
|
|
1481
|
+
if isinstance(analysis_name_or_id, int):
|
|
1482
|
+
search_tag = "id"
|
|
1483
|
+
elif isinstance(analysis_name_or_id, str):
|
|
1484
|
+
if analysis_name_or_id.isdigit():
|
|
1485
|
+
search_tag = "id"
|
|
1486
|
+
analysis_name_or_id = int(analysis_name_or_id)
|
|
1487
|
+
else:
|
|
1488
|
+
search_tag = "p_n"
|
|
1489
|
+
excluded_characters = ["\\", "[", "(", "+", "*"]
|
|
1490
|
+
excluded_bool = [character in analysis_name_or_id for character in excluded_characters]
|
|
1491
|
+
if any(excluded_bool):
|
|
1492
|
+
raise Exception(f"p_n does not allow characters {excluded_characters}")
|
|
1493
|
+
else:
|
|
1494
|
+
raise Exception("The analysis identifier must be its name or an " "integer")
|
|
1495
|
+
|
|
1496
|
+
search_condition = {
|
|
1497
|
+
search_tag: analysis_name_or_id,
|
|
1498
|
+
}
|
|
1499
|
+
response = platform.parse_response(
|
|
1500
|
+
platform.post(self._account.auth, "analysis_manager/get_analysis_list", data=search_condition)
|
|
1501
|
+
)
|
|
1502
|
+
|
|
1503
|
+
if len(response) > 1:
|
|
1504
|
+
raise Exception(f"multiple analyses with name " f"{analysis_name_or_id} found")
|
|
1505
|
+
elif len(response) == 1:
|
|
1506
|
+
return response[0]
|
|
1507
|
+
else:
|
|
1508
|
+
return None
|
|
1509
|
+
|
|
1510
|
+
def list_analysis(self, search_condition={}, items=(0, 9999)):
|
|
1499
1511
|
"""
|
|
1512
|
+
List the analysis available to the user.
|
|
1500
1513
|
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1514
|
+
Examples
|
|
1515
|
+
--------
|
|
1516
|
+
|
|
1517
|
+
>>> search_condition = {
|
|
1518
|
+
"secret_name":"014_S_6920",
|
|
1519
|
+
"from_d": "06.02.2025",
|
|
1520
|
+
"with_child_analysis": 1,
|
|
1521
|
+
"state": "completed"
|
|
1522
|
+
}
|
|
1523
|
+
list_analysis(search_condition=search_condition)
|
|
1504
1524
|
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
Copy a container to another project.
|
|
1525
|
+
Note the keys not needed for the search do not have to be included in
|
|
1526
|
+
the search condition.
|
|
1508
1527
|
|
|
1509
1528
|
Parameters
|
|
1510
1529
|
----------
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1530
|
+
search_condition : dict
|
|
1531
|
+
- p_n: str or None Analysis name
|
|
1532
|
+
- type: str or None Type
|
|
1533
|
+
- from_d: str or None dd.mm.yyyy Date from
|
|
1534
|
+
- to_d: str or None dd.mm.yyyy Date to
|
|
1535
|
+
- qa_status: str or None pass/fail/nd QC status
|
|
1536
|
+
- secret_name: str or None Subject ID
|
|
1537
|
+
- tags: str or None
|
|
1538
|
+
- with_child_analysis: 1 or None if 1, child analysis of workflows will appear
|
|
1539
|
+
- id: str or None ID
|
|
1540
|
+
- state: running, completed, pending, exception or None
|
|
1541
|
+
- username: str or None
|
|
1542
|
+
|
|
1543
|
+
items : List[int]
|
|
1544
|
+
list containing two elements [min, max] that correspond to the
|
|
1545
|
+
mininum and maximum range of analysis listed
|
|
1515
1546
|
|
|
1516
1547
|
Returns
|
|
1517
1548
|
-------
|
|
1518
|
-
|
|
1519
|
-
|
|
1549
|
+
dict
|
|
1550
|
+
List of analysis, each a dictionary
|
|
1520
1551
|
"""
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
"
|
|
1534
|
-
"
|
|
1552
|
+
assert len(items) == 2, f"The number of elements in items " f"'{len(items)}' should be equal to two."
|
|
1553
|
+
assert all([isinstance(item, int) for item in items]), f"All items elements '{items}' should be integers."
|
|
1554
|
+
search_keys = {
|
|
1555
|
+
"p_n": str,
|
|
1556
|
+
"type": str,
|
|
1557
|
+
"from_d": str,
|
|
1558
|
+
"to_d": str,
|
|
1559
|
+
"qa_status": str,
|
|
1560
|
+
"secret_name": str,
|
|
1561
|
+
"tags": str,
|
|
1562
|
+
"with_child_analysis": int,
|
|
1563
|
+
"id": int,
|
|
1564
|
+
"state": str,
|
|
1565
|
+
"username": str,
|
|
1535
1566
|
}
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1567
|
+
for key in search_condition.keys():
|
|
1568
|
+
if key not in search_keys.keys():
|
|
1569
|
+
raise Exception((f"This key '{key}' is not accepted by this" "search condition"))
|
|
1570
|
+
if not isinstance(search_condition[key], search_keys[key]) and search_condition[key] is not None:
|
|
1571
|
+
raise Exception((f"The key {key} in the search condition" f"is not type {search_keys[key]}"))
|
|
1572
|
+
if "p_n" == key:
|
|
1573
|
+
excluded_characters = ["\\", "[", "(", "+", "*"]
|
|
1574
|
+
excluded_bool = [character in search_condition["p_n"] for character in excluded_characters]
|
|
1575
|
+
if any(excluded_bool):
|
|
1576
|
+
raise Exception(f"p_n does not allow characters {excluded_characters}")
|
|
1577
|
+
req_headers = {"X-Range": f"items={items[0]}-{items[1] - 1}"}
|
|
1578
|
+
req_headers = {"X-Range": f"items={items[0]}-{items[1] - 1}"}
|
|
1579
|
+
return platform.parse_response(
|
|
1580
|
+
platform.post(
|
|
1581
|
+
auth=self._account.auth,
|
|
1582
|
+
endpoint="analysis_manager/get_analysis_list",
|
|
1583
|
+
headers=req_headers,
|
|
1584
|
+
data=search_condition,
|
|
1540
1585
|
)
|
|
1541
|
-
|
|
1542
|
-
logging.getLogger(logger_name).error("Couldn not copy container: {}".format(e))
|
|
1543
|
-
return False
|
|
1544
|
-
|
|
1545
|
-
return True
|
|
1586
|
+
)
|
|
1546
1587
|
|
|
1547
1588
|
def start_analysis(
|
|
1548
1589
|
self,
|
|
@@ -1650,94 +1691,7 @@ class Project:
|
|
|
1650
1691
|
|
|
1651
1692
|
return True
|
|
1652
1693
|
|
|
1653
|
-
|
|
1654
|
-
"""
|
|
1655
|
-
Handle the possible responses from the server after start_analysis.
|
|
1656
|
-
Sometimes we have to send a request again, and then check again the
|
|
1657
|
-
response. That"s why this function is separated from start_analysis.
|
|
1658
|
-
|
|
1659
|
-
Since this function sometimes calls itself, n_calls avoids entering an
|
|
1660
|
-
infinite loop due to some misbehaviour in the server.
|
|
1661
|
-
"""
|
|
1662
|
-
|
|
1663
|
-
call_limit = 10
|
|
1664
|
-
n_calls += 1
|
|
1665
|
-
|
|
1666
|
-
logger = logging.getLogger(logger_name)
|
|
1667
|
-
if n_calls > call_limit:
|
|
1668
|
-
logger.error(
|
|
1669
|
-
f"__handle_start_analysis_response called itself more\
|
|
1670
|
-
than {n_calls} times: aborting."
|
|
1671
|
-
)
|
|
1672
|
-
return None
|
|
1673
|
-
|
|
1674
|
-
try:
|
|
1675
|
-
response = platform.parse_response(
|
|
1676
|
-
platform.post(self._account.auth, "analysis_manager/analysis_registration", data=post_data)
|
|
1677
|
-
)
|
|
1678
|
-
logger.info(response["message"])
|
|
1679
|
-
return int(response["analysis_id"])
|
|
1680
|
-
except platform.ChooseDataError as choose_data:
|
|
1681
|
-
has_warning = False
|
|
1682
|
-
|
|
1683
|
-
# logging any warning that we have
|
|
1684
|
-
if choose_data.warning:
|
|
1685
|
-
has_warning = True
|
|
1686
|
-
logger.warning(response["warning"])
|
|
1687
|
-
|
|
1688
|
-
new_post = {
|
|
1689
|
-
"analysis_id": choose_data.analysis_id,
|
|
1690
|
-
"script_name": post_data["script_name"],
|
|
1691
|
-
"version": post_data["version"],
|
|
1692
|
-
}
|
|
1693
|
-
|
|
1694
|
-
if choose_data.data_to_choose:
|
|
1695
|
-
# in case we have data to choose
|
|
1696
|
-
chosen_files = {}
|
|
1697
|
-
for settings_key in choose_data.data_to_choose:
|
|
1698
|
-
chosen_files[settings_key] = {}
|
|
1699
|
-
filters = choose_data.data_to_choose[settings_key]["filters"]
|
|
1700
|
-
for filter_key in filters:
|
|
1701
|
-
filter_data = filters[filter_key]
|
|
1702
|
-
|
|
1703
|
-
# skip the filters that did not pass
|
|
1704
|
-
if not filter_data["passed"]:
|
|
1705
|
-
continue
|
|
1706
|
-
|
|
1707
|
-
number_of_files_to_select = 1
|
|
1708
|
-
if filter_data["range"][0] != 0:
|
|
1709
|
-
number_of_files_to_select = filter_data["range"][0]
|
|
1710
|
-
elif filter_data["range"][1] != 0:
|
|
1711
|
-
number_of_files_to_select = min(filter_data["range"][1], len(filter_data["files"]))
|
|
1712
|
-
else:
|
|
1713
|
-
number_of_files_to_select = len(filter_data["files"])
|
|
1714
|
-
|
|
1715
|
-
files_selection = [ff["_id"] for ff in filter_data["files"][:number_of_files_to_select]]
|
|
1716
|
-
chosen_files[settings_key][filter_key] = files_selection
|
|
1717
|
-
|
|
1718
|
-
new_post["user_preference"] = json.dumps(chosen_files)
|
|
1719
|
-
else:
|
|
1720
|
-
if has_warning and not ignore_warnings:
|
|
1721
|
-
logger.info("cancelling analysis due to warnings, " + 'set "ignore_warnings" to True to override')
|
|
1722
|
-
new_post["cancel"] = "1"
|
|
1723
|
-
else:
|
|
1724
|
-
logger.info("suppressing warnings")
|
|
1725
|
-
new_post["user_preference"] = "{}"
|
|
1726
|
-
new_post["_mint_only_warning"] = "1"
|
|
1727
|
-
|
|
1728
|
-
return self.__handle_start_analysis(new_post, ignore_warnings=ignore_warnings, n_calls=n_calls)
|
|
1729
|
-
except platform.ActionFailedError as e:
|
|
1730
|
-
logger.error(f"Unable to start the analysis: {e}")
|
|
1731
|
-
return None
|
|
1732
|
-
|
|
1733
|
-
@staticmethod
|
|
1734
|
-
def __get_modalities(files):
|
|
1735
|
-
modalities = []
|
|
1736
|
-
for file_ in files:
|
|
1737
|
-
modality = file_["metadata"]["modality"]
|
|
1738
|
-
if modality not in modalities:
|
|
1739
|
-
modalities.append(modality)
|
|
1740
|
-
return modalities
|
|
1694
|
+
""" QC Status Related Methods """
|
|
1741
1695
|
|
|
1742
1696
|
def set_qc_status_analysis(self, analysis_id,
|
|
1743
1697
|
status=QCStatus.UNDERTERMINED, comments=""):
|
|
@@ -1919,6 +1873,7 @@ class Project:
|
|
|
1919
1873
|
raise ValueError("Either 'patient_id' or 'subject_name' and 'ssid'"
|
|
1920
1874
|
" must not be empty.")
|
|
1921
1875
|
|
|
1876
|
+
""" Protocol Adherence Related Methods """
|
|
1922
1877
|
def set_project_pa_rules(self, rules_file_path, guidance_text=""):
|
|
1923
1878
|
"""
|
|
1924
1879
|
Updates the active project's protocol adherence rules using the
|
|
@@ -2004,6 +1959,96 @@ class Project:
|
|
|
2004
1959
|
|
|
2005
1960
|
return res["guidance_text"]
|
|
2006
1961
|
|
|
1962
|
+
""" Helper Methods """
|
|
1963
|
+
def __handle_start_analysis(self, post_data, ignore_warnings=False, n_calls=0):
|
|
1964
|
+
"""
|
|
1965
|
+
Handle the possible responses from the server after start_analysis.
|
|
1966
|
+
Sometimes we have to send a request again, and then check again the
|
|
1967
|
+
response. That"s why this function is separated from start_analysis.
|
|
1968
|
+
|
|
1969
|
+
Since this function sometimes calls itself, n_calls avoids entering an
|
|
1970
|
+
infinite loop due to some misbehaviour in the server.
|
|
1971
|
+
"""
|
|
1972
|
+
|
|
1973
|
+
call_limit = 10
|
|
1974
|
+
n_calls += 1
|
|
1975
|
+
|
|
1976
|
+
logger = logging.getLogger(logger_name)
|
|
1977
|
+
if n_calls > call_limit:
|
|
1978
|
+
logger.error(
|
|
1979
|
+
f"__handle_start_analysis_response called itself more\
|
|
1980
|
+
than {n_calls} times: aborting."
|
|
1981
|
+
)
|
|
1982
|
+
return None
|
|
1983
|
+
|
|
1984
|
+
try:
|
|
1985
|
+
response = platform.parse_response(
|
|
1986
|
+
platform.post(self._account.auth, "analysis_manager/analysis_registration", data=post_data)
|
|
1987
|
+
)
|
|
1988
|
+
logger.info(response["message"])
|
|
1989
|
+
return int(response["analysis_id"])
|
|
1990
|
+
except platform.ChooseDataError as choose_data:
|
|
1991
|
+
has_warning = False
|
|
1992
|
+
|
|
1993
|
+
# logging any warning that we have
|
|
1994
|
+
if choose_data.warning:
|
|
1995
|
+
has_warning = True
|
|
1996
|
+
logger.warning(response["warning"])
|
|
1997
|
+
|
|
1998
|
+
new_post = {
|
|
1999
|
+
"analysis_id": choose_data.analysis_id,
|
|
2000
|
+
"script_name": post_data["script_name"],
|
|
2001
|
+
"version": post_data["version"],
|
|
2002
|
+
}
|
|
2003
|
+
|
|
2004
|
+
if choose_data.data_to_choose:
|
|
2005
|
+
# in case we have data to choose
|
|
2006
|
+
chosen_files = {}
|
|
2007
|
+
for settings_key in choose_data.data_to_choose:
|
|
2008
|
+
chosen_files[settings_key] = {}
|
|
2009
|
+
filters = choose_data.data_to_choose[settings_key]["filters"]
|
|
2010
|
+
for filter_key in filters:
|
|
2011
|
+
filter_data = filters[filter_key]
|
|
2012
|
+
|
|
2013
|
+
# skip the filters that did not pass
|
|
2014
|
+
if not filter_data["passed"]:
|
|
2015
|
+
continue
|
|
2016
|
+
|
|
2017
|
+
number_of_files_to_select = 1
|
|
2018
|
+
if filter_data["range"][0] != 0:
|
|
2019
|
+
number_of_files_to_select = filter_data["range"][0]
|
|
2020
|
+
elif filter_data["range"][1] != 0:
|
|
2021
|
+
number_of_files_to_select = min(filter_data["range"][1], len(filter_data["files"]))
|
|
2022
|
+
else:
|
|
2023
|
+
number_of_files_to_select = len(filter_data["files"])
|
|
2024
|
+
|
|
2025
|
+
files_selection = [ff["_id"] for ff in filter_data["files"][:number_of_files_to_select]]
|
|
2026
|
+
chosen_files[settings_key][filter_key] = files_selection
|
|
2027
|
+
|
|
2028
|
+
new_post["user_preference"] = json.dumps(chosen_files)
|
|
2029
|
+
else:
|
|
2030
|
+
if has_warning and not ignore_warnings:
|
|
2031
|
+
logger.info("cancelling analysis due to warnings, " + 'set "ignore_warnings" to True to override')
|
|
2032
|
+
new_post["cancel"] = "1"
|
|
2033
|
+
else:
|
|
2034
|
+
logger.info("suppressing warnings")
|
|
2035
|
+
new_post["user_preference"] = "{}"
|
|
2036
|
+
new_post["_mint_only_warning"] = "1"
|
|
2037
|
+
|
|
2038
|
+
return self.__handle_start_analysis(new_post, ignore_warnings=ignore_warnings, n_calls=n_calls)
|
|
2039
|
+
except platform.ActionFailedError as e:
|
|
2040
|
+
logger.error(f"Unable to start the analysis: {e}")
|
|
2041
|
+
return None
|
|
2042
|
+
|
|
2043
|
+
@staticmethod
|
|
2044
|
+
def __get_modalities(files):
|
|
2045
|
+
modalities = []
|
|
2046
|
+
for file_ in files:
|
|
2047
|
+
modality = file_["metadata"]["modality"]
|
|
2048
|
+
if modality not in modalities:
|
|
2049
|
+
modalities.append(modality)
|
|
2050
|
+
return modalities
|
|
2051
|
+
|
|
2007
2052
|
def __show_progress(self, done, total, finish=False):
|
|
2008
2053
|
bytes_in_mb = 1024 * 1024
|
|
2009
2054
|
progress_message = "\r[{:.2f} %] Uploaded {:.2f} of {:.2f} Mb".format(
|