qmenta-client 1.1.dev1382__py3-none-any.whl → 1.1.dev1389__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qmenta/client/Project.py +1139 -1104
- {qmenta_client-1.1.dev1382.dist-info → qmenta_client-1.1.dev1389.dist-info}/METADATA +1 -1
- {qmenta_client-1.1.dev1382.dist-info → qmenta_client-1.1.dev1389.dist-info}/RECORD +4 -4
- {qmenta_client-1.1.dev1382.dist-info → qmenta_client-1.1.dev1389.dist-info}/WHEEL +0 -0
qmenta/client/Project.py
CHANGED
|
@@ -75,6 +75,7 @@ class Project:
|
|
|
75
75
|
|
|
76
76
|
"""
|
|
77
77
|
|
|
78
|
+
""" Project Related Methods """
|
|
78
79
|
def __init__(self, account: Account, project_id, max_upload_retries=5):
|
|
79
80
|
# if project_id is a string (the name of the project), get the
|
|
80
81
|
# project id (int)
|
|
@@ -132,770 +133,306 @@ class Project:
|
|
|
132
133
|
rep = "<Project {}>".format(self._project_name)
|
|
133
134
|
return rep
|
|
134
135
|
|
|
135
|
-
|
|
136
|
-
def
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
""
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
""
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
Each element is a string and is built using the formatting
|
|
156
|
-
"type;value", or "type;operation|value"
|
|
157
|
-
|
|
158
|
-
Complete search_criteria Dictionary Explanation:
|
|
159
|
-
|
|
160
|
-
search_criteria = {
|
|
161
|
-
"pars_patient_secret_name": "string;SUBJECTID",
|
|
162
|
-
"pars_ssid": "integer;OPERATOR|SSID",
|
|
163
|
-
"pars_modalities": "string;MODALITY",
|
|
164
|
-
"pars_tags": "tags;TAGS",
|
|
165
|
-
"pars_age_at_scan": "integer;OPERATOR|AGE_AT_SCAN",
|
|
166
|
-
"pars_[dicom]_KEY": "KEYTYPE;KEYVALUE",
|
|
167
|
-
"pars_PROJECTMETADATA": "METADATATYPE;METADATAVALUE",
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
Where:
|
|
171
|
-
"pars_patient_secret_name": Applies the search to the 'Subject ID'.
|
|
172
|
-
SUBJECTID is a comma separated list of strings.
|
|
173
|
-
"pars_ssid": Applies the search to the 'Session ID'.
|
|
174
|
-
SSID is an integer.
|
|
175
|
-
OPERATOR is the operator to apply. One of:
|
|
176
|
-
- Equal: eq
|
|
177
|
-
- Different Than: ne
|
|
178
|
-
- Greater Than: gt
|
|
179
|
-
- Greater/Equal To: gte
|
|
180
|
-
- Lower Than: lt
|
|
181
|
-
- Lower/Equal To: lte
|
|
182
|
-
|
|
183
|
-
"pars_modalities": Applies the search to the file 'Modalities'
|
|
184
|
-
available within each Subject ID.
|
|
185
|
-
MODALITY is a comma separated list of string. A session is provided as
|
|
186
|
-
long as one MODALITY is available.
|
|
187
|
-
"pars_tags": Applies the search to the file 'Tags' available within
|
|
188
|
-
each Subject ID and to the subject-level 'Tags'.
|
|
189
|
-
TAGS is a comma separated list of strings. A session is provided as
|
|
190
|
-
long as one tag is available.
|
|
191
|
-
"pars_age_at_scan": Applies the search to the 'age_at_scan' metadata
|
|
192
|
-
field.
|
|
193
|
-
AGE_AT_SCAN is an integer.
|
|
194
|
-
"pars_[dicom]_KEY": Applies the search to the metadata fields
|
|
195
|
-
available within each file. KEY must be one of the
|
|
196
|
-
metadata keys of the files. The full list of KEYS is shown above via
|
|
197
|
-
'file_m["metadata"]["info"].keys()'.
|
|
198
|
-
KEYTYPE is the type of the KEY. One of:
|
|
199
|
-
- integer
|
|
200
|
-
- decimal
|
|
201
|
-
- string
|
|
202
|
-
- list
|
|
203
|
-
|
|
204
|
-
if 'integer' or 'decimal' you must also include an OPERATOR
|
|
205
|
-
(i.e., "integer;OPERATOR|KEYVALUE").
|
|
206
|
-
if 'list' the KEYVALUE should be a semicolon separated list of
|
|
207
|
-
values. (i.e., "list;KEYVALUE1;KEYVALUE2;KEYVALUE3)
|
|
208
|
-
KEYVALUEs must be strings.
|
|
209
|
-
KEYVALUE is the expected value of the KEY.
|
|
210
|
-
"pars_[dicom]_PROJECTMETADATA": Applies to the metadata defined
|
|
211
|
-
within the 'Metadata Manager' of the project.
|
|
212
|
-
PROJECTMETADATA is the ID of the metadata field.
|
|
213
|
-
METADATATYPE is the type of the metadata field. One of:
|
|
214
|
-
- string
|
|
215
|
-
- integer
|
|
216
|
-
- list
|
|
217
|
-
- decimal
|
|
218
|
-
- single_option
|
|
219
|
-
- multiple_option
|
|
220
|
-
|
|
221
|
-
if 'integer' or 'decimal' you must also include an OPERATOR
|
|
222
|
-
(i.e., "integer;OPERATOR|METADATAVALUE").
|
|
223
|
-
KEYVALUE is the expected value of the metadata.
|
|
224
|
-
|
|
225
|
-
1) Example:
|
|
226
|
-
search_criteria = {
|
|
227
|
-
"pars_patient_secret_name": "string;abide",
|
|
228
|
-
"pars_ssid": "integer;eq|2"
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
2) Example:
|
|
232
|
-
search_criteria = {
|
|
233
|
-
"pars_modalities": "string;T1",
|
|
234
|
-
"pars_tags": "tags;flair",
|
|
235
|
-
"pars_[dicom]_Manufacturer": "string;ge",
|
|
236
|
-
"pars_[dicom]_FlipAngle": "integer;gt|5",
|
|
237
|
-
"pars_[dicom]_ImageType": "list;PRIMARY;SECONDARY",
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
Note the search criteria applies to all the files included within a
|
|
241
|
-
session. Hence, it does not imply that all the criteria conditions
|
|
242
|
-
are applied to the same files. In example 2) above, it means that
|
|
243
|
-
any Subject ID/Session ID that has a file classified with a 'T1'
|
|
244
|
-
modality, a file with a 'flair' tag, a file whose Manufacturer
|
|
245
|
-
contains 'ge', a file whose FlipAngle is greater than '5º', and a
|
|
246
|
-
file with ImageType with any of the values: PRIMARY or SECONDARY
|
|
247
|
-
will be selected.
|
|
248
|
-
|
|
249
|
-
Returns
|
|
250
|
-
-------
|
|
251
|
-
dict
|
|
252
|
-
A list of dictionary of {"metadata_name": "metadata_value"}
|
|
253
|
-
|
|
254
|
-
"""
|
|
255
|
-
|
|
256
|
-
assert len(items) == 2, f"The number of elements in items " f"'{len(items)}' should be equal to two."
|
|
257
|
-
assert all([isinstance(item, int) for item in items]), f"All items elements '{items}' should be integers."
|
|
258
|
-
|
|
259
|
-
assert all([key[:5] == "pars_" for key in search_criteria.keys()]), (
|
|
260
|
-
f"All keys of the search_criteria dictionary " f"'{search_criteria.keys()}' must start with 'pars_'."
|
|
261
|
-
)
|
|
262
|
-
|
|
263
|
-
for key, value in search_criteria.items():
|
|
264
|
-
if value.split(";")[0] in ["integer", "decimal"]:
|
|
265
|
-
assert value.split(";")[1].split("|")[0] in OPERATOR_LIST, (
|
|
266
|
-
f"Search criteria of type '{value.split(';')[0]}' must "
|
|
267
|
-
f"include an operator ({', '.join(OPERATOR_LIST)})."
|
|
268
|
-
)
|
|
269
|
-
|
|
270
|
-
content = platform.parse_response(
|
|
271
|
-
platform.post(
|
|
272
|
-
self._account.auth,
|
|
273
|
-
"patient_manager/get_patient_list",
|
|
274
|
-
data=search_criteria,
|
|
275
|
-
headers={"X-Range": f"items={items[0]}-{items[1]}"},
|
|
276
|
-
)
|
|
277
|
-
)
|
|
278
|
-
return content
|
|
279
|
-
|
|
280
|
-
def get_subjects_files_metadata(self, search_criteria={}, items=(0, 9999)):
|
|
281
|
-
"""
|
|
282
|
-
List all Subject ID/Session ID from the selected project that meet the
|
|
283
|
-
defined search criteria at a file level.
|
|
284
|
-
|
|
285
|
-
Note, albeit the search criteria is similar to the one defined in
|
|
286
|
-
method 'get_subjects_metadata()' (see differences below), the
|
|
287
|
-
output is different as this method provides the sessions which
|
|
288
|
-
have a file that satisfy all the conditions of the search criteria.
|
|
289
|
-
This method is slow.
|
|
290
|
-
|
|
291
|
-
Parameters
|
|
292
|
-
----------
|
|
293
|
-
search_criteria: dict
|
|
294
|
-
Each element is a string and is built using the formatting
|
|
295
|
-
"type;value", or "type;operation|value"
|
|
296
|
-
|
|
297
|
-
Complete search_criteria Dictionary Explanation:
|
|
298
|
-
|
|
299
|
-
search_criteria = {
|
|
300
|
-
"pars_patient_secret_name": "string;SUBJECTID",
|
|
301
|
-
"pars_ssid": "integer;OPERATOR|SSID",
|
|
302
|
-
"pars_modalities": "string;MODALITY",
|
|
303
|
-
"pars_tags": "tags;TAGS",
|
|
304
|
-
"pars_age_at_scan": "integer;OPERATOR|AGE_AT_SCAN",
|
|
305
|
-
"pars_[dicom]_KEY": "KEYTYPE;KEYVALUE",
|
|
306
|
-
"pars_PROJECTMETADATA": "METADATATYPE;METADATAVALUE",
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
Where:
|
|
310
|
-
"pars_patient_secret_name": Applies the search to the 'Subject ID'.
|
|
311
|
-
SUBJECTID is a comma separated list of strings.
|
|
312
|
-
"pars_ssid": Applies the search to the 'Session ID'.
|
|
313
|
-
SSID is an integer.
|
|
314
|
-
OPERATOR is the operator to apply. One of:
|
|
315
|
-
- Equal: eq
|
|
316
|
-
- Different Than: ne
|
|
317
|
-
- Greater Than: gt
|
|
318
|
-
- Greater/Equal To: gte
|
|
319
|
-
- Lower Than: lt
|
|
320
|
-
- Lower/Equal To: lte
|
|
321
|
-
|
|
322
|
-
"pars_modalities": Applies the search to the file 'Modalities'
|
|
323
|
-
available within each Subject ID.
|
|
324
|
-
MODALITY is a string.
|
|
325
|
-
"pars_tags": Applies only the search to the file 'Tags' available
|
|
326
|
-
within each Subject ID.
|
|
327
|
-
TAGS is a comma separated list of strings. All tags must be present in
|
|
328
|
-
the same file.
|
|
329
|
-
"pars_age_at_scan": Applies the search to the 'age_at_scan' metadata
|
|
330
|
-
field.
|
|
331
|
-
AGE_AT_SCAN is an integer.
|
|
332
|
-
"pars_[dicom]_KEY": Applies the search to the metadata fields
|
|
333
|
-
available within each file. KEY must be one of the
|
|
334
|
-
metadata keys of the files. The full list of KEYS is shown above via
|
|
335
|
-
'file_m["metadata"]["info"].keys()'. # TODO: SEE HOW TO WRITE THIS
|
|
336
|
-
KEYTYPE is the type of the KEY. One of:
|
|
337
|
-
- integer
|
|
338
|
-
- decimal
|
|
339
|
-
- string
|
|
340
|
-
- list
|
|
341
|
-
|
|
342
|
-
if 'integer' or 'decimal' you must also include an OPERATOR
|
|
343
|
-
(i.e., "integer;OPERATOR|KEYVALUE").
|
|
344
|
-
if 'list' the KEYVALUE should be a semicolon separated list of
|
|
345
|
-
values. (i.e., "list;KEYVALUE1;KEYVALUE2;KEYVALUE3)
|
|
346
|
-
KEYVALUEs must be strings.
|
|
347
|
-
KEYVALUE is the expected value of the KEY.
|
|
348
|
-
"pars_[dicom]_PROJECTMETADATA": Applies to the metadata defined
|
|
349
|
-
within the 'Metadata Manager' of the project.
|
|
350
|
-
PROJECTMETADATA is the ID of the metadata field.
|
|
351
|
-
METADATATYPE is the type of the metadata field. One of:
|
|
352
|
-
- string
|
|
353
|
-
- integer
|
|
354
|
-
- list
|
|
355
|
-
- decimal
|
|
356
|
-
- single_option
|
|
357
|
-
- multiple_option
|
|
358
|
-
|
|
359
|
-
if 'integer' or 'decimal' you must also include an OPERATOR
|
|
360
|
-
(i.e., "integer;OPERATOR|METADATAVALUE").
|
|
361
|
-
KEYVALUE is the expected value of the metadata.
|
|
362
|
-
|
|
363
|
-
1) Example:
|
|
364
|
-
search_criteria = {
|
|
365
|
-
"pars_patient_secret_name": "string;abide",
|
|
366
|
-
"pars_ssid": "integer;eq|2"
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
2) Example:
|
|
370
|
-
search_criteria = {
|
|
371
|
-
"pars_patient_secret_name": "string;001"
|
|
372
|
-
"pars_modalities": "string;T2",
|
|
373
|
-
"pars_tags": "tags;flair",
|
|
374
|
-
"pars_[dicom]_Manufacturer": "string;ge",
|
|
375
|
-
"pars_[dicom]_FlipAngle": "integer;gt|5",
|
|
376
|
-
"pars_[dicom]_ImageType": "list;PRIMARY;SECONDARY",
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
Note the search criteria might apply to both the files metadata
|
|
380
|
-
information available within a session and the metadata of the
|
|
381
|
-
session. And the method provides a session only if all the file
|
|
382
|
-
related conditions are satisfied within the same file.
|
|
383
|
-
In example 2) above, it means that the output will contain any
|
|
384
|
-
session whose Subject ID contains '001', and there is a file with
|
|
385
|
-
modality 'T2', tag 'flair', FlipAngle greater than 5º, and
|
|
386
|
-
ImageType with both values PRIMARY and SECONDARY.
|
|
387
|
-
Further, the acquisition had to be performed in a Manufacturer
|
|
388
|
-
containing 'ge'.
|
|
389
|
-
|
|
390
|
-
Returns
|
|
391
|
-
-------
|
|
392
|
-
dict
|
|
393
|
-
A list of dictionary of {"metadata_name": "metadata_value"}
|
|
394
|
-
|
|
395
|
-
"""
|
|
396
|
-
|
|
397
|
-
content = self.get_subjects_metadata(search_criteria, items=(0, 9999))
|
|
398
|
-
|
|
399
|
-
# Wrap search criteria.
|
|
400
|
-
modality, tags, dicoms = self.__wrap_search_criteria(search_criteria)
|
|
401
|
-
|
|
402
|
-
# Iterate over the files of each subject selected to include/exclude
|
|
403
|
-
# them from the results.
|
|
404
|
-
subjects = list()
|
|
405
|
-
for subject in content:
|
|
406
|
-
files = platform.parse_response(platform.post(
|
|
407
|
-
self._account.auth, "file_manager/get_container_files",
|
|
408
|
-
data={"container_id": str(int(subject["container_id"]))}
|
|
409
|
-
))
|
|
410
|
-
|
|
411
|
-
for file in files["meta"]:
|
|
412
|
-
if modality and \
|
|
413
|
-
modality != (file.get("metadata") or {}).get("modality"):
|
|
414
|
-
continue
|
|
415
|
-
if tags and not all([tag in file.get("tags") for tag in tags]):
|
|
416
|
-
continue
|
|
417
|
-
if dicoms:
|
|
418
|
-
result_values = list()
|
|
419
|
-
for key, dict_value in dicoms.items():
|
|
420
|
-
f_value = ((file.get("metadata") or {})
|
|
421
|
-
.get("info") or {}).get(key)
|
|
422
|
-
d_operator = dict_value["operation"]
|
|
423
|
-
d_value = dict_value["value"]
|
|
424
|
-
result_values.append(
|
|
425
|
-
self.__operation(d_value, d_operator, f_value)
|
|
426
|
-
)
|
|
427
|
-
|
|
428
|
-
if not all(result_values):
|
|
429
|
-
continue
|
|
430
|
-
subjects.append(subject)
|
|
431
|
-
break
|
|
432
|
-
return subjects
|
|
433
|
-
|
|
434
|
-
@property
|
|
435
|
-
def subjects(self):
|
|
436
|
-
"""
|
|
437
|
-
Return the list of subject names (Subject ID) from the selected
|
|
438
|
-
project.
|
|
439
|
-
|
|
440
|
-
:return: a list of subject names
|
|
441
|
-
:rtype: List(Strings)
|
|
442
|
-
"""
|
|
443
|
-
|
|
444
|
-
subjects = self.subjects_metadata
|
|
445
|
-
names = [s["patient_secret_name"] for s in subjects]
|
|
446
|
-
return list(set(names))
|
|
447
|
-
|
|
448
|
-
def check_subject_name(self, subject_name):
|
|
449
|
-
"""
|
|
450
|
-
Check if a given subject name (Subject ID) exists in the selected
|
|
451
|
-
project.
|
|
452
|
-
|
|
453
|
-
Parameters
|
|
454
|
-
----------
|
|
455
|
-
subject_name : str
|
|
456
|
-
Subject ID of the subject to check
|
|
457
|
-
|
|
458
|
-
Returns
|
|
459
|
-
-------
|
|
460
|
-
bool
|
|
461
|
-
True if subject name exists in project, False otherwise
|
|
462
|
-
"""
|
|
463
|
-
|
|
464
|
-
return subject_name in self.subjects
|
|
465
|
-
|
|
466
|
-
@property
|
|
467
|
-
def metadata_parameters(self):
|
|
468
|
-
"""
|
|
469
|
-
List all the parameters in the subject-level metadata.
|
|
470
|
-
|
|
471
|
-
Each project has a set of parameters that define the subjects-level
|
|
472
|
-
metadata. This function returns all these parameters and its
|
|
473
|
-
properties. New subject-level metadata parameters can be creted in the
|
|
474
|
-
QMENTA Platform via the Metadata Manager. The API only allow
|
|
475
|
-
modification of these subject-level metadata parameters via the
|
|
476
|
-
'change_subject_metadata()' method.
|
|
477
|
-
|
|
478
|
-
Returns
|
|
479
|
-
-------
|
|
480
|
-
dict[str] -> dict[str] -> x
|
|
481
|
-
dictionary {"param_name":
|
|
482
|
-
{ "order": Int,
|
|
483
|
-
"tags": [tag1, tag2, ..., ],
|
|
484
|
-
"title: "Title",
|
|
485
|
-
"type": "integer|string|date|list|decimal",
|
|
486
|
-
"visible": 0|1
|
|
487
|
-
}}
|
|
488
|
-
"""
|
|
489
|
-
logger = logging.getLogger(logger_name)
|
|
490
|
-
try:
|
|
491
|
-
data = platform.parse_response(platform.post(self._account.auth, "patient_manager/module_config"))
|
|
492
|
-
except errors.PlatformError:
|
|
493
|
-
logger.error("Could not retrieve metadata parameters.")
|
|
494
|
-
return None
|
|
495
|
-
return data["fields"]
|
|
496
|
-
|
|
497
|
-
def get_analysis(self, analysis_name_or_id):
|
|
136
|
+
""" Upload / Download Data Related Methods """
|
|
137
|
+
def _upload_chunk(
|
|
138
|
+
self,
|
|
139
|
+
data,
|
|
140
|
+
range_str,
|
|
141
|
+
length,
|
|
142
|
+
session_id,
|
|
143
|
+
disposition,
|
|
144
|
+
last_chunk,
|
|
145
|
+
name="",
|
|
146
|
+
date_of_scan="",
|
|
147
|
+
description="",
|
|
148
|
+
subject_name="",
|
|
149
|
+
ssid="",
|
|
150
|
+
filename="DATA.zip",
|
|
151
|
+
input_data_type="mri_brain_data:1.0",
|
|
152
|
+
result=False,
|
|
153
|
+
add_to_container_id=0,
|
|
154
|
+
split_data=False,
|
|
155
|
+
):
|
|
498
156
|
"""
|
|
499
|
-
|
|
157
|
+
Upload a chunk of a file to the platform.
|
|
500
158
|
|
|
501
159
|
Parameters
|
|
502
160
|
----------
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
161
|
+
data
|
|
162
|
+
The file chunk to upload
|
|
163
|
+
range_str
|
|
164
|
+
The string to send that describes the content range
|
|
165
|
+
length
|
|
166
|
+
The content length of the chunk to send
|
|
167
|
+
session_id
|
|
168
|
+
The session ID from the file path
|
|
169
|
+
filename
|
|
170
|
+
The name of the file to be sent
|
|
171
|
+
disposition
|
|
172
|
+
The disposition of the content
|
|
173
|
+
last_chunk
|
|
174
|
+
Set this only for the last chunk to be uploaded.
|
|
175
|
+
All following parameters are ignored when False.
|
|
176
|
+
split_data
|
|
177
|
+
Sets the header that informs the platform to split
|
|
178
|
+
the uploaded file into multiple sessions.
|
|
510
179
|
"""
|
|
511
|
-
if isinstance(analysis_name_or_id, int):
|
|
512
|
-
search_tag = "id"
|
|
513
|
-
elif isinstance(analysis_name_or_id, str):
|
|
514
|
-
if analysis_name_or_id.isdigit():
|
|
515
|
-
search_tag = "id"
|
|
516
|
-
analysis_name_or_id = int(analysis_name_or_id)
|
|
517
|
-
else:
|
|
518
|
-
search_tag = "p_n"
|
|
519
|
-
else:
|
|
520
|
-
raise Exception("The analysis identifier must be its name or an " "integer")
|
|
521
180
|
|
|
522
|
-
|
|
523
|
-
|
|
181
|
+
request_headers = {
|
|
182
|
+
"Content-Type": "application/zip",
|
|
183
|
+
"Content-Range": range_str,
|
|
184
|
+
"Session-ID": str(session_id),
|
|
185
|
+
"Content-Length": str(length),
|
|
186
|
+
"Content-Disposition": disposition,
|
|
524
187
|
}
|
|
525
|
-
response = platform.parse_response(
|
|
526
|
-
platform.post(self._account.auth, "analysis_manager/get_analysis_list", data=search_condition)
|
|
527
|
-
)
|
|
528
|
-
|
|
529
|
-
if len(response) > 1:
|
|
530
|
-
raise Exception(f"multiple analyses with name " f"{analysis_name_or_id} found")
|
|
531
|
-
elif len(response) == 1:
|
|
532
|
-
return response[0]
|
|
533
|
-
else:
|
|
534
|
-
return None
|
|
535
188
|
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
189
|
+
if last_chunk:
|
|
190
|
+
request_headers["X-Mint-Name"] = name
|
|
191
|
+
request_headers["X-Mint-Date"] = date_of_scan
|
|
192
|
+
request_headers["X-Mint-Description"] = description
|
|
193
|
+
request_headers["X-Mint-Patient-Secret"] = subject_name
|
|
194
|
+
request_headers["X-Mint-SSID"] = ssid
|
|
195
|
+
request_headers["X-Mint-Filename"] = filename
|
|
196
|
+
request_headers["X-Mint-Project-Id"] = str(self._project_id)
|
|
197
|
+
request_headers["X-Mint-Split-Data"] = str(int(split_data))
|
|
539
198
|
|
|
540
|
-
|
|
541
|
-
|
|
199
|
+
if input_data_type:
|
|
200
|
+
request_headers["X-Mint-Type"] = input_data_type
|
|
542
201
|
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
"state": "completed"
|
|
548
|
-
}
|
|
549
|
-
list_analysis(search_condition=search_condition)
|
|
202
|
+
if result:
|
|
203
|
+
request_headers["X-Mint-In-Out"] = "out"
|
|
204
|
+
else:
|
|
205
|
+
request_headers["X-Mint-In-Out"] = "in"
|
|
550
206
|
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
search_condition : dict
|
|
554
|
-
- p_n: str or None Analysis name
|
|
555
|
-
- type: str or None Type
|
|
556
|
-
- from_d: str or None dd.mm.yyyy Date from
|
|
557
|
-
- to_d: str or None dd.mm.yyyy Date to
|
|
558
|
-
- qa_status: str or None pass/fail/nd QC status
|
|
559
|
-
- secret_name: str or None Subject ID
|
|
560
|
-
- tags: str or None
|
|
561
|
-
- with_child_analysis: 1 or None if 1, child analysis of workflows will appear
|
|
562
|
-
- id: str or None ID
|
|
563
|
-
- state: running, completed, pending, exception or None
|
|
564
|
-
- username: str or None
|
|
207
|
+
if add_to_container_id > 0:
|
|
208
|
+
request_headers["X-Mint-Add-To"] = str(add_to_container_id)
|
|
565
209
|
|
|
566
|
-
|
|
567
|
-
list containing two elements [min, max] that correspond to the
|
|
568
|
-
mininum and maximum range of analysis listed
|
|
210
|
+
request_headers["X-Requested-With"] = "XMLHttpRequest"
|
|
569
211
|
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
List of analysis, each a dictionary
|
|
574
|
-
"""
|
|
575
|
-
assert len(items) == 2, f"The number of elements in items " f"'{len(items)}' should be equal to two."
|
|
576
|
-
assert all([isinstance(item, int) for item in items]), f"All items elements '{items}' should be integers."
|
|
577
|
-
search_keys = {
|
|
578
|
-
"p_n": str,
|
|
579
|
-
"type": str,
|
|
580
|
-
"from_d": str,
|
|
581
|
-
"to_d": str,
|
|
582
|
-
"qa_status": str,
|
|
583
|
-
"secret_name": str,
|
|
584
|
-
"tags": str,
|
|
585
|
-
"with_child_analysis": int,
|
|
586
|
-
"id": int,
|
|
587
|
-
"state": str,
|
|
588
|
-
"username": str,
|
|
589
|
-
}
|
|
590
|
-
for key in search_condition.keys():
|
|
591
|
-
if key not in search_keys.keys():
|
|
592
|
-
raise Exception((f"This key '{key}' is not accepted by this" "search condition"))
|
|
593
|
-
if not isinstance(search_condition[key], search_keys[key]) and search_condition[key] is not None:
|
|
594
|
-
raise Exception((f"The key {key} in the search condition" f"is not type {search_keys[key]}"))
|
|
595
|
-
req_headers = {"X-Range": f"items={items[0]}-{items[1] - 1}"}
|
|
596
|
-
return platform.parse_response(
|
|
597
|
-
platform.post(
|
|
598
|
-
auth=self._account.auth,
|
|
599
|
-
endpoint="analysis_manager/get_analysis_list",
|
|
600
|
-
headers=req_headers,
|
|
601
|
-
data=search_condition,
|
|
602
|
-
)
|
|
212
|
+
response_time = 900.0 if last_chunk else 120.0
|
|
213
|
+
response = platform.post(
|
|
214
|
+
auth=self._account.auth, endpoint="upload", data=data, headers=request_headers, timeout=response_time
|
|
603
215
|
)
|
|
604
216
|
|
|
605
|
-
|
|
217
|
+
return response
|
|
218
|
+
|
|
219
|
+
def upload_file(
|
|
220
|
+
self,
|
|
221
|
+
file_path,
|
|
222
|
+
subject_name,
|
|
223
|
+
ssid="",
|
|
224
|
+
date_of_scan="",
|
|
225
|
+
description="",
|
|
226
|
+
result=False,
|
|
227
|
+
name="",
|
|
228
|
+
input_data_type="qmenta_mri_brain_data:1.0",
|
|
229
|
+
add_to_container_id=0,
|
|
230
|
+
chunk_size=2**9,
|
|
231
|
+
split_data=False,
|
|
232
|
+
):
|
|
606
233
|
"""
|
|
607
|
-
|
|
234
|
+
Upload a ZIP file to the platform.
|
|
608
235
|
|
|
609
236
|
Parameters
|
|
610
237
|
----------
|
|
238
|
+
file_path : str
|
|
239
|
+
Path to the ZIP file to upload.
|
|
611
240
|
subject_name : str
|
|
612
|
-
Subject ID of the
|
|
241
|
+
Subject ID of the data to upload in the project in QMENTA Platform.
|
|
613
242
|
ssid : str
|
|
614
|
-
Session ID of the
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
the
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
Retrieve the list of input containers available to the user under a
|
|
634
|
-
certain search criteria.
|
|
635
|
-
|
|
636
|
-
Parameters
|
|
637
|
-
----------
|
|
638
|
-
search_criteria : dict
|
|
639
|
-
Each element is a string and is built using the formatting
|
|
640
|
-
"type;value".
|
|
641
|
-
|
|
642
|
-
List of possible keys:
|
|
643
|
-
d_n: container_name # TODO: WHAT IS THIS???
|
|
644
|
-
s_n: subject_id
|
|
645
|
-
Subject ID of the subject in the platform.
|
|
646
|
-
ssid: session_id
|
|
647
|
-
Session ID of the subejct in the platform.
|
|
648
|
-
from_d: from date
|
|
649
|
-
Starting date in which perform the search. Format: DD.MM.YYYY
|
|
650
|
-
to_d: to date
|
|
651
|
-
End date in which perform the search. Format: DD.MM.YYYY
|
|
652
|
-
sets: data sets (modalities) # TODO: WHAT IS THIS???
|
|
653
|
-
|
|
654
|
-
items: Tuple(int, int)
|
|
655
|
-
Starting and ending element of the search.
|
|
243
|
+
Session ID of the Subject ID (i.e., ID of the timepoint).
|
|
244
|
+
date_of_scan : str
|
|
245
|
+
Date of scan/creation of the file
|
|
246
|
+
description : str
|
|
247
|
+
Description of the file
|
|
248
|
+
result : bool
|
|
249
|
+
If result=True then the upload will be taken as an offline analysis
|
|
250
|
+
name : str
|
|
251
|
+
Name of the file in the platform
|
|
252
|
+
input_data_type : str
|
|
253
|
+
qmenta_medical_image_data:3.11
|
|
254
|
+
add_to_container_id : int
|
|
255
|
+
ID of the container to which this file should be added (if id > 0)
|
|
256
|
+
chunk_size : int
|
|
257
|
+
Size in kB of each chunk. Should be expressed as
|
|
258
|
+
a power of 2: 2**x. Default value of x is 9 (chunk_size = 512 kB)
|
|
259
|
+
split_data : bool
|
|
260
|
+
If True, the platform will try to split the uploaded file into
|
|
261
|
+
different sessions. It will be ignored when the ssid is given.
|
|
656
262
|
|
|
657
263
|
Returns
|
|
658
264
|
-------
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
information:
|
|
662
|
-
{"container_name", "container_id", "patient_secret_name", "ssid"}
|
|
265
|
+
bool
|
|
266
|
+
True if correctly uploaded, False otherwise.
|
|
663
267
|
"""
|
|
664
268
|
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
response = platform.parse_response(
|
|
669
|
-
platform.post(
|
|
670
|
-
self._account.auth,
|
|
671
|
-
"file_manager/get_container_list",
|
|
672
|
-
data=search_criteria,
|
|
673
|
-
headers={"X-Range": f"items={items[0]}-{items[1]}"},
|
|
674
|
-
)
|
|
675
|
-
)
|
|
676
|
-
containers = [
|
|
677
|
-
{
|
|
678
|
-
"patient_secret_name": container_item["patient_secret_name"],
|
|
679
|
-
"container_name": container_item["name"],
|
|
680
|
-
"container_id": container_item["_id"],
|
|
681
|
-
"ssid": container_item["ssid"],
|
|
682
|
-
}
|
|
683
|
-
for container_item in response
|
|
684
|
-
]
|
|
685
|
-
return containers
|
|
269
|
+
filename = os.path.split(file_path)[1]
|
|
270
|
+
input_data_type = "offline_analysis:1.0" if result else input_data_type
|
|
686
271
|
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
List the result containers available to the user.
|
|
690
|
-
Examples
|
|
691
|
-
--------
|
|
272
|
+
chunk_size *= 1024
|
|
273
|
+
max_retries = 10
|
|
692
274
|
|
|
693
|
-
|
|
694
|
-
"secret_name":"014_S_6920",
|
|
695
|
-
"from_d": "06.02.2025",
|
|
696
|
-
"with_child_analysis": 1,
|
|
697
|
-
"state": "completed"
|
|
698
|
-
}
|
|
699
|
-
list_result_containers(search_condition=search_condition)
|
|
275
|
+
name = name or os.path.split(file_path)[1]
|
|
700
276
|
|
|
701
|
-
|
|
702
|
-
----------
|
|
703
|
-
search_condition : dict
|
|
704
|
-
- p_n: str or None Analysis name
|
|
705
|
-
- type: str or None Type
|
|
706
|
-
- from_d: str or None dd.mm.yyyy Date from
|
|
707
|
-
- to_d: str or None dd.mm.yyyy Date to
|
|
708
|
-
- qa_status: str or None pass/fail/nd QC status
|
|
709
|
-
- secret_name: str or None Subject ID
|
|
710
|
-
- tags: str or None
|
|
711
|
-
- with_child_analysis: 1 or None if 1, child analysis of workflows will appear
|
|
712
|
-
- id: str or None ID
|
|
713
|
-
- state: running, completed, pending, exception or None
|
|
714
|
-
- username: str or None
|
|
715
|
-
items : List[int]
|
|
716
|
-
list containing two elements [min, max] that correspond to the
|
|
717
|
-
mininum and maximum range of analysis listed
|
|
277
|
+
total_bytes = os.path.getsize(file_path)
|
|
718
278
|
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
List of containers, each a dictionary
|
|
723
|
-
{"name": "container-name", "id": "container_id"}
|
|
724
|
-
if "id": None, that analysis did not had an output container,
|
|
725
|
-
probably it is a workflow
|
|
726
|
-
"""
|
|
727
|
-
analyses = self.list_analysis(search_condition, items)
|
|
728
|
-
return [{"name": analysis["name"], "id": (analysis.get("out_container_id") or None)} for analysis in analyses]
|
|
279
|
+
# making chunks of the file and sending one by one
|
|
280
|
+
logger = logging.getLogger(logger_name)
|
|
281
|
+
with open(file_path, "rb") as file_object:
|
|
729
282
|
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
283
|
+
file_size = os.path.getsize(file_path)
|
|
284
|
+
if file_size == 0:
|
|
285
|
+
logger.error("Cannot upload empty file {}".format(file_path))
|
|
286
|
+
return False
|
|
287
|
+
uploaded = 0
|
|
288
|
+
session_id = self.__get_session_id(file_path)
|
|
289
|
+
chunk_num = 0
|
|
290
|
+
retries_count = 0
|
|
291
|
+
uploaded_bytes = 0
|
|
292
|
+
response = None
|
|
293
|
+
last_chunk = False
|
|
740
294
|
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
List of file names (strings)
|
|
745
|
-
"""
|
|
746
|
-
try:
|
|
747
|
-
content = platform.parse_response(
|
|
748
|
-
platform.post(
|
|
749
|
-
self._account.auth, "file_manager/get_container_files", data={"container_id": container_id}
|
|
750
|
-
)
|
|
751
|
-
)
|
|
752
|
-
except errors.PlatformError as e:
|
|
753
|
-
logging.getLogger(logger_name).error(e)
|
|
754
|
-
return False
|
|
755
|
-
if "files" not in content.keys():
|
|
756
|
-
logging.getLogger(logger_name).error("Could not get files")
|
|
757
|
-
return False
|
|
758
|
-
return content["files"]
|
|
295
|
+
if ssid and split_data:
|
|
296
|
+
logger.warning("split-data argument will be ignored because" + " ssid has been specified")
|
|
297
|
+
split_data = False
|
|
759
298
|
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
metadata_info={},
|
|
765
|
-
tags=[]
|
|
766
|
-
):
|
|
767
|
-
"""
|
|
768
|
-
List the name of the files available inside a given container.
|
|
769
|
-
search condition example:
|
|
770
|
-
"metadata": {"SliceThickness":1},
|
|
771
|
-
}
|
|
772
|
-
Parameters
|
|
773
|
-
----------
|
|
774
|
-
container_id : str or int
|
|
775
|
-
Container identifier.
|
|
299
|
+
while True:
|
|
300
|
+
data = file_object.read(chunk_size)
|
|
301
|
+
if not data:
|
|
302
|
+
break
|
|
776
303
|
|
|
777
|
-
|
|
778
|
-
|
|
304
|
+
start_position = chunk_num * chunk_size
|
|
305
|
+
end_position = start_position + chunk_size - 1
|
|
306
|
+
bytes_to_send = chunk_size
|
|
779
307
|
|
|
780
|
-
|
|
781
|
-
|
|
308
|
+
if end_position >= total_bytes:
|
|
309
|
+
last_chunk = True
|
|
310
|
+
end_position = total_bytes - 1
|
|
311
|
+
bytes_to_send = total_bytes - uploaded_bytes
|
|
782
312
|
|
|
783
|
-
|
|
784
|
-
List of strings containing the tags of the files being filtered
|
|
313
|
+
bytes_range = "bytes " + str(start_position) + "-" + str(end_position) + "/" + str(total_bytes)
|
|
785
314
|
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
315
|
+
dispstr = f"attachment; filename={filename}"
|
|
316
|
+
response = self._upload_chunk(
|
|
317
|
+
data,
|
|
318
|
+
bytes_range,
|
|
319
|
+
bytes_to_send,
|
|
320
|
+
session_id,
|
|
321
|
+
dispstr,
|
|
322
|
+
last_chunk,
|
|
323
|
+
name,
|
|
324
|
+
date_of_scan,
|
|
325
|
+
description,
|
|
326
|
+
subject_name,
|
|
327
|
+
ssid,
|
|
328
|
+
filename,
|
|
329
|
+
input_data_type,
|
|
330
|
+
result,
|
|
331
|
+
add_to_container_id,
|
|
332
|
+
split_data,
|
|
804
333
|
)
|
|
805
|
-
for key in metadata_info.keys():
|
|
806
|
-
meta_key = (
|
|
807
|
-
(
|
|
808
|
-
metadata_file.get("metadata") or {}
|
|
809
|
-
).get("info") or {}).get(
|
|
810
|
-
key
|
|
811
|
-
)
|
|
812
|
-
if meta_key is None:
|
|
813
|
-
logging.getLogger(logger_name).warning(
|
|
814
|
-
f"{key} is not in file_info from file {file}"
|
|
815
|
-
)
|
|
816
|
-
info_bool.append(
|
|
817
|
-
metadata_info[key] == meta_key
|
|
818
|
-
)
|
|
819
|
-
if all(tags_bool) and all(info_bool) and modality_bool:
|
|
820
|
-
selected_files.append(file)
|
|
821
|
-
return selected_files
|
|
822
334
|
|
|
823
|
-
|
|
335
|
+
if response is None:
|
|
336
|
+
retries_count += 1
|
|
337
|
+
time.sleep(retries_count * 5)
|
|
338
|
+
if retries_count > max_retries:
|
|
339
|
+
error_message = "HTTP Connection Problem"
|
|
340
|
+
logger.error(error_message)
|
|
341
|
+
break
|
|
342
|
+
elif int(response.status_code) == 201:
|
|
343
|
+
chunk_num += 1
|
|
344
|
+
retries_count = 0
|
|
345
|
+
uploaded_bytes += chunk_size
|
|
346
|
+
elif int(response.status_code) == 200:
|
|
347
|
+
self.__show_progress(file_size, file_size, finish=True)
|
|
348
|
+
break
|
|
349
|
+
elif int(response.status_code) == 416:
|
|
350
|
+
retries_count += 1
|
|
351
|
+
time.sleep(retries_count * 5)
|
|
352
|
+
if retries_count > self.max_retries:
|
|
353
|
+
error_message = "Error Code: 416; " "Requested Range Not Satisfiable (NGINX)"
|
|
354
|
+
logger.error(error_message)
|
|
355
|
+
break
|
|
356
|
+
else:
|
|
357
|
+
retries_count += 1
|
|
358
|
+
time.sleep(retries_count * 5)
|
|
359
|
+
if retries_count > max_retries:
|
|
360
|
+
error_message = "Number of retries has been reached. " "Upload process stops here !"
|
|
361
|
+
logger.error(error_message)
|
|
362
|
+
break
|
|
363
|
+
|
|
364
|
+
uploaded += chunk_size
|
|
365
|
+
self.__show_progress(uploaded, file_size)
|
|
366
|
+
|
|
367
|
+
try:
|
|
368
|
+
platform.parse_response(response)
|
|
369
|
+
except errors.PlatformError as error:
|
|
370
|
+
logger.error(error)
|
|
371
|
+
return False
|
|
372
|
+
|
|
373
|
+
message = "Your data was successfully uploaded."
|
|
374
|
+
message += "The uploaded file will be soon processed !"
|
|
375
|
+
logger.info(message)
|
|
376
|
+
return True
|
|
377
|
+
|
|
378
|
+
def upload_mri(self, file_path, subject_name):
|
|
824
379
|
"""
|
|
825
|
-
|
|
380
|
+
Upload new MRI data to the subject.
|
|
826
381
|
|
|
827
382
|
Parameters
|
|
828
383
|
----------
|
|
829
|
-
|
|
830
|
-
|
|
384
|
+
file_path : str
|
|
385
|
+
Path to the file to upload
|
|
386
|
+
subject_name: str
|
|
831
387
|
|
|
832
388
|
Returns
|
|
833
389
|
-------
|
|
834
|
-
|
|
835
|
-
|
|
390
|
+
bool
|
|
391
|
+
True if upload was correctly done, False otherwise.
|
|
836
392
|
"""
|
|
837
393
|
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
platform.post(
|
|
841
|
-
self._account.auth, "file_manager/get_container_files", data={"container_id": container_id}
|
|
842
|
-
)
|
|
843
|
-
)
|
|
844
|
-
except errors.PlatformError as e:
|
|
845
|
-
logging.getLogger(logger_name).error(e)
|
|
846
|
-
return False
|
|
847
|
-
|
|
848
|
-
return data["meta"]
|
|
394
|
+
if self.__check_upload_file(file_path):
|
|
395
|
+
return self.upload_file(file_path, subject_name)
|
|
849
396
|
|
|
850
|
-
def
|
|
397
|
+
def upload_gametection(self, file_path, subject_name):
|
|
851
398
|
"""
|
|
852
|
-
|
|
399
|
+
Upload new Gametection data to the subject.
|
|
853
400
|
|
|
854
401
|
Parameters
|
|
855
402
|
----------
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
Name of the file.
|
|
403
|
+
file_path : str
|
|
404
|
+
Path to the file to upload
|
|
405
|
+
subject_name: str
|
|
860
406
|
|
|
861
407
|
Returns
|
|
862
408
|
-------
|
|
863
|
-
|
|
864
|
-
|
|
409
|
+
bool
|
|
410
|
+
True if upload was correctly done, False otherwise.
|
|
865
411
|
"""
|
|
866
|
-
all_metadata = self.list_container_files_metadata(container_id)
|
|
867
|
-
if all_metadata:
|
|
868
|
-
for file_meta in all_metadata:
|
|
869
|
-
if file_meta["name"] == filename:
|
|
870
|
-
return file_meta
|
|
871
|
-
else:
|
|
872
|
-
return False
|
|
873
412
|
|
|
874
|
-
|
|
413
|
+
if self.__check_upload_file(file_path):
|
|
414
|
+
return self.upload_file(file_path, subject_name, input_data_type="parkinson_gametection")
|
|
415
|
+
return False
|
|
416
|
+
|
|
417
|
+
def upload_result(self, file_path, subject_name):
|
|
875
418
|
"""
|
|
876
|
-
|
|
419
|
+
Upload new result data to the subject.
|
|
877
420
|
|
|
878
421
|
Parameters
|
|
879
422
|
----------
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
Name of the file to be edited.
|
|
884
|
-
modality : str or None
|
|
885
|
-
Modality identifier, or None if the file shouldn't have
|
|
886
|
-
any modality
|
|
887
|
-
tags : list[str] or None
|
|
888
|
-
List of tags, or None if the filename shouldn't have any tags
|
|
889
|
-
"""
|
|
423
|
+
file_path : str
|
|
424
|
+
Path to the file to upload
|
|
425
|
+
subject_name: str
|
|
890
426
|
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
427
|
+
Returns
|
|
428
|
+
-------
|
|
429
|
+
bool
|
|
430
|
+
True if upload was correctly done, False otherwise.
|
|
431
|
+
"""
|
|
432
|
+
|
|
433
|
+
if self.__check_upload_file(file_path):
|
|
434
|
+
return self.upload_file(file_path, subject_name, result=True)
|
|
435
|
+
return False
|
|
899
436
|
|
|
900
437
|
def download_file(self, container_id, file_name, local_filename=False, overwrite=False):
|
|
901
438
|
"""
|
|
@@ -913,6 +450,13 @@ class Project:
|
|
|
913
450
|
Whether to overwrite the file if existing.
|
|
914
451
|
"""
|
|
915
452
|
logger = logging.getLogger(logger_name)
|
|
453
|
+
if not isinstance(file_name, str):
|
|
454
|
+
raise ValueError("The name of the file to download (file_name) "
|
|
455
|
+
"should be of type string.")
|
|
456
|
+
if not isinstance(file_name, str):
|
|
457
|
+
raise ValueError("The name of the output file (local_filename) "
|
|
458
|
+
"should be of type string.")
|
|
459
|
+
|
|
916
460
|
if file_name not in self.list_container_files(container_id):
|
|
917
461
|
msg = f'File "{file_name}" does not exist in container ' f"{container_id}"
|
|
918
462
|
logger.error(msg)
|
|
@@ -954,6 +498,14 @@ class Project:
|
|
|
954
498
|
Name of the zip where the downloaded files are stored.
|
|
955
499
|
"""
|
|
956
500
|
logger = logging.getLogger(logger_name)
|
|
501
|
+
|
|
502
|
+
if not all([isinstance(file_name, str) for file_name in filenames]):
|
|
503
|
+
raise ValueError("The name of the files to download (filenames) "
|
|
504
|
+
"should be of type string.")
|
|
505
|
+
if not isinstance(zip_name, str):
|
|
506
|
+
raise ValueError("The name of the output ZIP file (zip_name) "
|
|
507
|
+
"should be of type string.")
|
|
508
|
+
|
|
957
509
|
files_not_in_container = list(filter(lambda f: f not in self.list_container_files(container_id), filenames))
|
|
958
510
|
|
|
959
511
|
if files_not_in_container:
|
|
@@ -980,29 +532,307 @@ class Project:
|
|
|
980
532
|
logger.info("Files from container {} saved to {}".format(container_id, zip_name))
|
|
981
533
|
return True
|
|
982
534
|
|
|
983
|
-
def
|
|
984
|
-
"""
|
|
985
|
-
|
|
986
|
-
|
|
535
|
+
def copy_container_to_project(self, container_id, project_id):
|
|
536
|
+
"""
|
|
537
|
+
Copy a container to another project.
|
|
538
|
+
|
|
539
|
+
Parameters
|
|
540
|
+
----------
|
|
541
|
+
container_id : int
|
|
542
|
+
ID of the container to copy.
|
|
543
|
+
project_id : int or str
|
|
544
|
+
ID of the project to retrieve, either the numeric ID or the name
|
|
545
|
+
|
|
546
|
+
Returns
|
|
547
|
+
-------
|
|
548
|
+
bool
|
|
549
|
+
True on success, False on fail
|
|
550
|
+
"""
|
|
551
|
+
|
|
552
|
+
if type(project_id) == int or type(project_id) == float:
|
|
553
|
+
p_id = int(project_id)
|
|
554
|
+
elif type(project_id) == str:
|
|
555
|
+
projects = self._account.projects
|
|
556
|
+
projects_match = [proj for proj in projects if proj["name"] == project_id]
|
|
557
|
+
if not projects_match:
|
|
558
|
+
raise Exception(f"Project {project_id}" + " does not exist or is not available for this user.")
|
|
559
|
+
p_id = int(projects_match[0]["id"])
|
|
560
|
+
else:
|
|
561
|
+
raise TypeError("project_id")
|
|
562
|
+
data = {
|
|
563
|
+
"container_id": container_id,
|
|
564
|
+
"project_id": p_id,
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
try:
|
|
568
|
+
platform.parse_response(
|
|
569
|
+
platform.post(self._account.auth, "file_manager/copy_container_to_another_project", data=data)
|
|
570
|
+
)
|
|
571
|
+
except errors.PlatformError as e:
|
|
572
|
+
logging.getLogger(logger_name).error("Couldn not copy container: {}".format(e))
|
|
573
|
+
return False
|
|
574
|
+
|
|
575
|
+
return True
|
|
576
|
+
|
|
577
|
+
""" Subject/Session Related Methods """
|
|
578
|
+
@property
|
|
579
|
+
def subjects(self):
|
|
580
|
+
"""
|
|
581
|
+
Return the list of subject names (Subject ID) from the selected
|
|
582
|
+
project.
|
|
583
|
+
|
|
584
|
+
:return: a list of subject names
|
|
585
|
+
:rtype: List(Strings)
|
|
586
|
+
"""
|
|
587
|
+
|
|
588
|
+
subjects = self.subjects_metadata
|
|
589
|
+
names = [s["patient_secret_name"] for s in subjects]
|
|
590
|
+
return list(set(names))
|
|
591
|
+
|
|
592
|
+
@property
|
|
593
|
+
def subjects_metadata(self):
|
|
594
|
+
"""
|
|
595
|
+
List all subject data from the selected project.
|
|
596
|
+
|
|
597
|
+
Returns
|
|
598
|
+
-------
|
|
599
|
+
dict
|
|
600
|
+
A list of dictionary of {"metadata_name": "metadata_value"}
|
|
601
|
+
"""
|
|
602
|
+
return self.get_subjects_metadata()
|
|
603
|
+
|
|
604
|
+
@property
|
|
605
|
+
def metadata_parameters(self):
|
|
606
|
+
"""
|
|
607
|
+
List all the parameters in the subject-level metadata.
|
|
608
|
+
|
|
609
|
+
Each project has a set of parameters that define the subjects-level
|
|
610
|
+
metadata. This function returns all these parameters and its
|
|
611
|
+
properties. New subject-level metadata parameters can be creted in the
|
|
612
|
+
QMENTA Platform via the Metadata Manager. The API only allow
|
|
613
|
+
modification of these subject-level metadata parameters via the
|
|
614
|
+
'change_subject_metadata()' method.
|
|
615
|
+
|
|
616
|
+
Returns
|
|
617
|
+
-------
|
|
618
|
+
dict[str] -> dict[str] -> x
|
|
619
|
+
dictionary {"param_name":
|
|
620
|
+
{ "order": Int,
|
|
621
|
+
"tags": [tag1, tag2, ..., ],
|
|
622
|
+
"title: "Title",
|
|
623
|
+
"type": "integer|string|date|list|decimal",
|
|
624
|
+
"visible": 0|1
|
|
625
|
+
}}
|
|
626
|
+
"""
|
|
627
|
+
logger = logging.getLogger(logger_name)
|
|
628
|
+
try:
|
|
629
|
+
data = platform.parse_response(platform.post(self._account.auth, "patient_manager/module_config"))
|
|
630
|
+
except errors.PlatformError:
|
|
631
|
+
logger.error("Could not retrieve metadata parameters.")
|
|
632
|
+
return None
|
|
633
|
+
return data["fields"]
|
|
634
|
+
|
|
635
|
+
def check_subject_name(self, subject_name):
|
|
636
|
+
"""
|
|
637
|
+
Check if a given subject name (Subject ID) exists in the selected
|
|
638
|
+
project.
|
|
639
|
+
|
|
640
|
+
Parameters
|
|
641
|
+
----------
|
|
642
|
+
subject_name : str
|
|
643
|
+
Subject ID of the subject to check
|
|
644
|
+
|
|
645
|
+
Returns
|
|
646
|
+
-------
|
|
647
|
+
bool
|
|
648
|
+
True if subject name exists in project, False otherwise
|
|
649
|
+
"""
|
|
650
|
+
|
|
651
|
+
return subject_name in self.subjects
|
|
652
|
+
|
|
653
|
+
def get_subject_container_id(self, subject_name, ssid):
|
|
654
|
+
"""
|
|
655
|
+
Given a Subject ID and Session ID, return its Container ID.
|
|
656
|
+
|
|
657
|
+
Parameters
|
|
658
|
+
----------
|
|
659
|
+
subject_name : str
|
|
660
|
+
Subject ID of the subject in the project.
|
|
661
|
+
ssid : str
|
|
662
|
+
Session ID of the subject in the project.
|
|
663
|
+
|
|
664
|
+
Returns
|
|
665
|
+
-------
|
|
666
|
+
int or bool
|
|
667
|
+
The Container ID of the subject in the project, or False if
|
|
668
|
+
the subject is not found.
|
|
669
|
+
"""
|
|
670
|
+
|
|
671
|
+
search_criteria = {"s_n": subject_name, "ssid": ssid}
|
|
672
|
+
response = self.list_input_containers(search_criteria=search_criteria)
|
|
673
|
+
|
|
674
|
+
for subject in response:
|
|
675
|
+
if subject["patient_secret_name"] == subject_name and subject["ssid"] == ssid:
|
|
676
|
+
return subject["container_id"]
|
|
677
|
+
return False
|
|
678
|
+
|
|
679
|
+
def get_subject_id(self, subject_name, ssid):
|
|
680
|
+
"""
|
|
681
|
+
Given a Subject ID and Session ID, return its Patient ID in the
|
|
682
|
+
project.
|
|
683
|
+
|
|
684
|
+
Parameters
|
|
685
|
+
----------
|
|
686
|
+
subject_name : str
|
|
687
|
+
Subject ID of the subject in the project.
|
|
688
|
+
ssid : str
|
|
689
|
+
Session ID of the subject in the project.
|
|
690
|
+
|
|
691
|
+
Returns
|
|
692
|
+
-------
|
|
693
|
+
int or bool
|
|
694
|
+
The ID of the subject in the project, or False if
|
|
695
|
+
the subject is not found.
|
|
696
|
+
"""
|
|
697
|
+
|
|
698
|
+
for user in self.get_subjects_metadata():
|
|
699
|
+
if user["patient_secret_name"] == str(subject_name) and user["ssid"] == str(ssid):
|
|
700
|
+
return int(user["_id"])
|
|
701
|
+
return False
|
|
702
|
+
|
|
703
|
+
def get_subjects_metadata(self, search_criteria={}, items=(0, 9999)):
|
|
704
|
+
"""
|
|
705
|
+
List all Subject ID/Session ID from the selected project that meet the
|
|
706
|
+
defined search criteria at a session level.
|
|
707
|
+
|
|
708
|
+
Parameters
|
|
709
|
+
----------
|
|
710
|
+
search_criteria: dict
|
|
711
|
+
Each element is a string and is built using the formatting
|
|
712
|
+
"type;value", or "type;operation|value"
|
|
713
|
+
|
|
714
|
+
Complete search_criteria Dictionary Explanation:
|
|
715
|
+
|
|
716
|
+
search_criteria = {
|
|
717
|
+
"pars_patient_secret_name": "string;SUBJECTID",
|
|
718
|
+
"pars_ssid": "integer;OPERATOR|SSID",
|
|
719
|
+
"pars_modalities": "string;MODALITY",
|
|
720
|
+
"pars_tags": "tags;TAGS",
|
|
721
|
+
"pars_age_at_scan": "integer;OPERATOR|AGE_AT_SCAN",
|
|
722
|
+
"pars_[dicom]_KEY": "KEYTYPE;KEYVALUE",
|
|
723
|
+
"pars_PROJECTMETADATA": "METADATATYPE;METADATAVALUE",
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
Where:
|
|
727
|
+
"pars_patient_secret_name": Applies the search to the 'Subject ID'.
|
|
728
|
+
SUBJECTID is a comma separated list of strings.
|
|
729
|
+
"pars_ssid": Applies the search to the 'Session ID'.
|
|
730
|
+
SSID is an integer.
|
|
731
|
+
OPERATOR is the operator to apply. One of:
|
|
732
|
+
- Equal: eq
|
|
733
|
+
- Different Than: ne
|
|
734
|
+
- Greater Than: gt
|
|
735
|
+
- Greater/Equal To: gte
|
|
736
|
+
- Lower Than: lt
|
|
737
|
+
- Lower/Equal To: lte
|
|
738
|
+
|
|
739
|
+
"pars_modalities": Applies the search to the file 'Modalities'
|
|
740
|
+
available within each Subject ID.
|
|
741
|
+
MODALITY is a comma separated list of string. A session is provided as
|
|
742
|
+
long as one MODALITY is available.
|
|
743
|
+
"pars_tags": Applies the search to the file 'Tags' available within
|
|
744
|
+
each Subject ID and to the subject-level 'Tags'.
|
|
745
|
+
TAGS is a comma separated list of strings. A session is provided as
|
|
746
|
+
long as one tag is available.
|
|
747
|
+
"pars_age_at_scan": Applies the search to the 'age_at_scan' metadata
|
|
748
|
+
field.
|
|
749
|
+
AGE_AT_SCAN is an integer.
|
|
750
|
+
"pars_[dicom]_KEY": Applies the search to the metadata fields
|
|
751
|
+
available within each file. KEY must be one of the
|
|
752
|
+
metadata keys of the files. The full list of KEYS for a given
|
|
753
|
+
file is shown in the QMENTA Platform within the File Information
|
|
754
|
+
of such File.
|
|
755
|
+
KEYTYPE is the type of the KEY. One of:
|
|
756
|
+
- integer
|
|
757
|
+
- decimal
|
|
758
|
+
- string
|
|
759
|
+
- list
|
|
760
|
+
|
|
761
|
+
if 'integer' or 'decimal' you must also include an OPERATOR
|
|
762
|
+
(i.e., "integer;OPERATOR|KEYVALUE").
|
|
763
|
+
if 'list' the KEYVALUE should be a semicolon separated list of
|
|
764
|
+
values. (i.e., "list;KEYVALUE1;KEYVALUE2;KEYVALUE3)
|
|
765
|
+
KEYVALUEs must be strings.
|
|
766
|
+
KEYVALUE is the expected value of the KEY.
|
|
767
|
+
"pars_[dicom]_PROJECTMETADATA": Applies to the metadata defined
|
|
768
|
+
within the 'Metadata Manager' of the project.
|
|
769
|
+
PROJECTMETADATA is the ID of the metadata field.
|
|
770
|
+
METADATATYPE is the type of the metadata field. One of:
|
|
771
|
+
- string
|
|
772
|
+
- integer
|
|
773
|
+
- list
|
|
774
|
+
- decimal
|
|
775
|
+
- single_option
|
|
776
|
+
- multiple_option
|
|
777
|
+
|
|
778
|
+
if 'integer' or 'decimal' you must also include an OPERATOR
|
|
779
|
+
(i.e., "integer;OPERATOR|METADATAVALUE").
|
|
780
|
+
KEYVALUE is the expected value of the metadata.
|
|
781
|
+
|
|
782
|
+
1) Example:
|
|
783
|
+
search_criteria = {
|
|
784
|
+
"pars_patient_secret_name": "string;abide",
|
|
785
|
+
"pars_ssid": "integer;eq|2"
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
2) Example:
|
|
789
|
+
search_criteria = {
|
|
790
|
+
"pars_modalities": "string;T1",
|
|
791
|
+
"pars_tags": "tags;flair",
|
|
792
|
+
"pars_[dicom]_Manufacturer": "string;ge",
|
|
793
|
+
"pars_[dicom]_FlipAngle": "integer;gt|5",
|
|
794
|
+
"pars_[dicom]_ImageType": "list;PRIMARY;SECONDARY",
|
|
795
|
+
}
|
|
987
796
|
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
Subject ID
|
|
992
|
-
|
|
993
|
-
|
|
797
|
+
Note the search criteria applies to all the files included within a
|
|
798
|
+
session. Hence, it does not imply that all the criteria conditions
|
|
799
|
+
are applied to the same files. In example 2) above, it means that
|
|
800
|
+
any Subject ID/Session ID that has a file classified with a 'T1'
|
|
801
|
+
modality, a file with a 'flair' tag, a file whose Manufacturer
|
|
802
|
+
contains 'ge', a file whose FlipAngle is greater than '5º', and a
|
|
803
|
+
file with ImageType with any of the values: PRIMARY or SECONDARY
|
|
804
|
+
will be selected.
|
|
994
805
|
|
|
995
806
|
Returns
|
|
996
807
|
-------
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
808
|
+
dict
|
|
809
|
+
A list of dictionary of {"metadata_name": "metadata_value"}
|
|
810
|
+
|
|
1000
811
|
"""
|
|
1001
812
|
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
813
|
+
assert len(items) == 2, f"The number of elements in items " f"'{len(items)}' should be equal to two."
|
|
814
|
+
assert all([isinstance(item, int) for item in items]), f"All items elements '{items}' should be integers."
|
|
815
|
+
|
|
816
|
+
assert all([key[:5] == "pars_" for key in search_criteria.keys()]), (
|
|
817
|
+
f"All keys of the search_criteria dictionary " f"'{search_criteria.keys()}' must start with 'pars_'."
|
|
818
|
+
)
|
|
819
|
+
|
|
820
|
+
for key, value in search_criteria.items():
|
|
821
|
+
if value.split(";")[0] in ["integer", "decimal"]:
|
|
822
|
+
assert value.split(";")[1].split("|")[0] in OPERATOR_LIST, (
|
|
823
|
+
f"Search criteria of type '{value.split(';')[0]}' must "
|
|
824
|
+
f"include an operator ({', '.join(OPERATOR_LIST)})."
|
|
825
|
+
)
|
|
826
|
+
|
|
827
|
+
content = platform.parse_response(
|
|
828
|
+
platform.post(
|
|
829
|
+
self._account.auth,
|
|
830
|
+
"patient_manager/get_patient_list",
|
|
831
|
+
data=search_criteria,
|
|
832
|
+
headers={"X-Range": f"items={items[0]}-{items[1]}"},
|
|
833
|
+
)
|
|
834
|
+
)
|
|
835
|
+
return content
|
|
1006
836
|
|
|
1007
837
|
def change_subject_metadata(self, patient_id, subject_name, ssid, tags, age_at_scan, metadata):
|
|
1008
838
|
"""
|
|
@@ -1070,25 +900,230 @@ class Project:
|
|
|
1070
900
|
f"interface (GUI)."
|
|
1071
901
|
)
|
|
1072
902
|
|
|
1073
|
-
post_data = {
|
|
1074
|
-
"patient_id": patient_id,
|
|
1075
|
-
"secret_name": str(subject_name),
|
|
1076
|
-
"ssid": str(ssid),
|
|
1077
|
-
"tags": ",".join(tags),
|
|
1078
|
-
"age_at_scan": age_at_scan,
|
|
1079
|
-
}
|
|
1080
|
-
for key, value in metadata.items():
|
|
1081
|
-
id = key[3:] if "md_" == key[:3] else key
|
|
1082
|
-
post_data[f"last_vals.{id}"] = value
|
|
903
|
+
post_data = {
|
|
904
|
+
"patient_id": patient_id,
|
|
905
|
+
"secret_name": str(subject_name),
|
|
906
|
+
"ssid": str(ssid),
|
|
907
|
+
"tags": ",".join(tags),
|
|
908
|
+
"age_at_scan": age_at_scan,
|
|
909
|
+
}
|
|
910
|
+
for key, value in metadata.items():
|
|
911
|
+
id = key[3:] if "md_" == key[:3] else key
|
|
912
|
+
post_data[f"last_vals.{id}"] = value
|
|
913
|
+
|
|
914
|
+
try:
|
|
915
|
+
platform.parse_response(platform.post(self._account.auth, "patient_manager/upsert_patient", data=post_data))
|
|
916
|
+
except errors.PlatformError:
|
|
917
|
+
logger.error(f"Patient ID '{patient_id}' could not be modified.")
|
|
918
|
+
return False
|
|
919
|
+
|
|
920
|
+
logger.info(f"Patient ID '{patient_id}' successfully modified.")
|
|
921
|
+
return True
|
|
922
|
+
|
|
923
|
+
def get_subjects_files_metadata(self, search_criteria={}, items=(0, 9999)):
|
|
924
|
+
"""
|
|
925
|
+
List all Subject ID/Session ID from the selected project that meet the
|
|
926
|
+
defined search criteria at a file level.
|
|
927
|
+
|
|
928
|
+
Note, albeit the search criteria is similar to the one defined in
|
|
929
|
+
method 'get_subjects_metadata()' (see differences below), the
|
|
930
|
+
output is different as this method provides the sessions which
|
|
931
|
+
have a file that satisfy all the conditions of the search criteria.
|
|
932
|
+
This method is slow.
|
|
933
|
+
|
|
934
|
+
Parameters
|
|
935
|
+
----------
|
|
936
|
+
search_criteria: dict
|
|
937
|
+
Each element is a string and is built using the formatting
|
|
938
|
+
"type;value", or "type;operation|value"
|
|
939
|
+
|
|
940
|
+
Complete search_criteria Dictionary Explanation:
|
|
941
|
+
|
|
942
|
+
search_criteria = {
|
|
943
|
+
"pars_patient_secret_name": "string;SUBJECTID",
|
|
944
|
+
"pars_ssid": "integer;OPERATOR|SSID",
|
|
945
|
+
"pars_modalities": "string;MODALITY",
|
|
946
|
+
"pars_tags": "tags;TAGS",
|
|
947
|
+
"pars_age_at_scan": "integer;OPERATOR|AGE_AT_SCAN",
|
|
948
|
+
"pars_[dicom]_KEY": "KEYTYPE;KEYVALUE",
|
|
949
|
+
"pars_PROJECTMETADATA": "METADATATYPE;METADATAVALUE",
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
Where:
|
|
953
|
+
"pars_patient_secret_name": Applies the search to the 'Subject ID'.
|
|
954
|
+
SUBJECTID is a comma separated list of strings.
|
|
955
|
+
"pars_ssid": Applies the search to the 'Session ID'.
|
|
956
|
+
SSID is an integer.
|
|
957
|
+
OPERATOR is the operator to apply. One of:
|
|
958
|
+
- Equal: eq
|
|
959
|
+
- Different Than: ne
|
|
960
|
+
- Greater Than: gt
|
|
961
|
+
- Greater/Equal To: gte
|
|
962
|
+
- Lower Than: lt
|
|
963
|
+
- Lower/Equal To: lte
|
|
964
|
+
|
|
965
|
+
"pars_modalities": Applies the search to the file 'Modalities'
|
|
966
|
+
available within each Subject ID.
|
|
967
|
+
MODALITY is a string.
|
|
968
|
+
"pars_tags": Applies only the search to the file 'Tags' available
|
|
969
|
+
within each Subject ID.
|
|
970
|
+
TAGS is a comma separated list of strings. All tags must be present in
|
|
971
|
+
the same file.
|
|
972
|
+
"pars_age_at_scan": Applies the search to the 'age_at_scan' metadata
|
|
973
|
+
field.
|
|
974
|
+
AGE_AT_SCAN is an integer.
|
|
975
|
+
"pars_[dicom]_KEY": Applies the search to the metadata fields
|
|
976
|
+
available within each file. KEY must be one of the
|
|
977
|
+
metadata keys of the files. The full list of KEYS for a given
|
|
978
|
+
file is shown in the QMENTA Platform within the File Information
|
|
979
|
+
of such File.
|
|
980
|
+
KEYTYPE is the type of the KEY. One of:
|
|
981
|
+
- integer
|
|
982
|
+
- decimal
|
|
983
|
+
- string
|
|
984
|
+
- list
|
|
985
|
+
|
|
986
|
+
if 'integer' or 'decimal' you must also include an OPERATOR
|
|
987
|
+
(i.e., "integer;OPERATOR|KEYVALUE").
|
|
988
|
+
if 'list' the KEYVALUE should be a semicolon separated list of
|
|
989
|
+
values. (i.e., "list;KEYVALUE1;KEYVALUE2;KEYVALUE3)
|
|
990
|
+
KEYVALUEs must be strings.
|
|
991
|
+
KEYVALUE is the expected value of the KEY.
|
|
992
|
+
"pars_[dicom]_PROJECTMETADATA": Applies to the metadata defined
|
|
993
|
+
within the 'Metadata Manager' of the project.
|
|
994
|
+
PROJECTMETADATA is the ID of the metadata field.
|
|
995
|
+
METADATATYPE is the type of the metadata field. One of:
|
|
996
|
+
- string
|
|
997
|
+
- integer
|
|
998
|
+
- list
|
|
999
|
+
- decimal
|
|
1000
|
+
- single_option
|
|
1001
|
+
- multiple_option
|
|
1002
|
+
|
|
1003
|
+
if 'integer' or 'decimal' you must also include an OPERATOR
|
|
1004
|
+
(i.e., "integer;OPERATOR|METADATAVALUE").
|
|
1005
|
+
KEYVALUE is the expected value of the metadata.
|
|
1006
|
+
|
|
1007
|
+
1) Example:
|
|
1008
|
+
search_criteria = {
|
|
1009
|
+
"pars_patient_secret_name": "string;abide",
|
|
1010
|
+
"pars_ssid": "integer;eq|2"
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
2) Example:
|
|
1014
|
+
search_criteria = {
|
|
1015
|
+
"pars_patient_secret_name": "string;001"
|
|
1016
|
+
"pars_modalities": "string;T2",
|
|
1017
|
+
"pars_tags": "tags;flair",
|
|
1018
|
+
"pars_[dicom]_Manufacturer": "string;ge",
|
|
1019
|
+
"pars_[dicom]_FlipAngle": "integer;gt|5",
|
|
1020
|
+
"pars_[dicom]_ImageType": "list;PRIMARY;SECONDARY",
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
Note the search criteria might apply to both the files metadata
|
|
1024
|
+
information available within a session and the metadata of the
|
|
1025
|
+
session. And the method provides a session only if all the file
|
|
1026
|
+
related conditions are satisfied within the same file.
|
|
1027
|
+
In example 2) above, it means that the output will contain any
|
|
1028
|
+
session whose Subject ID contains '001', and there is a file with
|
|
1029
|
+
modality 'T2', tag 'flair', FlipAngle greater than 5º, and
|
|
1030
|
+
ImageType with both values PRIMARY and SECONDARY.
|
|
1031
|
+
Further, the acquisition had to be performed in a Manufacturer
|
|
1032
|
+
containing 'ge'.
|
|
1033
|
+
|
|
1034
|
+
Returns
|
|
1035
|
+
-------
|
|
1036
|
+
dict
|
|
1037
|
+
A list of dictionary of {"metadata_name": "metadata_value"}
|
|
1038
|
+
|
|
1039
|
+
"""
|
|
1040
|
+
|
|
1041
|
+
content = self.get_subjects_metadata(search_criteria, items=(0, 9999))
|
|
1042
|
+
|
|
1043
|
+
# Wrap search criteria.
|
|
1044
|
+
modality, tags, dicoms = self.__wrap_search_criteria(search_criteria)
|
|
1045
|
+
|
|
1046
|
+
# Iterate over the files of each subject selected to include/exclude
|
|
1047
|
+
# them from the results.
|
|
1048
|
+
subjects = list()
|
|
1049
|
+
for subject in content:
|
|
1050
|
+
files = platform.parse_response(platform.post(
|
|
1051
|
+
self._account.auth, "file_manager/get_container_files",
|
|
1052
|
+
data={"container_id": str(int(subject["container_id"]))}
|
|
1053
|
+
))
|
|
1054
|
+
|
|
1055
|
+
for file in files["meta"]:
|
|
1056
|
+
if modality and \
|
|
1057
|
+
modality != (file.get("metadata") or {}).get("modality"):
|
|
1058
|
+
continue
|
|
1059
|
+
if tags and not all([tag in file.get("tags") for tag in tags]):
|
|
1060
|
+
continue
|
|
1061
|
+
if dicoms:
|
|
1062
|
+
result_values = list()
|
|
1063
|
+
for key, dict_value in dicoms.items():
|
|
1064
|
+
f_value = ((file.get("metadata") or {})
|
|
1065
|
+
.get("info") or {}).get(key)
|
|
1066
|
+
d_operator = dict_value["operation"]
|
|
1067
|
+
d_value = dict_value["value"]
|
|
1068
|
+
result_values.append(
|
|
1069
|
+
self.__operation(d_value, d_operator, f_value)
|
|
1070
|
+
)
|
|
1071
|
+
|
|
1072
|
+
if not all(result_values):
|
|
1073
|
+
continue
|
|
1074
|
+
subjects.append(subject)
|
|
1075
|
+
break
|
|
1076
|
+
return subjects
|
|
1077
|
+
|
|
1078
|
+
def get_file_metadata(self, container_id, filename):
|
|
1079
|
+
"""
|
|
1080
|
+
Retrieve the metadata from a particular file in a particular container.
|
|
1081
|
+
|
|
1082
|
+
Parameters
|
|
1083
|
+
----------
|
|
1084
|
+
container_id : str
|
|
1085
|
+
Container identifier.
|
|
1086
|
+
filename : str
|
|
1087
|
+
Name of the file.
|
|
1088
|
+
|
|
1089
|
+
Returns
|
|
1090
|
+
-------
|
|
1091
|
+
dict
|
|
1092
|
+
Dictionary with the metadata. False otherwise.
|
|
1093
|
+
"""
|
|
1094
|
+
all_metadata = self.list_container_files_metadata(container_id)
|
|
1095
|
+
if all_metadata:
|
|
1096
|
+
for file_meta in all_metadata:
|
|
1097
|
+
if file_meta["name"] == filename:
|
|
1098
|
+
return file_meta
|
|
1099
|
+
else:
|
|
1100
|
+
return False
|
|
1101
|
+
|
|
1102
|
+
def change_file_metadata(self, container_id, filename, modality, tags):
|
|
1103
|
+
"""
|
|
1104
|
+
Change modality and tags of `filename` in `container_id`
|
|
1083
1105
|
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1106
|
+
Parameters
|
|
1107
|
+
----------
|
|
1108
|
+
container_id : int
|
|
1109
|
+
Container identifier.
|
|
1110
|
+
filename : str
|
|
1111
|
+
Name of the file to be edited.
|
|
1112
|
+
modality : str or None
|
|
1113
|
+
Modality identifier, or None if the file shouldn't have
|
|
1114
|
+
any modality
|
|
1115
|
+
tags : list[str] or None
|
|
1116
|
+
List of tags, or None if the filename shouldn't have any tags
|
|
1117
|
+
"""
|
|
1089
1118
|
|
|
1090
|
-
|
|
1091
|
-
|
|
1119
|
+
tags_str = "" if tags is None else ";".join(tags)
|
|
1120
|
+
platform.parse_response(
|
|
1121
|
+
platform.post(
|
|
1122
|
+
self._account.auth,
|
|
1123
|
+
"file_manager/edit_file",
|
|
1124
|
+
data={"container_id": container_id, "filename": filename, "tags": tags_str, "modality": modality},
|
|
1125
|
+
)
|
|
1126
|
+
)
|
|
1092
1127
|
|
|
1093
1128
|
def delete_session(self, subject_name, session_id):
|
|
1094
1129
|
"""
|
|
@@ -1202,347 +1237,343 @@ class Project:
|
|
|
1202
1237
|
return False
|
|
1203
1238
|
return True
|
|
1204
1239
|
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
range_str,
|
|
1209
|
-
length,
|
|
1210
|
-
session_id,
|
|
1211
|
-
disposition,
|
|
1212
|
-
last_chunk,
|
|
1213
|
-
name="",
|
|
1214
|
-
date_of_scan="",
|
|
1215
|
-
description="",
|
|
1216
|
-
subject_name="",
|
|
1217
|
-
ssid="",
|
|
1218
|
-
filename="DATA.zip",
|
|
1219
|
-
input_data_type="mri_brain_data:1.0",
|
|
1220
|
-
result=False,
|
|
1221
|
-
add_to_container_id=0,
|
|
1222
|
-
split_data=False,
|
|
1223
|
-
):
|
|
1240
|
+
""" Container Related Methods """
|
|
1241
|
+
|
|
1242
|
+
def list_input_containers(self, search_criteria={}, items=(0, 9999)):
|
|
1224
1243
|
"""
|
|
1225
|
-
|
|
1244
|
+
Retrieve the list of input containers available to the user under a
|
|
1245
|
+
certain search criteria.
|
|
1226
1246
|
|
|
1227
1247
|
Parameters
|
|
1228
1248
|
----------
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
The string to send that describes the content range
|
|
1233
|
-
length
|
|
1234
|
-
The content length of the chunk to send
|
|
1235
|
-
session_id
|
|
1236
|
-
The session ID from the file path
|
|
1237
|
-
filename
|
|
1238
|
-
The name of the file to be sent
|
|
1239
|
-
disposition
|
|
1240
|
-
The disposition of the content
|
|
1241
|
-
last_chunk
|
|
1242
|
-
Set this only for the last chunk to be uploaded.
|
|
1243
|
-
All following parameters are ignored when False.
|
|
1244
|
-
split_data
|
|
1245
|
-
Sets the header that informs the platform to split
|
|
1246
|
-
the uploaded file into multiple sessions.
|
|
1247
|
-
"""
|
|
1248
|
-
|
|
1249
|
-
request_headers = {
|
|
1250
|
-
"Content-Type": "application/zip",
|
|
1251
|
-
"Content-Range": range_str,
|
|
1252
|
-
"Session-ID": str(session_id),
|
|
1253
|
-
"Content-Length": str(length),
|
|
1254
|
-
"Content-Disposition": disposition,
|
|
1255
|
-
}
|
|
1256
|
-
|
|
1257
|
-
if last_chunk:
|
|
1258
|
-
request_headers["X-Mint-Name"] = name
|
|
1259
|
-
request_headers["X-Mint-Date"] = date_of_scan
|
|
1260
|
-
request_headers["X-Mint-Description"] = description
|
|
1261
|
-
request_headers["X-Mint-Patient-Secret"] = subject_name
|
|
1262
|
-
request_headers["X-Mint-SSID"] = ssid
|
|
1263
|
-
request_headers["X-Mint-Filename"] = filename
|
|
1264
|
-
request_headers["X-Mint-Project-Id"] = str(self._project_id)
|
|
1265
|
-
request_headers["X-Mint-Split-Data"] = str(int(split_data))
|
|
1266
|
-
|
|
1267
|
-
if input_data_type:
|
|
1268
|
-
request_headers["X-Mint-Type"] = input_data_type
|
|
1269
|
-
|
|
1270
|
-
if result:
|
|
1271
|
-
request_headers["X-Mint-In-Out"] = "out"
|
|
1272
|
-
else:
|
|
1273
|
-
request_headers["X-Mint-In-Out"] = "in"
|
|
1274
|
-
|
|
1275
|
-
if add_to_container_id > 0:
|
|
1276
|
-
request_headers["X-Mint-Add-To"] = str(add_to_container_id)
|
|
1277
|
-
|
|
1278
|
-
request_headers["X-Requested-With"] = "XMLHttpRequest"
|
|
1279
|
-
|
|
1280
|
-
response_time = 900.0 if last_chunk else 120.0
|
|
1281
|
-
response = platform.post(
|
|
1282
|
-
auth=self._account.auth, endpoint="upload", data=data, headers=request_headers, timeout=response_time
|
|
1283
|
-
)
|
|
1284
|
-
|
|
1285
|
-
return response
|
|
1249
|
+
search_criteria : dict
|
|
1250
|
+
Each element is a string and is built using the formatting
|
|
1251
|
+
"type;value".
|
|
1286
1252
|
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
ssid
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
chunk_size=2**9,
|
|
1299
|
-
split_data=False,
|
|
1300
|
-
):
|
|
1301
|
-
"""
|
|
1302
|
-
Upload a ZIP file to the platform.
|
|
1253
|
+
List of possible keys:
|
|
1254
|
+
d_n: container_name # TODO: WHAT IS THIS???
|
|
1255
|
+
s_n: subject_id
|
|
1256
|
+
Subject ID of the subject in the platform.
|
|
1257
|
+
ssid: session_id
|
|
1258
|
+
Session ID of the subejct in the platform.
|
|
1259
|
+
from_d: from date
|
|
1260
|
+
Starting date in which perform the search. Format: DD.MM.YYYY
|
|
1261
|
+
to_d: to date
|
|
1262
|
+
End date in which perform the search. Format: DD.MM.YYYY
|
|
1263
|
+
sets: data sets (modalities) # TODO: WHAT IS THIS???
|
|
1303
1264
|
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
file_path : str
|
|
1307
|
-
Path to the ZIP file to upload.
|
|
1308
|
-
subject_name : str
|
|
1309
|
-
Subject ID of the data to upload in the project in QMENTA Platform.
|
|
1310
|
-
ssid : str
|
|
1311
|
-
Session ID of the Subject ID (i.e., ID of the timepoint).
|
|
1312
|
-
date_of_scan : str
|
|
1313
|
-
Date of scan/creation of the file
|
|
1314
|
-
description : str
|
|
1315
|
-
Description of the file
|
|
1316
|
-
result : bool
|
|
1317
|
-
If result=True then the upload will be taken as an offline analysis
|
|
1318
|
-
name : str
|
|
1319
|
-
Name of the file in the platform
|
|
1320
|
-
input_data_type : str
|
|
1321
|
-
qmenta_medical_image_data:3.11
|
|
1322
|
-
add_to_container_id : int
|
|
1323
|
-
ID of the container to which this file should be added (if id > 0)
|
|
1324
|
-
chunk_size : int
|
|
1325
|
-
Size in kB of each chunk. Should be expressed as
|
|
1326
|
-
a power of 2: 2**x. Default value of x is 9 (chunk_size = 512 kB)
|
|
1327
|
-
split_data : bool
|
|
1328
|
-
If True, the platform will try to split the uploaded file into
|
|
1329
|
-
different sessions. It will be ignored when the ssid is given.
|
|
1265
|
+
items: Tuple(int, int)
|
|
1266
|
+
Starting and ending element of the search.
|
|
1330
1267
|
|
|
1331
1268
|
Returns
|
|
1332
1269
|
-------
|
|
1333
|
-
|
|
1334
|
-
|
|
1270
|
+
dict
|
|
1271
|
+
List of containers, each a dictionary containing the following
|
|
1272
|
+
information:
|
|
1273
|
+
{"container_name", "container_id", "patient_secret_name", "ssid"}
|
|
1335
1274
|
"""
|
|
1336
1275
|
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
chunk_size *= 1024
|
|
1341
|
-
max_retries = 10
|
|
1342
|
-
|
|
1343
|
-
name = name or os.path.split(file_path)[1]
|
|
1344
|
-
|
|
1345
|
-
total_bytes = os.path.getsize(file_path)
|
|
1346
|
-
|
|
1347
|
-
# making chunks of the file and sending one by one
|
|
1348
|
-
logger = logging.getLogger(logger_name)
|
|
1349
|
-
with open(file_path, "rb") as file_object:
|
|
1350
|
-
|
|
1351
|
-
file_size = os.path.getsize(file_path)
|
|
1352
|
-
if file_size == 0:
|
|
1353
|
-
logger.error("Cannot upload empty file {}".format(file_path))
|
|
1354
|
-
return False
|
|
1355
|
-
uploaded = 0
|
|
1356
|
-
session_id = self.__get_session_id(file_path)
|
|
1357
|
-
chunk_num = 0
|
|
1358
|
-
retries_count = 0
|
|
1359
|
-
uploaded_bytes = 0
|
|
1360
|
-
response = None
|
|
1361
|
-
last_chunk = False
|
|
1276
|
+
assert len(items) == 2, f"The number of elements in items " f"'{len(items)}' should be equal to two."
|
|
1277
|
+
assert all([isinstance(item, int) for item in items]), f"All items elements '{items}' should be integers."
|
|
1362
1278
|
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1279
|
+
response = platform.parse_response(
|
|
1280
|
+
platform.post(
|
|
1281
|
+
self._account.auth,
|
|
1282
|
+
"file_manager/get_container_list",
|
|
1283
|
+
data=search_criteria,
|
|
1284
|
+
headers={"X-Range": f"items={items[0]}-{items[1]}"},
|
|
1285
|
+
)
|
|
1286
|
+
)
|
|
1287
|
+
containers = [
|
|
1288
|
+
{
|
|
1289
|
+
"patient_secret_name": container_item["patient_secret_name"],
|
|
1290
|
+
"container_name": container_item["name"],
|
|
1291
|
+
"container_id": container_item["_id"],
|
|
1292
|
+
"ssid": container_item["ssid"],
|
|
1293
|
+
}
|
|
1294
|
+
for container_item in response
|
|
1295
|
+
]
|
|
1296
|
+
return containers
|
|
1366
1297
|
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1298
|
+
def list_result_containers(self, search_condition={}, items=(0, 9999)):
|
|
1299
|
+
"""
|
|
1300
|
+
List the result containers available to the user.
|
|
1301
|
+
Examples
|
|
1302
|
+
--------
|
|
1371
1303
|
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1304
|
+
>>> search_condition = {
|
|
1305
|
+
"secret_name":"014_S_6920",
|
|
1306
|
+
"from_d": "06.02.2025",
|
|
1307
|
+
"with_child_analysis": 1,
|
|
1308
|
+
"state": "completed"
|
|
1309
|
+
}
|
|
1310
|
+
list_result_containers(search_condition=search_condition)
|
|
1375
1311
|
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
end_position = total_bytes - 1
|
|
1379
|
-
bytes_to_send = total_bytes - uploaded_bytes
|
|
1312
|
+
Note the keys not needed for the search do not have to be included in
|
|
1313
|
+
the search condition.
|
|
1380
1314
|
|
|
1381
|
-
|
|
1315
|
+
Parameters
|
|
1316
|
+
----------
|
|
1317
|
+
search_condition : dict
|
|
1318
|
+
- p_n: str or None Analysis name
|
|
1319
|
+
- type: str or None Type
|
|
1320
|
+
- from_d: str or None dd.mm.yyyy Date from
|
|
1321
|
+
- to_d: str or None dd.mm.yyyy Date to
|
|
1322
|
+
- qa_status: str or None pass/fail/nd QC status
|
|
1323
|
+
- secret_name: str or None Subject ID
|
|
1324
|
+
- tags: str or None
|
|
1325
|
+
- with_child_analysis: 1 or None if 1, child analysis of workflows will appear
|
|
1326
|
+
- id: str or None ID
|
|
1327
|
+
- state: running, completed, pending, exception or None
|
|
1328
|
+
- username: str or None
|
|
1382
1329
|
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
bytes_range,
|
|
1387
|
-
bytes_to_send,
|
|
1388
|
-
session_id,
|
|
1389
|
-
dispstr,
|
|
1390
|
-
last_chunk,
|
|
1391
|
-
name,
|
|
1392
|
-
date_of_scan,
|
|
1393
|
-
description,
|
|
1394
|
-
subject_name,
|
|
1395
|
-
ssid,
|
|
1396
|
-
filename,
|
|
1397
|
-
input_data_type,
|
|
1398
|
-
result,
|
|
1399
|
-
add_to_container_id,
|
|
1400
|
-
split_data,
|
|
1401
|
-
)
|
|
1330
|
+
items : List[int]
|
|
1331
|
+
list containing two elements [min, max] that correspond to the
|
|
1332
|
+
mininum and maximum range of analysis listed
|
|
1402
1333
|
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
uploaded_bytes += chunk_size
|
|
1414
|
-
elif int(response.status_code) == 200:
|
|
1415
|
-
self.__show_progress(file_size, file_size, finish=True)
|
|
1416
|
-
break
|
|
1417
|
-
elif int(response.status_code) == 416:
|
|
1418
|
-
retries_count += 1
|
|
1419
|
-
time.sleep(retries_count * 5)
|
|
1420
|
-
if retries_count > self.max_retries:
|
|
1421
|
-
error_message = "Error Code: 416; " "Requested Range Not Satisfiable (NGINX)"
|
|
1422
|
-
logger.error(error_message)
|
|
1423
|
-
break
|
|
1424
|
-
else:
|
|
1425
|
-
retries_count += 1
|
|
1426
|
-
time.sleep(retries_count * 5)
|
|
1427
|
-
if retries_count > max_retries:
|
|
1428
|
-
error_message = "Number of retries has been reached. " "Upload process stops here !"
|
|
1429
|
-
logger.error(error_message)
|
|
1430
|
-
break
|
|
1334
|
+
Returns
|
|
1335
|
+
-------
|
|
1336
|
+
dict
|
|
1337
|
+
List of containers, each a dictionary
|
|
1338
|
+
{"name": "container-name", "id": "container_id"}
|
|
1339
|
+
if "id": None, that analysis did not had an output container,
|
|
1340
|
+
probably it is a workflow
|
|
1341
|
+
"""
|
|
1342
|
+
analyses = self.list_analysis(search_condition, items)
|
|
1343
|
+
return [{"name": analysis["name"], "id": (analysis.get("out_container_id") or None)} for analysis in analyses]
|
|
1431
1344
|
|
|
1432
|
-
|
|
1433
|
-
|
|
1345
|
+
def list_container_files(
|
|
1346
|
+
self,
|
|
1347
|
+
container_id,
|
|
1348
|
+
):
|
|
1349
|
+
"""
|
|
1350
|
+
List the name of the files available inside a given container.
|
|
1351
|
+
Parameters
|
|
1352
|
+
----------
|
|
1353
|
+
container_id : str or int
|
|
1354
|
+
Container identifier.
|
|
1434
1355
|
|
|
1356
|
+
Returns
|
|
1357
|
+
-------
|
|
1358
|
+
list[str]
|
|
1359
|
+
List of file names (strings)
|
|
1360
|
+
"""
|
|
1435
1361
|
try:
|
|
1436
|
-
platform.parse_response(
|
|
1437
|
-
|
|
1438
|
-
|
|
1362
|
+
content = platform.parse_response(
|
|
1363
|
+
platform.post(
|
|
1364
|
+
self._account.auth, "file_manager/get_container_files", data={"container_id": container_id}
|
|
1365
|
+
)
|
|
1366
|
+
)
|
|
1367
|
+
except errors.PlatformError as e:
|
|
1368
|
+
logging.getLogger(logger_name).error(e)
|
|
1439
1369
|
return False
|
|
1370
|
+
if "files" not in content.keys():
|
|
1371
|
+
logging.getLogger(logger_name).error("Could not get files")
|
|
1372
|
+
return False
|
|
1373
|
+
return content["files"]
|
|
1440
1374
|
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1375
|
+
def list_container_filter_files(
|
|
1376
|
+
self,
|
|
1377
|
+
container_id,
|
|
1378
|
+
modality="",
|
|
1379
|
+
metadata_info={},
|
|
1380
|
+
tags=[]
|
|
1381
|
+
):
|
|
1447
1382
|
"""
|
|
1448
|
-
|
|
1449
|
-
|
|
1383
|
+
List the name of the files available inside a given container.
|
|
1384
|
+
search condition example:
|
|
1385
|
+
"metadata": {"SliceThickness":1},
|
|
1386
|
+
}
|
|
1450
1387
|
Parameters
|
|
1451
1388
|
----------
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1389
|
+
container_id : str or int
|
|
1390
|
+
Container identifier.
|
|
1391
|
+
|
|
1392
|
+
modality: str
|
|
1393
|
+
String containing the modality of the files being filtered
|
|
1394
|
+
|
|
1395
|
+
metadata_info: dict
|
|
1396
|
+
Dictionary containing the file metadata of the files being filtered
|
|
1397
|
+
|
|
1398
|
+
tags: list[str]
|
|
1399
|
+
List of strings containing the tags of the files being filtered
|
|
1455
1400
|
|
|
1456
1401
|
Returns
|
|
1457
1402
|
-------
|
|
1458
|
-
|
|
1459
|
-
|
|
1403
|
+
selected_files: list[str]
|
|
1404
|
+
List of file names (strings)
|
|
1460
1405
|
"""
|
|
1406
|
+
content_files = self.list_container_files(container_id)
|
|
1407
|
+
content_meta = self.list_container_files_metadata(container_id)
|
|
1408
|
+
selected_files = []
|
|
1409
|
+
for index, file in enumerate(content_files):
|
|
1410
|
+
metadata_file = content_meta[index]
|
|
1411
|
+
tags_file = metadata_file.get("tags")
|
|
1412
|
+
tags_bool = [tag in tags_file for tag in tags]
|
|
1413
|
+
info_bool = []
|
|
1414
|
+
if modality == "":
|
|
1415
|
+
modality_bool = True
|
|
1416
|
+
else:
|
|
1417
|
+
modality_bool = modality == metadata_file["metadata"].get(
|
|
1418
|
+
"modality"
|
|
1419
|
+
)
|
|
1420
|
+
for key in metadata_info.keys():
|
|
1421
|
+
meta_key = (
|
|
1422
|
+
(
|
|
1423
|
+
metadata_file.get("metadata") or {}
|
|
1424
|
+
).get("info") or {}).get(
|
|
1425
|
+
key
|
|
1426
|
+
)
|
|
1427
|
+
if meta_key is None:
|
|
1428
|
+
logging.getLogger(logger_name).warning(
|
|
1429
|
+
f"{key} is not in file_info from file {file}"
|
|
1430
|
+
)
|
|
1431
|
+
info_bool.append(
|
|
1432
|
+
metadata_info[key] == meta_key
|
|
1433
|
+
)
|
|
1434
|
+
if all(tags_bool) and all(info_bool) and modality_bool:
|
|
1435
|
+
selected_files.append(file)
|
|
1436
|
+
return selected_files
|
|
1461
1437
|
|
|
1462
|
-
|
|
1463
|
-
return self.upload_file(file_path, subject_name)
|
|
1464
|
-
|
|
1465
|
-
def upload_gametection(self, file_path, subject_name):
|
|
1438
|
+
def list_container_files_metadata(self, container_id):
|
|
1466
1439
|
"""
|
|
1467
|
-
|
|
1440
|
+
List all the metadata of the files available inside a given container.
|
|
1468
1441
|
|
|
1469
1442
|
Parameters
|
|
1470
1443
|
----------
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
subject_name: str
|
|
1444
|
+
container_id : str
|
|
1445
|
+
Container identifier.
|
|
1474
1446
|
|
|
1475
1447
|
Returns
|
|
1476
1448
|
-------
|
|
1477
|
-
|
|
1478
|
-
|
|
1449
|
+
dict
|
|
1450
|
+
Dictionary of {"metadata_name": "metadata_value"}
|
|
1479
1451
|
"""
|
|
1480
1452
|
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1453
|
+
try:
|
|
1454
|
+
data = platform.parse_response(
|
|
1455
|
+
platform.post(
|
|
1456
|
+
self._account.auth, "file_manager/get_container_files", data={"container_id": container_id}
|
|
1457
|
+
)
|
|
1458
|
+
)
|
|
1459
|
+
except errors.PlatformError as e:
|
|
1460
|
+
logging.getLogger(logger_name).error(e)
|
|
1461
|
+
return False
|
|
1484
1462
|
|
|
1485
|
-
|
|
1463
|
+
return data["meta"]
|
|
1464
|
+
|
|
1465
|
+
""" Analysis Related Methods """
|
|
1466
|
+
|
|
1467
|
+
def get_analysis(self, analysis_name_or_id):
|
|
1486
1468
|
"""
|
|
1487
|
-
|
|
1469
|
+
Returns the analysis corresponding with the analysis id or analysis name
|
|
1488
1470
|
|
|
1489
1471
|
Parameters
|
|
1490
1472
|
----------
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
subject_name: str
|
|
1473
|
+
analysis_name_or_id : int, str
|
|
1474
|
+
analysis_id or analysis name to search for
|
|
1494
1475
|
|
|
1495
1476
|
Returns
|
|
1496
1477
|
-------
|
|
1497
|
-
|
|
1498
|
-
|
|
1478
|
+
analysis: dict
|
|
1479
|
+
A dictionary containing the analysis information
|
|
1480
|
+
"""
|
|
1481
|
+
if isinstance(analysis_name_or_id, int):
|
|
1482
|
+
search_tag = "id"
|
|
1483
|
+
elif isinstance(analysis_name_or_id, str):
|
|
1484
|
+
if analysis_name_or_id.isdigit():
|
|
1485
|
+
search_tag = "id"
|
|
1486
|
+
analysis_name_or_id = int(analysis_name_or_id)
|
|
1487
|
+
else:
|
|
1488
|
+
search_tag = "p_n"
|
|
1489
|
+
else:
|
|
1490
|
+
raise Exception("The analysis identifier must be its name or an " "integer")
|
|
1491
|
+
|
|
1492
|
+
search_condition = {
|
|
1493
|
+
search_tag: analysis_name_or_id,
|
|
1494
|
+
}
|
|
1495
|
+
response = platform.parse_response(
|
|
1496
|
+
platform.post(self._account.auth, "analysis_manager/get_analysis_list", data=search_condition)
|
|
1497
|
+
)
|
|
1498
|
+
|
|
1499
|
+
if len(response) > 1:
|
|
1500
|
+
raise Exception(f"multiple analyses with name " f"{analysis_name_or_id} found")
|
|
1501
|
+
elif len(response) == 1:
|
|
1502
|
+
return response[0]
|
|
1503
|
+
else:
|
|
1504
|
+
return None
|
|
1505
|
+
|
|
1506
|
+
def list_analysis(self, search_condition={}, items=(0, 9999)):
|
|
1499
1507
|
"""
|
|
1508
|
+
List the analysis available to the user.
|
|
1500
1509
|
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1510
|
+
Examples
|
|
1511
|
+
--------
|
|
1512
|
+
|
|
1513
|
+
>>> search_condition = {
|
|
1514
|
+
"secret_name":"014_S_6920",
|
|
1515
|
+
"from_d": "06.02.2025",
|
|
1516
|
+
"with_child_analysis": 1,
|
|
1517
|
+
"state": "completed"
|
|
1518
|
+
}
|
|
1519
|
+
list_analysis(search_condition=search_condition)
|
|
1504
1520
|
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
Copy a container to another project.
|
|
1521
|
+
Note the keys not needed for the search do not have to be included in
|
|
1522
|
+
the search condition.
|
|
1508
1523
|
|
|
1509
1524
|
Parameters
|
|
1510
1525
|
----------
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1526
|
+
search_condition : dict
|
|
1527
|
+
- p_n: str or None Analysis name
|
|
1528
|
+
- type: str or None Type
|
|
1529
|
+
- from_d: str or None dd.mm.yyyy Date from
|
|
1530
|
+
- to_d: str or None dd.mm.yyyy Date to
|
|
1531
|
+
- qa_status: str or None pass/fail/nd QC status
|
|
1532
|
+
- secret_name: str or None Subject ID
|
|
1533
|
+
- tags: str or None
|
|
1534
|
+
- with_child_analysis: 1 or None if 1, child analysis of workflows will appear
|
|
1535
|
+
- id: str or None ID
|
|
1536
|
+
- state: running, completed, pending, exception or None
|
|
1537
|
+
- username: str or None
|
|
1538
|
+
|
|
1539
|
+
items : List[int]
|
|
1540
|
+
list containing two elements [min, max] that correspond to the
|
|
1541
|
+
mininum and maximum range of analysis listed
|
|
1515
1542
|
|
|
1516
1543
|
Returns
|
|
1517
1544
|
-------
|
|
1518
|
-
|
|
1519
|
-
|
|
1545
|
+
dict
|
|
1546
|
+
List of analysis, each a dictionary
|
|
1520
1547
|
"""
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
"
|
|
1534
|
-
"
|
|
1548
|
+
assert len(items) == 2, f"The number of elements in items " f"'{len(items)}' should be equal to two."
|
|
1549
|
+
assert all([isinstance(item, int) for item in items]), f"All items elements '{items}' should be integers."
|
|
1550
|
+
search_keys = {
|
|
1551
|
+
"p_n": str,
|
|
1552
|
+
"type": str,
|
|
1553
|
+
"from_d": str,
|
|
1554
|
+
"to_d": str,
|
|
1555
|
+
"qa_status": str,
|
|
1556
|
+
"secret_name": str,
|
|
1557
|
+
"tags": str,
|
|
1558
|
+
"with_child_analysis": int,
|
|
1559
|
+
"id": int,
|
|
1560
|
+
"state": str,
|
|
1561
|
+
"username": str,
|
|
1535
1562
|
}
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1563
|
+
for key in search_condition.keys():
|
|
1564
|
+
if key not in search_keys.keys():
|
|
1565
|
+
raise Exception((f"This key '{key}' is not accepted by this" "search condition"))
|
|
1566
|
+
if not isinstance(search_condition[key], search_keys[key]) and search_condition[key] is not None:
|
|
1567
|
+
raise Exception((f"The key {key} in the search condition" f"is not type {search_keys[key]}"))
|
|
1568
|
+
req_headers = {"X-Range": f"items={items[0]}-{items[1] - 1}"}
|
|
1569
|
+
return platform.parse_response(
|
|
1570
|
+
platform.post(
|
|
1571
|
+
auth=self._account.auth,
|
|
1572
|
+
endpoint="analysis_manager/get_analysis_list",
|
|
1573
|
+
headers=req_headers,
|
|
1574
|
+
data=search_condition,
|
|
1540
1575
|
)
|
|
1541
|
-
|
|
1542
|
-
logging.getLogger(logger_name).error("Couldn not copy container: {}".format(e))
|
|
1543
|
-
return False
|
|
1544
|
-
|
|
1545
|
-
return True
|
|
1576
|
+
)
|
|
1546
1577
|
|
|
1547
1578
|
def start_analysis(
|
|
1548
1579
|
self,
|
|
@@ -1650,94 +1681,7 @@ class Project:
|
|
|
1650
1681
|
|
|
1651
1682
|
return True
|
|
1652
1683
|
|
|
1653
|
-
|
|
1654
|
-
"""
|
|
1655
|
-
Handle the possible responses from the server after start_analysis.
|
|
1656
|
-
Sometimes we have to send a request again, and then check again the
|
|
1657
|
-
response. That"s why this function is separated from start_analysis.
|
|
1658
|
-
|
|
1659
|
-
Since this function sometimes calls itself, n_calls avoids entering an
|
|
1660
|
-
infinite loop due to some misbehaviour in the server.
|
|
1661
|
-
"""
|
|
1662
|
-
|
|
1663
|
-
call_limit = 10
|
|
1664
|
-
n_calls += 1
|
|
1665
|
-
|
|
1666
|
-
logger = logging.getLogger(logger_name)
|
|
1667
|
-
if n_calls > call_limit:
|
|
1668
|
-
logger.error(
|
|
1669
|
-
f"__handle_start_analysis_response called itself more\
|
|
1670
|
-
than {n_calls} times: aborting."
|
|
1671
|
-
)
|
|
1672
|
-
return None
|
|
1673
|
-
|
|
1674
|
-
try:
|
|
1675
|
-
response = platform.parse_response(
|
|
1676
|
-
platform.post(self._account.auth, "analysis_manager/analysis_registration", data=post_data)
|
|
1677
|
-
)
|
|
1678
|
-
logger.info(response["message"])
|
|
1679
|
-
return int(response["analysis_id"])
|
|
1680
|
-
except platform.ChooseDataError as choose_data:
|
|
1681
|
-
has_warning = False
|
|
1682
|
-
|
|
1683
|
-
# logging any warning that we have
|
|
1684
|
-
if choose_data.warning:
|
|
1685
|
-
has_warning = True
|
|
1686
|
-
logger.warning(response["warning"])
|
|
1687
|
-
|
|
1688
|
-
new_post = {
|
|
1689
|
-
"analysis_id": choose_data.analysis_id,
|
|
1690
|
-
"script_name": post_data["script_name"],
|
|
1691
|
-
"version": post_data["version"],
|
|
1692
|
-
}
|
|
1693
|
-
|
|
1694
|
-
if choose_data.data_to_choose:
|
|
1695
|
-
# in case we have data to choose
|
|
1696
|
-
chosen_files = {}
|
|
1697
|
-
for settings_key in choose_data.data_to_choose:
|
|
1698
|
-
chosen_files[settings_key] = {}
|
|
1699
|
-
filters = choose_data.data_to_choose[settings_key]["filters"]
|
|
1700
|
-
for filter_key in filters:
|
|
1701
|
-
filter_data = filters[filter_key]
|
|
1702
|
-
|
|
1703
|
-
# skip the filters that did not pass
|
|
1704
|
-
if not filter_data["passed"]:
|
|
1705
|
-
continue
|
|
1706
|
-
|
|
1707
|
-
number_of_files_to_select = 1
|
|
1708
|
-
if filter_data["range"][0] != 0:
|
|
1709
|
-
number_of_files_to_select = filter_data["range"][0]
|
|
1710
|
-
elif filter_data["range"][1] != 0:
|
|
1711
|
-
number_of_files_to_select = min(filter_data["range"][1], len(filter_data["files"]))
|
|
1712
|
-
else:
|
|
1713
|
-
number_of_files_to_select = len(filter_data["files"])
|
|
1714
|
-
|
|
1715
|
-
files_selection = [ff["_id"] for ff in filter_data["files"][:number_of_files_to_select]]
|
|
1716
|
-
chosen_files[settings_key][filter_key] = files_selection
|
|
1717
|
-
|
|
1718
|
-
new_post["user_preference"] = json.dumps(chosen_files)
|
|
1719
|
-
else:
|
|
1720
|
-
if has_warning and not ignore_warnings:
|
|
1721
|
-
logger.info("cancelling analysis due to warnings, " + 'set "ignore_warnings" to True to override')
|
|
1722
|
-
new_post["cancel"] = "1"
|
|
1723
|
-
else:
|
|
1724
|
-
logger.info("suppressing warnings")
|
|
1725
|
-
new_post["user_preference"] = "{}"
|
|
1726
|
-
new_post["_mint_only_warning"] = "1"
|
|
1727
|
-
|
|
1728
|
-
return self.__handle_start_analysis(new_post, ignore_warnings=ignore_warnings, n_calls=n_calls)
|
|
1729
|
-
except platform.ActionFailedError as e:
|
|
1730
|
-
logger.error(f"Unable to start the analysis: {e}")
|
|
1731
|
-
return None
|
|
1732
|
-
|
|
1733
|
-
@staticmethod
|
|
1734
|
-
def __get_modalities(files):
|
|
1735
|
-
modalities = []
|
|
1736
|
-
for file_ in files:
|
|
1737
|
-
modality = file_["metadata"]["modality"]
|
|
1738
|
-
if modality not in modalities:
|
|
1739
|
-
modalities.append(modality)
|
|
1740
|
-
return modalities
|
|
1684
|
+
""" QC Status Related Methods """
|
|
1741
1685
|
|
|
1742
1686
|
def set_qc_status_analysis(self, analysis_id,
|
|
1743
1687
|
status=QCStatus.UNDERTERMINED, comments=""):
|
|
@@ -1919,6 +1863,7 @@ class Project:
|
|
|
1919
1863
|
raise ValueError("Either 'patient_id' or 'subject_name' and 'ssid'"
|
|
1920
1864
|
" must not be empty.")
|
|
1921
1865
|
|
|
1866
|
+
""" Protocol Adherence Related Methods """
|
|
1922
1867
|
def set_project_pa_rules(self, rules_file_path, guidance_text=""):
|
|
1923
1868
|
"""
|
|
1924
1869
|
Updates the active project's protocol adherence rules using the
|
|
@@ -2004,6 +1949,96 @@ class Project:
|
|
|
2004
1949
|
|
|
2005
1950
|
return res["guidance_text"]
|
|
2006
1951
|
|
|
1952
|
+
""" Helper Methods """
|
|
1953
|
+
def __handle_start_analysis(self, post_data, ignore_warnings=False, n_calls=0):
|
|
1954
|
+
"""
|
|
1955
|
+
Handle the possible responses from the server after start_analysis.
|
|
1956
|
+
Sometimes we have to send a request again, and then check again the
|
|
1957
|
+
response. That"s why this function is separated from start_analysis.
|
|
1958
|
+
|
|
1959
|
+
Since this function sometimes calls itself, n_calls avoids entering an
|
|
1960
|
+
infinite loop due to some misbehaviour in the server.
|
|
1961
|
+
"""
|
|
1962
|
+
|
|
1963
|
+
call_limit = 10
|
|
1964
|
+
n_calls += 1
|
|
1965
|
+
|
|
1966
|
+
logger = logging.getLogger(logger_name)
|
|
1967
|
+
if n_calls > call_limit:
|
|
1968
|
+
logger.error(
|
|
1969
|
+
f"__handle_start_analysis_response called itself more\
|
|
1970
|
+
than {n_calls} times: aborting."
|
|
1971
|
+
)
|
|
1972
|
+
return None
|
|
1973
|
+
|
|
1974
|
+
try:
|
|
1975
|
+
response = platform.parse_response(
|
|
1976
|
+
platform.post(self._account.auth, "analysis_manager/analysis_registration", data=post_data)
|
|
1977
|
+
)
|
|
1978
|
+
logger.info(response["message"])
|
|
1979
|
+
return int(response["analysis_id"])
|
|
1980
|
+
except platform.ChooseDataError as choose_data:
|
|
1981
|
+
has_warning = False
|
|
1982
|
+
|
|
1983
|
+
# logging any warning that we have
|
|
1984
|
+
if choose_data.warning:
|
|
1985
|
+
has_warning = True
|
|
1986
|
+
logger.warning(response["warning"])
|
|
1987
|
+
|
|
1988
|
+
new_post = {
|
|
1989
|
+
"analysis_id": choose_data.analysis_id,
|
|
1990
|
+
"script_name": post_data["script_name"],
|
|
1991
|
+
"version": post_data["version"],
|
|
1992
|
+
}
|
|
1993
|
+
|
|
1994
|
+
if choose_data.data_to_choose:
|
|
1995
|
+
# in case we have data to choose
|
|
1996
|
+
chosen_files = {}
|
|
1997
|
+
for settings_key in choose_data.data_to_choose:
|
|
1998
|
+
chosen_files[settings_key] = {}
|
|
1999
|
+
filters = choose_data.data_to_choose[settings_key]["filters"]
|
|
2000
|
+
for filter_key in filters:
|
|
2001
|
+
filter_data = filters[filter_key]
|
|
2002
|
+
|
|
2003
|
+
# skip the filters that did not pass
|
|
2004
|
+
if not filter_data["passed"]:
|
|
2005
|
+
continue
|
|
2006
|
+
|
|
2007
|
+
number_of_files_to_select = 1
|
|
2008
|
+
if filter_data["range"][0] != 0:
|
|
2009
|
+
number_of_files_to_select = filter_data["range"][0]
|
|
2010
|
+
elif filter_data["range"][1] != 0:
|
|
2011
|
+
number_of_files_to_select = min(filter_data["range"][1], len(filter_data["files"]))
|
|
2012
|
+
else:
|
|
2013
|
+
number_of_files_to_select = len(filter_data["files"])
|
|
2014
|
+
|
|
2015
|
+
files_selection = [ff["_id"] for ff in filter_data["files"][:number_of_files_to_select]]
|
|
2016
|
+
chosen_files[settings_key][filter_key] = files_selection
|
|
2017
|
+
|
|
2018
|
+
new_post["user_preference"] = json.dumps(chosen_files)
|
|
2019
|
+
else:
|
|
2020
|
+
if has_warning and not ignore_warnings:
|
|
2021
|
+
logger.info("cancelling analysis due to warnings, " + 'set "ignore_warnings" to True to override')
|
|
2022
|
+
new_post["cancel"] = "1"
|
|
2023
|
+
else:
|
|
2024
|
+
logger.info("suppressing warnings")
|
|
2025
|
+
new_post["user_preference"] = "{}"
|
|
2026
|
+
new_post["_mint_only_warning"] = "1"
|
|
2027
|
+
|
|
2028
|
+
return self.__handle_start_analysis(new_post, ignore_warnings=ignore_warnings, n_calls=n_calls)
|
|
2029
|
+
except platform.ActionFailedError as e:
|
|
2030
|
+
logger.error(f"Unable to start the analysis: {e}")
|
|
2031
|
+
return None
|
|
2032
|
+
|
|
2033
|
+
@staticmethod
|
|
2034
|
+
def __get_modalities(files):
|
|
2035
|
+
modalities = []
|
|
2036
|
+
for file_ in files:
|
|
2037
|
+
modality = file_["metadata"]["modality"]
|
|
2038
|
+
if modality not in modalities:
|
|
2039
|
+
modalities.append(modality)
|
|
2040
|
+
return modalities
|
|
2041
|
+
|
|
2007
2042
|
def __show_progress(self, done, total, finish=False):
|
|
2008
2043
|
bytes_in_mb = 1024 * 1024
|
|
2009
2044
|
progress_message = "\r[{:.2f} %] Uploaded {:.2f} of {:.2f} Mb".format(
|