rucio-clients 37.3.0__py3-none-any.whl → 37.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio-clients might be problematic. Click here for more details.
- rucio/cli/rule.py +1 -1
- rucio/client/accountclient.py +205 -60
- rucio/client/accountlimitclient.py +84 -25
- rucio/client/baseclient.py +85 -48
- rucio/client/client.py +49 -41
- rucio/client/configclient.py +36 -13
- rucio/client/credentialclient.py +16 -6
- rucio/client/didclient.py +321 -133
- rucio/client/diracclient.py +13 -6
- rucio/client/downloadclient.py +435 -165
- rucio/client/exportclient.py +8 -2
- rucio/client/fileclient.py +10 -3
- rucio/client/importclient.py +4 -1
- rucio/client/lifetimeclient.py +48 -31
- rucio/client/lockclient.py +22 -7
- rucio/client/metaconventionsclient.py +59 -21
- rucio/client/pingclient.py +3 -1
- rucio/client/replicaclient.py +213 -96
- rucio/client/requestclient.py +123 -16
- rucio/client/rseclient.py +385 -160
- rucio/client/ruleclient.py +147 -51
- rucio/client/scopeclient.py +35 -10
- rucio/client/subscriptionclient.py +60 -27
- rucio/client/touchclient.py +16 -7
- rucio/vcsversion.py +3 -3
- {rucio_clients-37.3.0.data → rucio_clients-37.4.0.data}/data/etc/rucio.cfg.template +0 -1
- {rucio_clients-37.3.0.dist-info → rucio_clients-37.4.0.dist-info}/METADATA +1 -1
- {rucio_clients-37.3.0.dist-info → rucio_clients-37.4.0.dist-info}/RECORD +38 -38
- {rucio_clients-37.3.0.dist-info → rucio_clients-37.4.0.dist-info}/WHEEL +1 -1
- {rucio_clients-37.3.0.data → rucio_clients-37.4.0.data}/data/etc/rse-accounts.cfg.template +0 -0
- {rucio_clients-37.3.0.data → rucio_clients-37.4.0.data}/data/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio_clients-37.3.0.data → rucio_clients-37.4.0.data}/data/requirements.client.txt +0 -0
- {rucio_clients-37.3.0.data → rucio_clients-37.4.0.data}/data/rucio_client/merge_rucio_configs.py +0 -0
- {rucio_clients-37.3.0.data → rucio_clients-37.4.0.data}/scripts/rucio +0 -0
- {rucio_clients-37.3.0.data → rucio_clients-37.4.0.data}/scripts/rucio-admin +0 -0
- {rucio_clients-37.3.0.dist-info → rucio_clients-37.4.0.dist-info}/licenses/AUTHORS.rst +0 -0
- {rucio_clients-37.3.0.dist-info → rucio_clients-37.4.0.dist-info}/licenses/LICENSE +0 -0
- {rucio_clients-37.3.0.dist-info → rucio_clients-37.4.0.dist-info}/top_level.txt +0 -0
rucio/client/downloadclient.py
CHANGED
|
@@ -71,12 +71,19 @@ class BaseExtractionTool:
|
|
|
71
71
|
logger: "LoggerFunction" = logging.log
|
|
72
72
|
):
|
|
73
73
|
"""
|
|
74
|
-
|
|
74
|
+
Initializes a extraction tool object
|
|
75
|
+
|
|
76
|
+
Parameters
|
|
77
|
+
----------
|
|
78
|
+
program_name :
|
|
79
|
+
the name of the archive extraction program, e.g., unzip
|
|
80
|
+
useability_check_args :
|
|
81
|
+
the arguments of the extraction program to test if it's installed, e.g., --version
|
|
82
|
+
extract_args :
|
|
83
|
+
the arguments that will be passed to the program for extraction
|
|
84
|
+
logger :
|
|
85
|
+
optional decorated logging.log object that can be passed from the calling daemon or client.
|
|
75
86
|
|
|
76
|
-
:param program_name: the name of the archive extraction program, e.g., unzip
|
|
77
|
-
:param useability_check_args: the arguments of the extraction program to test if its installed, e.g., --version
|
|
78
|
-
:param extract_args: the arguments that will be passed to the program for extraction
|
|
79
|
-
:param logger: optional decorated logging.log object that can be passed from the calling daemon or client.
|
|
80
87
|
"""
|
|
81
88
|
self.program_name = program_name
|
|
82
89
|
self.useability_check_args = useability_check_args
|
|
@@ -88,7 +95,9 @@ class BaseExtractionTool:
|
|
|
88
95
|
"""
|
|
89
96
|
Checks if the extraction tool is installed and usable
|
|
90
97
|
|
|
91
|
-
|
|
98
|
+
Returns
|
|
99
|
+
-------
|
|
100
|
+
True if it is usable otherwise False
|
|
92
101
|
"""
|
|
93
102
|
if self.is_useable_result is not None:
|
|
94
103
|
return self.is_useable_result
|
|
@@ -113,11 +122,19 @@ class BaseExtractionTool:
|
|
|
113
122
|
"""
|
|
114
123
|
Calls the extraction program to extract a file from an archive
|
|
115
124
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
125
|
+
Parameters
|
|
126
|
+
----------
|
|
127
|
+
archive_file_path :
|
|
128
|
+
path to the archive
|
|
129
|
+
file_to_extract :
|
|
130
|
+
file name to extract from the archive
|
|
131
|
+
dest_dir_pat :
|
|
132
|
+
destination directory where the extracted file will be stored
|
|
133
|
+
|
|
134
|
+
Returns
|
|
135
|
+
-------
|
|
136
|
+
True on success otherwise False
|
|
119
137
|
|
|
120
|
-
:returns: True on success otherwise False
|
|
121
138
|
"""
|
|
122
139
|
if not self.is_useable():
|
|
123
140
|
return False
|
|
@@ -148,11 +165,16 @@ class DownloadClient:
|
|
|
148
165
|
check_pcache: bool = False
|
|
149
166
|
):
|
|
150
167
|
"""
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
168
|
+
Initializes the basic settings for an DownloadClient object
|
|
169
|
+
|
|
170
|
+
Parameters
|
|
171
|
+
----------
|
|
172
|
+
client :
|
|
173
|
+
Optional: rucio.client.client.Client object. If None, a new object will be created.
|
|
174
|
+
logger :
|
|
175
|
+
Optional: If None, default logger will be used.
|
|
176
|
+
external_traces :
|
|
177
|
+
Optional: reference to a list where traces can be added
|
|
156
178
|
"""
|
|
157
179
|
self.check_pcache = check_pcache
|
|
158
180
|
if logger is None:
|
|
@@ -219,29 +241,63 @@ class DownloadClient:
|
|
|
219
241
|
"""
|
|
220
242
|
Download items with a given PFN. This function can only download files, no datasets.
|
|
221
243
|
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
244
|
+
Parameters
|
|
245
|
+
----------
|
|
246
|
+
items :
|
|
247
|
+
List of dictionaries. Each dictionary describing a file to download. Dictionary keys:
|
|
248
|
+
* **pfn** : str
|
|
249
|
+
PFN string of this file
|
|
250
|
+
|
|
251
|
+
* **did** : str
|
|
252
|
+
DID string of this file (e.g. 'scope:file.name'). Wildcards are not allowed
|
|
253
|
+
|
|
254
|
+
* **rse** : str
|
|
255
|
+
rse name (e.g. 'CERN-PROD_DATADISK'). RSE Expressions are not allowed
|
|
256
|
+
|
|
257
|
+
* **base_dir** : str, optional
|
|
258
|
+
Base directory where the downloaded files will be stored. Default: '.'
|
|
259
|
+
|
|
260
|
+
* **no_subdir** : bool, optional
|
|
261
|
+
If true, files are written directly into base_dir. Default: False
|
|
262
|
+
|
|
263
|
+
* **adler32** : str, optional
|
|
264
|
+
The adler32 checksum to compare the downloaded file's adler32 checksum with
|
|
265
|
+
|
|
266
|
+
* **md5** : str, optional
|
|
267
|
+
The md5 checksum to compare the downloaded file's md5 checksum with
|
|
268
|
+
|
|
269
|
+
* **transfer_timeout** : int, optional
|
|
270
|
+
Timeout time for the download protocols. Default: None
|
|
271
|
+
|
|
272
|
+
* **check_local_with_filesize_only** : bool, optional
|
|
273
|
+
If true, already downloaded files will not be validated by checksum. Default: False
|
|
274
|
+
|
|
275
|
+
num_threads :
|
|
276
|
+
Suggestion of number of threads to use for the download. It will be lowered if it's too high.
|
|
277
|
+
trace_custom_fields :
|
|
278
|
+
Custom key value pairs to send with the traces
|
|
279
|
+
traces_copy_out :
|
|
280
|
+
Reference to an external list, where the traces should be uploaded
|
|
281
|
+
deactivate_file_download_exceptions :
|
|
282
|
+
If file download exceptions shouldn't be raised. Default: False
|
|
283
|
+
|
|
284
|
+
Returns
|
|
285
|
+
-------
|
|
286
|
+
|
|
287
|
+
A list of dictionaries with an entry for each file, containing the input options,
|
|
288
|
+
the did, and the clientState. clientState can be one of the following:
|
|
289
|
+
ALREADY_DONE, DONE, FILE_NOT_FOUND, FAIL_VALIDATE, FAILED
|
|
290
|
+
|
|
291
|
+
Raises
|
|
292
|
+
------
|
|
293
|
+
InputValidationError
|
|
294
|
+
If one of the input items is in the wrong format
|
|
295
|
+
NoFilesDownloaded
|
|
296
|
+
If no files could be downloaded
|
|
297
|
+
NotAllFilesDownloaded
|
|
298
|
+
If not all files could be downloaded
|
|
299
|
+
RucioException
|
|
300
|
+
If something unexpected went wrong during the download
|
|
245
301
|
"""
|
|
246
302
|
trace_custom_fields = trace_custom_fields or {}
|
|
247
303
|
logger = self.logger
|
|
@@ -304,34 +360,80 @@ class DownloadClient:
|
|
|
304
360
|
"""
|
|
305
361
|
Download items with given DIDs. This function can also download datasets and wildcarded DIDs.
|
|
306
362
|
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
363
|
+
Parameters
|
|
364
|
+
----------
|
|
365
|
+
items :
|
|
366
|
+
List of dictionaries. Each dictionary describing an item to download. Dictionary keys:
|
|
367
|
+
* **did** : str
|
|
368
|
+
DID string of this file (e.g. 'scope:file.name')
|
|
369
|
+
|
|
370
|
+
* **filters** : dict, optional
|
|
371
|
+
Filter to select DIDs for download. Optional if DID is given
|
|
372
|
+
|
|
373
|
+
* **rse** : str, optional
|
|
374
|
+
rse name (e.g. 'CERN-PROD_DATADISK') or rse expression from where to download
|
|
375
|
+
|
|
376
|
+
* **impl** : str, optional
|
|
377
|
+
name of the protocol implementation to be used to download this item
|
|
378
|
+
|
|
379
|
+
* **no_resolve_archives** : bool, optional
|
|
380
|
+
bool indicating whether archives should not be considered for download. Default: False
|
|
381
|
+
|
|
382
|
+
* **resolve_archives** : bool, optional
|
|
383
|
+
Deprecated: Use no_resolve_archives instead
|
|
384
|
+
|
|
385
|
+
* **force_scheme** : str or list[str], optional
|
|
386
|
+
force a specific scheme to download this item. Default: None
|
|
387
|
+
|
|
388
|
+
* **base_dir** : str, optional
|
|
389
|
+
base directory where the downloaded files will be stored. Default: '.'
|
|
390
|
+
|
|
391
|
+
* **no_subdir** : bool, optional
|
|
392
|
+
If true, files are written directly into base_dir. Default: False
|
|
393
|
+
|
|
394
|
+
* **nrandom** : int, optional
|
|
395
|
+
if the DID addresses a dataset, nrandom files will be randomly chosen for download from the dataset
|
|
396
|
+
|
|
397
|
+
* **ignore_checksum** : bool, optional
|
|
398
|
+
If true, skips the checksum validation between the downloaded file and the rucio catalouge. Default: False
|
|
399
|
+
|
|
400
|
+
* **transfer_timeout** : int, optional
|
|
401
|
+
Timeout time for the download protocols. Default: None
|
|
402
|
+
|
|
403
|
+
* **transfer_speed_timeout** : int, optional
|
|
404
|
+
Minimum allowed transfer speed (in KBps). Ignored if transfer_timeout set. Otherwise, used to compute default timeout. Default: 500
|
|
405
|
+
|
|
406
|
+
* **check_local_with_filesize_only** : bool, optional
|
|
407
|
+
If true, already downloaded files will not be validated by checksum. Default: False
|
|
408
|
+
|
|
409
|
+
num_threads :
|
|
410
|
+
Suggestion of number of threads to use for the download. It will be lowered if it's too high.
|
|
411
|
+
trace_custom_fields :
|
|
412
|
+
Custom key value pairs to send with the traces
|
|
413
|
+
traces_copy_out :
|
|
414
|
+
Reference to an external list, where the traces should be uploaded
|
|
415
|
+
deactivate_file_download_exceptions :
|
|
416
|
+
If file download exceptions shouldn't be raised. Default: False
|
|
417
|
+
sort :
|
|
418
|
+
Select best replica by replica sorting algorithm. Available algorithms:
|
|
419
|
+
``geoip`` - based on src/dst IP topographical distance
|
|
420
|
+
|
|
421
|
+
Returns
|
|
422
|
+
-------
|
|
423
|
+
|
|
424
|
+
A list of dictionaries with an entry for each file, containing the input options,
|
|
425
|
+
the did, and the clientState.
|
|
426
|
+
|
|
427
|
+
Raises
|
|
428
|
+
------
|
|
429
|
+
InputValidationError
|
|
430
|
+
If one of the input items is in the wrong format
|
|
431
|
+
NoFilesDownloaded
|
|
432
|
+
If no files could be downloaded
|
|
433
|
+
NotAllFilesDownloaded
|
|
434
|
+
If not all files could be downloaded
|
|
435
|
+
RucioException
|
|
436
|
+
If something unexpected went wrong during the download
|
|
335
437
|
"""
|
|
336
438
|
trace_custom_fields = trace_custom_fields or {}
|
|
337
439
|
logger = self.logger
|
|
@@ -364,24 +466,51 @@ class DownloadClient:
|
|
|
364
466
|
"""
|
|
365
467
|
Download items using a given metalink file.
|
|
366
468
|
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
469
|
+
Parameters
|
|
470
|
+
----------
|
|
471
|
+
item :
|
|
472
|
+
Dictionary describing an item to download. Dictionary keys:
|
|
473
|
+
* **base_dir** : str, optional
|
|
474
|
+
Base directory where the downloaded files will be stored. Default: '.'
|
|
475
|
+
|
|
476
|
+
* **no_subdir** : bool, optional
|
|
477
|
+
If true, files are written directly into base_dir. Default: False
|
|
478
|
+
|
|
479
|
+
* **ignore_checksum** : bool, optional
|
|
480
|
+
If true, skips the checksum validation between the downloaded file and the rucio catalogue. Default: False
|
|
481
|
+
|
|
482
|
+
* **transfer_timeout** : int, optional
|
|
483
|
+
Timeout time for the download protocols. Default: None
|
|
484
|
+
|
|
485
|
+
* **check_local_with_filesize_only** : bool, optional
|
|
486
|
+
If true, already downloaded files will not be validated by checksum. Default: False
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
num_threads :
|
|
490
|
+
Suggestion of number of threads to use for the download. It will be lowered if it's too high.
|
|
491
|
+
trace_custom_fields :
|
|
492
|
+
Custom key value pairs to send with the traces
|
|
493
|
+
traces_copy_out :
|
|
494
|
+
Reference to an external list, where the traces should be uploaded
|
|
495
|
+
deactivate_file_download_exceptions :
|
|
496
|
+
If file download exceptions shouldn't be raised. Default: False
|
|
497
|
+
|
|
498
|
+
Returns
|
|
499
|
+
-------
|
|
500
|
+
|
|
501
|
+
A list of dictionaries with an entry for each file, containing the input options,
|
|
502
|
+
the did, and the clientState.
|
|
503
|
+
|
|
504
|
+
Raises
|
|
505
|
+
------
|
|
506
|
+
InputValidationError
|
|
507
|
+
If one of the input items is in the wrong format
|
|
508
|
+
NoFilesDownloaded
|
|
509
|
+
If no files could be downloaded
|
|
510
|
+
NotAllFilesDownloaded
|
|
511
|
+
If not all files could be downloaded
|
|
512
|
+
RucioException
|
|
513
|
+
If something unexpected went wrong during the download
|
|
385
514
|
"""
|
|
386
515
|
trace_custom_fields = trace_custom_fields or {}
|
|
387
516
|
logger = self.logger
|
|
@@ -419,12 +548,21 @@ class DownloadClient:
|
|
|
419
548
|
Starts an appropriate number of threads to download items from the input list.
|
|
420
549
|
(This function is meant to be used as class internal only)
|
|
421
550
|
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
551
|
+
Parameters
|
|
552
|
+
----------
|
|
553
|
+
input_items :
|
|
554
|
+
List containing the input items to download
|
|
555
|
+
num_threads :
|
|
556
|
+
Suggestion of how many threads should be started
|
|
557
|
+
trace_custom_fields :
|
|
558
|
+
Custom key value pairs to send with the traces
|
|
559
|
+
traces_copy_out :
|
|
560
|
+
Reference to an external list, where the traces should be uploaded
|
|
561
|
+
|
|
562
|
+
Returns
|
|
563
|
+
-------
|
|
564
|
+
|
|
565
|
+
List with output items as dictionaries
|
|
428
566
|
"""
|
|
429
567
|
trace_custom_fields = trace_custom_fields or {}
|
|
430
568
|
logger = self.logger
|
|
@@ -483,11 +621,18 @@ class DownloadClient:
|
|
|
483
621
|
downloads them and stores the output in the output queue.
|
|
484
622
|
(This function is meant to be used as class internal only)
|
|
485
623
|
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
624
|
+
Parameters
|
|
625
|
+
----------
|
|
626
|
+
input_queue :
|
|
627
|
+
Queue containing the input items to download
|
|
628
|
+
output_queue :
|
|
629
|
+
Queue where the output items will be stored
|
|
630
|
+
trace_custom_fields :
|
|
631
|
+
Custom key value pairs to send with the traces
|
|
632
|
+
traces_copy_out :
|
|
633
|
+
Reference to an external list, where the traces should be uploaded
|
|
634
|
+
log_prefix :
|
|
635
|
+
String that will be put at the beginning of every log message
|
|
491
636
|
"""
|
|
492
637
|
logger = self.logger
|
|
493
638
|
|
|
@@ -516,8 +661,16 @@ class DownloadClient:
|
|
|
516
661
|
def _compute_actual_transfer_timeout(item: dict[str, Any]) -> int:
|
|
517
662
|
"""
|
|
518
663
|
Merge the two options related to timeout into the value which will be used for protocol download.
|
|
519
|
-
|
|
520
|
-
|
|
664
|
+
|
|
665
|
+
Parameters
|
|
666
|
+
----------
|
|
667
|
+
item :
|
|
668
|
+
Dictionary that describes the item to download
|
|
669
|
+
|
|
670
|
+
Returns
|
|
671
|
+
-------
|
|
672
|
+
|
|
673
|
+
Timeout in seconds
|
|
521
674
|
"""
|
|
522
675
|
default_transfer_timeout = 360
|
|
523
676
|
default_transfer_speed_timeout = 500 # KBps
|
|
@@ -553,12 +706,21 @@ class DownloadClient:
|
|
|
553
706
|
Downloads the given item and sends traces for success/failure.
|
|
554
707
|
(This function is meant to be used as class internal only)
|
|
555
708
|
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
709
|
+
Parameters
|
|
710
|
+
-----------
|
|
711
|
+
item :
|
|
712
|
+
Dictionary describing the item to download
|
|
713
|
+
trace :
|
|
714
|
+
Dictionary representing a pattern of trace that will be send
|
|
715
|
+
traces_copy_out :
|
|
716
|
+
Reference to an external list, where the traces should be uploaded
|
|
717
|
+
log_prefix :
|
|
718
|
+
String that will be put at the beginning of every log message
|
|
719
|
+
|
|
720
|
+
Returns
|
|
721
|
+
-------
|
|
722
|
+
|
|
723
|
+
Dictionary with all attributes from the input item and a clientState attribute
|
|
562
724
|
"""
|
|
563
725
|
logger = self.logger
|
|
564
726
|
pcache = Pcache() if self.check_pcache and len(item.get('archive_items', [])) == 0 else None
|
|
@@ -820,27 +982,58 @@ class DownloadClient:
|
|
|
820
982
|
It only can download files that are available via https/davs.
|
|
821
983
|
Aria2c needs to be installed and X509_USER_PROXY needs to be set!
|
|
822
984
|
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
985
|
+
Parameters
|
|
986
|
+
----------
|
|
987
|
+
items :
|
|
988
|
+
List of dictionaries. Each dictionary describing an item to download. Dictionary keys:
|
|
989
|
+
* **did** : str
|
|
990
|
+
DID string of this file (e.g. 'scope:file.name'). Wildcards are not allowed
|
|
991
|
+
|
|
992
|
+
* **rse** : str, optional
|
|
993
|
+
rse name (e.g. 'CERN-PROD_DATADISK') or rse expression from where to download
|
|
994
|
+
|
|
995
|
+
* **base_dir** : str, optional
|
|
996
|
+
base directory where the downloaded files will be stored. (Default: '.')
|
|
997
|
+
|
|
998
|
+
* **no_subdir** : bool, optional
|
|
999
|
+
If true, files are written directly into base_dir. (Default: False)
|
|
1000
|
+
|
|
1001
|
+
* **nrandom** : int, optional
|
|
1002
|
+
if the DID addresses a dataset, nrandom files will be randomly chosen for download from the dataset
|
|
1003
|
+
|
|
1004
|
+
* **ignore_checksum** : bool, optional
|
|
1005
|
+
If true, skips the checksum validation between the downloaded file and the rucio catalogue. (Default: False)
|
|
1006
|
+
|
|
1007
|
+
* **check_local_with_filesize_only** : bool, optional
|
|
1008
|
+
If true, already downloaded files will not be validated by checksum. (Default: False)
|
|
1009
|
+
|
|
1010
|
+
trace_custom_fields :
|
|
1011
|
+
Custom key value pairs to send with the traces
|
|
1012
|
+
filters :
|
|
1013
|
+
Filter to select DIDs for download
|
|
1014
|
+
deactivate_file_download_exceptions :
|
|
1015
|
+
If file download exceptions shouldn't be raised. Default: False
|
|
1016
|
+
sort :
|
|
1017
|
+
Select best replica by replica sorting algorithm. Available algorithms:
|
|
1018
|
+
* **geoip** - based on src/dst IP topographical distance
|
|
1019
|
+
|
|
1020
|
+
Returns
|
|
1021
|
+
-------
|
|
1022
|
+
|
|
1023
|
+
A list of dictionaries with an entry for each file, containing the input options,
|
|
1024
|
+
the did, and the clientState.
|
|
1025
|
+
|
|
1026
|
+
|
|
1027
|
+
Raises
|
|
1028
|
+
------
|
|
1029
|
+
InputValidationError
|
|
1030
|
+
If one of the input items is in the wrong format
|
|
1031
|
+
NoFilesDownloaded
|
|
1032
|
+
If no files could be downloaded
|
|
1033
|
+
NotAllFilesDownloaded
|
|
1034
|
+
If not all files could be downloaded
|
|
1035
|
+
RucioException
|
|
1036
|
+
If something unexpected went wrong during the download (e.g. aria2c could not be started)
|
|
844
1037
|
"""
|
|
845
1038
|
trace_custom_fields = trace_custom_fields or {}
|
|
846
1039
|
filters = filters or {}
|
|
@@ -880,11 +1073,20 @@ class DownloadClient:
|
|
|
880
1073
|
the RPC proxy instance.
|
|
881
1074
|
(This function is meant to be used as class internal only)
|
|
882
1075
|
|
|
883
|
-
|
|
1076
|
+
Parameters
|
|
1077
|
+
----------
|
|
1078
|
+
rpc_secret :
|
|
1079
|
+
The secret for the RPC proxy
|
|
1080
|
+
|
|
1081
|
+
Returns
|
|
1082
|
+
-------
|
|
884
1083
|
|
|
885
|
-
|
|
1084
|
+
A tuple with the process and the rpc proxy objects
|
|
886
1085
|
|
|
887
|
-
|
|
1086
|
+
Raises
|
|
1087
|
+
------
|
|
1088
|
+
RucioException
|
|
1089
|
+
If the process or the proxy could not be created
|
|
888
1090
|
"""
|
|
889
1091
|
logger = self.logger
|
|
890
1092
|
from xmlrpc.client import ServerProxy as RPCServerProxy
|
|
@@ -956,12 +1158,22 @@ class DownloadClient:
|
|
|
956
1158
|
as RPC background process first and a RPC proxy is needed.
|
|
957
1159
|
(This function is meant to be used as class internal only)
|
|
958
1160
|
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
1161
|
+
Parameters
|
|
1162
|
+
----------
|
|
1163
|
+
items :
|
|
1164
|
+
List of dictionaries. Each dictionary describing an item to download
|
|
1165
|
+
aria_rpc :
|
|
1166
|
+
RPC proxy to the aria2c process
|
|
1167
|
+
rpc_auth :
|
|
1168
|
+
The rpc authentication token
|
|
1169
|
+
trace_custom_fields :
|
|
1170
|
+
Custom key value pairs to send with the traces
|
|
1171
|
+
|
|
1172
|
+
Returns
|
|
1173
|
+
-------
|
|
1174
|
+
|
|
1175
|
+
A list of dictionaries with an entry for each file, containing the input options,
|
|
1176
|
+
the did, and the clientState.
|
|
965
1177
|
"""
|
|
966
1178
|
trace_custom_fields = trace_custom_fields or {}
|
|
967
1179
|
logger = self.logger
|
|
@@ -1111,7 +1323,11 @@ class DownloadClient:
|
|
|
1111
1323
|
def _resolve_one_item_dids(self, item: dict[str, Any]) -> "Iterator[dict[str, Any]]":
|
|
1112
1324
|
"""
|
|
1113
1325
|
Resolve scopes or wildcard DIDs to lists of full did names:
|
|
1114
|
-
|
|
1326
|
+
|
|
1327
|
+
Parameters
|
|
1328
|
+
----------
|
|
1329
|
+
item :
|
|
1330
|
+
One input item
|
|
1115
1331
|
"""
|
|
1116
1332
|
dids = item.get('did')
|
|
1117
1333
|
filters = item.get('filters', {})
|
|
@@ -1150,21 +1366,30 @@ class DownloadClient:
|
|
|
1150
1366
|
This function takes the input items given to download_dids etc.
|
|
1151
1367
|
and resolves the sources.
|
|
1152
1368
|
|
|
1153
|
-
|
|
1369
|
+
1. It first performs a list_dids call to dereference any wildcards and
|
|
1154
1370
|
retrieve DID stats (size, length, type).
|
|
1155
|
-
|
|
1371
|
+
|
|
1372
|
+
2. Next, input items are grouped together by common list_replicas options.
|
|
1156
1373
|
For each group, a single list_replicas call is performed.
|
|
1157
|
-
|
|
1374
|
+
|
|
1375
|
+
3. The resolved File DIDs with sources are finally mapped back to initial
|
|
1158
1376
|
input items to be able to correctly retrieve download options
|
|
1159
1377
|
(timeout, destination directories, etc)
|
|
1160
1378
|
|
|
1161
|
-
|
|
1379
|
+
Parameters
|
|
1380
|
+
----------
|
|
1381
|
+
input_items :
|
|
1382
|
+
List of dictionaries. Each dictionary describing an input item
|
|
1162
1383
|
|
|
1163
|
-
:
|
|
1164
|
-
|
|
1165
|
-
|
|
1384
|
+
Returns:
|
|
1385
|
+
-------
|
|
1386
|
+
* a dictionary that maps the dereferenced(w/o wildcards) input DIDs to a list of input items
|
|
1387
|
+
* and a list with a dictionary for each file DID which has to be downloaded
|
|
1166
1388
|
|
|
1167
|
-
|
|
1389
|
+
Raises
|
|
1390
|
+
------
|
|
1391
|
+
InputValidationError
|
|
1392
|
+
If one of the input items is in the wrong format
|
|
1168
1393
|
"""
|
|
1169
1394
|
logger = self.logger
|
|
1170
1395
|
|
|
@@ -1372,12 +1597,22 @@ class DownloadClient:
|
|
|
1372
1597
|
Optimises the amount of files to download
|
|
1373
1598
|
(This function is meant to be used as class internal only)
|
|
1374
1599
|
|
|
1375
|
-
|
|
1376
|
-
|
|
1600
|
+
Parameters
|
|
1601
|
+
----------
|
|
1602
|
+
did_to_input_items :
|
|
1603
|
+
dictionary that maps resolved input DIDs to input items
|
|
1604
|
+
file_items :
|
|
1605
|
+
list of dictionaries. Each dictionary describes a File DID to download
|
|
1377
1606
|
|
|
1378
|
-
|
|
1607
|
+
Returns
|
|
1608
|
+
-------
|
|
1379
1609
|
|
|
1380
|
-
|
|
1610
|
+
list of dictionaries. Each dictionary describes an element to download
|
|
1611
|
+
|
|
1612
|
+
Raises
|
|
1613
|
+
------
|
|
1614
|
+
InputValidationError
|
|
1615
|
+
If the given input is not valid or incomplete
|
|
1381
1616
|
"""
|
|
1382
1617
|
logger = self.logger
|
|
1383
1618
|
|
|
@@ -1599,11 +1834,20 @@ class DownloadClient:
|
|
|
1599
1834
|
Splits a given DID string (e.g. 'scope1:name.file') into its scope and name part
|
|
1600
1835
|
(This function is meant to be used as class internal only)
|
|
1601
1836
|
|
|
1602
|
-
|
|
1837
|
+
Parameters
|
|
1838
|
+
----------
|
|
1839
|
+
did_str :
|
|
1840
|
+
the DID string that will be split
|
|
1841
|
+
|
|
1842
|
+
Returns
|
|
1843
|
+
-------
|
|
1603
1844
|
|
|
1604
|
-
|
|
1845
|
+
the scope- and name part of the given DID
|
|
1605
1846
|
|
|
1606
|
-
|
|
1847
|
+
Raises
|
|
1848
|
+
------
|
|
1849
|
+
InputValidationError
|
|
1850
|
+
If the given DID string is not valid
|
|
1607
1851
|
"""
|
|
1608
1852
|
did = did_str.split(':')
|
|
1609
1853
|
if len(did) == 2:
|
|
@@ -1638,11 +1882,19 @@ class DownloadClient:
|
|
|
1638
1882
|
destination directory if it's not existent.
|
|
1639
1883
|
(This function is meant to be used as class internal only)
|
|
1640
1884
|
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1885
|
+
Parameters
|
|
1886
|
+
----------
|
|
1887
|
+
base_dir :
|
|
1888
|
+
base directory part
|
|
1889
|
+
dest_dir_name :
|
|
1890
|
+
name of the destination directory
|
|
1891
|
+
no_subdir :
|
|
1892
|
+
if no subdirectory should be created
|
|
1644
1893
|
|
|
1645
|
-
|
|
1894
|
+
Returns
|
|
1895
|
+
-------
|
|
1896
|
+
|
|
1897
|
+
the absolute path of the destination directory
|
|
1646
1898
|
"""
|
|
1647
1899
|
# append dest_dir_name, if subdir should be used
|
|
1648
1900
|
if dest_dir_name.startswith('/'):
|
|
@@ -1663,13 +1915,22 @@ class DownloadClient:
|
|
|
1663
1915
|
Checks if all files were successfully downloaded
|
|
1664
1916
|
(This function is meant to be used as class internal only)
|
|
1665
1917
|
|
|
1666
|
-
|
|
1667
|
-
|
|
1918
|
+
Parameters
|
|
1919
|
+
----------
|
|
1920
|
+
output_items :
|
|
1921
|
+
list of dictionaries describing the downloaded files
|
|
1922
|
+
deactivate_file_download_exceptions :
|
|
1923
|
+
Boolean, if file download exceptions shouldn't be raised
|
|
1924
|
+
|
|
1925
|
+
Returns
|
|
1926
|
+
-------
|
|
1668
1927
|
|
|
1669
|
-
|
|
1928
|
+
output_items list
|
|
1670
1929
|
|
|
1671
|
-
|
|
1672
|
-
|
|
1930
|
+
Raises
|
|
1931
|
+
------
|
|
1932
|
+
NoFilesDownloaded
|
|
1933
|
+
NotAllFilesDownloaded
|
|
1673
1934
|
"""
|
|
1674
1935
|
success_states = [FileDownloadState.ALREADY_DONE, FileDownloadState.DONE, FileDownloadState.FOUND_IN_PCACHE]
|
|
1675
1936
|
num_successful = 0
|
|
@@ -1692,19 +1953,28 @@ class DownloadClient:
|
|
|
1692
1953
|
"""
|
|
1693
1954
|
Checks if sending trace is allowed and send the trace.
|
|
1694
1955
|
|
|
1695
|
-
|
|
1956
|
+
Parameters
|
|
1957
|
+
----------
|
|
1958
|
+
trace :
|
|
1959
|
+
the trace to send
|
|
1696
1960
|
"""
|
|
1697
1961
|
if self.tracing:
|
|
1698
1962
|
send_trace(trace, self.client.trace_host, self.client.user_agent)
|
|
1699
1963
|
|
|
1700
1964
|
def preferred_impl(self, sources: list[dict[str, Any]]) -> Optional[str]:
|
|
1701
1965
|
"""
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1966
|
+
Finds the optimum protocol impl preferred by the client and
|
|
1967
|
+
supported by the remote RSE.
|
|
1968
|
+
|
|
1969
|
+
Parameters
|
|
1970
|
+
----------
|
|
1971
|
+
sources :
|
|
1972
|
+
List of sources for a given DID
|
|
1973
|
+
|
|
1974
|
+
Raises
|
|
1975
|
+
-------
|
|
1976
|
+
RucioException
|
|
1977
|
+
General exception with msg for more details
|
|
1708
1978
|
"""
|
|
1709
1979
|
|
|
1710
1980
|
preferred_protocols = []
|