rucio-clients 37.3.0__py3-none-any.whl → 37.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio-clients might be problematic. Click here for more details.

Files changed (41) hide show
  1. rucio/cli/rule.py +1 -1
  2. rucio/client/accountclient.py +205 -60
  3. rucio/client/accountlimitclient.py +84 -25
  4. rucio/client/baseclient.py +85 -48
  5. rucio/client/client.py +49 -41
  6. rucio/client/configclient.py +36 -13
  7. rucio/client/credentialclient.py +16 -6
  8. rucio/client/didclient.py +321 -133
  9. rucio/client/diracclient.py +13 -6
  10. rucio/client/downloadclient.py +435 -165
  11. rucio/client/exportclient.py +8 -2
  12. rucio/client/fileclient.py +10 -3
  13. rucio/client/importclient.py +4 -1
  14. rucio/client/lifetimeclient.py +48 -31
  15. rucio/client/lockclient.py +22 -7
  16. rucio/client/metaconventionsclient.py +59 -21
  17. rucio/client/pingclient.py +3 -1
  18. rucio/client/replicaclient.py +213 -96
  19. rucio/client/requestclient.py +124 -16
  20. rucio/client/rseclient.py +385 -160
  21. rucio/client/ruleclient.py +147 -51
  22. rucio/client/scopeclient.py +35 -10
  23. rucio/client/subscriptionclient.py +60 -27
  24. rucio/client/touchclient.py +16 -7
  25. rucio/common/constants.py +14 -17
  26. rucio/common/utils.py +18 -2
  27. rucio/rse/rsemanager.py +2 -2
  28. rucio/vcsversion.py +3 -3
  29. {rucio_clients-37.3.0.data → rucio_clients-37.5.0.data}/data/etc/rucio.cfg.template +0 -1
  30. {rucio_clients-37.3.0.dist-info → rucio_clients-37.5.0.dist-info}/METADATA +1 -1
  31. {rucio_clients-37.3.0.dist-info → rucio_clients-37.5.0.dist-info}/RECORD +41 -41
  32. {rucio_clients-37.3.0.dist-info → rucio_clients-37.5.0.dist-info}/WHEEL +1 -1
  33. {rucio_clients-37.3.0.data → rucio_clients-37.5.0.data}/data/etc/rse-accounts.cfg.template +0 -0
  34. {rucio_clients-37.3.0.data → rucio_clients-37.5.0.data}/data/etc/rucio.cfg.atlas.client.template +0 -0
  35. {rucio_clients-37.3.0.data → rucio_clients-37.5.0.data}/data/requirements.client.txt +0 -0
  36. {rucio_clients-37.3.0.data → rucio_clients-37.5.0.data}/data/rucio_client/merge_rucio_configs.py +0 -0
  37. {rucio_clients-37.3.0.data → rucio_clients-37.5.0.data}/scripts/rucio +0 -0
  38. {rucio_clients-37.3.0.data → rucio_clients-37.5.0.data}/scripts/rucio-admin +0 -0
  39. {rucio_clients-37.3.0.dist-info → rucio_clients-37.5.0.dist-info}/licenses/AUTHORS.rst +0 -0
  40. {rucio_clients-37.3.0.dist-info → rucio_clients-37.5.0.dist-info}/licenses/LICENSE +0 -0
  41. {rucio_clients-37.3.0.dist-info → rucio_clients-37.5.0.dist-info}/top_level.txt +0 -0
@@ -71,12 +71,19 @@ class BaseExtractionTool:
71
71
  logger: "LoggerFunction" = logging.log
72
72
  ):
73
73
  """
74
- Initialises a extraction tool object
74
+ Initializes a extraction tool object
75
+
76
+ Parameters
77
+ ----------
78
+ program_name :
79
+ the name of the archive extraction program, e.g., unzip
80
+ useability_check_args :
81
+ the arguments of the extraction program to test if it's installed, e.g., --version
82
+ extract_args :
83
+ the arguments that will be passed to the program for extraction
84
+ logger :
85
+ optional decorated logging.log object that can be passed from the calling daemon or client.
75
86
 
76
- :param program_name: the name of the archive extraction program, e.g., unzip
77
- :param useability_check_args: the arguments of the extraction program to test if its installed, e.g., --version
78
- :param extract_args: the arguments that will be passed to the program for extraction
79
- :param logger: optional decorated logging.log object that can be passed from the calling daemon or client.
80
87
  """
81
88
  self.program_name = program_name
82
89
  self.useability_check_args = useability_check_args
@@ -88,7 +95,9 @@ class BaseExtractionTool:
88
95
  """
89
96
  Checks if the extraction tool is installed and usable
90
97
 
91
- :returns: True if it is usable otherwise False
98
+ Returns
99
+ -------
100
+ True if it is usable otherwise False
92
101
  """
93
102
  if self.is_useable_result is not None:
94
103
  return self.is_useable_result
@@ -113,11 +122,19 @@ class BaseExtractionTool:
113
122
  """
114
123
  Calls the extraction program to extract a file from an archive
115
124
 
116
- :param archive_file_path: path to the archive
117
- :param file_to_extract: file name to extract from the archive
118
- :param dest_dir_path: destination directory where the extracted file will be stored
125
+ Parameters
126
+ ----------
127
+ archive_file_path :
128
+ path to the archive
129
+ file_to_extract :
130
+ file name to extract from the archive
131
+ dest_dir_pat :
132
+ destination directory where the extracted file will be stored
133
+
134
+ Returns
135
+ -------
136
+ True on success otherwise False
119
137
 
120
- :returns: True on success otherwise False
121
138
  """
122
139
  if not self.is_useable():
123
140
  return False
@@ -148,11 +165,16 @@ class DownloadClient:
148
165
  check_pcache: bool = False
149
166
  ):
150
167
  """
151
- Initialises the basic settings for an DownloadClient object
152
-
153
- :param client: Optional: rucio.client.client.Client object. If None, a new object will be created.
154
- :param external_traces: Optional: reference to a list where traces can be added
155
- :param logger: Optional: logging.Logger object. If None, default logger will be used.
168
+ Initializes the basic settings for an DownloadClient object
169
+
170
+ Parameters
171
+ ----------
172
+ client :
173
+ Optional: rucio.client.client.Client object. If None, a new object will be created.
174
+ logger :
175
+ Optional: If None, default logger will be used.
176
+ external_traces :
177
+ Optional: reference to a list where traces can be added
156
178
  """
157
179
  self.check_pcache = check_pcache
158
180
  if logger is None:
@@ -219,29 +241,63 @@ class DownloadClient:
219
241
  """
220
242
  Download items with a given PFN. This function can only download files, no datasets.
221
243
 
222
- :param items: List of dictionaries. Each dictionary describing a file to download. Keys:
223
- pfn - PFN string of this file
224
- did - DID string of this file (e.g. 'scope:file.name'). Wildcards are not allowed
225
- rse - rse name (e.g. 'CERN-PROD_DATADISK'). RSE Expressions are not allowed
226
- base_dir - Optional: Base directory where the downloaded files will be stored. (Default: '.')
227
- no_subdir - Optional: If true, files are written directly into base_dir. (Default: False)
228
- adler32 - Optional: The adler32 checmsum to compare the downloaded files adler32 checksum with
229
- md5 - Optional: The md5 checksum to compare the downloaded files md5 checksum with
230
- transfer_timeout - Optional: Timeout time for the download protocols. (Default: None)
231
- check_local_with_filesize_only - Optional: If true, already downloaded files will not be validated by checksum.
232
- :param num_threads: Suggestion of number of threads to use for the download. It will be lowered if it's too high.
233
- :param trace_custom_fields: Custom key value pairs to send with the traces
234
- :param traces_copy_out: reference to an external list, where the traces should be uploaded
235
- :param deactivate_file_download_exceptions: Boolean, if file download exceptions shouldn't be raised
236
-
237
-
238
- :returns: a list of dictionaries with an entry for each file, containing the input options, the did, and the clientState
239
- clientState can be one of the following: ALREADY_DONE, DONE, FILE_NOT_FOUND, FAIL_VALIDATE, FAILED
240
-
241
- :raises InputValidationError: if one of the input items is in the wrong format
242
- :raises NoFilesDownloaded: if no files could be downloaded
243
- :raises NotAllFilesDownloaded: if not all files could be downloaded
244
- :raises RucioException: if something unexpected went wrong during the download
244
+ Parameters
245
+ ----------
246
+ items :
247
+ List of dictionaries. Each dictionary describing a file to download. Dictionary keys:
248
+ * **pfn** : str
249
+ PFN string of this file
250
+
251
+ * **did** : str
252
+ DID string of this file (e.g. 'scope:file.name'). Wildcards are not allowed
253
+
254
+ * **rse** : str
255
+ rse name (e.g. 'CERN-PROD_DATADISK'). RSE Expressions are not allowed
256
+
257
+ * **base_dir** : str, optional
258
+ Base directory where the downloaded files will be stored. Default: '.'
259
+
260
+ * **no_subdir** : bool, optional
261
+ If true, files are written directly into base_dir. Default: False
262
+
263
+ * **adler32** : str, optional
264
+ The adler32 checksum to compare the downloaded file's adler32 checksum with
265
+
266
+ * **md5** : str, optional
267
+ The md5 checksum to compare the downloaded file's md5 checksum with
268
+
269
+ * **transfer_timeout** : int, optional
270
+ Timeout time for the download protocols. Default: None
271
+
272
+ * **check_local_with_filesize_only** : bool, optional
273
+ If true, already downloaded files will not be validated by checksum. Default: False
274
+
275
+ num_threads :
276
+ Suggestion of number of threads to use for the download. It will be lowered if it's too high.
277
+ trace_custom_fields :
278
+ Custom key value pairs to send with the traces
279
+ traces_copy_out :
280
+ Reference to an external list, where the traces should be uploaded
281
+ deactivate_file_download_exceptions :
282
+ If file download exceptions shouldn't be raised. Default: False
283
+
284
+ Returns
285
+ -------
286
+
287
+ A list of dictionaries with an entry for each file, containing the input options,
288
+ the did, and the clientState. clientState can be one of the following:
289
+ ALREADY_DONE, DONE, FILE_NOT_FOUND, FAIL_VALIDATE, FAILED
290
+
291
+ Raises
292
+ ------
293
+ InputValidationError
294
+ If one of the input items is in the wrong format
295
+ NoFilesDownloaded
296
+ If no files could be downloaded
297
+ NotAllFilesDownloaded
298
+ If not all files could be downloaded
299
+ RucioException
300
+ If something unexpected went wrong during the download
245
301
  """
246
302
  trace_custom_fields = trace_custom_fields or {}
247
303
  logger = self.logger
@@ -304,34 +360,80 @@ class DownloadClient:
304
360
  """
305
361
  Download items with given DIDs. This function can also download datasets and wildcarded DIDs.
306
362
 
307
- :param items: List of dictionaries. Each dictionary describing an item to download. Keys:
308
- did - DID string of this file (e.g. 'scope:file.name')
309
- filters - Filter to select DIDs for download. Optional if DID is given
310
- rse - Optional: rse name (e.g. 'CERN-PROD_DATADISK') or rse expression from where to download
311
- impl - Optional: name of the protocol implementation to be used to download this item.
312
- no_resolve_archives - Optional: bool indicating whether archives should not be considered for download (Default: False)
313
- resolve_archives - Deprecated: Use no_resolve_archives instead
314
- force_scheme - Optional: force a specific scheme to download this item. (Default: None)
315
- base_dir - Optional: base directory where the downloaded files will be stored. (Default: '.')
316
- no_subdir - Optional: If true, files are written directly into base_dir. (Default: False)
317
- nrandom - Optional: if the DID addresses a dataset, nrandom files will be randomly chosen for download from the dataset
318
- ignore_checksum - Optional: If true, skips the checksum validation between the downloaded file and the rucio catalouge. (Default: False)
319
- transfer_timeout - Optional: Timeout time for the download protocols. (Default: None)
320
- transfer_speed_timeout - Optional: Minimum allowed transfer speed (in KBps). Ignored if transfer_timeout set. Otherwise, used to compute default timeout (Default: 500)
321
- check_local_with_filesize_only - Optional: If true, already downloaded files will not be validated by checksum.
322
- :param num_threads: Suggestion of number of threads to use for the download. It will be lowered if it's too high.
323
- :param trace_custom_fields: Custom key value pairs to send with the traces.
324
- :param traces_copy_out: reference to an external list, where the traces should be uploaded
325
- :param deactivate_file_download_exceptions: Boolean, if file download exceptions shouldn't be raised
326
- :param sort: Select best replica by replica sorting algorithm. Available algorithms:
327
- ``geoip`` - based on src/dst IP topographical distance
328
-
329
- :returns: a list of dictionaries with an entry for each file, containing the input options, the did, and the clientState
330
-
331
- :raises InputValidationError: if one of the input items is in the wrong format
332
- :raises NoFilesDownloaded: if no files could be downloaded
333
- :raises NotAllFilesDownloaded: if not all files could be downloaded
334
- :raises RucioException: if something unexpected went wrong during the download
363
+ Parameters
364
+ ----------
365
+ items :
366
+ List of dictionaries. Each dictionary describing an item to download. Dictionary keys:
367
+ * **did** : str
368
+ DID string of this file (e.g. 'scope:file.name')
369
+
370
+ * **filters** : dict, optional
371
+ Filter to select DIDs for download. Optional if DID is given
372
+
373
+ * **rse** : str, optional
374
+ rse name (e.g. 'CERN-PROD_DATADISK') or rse expression from where to download
375
+
376
+ * **impl** : str, optional
377
+ name of the protocol implementation to be used to download this item
378
+
379
+ * **no_resolve_archives** : bool, optional
380
+ bool indicating whether archives should not be considered for download. Default: False
381
+
382
+ * **resolve_archives** : bool, optional
383
+ Deprecated: Use no_resolve_archives instead
384
+
385
+ * **force_scheme** : str or list[str], optional
386
+ force a specific scheme to download this item. Default: None
387
+
388
+ * **base_dir** : str, optional
389
+ base directory where the downloaded files will be stored. Default: '.'
390
+
391
+ * **no_subdir** : bool, optional
392
+ If true, files are written directly into base_dir. Default: False
393
+
394
+ * **nrandom** : int, optional
395
+ if the DID addresses a dataset, nrandom files will be randomly chosen for download from the dataset
396
+
397
+ * **ignore_checksum** : bool, optional
398
+ If true, skips the checksum validation between the downloaded file and the rucio catalouge. Default: False
399
+
400
+ * **transfer_timeout** : int, optional
401
+ Timeout time for the download protocols. Default: None
402
+
403
+ * **transfer_speed_timeout** : int, optional
404
+ Minimum allowed transfer speed (in KBps). Ignored if transfer_timeout set. Otherwise, used to compute default timeout. Default: 500
405
+
406
+ * **check_local_with_filesize_only** : bool, optional
407
+ If true, already downloaded files will not be validated by checksum. Default: False
408
+
409
+ num_threads :
410
+ Suggestion of number of threads to use for the download. It will be lowered if it's too high.
411
+ trace_custom_fields :
412
+ Custom key value pairs to send with the traces
413
+ traces_copy_out :
414
+ Reference to an external list, where the traces should be uploaded
415
+ deactivate_file_download_exceptions :
416
+ If file download exceptions shouldn't be raised. Default: False
417
+ sort :
418
+ Select best replica by replica sorting algorithm. Available algorithms:
419
+ ``geoip`` - based on src/dst IP topographical distance
420
+
421
+ Returns
422
+ -------
423
+
424
+ A list of dictionaries with an entry for each file, containing the input options,
425
+ the did, and the clientState.
426
+
427
+ Raises
428
+ ------
429
+ InputValidationError
430
+ If one of the input items is in the wrong format
431
+ NoFilesDownloaded
432
+ If no files could be downloaded
433
+ NotAllFilesDownloaded
434
+ If not all files could be downloaded
435
+ RucioException
436
+ If something unexpected went wrong during the download
335
437
  """
336
438
  trace_custom_fields = trace_custom_fields or {}
337
439
  logger = self.logger
@@ -364,24 +466,51 @@ class DownloadClient:
364
466
  """
365
467
  Download items using a given metalink file.
366
468
 
367
- :param item: dictionary describing an item to download. Keys:
368
- base_dir - Optional: base directory where the downloaded files will be stored. (Default: '.')
369
- no_subdir - Optional: If true, files are written directly into base_dir. (Default: False)
370
- ignore_checksum - Optional: If true, skips the checksum validation between the downloaded file and the rucio catalouge. (Default: False)
371
- transfer_timeout - Optional: Timeout time for the download protocols. (Default: None)
372
- check_local_with_filesize_only - Optional: If true, already downloaded files will not be validated by checksum.
373
-
374
- :param num_threads: Suggestion of number of threads to use for the download. It will be lowered if it's too high.
375
- :param trace_custom_fields: Custom key value pairs to send with the traces.
376
- :param traces_copy_out: reference to an external list, where the traces should be uploaded
377
- :param deactivate_file_download_exceptions: Boolean, if file download exceptions shouldn't be raised
378
-
379
- :returns: a list of dictionaries with an entry for each file, containing the input options, the did, and the clientState
380
-
381
- :raises InputValidationError: if one of the input items is in the wrong format
382
- :raises NoFilesDownloaded: if no files could be downloaded
383
- :raises NotAllFilesDownloaded: if not all files could be downloaded
384
- :raises RucioException: if something unexpected went wrong during the download
469
+ Parameters
470
+ ----------
471
+ item :
472
+ Dictionary describing an item to download. Dictionary keys:
473
+ * **base_dir** : str, optional
474
+ Base directory where the downloaded files will be stored. Default: '.'
475
+
476
+ * **no_subdir** : bool, optional
477
+ If true, files are written directly into base_dir. Default: False
478
+
479
+ * **ignore_checksum** : bool, optional
480
+ If true, skips the checksum validation between the downloaded file and the rucio catalogue. Default: False
481
+
482
+ * **transfer_timeout** : int, optional
483
+ Timeout time for the download protocols. Default: None
484
+
485
+ * **check_local_with_filesize_only** : bool, optional
486
+ If true, already downloaded files will not be validated by checksum. Default: False
487
+
488
+
489
+ num_threads :
490
+ Suggestion of number of threads to use for the download. It will be lowered if it's too high.
491
+ trace_custom_fields :
492
+ Custom key value pairs to send with the traces
493
+ traces_copy_out :
494
+ Reference to an external list, where the traces should be uploaded
495
+ deactivate_file_download_exceptions :
496
+ If file download exceptions shouldn't be raised. Default: False
497
+
498
+ Returns
499
+ -------
500
+
501
+ A list of dictionaries with an entry for each file, containing the input options,
502
+ the did, and the clientState.
503
+
504
+ Raises
505
+ ------
506
+ InputValidationError
507
+ If one of the input items is in the wrong format
508
+ NoFilesDownloaded
509
+ If no files could be downloaded
510
+ NotAllFilesDownloaded
511
+ If not all files could be downloaded
512
+ RucioException
513
+ If something unexpected went wrong during the download
385
514
  """
386
515
  trace_custom_fields = trace_custom_fields or {}
387
516
  logger = self.logger
@@ -419,12 +548,21 @@ class DownloadClient:
419
548
  Starts an appropriate number of threads to download items from the input list.
420
549
  (This function is meant to be used as class internal only)
421
550
 
422
- :param input_items: list containing the input items to download
423
- :param num_threads: suggestion of how many threads should be started
424
- :param trace_custom_fields: Custom key value pairs to send with the traces
425
- :param traces_copy_out: reference to an external list, where the traces should be uploaded
426
-
427
- :returns: list with output items as dictionaries
551
+ Parameters
552
+ ----------
553
+ input_items :
554
+ List containing the input items to download
555
+ num_threads :
556
+ Suggestion of how many threads should be started
557
+ trace_custom_fields :
558
+ Custom key value pairs to send with the traces
559
+ traces_copy_out :
560
+ Reference to an external list, where the traces should be uploaded
561
+
562
+ Returns
563
+ -------
564
+
565
+ List with output items as dictionaries
428
566
  """
429
567
  trace_custom_fields = trace_custom_fields or {}
430
568
  logger = self.logger
@@ -483,11 +621,18 @@ class DownloadClient:
483
621
  downloads them and stores the output in the output queue.
484
622
  (This function is meant to be used as class internal only)
485
623
 
486
- :param input_queue: queue containing the input items to download
487
- :param output_queue: queue where the output items will be stored
488
- :param trace_custom_fields: Custom key value pairs to send with the traces
489
- :param traces_copy_out: reference to an external list, where the traces should be uploaded
490
- :param log_prefix: string that will be put at the beginning of every log message
624
+ Parameters
625
+ ----------
626
+ input_queue :
627
+ Queue containing the input items to download
628
+ output_queue :
629
+ Queue where the output items will be stored
630
+ trace_custom_fields :
631
+ Custom key value pairs to send with the traces
632
+ traces_copy_out :
633
+ Reference to an external list, where the traces should be uploaded
634
+ log_prefix :
635
+ String that will be put at the beginning of every log message
491
636
  """
492
637
  logger = self.logger
493
638
 
@@ -516,8 +661,16 @@ class DownloadClient:
516
661
  def _compute_actual_transfer_timeout(item: dict[str, Any]) -> int:
517
662
  """
518
663
  Merge the two options related to timeout into the value which will be used for protocol download.
519
- :param item: dictionary that describes the item to download
520
- :return: timeout in seconds
664
+
665
+ Parameters
666
+ ----------
667
+ item :
668
+ Dictionary that describes the item to download
669
+
670
+ Returns
671
+ -------
672
+
673
+ Timeout in seconds
521
674
  """
522
675
  default_transfer_timeout = 360
523
676
  default_transfer_speed_timeout = 500 # KBps
@@ -553,12 +706,21 @@ class DownloadClient:
553
706
  Downloads the given item and sends traces for success/failure.
554
707
  (This function is meant to be used as class internal only)
555
708
 
556
- :param item: dictionary that describes the item to download
557
- :param trace: dictionary representing a pattern of trace that will be send
558
- :param traces_copy_out: reference to an external list, where the traces should be uploaded
559
- :param log_prefix: string that will be put at the beginning of every log message
560
-
561
- :returns: dictionary with all attributes from the input item and a clientState attribute
709
+ Parameters
710
+ -----------
711
+ item :
712
+ Dictionary describing the item to download
713
+ trace :
714
+ Dictionary representing a pattern of trace that will be send
715
+ traces_copy_out :
716
+ Reference to an external list, where the traces should be uploaded
717
+ log_prefix :
718
+ String that will be put at the beginning of every log message
719
+
720
+ Returns
721
+ -------
722
+
723
+ Dictionary with all attributes from the input item and a clientState attribute
562
724
  """
563
725
  logger = self.logger
564
726
  pcache = Pcache() if self.check_pcache and len(item.get('archive_items', [])) == 0 else None
@@ -820,27 +982,58 @@ class DownloadClient:
820
982
  It only can download files that are available via https/davs.
821
983
  Aria2c needs to be installed and X509_USER_PROXY needs to be set!
822
984
 
823
- :param items: List of dictionaries. Each dictionary describing an item to download. Keys:
824
- did - DID string of this file (e.g. 'scope:file.name'). Wildcards are not allowed
825
- rse - Optional: rse name (e.g. 'CERN-PROD_DATADISK') or rse expression from where to download
826
- base_dir - Optional: base directory where the downloaded files will be stored. (Default: '.')
827
- no_subdir - Optional: If true, files are written directly into base_dir. (Default: False)
828
- nrandom - Optional: if the DID addresses a dataset, nrandom files will be randomly chosen for download from the dataset
829
- ignore_checksum - Optional: If true, skips the checksum validation between the downloaded file and the rucio catalouge. (Default: False)
830
- check_local_with_filesize_only - Optional: If true, already downloaded files will not be validated by checksum.
831
-
832
- :param trace_custom_fields: Custom key value pairs to send with the traces
833
- :param filters: dictionary containing filter options
834
- :param deactivate_file_download_exceptions: Boolean, if file download exceptions shouldn't be raised
835
- :param sort: Select best replica by replica sorting algorithm. Available algorithms:
836
- ``geoip`` - based on src/dst IP topographical distance
837
-
838
- :returns: a list of dictionaries with an entry for each file, containing the input options, the did, and the clientState
839
-
840
- :raises InputValidationError: if one of the input items is in the wrong format
841
- :raises NoFilesDownloaded: if no files could be downloaded
842
- :raises NotAllFilesDownloaded: if not all files could be downloaded
843
- :raises RucioException: if something went wrong during the download (e.g. aria2c could not be started)
985
+ Parameters
986
+ ----------
987
+ items :
988
+ List of dictionaries. Each dictionary describing an item to download. Dictionary keys:
989
+ * **did** : str
990
+ DID string of this file (e.g. 'scope:file.name'). Wildcards are not allowed
991
+
992
+ * **rse** : str, optional
993
+ rse name (e.g. 'CERN-PROD_DATADISK') or rse expression from where to download
994
+
995
+ * **base_dir** : str, optional
996
+ base directory where the downloaded files will be stored. (Default: '.')
997
+
998
+ * **no_subdir** : bool, optional
999
+ If true, files are written directly into base_dir. (Default: False)
1000
+
1001
+ * **nrandom** : int, optional
1002
+ if the DID addresses a dataset, nrandom files will be randomly chosen for download from the dataset
1003
+
1004
+ * **ignore_checksum** : bool, optional
1005
+ If true, skips the checksum validation between the downloaded file and the rucio catalogue. (Default: False)
1006
+
1007
+ * **check_local_with_filesize_only** : bool, optional
1008
+ If true, already downloaded files will not be validated by checksum. (Default: False)
1009
+
1010
+ trace_custom_fields :
1011
+ Custom key value pairs to send with the traces
1012
+ filters :
1013
+ Filter to select DIDs for download
1014
+ deactivate_file_download_exceptions :
1015
+ If file download exceptions shouldn't be raised. Default: False
1016
+ sort :
1017
+ Select best replica by replica sorting algorithm. Available algorithms:
1018
+ * **geoip** - based on src/dst IP topographical distance
1019
+
1020
+ Returns
1021
+ -------
1022
+
1023
+ A list of dictionaries with an entry for each file, containing the input options,
1024
+ the did, and the clientState.
1025
+
1026
+
1027
+ Raises
1028
+ ------
1029
+ InputValidationError
1030
+ If one of the input items is in the wrong format
1031
+ NoFilesDownloaded
1032
+ If no files could be downloaded
1033
+ NotAllFilesDownloaded
1034
+ If not all files could be downloaded
1035
+ RucioException
1036
+ If something unexpected went wrong during the download (e.g. aria2c could not be started)
844
1037
  """
845
1038
  trace_custom_fields = trace_custom_fields or {}
846
1039
  filters = filters or {}
@@ -880,11 +1073,20 @@ class DownloadClient:
880
1073
  the RPC proxy instance.
881
1074
  (This function is meant to be used as class internal only)
882
1075
 
883
- :param rpc_secret: the secret for the RPC proxy
1076
+ Parameters
1077
+ ----------
1078
+ rpc_secret :
1079
+ The secret for the RPC proxy
884
1080
 
885
- :returns: a tuple with the process and the rpc proxy objects
1081
+ Returns
1082
+ -------
886
1083
 
887
- :raises RucioException: if the process or the proxy could not be created
1084
+ A tuple with the process and the rpc proxy objects
1085
+
1086
+ Raises
1087
+ ------
1088
+ RucioException
1089
+ If the process or the proxy could not be created
888
1090
  """
889
1091
  logger = self.logger
890
1092
  from xmlrpc.client import ServerProxy as RPCServerProxy
@@ -956,12 +1158,22 @@ class DownloadClient:
956
1158
  as RPC background process first and a RPC proxy is needed.
957
1159
  (This function is meant to be used as class internal only)
958
1160
 
959
- :param items: list of dictionaries containing one dict for each file to download
960
- :param aria_rcp: RPCProxy to the aria2c process
961
- :param rpc_auth: the rpc authentication token
962
- :param trace_custom_fields: Custom key value pairs to send with the traces
963
-
964
- :returns: a list of dictionaries with an entry for each file, containing the input options, the did, and the clientState
1161
+ Parameters
1162
+ ----------
1163
+ items :
1164
+ List of dictionaries. Each dictionary describing an item to download
1165
+ aria_rpc :
1166
+ RPC proxy to the aria2c process
1167
+ rpc_auth :
1168
+ The rpc authentication token
1169
+ trace_custom_fields :
1170
+ Custom key value pairs to send with the traces
1171
+
1172
+ Returns
1173
+ -------
1174
+
1175
+ A list of dictionaries with an entry for each file, containing the input options,
1176
+ the did, and the clientState.
965
1177
  """
966
1178
  trace_custom_fields = trace_custom_fields or {}
967
1179
  logger = self.logger
@@ -1111,7 +1323,11 @@ class DownloadClient:
1111
1323
  def _resolve_one_item_dids(self, item: dict[str, Any]) -> "Iterator[dict[str, Any]]":
1112
1324
  """
1113
1325
  Resolve scopes or wildcard DIDs to lists of full did names:
1114
- :param item: One input item
1326
+
1327
+ Parameters
1328
+ ----------
1329
+ item :
1330
+ One input item
1115
1331
  """
1116
1332
  dids = item.get('did')
1117
1333
  filters = item.get('filters', {})
@@ -1150,21 +1366,30 @@ class DownloadClient:
1150
1366
  This function takes the input items given to download_dids etc.
1151
1367
  and resolves the sources.
1152
1368
 
1153
- - It first performs a list_dids call to dereference any wildcards and
1369
+ 1. It first performs a list_dids call to dereference any wildcards and
1154
1370
  retrieve DID stats (size, length, type).
1155
- - Next, input items are grouped together by common list_replicas options.
1371
+
1372
+ 2. Next, input items are grouped together by common list_replicas options.
1156
1373
  For each group, a single list_replicas call is performed.
1157
- - The resolved File DIDs with sources are finally mapped back to initial
1374
+
1375
+ 3. The resolved File DIDs with sources are finally mapped back to initial
1158
1376
  input items to be able to correctly retrieve download options
1159
1377
  (timeout, destination directories, etc)
1160
1378
 
1161
- :param input_items: List of dictionaries. Each dictionary describing an input item
1379
+ Parameters
1380
+ ----------
1381
+ input_items :
1382
+ List of dictionaries. Each dictionary describing an input item
1162
1383
 
1163
- :returns: a tuple:
1164
- - a dictionary that maps the dereferenced(w/o wildcards) input DIDs to a list of input items
1165
- - and a list with a dictionary for each file DID which has to be downloaded
1384
+ Returns:
1385
+ -------
1386
+ * a dictionary that maps the dereferenced(w/o wildcards) input DIDs to a list of input items
1387
+ * and a list with a dictionary for each file DID which has to be downloaded
1166
1388
 
1167
- :raises InputValidationError: if one of the input items is in the wrong format
1389
+ Raises
1390
+ ------
1391
+ InputValidationError
1392
+ If one of the input items is in the wrong format
1168
1393
  """
1169
1394
  logger = self.logger
1170
1395
 
@@ -1372,12 +1597,22 @@ class DownloadClient:
1372
1597
  Optimises the amount of files to download
1373
1598
  (This function is meant to be used as class internal only)
1374
1599
 
1375
- :param did_to_input_items: dictionary that maps resolved input DIDs to input items
1376
- :param file_items: list of dictionaries. Each dictionary describes a File DID to download
1600
+ Parameters
1601
+ ----------
1602
+ did_to_input_items :
1603
+ dictionary that maps resolved input DIDs to input items
1604
+ file_items :
1605
+ list of dictionaries. Each dictionary describes a File DID to download
1377
1606
 
1378
- :returns: list of dictionaries. Each dictionary describes an element to download
1607
+ Returns
1608
+ -------
1379
1609
 
1380
- :raises InputValidationError: if the given input is not valid or incomplete
1610
+ list of dictionaries. Each dictionary describes an element to download
1611
+
1612
+ Raises
1613
+ ------
1614
+ InputValidationError
1615
+ If the given input is not valid or incomplete
1381
1616
  """
1382
1617
  logger = self.logger
1383
1618
 
@@ -1599,11 +1834,20 @@ class DownloadClient:
1599
1834
  Splits a given DID string (e.g. 'scope1:name.file') into its scope and name part
1600
1835
  (This function is meant to be used as class internal only)
1601
1836
 
1602
- :param did_str: the DID string that will be split
1837
+ Parameters
1838
+ ----------
1839
+ did_str :
1840
+ the DID string that will be split
1841
+
1842
+ Returns
1843
+ -------
1603
1844
 
1604
- :returns: the scope- and name part of the given DID
1845
+ the scope- and name part of the given DID
1605
1846
 
1606
- :raises InputValidationError: if the given DID string is not valid
1847
+ Raises
1848
+ ------
1849
+ InputValidationError
1850
+ If the given DID string is not valid
1607
1851
  """
1608
1852
  did = did_str.split(':')
1609
1853
  if len(did) == 2:
@@ -1638,11 +1882,19 @@ class DownloadClient:
1638
1882
  destination directory if it's not existent.
1639
1883
  (This function is meant to be used as class internal only)
1640
1884
 
1641
- :param base_dir: base directory part
1642
- :param dest_dir_name: name of the destination directory
1643
- :param no_subdir: if no subdirectory should be created
1885
+ Parameters
1886
+ ----------
1887
+ base_dir :
1888
+ base directory part
1889
+ dest_dir_name :
1890
+ name of the destination directory
1891
+ no_subdir :
1892
+ if no subdirectory should be created
1644
1893
 
1645
- :returns: the absolute path of the destination directory
1894
+ Returns
1895
+ -------
1896
+
1897
+ the absolute path of the destination directory
1646
1898
  """
1647
1899
  # append dest_dir_name, if subdir should be used
1648
1900
  if dest_dir_name.startswith('/'):
@@ -1663,13 +1915,22 @@ class DownloadClient:
1663
1915
  Checks if all files were successfully downloaded
1664
1916
  (This function is meant to be used as class internal only)
1665
1917
 
1666
- :param output_items: list of dictionaries describing the downloaded files
1667
- :param deactivate_file_download_exceptions: Boolean, if file download exceptions shouldn't be raised
1918
+ Parameters
1919
+ ----------
1920
+ output_items :
1921
+ list of dictionaries describing the downloaded files
1922
+ deactivate_file_download_exceptions :
1923
+ Boolean, if file download exceptions shouldn't be raised
1924
+
1925
+ Returns
1926
+ -------
1668
1927
 
1669
- :returns: output_items list
1928
+ output_items list
1670
1929
 
1671
- :raises NoFilesDownloaded:
1672
- :raises NotAllFilesDownloaded:
1930
+ Raises
1931
+ ------
1932
+ NoFilesDownloaded
1933
+ NotAllFilesDownloaded
1673
1934
  """
1674
1935
  success_states = [FileDownloadState.ALREADY_DONE, FileDownloadState.DONE, FileDownloadState.FOUND_IN_PCACHE]
1675
1936
  num_successful = 0
@@ -1692,19 +1953,28 @@ class DownloadClient:
1692
1953
  """
1693
1954
  Checks if sending trace is allowed and send the trace.
1694
1955
 
1695
- :param trace: the trace
1956
+ Parameters
1957
+ ----------
1958
+ trace :
1959
+ the trace to send
1696
1960
  """
1697
1961
  if self.tracing:
1698
1962
  send_trace(trace, self.client.trace_host, self.client.user_agent)
1699
1963
 
1700
1964
  def preferred_impl(self, sources: list[dict[str, Any]]) -> Optional[str]:
1701
1965
  """
1702
- Finds the optimum protocol impl preferred by the client and
1703
- supported by the remote RSE.
1704
-
1705
- :param sources: List of sources for a given DID
1706
-
1707
- :raises RucioException(msg): general exception with msg for more details.
1966
+ Finds the optimum protocol impl preferred by the client and
1967
+ supported by the remote RSE.
1968
+
1969
+ Parameters
1970
+ ----------
1971
+ sources :
1972
+ List of sources for a given DID
1973
+
1974
+ Raises
1975
+ -------
1976
+ RucioException
1977
+ General exception with msg for more details
1708
1978
  """
1709
1979
 
1710
1980
  preferred_protocols = []