eodag 3.0.0b1__py3-none-any.whl → 3.0.0b3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. eodag/__init__.py +6 -8
  2. eodag/api/core.py +119 -171
  3. eodag/api/product/__init__.py +10 -4
  4. eodag/api/product/_assets.py +52 -14
  5. eodag/api/product/_product.py +59 -30
  6. eodag/api/product/drivers/__init__.py +7 -2
  7. eodag/api/product/drivers/base.py +0 -3
  8. eodag/api/product/metadata_mapping.py +0 -28
  9. eodag/api/search_result.py +31 -9
  10. eodag/config.py +45 -41
  11. eodag/plugins/apis/base.py +3 -3
  12. eodag/plugins/apis/ecmwf.py +2 -3
  13. eodag/plugins/apis/usgs.py +43 -14
  14. eodag/plugins/authentication/aws_auth.py +11 -2
  15. eodag/plugins/authentication/openid_connect.py +5 -4
  16. eodag/plugins/authentication/token.py +2 -1
  17. eodag/plugins/crunch/base.py +3 -1
  18. eodag/plugins/crunch/filter_date.py +3 -9
  19. eodag/plugins/crunch/filter_latest_intersect.py +0 -3
  20. eodag/plugins/crunch/filter_latest_tpl_name.py +1 -4
  21. eodag/plugins/crunch/filter_overlap.py +4 -8
  22. eodag/plugins/crunch/filter_property.py +5 -11
  23. eodag/plugins/download/aws.py +46 -78
  24. eodag/plugins/download/base.py +27 -68
  25. eodag/plugins/download/http.py +48 -57
  26. eodag/plugins/download/s3rest.py +17 -25
  27. eodag/plugins/manager.py +6 -18
  28. eodag/plugins/search/__init__.py +9 -9
  29. eodag/plugins/search/base.py +7 -26
  30. eodag/plugins/search/build_search_result.py +0 -13
  31. eodag/plugins/search/cop_marine.py +1 -3
  32. eodag/plugins/search/creodias_s3.py +0 -3
  33. eodag/plugins/search/data_request_search.py +10 -5
  34. eodag/plugins/search/qssearch.py +95 -53
  35. eodag/plugins/search/static_stac_search.py +6 -3
  36. eodag/resources/ext_product_types.json +1 -1
  37. eodag/resources/product_types.yml +24 -0
  38. eodag/resources/providers.yml +198 -154
  39. eodag/resources/user_conf_template.yml +27 -27
  40. eodag/rest/core.py +11 -43
  41. eodag/rest/server.py +1 -6
  42. eodag/rest/stac.py +13 -87
  43. eodag/rest/types/eodag_search.py +4 -7
  44. eodag/rest/types/queryables.py +4 -12
  45. eodag/rest/types/stac_search.py +7 -11
  46. eodag/rest/utils/rfc3339.py +0 -1
  47. eodag/types/__init__.py +9 -3
  48. eodag/types/download_args.py +14 -5
  49. eodag/types/search_args.py +7 -8
  50. eodag/types/whoosh.py +0 -2
  51. eodag/utils/__init__.py +20 -79
  52. eodag/utils/constraints.py +0 -8
  53. eodag/utils/import_system.py +0 -4
  54. eodag/utils/logging.py +0 -3
  55. eodag/utils/notebook.py +4 -4
  56. eodag/utils/repr.py +113 -0
  57. eodag/utils/requests.py +12 -20
  58. eodag/utils/rest.py +0 -4
  59. eodag/utils/stac_reader.py +2 -14
  60. {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/METADATA +33 -14
  61. eodag-3.0.0b3.dist-info/RECORD +110 -0
  62. {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/WHEEL +1 -1
  63. eodag-3.0.0b1.dist-info/RECORD +0 -109
  64. {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/LICENSE +0 -0
  65. {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/entry_points.txt +0 -0
  66. {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/top_level.txt +0 -0
eodag/config.py CHANGED
@@ -47,6 +47,7 @@ from pkg_resources import resource_filename
47
47
  from requests.auth import AuthBase
48
48
  from typing_extensions import Doc
49
49
 
50
+ from eodag.api.product.metadata_mapping import mtd_cfg_as_conversion_and_querypath
50
51
  from eodag.utils import (
51
52
  HTTP_REQ_TIMEOUT,
52
53
  USER_AGENT,
@@ -110,22 +111,14 @@ class ProviderConfig(yaml.YAMLObject):
110
111
  """Representation of eodag configuration.
111
112
 
112
113
  :param name: The name of the provider
113
- :type name: str
114
114
  :param priority: (optional) The priority of the provider while searching a product.
115
115
  Lower value means lower priority. (Default: 0)
116
- :type priority: int
117
116
  :param api: (optional) The configuration of a plugin of type Api
118
- :type api: :class:`~eodag.config.PluginConfig`
119
117
  :param search: (optional) The configuration of a plugin of type Search
120
- :type search: :class:`~eodag.config.PluginConfig`
121
118
  :param products: (optional) The products types supported by the provider
122
- :type products: dict
123
119
  :param download: (optional) The configuration of a plugin of type Download
124
- :type download: :class:`~eodag.config.PluginConfig`
125
120
  :param auth: (optional) The configuration of a plugin of type Authentication
126
- :type auth: :class:`~eodag.config.PluginConfig`
127
121
  :param kwargs: Additional configuration variables for this provider
128
- :type kwargs: Any
129
122
  """
130
123
 
131
124
  name: str
@@ -171,7 +164,6 @@ class ProviderConfig(yaml.YAMLObject):
171
164
  """Validate a :class:`~eodag.config.ProviderConfig`
172
165
 
173
166
  :param config_keys: The configurations keys to validate
174
- :type config_keys: dict
175
167
  """
176
168
  if "name" not in config_keys:
177
169
  raise ValidationError("Provider config must have name key")
@@ -189,7 +181,6 @@ class ProviderConfig(yaml.YAMLObject):
189
181
  """Update the configuration parameters with values from `mapping`
190
182
 
191
183
  :param mapping: The mapping from which to override configuration parameters
192
- :type mapping: dict
193
184
  """
194
185
  if mapping is None:
195
186
  mapping = {}
@@ -215,12 +206,9 @@ class PluginConfig(yaml.YAMLObject):
215
206
  """Representation of a plugin config
216
207
 
217
208
  :param name: The name of the plugin class to use to instantiate the plugin object
218
- :type name: str
219
209
  :param metadata_mapping: (optional) The mapping between eodag metadata and
220
210
  the plugin specific metadata
221
- :type metadata_mapping: dict
222
211
  :param free_params: (optional) Additional configuration parameters
223
- :type free_params: dict
224
212
  """
225
213
 
226
214
  class Pagination(TypedDict):
@@ -246,6 +234,14 @@ class PluginConfig(yaml.YAMLObject):
246
234
  sort_order_mapping: Dict[Literal["ascending", "descending"], str]
247
235
  max_sort_params: Annotated[int, Gt(0)]
248
236
 
237
+ class DiscoverMetadata(TypedDict):
238
+ """Configuration for metadata discovery"""
239
+
240
+ auto_discovery: bool
241
+ metadata_pattern: str
242
+ search_param: str
243
+ metadata_path: str
244
+
249
245
  class OrderOnResponse(TypedDict):
250
246
  """Configuration for order on-response during download"""
251
247
 
@@ -319,7 +315,7 @@ class PluginConfig(yaml.YAMLObject):
319
315
  pagination: PluginConfig.Pagination
320
316
  sort: PluginConfig.Sort
321
317
  query_params_key: str
322
- discover_metadata: Dict[str, Union[str, bool]]
318
+ discover_metadata: PluginConfig.DiscoverMetadata
323
319
  discover_product_types: Dict[str, Any]
324
320
  discover_queryables: Dict[str, Any]
325
321
  metadata_mapping: Dict[str, Union[str, List[str]]]
@@ -343,9 +339,9 @@ class PluginConfig(yaml.YAMLObject):
343
339
 
344
340
  # download -------------------------------------------------------------------------
345
341
  base_uri: str
346
- outputs_prefix: str
342
+ output_dir: str
347
343
  extract: bool
348
- outputs_extension: str
344
+ output_extension: str
349
345
  order_enabled: bool # HTTPDownload
350
346
  order_method: str # HTTPDownload
351
347
  order_headers: Dict[str, str] # HTTPDownload
@@ -419,7 +415,6 @@ class PluginConfig(yaml.YAMLObject):
419
415
  """Update the configuration parameters with values from `mapping`
420
416
 
421
417
  :param mapping: The mapping from which to override configuration parameters
422
- :type mapping: dict
423
418
  """
424
419
  if mapping is None:
425
420
  mapping = {}
@@ -435,7 +430,6 @@ def load_default_config() -> Dict[str, ProviderConfig]:
435
430
  variable if exists.
436
431
 
437
432
  :returns: The default provider's configuration
438
- :rtype: dict
439
433
  """
440
434
  eodag_providers_cfg_file = os.getenv(
441
435
  "EODAG_PROVIDERS_CFG_FILE"
@@ -447,9 +441,7 @@ def load_config(config_path: str) -> Dict[str, ProviderConfig]:
447
441
  """Load the providers configuration into a dictionnary from a given file
448
442
 
449
443
  :param config_path: The path to the provider config file
450
- :type config_path: str
451
444
  :returns: The default provider's configuration
452
- :rtype: dict
453
445
  """
454
446
  logger.debug("Loading configuration from %s", config_path)
455
447
  config: Dict[str, ProviderConfig] = {}
@@ -475,17 +467,15 @@ def provider_config_init(
475
467
  """Applies some default values to provider config
476
468
 
477
469
  :param provider_config: An eodag provider configuration
478
- :type provider_config: :class:`~eodag.config.ProviderConfig`
479
470
  :param stac_search_default_conf: default conf to overwrite with provider_config if STAC
480
- :type stac_search_default_conf: dict
481
471
  """
482
- # For the provider, set the default outputs_prefix of its download plugin
472
+ # For the provider, set the default output_dir of its download plugin
483
473
  # as tempdir in a portable way
484
474
  for param_name in ("download", "api"):
485
475
  if param_name in vars(provider_config):
486
476
  param_value = getattr(provider_config, param_name)
487
- if not getattr(param_value, "outputs_prefix", None):
488
- param_value.outputs_prefix = tempfile.gettempdir()
477
+ if not getattr(param_value, "output_dir", None):
478
+ param_value.output_dir = tempfile.gettempdir()
489
479
  if not getattr(param_value, "delete_archive", None):
490
480
  param_value.delete_archive = True
491
481
 
@@ -514,9 +504,7 @@ def override_config_from_file(config: Dict[str, Any], file_path: str) -> None:
514
504
  """Override a configuration with the values in a file
515
505
 
516
506
  :param config: An eodag providers configuration dictionary
517
- :type config: dict
518
507
  :param file_path: The path to the file from where the new values will be read
519
- :type file_path: str
520
508
  """
521
509
  logger.info("Loading user configuration from: %s", os.path.abspath(file_path))
522
510
  with open(os.path.abspath(os.path.realpath(file_path)), "r") as fh:
@@ -534,7 +522,6 @@ def override_config_from_env(config: Dict[str, Any]) -> None:
534
522
  """Override a configuration with environment variables values
535
523
 
536
524
  :param config: An eodag providers configuration dictionary
537
- :type config: dict
538
525
  """
539
526
 
540
527
  def build_mapping_from_env(
@@ -554,11 +541,8 @@ def override_config_from_env(config: Dict[str, Any]) -> None:
554
541
  }
555
542
 
556
543
  :param env_var: The environment variable to be transformed into a dictionary
557
- :type env_var: str
558
544
  :param env_value: The value from environment variable
559
- :type env_value: str
560
545
  :param mapping: The mapping in which the value will be created
561
- :type mapping: dict
562
546
  """
563
547
  parts = env_var.split("__")
564
548
  iter_parts = iter(parts)
@@ -610,11 +594,39 @@ def override_config_from_mapping(
610
594
  """Override a configuration with the values in a mapping
611
595
 
612
596
  :param config: An eodag providers configuration dictionary
613
- :type config: dict
614
597
  :param mapping: The mapping containing the values to be overriden
615
- :type mapping: dict
616
598
  """
617
599
  for provider, new_conf in mapping.items():
600
+ # check if metada-mapping as already been built as jsonpath in providers_config
601
+ if not isinstance(new_conf, dict):
602
+ continue
603
+ new_conf_search = new_conf.get("search", {}) or {}
604
+ new_conf_api = new_conf.get("api", {}) or {}
605
+ if provider in config and "metadata_mapping" in {
606
+ **new_conf_search,
607
+ **new_conf_api,
608
+ }:
609
+ search_plugin_key = (
610
+ "search" if "metadata_mapping" in new_conf_search else "api"
611
+ )
612
+ # get some already configured value
613
+ configured_metadata_mapping = getattr(
614
+ config[provider], search_plugin_key
615
+ ).metadata_mapping
616
+ some_configured_value = next(iter(configured_metadata_mapping.values()))
617
+ # check if the configured value has already been built as jsonpath
618
+ if (
619
+ isinstance(some_configured_value, list)
620
+ and isinstance(some_configured_value[1], tuple)
621
+ or isinstance(some_configured_value, tuple)
622
+ ):
623
+ # also build as jsonpath the incoming conf
624
+ mtd_cfg_as_conversion_and_querypath(
625
+ deepcopy(mapping[provider][search_plugin_key]["metadata_mapping"]),
626
+ mapping[provider][search_plugin_key]["metadata_mapping"],
627
+ )
628
+
629
+ # try overriding conf
618
630
  old_conf: Optional[Dict[str, Any]] = config.get(provider)
619
631
  if old_conf is not None:
620
632
  old_conf.update(new_conf)
@@ -640,9 +652,7 @@ def merge_configs(config: Dict[str, Any], other_config: Dict[str, Any]) -> None:
640
652
  """Override a configuration with the values of another configuration
641
653
 
642
654
  :param config: An eodag providers configuration dictionary
643
- :type config: dict
644
655
  :param other_config: An eodag providers configuration dictionary
645
- :type other_config: dict
646
656
  """
647
657
  # configs union with other_config values as default
648
658
  other_config = dict(config, **other_config)
@@ -674,7 +684,6 @@ def load_yml_config(yml_path: str) -> Dict[Any, Any]:
674
684
  """Load a conf dictionnary from given yml absolute path
675
685
 
676
686
  :returns: The yml configuration file
677
- :rtype: dict
678
687
  """
679
688
  config = SimpleYamlProxyConfig(yml_path)
680
689
  return dict_items_recursive_apply(config.source, string_to_jsonpath)
@@ -684,7 +693,6 @@ def load_stac_config() -> Dict[str, Any]:
684
693
  """Load the stac configuration into a dictionnary
685
694
 
686
695
  :returns: The stac configuration
687
- :rtype: dict
688
696
  """
689
697
  return load_yml_config(
690
698
  resource_filename("eodag", os.path.join("resources/", "stac.yml"))
@@ -695,7 +703,6 @@ def load_stac_api_config() -> Dict[str, Any]:
695
703
  """Load the stac API configuration into a dictionnary
696
704
 
697
705
  :returns: The stac API configuration
698
- :rtype: dict
699
706
  """
700
707
  return load_yml_config(
701
708
  resource_filename("eodag", os.path.join("resources/", "stac_api.yml"))
@@ -706,7 +713,6 @@ def load_stac_provider_config() -> Dict[str, Any]:
706
713
  """Load the stac provider configuration into a dictionnary
707
714
 
708
715
  :returns: The stac provider configuration
709
- :rtype: dict
710
716
  """
711
717
  return SimpleYamlProxyConfig(
712
718
  resource_filename("eodag", os.path.join("resources/", "stac_provider.yml"))
@@ -719,9 +725,7 @@ def get_ext_product_types_conf(
719
725
  """Read external product types conf
720
726
 
721
727
  :param conf_uri: URI to local or remote configuration file
722
- :type conf_uri: str
723
728
  :returns: The external product types configuration
724
- :rtype: dict
725
729
  """
726
730
  logger.info("Fetching external product types from %s", conf_uri)
727
731
  if conf_uri.lower().startswith("http"):
@@ -33,17 +33,17 @@ class Api(Search, Download):
33
33
 
34
34
  The download methods must:
35
35
 
36
- - download data in the ``outputs_prefix`` folder defined in the plugin's
36
+ - download data in the ``output_dir`` folder defined in the plugin's
37
37
  configuration or passed through kwargs
38
38
  - extract products from their archive (if relevant) if ``extract`` is set to True
39
39
  (True by default)
40
- - save a product in an archive/directory (in ``outputs_prefix``) whose name must be
40
+ - save a product in an archive/directory (in ``output_dir``) whose name must be
41
41
  the product's ``title`` property
42
42
  - update the product's ``location`` attribute once its data is downloaded (and
43
43
  eventually after it's extracted) to the product's location given as a file URI
44
44
  (e.g. 'file:///tmp/product_folder' on Linux or
45
45
  'file:///C:/Users/username/AppData/LOcal/Temp' on Windows)
46
- - save a *record* file in the directory ``outputs_prefix/.downloaded`` whose name
46
+ - save a *record* file in the directory ``output_dir/.downloaded`` whose name
47
47
  is built on the MD5 hash of the product's ``product_type`` and ``properties['id']``
48
48
  attributes (``hashlib.md5((product.product_type+"-"+product.properties['id']).encode("utf-8")).hexdigest()``)
49
49
  and whose content is the product's ``remote_location`` attribute itself.
@@ -128,7 +128,6 @@ class EcmwfApi(Api, BuildPostSearchResult):
128
128
  """Check credentials and returns information needed for auth
129
129
 
130
130
  :returns: {key, url, email} dictionary
131
- :rtype: dict
132
131
  :raises: :class:`~eodag.utils.exceptions.AuthenticationError`
133
132
  """
134
133
  # Get credentials from eodag or using ecmwf conf
@@ -165,8 +164,8 @@ class EcmwfApi(Api, BuildPostSearchResult):
165
164
  """Download data from ECMWF MARS"""
166
165
  product_format = product.properties.get("format", "grib")
167
166
  product_extension = ECMWF_MARS_KNOWN_FORMATS.get(product_format, product_format)
168
- kwargs["outputs_extension"] = kwargs.get(
169
- "outputs_extension", f".{product_extension}"
167
+ kwargs["output_extension"] = kwargs.get(
168
+ "output_extension", f".{product_extension}"
170
169
  )
171
170
 
172
171
  # Prepare download
@@ -128,13 +128,14 @@ class UsgsApi(Api):
128
128
  raise NoMatchingProductType(
129
129
  "Cannot search on USGS without productType specified"
130
130
  )
131
- if kwargs.get("sortBy"):
131
+ if kwargs.get("sort_by"):
132
132
  raise ValidationError("USGS does not support sorting feature")
133
133
 
134
134
  self.authenticate()
135
135
 
136
136
  product_type_def_params = self.config.products.get( # type: ignore
137
- product_type, self.config.products[GENERIC_PRODUCT_TYPE] # type: ignore
137
+ product_type,
138
+ self.config.products[GENERIC_PRODUCT_TYPE], # type: ignore
138
139
  )
139
140
  usgs_dataset = format_dict_items(product_type_def_params, **kwargs)["dataset"]
140
141
  start_date = kwargs.pop("startTimeFromAscendingNode", None)
@@ -172,11 +173,39 @@ class UsgsApi(Api):
172
173
  max_results=items_per_page,
173
174
  starting_number=(1 + (page - 1) * items_per_page),
174
175
  )
175
- logger.info(
176
- f"Sending search request for {usgs_dataset} with {api_search_kwargs}"
177
- )
178
176
 
179
- results = api.scene_search(usgs_dataset, **api_search_kwargs)
177
+ # search by id
178
+ if searched_id := kwargs.get("id"):
179
+ dataset_filters = api.dataset_filters(usgs_dataset)
180
+ # ip pattern set as parameter queryable (first element of param conf list)
181
+ id_pattern = self.config.metadata_mapping["id"][0]
182
+ # loop on matching dataset_filters until one returns expected results
183
+ for dataset_filter in dataset_filters["data"]:
184
+ if id_pattern in dataset_filter["searchSql"]:
185
+ logger.debug(
186
+ f"Try using {dataset_filter['searchSql']} dataset filter to search by id on {usgs_dataset}"
187
+ )
188
+ full_api_search_kwargs = {
189
+ "where": {
190
+ "filter_id": dataset_filter["id"],
191
+ "value": searched_id,
192
+ },
193
+ **api_search_kwargs,
194
+ }
195
+ logger.info(
196
+ f"Sending search request for {usgs_dataset} with {full_api_search_kwargs}"
197
+ )
198
+ results = api.scene_search(
199
+ usgs_dataset, **full_api_search_kwargs
200
+ )
201
+ if len(results["data"]["results"]) == 1:
202
+ # search by id using this dataset_filter succeeded
203
+ break
204
+ else:
205
+ logger.info(
206
+ f"Sending search request for {usgs_dataset} with {api_search_kwargs}"
207
+ )
208
+ results = api.scene_search(usgs_dataset, **api_search_kwargs)
180
209
 
181
210
  # update results with storage info from download_options()
182
211
  results_by_entity_id = {
@@ -257,13 +286,13 @@ class UsgsApi(Api):
257
286
  )
258
287
  progress_callback = ProgressCallback(disable=True)
259
288
 
260
- outputs_extension = cast(
289
+ output_extension = cast(
261
290
  str,
262
291
  self.config.products.get( # type: ignore
263
292
  product.product_type, self.config.products[GENERIC_PRODUCT_TYPE] # type: ignore
264
- ).get("outputs_extension", ".tar.gz"),
293
+ ).get("output_extension", ".tar.gz"),
265
294
  )
266
- kwargs["outputs_extension"] = kwargs.get("outputs_extension", outputs_extension)
295
+ kwargs["output_extension"] = kwargs.get("output_extension", output_extension)
267
296
 
268
297
  fs_path, record_filename = self._prepare_download(
269
298
  product,
@@ -375,8 +404,8 @@ class UsgsApi(Api):
375
404
 
376
405
  # Check downloaded file format
377
406
  if (
378
- kwargs["outputs_extension"] == ".tar.gz" and tarfile.is_tarfile(fs_path)
379
- ) or (kwargs["outputs_extension"] == ".zip" and zipfile.is_zipfile(fs_path)):
407
+ kwargs["output_extension"] == ".tar.gz" and tarfile.is_tarfile(fs_path)
408
+ ) or (kwargs["output_extension"] == ".zip" and zipfile.is_zipfile(fs_path)):
380
409
  product_path = self._finalize(
381
410
  fs_path,
382
411
  progress_callback=progress_callback,
@@ -388,7 +417,7 @@ class UsgsApi(Api):
388
417
  logger.info(
389
418
  "Downloaded product detected as a tar File, but was was expected to be a zip file"
390
419
  )
391
- new_fs_path = fs_path[: fs_path.index(outputs_extension)] + ".tar.gz"
420
+ new_fs_path = fs_path[: fs_path.index(output_extension)] + ".tar.gz"
392
421
  shutil.move(fs_path, new_fs_path)
393
422
  product.location = path_to_uri(new_fs_path)
394
423
  return new_fs_path
@@ -396,7 +425,7 @@ class UsgsApi(Api):
396
425
  logger.info(
397
426
  "Downloaded product detected as a zip File, but was was expected to be a tar file"
398
427
  )
399
- new_fs_path = fs_path[: fs_path.index(outputs_extension)] + ".zip"
428
+ new_fs_path = fs_path[: fs_path.index(output_extension)] + ".zip"
400
429
  shutil.move(fs_path, new_fs_path)
401
430
  product.location = path_to_uri(new_fs_path)
402
431
  return new_fs_path
@@ -404,7 +433,7 @@ class UsgsApi(Api):
404
433
  logger.warning(
405
434
  "Downloaded product is not a tar or a zip File. Please check its file type before using it"
406
435
  )
407
- new_fs_path = fs_path[: fs_path.index(outputs_extension)]
436
+ new_fs_path = fs_path[: fs_path.index(output_extension)]
408
437
  shutil.move(fs_path, new_fs_path)
409
438
  product.location = path_to_uri(new_fs_path)
410
439
  return new_fs_path
@@ -35,6 +35,7 @@ class AwsAuth(Authentication):
35
35
  - auth anonymously using no-sign-request
36
36
  - auth using ``aws_profile``
37
37
  - auth using ``aws_access_key_id`` and ``aws_secret_access_key``
38
+ (optionally ``aws_session_token``)
38
39
  - auth using current environment (AWS environment variables and/or ``~/aws/*``),
39
40
  will be skipped if AWS credentials are filled in eodag conf
40
41
  """
@@ -45,13 +46,13 @@ class AwsAuth(Authentication):
45
46
  super(AwsAuth, self).__init__(provider, config)
46
47
  self.aws_access_key_id = None
47
48
  self.aws_secret_access_key = None
49
+ self.aws_session_token = None
48
50
  self.profile_name = None
49
51
 
50
52
  def authenticate(self) -> Dict[str, str]:
51
53
  """Authenticate
52
54
 
53
55
  :returns: dict containing AWS/boto3 non-empty credentials
54
- :rtype: dict
55
56
  """
56
57
  credentials = getattr(self.config, "credentials", {}) or {}
57
58
  self.aws_access_key_id = credentials.get(
@@ -60,7 +61,15 @@ class AwsAuth(Authentication):
60
61
  self.aws_secret_access_key = credentials.get(
61
62
  "aws_secret_access_key", self.aws_secret_access_key
62
63
  )
64
+ self.aws_session_token = credentials.get(
65
+ "aws_session_token", self.aws_session_token
66
+ )
63
67
  self.profile_name = credentials.get("aws_profile", self.profile_name)
64
68
 
65
- auth_keys = ["aws_access_key_id", "aws_secret_access_key", "profile_name"]
69
+ auth_keys = [
70
+ "aws_access_key_id",
71
+ "aws_secret_access_key",
72
+ "aws_session_token",
73
+ "profile_name",
74
+ ]
66
75
  return {k: getattr(self, k) for k in auth_keys if getattr(self, k)}
@@ -482,7 +482,7 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
482
482
  if not match:
483
483
  return value
484
484
  value_from_xpath = form_element.xpath(
485
- self.CONFIG_XPATH_REGEX.match(value).groupdict("xpath_value")["xpath_value"]
485
+ match.groupdict("xpath_value")["xpath_value"]
486
486
  )
487
487
  if len(value_from_xpath) == 1:
488
488
  return value_from_xpath[0]
@@ -512,10 +512,11 @@ class CodeAuthorizedAuth(AuthBase):
512
512
  def __call__(self, request: PreparedRequest) -> PreparedRequest:
513
513
  """Perform the actual authentication"""
514
514
  if self.where == "qs":
515
- parts = urlparse(request.url)
515
+ parts = urlparse(str(request.url))
516
516
  query_dict = parse_qs(parts.query)
517
- query_dict.update({self.key: self.token})
518
- url_without_args = parts._replace(query=None).geturl()
517
+ if self.key is not None:
518
+ query_dict.update({self.key: [self.token]})
519
+ url_without_args = parts._replace(query="").geturl()
519
520
 
520
521
  request.prepare_url(url_without_args, query_dict)
521
522
 
@@ -197,7 +197,8 @@ class RequestsTokenAuth(AuthBase):
197
197
  if self.where == "qs":
198
198
  parts = urlparse(str(request.url))
199
199
  qs = parse_qs(parts.query)
200
- qs[self.qs_key] = self.token # type: ignore
200
+ if self.qs_key is not None:
201
+ qs[self.qs_key] = [self.token]
201
202
  request.url = urlunparse(
202
203
  (
203
204
  parts.scheme,
@@ -19,6 +19,7 @@ from __future__ import annotations
19
19
 
20
20
  from typing import TYPE_CHECKING, Any, Dict, List, Optional
21
21
 
22
+ from eodag.config import PluginConfig
22
23
  from eodag.plugins.base import PluginTopic
23
24
 
24
25
  if TYPE_CHECKING:
@@ -29,7 +30,8 @@ class Crunch(PluginTopic):
29
30
  """Base cruncher"""
30
31
 
31
32
  def __init__(self, config: Optional[Dict[str, Any]]) -> None:
32
- self.config = config if config is not None else {}
33
+ self.config = PluginConfig()
34
+ self.config.__dict__ = config if config is not None else {}
33
35
 
34
36
  def proceed(
35
37
  self, products: List[EOProduct], **search_params: Any
@@ -21,7 +21,7 @@ import datetime
21
21
  import logging
22
22
  import time
23
23
  from datetime import datetime as dt
24
- from typing import TYPE_CHECKING, Any, Dict, List
24
+ from typing import TYPE_CHECKING, Any, List
25
25
 
26
26
  import dateutil.parser
27
27
  from dateutil import tz
@@ -41,12 +41,8 @@ class FilterDate(Crunch):
41
41
 
42
42
  - `start`: (optional) start sensing time in iso format
43
43
  - `end`: (optional) end sensing time in iso format
44
-
45
- :type config: dict
46
44
  """
47
45
 
48
- config: Dict[str, str]
49
-
50
46
  @staticmethod
51
47
  def sort_product_by_start_date(product: EOProduct) -> dt:
52
48
  """Get product start date"""
@@ -63,16 +59,14 @@ class FilterDate(Crunch):
63
59
  """Execute crunch: Filter products between start and end dates.
64
60
 
65
61
  :param products: A list of products resulting from a search
66
- :type products: list(:class:`~eodag.api.product._product.EOProduct`)
67
62
  :returns: The filtered products
68
- :rtype: list(:class:`~eodag.api.product._product.EOProduct`)
69
63
  """
70
64
  logger.debug("Start filtering by date")
71
65
  if not products:
72
66
  return []
73
67
 
74
68
  # filter start date
75
- filter_start_str = self.config.get("start", None)
69
+ filter_start_str = self.config.__dict__.get("start", None)
76
70
  if filter_start_str:
77
71
  filter_start = dateutil.parser.parse(filter_start_str)
78
72
  if not filter_start.tzinfo:
@@ -81,7 +75,7 @@ class FilterDate(Crunch):
81
75
  filter_start = None
82
76
 
83
77
  # filter end date
84
- filter_end_str = self.config.get("end", None)
78
+ filter_end_str = self.config.__dict__.get("end", None)
85
79
  if filter_end_str:
86
80
  filter_end = dateutil.parser.parse(filter_end_str)
87
81
  if not filter_end.tzinfo:
@@ -59,12 +59,9 @@ class FilterLatestIntersect(Crunch):
59
59
  Filter latest products (the ones with a the highest start date) that intersect search extent.
60
60
 
61
61
  :param products: A list of products resulting from a search
62
- :type products: list(:class:`~eodag.api.product._product.EOProduct`)
63
62
  :param search_params: Search criteria that must contain `geometry` (dict)
64
63
  or search `geom` (:class:`shapely.geometry.base.BaseGeometry`) argument will be used
65
- :type search_params: dict
66
64
  :returns: The filtered products
67
- :rtype: list(:class:`~eodag.api.product._product.EOProduct`)
68
65
  """
69
66
  logger.debug("Start filtering for latest products")
70
67
  if not products:
@@ -26,6 +26,7 @@ from eodag.utils.exceptions import ValidationError
26
26
 
27
27
  if TYPE_CHECKING:
28
28
  from eodag.api.product import EOProduct
29
+
29
30
  logger = logging.getLogger("eodag.crunch.latest_tpl_name")
30
31
 
31
32
 
@@ -37,8 +38,6 @@ class FilterLatestByName(Crunch):
37
38
  :param config: Crunch configuration, must contain :
38
39
 
39
40
  - `name_pattern` : product name pattern
40
-
41
- :type config: dict
42
41
  """
43
42
 
44
43
  NAME_PATTERN_CONSTRAINT = re.compile(r"\(\?P<tileid>\\d\{6\}\)")
@@ -60,9 +59,7 @@ class FilterLatestByName(Crunch):
60
59
  """Execute crunch: Filter Search results to get only the latest product, based on the name of the product
61
60
 
62
61
  :param products: A list of products resulting from a search
63
- :type products: list(:class:`~eodag.api.product._product.EOProduct`)
64
62
  :returns: The filtered products
65
- :rtype: list(:class:`~eodag.api.product._product.EOProduct`)
66
63
  """
67
64
  logger.debug("Starting products filtering")
68
65
  processed: List[str] = []
@@ -48,7 +48,6 @@ class FilterOverlap(Crunch):
48
48
  - `within` : True if product geometry is within the search area
49
49
 
50
50
  These configuration parameters are mutually exclusive.
51
- :type config: dict
52
51
  """
53
52
 
54
53
  def proceed(
@@ -57,11 +56,8 @@ class FilterOverlap(Crunch):
57
56
  """Execute crunch: Filter products, retaining only those that are overlapping with the search_extent
58
57
 
59
58
  :param products: A list of products resulting from a search
60
- :type products: list(:class:`~eodag.api.product._product.EOProduct`)
61
59
  :param search_params: Search criteria that must contain `geometry`
62
- :type search_params: dict
63
60
  :returns: The filtered products
64
- :rtype: list(:class:`~eodag.api.product._product.EOProduct`)
65
61
  """
66
62
  logger.debug("Start filtering for overlapping products")
67
63
  filtered: List[EOProduct] = []
@@ -73,10 +69,10 @@ class FilterOverlap(Crunch):
73
69
  "geometry not found in cruncher arguments, filtering disabled."
74
70
  )
75
71
  return products
76
- minimum_overlap = float(self.config.get("minimum_overlap", "0"))
77
- contains = self.config.get("contains", False)
78
- intersects = self.config.get("intersects", False)
79
- within = self.config.get("within", False)
72
+ minimum_overlap = float(self.config.__dict__.get("minimum_overlap", "0"))
73
+ contains = self.config.__dict__.get("contains", False)
74
+ intersects = self.config.__dict__.get("intersects", False)
75
+ within = self.config.__dict__.get("within", False)
80
76
 
81
77
  if contains and (within or intersects) or (within and intersects):
82
78
  logger.warning(