eodag 3.1.0b1__py3-none-any.whl → 3.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. eodag/api/core.py +69 -63
  2. eodag/api/product/_assets.py +49 -13
  3. eodag/api/product/_product.py +41 -30
  4. eodag/api/product/drivers/__init__.py +81 -4
  5. eodag/api/product/drivers/base.py +65 -4
  6. eodag/api/product/drivers/generic.py +65 -0
  7. eodag/api/product/drivers/sentinel1.py +97 -0
  8. eodag/api/product/drivers/sentinel2.py +95 -0
  9. eodag/api/product/metadata_mapping.py +85 -79
  10. eodag/api/search_result.py +13 -23
  11. eodag/cli.py +4 -4
  12. eodag/config.py +77 -80
  13. eodag/plugins/apis/base.py +1 -1
  14. eodag/plugins/apis/ecmwf.py +12 -15
  15. eodag/plugins/apis/usgs.py +12 -11
  16. eodag/plugins/authentication/aws_auth.py +16 -13
  17. eodag/plugins/authentication/base.py +5 -3
  18. eodag/plugins/authentication/header.py +3 -3
  19. eodag/plugins/authentication/keycloak.py +4 -4
  20. eodag/plugins/authentication/oauth.py +7 -3
  21. eodag/plugins/authentication/openid_connect.py +20 -14
  22. eodag/plugins/authentication/sas_auth.py +4 -4
  23. eodag/plugins/authentication/token.py +7 -7
  24. eodag/plugins/authentication/token_exchange.py +1 -1
  25. eodag/plugins/base.py +4 -4
  26. eodag/plugins/crunch/base.py +4 -4
  27. eodag/plugins/crunch/filter_date.py +4 -4
  28. eodag/plugins/crunch/filter_latest_intersect.py +6 -6
  29. eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
  30. eodag/plugins/crunch/filter_overlap.py +4 -4
  31. eodag/plugins/crunch/filter_property.py +4 -4
  32. eodag/plugins/download/aws.py +137 -77
  33. eodag/plugins/download/base.py +8 -17
  34. eodag/plugins/download/creodias_s3.py +2 -2
  35. eodag/plugins/download/http.py +30 -32
  36. eodag/plugins/download/s3rest.py +5 -4
  37. eodag/plugins/manager.py +10 -20
  38. eodag/plugins/search/__init__.py +6 -5
  39. eodag/plugins/search/base.py +38 -42
  40. eodag/plugins/search/build_search_result.py +286 -336
  41. eodag/plugins/search/cop_marine.py +22 -12
  42. eodag/plugins/search/creodias_s3.py +8 -78
  43. eodag/plugins/search/csw.py +11 -11
  44. eodag/plugins/search/data_request_search.py +19 -18
  45. eodag/plugins/search/qssearch.py +84 -151
  46. eodag/plugins/search/stac_list_assets.py +85 -0
  47. eodag/plugins/search/static_stac_search.py +4 -4
  48. eodag/resources/ext_product_types.json +1 -1
  49. eodag/resources/product_types.yml +848 -398
  50. eodag/resources/providers.yml +1038 -1115
  51. eodag/resources/stac_api.yml +2 -2
  52. eodag/resources/user_conf_template.yml +10 -9
  53. eodag/rest/cache.py +2 -2
  54. eodag/rest/config.py +3 -3
  55. eodag/rest/core.py +24 -24
  56. eodag/rest/errors.py +5 -5
  57. eodag/rest/server.py +3 -11
  58. eodag/rest/stac.py +41 -38
  59. eodag/rest/types/collections_search.py +3 -3
  60. eodag/rest/types/eodag_search.py +23 -23
  61. eodag/rest/types/queryables.py +40 -28
  62. eodag/rest/types/stac_search.py +15 -25
  63. eodag/rest/utils/__init__.py +11 -21
  64. eodag/rest/utils/cql_evaluate.py +6 -6
  65. eodag/rest/utils/rfc3339.py +2 -2
  66. eodag/types/__init__.py +97 -29
  67. eodag/types/bbox.py +2 -2
  68. eodag/types/download_args.py +2 -2
  69. eodag/types/queryables.py +5 -2
  70. eodag/types/search_args.py +4 -4
  71. eodag/types/whoosh.py +1 -3
  72. eodag/utils/__init__.py +82 -41
  73. eodag/utils/exceptions.py +2 -2
  74. eodag/utils/import_system.py +2 -2
  75. eodag/utils/requests.py +2 -2
  76. eodag/utils/rest.py +2 -2
  77. eodag/utils/s3.py +231 -0
  78. eodag/utils/stac_reader.py +10 -10
  79. {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/METADATA +12 -10
  80. eodag-3.2.0.dist-info/RECORD +113 -0
  81. {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/WHEEL +1 -1
  82. {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/entry_points.txt +1 -0
  83. eodag-3.1.0b1.dist-info/RECORD +0 -108
  84. {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info/licenses}/LICENSE +0 -0
  85. {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/top_level.txt +0 -0
@@ -19,19 +19,15 @@ from __future__ import annotations
19
19
 
20
20
  import logging
21
21
  import re
22
+ import socket
22
23
  from copy import copy as copy_copy
23
- from datetime import datetime, timedelta
24
24
  from typing import (
25
25
  TYPE_CHECKING,
26
26
  Annotated,
27
27
  Any,
28
28
  Callable,
29
- Dict,
30
- List,
31
29
  Optional,
32
30
  Sequence,
33
- Set,
34
- Tuple,
35
31
  TypedDict,
36
32
  cast,
37
33
  get_args,
@@ -52,7 +48,6 @@ import geojson
52
48
  import orjson
53
49
  import requests
54
50
  import yaml
55
- from dateutil.utils import today
56
51
  from jsonpath_ng import JSONPath
57
52
  from lxml import etree
58
53
  from pydantic import create_model
@@ -77,7 +72,7 @@ from eodag.plugins.search.base import Search
77
72
  from eodag.types import json_field_definition_to_python, model_fields_to_annotated
78
73
  from eodag.types.search_args import SortByList
79
74
  from eodag.utils import (
80
- DEFAULT_MISSION_START_DATE,
75
+ DEFAULT_SEARCH_TIMEOUT,
81
76
  GENERIC_PRODUCT_TYPE,
82
77
  HTTP_REQ_TIMEOUT,
83
78
  REQ_RETRY_BACKOFF_FACTOR,
@@ -128,7 +123,9 @@ class QueryStringSearch(Search):
128
123
  authentication error; only used if ``need_auth=true``
129
124
  * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in
130
125
  requests; default: ``True``
131
- * :attr:`~eodag.config.PluginConfig.dont_quote` (``List[str]``): characters that should not be quoted in the
126
+ * :attr:`~eodag.config.PluginConfig.asset_key_from_href` (``bool``): guess assets keys using their ``href``. Use
127
+ their original key if ``False``; default: ``True``
128
+ * :attr:`~eodag.config.PluginConfig.dont_quote` (``list[str]``): characters that should not be quoted in the
132
129
  url params
133
130
  * :attr:`~eodag.config.PluginConfig.timeout` (``int``): time to wait until request timeout in seconds;
134
131
  default: ``5``
@@ -136,10 +133,10 @@ class QueryStringSearch(Search):
136
133
  total number of retries to allow; default: ``3``
137
134
  * :attr:`~eodag.config.PluginConfig.retry_backoff_factor` (``int``): :class:`urllib3.util.Retry`
138
135
  ``backoff_factor`` parameter, backoff factor to apply between attempts after the second try; default: ``2``
139
- * :attr:`~eodag.config.PluginConfig.retry_status_forcelist` (``List[int]``): :class:`urllib3.util.Retry`
136
+ * :attr:`~eodag.config.PluginConfig.retry_status_forcelist` (``list[int]``): :class:`urllib3.util.Retry`
140
137
  ``status_forcelist`` parameter, list of integer HTTP status codes that we should force a retry on; default:
141
138
  ``[401, 429, 500, 502, 503, 504]``
142
- * :attr:`~eodag.config.PluginConfig.literal_search_params` (``Dict[str, str]``): A mapping of (search_param =>
139
+ * :attr:`~eodag.config.PluginConfig.literal_search_params` (``dict[str, str]``): A mapping of (search_param =>
143
140
  search_value) pairs giving search parameters to be passed as is in the search url query string. This is useful
144
141
  for example in situations where the user wants to add a fixed search query parameter exactly
145
142
  as it is done on the provider interface.
@@ -183,13 +180,13 @@ class QueryStringSearch(Search):
183
180
  * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_id` (``str``): mapping for the
184
181
  product type id
185
182
  * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_metadata`
186
- (``Dict[str, str]``): mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed
183
+ (``dict[str, str]``): mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed
187
184
  from the provider result
188
185
  * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_properties`
189
- (``Dict[str, str]``): mapping for product type properties which can be parsed from the result and are not
186
+ (``dict[str, str]``): mapping for product type properties which can be parsed from the result and are not
190
187
  product type metadata
191
188
  * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_unparsable_properties`
192
- (``Dict[str, str]``): mapping for product type properties which cannot be parsed from the result and are not
189
+ (``dict[str, str]``): mapping for product type properties which cannot be parsed from the result and are not
193
190
  product type metadata
194
191
  * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url` (``str``): url to fetch
195
192
  data for a single collection; used if product type metadata is not available from the endpoint given in
@@ -198,13 +195,13 @@ class QueryStringSearch(Search):
198
195
  to be added to the :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` to filter for a
199
196
  collection
200
197
  * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_product_type_parsable_metadata`
201
- (``Dict[str, str]``): mapping for product type metadata returned by the endpoint given in
198
+ (``dict[str, str]``): mapping for product type metadata returned by the endpoint given in
202
199
  :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url`.
203
200
 
204
201
  * :attr:`~eodag.config.PluginConfig.sort` (:class:`~eodag.config.PluginConfig.Sort`): configuration for sorting
205
202
  the results. It contains the keys:
206
203
 
207
- * :attr:`~eodag.config.PluginConfig.Sort.sort_by_default` (``List[Tuple(str, Literal["ASC", "DESC"])]``):
204
+ * :attr:`~eodag.config.PluginConfig.Sort.sort_by_default` (``list[Tuple(str, Literal["ASC", "DESC"])]``):
208
205
  parameter and sort order by which the result will be sorted by default (if the user does not enter a
209
206
  ``sort_by`` parameter); if not given the result will use the default sorting of the provider; Attention:
210
207
  for some providers sorting might cause a timeout if no filters are used. In that case no default
@@ -220,12 +217,12 @@ class QueryStringSearch(Search):
220
217
  * :attr:`~eodag.config.PluginConfig.Sort.sort_param_mapping` (``Dict [str, str]``): mapping for the parameters
221
218
  available for sorting
222
219
  * :attr:`~eodag.config.PluginConfig.Sort.sort_order_mapping`
223
- (``Dict[Literal["ascending", "descending"], str]``): mapping for the sort order
220
+ (``dict[Literal["ascending", "descending"], str]``): mapping for the sort order
224
221
  * :attr:`~eodag.config.PluginConfig.Sort.max_sort_params` (``int``): maximum number of sort parameters
225
222
  supported by the provider; used to validate the user input to avoid failed requests or unexpected behaviour
226
223
  (not all parameters are used in the request)
227
224
 
228
- * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Any]``): The search plugins of this kind can
225
+ * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Any]``): The search plugins of this kind can
229
226
  detect when a metadata mapping is "query-able", and get the semantics of how to format the query string
230
227
  parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
231
228
  just configure it in the metadata mapping to be a list of 2 items, the first one being the
@@ -258,7 +255,7 @@ class QueryStringSearch(Search):
258
255
  metadata is activated; default: ``False``; if false, the other parameters are not used;
259
256
  * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.metadata_pattern` (``str``): regex string a parameter in
260
257
  the result should match so that is used
261
- * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.search_param` (``Union [str, Dict[str, Any]]``): format
258
+ * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.search_param` (``Union [str, dict[str, Any]]``): format
262
259
  to add a query param given by the user and not in the metadata mapping to the requests, 'metadata' will be
263
260
  replaced by the search param; can be a string or a dict containing
264
261
  :attr:`~eodag.config.PluginConfig.free_text_search_operations`
@@ -286,7 +283,7 @@ class QueryStringSearch(Search):
286
283
  the result is an array of constraints
287
284
  """
288
285
 
289
- extract_properties: Dict[str, Callable[..., Dict[str, Any]]] = {
286
+ extract_properties: dict[str, Callable[..., dict[str, Any]]] = {
290
287
  "xml": properties_from_xml,
291
288
  "json": properties_from_json,
292
289
  }
@@ -297,8 +294,8 @@ class QueryStringSearch(Search):
297
294
  self.config.__dict__.setdefault("results_entry", "features")
298
295
  self.config.__dict__.setdefault("pagination", {})
299
296
  self.config.__dict__.setdefault("free_text_search_operations", {})
300
- self.search_urls: List[str] = []
301
- self.query_params: Dict[str, str] = dict()
297
+ self.search_urls: list[str] = []
298
+ self.query_params: dict[str, str] = dict()
302
299
  self.query_string = ""
303
300
  self.next_page_url = None
304
301
  self.next_page_query_obj = None
@@ -443,7 +440,7 @@ class QueryStringSearch(Search):
443
440
  self.next_page_query_obj = None
444
441
  self.next_page_merge = None
445
442
 
446
- def discover_product_types(self, **kwargs: Any) -> Optional[Dict[str, Any]]:
443
+ def discover_product_types(self, **kwargs: Any) -> Optional[dict[str, Any]]:
447
444
  """Fetch product types list from provider using `discover_product_types` conf
448
445
 
449
446
  :returns: configuration dict containing fetched product types information
@@ -460,7 +457,7 @@ class QueryStringSearch(Search):
460
457
  # no pagination
461
458
  return self.discover_product_types_per_page(**kwargs)
462
459
 
463
- conf_update_dict: Dict[str, Any] = {
460
+ conf_update_dict: dict[str, Any] = {
464
461
  "providers_config": {},
465
462
  "product_types_config": {},
466
463
  }
@@ -493,7 +490,7 @@ class QueryStringSearch(Search):
493
490
 
494
491
  def discover_product_types_per_page(
495
492
  self, **kwargs: Any
496
- ) -> Optional[Dict[str, Any]]:
493
+ ) -> Optional[dict[str, Any]]:
497
494
  """Fetch product types list from provider using `discover_product_types` conf
498
495
  using paginated ``kwargs["fetch_url"]``
499
496
 
@@ -536,7 +533,7 @@ class QueryStringSearch(Search):
536
533
 
537
534
  prep.info_message = "Fetching product types: {}".format(prep.url)
538
535
  prep.exception_message = (
539
- "Skipping error while fetching product types for " "{} {} instance:"
536
+ "Skipping error while fetching product types for {} {} instance:"
540
537
  ).format(self.provider, self.__class__.__name__)
541
538
 
542
539
  # Query using appropriate method
@@ -551,7 +548,7 @@ class QueryStringSearch(Search):
551
548
  return None
552
549
  else:
553
550
  try:
554
- conf_update_dict: Dict[str, Any] = {
551
+ conf_update_dict: dict[str, Any] = {
555
552
  "providers_config": {},
556
553
  "product_types_config": {},
557
554
  }
@@ -570,7 +567,7 @@ class QueryStringSearch(Search):
570
567
  result = result[0]
571
568
 
572
569
  def conf_update_from_product_type_result(
573
- product_type_result: Dict[str, Any]
570
+ product_type_result: dict[str, Any],
574
571
  ) -> None:
575
572
  """Update ``conf_update_dict`` using given product type json response"""
576
573
  # providers_config extraction
@@ -698,7 +695,7 @@ class QueryStringSearch(Search):
698
695
 
699
696
  def _get_product_type_metadata_from_single_collection_endpoint(
700
697
  self, product_type: str
701
- ) -> Dict[str, Any]:
698
+ ) -> dict[str, Any]:
702
699
  """
703
700
  retrieves additional product type information from an endpoint returning data for a single collection
704
701
  :param product_type: product type
@@ -726,7 +723,7 @@ class QueryStringSearch(Search):
726
723
  self,
727
724
  prep: PreparedSearch = PreparedSearch(),
728
725
  **kwargs: Any,
729
- ) -> Tuple[List[EOProduct], Optional[int]]:
726
+ ) -> tuple[list[EOProduct], Optional[int]]:
730
727
  """Perform a search on an OpenSearch-like interface
731
728
 
732
729
  :param prep: Object collecting needed information for search.
@@ -754,7 +751,7 @@ class QueryStringSearch(Search):
754
751
 
755
752
  # provider product type specific conf
756
753
  prep.product_type_def_params = (
757
- self.get_product_type_def_params(product_type, **kwargs)
754
+ self.get_product_type_def_params(product_type, format_variables=kwargs)
758
755
  if product_type is not None
759
756
  else {}
760
757
  )
@@ -778,7 +775,7 @@ class QueryStringSearch(Search):
778
775
  }
779
776
  )
780
777
 
781
- qp, qs = self.build_query_string(product_type, **keywords)
778
+ qp, qs = self.build_query_string(product_type, keywords)
782
779
 
783
780
  prep.query_params = qp
784
781
  prep.query_string = qs
@@ -806,21 +803,21 @@ class QueryStringSearch(Search):
806
803
  reason="Simply run `self.config.metadata_mapping.update(metadata_mapping)` instead",
807
804
  version="2.10.0",
808
805
  )
809
- def update_metadata_mapping(self, metadata_mapping: Dict[str, Any]) -> None:
806
+ def update_metadata_mapping(self, metadata_mapping: dict[str, Any]) -> None:
810
807
  """Update plugin metadata_mapping with input metadata_mapping configuration"""
811
808
  if self.config.metadata_mapping:
812
809
  self.config.metadata_mapping.update(metadata_mapping)
813
810
 
814
811
  def build_query_string(
815
- self, product_type: str, **kwargs: Any
816
- ) -> Tuple[Dict[str, Any], str]:
812
+ self, product_type: str, query_dict: dict[str, Any]
813
+ ) -> tuple[dict[str, Any], str]:
817
814
  """Build The query string using the search parameters"""
818
815
  logger.debug("Building the query string that will be used for search")
819
- query_params = format_query_params(product_type, self.config, kwargs)
816
+ query_params = format_query_params(product_type, self.config, query_dict)
820
817
 
821
818
  # Build the final query string, in one go without quoting it
822
819
  # (some providers do not operate well with urlencoded and quoted query strings)
823
- def quote_via(x: Any, *_args, **_kwargs) -> str:
820
+ def quote_via(x: Any, *_args: Any, **_kwargs: Any) -> str:
824
821
  return x
825
822
 
826
823
  return (
@@ -832,7 +829,7 @@ class QueryStringSearch(Search):
832
829
  self,
833
830
  prep: PreparedSearch = PreparedSearch(page=None, items_per_page=None),
834
831
  **kwargs: Any,
835
- ) -> Tuple[List[str], Optional[int]]:
832
+ ) -> tuple[list[str], Optional[int]]:
836
833
  """Build paginated urls"""
837
834
  page = prep.page
838
835
  items_per_page = prep.items_per_page
@@ -901,7 +898,7 @@ class QueryStringSearch(Search):
901
898
 
902
899
  def do_search(
903
900
  self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
904
- ) -> List[Any]:
901
+ ) -> list[Any]:
905
902
  """Perform the actual search request.
906
903
 
907
904
  If there is a specified number of items per page, return the results as soon
@@ -918,7 +915,7 @@ class QueryStringSearch(Search):
918
915
  "total_items_nb_key_path"
919
916
  ]
920
917
 
921
- results: List[Any] = []
918
+ results: list[Any] = []
922
919
  for search_url in prep.search_urls:
923
920
  single_search_prep = copy_copy(prep)
924
921
  single_search_prep.url = search_url
@@ -1069,14 +1066,15 @@ class QueryStringSearch(Search):
1069
1066
 
1070
1067
  def normalize_results(
1071
1068
  self, results: RawSearchResult, **kwargs: Any
1072
- ) -> List[EOProduct]:
1069
+ ) -> list[EOProduct]:
1073
1070
  """Build EOProducts from provider results"""
1074
1071
  normalize_remaining_count = len(results)
1075
1072
  logger.debug(
1076
1073
  "Adapting %s plugin results to eodag product representation"
1077
1074
  % normalize_remaining_count
1078
1075
  )
1079
- products: List[EOProduct] = []
1076
+ products: list[EOProduct] = []
1077
+ asset_key_from_href = getattr(self.config, "asset_key_from_href", True)
1080
1078
  for result in results:
1081
1079
  product = EOProduct(
1082
1080
  self.provider,
@@ -1091,8 +1089,16 @@ class QueryStringSearch(Search):
1091
1089
  product.properties = dict(
1092
1090
  getattr(self.config, "product_type_config", {}), **product.properties
1093
1091
  )
1094
- # move assets from properties to product's attr
1095
- product.assets.update(product.properties.pop("assets", {}))
1092
+ # move assets from properties to product's attr, normalize keys & roles
1093
+ for key, asset in product.properties.pop("assets", {}).items():
1094
+ norm_key, asset["roles"] = product.driver.guess_asset_key_and_roles(
1095
+ asset.get("href", "") if asset_key_from_href else key,
1096
+ product,
1097
+ )
1098
+ if norm_key:
1099
+ product.assets[norm_key] = asset
1100
+ # sort assets
1101
+ product.assets.data = dict(sorted(product.assets.data.items()))
1096
1102
  products.append(product)
1097
1103
  return products
1098
1104
 
@@ -1134,7 +1140,7 @@ class QueryStringSearch(Search):
1134
1140
  total_results = int(count_results)
1135
1141
  return total_results
1136
1142
 
1137
- def get_collections(self, prep: PreparedSearch, **kwargs: Any) -> Tuple[str, ...]:
1143
+ def get_collections(self, prep: PreparedSearch, **kwargs: Any) -> tuple[str, ...]:
1138
1144
  """Get the collection to which the product belongs"""
1139
1145
  # See https://earth.esa.int/web/sentinel/missions/sentinel-2/news/-
1140
1146
  # /asset_publisher/Ac0d/content/change-of
@@ -1145,7 +1151,7 @@ class QueryStringSearch(Search):
1145
1151
  not hasattr(prep, "product_type_def_params")
1146
1152
  or not prep.product_type_def_params
1147
1153
  ):
1148
- collections: Set[str] = set()
1154
+ collections: set[str] = set()
1149
1155
  collection = getattr(self.config, "collection", None)
1150
1156
  if collection is None:
1151
1157
  try:
@@ -1187,7 +1193,7 @@ class QueryStringSearch(Search):
1187
1193
  info_message = prep.info_message
1188
1194
  exception_message = prep.exception_message
1189
1195
  try:
1190
- timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
1196
+ timeout = getattr(self.config, "timeout", DEFAULT_SEARCH_TIMEOUT)
1191
1197
  ssl_verify = getattr(self.config, "ssl_verify", True)
1192
1198
 
1193
1199
  retry_total = getattr(self.config, "retry_total", REQ_RETRY_TOTAL)
@@ -1200,7 +1206,7 @@ class QueryStringSearch(Search):
1200
1206
 
1201
1207
  ssl_ctx = get_ssl_context(ssl_verify)
1202
1208
  # auth if needed
1203
- kwargs: Dict[str, Any] = {}
1209
+ kwargs: dict[str, Any] = {}
1204
1210
  if (
1205
1211
  getattr(self.config, "need_auth", False)
1206
1212
  and hasattr(prep, "auth")
@@ -1255,6 +1261,9 @@ class QueryStringSearch(Search):
1255
1261
  response.raise_for_status()
1256
1262
  except requests.exceptions.Timeout as exc:
1257
1263
  raise TimeOutError(exc, timeout=timeout) from exc
1264
+ except socket.timeout:
1265
+ err = requests.exceptions.Timeout(request=requests.Request(url=url))
1266
+ raise TimeOutError(err, timeout=timeout)
1258
1267
  except (requests.RequestException, URLError) as err:
1259
1268
  err_msg = err.readlines() if hasattr(err, "readlines") else ""
1260
1269
  if exception_message:
@@ -1331,7 +1340,7 @@ class ODataV4Search(QueryStringSearch):
1331
1340
 
1332
1341
  def do_search(
1333
1342
  self, prep: PreparedSearch = PreparedSearch(), **kwargs: Any
1334
- ) -> List[Any]:
1343
+ ) -> list[Any]:
1335
1344
  """A two step search can be performed if the metadata are not given into the search result"""
1336
1345
 
1337
1346
  if getattr(self.config, "per_product_metadata_query", False):
@@ -1366,7 +1375,7 @@ class ODataV4Search(QueryStringSearch):
1366
1375
  else:
1367
1376
  return super(ODataV4Search, self).do_search(prep, **kwargs)
1368
1377
 
1369
- def get_metadata_search_url(self, entity: Dict[str, Any]) -> str:
1378
+ def get_metadata_search_url(self, entity: dict[str, Any]) -> str:
1370
1379
  """Build the metadata link for the given entity"""
1371
1380
  return "{}({})/Metadata".format(
1372
1381
  self.config.api_endpoint.rstrip("/"), entity["id"]
@@ -1374,7 +1383,7 @@ class ODataV4Search(QueryStringSearch):
1374
1383
 
1375
1384
  def normalize_results(
1376
1385
  self, results: RawSearchResult, **kwargs: Any
1377
- ) -> List[EOProduct]:
1386
+ ) -> list[EOProduct]:
1378
1387
  """Build EOProducts from provider results
1379
1388
 
1380
1389
  If configured, a metadata pre-mapping can be applied to simplify further metadata extraction.
@@ -1430,87 +1439,11 @@ class PostJsonSearch(QueryStringSearch):
1430
1439
 
1431
1440
  """
1432
1441
 
1433
- def _get_default_end_date_from_start_date(
1434
- self, start_datetime: str, product_type_conf: Dict[str, Any]
1435
- ) -> str:
1436
- try:
1437
- start_date = datetime.fromisoformat(start_datetime)
1438
- except ValueError:
1439
- start_date = datetime.strptime(start_datetime, "%Y-%m-%dT%H:%M:%SZ")
1440
- if "completionTimeFromAscendingNode" in product_type_conf:
1441
- mapping = product_type_conf["completionTimeFromAscendingNode"]
1442
- # if date is mapped to year/month/(day), use end_date = start_date else start_date + 1 day
1443
- # (default dates are only needed for ecmwf products where selected timespans should not be too large)
1444
- if isinstance(mapping, list) and "year" in mapping[0]:
1445
- end_date = start_date
1446
- else:
1447
- end_date = start_date + timedelta(days=1)
1448
- return end_date.isoformat()
1449
- return self.get_product_type_cfg_value("missionEndDate", today().isoformat())
1450
-
1451
- def _check_date_params(
1452
- self, keywords: Dict[str, Any], product_type: Optional[str]
1453
- ) -> None:
1454
- """checks if start and end date are present in the keywords and adds them if not"""
1455
- if (
1456
- "startTimeFromAscendingNode"
1457
- and "completionTimeFromAscendingNode" in keywords
1458
- ):
1459
- return
1460
-
1461
- product_type_conf = getattr(self.config, "metadata_mapping", {})
1462
- if (
1463
- product_type
1464
- and product_type in self.config.products
1465
- and "metadata_mapping" in self.config.products[product_type]
1466
- ):
1467
- product_type_conf = self.config.products[product_type]["metadata_mapping"]
1468
- # start time given, end time missing
1469
- if "startTimeFromAscendingNode" in keywords:
1470
- keywords[
1471
- "completionTimeFromAscendingNode"
1472
- ] = self._get_default_end_date_from_start_date(
1473
- keywords["startTimeFromAscendingNode"], product_type_conf
1474
- )
1475
- return
1476
-
1477
- if "completionTimeFromAscendingNode" in product_type_conf:
1478
- mapping = product_type_conf["startTimeFromAscendingNode"]
1479
- if not isinstance(mapping, list):
1480
- mapping = product_type_conf["completionTimeFromAscendingNode"]
1481
- if isinstance(mapping, list):
1482
- # get time parameters (date, year, month, ...) from metadata mapping
1483
- input_mapping = mapping[0].replace("{{", "").replace("}}", "")
1484
- time_params = [
1485
- values.split(":")[0].strip() for values in input_mapping.split(",")
1486
- ]
1487
- time_params = [
1488
- tp.replace('"', "").replace("'", "") for tp in time_params
1489
- ]
1490
- # if startTime is not given but other time params (e.g. year/month/(day)) are given,
1491
- # no default date is required
1492
- in_keywords = True
1493
- for tp in time_params:
1494
- if tp not in keywords:
1495
- in_keywords = False
1496
- break
1497
- if not in_keywords:
1498
- keywords[
1499
- "startTimeFromAscendingNode"
1500
- ] = self.get_product_type_cfg_value(
1501
- "missionStartDate", DEFAULT_MISSION_START_DATE
1502
- )
1503
- keywords[
1504
- "completionTimeFromAscendingNode"
1505
- ] = self._get_default_end_date_from_start_date(
1506
- keywords["startTimeFromAscendingNode"], product_type_conf
1507
- )
1508
-
1509
1442
  def query(
1510
1443
  self,
1511
1444
  prep: PreparedSearch = PreparedSearch(),
1512
1445
  **kwargs: Any,
1513
- ) -> Tuple[List[EOProduct], Optional[int]]:
1446
+ ) -> tuple[list[EOProduct], Optional[int]]:
1514
1447
  """Perform a search on an OpenSearch-like interface"""
1515
1448
  product_type = kwargs.get("productType", "")
1516
1449
  count = prep.count
@@ -1528,7 +1461,7 @@ class PostJsonSearch(QueryStringSearch):
1528
1461
 
1529
1462
  # provider product type specific conf
1530
1463
  prep.product_type_def_params = self.get_product_type_def_params(
1531
- product_type, **kwargs
1464
+ product_type, format_variables=kwargs
1532
1465
  )
1533
1466
  else:
1534
1467
  keywords = {
@@ -1542,7 +1475,7 @@ class PostJsonSearch(QueryStringSearch):
1542
1475
 
1543
1476
  # provider product type specific conf
1544
1477
  prep.product_type_def_params = self.get_product_type_def_params(
1545
- product_type, **kwargs
1478
+ product_type, format_variables=kwargs
1546
1479
  )
1547
1480
 
1548
1481
  # Add to the query, the queryable parameters set in the provider product type definition
@@ -1555,10 +1488,8 @@ class PostJsonSearch(QueryStringSearch):
1555
1488
  and isinstance(self.config.metadata_mapping[k], list)
1556
1489
  }
1557
1490
  )
1558
- if getattr(self.config, "dates_required", False):
1559
- self._check_date_params(keywords, product_type)
1560
1491
 
1561
- qp, _ = self.build_query_string(product_type, **keywords)
1492
+ qp, _ = self.build_query_string(product_type, keywords)
1562
1493
 
1563
1494
  for query_param, query_value in qp.items():
1564
1495
  if (
@@ -1641,7 +1572,7 @@ class PostJsonSearch(QueryStringSearch):
1641
1572
 
1642
1573
  def normalize_results(
1643
1574
  self, results: RawSearchResult, **kwargs: Any
1644
- ) -> List[EOProduct]:
1575
+ ) -> list[EOProduct]:
1645
1576
  """Build EOProducts from provider results"""
1646
1577
  normalized = super().normalize_results(results, **kwargs)
1647
1578
  for product in normalized:
@@ -1676,12 +1607,12 @@ class PostJsonSearch(QueryStringSearch):
1676
1607
  self,
1677
1608
  prep: PreparedSearch = PreparedSearch(),
1678
1609
  **kwargs: Any,
1679
- ) -> Tuple[List[str], Optional[int]]:
1610
+ ) -> tuple[list[str], Optional[int]]:
1680
1611
  """Adds pagination to query parameters, and auth to url"""
1681
1612
  page = prep.page
1682
1613
  items_per_page = prep.items_per_page
1683
1614
  count = prep.count
1684
- urls: List[str] = []
1615
+ urls: list[str] = []
1685
1616
  total_results = 0 if count else None
1686
1617
 
1687
1618
  if "count_endpoint" not in self.config.pagination:
@@ -1750,7 +1681,7 @@ class PostJsonSearch(QueryStringSearch):
1750
1681
  raise ValidationError("Cannot request empty URL")
1751
1682
  info_message = prep.info_message
1752
1683
  exception_message = prep.exception_message
1753
- timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
1684
+ timeout = getattr(self.config, "timeout", DEFAULT_SEARCH_TIMEOUT)
1754
1685
  ssl_verify = getattr(self.config, "ssl_verify", True)
1755
1686
  try:
1756
1687
  # auth if needed
@@ -1841,24 +1772,24 @@ class StacSearch(PostJsonSearch):
1841
1772
  self.config.results_entry = results_entry
1842
1773
 
1843
1774
  def build_query_string(
1844
- self, product_type: str, **kwargs: Any
1845
- ) -> Tuple[Dict[str, Any], str]:
1775
+ self, product_type: str, query_dict: dict[str, Any]
1776
+ ) -> tuple[dict[str, Any], str]:
1846
1777
  """Build The query string using the search parameters"""
1847
1778
  logger.debug("Building the query string that will be used for search")
1848
1779
 
1849
1780
  # handle opened time intervals
1850
1781
  if any(
1851
- k in kwargs
1852
- for k in ("startTimeFromAscendingNode", "completionTimeFromAscendingNode")
1782
+ q in query_dict
1783
+ for q in ("startTimeFromAscendingNode", "completionTimeFromAscendingNode")
1853
1784
  ):
1854
- kwargs.setdefault("startTimeFromAscendingNode", "..")
1855
- kwargs.setdefault("completionTimeFromAscendingNode", "..")
1785
+ query_dict.setdefault("startTimeFromAscendingNode", "..")
1786
+ query_dict.setdefault("completionTimeFromAscendingNode", "..")
1856
1787
 
1857
- query_params = format_query_params(product_type, self.config, kwargs)
1788
+ query_params = format_query_params(product_type, self.config, query_dict)
1858
1789
 
1859
1790
  # Build the final query string, in one go without quoting it
1860
1791
  # (some providers do not operate well with urlencoded and quoted query strings)
1861
- def quote_via(x: Any, *_args, **_kwargs) -> str:
1792
+ def quote_via(x: Any, *_args: Any, **_kwargs: Any) -> str:
1862
1793
  return x
1863
1794
 
1864
1795
  return (
@@ -1868,7 +1799,7 @@ class StacSearch(PostJsonSearch):
1868
1799
 
1869
1800
  def discover_queryables(
1870
1801
  self, **kwargs: Any
1871
- ) -> Optional[Dict[str, Annotated[Any, FieldInfo]]]:
1802
+ ) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
1872
1803
  """Fetch queryables list from provider using `discover_queryables` conf
1873
1804
 
1874
1805
  :param kwargs: additional filters for queryables (`productType` and other search
@@ -1915,7 +1846,8 @@ class StacSearch(PostJsonSearch):
1915
1846
  return None
1916
1847
 
1917
1848
  fetch_url = unparsed_fetch_url.format(
1918
- provider_product_type=provider_product_type, **self.config.__dict__
1849
+ provider_product_type=provider_product_type,
1850
+ **self.config.__dict__,
1919
1851
  )
1920
1852
  auth = (
1921
1853
  self.auth
@@ -1932,7 +1864,8 @@ class StacSearch(PostJsonSearch):
1932
1864
  "{} {} instance:".format(self.provider, self.__class__.__name__),
1933
1865
  ),
1934
1866
  )
1935
- except (RequestError, KeyError, AttributeError):
1867
+ except (RequestError, KeyError, AttributeError) as e:
1868
+ logger.warning("failure in queryables discovery: %s", e)
1936
1869
  return None
1937
1870
  else:
1938
1871
  json_queryables = dict()
@@ -1964,7 +1897,7 @@ class StacSearch(PostJsonSearch):
1964
1897
  return None
1965
1898
 
1966
1899
  # convert json results to pydantic model fields
1967
- field_definitions: Dict[str, Any] = dict()
1900
+ field_definitions: dict[str, Any] = dict()
1968
1901
  for json_param, json_mtd in json_queryables.items():
1969
1902
  param = (
1970
1903
  get_queryable_from_provider(
@@ -1997,7 +1930,7 @@ class PostJsonSearchWithStacQueryables(StacSearch, PostJsonSearch):
1997
1930
  PostJsonSearch.__init__(self, provider, config)
1998
1931
 
1999
1932
  def build_query_string(
2000
- self, product_type: str, **kwargs: Any
2001
- ) -> Tuple[Dict[str, Any], str]:
1933
+ self, product_type: str, query_dict: dict[str, Any]
1934
+ ) -> tuple[dict[str, Any], str]:
2002
1935
  """Build The query string using the search parameters"""
2003
- return PostJsonSearch.build_query_string(self, product_type, **kwargs)
1936
+ return PostJsonSearch.build_query_string(self, product_type, query_dict)