rs-client-libraries 1.0a8.post16.dev0__tar.gz → 1.0a9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/PKG-INFO +3 -3
  2. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/pyproject.toml +9 -7
  3. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/__init__.py +1 -1
  4. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/ogcapi/dpr_client.py +4 -0
  5. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/ogcapi/ogcapi_client.py +1 -4
  6. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/ogcapi/staging_client.py +22 -13
  7. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_common/prefect_utils.py +16 -16
  8. rs_client_libraries-1.0a9/rs_common/utils.py +290 -0
  9. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/auxip_flow.py +149 -5
  10. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/cadip_flow.py +4 -1
  11. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/catalog_flow.py +20 -0
  12. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/dpr_flow.py +8 -3
  13. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/flow_utils.py +2 -0
  14. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/common/l0_last_steps.py +1 -1
  15. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/sentinel3/s3_l0.py +1 -1
  16. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand_processing.py +215 -48
  17. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/payload_builder.py +17 -5
  18. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/payload_generator.py +36 -16
  19. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/record_performance.py +12 -5
  20. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/staging_flow.py +5 -7
  21. rs_client_libraries-1.0a8.post16.dev0/rs_common/utils.py +0 -122
  22. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/LICENSE +0 -0
  23. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/README.md +0 -0
  24. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/ogcapi/__init__.py +0 -0
  25. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/osam_client.py +0 -0
  26. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/rs_client.py +0 -0
  27. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/stac/__init__.py +0 -0
  28. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/stac/auxip_client.py +0 -0
  29. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/stac/cadip_client.py +0 -0
  30. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/stac/catalog_client.py +0 -0
  31. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/stac/prip_client.py +0 -0
  32. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/stac/stac_base.py +0 -0
  33. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_common/__init__.py +0 -0
  34. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_common/config.py +0 -0
  35. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_common/init_opentelemetry.py +0 -0
  36. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_common/logging.py +0 -0
  37. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/__init__.py +0 -0
  38. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/init_pi_db_flow.py +0 -0
  39. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/__init__.py +0 -0
  40. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/common/__init__.py +0 -0
  41. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/common/l0.py +0 -0
  42. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/common/staging.py +0 -0
  43. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/common/types.py +0 -0
  44. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/sentinel1/__init__.py +0 -0
  45. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/sentinel1/s1_l0.py +0 -0
  46. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/sentinel3/__init__.py +0 -0
  47. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/operation/__init__.py +0 -0
  48. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/operation/init_quota_monitoring_flow.py +0 -0
  49. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/operation/osam_flows.py +0 -0
  50. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/operation/quota_monitoring_db_models.py +0 -0
  51. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/operation/quota_monitoring_flow.py +0 -0
  52. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/payload_template.py +0 -0
  53. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/pi_db_models.py +0 -0
  54. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/prip_flow.py +0 -0
  55. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/storage_configuration.py +0 -0
  56. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/utils/__init__.py +0 -0
  57. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/utils/artifact_verbose.py +0 -0
  58. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/utils/cadip.py +0 -0
  59. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/utils/catalog.py +0 -0
  60. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/utils/dask.py +0 -0
  61. {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/utils/dpr.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rs-client-libraries
3
- Version: 1.0a8.post16.dev0
3
+ Version: 1.0a9
4
4
  Summary: Copernicus Sentinel rs-client library and processing workflows.
5
5
  License: Apache-2.0
6
6
  License-File: LICENSE
@@ -25,8 +25,8 @@ Requires-Dist: cachetools (>=5.5.2)
25
25
  Requires-Dist: click (==8.3.1)
26
26
  Requires-Dist: dask-gateway (==2025.4.0)
27
27
  Requires-Dist: openapi-core (==0.22.0)
28
- Requires-Dist: opentelemetry-distro (>=0.61b0,<0.62)
29
- Requires-Dist: opentelemetry-exporter-otlp (>=1.40.0,<2.0.0)
28
+ Requires-Dist: opentelemetry-distro (>=0.62b0,<0.63)
29
+ Requires-Dist: opentelemetry-exporter-otlp (>=1.41.0,<2.0.0)
30
30
  Requires-Dist: prefect-aws (>=0.7.5)
31
31
  Requires-Dist: prefect[aws] (==3.6.20)
32
32
  Requires-Dist: protobuf (>=5.29.5)
@@ -18,7 +18,7 @@ dynamic = ["version"] # see: https://github.com/mtkennerly/poetry-dynamic-versio
18
18
 
19
19
  [tool.poetry]
20
20
  name = "rs-client-libraries"
21
- version = "1.0a8.post16.dev0"
21
+ version = "1.0a9"
22
22
  description = "Copernicus Sentinel rs-client library and processing workflows."
23
23
  license = "Apache-2.0"
24
24
  authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
@@ -51,8 +51,8 @@ include = [
51
51
 
52
52
  [tool.poetry.dependencies]
53
53
  python = ">=3.11,<3.15"
54
- opentelemetry-distro = "^0.61b0" # then run 'poetry run opentelemetry-bootstrap -a install'
55
- opentelemetry-exporter-otlp = "^1.40.0"
54
+ opentelemetry-distro = "^0.62b0" # then run 'poetry run opentelemetry-bootstrap -a install'
55
+ opentelemetry-exporter-otlp = "^1.41.0"
56
56
  protobuf = ">=5.29.5"
57
57
  cachetools = ">=5.5.2"
58
58
  pystac = "^1.13.0"
@@ -140,13 +140,15 @@ max-line-length = 120
140
140
  [tool.pylint.messages_control]
141
141
  max-line-length=120
142
142
  disable = [
143
- "logging-fstring-interpolation",
144
143
  "global-statement",
145
- "too-many-arguments",
146
- "too-many-positional-arguments",
144
+ "import-error",
145
+ "logging-fstring-interpolation",
147
146
  "not-context-manager",
148
- "too-many-locals",
147
+ "too-few-public-methods",
148
+ "too-many-arguments",
149
149
  "too-many-branches",
150
+ "too-many-locals",
151
+ "too-many-positional-arguments",
150
152
  "too-many-statements",
151
153
  ]
152
154
 
@@ -15,4 +15,4 @@
15
15
  """rsclient main module."""
16
16
 
17
17
  # Set automatically by running `poetry dynamic-versioning`
18
- __version__ = "1.0a8.post16.dev0"
18
+ __version__ = "1.0a9"
@@ -47,6 +47,8 @@ class DprProcessor(str, Enum):
47
47
  S1L0 = "s1_l0"
48
48
  S3L0 = "s3_l0"
49
49
  S1ARD = "s1_ard"
50
+ S3L1OLCI = "s3_l1olci"
51
+ # Add S3L2OLCI as "s3_l2olci" when implementation needed (already exists in rs-dpr-service)
50
52
 
51
53
 
52
54
  class DprPipeline(str, Enum):
@@ -56,6 +58,8 @@ class DprPipeline(str, Enum):
56
58
  S1L0FULL = "s1_l0_full"
57
59
  S3L0FULL = "s3_l0_full"
58
60
  S1ARDFULL = "s1_ard_full"
61
+ S3L1OLCIFULL = "ol1_eo"
62
+ # Add S3L2OLCIFULL as "ol2_fr" when implementation needed (already exists in rs-dpr-service)
59
63
 
60
64
 
61
65
  @dataclass
@@ -80,7 +80,7 @@ class OgcApiClient(RsClient):
80
80
 
81
81
  if result.errors:
82
82
  raise OgcValidationException(
83
- f"Error validating the request of the endpoint "
83
+ f"Error validating the request body {request.body!r} of the endpoint "
84
84
  f"{openapi_request.path}: {', '.join(str(x) for x in result.errors)}",
85
85
  )
86
86
  if not result.body:
@@ -226,7 +226,6 @@ class OgcApiClient(RsClient):
226
226
  job_status: dict,
227
227
  logger=None,
228
228
  job_name: str = "",
229
- # timeout: int | float = math.inf,
230
229
  poll_interval: int = 2,
231
230
  ) -> dict:
232
231
  """
@@ -236,8 +235,6 @@ class OgcApiClient(RsClient):
236
235
  job_status: Returned by `_run_process`
237
236
  logger: To show advancement in logger
238
237
  job_name: Job name to show in the logger
239
- timeout: Job completion timeout in seconds.
240
- NOTE: This argument has been disabled, see the comment bellow !
241
238
  poll_interval: When to check again for job completion in seconds
242
239
 
243
240
  Returns:
@@ -62,6 +62,7 @@ class StagingClient(OgcApiClient):
62
62
  self,
63
63
  stac_input: dict[Any, Any] | str,
64
64
  out_coll_name: str,
65
+ asset_names: set[str] | None = None,
65
66
  ) -> dict[str, dict]:
66
67
  """Method to start the staging process from rs-client - Call the endpoint /processes/staging/execution
67
68
 
@@ -74,7 +75,9 @@ class StagingClient(OgcApiClient):
74
75
  - A single link that returns a STAC itemCollection: this link should be an url to search a
75
76
  itemCollection, for example:
76
77
  http://localhost:8002/cadip/search?ids=S1A_20231120061537234567&collections=cadip_sentinel1
77
- out_coll_name (str): name of the output collection
78
+ out_coll_name (str): name of the output collection
79
+ asset_names (set[str]): An optional set to keep only selected asset names.
80
+ If not provided, all assets are staged.
78
81
  Return:
79
82
  dict(hostname, job_id (int, str)): Returns the status codes of the staging requests (one per URL hostname) +
80
83
  the identifiers
@@ -83,11 +86,10 @@ class StagingClient(OgcApiClient):
83
86
 
84
87
  # ----- Case 1: we only load a link (that refers to a STAC itemCollection) in the staging request body
85
88
  if isinstance(stac_input, str):
86
-
87
89
  # Check that it's an url
88
90
  parsed = urlparse(stac_input)
89
91
  if parsed.scheme and parsed.netloc and parsed.hostname:
90
- staging_body = {"inputs": {"collection": out_coll_name, "items": {"href": stac_input}}}
92
+ staging_body = self.staging_body(out_coll_name, {"href": stac_input}, asset_names)
91
93
  return {parsed.hostname: super()._run_process(RESOURCE, staging_body)}
92
94
 
93
95
  # ----- Case 2: we directly load a STAC ItemCollection in the staging request body
@@ -139,21 +141,31 @@ class StagingClient(OgcApiClient):
139
141
  hostname_triggers: dict[str, dict] = {}
140
142
  for hostname, items in hostname_items.items():
141
143
  stac_item_collection = ItemCollection(type="FeatureCollection", features=items)
142
- staging_body = {
143
- "inputs": {
144
- "collection": out_coll_name,
145
- "items": {"value": stac_item_collection.model_dump(mode="json")},
146
- },
147
- }
144
+ staging_body = self.staging_body(
145
+ out_coll_name,
146
+ {"value": stac_item_collection.model_dump(mode="json")},
147
+ asset_names,
148
+ )
148
149
  hostname_triggers[hostname] = super()._run_process(RESOURCE, staging_body)
149
150
 
150
151
  return hostname_triggers
151
152
 
153
+ def staging_body(
154
+ self,
155
+ collection: str,
156
+ items: dict[str, Any],
157
+ asset_names: set[str] | None = None,
158
+ ) -> dict[str, Any]:
159
+ """Build the staging request body with provided arguments"""
160
+ inputs = {"collection": collection, "items": items}
161
+ if asset_names:
162
+ inputs["asset_names"] = list(asset_names)
163
+ return {"inputs": inputs}
164
+
152
165
  def wait_for_jobs(
153
166
  self,
154
167
  all_job_status: dict[str, dict],
155
168
  logger=None,
156
- # timeout: int | float = math.inf,
157
169
  poll_interval: int = 2,
158
170
  ) -> dict[str, dict]:
159
171
  """
@@ -162,8 +174,6 @@ class StagingClient(OgcApiClient):
162
174
  Args:
163
175
  job_status: Returned by `run_staging`
164
176
  logger: To show advancement in logger
165
- timeout: Job completion timeout in seconds
166
- NOTE: This argument has been disabled, see the comment in super().wait_for_job() function
167
177
  poll_interval: When to check again for job completion in seconds
168
178
 
169
179
  Returns:
@@ -179,7 +189,6 @@ class StagingClient(OgcApiClient):
179
189
  job_status,
180
190
  logger,
181
191
  f"Staging from {hostname!r}",
182
- # timeout,
183
192
  poll_interval,
184
193
  )
185
194
  return job_results
@@ -45,11 +45,11 @@ from rs_common.utils import env_bool
45
45
  # In local mode, all your services are running locally.
46
46
  # In cluster mode, we use the services deployed on the RS-Server website.
47
47
  # This configuration is set in an environment variable.
48
- local_mode: bool = False
49
- cluster_mode: bool = not local_mode
48
+ LOCAL_MODE: bool = False
49
+ CLUSTER_MODE: bool = not LOCAL_MODE
50
50
 
51
51
  # Current user
52
- owner_id: str = ""
52
+ OWNER_ID: str = ""
53
53
 
54
54
  # Prefect block names
55
55
  BLOCK_NAME_ENV_GLOBAL: str = "env-vars"
@@ -73,16 +73,16 @@ def init_global_env(new_owner_id: str | None = None):
73
73
  Args:
74
74
  new_owner_id: new ower/user id, if known.
75
75
  """
76
- global local_mode, cluster_mode, owner_id
76
+ global LOCAL_MODE, CLUSTER_MODE, OWNER_ID
77
77
 
78
- local_mode = env_bool("RSPY_LOCAL_MODE", default=False)
79
- cluster_mode = not local_mode
78
+ LOCAL_MODE = env_bool("RSPY_LOCAL_MODE", default=False)
79
+ CLUSTER_MODE = not LOCAL_MODE
80
80
 
81
81
  # Override the owner id or read it from the env vars
82
82
  if new_owner_id:
83
- owner_id = new_owner_id
83
+ OWNER_ID = new_owner_id
84
84
  else:
85
- owner_id = os.getenv("JUPYTERHUB_USER", "") if cluster_mode else os.getenv("RSPY_HOST_USER", "")
85
+ OWNER_ID = os.getenv("JUPYTERHUB_USER", "") if CLUSTER_MODE else os.getenv("RSPY_HOST_USER", "")
86
86
 
87
87
 
88
88
  # Init the global variables
@@ -110,7 +110,7 @@ async def read_apikey(optional: bool = False, save_to_env: bool = True) -> None:
110
110
  save_to_env (bool): If True, saves the API key to the ~/.env file.
111
111
  """
112
112
  # No API key in local mode
113
- if local_mode:
113
+ if LOCAL_MODE:
114
114
  return
115
115
 
116
116
  # If the API is saved as an env var in the ~/.env file, then it has already
@@ -177,7 +177,7 @@ async def init_prefect_blocks():
177
177
 
178
178
  # In local mode, create the block that contains the environment variables for all users.
179
179
  # In cluster mode, this block has already been created by an admin.
180
- if local_mode:
180
+ if LOCAL_MODE:
181
181
 
182
182
  # Get all env var names that start with DASK_GATEWAY_
183
183
  regex = re.compile("^DASK_GATEWAY_.*")
@@ -207,7 +207,7 @@ async def init_prefect_blocks():
207
207
  # Env vars for current user/owner_id
208
208
 
209
209
  # In cluster mode, read the API key
210
- if cluster_mode:
210
+ if CLUSTER_MODE:
211
211
  await read_apikey()
212
212
 
213
213
  # Read env vars that are available from the client env in both local and cluster mode.
@@ -227,10 +227,10 @@ async def init_prefect_blocks():
227
227
  user_env_vars[key] = "1"
228
228
 
229
229
  # Save env vars in a secret block for the current user
230
- await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER, owner_id), user_env_vars)
230
+ await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER, OWNER_ID), user_env_vars)
231
231
 
232
232
  # Now read back the blocks so we are sure our env vars are up-to-date
233
- await read_prefect_blocks(owner_id)
233
+ await read_prefect_blocks(OWNER_ID)
234
234
 
235
235
 
236
236
  @sync_compatible
@@ -277,7 +277,7 @@ async def save_bucket_credentials(
277
277
  )
278
278
 
279
279
  # Update the prefect block
280
- await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER, owner_id), env_vars)
280
+ await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER, OWNER_ID), env_vars)
281
281
 
282
282
  # Save the env vars
283
283
  os.environ.update(env_vars)
@@ -303,7 +303,7 @@ async def remove_bucket_credentials(obs_ids: list[str] | str):
303
303
  env_vars += [f"S3_{key}_ACCESSKEY", f"S3_{key}_SECRETKEY", f"S3_{key}_ENDPOINT", f"S3_{key}_REGION"]
304
304
 
305
305
  # Update the prefect block
306
- await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER, owner_id), remove_keys=env_vars)
306
+ await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER, OWNER_ID), remove_keys=env_vars)
307
307
 
308
308
  # NOTE: I'm not sure we should remove the env vars from os.environ, it's risky because if the env vars
309
309
  # are also defined in other blocks, then maybe we don't want to remove them from os.environ.
@@ -425,7 +425,7 @@ async def get_share_bucket(sub_folder: str = "") -> tuple[S3Bucket, str]:
425
425
 
426
426
  # Use a specific block for the current user. Always overwrite it, because the subfolder may have changed.
427
427
  else:
428
- block_name = format_env_user(BLOCK_NAME_SHARE_BUCKET_USER, owner_id) # type: ignore
428
+ block_name = format_env_user(BLOCK_NAME_SHARE_BUCKET_USER, OWNER_ID) # type: ignore
429
429
 
430
430
  # Read the prefect blocks that contain the S3 authentication
431
431
  bucket_name = os.environ["PREFECT_BUCKET_NAME"]
@@ -0,0 +1,290 @@
1
+ # Copyright 2023-2026 Airbus, CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """This module is used to share common functions between apis"""
16
+
17
+ import logging
18
+ import os
19
+ import shutil
20
+ import tarfile
21
+ import zipfile
22
+ from dataclasses import dataclass
23
+ from datetime import datetime
24
+ from pathlib import Path
25
+ from typing import Any
26
+
27
+ from prefect import get_run_logger
28
+ from prefect.exceptions import MissingContextError
29
+
30
+
31
+ @dataclass
32
+ class AuthInfo:
33
+ """User authentication information in Keycloak."""
34
+
35
+ # User login (preferred username)
36
+ user_login: str
37
+
38
+ # IAM roles
39
+ iam_roles: list[str]
40
+
41
+ # Oauth2 attributes and/or custom `config` associated to the API key
42
+ attributes: dict[str, Any]
43
+
44
+
45
+ def read_response_error(response) -> str:
46
+ """Read and return an HTTP response error detail."""
47
+
48
+ # Try to read the response detail or error
49
+ try:
50
+ json = response.json()
51
+ if isinstance(json, str):
52
+ return json
53
+ return json.get("detail") or json.get("description") or json["error"]
54
+
55
+ # If this fail, get the full response content
56
+ except Exception: # pylint: disable=broad-exception-caught
57
+ return response.content.decode("utf-8", errors="ignore")
58
+
59
+
60
+ def get_href_service(rs_server_href, env_var) -> str:
61
+ """Get specific href link."""
62
+ if from_env := os.getenv(env_var, None):
63
+ return from_env.rstrip("/")
64
+ if not rs_server_href:
65
+ raise RuntimeError("RS-Server URL is undefined")
66
+ return rs_server_href.rstrip("/")
67
+
68
+
69
+ def env_bool(var: str, default: bool) -> bool:
70
+ """
71
+ Return True if an environemnt variable is set to 1, true or yes (case insensitive).
72
+ Return False if set to 0, false or no (case insensitive).
73
+ Return the default value if not set or set to a different value.
74
+ """
75
+ val = os.getenv(var, str(default)).lower()
76
+ if val in ("y", "yes", "t", "true", "on", "1"):
77
+ return True
78
+ if val in ("n", "no", "f", "false", "off", "0"):
79
+ return False
80
+ return default
81
+
82
+
83
+ def strftime_millis(date: datetime):
84
+ """Format datetime with milliseconds precision"""
85
+ return date.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
86
+
87
+
88
+ def create_valcover_filter(
89
+ start_datetime: datetime | str,
90
+ end_datetime: datetime | str,
91
+ product_type: str,
92
+ ) -> dict:
93
+ """Creates a ValCover filter from the input values to be used in flows
94
+
95
+ Args:
96
+ start_datetime: Start datetime for the time interval used to filter the files
97
+ end_datetime: End datetime for the time interval used to filter the files
98
+ product_type: Auxiliary file type wanted
99
+
100
+ Returns:
101
+ dict: ValCover filter
102
+ """
103
+ # Convert datetime inputs to str
104
+ if isinstance(start_datetime, datetime):
105
+ start_datetime = strftime_millis(start_datetime)
106
+ if isinstance(end_datetime, datetime):
107
+ end_datetime = strftime_millis(end_datetime)
108
+
109
+ # CQL2 filter: we use a filter combining a ValCover filter and a product type filter
110
+ return {
111
+ "op": "and",
112
+ "args": [
113
+ {"op": "=", "args": [{"property": "product:type"}, product_type]},
114
+ {
115
+ "op": "t_contains",
116
+ "args": [
117
+ {"interval": [{"property": "start_datetime"}, {"property": "end_datetime"}]},
118
+ {"interval": [start_datetime, end_datetime]},
119
+ ],
120
+ },
121
+ ],
122
+ }
123
+
124
+
125
+ # True if the 'RSPY_LOCAL_MODE' environemnt variable is set to 1, true or yes (case insensitive).
126
+ # By default: if not set or set to a different value, return False.
127
+ LOCAL_MODE: bool = env_bool("RSPY_LOCAL_MODE", default=False)
128
+
129
+ # Cluster mode is the opposite of local mode
130
+ CLUSTER_MODE: bool = not LOCAL_MODE
131
+
132
+
133
+ def _get_logger():
134
+ """Return the Prefect run logger when available, otherwise a module logger."""
135
+ try:
136
+ return get_run_logger()
137
+ except MissingContextError:
138
+ return logging.getLogger(__name__)
139
+
140
+
141
+ def _is_safe_extract_path(extract_to: Path, member_name: str) -> bool:
142
+ """Return whether an archive member stays within the target directory."""
143
+ member_path = Path(member_name.replace("\\", "/"))
144
+ if member_path.is_absolute():
145
+ return False
146
+ try:
147
+ (extract_to / member_path).resolve().relative_to(extract_to.resolve())
148
+ except ValueError:
149
+ return False
150
+ return True
151
+
152
+
153
+ def extract_zip(zip_path: Path, extract_to: Path):
154
+ """Extract a ZIP archive into the target directory."""
155
+ logger = get_run_logger()
156
+ logger.info(f"Extracting ZIP: {zip_path} -> {extract_to}")
157
+
158
+ with zipfile.ZipFile(zip_path, "r") as zip_ref:
159
+ count = 0
160
+ for name in zip_ref.namelist():
161
+ if not _is_safe_extract_path(extract_to, name):
162
+ logger.warning(f"Skipping unsafe ZIP member: {name}")
163
+ continue
164
+ destination = (extract_to / name.replace("\\", "/")).resolve()
165
+ if name.endswith("/"):
166
+ destination.mkdir(parents=True, exist_ok=True)
167
+ else:
168
+ destination.parent.mkdir(parents=True, exist_ok=True)
169
+ with zip_ref.open(name) as src, destination.open("wb") as dst:
170
+ shutil.copyfileobj(src, dst)
171
+ count += 1
172
+ logger.info(f"ZIP contains {count} safe entries")
173
+
174
+
175
+ def extract_tar(file_path: Path, extract_to: Path) -> int:
176
+ """Extract a TAR-compatible archive into the target directory."""
177
+ logger = get_run_logger()
178
+ logger.info(f"Extracting TAR archive: {file_path} -> {extract_to}")
179
+
180
+ with tarfile.open(file_path, "r:*") as tar:
181
+ count = 0
182
+ for member in tar.getmembers():
183
+ if member.issym() or member.islnk() or member.isdev():
184
+ logger.warning(f"Skipping unsafe TAR member: {member.name}")
185
+ continue
186
+ if not _is_safe_extract_path(extract_to, member.name):
187
+ logger.warning(f"Skipping unsafe TAR member: {member.name}")
188
+ continue
189
+ destination = (extract_to / member.name).resolve()
190
+ if member.isdir():
191
+ destination.mkdir(parents=True, exist_ok=True)
192
+ else:
193
+ extracted = tar.extractfile(member)
194
+ if extracted is None:
195
+ continue
196
+ destination.parent.mkdir(parents=True, exist_ok=True)
197
+ with extracted, destination.open("wb") as dst:
198
+ shutil.copyfileobj(extracted, dst)
199
+ count += 1
200
+ logger.info(f"TAR archive contains {count} safe entries")
201
+ return count
202
+
203
+
204
+ def strip_archive_suffix(name: str) -> str:
205
+ """Return the asset name without its supported archive suffix."""
206
+ normalized_name = name.lower()
207
+ for suffix in (".tar.gz", ".tgz", ".zip", ".tar"):
208
+ if normalized_name.endswith(suffix):
209
+ return name[: -len(suffix)]
210
+ return name
211
+
212
+
213
+ def get_upload_prefix(asset_href: str, asset_name: str) -> str:
214
+ """Return the S3 prefix where extracted content should be uploaded."""
215
+ logger = _get_logger()
216
+ parent_prefix = asset_href.rsplit("/", 1)[0]
217
+
218
+ if asset_name.lower().endswith(".zip"):
219
+ result = parent_prefix + "/"
220
+ logger.info(f"get_upload_prefix returning ZIP prefix={result}")
221
+ return result
222
+
223
+ last_segment = parent_prefix.rsplit("/", 1)[-1]
224
+ normalized_segment = strip_archive_suffix(last_segment)
225
+
226
+ if normalized_segment != last_segment:
227
+ base_prefix = parent_prefix.rsplit("/", 1)[0]
228
+ result = base_prefix + "/" + normalized_segment + "/"
229
+ logger.info(f"get_upload_prefix returning normalized prefix={result}")
230
+ return result
231
+
232
+ result = parent_prefix + "/"
233
+ logger.info(f"get_upload_prefix returning parent prefix={result}")
234
+ return result
235
+
236
+
237
+ def _extract_nested_archive(full_path: Path) -> bool:
238
+ """Extract a nested TAR-compatible archive and delete it on success."""
239
+ logger = get_run_logger()
240
+ logger.info(f"Found nested archive: {full_path}")
241
+ extracted_members = extract_tar(full_path, full_path.parent)
242
+ if not extracted_members:
243
+ logger.warning("Skipping removal of nested archive because no members were extracted: " f"{full_path}")
244
+ return False
245
+ full_path.unlink()
246
+ return True
247
+
248
+
249
+ def recursive_extract(folder: Path) -> int:
250
+ """Extract nested TAR-compatible archives found anywhere under ``folder``."""
251
+ logger = get_run_logger()
252
+ extracted_count = 0
253
+ extracted = True
254
+
255
+ while extracted:
256
+ extracted = False
257
+
258
+ for root, _, files in os.walk(folder):
259
+ for file in files:
260
+ full_path = Path(root) / file
261
+
262
+ if file.lower().endswith((".tar", ".tgz", ".tar.gz")) and _extract_nested_archive(full_path):
263
+ extracted = True
264
+ extracted_count += 1
265
+
266
+ logger.info(f"Processed {extracted_count} nested archive(s)")
267
+ return extracted_count
268
+
269
+
270
+ def normalize_extract_dir(extract_dir: Path) -> Path:
271
+ """
272
+ Return the directory that should be used as the upload root.
273
+
274
+ If the extraction produced a single top-level directory, descend into it to
275
+ avoid creating an unnecessary extra folder level in S3.
276
+ """
277
+ logger = get_run_logger()
278
+ root_items = list(extract_dir.iterdir())
279
+
280
+ if len(root_items) != 1:
281
+ logger.info("No normalization needed, multiple root items found")
282
+ return extract_dir
283
+
284
+ root_item = root_items[0]
285
+ if not root_item.is_dir():
286
+ logger.info("No normalization needed, single root item is not a directory")
287
+ return extract_dir
288
+
289
+ logger.info(f"Normalizing folder structure: entering {root_item}")
290
+ return root_item