rs-client-libraries 1.0a8.post16.dev0__tar.gz → 1.0a10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/PKG-INFO +9 -8
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/pyproject.toml +39 -31
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_client/__init__.py +1 -1
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_client/ogcapi/dpr_client.py +17 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_client/ogcapi/ogcapi_client.py +1 -4
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_client/ogcapi/staging_client.py +22 -13
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_client/stac/catalog_client.py +3 -3
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_client/stac/stac_base.py +10 -9
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_common/prefect_utils.py +16 -16
- rs_client_libraries-1.0a10/rs_common/utils.py +290 -0
- rs_client_libraries-1.0a10/rs_workflows/adf_conversion/S00__ADF_ECMWA.py +319 -0
- rs_client_libraries-1.0a10/rs_workflows/adf_conversion/S00__ADF_ECMWF.py +327 -0
- rs_client_libraries-1.0a10/rs_workflows/adf_conversion/S00__ADF_GETAS.py +198 -0
- rs_client_libraries-1.0a10/rs_workflows/adf_conversion/S00__ADF_WATER.py +330 -0
- rs_client_libraries-1.0a10/rs_workflows/adf_conversion/cop_dem_utils/Coordinates.py +23 -0
- rs_client_libraries-1.0a10/rs_workflows/adf_conversion/cop_dem_utils/CopDemConverter.py +123 -0
- rs_client_libraries-1.0a10/rs_workflows/adf_conversion/cop_dem_utils/Grid.py +20 -0
- rs_client_libraries-1.0a10/rs_workflows/adf_conversion/cop_dem_utils/Progression.py +28 -0
- rs_client_libraries-1.0a10/rs_workflows/adf_conversion/cop_dem_utils/Tile.py +36 -0
- rs_client_libraries-1.0a10/rs_workflows/adf_conversion/cop_dem_utils/__init__.py +5 -0
- rs_client_libraries-1.0a10/rs_workflows/adf_conversion/util.py +682 -0
- rs_client_libraries-1.0a10/rs_workflows/adf_flow.py +490 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/auxip_flow.py +23 -13
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/cadip_flow.py +7 -4
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/catalog_flow.py +28 -11
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/dpr_flow.py +48 -29
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/flow_utils.py +152 -2
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/init_pi_db_flow.py +1 -1
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/on_demand/common/l0.py +5 -5
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/on_demand/common/l0_last_steps.py +10 -21
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/on_demand/common/staging.py +18 -9
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/on_demand/common/types.py +91 -36
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/on_demand/sentinel1/s1_l0.py +3 -3
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/on_demand/sentinel3/s3_l0.py +4 -4
- rs_client_libraries-1.0a10/rs_workflows/on_demand_conversion.py +357 -0
- rs_client_libraries-1.0a10/rs_workflows/on_demand_processing.py +527 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/operation/init_quota_monitoring_flow.py +1 -1
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/operation/osam_flows.py +4 -4
- rs_client_libraries-1.0a10/rs_workflows/payload_builder.py +455 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/payload_generator.py +213 -71
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/payload_template.py +0 -13
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/prip_flow.py +3 -3
- rs_client_libraries-1.0a10/rs_workflows/product_type_mapping.py +61 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/record_performance.py +12 -5
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/staging_flow.py +8 -10
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/utils/cadip.py +2 -2
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/utils/catalog.py +22 -12
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/utils/dask.py +5 -2
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/utils/dpr.py +8 -9
- rs_client_libraries-1.0a10/rs_workflows/utils/utils.py +406 -0
- rs_client_libraries-1.0a8.post16.dev0/rs_common/utils.py +0 -122
- rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand_processing.py +0 -268
- rs_client_libraries-1.0a8.post16.dev0/rs_workflows/payload_builder.py +0 -322
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/LICENSE +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/README.md +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_client/ogcapi/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_client/osam_client.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_client/rs_client.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_client/stac/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_client/stac/auxip_client.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_client/stac/cadip_client.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_client/stac/prip_client.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_common/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_common/config.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_common/init_opentelemetry.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_common/logging.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/on_demand/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/on_demand/common/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/on_demand/sentinel1/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/on_demand/sentinel3/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/operation/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/operation/quota_monitoring_db_models.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/operation/quota_monitoring_flow.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/pi_db_models.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/storage_configuration.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/utils/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_workflows/utils/artifact_verbose.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rs-client-libraries
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.0a10
|
|
4
4
|
Summary: Copernicus Sentinel rs-client library and processing workflows.
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
License-File: LICENSE
|
|
@@ -22,13 +22,14 @@ Classifier: Topic :: Scientific/Engineering :: Image Processing
|
|
|
22
22
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
23
|
Requires-Dist: SQLAlchemy (>=2.0.37,<3.0.0)
|
|
24
24
|
Requires-Dist: cachetools (>=5.5.2)
|
|
25
|
-
Requires-Dist: click (==8.
|
|
25
|
+
Requires-Dist: click (==8.4.1)
|
|
26
26
|
Requires-Dist: dask-gateway (==2025.4.0)
|
|
27
|
-
Requires-Dist:
|
|
28
|
-
Requires-Dist:
|
|
29
|
-
Requires-Dist: opentelemetry-
|
|
30
|
-
Requires-Dist:
|
|
31
|
-
Requires-Dist: prefect
|
|
27
|
+
Requires-Dist: importlib_metadata (>=7.0)
|
|
28
|
+
Requires-Dist: openapi-core (>=0.23.1)
|
|
29
|
+
Requires-Dist: opentelemetry-distro (>=0.63b1,<0.64)
|
|
30
|
+
Requires-Dist: opentelemetry-exporter-otlp (>=1.42.1,<2.0.0)
|
|
31
|
+
Requires-Dist: prefect-aws (>=0.7.7)
|
|
32
|
+
Requires-Dist: prefect[aws] (==3.6.29)
|
|
32
33
|
Requires-Dist: protobuf (>=5.29.5)
|
|
33
34
|
Requires-Dist: psycopg2-binary (>=2.9.10)
|
|
34
35
|
Requires-Dist: pystac (>=1.13.0,<2.0.0)
|
|
@@ -36,7 +37,7 @@ Requires-Dist: pystac-client (>=0.9.0,<0.10.0)
|
|
|
36
37
|
Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
|
|
37
38
|
Requires-Dist: requests (>=2.32.4,<3.0.0)
|
|
38
39
|
Requires-Dist: stac-pydantic (>=3.4.0,<4.0.0)
|
|
39
|
-
Requires-Dist: urllib3 (>=2.
|
|
40
|
+
Requires-Dist: urllib3 (>=2.7.0)
|
|
40
41
|
Requires-Dist: uv (>=0.9.7)
|
|
41
42
|
Requires-Dist: werkzeug (>=3.1.6)
|
|
42
43
|
Project-URL: Documentation, https://home.rs-python.eu/rs-documentation/rs-client-libraries/docs/doc/
|
|
@@ -18,7 +18,7 @@ dynamic = ["version"] # see: https://github.com/mtkennerly/poetry-dynamic-versio
|
|
|
18
18
|
|
|
19
19
|
[tool.poetry]
|
|
20
20
|
name = "rs-client-libraries"
|
|
21
|
-
version = "1.
|
|
21
|
+
version = "1.0a10"
|
|
22
22
|
description = "Copernicus Sentinel rs-client library and processing workflows."
|
|
23
23
|
license = "Apache-2.0"
|
|
24
24
|
authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
|
|
@@ -51,20 +51,22 @@ include = [
|
|
|
51
51
|
|
|
52
52
|
[tool.poetry.dependencies]
|
|
53
53
|
python = ">=3.11,<3.15"
|
|
54
|
-
opentelemetry-distro = "^0.
|
|
55
|
-
opentelemetry-exporter-otlp = "^1.
|
|
54
|
+
opentelemetry-distro = "^0.63b1" # then run 'poetry run opentelemetry-bootstrap -a install'
|
|
55
|
+
opentelemetry-exporter-otlp = "^1.42.1"
|
|
56
56
|
protobuf = ">=5.29.5"
|
|
57
57
|
cachetools = ">=5.5.2"
|
|
58
58
|
pystac = "^1.13.0"
|
|
59
59
|
pystac-client = "^0.9.0"
|
|
60
60
|
python-dateutil = "^2.9.0.post0"
|
|
61
61
|
stac-pydantic = "^3.4.0"
|
|
62
|
-
openapi-core = "0.
|
|
63
|
-
click = "8.
|
|
64
|
-
prefect = { version = "3.6.
|
|
65
|
-
prefect-aws = ">=0.7.
|
|
62
|
+
openapi-core = ">=0.23.1"
|
|
63
|
+
click = "8.4.1"
|
|
64
|
+
prefect = { version = "3.6.29", extras = ["aws"] }
|
|
65
|
+
prefect-aws = ">=0.7.7"
|
|
66
|
+
# importlib_metadata required with prefect < 3.7.2 see https://github.com/PrefectHQ/prefect/pull/22012
|
|
67
|
+
importlib_metadata = ">=7.0"
|
|
66
68
|
requests = "^2.32.4"
|
|
67
|
-
urllib3 = ">=2.
|
|
69
|
+
urllib3 = ">=2.7.0"
|
|
68
70
|
SQLAlchemy = "^2.0.37"
|
|
69
71
|
dask-gateway = "2025.4.0"
|
|
70
72
|
uv = ">=0.9.7"
|
|
@@ -75,30 +77,30 @@ werkzeug = ">=3.1.6"
|
|
|
75
77
|
optional = true
|
|
76
78
|
|
|
77
79
|
[tool.poetry.group.dev.dependencies]
|
|
78
|
-
pytest = "^
|
|
79
|
-
pytest-asyncio = "^1.
|
|
80
|
-
pytest-cov = "^7.
|
|
81
|
-
pytest-docker = "^3.2.
|
|
80
|
+
pytest = "^9.0.3"
|
|
81
|
+
pytest-asyncio = "^1.3.0"
|
|
82
|
+
pytest-cov = "^7.1.0"
|
|
83
|
+
pytest-docker = "^3.2.5"
|
|
82
84
|
pytest-error-for-skips = "^2.0.2"
|
|
83
85
|
pytest-mock = "^3.15.1"
|
|
84
86
|
black = ">=26.3.1"
|
|
85
|
-
mypy = "^1.
|
|
86
|
-
pre-commit = "^4.
|
|
87
|
-
bandit = "^1.
|
|
88
|
-
safety = "^3.
|
|
89
|
-
pylint = "^
|
|
90
|
-
sphinx = "
|
|
91
|
-
sphinx-rtd-theme = "
|
|
92
|
-
flake8 = "^7.
|
|
93
|
-
flake8-pyproject = "^1.2.
|
|
94
|
-
responses = "^0.
|
|
95
|
-
starlette = "^0.
|
|
87
|
+
mypy = "^1.20.2"
|
|
88
|
+
pre-commit = "^4.6.0"
|
|
89
|
+
bandit = "^1.9.4"
|
|
90
|
+
safety = "^3.7.0"
|
|
91
|
+
pylint = "^4.0.5"
|
|
92
|
+
sphinx = ">=8.1.3"
|
|
93
|
+
sphinx-rtd-theme = ">=3.0.2"
|
|
94
|
+
flake8 = "^7.3.0"
|
|
95
|
+
flake8-pyproject = "^1.2.4"
|
|
96
|
+
responses = "^0.26.0"
|
|
97
|
+
starlette = "^1.0.0"
|
|
96
98
|
pytest-responses = "^0.5.1"
|
|
97
|
-
moto = "^5.1.
|
|
99
|
+
moto = "^5.1.22"
|
|
98
100
|
|
|
99
101
|
[tool.poetry.group.demo.dependencies] # for hybrid mode
|
|
100
102
|
jupyterlab = "^4.5.0"
|
|
101
|
-
boto3 = "^1.
|
|
103
|
+
boto3 = "^1.43.0"
|
|
102
104
|
papermill = "^2.6.0"
|
|
103
105
|
tornado = "^6.5" # fix trivy vulnerability
|
|
104
106
|
|
|
@@ -115,6 +117,7 @@ build-backend = "poetry_dynamic_versioning.backend"
|
|
|
115
117
|
|
|
116
118
|
[tool.black]
|
|
117
119
|
line-length = 120
|
|
120
|
+
extend-exclude = 'rs_workflows/adf_conversion/'
|
|
118
121
|
|
|
119
122
|
[tool.pytest.ini_options]
|
|
120
123
|
markers = [
|
|
@@ -130,9 +133,11 @@ warn_redundant_casts = true
|
|
|
130
133
|
check_untyped_defs = true
|
|
131
134
|
ignore_missing_imports = true
|
|
132
135
|
disable_error_code = ["call-overload"]
|
|
136
|
+
exclude = ["rs_workflows/adf_conversion/"]
|
|
133
137
|
|
|
134
138
|
[tool.flake8]
|
|
135
139
|
max-line-length = 120
|
|
140
|
+
exclude = ["rs_workflows/adf_conversion/"]
|
|
136
141
|
# per-file-ignores = [
|
|
137
142
|
# '__init__.py:F401',
|
|
138
143
|
# ]
|
|
@@ -140,18 +145,21 @@ max-line-length = 120
|
|
|
140
145
|
[tool.pylint.messages_control]
|
|
141
146
|
max-line-length=120
|
|
142
147
|
disable = [
|
|
143
|
-
"logging-fstring-interpolation",
|
|
144
148
|
"global-statement",
|
|
145
|
-
"
|
|
146
|
-
"
|
|
149
|
+
"import-error",
|
|
150
|
+
"logging-fstring-interpolation",
|
|
147
151
|
"not-context-manager",
|
|
148
|
-
"too-
|
|
152
|
+
"too-few-public-methods",
|
|
153
|
+
"too-many-arguments",
|
|
149
154
|
"too-many-branches",
|
|
155
|
+
"too-many-locals",
|
|
156
|
+
"too-many-positional-arguments",
|
|
150
157
|
"too-many-statements",
|
|
158
|
+
"too-many-lines",
|
|
151
159
|
]
|
|
152
160
|
|
|
153
161
|
[tool.pylint.MASTER]
|
|
154
|
-
ignore-paths = '^docs/.*$'
|
|
162
|
+
ignore-paths = ['^docs/.*$', '^rs_workflows/adf_conversion/.*$']
|
|
155
163
|
|
|
156
164
|
[tool.pylint.MISCELLANEOUS]
|
|
157
165
|
notes = ["XXX"] # no errors on TODO and FIXME
|
|
@@ -161,4 +169,4 @@ notes = ["XXX"] # no errors on TODO and FIXME
|
|
|
161
169
|
min-similarity-lines=10
|
|
162
170
|
|
|
163
171
|
[tool.bandit]
|
|
164
|
-
exclude_dirs = ["tests", ".poetry/plugins"]
|
|
172
|
+
exclude_dirs = ["tests", ".poetry/plugins", "rs_workflows/adf_conversion"]
|
{rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_client/ogcapi/dpr_client.py
RENAMED
|
@@ -47,6 +47,8 @@ class DprProcessor(str, Enum):
|
|
|
47
47
|
S1L0 = "s1_l0"
|
|
48
48
|
S3L0 = "s3_l0"
|
|
49
49
|
S1ARD = "s1_ard"
|
|
50
|
+
S3L1OLCI = "s3_l1olci"
|
|
51
|
+
# Add S3L2OLCI as "s3_l2olci" when implementation needed (already exists in rs-dpr-service)
|
|
50
52
|
|
|
51
53
|
|
|
52
54
|
class DprPipeline(str, Enum):
|
|
@@ -56,6 +58,8 @@ class DprPipeline(str, Enum):
|
|
|
56
58
|
S1L0FULL = "s1_l0_full"
|
|
57
59
|
S3L0FULL = "s3_l0_full"
|
|
58
60
|
S1ARDFULL = "s1_ard_full"
|
|
61
|
+
S3L1OLCIFULL = "ol1_eo"
|
|
62
|
+
# Add S3L2OLCIFULL as "ol2_fr" when implementation needed (already exists in rs-dpr-service)
|
|
59
63
|
|
|
60
64
|
|
|
61
65
|
@dataclass
|
|
@@ -75,6 +79,19 @@ class ClusterInfo:
|
|
|
75
79
|
cluster_instance: str | None = ""
|
|
76
80
|
|
|
77
81
|
|
|
82
|
+
default_cluster_info_repr = ClusterInfo.__repr__
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def custom_cluster_info_repr(self):
|
|
86
|
+
"""Obfuscate the jupyter token in the string representation of the dataclass"""
|
|
87
|
+
return default_cluster_info_repr(
|
|
88
|
+
ClusterInfo(f"{self.jupyter_token[:8]}***", self.cluster_label, self.cluster_instance),
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
ClusterInfo.__repr__ = custom_cluster_info_repr # type: ignore
|
|
93
|
+
|
|
94
|
+
|
|
78
95
|
class DprClient(OgcApiClient):
|
|
79
96
|
"""Implement the OGC API client for 'DPR as a service'."""
|
|
80
97
|
|
|
@@ -80,7 +80,7 @@ class OgcApiClient(RsClient):
|
|
|
80
80
|
|
|
81
81
|
if result.errors:
|
|
82
82
|
raise OgcValidationException(
|
|
83
|
-
f"Error validating the request of the endpoint "
|
|
83
|
+
f"Error validating the request body {request.body!r} of the endpoint "
|
|
84
84
|
f"{openapi_request.path}: {', '.join(str(x) for x in result.errors)}",
|
|
85
85
|
)
|
|
86
86
|
if not result.body:
|
|
@@ -226,7 +226,6 @@ class OgcApiClient(RsClient):
|
|
|
226
226
|
job_status: dict,
|
|
227
227
|
logger=None,
|
|
228
228
|
job_name: str = "",
|
|
229
|
-
# timeout: int | float = math.inf,
|
|
230
229
|
poll_interval: int = 2,
|
|
231
230
|
) -> dict:
|
|
232
231
|
"""
|
|
@@ -236,8 +235,6 @@ class OgcApiClient(RsClient):
|
|
|
236
235
|
job_status: Returned by `_run_process`
|
|
237
236
|
logger: To show advancement in logger
|
|
238
237
|
job_name: Job name to show in the logger
|
|
239
|
-
timeout: Job completion timeout in seconds.
|
|
240
|
-
NOTE: This argument has been disabled, see the comment bellow !
|
|
241
238
|
poll_interval: When to check again for job completion in seconds
|
|
242
239
|
|
|
243
240
|
Returns:
|
|
@@ -62,6 +62,7 @@ class StagingClient(OgcApiClient):
|
|
|
62
62
|
self,
|
|
63
63
|
stac_input: dict[Any, Any] | str,
|
|
64
64
|
out_coll_name: str,
|
|
65
|
+
asset_names: set[str] | None = None,
|
|
65
66
|
) -> dict[str, dict]:
|
|
66
67
|
"""Method to start the staging process from rs-client - Call the endpoint /processes/staging/execution
|
|
67
68
|
|
|
@@ -74,7 +75,9 @@ class StagingClient(OgcApiClient):
|
|
|
74
75
|
- A single link that returns a STAC itemCollection: this link should be an url to search a
|
|
75
76
|
itemCollection, for example:
|
|
76
77
|
http://localhost:8002/cadip/search?ids=S1A_20231120061537234567&collections=cadip_sentinel1
|
|
77
|
-
|
|
78
|
+
out_coll_name (str): name of the output collection
|
|
79
|
+
asset_names (set[str]): An optional set to keep only selected asset names.
|
|
80
|
+
If not provided, all assets are staged.
|
|
78
81
|
Return:
|
|
79
82
|
dict(hostname, job_id (int, str)): Returns the status codes of the staging requests (one per URL hostname) +
|
|
80
83
|
the identifiers
|
|
@@ -83,11 +86,10 @@ class StagingClient(OgcApiClient):
|
|
|
83
86
|
|
|
84
87
|
# ----- Case 1: we only load a link (that refers to a STAC itemCollection) in the staging request body
|
|
85
88
|
if isinstance(stac_input, str):
|
|
86
|
-
|
|
87
89
|
# Check that it's an url
|
|
88
90
|
parsed = urlparse(stac_input)
|
|
89
91
|
if parsed.scheme and parsed.netloc and parsed.hostname:
|
|
90
|
-
staging_body =
|
|
92
|
+
staging_body = self.staging_body(out_coll_name, {"href": stac_input}, asset_names)
|
|
91
93
|
return {parsed.hostname: super()._run_process(RESOURCE, staging_body)}
|
|
92
94
|
|
|
93
95
|
# ----- Case 2: we directly load a STAC ItemCollection in the staging request body
|
|
@@ -139,21 +141,31 @@ class StagingClient(OgcApiClient):
|
|
|
139
141
|
hostname_triggers: dict[str, dict] = {}
|
|
140
142
|
for hostname, items in hostname_items.items():
|
|
141
143
|
stac_item_collection = ItemCollection(type="FeatureCollection", features=items)
|
|
142
|
-
staging_body =
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
}
|
|
144
|
+
staging_body = self.staging_body(
|
|
145
|
+
out_coll_name,
|
|
146
|
+
{"value": stac_item_collection.model_dump(mode="json")},
|
|
147
|
+
asset_names,
|
|
148
|
+
)
|
|
148
149
|
hostname_triggers[hostname] = super()._run_process(RESOURCE, staging_body)
|
|
149
150
|
|
|
150
151
|
return hostname_triggers
|
|
151
152
|
|
|
153
|
+
def staging_body(
|
|
154
|
+
self,
|
|
155
|
+
collection: str,
|
|
156
|
+
items: dict[str, Any],
|
|
157
|
+
asset_names: set[str] | None = None,
|
|
158
|
+
) -> dict[str, Any]:
|
|
159
|
+
"""Build the staging request body with provided arguments"""
|
|
160
|
+
inputs = {"collection": collection, "items": items}
|
|
161
|
+
if asset_names:
|
|
162
|
+
inputs["asset_names"] = list(asset_names)
|
|
163
|
+
return {"inputs": inputs}
|
|
164
|
+
|
|
152
165
|
def wait_for_jobs(
|
|
153
166
|
self,
|
|
154
167
|
all_job_status: dict[str, dict],
|
|
155
168
|
logger=None,
|
|
156
|
-
# timeout: int | float = math.inf,
|
|
157
169
|
poll_interval: int = 2,
|
|
158
170
|
) -> dict[str, dict]:
|
|
159
171
|
"""
|
|
@@ -162,8 +174,6 @@ class StagingClient(OgcApiClient):
|
|
|
162
174
|
Args:
|
|
163
175
|
job_status: Returned by `run_staging`
|
|
164
176
|
logger: To show advancement in logger
|
|
165
|
-
timeout: Job completion timeout in seconds
|
|
166
|
-
NOTE: This argument has been disabled, see the comment in super().wait_for_job() function
|
|
167
177
|
poll_interval: When to check again for job completion in seconds
|
|
168
178
|
|
|
169
179
|
Returns:
|
|
@@ -179,7 +189,6 @@ class StagingClient(OgcApiClient):
|
|
|
179
189
|
job_status,
|
|
180
190
|
logger,
|
|
181
191
|
f"Staging from {hostname!r}",
|
|
182
|
-
# timeout,
|
|
183
192
|
poll_interval,
|
|
184
193
|
)
|
|
185
194
|
return job_results
|
|
@@ -226,7 +226,7 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
|
|
|
226
226
|
) -> ItemCollection | None:
|
|
227
227
|
"""Search items inside a specific collection."""
|
|
228
228
|
|
|
229
|
-
if "collections" in kwargs:
|
|
229
|
+
if "collections" in kwargs and kwargs["collections"]:
|
|
230
230
|
kwargs["collections"] = [
|
|
231
231
|
self.full_collection_id(kwargs.get("owner_id"), collection, "_") for collection in kwargs["collections"]
|
|
232
232
|
] # type: ignore
|
|
@@ -501,8 +501,8 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
|
|
|
501
501
|
self,
|
|
502
502
|
collection_id: str,
|
|
503
503
|
item_id: str,
|
|
504
|
-
owner_id: str,
|
|
505
504
|
patch_values: dict,
|
|
505
|
+
owner_id: str | None = None,
|
|
506
506
|
timeout: int = TIMEOUT,
|
|
507
507
|
raise_for_status: bool = True,
|
|
508
508
|
) -> Response:
|
|
@@ -512,7 +512,7 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
|
|
|
512
512
|
Args:
|
|
513
513
|
collection_id (str): ID of the Collection containing the item.
|
|
514
514
|
item_id (str): ID of the item to patch.
|
|
515
|
-
owner_id (str): Name of the item's owner.
|
|
515
|
+
owner_id (str): Name of the item's owner. If missing, we use self.owner_id.
|
|
516
516
|
patch_values (dict): Dictionary of values to patch in the STAC description of the item.
|
|
517
517
|
timeout (int): Timeout value for the HTTP request (optional, defaults to 30s).
|
|
518
518
|
raise_for_status (bool): If True, raise HTTPError in case of server error (optional, defaults to True).
|
{rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_client/stac/stac_base.py
RENAMED
|
@@ -52,7 +52,7 @@ def handle_api_error(func):
|
|
|
52
52
|
try:
|
|
53
53
|
return func(self, *args, **kwargs)
|
|
54
54
|
except APIError as e:
|
|
55
|
-
error_message = f"Pystac client returned exception: {e}"
|
|
55
|
+
error_message = f"❌ Pystac client returned exception: {e}"
|
|
56
56
|
if hasattr(self, "logger") and self.logger:
|
|
57
57
|
self.logger.exception(error_message)
|
|
58
58
|
else:
|
|
@@ -216,7 +216,7 @@ class StacBase(RsClient):
|
|
|
216
216
|
if items_ids and "ids" not in params:
|
|
217
217
|
params["ids"] = ",".join(items_ids)
|
|
218
218
|
self.logger.info(
|
|
219
|
-
"Retrieving items from collection '%s' with query params: %s.",
|
|
219
|
+
"🔍 Retrieving items from collection '%s' with query params: %s.",
|
|
220
220
|
collection_id,
|
|
221
221
|
params,
|
|
222
222
|
)
|
|
@@ -247,7 +247,7 @@ class StacBase(RsClient):
|
|
|
247
247
|
|
|
248
248
|
# Retrieve a list of items
|
|
249
249
|
if items_ids:
|
|
250
|
-
self.logger.info(f"Retrieving specific items from collection '{collection_id}'.")
|
|
250
|
+
self.logger.info(f"🔍 Retrieving specific items from collection '{collection_id}'.")
|
|
251
251
|
|
|
252
252
|
try:
|
|
253
253
|
return collection.get_items(*items_ids)
|
|
@@ -255,7 +255,7 @@ class StacBase(RsClient):
|
|
|
255
255
|
# Avoid pystac-client fallback through /search ; fetch items one by one.
|
|
256
256
|
# collection.get_items(*ids) internally triggers a search request.
|
|
257
257
|
except (APIError, requests.HTTPError, STACError, ValueError, TypeError) as exc:
|
|
258
|
-
self.logger.debug("Direct retrieval failed, fallback to per-item: %s", exc)
|
|
258
|
+
self.logger.debug("❌ Direct retrieval failed, fallback to per-item: %s", exc)
|
|
259
259
|
|
|
260
260
|
def iter_items():
|
|
261
261
|
for item_id in items_ids:
|
|
@@ -333,17 +333,17 @@ class StacBase(RsClient):
|
|
|
333
333
|
timeout=TIMEOUT,
|
|
334
334
|
)
|
|
335
335
|
except (requests.exceptions.RequestException, requests.exceptions.Timeout) as e:
|
|
336
|
-
self.logger.exception(f"Could not get the response from the endpoint {href_queryables}: {e}")
|
|
336
|
+
self.logger.exception(f"❌ Could not get the response from the endpoint {href_queryables}: {e}")
|
|
337
337
|
raise RuntimeError(
|
|
338
|
-
f"Could not get the response from the endpoint {href_queryables}",
|
|
338
|
+
f"❌ Could not get the response from the endpoint {href_queryables}",
|
|
339
339
|
) from e
|
|
340
340
|
if not response.ok:
|
|
341
|
-
raise RuntimeError(f"Could not get queryables from {href_queryables}")
|
|
341
|
+
raise RuntimeError(f"❌ Could not get queryables from {href_queryables}")
|
|
342
342
|
try:
|
|
343
343
|
json_data = response.json()
|
|
344
344
|
return cast(dict[str, Any], json_data) # Explicitly cast to Dict[str, Any]
|
|
345
345
|
except ValueError as e:
|
|
346
|
-
raise RuntimeError(f"Invalid JSON response from {href_queryables}") from e
|
|
346
|
+
raise RuntimeError(f"❌ Invalid JSON response from {href_queryables}") from e
|
|
347
347
|
|
|
348
348
|
@handle_api_error
|
|
349
349
|
def search( # pylint: disable=too-many-arguments, too-many-positional-arguments
|
|
@@ -362,6 +362,7 @@ class StacBase(RsClient):
|
|
|
362
362
|
kwargs.pop("owner_id", None)
|
|
363
363
|
kwargs["datetime"] = kwargs.pop("timestamp", None)
|
|
364
364
|
kwargs["filter"] = kwargs.pop("stac_filter", None)
|
|
365
|
+
self.logger.info(f"🔍 Performing STAC search with parameters: {kwargs}")
|
|
365
366
|
|
|
366
367
|
try:
|
|
367
368
|
items_search = self.ps_client.search(**kwargs)
|
|
@@ -369,7 +370,7 @@ class StacBase(RsClient):
|
|
|
369
370
|
return items_search.item_collection()
|
|
370
371
|
except NotImplementedError:
|
|
371
372
|
self.logger.exception(
|
|
372
|
-
"The API does not conform to the STAC API Item Search spec"
|
|
373
|
+
"❌ The API does not conform to the STAC API Item Search spec"
|
|
373
374
|
"or does not have a link with a 'rel' type of 'search' ",
|
|
374
375
|
)
|
|
375
376
|
return None
|
{rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a10}/rs_common/prefect_utils.py
RENAMED
|
@@ -45,11 +45,11 @@ from rs_common.utils import env_bool
|
|
|
45
45
|
# In local mode, all your services are running locally.
|
|
46
46
|
# In cluster mode, we use the services deployed on the RS-Server website.
|
|
47
47
|
# This configuration is set in an environment variable.
|
|
48
|
-
|
|
49
|
-
|
|
48
|
+
LOCAL_MODE: bool = False
|
|
49
|
+
CLUSTER_MODE: bool = not LOCAL_MODE
|
|
50
50
|
|
|
51
51
|
# Current user
|
|
52
|
-
|
|
52
|
+
OWNER_ID: str = ""
|
|
53
53
|
|
|
54
54
|
# Prefect block names
|
|
55
55
|
BLOCK_NAME_ENV_GLOBAL: str = "env-vars"
|
|
@@ -73,16 +73,16 @@ def init_global_env(new_owner_id: str | None = None):
|
|
|
73
73
|
Args:
|
|
74
74
|
new_owner_id: new ower/user id, if known.
|
|
75
75
|
"""
|
|
76
|
-
global
|
|
76
|
+
global LOCAL_MODE, CLUSTER_MODE, OWNER_ID
|
|
77
77
|
|
|
78
|
-
|
|
79
|
-
|
|
78
|
+
LOCAL_MODE = env_bool("RSPY_LOCAL_MODE", default=False)
|
|
79
|
+
CLUSTER_MODE = not LOCAL_MODE
|
|
80
80
|
|
|
81
81
|
# Override the owner id or read it from the env vars
|
|
82
82
|
if new_owner_id:
|
|
83
|
-
|
|
83
|
+
OWNER_ID = new_owner_id
|
|
84
84
|
else:
|
|
85
|
-
|
|
85
|
+
OWNER_ID = os.getenv("JUPYTERHUB_USER", "") if CLUSTER_MODE else os.getenv("RSPY_HOST_USER", "")
|
|
86
86
|
|
|
87
87
|
|
|
88
88
|
# Init the global variables
|
|
@@ -110,7 +110,7 @@ async def read_apikey(optional: bool = False, save_to_env: bool = True) -> None:
|
|
|
110
110
|
save_to_env (bool): If True, saves the API key to the ~/.env file.
|
|
111
111
|
"""
|
|
112
112
|
# No API key in local mode
|
|
113
|
-
if
|
|
113
|
+
if LOCAL_MODE:
|
|
114
114
|
return
|
|
115
115
|
|
|
116
116
|
# If the API is saved as an env var in the ~/.env file, then it has already
|
|
@@ -177,7 +177,7 @@ async def init_prefect_blocks():
|
|
|
177
177
|
|
|
178
178
|
# In local mode, create the block that contains the environment variables for all users.
|
|
179
179
|
# In cluster mode, this block has already been created by an admin.
|
|
180
|
-
if
|
|
180
|
+
if LOCAL_MODE:
|
|
181
181
|
|
|
182
182
|
# Get all env var names that start with DASK_GATEWAY_
|
|
183
183
|
regex = re.compile("^DASK_GATEWAY_.*")
|
|
@@ -207,7 +207,7 @@ async def init_prefect_blocks():
|
|
|
207
207
|
# Env vars for current user/owner_id
|
|
208
208
|
|
|
209
209
|
# In cluster mode, read the API key
|
|
210
|
-
if
|
|
210
|
+
if CLUSTER_MODE:
|
|
211
211
|
await read_apikey()
|
|
212
212
|
|
|
213
213
|
# Read env vars that are available from the client env in both local and cluster mode.
|
|
@@ -227,10 +227,10 @@ async def init_prefect_blocks():
|
|
|
227
227
|
user_env_vars[key] = "1"
|
|
228
228
|
|
|
229
229
|
# Save env vars in a secret block for the current user
|
|
230
|
-
await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER,
|
|
230
|
+
await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER, OWNER_ID), user_env_vars)
|
|
231
231
|
|
|
232
232
|
# Now read back the blocks so we are sure our env vars are up-to-date
|
|
233
|
-
await read_prefect_blocks(
|
|
233
|
+
await read_prefect_blocks(OWNER_ID)
|
|
234
234
|
|
|
235
235
|
|
|
236
236
|
@sync_compatible
|
|
@@ -277,7 +277,7 @@ async def save_bucket_credentials(
|
|
|
277
277
|
)
|
|
278
278
|
|
|
279
279
|
# Update the prefect block
|
|
280
|
-
await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER,
|
|
280
|
+
await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER, OWNER_ID), env_vars)
|
|
281
281
|
|
|
282
282
|
# Save the env vars
|
|
283
283
|
os.environ.update(env_vars)
|
|
@@ -303,7 +303,7 @@ async def remove_bucket_credentials(obs_ids: list[str] | str):
|
|
|
303
303
|
env_vars += [f"S3_{key}_ACCESSKEY", f"S3_{key}_SECRETKEY", f"S3_{key}_ENDPOINT", f"S3_{key}_REGION"]
|
|
304
304
|
|
|
305
305
|
# Update the prefect block
|
|
306
|
-
await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER,
|
|
306
|
+
await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER, OWNER_ID), remove_keys=env_vars)
|
|
307
307
|
|
|
308
308
|
# NOTE: I'm not sure we should remove the env vars from os.environ, it's risky because if the env vars
|
|
309
309
|
# are also defined in other blocks, then maybe we don't want to remove them from os.environ.
|
|
@@ -425,7 +425,7 @@ async def get_share_bucket(sub_folder: str = "") -> tuple[S3Bucket, str]:
|
|
|
425
425
|
|
|
426
426
|
# Use a specific block for the current user. Always overwrite it, because the subfolder may have changed.
|
|
427
427
|
else:
|
|
428
|
-
block_name = format_env_user(BLOCK_NAME_SHARE_BUCKET_USER,
|
|
428
|
+
block_name = format_env_user(BLOCK_NAME_SHARE_BUCKET_USER, OWNER_ID) # type: ignore
|
|
429
429
|
|
|
430
430
|
# Read the prefect blocks that contain the S3 authentication
|
|
431
431
|
bucket_name = os.environ["PREFECT_BUCKET_NAME"]
|