wraptile 0.1.1__tar.gz → 0.2.0.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/.gitignore +1 -1
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/PKG-INFO +4 -4
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/pyproject.toml +4 -4
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/app.py +0 -1
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/exceptions.py +29 -14
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/services/airflow/airflow_service.py +47 -46
- wraptile-0.2.0.dev1/wraptile/services/airflow/tokens.py +339 -0
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/services/local/local_service.py +19 -3
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/services/local/testing.py +108 -4
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/LICENSE +0 -0
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/README.md +0 -0
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/__init__.py +0 -0
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/cli.py +0 -0
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/constants.py +0 -0
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/logging.py +0 -0
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/main.py +0 -0
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/provider.py +0 -0
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/py.typed +0 -0
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/routes.py +0 -0
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/services/__init__.py +0 -0
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/services/airflow/__init__.py +0 -0
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/services/base/__init__.py +0 -0
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/services/base/service_base.py +0 -0
- {wraptile-0.1.1 → wraptile-0.2.0.dev1}/wraptile/services/local/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: wraptile
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0.dev1
|
|
4
4
|
Summary: FastAPI server that implements the OGC API - Processes
|
|
5
5
|
Project-URL: Documentation, https://eo-tools.github.io/eozilla
|
|
6
6
|
Project-URL: Repository, https://github.com/eo-tools/eozilla
|
|
@@ -28,11 +28,11 @@ Classifier: Topic :: Software Development
|
|
|
28
28
|
Classifier: Typing :: Typed
|
|
29
29
|
Requires-Python: >=3.11
|
|
30
30
|
Requires-Dist: fastapi
|
|
31
|
-
Requires-Dist: gavicore>=0.
|
|
32
|
-
Requires-Dist: procodile>=0.
|
|
31
|
+
Requires-Dist: gavicore>=0.2.0.dev1
|
|
32
|
+
Requires-Dist: procodile>=0.2.0.dev1
|
|
33
33
|
Requires-Dist: pydantic
|
|
34
34
|
Requires-Dist: pyyaml
|
|
35
|
-
Requires-Dist: typer
|
|
35
|
+
Requires-Dist: typer<1,>=0.26.0
|
|
36
36
|
Requires-Dist: uvicorn
|
|
37
37
|
Description-Content-Type: text/markdown
|
|
38
38
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "wraptile"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0.dev1"
|
|
8
8
|
description = "FastAPI server that implements the OGC API - Processes"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
authors = [
|
|
@@ -43,11 +43,11 @@ dependencies = [
|
|
|
43
43
|
"fastapi",
|
|
44
44
|
"pydantic",
|
|
45
45
|
"pyyaml",
|
|
46
|
-
"typer",
|
|
46
|
+
"typer >=0.26.0,<1",
|
|
47
47
|
"uvicorn",
|
|
48
48
|
# local dependency
|
|
49
|
-
"gavicore >=0.
|
|
50
|
-
"procodile >=0.
|
|
49
|
+
"gavicore >=0.2.0.dev1",
|
|
50
|
+
"procodile >=0.2.0.dev1",
|
|
51
51
|
]
|
|
52
52
|
|
|
53
53
|
[project.scripts]
|
|
@@ -2,39 +2,49 @@
|
|
|
2
2
|
# Permissions are hereby granted under the terms of the Apache 2.0 License:
|
|
3
3
|
# https://opensource.org/license/apache-2-0.
|
|
4
4
|
|
|
5
|
-
import traceback
|
|
6
5
|
from http import HTTPStatus
|
|
7
6
|
from typing import Optional
|
|
8
7
|
|
|
9
8
|
from fastapi import HTTPException
|
|
10
9
|
|
|
11
10
|
from gavicore.models import ApiError
|
|
11
|
+
from gavicore.service.errors import (
|
|
12
|
+
ErrorTypeId,
|
|
13
|
+
create_api_error,
|
|
14
|
+
get_error_type_id,
|
|
15
|
+
)
|
|
12
16
|
|
|
13
17
|
|
|
14
18
|
class ServiceException(HTTPException):
|
|
15
19
|
"""Raised if a service error occurred."""
|
|
16
20
|
|
|
21
|
+
content: ApiError
|
|
22
|
+
|
|
17
23
|
def __init__(
|
|
18
24
|
self,
|
|
19
25
|
status_code: int,
|
|
20
26
|
detail: str,
|
|
21
27
|
exception: Optional[Exception] = None,
|
|
28
|
+
type_id: ErrorTypeId | None = None,
|
|
29
|
+
is_job_problem: bool = False,
|
|
22
30
|
):
|
|
23
31
|
super().__init__(status_code=status_code, detail=detail)
|
|
24
|
-
|
|
25
|
-
|
|
32
|
+
|
|
33
|
+
if type_id is None:
|
|
34
|
+
type_id = get_error_type_id(
|
|
35
|
+
status_code,
|
|
36
|
+
is_job_problem=is_job_problem,
|
|
37
|
+
default="internal-server-error",
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
title = str(exception) if exception else HTTPStatus(status_code).phrase
|
|
41
|
+
|
|
42
|
+
self.content = create_api_error(
|
|
43
|
+
type_id,
|
|
26
44
|
status=status_code,
|
|
27
|
-
title=
|
|
45
|
+
title=title,
|
|
28
46
|
detail=detail,
|
|
29
|
-
|
|
30
|
-
"x-traceback": (
|
|
31
|
-
traceback.format_exception(
|
|
32
|
-
type(exception), exception, exception.__traceback__
|
|
33
|
-
)
|
|
34
|
-
if exception is not None
|
|
35
|
-
else None
|
|
36
|
-
)
|
|
37
|
-
},
|
|
47
|
+
exception=exception,
|
|
38
48
|
)
|
|
39
49
|
|
|
40
50
|
|
|
@@ -42,4 +52,9 @@ class ServiceConfigException(ServiceException):
|
|
|
42
52
|
"""Raised if a service configuration error occurred."""
|
|
43
53
|
|
|
44
54
|
def __init__(self, message: str):
|
|
45
|
-
super().__init__(
|
|
55
|
+
super().__init__(
|
|
56
|
+
detail=message,
|
|
57
|
+
status_code=500,
|
|
58
|
+
type_id="internal-server-config-error",
|
|
59
|
+
exception=self,
|
|
60
|
+
)
|
|
@@ -5,11 +5,9 @@
|
|
|
5
5
|
import datetime
|
|
6
6
|
import logging
|
|
7
7
|
import os
|
|
8
|
-
from functools import cached_property
|
|
9
8
|
from typing import Any, Optional
|
|
10
9
|
|
|
11
10
|
import fastapi
|
|
12
|
-
import requests
|
|
13
11
|
from airflow_client.client import (
|
|
14
12
|
ApiClient,
|
|
15
13
|
ApiException,
|
|
@@ -38,6 +36,7 @@ from gavicore.models import (
|
|
|
38
36
|
)
|
|
39
37
|
from procodile.workflow import FINAL_STEP_ID
|
|
40
38
|
from wraptile.exceptions import ServiceException
|
|
39
|
+
from wraptile.services.airflow.tokens import TokenProvider, create_token_provider
|
|
41
40
|
from wraptile.services.base import ServiceBase
|
|
42
41
|
|
|
43
42
|
_log = logging.getLogger(__name__)
|
|
@@ -56,6 +55,8 @@ class AirflowService(ServiceBase):
|
|
|
56
55
|
self._airflow_base_url: Optional[str] = None
|
|
57
56
|
self._airflow_username: Optional[str] = None
|
|
58
57
|
self._airflow_password: Optional[str] = None
|
|
58
|
+
self._token_provider_instance: Optional[TokenProvider] = None
|
|
59
|
+
self._api_client: Optional[ApiClient] = None
|
|
59
60
|
|
|
60
61
|
def configure(
|
|
61
62
|
self,
|
|
@@ -144,7 +145,10 @@ class AirflowService(ServiceBase):
|
|
|
144
145
|
title = schema_dict.get("title")
|
|
145
146
|
description = param_value.get("description") or schema_dict.get("description")
|
|
146
147
|
nullable = bool(schema_dict.get("nullable", False))
|
|
147
|
-
schema_override: dict[str, Any] = {
|
|
148
|
+
schema_override: dict[str, Any] = {
|
|
149
|
+
"default": default,
|
|
150
|
+
"description": description,
|
|
151
|
+
}
|
|
148
152
|
if title is not None:
|
|
149
153
|
schema_override["title"] = title
|
|
150
154
|
return InputDescription.model_validate(
|
|
@@ -169,7 +173,9 @@ class AirflowService(ServiceBase):
|
|
|
169
173
|
try:
|
|
170
174
|
dag_run = self.airflow_dag_run_api.trigger_dag_run(process_id, dag_run_body)
|
|
171
175
|
except ApiException as e:
|
|
172
|
-
raise ServiceException(
|
|
176
|
+
raise ServiceException(
|
|
177
|
+
e.status, e.reason, exception=e, is_job_problem=True
|
|
178
|
+
) from e
|
|
173
179
|
return self.dag_run_to_job_info(dag_run)
|
|
174
180
|
|
|
175
181
|
async def get_jobs(self, request: fastapi.Request, *args, **kwargs) -> JobList:
|
|
@@ -180,14 +186,18 @@ class AirflowService(ServiceBase):
|
|
|
180
186
|
owners=None, # TODO important, get for current user only
|
|
181
187
|
)
|
|
182
188
|
except ApiException as e:
|
|
183
|
-
raise ServiceException(
|
|
189
|
+
raise ServiceException(
|
|
190
|
+
e.status, detail=e.reason, exception=e, is_job_problem=True
|
|
191
|
+
) from e
|
|
184
192
|
|
|
185
193
|
jobs: list[JobInfo] = []
|
|
186
194
|
for dag in dag_collection.dags:
|
|
187
195
|
try:
|
|
188
196
|
dag_run_collection = self.airflow_dag_run_api.get_dag_runs(dag.dag_id)
|
|
189
197
|
except ApiException as e:
|
|
190
|
-
raise ServiceException(
|
|
198
|
+
raise ServiceException(
|
|
199
|
+
e.status, e.reason, exception=e, is_job_problem=True
|
|
200
|
+
) from e
|
|
191
201
|
jobs.extend(
|
|
192
202
|
self.dag_run_to_job_info(dag_run)
|
|
193
203
|
for dag_run in dag_run_collection.dag_runs
|
|
@@ -199,7 +209,9 @@ class AirflowService(ServiceBase):
|
|
|
199
209
|
try:
|
|
200
210
|
dag_run = self.airflow_dag_run_api.get_dag_run(dag_id, job_id)
|
|
201
211
|
except ApiException as e:
|
|
202
|
-
raise ServiceException(
|
|
212
|
+
raise ServiceException(
|
|
213
|
+
e.status, e.reason, exception=e, is_job_problem=True
|
|
214
|
+
) from e
|
|
203
215
|
return self.dag_run_to_job_info(dag_run)
|
|
204
216
|
|
|
205
217
|
async def dismiss_job(self, job_id: str, *args, **kwargs) -> JobInfo:
|
|
@@ -213,7 +225,9 @@ class AirflowService(ServiceBase):
|
|
|
213
225
|
dag_id, job_id, dag_run_patch
|
|
214
226
|
)
|
|
215
227
|
except ApiException as e:
|
|
216
|
-
raise ServiceException(
|
|
228
|
+
raise ServiceException(
|
|
229
|
+
e.status, e.reason, exception=e, is_job_problem=True
|
|
230
|
+
) from e
|
|
217
231
|
return self.dag_run_to_job_info(dag_run)
|
|
218
232
|
|
|
219
233
|
async def get_job_results(self, job_id: str, *args, **kwargs) -> JobResults:
|
|
@@ -266,58 +280,45 @@ class AirflowService(ServiceBase):
|
|
|
266
280
|
}
|
|
267
281
|
return mapping[dag_run_state]
|
|
268
282
|
|
|
269
|
-
|
|
283
|
+
# NOTE: these are plain properties (not cached_property) so every call
|
|
284
|
+
# re-reads airflow_client, which refreshes the bearer token on expiry. A
|
|
285
|
+
# cached_property would pin the first token for the life of the service.
|
|
286
|
+
@property
|
|
270
287
|
def airflow_dag_api(self) -> DAGApi:
|
|
271
288
|
return DAGApi(self.airflow_client)
|
|
272
289
|
|
|
273
|
-
@
|
|
290
|
+
@property
|
|
274
291
|
def airflow_dag_run_api(self) -> DAGRunApi:
|
|
275
292
|
return DAGRunApi(self.airflow_client)
|
|
276
293
|
|
|
277
|
-
@
|
|
294
|
+
@property
|
|
278
295
|
def airflow_xcom_api(self) -> XComApi:
|
|
279
296
|
return XComApi(self.airflow_client)
|
|
280
297
|
|
|
281
|
-
@
|
|
298
|
+
@property
|
|
282
299
|
def airflow_client(self) -> ApiClient:
|
|
283
300
|
airflow_base_url: str = (
|
|
284
301
|
self._airflow_base_url
|
|
285
302
|
or os.getenv("AIRFLOW_API_BASE_URL")
|
|
286
303
|
or DEFAULT_AIRFLOW_BASE_URL
|
|
287
304
|
)
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
if not airflow_password:
|
|
293
|
-
raise RuntimeError(
|
|
294
|
-
"missing Airflow password; please set env var AIRFLOW_PASSWORD"
|
|
305
|
+
access_token = self._token_provider(airflow_base_url).get_token()
|
|
306
|
+
if self._api_client is None:
|
|
307
|
+
self._api_client = ApiClient(
|
|
308
|
+
Configuration(host=airflow_base_url, access_token=access_token)
|
|
295
309
|
)
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
return ApiClient(configuration)
|
|
310
|
+
else:
|
|
311
|
+
# The generated Airflow client resolves the bearer header from
|
|
312
|
+
# configuration.access_token at call time, so mutating it in place
|
|
313
|
+
# refreshes auth without discarding the connection pool.
|
|
314
|
+
self._api_client.configuration.access_token = access_token
|
|
315
|
+
return self._api_client
|
|
303
316
|
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
f"{base_url}/auth/token",
|
|
313
|
-
json={"username": username, "password": password},
|
|
314
|
-
timeout=10,
|
|
315
|
-
)
|
|
316
|
-
try:
|
|
317
|
-
response.raise_for_status()
|
|
318
|
-
token_data = response.json()
|
|
319
|
-
return token_data.get("access_token")
|
|
320
|
-
except requests.exceptions.HTTPError as e:
|
|
321
|
-
raise ServiceException(
|
|
322
|
-
response.status_code, detail=response.reason or str(e), exception=e
|
|
323
|
-
) from e
|
|
317
|
+
def _token_provider(self, base_url: str) -> TokenProvider:
|
|
318
|
+
if self._token_provider_instance is None:
|
|
319
|
+
self._token_provider_instance = create_token_provider(
|
|
320
|
+
base_url,
|
|
321
|
+
username=self._airflow_username,
|
|
322
|
+
password=self._airflow_password,
|
|
323
|
+
)
|
|
324
|
+
return self._token_provider_instance
|
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
# Copyright (c) 2025-2026 by the Eozilla team and contributors
|
|
2
|
+
# Permissions are hereby granted under the terms of the Apache 2.0 License:
|
|
3
|
+
# https://opensource.org/license/apache-2-0.
|
|
4
|
+
|
|
5
|
+
"""Bearer tokens for the wraptile->airflow hop.
|
|
6
|
+
|
|
7
|
+
Airflow's API accepts **only tokens Airflow itself issued**. An IdP token is not
|
|
8
|
+
an Airflow bearer, even when Airflow is configured against that IdP: an auth
|
|
9
|
+
manager uses the IdP to authenticate a login and to answer authorization
|
|
10
|
+
queries, then mints its own JWT. Sending the IdP token to ``/api/v2/*`` yields
|
|
11
|
+
``403 "Invalid JWT token"``.
|
|
12
|
+
|
|
13
|
+
So the token the *gateway* wants and the token *Airflow* wants are different,
|
|
14
|
+
and both travel on the same ``Authorization`` header. The hop is therefore a
|
|
15
|
+
two-step exchange:
|
|
16
|
+
|
|
17
|
+
1. mint an IdP token via ``client_credentials`` — the gateway validates this
|
|
18
|
+
(issuer, audience, roles) on the ``/auth/token`` request;
|
|
19
|
+
2. exchange it at Airflow's ``/auth/token`` for an **Airflow** JWT, which
|
|
20
|
+
authenticates every subsequent API call.
|
|
21
|
+
|
|
22
|
+
Providers, selected by
|
|
23
|
+
[`TokenConfig.create_token_provider`][wraptile.services.airflow.tokens.TokenConfig.create_token_provider]:
|
|
24
|
+
|
|
25
|
+
* [`AirflowGatewayTokenProvider`][wraptile.services.airflow.tokens.AirflowGatewayTokenProvider] — the two-step exchange above; the
|
|
26
|
+
correct path whenever an IdP is configured.
|
|
27
|
+
* [`ClientCredentialsTokenProvider`][wraptile.services.airflow.tokens.ClientCredentialsTokenProvider] — step 1 alone. It is a *component*
|
|
28
|
+
of the exchange, not a way to talk to Airflow: its token is IdP currency.
|
|
29
|
+
* [`AirflowNativeTokenProvider`][wraptile.services.airflow.tokens.AirflowNativeTokenProvider] — Airflow's ``/auth/token`` by password,
|
|
30
|
+
for deployments with neither an IdP nor a gateway.
|
|
31
|
+
|
|
32
|
+
The environment is read in exactly one place,
|
|
33
|
+
[`TokenConfig.from_env`][wraptile.services.airflow.tokens.TokenConfig.from_env];
|
|
34
|
+
everything downstream of it takes a
|
|
35
|
+
[`TokenConfig`][wraptile.services.airflow.tokens.TokenConfig] and is testable
|
|
36
|
+
without touching ``os.environ``.
|
|
37
|
+
|
|
38
|
+
Nothing here is specific to a particular identity provider: only the standard
|
|
39
|
+
OAuth2 token endpoint and grants are used.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
import abc
|
|
43
|
+
import os
|
|
44
|
+
import time
|
|
45
|
+
from collections.abc import Mapping
|
|
46
|
+
from dataclasses import dataclass
|
|
47
|
+
from typing import Optional
|
|
48
|
+
|
|
49
|
+
import requests
|
|
50
|
+
|
|
51
|
+
from wraptile.exceptions import ServiceException
|
|
52
|
+
|
|
53
|
+
# Re-mint the service token this many seconds before it expires, so a request
|
|
54
|
+
# never carries an already-expired bearer (a typical provider TTL is 300s).
|
|
55
|
+
TOKEN_REFRESH_MARGIN = 30
|
|
56
|
+
|
|
57
|
+
# Used when the token response omits "expires_in".
|
|
58
|
+
DEFAULT_TOKEN_LIFETIME = 300
|
|
59
|
+
|
|
60
|
+
TOKEN_REQUEST_TIMEOUT = 10
|
|
61
|
+
|
|
62
|
+
DEFAULT_AUDIENCE = "airflow"
|
|
63
|
+
|
|
64
|
+
DEFAULT_AIRFLOW_USERNAME = "admin"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class TokenProvider(abc.ABC):
|
|
68
|
+
"""Supplies the bearer token for calls to the Airflow API."""
|
|
69
|
+
|
|
70
|
+
@abc.abstractmethod
|
|
71
|
+
def get_token(self) -> str:
|
|
72
|
+
"""Return a currently valid access token."""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class ClientCredentialsTokenProvider(TokenProvider):
|
|
76
|
+
"""OAuth2 ``client_credentials`` grant against an OIDC provider.
|
|
77
|
+
|
|
78
|
+
The token is cached and re-minted shortly before expiry so it is not
|
|
79
|
+
re-fetched on every request.
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
def __init__(
|
|
83
|
+
self,
|
|
84
|
+
token_url: str,
|
|
85
|
+
client_id: str,
|
|
86
|
+
client_secret: Optional[str] = None,
|
|
87
|
+
audience: Optional[str] = None,
|
|
88
|
+
):
|
|
89
|
+
self._token_url = token_url
|
|
90
|
+
self._client_id = client_id
|
|
91
|
+
self._client_secret = client_secret
|
|
92
|
+
self._audience = audience
|
|
93
|
+
self._token: Optional[str] = None
|
|
94
|
+
self._expiry: float = 0.0
|
|
95
|
+
|
|
96
|
+
def get_token(self) -> str:
|
|
97
|
+
now = time.monotonic()
|
|
98
|
+
if self._token and now < self._expiry:
|
|
99
|
+
return self._token
|
|
100
|
+
|
|
101
|
+
data = {"grant_type": "client_credentials", "client_id": self._client_id}
|
|
102
|
+
if self._client_secret:
|
|
103
|
+
data["client_secret"] = self._client_secret
|
|
104
|
+
if self._audience:
|
|
105
|
+
# The client's audience mapper normally stamps this already;
|
|
106
|
+
# sending it explicitly is harmless and defensive.
|
|
107
|
+
data["audience"] = self._audience
|
|
108
|
+
|
|
109
|
+
response = requests.post(
|
|
110
|
+
self._token_url, data=data, timeout=TOKEN_REQUEST_TIMEOUT
|
|
111
|
+
)
|
|
112
|
+
token_data = _json_or_raise(response)
|
|
113
|
+
self._token = token_data["access_token"]
|
|
114
|
+
self._expiry = (
|
|
115
|
+
now
|
|
116
|
+
+ token_data.get("expires_in", DEFAULT_TOKEN_LIFETIME)
|
|
117
|
+
- TOKEN_REFRESH_MARGIN
|
|
118
|
+
)
|
|
119
|
+
return self._token
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class AirflowGatewayTokenProvider(TokenProvider):
|
|
123
|
+
"""Exchange an IdP service token for an Airflow-issued JWT.
|
|
124
|
+
|
|
125
|
+
Airflow's API rejects IdP tokens (``403 "Invalid JWT token"``) — it accepts
|
|
126
|
+
only JWTs it minted. Airflow's ``/auth/token`` performs the exchange, and a
|
|
127
|
+
gateway in front of it validates the IdP token on *that* request. Hence two
|
|
128
|
+
tokens, each authenticating a different leg:
|
|
129
|
+
|
|
130
|
+
* the IdP token goes on the ``Authorization`` header, satisfying the gateway;
|
|
131
|
+
* the client credentials go in the body, satisfying Airflow.
|
|
132
|
+
|
|
133
|
+
Both are needed simultaneously. Without a gateway the header is simply
|
|
134
|
+
ignored, so this provider is also correct when Airflow is reached directly.
|
|
135
|
+
|
|
136
|
+
The Airflow JWT is cached against **its own** expiry, which is unrelated to
|
|
137
|
+
the IdP token's: ``inner`` re-mints independently on its own schedule.
|
|
138
|
+
Deriving one lifetime from the other silently breaks the hop the moment the
|
|
139
|
+
shorter one lapses.
|
|
140
|
+
"""
|
|
141
|
+
|
|
142
|
+
def __init__(
|
|
143
|
+
self,
|
|
144
|
+
base_url: str,
|
|
145
|
+
client_id: str,
|
|
146
|
+
client_secret: Optional[str],
|
|
147
|
+
inner: TokenProvider,
|
|
148
|
+
):
|
|
149
|
+
self._base_url = base_url
|
|
150
|
+
self._client_id = client_id
|
|
151
|
+
self._client_secret = client_secret
|
|
152
|
+
self._inner = inner
|
|
153
|
+
self._token: Optional[str] = None
|
|
154
|
+
self._expiry: float = 0.0
|
|
155
|
+
|
|
156
|
+
def get_token(self) -> str:
|
|
157
|
+
now = time.monotonic()
|
|
158
|
+
if self._token and now < self._expiry:
|
|
159
|
+
return self._token
|
|
160
|
+
|
|
161
|
+
body = {"grant_type": "client_credentials", "client_id": self._client_id}
|
|
162
|
+
if self._client_secret:
|
|
163
|
+
body["client_secret"] = self._client_secret
|
|
164
|
+
|
|
165
|
+
response = requests.post(
|
|
166
|
+
f"{self._base_url}/auth/token",
|
|
167
|
+
json=body,
|
|
168
|
+
# The IdP token authenticates this request to the gateway, not to
|
|
169
|
+
# Airflow; Airflow authenticates it from the body above.
|
|
170
|
+
headers={"Authorization": f"Bearer {self._inner.get_token()}"},
|
|
171
|
+
timeout=TOKEN_REQUEST_TIMEOUT,
|
|
172
|
+
)
|
|
173
|
+
token_data = _json_or_raise(response)
|
|
174
|
+
self._token = token_data["access_token"]
|
|
175
|
+
# Airflow's response carries no "expires_in"; its lifetime comes from
|
|
176
|
+
# the server's api_auth.jwt_expiration_time. Assume the conservative
|
|
177
|
+
# default and re-mint early — an over-eager exchange costs one request,
|
|
178
|
+
# a stale JWT costs a 403 mid-flight.
|
|
179
|
+
self._expiry = (
|
|
180
|
+
now
|
|
181
|
+
+ token_data.get("expires_in", DEFAULT_TOKEN_LIFETIME)
|
|
182
|
+
- TOKEN_REFRESH_MARGIN
|
|
183
|
+
)
|
|
184
|
+
return self._token
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
class AirflowNativeTokenProvider(TokenProvider):
|
|
188
|
+
"""Airflow's own ``/auth/token`` endpoint, authenticated by password.
|
|
189
|
+
|
|
190
|
+
Only for deployments with **no IdP and no gateway**: the password grant
|
|
191
|
+
requires direct access grants enabled on the IdP client (normally, and
|
|
192
|
+
correctly, disabled), and this sends no ``Authorization`` header, so a
|
|
193
|
+
gateway enforcing JWT on ``/auth/token`` rejects it with 401 before Airflow
|
|
194
|
+
ever sees it. When an IdP is configured, use
|
|
195
|
+
[`AirflowGatewayTokenProvider`][wraptile.services.airflow.tokens.AirflowGatewayTokenProvider]
|
|
196
|
+
instead.
|
|
197
|
+
"""
|
|
198
|
+
|
|
199
|
+
def __init__(self, base_url: str, username: str, password: str):
|
|
200
|
+
self._base_url = base_url
|
|
201
|
+
self._username = username
|
|
202
|
+
self._password = password
|
|
203
|
+
|
|
204
|
+
def get_token(self) -> str:
|
|
205
|
+
response = requests.post(
|
|
206
|
+
f"{self._base_url}/auth/token",
|
|
207
|
+
json={"username": self._username, "password": self._password},
|
|
208
|
+
timeout=TOKEN_REQUEST_TIMEOUT,
|
|
209
|
+
)
|
|
210
|
+
return _json_or_raise(response)["access_token"]
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
@dataclass(frozen=True)
|
|
214
|
+
class TokenConfig:
|
|
215
|
+
"""Everything needed to decide *how* to obtain an Airflow bearer token.
|
|
216
|
+
|
|
217
|
+
Construct it directly in tests, or via
|
|
218
|
+
[`from_env`][wraptile.services.airflow.tokens.TokenConfig.from_env] in
|
|
219
|
+
production.
|
|
220
|
+
The OIDC path wins whenever it is configured; the Airflow-native path is
|
|
221
|
+
the fallback.
|
|
222
|
+
"""
|
|
223
|
+
|
|
224
|
+
airflow_base_url: str
|
|
225
|
+
oidc_token_url: Optional[str] = None
|
|
226
|
+
oidc_client_id: Optional[str] = None
|
|
227
|
+
oidc_client_secret: Optional[str] = None
|
|
228
|
+
oidc_audience: str = DEFAULT_AUDIENCE
|
|
229
|
+
airflow_username: str = DEFAULT_AIRFLOW_USERNAME
|
|
230
|
+
airflow_password: Optional[str] = None
|
|
231
|
+
|
|
232
|
+
@classmethod
|
|
233
|
+
def from_env(
|
|
234
|
+
cls,
|
|
235
|
+
base_url: str,
|
|
236
|
+
username: Optional[str] = None,
|
|
237
|
+
password: Optional[str] = None,
|
|
238
|
+
env: Optional[Mapping[str, str]] = None,
|
|
239
|
+
# Airflow credentials passed by the caller take precedence over the
|
|
240
|
+
# environment; the OIDC settings are environment-only (deployment
|
|
241
|
+
# decides whether a gateway is in the path, not the call site).
|
|
242
|
+
) -> "TokenConfig":
|
|
243
|
+
"""Read the token settings from a mapping, `os.environ` by default.
|
|
244
|
+
|
|
245
|
+
Args:
|
|
246
|
+
base_url: Base URL of the Airflow web API, used by the native
|
|
247
|
+
fallback.
|
|
248
|
+
username: Airflow username, defaults to env `AIRFLOW_USERNAME`,
|
|
249
|
+
then `admin`.
|
|
250
|
+
password: Airflow password, defaults to env `AIRFLOW_PASSWORD`.
|
|
251
|
+
env: The environment to read; injected by tests.
|
|
252
|
+
"""
|
|
253
|
+
env = os.environ if env is None else env
|
|
254
|
+
return cls(
|
|
255
|
+
airflow_base_url=base_url,
|
|
256
|
+
oidc_token_url=env.get("OIDC_TOKEN_URL") or None,
|
|
257
|
+
oidc_client_id=env.get("OIDC_CLIENT_ID") or None,
|
|
258
|
+
oidc_client_secret=env.get("OIDC_CLIENT_SECRET") or None,
|
|
259
|
+
oidc_audience=env.get("OIDC_AUDIENCE") or DEFAULT_AUDIENCE,
|
|
260
|
+
airflow_username=(
|
|
261
|
+
username or env.get("AIRFLOW_USERNAME") or DEFAULT_AIRFLOW_USERNAME
|
|
262
|
+
),
|
|
263
|
+
airflow_password=password or env.get("AIRFLOW_PASSWORD") or None,
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
@property
|
|
267
|
+
def uses_client_credentials(self) -> bool:
|
|
268
|
+
"""Whether the OIDC client-credentials grant is configured at all."""
|
|
269
|
+
return bool(self.oidc_token_url or self.oidc_client_id)
|
|
270
|
+
|
|
271
|
+
def create_token_provider(self) -> TokenProvider:
|
|
272
|
+
"""Build the token provider this configuration selects.
|
|
273
|
+
|
|
274
|
+
With OIDC configured this returns
|
|
275
|
+
[`AirflowGatewayTokenProvider`][wraptile.services.airflow.tokens.AirflowGatewayTokenProvider],
|
|
276
|
+
never the bare
|
|
277
|
+
[`ClientCredentialsTokenProvider`][wraptile.services.airflow.tokens.ClientCredentialsTokenProvider] — the latter's
|
|
278
|
+
token is IdP currency, which Airflow rejects with
|
|
279
|
+
``403 "Invalid JWT token"``. The exchange is not an extra hop to be
|
|
280
|
+
optimised away; it is the only thing Airflow's API accepts.
|
|
281
|
+
|
|
282
|
+
Raises:
|
|
283
|
+
RuntimeError: If the OIDC settings are only half-filled, or if the
|
|
284
|
+
native fallback has no Airflow password.
|
|
285
|
+
"""
|
|
286
|
+
if self.uses_client_credentials:
|
|
287
|
+
if not (self.oidc_token_url and self.oidc_client_id):
|
|
288
|
+
# Half-configuring OIDC is a deployment mistake. Falling back to
|
|
289
|
+
# the native endpoint here would paper over it and send Airflow
|
|
290
|
+
# a token the gateway was meant to validate.
|
|
291
|
+
raise RuntimeError(
|
|
292
|
+
"incomplete OIDC configuration; please set both env vars"
|
|
293
|
+
" OIDC_TOKEN_URL and OIDC_CLIENT_ID"
|
|
294
|
+
)
|
|
295
|
+
return AirflowGatewayTokenProvider(
|
|
296
|
+
self.airflow_base_url,
|
|
297
|
+
client_id=self.oidc_client_id,
|
|
298
|
+
client_secret=self.oidc_client_secret,
|
|
299
|
+
inner=ClientCredentialsTokenProvider(
|
|
300
|
+
token_url=self.oidc_token_url,
|
|
301
|
+
client_id=self.oidc_client_id,
|
|
302
|
+
client_secret=self.oidc_client_secret,
|
|
303
|
+
audience=self.oidc_audience,
|
|
304
|
+
),
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
if not self.airflow_password:
|
|
308
|
+
raise RuntimeError(
|
|
309
|
+
"missing Airflow password; please set env var AIRFLOW_PASSWORD"
|
|
310
|
+
)
|
|
311
|
+
return AirflowNativeTokenProvider(
|
|
312
|
+
self.airflow_base_url,
|
|
313
|
+
username=self.airflow_username,
|
|
314
|
+
password=self.airflow_password,
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def create_token_provider(
|
|
319
|
+
base_url: str,
|
|
320
|
+
username: Optional[str] = None,
|
|
321
|
+
password: Optional[str] = None,
|
|
322
|
+
) -> TokenProvider:
|
|
323
|
+
"""Select the token provider from the environment.
|
|
324
|
+
|
|
325
|
+
Convenience wrapper over ``TokenConfig.from_env(...).create_token_provider()``.
|
|
326
|
+
"""
|
|
327
|
+
return TokenConfig.from_env(
|
|
328
|
+
base_url, username=username, password=password
|
|
329
|
+
).create_token_provider()
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _json_or_raise(response: requests.Response) -> dict:
|
|
333
|
+
try:
|
|
334
|
+
response.raise_for_status()
|
|
335
|
+
except requests.exceptions.HTTPError as e:
|
|
336
|
+
raise ServiceException(
|
|
337
|
+
response.status_code, detail=response.reason or str(e), exception=e
|
|
338
|
+
) from e
|
|
339
|
+
return response.json()
|
|
@@ -21,6 +21,7 @@ from gavicore.models import (
|
|
|
21
21
|
ProcessRequest,
|
|
22
22
|
ProcessSummary,
|
|
23
23
|
)
|
|
24
|
+
from gavicore.service.errors import ErrorTypeId
|
|
24
25
|
from gavicore.util.dynimp import import_value
|
|
25
26
|
from procodile import Job, Process, ProcessRegistry
|
|
26
27
|
from wraptile.exceptions import ServiceException
|
|
@@ -102,6 +103,7 @@ class LocalService(ServiceBase):
|
|
|
102
103
|
400,
|
|
103
104
|
detail=f"Invalid parameterization for process {process_id!r}: {e}",
|
|
104
105
|
exception=e,
|
|
106
|
+
type_id="bad-request",
|
|
105
107
|
) from e
|
|
106
108
|
executor = self._ensure_executor()
|
|
107
109
|
use_processes = isinstance(executor, ProcessPoolExecutor)
|
|
@@ -239,7 +241,11 @@ class LocalService(ServiceBase):
|
|
|
239
241
|
def _get_process(self, process_id: str) -> Process:
|
|
240
242
|
process = self.process_registry.get(process_id)
|
|
241
243
|
if process is None:
|
|
242
|
-
raise ServiceException(
|
|
244
|
+
raise ServiceException(
|
|
245
|
+
404,
|
|
246
|
+
detail=f"Process {process_id!r} does not exist",
|
|
247
|
+
type_id="no-such-process",
|
|
248
|
+
)
|
|
243
249
|
return process
|
|
244
250
|
|
|
245
251
|
def _get_job(
|
|
@@ -247,10 +253,20 @@ class LocalService(ServiceBase):
|
|
|
247
253
|
) -> Job:
|
|
248
254
|
job = self.jobs.get(job_id)
|
|
249
255
|
if job is None:
|
|
250
|
-
raise ServiceException(
|
|
256
|
+
raise ServiceException(
|
|
257
|
+
404, detail=f"Job {job_id!r} does not exist", type_id="no-such-job"
|
|
258
|
+
)
|
|
251
259
|
message = forbidden_status_codes.get(job.job_info.status)
|
|
260
|
+
type_id: ErrorTypeId | None = None
|
|
261
|
+
if job.job_info.status in (JobStatus.accepted, JobStatus.running):
|
|
262
|
+
type_id = "result-not-ready"
|
|
252
263
|
if message:
|
|
253
|
-
raise ServiceException(
|
|
264
|
+
raise ServiceException(
|
|
265
|
+
403,
|
|
266
|
+
detail=f"Job {job_id!r} {message}",
|
|
267
|
+
is_job_problem=True,
|
|
268
|
+
type_id=type_id,
|
|
269
|
+
)
|
|
254
270
|
return job
|
|
255
271
|
|
|
256
272
|
|
|
@@ -10,7 +10,7 @@ from typing import Annotated, Optional
|
|
|
10
10
|
import pydantic
|
|
11
11
|
from pydantic import Field
|
|
12
12
|
|
|
13
|
-
from gavicore.models import InputDescription, Link, Schema
|
|
13
|
+
from gavicore.models import InputDescription, Link, OutputDescription, Schema
|
|
14
14
|
from procodile import FromMain, FromStep, JobContext
|
|
15
15
|
from wraptile.services.local import LocalService
|
|
16
16
|
|
|
@@ -107,20 +107,20 @@ def primes_between(
|
|
|
107
107
|
title="Variable names",
|
|
108
108
|
description="Comma-separated list of variable names.",
|
|
109
109
|
schema=Schema(), # type: ignore[call-arg]
|
|
110
|
-
**{"x-ui:advanced": True},
|
|
111
110
|
),
|
|
112
111
|
"bbox": Field(
|
|
113
112
|
title="Bounding box",
|
|
114
113
|
description="Bounding box in geographical coordinates.",
|
|
115
|
-
json_schema_extra={"x-ui-widget": "map"},
|
|
116
114
|
min_length=4,
|
|
117
115
|
max_length=4,
|
|
116
|
+
json_schema_extra={"x-ui-widget": "map"},
|
|
118
117
|
),
|
|
119
118
|
"resolution": Field(
|
|
120
119
|
title="Spatial resolution",
|
|
121
120
|
description="Spatial resolution in degree.",
|
|
122
121
|
ge=0.01,
|
|
123
122
|
le=1.0,
|
|
123
|
+
json_schema_extra={"x-ui-advanced": True},
|
|
124
124
|
),
|
|
125
125
|
"start_date": Field(
|
|
126
126
|
title="Start date",
|
|
@@ -135,12 +135,12 @@ def primes_between(
|
|
|
135
135
|
description="Size of time steps in days.",
|
|
136
136
|
ge=1,
|
|
137
137
|
le=10,
|
|
138
|
+
json_schema_extra={"x-ui-advanced": True},
|
|
138
139
|
),
|
|
139
140
|
"output_path": InputDescription(
|
|
140
141
|
title="Output path",
|
|
141
142
|
description="Local output path or URI.",
|
|
142
143
|
schema=Schema(minLength=1), # type: ignore[call-arg]
|
|
143
|
-
**{"x-ui:advanced": True},
|
|
144
144
|
),
|
|
145
145
|
},
|
|
146
146
|
)
|
|
@@ -309,3 +309,107 @@ def store_data(
|
|
|
309
309
|
) -> tuple[tuple[str, str], str]:
|
|
310
310
|
print("Storing data")
|
|
311
311
|
return id, second_input
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
@registry.process(
|
|
315
|
+
id="218",
|
|
316
|
+
title="L3B AOI Indicators Processor",
|
|
317
|
+
version="5.0.4",
|
|
318
|
+
description="Site Processing L3B NDVI Processor (demo from ESA Sen3CAP project).",
|
|
319
|
+
inputs={
|
|
320
|
+
"start_date": InputDescription(
|
|
321
|
+
title="Start date",
|
|
322
|
+
schema=Schema(
|
|
323
|
+
**{
|
|
324
|
+
"type": "string",
|
|
325
|
+
"format": "date",
|
|
326
|
+
"nullable": False,
|
|
327
|
+
"x-ui-order": 10,
|
|
328
|
+
}
|
|
329
|
+
),
|
|
330
|
+
),
|
|
331
|
+
"end_date": InputDescription(
|
|
332
|
+
title="End date",
|
|
333
|
+
schema=Schema(
|
|
334
|
+
**{
|
|
335
|
+
"type": "string",
|
|
336
|
+
"format": "date",
|
|
337
|
+
"nullable": False,
|
|
338
|
+
"x-ui-order": 11,
|
|
339
|
+
}
|
|
340
|
+
),
|
|
341
|
+
),
|
|
342
|
+
"geometry": InputDescription(
|
|
343
|
+
title="Geometry (WKT)",
|
|
344
|
+
schema=Schema(
|
|
345
|
+
**{
|
|
346
|
+
"type": "string",
|
|
347
|
+
"format": "wkt",
|
|
348
|
+
"nullable": False,
|
|
349
|
+
"x-ui-widget": "map",
|
|
350
|
+
"x-ui-order": 20,
|
|
351
|
+
}
|
|
352
|
+
),
|
|
353
|
+
),
|
|
354
|
+
"indicator_name": InputDescription(
|
|
355
|
+
title="L3B Indicator Name",
|
|
356
|
+
schema=Schema(
|
|
357
|
+
**{
|
|
358
|
+
"type": "string",
|
|
359
|
+
"enum": ["NDVI", "LAI", "FAPAR", "FCOVER", "NDWI"],
|
|
360
|
+
"nullable": True,
|
|
361
|
+
"x-ui-widget": "radio",
|
|
362
|
+
"x-ui-advanced": True,
|
|
363
|
+
"x-ui-order": 30,
|
|
364
|
+
}
|
|
365
|
+
),
|
|
366
|
+
),
|
|
367
|
+
"site_extend": InputDescription(
|
|
368
|
+
title="Site extent",
|
|
369
|
+
schema=Schema(
|
|
370
|
+
**{
|
|
371
|
+
"type": "string",
|
|
372
|
+
"format": "wkt",
|
|
373
|
+
"nullable": True,
|
|
374
|
+
"x-ui-order": 40,
|
|
375
|
+
}
|
|
376
|
+
),
|
|
377
|
+
),
|
|
378
|
+
},
|
|
379
|
+
outputs={
|
|
380
|
+
"return_value": OutputDescription(
|
|
381
|
+
title="stacitemsfile",
|
|
382
|
+
schema=Schema(
|
|
383
|
+
**{
|
|
384
|
+
"type": "string",
|
|
385
|
+
"format": "uri",
|
|
386
|
+
"nullable": False,
|
|
387
|
+
}
|
|
388
|
+
),
|
|
389
|
+
)
|
|
390
|
+
},
|
|
391
|
+
)
|
|
392
|
+
def processor(
|
|
393
|
+
start_date: str,
|
|
394
|
+
end_date: str,
|
|
395
|
+
geometry: str,
|
|
396
|
+
indicator_name: str | None,
|
|
397
|
+
site_extend: str | None,
|
|
398
|
+
) -> dict[str, str]:
|
|
399
|
+
ctx = JobContext.get()
|
|
400
|
+
ctx.report_progress(message="Started processing")
|
|
401
|
+
for i in range(10):
|
|
402
|
+
time.sleep(0.5)
|
|
403
|
+
ctx.report_progress(progress=(i + 1) * 10)
|
|
404
|
+
ctx.report_progress(message="Ended processing")
|
|
405
|
+
return {
|
|
406
|
+
k: v
|
|
407
|
+
for k, v in dict(
|
|
408
|
+
start_date=start_date,
|
|
409
|
+
end_date=end_date,
|
|
410
|
+
geometry=geometry,
|
|
411
|
+
indicator_name=indicator_name,
|
|
412
|
+
site_extend=site_extend,
|
|
413
|
+
).items()
|
|
414
|
+
if v is not None
|
|
415
|
+
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|