atlan-application-sdk-api 3.51.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- application_sdk/__init__.py +7 -0
- application_sdk/_build_info.py +62 -0
- application_sdk/_context_errors.py +65 -0
- application_sdk/_install.py +130 -0
- application_sdk/_logging.py +171 -0
- application_sdk/_runtime/__init__.py +29 -0
- application_sdk/_runtime/offload.py +1020 -0
- application_sdk/_runtime/progress.py +757 -0
- application_sdk/clients/__init__.py +63 -0
- application_sdk/clients/_interface.py +58 -0
- application_sdk/clients/_sql_failure_classifier.py +112 -0
- application_sdk/clients/base.py +565 -0
- application_sdk/clients/http_errors.py +28 -0
- application_sdk/clients/models.py +49 -0
- application_sdk/clients/sql.py +1218 -0
- application_sdk/clients/sql_errors.py +192 -0
- application_sdk/clients/sql_typecasters.py +431 -0
- application_sdk/clients/ssl_utils.py +247 -0
- application_sdk/common/__init__.py +53 -0
- application_sdk/common/_generated_tree.py +321 -0
- application_sdk/common/_listing.py +192 -0
- application_sdk/common/aws_utils.py +522 -0
- application_sdk/common/aws_utils_errors.py +88 -0
- application_sdk/common/build_identity.py +120 -0
- application_sdk/common/concurrency.py +45 -0
- application_sdk/common/errors.py +99 -0
- application_sdk/common/filter_matching.py +261 -0
- application_sdk/common/logger.py +11 -0
- application_sdk/common/models.py +31 -0
- application_sdk/common/sql_filters.py +719 -0
- application_sdk/common/sql_filters_errors.py +17 -0
- application_sdk/common/task_queue.py +431 -0
- application_sdk/common/types.py +51 -0
- application_sdk/common/utils.py +87 -0
- application_sdk/constants.py +1120 -0
- application_sdk/contracts/__init__.py +158 -0
- application_sdk/contracts/base.py +1178 -0
- application_sdk/contracts/events.py +251 -0
- application_sdk/contracts/types.py +801 -0
- application_sdk/contracts/types_errors.py +17 -0
- application_sdk/credentials/__init__.py +326 -0
- application_sdk/credentials/errors.py +226 -0
- application_sdk/credentials/extra.py +114 -0
- application_sdk/credentials/ingress.py +283 -0
- application_sdk/credentials/ref.py +376 -0
- application_sdk/credentials/spec.py +182 -0
- application_sdk/credentials/types.py +339 -0
- application_sdk/credentials/utils.py +257 -0
- application_sdk/errors/__init__.py +224 -0
- application_sdk/errors/base.py +504 -0
- application_sdk/errors/categories.py +118 -0
- application_sdk/errors/http.py +81 -0
- application_sdk/errors/leaves.py +526 -0
- application_sdk/errors/wire.py +124 -0
- application_sdk/handler/__init__.py +97 -0
- application_sdk/handler/_warmup.py +198 -0
- application_sdk/handler/asgi.py +252 -0
- application_sdk/handler/base.py +281 -0
- application_sdk/handler/context.py +212 -0
- application_sdk/handler/contracts.py +1166 -0
- application_sdk/handler/manifest.py +81 -0
- application_sdk/handler/metadata_routes.py +699 -0
- application_sdk/handler/object_routes.py +418 -0
- application_sdk/handler/request_contract.py +91 -0
- application_sdk/handler/routes.py +1118 -0
- application_sdk/handler/service_errors.py +37 -0
- application_sdk/server/__init__.py +15 -0
- application_sdk/server/fastapi/models.py +338 -0
- application_sdk/templates/__init__.py +52 -0
- application_sdk/templates/contracts/__init__.py +111 -0
- application_sdk/templates/contracts/base_metadata_extraction.py +32 -0
- application_sdk/templates/contracts/incremental_sql.py +563 -0
- application_sdk/templates/contracts/sql_metadata.py +824 -0
- application_sdk/templates/contracts/sql_query.py +105 -0
- application_sdk/version.py +40 -0
- atlan_application_sdk_api-3.51.0.dist-info/METADATA +230 -0
- atlan_application_sdk_api-3.51.0.dist-info/RECORD +78 -0
- atlan_application_sdk_api-3.51.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,565 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import warnings
|
|
3
|
+
from typing import Any, Literal, overload
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
from httpx import Headers
|
|
7
|
+
from httpx._types import (
|
|
8
|
+
AuthTypes,
|
|
9
|
+
HeaderTypes,
|
|
10
|
+
QueryParamTypes,
|
|
11
|
+
RequestData,
|
|
12
|
+
RequestFiles,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
from application_sdk._logging import get_logger
|
|
16
|
+
from application_sdk.clients._interface import ClientInterface
|
|
17
|
+
from application_sdk.clients.http_errors import HttpClientSourceUnavailableError
|
|
18
|
+
from application_sdk.clients.sql_errors import AbstractClientLoadError
|
|
19
|
+
from application_sdk.clients.ssl_utils import get_ssl_context
|
|
20
|
+
from application_sdk.constants import (
|
|
21
|
+
_HTTP_CONNECT_RETRIES,
|
|
22
|
+
_HTTP_POOL_LIMITS,
|
|
23
|
+
_HTTP_POOL_TIMEOUT_SECONDS,
|
|
24
|
+
)
|
|
25
|
+
from application_sdk.errors.http import classify_http_exception
|
|
26
|
+
from application_sdk.errors.leaves import SourceUnavailableError
|
|
27
|
+
|
|
28
|
+
logger = get_logger(__name__)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class _SharedTransport(httpx.AsyncBaseTransport):
|
|
32
|
+
"""Lends a long-lived transport to a short-lived ``httpx.AsyncClient``.
|
|
33
|
+
|
|
34
|
+
Each request builds its own ``AsyncClient``, and ``AsyncClient.__aexit__``
|
|
35
|
+
closes its transport. When that transport is the client's shared pool,
|
|
36
|
+
the first request to finish closes the sockets every concurrent request
|
|
37
|
+
is still reading from, and they fail with ``httpx.ReadError``. This
|
|
38
|
+
wrapper forwards requests and ignores the per-call enter and close; the
|
|
39
|
+
owning ``BaseClient`` enters the real transport once, before its first
|
|
40
|
+
request, and ``BaseClient.close()`` exits it.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
def __init__(self, transport: httpx.AsyncBaseTransport) -> None:
|
|
44
|
+
self._transport = transport
|
|
45
|
+
|
|
46
|
+
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
|
47
|
+
return await self._transport.handle_async_request(request)
|
|
48
|
+
|
|
49
|
+
async def __aenter__(self) -> "_SharedTransport":
|
|
50
|
+
return self
|
|
51
|
+
|
|
52
|
+
async def __aexit__(self, *args: Any) -> None:
|
|
53
|
+
return None
|
|
54
|
+
|
|
55
|
+
async def aclose(self) -> None:
|
|
56
|
+
return None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class BaseClient(ClientInterface):
|
|
60
|
+
"""
|
|
61
|
+
Base client for non-SQL based applications.
|
|
62
|
+
|
|
63
|
+
This class provides a base implementation for clients that need to connect
|
|
64
|
+
to non-SQL data sources. It implements the ClientInterface and provides
|
|
65
|
+
basic functionality that can be extended by subclasses.
|
|
66
|
+
|
|
67
|
+
Attributes:
|
|
68
|
+
credentials (Dict[str, Any]): Client credentials for authentication.
|
|
69
|
+
http_headers (HeaderTypes): HTTP headers for all http requests made by this client. Supports dict, Headers object, or list of tuples.
|
|
70
|
+
http_retry_transport (httpx.AsyncBaseTransport): HTTP transport for requests. Defaults to an
|
|
71
|
+
httpx transport that retries connection setup up to 3 times. Can be overridden in load()
|
|
72
|
+
for custom retry behavior.
|
|
73
|
+
|
|
74
|
+
Extending the Client:
|
|
75
|
+
To customize retry behavior, subclasses can override the http_retry_transport
|
|
76
|
+
in the load() method, similar to how http_headers is set:
|
|
77
|
+
|
|
78
|
+
Example:
|
|
79
|
+
>>> class MyClient(BaseClient):
|
|
80
|
+
... async def load(self, **kwargs):
|
|
81
|
+
... # Set up HTTP headers in load method for better modularity
|
|
82
|
+
... credentials = kwargs.get("credentials", {})
|
|
83
|
+
... # Can use dict, Headers object, or list of tuples
|
|
84
|
+
... self.http_headers = {
|
|
85
|
+
... "Authorization": f"Bearer {credentials.get('token')}",
|
|
86
|
+
... "User-Agent": "MyApp/1.0"
|
|
87
|
+
... }
|
|
88
|
+
... # Optionally override retry transport with custom configuration
|
|
89
|
+
... # For advanced retry logic with status code handling, use httpx-retries:
|
|
90
|
+
... # from httpx_retries import Retry, RetryTransport
|
|
91
|
+
... # retry = Retry(total=5, backoff_factor=20)
|
|
92
|
+
... # self.http_retry_transport = RetryTransport(retry=retry) #replace transport with custom transport if needed
|
|
93
|
+
|
|
94
|
+
Advanced Retry Configuration:
|
|
95
|
+
For applications requiring advanced retry logic (e.g., status code-based retries,
|
|
96
|
+
rate limiting, custom backoff strategies), consider using httpx-retries library:
|
|
97
|
+
|
|
98
|
+
>>> class MyClient(BaseClient):
|
|
99
|
+
... async def load(self, **kwargs):
|
|
100
|
+
... # Set up headers
|
|
101
|
+
... self.http_headers = {"Authorization": f"Bearer {kwargs.get('token')}"}
|
|
102
|
+
...
|
|
103
|
+
... # Install httpx-retries: pip install httpx-retries
|
|
104
|
+
... from httpx_retries import Retry, RetryTransport
|
|
105
|
+
...
|
|
106
|
+
... # Configure retry for status codes and network errors
|
|
107
|
+
... retry = Retry(
|
|
108
|
+
... total=5,
|
|
109
|
+
... backoff_factor=10,
|
|
110
|
+
... status_forcelist=[429, 500, 502, 503, 504]
|
|
111
|
+
... )
|
|
112
|
+
... self.http_retry_transport = RetryTransport(retry=retry)
|
|
113
|
+
|
|
114
|
+
Header Management:
|
|
115
|
+
The client supports a two-level header system using httpx Headers for merging headers:
|
|
116
|
+
- Client-level headers: Set in the load() method and used for all requests
|
|
117
|
+
- Method-level headers: Passed to individual methods and override/add to client headers
|
|
118
|
+
|
|
119
|
+
Example:
|
|
120
|
+
>>> client = MyClient()
|
|
121
|
+
>>> await client.load(credentials={"token": "initial_token"})
|
|
122
|
+
>>> # This request will use: {"Authorization": "Bearer initial_token", "User-Agent": "MyApp/1.0", "Content-Type": "application/json"}
|
|
123
|
+
>>> response = await client.execute_http_post_request(
|
|
124
|
+
... url="https://api.example.com/data",
|
|
125
|
+
... headers={"Content-Type": "application/json"}
|
|
126
|
+
... )
|
|
127
|
+
"""
|
|
128
|
+
|
|
129
|
+
def __init__(
|
|
130
|
+
self,
|
|
131
|
+
credentials: dict[str, Any] = {},
|
|
132
|
+
http_headers: HeaderTypes = {},
|
|
133
|
+
):
|
|
134
|
+
"""
|
|
135
|
+
Initialize the base client.
|
|
136
|
+
|
|
137
|
+
Args:
|
|
138
|
+
credentials (Dict[str, Any], optional): Client credentials for authentication. Defaults to {}.
|
|
139
|
+
http_headers (HeaderTypes, optional): HTTP headers for all requests. Defaults to {}.
|
|
140
|
+
"""
|
|
141
|
+
self.credentials = credentials
|
|
142
|
+
self.http_headers = http_headers
|
|
143
|
+
|
|
144
|
+
# Pool limits prevent CLOSE_WAIT zombie socket accumulation (see
|
|
145
|
+
# BLDX-1153). httpcore retries only connection setup (ConnectError,
|
|
146
|
+
# ConnectTimeout), before any request bytes are sent, so the retries
|
|
147
|
+
# are safe for POST.
|
|
148
|
+
self.http_retry_transport: httpx.AsyncBaseTransport = httpx.AsyncHTTPTransport(
|
|
149
|
+
limits=_HTTP_POOL_LIMITS,
|
|
150
|
+
retries=_HTTP_CONNECT_RETRIES,
|
|
151
|
+
)
|
|
152
|
+
# The transport entered by the first request and not yet exited by
|
|
153
|
+
# close(). While set, its connection pool may hold sockets.
|
|
154
|
+
self._entered_transport: httpx.AsyncBaseTransport | None = None
|
|
155
|
+
self._enter_lock = asyncio.Lock()
|
|
156
|
+
|
|
157
|
+
def __del__(self) -> None:
|
|
158
|
+
"""Warn when a client that made requests is collected unclosed.
|
|
159
|
+
|
|
160
|
+
Guarded against interpreter teardown, when module globals such as
|
|
161
|
+
``warnings`` may already be cleared: the warning is best-effort and an
|
|
162
|
+
exception from a finalizer can only surface as an unraisable error.
|
|
163
|
+
"""
|
|
164
|
+
try:
|
|
165
|
+
if getattr(self, "_entered_transport", None) is not None:
|
|
166
|
+
warnings.warn(
|
|
167
|
+
f"{type(self).__name__} was not closed; its connection pool "
|
|
168
|
+
"may still hold sockets. Use 'async with' or call close(), "
|
|
169
|
+
"and call super().close() from a close() override.",
|
|
170
|
+
ResourceWarning,
|
|
171
|
+
source=self,
|
|
172
|
+
stacklevel=2,
|
|
173
|
+
)
|
|
174
|
+
except Exception: # noqa: S110, BLE001 — teardown-only; see docstring
|
|
175
|
+
pass
|
|
176
|
+
|
|
177
|
+
async def _borrow_transport(self) -> _SharedTransport:
|
|
178
|
+
"""Enter ``http_retry_transport`` once, then lend it to a per-call client.
|
|
179
|
+
|
|
180
|
+
A per-call ``AsyncClient`` used to enter and exit the transport on
|
|
181
|
+
every request. The transport is now shared, so it is entered once here
|
|
182
|
+
(again if ``load()`` swapped it) and exited by ``close()``.
|
|
183
|
+
"""
|
|
184
|
+
transport = self.http_retry_transport
|
|
185
|
+
if self._entered_transport is not transport:
|
|
186
|
+
async with self._enter_lock:
|
|
187
|
+
if self._entered_transport is not transport:
|
|
188
|
+
await transport.__aenter__()
|
|
189
|
+
self._entered_transport = transport
|
|
190
|
+
return _SharedTransport(transport)
|
|
191
|
+
|
|
192
|
+
async def load(self, **kwargs: Any) -> None:
|
|
193
|
+
"""
|
|
194
|
+
Initialize the client with credentials and necessary attributes for the client to work.
|
|
195
|
+
|
|
196
|
+
This method should be implemented by subclasses to:
|
|
197
|
+
- Set up authentication headers in self.http_headers in case of http requestss
|
|
198
|
+
- Initialize any required client state
|
|
199
|
+
- Handle credential processing
|
|
200
|
+
- Optionally override self.http_retry_transport for custom retry behavior
|
|
201
|
+
|
|
202
|
+
For advanced retry logic (status code-based retries, rate limiting, custom backoff),
|
|
203
|
+
consider using httpx-retries library and overriding http_retry_transport:
|
|
204
|
+
|
|
205
|
+
Example:
|
|
206
|
+
>>> async def load(self, **kwargs):
|
|
207
|
+
... # Set up headers
|
|
208
|
+
... self.http_headers = {"Authorization": f"Bearer {kwargs.get('token')}"}
|
|
209
|
+
...
|
|
210
|
+
... # For advanced retry logic, install httpx-retries: pip install httpx-retries
|
|
211
|
+
... from httpx_retries import Retry, RetryTransport
|
|
212
|
+
... retry = Retry(total=5, backoff_factor=10, status_forcelist=[429, 500, 502, 503, 504])
|
|
213
|
+
... self.http_retry_transport = RetryTransport(retry=retry)
|
|
214
|
+
|
|
215
|
+
Args:
|
|
216
|
+
**kwargs: Additional keyword arguments, typically including credentials.
|
|
217
|
+
May also include retry configuration parameters that can be used to
|
|
218
|
+
create a custom http_retry_transport.
|
|
219
|
+
|
|
220
|
+
Raises:
|
|
221
|
+
AbstractClientLoadError: If the subclass does not implement this method.
|
|
222
|
+
"""
|
|
223
|
+
|
|
224
|
+
raise AbstractClientLoadError()
|
|
225
|
+
|
|
226
|
+
async def close(self, *args: Any, **kwargs: Any) -> None:
|
|
227
|
+
"""Release the HTTP transport and its connection pool.
|
|
228
|
+
|
|
229
|
+
Requests borrow ``http_retry_transport`` without closing it, so that
|
|
230
|
+
concurrent requests on one client do not tear down each other's
|
|
231
|
+
connections. Call this once the client is no longer needed, or use
|
|
232
|
+
the client as an async context manager (``async with``), which calls it
|
|
233
|
+
on exit. A subclass that overrides ``close()`` must call
|
|
234
|
+
``super().close()``, or the connection pool stays open.
|
|
235
|
+
"""
|
|
236
|
+
entered, self._entered_transport = self._entered_transport, None
|
|
237
|
+
if entered is not None:
|
|
238
|
+
await entered.__aexit__(None, None, None)
|
|
239
|
+
if entered is not self.http_retry_transport:
|
|
240
|
+
await self.http_retry_transport.aclose()
|
|
241
|
+
|
|
242
|
+
@overload
|
|
243
|
+
async def execute_http_get_request(
|
|
244
|
+
self,
|
|
245
|
+
url: str,
|
|
246
|
+
headers: HeaderTypes | None = None,
|
|
247
|
+
params: QueryParamTypes | None = None,
|
|
248
|
+
auth: AuthTypes | None = None,
|
|
249
|
+
timeout: int = 10,
|
|
250
|
+
*,
|
|
251
|
+
raise_on_error: Literal[True],
|
|
252
|
+
) -> httpx.Response: ...
|
|
253
|
+
|
|
254
|
+
@overload
|
|
255
|
+
async def execute_http_get_request(
|
|
256
|
+
self,
|
|
257
|
+
url: str,
|
|
258
|
+
headers: HeaderTypes | None = None,
|
|
259
|
+
params: QueryParamTypes | None = None,
|
|
260
|
+
auth: AuthTypes | None = None,
|
|
261
|
+
timeout: int = 10,
|
|
262
|
+
*,
|
|
263
|
+
raise_on_error: bool | None = None,
|
|
264
|
+
) -> httpx.Response | None: ...
|
|
265
|
+
|
|
266
|
+
async def execute_http_get_request(
|
|
267
|
+
self,
|
|
268
|
+
url: str,
|
|
269
|
+
headers: HeaderTypes | None = None,
|
|
270
|
+
params: QueryParamTypes | None = None,
|
|
271
|
+
auth: AuthTypes | None = None,
|
|
272
|
+
timeout: int = 10,
|
|
273
|
+
*,
|
|
274
|
+
raise_on_error: bool | None = None,
|
|
275
|
+
) -> httpx.Response | None:
|
|
276
|
+
"""
|
|
277
|
+
Perform an HTTP GET request using the configured transport.
|
|
278
|
+
|
|
279
|
+
The default transport retries connection setup (connect errors and
|
|
280
|
+
connect timeouts) up to 3 times. It does not retry read errors, read
|
|
281
|
+
timeouts or HTTP status codes, and any response, including 4xx and
|
|
282
|
+
5xx, is returned as is. For status code-based retries (429, 500,
|
|
283
|
+
etc.), override http_retry_transport in load(), for example with the
|
|
284
|
+
httpx-retries library.
|
|
285
|
+
|
|
286
|
+
Args:
|
|
287
|
+
url (str): The URL to make the GET request to
|
|
288
|
+
headers (Optional[HeaderTypes]): HTTP headers to include in the request. Supports dict, Headers object, or list of tuples. These headers will override/add to any client-level headers set in the load() method.
|
|
289
|
+
params (Optional[QueryParamTypes]): Query parameters to include in the request. Supports dict, list of tuples, or string.
|
|
290
|
+
auth (Optional[AuthTypes]): Authentication to use for the request. Supports BasicAuth, DigestAuth, custom auth classes, or tuples for basic auth.
|
|
291
|
+
timeout (int): Request timeout in seconds. Defaults to 10.
|
|
292
|
+
raise_on_error (Optional[bool]): When True, a request the source
|
|
293
|
+
did not answer (network failure, timeout, dropped connection)
|
|
294
|
+
raises HttpClientSourceUnavailableError, and any other exception,
|
|
295
|
+
including httpx.PoolTimeout, propagates unchanged.
|
|
296
|
+
When False, every exception is logged and None is returned.
|
|
297
|
+
Leaving it unset behaves like False and emits a
|
|
298
|
+
DeprecationWarning: the default becomes True in v4.0.
|
|
299
|
+
|
|
300
|
+
Returns:
|
|
301
|
+
Optional[httpx.Response]: The HTTP response. None if the request
|
|
302
|
+
failed and raise_on_error is not True.
|
|
303
|
+
|
|
304
|
+
Raises:
|
|
305
|
+
HttpClientSourceUnavailableError: A SourceUnavailableError, if
|
|
306
|
+
raise_on_error is True and the source did not answer. The
|
|
307
|
+
httpx error is carried as ``cause``.
|
|
308
|
+
|
|
309
|
+
Example:
|
|
310
|
+
>>> # Using Basic Authentication
|
|
311
|
+
>>> from httpx import BasicAuth
|
|
312
|
+
>>> response = await client.execute_http_get_request(
|
|
313
|
+
... url="https://api.example.com/data",
|
|
314
|
+
... auth=BasicAuth("username", "password"),
|
|
315
|
+
... params={"limit": 100},
|
|
316
|
+
... raise_on_error=True,
|
|
317
|
+
... )
|
|
318
|
+
>>>
|
|
319
|
+
>>> # Using tuple for basic auth (username, password)
|
|
320
|
+
>>> response = await client.execute_http_get_request(
|
|
321
|
+
... url="https://api.example.com/data",
|
|
322
|
+
... auth=("username", "password"),
|
|
323
|
+
... params={"limit": 100},
|
|
324
|
+
... raise_on_error=True,
|
|
325
|
+
... )
|
|
326
|
+
>>>
|
|
327
|
+
>>> # Using custom headers for Bearer token
|
|
328
|
+
>>> response = await client.execute_http_get_request(
|
|
329
|
+
... url="https://api.example.com/data",
|
|
330
|
+
... headers={"Authorization": "Bearer token"},
|
|
331
|
+
... params={"limit": 100},
|
|
332
|
+
... raise_on_error=True,
|
|
333
|
+
... )
|
|
334
|
+
"""
|
|
335
|
+
_warn_if_raise_on_error_unset("execute_http_get_request", raise_on_error)
|
|
336
|
+
ssl_context = get_ssl_context()
|
|
337
|
+
transport = await self._borrow_transport()
|
|
338
|
+
async with httpx.AsyncClient(
|
|
339
|
+
timeout=httpx.Timeout(timeout, pool=_HTTP_POOL_TIMEOUT_SECONDS),
|
|
340
|
+
transport=transport,
|
|
341
|
+
verify=ssl_context,
|
|
342
|
+
) as client:
|
|
343
|
+
merged_headers = Headers(self.http_headers)
|
|
344
|
+
if headers:
|
|
345
|
+
merged_headers.update(headers)
|
|
346
|
+
|
|
347
|
+
try:
|
|
348
|
+
response = await client.get(
|
|
349
|
+
url,
|
|
350
|
+
headers=merged_headers,
|
|
351
|
+
params=params,
|
|
352
|
+
auth=auth if auth is not None else httpx.USE_CLIENT_DEFAULT,
|
|
353
|
+
)
|
|
354
|
+
return response
|
|
355
|
+
except Exception as exc: # conformance: ignore[E004] _request_failed logs with exc_info or raises typed
|
|
356
|
+
# conformance: ignore[E007] deprecated None-on-failure contract until v4.0
|
|
357
|
+
return _request_failed("GET", url, exc, raise_on_error)
|
|
358
|
+
|
|
359
|
+
@overload
|
|
360
|
+
async def execute_http_post_request(
|
|
361
|
+
self,
|
|
362
|
+
url: str,
|
|
363
|
+
data: RequestData | None = None,
|
|
364
|
+
json_data: Any | None = None,
|
|
365
|
+
content: bytes | None = None,
|
|
366
|
+
files: RequestFiles | None = None,
|
|
367
|
+
headers: HeaderTypes | None = None,
|
|
368
|
+
params: QueryParamTypes | None = None,
|
|
369
|
+
cookies: dict[str, str] | None = None,
|
|
370
|
+
auth: AuthTypes | None = None,
|
|
371
|
+
follow_redirects: bool = True,
|
|
372
|
+
timeout: int = 30,
|
|
373
|
+
*,
|
|
374
|
+
raise_on_error: Literal[True],
|
|
375
|
+
) -> httpx.Response: ...
|
|
376
|
+
|
|
377
|
+
@overload
|
|
378
|
+
async def execute_http_post_request(
|
|
379
|
+
self,
|
|
380
|
+
url: str,
|
|
381
|
+
data: RequestData | None = None,
|
|
382
|
+
json_data: Any | None = None,
|
|
383
|
+
content: bytes | None = None,
|
|
384
|
+
files: RequestFiles | None = None,
|
|
385
|
+
headers: HeaderTypes | None = None,
|
|
386
|
+
params: QueryParamTypes | None = None,
|
|
387
|
+
cookies: dict[str, str] | None = None,
|
|
388
|
+
auth: AuthTypes | None = None,
|
|
389
|
+
follow_redirects: bool = True,
|
|
390
|
+
timeout: int = 30,
|
|
391
|
+
*,
|
|
392
|
+
raise_on_error: bool | None = None,
|
|
393
|
+
) -> httpx.Response | None: ...
|
|
394
|
+
|
|
395
|
+
async def execute_http_post_request(
|
|
396
|
+
self,
|
|
397
|
+
url: str,
|
|
398
|
+
data: RequestData | None = None,
|
|
399
|
+
json_data: Any | None = None,
|
|
400
|
+
content: bytes | None = None,
|
|
401
|
+
files: RequestFiles | None = None,
|
|
402
|
+
headers: HeaderTypes | None = None,
|
|
403
|
+
params: QueryParamTypes | None = None,
|
|
404
|
+
cookies: dict[str, str] | None = None,
|
|
405
|
+
auth: AuthTypes | None = None,
|
|
406
|
+
follow_redirects: bool = True,
|
|
407
|
+
timeout: int = 30,
|
|
408
|
+
*,
|
|
409
|
+
raise_on_error: bool | None = None,
|
|
410
|
+
) -> httpx.Response | None:
|
|
411
|
+
"""
|
|
412
|
+
Perform an HTTP POST request using the configured transport.
|
|
413
|
+
|
|
414
|
+
The default transport retries connection setup (connect errors and
|
|
415
|
+
connect timeouts) up to 3 times; no request bytes have been sent at
|
|
416
|
+
that point, so the retry is safe for POST. It does not retry read
|
|
417
|
+
errors, read timeouts or HTTP status codes, and any response,
|
|
418
|
+
including 4xx and 5xx, is returned as is. For status code-based
|
|
419
|
+
retries (429, 500, etc.), override http_retry_transport in load(),
|
|
420
|
+
for example with the httpx-retries library.
|
|
421
|
+
|
|
422
|
+
Args:
|
|
423
|
+
url (str): The URL to make the POST request to
|
|
424
|
+
data (Optional[RequestData]): Form data to send in the request body. Supports dict, list of tuples, or other httpx-compatible formats.
|
|
425
|
+
json_data (Optional[Any]): JSON data to send in the request body. Any JSON-serializable object.
|
|
426
|
+
content (Optional[bytes]): Raw binary content to send in the request body
|
|
427
|
+
files (Optional[RequestFiles]): Files to upload in the request body. Supports various file formats and tuples.
|
|
428
|
+
headers (Optional[HeaderTypes]): HTTP headers to include in the request. Supports dict, Headers object, or list of tuples. These headers will override/add to any client-level headers set in the load() method.
|
|
429
|
+
params (Optional[QueryParamTypes]): Query parameters to include in the request. Supports dict, list of tuples, or string.
|
|
430
|
+
cookies (Optional[Dict[str, str]]): Cookies to include in the request
|
|
431
|
+
auth (Optional[AuthTypes]): Authentication to use for the request. Supports BasicAuth, DigestAuth, custom auth classes, or tuples for basic auth.
|
|
432
|
+
follow_redirects (bool): Whether to follow HTTP redirects. Defaults to True.
|
|
433
|
+
timeout (int): Request timeout in seconds. Defaults to 30.
|
|
434
|
+
raise_on_error (Optional[bool]): When True, a request the source
|
|
435
|
+
did not answer (network failure, timeout, dropped connection)
|
|
436
|
+
raises HttpClientSourceUnavailableError, and any other exception,
|
|
437
|
+
including httpx.PoolTimeout, propagates unchanged.
|
|
438
|
+
When False, every exception is logged and None is returned.
|
|
439
|
+
Leaving it unset behaves like False and emits a
|
|
440
|
+
DeprecationWarning: the default becomes True in v4.0.
|
|
441
|
+
|
|
442
|
+
Returns:
|
|
443
|
+
Optional[httpx.Response]: The HTTP response. None if the request
|
|
444
|
+
failed and raise_on_error is not True.
|
|
445
|
+
|
|
446
|
+
Raises:
|
|
447
|
+
HttpClientSourceUnavailableError: A SourceUnavailableError, if
|
|
448
|
+
raise_on_error is True and the source did not answer. The
|
|
449
|
+
httpx error is carried as ``cause``.
|
|
450
|
+
|
|
451
|
+
Example:
|
|
452
|
+
>>> # Basic JSON POST request with authentication
|
|
453
|
+
>>> from httpx import BasicAuth
|
|
454
|
+
>>> response = await client.execute_http_post_request(
|
|
455
|
+
... url="https://api.example.com/data",
|
|
456
|
+
... json_data={"name": "test", "value": 123},
|
|
457
|
+
... headers={"Content-Type": "application/json"},
|
|
458
|
+
... auth=BasicAuth("username", "password"),
|
|
459
|
+
... raise_on_error=True,
|
|
460
|
+
... )
|
|
461
|
+
>>>
|
|
462
|
+
>>> # File upload with basic auth tuple
|
|
463
|
+
>>> with open("file.txt", "rb") as f:
|
|
464
|
+
... response = await client.execute_http_post_request(
|
|
465
|
+
... url="https://api.example.com/upload",
|
|
466
|
+
... data={"description": "My file"},
|
|
467
|
+
... files={"file": ("file.txt", f.read(), "text/plain")},
|
|
468
|
+
... auth=("username", "password"),
|
|
469
|
+
... raise_on_error=True,
|
|
470
|
+
... )
|
|
471
|
+
"""
|
|
472
|
+
_warn_if_raise_on_error_unset("execute_http_post_request", raise_on_error)
|
|
473
|
+
ssl_context = get_ssl_context()
|
|
474
|
+
transport = await self._borrow_transport()
|
|
475
|
+
async with httpx.AsyncClient(
|
|
476
|
+
timeout=httpx.Timeout(timeout, pool=_HTTP_POOL_TIMEOUT_SECONDS),
|
|
477
|
+
transport=transport,
|
|
478
|
+
verify=ssl_context,
|
|
479
|
+
) as client:
|
|
480
|
+
merged_headers = Headers(self.http_headers)
|
|
481
|
+
if headers:
|
|
482
|
+
merged_headers.update(headers)
|
|
483
|
+
|
|
484
|
+
try:
|
|
485
|
+
response = await client.post(
|
|
486
|
+
url,
|
|
487
|
+
data=data,
|
|
488
|
+
json=json_data,
|
|
489
|
+
content=content,
|
|
490
|
+
files=files,
|
|
491
|
+
headers=merged_headers,
|
|
492
|
+
params=params,
|
|
493
|
+
cookies=cookies,
|
|
494
|
+
auth=auth if auth is not None else httpx.USE_CLIENT_DEFAULT,
|
|
495
|
+
follow_redirects=follow_redirects,
|
|
496
|
+
)
|
|
497
|
+
return response
|
|
498
|
+
except Exception as exc: # conformance: ignore[E004] _request_failed logs with exc_info or raises typed
|
|
499
|
+
# conformance: ignore[E007] deprecated None-on-failure contract until v4.0
|
|
500
|
+
return _request_failed("POST", url, exc, raise_on_error)
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _warn_if_raise_on_error_unset(
|
|
504
|
+
method_name: str, raise_on_error: bool | None
|
|
505
|
+
) -> None:
|
|
506
|
+
"""Warn a caller still relying on the deprecated ``None``-on-failure default."""
|
|
507
|
+
if raise_on_error is None:
|
|
508
|
+
warnings.warn(
|
|
509
|
+
f"BaseClient.{method_name} returns None when a request fails unless "
|
|
510
|
+
"raise_on_error=True is passed, which raises SourceUnavailableError "
|
|
511
|
+
"instead. The default becomes raise_on_error=True in v4.0; pass "
|
|
512
|
+
"raise_on_error=True (or False to keep None) explicitly.",
|
|
513
|
+
DeprecationWarning,
|
|
514
|
+
stacklevel=3,
|
|
515
|
+
)
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def _endpoint(url: str) -> str:
|
|
519
|
+
"""The request URL's scheme, host and port only.
|
|
520
|
+
|
|
521
|
+
Userinfo, query and fragment can carry secrets, and so can a path segment
|
|
522
|
+
(``/keys/<token>/``), which no redaction pattern recognises. Every failure
|
|
523
|
+
this names got no response from the source, so the host is what locates
|
|
524
|
+
it; the caller's own frames say which call it was.
|
|
525
|
+
"""
|
|
526
|
+
try:
|
|
527
|
+
parsed = httpx.URL(url)
|
|
528
|
+
if not parsed.host:
|
|
529
|
+
return "<invalid url>"
|
|
530
|
+
return str(httpx.URL(scheme=parsed.scheme, host=parsed.host, port=parsed.port))
|
|
531
|
+
except httpx.InvalidURL:
|
|
532
|
+
# No exc_info and no URL: the unparsed URL may hold a credential.
|
|
533
|
+
logger.debug("Request URL could not be parsed; logged as <invalid url>")
|
|
534
|
+
return "<invalid url>"
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
def _request_failed(
|
|
538
|
+
method: str, url: str, exc: Exception, raise_on_error: bool | None
|
|
539
|
+
) -> None:
|
|
540
|
+
"""Handle an exception from an HTTP helper request.
|
|
541
|
+
|
|
542
|
+
With ``raise_on_error``, a failure that :func:`classify_http_exception`
|
|
543
|
+
attributes to the source (a network failure, a timeout, a dropped
|
|
544
|
+
connection) becomes :class:`HttpClientSourceUnavailableError`; anything else,
|
|
545
|
+
including ``httpx.PoolTimeout``, an unsupported scheme and a redirect
|
|
546
|
+
loop, is re-raised unchanged. Without it, the exception is logged and the
|
|
547
|
+
helper returns ``None``.
|
|
548
|
+
|
|
549
|
+
The log carries the full traceback through ``exc_info``. The logger
|
|
550
|
+
redacts URL userinfo, query and fragment values in it, in the message,
|
|
551
|
+
the chained causes and the source lines, before anything leaves.
|
|
552
|
+
"""
|
|
553
|
+
endpoint = _endpoint(url)
|
|
554
|
+
if raise_on_error:
|
|
555
|
+
if classify_http_exception(exc) is SourceUnavailableError:
|
|
556
|
+
network_error = type(exc).__name__
|
|
557
|
+
raise HttpClientSourceUnavailableError(
|
|
558
|
+
message=f"HTTP {method} {endpoint} failed: {network_error}",
|
|
559
|
+
endpoint=endpoint,
|
|
560
|
+
network_error=network_error,
|
|
561
|
+
cause=exc,
|
|
562
|
+
) from exc
|
|
563
|
+
raise exc
|
|
564
|
+
logger.error("Request failed for %s %s", method, endpoint, exc_info=exc)
|
|
565
|
+
return None
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Typed error leaves for the BaseClient HTTP helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import ClassVar
|
|
7
|
+
|
|
8
|
+
from application_sdk.errors.leaves import SourceUnavailableError
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass(kw_only=True)
|
|
12
|
+
class HttpClientSourceUnavailableError(SourceUnavailableError):
|
|
13
|
+
"""An HTTP helper request got no response from the source.
|
|
14
|
+
|
|
15
|
+
A network failure, a timeout or a dropped connection: the request never
|
|
16
|
+
produced an HTTP status, so this is not a 4xx or 5xx from the source.
|
|
17
|
+
Raised by ``BaseClient.execute_http_get_request`` and
|
|
18
|
+
``execute_http_post_request`` when called with ``raise_on_error=True``.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
code: ClassVar[str] = "SOURCE_UNAVAILABLE_HTTP_CLIENT"
|
|
22
|
+
message: str = "HTTP client could not reach the source"
|
|
23
|
+
suggested_action: str | None = (
|
|
24
|
+
"Check DNS resolution, firewall and private-link rules, TLS "
|
|
25
|
+
"configuration, and that the source endpoint is up and accepting "
|
|
26
|
+
"connections from the worker network."
|
|
27
|
+
)
|
|
28
|
+
source_type: str | None = "http"
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Pydantic models for database client configurations.
|
|
3
|
+
This module provides Pydantic models for database connection configurations,
|
|
4
|
+
ensuring type safety and validation for database client settings.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from typing import Any, Dict, List, Optional
|
|
8
|
+
|
|
9
|
+
from pydantic import BaseModel, Field
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class DatabaseConfig(BaseModel):
|
|
13
|
+
"""
|
|
14
|
+
Pydantic model for database connection configuration.
|
|
15
|
+
This model defines the structure for database connection configurations,
|
|
16
|
+
including connection templates, required parameters, defaults, and additional
|
|
17
|
+
connection parameters.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
template: str = Field(
|
|
21
|
+
...,
|
|
22
|
+
description="SQLAlchemy connection string template with placeholders for connection parameters",
|
|
23
|
+
)
|
|
24
|
+
required: List[str] = Field(
|
|
25
|
+
default=[],
|
|
26
|
+
description="List of required connection parameters that must be provided",
|
|
27
|
+
)
|
|
28
|
+
defaults: Optional[Dict[str, Any]] = Field(
|
|
29
|
+
default=None,
|
|
30
|
+
description="Default connection parameters to be added to the connection string",
|
|
31
|
+
)
|
|
32
|
+
parameters: Optional[List[str]] = Field(
|
|
33
|
+
default=None,
|
|
34
|
+
description="List of additional connection parameter names that can be dynamically added from credentials to the connection string. ex: ['ssl_mode'] will be added to the connection string as ?ssl_mode=require",
|
|
35
|
+
)
|
|
36
|
+
connect_args: Dict[str, Any] = Field(
|
|
37
|
+
default_factory=dict,
|
|
38
|
+
description="Additional connection arguments to be passed to SQLAlchemy. ex: {'sslmode': 'require'}",
|
|
39
|
+
)
|
|
40
|
+
pool_pre_ping: bool = Field(
|
|
41
|
+
default=True,
|
|
42
|
+
description="Whether SQLAlchemy should test pooled connections for liveness before checkout. Disable for dialects or endpoints where the driver's ping/close path is unsafe.",
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
model_config = {
|
|
46
|
+
"extra": "forbid",
|
|
47
|
+
"validate_assignment": True,
|
|
48
|
+
"use_enum_values": True,
|
|
49
|
+
}
|