ccflow-http 0.1.0__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ccflow_http-0.1.0 → ccflow_http-0.1.1}/.gitignore +2 -2
- {ccflow_http-0.1.0 → ccflow_http-0.1.1}/PKG-INFO +2 -3
- ccflow_http-0.1.1/ccflow_http/__init__.py +3 -0
- {ccflow_http-0.1.0 → ccflow_http-0.1.1}/ccflow_http/base.py +94 -68
- {ccflow_http-0.1.0 → ccflow_http-0.1.1}/ccflow_http/tests/test_all.py +1 -1
- {ccflow_http-0.1.0 → ccflow_http-0.1.1}/ccflow_http/tests/test_model.py +134 -0
- {ccflow_http-0.1.0 → ccflow_http-0.1.1}/pyproject.toml +3 -4
- ccflow_http-0.1.0/ccflow_http/__init__.py +0 -3
- {ccflow_http-0.1.0 → ccflow_http-0.1.1}/LICENSE +0 -0
- {ccflow_http-0.1.0 → ccflow_http-0.1.1}/README.md +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ccflow-http
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: ccflow models for HTTP
|
|
5
5
|
Project-URL: Repository, https://github.com/1kbgz/ccflow-http
|
|
6
6
|
Project-URL: Homepage, https://github.com/1kbgz/ccflow-http
|
|
@@ -10,14 +10,13 @@ License-File: LICENSE
|
|
|
10
10
|
Classifier: Development Status :: 3 - Alpha
|
|
11
11
|
Classifier: Programming Language :: Python
|
|
12
12
|
Classifier: Programming Language :: Python :: 3
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
14
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
15
14
|
Classifier: Programming Language :: Python :: 3.12
|
|
16
15
|
Classifier: Programming Language :: Python :: 3.13
|
|
17
16
|
Classifier: Programming Language :: Python :: 3.14
|
|
18
17
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
19
18
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
20
|
-
Requires-Python: >=3.
|
|
19
|
+
Requires-Python: >=3.11
|
|
21
20
|
Requires-Dist: ccflow-etl>=0.1.0
|
|
22
21
|
Requires-Dist: ccflow>=0.8.5
|
|
23
22
|
Requires-Dist: httpx
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
from base64 import b64encode
|
|
2
2
|
from csv import DictReader
|
|
3
|
+
from datetime import UTC, datetime
|
|
4
|
+
from email.utils import parsedate_to_datetime
|
|
3
5
|
from gzip import decompress
|
|
4
6
|
from io import StringIO
|
|
5
7
|
from time import monotonic, sleep
|
|
6
|
-
from typing import Any,
|
|
8
|
+
from typing import Any, Literal
|
|
7
9
|
from urllib.parse import parse_qsl, urlsplit, urlunsplit
|
|
8
10
|
|
|
9
11
|
import httpx
|
|
@@ -14,15 +16,15 @@ from jinja2 import Environment
|
|
|
14
16
|
from pydantic import Field
|
|
15
17
|
|
|
16
18
|
__all__ = (
|
|
17
|
-
"HTTPConfig",
|
|
18
19
|
"HTTPAuth",
|
|
20
|
+
"HTTPConfig",
|
|
19
21
|
"HTTPContext",
|
|
20
|
-
"
|
|
22
|
+
"HTTPModel",
|
|
21
23
|
"HTTPRequest",
|
|
22
|
-
"
|
|
24
|
+
"HTTPRequestContext",
|
|
23
25
|
"HTTPResponseResult",
|
|
24
26
|
"HTTPResult",
|
|
25
|
-
"
|
|
27
|
+
"HTTPRetryPolicy",
|
|
26
28
|
"redact_mapping",
|
|
27
29
|
"safe_request_dump",
|
|
28
30
|
)
|
|
@@ -38,27 +40,27 @@ class HTTPConfig(BaseModel):
|
|
|
38
40
|
base_url: str = ""
|
|
39
41
|
timeout: float = 30.0
|
|
40
42
|
follow_redirects: bool = True
|
|
41
|
-
headers:
|
|
42
|
-
transport:
|
|
43
|
+
headers: dict[str, str] = Field(default_factory=dict)
|
|
44
|
+
transport: Any | None = None
|
|
43
45
|
|
|
44
46
|
|
|
45
47
|
class HTTPAuth(BaseModel):
|
|
46
48
|
strategy: HTTPAuthStrategy = "none"
|
|
47
|
-
token:
|
|
48
|
-
name:
|
|
49
|
-
value:
|
|
50
|
-
username:
|
|
51
|
-
password:
|
|
49
|
+
token: str | None = None
|
|
50
|
+
name: str | None = None
|
|
51
|
+
value: str | None = None
|
|
52
|
+
username: str | None = None
|
|
53
|
+
password: str | None = None
|
|
52
54
|
scheme: str = "Bearer"
|
|
53
55
|
|
|
54
56
|
|
|
55
57
|
class HTTPContext(ContextBase):
|
|
56
|
-
path:
|
|
57
|
-
query:
|
|
58
|
-
headers:
|
|
59
|
-
template_values:
|
|
60
|
-
json_body:
|
|
61
|
-
content:
|
|
58
|
+
path: str | None = None
|
|
59
|
+
query: dict[str, Any] = Field(default_factory=dict)
|
|
60
|
+
headers: dict[str, str] = Field(default_factory=dict)
|
|
61
|
+
template_values: dict[str, Any] = Field(default_factory=dict)
|
|
62
|
+
json_body: Any | None = None
|
|
63
|
+
content: bytes | str | None = None
|
|
62
64
|
|
|
63
65
|
|
|
64
66
|
class HTTPRequestContext(HTTPContext): ...
|
|
@@ -67,13 +69,13 @@ class HTTPRequestContext(HTTPContext): ...
|
|
|
67
69
|
class HTTPRequest(BaseModel):
|
|
68
70
|
method: HTTPMethod
|
|
69
71
|
url: str
|
|
70
|
-
params:
|
|
71
|
-
headers:
|
|
72
|
-
json_data:
|
|
73
|
-
content:
|
|
72
|
+
params: dict[str, Any] = Field(default_factory=dict)
|
|
73
|
+
headers: dict[str, str] = Field(default_factory=dict)
|
|
74
|
+
json_data: Any | None = None
|
|
75
|
+
content: bytes | str | None = None
|
|
74
76
|
|
|
75
77
|
|
|
76
|
-
def redact_mapping(values:
|
|
78
|
+
def redact_mapping(values: dict[str, Any]) -> dict[str, Any]:
|
|
77
79
|
redacted = {}
|
|
78
80
|
for key, value in values.items():
|
|
79
81
|
normalized_key = key.lower().replace("_", "").replace("-", "")
|
|
@@ -84,7 +86,7 @@ def redact_mapping(values: Dict[str, Any]) -> Dict[str, Any]:
|
|
|
84
86
|
return redacted
|
|
85
87
|
|
|
86
88
|
|
|
87
|
-
def safe_request_dump(request: HTTPRequest) ->
|
|
89
|
+
def safe_request_dump(request: HTTPRequest) -> dict[str, Any]:
|
|
88
90
|
request_data = request.model_dump(exclude={"type_"})
|
|
89
91
|
request_data["params"] = redact_mapping(request_data.get("params", {}))
|
|
90
92
|
request_data["headers"] = redact_mapping(request_data.get("headers", {}))
|
|
@@ -95,33 +97,54 @@ class HTTPRetryEvent(BaseModel):
|
|
|
95
97
|
attempt: int
|
|
96
98
|
outcome: HTTPRetryOutcome
|
|
97
99
|
delay_seconds: float = 0.0
|
|
98
|
-
status_code:
|
|
99
|
-
exception_type:
|
|
100
|
-
category:
|
|
101
|
-
message:
|
|
100
|
+
status_code: int | None = None
|
|
101
|
+
exception_type: str | None = None
|
|
102
|
+
category: str | None = None
|
|
103
|
+
message: str | None = None
|
|
102
104
|
|
|
103
105
|
|
|
104
106
|
class HTTPRetryPolicy(RetryPolicy):
|
|
105
|
-
retry_status_codes:
|
|
106
|
-
retry_exceptions:
|
|
107
|
+
retry_status_codes: list[int] = Field(default_factory=lambda: [429, 500, 502, 503, 504])
|
|
108
|
+
retry_exceptions: list[PyObjectPath] = Field(
|
|
107
109
|
default_factory=lambda: [PyObjectPath.validate(httpx.TimeoutException), PyObjectPath.validate(httpx.ConnectError)]
|
|
108
110
|
)
|
|
109
|
-
timeout_exception_types:
|
|
111
|
+
timeout_exception_types: list[str] = Field(
|
|
110
112
|
default_factory=lambda: ["TimeoutError", "TimeoutException", "ConnectTimeout", "ReadTimeout", "WriteTimeout", "PoolTimeout"]
|
|
111
113
|
)
|
|
114
|
+
retry_after_max: float | None = Field(
|
|
115
|
+
default=300.0,
|
|
116
|
+
ge=0.0,
|
|
117
|
+
description="Cap (in seconds) on honoring a server-provided Retry-After header. None honors it unbounded.",
|
|
118
|
+
)
|
|
112
119
|
|
|
113
|
-
def should_retry_status(self, status_code:
|
|
120
|
+
def should_retry_status(self, status_code: int | None, attempt: int) -> bool:
|
|
114
121
|
return status_code in self.retry_status_codes and attempt < self.max_attempts
|
|
115
122
|
|
|
123
|
+
def retry_after_seconds(self, headers: dict[str, str]) -> float | None:
|
|
124
|
+
value = next((v for k, v in headers.items() if k.lower() == "retry-after"), None)
|
|
125
|
+
if value is None:
|
|
126
|
+
return None
|
|
127
|
+
try:
|
|
128
|
+
delay_seconds = float(value)
|
|
129
|
+
except ValueError:
|
|
130
|
+
try:
|
|
131
|
+
delay_seconds = (parsedate_to_datetime(value) - datetime.now(UTC)).total_seconds()
|
|
132
|
+
except (TypeError, ValueError):
|
|
133
|
+
return None
|
|
134
|
+
delay_seconds = max(delay_seconds, 0.0)
|
|
135
|
+
if self.retry_after_max is not None:
|
|
136
|
+
delay_seconds = min(delay_seconds, self.retry_after_max)
|
|
137
|
+
return delay_seconds
|
|
138
|
+
|
|
116
139
|
def should_retry_exception(self, exception: BaseException, attempt: int) -> bool:
|
|
117
140
|
return self._should_retry(exception) and attempt < self.max_attempts
|
|
118
141
|
|
|
119
|
-
def delay_seconds(self, attempt: int, jitter_value:
|
|
142
|
+
def delay_seconds(self, attempt: int, jitter_value: float | None = None) -> float:
|
|
120
143
|
if attempt < 1:
|
|
121
144
|
raise ValueError("attempt must be greater than or equal to 1")
|
|
122
145
|
return self._compute_delay(attempt, jitter_value=jitter_value)
|
|
123
146
|
|
|
124
|
-
def retry_delay_seconds(self, attempt: int, total_wait_seconds: float) ->
|
|
147
|
+
def retry_delay_seconds(self, attempt: int, total_wait_seconds: float) -> float | None:
|
|
125
148
|
delay_seconds = self.delay_seconds(attempt)
|
|
126
149
|
if self.max_delay is not None and total_wait_seconds + delay_seconds > self.max_delay:
|
|
127
150
|
return None
|
|
@@ -136,7 +159,7 @@ class HTTPRetryPolicy(RetryPolicy):
|
|
|
136
159
|
return "connection"
|
|
137
160
|
return "exception"
|
|
138
161
|
|
|
139
|
-
def status_category(self, status_code:
|
|
162
|
+
def status_category(self, status_code: int | None) -> str:
|
|
140
163
|
if status_code == 429:
|
|
141
164
|
return "rate_limit"
|
|
142
165
|
if status_code == 408:
|
|
@@ -148,48 +171,48 @@ class HTTPRetryPolicy(RetryPolicy):
|
|
|
148
171
|
|
|
149
172
|
class HTTPResponseResult(GenericResult[Any]):
|
|
150
173
|
status_code: int
|
|
151
|
-
headers:
|
|
174
|
+
headers: dict[str, str] = Field(default_factory=dict)
|
|
152
175
|
url: str = ""
|
|
153
176
|
attempts: int = 1
|
|
154
177
|
pages: int = 1
|
|
155
|
-
rate_limit:
|
|
156
|
-
retry_events:
|
|
157
|
-
retry_summary:
|
|
178
|
+
rate_limit: dict[str, str] = Field(default_factory=dict)
|
|
179
|
+
retry_events: list[dict[str, Any]] = Field(default_factory=list)
|
|
180
|
+
retry_summary: dict[str, int] = Field(default_factory=dict)
|
|
158
181
|
|
|
159
182
|
|
|
160
183
|
class HTTPResult(HTTPResponseResult): ...
|
|
161
184
|
|
|
162
185
|
|
|
163
186
|
class HTTPModel(CallableModel):
|
|
164
|
-
config:
|
|
187
|
+
config: HTTPConfig | None = None
|
|
165
188
|
auth: HTTPAuth = Field(default_factory=HTTPAuth)
|
|
166
189
|
method: HTTPMethod = "GET"
|
|
167
190
|
base_url: str = ""
|
|
168
191
|
path: str = ""
|
|
169
|
-
query:
|
|
170
|
-
headers:
|
|
192
|
+
query: dict[str, Any] = Field(default_factory=dict)
|
|
193
|
+
headers: dict[str, str] = Field(default_factory=dict)
|
|
171
194
|
timeout: float = 30.0
|
|
172
195
|
follow_redirects: bool = True
|
|
173
196
|
response_format: ResponseFormat = "json"
|
|
174
|
-
json_body:
|
|
175
|
-
content:
|
|
197
|
+
json_body: Any | None = None
|
|
198
|
+
content: bytes | str | None = None
|
|
176
199
|
max_attempts: int = 1
|
|
177
|
-
retry_status_codes:
|
|
178
|
-
retry_policy:
|
|
179
|
-
execution_policy:
|
|
200
|
+
retry_status_codes: list[int] = Field(default_factory=lambda: [429, 500, 502, 503, 504])
|
|
201
|
+
retry_policy: HTTPRetryPolicy | None = None
|
|
202
|
+
execution_policy: ExecutionPolicy | None = None
|
|
180
203
|
paginate: bool = False
|
|
181
204
|
max_pages: int = 100
|
|
182
205
|
pagination_mode: HTTPPaginationMode = "next_url"
|
|
183
206
|
next_url_field: str = "next_url"
|
|
184
207
|
next_cursor_field: str = "next_cursor"
|
|
185
208
|
cursor_param: str = "cursor"
|
|
186
|
-
cursor_start:
|
|
209
|
+
cursor_start: str | None = None
|
|
187
210
|
page_param: str = "page"
|
|
188
211
|
page_start: int = 1
|
|
189
212
|
offset_param: str = "offset"
|
|
190
213
|
offset_start: int = 0
|
|
191
214
|
limit_param: str = "limit"
|
|
192
|
-
limit:
|
|
215
|
+
limit: int | None = None
|
|
193
216
|
results_field: str = "results"
|
|
194
217
|
|
|
195
218
|
@property
|
|
@@ -200,25 +223,25 @@ class HTTPModel(CallableModel):
|
|
|
200
223
|
def result_type(self):
|
|
201
224
|
return HTTPResult
|
|
202
225
|
|
|
203
|
-
def _template_data(self, context: HTTPContext) ->
|
|
226
|
+
def _template_data(self, context: HTTPContext) -> dict[str, Any]:
|
|
204
227
|
data = context.model_dump(exclude_none=True)
|
|
205
228
|
data.update(context.template_values)
|
|
206
229
|
return data
|
|
207
230
|
|
|
208
|
-
def _render(self, value: Any, data:
|
|
231
|
+
def _render(self, value: Any, data: dict[str, Any]) -> Any:
|
|
209
232
|
if isinstance(value, str):
|
|
210
233
|
return Environment().from_string(value).render(**data)
|
|
211
234
|
return value
|
|
212
235
|
|
|
213
|
-
def _render_mapping(self, values:
|
|
236
|
+
def _render_mapping(self, values: dict[str, Any], data: dict[str, Any]) -> dict[str, Any]:
|
|
214
237
|
return {key: self._render(value, data) for key, value in values.items() if value is not None}
|
|
215
238
|
|
|
216
|
-
def _render_required_auth_value(self, value:
|
|
239
|
+
def _render_required_auth_value(self, value: str | None, field_name: str, data: dict[str, Any]) -> str:
|
|
217
240
|
if value is None:
|
|
218
241
|
raise ValueError(f"HTTP auth strategy {self.auth.strategy!r} requires {field_name}.")
|
|
219
242
|
return str(self._render(value, data))
|
|
220
243
|
|
|
221
|
-
def _apply_auth(self, headers:
|
|
244
|
+
def _apply_auth(self, headers: dict[str, str], query: dict[str, Any], data: dict[str, Any]) -> None:
|
|
222
245
|
match self.auth.strategy:
|
|
223
246
|
case "none":
|
|
224
247
|
return
|
|
@@ -236,7 +259,7 @@ class HTTPModel(CallableModel):
|
|
|
236
259
|
case "basic":
|
|
237
260
|
username = self._render_required_auth_value(self.auth.username, "username", data)
|
|
238
261
|
password = self._render_required_auth_value(self.auth.password, "password", data)
|
|
239
|
-
encoded = b64encode(f"{username}:{password}".encode(
|
|
262
|
+
encoded = b64encode(f"{username}:{password}".encode()).decode("ascii")
|
|
240
263
|
headers["Authorization"] = f"Basic {encoded}"
|
|
241
264
|
case _:
|
|
242
265
|
raise ValueError(f"Unsupported HTTP auth strategy: {self.auth.strategy}")
|
|
@@ -254,7 +277,7 @@ class HTTPModel(CallableModel):
|
|
|
254
277
|
return self.config.follow_redirects
|
|
255
278
|
return self.follow_redirects
|
|
256
279
|
|
|
257
|
-
def _client_kwargs(self) ->
|
|
280
|
+
def _client_kwargs(self) -> dict[str, Any]:
|
|
258
281
|
kwargs = {
|
|
259
282
|
"base_url": self._base_url(),
|
|
260
283
|
"timeout": self._timeout(),
|
|
@@ -264,7 +287,7 @@ class HTTPModel(CallableModel):
|
|
|
264
287
|
kwargs["transport"] = self.config.transport
|
|
265
288
|
return kwargs
|
|
266
289
|
|
|
267
|
-
def build_request(self, context:
|
|
290
|
+
def build_request(self, context: HTTPContext | None = None) -> HTTPRequest:
|
|
268
291
|
context = context or HTTPRequestContext()
|
|
269
292
|
data = self._template_data(context)
|
|
270
293
|
|
|
@@ -304,7 +327,7 @@ class HTTPModel(CallableModel):
|
|
|
304
327
|
def _safe_url(self, request: HTTPRequest) -> str:
|
|
305
328
|
return request.url.split("?", 1)[0]
|
|
306
329
|
|
|
307
|
-
def _rate_limit_headers(self, headers:
|
|
330
|
+
def _rate_limit_headers(self, headers: dict[str, str]) -> dict[str, str]:
|
|
308
331
|
rate_limit = {}
|
|
309
332
|
for key, value in headers.items():
|
|
310
333
|
normalized_key = key.lower()
|
|
@@ -322,7 +345,7 @@ class HTTPModel(CallableModel):
|
|
|
322
345
|
def _now(self) -> float:
|
|
323
346
|
return monotonic()
|
|
324
347
|
|
|
325
|
-
def _throttle(self, previous_started_at:
|
|
348
|
+
def _throttle(self, previous_started_at: float | None) -> float:
|
|
326
349
|
now = self._now()
|
|
327
350
|
if self.execution_policy is None:
|
|
328
351
|
return now
|
|
@@ -331,10 +354,10 @@ class HTTPModel(CallableModel):
|
|
|
331
354
|
self._sleep(delay_seconds)
|
|
332
355
|
return now + delay_seconds
|
|
333
356
|
|
|
334
|
-
def _retry_event(self, **values: Any) ->
|
|
357
|
+
def _retry_event(self, **values: Any) -> dict[str, Any]:
|
|
335
358
|
return HTTPRetryEvent(**values).model_dump(exclude={"type_"}, exclude_none=True)
|
|
336
359
|
|
|
337
|
-
def _retry_summary(self, events:
|
|
360
|
+
def _retry_summary(self, events: list[dict[str, Any]], attempts: int, succeeded: bool) -> dict[str, int]:
|
|
338
361
|
return {
|
|
339
362
|
"attempts": attempts,
|
|
340
363
|
"retried": sum(1 for event in events if event["outcome"] == "retry"),
|
|
@@ -351,10 +374,10 @@ class HTTPModel(CallableModel):
|
|
|
351
374
|
return current
|
|
352
375
|
|
|
353
376
|
def _request_once(
|
|
354
|
-
self, client: httpx.Client, request: HTTPRequest, previous_started_at:
|
|
355
|
-
) ->
|
|
377
|
+
self, client: httpx.Client, request: HTTPRequest, previous_started_at: float | None
|
|
378
|
+
) -> tuple[httpx.Response, int, list[dict[str, Any]], float]:
|
|
356
379
|
attempts = 0
|
|
357
|
-
events:
|
|
380
|
+
events: list[dict[str, Any]] = []
|
|
358
381
|
retry_policy = self._retry_policy()
|
|
359
382
|
total_retry_wait_seconds = 0.0
|
|
360
383
|
while True:
|
|
@@ -376,6 +399,9 @@ class HTTPModel(CallableModel):
|
|
|
376
399
|
if retry_policy.should_retry_status(status_code, attempts):
|
|
377
400
|
delay_seconds = retry_policy.retry_delay_seconds(attempts, total_wait_seconds=total_retry_wait_seconds)
|
|
378
401
|
if delay_seconds is not None:
|
|
402
|
+
retry_after = retry_policy.retry_after_seconds(dict(exc.response.headers or {})) if exc.response is not None else None
|
|
403
|
+
if retry_after is not None:
|
|
404
|
+
delay_seconds = max(delay_seconds, retry_after)
|
|
379
405
|
events.append(
|
|
380
406
|
self._retry_event(
|
|
381
407
|
attempt=attempts,
|
|
@@ -430,7 +456,7 @@ class HTTPModel(CallableModel):
|
|
|
430
456
|
except httpx.HTTPError as exc:
|
|
431
457
|
raise RuntimeError(f"HTTP {request.method} {self._safe_url(request)} failed with {type(exc).__name__}") from exc
|
|
432
458
|
|
|
433
|
-
def _merge_page_values(self, values:
|
|
459
|
+
def _merge_page_values(self, values: list[Any]) -> Any:
|
|
434
460
|
if not values:
|
|
435
461
|
return []
|
|
436
462
|
if all(isinstance(value, dict) and isinstance(value.get(self.results_field), list) for value in values):
|
|
@@ -441,11 +467,11 @@ class HTTPModel(CallableModel):
|
|
|
441
467
|
return merged
|
|
442
468
|
return values
|
|
443
469
|
|
|
444
|
-
def _page_items(self, value: Any) ->
|
|
470
|
+
def _page_items(self, value: Any) -> list[Any] | None:
|
|
445
471
|
items = self._extract_field(value, self.results_field)
|
|
446
472
|
return items if isinstance(items, list) else None
|
|
447
473
|
|
|
448
|
-
def _request_with_params(self, request: HTTPRequest, params:
|
|
474
|
+
def _request_with_params(self, request: HTTPRequest, params: dict[str, Any]) -> HTTPRequest:
|
|
449
475
|
return request.model_copy(update={"params": {**request.params, **params}})
|
|
450
476
|
|
|
451
477
|
def _is_sensitive_query_param(self, key: str) -> bool:
|
|
@@ -484,7 +510,7 @@ class HTTPModel(CallableModel):
|
|
|
484
510
|
case _:
|
|
485
511
|
raise ValueError(f"Unsupported pagination mode: {self.pagination_mode}")
|
|
486
512
|
|
|
487
|
-
def _next_paginated_request(self, request: HTTPRequest, value: Any) ->
|
|
513
|
+
def _next_paginated_request(self, request: HTTPRequest, value: Any) -> HTTPRequest | None:
|
|
488
514
|
if self._page_items(value) == []:
|
|
489
515
|
return None
|
|
490
516
|
match self.pagination_mode:
|
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
from base64 import b64decode
|
|
4
4
|
from dataclasses import dataclass
|
|
5
|
+
from datetime import UTC
|
|
5
6
|
from gzip import compress
|
|
6
7
|
from typing import Any
|
|
7
8
|
|
|
@@ -522,3 +523,136 @@ def test_http_model_supports_cursor_page_and_offset_pagination_with_mock_transpo
|
|
|
522
523
|
assert page_result.value["results"] == [{"id": 1}, {"id": 2}]
|
|
523
524
|
assert offset_requests == [{"offset": "0", "limit": "2"}, {"offset": "2", "limit": "2"}, {"offset": "4", "limit": "2"}]
|
|
524
525
|
assert offset_result.value["results"] == [{"id": 1}, {"id": 2}]
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
def _retrying_client(calls, response_factory):
|
|
529
|
+
class FakeClient:
|
|
530
|
+
def __init__(self, **kwargs):
|
|
531
|
+
pass
|
|
532
|
+
|
|
533
|
+
def __enter__(self):
|
|
534
|
+
return self
|
|
535
|
+
|
|
536
|
+
def __exit__(self, exc_type, exc, traceback):
|
|
537
|
+
return False
|
|
538
|
+
|
|
539
|
+
def request(self, **kwargs):
|
|
540
|
+
calls.append(kwargs)
|
|
541
|
+
if len(calls) == 1:
|
|
542
|
+
response_factory()
|
|
543
|
+
return FakeResponse(value={"status": "OK"})
|
|
544
|
+
|
|
545
|
+
return FakeClient
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def _raise_status(status_code, headers=None):
|
|
549
|
+
request = httpx.Request("GET", "https://api.example.test/v1/tickers")
|
|
550
|
+
response = httpx.Response(status_code, request=request, headers=headers or {})
|
|
551
|
+
raise httpx.HTTPStatusError("error", request=request, response=response)
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
def test_http_model_retries_connect_error(monkeypatch):
|
|
555
|
+
calls = []
|
|
556
|
+
|
|
557
|
+
def fail_once():
|
|
558
|
+
raise httpx.ConnectError("connection failed")
|
|
559
|
+
|
|
560
|
+
monkeypatch.setattr("ccflow_http.base.httpx.Client", _retrying_client(calls, fail_once))
|
|
561
|
+
|
|
562
|
+
result = HTTPModel(base_url="https://api.example.test", path="/v1/tickers", max_attempts=2)(HTTPRequestContext())
|
|
563
|
+
|
|
564
|
+
assert len(calls) == 2
|
|
565
|
+
assert result.attempts == 2
|
|
566
|
+
assert result.retry_events[0]["category"] == "connection"
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
def test_http_model_honors_retry_after_seconds_over_backoff(monkeypatch):
|
|
570
|
+
calls = []
|
|
571
|
+
monkeypatch.setattr("ccflow_http.base.httpx.Client", _retrying_client(calls, lambda: _raise_status(429, {"retry-after": "45"})))
|
|
572
|
+
|
|
573
|
+
model = HTTPModel(
|
|
574
|
+
base_url="https://api.example.test",
|
|
575
|
+
path="/v1/tickers",
|
|
576
|
+
retry_policy=HTTPRetryPolicy(max_attempts=2, wait_initial=1.0),
|
|
577
|
+
)
|
|
578
|
+
sleeps = []
|
|
579
|
+
monkeypatch.setattr(model, "_sleep", sleeps.append)
|
|
580
|
+
|
|
581
|
+
result = model(HTTPRequestContext())
|
|
582
|
+
|
|
583
|
+
assert len(calls) == 2
|
|
584
|
+
assert sleeps == [45.0]
|
|
585
|
+
assert result.retry_events[0]["delay_seconds"] == 45.0
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
def test_http_model_uses_backoff_when_it_exceeds_retry_after(monkeypatch):
|
|
589
|
+
calls = []
|
|
590
|
+
monkeypatch.setattr("ccflow_http.base.httpx.Client", _retrying_client(calls, lambda: _raise_status(429, {"retry-after": "1"})))
|
|
591
|
+
|
|
592
|
+
model = HTTPModel(
|
|
593
|
+
base_url="https://api.example.test",
|
|
594
|
+
path="/v1/tickers",
|
|
595
|
+
retry_policy=HTTPRetryPolicy(max_attempts=2, wait_initial=5.0),
|
|
596
|
+
)
|
|
597
|
+
sleeps = []
|
|
598
|
+
monkeypatch.setattr(model, "_sleep", sleeps.append)
|
|
599
|
+
|
|
600
|
+
model(HTTPRequestContext())
|
|
601
|
+
|
|
602
|
+
assert sleeps == [5.0]
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def test_http_model_caps_retry_after_at_retry_after_max(monkeypatch):
|
|
606
|
+
calls = []
|
|
607
|
+
monkeypatch.setattr("ccflow_http.base.httpx.Client", _retrying_client(calls, lambda: _raise_status(429, {"retry-after": "9999"})))
|
|
608
|
+
|
|
609
|
+
model = HTTPModel(
|
|
610
|
+
base_url="https://api.example.test",
|
|
611
|
+
path="/v1/tickers",
|
|
612
|
+
retry_policy=HTTPRetryPolicy(max_attempts=2, wait_initial=1.0, retry_after_max=300.0),
|
|
613
|
+
)
|
|
614
|
+
sleeps = []
|
|
615
|
+
monkeypatch.setattr(model, "_sleep", sleeps.append)
|
|
616
|
+
|
|
617
|
+
model(HTTPRequestContext())
|
|
618
|
+
|
|
619
|
+
assert sleeps == [300.0]
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
def test_http_model_honors_retry_after_http_date(monkeypatch):
|
|
623
|
+
from datetime import datetime, timedelta
|
|
624
|
+
from email.utils import format_datetime
|
|
625
|
+
|
|
626
|
+
calls = []
|
|
627
|
+
retry_at = format_datetime(datetime.now(UTC) + timedelta(seconds=60))
|
|
628
|
+
monkeypatch.setattr("ccflow_http.base.httpx.Client", _retrying_client(calls, lambda: _raise_status(429, {"retry-after": retry_at})))
|
|
629
|
+
|
|
630
|
+
model = HTTPModel(
|
|
631
|
+
base_url="https://api.example.test",
|
|
632
|
+
path="/v1/tickers",
|
|
633
|
+
retry_policy=HTTPRetryPolicy(max_attempts=2, wait_initial=1.0),
|
|
634
|
+
)
|
|
635
|
+
sleeps = []
|
|
636
|
+
monkeypatch.setattr(model, "_sleep", sleeps.append)
|
|
637
|
+
|
|
638
|
+
model(HTTPRequestContext())
|
|
639
|
+
|
|
640
|
+
assert len(sleeps) == 1
|
|
641
|
+
assert 50.0 <= sleeps[0] <= 60.0
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
def test_http_model_ignores_invalid_retry_after(monkeypatch):
|
|
645
|
+
calls = []
|
|
646
|
+
monkeypatch.setattr("ccflow_http.base.httpx.Client", _retrying_client(calls, lambda: _raise_status(429, {"retry-after": "soon"})))
|
|
647
|
+
|
|
648
|
+
model = HTTPModel(
|
|
649
|
+
base_url="https://api.example.test",
|
|
650
|
+
path="/v1/tickers",
|
|
651
|
+
retry_policy=HTTPRetryPolicy(max_attempts=2, wait_initial=1.0),
|
|
652
|
+
)
|
|
653
|
+
sleeps = []
|
|
654
|
+
monkeypatch.setattr(model, "_sleep", sleeps.append)
|
|
655
|
+
|
|
656
|
+
model(HTTPRequestContext())
|
|
657
|
+
|
|
658
|
+
assert sleeps == [1.0]
|
|
@@ -12,8 +12,8 @@ authors = [
|
|
|
12
12
|
description = "ccflow models for HTTP"
|
|
13
13
|
readme = "README.md"
|
|
14
14
|
license = { text = "Apache-2.0" }
|
|
15
|
-
version = "0.1.
|
|
16
|
-
requires-python = ">=3.
|
|
15
|
+
version = "0.1.1"
|
|
16
|
+
requires-python = ">=3.11"
|
|
17
17
|
keywords = []
|
|
18
18
|
|
|
19
19
|
classifiers = [
|
|
@@ -22,7 +22,6 @@ classifiers = [
|
|
|
22
22
|
"Programming Language :: Python :: Implementation :: CPython",
|
|
23
23
|
"Programming Language :: Python :: Implementation :: PyPy",
|
|
24
24
|
"Programming Language :: Python :: 3",
|
|
25
|
-
"Programming Language :: Python :: 3.10",
|
|
26
25
|
"Programming Language :: Python :: 3.11",
|
|
27
26
|
"Programming Language :: Python :: 3.12",
|
|
28
27
|
"Programming Language :: Python :: 3.13",
|
|
@@ -61,7 +60,7 @@ Repository = "https://github.com/1kbgz/ccflow-http"
|
|
|
61
60
|
Homepage = "https://github.com/1kbgz/ccflow-http"
|
|
62
61
|
|
|
63
62
|
[tool.bumpversion]
|
|
64
|
-
current_version = "0.1.
|
|
63
|
+
current_version = "0.1.1"
|
|
65
64
|
commit = true
|
|
66
65
|
tag = true
|
|
67
66
|
commit_args = "-s"
|
|
File without changes
|
|
File without changes
|