ccflow-http 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ccflow_http/__init__.py +3 -0
- ccflow_http/base.py +542 -0
- ccflow_http/tests/test_all.py +5 -0
- ccflow_http/tests/test_model.py +524 -0
- ccflow_http-0.1.0.dist-info/METADATA +74 -0
- ccflow_http-0.1.0.dist-info/RECORD +8 -0
- ccflow_http-0.1.0.dist-info/WHEEL +4 -0
- ccflow_http-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,524 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from base64 import b64decode
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from gzip import compress
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import httpx
|
|
9
|
+
import pytest
|
|
10
|
+
from ccflow_etl import ExecutionPolicy
|
|
11
|
+
|
|
12
|
+
from ccflow_http import HTTPAuth, HTTPConfig, HTTPModel, HTTPRequest, HTTPRequestContext, HTTPResponseResult, HTTPRetryPolicy, safe_request_dump
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class FakeResponse:
|
|
17
|
+
value: Any
|
|
18
|
+
status_code: int = 200
|
|
19
|
+
headers: dict[str, str] | None = None
|
|
20
|
+
url: str = "https://api.example.test/v1/tickers/AAA"
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def content(self) -> bytes:
|
|
24
|
+
return b"payload"
|
|
25
|
+
|
|
26
|
+
@property
|
|
27
|
+
def text(self) -> str:
|
|
28
|
+
return "payload"
|
|
29
|
+
|
|
30
|
+
def json(self) -> Any:
|
|
31
|
+
return self.value
|
|
32
|
+
|
|
33
|
+
def raise_for_status(self) -> None:
|
|
34
|
+
return None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_http_model_renders_request_and_returns_json(monkeypatch):
|
|
38
|
+
calls = []
|
|
39
|
+
|
|
40
|
+
class FakeClient:
|
|
41
|
+
def __init__(self, **kwargs):
|
|
42
|
+
self.kwargs = kwargs
|
|
43
|
+
|
|
44
|
+
def __enter__(self):
|
|
45
|
+
return self
|
|
46
|
+
|
|
47
|
+
def __exit__(self, exc_type, exc, traceback):
|
|
48
|
+
return False
|
|
49
|
+
|
|
50
|
+
def request(self, **kwargs):
|
|
51
|
+
calls.append({"client": self.kwargs, "request": kwargs})
|
|
52
|
+
return FakeResponse(value={"status": "OK", "results": [{"ticker": "AAA"}]}, headers={"x-limit-remaining": "9"})
|
|
53
|
+
|
|
54
|
+
monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
|
|
55
|
+
|
|
56
|
+
model = HTTPModel(
|
|
57
|
+
base_url="https://api.example.test",
|
|
58
|
+
path="/v1/tickers/{{ ticker }}",
|
|
59
|
+
query={"date": "{{ date }}", "apiKey": "{{ api_key }}"},
|
|
60
|
+
headers={"Authorization": "Bearer {{ token }}"},
|
|
61
|
+
response_format="json",
|
|
62
|
+
timeout=12.5,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
result = model(
|
|
66
|
+
HTTPRequestContext(
|
|
67
|
+
template_values={"ticker": "AAA", "date": "2024-01-03", "api_key": "secret", "token": "abc"},
|
|
68
|
+
query={"adjusted": True},
|
|
69
|
+
)
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
assert result.value == {"status": "OK", "results": [{"ticker": "AAA"}]}
|
|
73
|
+
assert result.status_code == 200
|
|
74
|
+
assert result.headers == {"x-limit-remaining": "9"}
|
|
75
|
+
assert result.url == "https://api.example.test/v1/tickers/AAA"
|
|
76
|
+
assert calls == [
|
|
77
|
+
{
|
|
78
|
+
"client": {"base_url": "https://api.example.test", "timeout": 12.5, "follow_redirects": True},
|
|
79
|
+
"request": {
|
|
80
|
+
"method": "GET",
|
|
81
|
+
"url": "/v1/tickers/AAA",
|
|
82
|
+
"params": {"date": "2024-01-03", "apiKey": "secret", "adjusted": True},
|
|
83
|
+
"headers": {"Authorization": "Bearer abc"},
|
|
84
|
+
"json": None,
|
|
85
|
+
"content": None,
|
|
86
|
+
},
|
|
87
|
+
}
|
|
88
|
+
]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def test_http_model_can_explain_request_without_network():
|
|
92
|
+
model = HTTPModel(
|
|
93
|
+
base_url="https://api.example.test",
|
|
94
|
+
path="/v1/tickers/{{ ticker }}",
|
|
95
|
+
query={"date": "{{ date }}"},
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
request = model.build_request(HTTPRequestContext(template_values={"ticker": "AAA", "date": "2024-01-03"}))
|
|
99
|
+
|
|
100
|
+
assert request.method == "GET"
|
|
101
|
+
assert request.url == "/v1/tickers/AAA"
|
|
102
|
+
assert request.params == {"date": "2024-01-03"}
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_safe_request_dump_redacts_secret_params_and_headers():
|
|
106
|
+
request = HTTPRequest(
|
|
107
|
+
method="GET",
|
|
108
|
+
url="/v1/tickers/AAA",
|
|
109
|
+
params={"date": "2024-01-03", "apiKey": "query-secret", "page_token": "cursor-secret"},
|
|
110
|
+
headers={"Authorization": "Bearer header-secret", "X-Request-ID": "abc"},
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
assert safe_request_dump(request) == {
|
|
114
|
+
"method": "GET",
|
|
115
|
+
"url": "/v1/tickers/AAA",
|
|
116
|
+
"params": {"date": "2024-01-03", "apiKey": "***", "page_token": "***"},
|
|
117
|
+
"headers": {"Authorization": "***", "X-Request-ID": "abc"},
|
|
118
|
+
"json_data": None,
|
|
119
|
+
"content": None,
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def test_http_model_error_message_omits_secret_query_values(monkeypatch):
|
|
124
|
+
class FakeClient:
|
|
125
|
+
def __init__(self, **kwargs):
|
|
126
|
+
pass
|
|
127
|
+
|
|
128
|
+
def __enter__(self):
|
|
129
|
+
return self
|
|
130
|
+
|
|
131
|
+
def __exit__(self, exc_type, exc, traceback):
|
|
132
|
+
return False
|
|
133
|
+
|
|
134
|
+
def request(self, **kwargs):
|
|
135
|
+
request = httpx.Request("GET", "https://api.example.test/v1/tickers/AAA?apiKey=secret")
|
|
136
|
+
response = httpx.Response(429, request=request)
|
|
137
|
+
raise httpx.HTTPStatusError("rate limited with apiKey=secret", request=request, response=response)
|
|
138
|
+
|
|
139
|
+
monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
|
|
140
|
+
|
|
141
|
+
model = HTTPModel(base_url="https://api.example.test", path="/v1/tickers/{{ ticker }}", query={"apiKey": "{{ api_key }}"})
|
|
142
|
+
|
|
143
|
+
with pytest.raises(RuntimeError, match="HTTP GET /v1/tickers/AAA failed with status 429") as error:
|
|
144
|
+
model(HTTPRequestContext(template_values={"ticker": "AAA", "api_key": "secret"}))
|
|
145
|
+
|
|
146
|
+
assert "secret" not in str(error.value)
|
|
147
|
+
assert "apiKey" not in str(error.value)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def test_http_model_retries_retryable_status_and_captures_rate_limit(monkeypatch):
|
|
151
|
+
calls = []
|
|
152
|
+
|
|
153
|
+
class FakeClient:
|
|
154
|
+
def __init__(self, **kwargs):
|
|
155
|
+
pass
|
|
156
|
+
|
|
157
|
+
def __enter__(self):
|
|
158
|
+
return self
|
|
159
|
+
|
|
160
|
+
def __exit__(self, exc_type, exc, traceback):
|
|
161
|
+
return False
|
|
162
|
+
|
|
163
|
+
def request(self, **kwargs):
|
|
164
|
+
calls.append(kwargs)
|
|
165
|
+
if len(calls) == 1:
|
|
166
|
+
request = httpx.Request("GET", "https://api.example.test/v1/tickers")
|
|
167
|
+
response = httpx.Response(429, request=request, headers={"retry-after": "1"})
|
|
168
|
+
raise httpx.HTTPStatusError("rate limited", request=request, response=response)
|
|
169
|
+
return FakeResponse(value={"status": "OK"}, headers={"x-ratelimit-remaining": "8"})
|
|
170
|
+
|
|
171
|
+
monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
|
|
172
|
+
|
|
173
|
+
result = HTTPModel(base_url="https://api.example.test", path="/v1/tickers", max_attempts=2)(HTTPRequestContext())
|
|
174
|
+
|
|
175
|
+
assert len(calls) == 2
|
|
176
|
+
assert result.value == {"status": "OK"}
|
|
177
|
+
assert result.attempts == 2
|
|
178
|
+
assert result.rate_limit == {"x-ratelimit-remaining": "8"}
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def test_http_model_consumes_shared_retry_policy(monkeypatch):
|
|
182
|
+
calls = []
|
|
183
|
+
|
|
184
|
+
class FakeClient:
|
|
185
|
+
def __init__(self, **kwargs):
|
|
186
|
+
pass
|
|
187
|
+
|
|
188
|
+
def __enter__(self):
|
|
189
|
+
return self
|
|
190
|
+
|
|
191
|
+
def __exit__(self, exc_type, exc, traceback):
|
|
192
|
+
return False
|
|
193
|
+
|
|
194
|
+
def request(self, **kwargs):
|
|
195
|
+
calls.append(kwargs)
|
|
196
|
+
if len(calls) == 1:
|
|
197
|
+
request = httpx.Request("GET", "https://api.example.test/v1/tickers")
|
|
198
|
+
response = httpx.Response(503, request=request)
|
|
199
|
+
raise httpx.HTTPStatusError("unavailable", request=request, response=response)
|
|
200
|
+
return FakeResponse(value={"status": "OK"})
|
|
201
|
+
|
|
202
|
+
monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
|
|
203
|
+
|
|
204
|
+
model = HTTPModel(base_url="https://api.example.test", path="/v1/tickers", retry_policy=HTTPRetryPolicy(max_attempts=2, retry_status_codes=[503]))
|
|
205
|
+
|
|
206
|
+
assert model(HTTPRequestContext()).attempts == 2
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def test_http_model_consumes_shared_retry_delay_and_execution_policy(monkeypatch):
|
|
210
|
+
calls = []
|
|
211
|
+
|
|
212
|
+
class FakeClient:
|
|
213
|
+
def __init__(self, **kwargs):
|
|
214
|
+
pass
|
|
215
|
+
|
|
216
|
+
def __enter__(self):
|
|
217
|
+
return self
|
|
218
|
+
|
|
219
|
+
def __exit__(self, exc_type, exc, traceback):
|
|
220
|
+
return False
|
|
221
|
+
|
|
222
|
+
def request(self, **kwargs):
|
|
223
|
+
calls.append(kwargs)
|
|
224
|
+
if len(calls) == 1:
|
|
225
|
+
request = httpx.Request("GET", "https://api.example.test/v1/tickers")
|
|
226
|
+
response = httpx.Response(429, request=request)
|
|
227
|
+
raise httpx.HTTPStatusError("rate limited", request=request, response=response)
|
|
228
|
+
return FakeResponse(value={"status": "OK"})
|
|
229
|
+
|
|
230
|
+
monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
|
|
231
|
+
|
|
232
|
+
model = HTTPModel(
|
|
233
|
+
base_url="https://api.example.test",
|
|
234
|
+
path="/v1/tickers",
|
|
235
|
+
retry_policy=HTTPRetryPolicy(max_attempts=2, retry_status_codes=[429], wait_initial=1.25),
|
|
236
|
+
execution_policy=ExecutionPolicy(requests_per_interval=1, interval_seconds=2.0),
|
|
237
|
+
)
|
|
238
|
+
sleeps = []
|
|
239
|
+
request_times = iter([100.0, 101.25])
|
|
240
|
+
monkeypatch.setattr(model, "_sleep", sleeps.append)
|
|
241
|
+
monkeypatch.setattr(model, "_now", lambda: next(request_times))
|
|
242
|
+
|
|
243
|
+
result = model(HTTPRequestContext())
|
|
244
|
+
|
|
245
|
+
assert len(calls) == 2
|
|
246
|
+
assert sleeps == [1.25, 0.75]
|
|
247
|
+
assert result.retry_events == [
|
|
248
|
+
{
|
|
249
|
+
"attempt": 1,
|
|
250
|
+
"outcome": "retry",
|
|
251
|
+
"delay_seconds": 1.25,
|
|
252
|
+
"status_code": 429,
|
|
253
|
+
"category": "rate_limit",
|
|
254
|
+
"message": "retryable status code 429",
|
|
255
|
+
}
|
|
256
|
+
]
|
|
257
|
+
assert result.retry_summary == {"attempts": 2, "retried": 1, "failed": 0, "succeeded": 1}
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def test_http_model_retries_timeout_exception(monkeypatch):
|
|
261
|
+
calls = []
|
|
262
|
+
|
|
263
|
+
class FakeClient:
|
|
264
|
+
def __init__(self, **kwargs):
|
|
265
|
+
pass
|
|
266
|
+
|
|
267
|
+
def __enter__(self):
|
|
268
|
+
return self
|
|
269
|
+
|
|
270
|
+
def __exit__(self, exc_type, exc, traceback):
|
|
271
|
+
return False
|
|
272
|
+
|
|
273
|
+
def request(self, **kwargs):
|
|
274
|
+
calls.append(kwargs)
|
|
275
|
+
if len(calls) == 1:
|
|
276
|
+
raise httpx.TimeoutException("timed out")
|
|
277
|
+
return FakeResponse(value={"status": "OK"})
|
|
278
|
+
|
|
279
|
+
monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
|
|
280
|
+
|
|
281
|
+
result = HTTPModel(base_url="https://api.example.test", path="/v1/tickers", max_attempts=2)(HTTPRequestContext())
|
|
282
|
+
|
|
283
|
+
assert len(calls) == 2
|
|
284
|
+
assert result.value == {"status": "OK"}
|
|
285
|
+
assert result.attempts == 2
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def test_http_model_retries_5xx_until_attempts_are_exhausted(monkeypatch):
|
|
289
|
+
calls = []
|
|
290
|
+
|
|
291
|
+
class FakeClient:
|
|
292
|
+
def __init__(self, **kwargs):
|
|
293
|
+
pass
|
|
294
|
+
|
|
295
|
+
def __enter__(self):
|
|
296
|
+
return self
|
|
297
|
+
|
|
298
|
+
def __exit__(self, exc_type, exc, traceback):
|
|
299
|
+
return False
|
|
300
|
+
|
|
301
|
+
def request(self, **kwargs):
|
|
302
|
+
calls.append(kwargs)
|
|
303
|
+
request = httpx.Request("GET", "https://api.example.test/v1/tickers?apiKey=secret")
|
|
304
|
+
response = httpx.Response(500, request=request)
|
|
305
|
+
raise httpx.HTTPStatusError("server error with apiKey=secret", request=request, response=response)
|
|
306
|
+
|
|
307
|
+
monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
|
|
308
|
+
|
|
309
|
+
model = HTTPModel(base_url="https://api.example.test", path="/v1/tickers", query={"apiKey": "secret"}, max_attempts=2)
|
|
310
|
+
|
|
311
|
+
with pytest.raises(RuntimeError, match="HTTP GET /v1/tickers failed with status 500") as error:
|
|
312
|
+
model(HTTPRequestContext())
|
|
313
|
+
|
|
314
|
+
assert len(calls) == 2
|
|
315
|
+
assert "secret" not in str(error.value)
|
|
316
|
+
assert "apiKey" not in str(error.value)
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def test_http_model_paginates_massive_style_next_url(monkeypatch):
|
|
320
|
+
calls = []
|
|
321
|
+
|
|
322
|
+
class FakeClient:
|
|
323
|
+
def __init__(self, **kwargs):
|
|
324
|
+
pass
|
|
325
|
+
|
|
326
|
+
def __enter__(self):
|
|
327
|
+
return self
|
|
328
|
+
|
|
329
|
+
def __exit__(self, exc_type, exc, traceback):
|
|
330
|
+
return False
|
|
331
|
+
|
|
332
|
+
def request(self, **kwargs):
|
|
333
|
+
calls.append(kwargs)
|
|
334
|
+
if len(calls) == 1:
|
|
335
|
+
return FakeResponse(value={"results": [{"ticker": "AAA"}], "next_url": "/v3/reference/tickers?cursor=2"})
|
|
336
|
+
return FakeResponse(value={"results": [{"ticker": "BBB"}]})
|
|
337
|
+
|
|
338
|
+
monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
|
|
339
|
+
|
|
340
|
+
result = HTTPModel(base_url="https://api.example.test", path="/v3/reference/tickers", paginate=True)(HTTPRequestContext())
|
|
341
|
+
|
|
342
|
+
assert [call["url"] for call in calls] == ["/v3/reference/tickers", "/v3/reference/tickers"]
|
|
343
|
+
assert calls[1]["params"] == {"cursor": "2"}
|
|
344
|
+
assert result.value["results"] == [{"ticker": "AAA"}, {"ticker": "BBB"}]
|
|
345
|
+
assert result.pages == 2
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def test_http_model_preserves_query_auth_for_next_url_pagination(monkeypatch):
|
|
349
|
+
calls = []
|
|
350
|
+
|
|
351
|
+
class FakeClient:
|
|
352
|
+
def __init__(self, **kwargs):
|
|
353
|
+
pass
|
|
354
|
+
|
|
355
|
+
def __enter__(self):
|
|
356
|
+
return self
|
|
357
|
+
|
|
358
|
+
def __exit__(self, exc_type, exc, traceback):
|
|
359
|
+
return False
|
|
360
|
+
|
|
361
|
+
def request(self, **kwargs):
|
|
362
|
+
calls.append(kwargs)
|
|
363
|
+
if len(calls) == 1:
|
|
364
|
+
return FakeResponse(value={"results": [{"ticker": "AAA"}], "next_url": "/v3/reference/tickers?cursor=2"})
|
|
365
|
+
return FakeResponse(value={"results": [{"ticker": "BBB"}]})
|
|
366
|
+
|
|
367
|
+
monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
|
|
368
|
+
|
|
369
|
+
HTTPModel(
|
|
370
|
+
base_url="https://api.example.test",
|
|
371
|
+
path="/v3/reference/tickers",
|
|
372
|
+
query={"apiKey": "secret"},
|
|
373
|
+
paginate=True,
|
|
374
|
+
)(HTTPRequestContext())
|
|
375
|
+
|
|
376
|
+
assert calls[1]["url"] == "/v3/reference/tickers"
|
|
377
|
+
assert calls[1]["params"] == {"cursor": "2", "apiKey": "secret"}
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def test_http_model_sends_next_url_query_and_preserved_auth_with_httpx_transport():
|
|
381
|
+
seen_urls = []
|
|
382
|
+
|
|
383
|
+
def handler(request: httpx.Request) -> httpx.Response:
|
|
384
|
+
seen_urls.append(str(request.url))
|
|
385
|
+
if len(seen_urls) == 1:
|
|
386
|
+
return httpx.Response(200, json={"results": [{"ticker": "AAA"}], "next_url": "/v3/reference/tickers?cursor=2"})
|
|
387
|
+
return httpx.Response(200, json={"results": [{"ticker": "BBB"}]})
|
|
388
|
+
|
|
389
|
+
model = HTTPModel(
|
|
390
|
+
config=HTTPConfig(base_url="https://api.example.test", transport=httpx.MockTransport(handler)),
|
|
391
|
+
path="/v3/reference/tickers",
|
|
392
|
+
query={"apiKey": "secret"},
|
|
393
|
+
paginate=True,
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
result = model(HTTPRequestContext())
|
|
397
|
+
|
|
398
|
+
assert seen_urls == [
|
|
399
|
+
"https://api.example.test/v3/reference/tickers?apiKey=secret",
|
|
400
|
+
"https://api.example.test/v3/reference/tickers?cursor=2&apiKey=secret",
|
|
401
|
+
]
|
|
402
|
+
assert result.value["results"] == [{"ticker": "AAA"}, {"ticker": "BBB"}]
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def test_http_model_applies_config_and_all_auth_strategies_with_mock_transport():
|
|
406
|
+
seen_requests = []
|
|
407
|
+
|
|
408
|
+
def handler(request: httpx.Request) -> httpx.Response:
|
|
409
|
+
seen_requests.append(request)
|
|
410
|
+
return httpx.Response(200, json={"status": "OK"}, headers={"x-ratelimit-remaining": "7"})
|
|
411
|
+
|
|
412
|
+
transport = httpx.MockTransport(handler)
|
|
413
|
+
config = HTTPConfig(base_url="https://api.example.test", timeout=4.0, headers={"Accept": "application/json"}, transport=transport)
|
|
414
|
+
|
|
415
|
+
bearer = HTTPModel(config=config, path="/v1/{{ resource }}", auth=HTTPAuth(strategy="bearer", token="{{ token }}"))
|
|
416
|
+
bearer_result = bearer(HTTPRequestContext(template_values={"resource": "tickers", "token": "bearer-token"}))
|
|
417
|
+
|
|
418
|
+
api_header = HTTPModel(config=config, path="/v1/tickers", auth=HTTPAuth(strategy="api_key_header", name="X-API-Key", value="{{ api_key }}"))
|
|
419
|
+
api_header(HTTPRequestContext(template_values={"api_key": "header-key"}))
|
|
420
|
+
|
|
421
|
+
api_query = HTTPModel(config=config, path="/v1/tickers", auth=HTTPAuth(strategy="api_key_query", name="apiKey", value="{{ api_key }}"))
|
|
422
|
+
api_query(HTTPRequestContext(template_values={"api_key": "query-key"}))
|
|
423
|
+
|
|
424
|
+
basic = HTTPModel(config=config, path="/v1/tickers", auth=HTTPAuth(strategy="basic", username="{{ user }}", password="{{ password }}"))
|
|
425
|
+
basic(HTTPRequestContext(template_values={"user": "svc", "password": "secret"}))
|
|
426
|
+
|
|
427
|
+
no_auth = HTTPModel(config=config, path="/v1/tickers", auth=HTTPAuth(strategy="none"))
|
|
428
|
+
no_auth(HTTPRequestContext())
|
|
429
|
+
|
|
430
|
+
assert isinstance(bearer_result, HTTPResponseResult)
|
|
431
|
+
assert str(seen_requests[0].url) == "https://api.example.test/v1/tickers"
|
|
432
|
+
assert seen_requests[0].headers["accept"] == "application/json"
|
|
433
|
+
assert seen_requests[0].headers["authorization"] == "Bearer bearer-token"
|
|
434
|
+
assert seen_requests[1].headers["x-api-key"] == "header-key"
|
|
435
|
+
assert dict(seen_requests[2].url.params) == {"apiKey": "query-key"}
|
|
436
|
+
assert seen_requests[3].headers["authorization"].startswith("Basic ")
|
|
437
|
+
assert b64decode(seen_requests[3].headers["authorization"].removeprefix("Basic ")).decode("utf-8") == "svc:secret"
|
|
438
|
+
assert "authorization" not in seen_requests[4].headers
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def test_http_model_parses_csv_and_gzip_responses_with_mock_transport():
|
|
442
|
+
def csv_handler(request: httpx.Request) -> httpx.Response:
|
|
443
|
+
return httpx.Response(200, text="ticker,volume\nAAA,10\nBBB,20\n")
|
|
444
|
+
|
|
445
|
+
csv_result = HTTPModel(
|
|
446
|
+
config=HTTPConfig(base_url="https://api.example.test", transport=httpx.MockTransport(csv_handler)),
|
|
447
|
+
path="/csv",
|
|
448
|
+
response_format="csv",
|
|
449
|
+
)(HTTPRequestContext())
|
|
450
|
+
|
|
451
|
+
def gzip_handler(request: httpx.Request) -> httpx.Response:
|
|
452
|
+
return httpx.Response(200, content=compress(b'{"status":"OK"}'))
|
|
453
|
+
|
|
454
|
+
gzip_result = HTTPModel(
|
|
455
|
+
config=HTTPConfig(base_url="https://api.example.test", transport=httpx.MockTransport(gzip_handler)),
|
|
456
|
+
path="/gzip",
|
|
457
|
+
response_format="gzip",
|
|
458
|
+
)(HTTPRequestContext())
|
|
459
|
+
|
|
460
|
+
assert csv_result.value == [{"ticker": "AAA", "volume": "10"}, {"ticker": "BBB", "volume": "20"}]
|
|
461
|
+
assert gzip_result.value == b'{"status":"OK"}'
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def test_http_model_supports_cursor_page_and_offset_pagination_with_mock_transport():
|
|
465
|
+
cursor_requests = []
|
|
466
|
+
|
|
467
|
+
def cursor_handler(request: httpx.Request) -> httpx.Response:
|
|
468
|
+
cursor_requests.append(dict(request.url.params))
|
|
469
|
+
if "cursor" not in request.url.params:
|
|
470
|
+
return httpx.Response(200, json={"results": [{"id": 1}], "next_cursor": "abc"})
|
|
471
|
+
return httpx.Response(200, json={"results": [{"id": 2}]})
|
|
472
|
+
|
|
473
|
+
cursor_result = HTTPModel(
|
|
474
|
+
config=HTTPConfig(base_url="https://api.example.test", transport=httpx.MockTransport(cursor_handler)),
|
|
475
|
+
path="/items",
|
|
476
|
+
paginate=True,
|
|
477
|
+
pagination_mode="cursor",
|
|
478
|
+
next_cursor_field="next_cursor",
|
|
479
|
+
cursor_param="cursor",
|
|
480
|
+
)(HTTPRequestContext())
|
|
481
|
+
|
|
482
|
+
page_requests = []
|
|
483
|
+
|
|
484
|
+
def page_handler(request: httpx.Request) -> httpx.Response:
|
|
485
|
+
page_requests.append(dict(request.url.params))
|
|
486
|
+
page = int(request.url.params["page"])
|
|
487
|
+
payload = {1: [{"id": 1}], 2: [{"id": 2}]}.get(page, [])
|
|
488
|
+
return httpx.Response(200, json={"results": payload})
|
|
489
|
+
|
|
490
|
+
page_result = HTTPModel(
|
|
491
|
+
config=HTTPConfig(base_url="https://api.example.test", transport=httpx.MockTransport(page_handler)),
|
|
492
|
+
path="/items",
|
|
493
|
+
paginate=True,
|
|
494
|
+
pagination_mode="page",
|
|
495
|
+
page_param="page",
|
|
496
|
+
page_start=1,
|
|
497
|
+
max_pages=5,
|
|
498
|
+
)(HTTPRequestContext())
|
|
499
|
+
|
|
500
|
+
offset_requests = []
|
|
501
|
+
|
|
502
|
+
def offset_handler(request: httpx.Request) -> httpx.Response:
|
|
503
|
+
offset_requests.append(dict(request.url.params))
|
|
504
|
+
offset = int(request.url.params["offset"])
|
|
505
|
+
payload = {0: [{"id": 1}], 2: [{"id": 2}]}.get(offset, [])
|
|
506
|
+
return httpx.Response(200, json={"results": payload})
|
|
507
|
+
|
|
508
|
+
offset_result = HTTPModel(
|
|
509
|
+
config=HTTPConfig(base_url="https://api.example.test", transport=httpx.MockTransport(offset_handler)),
|
|
510
|
+
path="/items",
|
|
511
|
+
paginate=True,
|
|
512
|
+
pagination_mode="offset",
|
|
513
|
+
offset_param="offset",
|
|
514
|
+
limit_param="limit",
|
|
515
|
+
limit=2,
|
|
516
|
+
max_pages=5,
|
|
517
|
+
)(HTTPRequestContext())
|
|
518
|
+
|
|
519
|
+
assert cursor_requests == [{}, {"cursor": "abc"}]
|
|
520
|
+
assert cursor_result.value["results"] == [{"id": 1}, {"id": 2}]
|
|
521
|
+
assert page_requests == [{"page": "1"}, {"page": "2"}, {"page": "3"}]
|
|
522
|
+
assert page_result.value["results"] == [{"id": 1}, {"id": 2}]
|
|
523
|
+
assert offset_requests == [{"offset": "0", "limit": "2"}, {"offset": "2", "limit": "2"}, {"offset": "4", "limit": "2"}]
|
|
524
|
+
assert offset_result.value["results"] == [{"id": 1}, {"id": 2}]
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ccflow-http
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: ccflow models for HTTP
|
|
5
|
+
Project-URL: Repository, https://github.com/1kbgz/ccflow-http
|
|
6
|
+
Project-URL: Homepage, https://github.com/1kbgz/ccflow-http
|
|
7
|
+
Author-email: 1kbgz <dev@1kbgz.com>
|
|
8
|
+
License: Apache-2.0
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Programming Language :: Python
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
19
|
+
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Requires-Dist: ccflow-etl>=0.1.0
|
|
22
|
+
Requires-Dist: ccflow>=0.8.5
|
|
23
|
+
Requires-Dist: httpx
|
|
24
|
+
Requires-Dist: jinja2
|
|
25
|
+
Provides-Extra: develop
|
|
26
|
+
Requires-Dist: build; extra == 'develop'
|
|
27
|
+
Requires-Dist: bump-my-version; extra == 'develop'
|
|
28
|
+
Requires-Dist: check-dist; extra == 'develop'
|
|
29
|
+
Requires-Dist: codespell; extra == 'develop'
|
|
30
|
+
Requires-Dist: hatchling; extra == 'develop'
|
|
31
|
+
Requires-Dist: mdformat; extra == 'develop'
|
|
32
|
+
Requires-Dist: mdformat-tables>=1; extra == 'develop'
|
|
33
|
+
Requires-Dist: pytest; extra == 'develop'
|
|
34
|
+
Requires-Dist: pytest-cov; extra == 'develop'
|
|
35
|
+
Requires-Dist: ruff; extra == 'develop'
|
|
36
|
+
Requires-Dist: twine; extra == 'develop'
|
|
37
|
+
Requires-Dist: ty; extra == 'develop'
|
|
38
|
+
Requires-Dist: uv; extra == 'develop'
|
|
39
|
+
Requires-Dist: wheel; extra == 'develop'
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
# ccflow-http
|
|
43
|
+
|
|
44
|
+
ccflow models for HTTP
|
|
45
|
+
|
|
46
|
+
[](https://github.com/1kbgz/ccflow-http/actions/workflows/build.yaml)
|
|
47
|
+
[](https://codecov.io/gh/1kbgz/ccflow-http)
|
|
48
|
+
[](https://github.com/1kbgz/ccflow-http)
|
|
49
|
+
[](https://pypi.python.org/pypi/ccflow-http)
|
|
50
|
+
|
|
51
|
+
## Overview
|
|
52
|
+
|
|
53
|
+
`ccflow-http` provides public, domain-neutral HTTP callable models for `ccflow` workflows. It should own request configuration, auth strategies, request templating, pagination, response parsing, timeout handling, retry/rate-limit integration, and HTTP result metadata.
|
|
54
|
+
|
|
55
|
+
It should not contain provider-specific endpoint catalogs. Domain packages can configure or subclass these generic models for particular APIs.
|
|
56
|
+
|
|
57
|
+
## Current Status
|
|
58
|
+
|
|
59
|
+
- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, and mocked `httpx` transport tests.
|
|
60
|
+
- Partial: the shared execution policy is currently consumed inside `HTTPModel` for sequential request spacing; broader evaluator-level concurrency coordination still belongs in evaluator integrations.
|
|
61
|
+
- Missing: broader integration examples and provider-specific subclasses/configs in downstream packages.
|
|
62
|
+
|
|
63
|
+
## Dependency Contract
|
|
64
|
+
|
|
65
|
+
- Depends on `ccflow` for callable model, context, and result interfaces.
|
|
66
|
+
- Depends on `ccflow` retry policy semantics and `ccflow-etl` execution policy models.
|
|
67
|
+
- Must not depend on finance packages or application-specific packages.
|
|
68
|
+
|
|
69
|
+
## Test Convention
|
|
70
|
+
|
|
71
|
+
Default tests should use mocked `httpx` transports or local fixtures. They should not require live network calls or provider credentials.
|
|
72
|
+
|
|
73
|
+
> [!NOTE]
|
|
74
|
+
> This library was generated using [copier](https://copier.readthedocs.io/en/stable/) from the [Base Python Project Template repository](https://github.com/python-project-templates/base).
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
ccflow_http/__init__.py,sha256=GZa5kRIfRkSwASZsKw2rf3LPXqLZmmppxKbLjKZC2UU,43
|
|
2
|
+
ccflow_http/base.py,sha256=r2i7VAMCoCxTbmmJGCNABunAWoRUsHvPs494MbR3Pfc,23748
|
|
3
|
+
ccflow_http/tests/test_all.py,sha256=GX3PrXlvg63k7vJy3j14mSLgiMaA8yfBWeGc2AbLbrc,68
|
|
4
|
+
ccflow_http/tests/test_model.py,sha256=NftVa2joIEdirfh4-vPGOMafyNWycs8tNmev8ok_6Qg,19193
|
|
5
|
+
ccflow_http-0.1.0.dist-info/METADATA,sha256=icc3x5AEZJn9hs3p-LBGvbkq_GEQbeycw9alLNQVjBc,4149
|
|
6
|
+
ccflow_http-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
7
|
+
ccflow_http-0.1.0.dist-info/licenses/LICENSE,sha256=RcPqVYf2JyhlV4FlBQud0104R3fT1_-7XbT-vp4eLyE,11335
|
|
8
|
+
ccflow_http-0.1.0.dist-info/RECORD,,
|