ccflow-http 0.3.0__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ccflow_http-0.3.0 → ccflow_http-0.4.1}/PKG-INFO +3 -3
- {ccflow_http-0.3.0 → ccflow_http-0.4.1}/README.md +1 -1
- {ccflow_http-0.3.0 → ccflow_http-0.4.1}/ccflow_http/__init__.py +1 -1
- {ccflow_http-0.3.0 → ccflow_http-0.4.1}/ccflow_http/base.py +86 -39
- {ccflow_http-0.3.0 → ccflow_http-0.4.1}/ccflow_http/download.py +5 -5
- {ccflow_http-0.3.0 → ccflow_http-0.4.1}/ccflow_http/tests/test_model.py +102 -2
- {ccflow_http-0.3.0 → ccflow_http-0.4.1}/pyproject.toml +3 -3
- {ccflow_http-0.3.0 → ccflow_http-0.4.1}/.gitignore +0 -0
- {ccflow_http-0.3.0 → ccflow_http-0.4.1}/LICENSE +0 -0
- {ccflow_http-0.3.0 → ccflow_http-0.4.1}/ccflow_http/tests/test_all.py +0 -0
- {ccflow_http-0.3.0 → ccflow_http-0.4.1}/ccflow_http/tests/test_download.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ccflow-http
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: ccflow models for HTTP
|
|
5
5
|
Project-URL: Repository, https://github.com/1kbgz/ccflow-http
|
|
6
6
|
Project-URL: Homepage, https://github.com/1kbgz/ccflow-http
|
|
@@ -28,7 +28,7 @@ Requires-Dist: check-dist; extra == 'develop'
|
|
|
28
28
|
Requires-Dist: codespell; extra == 'develop'
|
|
29
29
|
Requires-Dist: hatchling; extra == 'develop'
|
|
30
30
|
Requires-Dist: mdformat; extra == 'develop'
|
|
31
|
-
Requires-Dist: mdformat-
|
|
31
|
+
Requires-Dist: mdformat-myst; extra == 'develop'
|
|
32
32
|
Requires-Dist: mdformat-tables>=1; extra == 'develop'
|
|
33
33
|
Requires-Dist: pytest; extra == 'develop'
|
|
34
34
|
Requires-Dist: pytest-cov; extra == 'develop'
|
|
@@ -56,7 +56,7 @@ It should not contain provider-specific endpoint catalogs. Domain packages can c
|
|
|
56
56
|
|
|
57
57
|
## Current Status
|
|
58
58
|
|
|
59
|
-
- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, redaction
|
|
59
|
+
- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, a configurable `RedactionPolicy` (`HTTPModel.redaction`: exact names, substrings, and patterns, with `auth` header and query names added automatically) that masks secret header and query values in `httpx2` request logs, request dumps, result URLs and headers, retry events, and errors, `HTTPDownloadModel` for streaming a response to a local file with size and SHA-256, `HTTPArtifactCaptureModel` for storing a dated download plus a manifest through a `ccflow-etl` file writer, and mocked `httpx2` transport tests.
|
|
60
60
|
- Partial: the shared execution policy is currently consumed inside `HTTPModel` for sequential request spacing; broader evaluator-level concurrency coordination still belongs in evaluator integrations.
|
|
61
61
|
- Missing: broader integration examples and provider-specific subclasses/configs in downstream packages.
|
|
62
62
|
|
|
@@ -15,7 +15,7 @@ It should not contain provider-specific endpoint catalogs. Domain packages can c
|
|
|
15
15
|
|
|
16
16
|
## Current Status
|
|
17
17
|
|
|
18
|
-
- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, redaction
|
|
18
|
+
- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, a configurable `RedactionPolicy` (`HTTPModel.redaction`: exact names, substrings, and patterns, with `auth` header and query names added automatically) that masks secret header and query values in `httpx2` request logs, request dumps, result URLs and headers, retry events, and errors, `HTTPDownloadModel` for streaming a response to a local file with size and SHA-256, `HTTPArtifactCaptureModel` for storing a dated download plus a manifest through a `ccflow-etl` file writer, and mocked `httpx2` transport tests.
|
|
19
19
|
- Partial: the shared execution policy is currently consumed inside `HTTPModel` for sequential request spacing; broader evaluator-level concurrency coordination still belongs in evaluator integrations.
|
|
20
20
|
- Missing: broader integration examples and provider-specific subclasses/configs in downstream packages.
|
|
21
21
|
|
|
@@ -17,7 +17,7 @@ from ccflow import BaseModel, CallableModel, ContextBase, Flow, GenericResult, P
|
|
|
17
17
|
from ccflow.utils.retry import RetryPolicy
|
|
18
18
|
from ccflow_etl import ExecutionPolicy
|
|
19
19
|
from jinja2 import Environment
|
|
20
|
-
from pydantic import Field, PrivateAttr
|
|
20
|
+
from pydantic import Field, PrivateAttr, field_validator
|
|
21
21
|
|
|
22
22
|
__all__ = (
|
|
23
23
|
"HTTPAuth",
|
|
@@ -29,6 +29,7 @@ __all__ = (
|
|
|
29
29
|
"HTTPResponseResult",
|
|
30
30
|
"HTTPResult",
|
|
31
31
|
"HTTPRetryPolicy",
|
|
32
|
+
"RedactionPolicy",
|
|
32
33
|
"redact_mapping",
|
|
33
34
|
"redact_url",
|
|
34
35
|
"safe_request_dump",
|
|
@@ -51,11 +52,11 @@ class HTTPConfig(BaseModel):
|
|
|
51
52
|
|
|
52
53
|
class HTTPAuth(BaseModel):
|
|
53
54
|
strategy: HTTPAuthStrategy = "none"
|
|
54
|
-
token: str | None = None
|
|
55
|
+
token: str | None = Field(default=None, repr=False)
|
|
55
56
|
name: str | None = None
|
|
56
|
-
value: str | None = None
|
|
57
|
+
value: str | None = Field(default=None, repr=False)
|
|
57
58
|
username: str | None = None
|
|
58
|
-
password: str | None = None
|
|
59
|
+
password: str | None = Field(default=None, repr=False)
|
|
59
60
|
scheme: str = "Bearer"
|
|
60
61
|
|
|
61
62
|
|
|
@@ -78,38 +79,79 @@ class HTTPRequest(BaseModel):
|
|
|
78
79
|
headers: dict[str, str] = Field(default_factory=dict)
|
|
79
80
|
json_data: Any | None = None
|
|
80
81
|
content: bytes | str | None = None
|
|
81
|
-
#
|
|
82
|
-
|
|
82
|
+
# Redaction for this request: the model's policy plus any names its auth places credentials under.
|
|
83
|
+
_redaction: "RedactionPolicy" = PrivateAttr(default_factory=lambda: _DEFAULT_REDACTION)
|
|
83
84
|
|
|
84
85
|
|
|
85
|
-
# Extra sensitive query names for the request in flight, so the httpx2 log filter can redact them.
|
|
86
|
-
_REQUEST_SENSITIVE_PARAMS: ContextVar[frozenset[str]] = ContextVar("ccflow_http_request_sensitive_params", default=frozenset())
|
|
87
86
|
_QUERY_PAIR = re.compile(r"([?&])([^=&\s'\"]+)=([^&\s'\"]*)")
|
|
88
87
|
|
|
89
88
|
|
|
90
|
-
def
|
|
91
|
-
|
|
92
|
-
return key in extra or normalized_key in {"apikey", "authorization", "password"} or "token" in normalized_key or "secret" in normalized_key
|
|
89
|
+
def _normalize_name(name: str) -> str:
|
|
90
|
+
return name.lower().replace("_", "").replace("-", "")
|
|
93
91
|
|
|
94
92
|
|
|
95
|
-
|
|
96
|
-
|
|
93
|
+
class RedactionPolicy(BaseModel):
|
|
94
|
+
"""Which header and query parameter names carry secrets, and how their values are masked.
|
|
97
95
|
|
|
96
|
+
Names are compared after lowercasing and removing hyphens and underscores, so ``X-Api-Key`` and ``x_api_key``
|
|
97
|
+
are the same name. A name is sensitive when it is one of ``names``, contains one of ``substrings``, or matches one
|
|
98
|
+
of ``patterns`` (regular expressions searched in the normalized name). Set it on ``HTTPModel.redaction`` to cover
|
|
99
|
+
a provider's own header or parameter names; names used by ``auth`` are added automatically.
|
|
100
|
+
"""
|
|
98
101
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
pairs = []
|
|
104
|
-
for pair in parts.query.split("&"):
|
|
105
|
-
key = pair.split("=", 1)[0]
|
|
106
|
-
pairs.append(f"{key}=***" if _is_sensitive_key(unquote_plus(key), extra) else pair)
|
|
107
|
-
return urlunsplit(parts._replace(query="&".join(pairs)))
|
|
102
|
+
names: set[str] = Field(default_factory=lambda: {"authorization", "proxyauthorization", "cookie", "setcookie", "password"})
|
|
103
|
+
substrings: set[str] = Field(default_factory=lambda: {"apikey", "token", "secret", "password"})
|
|
104
|
+
patterns: list[str] = Field(default_factory=list)
|
|
105
|
+
mask: str = "***"
|
|
108
106
|
|
|
107
|
+
@field_validator("names", "substrings", mode="after")
|
|
108
|
+
@classmethod
|
|
109
|
+
def _normalize(cls, values: set[str]) -> set[str]:
|
|
110
|
+
return {_normalize_name(value) for value in values}
|
|
109
111
|
|
|
110
|
-
def
|
|
111
|
-
|
|
112
|
-
|
|
112
|
+
def with_names(self, *names: str) -> "RedactionPolicy":
|
|
113
|
+
"""Return a copy that also treats ``names`` as sensitive."""
|
|
114
|
+
if not names:
|
|
115
|
+
return self
|
|
116
|
+
return self.model_copy(update={"names": self.names | {_normalize_name(name) for name in names}})
|
|
117
|
+
|
|
118
|
+
def is_sensitive(self, name: str) -> bool:
|
|
119
|
+
normalized = _normalize_name(name)
|
|
120
|
+
return (
|
|
121
|
+
normalized in self.names
|
|
122
|
+
or any(substring in normalized for substring in self.substrings)
|
|
123
|
+
or any(re.search(pattern, normalized) for pattern in self.patterns)
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
def redact_mapping(self, values: dict[str, Any]) -> dict[str, Any]:
|
|
127
|
+
return {key: self.mask if self.is_sensitive(key) else value for key, value in values.items()}
|
|
128
|
+
|
|
129
|
+
def redact_url(self, url: str) -> str:
|
|
130
|
+
parts = urlsplit(url)
|
|
131
|
+
if not parts.query:
|
|
132
|
+
return url
|
|
133
|
+
pairs = []
|
|
134
|
+
for pair in parts.query.split("&"):
|
|
135
|
+
key = pair.split("=", 1)[0]
|
|
136
|
+
pairs.append(f"{key}={self.mask}" if self.is_sensitive(unquote_plus(key)) else pair)
|
|
137
|
+
return urlunsplit(parts._replace(query="&".join(pairs)))
|
|
138
|
+
|
|
139
|
+
def redact_text(self, text: str) -> str:
|
|
140
|
+
"""Redact secret query values that appear anywhere in free text, such as an exception message quoting a URL."""
|
|
141
|
+
return _QUERY_PAIR.sub(lambda match: f"{match[1]}{match[2]}={self.mask}" if self.is_sensitive(unquote_plus(match[2])) else match[0], text)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
_DEFAULT_REDACTION = RedactionPolicy()
|
|
145
|
+
# Redaction policy of the request in flight, so the httpx2 log filter can apply it.
|
|
146
|
+
_REQUEST_REDACTION: ContextVar[RedactionPolicy | None] = ContextVar("ccflow_http_request_redaction", default=None)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def redact_mapping(values: dict[str, Any], extra: Collection[str] = (), policy: RedactionPolicy | None = None) -> dict[str, Any]:
|
|
150
|
+
return (policy or _DEFAULT_REDACTION).with_names(*extra).redact_mapping(values)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def redact_url(url: str, extra: Collection[str] = (), policy: RedactionPolicy | None = None) -> str:
|
|
154
|
+
return (policy or _DEFAULT_REDACTION).with_names(*extra).redact_url(url)
|
|
113
155
|
|
|
114
156
|
|
|
115
157
|
class _RedactURLFilter(Filter):
|
|
@@ -117,8 +159,8 @@ class _RedactURLFilter(Filter):
|
|
|
117
159
|
|
|
118
160
|
def filter(self, record: LogRecord) -> bool:
|
|
119
161
|
if isinstance(record.args, tuple):
|
|
120
|
-
|
|
121
|
-
record.args = tuple(redact_url(str(arg)
|
|
162
|
+
policy = _REQUEST_REDACTION.get() or _DEFAULT_REDACTION
|
|
163
|
+
record.args = tuple(policy.redact_url(str(arg)) if isinstance(arg, httpx2.URL) else arg for arg in record.args)
|
|
122
164
|
return True
|
|
123
165
|
|
|
124
166
|
|
|
@@ -127,8 +169,8 @@ getLogger("httpx2").addFilter(_RedactURLFilter())
|
|
|
127
169
|
|
|
128
170
|
def safe_request_dump(request: HTTPRequest) -> dict[str, Any]:
|
|
129
171
|
request_data = request.model_dump(exclude={"type_"})
|
|
130
|
-
request_data["params"] = redact_mapping(request_data.get("params", {})
|
|
131
|
-
request_data["headers"] = redact_mapping(request_data.get("headers", {}))
|
|
172
|
+
request_data["params"] = request._redaction.redact_mapping(request_data.get("params", {}))
|
|
173
|
+
request_data["headers"] = request._redaction.redact_mapping(request_data.get("headers", {}))
|
|
132
174
|
return request_data
|
|
133
175
|
|
|
134
176
|
|
|
@@ -241,6 +283,9 @@ class HTTPModel(CallableModel):
|
|
|
241
283
|
description="Error status codes returned as results with value None instead of raising, such as 404 for lookups that may miss.",
|
|
242
284
|
)
|
|
243
285
|
execution_policy: ExecutionPolicy | None = None
|
|
286
|
+
redaction: RedactionPolicy = Field(
|
|
287
|
+
default_factory=RedactionPolicy, description="Header and query parameter names whose values are masked in logs, results, and errors."
|
|
288
|
+
)
|
|
244
289
|
paginate: bool = False
|
|
245
290
|
max_pages: int = 100
|
|
246
291
|
pagination_mode: HTTPPaginationMode = "next_url"
|
|
@@ -349,8 +394,10 @@ class HTTPModel(CallableModel):
|
|
|
349
394
|
json_data=context.json_body if context.json_body is not None else self.json_body,
|
|
350
395
|
content=context.content if context.content is not None else self.content,
|
|
351
396
|
)
|
|
352
|
-
|
|
353
|
-
|
|
397
|
+
auth_names = []
|
|
398
|
+
if self.auth.strategy in ("api_key_query", "api_key_header"):
|
|
399
|
+
auth_names.append(self._render_required_auth_value(self.auth.name, "name", data))
|
|
400
|
+
request._redaction = self.redaction.with_names(*auth_names)
|
|
354
401
|
return request
|
|
355
402
|
|
|
356
403
|
def _response_value(self, response: httpx2.Response) -> Any:
|
|
@@ -489,7 +536,7 @@ class HTTPModel(CallableModel):
|
|
|
489
536
|
delay_seconds=delay_seconds,
|
|
490
537
|
exception_type=type(exc).__name__,
|
|
491
538
|
category=retry_policy.exception_category(exc),
|
|
492
|
-
message=
|
|
539
|
+
message=request._redaction.redact_text(str(exc)),
|
|
493
540
|
)
|
|
494
541
|
)
|
|
495
542
|
self._sleep(delay_seconds)
|
|
@@ -501,7 +548,7 @@ class HTTPModel(CallableModel):
|
|
|
501
548
|
outcome="failed",
|
|
502
549
|
exception_type=type(exc).__name__,
|
|
503
550
|
category=retry_policy.exception_category(exc),
|
|
504
|
-
message=
|
|
551
|
+
message=request._redaction.redact_text(str(exc)),
|
|
505
552
|
)
|
|
506
553
|
)
|
|
507
554
|
# httpx2 exceptions keep the full request, including secret query values, so do not chain them.
|
|
@@ -531,7 +578,7 @@ class HTTPModel(CallableModel):
|
|
|
531
578
|
next_url_parts = urlsplit(next_url)
|
|
532
579
|
next_url_params = dict(parse_qsl(next_url_parts.query, keep_blank_values=True))
|
|
533
580
|
for key, value in request.params.items():
|
|
534
|
-
if key not in next_url_params and
|
|
581
|
+
if key not in next_url_params and request._redaction.is_sensitive(key):
|
|
535
582
|
next_url_params[key] = value
|
|
536
583
|
next_url_without_query = urlunsplit((next_url_parts.scheme, next_url_parts.netloc, next_url_parts.path, "", next_url_parts.fragment))
|
|
537
584
|
return request.model_copy(update={"url": next_url_without_query, "params": next_url_params})
|
|
@@ -582,9 +629,9 @@ class HTTPModel(CallableModel):
|
|
|
582
629
|
@Flow.call
|
|
583
630
|
def __call__(self, context: HTTPRequestContext) -> HTTPResult:
|
|
584
631
|
request = self._initial_paginated_request(self.build_request(context))
|
|
585
|
-
|
|
632
|
+
redaction = request._redaction
|
|
586
633
|
|
|
587
|
-
token =
|
|
634
|
+
token = _REQUEST_REDACTION.set(redaction)
|
|
588
635
|
try:
|
|
589
636
|
with httpx2.Client(**self._client_kwargs()) as client:
|
|
590
637
|
values = []
|
|
@@ -615,13 +662,13 @@ class HTTPModel(CallableModel):
|
|
|
615
662
|
break
|
|
616
663
|
request = next_request
|
|
617
664
|
finally:
|
|
618
|
-
|
|
665
|
+
_REQUEST_REDACTION.reset(token)
|
|
619
666
|
|
|
620
667
|
return HTTPResult(
|
|
621
668
|
value=None if accepted_error else (self._merge_page_values(values) if self.paginate else values[-1]),
|
|
622
669
|
status_code=response.status_code,
|
|
623
|
-
headers=dict(response.headers or {}),
|
|
624
|
-
url=redact_url(str(response.url)
|
|
670
|
+
headers=redaction.redact_mapping(dict(response.headers or {})),
|
|
671
|
+
url=redaction.redact_url(str(response.url)),
|
|
625
672
|
attempts=total_attempts,
|
|
626
673
|
pages=pages,
|
|
627
674
|
rate_limit=self._rate_limit_headers(dict(response.headers or {})),
|
|
@@ -10,7 +10,7 @@ from ccflow import CallableModel, DateContext, Flow, ResultBase
|
|
|
10
10
|
from ccflow_etl import ArtifactWriteFileContext, ArtifactWriteFileModel, ArtifactWriteFileResult
|
|
11
11
|
from pydantic import Field
|
|
12
12
|
|
|
13
|
-
from .base import
|
|
13
|
+
from .base import _REQUEST_REDACTION, HTTPModel, HTTPRequest, HTTPRequestContext
|
|
14
14
|
|
|
15
15
|
__all__ = (
|
|
16
16
|
"HTTPArtifactCaptureModel",
|
|
@@ -69,13 +69,13 @@ class HTTPDownloadModel(HTTPModel):
|
|
|
69
69
|
@Flow.call
|
|
70
70
|
def __call__(self, context: HTTPDownloadContext) -> HTTPDownloadResult:
|
|
71
71
|
request = self.build_request(context)
|
|
72
|
-
|
|
72
|
+
redaction = request._redaction
|
|
73
73
|
path = context.output_path
|
|
74
74
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
75
75
|
temp_path = path.with_name(f".{path.name}.{uuid4().hex}.tmp")
|
|
76
76
|
digest = sha256()
|
|
77
77
|
size = 0
|
|
78
|
-
token =
|
|
78
|
+
token = _REQUEST_REDACTION.set(redaction)
|
|
79
79
|
try:
|
|
80
80
|
with httpx2.Client(**self._client_kwargs()) as client:
|
|
81
81
|
response, attempts, events, _ = self._request_once(client, request, None)
|
|
@@ -92,13 +92,13 @@ class HTTPDownloadModel(HTTPModel):
|
|
|
92
92
|
temp_path.unlink(missing_ok=True)
|
|
93
93
|
raise
|
|
94
94
|
finally:
|
|
95
|
-
|
|
95
|
+
_REQUEST_REDACTION.reset(token)
|
|
96
96
|
return HTTPDownloadResult(
|
|
97
97
|
path=str(path),
|
|
98
98
|
size=size,
|
|
99
99
|
sha256=digest.hexdigest(),
|
|
100
100
|
status_code=response.status_code,
|
|
101
|
-
url=redact_url(str(response.url)
|
|
101
|
+
url=redaction.redact_url(str(response.url)),
|
|
102
102
|
etag=response.headers.get("etag"),
|
|
103
103
|
last_modified=response.headers.get("last-modified"),
|
|
104
104
|
content_type=response.headers.get("content-type"),
|
|
@@ -18,6 +18,7 @@ from ccflow_http import (
|
|
|
18
18
|
HTTPRequestContext,
|
|
19
19
|
HTTPResponseResult,
|
|
20
20
|
HTTPRetryPolicy,
|
|
21
|
+
RedactionPolicy,
|
|
21
22
|
redact_url,
|
|
22
23
|
safe_request_dump,
|
|
23
24
|
)
|
|
@@ -118,14 +119,14 @@ def test_safe_request_dump_redacts_secret_params_and_headers():
|
|
|
118
119
|
method="GET",
|
|
119
120
|
url="/v1/tickers/AAA",
|
|
120
121
|
params={"date": "2024-01-03", "apiKey": "query-secret", "page_token": "cursor-secret"},
|
|
121
|
-
headers={"Authorization": "Bearer header-secret", "X-Request-ID": "abc"},
|
|
122
|
+
headers={"Authorization": "Bearer header-secret", "X-OPENFIGI-APIKEY": "vendor-secret", "X-Request-ID": "abc"},
|
|
122
123
|
)
|
|
123
124
|
|
|
124
125
|
assert safe_request_dump(request) == {
|
|
125
126
|
"method": "GET",
|
|
126
127
|
"url": "/v1/tickers/AAA",
|
|
127
128
|
"params": {"date": "2024-01-03", "apiKey": "***", "page_token": "***"},
|
|
128
|
-
"headers": {"Authorization": "***", "X-Request-ID": "abc"},
|
|
129
|
+
"headers": {"Authorization": "***", "X-OPENFIGI-APIKEY": "***", "X-Request-ID": "abc"},
|
|
129
130
|
"json_data": None,
|
|
130
131
|
"content": None,
|
|
131
132
|
}
|
|
@@ -797,3 +798,102 @@ def test_http_model_ignores_invalid_retry_after(monkeypatch):
|
|
|
797
798
|
model(HTTPRequestContext())
|
|
798
799
|
|
|
799
800
|
assert sleeps == [1.0]
|
|
801
|
+
|
|
802
|
+
|
|
803
|
+
def test_redaction_policy_matches_normalized_names_substrings_and_patterns():
|
|
804
|
+
policy = RedactionPolicy(names={"X-Session"}, substrings={"Signature"}, patterns=[r"^sig\d+$"], mask="<redacted>")
|
|
805
|
+
|
|
806
|
+
assert policy.is_sensitive("x_session") and policy.is_sensitive("X-SESSION")
|
|
807
|
+
assert policy.is_sensitive("X-Amz-Signature")
|
|
808
|
+
assert policy.is_sensitive("sig2") and not policy.is_sensitive("signal")
|
|
809
|
+
assert not policy.is_sensitive("apiKey")
|
|
810
|
+
assert policy.with_names("apiKey").is_sensitive("API_KEY")
|
|
811
|
+
assert policy.redact_mapping({"X-Session": "s", "date": "d"}) == {"X-Session": "<redacted>", "date": "d"}
|
|
812
|
+
assert policy.redact_url("https://h/p?sig2=a&date=d") == "https://h/p?sig2=<redacted>&date=d"
|
|
813
|
+
assert policy.redact_text("failed for https://h/p?x-session=a&date=d") == "failed for https://h/p?x-session=<redacted>&date=d"
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
def test_default_redaction_policy_covers_common_credential_names():
|
|
817
|
+
policy = RedactionPolicy()
|
|
818
|
+
|
|
819
|
+
for name in [
|
|
820
|
+
"Authorization",
|
|
821
|
+
"Proxy-Authorization",
|
|
822
|
+
"Cookie",
|
|
823
|
+
"Set-Cookie",
|
|
824
|
+
"password",
|
|
825
|
+
"apiKey",
|
|
826
|
+
"X-OPENFIGI-APIKEY",
|
|
827
|
+
"page_token",
|
|
828
|
+
"client_secret",
|
|
829
|
+
]:
|
|
830
|
+
assert policy.is_sensitive(name), name
|
|
831
|
+
for name in ["date", "ticker", "X-Request-ID", "limit"]:
|
|
832
|
+
assert not policy.is_sensitive(name), name
|
|
833
|
+
|
|
834
|
+
|
|
835
|
+
def test_http_model_applies_configured_and_auth_redaction_names_everywhere(caplog, monkeypatch):
|
|
836
|
+
monkeypatch.setattr("ccflow_http.base.sleep", lambda seconds: None)
|
|
837
|
+
seen = []
|
|
838
|
+
|
|
839
|
+
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
840
|
+
seen.append(request)
|
|
841
|
+
if len(seen) == 1:
|
|
842
|
+
raise httpx2.ConnectError(f"connection reset for {request.url}", request=request)
|
|
843
|
+
return httpx2.Response(200, json={"ok": True})
|
|
844
|
+
|
|
845
|
+
model = HTTPModel(
|
|
846
|
+
config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(handler)),
|
|
847
|
+
path="/v1/items",
|
|
848
|
+
query={"sig": "query-secret", "date": "2025-01-02"},
|
|
849
|
+
headers={"X-Vendor-Session": "header-secret"},
|
|
850
|
+
auth=HTTPAuth(strategy="api_key_header", name="X-Partner", value="auth-secret"),
|
|
851
|
+
redaction=RedactionPolicy(names={"X-Vendor-Session"}, patterns=[r"^sig$"]),
|
|
852
|
+
retry_policy=HTTPRetryPolicy(max_attempts=2, wait_initial=0.0, wait_max=0.0, wait_jitter=0.0),
|
|
853
|
+
)
|
|
854
|
+
|
|
855
|
+
with caplog.at_level("INFO", logger="httpx2"):
|
|
856
|
+
result = model(HTTPRequestContext())
|
|
857
|
+
dump = safe_request_dump(model.build_request(HTTPRequestContext()))
|
|
858
|
+
|
|
859
|
+
assert seen[-1].headers["X-Partner"] == "auth-secret" and seen[-1].headers["X-Vendor-Session"] == "header-secret"
|
|
860
|
+
assert dump["headers"]["X-Partner"] == "***" and dump["headers"]["X-Vendor-Session"] == "***"
|
|
861
|
+
assert dump["params"] == {"sig": "***", "date": "2025-01-02"}
|
|
862
|
+
assert result.url == "https://api.example.test/v1/items?sig=***&date=2025-01-02"
|
|
863
|
+
assert "query-secret" not in caplog.text
|
|
864
|
+
assert "query-secret" not in result.retry_events[0]["message"]
|
|
865
|
+
|
|
866
|
+
|
|
867
|
+
def test_http_model_redacts_sensitive_response_headers():
|
|
868
|
+
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
869
|
+
return httpx2.Response(
|
|
870
|
+
200,
|
|
871
|
+
json={"ok": True},
|
|
872
|
+
headers={"Set-Cookie": "session=cookie-secret", "X-Auth-Token": "token-secret", "X-Request-ID": "req-1"},
|
|
873
|
+
)
|
|
874
|
+
|
|
875
|
+
model = HTTPModel(
|
|
876
|
+
config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(handler)),
|
|
877
|
+
path="/v1/items",
|
|
878
|
+
)
|
|
879
|
+
|
|
880
|
+
result = model(HTTPRequestContext())
|
|
881
|
+
|
|
882
|
+
assert result.headers["set-cookie"] == "***"
|
|
883
|
+
assert result.headers["x-auth-token"] == "***"
|
|
884
|
+
assert result.headers["x-request-id"] == "req-1"
|
|
885
|
+
assert "secret" not in repr(result)
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
def test_http_auth_repr_omits_secrets():
|
|
889
|
+
auths = [
|
|
890
|
+
HTTPAuth(strategy="bearer", token="bearer-secret"),
|
|
891
|
+
HTTPAuth(strategy="api_key_header", name="X-Api-Key", value="key-secret"),
|
|
892
|
+
HTTPAuth(strategy="basic", username="svc", password="basic-secret"),
|
|
893
|
+
]
|
|
894
|
+
|
|
895
|
+
for auth in auths:
|
|
896
|
+
assert "secret" not in repr(auth)
|
|
897
|
+
assert "secret" not in str(auth)
|
|
898
|
+
assert "X-Api-Key" in repr(auths[1])
|
|
899
|
+
assert "svc" in repr(auths[2])
|
|
@@ -12,7 +12,7 @@ authors = [
|
|
|
12
12
|
description = "ccflow models for HTTP"
|
|
13
13
|
readme = "README.md"
|
|
14
14
|
license = { text = "Apache-2.0" }
|
|
15
|
-
version = "0.
|
|
15
|
+
version = "0.4.1"
|
|
16
16
|
requires-python = ">=3.11"
|
|
17
17
|
keywords = []
|
|
18
18
|
|
|
@@ -43,7 +43,7 @@ develop = [
|
|
|
43
43
|
"codespell",
|
|
44
44
|
"hatchling",
|
|
45
45
|
"mdformat",
|
|
46
|
-
"mdformat-
|
|
46
|
+
"mdformat-myst",
|
|
47
47
|
"mdformat-tables>=1",
|
|
48
48
|
"pytest",
|
|
49
49
|
"pytest-cov",
|
|
@@ -61,7 +61,7 @@ Repository = "https://github.com/1kbgz/ccflow-http"
|
|
|
61
61
|
Homepage = "https://github.com/1kbgz/ccflow-http"
|
|
62
62
|
|
|
63
63
|
[tool.bumpversion]
|
|
64
|
-
current_version = "0.
|
|
64
|
+
current_version = "0.4.1"
|
|
65
65
|
commit = true
|
|
66
66
|
tag = true
|
|
67
67
|
commit_args = "-s"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|