ccflow-http 0.3.0__tar.gz → 0.4.0__tar.gz

{ccflow_http-0.3.0 → ccflow_http-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ccflow-http
-Version: 0.3.0
+Version: 0.4.0
 Summary: ccflow models for HTTP
 Project-URL: Repository, https://github.com/1kbgz/ccflow-http
 Project-URL: Homepage, https://github.com/1kbgz/ccflow-http
@@ -56,7 +56,7 @@ It should not contain provider-specific endpoint catalogs. Domain packages can c
 
 ## Current Status
 
-- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, redaction of secret query values (including the configured `api_key_query` parameter) from `httpx2` request logs, request dumps, result URLs, retry events, and errors, `HTTPDownloadModel` for streaming a response to a local file with size and SHA-256, `HTTPArtifactCaptureModel` for storing a dated download plus a manifest through a `ccflow-etl` file writer, and mocked `httpx2` transport tests.
+- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, a configurable `RedactionPolicy` (`HTTPModel.redaction`: exact names, substrings, and patterns, with `auth` header and query names added automatically) that masks secret header and query values in `httpx2` request logs, request dumps, result URLs, retry events, and errors, `HTTPDownloadModel` for streaming a response to a local file with size and SHA-256, `HTTPArtifactCaptureModel` for storing a dated download plus a manifest through a `ccflow-etl` file writer, and mocked `httpx2` transport tests.
 - Partial: the shared execution policy is currently consumed inside `HTTPModel` for sequential request spacing; broader evaluator-level concurrency coordination still belongs in evaluator integrations.
 - Missing: broader integration examples and provider-specific subclasses/configs in downstream packages.
 

{ccflow_http-0.3.0 → ccflow_http-0.4.0}/README.md

@@ -15,7 +15,7 @@ It should not contain provider-specific endpoint catalogs. Domain packages can c
 
 ## Current Status
 
-- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, redaction of secret query values (including the configured `api_key_query` parameter) from `httpx2` request logs, request dumps, result URLs, retry events, and errors, `HTTPDownloadModel` for streaming a response to a local file with size and SHA-256, `HTTPArtifactCaptureModel` for storing a dated download plus a manifest through a `ccflow-etl` file writer, and mocked `httpx2` transport tests.
+- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, a configurable `RedactionPolicy` (`HTTPModel.redaction`: exact names, substrings, and patterns, with `auth` header and query names added automatically) that masks secret header and query values in `httpx2` request logs, request dumps, result URLs, retry events, and errors, `HTTPDownloadModel` for streaming a response to a local file with size and SHA-256, `HTTPArtifactCaptureModel` for storing a dated download plus a manifest through a `ccflow-etl` file writer, and mocked `httpx2` transport tests.
 - Partial: the shared execution policy is currently consumed inside `HTTPModel` for sequential request spacing; broader evaluator-level concurrency coordination still belongs in evaluator integrations.
 - Missing: broader integration examples and provider-specific subclasses/configs in downstream packages.
 

{ccflow_http-0.3.0 → ccflow_http-0.4.0}/ccflow_http/__init__.py

@@ -1,4 +1,4 @@
 from .base import *
 from .download import *
 
-__version__ = "0.3.0"
+__version__ = "0.4.0"

{ccflow_http-0.3.0 → ccflow_http-0.4.0}/ccflow_http/base.py

@@ -17,7 +17,7 @@ from ccflow import BaseModel, CallableModel, ContextBase, Flow, GenericResult, P
 from ccflow.utils.retry import RetryPolicy
 from ccflow_etl import ExecutionPolicy
 from jinja2 import Environment
-from pydantic import Field, PrivateAttr
+from pydantic import Field, PrivateAttr, field_validator
 
 __all__ = (
     "HTTPAuth",
@@ -29,6 +29,7 @@ __all__ = (
     "HTTPResponseResult",
     "HTTPResult",
     "HTTPRetryPolicy",
+    "RedactionPolicy",
     "redact_mapping",
     "redact_url",
     "safe_request_dump",
@@ -78,38 +79,79 @@ class HTTPRequest(BaseModel):
     headers: dict[str, str] = Field(default_factory=dict)
     json_data: Any | None = None
     content: bytes | str | None = None
-    # Query parameter names that carry credentials under names the key heuristic may not recognize, such as api_key_query auth.
-    _sensitive_params: frozenset[str] = PrivateAttr(default_factory=frozenset)
+    # Redaction for this request: the model's policy plus any names its auth places credentials under.
+    _redaction: "RedactionPolicy" = PrivateAttr(default_factory=lambda: _DEFAULT_REDACTION)
 
 
-# Extra sensitive query names for the request in flight, so the httpx2 log filter can redact them.
-_REQUEST_SENSITIVE_PARAMS: ContextVar[frozenset[str]] = ContextVar("ccflow_http_request_sensitive_params", default=frozenset())
 _QUERY_PAIR = re.compile(r"([?&])([^=&\s'\"]+)=([^&\s'\"]*)")
 
 
-def _is_sensitive_key(key: str, extra: Collection[str] = ()) -> bool:
-    normalized_key = key.lower().replace("_", "").replace("-", "")
-    return key in extra or normalized_key in {"apikey", "authorization", "password"} or "token" in normalized_key or "secret" in normalized_key
+def _normalize_name(name: str) -> str:
+    return name.lower().replace("_", "").replace("-", "")
 
 
-def redact_mapping(values: dict[str, Any], extra: Collection[str] = ()) -> dict[str, Any]:
-    return {key: "***" if _is_sensitive_key(key, extra) else value for key, value in values.items()}
+class RedactionPolicy(BaseModel):
+    """Which header and query parameter names carry secrets, and how their values are masked.
 
+    Names are compared after lowercasing and removing hyphens and underscores, so ``X-Api-Key`` and ``x_api_key``
+    are the same name. A name is sensitive when it is one of ``names``, contains one of ``substrings``, or matches one
+    of ``patterns`` (regular expressions searched in the normalized name). Set it on ``HTTPModel.redaction`` to cover
+    a provider's own header or parameter names; names used by ``auth`` are added automatically.
+    """
 
-def redact_url(url: str, extra: Collection[str] = ()) -> str:
-    parts = urlsplit(url)
-    if not parts.query:
-        return url
-    pairs = []
-    for pair in parts.query.split("&"):
-        key = pair.split("=", 1)[0]
-        pairs.append(f"{key}=***" if _is_sensitive_key(unquote_plus(key), extra) else pair)
-    return urlunsplit(parts._replace(query="&".join(pairs)))
+    names: set[str] = Field(default_factory=lambda: {"authorization", "proxyauthorization", "cookie", "setcookie", "password"})
+    substrings: set[str] = Field(default_factory=lambda: {"apikey", "token", "secret", "password"})
+    patterns: list[str] = Field(default_factory=list)
+    mask: str = "***"
 
+    @field_validator("names", "substrings", mode="after")
+    @classmethod
+    def _normalize(cls, values: set[str]) -> set[str]:
+        return {_normalize_name(value) for value in values}
 
-def _redact_query_text(text: str, extra: Collection[str] = ()) -> str:
-    """Redact secret query values that appear anywhere in free text, such as an exception message quoting a URL."""
-    return _QUERY_PAIR.sub(lambda match: f"{match[1]}{match[2]}=***" if _is_sensitive_key(unquote_plus(match[2]), extra) else match[0], text)
+    def with_names(self, *names: str) -> "RedactionPolicy":
+        """Return a copy that also treats ``names`` as sensitive."""
+        if not names:
+            return self
+        return self.model_copy(update={"names": self.names | {_normalize_name(name) for name in names}})
+
+    def is_sensitive(self, name: str) -> bool:
+        normalized = _normalize_name(name)
+        return (
+            normalized in self.names
+            or any(substring in normalized for substring in self.substrings)
+            or any(re.search(pattern, normalized) for pattern in self.patterns)
+        )
+
+    def redact_mapping(self, values: dict[str, Any]) -> dict[str, Any]:
+        return {key: self.mask if self.is_sensitive(key) else value for key, value in values.items()}
+
+    def redact_url(self, url: str) -> str:
+        parts = urlsplit(url)
+        if not parts.query:
+            return url
+        pairs = []
+        for pair in parts.query.split("&"):
+            key = pair.split("=", 1)[0]
+            pairs.append(f"{key}={self.mask}" if self.is_sensitive(unquote_plus(key)) else pair)
+        return urlunsplit(parts._replace(query="&".join(pairs)))
+
+    def redact_text(self, text: str) -> str:
+        """Redact secret query values that appear anywhere in free text, such as an exception message quoting a URL."""
+        return _QUERY_PAIR.sub(lambda match: f"{match[1]}{match[2]}={self.mask}" if self.is_sensitive(unquote_plus(match[2])) else match[0], text)
+
+
+_DEFAULT_REDACTION = RedactionPolicy()
+# Redaction policy of the request in flight, so the httpx2 log filter can apply it.
+_REQUEST_REDACTION: ContextVar[RedactionPolicy | None] = ContextVar("ccflow_http_request_redaction", default=None)
+
+
+def redact_mapping(values: dict[str, Any], extra: Collection[str] = (), policy: RedactionPolicy | None = None) -> dict[str, Any]:
+    return (policy or _DEFAULT_REDACTION).with_names(*extra).redact_mapping(values)
+
+
+def redact_url(url: str, extra: Collection[str] = (), policy: RedactionPolicy | None = None) -> str:
+    return (policy or _DEFAULT_REDACTION).with_names(*extra).redact_url(url)
 
 
 class _RedactURLFilter(Filter):
@@ -117,8 +159,8 @@ class _RedactURLFilter(Filter):
 
     def filter(self, record: LogRecord) -> bool:
         if isinstance(record.args, tuple):
-            extra = _REQUEST_SENSITIVE_PARAMS.get()
-            record.args = tuple(redact_url(str(arg), extra) if isinstance(arg, httpx2.URL) else arg for arg in record.args)
+            policy = _REQUEST_REDACTION.get() or _DEFAULT_REDACTION
+            record.args = tuple(policy.redact_url(str(arg)) if isinstance(arg, httpx2.URL) else arg for arg in record.args)
         return True
 
 
@@ -127,8 +169,8 @@ getLogger("httpx2").addFilter(_RedactURLFilter())
 
 def safe_request_dump(request: HTTPRequest) -> dict[str, Any]:
     request_data = request.model_dump(exclude={"type_"})
-    request_data["params"] = redact_mapping(request_data.get("params", {}), request._sensitive_params)
-    request_data["headers"] = redact_mapping(request_data.get("headers", {}))
+    request_data["params"] = request._redaction.redact_mapping(request_data.get("params", {}))
+    request_data["headers"] = request._redaction.redact_mapping(request_data.get("headers", {}))
     return request_data
 
 
@@ -241,6 +283,9 @@ class HTTPModel(CallableModel):
         description="Error status codes returned as results with value None instead of raising, such as 404 for lookups that may miss.",
     )
     execution_policy: ExecutionPolicy | None = None
+    redaction: RedactionPolicy = Field(
+        default_factory=RedactionPolicy, description="Header and query parameter names whose values are masked in logs, results, and errors."
+    )
     paginate: bool = False
     max_pages: int = 100
     pagination_mode: HTTPPaginationMode = "next_url"
@@ -349,8 +394,10 @@ class HTTPModel(CallableModel):
             json_data=context.json_body if context.json_body is not None else self.json_body,
             content=context.content if context.content is not None else self.content,
         )
-        if self.auth.strategy == "api_key_query":
-            request._sensitive_params = frozenset({self._render_required_auth_value(self.auth.name, "name", data)})
+        auth_names = []
+        if self.auth.strategy in ("api_key_query", "api_key_header"):
+            auth_names.append(self._render_required_auth_value(self.auth.name, "name", data))
+        request._redaction = self.redaction.with_names(*auth_names)
         return request
 
     def _response_value(self, response: httpx2.Response) -> Any:
@@ -489,7 +536,7 @@ class HTTPModel(CallableModel):
                                 delay_seconds=delay_seconds,
                                 exception_type=type(exc).__name__,
                                 category=retry_policy.exception_category(exc),
-                                message=_redact_query_text(str(exc), request._sensitive_params),
+                                message=request._redaction.redact_text(str(exc)),
                             )
                         )
                         self._sleep(delay_seconds)
@@ -501,7 +548,7 @@ class HTTPModel(CallableModel):
                         outcome="failed",
                         exception_type=type(exc).__name__,
                         category=retry_policy.exception_category(exc),
-                        message=_redact_query_text(str(exc), request._sensitive_params),
+                        message=request._redaction.redact_text(str(exc)),
                     )
                 )
                 # httpx2 exceptions keep the full request, including secret query values, so do not chain them.
@@ -531,7 +578,7 @@ class HTTPModel(CallableModel):
         next_url_parts = urlsplit(next_url)
         next_url_params = dict(parse_qsl(next_url_parts.query, keep_blank_values=True))
         for key, value in request.params.items():
-            if key not in next_url_params and _is_sensitive_key(key, request._sensitive_params):
+            if key not in next_url_params and request._redaction.is_sensitive(key):
                 next_url_params[key] = value
         next_url_without_query = urlunsplit((next_url_parts.scheme, next_url_parts.netloc, next_url_parts.path, "", next_url_parts.fragment))
         return request.model_copy(update={"url": next_url_without_query, "params": next_url_params})
@@ -582,9 +629,9 @@ class HTTPModel(CallableModel):
     @Flow.call
     def __call__(self, context: HTTPRequestContext) -> HTTPResult:
         request = self._initial_paginated_request(self.build_request(context))
-        sensitive_params = request._sensitive_params
+        redaction = request._redaction
 
-        token = _REQUEST_SENSITIVE_PARAMS.set(sensitive_params)
+        token = _REQUEST_REDACTION.set(redaction)
         try:
             with httpx2.Client(**self._client_kwargs()) as client:
                 values = []
@@ -615,13 +662,13 @@ class HTTPModel(CallableModel):
                         break
                     request = next_request
         finally:
-            _REQUEST_SENSITIVE_PARAMS.reset(token)
+            _REQUEST_REDACTION.reset(token)
 
         return HTTPResult(
             value=None if accepted_error else (self._merge_page_values(values) if self.paginate else values[-1]),
             status_code=response.status_code,
             headers=dict(response.headers or {}),
-            url=redact_url(str(response.url), sensitive_params),
+            url=redaction.redact_url(str(response.url)),
             attempts=total_attempts,
             pages=pages,
             rate_limit=self._rate_limit_headers(dict(response.headers or {})),

{ccflow_http-0.3.0 → ccflow_http-0.4.0}/ccflow_http/download.py

@@ -10,7 +10,7 @@ from ccflow import CallableModel, DateContext, Flow, ResultBase
 from ccflow_etl import ArtifactWriteFileContext, ArtifactWriteFileModel, ArtifactWriteFileResult
 from pydantic import Field
 
-from .base import _REQUEST_SENSITIVE_PARAMS, HTTPModel, HTTPRequest, HTTPRequestContext, redact_url
+from .base import _REQUEST_REDACTION, HTTPModel, HTTPRequest, HTTPRequestContext
 
 __all__ = (
     "HTTPArtifactCaptureModel",
@@ -69,13 +69,13 @@ class HTTPDownloadModel(HTTPModel):
     @Flow.call
     def __call__(self, context: HTTPDownloadContext) -> HTTPDownloadResult:
         request = self.build_request(context)
-        sensitive_params = request._sensitive_params
+        redaction = request._redaction
         path = context.output_path
         path.parent.mkdir(parents=True, exist_ok=True)
         temp_path = path.with_name(f".{path.name}.{uuid4().hex}.tmp")
         digest = sha256()
         size = 0
-        token = _REQUEST_SENSITIVE_PARAMS.set(sensitive_params)
+        token = _REQUEST_REDACTION.set(redaction)
         try:
             with httpx2.Client(**self._client_kwargs()) as client:
                 response, attempts, events, _ = self._request_once(client, request, None)
@@ -92,13 +92,13 @@ class HTTPDownloadModel(HTTPModel):
             temp_path.unlink(missing_ok=True)
             raise
         finally:
-            _REQUEST_SENSITIVE_PARAMS.reset(token)
+            _REQUEST_REDACTION.reset(token)
         return HTTPDownloadResult(
             path=str(path),
             size=size,
             sha256=digest.hexdigest(),
             status_code=response.status_code,
-            url=redact_url(str(response.url), sensitive_params),
+            url=redaction.redact_url(str(response.url)),
             etag=response.headers.get("etag"),
             last_modified=response.headers.get("last-modified"),
             content_type=response.headers.get("content-type"),

{ccflow_http-0.3.0 → ccflow_http-0.4.0}/ccflow_http/tests/test_model.py

@@ -18,6 +18,7 @@ from ccflow_http import (
     HTTPRequestContext,
     HTTPResponseResult,
     HTTPRetryPolicy,
+    RedactionPolicy,
     redact_url,
     safe_request_dump,
 )
@@ -118,14 +119,14 @@ def test_safe_request_dump_redacts_secret_params_and_headers():
         method="GET",
         url="/v1/tickers/AAA",
         params={"date": "2024-01-03", "apiKey": "query-secret", "page_token": "cursor-secret"},
-        headers={"Authorization": "Bearer header-secret", "X-Request-ID": "abc"},
+        headers={"Authorization": "Bearer header-secret", "X-OPENFIGI-APIKEY": "vendor-secret", "X-Request-ID": "abc"},
     )
 
     assert safe_request_dump(request) == {
         "method": "GET",
         "url": "/v1/tickers/AAA",
         "params": {"date": "2024-01-03", "apiKey": "***", "page_token": "***"},
-        "headers": {"Authorization": "***", "X-Request-ID": "abc"},
+        "headers": {"Authorization": "***", "X-OPENFIGI-APIKEY": "***", "X-Request-ID": "abc"},
         "json_data": None,
         "content": None,
     }
@@ -797,3 +798,67 @@ def test_http_model_ignores_invalid_retry_after(monkeypatch):
     model(HTTPRequestContext())
 
     assert sleeps == [1.0]
+
+
+def test_redaction_policy_matches_normalized_names_substrings_and_patterns():
+    policy = RedactionPolicy(names={"X-Session"}, substrings={"Signature"}, patterns=[r"^sig\d+$"], mask="<redacted>")
+
+    assert policy.is_sensitive("x_session") and policy.is_sensitive("X-SESSION")
+    assert policy.is_sensitive("X-Amz-Signature")
+    assert policy.is_sensitive("sig2") and not policy.is_sensitive("signal")
+    assert not policy.is_sensitive("apiKey")
+    assert policy.with_names("apiKey").is_sensitive("API_KEY")
+    assert policy.redact_mapping({"X-Session": "s", "date": "d"}) == {"X-Session": "<redacted>", "date": "d"}
+    assert policy.redact_url("https://h/p?sig2=a&date=d") == "https://h/p?sig2=<redacted>&date=d"
+    assert policy.redact_text("failed for https://h/p?x-session=a&date=d") == "failed for https://h/p?x-session=<redacted>&date=d"
+
+
+def test_default_redaction_policy_covers_common_credential_names():
+    policy = RedactionPolicy()
+
+    for name in [
+        "Authorization",
+        "Proxy-Authorization",
+        "Cookie",
+        "Set-Cookie",
+        "password",
+        "apiKey",
+        "X-OPENFIGI-APIKEY",
+        "page_token",
+        "client_secret",
+    ]:
+        assert policy.is_sensitive(name), name
+    for name in ["date", "ticker", "X-Request-ID", "limit"]:
+        assert not policy.is_sensitive(name), name
+
+
+def test_http_model_applies_configured_and_auth_redaction_names_everywhere(caplog, monkeypatch):
+    monkeypatch.setattr("ccflow_http.base.sleep", lambda seconds: None)
+    seen = []
+
+    def handler(request: httpx2.Request) -> httpx2.Response:
+        seen.append(request)
+        if len(seen) == 1:
+            raise httpx2.ConnectError(f"connection reset for {request.url}", request=request)
+        return httpx2.Response(200, json={"ok": True})
+
+    model = HTTPModel(
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(handler)),
+        path="/v1/items",
+        query={"sig": "query-secret", "date": "2025-01-02"},
+        headers={"X-Vendor-Session": "header-secret"},
+        auth=HTTPAuth(strategy="api_key_header", name="X-Partner", value="auth-secret"),
+        redaction=RedactionPolicy(names={"X-Vendor-Session"}, patterns=[r"^sig$"]),
+        retry_policy=HTTPRetryPolicy(max_attempts=2, wait_initial=0.0, wait_max=0.0, wait_jitter=0.0),
+    )
+
+    with caplog.at_level("INFO", logger="httpx2"):
+        result = model(HTTPRequestContext())
+    dump = safe_request_dump(model.build_request(HTTPRequestContext()))
+
+    assert seen[-1].headers["X-Partner"] == "auth-secret" and seen[-1].headers["X-Vendor-Session"] == "header-secret"
+    assert dump["headers"]["X-Partner"] == "***" and dump["headers"]["X-Vendor-Session"] == "***"
+    assert dump["params"] == {"sig": "***", "date": "2025-01-02"}
+    assert result.url == "https://api.example.test/v1/items?sig=***&date=2025-01-02"
+    assert "query-secret" not in caplog.text
+    assert "query-secret" not in result.retry_events[0]["message"]

{ccflow_http-0.3.0 → ccflow_http-0.4.0}/pyproject.toml

@@ -12,7 +12,7 @@ authors = [
 description = "ccflow models for HTTP"
 readme = "README.md"
 license = { text = "Apache-2.0" }
-version = "0.3.0"
+version = "0.4.0"
 requires-python = ">=3.11"
 keywords = []
 
@@ -61,7 +61,7 @@ Repository = "https://github.com/1kbgz/ccflow-http"
 Homepage = "https://github.com/1kbgz/ccflow-http"
 
 [tool.bumpversion]
-current_version = "0.3.0"
+current_version = "0.4.0"
 commit = true
 tag = true
 commit_args = "-s"