ccflow-http 0.1.1__tar.gz → 0.3.0__tar.gz

{ccflow_http-0.1.1 → ccflow_http-0.3.0}/.gitignore

@@ -13,6 +13,10 @@ __pycache__/
 *.exp
 *.lib
 
+# vcpkg
+vcpkg/
+vcpkg_installed/
+
 # Rust
 target
 target-capi
@@ -142,6 +146,7 @@ ccflow_http/labextension
 
 # Emscripten SDK (locally installed)
 emsdk
+.pyodide_build/
 
 # Mac
 .DS_Store

{ccflow_http-0.1.1 → ccflow_http-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: ccflow-http
-Version: 0.1.1
+Version: 0.3.0
 Summary: ccflow models for HTTP
 Project-URL: Repository, https://github.com/1kbgz/ccflow-http
 Project-URL: Homepage, https://github.com/1kbgz/ccflow-http
@@ -17,9 +17,9 @@ Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.11
-Requires-Dist: ccflow-etl>=0.1.0
+Requires-Dist: ccflow-etl>=0.4.0
 Requires-Dist: ccflow>=0.8.5
-Requires-Dist: httpx
+Requires-Dist: httpx2
 Requires-Dist: jinja2
 Provides-Extra: develop
 Requires-Dist: build; extra == 'develop'
@@ -28,6 +28,7 @@ Requires-Dist: check-dist; extra == 'develop'
 Requires-Dist: codespell; extra == 'develop'
 Requires-Dist: hatchling; extra == 'develop'
 Requires-Dist: mdformat; extra == 'develop'
+Requires-Dist: mdformat-frontmatter; extra == 'develop'
 Requires-Dist: mdformat-tables>=1; extra == 'develop'
 Requires-Dist: pytest; extra == 'develop'
 Requires-Dist: pytest-cov; extra == 'develop'
@@ -55,7 +56,7 @@ It should not contain provider-specific endpoint catalogs. Domain packages can c
 
 ## Current Status
 
-- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, and mocked `httpx` transport tests.
+- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, redaction of secret query values (including the configured `api_key_query` parameter) from `httpx2` request logs, request dumps, result URLs, retry events, and errors, `HTTPDownloadModel` for streaming a response to a local file with size and SHA-256, `HTTPArtifactCaptureModel` for storing a dated download plus a manifest through a `ccflow-etl` file writer, and mocked `httpx2` transport tests.
 - Partial: the shared execution policy is currently consumed inside `HTTPModel` for sequential request spacing; broader evaluator-level concurrency coordination still belongs in evaluator integrations.
 - Missing: broader integration examples and provider-specific subclasses/configs in downstream packages.
 
@@ -67,7 +68,7 @@ It should not contain provider-specific endpoint catalogs. Domain packages can c
 
 ## Test Convention
 
-Default tests should use mocked `httpx` transports or local fixtures. They should not require live network calls or provider credentials.
+Default tests should use mocked `httpx2` transports or local fixtures. They should not require live network calls or provider credentials.
 
 > [!NOTE]
 > This library was generated using [copier](https://copier.readthedocs.io/en/stable/) from the [Base Python Project Template repository](https://github.com/python-project-templates/base).

{ccflow_http-0.1.1 → ccflow_http-0.3.0}/README.md

@@ -15,7 +15,7 @@ It should not contain provider-specific endpoint catalogs. Domain packages can c
 
 ## Current Status
 
-- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, and mocked `httpx` transport tests.
+- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, redaction of secret query values (including the configured `api_key_query` parameter) from `httpx2` request logs, request dumps, result URLs, retry events, and errors, `HTTPDownloadModel` for streaming a response to a local file with size and SHA-256, `HTTPArtifactCaptureModel` for storing a dated download plus a manifest through a `ccflow-etl` file writer, and mocked `httpx2` transport tests.
 - Partial: the shared execution policy is currently consumed inside `HTTPModel` for sequential request spacing; broader evaluator-level concurrency coordination still belongs in evaluator integrations.
 - Missing: broader integration examples and provider-specific subclasses/configs in downstream packages.
 
@@ -27,7 +27,7 @@ It should not contain provider-specific endpoint catalogs. Domain packages can c
 
 ## Test Convention
 
-Default tests should use mocked `httpx` transports or local fixtures. They should not require live network calls or provider credentials.
+Default tests should use mocked `httpx2` transports or local fixtures. They should not require live network calls or provider credentials.
 
 > [!NOTE]
 > This library was generated using [copier](https://copier.readthedocs.io/en/stable/) from the [Base Python Project Template repository](https://github.com/python-project-templates/base).

ccflow_http-0.3.0/ccflow_http/__init__.py

@@ -0,0 +1,4 @@
+from .base import *
+from .download import *
+
+__version__ = "0.3.0"

{ccflow_http-0.1.1 → ccflow_http-0.3.0}/ccflow_http/base.py

@@ -1,19 +1,23 @@
+import re
 from base64 import b64encode
+from collections.abc import Collection
+from contextvars import ContextVar
 from csv import DictReader
 from datetime import UTC, datetime
 from email.utils import parsedate_to_datetime
 from gzip import decompress
 from io import StringIO
+from logging import Filter, LogRecord, getLogger
 from time import monotonic, sleep
 from typing import Any, Literal
-from urllib.parse import parse_qsl, urlsplit, urlunsplit
+from urllib.parse import parse_qsl, unquote_plus, urlsplit, urlunsplit
 
-import httpx
+import httpx2
 from ccflow import BaseModel, CallableModel, ContextBase, Flow, GenericResult, PyObjectPath
 from ccflow.utils.retry import RetryPolicy
 from ccflow_etl import ExecutionPolicy
 from jinja2 import Environment
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 
 __all__ = (
     "HTTPAuth",
@@ -26,6 +30,7 @@ __all__ = (
     "HTTPResult",
     "HTTPRetryPolicy",
     "redact_mapping",
+    "redact_url",
     "safe_request_dump",
 )
 
@@ -73,22 +78,56 @@ class HTTPRequest(BaseModel):
     headers: dict[str, str] = Field(default_factory=dict)
     json_data: Any | None = None
     content: bytes | str | None = None
+    # Query parameter names that carry credentials under names the key heuristic may not recognize, such as api_key_query auth.
+    _sensitive_params: frozenset[str] = PrivateAttr(default_factory=frozenset)
 
 
-def redact_mapping(values: dict[str, Any]) -> dict[str, Any]:
-    redacted = {}
-    for key, value in values.items():
-        normalized_key = key.lower().replace("_", "").replace("-", "")
-        if normalized_key in {"apikey", "authorization", "password"} or "token" in normalized_key or "secret" in normalized_key:
-            redacted[key] = "***"
-        else:
-            redacted[key] = value
-    return redacted
+# Extra sensitive query names for the request in flight, so the httpx2 log filter can redact them.
+_REQUEST_SENSITIVE_PARAMS: ContextVar[frozenset[str]] = ContextVar("ccflow_http_request_sensitive_params", default=frozenset())
+_QUERY_PAIR = re.compile(r"([?&])([^=&\s'\"]+)=([^&\s'\"]*)")
+
+
+def _is_sensitive_key(key: str, extra: Collection[str] = ()) -> bool:
+    normalized_key = key.lower().replace("_", "").replace("-", "")
+    return key in extra or normalized_key in {"apikey", "authorization", "password"} or "token" in normalized_key or "secret" in normalized_key
+
+
+def redact_mapping(values: dict[str, Any], extra: Collection[str] = ()) -> dict[str, Any]:
+    return {key: "***" if _is_sensitive_key(key, extra) else value for key, value in values.items()}
+
+
+def redact_url(url: str, extra: Collection[str] = ()) -> str:
+    parts = urlsplit(url)
+    if not parts.query:
+        return url
+    pairs = []
+    for pair in parts.query.split("&"):
+        key = pair.split("=", 1)[0]
+        pairs.append(f"{key}=***" if _is_sensitive_key(unquote_plus(key), extra) else pair)
+    return urlunsplit(parts._replace(query="&".join(pairs)))
+
+
+def _redact_query_text(text: str, extra: Collection[str] = ()) -> str:
+    """Redact secret query values that appear anywhere in free text, such as an exception message quoting a URL."""
+    return _QUERY_PAIR.sub(lambda match: f"{match[1]}{match[2]}=***" if _is_sensitive_key(unquote_plus(match[2]), extra) else match[0], text)
+
+
+class _RedactURLFilter(Filter):
+    """Redact secret query values from URLs that httpx2 logs for each request."""
+
+    def filter(self, record: LogRecord) -> bool:
+        if isinstance(record.args, tuple):
+            extra = _REQUEST_SENSITIVE_PARAMS.get()
+            record.args = tuple(redact_url(str(arg), extra) if isinstance(arg, httpx2.URL) else arg for arg in record.args)
+        return True
+
+
+getLogger("httpx2").addFilter(_RedactURLFilter())
 
 
 def safe_request_dump(request: HTTPRequest) -> dict[str, Any]:
     request_data = request.model_dump(exclude={"type_"})
-    request_data["params"] = redact_mapping(request_data.get("params", {}))
+    request_data["params"] = redact_mapping(request_data.get("params", {}), request._sensitive_params)
     request_data["headers"] = redact_mapping(request_data.get("headers", {}))
     return request_data
 
@@ -105,9 +144,7 @@ class HTTPRetryEvent(BaseModel):
 
 class HTTPRetryPolicy(RetryPolicy):
     retry_status_codes: list[int] = Field(default_factory=lambda: [429, 500, 502, 503, 504])
-    retry_exceptions: list[PyObjectPath] = Field(
-        default_factory=lambda: [PyObjectPath.validate(httpx.TimeoutException), PyObjectPath.validate(httpx.ConnectError)]
-    )
+    retry_exceptions: list[PyObjectPath] = Field(default_factory=lambda: [PyObjectPath.validate(httpx2.TransportError)])
     timeout_exception_types: list[str] = Field(
         default_factory=lambda: ["TimeoutError", "TimeoutException", "ConnectTimeout", "ReadTimeout", "WriteTimeout", "PoolTimeout"]
     )
@@ -199,6 +236,10 @@ class HTTPModel(CallableModel):
     max_attempts: int = 1
     retry_status_codes: list[int] = Field(default_factory=lambda: [429, 500, 502, 503, 504])
     retry_policy: HTTPRetryPolicy | None = None
+    accepted_status_codes: list[int] = Field(
+        default_factory=list,
+        description="Error status codes returned as results with value None instead of raising, such as 404 for lookups that may miss.",
+    )
     execution_policy: ExecutionPolicy | None = None
     paginate: bool = False
     max_pages: int = 100
@@ -300,7 +341,7 @@ class HTTPModel(CallableModel):
         rendered_headers = self._render_mapping(headers, data)
         self._apply_auth(rendered_headers, rendered_query, data)
 
-        return HTTPRequest(
+        request = HTTPRequest(
             method=self.method,
             url=self._render(path, data),
             params=rendered_query,
@@ -308,8 +349,11 @@ class HTTPModel(CallableModel):
             json_data=context.json_body if context.json_body is not None else self.json_body,
             content=context.content if context.content is not None else self.content,
         )
+        if self.auth.strategy == "api_key_query":
+            request._sensitive_params = frozenset({self._render_required_auth_value(self.auth.name, "name", data)})
+        return request
 
-    def _response_value(self, response: httpx.Response) -> Any:
+    def _response_value(self, response: httpx2.Response) -> Any:
         match self.response_format:
             case "json":
                 return response.json()
@@ -373,9 +417,19 @@ class HTTPModel(CallableModel):
             current = current.get(part)
         return current
 
+    def _send(self, client: httpx2.Client, request: HTTPRequest) -> httpx2.Response:
+        return client.request(
+            method=request.method,
+            url=request.url,
+            params=request.params,
+            headers=request.headers,
+            json=request.json_data,
+            content=request.content,
+        )
+
     def _request_once(
-        self, client: httpx.Client, request: HTTPRequest, previous_started_at: float | None
-    ) -> tuple[httpx.Response, int, list[dict[str, Any]], float]:
+        self, client: httpx2.Client, request: HTTPRequest, previous_started_at: float | None
+    ) -> tuple[httpx2.Response, int, list[dict[str, Any]], float]:
         attempts = 0
         events: list[dict[str, Any]] = []
         retry_policy = self._retry_policy()
@@ -384,17 +438,14 @@ class HTTPModel(CallableModel):
             attempts += 1
             previous_started_at = self._throttle(previous_started_at)
             try:
-                response = client.request(
-                    method=request.method,
-                    url=request.url,
-                    params=request.params,
-                    headers=request.headers,
-                    json=request.json_data,
-                    content=request.content,
-                )
+                response = self._send(client, request)
+                if response.status_code in self.accepted_status_codes:
+                    return response, attempts, events, previous_started_at
                 response.raise_for_status()
                 return response, attempts, events, previous_started_at
-            except httpx.HTTPStatusError as exc:
+            except httpx2.HTTPStatusError as exc:
+                if exc.response is not None:
+                    exc.response.close()
                 status_code = exc.response.status_code if exc.response is not None else None
                 if retry_policy.should_retry_status(status_code, attempts):
                     delay_seconds = retry_policy.retry_delay_seconds(attempts, total_wait_seconds=total_retry_wait_seconds)
@@ -425,8 +476,9 @@ class HTTPModel(CallableModel):
                     )
                 )
                 status_label = status_code if status_code is not None else "unknown"
-                raise RuntimeError(f"HTTP {request.method} {self._safe_url(request)} failed with status {status_label}") from exc
-            except (httpx.TimeoutException, httpx.ConnectError) as exc:
+                # httpx2 status errors embed the full request URL, including secret query values.
+                raise RuntimeError(f"HTTP {request.method} {self._safe_url(request)} failed with status {status_label}") from None
+            except httpx2.TransportError as exc:
                 if retry_policy.should_retry_exception(exc, attempts):
                     delay_seconds = retry_policy.retry_delay_seconds(attempts, total_wait_seconds=total_retry_wait_seconds)
                     if delay_seconds is not None:
@@ -437,7 +489,7 @@ class HTTPModel(CallableModel):
                                 delay_seconds=delay_seconds,
                                 exception_type=type(exc).__name__,
                                 category=retry_policy.exception_category(exc),
-                                message=str(exc),
+                                message=_redact_query_text(str(exc), request._sensitive_params),
                             )
                         )
                         self._sleep(delay_seconds)
@@ -449,12 +501,13 @@ class HTTPModel(CallableModel):
                         outcome="failed",
                         exception_type=type(exc).__name__,
                         category=retry_policy.exception_category(exc),
-                        message=str(exc),
+                        message=_redact_query_text(str(exc), request._sensitive_params),
                     )
                 )
-                raise RuntimeError(f"HTTP {request.method} {self._safe_url(request)} failed with {type(exc).__name__}") from exc
-            except httpx.HTTPError as exc:
-                raise RuntimeError(f"HTTP {request.method} {self._safe_url(request)} failed with {type(exc).__name__}") from exc
+                # httpx2 exceptions keep the full request, including secret query values, so do not chain them.
+                raise RuntimeError(f"HTTP {request.method} {self._safe_url(request)} failed with {type(exc).__name__}") from None
+            except httpx2.HTTPError as exc:
+                raise RuntimeError(f"HTTP {request.method} {self._safe_url(request)} failed with {type(exc).__name__}") from None
 
     def _merge_page_values(self, values: list[Any]) -> Any:
         if not values:
@@ -474,15 +527,11 @@ class HTTPModel(CallableModel):
     def _request_with_params(self, request: HTTPRequest, params: dict[str, Any]) -> HTTPRequest:
         return request.model_copy(update={"params": {**request.params, **params}})
 
-    def _is_sensitive_query_param(self, key: str) -> bool:
-        normalized_key = key.lower().replace("_", "").replace("-", "")
-        return normalized_key in {"apikey", "authorization", "password"} or "token" in normalized_key or "secret" in normalized_key
-
     def _next_url_request(self, request: HTTPRequest, next_url: str) -> HTTPRequest:
         next_url_parts = urlsplit(next_url)
         next_url_params = dict(parse_qsl(next_url_parts.query, keep_blank_values=True))
         for key, value in request.params.items():
-            if key not in next_url_params and self._is_sensitive_query_param(key):
+            if key not in next_url_params and _is_sensitive_key(key, request._sensitive_params):
                 next_url_params[key] = value
         next_url_without_query = urlunsplit((next_url_parts.scheme, next_url_parts.netloc, next_url_parts.path, "", next_url_parts.fragment))
         return request.model_copy(update={"url": next_url_without_query, "params": next_url_params})
@@ -533,36 +582,49 @@ class HTTPModel(CallableModel):
     @Flow.call
     def __call__(self, context: HTTPRequestContext) -> HTTPResult:
         request = self._initial_paginated_request(self.build_request(context))
+        sensitive_params = request._sensitive_params
 
-        with httpx.Client(**self._client_kwargs()) as client:
-            values = []
-            retry_events = []
-            total_attempts = 0
-            pages = 0
-            previous_started_at = None
-            while True:
-                response, attempts, events, previous_started_at = self._request_once(client, request, previous_started_at)
-                total_attempts += attempts
-                retry_events.extend(events)
-                pages += 1
-                value = self._response_value(response)
-                values.append(value)
-
-                if not self.paginate or pages >= self.max_pages:
-                    break
-                next_request = self._next_paginated_request(request, value)
-                if next_request is None:
-                    break
-                request = next_request
-
-            return HTTPResult(
-                value=self._merge_page_values(values) if self.paginate else values[-1],
-                status_code=response.status_code,
-                headers=dict(response.headers or {}),
-                url=str(response.url),
-                attempts=total_attempts,
-                pages=pages,
-                rate_limit=self._rate_limit_headers(dict(response.headers or {})),
-                retry_events=retry_events,
-                retry_summary=self._retry_summary(retry_events, attempts=total_attempts, succeeded=True),
-            )
+        token = _REQUEST_SENSITIVE_PARAMS.set(sensitive_params)
+        try:
+            with httpx2.Client(**self._client_kwargs()) as client:
+                values = []
+                retry_events = []
+                total_attempts = 0
+                pages = 0
+                previous_started_at = None
+                while True:
+                    response, attempts, events, previous_started_at = self._request_once(client, request, previous_started_at)
+                    total_attempts += attempts
+                    retry_events.extend(events)
+                    pages += 1
+                    accepted_error = response.status_code in self.accepted_status_codes
+                    if accepted_error:
+                        if pages > 1:
+                            raise RuntimeError(
+                                f"HTTP {request.method} {self._safe_url(request)} returned status {response.status_code} on page {pages}; "
+                                "accepted statuses are only returned for the first page"
+                            )
+                        break
+                    value = self._response_value(response)
+                    values.append(value)
+
+                    if not self.paginate or pages >= self.max_pages:
+                        break
+                    next_request = self._next_paginated_request(request, value)
+                    if next_request is None:
+                        break
+                    request = next_request
+        finally:
+            _REQUEST_SENSITIVE_PARAMS.reset(token)
+
+        return HTTPResult(
+            value=None if accepted_error else (self._merge_page_values(values) if self.paginate else values[-1]),
+            status_code=response.status_code,
+            headers=dict(response.headers or {}),
+            url=redact_url(str(response.url), sensitive_params),
+            attempts=total_attempts,
+            pages=pages,
+            rate_limit=self._rate_limit_headers(dict(response.headers or {})),
+            retry_events=retry_events,
+            retry_summary=self._retry_summary(retry_events, attempts=total_attempts, succeeded=True),
+        )

ccflow_http-0.3.0/ccflow_http/download.py

@@ -0,0 +1,205 @@
+import json
+from datetime import UTC, date, datetime
+from hashlib import sha256
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+import httpx2
+from ccflow import CallableModel, DateContext, Flow, ResultBase
+from ccflow_etl import ArtifactWriteFileContext, ArtifactWriteFileModel, ArtifactWriteFileResult
+from pydantic import Field
+
+from .base import _REQUEST_SENSITIVE_PARAMS, HTTPModel, HTTPRequest, HTTPRequestContext, redact_url
+
+__all__ = (
+    "HTTPArtifactCaptureModel",
+    "HTTPArtifactCaptureResult",
+    "HTTPDownloadContext",
+    "HTTPDownloadModel",
+    "HTTPDownloadResult",
+)
+
+
+class HTTPDownloadContext(HTTPRequestContext):
+    output_path: Path
+
+
+class HTTPDownloadResult(ResultBase):
+    path: str
+    size: int
+    sha256: str
+    status_code: int
+    url: str
+    etag: str | None = None
+    last_modified: str | None = None
+    content_type: str | None = None
+    attempts: int = 1
+    retry_events: list[dict[str, Any]] = Field(default_factory=list)
+
+
+class HTTPDownloadModel(HTTPModel):
+    """Stream a response body to a local file without holding it in memory.
+
+    The file is written to a temporary path beside the target and renamed into place only after the full body
+    has been received, so a failed download never leaves a partial file at the target path.
+    """
+
+    chunk_size: int = Field(default=1 << 20, ge=1)
+
+    @property
+    def context_type(self) -> type[HTTPDownloadContext]:
+        return HTTPDownloadContext
+
+    @property
+    def result_type(self) -> type[HTTPDownloadResult]:
+        return HTTPDownloadResult
+
+    def _send(self, client: httpx2.Client, request: HTTPRequest) -> httpx2.Response:
+        built = client.build_request(
+            method=request.method,
+            url=request.url,
+            params=request.params,
+            headers=request.headers,
+            json=request.json_data,
+            content=request.content,
+        )
+        return client.send(built, stream=True)
+
+    @Flow.call
+    def __call__(self, context: HTTPDownloadContext) -> HTTPDownloadResult:
+        request = self.build_request(context)
+        sensitive_params = request._sensitive_params
+        path = context.output_path
+        path.parent.mkdir(parents=True, exist_ok=True)
+        temp_path = path.with_name(f".{path.name}.{uuid4().hex}.tmp")
+        digest = sha256()
+        size = 0
+        token = _REQUEST_SENSITIVE_PARAMS.set(sensitive_params)
+        try:
+            with httpx2.Client(**self._client_kwargs()) as client:
+                response, attempts, events, _ = self._request_once(client, request, None)
+                try:
+                    with temp_path.open("wb") as file:
+                        for chunk in response.iter_bytes(self.chunk_size):
+                            file.write(chunk)
+                            digest.update(chunk)
+                            size += len(chunk)
+                finally:
+                    response.close()
+            temp_path.replace(path)
+        except BaseException:
+            temp_path.unlink(missing_ok=True)
+            raise
+        finally:
+            _REQUEST_SENSITIVE_PARAMS.reset(token)
+        return HTTPDownloadResult(
+            path=str(path),
+            size=size,
+            sha256=digest.hexdigest(),
+            status_code=response.status_code,
+            url=redact_url(str(response.url), sensitive_params),
+            etag=response.headers.get("etag"),
+            last_modified=response.headers.get("last-modified"),
+            content_type=response.headers.get("content-type"),
+            attempts=attempts,
+            retry_events=events,
+        )
+
+
+class HTTPArtifactCaptureResult(ResultBase):
+    date: date
+    key: str
+    manifest_key: str
+    status: str
+    download: HTTPDownloadResult | None = None
+    write: ArtifactWriteFileResult | None = None
+    manifest_write: ArtifactWriteFileResult | None = None
+
+
+class HTTPArtifactCaptureModel(CallableModel):
+    """Capture one HTTP resource for a date: download it, store it under a dated key, and store a manifest beside it.
+
+    The manifest records the redacted URL, capture time, status, response headers, size, and SHA-256 of the stored
+    payload. An existing payload is skipped before downloading unless ``overwrite`` is set.
+    """
+
+    download: HTTPDownloadModel
+    writer: ArtifactWriteFileModel
+    key_template: str = Field(description="Payload key with {date}, {year}, {month}, and {day} placeholders.")
+    workspace: Path = Path("cc-etl-workspace")
+    dataset: str | None = None
+    media_type: str | None = None
+    overwrite: bool = False
+    keep_local: bool = Field(default=False, description="Keep the downloaded file in the workspace after it is stored.")
+    explain: bool = False
+
+    @property
+    def context_type(self) -> type[DateContext]:
+        return DateContext
+
+    @property
+    def result_type(self) -> type[HTTPArtifactCaptureResult]:
+        return HTTPArtifactCaptureResult
+
+    def key(self, context: DateContext) -> str:
+        value = context.date
+        return self.key_template.format(date=value.isoformat(), year=f"{value.year:04d}", month=f"{value.month:02d}", day=f"{value.day:02d}")
+
+    def _write(self, key: str, path: Path, media_type: str | None, dry_run: bool, metadata: dict[str, Any]) -> ArtifactWriteFileResult:
+        return self.writer(
+            ArtifactWriteFileContext(
+                key=key,
+                path=path,
+                media_type=media_type,
+                dataset=self.dataset,
+                stage="extract",
+                overwrite=self.overwrite,
+                dry_run=dry_run,
+                metadata=metadata,
+            )
+        )
+
+    def _exists(self, key: str) -> bool:
+        exists = getattr(self.writer.store, "exists", None)
+        return bool(exists and exists(key))
+
+    @Flow.call
+    def __call__(self, context: DateContext) -> HTTPArtifactCaptureResult:
+        key = self.key(context)
+        manifest_key = f"{key}.manifest.json"
+        local_path = self.workspace / key
+        manifest_path = self.workspace / manifest_key
+        metadata = {"capture_date": context.date.isoformat()}
+        if self.explain:
+            return HTTPArtifactCaptureResult(
+                date=context.date,
+                key=key,
+                manifest_key=manifest_key,
+                status="planned",
+                write=self._write(key, local_path, self.media_type, True, metadata),
+                manifest_write=self._write(manifest_key, manifest_path, "application/json", True, metadata),
+            )
+        if not self.overwrite and self._exists(key):
+            return HTTPArtifactCaptureResult(date=context.date, key=key, manifest_key=manifest_key, status="exists")
+
+        download = self.download(HTTPDownloadContext(output_path=local_path))
+        manifest = {
+            "key": key,
+            "capture_date": context.date.isoformat(),
+            "captured_at": datetime.now(UTC).isoformat(),
+            **download.model_dump(mode="json", exclude={"type_", "path", "retry_events"}),
+        }
+        manifest_path.parent.mkdir(parents=True, exist_ok=True)
+        manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
+        try:
+            write = self._write(key, local_path, self.media_type or download.content_type, False, {**metadata, "sha256": download.sha256})
+            manifest_write = self._write(manifest_key, manifest_path, "application/json", False, metadata)
+        finally:
+            if not self.keep_local:
+                local_path.unlink(missing_ok=True)
+                manifest_path.unlink(missing_ok=True)
+        status = next((value for value in (write.status, manifest_write.status) if value != "exists"), "exists")
+        return HTTPArtifactCaptureResult(
+            date=context.date, key=key, manifest_key=manifest_key, status=status, download=download, write=write, manifest_write=manifest_write
+        )

ccflow_http-0.3.0/ccflow_http/tests/test_download.py

@@ -0,0 +1,110 @@
+import json
+from datetime import date
+from hashlib import sha256
+
+import httpx2
+import pytest
+from ccflow_etl import ArtifactWriteFileModel, LocalFileOutput
+
+from ccflow_http import HTTPArtifactCaptureModel, HTTPAuth, HTTPConfig, HTTPDownloadContext, HTTPDownloadModel, HTTPRetryPolicy
+
+PAYLOAD = b"x" * 5000
+
+
+def _download_model(handler, **kwargs):
+    return HTTPDownloadModel(
+        config=HTTPConfig(base_url="https://files.example.test", transport=httpx2.MockTransport(handler)),
+        path="/bulk/data.zip",
+        chunk_size=1024,
+        **kwargs,
+    )
+
+
+def test_http_download_streams_body_to_file_with_checksum_and_headers(tmp_path):
+    def handler(request):
+        return httpx2.Response(
+            200, content=PAYLOAD, headers={"ETag": '"abc"', "Last-Modified": "Fri, 09 Oct 2026 04:35:00 GMT", "Content-Type": "application/zip"}
+        )
+
+    target = tmp_path / "out" / "data.zip"
+    result = _download_model(handler, auth=HTTPAuth(strategy="api_key_query", name="key", value="secret"))(HTTPDownloadContext(output_path=target))
+
+    assert target.read_bytes() == PAYLOAD
+    assert result.size == len(PAYLOAD)
+    assert result.sha256 == sha256(PAYLOAD).hexdigest()
+    assert (result.etag, result.last_modified, result.content_type) == ('"abc"', "Fri, 09 Oct 2026 04:35:00 GMT", "application/zip")
+    assert result.url == "https://files.example.test/bulk/data.zip?key=***"
+    assert not list(target.parent.glob(".*.tmp"))
+
+
+def test_http_download_retries_status_and_leaves_no_partial_file_on_failure(tmp_path, monkeypatch):
+    monkeypatch.setattr("ccflow_http.base.sleep", lambda seconds: None)
+    calls = []
+
+    def flaky(request):
+        calls.append(request)
+        return httpx2.Response(503) if len(calls) == 1 else httpx2.Response(200, content=PAYLOAD)
+
+    policy = HTTPRetryPolicy(max_attempts=2, wait_initial=0.0, wait_max=0.0, wait_jitter=0.0)
+    result = _download_model(flaky, retry_policy=policy)(HTTPDownloadContext(output_path=tmp_path / "data.zip"))
+    assert result.attempts == 2
+
+    target = tmp_path / "failed.zip"
+    with pytest.raises(RuntimeError, match="failed with status 404"):
+        _download_model(lambda request: httpx2.Response(404))(HTTPDownloadContext(output_path=target))
+    assert not target.exists()
+    assert not list(tmp_path.glob(".failed.zip.*.tmp"))
+
+
+def _capture(tmp_path, handler, **kwargs):
+    store = LocalFileOutput(path=tmp_path / "store")
+    model = HTTPArtifactCaptureModel(
+        download=_download_model(handler),
+        writer=ArtifactWriteFileModel(store=store),
+        key_template="bulk/data/{year}/{month}/{date}.zip",
+        workspace=tmp_path / "workspace",
+        dataset="bulk-data",
+        **kwargs,
+    )
+    return model, store
+
+
+def test_http_capture_stores_payload_and_manifest_and_cleans_workspace(tmp_path):
+    model, store = _capture(tmp_path, lambda request: httpx2.Response(200, content=PAYLOAD, headers={"ETag": '"abc"'}))
+
+    result = model(date(2026, 10, 11))
+    manifest = json.loads(store.file_path("bulk/data/2026/10/2026-10-11.zip.manifest.json").read_text())
+
+    assert result.status == "written"
+    assert store.file_path("bulk/data/2026/10/2026-10-11.zip").read_bytes() == PAYLOAD
+    assert manifest["sha256"] == sha256(PAYLOAD).hexdigest()
+    assert manifest["size"] == len(PAYLOAD)
+    assert manifest["etag"] == '"abc"'
+    assert manifest["capture_date"] == "2026-10-11"
+    assert manifest["url"] == "https://files.example.test/bulk/data.zip"
+    assert not [path for path in (tmp_path / "workspace").rglob("*") if path.is_file()]
+
+
+def test_http_capture_skips_existing_payload_without_downloading(tmp_path):
+    calls = []
+    model, _ = _capture(tmp_path, lambda request: calls.append(request) or httpx2.Response(200, content=PAYLOAD))
+    model(date(2026, 10, 11))
+
+    result = model(date(2026, 10, 11))
+
+    assert result.status == "exists"
+    assert len(calls) == 1
+
+
+def test_http_capture_explain_does_no_network_or_storage_io(tmp_path):
+    def handler(request):
+        raise AssertionError("explain made a request")
+
+    model, _ = _capture(tmp_path, handler, explain=True)
+
+    result = model(date(2026, 10, 11))
+
+    assert result.status == "planned"
+    assert result.write.status == "planned"
+    assert not (tmp_path / "store").exists()
+    assert not (tmp_path / "workspace").exists()

{ccflow_http-0.1.1 → ccflow_http-0.3.0}/ccflow_http/tests/test_model.py

@@ -6,11 +6,21 @@ from datetime import UTC
 from gzip import compress
 from typing import Any
 
-import httpx
+import httpx2
 import pytest
 from ccflow_etl import ExecutionPolicy
 
-from ccflow_http import HTTPAuth, HTTPConfig, HTTPModel, HTTPRequest, HTTPRequestContext, HTTPResponseResult, HTTPRetryPolicy, safe_request_dump
+from ccflow_http import (
+    HTTPAuth,
+    HTTPConfig,
+    HTTPModel,
+    HTTPRequest,
+    HTTPRequestContext,
+    HTTPResponseResult,
+    HTTPRetryPolicy,
+    redact_url,
+    safe_request_dump,
+)
 
 
 @dataclass
@@ -52,7 +62,7 @@ def test_http_model_renders_request_and_returns_json(monkeypatch):
             calls.append({"client": self.kwargs, "request": kwargs})
             return FakeResponse(value={"status": "OK", "results": [{"ticker": "AAA"}]}, headers={"x-limit-remaining": "9"})
 
-    monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
+    monkeypatch.setattr("ccflow_http.base.httpx2.Client", FakeClient)
 
     model = HTTPModel(
         base_url="https://api.example.test",
@@ -133,11 +143,11 @@ def test_http_model_error_message_omits_secret_query_values(monkeypatch):
             return False
 
         def request(self, **kwargs):
-            request = httpx.Request("GET", "https://api.example.test/v1/tickers/AAA?apiKey=secret")
-            response = httpx.Response(429, request=request)
-            raise httpx.HTTPStatusError("rate limited with apiKey=secret", request=request, response=response)
+            request = httpx2.Request("GET", "https://api.example.test/v1/tickers/AAA?apiKey=secret")
+            response = httpx2.Response(429, request=request)
+            raise httpx2.HTTPStatusError("rate limited with apiKey=secret", request=request, response=response)
 
-    monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
+    monkeypatch.setattr("ccflow_http.base.httpx2.Client", FakeClient)
 
     model = HTTPModel(base_url="https://api.example.test", path="/v1/tickers/{{ ticker }}", query={"apiKey": "{{ api_key }}"})
 
@@ -146,6 +156,34 @@ def test_http_model_error_message_omits_secret_query_values(monkeypatch):
 
     assert "secret" not in str(error.value)
     assert "apiKey" not in str(error.value)
+    assert error.value.__cause__ is None
+    assert error.value.__suppress_context__
+
+
+def test_redact_url_masks_secret_query_values():
+    assert (
+        redact_url("https://api.example.test/v1/tickers?date=2024-01-03&apiKey=abc&page_token=def&limit=10")
+        == "https://api.example.test/v1/tickers?date=2024-01-03&apiKey=***&page_token=***&limit=10"
+    )
+    assert redact_url("https://api.example.test/v1/tickers") == "https://api.example.test/v1/tickers"
+    assert redact_url("/v1/tickers?cursor=2") == "/v1/tickers?cursor=2"
+
+
+def test_http_model_redacts_secret_query_values_from_request_log_and_result(caplog):
+    transport = httpx2.MockTransport(lambda request: httpx2.Response(200, json={"status": "OK"}))
+    model = HTTPModel(
+        config=HTTPConfig(base_url="https://api.example.test", transport=transport),
+        path="/v1/tickers",
+        query={"date": "2024-01-03"},
+        auth=HTTPAuth(strategy="api_key_query", name="apiKey", value="query-key"),
+    )
+
+    with caplog.at_level("INFO", logger="httpx2"):
+        result = model(HTTPRequestContext())
+
+    assert "HTTP Request: GET https://api.example.test/v1/tickers?date=2024-01-03&apiKey=***" in caplog.text
+    assert "query-key" not in caplog.text
+    assert result.url == "https://api.example.test/v1/tickers?date=2024-01-03&apiKey=***"
 
 
 def test_http_model_retries_retryable_status_and_captures_rate_limit(monkeypatch):
@@ -164,12 +202,12 @@ def test_http_model_retries_retryable_status_and_captures_rate_limit(monkeypatch
         def request(self, **kwargs):
             calls.append(kwargs)
             if len(calls) == 1:
-                request = httpx.Request("GET", "https://api.example.test/v1/tickers")
-                response = httpx.Response(429, request=request, headers={"retry-after": "1"})
-                raise httpx.HTTPStatusError("rate limited", request=request, response=response)
+                request = httpx2.Request("GET", "https://api.example.test/v1/tickers")
+                response = httpx2.Response(429, request=request, headers={"retry-after": "1"})
+                raise httpx2.HTTPStatusError("rate limited", request=request, response=response)
             return FakeResponse(value={"status": "OK"}, headers={"x-ratelimit-remaining": "8"})
 
-    monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
+    monkeypatch.setattr("ccflow_http.base.httpx2.Client", FakeClient)
 
     result = HTTPModel(base_url="https://api.example.test", path="/v1/tickers", max_attempts=2)(HTTPRequestContext())
 
@@ -195,12 +233,12 @@ def test_http_model_consumes_shared_retry_policy(monkeypatch):
         def request(self, **kwargs):
             calls.append(kwargs)
             if len(calls) == 1:
-                request = httpx.Request("GET", "https://api.example.test/v1/tickers")
-                response = httpx.Response(503, request=request)
-                raise httpx.HTTPStatusError("unavailable", request=request, response=response)
+                request = httpx2.Request("GET", "https://api.example.test/v1/tickers")
+                response = httpx2.Response(503, request=request)
+                raise httpx2.HTTPStatusError("unavailable", request=request, response=response)
             return FakeResponse(value={"status": "OK"})
 
-    monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
+    monkeypatch.setattr("ccflow_http.base.httpx2.Client", FakeClient)
 
     model = HTTPModel(base_url="https://api.example.test", path="/v1/tickers", retry_policy=HTTPRetryPolicy(max_attempts=2, retry_status_codes=[503]))
 
@@ -223,12 +261,12 @@ def test_http_model_consumes_shared_retry_delay_and_execution_policy(monkeypatch
         def request(self, **kwargs):
             calls.append(kwargs)
             if len(calls) == 1:
-                request = httpx.Request("GET", "https://api.example.test/v1/tickers")
-                response = httpx.Response(429, request=request)
-                raise httpx.HTTPStatusError("rate limited", request=request, response=response)
+                request = httpx2.Request("GET", "https://api.example.test/v1/tickers")
+                response = httpx2.Response(429, request=request)
+                raise httpx2.HTTPStatusError("rate limited", request=request, response=response)
             return FakeResponse(value={"status": "OK"})
 
-    monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
+    monkeypatch.setattr("ccflow_http.base.httpx2.Client", FakeClient)
 
     model = HTTPModel(
         base_url="https://api.example.test",
@@ -274,10 +312,10 @@ def test_http_model_retries_timeout_exception(monkeypatch):
         def request(self, **kwargs):
             calls.append(kwargs)
             if len(calls) == 1:
-                raise httpx.TimeoutException("timed out")
+                raise httpx2.TimeoutException("timed out")
             return FakeResponse(value={"status": "OK"})
 
-    monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
+    monkeypatch.setattr("ccflow_http.base.httpx2.Client", FakeClient)
 
     result = HTTPModel(base_url="https://api.example.test", path="/v1/tickers", max_attempts=2)(HTTPRequestContext())
 
@@ -301,11 +339,11 @@ def test_http_model_retries_5xx_until_attempts_are_exhausted(monkeypatch):
 
         def request(self, **kwargs):
             calls.append(kwargs)
-            request = httpx.Request("GET", "https://api.example.test/v1/tickers?apiKey=secret")
-            response = httpx.Response(500, request=request)
-            raise httpx.HTTPStatusError("server error with apiKey=secret", request=request, response=response)
+            request = httpx2.Request("GET", "https://api.example.test/v1/tickers?apiKey=secret")
+            response = httpx2.Response(500, request=request)
+            raise httpx2.HTTPStatusError("server error with apiKey=secret", request=request, response=response)
 
-    monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
+    monkeypatch.setattr("ccflow_http.base.httpx2.Client", FakeClient)
 
     model = HTTPModel(base_url="https://api.example.test", path="/v1/tickers", query={"apiKey": "secret"}, max_attempts=2)
 
@@ -336,7 +374,7 @@ def test_http_model_paginates_massive_style_next_url(monkeypatch):
                 return FakeResponse(value={"results": [{"ticker": "AAA"}], "next_url": "/v3/reference/tickers?cursor=2"})
             return FakeResponse(value={"results": [{"ticker": "BBB"}]})
 
-    monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
+    monkeypatch.setattr("ccflow_http.base.httpx2.Client", FakeClient)
 
     result = HTTPModel(base_url="https://api.example.test", path="/v3/reference/tickers", paginate=True)(HTTPRequestContext())
 
@@ -365,7 +403,7 @@ def test_http_model_preserves_query_auth_for_next_url_pagination(monkeypatch):
                 return FakeResponse(value={"results": [{"ticker": "AAA"}], "next_url": "/v3/reference/tickers?cursor=2"})
             return FakeResponse(value={"results": [{"ticker": "BBB"}]})
 
-    monkeypatch.setattr("ccflow_http.base.httpx.Client", FakeClient)
+    monkeypatch.setattr("ccflow_http.base.httpx2.Client", FakeClient)
 
     HTTPModel(
         base_url="https://api.example.test",
@@ -378,17 +416,17 @@ def test_http_model_preserves_query_auth_for_next_url_pagination(monkeypatch):
     assert calls[1]["params"] == {"cursor": "2", "apiKey": "secret"}
 
 
-def test_http_model_sends_next_url_query_and_preserved_auth_with_httpx_transport():
+def test_http_model_sends_next_url_query_and_preserved_auth_with_httpx2_transport():
     seen_urls = []
 
-    def handler(request: httpx.Request) -> httpx.Response:
+    def handler(request: httpx2.Request) -> httpx2.Response:
         seen_urls.append(str(request.url))
         if len(seen_urls) == 1:
-            return httpx.Response(200, json={"results": [{"ticker": "AAA"}], "next_url": "/v3/reference/tickers?cursor=2"})
-        return httpx.Response(200, json={"results": [{"ticker": "BBB"}]})
+            return httpx2.Response(200, json={"results": [{"ticker": "AAA"}], "next_url": "/v3/reference/tickers?cursor=2"})
+        return httpx2.Response(200, json={"results": [{"ticker": "BBB"}]})
 
     model = HTTPModel(
-        config=HTTPConfig(base_url="https://api.example.test", transport=httpx.MockTransport(handler)),
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(handler)),
         path="/v3/reference/tickers",
         query={"apiKey": "secret"},
         paginate=True,
@@ -406,11 +444,11 @@ def test_http_model_sends_next_url_query_and_preserved_auth_with_httpx_transport
 def test_http_model_applies_config_and_all_auth_strategies_with_mock_transport():
     seen_requests = []
 
-    def handler(request: httpx.Request) -> httpx.Response:
+    def handler(request: httpx2.Request) -> httpx2.Response:
         seen_requests.append(request)
-        return httpx.Response(200, json={"status": "OK"}, headers={"x-ratelimit-remaining": "7"})
+        return httpx2.Response(200, json={"status": "OK"}, headers={"x-ratelimit-remaining": "7"})
 
-    transport = httpx.MockTransport(handler)
+    transport = httpx2.MockTransport(handler)
     config = HTTPConfig(base_url="https://api.example.test", timeout=4.0, headers={"Accept": "application/json"}, transport=transport)
 
     bearer = HTTPModel(config=config, path="/v1/{{ resource }}", auth=HTTPAuth(strategy="bearer", token="{{ token }}"))
@@ -439,21 +477,123 @@ def test_http_model_applies_config_and_all_auth_strategies_with_mock_transport()
     assert "authorization" not in seen_requests[4].headers
 
 
+def test_http_model_returns_accepted_error_status_without_raising():
+    transport = httpx2.MockTransport(lambda request: httpx2.Response(404, json={"status": "NOT_FOUND"}))
+    config = HTTPConfig(base_url="https://api.example.test", transport=transport)
+
+    result = HTTPModel(config=config, path="/v1/tickers/ZZZZ", accepted_status_codes=[404])(HTTPRequestContext())
+
+    assert result.status_code == 404
+    assert result.value is None
+    assert result.retry_summary == {"attempts": 1, "retried": 0, "failed": 0, "succeeded": 1}
+    with pytest.raises(RuntimeError, match="failed with status 404"):
+        HTTPModel(config=config, path="/v1/tickers/ZZZZ")(HTTPRequestContext())
+
+
+def test_http_model_raises_on_accepted_error_status_after_first_page():
+    def handler(request: httpx2.Request) -> httpx2.Response:
+        if "cursor" in request.url.params:
+            return httpx2.Response(404)
+        return httpx2.Response(200, json={"results": [{"ticker": "AAA"}], "next_url": "/v1/tickers?cursor=2"})
+
+    model = HTTPModel(
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(handler)),
+        path="/v1/tickers",
+        paginate=True,
+        accepted_status_codes=[404],
+    )
+
+    with pytest.raises(RuntimeError, match="status 404 on page 2"):
+        model(HTTPRequestContext())
+
+
+def test_http_model_redacts_custom_query_auth_name_everywhere(caplog):
+    seen_urls = []
+
+    def handler(request: httpx2.Request) -> httpx2.Response:
+        seen_urls.append(str(request.url))
+        if len(seen_urls) == 1:
+            return httpx2.Response(200, json={"results": [{"ticker": "AAA"}], "next_url": "/v1/tickers?cursor=2"})
+        return httpx2.Response(200, json={"results": [{"ticker": "BBB"}]})
+
+    model = HTTPModel(
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(handler)),
+        path="/v1/tickers",
+        paginate=True,
+        auth=HTTPAuth(strategy="api_key_query", name="key", value="custom-secret"),
+    )
+
+    with caplog.at_level("INFO", logger="httpx2"):
+        result = model(HTTPRequestContext())
+
+    assert seen_urls == ["https://api.example.test/v1/tickers?key=custom-secret", "https://api.example.test/v1/tickers?cursor=2&key=custom-secret"]
+    assert "custom-secret" not in caplog.text
+    assert caplog.text.count("key=***") == 2
+    assert result.url == "https://api.example.test/v1/tickers?cursor=2&key=***"
+    assert safe_request_dump(model.build_request(HTTPRequestContext()))["params"] == {"key": "***"}
+
+
+def test_http_model_does_not_leak_query_secrets_through_transport_errors(monkeypatch):
+    monkeypatch.setattr("ccflow_http.base.sleep", lambda seconds: None)
+    calls = []
+
+    def handler(request: httpx2.Request) -> httpx2.Response:
+        calls.append(request)
+        if len(calls) == 1:
+            raise httpx2.ConnectError(f"connection reset for {request.url}", request=request)
+        raise httpx2.ReadTimeout(f"read timed out for {request.url}", request=request)
+
+    model = HTTPModel(
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(handler)),
+        path="/v1/tickers",
+        auth=HTTPAuth(strategy="api_key_query", name="key", value="custom-secret"),
+        retry_policy=HTTPRetryPolicy(max_attempts=2, wait_initial=0.0, wait_max=0.0, wait_jitter=0.0),
+    )
+
+    with pytest.raises(RuntimeError, match="failed with ReadTimeout") as error:
+        model(HTTPRequestContext())
+
+    assert error.value.__cause__ is None
+    assert error.value.__suppress_context__
+
+
+def test_http_model_redacts_query_secrets_in_retry_event_messages(monkeypatch):
+    monkeypatch.setattr("ccflow_http.base.sleep", lambda seconds: None)
+    calls = []
+
+    def handler(request: httpx2.Request) -> httpx2.Response:
+        calls.append(request)
+        if len(calls) == 1:
+            raise httpx2.ConnectError(f"connection reset for {request.url}", request=request)
+        return httpx2.Response(200, json={"status": "OK"})
+
+    model = HTTPModel(
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(handler)),
+        path="/v1/tickers",
+        auth=HTTPAuth(strategy="api_key_query", name="key", value="custom-secret"),
+        retry_policy=HTTPRetryPolicy(max_attempts=2, wait_initial=0.0, wait_max=0.0, wait_jitter=0.0),
+    )
+
+    result = model(HTTPRequestContext())
+
+    assert result.retry_events[0]["message"] == "connection reset for https://api.example.test/v1/tickers?key=***"
+
+
 def test_http_model_parses_csv_and_gzip_responses_with_mock_transport():
-    def csv_handler(request: httpx.Request) -> httpx.Response:
-        return httpx.Response(200, text="ticker,volume\nAAA,10\nBBB,20\n")
+    def csv_handler(request: httpx2.Request) -> httpx2.Response:
+        return httpx2.Response(200, text="ticker,volume\nAAA,10\nBBB,20\n")
 
     csv_result = HTTPModel(
-        config=HTTPConfig(base_url="https://api.example.test", transport=httpx.MockTransport(csv_handler)),
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(csv_handler)),
         path="/csv",
         response_format="csv",
     )(HTTPRequestContext())
 
-    def gzip_handler(request: httpx.Request) -> httpx.Response:
-        return httpx.Response(200, content=compress(b'{"status":"OK"}'))
+    def gzip_handler(request: httpx2.Request) -> httpx2.Response:
+        return httpx2.Response(200, content=compress(b'{"status":"OK"}'))
 
     gzip_result = HTTPModel(
-        config=HTTPConfig(base_url="https://api.example.test", transport=httpx.MockTransport(gzip_handler)),
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(gzip_handler)),
         path="/gzip",
         response_format="gzip",
     )(HTTPRequestContext())
@@ -465,14 +605,14 @@ def test_http_model_parses_csv_and_gzip_responses_with_mock_transport():
 def test_http_model_supports_cursor_page_and_offset_pagination_with_mock_transport():
     cursor_requests = []
 
-    def cursor_handler(request: httpx.Request) -> httpx.Response:
+    def cursor_handler(request: httpx2.Request) -> httpx2.Response:
         cursor_requests.append(dict(request.url.params))
         if "cursor" not in request.url.params:
-            return httpx.Response(200, json={"results": [{"id": 1}], "next_cursor": "abc"})
-        return httpx.Response(200, json={"results": [{"id": 2}]})
+            return httpx2.Response(200, json={"results": [{"id": 1}], "next_cursor": "abc"})
+        return httpx2.Response(200, json={"results": [{"id": 2}]})
 
     cursor_result = HTTPModel(
-        config=HTTPConfig(base_url="https://api.example.test", transport=httpx.MockTransport(cursor_handler)),
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(cursor_handler)),
         path="/items",
         paginate=True,
         pagination_mode="cursor",
@@ -482,14 +622,14 @@ def test_http_model_supports_cursor_page_and_offset_pagination_with_mock_transpo
 
     page_requests = []
 
-    def page_handler(request: httpx.Request) -> httpx.Response:
+    def page_handler(request: httpx2.Request) -> httpx2.Response:
         page_requests.append(dict(request.url.params))
         page = int(request.url.params["page"])
         payload = {1: [{"id": 1}], 2: [{"id": 2}]}.get(page, [])
-        return httpx.Response(200, json={"results": payload})
+        return httpx2.Response(200, json={"results": payload})
 
     page_result = HTTPModel(
-        config=HTTPConfig(base_url="https://api.example.test", transport=httpx.MockTransport(page_handler)),
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(page_handler)),
         path="/items",
         paginate=True,
         pagination_mode="page",
@@ -500,14 +640,14 @@ def test_http_model_supports_cursor_page_and_offset_pagination_with_mock_transpo
 
     offset_requests = []
 
-    def offset_handler(request: httpx.Request) -> httpx.Response:
+    def offset_handler(request: httpx2.Request) -> httpx2.Response:
         offset_requests.append(dict(request.url.params))
         offset = int(request.url.params["offset"])
         payload = {0: [{"id": 1}], 2: [{"id": 2}]}.get(offset, [])
-        return httpx.Response(200, json={"results": payload})
+        return httpx2.Response(200, json={"results": payload})
 
     offset_result = HTTPModel(
-        config=HTTPConfig(base_url="https://api.example.test", transport=httpx.MockTransport(offset_handler)),
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(offset_handler)),
         path="/items",
         paginate=True,
         pagination_mode="offset",
@@ -546,29 +686,30 @@ def _retrying_client(calls, response_factory):
 
 
 def _raise_status(status_code, headers=None):
-    request = httpx.Request("GET", "https://api.example.test/v1/tickers")
-    response = httpx.Response(status_code, request=request, headers=headers or {})
-    raise httpx.HTTPStatusError("error", request=request, response=response)
+    request = httpx2.Request("GET", "https://api.example.test/v1/tickers")
+    response = httpx2.Response(status_code, request=request, headers=headers or {})
+    raise httpx2.HTTPStatusError("error", request=request, response=response)
 
 
-def test_http_model_retries_connect_error(monkeypatch):
+@pytest.mark.parametrize(("exception_type", "category"), [(httpx2.ConnectError, "connection"), (httpx2.RemoteProtocolError, "exception")])
+def test_http_model_retries_transport_error(monkeypatch, exception_type, category):
     calls = []
 
     def fail_once():
-        raise httpx.ConnectError("connection failed")
+        raise exception_type("transport failed")
 
-    monkeypatch.setattr("ccflow_http.base.httpx.Client", _retrying_client(calls, fail_once))
+    monkeypatch.setattr("ccflow_http.base.httpx2.Client", _retrying_client(calls, fail_once))
 
     result = HTTPModel(base_url="https://api.example.test", path="/v1/tickers", max_attempts=2)(HTTPRequestContext())
 
     assert len(calls) == 2
     assert result.attempts == 2
-    assert result.retry_events[0]["category"] == "connection"
+    assert result.retry_events[0]["category"] == category
 
 
 def test_http_model_honors_retry_after_seconds_over_backoff(monkeypatch):
     calls = []
-    monkeypatch.setattr("ccflow_http.base.httpx.Client", _retrying_client(calls, lambda: _raise_status(429, {"retry-after": "45"})))
+    monkeypatch.setattr("ccflow_http.base.httpx2.Client", _retrying_client(calls, lambda: _raise_status(429, {"retry-after": "45"})))
 
     model = HTTPModel(
         base_url="https://api.example.test",
@@ -587,7 +728,7 @@ def test_http_model_honors_retry_after_seconds_over_backoff(monkeypatch):
 
 def test_http_model_uses_backoff_when_it_exceeds_retry_after(monkeypatch):
     calls = []
-    monkeypatch.setattr("ccflow_http.base.httpx.Client", _retrying_client(calls, lambda: _raise_status(429, {"retry-after": "1"})))
+    monkeypatch.setattr("ccflow_http.base.httpx2.Client", _retrying_client(calls, lambda: _raise_status(429, {"retry-after": "1"})))
 
     model = HTTPModel(
         base_url="https://api.example.test",
@@ -604,7 +745,7 @@ def test_http_model_uses_backoff_when_it_exceeds_retry_after(monkeypatch):
 
 def test_http_model_caps_retry_after_at_retry_after_max(monkeypatch):
     calls = []
-    monkeypatch.setattr("ccflow_http.base.httpx.Client", _retrying_client(calls, lambda: _raise_status(429, {"retry-after": "9999"})))
+    monkeypatch.setattr("ccflow_http.base.httpx2.Client", _retrying_client(calls, lambda: _raise_status(429, {"retry-after": "9999"})))
 
     model = HTTPModel(
         base_url="https://api.example.test",
@@ -625,7 +766,7 @@ def test_http_model_honors_retry_after_http_date(monkeypatch):
 
     calls = []
     retry_at = format_datetime(datetime.now(UTC) + timedelta(seconds=60))
-    monkeypatch.setattr("ccflow_http.base.httpx.Client", _retrying_client(calls, lambda: _raise_status(429, {"retry-after": retry_at})))
+    monkeypatch.setattr("ccflow_http.base.httpx2.Client", _retrying_client(calls, lambda: _raise_status(429, {"retry-after": retry_at})))
 
     model = HTTPModel(
         base_url="https://api.example.test",
@@ -643,7 +784,7 @@ def test_http_model_honors_retry_after_http_date(monkeypatch):
 
 def test_http_model_ignores_invalid_retry_after(monkeypatch):
     calls = []
-    monkeypatch.setattr("ccflow_http.base.httpx.Client", _retrying_client(calls, lambda: _raise_status(429, {"retry-after": "soon"})))
+    monkeypatch.setattr("ccflow_http.base.httpx2.Client", _retrying_client(calls, lambda: _raise_status(429, {"retry-after": "soon"})))
 
     model = HTTPModel(
         base_url="https://api.example.test",

{ccflow_http-0.1.1 → ccflow_http-0.3.0}/pyproject.toml

@@ -12,7 +12,7 @@ authors = [
 description = "ccflow models for HTTP"
 readme = "README.md"
 license = { text = "Apache-2.0" }
-version = "0.1.1"
+version = "0.3.0"
 requires-python = ">=3.11"
 keywords = []
 
@@ -30,8 +30,8 @@ classifiers = [
 
 dependencies = [
     "ccflow>=0.8.5",
-    "ccflow-etl>=0.1.0",
-    "httpx",
+    "ccflow-etl>=0.4.0",
+    "httpx2",
     "jinja2",
 ]
 
@@ -43,6 +43,7 @@ develop = [
     "codespell",
     "hatchling",
     "mdformat",
+    "mdformat-frontmatter",
     "mdformat-tables>=1",
     "pytest",
     "pytest-cov",
@@ -60,7 +61,7 @@ Repository = "https://github.com/1kbgz/ccflow-http"
 Homepage = "https://github.com/1kbgz/ccflow-http"
 
 [tool.bumpversion]
-current_version = "0.1.1"
+current_version = "0.3.0"
 commit = true
 tag = true
 commit_args = "-s"
@@ -78,7 +79,7 @@ replace = 'version = "{new_version}"'
 [tool.coverage.run]
 branch = true
 omit = [
-    "ccflow_http/tests/integration/",
+    "ccflow_http/tests/integration/*",
 ]
 
 [tool.coverage.report]

ccflow_http-0.1.1/ccflow_http/__init__.py

@@ -1,3 +0,0 @@
-from .base import *
-
-__version__ = "0.1.1"