ccflow-http 0.2.0__tar.gz → 0.4.0__tar.gz

{ccflow_http-0.2.0 → ccflow_http-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ccflow-http
-Version: 0.2.0
+Version: 0.4.0
 Summary: ccflow models for HTTP
 Project-URL: Repository, https://github.com/1kbgz/ccflow-http
 Project-URL: Homepage, https://github.com/1kbgz/ccflow-http
@@ -17,7 +17,7 @@ Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.11
-Requires-Dist: ccflow-etl>=0.1.0
+Requires-Dist: ccflow-etl>=0.4.0
 Requires-Dist: ccflow>=0.8.5
 Requires-Dist: httpx2
 Requires-Dist: jinja2
@@ -28,6 +28,7 @@ Requires-Dist: check-dist; extra == 'develop'
 Requires-Dist: codespell; extra == 'develop'
 Requires-Dist: hatchling; extra == 'develop'
 Requires-Dist: mdformat; extra == 'develop'
+Requires-Dist: mdformat-frontmatter; extra == 'develop'
 Requires-Dist: mdformat-tables>=1; extra == 'develop'
 Requires-Dist: pytest; extra == 'develop'
 Requires-Dist: pytest-cov; extra == 'develop'
@@ -55,7 +56,7 @@ It should not contain provider-specific endpoint catalogs. Domain packages can c
 
 ## Current Status
 
-- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, redaction of secret query values from `httpx2` request logs, result URLs, and status errors, and mocked `httpx2` transport tests.
+- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, a configurable `RedactionPolicy` (`HTTPModel.redaction`: exact names, substrings, and patterns, with `auth` header and query names added automatically) that masks secret header and query values in `httpx2` request logs, request dumps, result URLs, retry events, and errors, `HTTPDownloadModel` for streaming a response to a local file with size and SHA-256, `HTTPArtifactCaptureModel` for storing a dated download plus a manifest through a `ccflow-etl` file writer, and mocked `httpx2` transport tests.
 - Partial: the shared execution policy is currently consumed inside `HTTPModel` for sequential request spacing; broader evaluator-level concurrency coordination still belongs in evaluator integrations.
 - Missing: broader integration examples and provider-specific subclasses/configs in downstream packages.
 

{ccflow_http-0.2.0 → ccflow_http-0.4.0}/README.md

@@ -15,7 +15,7 @@ It should not contain provider-specific endpoint catalogs. Domain packages can c
 
 ## Current Status
 
-- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, redaction of secret query values from `httpx2` request logs, result URLs, and status errors, and mocked `httpx2` transport tests.
+- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, a configurable `RedactionPolicy` (`HTTPModel.redaction`: exact names, substrings, and patterns, with `auth` header and query names added automatically) that masks secret header and query values in `httpx2` request logs, request dumps, result URLs, retry events, and errors, `HTTPDownloadModel` for streaming a response to a local file with size and SHA-256, `HTTPArtifactCaptureModel` for storing a dated download plus a manifest through a `ccflow-etl` file writer, and mocked `httpx2` transport tests.
 - Partial: the shared execution policy is currently consumed inside `HTTPModel` for sequential request spacing; broader evaluator-level concurrency coordination still belongs in evaluator integrations.
 - Missing: broader integration examples and provider-specific subclasses/configs in downstream packages.
 

ccflow_http-0.4.0/ccflow_http/__init__.py

@@ -0,0 +1,4 @@
+from .base import *
+from .download import *
+
+__version__ = "0.4.0"

{ccflow_http-0.2.0 → ccflow_http-0.4.0}/ccflow_http/base.py

@@ -1,4 +1,7 @@
+import re
 from base64 import b64encode
+from collections.abc import Collection
+from contextvars import ContextVar
 from csv import DictReader
 from datetime import UTC, datetime
 from email.utils import parsedate_to_datetime
@@ -14,7 +17,7 @@ from ccflow import BaseModel, CallableModel, ContextBase, Flow, GenericResult, P
 from ccflow.utils.retry import RetryPolicy
 from ccflow_etl import ExecutionPolicy
 from jinja2 import Environment
-from pydantic import Field
+from pydantic import Field, PrivateAttr, field_validator
 
 __all__ = (
     "HTTPAuth",
@@ -26,6 +29,7 @@ __all__ = (
     "HTTPResponseResult",
     "HTTPResult",
     "HTTPRetryPolicy",
+    "RedactionPolicy",
     "redact_mapping",
     "redact_url",
     "safe_request_dump",
@@ -75,26 +79,79 @@ class HTTPRequest(BaseModel):
     headers: dict[str, str] = Field(default_factory=dict)
     json_data: Any | None = None
     content: bytes | str | None = None
+    # Redaction for this request: the model's policy plus any names its auth places credentials under.
+    _redaction: "RedactionPolicy" = PrivateAttr(default_factory=lambda: _DEFAULT_REDACTION)
 
 
-def _is_sensitive_key(key: str) -> bool:
-    normalized_key = key.lower().replace("_", "").replace("-", "")
-    return normalized_key in {"apikey", "authorization", "password"} or "token" in normalized_key or "secret" in normalized_key
+_QUERY_PAIR = re.compile(r"([?&])([^=&\s'\"]+)=([^&\s'\"]*)")
 
 
-def redact_mapping(values: dict[str, Any]) -> dict[str, Any]:
-    return {key: "***" if _is_sensitive_key(key) else value for key, value in values.items()}
+def _normalize_name(name: str) -> str:
+    return name.lower().replace("_", "").replace("-", "")
 
 
-def redact_url(url: str) -> str:
-    parts = urlsplit(url)
-    if not parts.query:
-        return url
-    pairs = []
-    for pair in parts.query.split("&"):
-        key = pair.split("=", 1)[0]
-        pairs.append(f"{key}=***" if _is_sensitive_key(unquote_plus(key)) else pair)
-    return urlunsplit(parts._replace(query="&".join(pairs)))
+class RedactionPolicy(BaseModel):
+    """Which header and query parameter names carry secrets, and how their values are masked.
+
+    Names are compared after lowercasing and removing hyphens and underscores, so ``X-Api-Key`` and ``x_api_key``
+    are the same name. A name is sensitive when it is one of ``names``, contains one of ``substrings``, or matches one
+    of ``patterns`` (regular expressions searched in the normalized name). Set it on ``HTTPModel.redaction`` to cover
+    a provider's own header or parameter names; names used by ``auth`` are added automatically.
+    """
+
+    names: set[str] = Field(default_factory=lambda: {"authorization", "proxyauthorization", "cookie", "setcookie", "password"})
+    substrings: set[str] = Field(default_factory=lambda: {"apikey", "token", "secret", "password"})
+    patterns: list[str] = Field(default_factory=list)
+    mask: str = "***"
+
+    @field_validator("names", "substrings", mode="after")
+    @classmethod
+    def _normalize(cls, values: set[str]) -> set[str]:
+        return {_normalize_name(value) for value in values}
+
+    def with_names(self, *names: str) -> "RedactionPolicy":
+        """Return a copy that also treats ``names`` as sensitive."""
+        if not names:
+            return self
+        return self.model_copy(update={"names": self.names | {_normalize_name(name) for name in names}})
+
+    def is_sensitive(self, name: str) -> bool:
+        normalized = _normalize_name(name)
+        return (
+            normalized in self.names
+            or any(substring in normalized for substring in self.substrings)
+            or any(re.search(pattern, normalized) for pattern in self.patterns)
+        )
+
+    def redact_mapping(self, values: dict[str, Any]) -> dict[str, Any]:
+        return {key: self.mask if self.is_sensitive(key) else value for key, value in values.items()}
+
+    def redact_url(self, url: str) -> str:
+        parts = urlsplit(url)
+        if not parts.query:
+            return url
+        pairs = []
+        for pair in parts.query.split("&"):
+            key = pair.split("=", 1)[0]
+            pairs.append(f"{key}={self.mask}" if self.is_sensitive(unquote_plus(key)) else pair)
+        return urlunsplit(parts._replace(query="&".join(pairs)))
+
+    def redact_text(self, text: str) -> str:
+        """Redact secret query values that appear anywhere in free text, such as an exception message quoting a URL."""
+        return _QUERY_PAIR.sub(lambda match: f"{match[1]}{match[2]}={self.mask}" if self.is_sensitive(unquote_plus(match[2])) else match[0], text)
+
+
+_DEFAULT_REDACTION = RedactionPolicy()
+# Redaction policy of the request in flight, so the httpx2 log filter can apply it.
+_REQUEST_REDACTION: ContextVar[RedactionPolicy | None] = ContextVar("ccflow_http_request_redaction", default=None)
+
+
+def redact_mapping(values: dict[str, Any], extra: Collection[str] = (), policy: RedactionPolicy | None = None) -> dict[str, Any]:
+    return (policy or _DEFAULT_REDACTION).with_names(*extra).redact_mapping(values)
+
+
+def redact_url(url: str, extra: Collection[str] = (), policy: RedactionPolicy | None = None) -> str:
+    return (policy or _DEFAULT_REDACTION).with_names(*extra).redact_url(url)
 
 
 class _RedactURLFilter(Filter):
@@ -102,7 +159,8 @@ class _RedactURLFilter(Filter):
 
     def filter(self, record: LogRecord) -> bool:
         if isinstance(record.args, tuple):
-            record.args = tuple(redact_url(str(arg)) if isinstance(arg, httpx2.URL) else arg for arg in record.args)
+            policy = _REQUEST_REDACTION.get() or _DEFAULT_REDACTION
+            record.args = tuple(policy.redact_url(str(arg)) if isinstance(arg, httpx2.URL) else arg for arg in record.args)
         return True
 
 
@@ -111,8 +169,8 @@ getLogger("httpx2").addFilter(_RedactURLFilter())
 
 def safe_request_dump(request: HTTPRequest) -> dict[str, Any]:
     request_data = request.model_dump(exclude={"type_"})
-    request_data["params"] = redact_mapping(request_data.get("params", {}))
-    request_data["headers"] = redact_mapping(request_data.get("headers", {}))
+    request_data["params"] = request._redaction.redact_mapping(request_data.get("params", {}))
+    request_data["headers"] = request._redaction.redact_mapping(request_data.get("headers", {}))
     return request_data
 
 
@@ -225,6 +283,9 @@ class HTTPModel(CallableModel):
         description="Error status codes returned as results with value None instead of raising, such as 404 for lookups that may miss.",
     )
     execution_policy: ExecutionPolicy | None = None
+    redaction: RedactionPolicy = Field(
+        default_factory=RedactionPolicy, description="Header and query parameter names whose values are masked in logs, results, and errors."
+    )
     paginate: bool = False
     max_pages: int = 100
     pagination_mode: HTTPPaginationMode = "next_url"
@@ -325,7 +386,7 @@ class HTTPModel(CallableModel):
         rendered_headers = self._render_mapping(headers, data)
         self._apply_auth(rendered_headers, rendered_query, data)
 
-        return HTTPRequest(
+        request = HTTPRequest(
             method=self.method,
             url=self._render(path, data),
             params=rendered_query,
@@ -333,6 +394,11 @@ class HTTPModel(CallableModel):
             json_data=context.json_body if context.json_body is not None else self.json_body,
             content=context.content if context.content is not None else self.content,
         )
+        auth_names = []
+        if self.auth.strategy in ("api_key_query", "api_key_header"):
+            auth_names.append(self._render_required_auth_value(self.auth.name, "name", data))
+        request._redaction = self.redaction.with_names(*auth_names)
+        return request
 
     def _response_value(self, response: httpx2.Response) -> Any:
         match self.response_format:
@@ -398,6 +464,16 @@ class HTTPModel(CallableModel):
             current = current.get(part)
         return current
 
+    def _send(self, client: httpx2.Client, request: HTTPRequest) -> httpx2.Response:
+        return client.request(
+            method=request.method,
+            url=request.url,
+            params=request.params,
+            headers=request.headers,
+            json=request.json_data,
+            content=request.content,
+        )
+
     def _request_once(
         self, client: httpx2.Client, request: HTTPRequest, previous_started_at: float | None
     ) -> tuple[httpx2.Response, int, list[dict[str, Any]], float]:
@@ -409,19 +485,14 @@ class HTTPModel(CallableModel):
             attempts += 1
             previous_started_at = self._throttle(previous_started_at)
             try:
-                response = client.request(
-                    method=request.method,
-                    url=request.url,
-                    params=request.params,
-                    headers=request.headers,
-                    json=request.json_data,
-                    content=request.content,
-                )
+                response = self._send(client, request)
                 if response.status_code in self.accepted_status_codes:
                     return response, attempts, events, previous_started_at
                 response.raise_for_status()
                 return response, attempts, events, previous_started_at
             except httpx2.HTTPStatusError as exc:
+                if exc.response is not None:
+                    exc.response.close()
                 status_code = exc.response.status_code if exc.response is not None else None
                 if retry_policy.should_retry_status(status_code, attempts):
                     delay_seconds = retry_policy.retry_delay_seconds(attempts, total_wait_seconds=total_retry_wait_seconds)
@@ -465,7 +536,7 @@ class HTTPModel(CallableModel):
                                 delay_seconds=delay_seconds,
                                 exception_type=type(exc).__name__,
                                 category=retry_policy.exception_category(exc),
-                                message=str(exc),
+                                message=request._redaction.redact_text(str(exc)),
                             )
                         )
                         self._sleep(delay_seconds)
@@ -477,12 +548,13 @@ class HTTPModel(CallableModel):
                         outcome="failed",
                         exception_type=type(exc).__name__,
                         category=retry_policy.exception_category(exc),
-                        message=str(exc),
+                        message=request._redaction.redact_text(str(exc)),
                     )
                 )
-                raise RuntimeError(f"HTTP {request.method} {self._safe_url(request)} failed with {type(exc).__name__}") from exc
+                # httpx2 exceptions keep the full request, including secret query values, so do not chain them.
+                raise RuntimeError(f"HTTP {request.method} {self._safe_url(request)} failed with {type(exc).__name__}") from None
             except httpx2.HTTPError as exc:
-                raise RuntimeError(f"HTTP {request.method} {self._safe_url(request)} failed with {type(exc).__name__}") from exc
+                raise RuntimeError(f"HTTP {request.method} {self._safe_url(request)} failed with {type(exc).__name__}") from None
 
     def _merge_page_values(self, values: list[Any]) -> Any:
         if not values:
@@ -506,7 +578,7 @@ class HTTPModel(CallableModel):
         next_url_parts = urlsplit(next_url)
         next_url_params = dict(parse_qsl(next_url_parts.query, keep_blank_values=True))
         for key, value in request.params.items():
-            if key not in next_url_params and _is_sensitive_key(key):
+            if key not in next_url_params and request._redaction.is_sensitive(key):
                 next_url_params[key] = value
         next_url_without_query = urlunsplit((next_url_parts.scheme, next_url_parts.netloc, next_url_parts.path, "", next_url_parts.fragment))
         return request.model_copy(update={"url": next_url_without_query, "params": next_url_params})
@@ -557,39 +629,49 @@ class HTTPModel(CallableModel):
     @Flow.call
     def __call__(self, context: HTTPRequestContext) -> HTTPResult:
         request = self._initial_paginated_request(self.build_request(context))
+        redaction = request._redaction
 
-        with httpx2.Client(**self._client_kwargs()) as client:
-            values = []
-            retry_events = []
-            total_attempts = 0
-            pages = 0
-            previous_started_at = None
-            while True:
-                response, attempts, events, previous_started_at = self._request_once(client, request, previous_started_at)
-                total_attempts += attempts
-                retry_events.extend(events)
-                pages += 1
-                accepted_error = response.status_code in self.accepted_status_codes
-                if accepted_error:
-                    break
-                value = self._response_value(response)
-                values.append(value)
-
-                if not self.paginate or pages >= self.max_pages:
-                    break
-                next_request = self._next_paginated_request(request, value)
-                if next_request is None:
-                    break
-                request = next_request
-
-            return HTTPResult(
-                value=None if accepted_error else (self._merge_page_values(values) if self.paginate else values[-1]),
-                status_code=response.status_code,
-                headers=dict(response.headers or {}),
-                url=redact_url(str(response.url)),
-                attempts=total_attempts,
-                pages=pages,
-                rate_limit=self._rate_limit_headers(dict(response.headers or {})),
-                retry_events=retry_events,
-                retry_summary=self._retry_summary(retry_events, attempts=total_attempts, succeeded=True),
-            )
+        token = _REQUEST_REDACTION.set(redaction)
+        try:
+            with httpx2.Client(**self._client_kwargs()) as client:
+                values = []
+                retry_events = []
+                total_attempts = 0
+                pages = 0
+                previous_started_at = None
+                while True:
+                    response, attempts, events, previous_started_at = self._request_once(client, request, previous_started_at)
+                    total_attempts += attempts
+                    retry_events.extend(events)
+                    pages += 1
+                    accepted_error = response.status_code in self.accepted_status_codes
+                    if accepted_error:
+                        if pages > 1:
+                            raise RuntimeError(
+                                f"HTTP {request.method} {self._safe_url(request)} returned status {response.status_code} on page {pages}; "
+                                "accepted statuses are only returned for the first page"
+                            )
+                        break
+                    value = self._response_value(response)
+                    values.append(value)
+
+                    if not self.paginate or pages >= self.max_pages:
+                        break
+                    next_request = self._next_paginated_request(request, value)
+                    if next_request is None:
+                        break
+                    request = next_request
+        finally:
+            _REQUEST_REDACTION.reset(token)
+
+        return HTTPResult(
+            value=None if accepted_error else (self._merge_page_values(values) if self.paginate else values[-1]),
+            status_code=response.status_code,
+            headers=dict(response.headers or {}),
+            url=redaction.redact_url(str(response.url)),
+            attempts=total_attempts,
+            pages=pages,
+            rate_limit=self._rate_limit_headers(dict(response.headers or {})),
+            retry_events=retry_events,
+            retry_summary=self._retry_summary(retry_events, attempts=total_attempts, succeeded=True),
+        )

ccflow_http-0.4.0/ccflow_http/download.py

@@ -0,0 +1,205 @@
+import json
+from datetime import UTC, date, datetime
+from hashlib import sha256
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+import httpx2
+from ccflow import CallableModel, DateContext, Flow, ResultBase
+from ccflow_etl import ArtifactWriteFileContext, ArtifactWriteFileModel, ArtifactWriteFileResult
+from pydantic import Field
+
+from .base import _REQUEST_REDACTION, HTTPModel, HTTPRequest, HTTPRequestContext
+
+__all__ = (
+    "HTTPArtifactCaptureModel",
+    "HTTPArtifactCaptureResult",
+    "HTTPDownloadContext",
+    "HTTPDownloadModel",
+    "HTTPDownloadResult",
+)
+
+
+class HTTPDownloadContext(HTTPRequestContext):
+    output_path: Path
+
+
+class HTTPDownloadResult(ResultBase):
+    path: str
+    size: int
+    sha256: str
+    status_code: int
+    url: str
+    etag: str | None = None
+    last_modified: str | None = None
+    content_type: str | None = None
+    attempts: int = 1
+    retry_events: list[dict[str, Any]] = Field(default_factory=list)
+
+
+class HTTPDownloadModel(HTTPModel):
+    """Stream a response body to a local file without holding it in memory.
+
+    The file is written to a temporary path beside the target and renamed into place only after the full body
+    has been received, so a failed download never leaves a partial file at the target path.
+    """
+
+    chunk_size: int = Field(default=1 << 20, ge=1)
+
+    @property
+    def context_type(self) -> type[HTTPDownloadContext]:
+        return HTTPDownloadContext
+
+    @property
+    def result_type(self) -> type[HTTPDownloadResult]:
+        return HTTPDownloadResult
+
+    def _send(self, client: httpx2.Client, request: HTTPRequest) -> httpx2.Response:
+        built = client.build_request(
+            method=request.method,
+            url=request.url,
+            params=request.params,
+            headers=request.headers,
+            json=request.json_data,
+            content=request.content,
+        )
+        return client.send(built, stream=True)
+
+    @Flow.call
+    def __call__(self, context: HTTPDownloadContext) -> HTTPDownloadResult:
+        request = self.build_request(context)
+        redaction = request._redaction
+        path = context.output_path
+        path.parent.mkdir(parents=True, exist_ok=True)
+        temp_path = path.with_name(f".{path.name}.{uuid4().hex}.tmp")
+        digest = sha256()
+        size = 0
+        token = _REQUEST_REDACTION.set(redaction)
+        try:
+            with httpx2.Client(**self._client_kwargs()) as client:
+                response, attempts, events, _ = self._request_once(client, request, None)
+                try:
+                    with temp_path.open("wb") as file:
+                        for chunk in response.iter_bytes(self.chunk_size):
+                            file.write(chunk)
+                            digest.update(chunk)
+                            size += len(chunk)
+                finally:
+                    response.close()
+            temp_path.replace(path)
+        except BaseException:
+            temp_path.unlink(missing_ok=True)
+            raise
+        finally:
+            _REQUEST_REDACTION.reset(token)
+        return HTTPDownloadResult(
+            path=str(path),
+            size=size,
+            sha256=digest.hexdigest(),
+            status_code=response.status_code,
+            url=redaction.redact_url(str(response.url)),
+            etag=response.headers.get("etag"),
+            last_modified=response.headers.get("last-modified"),
+            content_type=response.headers.get("content-type"),
+            attempts=attempts,
+            retry_events=events,
+        )
+
+
+class HTTPArtifactCaptureResult(ResultBase):
+    date: date
+    key: str
+    manifest_key: str
+    status: str
+    download: HTTPDownloadResult | None = None
+    write: ArtifactWriteFileResult | None = None
+    manifest_write: ArtifactWriteFileResult | None = None
+
+
+class HTTPArtifactCaptureModel(CallableModel):
+    """Capture one HTTP resource for a date: download it, store it under a dated key, and store a manifest beside it.
+
+    The manifest records the redacted URL, capture time, status, response headers, size, and SHA-256 of the stored
+    payload. An existing payload is skipped before downloading unless ``overwrite`` is set.
+    """
+
+    download: HTTPDownloadModel
+    writer: ArtifactWriteFileModel
+    key_template: str = Field(description="Payload key with {date}, {year}, {month}, and {day} placeholders.")
+    workspace: Path = Path("cc-etl-workspace")
+    dataset: str | None = None
+    media_type: str | None = None
+    overwrite: bool = False
+    keep_local: bool = Field(default=False, description="Keep the downloaded file in the workspace after it is stored.")
+    explain: bool = False
+
+    @property
+    def context_type(self) -> type[DateContext]:
+        return DateContext
+
+    @property
+    def result_type(self) -> type[HTTPArtifactCaptureResult]:
+        return HTTPArtifactCaptureResult
+
+    def key(self, context: DateContext) -> str:
+        value = context.date
+        return self.key_template.format(date=value.isoformat(), year=f"{value.year:04d}", month=f"{value.month:02d}", day=f"{value.day:02d}")
+
+    def _write(self, key: str, path: Path, media_type: str | None, dry_run: bool, metadata: dict[str, Any]) -> ArtifactWriteFileResult:
+        return self.writer(
+            ArtifactWriteFileContext(
+                key=key,
+                path=path,
+                media_type=media_type,
+                dataset=self.dataset,
+                stage="extract",
+                overwrite=self.overwrite,
+                dry_run=dry_run,
+                metadata=metadata,
+            )
+        )
+
+    def _exists(self, key: str) -> bool:
+        exists = getattr(self.writer.store, "exists", None)
+        return bool(exists and exists(key))
+
+    @Flow.call
+    def __call__(self, context: DateContext) -> HTTPArtifactCaptureResult:
+        key = self.key(context)
+        manifest_key = f"{key}.manifest.json"
+        local_path = self.workspace / key
+        manifest_path = self.workspace / manifest_key
+        metadata = {"capture_date": context.date.isoformat()}
+        if self.explain:
+            return HTTPArtifactCaptureResult(
+                date=context.date,
+                key=key,
+                manifest_key=manifest_key,
+                status="planned",
+                write=self._write(key, local_path, self.media_type, True, metadata),
+                manifest_write=self._write(manifest_key, manifest_path, "application/json", True, metadata),
+            )
+        if not self.overwrite and self._exists(key):
+            return HTTPArtifactCaptureResult(date=context.date, key=key, manifest_key=manifest_key, status="exists")
+
+        download = self.download(HTTPDownloadContext(output_path=local_path))
+        manifest = {
+            "key": key,
+            "capture_date": context.date.isoformat(),
+            "captured_at": datetime.now(UTC).isoformat(),
+            **download.model_dump(mode="json", exclude={"type_", "path", "retry_events"}),
+        }
+        manifest_path.parent.mkdir(parents=True, exist_ok=True)
+        manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
+        try:
+            write = self._write(key, local_path, self.media_type or download.content_type, False, {**metadata, "sha256": download.sha256})
+            manifest_write = self._write(manifest_key, manifest_path, "application/json", False, metadata)
+        finally:
+            if not self.keep_local:
+                local_path.unlink(missing_ok=True)
+                manifest_path.unlink(missing_ok=True)
+        status = next((value for value in (write.status, manifest_write.status) if value != "exists"), "exists")
+        return HTTPArtifactCaptureResult(
+            date=context.date, key=key, manifest_key=manifest_key, status=status, download=download, write=write, manifest_write=manifest_write
+        )

ccflow_http-0.4.0/ccflow_http/tests/test_download.py

@@ -0,0 +1,110 @@
+import json
+from datetime import date
+from hashlib import sha256
+
+import httpx2
+import pytest
+from ccflow_etl import ArtifactWriteFileModel, LocalFileOutput
+
+from ccflow_http import HTTPArtifactCaptureModel, HTTPAuth, HTTPConfig, HTTPDownloadContext, HTTPDownloadModel, HTTPRetryPolicy
+
+PAYLOAD = b"x" * 5000
+
+
+def _download_model(handler, **kwargs):
+    return HTTPDownloadModel(
+        config=HTTPConfig(base_url="https://files.example.test", transport=httpx2.MockTransport(handler)),
+        path="/bulk/data.zip",
+        chunk_size=1024,
+        **kwargs,
+    )
+
+
+def test_http_download_streams_body_to_file_with_checksum_and_headers(tmp_path):
+    def handler(request):
+        return httpx2.Response(
+            200, content=PAYLOAD, headers={"ETag": '"abc"', "Last-Modified": "Fri, 09 Oct 2026 04:35:00 GMT", "Content-Type": "application/zip"}
+        )
+
+    target = tmp_path / "out" / "data.zip"
+    result = _download_model(handler, auth=HTTPAuth(strategy="api_key_query", name="key", value="secret"))(HTTPDownloadContext(output_path=target))
+
+    assert target.read_bytes() == PAYLOAD
+    assert result.size == len(PAYLOAD)
+    assert result.sha256 == sha256(PAYLOAD).hexdigest()
+    assert (result.etag, result.last_modified, result.content_type) == ('"abc"', "Fri, 09 Oct 2026 04:35:00 GMT", "application/zip")
+    assert result.url == "https://files.example.test/bulk/data.zip?key=***"
+    assert not list(target.parent.glob(".*.tmp"))
+
+
+def test_http_download_retries_status_and_leaves_no_partial_file_on_failure(tmp_path, monkeypatch):
+    monkeypatch.setattr("ccflow_http.base.sleep", lambda seconds: None)
+    calls = []
+
+    def flaky(request):
+        calls.append(request)
+        return httpx2.Response(503) if len(calls) == 1 else httpx2.Response(200, content=PAYLOAD)
+
+    policy = HTTPRetryPolicy(max_attempts=2, wait_initial=0.0, wait_max=0.0, wait_jitter=0.0)
+    result = _download_model(flaky, retry_policy=policy)(HTTPDownloadContext(output_path=tmp_path / "data.zip"))
+    assert result.attempts == 2
+
+    target = tmp_path / "failed.zip"
+    with pytest.raises(RuntimeError, match="failed with status 404"):
+        _download_model(lambda request: httpx2.Response(404))(HTTPDownloadContext(output_path=target))
+    assert not target.exists()
+    assert not list(tmp_path.glob(".failed.zip.*.tmp"))
+
+
+def _capture(tmp_path, handler, **kwargs):
+    store = LocalFileOutput(path=tmp_path / "store")
+    model = HTTPArtifactCaptureModel(
+        download=_download_model(handler),
+        writer=ArtifactWriteFileModel(store=store),
+        key_template="bulk/data/{year}/{month}/{date}.zip",
+        workspace=tmp_path / "workspace",
+        dataset="bulk-data",
+        **kwargs,
+    )
+    return model, store
+
+
+def test_http_capture_stores_payload_and_manifest_and_cleans_workspace(tmp_path):
+    model, store = _capture(tmp_path, lambda request: httpx2.Response(200, content=PAYLOAD, headers={"ETag": '"abc"'}))
+
+    result = model(date(2026, 10, 11))
+    manifest = json.loads(store.file_path("bulk/data/2026/10/2026-10-11.zip.manifest.json").read_text())
+
+    assert result.status == "written"
+    assert store.file_path("bulk/data/2026/10/2026-10-11.zip").read_bytes() == PAYLOAD
+    assert manifest["sha256"] == sha256(PAYLOAD).hexdigest()
+    assert manifest["size"] == len(PAYLOAD)
+    assert manifest["etag"] == '"abc"'
+    assert manifest["capture_date"] == "2026-10-11"
+    assert manifest["url"] == "https://files.example.test/bulk/data.zip"
+    assert not [path for path in (tmp_path / "workspace").rglob("*") if path.is_file()]
+
+
+def test_http_capture_skips_existing_payload_without_downloading(tmp_path):
+    calls = []
+    model, _ = _capture(tmp_path, lambda request: calls.append(request) or httpx2.Response(200, content=PAYLOAD))
+    model(date(2026, 10, 11))
+
+    result = model(date(2026, 10, 11))
+
+    assert result.status == "exists"
+    assert len(calls) == 1
+
+
+def test_http_capture_explain_does_no_network_or_storage_io(tmp_path):
+    def handler(request):
+        raise AssertionError("explain made a request")
+
+    model, _ = _capture(tmp_path, handler, explain=True)
+
+    result = model(date(2026, 10, 11))
+
+    assert result.status == "planned"
+    assert result.write.status == "planned"
+    assert not (tmp_path / "store").exists()
+    assert not (tmp_path / "workspace").exists()

{ccflow_http-0.2.0 → ccflow_http-0.4.0}/ccflow_http/tests/test_model.py

@@ -18,6 +18,7 @@ from ccflow_http import (
     HTTPRequestContext,
     HTTPResponseResult,
     HTTPRetryPolicy,
+    RedactionPolicy,
     redact_url,
     safe_request_dump,
 )
@@ -118,14 +119,14 @@ def test_safe_request_dump_redacts_secret_params_and_headers():
         method="GET",
         url="/v1/tickers/AAA",
         params={"date": "2024-01-03", "apiKey": "query-secret", "page_token": "cursor-secret"},
-        headers={"Authorization": "Bearer header-secret", "X-Request-ID": "abc"},
+        headers={"Authorization": "Bearer header-secret", "X-OPENFIGI-APIKEY": "vendor-secret", "X-Request-ID": "abc"},
     )
 
     assert safe_request_dump(request) == {
         "method": "GET",
         "url": "/v1/tickers/AAA",
         "params": {"date": "2024-01-03", "apiKey": "***", "page_token": "***"},
-        "headers": {"Authorization": "***", "X-Request-ID": "abc"},
+        "headers": {"Authorization": "***", "X-OPENFIGI-APIKEY": "***", "X-Request-ID": "abc"},
         "json_data": None,
         "content": None,
     }
@@ -490,7 +491,7 @@ def test_http_model_returns_accepted_error_status_without_raising():
         HTTPModel(config=config, path="/v1/tickers/ZZZZ")(HTTPRequestContext())
 
 
-def test_http_model_stops_pagination_on_accepted_error_status():
+def test_http_model_raises_on_accepted_error_status_after_first_page():
     def handler(request: httpx2.Request) -> httpx2.Response:
         if "cursor" in request.url.params:
             return httpx2.Response(404)
@@ -503,11 +504,80 @@ def test_http_model_stops_pagination_on_accepted_error_status():
         accepted_status_codes=[404],
     )
 
+    with pytest.raises(RuntimeError, match="status 404 on page 2"):
+        model(HTTPRequestContext())
+
+
+def test_http_model_redacts_custom_query_auth_name_everywhere(caplog):
+    seen_urls = []
+
+    def handler(request: httpx2.Request) -> httpx2.Response:
+        seen_urls.append(str(request.url))
+        if len(seen_urls) == 1:
+            return httpx2.Response(200, json={"results": [{"ticker": "AAA"}], "next_url": "/v1/tickers?cursor=2"})
+        return httpx2.Response(200, json={"results": [{"ticker": "BBB"}]})
+
+    model = HTTPModel(
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(handler)),
+        path="/v1/tickers",
+        paginate=True,
+        auth=HTTPAuth(strategy="api_key_query", name="key", value="custom-secret"),
+    )
+
+    with caplog.at_level("INFO", logger="httpx2"):
+        result = model(HTTPRequestContext())
+
+    assert seen_urls == ["https://api.example.test/v1/tickers?key=custom-secret", "https://api.example.test/v1/tickers?cursor=2&key=custom-secret"]
+    assert "custom-secret" not in caplog.text
+    assert caplog.text.count("key=***") == 2
+    assert result.url == "https://api.example.test/v1/tickers?cursor=2&key=***"
+    assert safe_request_dump(model.build_request(HTTPRequestContext()))["params"] == {"key": "***"}
+
+
+def test_http_model_does_not_leak_query_secrets_through_transport_errors(monkeypatch):
+    monkeypatch.setattr("ccflow_http.base.sleep", lambda seconds: None)
+    calls = []
+
+    def handler(request: httpx2.Request) -> httpx2.Response:
+        calls.append(request)
+        if len(calls) == 1:
+            raise httpx2.ConnectError(f"connection reset for {request.url}", request=request)
+        raise httpx2.ReadTimeout(f"read timed out for {request.url}", request=request)
+
+    model = HTTPModel(
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(handler)),
+        path="/v1/tickers",
+        auth=HTTPAuth(strategy="api_key_query", name="key", value="custom-secret"),
+        retry_policy=HTTPRetryPolicy(max_attempts=2, wait_initial=0.0, wait_max=0.0, wait_jitter=0.0),
+    )
+
+    with pytest.raises(RuntimeError, match="failed with ReadTimeout") as error:
+        model(HTTPRequestContext())
+
+    assert error.value.__cause__ is None
+    assert error.value.__suppress_context__
+
+
+def test_http_model_redacts_query_secrets_in_retry_event_messages(monkeypatch):
+    monkeypatch.setattr("ccflow_http.base.sleep", lambda seconds: None)
+    calls = []
+
+    def handler(request: httpx2.Request) -> httpx2.Response:
+        calls.append(request)
+        if len(calls) == 1:
+            raise httpx2.ConnectError(f"connection reset for {request.url}", request=request)
+        return httpx2.Response(200, json={"status": "OK"})
+
+    model = HTTPModel(
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(handler)),
+        path="/v1/tickers",
+        auth=HTTPAuth(strategy="api_key_query", name="key", value="custom-secret"),
+        retry_policy=HTTPRetryPolicy(max_attempts=2, wait_initial=0.0, wait_max=0.0, wait_jitter=0.0),
+    )
+
     result = model(HTTPRequestContext())
 
-    assert result.status_code == 404
-    assert result.value is None
-    assert result.pages == 2
+    assert result.retry_events[0]["message"] == "connection reset for https://api.example.test/v1/tickers?key=***"
 
 
 def test_http_model_parses_csv_and_gzip_responses_with_mock_transport():
@@ -728,3 +798,67 @@ def test_http_model_ignores_invalid_retry_after(monkeypatch):
     model(HTTPRequestContext())
 
     assert sleeps == [1.0]
+
+
+def test_redaction_policy_matches_normalized_names_substrings_and_patterns():
+    policy = RedactionPolicy(names={"X-Session"}, substrings={"Signature"}, patterns=[r"^sig\d+$"], mask="<redacted>")
+
+    assert policy.is_sensitive("x_session") and policy.is_sensitive("X-SESSION")
+    assert policy.is_sensitive("X-Amz-Signature")
+    assert policy.is_sensitive("sig2") and not policy.is_sensitive("signal")
+    assert not policy.is_sensitive("apiKey")
+    assert policy.with_names("apiKey").is_sensitive("API_KEY")
+    assert policy.redact_mapping({"X-Session": "s", "date": "d"}) == {"X-Session": "<redacted>", "date": "d"}
+    assert policy.redact_url("https://h/p?sig2=a&date=d") == "https://h/p?sig2=<redacted>&date=d"
+    assert policy.redact_text("failed for https://h/p?x-session=a&date=d") == "failed for https://h/p?x-session=<redacted>&date=d"
+
+
+def test_default_redaction_policy_covers_common_credential_names():
+    policy = RedactionPolicy()
+
+    for name in [
+        "Authorization",
+        "Proxy-Authorization",
+        "Cookie",
+        "Set-Cookie",
+        "password",
+        "apiKey",
+        "X-OPENFIGI-APIKEY",
+        "page_token",
+        "client_secret",
+    ]:
+        assert policy.is_sensitive(name), name
+    for name in ["date", "ticker", "X-Request-ID", "limit"]:
+        assert not policy.is_sensitive(name), name
+
+
+def test_http_model_applies_configured_and_auth_redaction_names_everywhere(caplog, monkeypatch):
+    monkeypatch.setattr("ccflow_http.base.sleep", lambda seconds: None)
+    seen = []
+
+    def handler(request: httpx2.Request) -> httpx2.Response:
+        seen.append(request)
+        if len(seen) == 1:
+            raise httpx2.ConnectError(f"connection reset for {request.url}", request=request)
+        return httpx2.Response(200, json={"ok": True})
+
+    model = HTTPModel(
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(handler)),
+        path="/v1/items",
+        query={"sig": "query-secret", "date": "2025-01-02"},
+        headers={"X-Vendor-Session": "header-secret"},
+        auth=HTTPAuth(strategy="api_key_header", name="X-Partner", value="auth-secret"),
+        redaction=RedactionPolicy(names={"X-Vendor-Session"}, patterns=[r"^sig$"]),
+        retry_policy=HTTPRetryPolicy(max_attempts=2, wait_initial=0.0, wait_max=0.0, wait_jitter=0.0),
+    )
+
+    with caplog.at_level("INFO", logger="httpx2"):
+        result = model(HTTPRequestContext())
+    dump = safe_request_dump(model.build_request(HTTPRequestContext()))
+
+    assert seen[-1].headers["X-Partner"] == "auth-secret" and seen[-1].headers["X-Vendor-Session"] == "header-secret"
+    assert dump["headers"]["X-Partner"] == "***" and dump["headers"]["X-Vendor-Session"] == "***"
+    assert dump["params"] == {"sig": "***", "date": "2025-01-02"}
+    assert result.url == "https://api.example.test/v1/items?sig=***&date=2025-01-02"
+    assert "query-secret" not in caplog.text
+    assert "query-secret" not in result.retry_events[0]["message"]

{ccflow_http-0.2.0 → ccflow_http-0.4.0}/pyproject.toml

@@ -12,7 +12,7 @@ authors = [
 description = "ccflow models for HTTP"
 readme = "README.md"
 license = { text = "Apache-2.0" }
-version = "0.2.0"
+version = "0.4.0"
 requires-python = ">=3.11"
 keywords = []
 
@@ -30,7 +30,7 @@ classifiers = [
 
 dependencies = [
     "ccflow>=0.8.5",
-    "ccflow-etl>=0.1.0",
+    "ccflow-etl>=0.4.0",
     "httpx2",
     "jinja2",
 ]
@@ -43,6 +43,7 @@ develop = [
     "codespell",
     "hatchling",
     "mdformat",
+    "mdformat-frontmatter",
     "mdformat-tables>=1",
     "pytest",
     "pytest-cov",
@@ -60,7 +61,7 @@ Repository = "https://github.com/1kbgz/ccflow-http"
 Homepage = "https://github.com/1kbgz/ccflow-http"
 
 [tool.bumpversion]
-current_version = "0.2.0"
+current_version = "0.4.0"
 commit = true
 tag = true
 commit_args = "-s"
@@ -78,7 +79,7 @@ replace = 'version = "{new_version}"'
 [tool.coverage.run]
 branch = true
 omit = [
-    "ccflow_http/tests/integration/",
+    "ccflow_http/tests/integration/*",
 ]
 
 [tool.coverage.report]

ccflow_http-0.2.0/ccflow_http/__init__.py

@@ -1,3 +0,0 @@
-from .base import *
-
-__version__ = "0.2.0"