ccflow-http 0.2.0__tar.gz → 0.3.0__tar.gz

{ccflow_http-0.2.0 → ccflow_http-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ccflow-http
-Version: 0.2.0
+Version: 0.3.0
 Summary: ccflow models for HTTP
 Project-URL: Repository, https://github.com/1kbgz/ccflow-http
 Project-URL: Homepage, https://github.com/1kbgz/ccflow-http
@@ -17,7 +17,7 @@ Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.11
-Requires-Dist: ccflow-etl>=0.1.0
+Requires-Dist: ccflow-etl>=0.4.0
 Requires-Dist: ccflow>=0.8.5
 Requires-Dist: httpx2
 Requires-Dist: jinja2
@@ -28,6 +28,7 @@ Requires-Dist: check-dist; extra == 'develop'
 Requires-Dist: codespell; extra == 'develop'
 Requires-Dist: hatchling; extra == 'develop'
 Requires-Dist: mdformat; extra == 'develop'
+Requires-Dist: mdformat-frontmatter; extra == 'develop'
 Requires-Dist: mdformat-tables>=1; extra == 'develop'
 Requires-Dist: pytest; extra == 'develop'
 Requires-Dist: pytest-cov; extra == 'develop'
@@ -55,7 +56,7 @@ It should not contain provider-specific endpoint catalogs. Domain packages can c
 
 ## Current Status
 
-- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, redaction of secret query values from `httpx2` request logs, result URLs, and status errors, and mocked `httpx2` transport tests.
+- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, redaction of secret query values (including the configured `api_key_query` parameter) from `httpx2` request logs, request dumps, result URLs, retry events, and errors, `HTTPDownloadModel` for streaming a response to a local file with size and SHA-256, `HTTPArtifactCaptureModel` for storing a dated download plus a manifest through a `ccflow-etl` file writer, and mocked `httpx2` transport tests.
 - Partial: the shared execution policy is currently consumed inside `HTTPModel` for sequential request spacing; broader evaluator-level concurrency coordination still belongs in evaluator integrations.
 - Missing: broader integration examples and provider-specific subclasses/configs in downstream packages.
 

{ccflow_http-0.2.0 → ccflow_http-0.3.0}/README.md

@@ -15,7 +15,7 @@ It should not contain provider-specific endpoint catalogs. Domain packages can c
 
 ## Current Status
 
-- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, redaction of secret query values from `httpx2` request logs, result URLs, and status errors, and mocked `httpx2` transport tests.
+- Implemented: `HTTPConfig`, `HTTPAuth`, `HTTPContext`, `HTTPRequestContext`, `HTTPRequest`, `HTTPRetryPolicy`, `HTTPResponseResult`, compatibility `HTTPResult`, `HTTPModel`, templated path/query/header rendering, request explanation through `build_request`, no-auth/bearer/API-key/basic auth helpers, JSON/text/bytes/CSV/gzip response parsing, HTTP status retry classification over `ccflow` retry semantics, accepted error statuses returned as results instead of exceptions, retry event summaries on HTTP results, `ccflow-etl` `ExecutionPolicy` request spacing, `next_url`/cursor/page/offset pagination, rate-limit header capture, redaction of secret query values (including the configured `api_key_query` parameter) from `httpx2` request logs, request dumps, result URLs, retry events, and errors, `HTTPDownloadModel` for streaming a response to a local file with size and SHA-256, `HTTPArtifactCaptureModel` for storing a dated download plus a manifest through a `ccflow-etl` file writer, and mocked `httpx2` transport tests.
 - Partial: the shared execution policy is currently consumed inside `HTTPModel` for sequential request spacing; broader evaluator-level concurrency coordination still belongs in evaluator integrations.
 - Missing: broader integration examples and provider-specific subclasses/configs in downstream packages.
 

ccflow_http-0.3.0/ccflow_http/__init__.py

@@ -0,0 +1,4 @@
+from .base import *
+from .download import *
+
+__version__ = "0.3.0"

{ccflow_http-0.2.0 → ccflow_http-0.3.0}/ccflow_http/base.py

@@ -1,4 +1,7 @@
+import re
 from base64 import b64encode
+from collections.abc import Collection
+from contextvars import ContextVar
 from csv import DictReader
 from datetime import UTC, datetime
 from email.utils import parsedate_to_datetime
@@ -14,7 +17,7 @@ from ccflow import BaseModel, CallableModel, ContextBase, Flow, GenericResult, P
 from ccflow.utils.retry import RetryPolicy
 from ccflow_etl import ExecutionPolicy
 from jinja2 import Environment
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 
 __all__ = (
     "HTTPAuth",
@@ -75,34 +78,47 @@ class HTTPRequest(BaseModel):
     headers: dict[str, str] = Field(default_factory=dict)
     json_data: Any | None = None
     content: bytes | str | None = None
+    # Query parameter names that carry credentials under names the key heuristic may not recognize, such as api_key_query auth.
+    _sensitive_params: frozenset[str] = PrivateAttr(default_factory=frozenset)
 
 
-def _is_sensitive_key(key: str) -> bool:
+# Extra sensitive query names for the request in flight, so the httpx2 log filter can redact them.
+_REQUEST_SENSITIVE_PARAMS: ContextVar[frozenset[str]] = ContextVar("ccflow_http_request_sensitive_params", default=frozenset())
+_QUERY_PAIR = re.compile(r"([?&])([^=&\s'\"]+)=([^&\s'\"]*)")
+
+
+def _is_sensitive_key(key: str, extra: Collection[str] = ()) -> bool:
     normalized_key = key.lower().replace("_", "").replace("-", "")
-    return normalized_key in {"apikey", "authorization", "password"} or "token" in normalized_key or "secret" in normalized_key
+    return key in extra or normalized_key in {"apikey", "authorization", "password"} or "token" in normalized_key or "secret" in normalized_key
 
 
-def redact_mapping(values: dict[str, Any]) -> dict[str, Any]:
-    return {key: "***" if _is_sensitive_key(key) else value for key, value in values.items()}
+def redact_mapping(values: dict[str, Any], extra: Collection[str] = ()) -> dict[str, Any]:
+    return {key: "***" if _is_sensitive_key(key, extra) else value for key, value in values.items()}
 
 
-def redact_url(url: str) -> str:
+def redact_url(url: str, extra: Collection[str] = ()) -> str:
     parts = urlsplit(url)
     if not parts.query:
         return url
     pairs = []
     for pair in parts.query.split("&"):
         key = pair.split("=", 1)[0]
-        pairs.append(f"{key}=***" if _is_sensitive_key(unquote_plus(key)) else pair)
+        pairs.append(f"{key}=***" if _is_sensitive_key(unquote_plus(key), extra) else pair)
     return urlunsplit(parts._replace(query="&".join(pairs)))
 
 
+def _redact_query_text(text: str, extra: Collection[str] = ()) -> str:
+    """Redact secret query values that appear anywhere in free text, such as an exception message quoting a URL."""
+    return _QUERY_PAIR.sub(lambda match: f"{match[1]}{match[2]}=***" if _is_sensitive_key(unquote_plus(match[2]), extra) else match[0], text)
+
+
 class _RedactURLFilter(Filter):
     """Redact secret query values from URLs that httpx2 logs for each request."""
 
     def filter(self, record: LogRecord) -> bool:
         if isinstance(record.args, tuple):
-            record.args = tuple(redact_url(str(arg)) if isinstance(arg, httpx2.URL) else arg for arg in record.args)
+            extra = _REQUEST_SENSITIVE_PARAMS.get()
+            record.args = tuple(redact_url(str(arg), extra) if isinstance(arg, httpx2.URL) else arg for arg in record.args)
         return True
 
 
@@ -111,7 +127,7 @@ getLogger("httpx2").addFilter(_RedactURLFilter())
 
 def safe_request_dump(request: HTTPRequest) -> dict[str, Any]:
     request_data = request.model_dump(exclude={"type_"})
-    request_data["params"] = redact_mapping(request_data.get("params", {}))
+    request_data["params"] = redact_mapping(request_data.get("params", {}), request._sensitive_params)
     request_data["headers"] = redact_mapping(request_data.get("headers", {}))
     return request_data
 
@@ -325,7 +341,7 @@ class HTTPModel(CallableModel):
         rendered_headers = self._render_mapping(headers, data)
         self._apply_auth(rendered_headers, rendered_query, data)
 
-        return HTTPRequest(
+        request = HTTPRequest(
             method=self.method,
             url=self._render(path, data),
             params=rendered_query,
@@ -333,6 +349,9 @@ class HTTPModel(CallableModel):
             json_data=context.json_body if context.json_body is not None else self.json_body,
             content=context.content if context.content is not None else self.content,
         )
+        if self.auth.strategy == "api_key_query":
+            request._sensitive_params = frozenset({self._render_required_auth_value(self.auth.name, "name", data)})
+        return request
 
     def _response_value(self, response: httpx2.Response) -> Any:
         match self.response_format:
@@ -398,6 +417,16 @@ class HTTPModel(CallableModel):
             current = current.get(part)
         return current
 
+    def _send(self, client: httpx2.Client, request: HTTPRequest) -> httpx2.Response:
+        return client.request(
+            method=request.method,
+            url=request.url,
+            params=request.params,
+            headers=request.headers,
+            json=request.json_data,
+            content=request.content,
+        )
+
     def _request_once(
         self, client: httpx2.Client, request: HTTPRequest, previous_started_at: float | None
     ) -> tuple[httpx2.Response, int, list[dict[str, Any]], float]:
@@ -409,19 +438,14 @@ class HTTPModel(CallableModel):
             attempts += 1
             previous_started_at = self._throttle(previous_started_at)
             try:
-                response = client.request(
-                    method=request.method,
-                    url=request.url,
-                    params=request.params,
-                    headers=request.headers,
-                    json=request.json_data,
-                    content=request.content,
-                )
+                response = self._send(client, request)
                 if response.status_code in self.accepted_status_codes:
                     return response, attempts, events, previous_started_at
                 response.raise_for_status()
                 return response, attempts, events, previous_started_at
             except httpx2.HTTPStatusError as exc:
+                if exc.response is not None:
+                    exc.response.close()
                 status_code = exc.response.status_code if exc.response is not None else None
                 if retry_policy.should_retry_status(status_code, attempts):
                     delay_seconds = retry_policy.retry_delay_seconds(attempts, total_wait_seconds=total_retry_wait_seconds)
@@ -465,7 +489,7 @@ class HTTPModel(CallableModel):
                                 delay_seconds=delay_seconds,
                                 exception_type=type(exc).__name__,
                                 category=retry_policy.exception_category(exc),
-                                message=str(exc),
+                                message=_redact_query_text(str(exc), request._sensitive_params),
                             )
                         )
                         self._sleep(delay_seconds)
@@ -477,12 +501,13 @@ class HTTPModel(CallableModel):
                         outcome="failed",
                         exception_type=type(exc).__name__,
                         category=retry_policy.exception_category(exc),
-                        message=str(exc),
+                        message=_redact_query_text(str(exc), request._sensitive_params),
                     )
                 )
-                raise RuntimeError(f"HTTP {request.method} {self._safe_url(request)} failed with {type(exc).__name__}") from exc
+                # httpx2 exceptions keep the full request, including secret query values, so do not chain them.
+                raise RuntimeError(f"HTTP {request.method} {self._safe_url(request)} failed with {type(exc).__name__}") from None
             except httpx2.HTTPError as exc:
-                raise RuntimeError(f"HTTP {request.method} {self._safe_url(request)} failed with {type(exc).__name__}") from exc
+                raise RuntimeError(f"HTTP {request.method} {self._safe_url(request)} failed with {type(exc).__name__}") from None
 
     def _merge_page_values(self, values: list[Any]) -> Any:
         if not values:
@@ -506,7 +531,7 @@ class HTTPModel(CallableModel):
         next_url_parts = urlsplit(next_url)
         next_url_params = dict(parse_qsl(next_url_parts.query, keep_blank_values=True))
         for key, value in request.params.items():
-            if key not in next_url_params and _is_sensitive_key(key):
+            if key not in next_url_params and _is_sensitive_key(key, request._sensitive_params):
                 next_url_params[key] = value
         next_url_without_query = urlunsplit((next_url_parts.scheme, next_url_parts.netloc, next_url_parts.path, "", next_url_parts.fragment))
         return request.model_copy(update={"url": next_url_without_query, "params": next_url_params})
@@ -557,39 +582,49 @@ class HTTPModel(CallableModel):
     @Flow.call
     def __call__(self, context: HTTPRequestContext) -> HTTPResult:
         request = self._initial_paginated_request(self.build_request(context))
+        sensitive_params = request._sensitive_params
 
-        with httpx2.Client(**self._client_kwargs()) as client:
-            values = []
-            retry_events = []
-            total_attempts = 0
-            pages = 0
-            previous_started_at = None
-            while True:
-                response, attempts, events, previous_started_at = self._request_once(client, request, previous_started_at)
-                total_attempts += attempts
-                retry_events.extend(events)
-                pages += 1
-                accepted_error = response.status_code in self.accepted_status_codes
-                if accepted_error:
-                    break
-                value = self._response_value(response)
-                values.append(value)
-
-                if not self.paginate or pages >= self.max_pages:
-                    break
-                next_request = self._next_paginated_request(request, value)
-                if next_request is None:
-                    break
-                request = next_request
-
-            return HTTPResult(
-                value=None if accepted_error else (self._merge_page_values(values) if self.paginate else values[-1]),
-                status_code=response.status_code,
-                headers=dict(response.headers or {}),
-                url=redact_url(str(response.url)),
-                attempts=total_attempts,
-                pages=pages,
-                rate_limit=self._rate_limit_headers(dict(response.headers or {})),
-                retry_events=retry_events,
-                retry_summary=self._retry_summary(retry_events, attempts=total_attempts, succeeded=True),
-            )
+        token = _REQUEST_SENSITIVE_PARAMS.set(sensitive_params)
+        try:
+            with httpx2.Client(**self._client_kwargs()) as client:
+                values = []
+                retry_events = []
+                total_attempts = 0
+                pages = 0
+                previous_started_at = None
+                while True:
+                    response, attempts, events, previous_started_at = self._request_once(client, request, previous_started_at)
+                    total_attempts += attempts
+                    retry_events.extend(events)
+                    pages += 1
+                    accepted_error = response.status_code in self.accepted_status_codes
+                    if accepted_error:
+                        if pages > 1:
+                            raise RuntimeError(
+                                f"HTTP {request.method} {self._safe_url(request)} returned status {response.status_code} on page {pages}; "
+                                "accepted statuses are only returned for the first page"
+                            )
+                        break
+                    value = self._response_value(response)
+                    values.append(value)
+
+                    if not self.paginate or pages >= self.max_pages:
+                        break
+                    next_request = self._next_paginated_request(request, value)
+                    if next_request is None:
+                        break
+                    request = next_request
+        finally:
+            _REQUEST_SENSITIVE_PARAMS.reset(token)
+
+        return HTTPResult(
+            value=None if accepted_error else (self._merge_page_values(values) if self.paginate else values[-1]),
+            status_code=response.status_code,
+            headers=dict(response.headers or {}),
+            url=redact_url(str(response.url), sensitive_params),
+            attempts=total_attempts,
+            pages=pages,
+            rate_limit=self._rate_limit_headers(dict(response.headers or {})),
+            retry_events=retry_events,
+            retry_summary=self._retry_summary(retry_events, attempts=total_attempts, succeeded=True),
+        )

ccflow_http-0.3.0/ccflow_http/download.py

@@ -0,0 +1,205 @@
+import json
+from datetime import UTC, date, datetime
+from hashlib import sha256
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+import httpx2
+from ccflow import CallableModel, DateContext, Flow, ResultBase
+from ccflow_etl import ArtifactWriteFileContext, ArtifactWriteFileModel, ArtifactWriteFileResult
+from pydantic import Field
+
+from .base import _REQUEST_SENSITIVE_PARAMS, HTTPModel, HTTPRequest, HTTPRequestContext, redact_url
+
+__all__ = (
+    "HTTPArtifactCaptureModel",
+    "HTTPArtifactCaptureResult",
+    "HTTPDownloadContext",
+    "HTTPDownloadModel",
+    "HTTPDownloadResult",
+)
+
+
+class HTTPDownloadContext(HTTPRequestContext):
+    output_path: Path
+
+
+class HTTPDownloadResult(ResultBase):
+    path: str
+    size: int
+    sha256: str
+    status_code: int
+    url: str
+    etag: str | None = None
+    last_modified: str | None = None
+    content_type: str | None = None
+    attempts: int = 1
+    retry_events: list[dict[str, Any]] = Field(default_factory=list)
+
+
+class HTTPDownloadModel(HTTPModel):
+    """Stream a response body to a local file without holding it in memory.
+
+    The file is written to a temporary path beside the target and renamed into place only after the full body
+    has been received, so a failed download never leaves a partial file at the target path.
+    """
+
+    chunk_size: int = Field(default=1 << 20, ge=1)
+
+    @property
+    def context_type(self) -> type[HTTPDownloadContext]:
+        return HTTPDownloadContext
+
+    @property
+    def result_type(self) -> type[HTTPDownloadResult]:
+        return HTTPDownloadResult
+
+    def _send(self, client: httpx2.Client, request: HTTPRequest) -> httpx2.Response:
+        built = client.build_request(
+            method=request.method,
+            url=request.url,
+            params=request.params,
+            headers=request.headers,
+            json=request.json_data,
+            content=request.content,
+        )
+        return client.send(built, stream=True)
+
+    @Flow.call
+    def __call__(self, context: HTTPDownloadContext) -> HTTPDownloadResult:
+        request = self.build_request(context)
+        sensitive_params = request._sensitive_params
+        path = context.output_path
+        path.parent.mkdir(parents=True, exist_ok=True)
+        temp_path = path.with_name(f".{path.name}.{uuid4().hex}.tmp")
+        digest = sha256()
+        size = 0
+        token = _REQUEST_SENSITIVE_PARAMS.set(sensitive_params)
+        try:
+            with httpx2.Client(**self._client_kwargs()) as client:
+                response, attempts, events, _ = self._request_once(client, request, None)
+                try:
+                    with temp_path.open("wb") as file:
+                        for chunk in response.iter_bytes(self.chunk_size):
+                            file.write(chunk)
+                            digest.update(chunk)
+                            size += len(chunk)
+                finally:
+                    response.close()
+            temp_path.replace(path)
+        except BaseException:
+            temp_path.unlink(missing_ok=True)
+            raise
+        finally:
+            _REQUEST_SENSITIVE_PARAMS.reset(token)
+        return HTTPDownloadResult(
+            path=str(path),
+            size=size,
+            sha256=digest.hexdigest(),
+            status_code=response.status_code,
+            url=redact_url(str(response.url), sensitive_params),
+            etag=response.headers.get("etag"),
+            last_modified=response.headers.get("last-modified"),
+            content_type=response.headers.get("content-type"),
+            attempts=attempts,
+            retry_events=events,
+        )
+
+
+class HTTPArtifactCaptureResult(ResultBase):
+    date: date
+    key: str
+    manifest_key: str
+    status: str
+    download: HTTPDownloadResult | None = None
+    write: ArtifactWriteFileResult | None = None
+    manifest_write: ArtifactWriteFileResult | None = None
+
+
+class HTTPArtifactCaptureModel(CallableModel):
+    """Capture one HTTP resource for a date: download it, store it under a dated key, and store a manifest beside it.
+
+    The manifest records the redacted URL, capture time, status, response headers, size, and SHA-256 of the stored
+    payload. An existing payload is skipped before downloading unless ``overwrite`` is set.
+    """
+
+    download: HTTPDownloadModel
+    writer: ArtifactWriteFileModel
+    key_template: str = Field(description="Payload key with {date}, {year}, {month}, and {day} placeholders.")
+    workspace: Path = Path("cc-etl-workspace")
+    dataset: str | None = None
+    media_type: str | None = None
+    overwrite: bool = False
+    keep_local: bool = Field(default=False, description="Keep the downloaded file in the workspace after it is stored.")
+    explain: bool = False
+
+    @property
+    def context_type(self) -> type[DateContext]:
+        return DateContext
+
+    @property
+    def result_type(self) -> type[HTTPArtifactCaptureResult]:
+        return HTTPArtifactCaptureResult
+
+    def key(self, context: DateContext) -> str:
+        value = context.date
+        return self.key_template.format(date=value.isoformat(), year=f"{value.year:04d}", month=f"{value.month:02d}", day=f"{value.day:02d}")
+
+    def _write(self, key: str, path: Path, media_type: str | None, dry_run: bool, metadata: dict[str, Any]) -> ArtifactWriteFileResult:
+        return self.writer(
+            ArtifactWriteFileContext(
+                key=key,
+                path=path,
+                media_type=media_type,
+                dataset=self.dataset,
+                stage="extract",
+                overwrite=self.overwrite,
+                dry_run=dry_run,
+                metadata=metadata,
+            )
+        )
+
+    def _exists(self, key: str) -> bool:
+        exists = getattr(self.writer.store, "exists", None)
+        return bool(exists and exists(key))
+
+    @Flow.call
+    def __call__(self, context: DateContext) -> HTTPArtifactCaptureResult:
+        key = self.key(context)
+        manifest_key = f"{key}.manifest.json"
+        local_path = self.workspace / key
+        manifest_path = self.workspace / manifest_key
+        metadata = {"capture_date": context.date.isoformat()}
+        if self.explain:
+            return HTTPArtifactCaptureResult(
+                date=context.date,
+                key=key,
+                manifest_key=manifest_key,
+                status="planned",
+                write=self._write(key, local_path, self.media_type, True, metadata),
+                manifest_write=self._write(manifest_key, manifest_path, "application/json", True, metadata),
+            )
+        if not self.overwrite and self._exists(key):
+            return HTTPArtifactCaptureResult(date=context.date, key=key, manifest_key=manifest_key, status="exists")
+
+        download = self.download(HTTPDownloadContext(output_path=local_path))
+        manifest = {
+            "key": key,
+            "capture_date": context.date.isoformat(),
+            "captured_at": datetime.now(UTC).isoformat(),
+            **download.model_dump(mode="json", exclude={"type_", "path", "retry_events"}),
+        }
+        manifest_path.parent.mkdir(parents=True, exist_ok=True)
+        manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
+        try:
+            write = self._write(key, local_path, self.media_type or download.content_type, False, {**metadata, "sha256": download.sha256})
+            manifest_write = self._write(manifest_key, manifest_path, "application/json", False, metadata)
+        finally:
+            if not self.keep_local:
+                local_path.unlink(missing_ok=True)
+                manifest_path.unlink(missing_ok=True)
+        status = next((value for value in (write.status, manifest_write.status) if value != "exists"), "exists")
+        return HTTPArtifactCaptureResult(
+            date=context.date, key=key, manifest_key=manifest_key, status=status, download=download, write=write, manifest_write=manifest_write
+        )

ccflow_http-0.3.0/ccflow_http/tests/test_download.py

@@ -0,0 +1,110 @@
+import json
+from datetime import date
+from hashlib import sha256
+
+import httpx2
+import pytest
+from ccflow_etl import ArtifactWriteFileModel, LocalFileOutput
+
+from ccflow_http import HTTPArtifactCaptureModel, HTTPAuth, HTTPConfig, HTTPDownloadContext, HTTPDownloadModel, HTTPRetryPolicy
+
+PAYLOAD = b"x" * 5000
+
+
+def _download_model(handler, **kwargs):
+    return HTTPDownloadModel(
+        config=HTTPConfig(base_url="https://files.example.test", transport=httpx2.MockTransport(handler)),
+        path="/bulk/data.zip",
+        chunk_size=1024,
+        **kwargs,
+    )
+
+
+def test_http_download_streams_body_to_file_with_checksum_and_headers(tmp_path):
+    def handler(request):
+        return httpx2.Response(
+            200, content=PAYLOAD, headers={"ETag": '"abc"', "Last-Modified": "Fri, 09 Oct 2026 04:35:00 GMT", "Content-Type": "application/zip"}
+        )
+
+    target = tmp_path / "out" / "data.zip"
+    result = _download_model(handler, auth=HTTPAuth(strategy="api_key_query", name="key", value="secret"))(HTTPDownloadContext(output_path=target))
+
+    assert target.read_bytes() == PAYLOAD
+    assert result.size == len(PAYLOAD)
+    assert result.sha256 == sha256(PAYLOAD).hexdigest()
+    assert (result.etag, result.last_modified, result.content_type) == ('"abc"', "Fri, 09 Oct 2026 04:35:00 GMT", "application/zip")
+    assert result.url == "https://files.example.test/bulk/data.zip?key=***"
+    assert not list(target.parent.glob(".*.tmp"))
+
+
+def test_http_download_retries_status_and_leaves_no_partial_file_on_failure(tmp_path, monkeypatch):
+    monkeypatch.setattr("ccflow_http.base.sleep", lambda seconds: None)
+    calls = []
+
+    def flaky(request):
+        calls.append(request)
+        return httpx2.Response(503) if len(calls) == 1 else httpx2.Response(200, content=PAYLOAD)
+
+    policy = HTTPRetryPolicy(max_attempts=2, wait_initial=0.0, wait_max=0.0, wait_jitter=0.0)
+    result = _download_model(flaky, retry_policy=policy)(HTTPDownloadContext(output_path=tmp_path / "data.zip"))
+    assert result.attempts == 2
+
+    target = tmp_path / "failed.zip"
+    with pytest.raises(RuntimeError, match="failed with status 404"):
+        _download_model(lambda request: httpx2.Response(404))(HTTPDownloadContext(output_path=target))
+    assert not target.exists()
+    assert not list(tmp_path.glob(".failed.zip.*.tmp"))
+
+
+def _capture(tmp_path, handler, **kwargs):
+    store = LocalFileOutput(path=tmp_path / "store")
+    model = HTTPArtifactCaptureModel(
+        download=_download_model(handler),
+        writer=ArtifactWriteFileModel(store=store),
+        key_template="bulk/data/{year}/{month}/{date}.zip",
+        workspace=tmp_path / "workspace",
+        dataset="bulk-data",
+        **kwargs,
+    )
+    return model, store
+
+
+def test_http_capture_stores_payload_and_manifest_and_cleans_workspace(tmp_path):
+    model, store = _capture(tmp_path, lambda request: httpx2.Response(200, content=PAYLOAD, headers={"ETag": '"abc"'}))
+
+    result = model(date(2026, 10, 11))
+    manifest = json.loads(store.file_path("bulk/data/2026/10/2026-10-11.zip.manifest.json").read_text())
+
+    assert result.status == "written"
+    assert store.file_path("bulk/data/2026/10/2026-10-11.zip").read_bytes() == PAYLOAD
+    assert manifest["sha256"] == sha256(PAYLOAD).hexdigest()
+    assert manifest["size"] == len(PAYLOAD)
+    assert manifest["etag"] == '"abc"'
+    assert manifest["capture_date"] == "2026-10-11"
+    assert manifest["url"] == "https://files.example.test/bulk/data.zip"
+    assert not [path for path in (tmp_path / "workspace").rglob("*") if path.is_file()]
+
+
+def test_http_capture_skips_existing_payload_without_downloading(tmp_path):
+    calls = []
+    model, _ = _capture(tmp_path, lambda request: calls.append(request) or httpx2.Response(200, content=PAYLOAD))
+    model(date(2026, 10, 11))
+
+    result = model(date(2026, 10, 11))
+
+    assert result.status == "exists"
+    assert len(calls) == 1
+
+
+def test_http_capture_explain_does_no_network_or_storage_io(tmp_path):
+    def handler(request):
+        raise AssertionError("explain made a request")
+
+    model, _ = _capture(tmp_path, handler, explain=True)
+
+    result = model(date(2026, 10, 11))
+
+    assert result.status == "planned"
+    assert result.write.status == "planned"
+    assert not (tmp_path / "store").exists()
+    assert not (tmp_path / "workspace").exists()

{ccflow_http-0.2.0 → ccflow_http-0.3.0}/ccflow_http/tests/test_model.py

@@ -490,7 +490,7 @@ def test_http_model_returns_accepted_error_status_without_raising():
         HTTPModel(config=config, path="/v1/tickers/ZZZZ")(HTTPRequestContext())
 
 
-def test_http_model_stops_pagination_on_accepted_error_status():
+def test_http_model_raises_on_accepted_error_status_after_first_page():
     def handler(request: httpx2.Request) -> httpx2.Response:
         if "cursor" in request.url.params:
             return httpx2.Response(404)
@@ -503,11 +503,80 @@ def test_http_model_stops_pagination_on_accepted_error_status():
         accepted_status_codes=[404],
     )
 
+    with pytest.raises(RuntimeError, match="status 404 on page 2"):
+        model(HTTPRequestContext())
+
+
+def test_http_model_redacts_custom_query_auth_name_everywhere(caplog):
+    seen_urls = []
+
+    def handler(request: httpx2.Request) -> httpx2.Response:
+        seen_urls.append(str(request.url))
+        if len(seen_urls) == 1:
+            return httpx2.Response(200, json={"results": [{"ticker": "AAA"}], "next_url": "/v1/tickers?cursor=2"})
+        return httpx2.Response(200, json={"results": [{"ticker": "BBB"}]})
+
+    model = HTTPModel(
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(handler)),
+        path="/v1/tickers",
+        paginate=True,
+        auth=HTTPAuth(strategy="api_key_query", name="key", value="custom-secret"),
+    )
+
+    with caplog.at_level("INFO", logger="httpx2"):
+        result = model(HTTPRequestContext())
+
+    assert seen_urls == ["https://api.example.test/v1/tickers?key=custom-secret", "https://api.example.test/v1/tickers?cursor=2&key=custom-secret"]
+    assert "custom-secret" not in caplog.text
+    assert caplog.text.count("key=***") == 2
+    assert result.url == "https://api.example.test/v1/tickers?cursor=2&key=***"
+    assert safe_request_dump(model.build_request(HTTPRequestContext()))["params"] == {"key": "***"}
+
+
+def test_http_model_does_not_leak_query_secrets_through_transport_errors(monkeypatch):
+    monkeypatch.setattr("ccflow_http.base.sleep", lambda seconds: None)
+    calls = []
+
+    def handler(request: httpx2.Request) -> httpx2.Response:
+        calls.append(request)
+        if len(calls) == 1:
+            raise httpx2.ConnectError(f"connection reset for {request.url}", request=request)
+        raise httpx2.ReadTimeout(f"read timed out for {request.url}", request=request)
+
+    model = HTTPModel(
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(handler)),
+        path="/v1/tickers",
+        auth=HTTPAuth(strategy="api_key_query", name="key", value="custom-secret"),
+        retry_policy=HTTPRetryPolicy(max_attempts=2, wait_initial=0.0, wait_max=0.0, wait_jitter=0.0),
+    )
+
+    with pytest.raises(RuntimeError, match="failed with ReadTimeout") as error:
+        model(HTTPRequestContext())
+
+    assert error.value.__cause__ is None
+    assert error.value.__suppress_context__
+
+
+def test_http_model_redacts_query_secrets_in_retry_event_messages(monkeypatch):
+    monkeypatch.setattr("ccflow_http.base.sleep", lambda seconds: None)
+    calls = []
+
+    def handler(request: httpx2.Request) -> httpx2.Response:
+        calls.append(request)
+        if len(calls) == 1:
+            raise httpx2.ConnectError(f"connection reset for {request.url}", request=request)
+        return httpx2.Response(200, json={"status": "OK"})
+
+    model = HTTPModel(
+        config=HTTPConfig(base_url="https://api.example.test", transport=httpx2.MockTransport(handler)),
+        path="/v1/tickers",
+        auth=HTTPAuth(strategy="api_key_query", name="key", value="custom-secret"),
+        retry_policy=HTTPRetryPolicy(max_attempts=2, wait_initial=0.0, wait_max=0.0, wait_jitter=0.0),
+    )
+
     result = model(HTTPRequestContext())
 
-    assert result.status_code == 404
-    assert result.value is None
-    assert result.pages == 2
+    assert result.retry_events[0]["message"] == "connection reset for https://api.example.test/v1/tickers?key=***"
 
 
 def test_http_model_parses_csv_and_gzip_responses_with_mock_transport():

{ccflow_http-0.2.0 → ccflow_http-0.3.0}/pyproject.toml

@@ -12,7 +12,7 @@ authors = [
 description = "ccflow models for HTTP"
 readme = "README.md"
 license = { text = "Apache-2.0" }
-version = "0.2.0"
+version = "0.3.0"
 requires-python = ">=3.11"
 keywords = []
 
@@ -30,7 +30,7 @@ classifiers = [
 
 dependencies = [
     "ccflow>=0.8.5",
-    "ccflow-etl>=0.1.0",
+    "ccflow-etl>=0.4.0",
     "httpx2",
     "jinja2",
 ]
@@ -43,6 +43,7 @@ develop = [
     "codespell",
     "hatchling",
     "mdformat",
+    "mdformat-frontmatter",
     "mdformat-tables>=1",
     "pytest",
     "pytest-cov",
@@ -60,7 +61,7 @@ Repository = "https://github.com/1kbgz/ccflow-http"
 Homepage = "https://github.com/1kbgz/ccflow-http"
 
 [tool.bumpversion]
-current_version = "0.2.0"
+current_version = "0.3.0"
 commit = true
 tag = true
 commit_args = "-s"
@@ -78,7 +79,7 @@ replace = 'version = "{new_version}"'
 [tool.coverage.run]
 branch = true
 omit = [
-    "ccflow_http/tests/integration/",
+    "ccflow_http/tests/integration/*",
 ]
 
 [tool.coverage.report]

ccflow_http-0.2.0/ccflow_http/__init__.py

@@ -1,3 +0,0 @@
-from .base import *
-
-__version__ = "0.2.0"