hotdata-framework 0.12.0__tar.gz → 0.13.0__tar.gz

{hotdata_framework-0.12.0 → hotdata_framework-0.13.0}/.github/workflows/publish.yml

@@ -31,6 +31,21 @@ jobs:
       # The tag being released, whether it arrived by push or by dispatch.
       TAG: ${{ inputs.tag || github.ref_name }}
     steps:
+      # Before checkout, because checkout resolves the input as an arbitrary ref:
+      # a branch or SHA is fetched first and only rejected later by the version
+      # match below, which is also looser (`^v[0-9]`). Strict here so the dispatch
+      # contract matches release.yml — release.sh only ever produces X.Y.Z.
+      - name: Validate release tag format
+        if: github.event_name == 'workflow_dispatch'
+        env:
+          INPUT_TAG: ${{ inputs.tag }}
+        run: |
+          set -euo pipefail
+          if [[ ! "$INPUT_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "tag must look like vX.Y.Z, got: $INPUT_TAG" >&2
+            exit 1
+          fi
+
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
         with:
           ref: ${{ inputs.tag || github.ref_name }}

{hotdata_framework-0.12.0 → hotdata_framework-0.13.0}/.github/workflows/release.yml

@@ -25,6 +25,24 @@ jobs:
       # The tag being released, whether it arrived by push or by dispatch.
       TAG: ${{ inputs.tag || github.ref_name }}
     steps:
+      # Before checkout, because checkout resolves the input as an arbitrary ref
+      # — and this workflow needs the guard more than publish.yml does. It holds
+      # `contents: write`, and action-gh-release CREATES a tag when tag_name does
+      # not resolve to one, so an unvalidated `tag: main` would check out cleanly
+      # and leave refs/tags/main plus a release named for it. The push path is
+      # constrained by the v[0-9]* filter; the dispatch path was not constrained
+      # at all.
+      - name: Validate release tag format
+        if: github.event_name == 'workflow_dispatch'
+        env:
+          INPUT_TAG: ${{ inputs.tag }}
+        run: |
+          set -euo pipefail
+          if [[ ! "$INPUT_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "tag must look like vX.Y.Z, got: $INPUT_TAG" >&2
+            exit 1
+          fi
+
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
         with:
           ref: ${{ inputs.tag || github.ref_name }}

{hotdata_framework-0.12.0 → hotdata_framework-0.13.0}/CHANGELOG.md

@@ -7,6 +7,60 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.13.0] - 2026-08-27
+
+### Fixed
+
+- fix(load): retry an `append` load instead of running it at most once.
+
+  `append` was excluded from retries on the grounds that it is not idempotent:
+  if the server commits but the response is lost, a retry would duplicate rows.
+  That is not how the server behaves. It keys a receipt on `upload_id`, and a
+  re-POST of the same id replays the committed result instead of applying the
+  load again — so what makes a retry safe is re-sending the same upload, not
+  the mode. This client stages once, in `upload_parquet`, outside the retried
+  operation, so the invariant holds for every mode.
+
+  The exclusion cost real availability. The destination serialises writes per
+  table and refuses rather than queues, so concurrent writers to one table get
+  `409 RESOURCE_LOCKED` — and an append had no budget to wait it out, whatever
+  `max_retries` the caller had configured.
+
+  `HotdataClient.load_managed_table(file=...)` uploads inside the call and so
+  does not hold the invariant. It is unwrapped and unaffected.
+
+- fix(errors): classify a 409 by its `error.code` rather than by the status alone.
+
+  `CONFLICT` is now terminal: it means the request cannot succeed as posted, so
+  the previous behaviour spent the entire retry budget arriving at the same
+  answer. `RESOURCE_LOCKED` stays transient. A 409 with no error envelope — a
+  failed query result, say — is classified as before.
+
+- fix(retry): honour `Retry-After`, and jitter the backoff.
+
+  `Retry-After` is taken as a floor on the ramp, capped like the ramp so a bad
+  header cannot park an attempt for an hour. Jitter of up to +50% is added on
+  top and never subtracted, so a stated `Retry-After` is not undercut. Without
+  it, writers that collided on one table retry in lockstep and collide again.
+
+  This lengthens a 20-attempt budget from 285s to roughly 316-405s.
+
+- docs: scope the "a load is not idempotent" claim in the README and in
+  `test_retry_policy` to the transport layer, which is where it is still true
+  and where those two were always talking about. Left unscoped they read as
+  repo-wide and contradict the call-layer retry above.
+
+### Added
+
+- `HotdataError` carries `status_code`, `code` and `retry_after_seconds`. The
+  message is flattened and truncated for readability, so it could not serve as
+  a discriminator; these can.
+
+## [0.12.1] - 2026-08-18
+
+### Fixed
+
+- fix(load): submit managed loads as a job and poll, instead of holding one request open
 
 ## [0.12.0] - 2026-08-11
 

{hotdata_framework-0.12.0 → hotdata_framework-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: hotdata-framework
-Version: 0.12.0
+Version: 0.13.0
 Summary: Python framework for building Hotdata integrations: workspace runtime, query execution, and managed databases
 Project-URL: Homepage, https://www.hotdata.dev
 Project-URL: Documentation, https://www.hotdata.dev/docs
@@ -38,7 +38,7 @@ Runtime boundary and guarantees are defined in `CONTRACT.md`.
 
 - **Environment-driven client setup** — create clients from `HOTDATA_API_KEY`, optional `HOTDATA_API_URL`, and `HOTDATA_WORKSPACE`.
 - **Workspace resolution** — choose an explicit workspace from env, otherwise discover workspaces and select the active workspace or first available workspace.
-- **HTTP resilience** — retry SQL execution on stale pooled sockets. Transport-level retries are the SDK's own default, which this package leaves in place so a non-idempotent request is never replayed on a response status.
+- **HTTP resilience** — retry SQL execution on stale pooled sockets. Transport-level retries are the SDK's own default, which this package leaves in place so a request is never blindly replayed on a response status. That is a claim about the transport, which cannot know what it would be replaying. `ManagedDatabaseClient` retries at the call layer, which can: a managed load is safe to re-send because it carries the same `upload_id` and the API replays its receipt for that id rather than applying the load twice.
 - **SQL execution helper** — run SQL through `POST /v1/query`, poll async query runs when needed, and return a `QueryResult`.
 - **Result utilities** — convert query results to records, pandas DataFrames, or metadata dictionaries for adapter display layers.
 - **History helpers** — list recent results and query run history with normalized dataclasses.

{hotdata_framework-0.12.0 → hotdata_framework-0.13.0}/README.md

@@ -10,7 +10,7 @@ Runtime boundary and guarantees are defined in `CONTRACT.md`.
 
 - **Environment-driven client setup** — create clients from `HOTDATA_API_KEY`, optional `HOTDATA_API_URL`, and `HOTDATA_WORKSPACE`.
 - **Workspace resolution** — choose an explicit workspace from env, otherwise discover workspaces and select the active workspace or first available workspace.
-- **HTTP resilience** — retry SQL execution on stale pooled sockets. Transport-level retries are the SDK's own default, which this package leaves in place so a non-idempotent request is never replayed on a response status.
+- **HTTP resilience** — retry SQL execution on stale pooled sockets. Transport-level retries are the SDK's own default, which this package leaves in place so a request is never blindly replayed on a response status. That is a claim about the transport, which cannot know what it would be replaying. `ManagedDatabaseClient` retries at the call layer, which can: a managed load is safe to re-send because it carries the same `upload_id` and the API replays its receipt for that id rather than applying the load twice.
 - **SQL execution helper** — run SQL through `POST /v1/query`, poll async query runs when needed, and return a `QueryResult`.
 - **Result utilities** — convert query results to records, pandas DataFrames, or metadata dictionaries for adapter display layers.
 - **History helpers** — list recent results and query run history with normalized dataclasses.

{hotdata_framework-0.12.0 → hotdata_framework-0.13.0}/hotdata_framework/client.py

@@ -25,6 +25,7 @@ from hotdata.models.database_default_table_decl import DatabaseDefaultTableDecl
 from hotdata.models.index_info_response import IndexInfoResponse
 from hotdata.models.job_status_response import JobStatusResponse
 from hotdata.models.load_managed_table_request import LoadManagedTableRequest
+from hotdata.models.load_managed_table_response import LoadManagedTableResponse
 from hotdata.models.query_request import QueryRequest
 from hotdata.models.query_response import QueryResponse
 from hotdata.models.submit_job_response import SubmitJobResponse
@@ -81,6 +82,37 @@ _RESULT_FAILURE = frozenset({"failed", "cancelled"})
 # Jobs have no "cancelled" state; "partially_succeeded" carries an error_message.
 _JOB_TERMINAL = frozenset({"succeeded", "partially_succeeded", "failed"})
 
+# How long a load may finish INLINE before the server hands back a job instead.
+# Small enough that a slow load stops holding a request open, large enough that
+# the overwhelming majority never become jobs at all: dlt's bookkeeping tables
+# (`_dlt_version`, `_dlt_loads`, `_dlt_pipeline_state`) settle in under a second,
+# and paying a submit-then-poll round trip for those would be a regression.
+_LOAD_INLINE_WAIT_MS = 10_000
+
+# A load's own polling budget. Deliberately NOT the 300s used for queries and
+# results: a load is the one operation here whose duration scales with the data,
+# and reusing the query budget is what put a five-minute ceiling on it in the
+# first place. Bounded rather than unbounded so a wedged job still surfaces.
+_LOAD_JOB_TIMEOUT_S = 3600.0
+
+# How long a poll tolerates CONTINUOUS status-check failure before giving up.
+#
+# Time, not a count: a count means whatever the caller's `interval_s` makes it,
+# and callers differ -- five checks is eight seconds at the load interval and
+# something else at the index one. The thing being survived is a gateway blip,
+# and a rolling restart or a load balancer reconverging routinely serves 502s for
+# longer than a few seconds. Too short and the poll aborts, the caller's retry
+# re-submits the load, and the original job is still holding the table -- the
+# door this tolerance exists to close.
+#
+# Still bounded by the poll's own deadline, so this only decides how a stretch of
+# failures ends, never how long the wait can be.
+_JOB_POLL_ERROR_GRACE_S = 120.0
+
+# Failed checks back off rather than hammering at `interval_s`: whatever is
+# serving 502s does not need the extra traffic.
+_JOB_POLL_ERROR_MAX_BACKOFF_S = 15.0
+
 
 @dataclass(frozen=True)
 class ResultSummary:
@@ -418,10 +450,25 @@ class HotdataClient:
         else:
             assert file is not None
             resolved_upload_id = self.upload_parquet(file)
+        # ASKED FOR AS A JOB, not as a held-open request. A load's duration scales
+        # with the data, and a single request that must survive minutes has to
+        # survive every layer between here and the engine -- CDN, gateway, socket
+        # read timeout -- any one of which ends it. When it ends, the server logs
+        # the load `abandoned` and DISCARDS work it had already done, while the
+        # table's write lock is still held against the retry that follows; the
+        # retry then collides with it (409 RESOURCE_LOCKED) and the pair can spin
+        # indefinitely without the load ever completing. Observed in production on
+        # a table whose load runs past five minutes.
+        #
+        # `async_after_ms` keeps the common case unchanged: the server answers 200
+        # with the result if it finishes inside the window, and only falls back to
+        # a job when it does not. So nothing pays for polling that did not need it.
         request = LoadManagedTableRequest(
             mode=mode,
             upload_id=resolved_upload_id,
             key=key,
+            var_async=True,
+            async_after_ms=_LOAD_INLINE_WAIT_MS,
         )
         try:
             loaded = self.connections().load_managed_table(
@@ -432,12 +479,21 @@ class HotdataClient:
             )
         except ApiException as e:
             raise RuntimeError(api_error_message(e)) from e
+        # Only the job branch is type-checked. The index path can also assert its
+        # inline type because it names it positively first; here the inline shape is
+        # read duck-typed, which callers and tests already rely on, so asserting it
+        # would narrow an interface this change has no business narrowing.
+        job_id: str | None = None
+        if isinstance(loaded, SubmitJobResponse):
+            job_id = loaded.id
+            loaded = self._load_response_from_job(job_id)
         return LoadManagedTableResult(
             connection_id=loaded.connection_id,
             schema_name=loaded.schema_name,
             table_name=loaded.table_name,
             row_count=loaded.row_count,
             full_name=f"{db.id}.{loaded.schema_name}.{loaded.table_name}",
+            job_id=job_id,
         )
 
     def add_managed_table(
@@ -880,11 +936,40 @@ class HotdataClient:
         jobs = self._jobs_api()
         deadline = time.monotonic() + timeout_s
         last: JobStatusResponse | None = None
+        # When the current run of failures began, or None while checks succeed.
+        failing_since: float | None = None
+        error_backoff = interval_s
         while time.monotonic() < deadline:
             try:
                 last = jobs.get_job(job_id)
+                failing_since = None
+                error_backoff = interval_s
             except ApiException as e:
-                raise RuntimeError(api_error_message(e)) from e
+                # A failed STATUS CHECK is not a failed job. Aborting here throws
+                # away work that is still running, and the caller's retry then
+                # re-submits it while the original still holds its resources -- so
+                # one blip becomes a collision with the job it just abandoned. A
+                # load polls for up to `_LOAD_JOB_TIMEOUT_S`, so the longer the
+                # wait the more chances to hit it, which is exactly backwards.
+                #
+                # CONSECUTIVE failures are the signal: an isolated 502 is noise, a
+                # run of them means the API is gone and there is nothing to wait
+                # for. The poll's own deadline bounds the total wait regardless.
+                now = time.monotonic()
+                if failing_since is None:
+                    failing_since = now
+                elif now - failing_since >= _JOB_POLL_ERROR_GRACE_S:
+                    # Name the job. This is one of the two paths where the caller
+                    # cannot tell whether the load landed, so the id is the only
+                    # thing that makes the question answerable -- and it is exactly
+                    # what a message like "502: Bad Gateway" leaves out.
+                    raise RuntimeError(
+                        f"Job {job_id} status checks failed for "
+                        f"{_JOB_POLL_ERROR_GRACE_S:.0f}s: {api_error_message(e)}"
+                    ) from e
+                time.sleep(error_backoff)
+                error_backoff = min(error_backoff * 2, _JOB_POLL_ERROR_MAX_BACKOFF_S)
+                continue
             if last.status in _JOB_TERMINAL:
                 return last
             time.sleep(interval_s)
@@ -893,6 +978,41 @@ class HotdataClient:
             f"Job {job_id} did not finish within {timeout_s}s (last status: {last_status})"
         )
 
+    def _load_response_from_job(self, job_id: str) -> LoadManagedTableResponse:
+        """The result of a load the server chose to run as a job.
+
+        Polling replaces waiting on the request, so the outcome is read from
+        durable state rather than from a connection that has to stay alive. That
+        also gives a caller a handle: the job id is returned on
+        `LoadManagedTableResult`, so "did it land?" is answerable after a lost
+        response. That answer is a convenience rather than a precondition for
+        retrying: re-sending the same upload_id replays the server's receipt
+        instead of applying the load a second time, which is what makes a retry
+        safe in every mode. It stops being safe for a caller that re-stages the
+        upload, because a fresh upload id has no receipt to replay.
+
+        `partially_succeeded` is terminal and carries a message, so it is raised
+        rather than returned -- a caller asked for a table's contents to be
+        replaced or appended to, and "some of it" is not an answer it can use.
+        """
+        final = self._poll_job(job_id, timeout_s=_LOAD_JOB_TIMEOUT_S)
+        status = enum_value(final.status)
+        if status != "succeeded":
+            # The id goes in whether or not the server's message mentions it: the
+            # caller is being told the load did not succeed, and "which load" is
+            # the next thing it needs.
+            detail = final.error_message or f"finished {status}"
+            raise RuntimeError(f"load job {job_id} {status}: {detail}")
+        # `result` is a oneOf wrapper today; tolerate the model arriving directly,
+        # the same way the index path does rather than disagreeing with it.
+        payload = getattr(final.result, "actual_instance", final.result)
+        if not isinstance(payload, LoadManagedTableResponse):
+            raise RuntimeError(
+                f"load job {job_id} succeeded without a load result "
+                f"(got {type(payload).__name__})"
+            )
+        return payload
+
     def _wait_result_ready(
         self,
         result_id: str,

{hotdata_framework-0.12.0 → hotdata_framework-0.13.0}/hotdata_framework/databases.py

@@ -83,6 +83,11 @@ class LoadManagedTableResult:
     table_name: str
     row_count: int
     full_name: str
+    # Set when the server ran the load as a background job. Carried for the same
+    # reason CreateIndexResult carries it: it is the only handle a caller has to
+    # ask "did that land?" after a lost response, and without it the question is
+    # unanswerable. `None` when the load finished inline.
+    job_id: str | None = None
 
     def to_dict(self) -> dict[str, Any]:
         return asdict(self)

hotdata_framework-0.13.0/hotdata_framework/errors.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+
+from hotdata.rest import ApiException
+
+# The API explains a 409 with a machine-readable code, and the two it sends
+# mean opposite things to a retry policy. RESOURCE_LOCKED is a refusal taken
+# before any work: the insert that would have created the unit of work lost a
+# unique-constraint race, so nothing was claimed and nothing was written.
+# CONFLICT is the opposite — the request cannot succeed as posted, so retrying
+# spends the whole budget arriving at the same answer.
+_TERMINAL_CONFLICT_CODE = "CONFLICT"
+
+
+class HotdataError(RuntimeError):
+    """An API failure, carrying what a retry policy needs to decide.
+
+    The message cannot be the discriminator: it is flattened and truncated for
+    readability, so keying on it means substring-matching prose. ``status_code``
+    and ``code`` are the machine-readable form of the same answer, and
+    ``retry_after_seconds`` is the server's own estimate of how long the
+    condition it just refused will last.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        status_code: int | None = None,
+        code: str | None = None,
+        retry_after_seconds: float | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.status_code = status_code
+        self.code = code
+        self.retry_after_seconds = retry_after_seconds
+
+
+class HotdataTransientError(HotdataError):
+    pass
+
+
+class HotdataTerminalError(HotdataError):
+    pass
+
+
+def _error_code(body: object) -> str | None:
+    """The ``error.code`` an API error envelope carries, if this body is one.
+
+    Not every 409 comes from an endpoint that speaks the envelope — a failed
+    query result is reported as one and carries a result document instead — so
+    a missing code is ordinary, and callers fall back to the status.
+    """
+    if not isinstance(body, (str, bytes, bytearray)):
+        return None
+    try:
+        parsed: object = json.loads(body)
+    except ValueError:
+        return None
+    if not isinstance(parsed, Mapping):
+        return None
+    error: object = parsed.get("error")
+    if not isinstance(error, Mapping):
+        return None
+    code: object = error.get("code")
+    return code if isinstance(code, str) else None
+
+
+def _retry_after_seconds(headers: object) -> float | None:
+    """``Retry-After`` as a number of seconds, when the response states one.
+
+    Only the delta-seconds form is read. That is what the API sends, and the
+    HTTP-date form would need a comparison against a server clock we do not
+    have to be worth anything.
+    """
+    if not isinstance(headers, Mapping):
+        return None
+    raw: object = headers.get("Retry-After")
+    if raw is None:
+        # The SDK hands us urllib3's case-insensitive mapping and the API sends
+        # the header lower-cased, so the direct hit is what normally answers.
+        # Fall back for any plain dict that reaches us instead — a missed
+        # header is silent, and silence here reads as "the server asked for
+        # nothing".
+        raw = next((v for k, v in headers.items() if str(k).lower() == "retry-after"), None)
+    if raw is None:
+        return None
+    try:
+        seconds = float(str(raw).strip())
+    except ValueError:
+        return None
+    return seconds if seconds >= 0 else None
+
+
+def _error_class(status_code: int, code: str | None) -> type[HotdataError]:
+    if status_code == 409 and code == _TERMINAL_CONFLICT_CODE:
+        # The request cannot succeed as posted — an upload already consumed
+        # with nothing to replay, a receipt naming a different target, an
+        # incompatible column type. Every retry reaches the same 409.
+        return HotdataTerminalError
+    if status_code in (408, 409, 425, 429):
+        return HotdataTransientError
+    if status_code == 501:
+        # Not Implemented is a permanent capability gap (e.g. the storage
+        # backend cannot issue presigned URLs) — retrying cannot succeed.
+        return HotdataTerminalError
+    if 500 <= status_code <= 599:
+        return HotdataTransientError
+    return HotdataTerminalError
+
+
+def classify_sdk_error(error: Exception) -> HotdataError:
+    if isinstance(error, TimeoutError):
+        return HotdataTransientError(str(error))
+    if isinstance(error, ConnectionError):
+        return HotdataTransientError(str(error))
+    if isinstance(error, ApiException):
+        status_code = int(error.status or 0)
+        message = f"{status_code}: {error.reason or 'unknown error'}"
+        # The response body is where the API explains itself (e.g. which
+        # header is missing) — without it "400: Bad Request" is undebuggable.
+        body: object = getattr(error, "body", None)
+        if body:
+            message = f"{message} — {' '.join(str(body).split())[:500]}"
+        code = _error_code(body)
+        return _error_class(status_code, code)(
+            message,
+            status_code=status_code,
+            code=code,
+            retry_after_seconds=_retry_after_seconds(getattr(error, "headers", None)),
+        )
+    return HotdataTerminalError(str(error))

{hotdata_framework-0.12.0 → hotdata_framework-0.13.0}/hotdata_framework/managed_client.py

@@ -7,6 +7,7 @@ in one place rather than being duplicated per adapter.
 
 from __future__ import annotations
 
+import random
 import time
 from collections.abc import Callable
 from typing import Any, Protocol, TypeVar
@@ -53,6 +54,10 @@ class ManagedDatabaseClient:
     _QUERY_TIMEOUT_SECONDS = 300.0
     _POLL_INTERVAL_SECONDS = 0.4
     _MAX_BACKOFF_SECONDS = 30.0
+    # Spread as a fraction of the wait, added on top of it. Half an interval is
+    # enough to decorrelate writers that started together without materially
+    # changing how long the budget lasts.
+    _RETRY_JITTER_FRACTION = 0.5
 
     def __init__(
         self,
@@ -207,9 +212,16 @@ class ManagedDatabaseClient:
         mode: ManagedLoadMode = "replace",
         key: list[str] | None = None,
     ) -> LoadManagedTableResult:
-        # append is the only non-idempotent mode: if the server commits the load
-        # but the response is lost, a retry re-appends the same rows. Run it
-        # at-most-once; every other mode is safe to retry.
+        # Retryable in every mode, append included. A retry re-sends the SAME
+        # upload_id, and the server keys a receipt on it: a replay returns the
+        # committed result rather than applying the load a second time. So the
+        # invariant that makes this safe is the upload id, not the mode — a
+        # caller that re-stages the upload between attempts mints a new id,
+        # loses the receipt, and a retried append would then duplicate rows.
+        # This client stages once, in upload_parquet, outside the operation
+        # retried here. `HotdataClient.load_managed_table(file=...)` uploads
+        # inside the call and so does not hold the invariant; it is unwrapped,
+        # and retrying an append through it is the caller's to justify.
         #
         # `key` is the merge key for delete/update/upsert loads: when set it is
         # matched per-load instead of a key declared at table creation. Omit it
@@ -222,20 +234,40 @@ class ManagedDatabaseClient:
                 upload_id=upload_id,
                 mode=mode,
                 key=key,
-            ),
-            retryable=(mode != "append"),
+            )
         )
 
-    def _request_with_retry(self, operation: Callable[[], T], *, retryable: bool = True) -> T:
-        max_attempts = self._max_retries if retryable else 1
+    def _request_with_retry(self, operation: Callable[[], T]) -> T:
+        max_attempts = self._max_retries
         for attempt in range(1, max_attempts + 1):
             try:
                 return operation()
             except Exception as error:
                 mapped_error = classify_sdk_error(error.__cause__ or error)
                 if isinstance(mapped_error, HotdataTransientError) and attempt < max_attempts:
-                    backoff = min(self._retry_backoff_seconds * attempt, self._MAX_BACKOFF_SECONDS)
-                    time.sleep(backoff)
+                    time.sleep(self._retry_delay(attempt, mapped_error.retry_after_seconds))
                     continue
                 raise mapped_error from error
         raise RuntimeError("No retry attempts configured")
+
+    def _retry_delay(self, attempt: int, retry_after_seconds: float | None) -> float:
+        """A linear ramp, floored by the server's Retry-After and spread by jitter.
+
+        Retry-After is a floor rather than a replacement: it says how long the
+        condition just refused typically lasts, while the ramp is what gives up
+        eventually, and taking the larger of the two honours both. It is capped
+        like the ramp so a hostile or mistaken header cannot park an attempt for
+        an hour.
+
+        Jitter is added on top and never subtracted, so a stated Retry-After is
+        not undercut. It matters because the callers that collide are the ones
+        that started together: writers refused by one table's lock would retry
+        in lockstep on an identical ramp and re-collide every time.
+        _MAX_BACKOFF_SECONDS caps the ramp, deliberately not the jitter above
+        it — clamping the total would flatten every late attempt onto the same
+        value and re-correlate exactly the waits that most need spreading.
+        """
+        base = min(self._retry_backoff_seconds * attempt, self._MAX_BACKOFF_SECONDS)
+        if retry_after_seconds is not None:
+            base = max(base, min(retry_after_seconds, self._MAX_BACKOFF_SECONDS))
+        return base * (1.0 + random.random() * self._RETRY_JITTER_FRACTION)

{hotdata_framework-0.12.0 → hotdata_framework-0.13.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hotdata-framework"
-version = "0.12.0"
+version = "0.13.0"
 description = "Python framework for building Hotdata integrations: workspace runtime, query execution, and managed databases"
 readme = "README.md"
 requires-python = ">=3.10"