hotdata-framework 0.6.3__tar.gz → 0.7.1__tar.gz

{hotdata_framework-0.6.3 → hotdata_framework-0.7.1}/CHANGELOG.md

@@ -8,6 +8,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 
+## [0.7.1] - 2026-07-15
+
+### Changed
+
+- `upload_parquet()` now uses the presigned upload session API (`POST /v1/uploads`) instead of reading the entire file into memory before uploading. For multipart mode the file is streamed one `part_size` chunk at a time, eliminating the memory spike that caused OOM on large Parquet files. Falls back to `POST /v1/files` when the server returns 501.
+
+## [0.7.0] - 2026-07-14
+
+### Added
+
+- `load_managed_table(..., mode=...)` selects the load mode (`replace` (default), `append`, `delete`, `update`, `upsert`) instead of always replacing the table. `replace`/`append` apply the upload directly; `delete`/`update`/`upsert` match rows by the table's declared key. Backward compatible — omitting `mode` still replaces.
+- `create_managed_database(..., keys={table: [cols]})` and `add_managed_table(..., key=[cols])` declare a table's row-identity key, enabling the key-based load modes on it. Requires a `hotdata` client whose managed-table decl models carry `key` (see the dependency floor bump); tables declared without a key stay `replace`/`append`-only.
+
+### Fixed
+
+- `load_managed_table(..., mode="append")` is no longer retried on transient errors. Every other mode is idempotent, but retrying an `append` whose commit succeeded before the response was received would duplicate the uploaded rows; `append` now runs at most once. `mode` is also now typed as a literal of the accepted values.
+
 ## [0.6.3] - 2026-07-08
 
 ### Added

{hotdata_framework-0.6.3 → hotdata_framework-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hotdata-framework
-Version: 0.6.3
+Version: 0.7.1
 Summary: Python framework for building Hotdata integrations: workspace/session runtime, query execution, and managed databases
 Project-URL: Homepage, https://www.hotdata.dev
 Project-URL: Documentation, https://www.hotdata.dev/docs
@@ -21,7 +21,7 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: hotdata>=0.6.0
+Requires-Dist: hotdata>=0.7.0
 Requires-Dist: pandas>=2.0
 Requires-Dist: pyarrow>=14.0
 Description-Content-Type: text/markdown

{hotdata_framework-0.6.3 → hotdata_framework-0.7.1}/hotdata_framework/client.py

@@ -1,10 +1,12 @@
 from __future__ import annotations
 
 import functools
+import os
 import time
+import urllib3
 from collections.abc import Iterator
 from dataclasses import asdict, dataclass
-from typing import Any
+from typing import Any, Literal
 
 from hotdata import ApiClient, Configuration
 from hotdata.api.connections_api import ConnectionsApi
@@ -20,6 +22,9 @@ from hotdata.models.async_query_response import AsyncQueryResponse
 from hotdata.models.create_database_request import CreateDatabaseRequest
 from hotdata.models.database_default_schema_decl import DatabaseDefaultSchemaDecl
 from hotdata.models.database_default_table_decl import DatabaseDefaultTableDecl
+from hotdata.models.create_upload_request import CreateUploadRequest
+from hotdata.models.finalize_upload_part import FinalizeUploadPart
+from hotdata.models.finalize_upload_request import FinalizeUploadRequest
 from hotdata.models.load_managed_table_request import LoadManagedTableRequest
 from hotdata.models.query_request import QueryRequest
 from hotdata.models.query_response import QueryResponse
@@ -46,6 +51,10 @@ from hotdata_framework.env import (
 from hotdata_framework.http import default_http_retries
 from hotdata_framework.result import QueryResult
 
+# Load modes the managed-table endpoint accepts: replace overwrites, append adds
+# rows, delete/update/upsert match by the table's declared key.
+ManagedLoadMode = Literal["replace", "append", "delete", "update", "upsert"]
+
 _TERMINAL = frozenset({"succeeded", "failed", "cancelled"})
 _RESULT_FAILURE = frozenset({"failed", "cancelled"})
 
@@ -241,14 +250,21 @@ class HotdataClient:
         *,
         schema: str = DEFAULT_SCHEMA,
         tables: list[str] | None = None,
+        keys: dict[str, list[str]] | None = None,
         expires_at: str | None = None,
     ) -> ManagedDatabase:
+        """Create a managed database. ``keys`` maps a table to its key columns
+        (enabling delete/update/upsert on it); omitted tables are keyless."""
+        keys = keys or {}
         schemas = None
         if tables:
             schemas = [
                 DatabaseDefaultSchemaDecl(
                     name=schema,
-                    tables=[DatabaseDefaultTableDecl(name=t) for t in tables],
+                    tables=[
+                        DatabaseDefaultTableDecl(name=t, key=list(keys.get(t, [])))
+                        for t in tables
+                    ],
                 )
             ]
         request = CreateDatabaseRequest(
@@ -295,6 +311,71 @@ class HotdataClient:
     def upload_parquet(self, path: str) -> str:
         if not is_parquet_path(path):
             raise ValueError(f"Managed table loads require a parquet file (got {path!r})")
+        file_size = os.path.getsize(path)
+        try:
+            session = self.uploads().create_upload_session_handler(
+                CreateUploadRequest(
+                    declared_size_bytes=file_size,
+                    content_type="application/octet-stream",
+                )
+            )
+        except ApiException as e:
+            if e.status == 501:
+                return self._upload_parquet_post(path)
+            raise RuntimeError(api_error_message(e)) from e
+        http = urllib3.PoolManager()
+        parts: list[FinalizeUploadPart] | None = None
+        try:
+            if session.mode == "single":
+                with open(path, "rb") as f:
+                    data = f.read()
+                resp = http.request(
+                    "PUT",
+                    session.url,
+                    body=data,
+                    headers={"Content-Length": str(file_size), **session.headers},
+                )
+                if resp.status not in (200, 201, 204):
+                    raise RuntimeError(f"Storage PUT failed: HTTP {resp.status}")
+            else:
+                collected: list[FinalizeUploadPart] = []
+                with open(path, "rb") as f:
+                    for i, part_url in enumerate(session.part_urls):
+                        chunk = f.read(session.part_size)
+                        resp = http.request(
+                            "PUT",
+                            part_url,
+                            body=chunk,
+                            headers={
+                                "Content-Length": str(len(chunk)),
+                                **session.headers,
+                            },
+                        )
+                        if resp.status not in (200, 201, 204):
+                            raise RuntimeError(
+                                f"Part {i + 1} PUT failed: HTTP {resp.status}"
+                            )
+                        collected.append(
+                            FinalizeUploadPart(
+                                part_number=i + 1,
+                                e_tag=resp.headers["ETag"],
+                            )
+                        )
+                parts = collected
+        finally:
+            http.clear()
+        try:
+            finalized = self.uploads().finalize_upload_handler(
+                upload_id=session.upload_id,
+                x_upload_finalize_token=session.finalize_token,
+                finalize_upload_request=FinalizeUploadRequest(parts=parts),
+            )
+        except ApiException as e:
+            raise RuntimeError(api_error_message(e)) from e
+        return finalized.upload_id
+
+    def _upload_parquet_post(self, path: str) -> str:
+        """Fallback for storage backends that do not support presigned URLs (501)."""
         with open(path, "rb") as f:
             data = f.read()
         try:
@@ -314,6 +395,7 @@ class HotdataClient:
         schema: str = DEFAULT_SCHEMA,
         upload_id: str | None = None,
         file: str | None = None,
+        mode: ManagedLoadMode = "replace",
     ) -> LoadManagedTableResult:
         if (upload_id is None) == (file is None):
             raise ValueError("Exactly one of upload_id or file is required")
@@ -324,7 +406,7 @@ class HotdataClient:
             assert file is not None
             resolved_upload_id = self.upload_parquet(file)
         request = LoadManagedTableRequest(
-            mode="replace",
+            mode=mode,
             upload_id=resolved_upload_id,
         )
         try:
@@ -350,15 +432,17 @@ class HotdataClient:
         table: str,
         *,
         schema: str = DEFAULT_SCHEMA,
+        key: list[str] | None = None,
     ) -> ManagedTable:
         """Declare a new table on an existing managed database.
 
         The table is added empty (declared-but-unloaded); populate it with
         :meth:`load_managed_table`. Use this to evolve a managed database's
-        schema after creation without recreating it.
+        schema after creation without recreating it. ``key`` sets the
+        row-identity columns for delete/update/upsert; omit for keyless.
         """
         db = self.resolve_managed_database(database)
-        request = AddManagedTableRequest(name=table)
+        request = AddManagedTableRequest(name=table, key=list(key or []))
         try:
             self._databases_api().add_database_table(db.id, schema, request)
         except ApiException as e:

{hotdata_framework-0.6.3 → hotdata_framework-0.7.1}/hotdata_framework/managed_client.py

@@ -21,6 +21,7 @@ from hotdata.models.query_request import QueryRequest
 from hotdata.models.query_response import QueryResponse
 
 from hotdata_framework.client import HotdataClient as RuntimeClient
+from hotdata_framework.client import ManagedLoadMode
 from hotdata_framework.databases import LoadManagedTableResult, ManagedDatabase
 from hotdata_framework.errors import (
     HotdataTransientError,
@@ -203,23 +204,30 @@ class ManagedDatabaseClient:
         *,
         schema: str,
         upload_id: str,
+        mode: ManagedLoadMode = "replace",
     ) -> LoadManagedTableResult:
+        # append is the only non-idempotent mode: if the server commits the load
+        # but the response is lost, a retry re-appends the same rows. Run it
+        # at-most-once; every other mode is safe to retry.
         return self._request_with_retry(
             lambda: self._runtime.load_managed_table(
                 database,
                 table,
                 schema=schema,
                 upload_id=upload_id,
-            )
+                mode=mode,
+            ),
+            retryable=(mode != "append"),
         )
 
-    def _request_with_retry(self, operation: Callable[[], T]) -> T:
-        for attempt in range(1, self._max_retries + 1):
+    def _request_with_retry(self, operation: Callable[[], T], *, retryable: bool = True) -> T:
+        max_attempts = self._max_retries if retryable else 1
+        for attempt in range(1, max_attempts + 1):
             try:
                 return operation()
             except Exception as error:
                 mapped_error = classify_sdk_error(error.__cause__ or error)
-                if isinstance(mapped_error, HotdataTransientError) and attempt < self._max_retries:
+                if isinstance(mapped_error, HotdataTransientError) and attempt < max_attempts:
                     backoff = min(self._retry_backoff_seconds * attempt, self._MAX_BACKOFF_SECONDS)
                     time.sleep(backoff)
                     continue

{hotdata_framework-0.6.3 → hotdata_framework-0.7.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hotdata-framework"
-version = "0.6.3"
+version = "0.7.1"
 description = "Python framework for building Hotdata integrations: workspace/session runtime, query execution, and managed databases"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -26,7 +26,8 @@ classifiers = [
   "Typing :: Typed",
 ]
 dependencies = [
-  "hotdata>=0.6.0",
+  # 0.7.0 adds `key` to table decls (create_managed_database(keys=) / add_managed_table(key=))
+  "hotdata>=0.7.0",
   "pandas>=2.0",
   "pyarrow>=14.0",
 ]

{hotdata_framework-0.6.3 → hotdata_framework-0.7.1}/tests/test_databases.py

@@ -1,10 +1,12 @@
 from __future__ import annotations
 
+import io
 from types import SimpleNamespace
-from unittest.mock import mock_open, patch
+from unittest.mock import MagicMock, mock_open, patch
 
 import pytest
 from hotdata.exceptions import ApiException
+from hotdata.models.database_default_table_decl import DatabaseDefaultTableDecl
 
 from hotdata_framework.client import HotdataClient
 from hotdata_framework.databases import (
@@ -13,6 +15,20 @@ from hotdata_framework.databases import (
 )
 
 
+def _decl_key_supported() -> bool:
+    # `key` ships with the regenerated client; the key tests activate once it does.
+    try:
+        return DatabaseDefaultTableDecl(name="t", key=["k"]).key == ["k"]
+    except Exception:
+        return False
+
+
+requires_key_field = pytest.mark.skipif(
+    not _decl_key_supported(),
+    reason="hotdata client without `key` on managed-table decls",
+)
+
+
 def _client() -> HotdataClient:
     return HotdataClient("k", "ws", host="https://api.hotdata.dev")
 
@@ -179,16 +195,105 @@ def test_upload_parquet_rejects_non_parquet():
         client.upload_parquet("/tmp/data.csv")
 
 
-def test_upload_parquet_returns_upload_id():
+def _mock_open_bytes(data: bytes) -> MagicMock:
+    """Return an open() mock whose file handle supports read(n) via BytesIO."""
+    bio = io.BytesIO(data)
+    m = MagicMock()
+    m.__enter__ = lambda s: bio
+    m.__exit__ = MagicMock(return_value=False)
+    return MagicMock(return_value=m)
+
+
+def _session(mode: str, **kw) -> SimpleNamespace:
+    defaults = dict(
+        upload_id="upl_sess",
+        finalize_token="tok",
+        headers={},
+        part_size=None,
+        part_urls=None,
+        url=None,
+    )
+    return SimpleNamespace(mode=mode, **{**defaults, **kw})
+
+
+def _http_resp(status: int = 200, etag: str = '"abc"') -> SimpleNamespace:
+    return SimpleNamespace(status=status, headers={"ETag": etag})
+
+
+def test_upload_parquet_multipart():
+    client = _client()
+    data = b"PAR1" + b"\x00" * 6  # 10 bytes -> 2 parts of 5
+    session = _session("multipart", part_size=5, part_urls=["https://s/1", "https://s/2"])
+    finalized = SimpleNamespace(upload_id="upl_final")
+
+    with (
+        patch("builtins.open", _mock_open_bytes(data)),
+        patch("os.path.getsize", return_value=len(data)),
+        patch.object(client, "uploads") as uploads,
+        patch("hotdata_framework.client.urllib3.PoolManager") as MockPool,
+    ):
+        pool = MockPool.return_value
+        pool.request.return_value = _http_resp()
+        pool.clear.return_value = None
+        uploads.return_value.create_upload_session_handler.return_value = session
+        uploads.return_value.finalize_upload_handler.return_value = finalized
+
+        upload_id = client.upload_parquet("/tmp/data.parquet")
+
+    assert upload_id == "upl_final"
+    assert pool.request.call_count == 2
+    finalize_call = uploads.return_value.finalize_upload_handler.call_args
+    assert finalize_call.kwargs["upload_id"] == "upl_sess"
+    assert finalize_call.kwargs["x_upload_finalize_token"] == "tok"
+    parts = finalize_call.kwargs["finalize_upload_request"].parts
+    assert len(parts) == 2
+    assert parts[0].part_number == 1
+    assert parts[1].part_number == 2
+
+
+def test_upload_parquet_single_put():
+    client = _client()
+    data = b"PAR1tiny"
+    session = _session("single", url="https://s/put")
+    finalized = SimpleNamespace(upload_id="upl_single")
+
+    with (
+        patch("builtins.open", mock_open(read_data=data)),
+        patch("os.path.getsize", return_value=len(data)),
+        patch.object(client, "uploads") as uploads,
+        patch("hotdata_framework.client.urllib3.PoolManager") as MockPool,
+    ):
+        pool = MockPool.return_value
+        pool.request.return_value = _http_resp()
+        pool.clear.return_value = None
+        uploads.return_value.create_upload_session_handler.return_value = session
+        uploads.return_value.finalize_upload_handler.return_value = finalized
+
+        upload_id = client.upload_parquet("/tmp/data.parquet")
+
+    assert upload_id == "upl_single"
+    pool.request.assert_called_once()
+    call_args = pool.request.call_args
+    assert call_args.args[0] == "PUT"
+    assert call_args.args[1] == "https://s/put"
+
+
+def test_upload_parquet_fallback_on_501():
     client = _client()
-    uploaded = SimpleNamespace(id="upl_123")
+    uploaded = SimpleNamespace(id="upl_post")
+
     with (
         patch("builtins.open", mock_open(read_data=b"PAR1")),
+        patch("os.path.getsize", return_value=4),
         patch.object(client, "uploads") as uploads,
     ):
+        err = ApiException(status=501)
+        uploads.return_value.create_upload_session_handler.side_effect = err
         uploads.return_value.upload_file.return_value = uploaded
+
         upload_id = client.upload_parquet("/tmp/data.parquet")
-    assert upload_id == "upl_123"
+
+    assert upload_id == "upl_post"
 
 
 def test_load_managed_table_with_upload_id():
@@ -230,6 +335,69 @@ def test_load_managed_table_requires_exactly_one_source():
         )
 
 
+def _load_and_capture_request(client, **kwargs):
+    db = managed_database_from_detail(_detail())
+    loaded = SimpleNamespace(
+        connection_id="conn_1", schema_name="public", table_name="orders", row_count=1
+    )
+    with (
+        patch.object(client, "resolve_managed_database", return_value=db),
+        patch.object(client, "connections") as connections,
+    ):
+        connections.return_value.load_managed_table.return_value = loaded
+        client.load_managed_table("db_1", "orders", upload_id="upl_1", **kwargs)
+    return connections.return_value.load_managed_table.call_args.args[3]
+
+
+def test_load_managed_table_defaults_to_replace():
+    assert _load_and_capture_request(_client()).mode == "replace"
+
+
+@pytest.mark.parametrize("mode", ["append", "delete", "update", "upsert"])
+def test_load_managed_table_forwards_mode(mode: str):
+    assert _load_and_capture_request(_client(), mode=mode).mode == mode
+
+
+@requires_key_field
+def test_create_managed_database_declares_keys():
+    client = _client()
+    with patch.object(client, "_databases_api") as dbs:
+        dbs.return_value.create_database.return_value = _detail(id="db_new")
+        client.create_managed_database(
+            "mydb", tables=["orders", "events"], keys={"orders": ["id"]}
+        )
+    req = dbs.return_value.create_database.call_args.args[0]
+    declared = {t.name: list(t.key) for t in req.schemas[0].tables}
+    assert declared == {"orders": ["id"], "events": []}
+
+
+@requires_key_field
+def test_add_managed_table_declares_key():
+    client = _client()
+    db = managed_database_from_detail(_detail())
+    with (
+        patch.object(client, "resolve_managed_database", return_value=db),
+        patch.object(client, "_databases_api") as dbs,
+    ):
+        client.add_managed_table("db_1", "line_items", key=["order_id", "sku"])
+    req = dbs.return_value.add_database_table.call_args.args[2]
+    assert req.name == "line_items"
+    assert list(req.key) == ["order_id", "sku"]
+
+
+@requires_key_field
+def test_add_managed_table_keyless_by_default():
+    client = _client()
+    db = managed_database_from_detail(_detail())
+    with (
+        patch.object(client, "resolve_managed_database", return_value=db),
+        patch.object(client, "_databases_api") as dbs,
+    ):
+        client.add_managed_table("db_1", "orders")
+    req = dbs.return_value.add_database_table.call_args.args[2]
+    assert list(req.key) == []
+
+
 def test_delete_managed_table_uses_default_connection_id():
     client = _client()
     db = managed_database_from_detail(_detail())

{hotdata_framework-0.6.3 → hotdata_framework-0.7.1}/tests/test_managed_client.py

@@ -156,3 +156,56 @@ def test_fetch_table_carries_database_scope_on_result_reads(
     assert table is not None
     assert result_scopes == ["db1"]
     assert arrow_scopes == ["db1"]
+
+
+def _load_recording_runtime(calls: list[str]) -> SimpleNamespace:
+    """A runtime whose ``load_managed_table`` records each mode and always fails
+    with a transient error, so retry behaviour is observable via ``calls``."""
+
+    def load_managed_table(
+        database: str, table: str, *, schema: str, upload_id: str, mode: str
+    ) -> SimpleNamespace:
+        calls.append(mode)
+        raise TimeoutError("commit succeeded but response was lost")
+
+    runtime = _fake_runtime()
+    runtime.load_managed_table = load_managed_table
+    return runtime
+
+
+def _managed_client(max_retries: int) -> Any:
+    return mc.ManagedDatabaseClient(
+        api_key="k",
+        workspace_id="w",
+        api_base_url="https://example.test",
+        max_retries=max_retries,
+        retry_backoff_seconds=0.0,
+    )
+
+
+def test_append_load_runs_at_most_once(monkeypatch: pytest.MonkeyPatch) -> None:
+    """``append`` is not idempotent: retrying after a commit whose response was
+    lost would duplicate rows. A transient failure must surface immediately
+    without re-appending, even with retries budgeted."""
+    monkeypatch.setattr(mc.time, "sleep", lambda _seconds: None)
+    calls: list[str] = []
+    client = _managed_client(max_retries=8)
+    client._runtime = _load_recording_runtime(calls)
+
+    with pytest.raises(mc.HotdataTransientError):
+        client.load_managed_table("db", "orders", schema="public", upload_id="u1", mode="append")
+
+    assert calls == ["append"]  # tried once, never retried
+
+
+def test_idempotent_load_retries_on_transient(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Idempotent modes still exhaust the retry budget on transient errors."""
+    monkeypatch.setattr(mc.time, "sleep", lambda _seconds: None)
+    calls: list[str] = []
+    client = _managed_client(max_retries=3)
+    client._runtime = _load_recording_runtime(calls)
+
+    with pytest.raises(mc.HotdataTransientError):
+        client.load_managed_table("db", "orders", schema="public", upload_id="u1", mode="replace")
+
+    assert calls == ["replace", "replace", "replace"]  # retried up to max_retries

{hotdata_framework-0.6.3 → hotdata_framework-0.7.1}/uv.lock

@@ -86,7 +86,7 @@ wheels = [
 
 [[package]]
 name = "hotdata"
-version = "0.6.0"
+version = "0.7.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "pydantic" },
@@ -94,14 +94,14 @@ dependencies = [
     { name = "typing-extensions" },
     { name = "urllib3" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/d2/17/6d9e7cbe7e766cd52968653096ce4199d0cb1aa55b73fcfc2979fb9805ff/hotdata-0.6.0.tar.gz", hash = "sha256:c655fd9691a1d65dd4ed51b86f1c27bf8e73268b4c44696987a57a624da35143", size = 214162, upload-time = "2026-07-07T16:44:32.65Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/61/30/08681132e019f6c9ddb566dc38be6a302763c4b4848b2d97e3d7ace8d257/hotdata-0.7.0.tar.gz", hash = "sha256:ef4484a74c0f33ee543b0a0dbfb5a6c04e7b5812b277f42a6448000d41c2405d", size = 215130, upload-time = "2026-07-14T12:21:33.936Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/dc/0a/4c9f23e1fa85a8d0336681f608c13100e4ab2cf4204846d90374b1bb141b/hotdata-0.6.0-py3-none-any.whl", hash = "sha256:3324758eb0aa37bbcc505ddac524566a8c10fbe0787e226932e8b04d565200c1", size = 312820, upload-time = "2026-07-07T16:44:31.121Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/ef/6c4236640629688074892759b1a7593c08f30e9d916b6951c2bbfec19d7f/hotdata-0.7.0-py3-none-any.whl", hash = "sha256:d9e3008e3084d22bcc27b7bf1d08d8c2c255fe3726c252c777bb614a3cae0576", size = 314103, upload-time = "2026-07-14T12:21:32.154Z" },
 ]
 
 [[package]]
 name = "hotdata-framework"
-version = "0.6.3"
+version = "0.7.1"
 source = { editable = "." }
 dependencies = [
     { name = "hotdata" },
@@ -120,7 +120,7 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "hotdata", specifier = ">=0.6.0" },
+    { name = "hotdata", specifier = ">=0.7.0" },
     { name = "pandas", specifier = ">=2.0" },
     { name = "pyarrow", specifier = ">=14.0" },
 ]