hotdata-dlt-destination 0.3.0__py3-none-any.whl

hotdata_dlt_destination/__init__.py

@@ -0,0 +1,3 @@
+from hotdata_dlt_destination.destination import hotdata_destination
+
+__all__ = ["hotdata_destination"]

hotdata_dlt_destination/cli.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from hotdata_dlt_destination.config import HotdataDestinationConfig
+
+
+def main() -> None:
+    config = HotdataDestinationConfig.from_env()
+    print("hotdata-dlt-destination is configured")
+    print(f"api_base_url={config.api_base_url}")
+    print(f"workspace_id={config.workspace_id}")
+    print(f"database_name={config.database_name}")
+    print(f"schema={config.schema}")
+    print(f"write_disposition={config.write_disposition}")

hotdata_dlt_destination/config.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class HotdataDestinationConfig:
+    api_key: str
+    workspace_id: str
+    database_name: str
+    api_base_url: str = "https://api.hotdata.dev"
+    schema: str = "public"
+    write_disposition: str = "append"
+    create_database_if_missing: bool = True
+    declared_tables: tuple[str, ...] = ()
+    max_retries: int = 5
+    retry_backoff_seconds: float = 1.0
+
+    @classmethod
+    def from_env(cls) -> HotdataDestinationConfig:
+        declared = os.environ.get("HOTDATA_DECLARED_TABLES", "")
+        declared_tables = tuple(
+            table.strip() for table in declared.split(",") if table.strip()
+        )
+        return cls(
+            api_key=os.environ["HOTDATA_API_KEY"],
+            workspace_id=os.environ["HOTDATA_WORKSPACE"],
+            database_name=os.environ.get("HOTDATA_DATABASE", "dlt"),
+            api_base_url=os.environ.get("HOTDATA_API_BASE_URL", "https://api.hotdata.dev"),
+            schema=os.environ.get("HOTDATA_SCHEMA", "public"),
+            write_disposition=os.environ.get("HOTDATA_WRITE_DISPOSITION", "append"),
+            create_database_if_missing=os.environ.get(
+                "HOTDATA_CREATE_DATABASE_IF_MISSING", "true"
+            ).lower()
+            in {"1", "true", "yes"},
+            declared_tables=declared_tables,
+            max_retries=int(os.environ.get("HOTDATA_MAX_RETRIES", "5")),
+            retry_backoff_seconds=float(os.environ.get("HOTDATA_RETRY_BACKOFF_SECONDS", "1.0")),
+        )

hotdata_dlt_destination/contracts.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from dlt.common.schema import TTableSchema
+
+IDENTIFIER_RE = re.compile(r"[^a-zA-Z0-9_]")
+
+
+def normalize_identifier(value: str) -> str:
+    normalized = IDENTIFIER_RE.sub("_", value).strip("_").lower()
+    return normalized or "unnamed"
+
+
+@dataclass(frozen=True)
+class TableContract:
+    database_name: str
+    schema: str
+    table_name: str
+
+    @property
+    def qualified_target(self) -> str:
+        return f"{self.database_name}.{self.schema}.{self.table_name}"
+
+    @classmethod
+    def from_table_schema(
+        cls,
+        table: TTableSchema,
+        *,
+        database_name: str,
+        schema: str,
+    ) -> TableContract:
+        table_name = normalize_identifier(table["name"])
+        parent_name = table.get("parent")
+        parent_table_name = normalize_identifier(parent_name) if parent_name else ""
+
+        if parent_table_name:
+            normalized_table_name = f"{parent_table_name}__{table_name}"
+        else:
+            normalized_table_name = table_name
+
+        return cls(
+            database_name=normalize_identifier(database_name),
+            schema=normalize_identifier(schema),
+            table_name=normalized_table_name,
+        )
+
+    @classmethod
+    def declared_table_names(
+        cls,
+        *,
+        database_name: str,
+        schema: str,
+        table_names: list[str],
+    ) -> list[str]:
+        return sorted(
+            {
+                cls.from_table_schema(
+                    {"name": table_name},
+                    database_name=database_name,
+                    schema=schema,
+                ).table_name
+                for table_name in table_names
+            }
+        )

hotdata_dlt_destination/destination.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+import os
+import tempfile
+from datetime import UTC, datetime
+from typing import Any
+
+import dlt
+from dlt.common.destination.exceptions import DestinationTerminalException
+from dlt.common.schema import TTableSchema
+from dlt.common.typing import TDataItems
+
+from hotdata_dlt_destination.config import HotdataDestinationConfig
+from hotdata_dlt_destination.contracts import TableContract
+from hotdata_dlt_destination.errors import HotdataTerminalError
+from hotdata_dlt_destination.hotdata_client import HotdataClient
+from hotdata_dlt_destination.idempotency import compute_batch_key, compute_row_key
+from hotdata_dlt_destination.merge import (
+    combine_rows,
+    resolve_primary_key,
+    resolve_write_disposition,
+)
+from hotdata_dlt_destination.parquet import read_parquet_rows, write_rows_parquet
+
+
+def _load_batch_rows(items: TDataItems | str) -> list[dict[str, Any]]:
+    if isinstance(items, str):
+        return read_parquet_rows(items)
+    return [dict(item) for item in items]
+
+
+def _augment_rows(
+    *,
+    table_name: str,
+    items: TDataItems | str,
+) -> tuple[str, list[dict[str, Any]]]:
+    rows = _load_batch_rows(items)
+    batch_key = compute_batch_key(table_name, rows)
+    loaded_at = datetime.now(UTC).isoformat()
+    augmented_rows = [
+        {
+            **row,
+            "_hotdata_batch_key": batch_key,
+            "_hotdata_row_key": compute_row_key(table_name, row),
+            "_hotdata_loaded_at": loaded_at,
+        }
+        for row in rows
+    ]
+    return batch_key, augmented_rows
+
+
+def _declared_tables(
+    *,
+    contract: TableContract,
+    declared_tables: list[str] | None,
+) -> list[str]:
+    normalized_declared = TableContract.declared_table_names(
+        database_name=contract.database_name,
+        schema=contract.schema,
+        table_names=declared_tables or [],
+    )
+    return sorted({*normalized_declared, contract.table_name})
+
+
+@dlt.destination(
+    batch_size=0,
+    loader_file_format="parquet",
+    loader_parallelism_strategy="table-sequential",
+    name="hotdata",
+    naming_convention="direct",
+    max_table_nesting=0,
+    skip_dlt_columns_and_tables=True,
+)
+def hotdata_destination(
+    items: TDataItems | str,
+    table: TTableSchema,
+    api_key: str = dlt.secrets.value,
+    workspace_id: str = dlt.secrets.value,
+    api_base_url: str = "https://api.hotdata.dev",
+    database_name: str = "dlt",
+    schema: str = "public",
+    write_disposition: str = "append",
+    declared_tables: list[str] | None = None,
+    create_database_if_missing: bool = True,
+    max_retries: int = 5,
+    retry_backoff_seconds: float = 1.0,
+) -> None:
+    config = HotdataDestinationConfig(
+        api_key=api_key,
+        workspace_id=workspace_id,
+        api_base_url=api_base_url,
+        database_name=database_name,
+        schema=schema,
+        write_disposition=write_disposition,
+        declared_tables=tuple(declared_tables or ()),
+        create_database_if_missing=create_database_if_missing,
+        max_retries=max_retries,
+        retry_backoff_seconds=retry_backoff_seconds,
+    )
+    contract = TableContract.from_table_schema(
+        table,
+        database_name=config.database_name,
+        schema=config.schema,
+    )
+    disposition = resolve_write_disposition(table, config.write_disposition)
+    primary_key = resolve_primary_key(table)
+    _, batch_rows = _augment_rows(table_name=contract.table_name, items=items)
+
+    client = HotdataClient(
+        api_key=config.api_key,
+        workspace_id=config.workspace_id,
+        api_base_url=config.api_base_url,
+        max_retries=config.max_retries,
+        retry_backoff_seconds=config.retry_backoff_seconds,
+    )
+
+    parquet_path = ""
+    try:
+        client.ensure_managed_database(
+            contract.database_name,
+            schema=contract.schema,
+            tables=_declared_tables(
+                contract=contract,
+                declared_tables=list(config.declared_tables),
+            ),
+            create_if_missing=config.create_database_if_missing,
+        )
+
+        rows_to_load = batch_rows
+        if disposition != "replace":
+            existing_rows = client.fetch_table_rows(
+                database=contract.database_name,
+                schema=contract.schema,
+                table=contract.table_name,
+            )
+            rows_to_load = combine_rows(
+                disposition=disposition,
+                existing=existing_rows,
+                incoming=batch_rows,
+                primary_key=primary_key,
+            )
+
+        with tempfile.NamedTemporaryFile(suffix=".parquet", delete=False) as handle:
+            parquet_path = handle.name
+        write_rows_parquet(rows_to_load, parquet_path)
+
+        upload_id = client.upload_parquet(parquet_path)
+        client.load_managed_table(
+            contract.database_name,
+            contract.table_name,
+            schema=contract.schema,
+            upload_id=upload_id,
+        )
+    except HotdataTerminalError as error:
+        raise DestinationTerminalException(str(error)) from error
+    finally:
+        if parquet_path:
+            os.unlink(parquet_path)
+        client.close()

hotdata_dlt_destination/errors.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from hotdata.rest import ApiException
+
+
+class HotdataDestinationError(RuntimeError):
+    pass
+
+
+class HotdataTransientError(HotdataDestinationError):
+    pass
+
+
+class HotdataTerminalError(HotdataDestinationError):
+    pass
+
+
+def classify_sdk_error(error: Exception) -> HotdataDestinationError:
+    if isinstance(error, TimeoutError):
+        return HotdataTransientError(str(error))
+    if isinstance(error, ConnectionError):
+        return HotdataTransientError(str(error))
+    if isinstance(error, ApiException):
+        status_code = int(error.status or 0)
+        message = f"{status_code}: {error.body or error.reason}"
+        if status_code in (408, 409, 425, 429):
+            return HotdataTransientError(message)
+        if 500 <= status_code <= 599:
+            return HotdataTransientError(message)
+        return HotdataTerminalError(message)
+    return HotdataTerminalError(str(error))

hotdata_dlt_destination/hotdata_client.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from typing import Any, TypeVar
+
+from hotdata.rest import ApiException
+from hotdata_runtime.client import HotdataClient as RuntimeClient
+from hotdata_runtime.databases import LoadManagedTableResult, ManagedDatabase
+
+from hotdata_dlt_destination.errors import (
+    HotdataTerminalError,
+    HotdataTransientError,
+    classify_sdk_error,
+)
+
+T = TypeVar("T")
+
+
+class HotdataClient:
+    """Managed-database client with bounded retries over hotdata-runtime."""
+
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        workspace_id: str,
+        api_base_url: str,
+        max_retries: int,
+        retry_backoff_seconds: float,
+    ) -> None:
+        self._max_retries = max_retries
+        self._retry_backoff_seconds = retry_backoff_seconds
+        self._runtime = RuntimeClient(
+            api_key,
+            workspace_id,
+            host=api_base_url.rstrip("/"),
+        )
+
+    def close(self) -> None:
+        self._runtime.close()
+
+    def ensure_managed_database(
+        self,
+        name: str,
+        *,
+        schema: str,
+        tables: list[str],
+        create_if_missing: bool,
+    ) -> ManagedDatabase:
+        def operation() -> ManagedDatabase:
+            try:
+                return self._runtime.resolve_managed_database(name)
+            except KeyError:
+                if not create_if_missing:
+                    raise
+                return self._runtime.create_managed_database(
+                    name,
+                    schema=schema,
+                    tables=sorted(set(tables)),
+                )
+
+        return self._request_with_retry(operation)
+
+    def table_is_synced(
+        self,
+        database: str,
+        table: str,
+        *,
+        schema: str,
+    ) -> bool:
+        for managed_table in self._runtime.list_managed_tables(database, schema=schema):
+            if managed_table.table == table:
+                return managed_table.synced
+        return False
+
+    def fetch_table_rows(
+        self,
+        *,
+        database: str,
+        schema: str,
+        table: str,
+    ) -> list[dict[str, Any]]:
+        def operation() -> list[dict[str, Any]]:
+            if not self.table_is_synced(database, table, schema=schema):
+                return []
+            qualified_table = f"{database}.{schema}.{table}"
+            result = self._runtime.execute_sql(f"SELECT * FROM {qualified_table}")
+            return result.to_records()
+
+        return self._request_with_retry(operation)
+
+    def upload_parquet(self, path: str) -> str:
+        return self._request_with_retry(lambda: self._runtime.upload_parquet(path))
+
+    def load_managed_table(
+        self,
+        database: str,
+        table: str,
+        *,
+        schema: str,
+        upload_id: str,
+    ) -> LoadManagedTableResult:
+        return self._request_with_retry(
+            lambda: self._runtime.load_managed_table(
+                database,
+                table,
+                schema=schema,
+                upload_id=upload_id,
+            )
+        )
+
+    def _request_with_retry(self, operation: Callable[[], T]) -> T:
+        for attempt in range(1, self._max_retries + 1):
+            try:
+                return operation()
+            except Exception as error:
+                mapped_error = self._classify_error(error)
+                if isinstance(mapped_error, HotdataTransientError) and attempt < self._max_retries:
+                    time.sleep(self._retry_backoff_seconds * attempt)
+                    continue
+                if isinstance(mapped_error, HotdataTransientError):
+                    raise mapped_error from error
+                raise HotdataTerminalError(str(mapped_error)) from error
+
+        raise HotdataTerminalError("Unexpected retry state")
+
+    @staticmethod
+    def _classify_error(error: Exception) -> Exception:
+        cause = error.__cause__ or error
+        if isinstance(cause, ApiException):
+            return classify_sdk_error(cause)
+        if isinstance(error, ApiException):
+            return classify_sdk_error(error)
+        if isinstance(error, (TimeoutError, ConnectionError)):
+            return classify_sdk_error(error)
+        return classify_sdk_error(error)

hotdata_dlt_destination/idempotency.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Sequence
+from typing import Any
+
+
+def stable_json_dumps(value: Any) -> str:
+    return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
+
+
+def compute_batch_key(table_name: str, items: Sequence[dict[str, Any]]) -> str:
+    payload = {
+        "table": table_name,
+        "items": [stable_json_dumps(item) for item in items],
+    }
+    return hashlib.sha256(stable_json_dumps(payload).encode("utf-8")).hexdigest()
+
+
+def compute_row_key(table_name: str, row: dict[str, Any]) -> str:
+    payload = {
+        "table": table_name,
+        "row": stable_json_dumps(row),
+    }
+    return hashlib.sha256(stable_json_dumps(payload).encode("utf-8")).hexdigest()

hotdata_dlt_destination/merge.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from typing import Any
+
+from dlt.common.schema import TTableSchema
+
+SUPPORTED_WRITE_DISPOSITIONS = frozenset({"replace", "append", "merge", "upsert"})
+
+
+def resolve_write_disposition(table: TTableSchema, default: str) -> str:
+    disposition = table.get("write_disposition") or default
+    return str(disposition).lower()
+
+
+def resolve_primary_key(table: TTableSchema) -> list[str] | None:
+    primary_key = table.get("primary_key")
+    if primary_key is None:
+        return None
+    if isinstance(primary_key, str):
+        return [primary_key]
+    return [str(key) for key in primary_key]
+
+
+def row_key(row: dict[str, Any], keys: list[str]) -> tuple[Any, ...]:
+    return tuple(row.get(key) for key in keys)
+
+
+def append_rows(
+    existing: list[dict[str, Any]],
+    incoming: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    return [*existing, *incoming]
+
+
+def merge_rows(
+    existing: list[dict[str, Any]],
+    incoming: list[dict[str, Any]],
+    *,
+    primary_key: list[str],
+) -> list[dict[str, Any]]:
+    merged = list(existing)
+    index = {row_key(row, primary_key): position for position, row in enumerate(merged)}
+    for row in incoming:
+        key = row_key(row, primary_key)
+        if key in index:
+            merged[index[key]] = row
+        else:
+            index[key] = len(merged)
+            merged.append(row)
+    return merged
+
+
+def combine_rows(
+    *,
+    disposition: str,
+    existing: list[dict[str, Any]],
+    incoming: list[dict[str, Any]],
+    primary_key: list[str] | None,
+) -> list[dict[str, Any]]:
+    if disposition == "replace":
+        return incoming
+    if disposition in ("merge", "upsert"):
+        keys = primary_key or ["_hotdata_row_key"]
+        return merge_rows(existing, incoming, primary_key=keys)
+    if disposition == "append":
+        return append_rows(existing, incoming)
+    raise ValueError(
+        f"Unsupported write_disposition {disposition!r}. "
+        f"Expected one of: {', '.join(sorted(SUPPORTED_WRITE_DISPOSITIONS))}"
+    )

hotdata_dlt_destination/parquet.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+
+def read_parquet_rows(path: str | Path) -> list[dict[str, Any]]:
+    table = pq.read_table(Path(path))
+    return table.to_pylist()
+
+
+def write_rows_parquet(rows: list[dict[str, Any]], path: str | Path) -> None:
+    table = pa.Table.from_pylist(rows)
+    pq.write_table(table, Path(path))

hotdata_dlt_destination/pipelines/__init__.py

@@ -0,0 +1 @@
+__all__ = []

hotdata_dlt_destination/pipelines/basic_pipeline.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import dlt
+
+from hotdata_dlt_destination.destination import hotdata_destination
+
+
+@dlt.resource(name="customers", write_disposition="append")
+def customers_resource() -> list[dict[str, object]]:
+    return [
+        {"id": 1, "name": "Acme", "tier": "enterprise", "is_active": True},
+        {"id": 2, "name": "Globex", "tier": "startup", "is_active": True},
+        {"id": 3, "name": "Initech", "tier": "midmarket", "is_active": False},
+    ]
+
+
+def main() -> None:
+    pipeline = dlt.pipeline(
+        pipeline_name="hotdata_basic",
+        destination=hotdata_destination(
+            write_disposition="append",
+            declared_tables=["customers"],
+        ),  # type: ignore[call-arg]
+        dataset_name="hotdata_basic",
+    )
+    load_info = pipeline.run(customers_resource())
+    print(load_info)
+
+
+if __name__ == "__main__":
+    main()

hotdata_dlt_destination/pipelines/incremental_pipeline.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+import dlt
+
+from hotdata_dlt_destination.destination import hotdata_destination
+
+
+@dlt.resource(name="orders", write_disposition="merge", primary_key="id")
+def orders_resource(since: str = "2026-01-01T00:00:00+00:00") -> list[dict[str, object]]:
+    watermark = datetime.fromisoformat(since)
+    rows = [
+        {"id": 101, "status": "new", "updated_at": "2026-01-01T01:00:00+00:00", "amount": 120.50},
+        {"id": 102, "status": "paid", "updated_at": "2026-01-02T01:00:00+00:00", "amount": 50.00},
+        {"id": 101, "status": "paid", "updated_at": "2026-01-03T01:00:00+00:00", "amount": 120.50},
+    ]
+    return [
+        row
+        for row in rows
+        if datetime.fromisoformat(str(row["updated_at"])).replace(tzinfo=UTC) > watermark
+    ]
+
+
+def main() -> None:
+    pipeline = dlt.pipeline(
+        pipeline_name="hotdata_incremental",
+        destination=hotdata_destination(
+            write_disposition="upsert",
+            declared_tables=["orders"],
+        ),  # type: ignore[call-arg]
+        dataset_name="hotdata_incremental",
+    )
+    load_info = pipeline.run(orders_resource())
+    print(load_info)
+
+
+if __name__ == "__main__":
+    main()

hotdata_dlt_destination/pipelines/linear_pipeline.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import json
+import os
+from typing import Any
+from urllib import request
+
+import dlt
+
+from hotdata_dlt_destination.destination import hotdata_destination
+
+LINEAR_GRAPHQL_URL = "https://api.linear.app/graphql"
+
+
+def fetch_linear_issues(
+    *,
+    api_key: str,
+    issue_limit: int,
+    team_key: str | None,
+) -> list[dict[str, Any]]:
+    if team_key:
+        query = """
+        query Issues($first: Int!, $teamKey: String!) {
+          issues(
+            first: $first
+            orderBy: updatedAt
+            filter: {team: {key: {eq: $teamKey}}}
+          ) {
+            nodes {
+              id
+              identifier
+              title
+              priority
+              createdAt
+              updatedAt
+              url
+              state { name type }
+              team { id key name }
+            }
+          }
+        }
+        """
+        variables: dict[str, Any] = {"first": issue_limit, "teamKey": team_key}
+    else:
+        query = """
+        query Issues($first: Int!) {
+          issues(first: $first, orderBy: updatedAt) {
+            nodes {
+              id
+              identifier
+              title
+              priority
+              createdAt
+              updatedAt
+              url
+              state { name type }
+              team { id key name }
+            }
+          }
+        }
+        """
+        variables = {"first": issue_limit}
+
+    payload = json.dumps({"query": query, "variables": variables}).encode("utf-8")
+    req = request.Request(
+        LINEAR_GRAPHQL_URL,
+        data=payload,
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": api_key,
+        },
+        method="POST",
+    )
+    with request.urlopen(req) as response:
+        body = json.loads(response.read().decode("utf-8"))
+    return body["data"]["issues"]["nodes"]
+
+
+@dlt.resource(name="linear_issues", write_disposition="merge", primary_key="id")
+def linear_issues_resource(
+    linear_api_key: str,
+    linear_issue_limit: int = 25,
+    linear_team_key: str | None = None,
+) -> list[dict[str, Any]]:
+    return fetch_linear_issues(
+        api_key=linear_api_key,
+        issue_limit=linear_issue_limit,
+        team_key=linear_team_key,
+    )
+
+
+def main() -> None:
+    pipeline = dlt.pipeline(
+        pipeline_name="hotdata_linear",
+        destination=hotdata_destination(
+            write_disposition="upsert",
+            database_name="linear",
+            declared_tables=["linear_issues"],
+        ),  # type: ignore[call-arg]
+        dataset_name="hotdata_linear",
+    )
+    linear_api_key = os.environ["LINEAR_API_KEY"]
+    linear_team_key = os.environ.get("LINEAR_TEAM_KEY")
+    issue_limit = int(os.environ.get("LINEAR_ISSUE_LIMIT", "25"))
+    load_info = pipeline.run(
+        linear_issues_resource(
+            linear_api_key=linear_api_key,
+            linear_team_key=linear_team_key,
+            linear_issue_limit=issue_limit,
+        )
+    )
+    print(load_info)
+
+
+if __name__ == "__main__":
+    main()

hotdata_dlt_destination-0.3.0.dist-info/METADATA

@@ -0,0 +1,108 @@
+Metadata-Version: 2.3
+Name: hotdata-dlt-destination
+Version: 0.3.0
+Summary: dlt destination for loading data into Hotdata managed databases.
+Author-email: 669988+eddietejeda@users.noreply.github.com
+Requires-Dist: dlt>=1.26.0
+Requires-Dist: hotdata>=0.2.2
+Requires-Dist: hotdata-runtime>=0.1.1
+Requires-Dist: pyarrow>=14
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+
+# hotdata-dlt-destination
+
+`hotdata-dlt-destination` is a Python package that implements a custom [dlt destination](https://dlthub.com/docs/dlt-ecosystem/destinations/destination) for loading data into **Hotdata managed databases** with deterministic idempotency keys and explicit write semantics.
+
+## What this repo includes
+
+- Custom destination via `@dlt.destination` in `src/hotdata_dlt_destination/destination.py`
+- Managed-database ingestion through `hotdata-runtime` (`upload_parquet`, `load_managed_table`, `SELECT`)
+- Read-modify-write append/merge using only supported API operations
+- Deterministic batch and row idempotency keys
+- Example pipelines:
+  - `hotdata-dlt-basic-pipeline` (append)
+  - `hotdata-dlt-incremental-pipeline` (upsert/merge)
+  - `hotdata-dlt-linear-pipeline` (Linear issues → Hotdata)
+- Unit tests in `tests/`
+- Architecture and runbook docs in `docs/`
+
+## Data contract defaults
+
+- Managed database: `database_name` (default `dlt`, created on first load when missing)
+- Schema: `public`
+- Table name: normalized lowercase dlt table identifier
+- Nested table names: `{parent}__{child}`
+- Write semantics (all use `load_managed_table(replace)` under the hood):
+  - `replace`: upload batch parquet and replace the target table
+  - `append`: read existing target rows, append batch in Python, replace target
+  - `upsert`/`merge`: read existing rows, upsert by dlt `primary_key` (or `_hotdata_row_key`), replace target
+- Idempotency:
+  - Batch key `_hotdata_batch_key` = hash(table + full batch payload)
+  - Row key `_hotdata_row_key` = hash(table + canonical row payload)
+
+## Configure
+
+Set environment variables (or pass destination kwargs / dlt secrets):
+
+- `HOTDATA_API_KEY`
+- `HOTDATA_WORKSPACE`
+- `HOTDATA_DATABASE` (managed database name, default `dlt`)
+- optional: `HOTDATA_SCHEMA`, `HOTDATA_WRITE_DISPOSITION`, `HOTDATA_DECLARED_TABLES`, retry tuning
+
+For pipelines with multiple tables, declare every target table when the managed database is first created:
+
+```python
+hotdata_destination(
+    database_name="analytics",
+    declared_tables=["customers", "orders", "orders__items"],
+)
+```
+
+## Usage
+
+```python
+import dlt
+from hotdata_dlt_destination import hotdata_destination
+
+pipeline = dlt.pipeline(
+    pipeline_name="my_pipeline",
+    destination=hotdata_destination(
+        database_name="analytics",
+        write_disposition="append",
+        declared_tables=["customers"],
+    ),
+)
+pipeline.run(my_resource())
+```
+
+Per-resource `write_disposition` and `primary_key` from dlt take precedence over the destination default.
+
+## Developer workflow
+
+```bash
+uv sync
+uv run ruff check .
+uv run pytest
+uv run hotdata-dlt-destination
+```
+
+Run pipelines:
+
+```bash
+uv run hotdata-dlt-basic-pipeline
+uv run hotdata-dlt-incremental-pipeline
+uv run hotdata-dlt-linear-pipeline
+```
+
+Run the live end-to-end integration test (requires Hotdata + Linear env vars):
+
+```bash
+uv run pytest tests/test_e2e_linear_hotdata.py -m integration
+```
+
+## References
+
+- [Hotdata Python SDK](https://github.com/hotdata-dev/sdk-python)
+- [hotdata-runtime](https://github.com/hotdata-dev/hotdata-runtime)
+- [dlt custom destination](https://dlthub.com/docs/dlt-ecosystem/destinations/destination)

hotdata_dlt_destination-0.3.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+hotdata_dlt_destination/__init__.py,sha256=hff97-_J8MfYcy1CHJ0nEkSW5XphL2jNoYEKDIMSTWQ,103
+hotdata_dlt_destination/cli.py,sha256=aTdERzqgJiInAbxXrVTgPKM6NsYtVS7vxD_DDdYx_Hg,471
+hotdata_dlt_destination/config.py,sha256=77Y9e1RMVvQz6-pcgCHm92VDiPhpSamsqYt5fwu98gY,1556
+hotdata_dlt_destination/contracts.py,sha256=H3iMWSWPInT_173hxMhvTgMEmLR_Tbje5npJZDuMJHE,1732
+hotdata_dlt_destination/destination.py,sha256=-s3QrlyP5FwSNynVun_14NBXWUT2rohTbdkJSs2EaBg,5162
+hotdata_dlt_destination/errors.py,sha256=prj76YG0YzCY-vpcqCq2xvb1LT1JbWNZnNS7MDzYoSQ,936
+hotdata_dlt_destination/hotdata_client.py,sha256=KmxrEtTYSRZSV3P4RnCTAF9XwO943qmWJV9fWSBmKoo,4266
+hotdata_dlt_destination/idempotency.py,sha256=iW5Qhb1LmwtmpNmL3XN7xOoKt0p_hW7M3ZiGXMSOfPI,765
+hotdata_dlt_destination/merge.py,sha256=FwB6voZt8PvZ1DrqBZNyOsvmodaIlPubAVX9p3Iu9OE,2037
+hotdata_dlt_destination/parquet.py,sha256=GCZ7RzpiCXWMq-vR3id2mP3k3SwlKJ2u_F0p1zIdxfQ,426
+hotdata_dlt_destination/pipelines/__init__.py,sha256=da1PTClDMl-IBkrSvq6JC1lnS-K_BASzCvxVhNxN5Ls,13
+hotdata_dlt_destination/pipelines/basic_pipeline.py,sha256=rF_M1-aDl6sEVQQ4SPtJ6V8qWVzxwqbMz-yD21X4hN8,887
+hotdata_dlt_destination/pipelines/incremental_pipeline.py,sha256=doCDF8AWKznuhYpuVeTUkv8DDzVU5x6qJpJxOGJUZXg,1239
+hotdata_dlt_destination/pipelines/linear_pipeline.py,sha256=koc8-zJ2nIdMZ5isE7I6edD211cECEraAfW4z5Iq8vM,3034
+hotdata_dlt_destination-0.3.0.dist-info/WHEEL,sha256=i9aSRDivn5iP9LaR1BLQX2GNAuriQWPsFwbbWygTX2k,81
+hotdata_dlt_destination-0.3.0.dist-info/entry_points.txt,sha256=wzZqzxtbIJ7q17dFJNbGGmaC4xyLJvA3ZTiSYaF09Kc,341
+hotdata_dlt_destination-0.3.0.dist-info/METADATA,sha256=OdiSZqtqw9XoK6qrGHx4M_VOn27ewRRvf1dvNFQju3Y,3601
+hotdata_dlt_destination-0.3.0.dist-info/RECORD,,

hotdata_dlt_destination-0.3.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.11.15
+Root-Is-Purelib: true
+Tag: py3-none-any

hotdata_dlt_destination-0.3.0.dist-info/entry_points.txt

@@ -0,0 +1,6 @@
+[console_scripts]
+hotdata-dlt-basic-pipeline = hotdata_dlt_destination.pipelines.basic_pipeline:main
+hotdata-dlt-destination = hotdata_dlt_destination.cli:main
+hotdata-dlt-incremental-pipeline = hotdata_dlt_destination.pipelines.incremental_pipeline:main
+hotdata-dlt-linear-pipeline = hotdata_dlt_destination.pipelines.linear_pipeline:main
+