dpone-native-accel 0.29.0__tar.gz

dpone_native_accel-0.29.0/PKG-INFO

@@ -0,0 +1,31 @@
+Metadata-Version: 2.4
+Name: dpone-native-accel
+Version: 0.29.0
+Summary: Optional native acceleration provider for dpone native transfer
+Author: PaulKov
+License-Expression: Apache-2.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Typing :: Typed
+Maintainer: PaulKov
+Requires-Python: >=3.11, <3.13
+Project-URL: Homepage, https://github.com/PaulKov/dpone
+Project-URL: Repository, https://github.com/PaulKov/dpone
+Description-Content-Type: text/markdown
+
+# dpone-native-accel
+
+Optional provider package for dpone native-transfer acceleration.
+
+The package exposes the stable provider boundary used by `dpone[accel]`.
+Backends declare certified capabilities before dpone can select them for
+`native_transfer.wire.acceleration.mode: auto|required`.
+
+The v0.29 provider includes a fused MSSQL BCP native -> ClickHouse Native
+encoder for certified primitive, decimal, temporal, UUID, binary, and text
+layouts. It avoids the core row-dict reference pipeline while preserving
+byte-for-byte equivalence with the Python reference path. Unsupported layouts
+fall back to the reference path unless acceleration is explicitly required.

dpone_native_accel-0.29.0/README.md

@@ -0,0 +1,13 @@
+# dpone-native-accel
+
+Optional provider package for dpone native-transfer acceleration.
+
+The package exposes the stable provider boundary used by `dpone[accel]`.
+Backends declare certified capabilities before dpone can select them for
+`native_transfer.wire.acceleration.mode: auto|required`.
+
+The v0.29 provider includes a fused MSSQL BCP native -> ClickHouse Native
+encoder for certified primitive, decimal, temporal, UUID, binary, and text
+layouts. It avoids the core row-dict reference pipeline while preserving
+byte-for-byte equivalence with the Python reference path. Unsupported layouts
+fall back to the reference path unless acceleration is explicitly required.

dpone_native_accel-0.29.0/pyproject.toml

@@ -0,0 +1,25 @@
+[build-system]
+requires = ["uv_build>=0.9.18,<0.12.0"]
+build-backend = "uv_build"
+
+[project]
+name = "dpone-native-accel"
+version = "0.29.0"
+description = "Optional native acceleration provider for dpone native transfer"
+readme = "README.md"
+requires-python = ">=3.11,<3.13"
+license = "Apache-2.0"
+authors = [{ name = "PaulKov" }]
+maintainers = [{ name = "PaulKov" }]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "License :: OSI Approved :: Apache Software License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Typing :: Typed",
+]
+
+[project.urls]
+Homepage = "https://github.com/PaulKov/dpone"
+Repository = "https://github.com/PaulKov/dpone"

dpone_native_accel-0.29.0/src/dpone_native_accel/__init__.py

@@ -0,0 +1,7 @@
+"""Optional dpone native acceleration provider."""
+
+from __future__ import annotations
+
+from dpone_native_accel._provider import SCHEMA_VERSION, __version__, capabilities, transcode
+
+__all__ = ["SCHEMA_VERSION", "__version__", "capabilities", "transcode"]

dpone_native_accel-0.29.0/src/dpone_native_accel/_mssql_clickhouse_native.py

@@ -0,0 +1,336 @@
+"""Fused MSSQL BCP native to ClickHouse Native block encoder."""
+
+from __future__ import annotations
+
+import re
+import struct
+import uuid
+from collections.abc import Iterator, Mapping, Sequence
+from dataclasses import dataclass
+from datetime import date
+from pathlib import Path
+from typing import Any, BinaryIO
+
+_DAYS_TO_UNIX_EPOCH = (date(1970, 1, 1) - date(1, 1, 1)).days
+_DAYS_TO_SQL_SERVER_EPOCH = (date(1970, 1, 1) - date(1900, 1, 1)).days
+
+
+@dataclass(frozen=True, slots=True)
+class ColumnLayout:
+    name: str
+    storage_type: str
+    clickhouse_type: str
+    nullable: bool
+    prefix_width: int
+    fixed_length: int | None
+    precision: int | None
+    scale: int | None
+    encoding: str
+
+    @classmethod
+    def from_contract(cls, raw: Mapping[str, Any], target_type: str | None) -> ColumnLayout:
+        return cls(
+            name=str(raw["name"]),
+            storage_type=str(raw["storage_type"]).lower(),
+            clickhouse_type=str(target_type or raw["target_type"]),
+            nullable=bool(raw.get("nullable")),
+            prefix_width=int(raw.get("prefix_width") or 0),
+            fixed_length=int(raw["fixed_length"]) if raw.get("fixed_length") is not None else None,
+            precision=int(raw["precision"]) if raw.get("precision") is not None else None,
+            scale=int(raw["scale"]) if raw.get("scale") is not None else None,
+            encoding=str(raw.get("encoding") or "utf-8"),
+        )
+
+
+class MssqlBcpClickHouseNativeBackend:
+    """Encode supported MSSQL native rows into ClickHouse Native blocks."""
+
+    def __init__(
+        self,
+        *,
+        artifact_path: Path,
+        columns: Sequence[ColumnLayout],
+        block_rows: int,
+        block_bytes: int | None,
+    ) -> None:
+        self._artifact_path = artifact_path
+        self._columns = tuple(columns)
+        self._block_rows = max(1, block_rows)
+        self._block_bytes = block_bytes
+
+    @classmethod
+    def from_request(cls, request: Mapping[str, Any]) -> MssqlBcpClickHouseNativeBackend:
+        contract = _mapping(request.get("native_wire_contract"))
+        bulk = _mapping(request.get("bulk_wire_contract"))
+        target_schema = tuple((str(name), str(dtype)) for name, dtype in request.get("clickhouse_schema") or ())
+        target_by_position = tuple(dtype for _, dtype in target_schema)
+        raw_columns = tuple(_mapping(item) for item in contract.get("columns") or ())
+        columns = tuple(
+            ColumnLayout.from_contract(raw, target_by_position[index] if index < len(target_by_position) else None)
+            for index, raw in enumerate(raw_columns)
+        )
+        return cls(
+            artifact_path=Path(str(request["artifact_path"])),
+            columns=columns,
+            block_rows=int(bulk.get("block_rows") or 65_536),
+            block_bytes=_parse_bytes(bulk.get("block_bytes")),
+        )
+
+    def iter_blocks(self) -> Iterator[bytes]:
+        with self._artifact_path.open("rb") as handle:
+            while True:
+                block = _BlockBuilder(self._columns)
+                estimated_bytes = 0
+                while len(block) < self._block_rows and not _eof(handle):
+                    values = [_read_cell(handle, column) for column in self._columns]
+                    estimated_bytes += sum(len(value or b"") + 1 for value in values)
+                    block.append(values)
+                    if self._block_bytes is not None and estimated_bytes >= self._block_bytes:
+                        break
+                if len(block) == 0:
+                    return
+                yield block.to_bytes()
+
+
+class _BlockBuilder:
+    def __init__(self, columns: Sequence[ColumnLayout]) -> None:
+        self._columns = tuple(columns)
+        self._null_maps = [bytearray() for _ in columns]
+        self._data = [bytearray() for _ in columns]
+        self._rows = 0
+
+    def __len__(self) -> int:
+        return self._rows
+
+    def append(self, values: Sequence[bytes | None]) -> None:
+        for index, value in enumerate(values):
+            column = self._columns[index]
+            nullable, inner_type = _unwrap_nullable(column.clickhouse_type)
+            if nullable:
+                self._null_maps[index].append(1 if value is None else 0)
+                self._data[index].extend(
+                    _default_value(inner_type) if value is None else _encode_value(value, column, inner_type)
+                )
+            elif value is None:
+                raise ValueError(f"native_acceleration_null_for_non_nullable:{column.name}")
+            else:
+                self._data[index].extend(_encode_value(value, column, column.clickhouse_type))
+        self._rows += 1
+
+    def to_bytes(self) -> bytes:
+        payload = bytearray()
+        payload.extend(_var_uint(len(self._columns)))
+        payload.extend(_var_uint(self._rows))
+        for index, column in enumerate(self._columns):
+            payload.extend(_ch_string(column.name))
+            payload.extend(_ch_string(column.clickhouse_type))
+            payload.extend(self._null_maps[index])
+            payload.extend(self._data[index])
+        return bytes(payload)
+
+
+def _read_cell(handle: BinaryIO, column: ColumnLayout) -> bytes | None:
+    length = column.fixed_length
+    if column.prefix_width:
+        indicator = int.from_bytes(_read_exact(handle, column.prefix_width), byteorder="little", signed=True)
+        if indicator == -1:
+            return None
+        length = indicator
+    if length is None:
+        raise ValueError(f"native_acceleration_missing_length:{column.name}")
+    return _read_exact(handle, length)
+
+
+def _encode_value(payload: bytes, column: ColumnLayout, clickhouse_type: str) -> bytes:
+    root = _root_type(clickhouse_type)
+    storage = column.storage_type
+    if root == "bool":
+        return struct.pack("<B", 1 if payload[0] else 0)
+    if root in {"int8", "uint8", "int16", "uint16", "int32", "uint32", "int64", "uint64", "float32", "float64"}:
+        return payload
+    if root.startswith("decimal"):
+        return _encode_decimal(payload, column, clickhouse_type)
+    if root == "date":
+        return _encode_date(payload)
+    if root == "datetime64":
+        return _encode_datetime64(payload, column, _scale(clickhouse_type))
+    if root == "uuid":
+        return _encode_uuid(payload)
+    if root == "string":
+        if storage in {"nvarchar", "nchar"}:
+            value = payload.decode(column.encoding).encode("utf-8")
+        else:
+            value = payload
+        return _var_uint(len(value)) + value
+    raise ValueError(f"native_acceleration_unsupported_clickhouse_type:{clickhouse_type}")
+
+
+def _default_value(clickhouse_type: str) -> bytes:
+    root = _root_type(clickhouse_type)
+    widths = {
+        "bool": 1,
+        "int8": 1,
+        "uint8": 1,
+        "int16": 2,
+        "uint16": 2,
+        "int32": 4,
+        "uint32": 4,
+        "int64": 8,
+        "uint64": 8,
+        "float32": 4,
+        "float64": 8,
+    }
+    if root == "string":
+        return b"\x00"
+    if root == "uuid":
+        return b"\x00" * 16
+    if root == "date":
+        return b"\x00" * 2
+    if root == "datetime64":
+        return b"\x00" * 8
+    if root.startswith("decimal"):
+        precision, _ = _decimal_precision_scale(clickhouse_type)
+        return b"\x00" * _decimal_width(precision)
+    if root in widths:
+        return b"\x00" * widths[root]
+    raise ValueError(f"native_acceleration_unsupported_clickhouse_type:{clickhouse_type}")
+
+
+def _encode_decimal(payload: bytes, column: ColumnLayout, clickhouse_type: str) -> bytes:
+    precision, target_scale = _decimal_precision_scale(clickhouse_type)
+    storage = column.storage_type
+    if storage in {"money", "smallmoney"}:
+        scaled = _money_integer(payload) if storage == "money" else struct.unpack("<i", payload)[0]
+    elif storage in {"decimal", "numeric"}:
+        source_scale = int(column.scale if column.scale is not None else payload[1])
+        sign = 1 if payload[2] == 1 else -1
+        magnitude = int.from_bytes(payload[3:], byteorder="little", signed=False)
+        scaled = sign * magnitude
+        if target_scale > source_scale:
+            scaled *= 10 ** (target_scale - source_scale)
+        elif target_scale < source_scale:
+            scaled //= 10 ** (source_scale - target_scale)
+    else:
+        raise ValueError(f"native_acceleration_unsupported_decimal_source:{column.storage_type}")
+    return int(scaled).to_bytes(_decimal_width(precision), byteorder="little", signed=True)
+
+
+def _encode_date(payload: bytes) -> bytes:
+    days = int.from_bytes(payload, byteorder="little", signed=False) - _DAYS_TO_UNIX_EPOCH
+    return struct.pack("<H", days)
+
+
+def _encode_datetime64(payload: bytes, column: ColumnLayout, target_scale: int) -> bytes:
+    storage = column.storage_type
+    if storage == "datetime2":
+        time_length = (column.fixed_length or len(payload)) - 3
+        source_scale = int(column.scale or target_scale)
+        time_ticks = int.from_bytes(payload[:time_length], byteorder="little", signed=False)
+        days = int.from_bytes(payload[time_length:], byteorder="little", signed=False) - _DAYS_TO_UNIX_EPOCH
+        ticks = days * 86400 * (10**source_scale) + time_ticks
+        if target_scale > source_scale:
+            ticks *= 10 ** (target_scale - source_scale)
+        elif target_scale < source_scale:
+            ticks //= 10 ** (source_scale - target_scale)
+        return struct.pack("<q", ticks)
+    if storage == "datetime":
+        days, sql_ticks = struct.unpack("<ii", payload)
+        milliseconds = (sql_ticks * 1000 + 150) // 300
+        ticks = (days - _DAYS_TO_SQL_SERVER_EPOCH) * 86400 * 1000 + milliseconds
+        return struct.pack("<q", _rescale_ticks(ticks, 3, target_scale))
+    if storage == "smalldatetime":
+        days, minutes = struct.unpack("<HH", payload)
+        ticks = ((days - _DAYS_TO_SQL_SERVER_EPOCH) * 86400 + minutes * 60) * (10**target_scale)
+        return struct.pack("<q", ticks)
+    raise ValueError(f"native_acceleration_unsupported_datetime_source:{column.storage_type}")
+
+
+def _encode_uuid(payload: bytes) -> bytes:
+    raw = uuid.UUID(bytes_le=payload).bytes
+    return raw[:8][::-1] + raw[8:][::-1]
+
+
+def _rescale_ticks(value: int, source_scale: int, target_scale: int) -> int:
+    if target_scale > source_scale:
+        return value * (10 ** (target_scale - source_scale))
+    if target_scale < source_scale:
+        return value // (10 ** (source_scale - target_scale))
+    return value
+
+
+def _money_integer(payload: bytes) -> int:
+    high = int.from_bytes(payload[:4], byteorder="little", signed=True)
+    low = int.from_bytes(payload[4:], byteorder="little", signed=False)
+    return (high << 32) + low
+
+
+def _decimal_precision_scale(clickhouse_type: str) -> tuple[int, int]:
+    match = re.search(r"\((\d+)\s*,\s*(\d+)\)", clickhouse_type)
+    if not match:
+        return 38, 9
+    return int(match.group(1)), int(match.group(2))
+
+
+def _decimal_width(precision: int) -> int:
+    return 4 if precision <= 9 else 8 if precision <= 18 else 16 if precision <= 38 else 32
+
+
+def _scale(clickhouse_type: str) -> int:
+    match = re.search(r"\((\d+)", clickhouse_type)
+    return min(max(int(match.group(1)), 0), 9) if match else 0
+
+
+def _unwrap_nullable(clickhouse_type: str) -> tuple[bool, str]:
+    value = clickhouse_type.strip()
+    if value.lower().startswith("nullable(") and value.endswith(")"):
+        return True, value[len("Nullable(") : -1].strip()
+    return False, value
+
+
+def _root_type(clickhouse_type: str) -> str:
+    return clickhouse_type.strip().split("(", 1)[0].strip().lower()
+
+
+def _ch_string(value: str) -> bytes:
+    payload = value.encode("utf-8")
+    return _var_uint(len(payload)) + payload
+
+
+def _var_uint(value: int) -> bytes:
+    output = bytearray()
+    current = int(value)
+    while current >= 0x80:
+        output.append((current & 0x7F) | 0x80)
+        current >>= 7
+    output.append(current)
+    return bytes(output)
+
+
+def _read_exact(handle: BinaryIO, length: int) -> bytes:
+    payload = handle.read(length)
+    if len(payload) != length:
+        raise EOFError("native_acceleration_unexpected_eof")
+    return payload
+
+
+def _eof(handle: BinaryIO) -> bool:
+    position = handle.tell()
+    payload = handle.read(1)
+    handle.seek(position)
+    return not payload
+
+
+def _mapping(value: Any) -> Mapping[str, Any]:
+    if not isinstance(value, Mapping):
+        raise ValueError("native_acceleration_invalid_request")
+    return value
+
+
+def _parse_bytes(value: Any) -> int | None:
+    if value is None:
+        return None
+    text = str(value).strip()
+    units = {"kib": 1024, "mib": 1024**2, "gib": 1024**3, "kb": 1000, "mb": 1000**2, "gb": 1000**3}
+    suffix = "".join(re.findall(r"[A-Za-z]+", text)).lower()
+    number = text[: len(text) - len(suffix)] if suffix else text
+    return int(float(number.strip()) * units.get(suffix, 1))

dpone_native_accel-0.29.0/src/dpone_native_accel/_provider.py

@@ -0,0 +1,61 @@
+"""Public provider facade for dpone native acceleration."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from typing import Any
+
+from dpone_native_accel._mssql_clickhouse_native import MssqlBcpClickHouseNativeBackend
+
+__version__ = "0.29.0"
+SCHEMA_VERSION = "dpone.native_transfer.acceleration.v1"
+BACKEND_ID = "mssql_bcp_native_to_clickhouse_native"
+SUPPORTED_TYPES = (
+    "bit",
+    "tinyint",
+    "smallint",
+    "int",
+    "bigint",
+    "real",
+    "float",
+    "money",
+    "smallmoney",
+    "decimal",
+    "numeric",
+    "date",
+    "datetime",
+    "datetime2",
+    "smalldatetime",
+    "uniqueidentifier",
+    "binary",
+    "varbinary",
+    "char",
+    "varchar",
+    "nchar",
+    "nvarchar",
+)
+
+
+def capabilities() -> dict[str, Any]:
+    """Return certified native acceleration backends."""
+
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "package_version": __version__,
+        "backends": [
+            {
+                "backend_id": BACKEND_ID,
+                "source_format": "mssql-bcp-native",
+                "target_format": "Native",
+                "certified": True,
+                "supported_platforms": ["linux_x86_64", "macos_arm64", "macos_x86_64"],
+                "supported_types": list(SUPPORTED_TYPES),
+            }
+        ],
+    }
+
+
+def transcode(request: Mapping[str, Any]) -> Iterable[bytes]:
+    """Transcode one MSSQL BCP native artifact to ClickHouse Native blocks."""
+
+    return MssqlBcpClickHouseNativeBackend.from_request(request).iter_blocks()