machbaseapi 2.0.0__py3-none-any.whl

machbaseAPI/protocol.py

@@ -0,0 +1,661 @@
+from __future__ import annotations
+
+import datetime
+import ipaddress
+import socket
+import struct
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple
+
+from .constants import *
+from .errors import DatabaseError, OperationalError, ProgrammingError
+from .marshal import MarshalReader, MarshalWriter
+from .packet import PacketReader
+from .types import spiner_type_from_legacy_sql, spiner_type_from_value
+
+
+@dataclass
+class ColumnMeta:
+    name: str
+    cm_type: int
+    precision: int
+    scale: int
+    spiner_type: int
+    length: int
+    is_variable: bool
+
+
+@dataclass
+class QueryMeta:
+    statement_id: int
+    columns: List[ColumnMeta]
+    rows: List[Dict[str, Any]]
+    rows_affected: int
+    message: Optional[str]
+    more: bool
+
+
+@dataclass
+class _AppendSession:
+    statement_id: int
+    table: str
+    columns: List[ColumnMeta]
+    server_le: bool
+    user_types: Optional[List[int]] = None
+
+
+def extract_spiner_type(cm_type: int) -> int:
+    return (cm_type >> 56) & 0xFF
+
+
+def extract_precision(cm_type: int) -> int:
+    return (cm_type >> 28) & 0x0FFFFFFF
+
+
+def extract_scale(cm_type: int) -> int:
+    return cm_type & 0x0FFFFFFF
+
+
+def _compute_column_length(spiner_type: int, precision: int) -> int:
+    if spiner_type in {CMD_INT16_TYPE, CMD_UINT16_TYPE}:
+        return 2
+    if spiner_type in {CMD_INT32_TYPE, CMD_UINT32_TYPE, CMD_FLT32_TYPE}:
+        return 4
+    if spiner_type in {CMD_INT64_TYPE, CMD_UINT64_TYPE, CMD_DATE_TYPE, CMD_FLT64_TYPE}:
+        return 8
+    if spiner_type == CMD_IPV4_TYPE:
+        return 5
+    if spiner_type == CMD_IPV6_TYPE:
+        return 17
+    if spiner_type == CMD_BOOL_TYPE:
+        return 1
+    if spiner_type == CMD_NUL_TYPE:
+        return 0
+    return precision
+
+
+def _decode_ip_field(raw: bytes) -> str:
+    if len(raw) == 5 and raw[0] == 4:
+        return ".".join(str(b) for b in raw[1:])
+    if len(raw) == 17 and raw[0] == 6:
+        return str(ipaddress.IPv6Address(raw[1:]))
+    return ""
+
+
+def _format_datetime(value: int) -> Optional[str]:
+    if value in (0xFFFFFFFFFFFFFFFF, -1):
+        return None
+
+    dt: Optional[datetime.datetime] = None
+    for divisor in (1_000_000_000, 1_000_000, 1_000):
+        try:
+            candidate = datetime.datetime.fromtimestamp(value / divisor)
+        except (OverflowError, OSError, ValueError):
+            continue
+        if 1980 <= candidate.year <= 2500:
+            dt = candidate
+            break
+        if dt is None:
+            dt = candidate
+
+    if dt is None:
+        dt = datetime.datetime.fromtimestamp(0)
+
+    if dt.microsecond:
+        return dt.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+    return dt.strftime("%Y-%m-%d %H:%M:%S")
+
+
+def _decode_column_value(spiner_type: int, data: bytes) -> Any:
+    if spiner_type == CMD_INT16_TYPE and data == b"\x80\x00":
+        return None
+    if spiner_type == CMD_UINT16_TYPE and data == b"\xff\xff":
+        return None
+    if spiner_type == CMD_INT32_TYPE and data == b"\x80\x00\x00\x00":
+        return None
+    if spiner_type == CMD_UINT32_TYPE and data == b"\xff\xff\xff\xff":
+        return None
+    if spiner_type == CMD_INT64_TYPE and data == b"\x80\x00\x00\x00\x00\x00\x00\x00":
+        return None
+    if spiner_type == CMD_UINT64_TYPE and data == b"\xff\xff\xff\xff\xff\xff\xff\xff":
+        return None
+    if spiner_type == CMD_BOOL_TYPE:
+        return data[0] != 0 if data else None
+    if spiner_type == CMD_INT16_TYPE:
+        return int.from_bytes(data[:2], "big", signed=True)
+    if spiner_type == CMD_UINT16_TYPE:
+        return int.from_bytes(data[:2], "big", signed=False)
+    if spiner_type == CMD_INT32_TYPE:
+        return int.from_bytes(data[:4], "big", signed=True)
+    if spiner_type == CMD_UINT32_TYPE:
+        return int.from_bytes(data[:4], "big", signed=False)
+    if spiner_type == CMD_INT64_TYPE:
+        return int.from_bytes(data[:8], "big", signed=True)
+    if spiner_type == CMD_UINT64_TYPE:
+        return int.from_bytes(data[:8], "big", signed=False)
+    if spiner_type == CMD_FLT32_TYPE:
+        return struct.unpack(">f", data[:4])[0]
+    if spiner_type == CMD_FLT64_TYPE:
+        return struct.unpack(">d", data[:8])[0]
+    if spiner_type == CMD_DATE_TYPE:
+        return _format_datetime(int.from_bytes(data[:8], "big", signed=True))
+    if spiner_type == CMD_IPV4_TYPE:
+        return _decode_ip_field(data[:5])
+    if spiner_type == CMD_IPV6_TYPE:
+        return _decode_ip_field(data[:17])
+    if spiner_type in (CMD_VARCHAR_TYPE, CMD_CHAR_TYPE, CMD_TEXT_TYPE, CMD_CLOB_TYPE, CMD_JSON_TYPE):
+        try:
+            return data.decode("utf-8")
+        except Exception:
+            return bytes(data)
+    if spiner_type in (CMD_BINARY_TYPE, CMD_BLOB_TYPE):
+        return bytes(data)
+    if spiner_type == CMD_NUL_TYPE:
+        return None
+    if not data:
+        return None
+    try:
+        return data.decode("utf-8")
+    except Exception:
+        return bytes(data)
+
+
+def _consume_row_values(unit_data: bytes, columns: Sequence[ColumnMeta]) -> Dict[str, Any]:
+    row: Dict[str, Any] = {}
+    offset = 0
+    for column in columns:
+        if column.is_variable:
+            if offset + 4 > len(unit_data):
+                raise DatabaseError("Malformed variable-length field")
+            size = int.from_bytes(unit_data[offset:offset + 4], "big", signed=False)
+            offset += 4
+            data = unit_data[offset:offset + size]
+            offset += size
+            row[column.name] = _decode_column_value(column.spiner_type, data)
+            continue
+        data = unit_data[offset:offset + column.length]
+        offset += column.length
+        row[column.name] = _decode_column_value(column.spiner_type, data)
+    return row
+
+
+def collect_units(payload: bytes) -> Dict[int, List[Tuple[int, bytes]]]:
+    buckets: Dict[int, List[Tuple[int, bytes]]] = {}
+    for unit in MarshalReader(payload):
+        buckets.setdefault(unit.id, []).append((unit.type, unit.data))
+    return buckets
+
+
+def _first_unit(units: Mapping[int, List[Tuple[int, bytes]]], unit_id: int) -> Optional[Tuple[int, bytes]]:
+    values = units.get(unit_id)
+    return values[0] if values else None
+
+
+def _read_status(data: bytes) -> int:
+    return int.from_bytes(data[:8], "little", signed=False)
+
+
+def _status_error_code(status: int) -> int:
+    return status & 0xFFFFFFFF
+
+
+def _format_error_message(units: Mapping[int, List[Tuple[int, bytes]]]) -> str:
+    parts: List[str] = []
+    for unit_id in (CMI_R_MESSAGE_ID, CMI_R_EMESSAGE_ID):
+        item = _first_unit(units, unit_id)
+        if item:
+            parts.append(item[1].decode("utf-8", errors="replace"))
+    message = "; ".join(part for part in parts if part)
+    status_unit = _first_unit(units, CMI_R_RESULT_ID)
+    if status_unit:
+        error_no = _status_error_code(_read_status(status_unit[1]))
+        if error_no:
+            if message:
+                return f"{error_no} - {message}"
+            return str(error_no)
+    return message
+
+
+def _make_protocol_version(major: int, minor: int, patch: int) -> int:
+    # JVM 기반 클라이언트에서 사용하는 makeVersion(major, minor, patch) 포맷을 그대로 사용한다.
+    return ((major & 0xFFFF) << 48) | ((minor & 0xFFFF) << 32) | (patch & 0xFFFFFFFF)
+
+
+def _status_ok(status: int) -> bool:
+    masked = status & 0xFFFFFFFF00000000
+    return masked in {CMI_OK_RESULT, CMI_LAST_RESULT}
+
+
+def _status_last(status: int) -> bool:
+    return (status & 0xFFFFFFFF00000000) == CMI_LAST_RESULT
+
+
+def _build_column_meta(units: Mapping[int, List[Tuple[int, bytes]]]) -> List[ColumnMeta]:
+    names = [item[1].decode("utf-8", errors="replace") for item in units.get(CMI_P_COLNAME_ID, [])]
+    types = units.get(CMI_P_COLTYPE_ID, [])
+    count = min(len(names), len(types))
+    columns: List[ColumnMeta] = []
+    for index in range(count):
+        cm_type = int.from_bytes(types[index][1][:8], "little", signed=False)
+        spiner_type = extract_spiner_type(cm_type)
+        precision = extract_precision(cm_type)
+        scale = extract_scale(cm_type)
+        columns.append(
+            ColumnMeta(
+                name=names[index],
+                cm_type=cm_type,
+                precision=precision,
+                scale=scale,
+                spiner_type=spiner_type,
+                length=_compute_column_length(spiner_type, precision),
+                is_variable=is_variable_type(spiner_type),
+            )
+        )
+    return columns
+
+
+def _rows_from_units(units: Mapping[int, List[Tuple[int, bytes]]], columns: Sequence[ColumnMeta]) -> List[Dict[str, Any]]:
+    rows: List[Dict[str, Any]] = []
+    for _, data in units.get(CMI_F_VALUE_ID, []):
+        rows.append(_consume_row_values(data, columns))
+    return rows
+
+
+def _int_to_microseconds(value: Any) -> int:
+    if value is None:
+        return -1
+    if isinstance(value, datetime.datetime):
+        return int(value.timestamp() * 1_000_000_000)
+    if isinstance(value, datetime.date):
+        return int(
+            datetime.datetime.combine(value, datetime.time.min).timestamp() * 1_000_000_000
+        )
+    if isinstance(value, (int, float)):
+        return int(value)
+    text = str(value).strip()
+    if not text:
+        return -1
+    for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d"):
+        try:
+            dt = datetime.datetime.strptime(text, fmt)
+            return int(dt.timestamp() * 1_000_000_000)
+        except ValueError:
+            continue
+    try:
+        return int(text)
+    except ValueError:
+        return -1
+
+
+def _encode_length_prefixed(data: bytes, server_le: bool) -> bytes:
+    return len(data).to_bytes(4, "little" if server_le else "big") + data
+
+
+def _encode_fixed_int(value: int, size: int, signed: bool, server_le: bool) -> bytes:
+    return int(value).to_bytes(size, "little" if server_le else "big", signed=signed)
+
+
+def _encode_value(spiner_type: int, value: Any, server_le: bool) -> bytes:
+    if value is None:
+        if is_variable_type(spiner_type):
+            return _encode_length_prefixed(b"", server_le)
+        if spiner_type == CMD_DATE_TYPE:
+            return _encode_fixed_int(-1, 8, True, server_le)
+        if spiner_type == CMD_IPV4_TYPE:
+            return bytes([4, 0, 0, 0, 0])
+        if spiner_type == CMD_IPV6_TYPE:
+            return bytes([6]) + bytes(16)
+        return b""
+    if spiner_type == CMD_BOOL_TYPE:
+        return b"\x01" if bool(value) else b"\x00"
+    if spiner_type == CMD_INT16_TYPE:
+        return _encode_fixed_int(int(value), 2, True, server_le)
+    if spiner_type == CMD_UINT16_TYPE:
+        return _encode_fixed_int(int(value), 2, False, server_le)
+    if spiner_type == CMD_INT32_TYPE:
+        return _encode_fixed_int(int(value), 4, True, server_le)
+    if spiner_type == CMD_UINT32_TYPE:
+        return _encode_fixed_int(int(value), 4, False, server_le)
+    if spiner_type == CMD_INT64_TYPE:
+        return _encode_fixed_int(int(value), 8, True, server_le)
+    if spiner_type == CMD_UINT64_TYPE:
+        return _encode_fixed_int(int(value), 8, False, server_le)
+    if spiner_type == CMD_FLT32_TYPE:
+        return struct.pack("<f" if server_le else ">f", float(value))
+    if spiner_type == CMD_FLT64_TYPE:
+        return struct.pack("<d" if server_le else ">d", float(value))
+    if spiner_type == CMD_DATE_TYPE:
+        return _encode_fixed_int(_int_to_microseconds(value), 8, True, server_le)
+    if spiner_type == CMD_IPV4_TYPE:
+        packed = ipaddress.ip_address(str(value)).packed
+        return bytes([4]) + packed
+    if spiner_type == CMD_IPV6_TYPE:
+        packed = ipaddress.ip_address(str(value)).packed
+        return bytes([6]) + packed
+    if isinstance(value, (bytes, bytearray, memoryview)):
+        return _encode_length_prefixed(bytes(value), server_le)
+    return _encode_length_prefixed(str(value).encode("utf-8"), server_le)
+
+
+class MachbaseProtocolClient:
+    def __init__(
+        self,
+        *,
+        host: str = "127.0.0.1",
+        port: int = 5656,
+        user: str = "SYS",
+        password: str = "MANAGER",
+        database: str = "data",
+        timeout_ms: int = DEFAULT_CONNECT_TIMEOUT_MS,
+        query_timeout_ms: int = DEFAULT_QUERY_TIMEOUT_MS,
+        client_id: str = "PYMCB",
+        show_hidden_columns: bool = False,
+        timezone: str = "",
+    ):
+        self.host = host
+        self.port = int(port)
+        self.user = user
+        self.password = password
+        self.database = database
+        self.connect_timeout_ms = int(timeout_ms)
+        self.query_timeout_ms = int(query_timeout_ms)
+        self.client_id = client_id
+        self.show_hidden_columns = bool(show_hidden_columns)
+        self.timezone = timezone
+        self.socket: Optional[socket.socket] = None
+        self.reader: Optional[PacketReader] = None
+        self.connected = False
+        self.next_stmt = 1
+        self.session_id = 0
+        self.server_le = True
+
+    def connect(self) -> None:
+        if self.connected:
+            return
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+            sock.settimeout(self.connect_timeout_ms / 1000.0)
+            sock.connect((self.host, self.port))
+            sock.settimeout(None)
+            self._handshake(sock)
+            self.socket = sock
+            self.reader = PacketReader(sock)
+            self._send_connect()
+            self.connected = True
+        except Exception as exc:
+            try:
+                sock.close()
+            except Exception:
+                pass
+            self.socket = None
+            self.reader = None
+            raise OperationalError(str(exc)) from exc
+
+    def close(self) -> None:
+        if self.socket:
+            try:
+                self.socket.close()
+            finally:
+                self.socket = None
+                self.reader = None
+        self.connected = False
+
+    def _next_statement_id(self) -> int:
+        sid = self.next_stmt
+        self.next_stmt = (self.next_stmt + 1) & 0x7FFFFFFF
+        if self.next_stmt == 0:
+            self.next_stmt = 1
+        return sid
+
+    def _handshake(self, sock: socket.socket) -> None:
+        sock.sendall(f"{CMI_HANDSHAKE_PREFIX}{CMI_HANDSHAKE_ENDIAN_LITTLE}".encode("ascii"))
+        response = self._recv_exact(sock, CMI_PROTO_CNT)
+        if response.decode("ascii") != CMI_HANDSHAKE_READY:
+            raise OperationalError("Invalid handshake response")
+
+    def _send_connect(self) -> None:
+        version = _make_protocol_version(
+            CMI_PROTOCOL_MAJOR_VERSION,
+            CMI_PROTOCOL_MINOR_VERSION,
+            CMI_PROTOCOL_FIX_VERSION,
+        )
+        writer = MarshalWriter(CMI_CONNECT_PROTOCOL, 0)
+        writer.add_uint64(CMI_C_VERSION_ID, version)
+        writer.add_string(CMI_C_CLIENT_ID, self.client_id)
+        writer.add_string(CMI_C_DATABASE_ID, self.database)
+        writer.add_string(CMI_C_USER_ID, self.user)
+        writer.add_string(CMI_C_PASSWORD_ID, self.password)
+        writer.add_uint64(CMI_C_TIMEOUT_ID, max(1, self.query_timeout_ms // 1000))
+        writer.add_uint32(CMI_C_SHC_ID, 1 if self.show_hidden_columns else 0)
+        writer.add_string(CMI_C_IP_ID, self.host)
+        if self.timezone:
+            writer.add_string(CMI_C_TIMEZONE_ID, self.timezone)
+        self._send_packets(writer.finalize())
+        units = collect_units(self._read_protocol(CMI_CONNECT_PROTOCOL, self.query_timeout_ms))
+        status_unit = _first_unit(units, CMI_R_RESULT_ID)
+        if not status_unit:
+            raise OperationalError("CONNECT response missing status")
+        status = _read_status(status_unit[1])
+        if not _status_ok(status):
+            raise OperationalError(_format_error_message(units) or "CONNECT failed")
+        sid = _first_unit(units, CMI_C_SID_ID)
+        if sid:
+            self.session_id = int.from_bytes(sid[1][:8], "little", signed=False)
+        endian = _first_unit(units, CMI_C_ENDIAN_ID)
+        if endian:
+            self.server_le = int.from_bytes(endian[1][:4], "little", signed=False) == 0
+
+    def _send_packets(self, packets: Sequence[bytes]) -> None:
+        if not self.socket:
+            raise OperationalError("Not connected")
+        for packet in packets:
+            self.socket.sendall(packet)
+
+    def _read_protocol(self, protocol_id: int, timeout_ms: int) -> bytes:
+        if not self.reader:
+            raise OperationalError("Not connected")
+        chunks: List[bytes] = []
+        while True:
+            packet = self.reader.next(timeout_ms)
+            if packet.protocol != protocol_id:
+                raise OperationalError(f"Protocol mismatch: expected {protocol_id}, got {packet.protocol}")
+            chunks.append(packet.body)
+            if packet.flag in (0, 3):
+                return b"".join(chunks)
+
+    def exec_direct(self, sql: str) -> QueryMeta:
+        stmt_id = self._next_statement_id()
+        writer = MarshalWriter(CMI_EXECDIRECT_PROTOCOL, stmt_id)
+        writer.add_string(CMI_D_STATEMENT_ID, sql)
+        writer.add_uint64(CMI_P_ID_ID, stmt_id)
+        self._send_packets(writer.finalize())
+        units = collect_units(self._read_protocol(CMI_EXECDIRECT_PROTOCOL, self.query_timeout_ms))
+        status_unit = _first_unit(units, CMI_R_RESULT_ID)
+        if not status_unit:
+            raise DatabaseError("EXEC response missing status")
+        status = _read_status(status_unit[1])
+        if not _status_ok(status):
+            raise DatabaseError(_format_error_message(units) or "Execute failed")
+        rows = _first_unit(units, CMI_P_ROWS_ID)
+        rows_affected = int.from_bytes(rows[1][:8], "little", signed=False) if rows else 0
+        message = _format_error_message(units) or None
+        return QueryMeta(stmt_id, [], [], rows_affected, message, _status_last(status))
+
+    def _fetch_rows_loop(self, statement_id: int, columns: Sequence[ColumnMeta]) -> List[Dict[str, Any]]:
+        rows: List[Dict[str, Any]] = []
+        while True:
+            writer = MarshalWriter(CMI_FETCH_PROTOCOL, statement_id)
+            writer.add_uint32(CMI_F_ID_ID, statement_id)
+            writer.add_sint64(CMI_F_ROWS_ID, 1000)
+            self._send_packets(writer.finalize())
+            units = collect_units(self._read_protocol(CMI_FETCH_PROTOCOL, self.query_timeout_ms))
+            status_unit = _first_unit(units, CMI_R_RESULT_ID)
+            if not status_unit:
+                raise DatabaseError("FETCH response missing status")
+            status_units = units.get(CMI_R_RESULT_ID, [])
+            has_last = False
+            for _, status_bytes in status_units:
+                status = _read_status(status_bytes)
+                if not _status_ok(status):
+                    raise DatabaseError(_format_error_message(units) or "Fetch failed")
+                if _status_last(status):
+                    has_last = True
+            batch = _rows_from_units(units, columns)
+            rows.extend(batch)
+            row_count = _first_unit(units, CMI_F_ROWS_ID)
+            if has_last:
+                break
+            row_cnt = int.from_bytes(row_count[1][:8], "little", signed=False) if row_count else None
+            no_more_rows = (row_cnt is None or row_cnt <= 0) and not batch
+            if no_more_rows:
+                break
+        return rows
+
+    def query(self, sql: str) -> QueryMeta:
+        stmt_id = self._next_statement_id()
+        writer = MarshalWriter(CMI_EXECDIRECT_PROTOCOL, stmt_id)
+        writer.add_string(CMI_D_STATEMENT_ID, sql)
+        writer.add_uint64(CMI_P_ID_ID, stmt_id)
+        self._send_packets(writer.finalize())
+        units = collect_units(self._read_protocol(CMI_EXECDIRECT_PROTOCOL, self.query_timeout_ms))
+        status_unit = _first_unit(units, CMI_R_RESULT_ID)
+        if not status_unit:
+            raise DatabaseError("QUERY response missing status")
+        status = _read_status(status_unit[1])
+        if not _status_ok(status):
+            raise DatabaseError(_format_error_message(units) or "Query failed")
+        columns = _build_column_meta(units)
+        rows = _rows_from_units(units, columns)
+        if columns and not _status_last(status):
+            rows.extend(self._fetch_rows_loop(stmt_id, columns))
+        return QueryMeta(stmt_id, columns, rows, len(rows), _format_error_message(units) or None, _status_last(status))
+
+    def append_open(self, table: str) -> _AppendSession:
+        stmt_id = self._next_statement_id()
+        writer = MarshalWriter(CMI_APPEND_OPEN_PROTOCOL, stmt_id)
+        writer.add_uint64(CMI_P_ID_ID, stmt_id)
+        writer.add_string(CMI_P_TABLE_ID, table)
+        writer.add_uint64(CMI_E_ENDIAN_ID, 0)
+        self._send_packets(writer.finalize())
+        units = collect_units(self._read_protocol(CMI_APPEND_OPEN_PROTOCOL, self.query_timeout_ms))
+        status_unit = _first_unit(units, CMI_R_RESULT_ID)
+        if not status_unit:
+            raise DatabaseError("APPEND open response missing status")
+        status = _read_status(status_unit[1])
+        if not _status_ok(status):
+            raise DatabaseError(_format_error_message(units) or "Append open failed")
+        return _AppendSession(stmt_id, table, _build_column_meta(units), self.server_le)
+
+    def append_data(
+        self,
+        session: _AppendSession,
+        rows: Sequence[Sequence[Any]],
+        types: Optional[Sequence[int]] = None,
+        times: Optional[Sequence[Any]] = None,
+        on_ack: Optional[Callable[[Dict[int, List[Tuple[int, bytes]]]], None]] = None,
+    ) -> int:
+        writer = MarshalWriter(CMI_APPEND_DATA_PROTOCOL, session.statement_id, adds=session.statement_id & 0xFFFF)
+        metadata = session.columns
+        has_arrival = bool(metadata) and metadata[0].name.upper() in {"", "_ARRIVAL_TIME"}
+        null_bytes = (len(metadata) // 8) + 1 if metadata else 0
+        for row_index, row in enumerate(rows):
+            values = list(row)
+            if metadata and has_arrival:
+                if len(values) == len(metadata) - 1:
+                    if times and row_index < len(times):
+                        values = [times[row_index]] + values
+                    else:
+                        values = [0] + values
+                elif len(values) == len(metadata):
+                    if times and row_index < len(times) and values[0] is None:
+                        values[0] = times[row_index]
+                else:
+                    raise ProgrammingError("Append row length does not match append metadata")
+            elif metadata and len(values) != len(metadata):
+                raise ProgrammingError("Append row length does not match append metadata")
+            null_bits = bytearray(null_bytes)
+            payload = bytearray()
+            payload.append(0)
+            payload.extend(int(null_bytes).to_bytes(4, "little" if session.server_le else "big"))
+            payload.extend(null_bits)
+            for col_index, value in enumerate(values):
+                if has_arrival and col_index == 0:
+                    payload.extend(_encode_value(CMD_DATE_TYPE, value, session.server_le))
+                    continue
+                if metadata:
+                    spiner_type = metadata[col_index].spiner_type
+                elif types and col_index < len(types):
+                    spiner_type = spiner_type_from_legacy_sql(int(types[col_index]))
+                else:
+                    spiner_type = spiner_type_from_value(value)
+                if value is None and null_bytes:
+                    null_index = col_index
+                    if null_index < 0:
+                        payload.extend(_encode_value(spiner_type, value, session.server_le))
+                        continue
+                    null_offset = 5 + (null_index // 8)
+                    payload[null_offset] |= 1 << (7 - (null_index % 8))
+                    continue
+                payload.extend(_encode_value(spiner_type, value, session.server_le))
+            writer.add_binary(CMI_P_ROWS_ID, bytes(payload))
+        self._send_packets(writer.finalize())
+        try:
+            units = collect_units(self._read_protocol(CMI_APPEND_DATA_PROTOCOL, 0))
+        except TimeoutError:
+            return len(rows)
+        if on_ack is not None:
+            on_ack(units)
+        status_unit = _first_unit(units, CMI_R_RESULT_ID)
+        if status_unit:
+            status = _read_status(status_unit[1])
+            if not _status_ok(status):
+                raise DatabaseError(_format_error_message(units) or "Append data failed")
+        failures = _first_unit(units, CMI_X_APPEND_FAILURE_ID)
+        if failures and int.from_bytes(failures[1][:8], "little", signed=False) > 0:
+                raise DatabaseError(_format_error_message(units) or "Append data reported failures")
+        return len(rows)
+
+    def append_close(self, session: _AppendSession) -> int:
+        writer = MarshalWriter(CMI_APPEND_CLOSE_PROTOCOL, session.statement_id)
+        writer.add_uint64(CMI_P_ID_ID, session.statement_id)
+        self._send_packets(writer.finalize())
+        units = collect_units(self._read_protocol(CMI_APPEND_CLOSE_PROTOCOL, self.query_timeout_ms))
+        status_unit = _first_unit(units, CMI_R_RESULT_ID)
+        if not status_unit:
+            raise DatabaseError("APPEND close response missing status")
+        status = _read_status(status_unit[1])
+        if not _status_ok(status):
+            raise DatabaseError(_format_error_message(units) or "Append close failed")
+        success = _first_unit(units, CMI_X_APPEND_SUCCESS_ID)
+        return int.from_bytes(success[1][:8], "little", signed=False) if success else 0
+
+    def append(
+        self,
+        table: str,
+        rows: Sequence[Sequence[Any]],
+        types: Optional[Sequence[int]] = None,
+        times: Optional[Sequence[Any]] = None,
+    ) -> int:
+        session = self.append_open(table)
+        try:
+            self.append_data(session, rows, types=types, times=times)
+            return self.append_close(session)
+        except Exception:
+            try:
+                self.append_close(session)
+            except Exception:
+                pass
+            raise
+
+    @staticmethod
+    def _recv_exact(sock: socket.socket, length: int) -> bytes:
+        chunks: List[bytes] = []
+        remaining = length
+        while remaining > 0:
+            chunk = sock.recv(remaining)
+            if not chunk:
+                raise OperationalError("Socket closed while reading protocol")
+            chunks.append(chunk)
+            remaining -= len(chunk)
+        return b"".join(chunks)