chtypes 2.0.0.dev13__tar.gz → 2.0.0.dev14__tar.gz

{chtypes-2.0.0.dev13 → chtypes-2.0.0.dev14}/CHANGELOG.md

@@ -8,6 +8,8 @@ The four bindings in this repository are released together and give one answer,
 
 ### Added
 
+- `Value.null_input` (ABI v2): whether the input field was the format's NULL literal (a JSON `null`, a TSV `\N`), decoded from `cols[].null_input` in the row and batch documents; false when the document carries none. It differs from `null` where a NULL literal is read into a column that is not Nullable. Decode cells over the ABI v2 stub (variant `null-input`: a NULL literal into a non-Nullable UUID, FixedString, Int32, DateTime, LowCardinality(String) and Enum8 column, in JSONEachRow and TSV) run in all four bindings.
+- `Value.poison` and `Value.dup_dropped` (ABI v2): whether the stored value is one the server cannot read back (the batch is then `accepted_poisoned` and exports nothing), and whether the reader dropped a duplicate key, decoded from `cols[].poison` and `cols[].dup_dropped` in the row and batch documents; false when the document carries none. Decode cells over the ABI v2 stub (variant `value-flags`: a NULL read into a non-Nullable Enum8 in JSONEachRow and TSV, and a repeated top-level JSONEachRow key that arrives after every column has been read, which the reader skips so the first value is stored, with controls holding each member false and absent) run in all four bindings.
 - Tests: the shared revocation-statement table (`tests/fixtures/revocation-statement/cases.json`) runs against `verify_revocation` and `check_installed`: `superseded_by` must be a string (`""` = no fix), and the first verifying revocation in a listing wins. No behavior changed.
 - `column_defaults` is covered by unit tests (ABI v2): the exact `insert_options` JSON for a UTF-8 value, a non-UTF-8 value (`value_b64`), an empty value (sent as `""`) and two entries in order, and the decode of a `default_override` cell (`Source.DEFAULT_OVERRIDE`, stored). The batches guide gains a usage section. No behavior changed.
 - Docs: `docs/guides/batches.md` shows how a `column_defaults` value is read, in the body format's own field encoding (CSV quoting, TSV escapes, a JSON string or number), with a value that doesn't read refused at the call before any body byte.

{chtypes-2.0.0.dev13 → chtypes-2.0.0.dev14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: chtypes
-Version: 2.0.0.dev13
+Version: 2.0.0.dev14
 Summary: Python binding for the chtypes artifacts: ClickHouse's own type system, per version, behind the frozen chs_* C ABI
 Project-URL: Homepage, https://github.com/wave-rf/chtypes
 Project-URL: Repository, https://github.com/wave-rf/chtypes

{chtypes-2.0.0.dev13 → chtypes-2.0.0.dev14}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "chtypes"
-version = "2.0.0.dev13"
+version = "2.0.0.dev14"
 description = "Python binding for the chtypes artifacts: ClickHouse's own type system, per version, behind the frozen chs_* C ABI"
 readme = "README.md"
 license = "Apache-2.0"

{chtypes-2.0.0.dev13 → chtypes-2.0.0.dev14}/src/chtypes/_decode.py

@@ -322,6 +322,9 @@ def _value(obj: dict[str, Any], where: str) -> Value:
         source=source,
         is_stored=Source.is_stored(source),
         value=_opt_b64(obj, "value_b64", where),
+        null_input=_bool(obj, "null_input", where),
+        poison=_bool(obj, "poison", where),
+        dup_dropped=_bool(obj, "dup_dropped", where),
     )
 
 

{chtypes-2.0.0.dev13 → chtypes-2.0.0.dev14}/src/chtypes/results.py

@@ -73,7 +73,20 @@ class Value:
     bytes of a scalar String or FixedString leaf (from `value_b64`), None where
     the document carries none. `null`: the library's verdict, poison included.
     `source`: the `value_src` spelling (from `src`); `Source` lists them.
-    `is_stored`: the description's own fact for `source`.
+    `is_stored`: the description's own fact for `source`. `null_input`: whether
+    the input field was the format's NULL literal (a JSON null, a TSV `\\N`),
+    from `null_input`, False where the document carries none; a fact about the
+    input, where `null` is one about the stored value, so the two differ where
+    a NULL literal is read into a column that is not Nullable. `poison`: the
+    stored value is one the server cannot read back (the batch is then
+    accepted_poisoned and exports nothing), from `poison`; `dup_dropped`: the
+    reader dropped a duplicate key for this column, from `dup_dropped` (at default
+    settings a repeated top-level JSONEachRow key that arrives after every column
+    has been read is skipped: the first value is stored, this is True, and a
+    `duplicate_key_dropped` transform is reported; a repeat while a column is
+    still unread is refused with 117; a repeated key inside a Map value keeps
+    both keys). Each is
+    False where the document carries none.
     """
 
     column: bytes
@@ -82,6 +95,9 @@ class Value:
     source: str
     is_stored: bool
     value: bytes | None = None
+    null_input: bool = False
+    poison: bool = False
+    dup_dropped: bool = False
 
 
 @dataclass(frozen=True, slots=True)

chtypes-2.0.0.dev14/tests/abi2/test_null_input.py

@@ -0,0 +1,153 @@
+"""The null_input decode cells, through the public API, over the generated v2
+stub (variant "null-input").
+
+The stub answers scripts/abi-v1/emit/_stubshared.py's NULL_INPUT_DOCS: for a
+JSON null and a TSV `\\N` into each of six non-Nullable types (UUID,
+FixedString(4), Int32, DateTime, LowCardinality(String), Enum8), the entry
+carries `null_input` true; two controls carry it false and absent. Each entry's
+expected value is read off its name (a NULL_INPUT_CONTROL_PREFIX name expects
+False), and the cell count is the generator's NULL_INPUT_CELL_COUNT, so a
+document that lost its cells fails. The other three bindings pin that count as
+12; `test_null_input_cell_count_is_pinned` fails here first if the generator
+changes it. Without `$CHTYPES_ABI2_STUBS` the stub tests skip loudly by name.
+"""
+
+from __future__ import annotations
+
+import shutil
+import sys
+from pathlib import Path
+
+import pytest
+
+from chtypes import Format, Value, library
+
+from .conftest import REPO_ROOT, stub_path
+
+PINNED_CELL_COUNT = 12  # Go, TypeScript and Rust pin the same number
+
+
+def _shared():
+    sys.path.insert(0, str(REPO_ROOT / "scripts" / "abi-v1"))
+    from emit import _stubshared
+
+    return _stubshared
+
+
+@pytest.fixture
+def null_input_library(
+    stubs_dir: Path, stubs_manifest: dict, tmp_path: Path, clean_process, monkeypatch
+):
+    monkeypatch.setenv("CHTYPES_ALLOW_UNVERIFIED_LIBRARY", "1")
+    target = tmp_path / "null-input.so"
+    shutil.copyfile(stub_path(stubs_dir, stubs_manifest["variants"]["null-input"]), target)
+    target.chmod(0o755)
+    with pytest.warns(UserWarning, match="UNVERIFIED"):
+        return library.open_unverified(target, allow=True)
+
+
+def _check(where: str, cols: tuple[Value, ...], prefix: bytes) -> int:
+    cells = 0
+    for c in cols:
+        want = not c.column.startswith(prefix)
+        cells += want
+        assert c.null_input is want, (
+            f"{where}: column {c.column!r}: null_input = {c.null_input}, want {want}"
+        )
+        # A NULL into a non-Nullable Enum8 stores 0, outside the enum: poisoned.
+        want_poison = want and c.column.endswith(b".enum8")
+        assert c.poison is want_poison, (
+            f"{where}: column {c.column!r}: poison = {c.poison}, want {want_poison}"
+        )
+    return cells
+
+
+def test_null_input_cell_count_is_pinned() -> None:
+    assert _shared().NULL_INPUT_CELL_COUNT == PINNED_CELL_COUNT
+
+
+def test_stub_null_input(null_input_library) -> None:
+    shared = _shared()
+    prefix = shared.NULL_INPUT_CONTROL_PREFIX.encode()
+    schema = null_input_library.compile_table(b"CREATE TABLE t (x Int32)")
+
+    row = schema.row(Format.JSON_EACH_ROW, b'{"x":null}')
+    assert _check("row", row.columns, prefix) == shared.NULL_INPUT_CELL_COUNT
+
+    batch = schema.rows(Format.TSV, b"\\N\n")
+    cells = sum(_check(f"batch.rows[{i}]", r.columns, prefix) for i, r in enumerate(batch.rows))
+    assert cells == shared.NULL_INPUT_CELL_COUNT
+
+
+# The poison and dup_dropped cells, variant "value-flags"
+# (_stubshared.VALUE_FLAGS_DOCS): a JSON null or TSV \N into a non-Nullable
+# Enum8 (poison, in JSONEachRow and TSV) and one canned entry that carries
+# dup_dropped, plus four controls that hold each member false and absent. A
+# "poison." entry expects poison True and dup_dropped False, a "dup." entry the
+# reverse, a "control." entry both False, so a binding that swaps the members
+# fails. The other three bindings pin the counts as 2 and 1.
+PINNED_POISON_CELLS = 2
+PINNED_DUP_CELLS = 1
+
+
+@pytest.fixture
+def value_flags_library(
+    stubs_dir: Path, stubs_manifest: dict, tmp_path: Path, clean_process, monkeypatch
+):
+    monkeypatch.setenv("CHTYPES_ALLOW_UNVERIFIED_LIBRARY", "1")
+    target = tmp_path / "value-flags.so"
+    shutil.copyfile(stub_path(stubs_dir, stubs_manifest["variants"]["value-flags"]), target)
+    target.chmod(0o755)
+    with pytest.warns(UserWarning, match="UNVERIFIED"):
+        return library.open_unverified(target, allow=True)
+
+
+def _check_flags(where: str, cols: tuple[Value, ...], shared) -> tuple[int, int]:
+    poison = dup = 0
+    for c in cols:
+        name = c.column.decode()
+        want_p = name.startswith(shared.VALUE_FLAGS_POISON_PREFIX)
+        want_d = name.startswith(shared.VALUE_FLAGS_DUP_PREFIX)
+        if not (want_p or want_d):
+            assert name.startswith(shared.VALUE_FLAGS_CONTROL_PREFIX), (
+                f"{where}: unexpected {name!r}"
+            )
+        poison += want_p
+        dup += want_d
+        if want_d:
+            # The producing input: a repeated top-level JSONEachRow key after
+            # every column was read; the first value is stored.
+            assert (name, c.text) == ("dup.JSONEachRow.repeated_top_key", b"1"), where
+        assert (c.poison, c.dup_dropped) == (want_p, want_d), (
+            f"{where}: column {name!r}: poison = {c.poison}, dup_dropped = {c.dup_dropped}, "
+            f"want {want_p}, {want_d}"
+        )
+    return poison, dup
+
+
+def test_value_flags_cell_counts_are_pinned() -> None:
+    shared = _shared()
+    assert shared.VALUE_FLAGS_POISON_CELLS == PINNED_POISON_CELLS
+    assert shared.VALUE_FLAGS_DUP_CELLS == PINNED_DUP_CELLS
+
+
+def test_stub_value_flags(value_flags_library) -> None:
+    shared = _shared()
+    schema = value_flags_library.compile_table(b"CREATE TABLE t (x Int32)")
+
+    row = schema.row(Format.JSON_EACH_ROW, b'{"a":1,"a":2}')
+    assert _check_flags("row", row.columns, shared) == (PINNED_POISON_CELLS, PINNED_DUP_CELLS)
+
+    dup_transforms = [
+        t
+        for t in row.transformed
+        if t.reason == "duplicate_key_dropped" and t.column == b"dup.JSONEachRow.repeated_top_key"
+    ]
+    assert len(dup_transforms) == 1
+
+    batch = schema.rows(Format.TSV, b"1\n")
+    counts = [_check_flags(f"batch.rows[{i}]", r.columns, shared) for i, r in enumerate(batch.rows)]
+    assert (sum(p for p, _ in counts), sum(d for _, d in counts)) == (
+        PINNED_POISON_CELLS,
+        PINNED_DUP_CELLS,
+    )

{chtypes-2.0.0.dev13 → chtypes-2.0.0.dev14}/uv.lock

@@ -74,7 +74,7 @@ wheels = [
 
 [[package]]
 name = "chtypes"
-version = "2.0.0.dev13"
+version = "2.0.0.dev14"
 source = { editable = "." }
 dependencies = [
     { name = "backports-zstd", marker = "python_full_version < '3.14'" },