chtypes 2.0.0.dev7__tar.gz → 2.0.0.dev9__tar.gz

{chtypes-2.0.0.dev7 → chtypes-2.0.0.dev9}/CHANGELOG.md

@@ -54,6 +54,17 @@ The four bindings in this repository are released together and give one answer,
 - **Enforcement gate restored** for `26.7` on `darwin-arm64`.
 - **Enforcement gate restored** for `26.8` on `darwin-arm64`.
 - **Enforcement gate restored** for `26.9` on `darwin-arm64`.
+- **Enforcement gate lifted** for `26.3` on `linux-amd64`.
+- **Enforcement gate lifted** for `26.7` on `linux-amd64`.
+- **Enforcement gate lifted** for `26.8` on `linux-amd64`.
+- **Enforcement gate lifted** for `26.9` on `linux-amd64`.
+- **Enforcement gate lifted** for `26.3` on `linux-arm64`.
+- **Enforcement gate lifted** for `26.7` on `linux-arm64`.
+- **Enforcement gate lifted** for `26.8` on `linux-arm64`.
+- **Enforcement gate lifted** for `26.9` on `linux-arm64`.
+- **Enforcement gate lifted** for `26.7` on `darwin-arm64`.
+- **Enforcement gate lifted** for `26.8` on `darwin-arm64`.
+- **Enforcement gate lifted** for `26.9` on `darwin-arm64`.
 
 ### Removed
 
@@ -61,6 +72,8 @@ The four bindings in this repository are released together and give one answer,
 
 ### Fixed
 
+- A settings or query-parameter key or value that is not valid UTF-8 is refused as a usage error; it was silently replaced with U+FFFD, so the library saw a different value. A string holding a lone surrogate (what `surrogateescape` makes of undecodable bytes) was sent as a `\udXXX` escape; the same inputs are refused now, for the per-call zone, the setup defaults and the server profile too. Valid UTF-8 serializes as before.
+- A declined export's payload is absent, as documented: it was an empty payload when the library returned a zero-length buffer for a declined export. `BatchResult.payload` and `spans` are `None` for a declined export. Whether an export was declined is decided from the document's `export_declined`, never from the buffer's length.
 - The layout's `index.json` never gains a second entry for one manifest (#487): concurrent installs of one build each appended an entry (7 or 8 entries for the same digest after 8 concurrent installs, measured on 1.0.2). The merge now deduplicates by digest, as Go and TypeScript do.
 - `--frozen` with a warm, verified cache makes zero requests, and `--frozen --offline` verifies the installed build against the lock with zero network (#414). An installed build whose `verified.json` is for the platform and records exactly the lock's pinned manifest, layer and bundle digests is used as it is. `--frozen --offline` is `CHTYPES_ARTIFACT_MISSING` when the pinned build is not installed, and `CHTYPES_ARTIFACT_PINNED` when there is no lock entry or the installed record does not match the pin; `--offline` without `--frozen` is unchanged. The `frozen-warm-cache-zero-requests`, `frozen-offline-lock-match`, `frozen-offline-lock-mismatch` and `frozen-offline-lock-not-installed` conformance cases hold all four bindings to it, under both corpora. The Python frozen path also compares the record's layer and bundle digests to the pin, and looks in the system directories too.
 - A 2.0.0-dev SDK whose own-fingerprint alias has expired, with a cached build of that fingerprint for the request, now says so in the `CHTYPES_ARTIFACT_UNPUBLISHED` message of `fetch` (#581): `; a cached build for this fingerprint (build <id>) is still usable by open; upgrade the SDK to get newer builds` is appended. The code, the exit status (4) and the cache are unchanged, and all four bindings give the same message (`fp-ahead-cached-own-fp`).

{chtypes-2.0.0.dev7 → chtypes-2.0.0.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: chtypes
-Version: 2.0.0.dev7
+Version: 2.0.0.dev9
 Summary: Python binding for the chtypes artifacts: ClickHouse's own type system, per version, behind the frozen chs_* C ABI
 Project-URL: Homepage, https://github.com/wave-rf/chtypes
 Project-URL: Repository, https://github.com/wave-rf/chtypes

{chtypes-2.0.0.dev7 → chtypes-2.0.0.dev9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "chtypes"
-version = "2.0.0.dev7"
+version = "2.0.0.dev9"
 description = "Python binding for the chtypes artifacts: ClickHouse's own type system, per version, behind the frozen chs_* C ABI"
 readme = "README.md"
 license = "Apache-2.0"

{chtypes-2.0.0.dev7 → chtypes-2.0.0.dev9}/src/chtypes/_decode.py

@@ -428,6 +428,13 @@ def decode_batch(raw: bytes, export: bytes | None) -> BatchResult:
     framing = obj.get("framing")
     engine_rows = None if obj.get("engine_rows") is None else _engine_rows(obj, where)
     spans = None if obj.get("row_spans") is None else _spans(obj, "row_spans", where)
+    export_declined = _bytes(obj, "export_declined", where)
+    if export_declined:
+        # A declined export has no payload and no spans, whatever buffer the call
+        # returned: the library returns a NULL out_export for it (builds up to
+        # 20261011.105019 returned a zero-length buffer), and the document's
+        # export_declined decides, never the buffer.
+        export, spans = None, None
     return BatchResult(
         outcome=Outcome.of(_text(obj, "outcome", where)),
         err_code=_int(obj, "code", where),
@@ -439,7 +446,7 @@ def decode_batch(raw: bytes, export: bytes | None) -> BatchResult:
         engine_rows=engine_rows,
         payload=export,
         spans=spans,
-        export_declined=_bytes(obj, "export_declined", where),
+        export_declined=export_declined,
         rows_passed=_u64(obj, "rows_passed", where),
         rows_cut=_u64(obj, "rows_cut", where),
         partition_count=_opt_u64(obj, "partition_count", where),

{chtypes-2.0.0.dev7 → chtypes-2.0.0.dev9}/src/chtypes/_input.py

@@ -45,6 +45,31 @@ def body_bytes(value: bytes, what: str = "body") -> bytes:
     raise TypeError(f"{what} must be bytes, not {type(value).__name__}")
 
 
+_NON_UTF8_HINT = (
+    "this ABI generation cannot carry a non-UTF-8 value in a JSON input; "
+    "write the bytes as unhex('<hex>') in the expression instead"
+)
+
+
+def _is_utf8(text: str) -> bool:
+    """False for a `str` that cannot be UTF-8 encoded (a lone surrogate, as
+    `surrogateescape` makes of undecodable bytes)."""
+    try:
+        text.encode("utf-8")
+    except UnicodeEncodeError:
+        return False
+    return True
+
+
+def _refuse_non_utf8(what: str, part: str, key: str) -> Exception:
+    # The key is shown escaped (ASCII only), never as the raw text.
+    return misuse(
+        f"{what}: the key {json.dumps(key)} is not valid UTF-8, and {_NON_UTF8_HINT}"
+        if part == "key"
+        else f"{what}: the value for key {json.dumps(key)} is not valid UTF-8, and {_NON_UTF8_HINT}"
+    )
+
+
 def _string_map(mapping: Mapping[str, str], what: str) -> dict[str, str]:
     out: dict[str, str] = {}
     for key, value in mapping.items():
@@ -53,6 +78,12 @@ def _string_map(mapping: Mapping[str, str], what: str) -> dict[str, str]:
                 f"{what} maps strings to strings only; got {type(key).__name__} -> "
                 f"{type(value).__name__} for {key!r} (a value is never rewritten)"
             )
+        # json.dumps would write a lone surrogate as a \udXXX escape, a value
+        # the library does not read as the bytes the caller meant: refuse.
+        if not _is_utf8(key):
+            raise _refuse_non_utf8(what, "key", key)
+        if not _is_utf8(value):
+            raise _refuse_non_utf8(what, "value", key)
         out[key] = value
     return out
 
@@ -75,6 +106,8 @@ def settings_json(
     if session_timezone is not None:
         if not isinstance(session_timezone, str):
             raise TypeError(f"session_timezone must be str, not {type(session_timezone).__name__}")
+        if not _is_utf8(session_timezone):
+            raise misuse(f"the zone: the value is not valid UTF-8, and {_NON_UTF8_HINT}")
         if _ZONE_KEY in merged:
             raise misuse(
                 f"the {_ZONE_KEY} option and a {_ZONE_KEY} settings key were both given; "
@@ -141,10 +174,12 @@ def server_profile_json(profile: ServerProfile) -> bytes:
     if profile.timezone is not None:
         if not isinstance(profile.timezone, str):
             raise TypeError(f"timezone must be str, not {type(profile.timezone).__name__}")
+        if not _is_utf8(profile.timezone):
+            raise misuse(f"server profile: the timezone is not valid UTF-8, and {_NON_UTF8_HINT}")
         if profile.timezone != "":
             doc["timezone"] = profile.timezone
     if profile.settings is not None:
-        doc["settings"] = _string_map(profile.settings, "settings")
+        doc["settings"] = _string_map(profile.settings, "server profile settings")
     if profile.macros is not None:
-        doc["macros"] = _string_map(profile.macros, "macros")
+        doc["macros"] = _string_map(profile.macros, "server profile macros")
     return _dump(doc)

{chtypes-2.0.0.dev7 → chtypes-2.0.0.dev9}/src/chtypes/library.py

@@ -296,7 +296,7 @@ class Schema(_Handle):
         zone is its own, fixed here: its WHERE runs in the session this call's
         `session_timezone` (or `settings`) names, on every evaluation."""
         expr_bytes = to_bytes(expr, "expr")
-        params_bytes = string_map_json(params, "params")
+        params_bytes = string_map_json(params, "query parameters")
         settings_bytes = settings_json(settings, session_timezone)
         with _hold(self._guard):
             handle = self._library._api.filter_create(

{chtypes-2.0.0.dev7 → chtypes-2.0.0.dev9}/tests/api/test_decode.py

@@ -177,6 +177,28 @@ BATCH = {
 }
 
 
+def test_a_declined_exports_payload_is_absent_whatever_the_buffer() -> None:
+    # The library returns a NULL out_export for a declined export (builds up to
+    # 20261011.105019 returned a non-NULL, zero-length buffer): the document's
+    # export_declined decides, never the buffer.
+    for member in (
+        {"export_declined": "only accepted batches export"},
+        {"export_declined_b64": b64(b"\xff")},
+    ):
+        doc = {"outcome": "rejected", "row_spans": [{"off": 0, "len": 1}], **member}
+        for buffer in (b"", b"x"):
+            batch = decode_batch(dumps(doc), buffer)
+            assert batch.payload is None and batch.spans is None and batch.export_declined
+
+
+def test_an_accepted_zero_row_export_is_present_and_empty() -> None:
+    batch = decode_batch(
+        dumps({"outcome": "accepted", "row_spans": [], "export_declined": ""}), b""
+    )
+    assert batch.payload == b"" and batch.spans == () and batch.export_declined == b""
+    assert decode_batch(dumps({"outcome": "accepted"}), None).payload is None
+
+
 def test_a_batch_decodes_rows_payload_framing_and_spans() -> None:
     batch = decode_batch(dumps(BATCH), b"export bytes")
     assert batch.outcome is Outcome.ACCEPTED_POISONED

{chtypes-2.0.0.dev7 → chtypes-2.0.0.dev9}/tests/api/test_input.py

@@ -64,3 +64,41 @@ def test_text_inputs_are_utf8_and_bodies_never_accept_text() -> None:
     assert body_bytes(memoryview(b"ab")) == b"ab"
     with pytest.raises(TypeError):
         body_bytes("text")  # type: ignore[arg-type]
+
+
+_LONE = "\udcff"  # what surrogateescape makes of the byte 0xff
+
+
+@pytest.mark.parametrize(
+    ("call", "words"),
+    [
+        (lambda: settings_json({"k": _LONE}), ("settings", 'the value for key "k"')),
+        (lambda: settings_json({"k" + _LONE: "v"}), ("settings", 'the key "k\\udcff"')),
+        (lambda: settings_json({"k": _LONE}, "UTC"), ("settings", 'the value for key "k"')),
+        (lambda: settings_json(None, _LONE), ("the zone",)),
+        (
+            lambda: string_map_json({"p": _LONE}, "query parameters"),
+            ("query parameters", 'the value for key "p"'),
+        ),
+        (
+            lambda: string_map_json({_LONE: "v"}, "query parameters"),
+            ("query parameters", 'the key "\\udcff"'),
+        ),
+    ],
+)
+def test_a_non_utf8_key_or_value_is_a_usage_error_naming_the_input_and_the_key(call, words) -> None:
+    with pytest.raises(UsageError) as info:
+        call()
+    message = info.value.message.decode("utf-8")
+    for word in (*words, "not valid UTF-8", "unhex('<hex>')"):
+        assert word in message
+    assert info.value.status == 3 and info.value.ch_code == 0
+    assert message.isascii()  # the key is escaped, never raw
+
+
+def test_valid_non_ascii_serializes_byte_for_byte_as_before() -> None:
+    # Measured before the change on the same inputs: json.dumps, ASCII escapes.
+    assert string_map_json({"p": "é日本"}, "query parameters") == b'{"p":"\\u00e9\\u65e5\\u672c"}'
+    assert settings_json({"é": "\ufffd"}, "Asia/Tokyo") == (
+        b'{"\\u00e9":"\\ufffd","session_timezone":"Asia/Tokyo"}'
+    )

{chtypes-2.0.0.dev7 → chtypes-2.0.0.dev9}/uv.lock

@@ -74,7 +74,7 @@ wheels = [
 
 [[package]]
 name = "chtypes"
-version = "2.0.0.dev7"
+version = "2.0.0.dev9"
 source = { editable = "." }
 dependencies = [
     { name = "backports-zstd", marker = "python_full_version < '3.14'" },