chtypes 0.5.2__tar.gz → 1.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {chtypes-0.5.2 → chtypes-1.0.2}/CHANGELOG.md +40 -0
- {chtypes-0.5.2 → chtypes-1.0.2}/PKG-INFO +30 -24
- {chtypes-0.5.2 → chtypes-1.0.2}/README.md +28 -23
- {chtypes-0.5.2 → chtypes-1.0.2}/pyproject.toml +19 -10
- chtypes-1.0.2/src/chtypes/__init__.py +187 -0
- chtypes-1.0.2/src/chtypes/__main__.py +287 -0
- chtypes-1.0.2/src/chtypes/_abi1/__init__.py +17 -0
- chtypes-1.0.2/src/chtypes/_abi1/_decls.py +1097 -0
- chtypes-1.0.2/src/chtypes/_abi1/_errmap.py +70 -0
- chtypes-1.0.2/src/chtypes/_abi1/_errors.py +31 -0
- chtypes-1.0.2/src/chtypes/_abi1/_loader.py +302 -0
- chtypes-1.0.2/src/chtypes/_abi1/_vocab.py +254 -0
- chtypes-1.0.2/src/chtypes/_decode.py +559 -0
- chtypes-1.0.2/src/chtypes/_guard.py +64 -0
- chtypes-1.0.2/src/chtypes/_input.py +109 -0
- chtypes-1.0.2/src/chtypes/_ocifetch/__init__.py +38 -0
- chtypes-1.0.2/src/chtypes/_ocifetch/_constants.py +123 -0
- chtypes-1.0.2/src/chtypes/_ocifetch/_dsse.py +231 -0
- chtypes-1.0.2/src/chtypes/_ocifetch/_ensure.py +1441 -0
- chtypes-1.0.2/src/chtypes/_ocifetch/_errors.py +162 -0
- chtypes-1.0.2/src/chtypes/_ocifetch/_goldens.py +105 -0
- chtypes-1.0.2/src/chtypes/_ocifetch/_http.py +576 -0
- chtypes-1.0.2/src/chtypes/_ocifetch/_layout.py +396 -0
- chtypes-1.0.2/src/chtypes/_ocifetch/_lock.py +148 -0
- chtypes-1.0.2/src/chtypes/_ocifetch/_oci.py +342 -0
- chtypes-1.0.2/src/chtypes/_ocifetch/_referrers.py +137 -0
- chtypes-1.0.2/src/chtypes/_ocifetch/_unpack.py +132 -0
- chtypes-1.0.2/src/chtypes/_setup.py +113 -0
- chtypes-1.0.2/src/chtypes/errors.py +255 -0
- chtypes-1.0.2/src/chtypes/library.py +394 -0
- chtypes-1.0.2/src/chtypes/registry.py +200 -0
- chtypes-1.0.2/src/chtypes/results.py +287 -0
- chtypes-1.0.2/tests/abi1/__init__.py +9 -0
- chtypes-1.0.2/tests/abi1/_stub_tree.py +133 -0
- chtypes-1.0.2/tests/abi1/conftest.py +247 -0
- chtypes-1.0.2/tests/abi1/test_conformance.py +194 -0
- chtypes-1.0.2/tests/abi1/test_goldens_runner_stub.py +85 -0
- chtypes-1.0.2/tests/abi1/test_loader.py +97 -0
- chtypes-1.0.2/tests/abi1/test_public_library.py +322 -0
- chtypes-1.0.2/tests/abi1/test_public_open.py +180 -0
- chtypes-1.0.2/tests/abi1/test_public_registry.py +168 -0
- chtypes-1.0.2/tests/abi1/test_public_setup_cases.py +93 -0
- chtypes-1.0.2/tests/api/__init__.py +0 -0
- chtypes-1.0.2/tests/api/test_decode.py +299 -0
- chtypes-1.0.2/tests/api/test_input.py +66 -0
- chtypes-1.0.2/tests/api/test_vocab.py +90 -0
- chtypes-1.0.2/tests/cli/__init__.py +0 -0
- chtypes-1.0.2/tests/cli/test_main.py +193 -0
- chtypes-1.0.2/tests/conftest.py +6 -0
- chtypes-1.0.2/tests/goldens_v1/__init__.py +0 -0
- chtypes-1.0.2/tests/goldens_v1/data/stub-goldens.json +148 -0
- chtypes-1.0.2/tests/goldens_v1/test_runner.py +340 -0
- chtypes-1.0.2/tests/ocifetch/__init__.py +4 -0
- chtypes-1.0.2/tests/ocifetch/_bundle_support.py +88 -0
- chtypes-1.0.2/tests/ocifetch/_registry_support.py +218 -0
- chtypes-1.0.2/tests/ocifetch/_sign_support.py +72 -0
- chtypes-1.0.2/tests/ocifetch/test_conformance.py +625 -0
- chtypes-1.0.2/tests/ocifetch/test_dsse.py +193 -0
- chtypes-1.0.2/tests/ocifetch/test_ensure.py +295 -0
- chtypes-1.0.2/tests/ocifetch/test_goldens.py +82 -0
- chtypes-1.0.2/tests/ocifetch/test_http.py +570 -0
- chtypes-1.0.2/tests/ocifetch/test_layout.py +218 -0
- chtypes-1.0.2/tests/ocifetch/test_lock.py +107 -0
- chtypes-1.0.2/tests/ocifetch/test_oci.py +212 -0
- chtypes-1.0.2/tests/ocifetch/test_unpack.py +176 -0
- {chtypes-0.5.2 → chtypes-1.0.2}/tests/test_ed25519.py +4 -4
- chtypes-1.0.2/tests/test_examples_tour.py +40 -0
- chtypes-1.0.2/tests/test_fetch_options.py +31 -0
- chtypes-1.0.2/uv.lock +182 -0
- chtypes-0.5.2/src/chtypes/__init__.py +0 -223
- chtypes-0.5.2/src/chtypes/__main__.py +0 -300
- chtypes-0.5.2/src/chtypes/_document.py +0 -345
- chtypes-0.5.2/src/chtypes/_error_codes.py +0 -153
- chtypes-0.5.2/src/chtypes/_manifest.py +0 -165
- chtypes-0.5.2/src/chtypes/_native.py +0 -880
- chtypes-0.5.2/src/chtypes/_rawjson.py +0 -292
- chtypes-0.5.2/src/chtypes/discover.py +0 -263
- chtypes-0.5.2/src/chtypes/errors.py +0 -329
- chtypes-0.5.2/src/chtypes/fetch.py +0 -1844
- chtypes-0.5.2/src/chtypes/registry.py +0 -1853
- chtypes-0.5.2/src/chtypes/results.py +0 -660
- chtypes-0.5.2/src/chtypes/transform.py +0 -554
- chtypes-0.5.2/tests/conftest.py +0 -140
- chtypes-0.5.2/tests/test_abi_revision.py +0 -139
- chtypes-0.5.2/tests/test_cli.py +0 -300
- chtypes-0.5.2/tests/test_csv_reader.py +0 -445
- chtypes-0.5.2/tests/test_discover_reconstruct.py +0 -91
- chtypes-0.5.2/tests/test_error_codes.py +0 -250
- chtypes-0.5.2/tests/test_errors.py +0 -70
- chtypes-0.5.2/tests/test_exact_patch_resolution.py +0 -613
- chtypes-0.5.2/tests/test_fetch.py +0 -1085
- chtypes-0.5.2/tests/test_fetch_abi_revision.py +0 -248
- chtypes-0.5.2/tests/test_fetch_sh.py +0 -508
- chtypes-0.5.2/tests/test_formats_withnames.py +0 -458
- chtypes-0.5.2/tests/test_golden.py +0 -262
- chtypes-0.5.2/tests/test_image_identity.py +0 -115
- chtypes-0.5.2/tests/test_lazy.py +0 -251
- chtypes-0.5.2/tests/test_parity.py +0 -441
- chtypes-0.5.2/tests/test_partition_key.py +0 -169
- chtypes-0.5.2/tests/test_quoting_boundaries.py +0 -303
- chtypes-0.5.2/tests/test_rawjson.py +0 -168
- chtypes-0.5.2/tests/test_registry.py +0 -686
- chtypes-0.5.2/tests/test_results_rules.py +0 -150
- chtypes-0.5.2/tests/test_rows_export_with.py +0 -416
- chtypes-0.5.2/tests/test_transform.py +0 -231
- chtypes-0.5.2/uv.lock +0 -108
- {chtypes-0.5.2 → chtypes-1.0.2}/.gitignore +0 -0
- {chtypes-0.5.2 → chtypes-1.0.2}/LICENSE +0 -0
- {chtypes-0.5.2 → chtypes-1.0.2}/src/chtypes/_ed25519.py +0 -0
- {chtypes-0.5.2 → chtypes-1.0.2}/src/chtypes/py.typed +0 -0
|
@@ -6,6 +6,46 @@ The four bindings in this repository are released together and give one answer,
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [1.0.2] — 2026-10-06
|
|
10
|
+
|
|
11
|
+
There is no 1.0.1 of this binding: 1.0.1 was a Go-only fix to that module's metadata. From 1.0.2 the four bindings release together again.
|
|
12
|
+
|
|
13
|
+
### Fixed
|
|
14
|
+
|
|
15
|
+
- `setup` now latches only once an image completes load step 7 (#458). A zone the library refuses, such as `setup(timezone="Not/AZone")`, used to fix the process setup at the first open: every later open failed with ClickHouse's code 36, and a corrected `setup` was a `UsageError` until the process restarted. Now, if step 7 (`chs_initialize`, then `chs_set_defaults` when there are defaults) fails before any image has completed it, the setup record is cleared. That open still raises the library's own error, a corrected `setup` is accepted, and the next open runs step 7 again with it, on the same image too: a failed step 7 is never remembered (`open_unverified` and `Registry.for_version` alike). Once an image has completed step 7, a different setup is still a `UsageError`. No name or signature changes. All four bindings run the same case, `tests/fixtures/abi-v1/setup-cases.json`, and the rule is `docs/reference/bindings-v1.md` section 6.
|
|
16
|
+
|
|
17
|
+
## [1.0.0] — 2026-10-04
|
|
18
|
+
|
|
19
|
+
The Python binding's first v1 release, and a deliberate break from 0.x. The default branch switches to v1 shortly after release. The sections below are the entries written as each part landed; this is what they add up to.
|
|
20
|
+
|
|
21
|
+
- **A new ABI.** The binding speaks the generated ABI v1 layer, identified by an ABI fingerprint in `include/chtypes.h` rather than a revision number, so every FFI declaration is generated from `spec/abi-v1/abi.json`, not written by hand. The ABI stays provisional until the maintainer confirms it.
|
|
22
|
+
- **Fetch over OCI, with signed statements.** Libraries are fetched from an OCI registry and verified against a signed statement before they are loaded; the default trust is the release key only (`docs/guides/fetch-v1.md`).
|
|
23
|
+
- **The public API is `docs/reference/bindings-v1.md`**, the same four-binding contract with each language's own spelling.
|
|
24
|
+
- **The CLI** is `python -m chtypes` and the `chtypes` console script: `fetch`, `verify`, `list` and `where`.
|
|
25
|
+
- **The v0 surface is removed**; the list, and the reason for each removal, is `docs/reference/bindings-v1.md` section 7.
|
|
26
|
+
- **Known gaps** are listed in [`docs/limitations.md`](../docs/limitations.md#known-gaps-in-10), and support for a line this release does not mention is "support unknown", never "unsupported".
|
|
27
|
+
|
|
28
|
+
### Changed (v1, breaking)
|
|
29
|
+
|
|
30
|
+
- **The public API is the one in `docs/reference/bindings-v1.md`, over the generated ABI v1 layer and the OCI fetch layer.** `setup(timezone=, defaults=)` is the one process setup; `Registry(*, fetch=, autofetch=, preload=)` is keyword-only and opens nothing at construction; `Registry.for_version`, `installed()` and `libraries()` replace the directory-scanning registry; `Library.compile_table` (exactly one `CREATE TABLE`) replaces `compile_ddl`; `open_unverified(path, allow=)` replaces `Load`.
|
|
31
|
+
- **Names, SQL, messages and renderings are `bytes`.** Result types are one-to-one decodes of the library's documents (`RowResult.columns` is every entry, `values` the stored subset by the description's own `is_stored` fact), and every vocabulary (`Format`, `Outcome`, `Verdict`, `Reason`, `Source`, `DefaultKind`, `DocFlags`) is generated from the ABI description.
|
|
32
|
+
- **One error family.** `CallError` (under `ChtypesError`) carries `status`, `ch_code`, `ch_name`, `message` and `column`; `SchemaError`, `UnsupportedError`, `UsageError` and `InternalError` are peers. Fetch and loader errors are `ArtifactError` subclasses, one class per code.
|
|
33
|
+
- **Each `Schema`, `Filter` and `Block` has a close guard**: `close` waits for calls already inside the object, and a call after close is a `UsageError` raised before any C call. No lock is taken around a call.
|
|
34
|
+
|
|
35
|
+
- **`python -m chtypes` and the `chtypes` console script are back, over the v1 fetch layer**: `fetch <spelling>... | --all [--lock FILE] [--frozen] [--offline]`, `verify`, `list [--offline]` and `where`. Exit statuses are the `errors` table of `spec/fetch-v1/constants.json`, usage errors exit 2.
|
|
36
|
+
- **`FetchOptions.system_dirs`** names the read-only directories searched after the cache (`None` keeps the default list, an empty sequence searches none), and `FetchOptions.to_options` is private. `CHTYPES_TRUSTED_KEYS` (comma-separated hex public keys) replaces the default trust list when set, as `docs/guides/fetch-v1.md` §4 says.
|
|
37
|
+
|
|
38
|
+
### Removed (v1)
|
|
39
|
+
|
|
40
|
+
- `compile_ddl`, the engine, TTL and partition-key setters, `encode_settings`, the transform classifier, discovery reconstruction, the hand JSON readers, `quote_bare_denormals`, the hand error-code table parser, the manifest and revision machinery, the v0 fetch module and its `python -m chtypes` command line, `ABI_REVISION`, `COMPILE_DECLARED`, `Registry.shutdown`, `Library.close` and the `Registry` context manager.
|
|
41
|
+
|
|
42
|
+
### Fixed
|
|
43
|
+
|
|
44
|
+
- **The crash-guard refuse-list is now sourced identically to the other three bindings, and an artifact with neither source refuses rather than loading unguarded.** This binding already fell back to the manifest's `unsafe_families` field when `unsafe_families.txt` was absent, but a manifest predating the field (or one that simply omitted it) was indistinguishable from a present, empty field, so `chs_init` silently ran with no refuse-list. `Manifest.unsafe_families` is now `None` when the field is absent rather than defaulting to `""`, and a directory with neither the file nor the field is refused, naming both, instead of loading unguarded (`docs/reference/artifact.md` step 9).
|
|
45
|
+
- **`ensure`/`fetch` now retry a transient HTTP 5xx, 408 or 429 from the artifacts host, and a connection-level failure (refused, reset, timed out, DNS), within the existing retry budget and schedule** (`docs/guides/fetch.md` §3a: 5 attempts, delays doubling from 4s) — previously `_Source._open`/`download` ran their own separate, shorter retry (3 attempts, incrementing by whole seconds) before giving up, independent of the publish-window budget, and a 408 was never retried at all. A `Retry-After` on a 503 or 429 is honored, in both the delta-seconds and HTTP-date forms, capped so it never makes the total wait exceed the existing budget — one that does not fit fails at once, naming the requested delay. A 404 or 410, and a tarball hash or size mismatch, are still decided on the first attempt, never retried. `SourceUnreachableError` gains `.retryable`/`.retry_after` attributes (closes #365).
|
|
46
|
+
- **`fetch`/`ensure` now send `CHTYPES_DOWNLOAD_TOKEN` as a bearer token** to an HTTP source, matching the other three bindings and `scripts/fetch.sh` — found missing during the #365 audit.
|
|
47
|
+
- **A `Filter` or `Block` held in a reference cycle with its `Schema` could be freed AFTER the schema**, a use-after-free the C contract forbids (handle lifetime, §Filters/§Blocks). `Schema` tracked open children through a `weakref.WeakSet`, and CPython clears a `WeakSet`'s membership before running `Schema.__del__` when the schema and a child die together in one `gc.collect()`, so the safety net saw the set already empty and freed the schema handle first. `Schema`/`Filter`/`Block` now coordinate through a small native-half object per handle — the shape TypeScript has used since #334 — and the safety net is a `weakref.finalize` callback bound only to that native half, never `__del__`, so it holds no reference back to the Schema/Filter/Block and runs correctly regardless of a reference cycle or which object's finalizer the collector happens to run first (#381).
|
|
48
|
+
|
|
9
49
|
## [0.5.2] — 2026-10-01
|
|
10
50
|
|
|
11
51
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: chtypes
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 1.0.2
|
|
4
4
|
Summary: Python binding for the chtypes artifacts: ClickHouse's own type system, per version, behind the frozen chs_* C ABI
|
|
5
5
|
Project-URL: Homepage, https://github.com/wave-rf/chtypes
|
|
6
6
|
Project-URL: Repository, https://github.com/wave-rf/chtypes
|
|
@@ -14,54 +14,56 @@ Classifier: License :: OSI Approved :: Apache Software License
|
|
|
14
14
|
Classifier: Programming Language :: Python :: 3
|
|
15
15
|
Classifier: Topic :: Database
|
|
16
16
|
Requires-Python: >=3.11
|
|
17
|
+
Requires-Dist: backports-zstd>=1.7; python_version < '3.14'
|
|
17
18
|
Description-Content-Type: text/markdown
|
|
18
19
|
|
|
19
20
|
# chtypes — Python SDK
|
|
20
21
|
|
|
21
22
|
**If this row were inserted into this table on this ClickHouse version, what would happen?** chtypes answers with ClickHouse's own code: the real C++ type machinery, vendored per release into a native library behind the frozen `chs_*` C ABI and reached here through stdlib `ctypes`. Nothing semantic is reimplemented, so _"what does ClickHouse do with `256` into a `UInt8`?"_ is answered by ClickHouse rather than by a model of it. One peer binding among `{go, python, ts, rust}` — no language is privileged, and all four give one answer.
|
|
22
23
|
|
|
23
|
-
Pure
|
|
24
|
+
Pure ctypes: no compiler, no build step. The only dependency is a zstd decompressor for the artifact layer, which is the standard library's own from Python 3.14 and the `backports.zstd` package before that.
|
|
24
25
|
|
|
25
26
|
## Install
|
|
26
27
|
|
|
27
|
-
Two things: this package, and at least one **artifact** — the per-version native library it `dlopen`s at runtime.
|
|
28
|
+
Two things: this package, and at least one **artifact** — the per-version native library it `dlopen`s at runtime. The registry fetches the artifact on first use when autofetch is on (`Registry(autofetch=True)` or `CHTYPES_AUTOFETCH=1`), after verifying its signature and every byte against the signed statement; with autofetch off it answers only from what is already installed.
|
|
28
29
|
|
|
29
30
|
```sh
|
|
30
31
|
uv add chtypes # or: pip install chtypes
|
|
31
|
-
python -m chtypes fetch 26.8
|
|
32
32
|
```
|
|
33
33
|
|
|
34
|
-
The fetch lands in `~/.cache/chtypes/artifacts/abi<R>/<os>-<arch>/26.8/` — the per-user cache every chtypes binding reads by default, `<R>` the ABI revision this SDK speaks (fetch installs only artifacts built at it) — after checking an ed25519 signature over the release and the sha256 of every byte. `$CHTYPES_REGISTRY` overrides it. The ed25519 verifier is pure stdlib too.
|
|
35
|
-
|
|
36
34
|
## Quickstart
|
|
37
35
|
|
|
38
36
|
```python
|
|
37
|
+
import chtypes
|
|
39
38
|
from chtypes import Format, Registry
|
|
40
39
|
|
|
41
|
-
|
|
42
|
-
|
|
40
|
+
chtypes.setup(timezone="UTC") # once, before the first open (optional)
|
|
41
|
+
registry = Registry(autofetch=True)
|
|
42
|
+
library = registry.for_version("26.8") # a release line or an exact version
|
|
43
43
|
|
|
44
|
-
|
|
44
|
+
ddl = b"CREATE TABLE t (x UInt8, ts DateTime DEFAULT now()) ENGINE = Memory"
|
|
45
|
+
with library.compile_table(ddl) as schema:
|
|
45
46
|
batch = schema.rows(Format.JSON_EACH_ROW, b'{"x":256}\n')
|
|
46
47
|
|
|
47
48
|
row = batch.rows[0]
|
|
48
|
-
print(batch.outcome)
|
|
49
|
-
print(row.
|
|
50
|
-
print(row.transformed[0].reason)
|
|
51
|
-
print(row.substituted) # ts: send it explicitly, or preview != stored
|
|
49
|
+
print(batch.outcome) # accepted
|
|
50
|
+
print(row.values[0].text) # b'0' what would actually be stored
|
|
51
|
+
print(row.transformed[0].reason) # overflow_wrap which is the product
|
|
52
52
|
```
|
|
53
53
|
|
|
54
|
-
The row is **accepted** and `256` is silently stored as `0`. That report — `transformed` — is
|
|
54
|
+
The row is **accepted** and `256` is silently stored as `0`. That report — `transformed` — is why chtypes exists. Names, SQL, messages and renderings come back as `bytes`, never decoded for you: a column name that is not valid UTF-8 round-trips exactly.
|
|
55
55
|
|
|
56
|
-
|
|
56
|
+
A column whose DEFAULT calls a random or UUID generator is drawn by the library (`Source.DEFAULT_GENERATED`). Ask `rows` for an `export` and insert its `payload`, never the original body, or the server draws a different value than the one previewed.
|
|
57
57
|
|
|
58
|
-
##
|
|
58
|
+
## Errors
|
|
59
59
|
|
|
60
|
-
A bad **row** is a verdict, not an exception: `outcome` becomes `Outcome.REJECTED` with ClickHouse's own code. Exceptions are for
|
|
60
|
+
A bad **row** is a verdict, not an exception: `outcome` becomes `Outcome.REJECTED` with ClickHouse's own code. Exceptions are for the call and for the machinery. Every call error carries `status`, `ch_code`, `ch_name`, `message` (bytes) and `column` (bytes).
|
|
61
61
|
|
|
62
|
-
- `SchemaError` — the server itself would refuse this, and `.
|
|
62
|
+
- `SchemaError` — the server itself would refuse this, and `.ch_code` is a real ClickHouse code.
|
|
63
63
|
- `UnsupportedError` — this build declines to answer, and a real server might well have accepted. **Fall back to the server**; never tell a user they are wrong on the strength of a decline.
|
|
64
|
-
-
|
|
64
|
+
- `UsageError` — a misuse: a closed object, a conflicting `setup`, a zone given twice. `InternalError` — a library bug.
|
|
65
|
+
- **`UnsupportedError` is a peer of `SchemaError`, not a subclass**, so `except SchemaError` never catches a decline. Catch `CallError` for all four, deliberately.
|
|
66
|
+
- `ArtifactError` and its subclasses cover fetching and loading: `ArtifactCorruptError`, `ArtifactIncompatibleError`, `ArtifactMissingError` and the rest, one class per code.
|
|
65
67
|
|
|
66
68
|
## Documentation
|
|
67
69
|
|
|
@@ -76,15 +78,19 @@ A bad **row** is a verdict, not an exception: `outcome` becomes `Outcome.REJECTE
|
|
|
76
78
|
| [Filters](https://github.com/wave-rf/chtypes/blob/main/docs/guides/filters.md) · [Multi-version](https://github.com/wave-rf/chtypes/blob/main/docs/guides/multi-version.md) | boolean expressions over rows; several ClickHouse versions in one process |
|
|
77
79
|
| [Support matrix](https://github.com/wave-rf/chtypes/blob/main/docs/support.md) · [Limitations](https://github.com/wave-rf/chtypes/blob/main/docs/limitations.md) | what works where; what chtypes declines to answer |
|
|
78
80
|
|
|
79
|
-
##
|
|
81
|
+
## Fetching from the command line
|
|
82
|
+
|
|
83
|
+
`chtypes fetch 26.8` (or `python -m chtypes fetch 26.8`) resolves, verifies and installs a build into the per-user cache; `chtypes verify`, `chtypes list` and `chtypes where` complete the four commands. `--lock FILE` records what was installed and `--frozen` fetches only what the lock pins. See the [Python API reference](https://github.com/wave-rf/chtypes/blob/main/docs/reference/python.md#the-command).
|
|
84
|
+
|
|
85
|
+
## Things specific to this binding
|
|
80
86
|
|
|
81
|
-
**A settings value
|
|
87
|
+
**A settings value is a string, and only a string.** `settings={"flatten_nested": "0"}`, never `0`: a non-string value is a `TypeError`, and nothing rewrites one for you.
|
|
82
88
|
|
|
83
|
-
**`
|
|
89
|
+
**`ctypes` releases the GIL for the whole of a foreign call**, so calls on compiled handles run in parallel across threads. The package takes no lock around a call; it keeps a close guard per handle and one setup guard.
|
|
84
90
|
|
|
85
|
-
|
|
91
|
+
**The INSERT column list is a keyword-only `columns`** on `schema.row`, `schema.rows` and `schema.parse_block`: `schema.row(Format.JSON_EACH_ROW, body, columns=["id", "e"])`. `None` or an empty sequence is the no-list shape; a list makes the data supply exactly those columns, with a listed `EPHEMERAL` value read and in scope for the DEFAULTs that reference it but never stored.
|
|
86
92
|
|
|
87
|
-
**
|
|
93
|
+
**Known gaps in 1.0** are listed, with workarounds, in [`docs/limitations.md`](https://github.com/wave-rf/chtypes/blob/main/docs/limitations.md#known-gaps-in-10).
|
|
88
94
|
|
|
89
95
|
## Tests
|
|
90
96
|
|
|
@@ -2,48 +2,49 @@
|
|
|
2
2
|
|
|
3
3
|
**If this row were inserted into this table on this ClickHouse version, what would happen?** chtypes answers with ClickHouse's own code: the real C++ type machinery, vendored per release into a native library behind the frozen `chs_*` C ABI and reached here through stdlib `ctypes`. Nothing semantic is reimplemented, so _"what does ClickHouse do with `256` into a `UInt8`?"_ is answered by ClickHouse rather than by a model of it. One peer binding among `{go, python, ts, rust}` — no language is privileged, and all four give one answer.
|
|
4
4
|
|
|
5
|
-
Pure
|
|
5
|
+
Pure ctypes: no compiler, no build step. The only dependency is a zstd decompressor for the artifact layer, which is the standard library's own from Python 3.14 and the `backports.zstd` package before that.
|
|
6
6
|
|
|
7
7
|
## Install
|
|
8
8
|
|
|
9
|
-
Two things: this package, and at least one **artifact** — the per-version native library it `dlopen`s at runtime.
|
|
9
|
+
Two things: this package, and at least one **artifact** — the per-version native library it `dlopen`s at runtime. The registry fetches the artifact on first use when autofetch is on (`Registry(autofetch=True)` or `CHTYPES_AUTOFETCH=1`), after verifying its signature and every byte against the signed statement; with autofetch off it answers only from what is already installed.
|
|
10
10
|
|
|
11
11
|
```sh
|
|
12
12
|
uv add chtypes # or: pip install chtypes
|
|
13
|
-
python -m chtypes fetch 26.8
|
|
14
13
|
```
|
|
15
14
|
|
|
16
|
-
The fetch lands in `~/.cache/chtypes/artifacts/abi<R>/<os>-<arch>/26.8/` — the per-user cache every chtypes binding reads by default, `<R>` the ABI revision this SDK speaks (fetch installs only artifacts built at it) — after checking an ed25519 signature over the release and the sha256 of every byte. `$CHTYPES_REGISTRY` overrides it. The ed25519 verifier is pure stdlib too.
|
|
17
|
-
|
|
18
15
|
## Quickstart
|
|
19
16
|
|
|
20
17
|
```python
|
|
18
|
+
import chtypes
|
|
21
19
|
from chtypes import Format, Registry
|
|
22
20
|
|
|
23
|
-
|
|
24
|
-
|
|
21
|
+
chtypes.setup(timezone="UTC") # once, before the first open (optional)
|
|
22
|
+
registry = Registry(autofetch=True)
|
|
23
|
+
library = registry.for_version("26.8") # a release line or an exact version
|
|
25
24
|
|
|
26
|
-
|
|
25
|
+
ddl = b"CREATE TABLE t (x UInt8, ts DateTime DEFAULT now()) ENGINE = Memory"
|
|
26
|
+
with library.compile_table(ddl) as schema:
|
|
27
27
|
batch = schema.rows(Format.JSON_EACH_ROW, b'{"x":256}\n')
|
|
28
28
|
|
|
29
29
|
row = batch.rows[0]
|
|
30
|
-
print(batch.outcome)
|
|
31
|
-
print(row.
|
|
32
|
-
print(row.transformed[0].reason)
|
|
33
|
-
print(row.substituted) # ts: send it explicitly, or preview != stored
|
|
30
|
+
print(batch.outcome) # accepted
|
|
31
|
+
print(row.values[0].text) # b'0' what would actually be stored
|
|
32
|
+
print(row.transformed[0].reason) # overflow_wrap which is the product
|
|
34
33
|
```
|
|
35
34
|
|
|
36
|
-
The row is **accepted** and `256` is silently stored as `0`. That report — `transformed` — is
|
|
35
|
+
The row is **accepted** and `256` is silently stored as `0`. That report — `transformed` — is why chtypes exists. Names, SQL, messages and renderings come back as `bytes`, never decoded for you: a column name that is not valid UTF-8 round-trips exactly.
|
|
37
36
|
|
|
38
|
-
|
|
37
|
+
A column whose DEFAULT calls a random or UUID generator is drawn by the library (`Source.DEFAULT_GENERATED`). Ask `rows` for an `export` and insert its `payload`, never the original body, or the server draws a different value than the one previewed.
|
|
39
38
|
|
|
40
|
-
##
|
|
39
|
+
## Errors
|
|
41
40
|
|
|
42
|
-
A bad **row** is a verdict, not an exception: `outcome` becomes `Outcome.REJECTED` with ClickHouse's own code. Exceptions are for
|
|
41
|
+
A bad **row** is a verdict, not an exception: `outcome` becomes `Outcome.REJECTED` with ClickHouse's own code. Exceptions are for the call and for the machinery. Every call error carries `status`, `ch_code`, `ch_name`, `message` (bytes) and `column` (bytes).
|
|
43
42
|
|
|
44
|
-
- `SchemaError` — the server itself would refuse this, and `.
|
|
43
|
+
- `SchemaError` — the server itself would refuse this, and `.ch_code` is a real ClickHouse code.
|
|
45
44
|
- `UnsupportedError` — this build declines to answer, and a real server might well have accepted. **Fall back to the server**; never tell a user they are wrong on the strength of a decline.
|
|
46
|
-
-
|
|
45
|
+
- `UsageError` — a misuse: a closed object, a conflicting `setup`, a zone given twice. `InternalError` — a library bug.
|
|
46
|
+
- **`UnsupportedError` is a peer of `SchemaError`, not a subclass**, so `except SchemaError` never catches a decline. Catch `CallError` for all four, deliberately.
|
|
47
|
+
- `ArtifactError` and its subclasses cover fetching and loading: `ArtifactCorruptError`, `ArtifactIncompatibleError`, `ArtifactMissingError` and the rest, one class per code.
|
|
47
48
|
|
|
48
49
|
## Documentation
|
|
49
50
|
|
|
@@ -58,15 +59,19 @@ A bad **row** is a verdict, not an exception: `outcome` becomes `Outcome.REJECTE
|
|
|
58
59
|
| [Filters](https://github.com/wave-rf/chtypes/blob/main/docs/guides/filters.md) · [Multi-version](https://github.com/wave-rf/chtypes/blob/main/docs/guides/multi-version.md) | boolean expressions over rows; several ClickHouse versions in one process |
|
|
59
60
|
| [Support matrix](https://github.com/wave-rf/chtypes/blob/main/docs/support.md) · [Limitations](https://github.com/wave-rf/chtypes/blob/main/docs/limitations.md) | what works where; what chtypes declines to answer |
|
|
60
61
|
|
|
61
|
-
##
|
|
62
|
+
## Fetching from the command line
|
|
63
|
+
|
|
64
|
+
`chtypes fetch 26.8` (or `python -m chtypes fetch 26.8`) resolves, verifies and installs a build into the per-user cache; `chtypes verify`, `chtypes list` and `chtypes where` complete the four commands. `--lock FILE` records what was installed and `--frozen` fetches only what the lock pins. See the [Python API reference](https://github.com/wave-rf/chtypes/blob/main/docs/reference/python.md#the-command).
|
|
65
|
+
|
|
66
|
+
## Things specific to this binding
|
|
62
67
|
|
|
63
|
-
**A settings value
|
|
68
|
+
**A settings value is a string, and only a string.** `settings={"flatten_nested": "0"}`, never `0`: a non-string value is a `TypeError`, and nothing rewrites one for you.
|
|
64
69
|
|
|
65
|
-
**`
|
|
70
|
+
**`ctypes` releases the GIL for the whole of a foreign call**, so calls on compiled handles run in parallel across threads. The package takes no lock around a call; it keeps a close guard per handle and one setup guard.
|
|
66
71
|
|
|
67
|
-
|
|
72
|
+
**The INSERT column list is a keyword-only `columns`** on `schema.row`, `schema.rows` and `schema.parse_block`: `schema.row(Format.JSON_EACH_ROW, body, columns=["id", "e"])`. `None` or an empty sequence is the no-list shape; a list makes the data supply exactly those columns, with a listed `EPHEMERAL` value read and in scope for the DEFAULTs that reference it but never stored.
|
|
68
73
|
|
|
69
|
-
**
|
|
74
|
+
**Known gaps in 1.0** are listed, with workarounds, in [`docs/limitations.md`](https://github.com/wave-rf/chtypes/blob/main/docs/limitations.md#known-gaps-in-10).
|
|
70
75
|
|
|
71
76
|
## Tests
|
|
72
77
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "chtypes"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "1.0.2"
|
|
4
4
|
description = "Python binding for the chtypes artifacts: ClickHouse's own type system, per version, behind the frozen chs_* C ABI"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "Apache-2.0"
|
|
@@ -13,15 +13,15 @@ classifiers = [
|
|
|
13
13
|
"Programming Language :: Python :: 3",
|
|
14
14
|
"Topic :: Database",
|
|
15
15
|
]
|
|
16
|
-
# Zero runtime dependencies
|
|
17
|
-
# artifact's C ABI, so installing it needs no compiler, no
|
|
18
|
-
# no wheel per Python version
|
|
19
|
-
#
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
#
|
|
23
|
-
|
|
24
|
-
|
|
16
|
+
# Zero runtime dependencies was the rule through v0: the binding is stdlib
|
|
17
|
+
# `ctypes` over the artifact's C ABI, so installing it needs no compiler, no
|
|
18
|
+
# cffi build step and no wheel per Python version. v1's artifacts are zstd
|
|
19
|
+
# (docs/guides/fetch-v1.md "Bytes"), and CPython's own `compression.zstd`
|
|
20
|
+
# only exists from 3.14 (PEP 784); this binding's floor is 3.11, so 3.11-3.13
|
|
21
|
+
# need a backport of the same stdlib-shaped API. 3.14+ needs nothing extra —
|
|
22
|
+
# the marker below excludes it, so a 3.14 install still carries zero
|
|
23
|
+
# dependencies. See README "Why ctypes".
|
|
24
|
+
dependencies = ["backports.zstd>=1.7; python_version < '3.14'"]
|
|
25
25
|
|
|
26
26
|
[project.urls]
|
|
27
27
|
Homepage = "https://github.com/wave-rf/chtypes"
|
|
@@ -29,6 +29,9 @@ Repository = "https://github.com/wave-rf/chtypes"
|
|
|
29
29
|
Documentation = "https://github.com/wave-rf/chtypes/blob/main/python/README.md"
|
|
30
30
|
Changelog = "https://github.com/wave-rf/chtypes/releases"
|
|
31
31
|
|
|
32
|
+
[project.scripts]
|
|
33
|
+
chtypes = "chtypes.__main__:main"
|
|
34
|
+
|
|
32
35
|
[dependency-groups]
|
|
33
36
|
dev = [
|
|
34
37
|
"pytest>=8.0",
|
|
@@ -39,6 +42,12 @@ dev = [
|
|
|
39
42
|
requires = ["hatchling"]
|
|
40
43
|
build-backend = "hatchling.build"
|
|
41
44
|
|
|
45
|
+
[tool.hatch.build]
|
|
46
|
+
# Two v0 verification modules stay in the tree, unimported, so the security
|
|
47
|
+
# carve-out in scripts/policy-merge-check.py keeps covering them until their
|
|
48
|
+
# deletion is decided; they are not part of v1 and never ship.
|
|
49
|
+
exclude = ["/src/chtypes/fetch.py", "/src/chtypes/_manifest.py"]
|
|
50
|
+
|
|
42
51
|
[tool.hatch.build.targets.wheel]
|
|
43
52
|
packages = ["src/chtypes"]
|
|
44
53
|
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""chtypes: ClickHouse's own type system, per version, from Python.
|
|
2
|
+
|
|
3
|
+
One question, exactly: *if this row were inserted into this ClickHouse table on
|
|
4
|
+
this ClickHouse version, what would happen?* The answer comes from ClickHouse's
|
|
5
|
+
real C++ machinery, vendored per release into a shared library behind the frozen
|
|
6
|
+
C ABI. Nothing here reimplements a coercion rule, which is why the answers are
|
|
7
|
+
exact by construction rather than approximately right.
|
|
8
|
+
|
|
9
|
+
import chtypes
|
|
10
|
+
from chtypes import Format, Registry
|
|
11
|
+
|
|
12
|
+
chtypes.setup(timezone="UTC") # once, before the first open
|
|
13
|
+
registry = Registry(autofetch=True)
|
|
14
|
+
library = registry.for_version("26.8") # a release line or an exact version
|
|
15
|
+
ddl = b"CREATE TABLE t (ts DateTime, seq UInt8) ENGINE = Memory"
|
|
16
|
+
with library.compile_table(ddl) as schema:
|
|
17
|
+
batch = schema.rows(Format.JSON_EACH_ROW, b'{"ts":"2026-01-15 10:30:00","seq":256}\\n')
|
|
18
|
+
|
|
19
|
+
batch.outcome # Outcome.ACCEPTED: the insert would succeed
|
|
20
|
+
batch.transformed # the changes ClickHouse makes and reports as success
|
|
21
|
+
|
|
22
|
+
The specification is docs/reference/bindings-v1.md. Names, SQL, messages and
|
|
23
|
+
renderings come back as `bytes`: a name that is not valid UTF-8 round-trips
|
|
24
|
+
exactly, and nothing here decodes one for you.
|
|
25
|
+
|
|
26
|
+
Three things a caller must not skip:
|
|
27
|
+
|
|
28
|
+
* **`transformed` is the product.** ClickHouse returns success for every one of
|
|
29
|
+
those changes. Reading `outcome` alone says a row was accepted and nothing
|
|
30
|
+
about the value the table will hold.
|
|
31
|
+
* **`Outcome.UNSUPPORTED` is not a rejection.** It means a real server might well
|
|
32
|
+
have accepted this and this build declines to answer.
|
|
33
|
+
* **Insert the library's output, not your input**, when a row carries a
|
|
34
|
+
`Source.DEFAULT_GENERATED` column: ask `rows` for an `export` and insert its
|
|
35
|
+
`payload`.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
from __future__ import annotations
|
|
39
|
+
|
|
40
|
+
from importlib.metadata import PackageNotFoundError as _PackageNotFound
|
|
41
|
+
from importlib.metadata import version as _package_version
|
|
42
|
+
|
|
43
|
+
from ._abi1._vocab import (
|
|
44
|
+
EXPORT_NONE,
|
|
45
|
+
DefaultKind,
|
|
46
|
+
DocFlags,
|
|
47
|
+
FilterOutcome,
|
|
48
|
+
Format,
|
|
49
|
+
Outcome,
|
|
50
|
+
Reason,
|
|
51
|
+
Source,
|
|
52
|
+
Status,
|
|
53
|
+
Verdict,
|
|
54
|
+
)
|
|
55
|
+
from ._input import BytesIn, Settings
|
|
56
|
+
from ._setup import setup
|
|
57
|
+
from .errors import (
|
|
58
|
+
CODE_ARTIFACT_CORRUPT,
|
|
59
|
+
CODE_ARTIFACT_INCOMPATIBLE,
|
|
60
|
+
CODE_ARTIFACT_MISSING,
|
|
61
|
+
CODE_ARTIFACT_PINNED,
|
|
62
|
+
CODE_ARTIFACT_UNPUBLISHED,
|
|
63
|
+
CODE_ARTIFACT_UNTRUSTED,
|
|
64
|
+
CODE_SOURCE_FORBIDDEN,
|
|
65
|
+
CODE_SOURCE_INCOMPATIBLE,
|
|
66
|
+
CODE_SOURCE_UNAUTHORIZED,
|
|
67
|
+
CODE_SOURCE_UNREACHABLE,
|
|
68
|
+
ArtifactCorruptError,
|
|
69
|
+
ArtifactError,
|
|
70
|
+
ArtifactIncompatibleError,
|
|
71
|
+
ArtifactMissingError,
|
|
72
|
+
ArtifactPinnedError,
|
|
73
|
+
ArtifactUnpublishedError,
|
|
74
|
+
ArtifactUntrustedError,
|
|
75
|
+
CallError,
|
|
76
|
+
ChtypesError,
|
|
77
|
+
InternalError,
|
|
78
|
+
SchemaError,
|
|
79
|
+
SourceForbiddenError,
|
|
80
|
+
SourceIncompatibleError,
|
|
81
|
+
SourceUnauthorizedError,
|
|
82
|
+
SourceUnreachableError,
|
|
83
|
+
UnsupportedError,
|
|
84
|
+
UsageError,
|
|
85
|
+
)
|
|
86
|
+
from .library import Block, Filter, Library, Schema, open_unverified
|
|
87
|
+
from .registry import FetchOptions, Registry, Resolved, TrustedKey
|
|
88
|
+
from .results import (
|
|
89
|
+
BatchResult,
|
|
90
|
+
BuildInfo,
|
|
91
|
+
Capabilities,
|
|
92
|
+
Column,
|
|
93
|
+
Computed,
|
|
94
|
+
DiscoveredColumn,
|
|
95
|
+
Discovery,
|
|
96
|
+
EngineCell,
|
|
97
|
+
ErrorCodeEntry,
|
|
98
|
+
ErrorCodeTable,
|
|
99
|
+
FilterResult,
|
|
100
|
+
FilterRowError,
|
|
101
|
+
Framing,
|
|
102
|
+
Header,
|
|
103
|
+
RowResult,
|
|
104
|
+
SchemaDescription,
|
|
105
|
+
Span,
|
|
106
|
+
Transform,
|
|
107
|
+
Value,
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
__all__ = [
|
|
111
|
+
"CODE_ARTIFACT_CORRUPT",
|
|
112
|
+
"CODE_ARTIFACT_INCOMPATIBLE",
|
|
113
|
+
"CODE_ARTIFACT_MISSING",
|
|
114
|
+
"CODE_ARTIFACT_PINNED",
|
|
115
|
+
"CODE_ARTIFACT_UNPUBLISHED",
|
|
116
|
+
"CODE_ARTIFACT_UNTRUSTED",
|
|
117
|
+
"CODE_SOURCE_FORBIDDEN",
|
|
118
|
+
"CODE_SOURCE_INCOMPATIBLE",
|
|
119
|
+
"CODE_SOURCE_UNAUTHORIZED",
|
|
120
|
+
"CODE_SOURCE_UNREACHABLE",
|
|
121
|
+
"EXPORT_NONE",
|
|
122
|
+
"ArtifactCorruptError",
|
|
123
|
+
"ArtifactError",
|
|
124
|
+
"ArtifactIncompatibleError",
|
|
125
|
+
"ArtifactMissingError",
|
|
126
|
+
"ArtifactPinnedError",
|
|
127
|
+
"ArtifactUnpublishedError",
|
|
128
|
+
"ArtifactUntrustedError",
|
|
129
|
+
"BatchResult",
|
|
130
|
+
"Block",
|
|
131
|
+
"BuildInfo",
|
|
132
|
+
"BytesIn",
|
|
133
|
+
"CallError",
|
|
134
|
+
"Capabilities",
|
|
135
|
+
"ChtypesError",
|
|
136
|
+
"Column",
|
|
137
|
+
"Computed",
|
|
138
|
+
"DefaultKind",
|
|
139
|
+
"DiscoveredColumn",
|
|
140
|
+
"Discovery",
|
|
141
|
+
"EngineCell",
|
|
142
|
+
"DocFlags",
|
|
143
|
+
"ErrorCodeEntry",
|
|
144
|
+
"ErrorCodeTable",
|
|
145
|
+
"FetchOptions",
|
|
146
|
+
"Filter",
|
|
147
|
+
"FilterOutcome",
|
|
148
|
+
"FilterResult",
|
|
149
|
+
"FilterRowError",
|
|
150
|
+
"Format",
|
|
151
|
+
"Framing",
|
|
152
|
+
"Header",
|
|
153
|
+
"InternalError",
|
|
154
|
+
"Library",
|
|
155
|
+
"Outcome",
|
|
156
|
+
"Reason",
|
|
157
|
+
"Registry",
|
|
158
|
+
"Resolved",
|
|
159
|
+
"RowResult",
|
|
160
|
+
"Schema",
|
|
161
|
+
"SchemaDescription",
|
|
162
|
+
"SchemaError",
|
|
163
|
+
"Settings",
|
|
164
|
+
"Source",
|
|
165
|
+
"SourceForbiddenError",
|
|
166
|
+
"SourceIncompatibleError",
|
|
167
|
+
"SourceUnauthorizedError",
|
|
168
|
+
"SourceUnreachableError",
|
|
169
|
+
"Span",
|
|
170
|
+
"Status",
|
|
171
|
+
"Transform",
|
|
172
|
+
"TrustedKey",
|
|
173
|
+
"UnsupportedError",
|
|
174
|
+
"UsageError",
|
|
175
|
+
"Value",
|
|
176
|
+
"Verdict",
|
|
177
|
+
"open_unverified",
|
|
178
|
+
"setup",
|
|
179
|
+
]
|
|
180
|
+
|
|
181
|
+
# Derived, never written twice. pyproject.toml is the single source of truth and
|
|
182
|
+
# the build reads it from there; asking importlib for it means this attribute
|
|
183
|
+
# cannot drift from the package it names.
|
|
184
|
+
try:
|
|
185
|
+
__version__ = _package_version("chtypes")
|
|
186
|
+
except _PackageNotFound: # a source tree that was never installed
|
|
187
|
+
__version__ = "0.0.0+unknown"
|