chtypes 0.1.0__py3-none-any.whl

chtypes/__init__.py

@@ -0,0 +1,202 @@
+"""chtypes — ClickHouse's own type system, per version, from Python.
+
+One question, exactly: *if this row were inserted into this ClickHouse table on
+this ClickHouse version, what would happen?* The answer comes from ClickHouse's
+real C++ machinery (`DataTypeFactory`, `ISerialization`, `ReadHelpers`,
+`evaluateMissingDefaults`, the TTL algorithms, `MergeTreeDataWriter::mergeBlock`)
+vendored per release into a shared library behind the frozen `chs_*` C ABI.
+Nothing here reimplements a coercion rule, which is why the answers are exact by
+construction rather than approximately right.
+
+    from chtypes import Format, Registry
+
+    registry = Registry()                    # the search path: $CHTYPES_REGISTRY, the cache, …
+    library = registry.for_version("25.8")   # a minor line or an exact patch
+    with library.compile_ddl("ts DateTime, seq UInt8") as schema:
+        batch = schema.rows(Format.JSON_EACH_ROW, b'{"ts":"2026-01-15 10:30:00","seq":256}\\n')
+
+    batch.outcome                 # Outcome.ACCEPTED — the insert would succeed
+    batch.rows[0].value("seq")    # Value(text='0', ...) — and store 0
+    batch.transformed             # Transform(column='seq', input='256', stored='0',
+                                  #           reason='overflow_wrap', row=0)
+
+Three things a caller must not skip, each of which cost this repository
+something to learn:
+
+* **`transformed` is the product.** ClickHouse returns success for every one of
+  those changes. Reading `outcome` alone tells a tenant their row was accepted
+  and nothing about the value the table will hold.
+* **A row accepted per row may not be stored per batch.** `BatchResult.transformed`
+  folds in the storage layer's verdicts (`ttl_expired`, `ttl_column_expired`),
+  and `engine_rows` — when present — is the stored truth, not `rows`.
+* **`Outcome.UNSUPPORTED` is not a rejection.** It means a real server might well
+  have accepted this and this build declines to guess. Treating it as either a
+  rejection or an acceptance manufactures a wrong answer the product never gave.
+
+The specification in `spec/` is normative; `go/chtypes` (Go) is the reference
+implementation. Where this binding and that package disagree, the package is
+right.
+"""
+
+from __future__ import annotations
+
+from ._native import ABI_REVISION
+from ._rawjson import RawNumber, quote_bare_denormals
+from .discover import (
+    QUERY_CHANGED_SETTINGS,
+    QUERY_SERVER_VERSION,
+    QUERY_TABLE_COLUMNS,
+    DiscoveredColumn,
+    ServerProfile,
+    parse_changed_settings_result,
+    parse_columns_result,
+    parse_version_result,
+    reconstruct_ddl,
+)
+from .errors import (
+    CODE_ARTIFACT_CORRUPT,
+    CODE_ARTIFACT_MISSING,
+    CODE_ARTIFACT_PINNED,
+    CODE_ARTIFACT_UNPUBLISHED,
+    CODE_ARTIFACT_UNTRUSTED,
+    CODE_SOURCE_UNREACHABLE,
+    CODE_UNSUPPORTED,
+    ArtifactCorruptError,
+    ArtifactError,
+    ArtifactMissingError,
+    ArtifactPinnedError,
+    ArtifactUnpublishedError,
+    ArtifactUntrustedError,
+    ChtypesError,
+    RegistryError,
+    SchemaError,
+    SourceUnreachableError,
+    UnsignedArtifactWarning,
+    UnsupportedError,
+)
+from .fetch import (
+    ENV_AUTOFETCH,
+    RELEASE_KEY_ID,
+    RELEASE_PUBLIC_KEY,
+    ensure,
+    fetch_destination,
+    fetch_lines,
+    registry_search_path,
+)
+from .registry import (
+    ENV_REGISTRY,
+    Block,
+    Filter,
+    Library,
+    Manifest,
+    Registry,
+    Schema,
+    default_registry_dir,
+    host_platform,
+    minor_of,
+    read_manifest,
+    verify_library,
+)
+from .results import (
+    COMPILE_DECLARED,
+    DOC_ALL,
+    DOC_DEFAULTS,
+    DOC_TRANSFORMS,
+    DOC_VALUES,
+    EXPORT_NONE,
+    LOSSLESS_REASONS,
+    BatchResult,
+    Column,
+    Computed,
+    DefaultKind,
+    FilterOutcome,
+    FilterResult,
+    FilterRowError,
+    Format,
+    Outcome,
+    Reason,
+    RowResult,
+    Span,
+    Substitution,
+    Transform,
+    Value,
+    Verdict,
+)
+
+__all__ = [
+    "ABI_REVISION",
+    "CODE_ARTIFACT_CORRUPT",
+    "CODE_ARTIFACT_MISSING",
+    "CODE_ARTIFACT_PINNED",
+    "CODE_ARTIFACT_UNPUBLISHED",
+    "CODE_ARTIFACT_UNTRUSTED",
+    "CODE_SOURCE_UNREACHABLE",
+    "CODE_UNSUPPORTED",
+    "COMPILE_DECLARED",
+    "DOC_ALL",
+    "DOC_DEFAULTS",
+    "DOC_TRANSFORMS",
+    "DOC_VALUES",
+    "ENV_AUTOFETCH",
+    "ENV_REGISTRY",
+    "default_registry_dir",
+    "EXPORT_NONE",
+    "RELEASE_KEY_ID",
+    "RELEASE_PUBLIC_KEY",
+    "LOSSLESS_REASONS",
+    "QUERY_CHANGED_SETTINGS",
+    "QUERY_SERVER_VERSION",
+    "QUERY_TABLE_COLUMNS",
+    "ArtifactCorruptError",
+    "ArtifactError",
+    "ArtifactMissingError",
+    "ArtifactPinnedError",
+    "ArtifactUnpublishedError",
+    "ArtifactUntrustedError",
+    "BatchResult",
+    "Block",
+    "ChtypesError",
+    "Column",
+    "Computed",
+    "DefaultKind",
+    "DiscoveredColumn",
+    "Filter",
+    "FilterOutcome",
+    "FilterResult",
+    "FilterRowError",
+    "Format",
+    "Library",
+    "Manifest",
+    "Outcome",
+    "RawNumber",
+    "Reason",
+    "Registry",
+    "RegistryError",
+    "RowResult",
+    "Schema",
+    "SchemaError",
+    "ServerProfile",
+    "SourceUnreachableError",
+    "Span",
+    "Substitution",
+    "Transform",
+    "UnsignedArtifactWarning",
+    "UnsupportedError",
+    "Value",
+    "Verdict",
+    "ensure",
+    "fetch_destination",
+    "fetch_lines",
+    "host_platform",
+    "minor_of",
+    "parse_changed_settings_result",
+    "parse_columns_result",
+    "parse_version_result",
+    "quote_bare_denormals",
+    "read_manifest",
+    "reconstruct_ddl",
+    "registry_search_path",
+    "verify_library",
+]
+
+__version__ = "0.1.0"

chtypes/__main__.py

@@ -0,0 +1,268 @@
+"""``python -m chtypes`` and the ``chtypes`` console script — docs/fetch.md §6.
+
+    chtypes fetch <line>... [--all] [--platform <os-arch>] [--dest <dir>]
+                            [--tag <t> | --url <base>] [--lock <file>] [--frozen]
+                            [--force] [--offline]
+    chtypes verify [--dest <dir>]        re-hash every installed line against its manifest
+    chtypes list   [--dest <dir>]        what is installed, and what the release offers
+    chtypes where                        the registry directory fetch would write to
+
+Exit codes: 0 ok · 1 verification failed · 2 usage · 3 source unreachable ·
+4 not published for this platform/line. Progress goes to stderr; ``fetch``
+prints the installed directory alone on stdout, one per line.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import warnings
+from collections.abc import Sequence
+
+from ._manifest import Manifest
+from .errors import (
+    CODE_ARTIFACT_UNPUBLISHED,
+    CODE_SOURCE_UNREACHABLE,
+    ArtifactError,
+    ChtypesError,
+    UnsignedArtifactWarning,
+)
+from .fetch import DEFAULT_ARTIFACTS_URL, DEFAULT_TAG, PLATFORMS, ReleaseEntry
+
+__all__ = ["main"]
+
+EXIT_OK = 0
+EXIT_VERIFY_FAILED = 1
+EXIT_USAGE = 2
+EXIT_UNREACHABLE = 3
+EXIT_UNPUBLISHED = 4
+
+_EXIT_FOR_CODE = {
+    CODE_SOURCE_UNREACHABLE: EXIT_UNREACHABLE,
+    CODE_ARTIFACT_UNPUBLISHED: EXIT_UNPUBLISHED,
+}
+
+
+def _say(line: str) -> None:
+    sys.stderr.write(line + "\n")
+    sys.stderr.flush()
+
+
+class _Parser(argparse.ArgumentParser):
+    """argparse exits 2 on usage errors already; this keeps the message shape ours."""
+
+    def error(self, message: str) -> None:  # type: ignore[override]
+        self.print_usage(sys.stderr)
+        _say(f"chtypes: {message}")
+        raise SystemExit(EXIT_USAGE)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = _Parser(
+        prog="chtypes",
+        description="Fetch, verify and locate chtypes artifacts (docs/fetch.md).",
+    )
+    sub = parser.add_subparsers(dest="command", metavar="<command>")
+    sub.required = True
+
+    def dest_option(p: argparse.ArgumentParser) -> None:
+        p.add_argument(
+            "--dest",
+            metavar="<dir>",
+            help="the registry directory (default: $CHTYPES_REGISTRY, else the per-user cache)",
+        )
+
+    fetch = sub.add_parser(
+        "fetch",
+        help="install one or more ClickHouse lines, verified",
+        description="Install ClickHouse lines through the verification chain (docs/fetch.md §3).",
+    )
+    fetch.add_argument(
+        "lines",
+        nargs="*",
+        metavar="<line>",
+        help="a minor line (25.8) or an exact patch (25.8.28.1-lts, a hard requirement)",
+    )
+    fetch.add_argument("--all", action="store_true", help="every line the release publishes")
+    fetch.add_argument(
+        "--platform",
+        metavar="<os-arch>",
+        choices=PLATFORMS,
+        help=f"one of {', '.join(PLATFORMS)} (default: this host)",
+    )
+    dest_option(fetch)
+    where_from = fetch.add_mutually_exclusive_group()
+    where_from.add_argument(
+        "--tag",
+        metavar="<t>",
+        help=f"a release tag on the artifacts host (default: the rolling '{DEFAULT_TAG}')",
+    )
+    where_from.add_argument(
+        "--url",
+        metavar="<base>",
+        help=f"any base: an http(s) URL, a file:// URL or a directory "
+        f"(default: $CHTYPES_ARTIFACTS_URL or {DEFAULT_ARTIFACTS_URL}, plus the tag)",
+    )
+    fetch.add_argument(
+        "--lock", metavar="<file>", help="record what was installed in this lock file (§5)"
+    )
+    fetch.add_argument(
+        "--frozen",
+        action="store_true",
+        help="refuse anything the lock file does not pin (default lock: chtypes.lock)",
+    )
+    fetch.add_argument("--force", action="store_true", help="re-download an installed line")
+    fetch.add_argument(
+        "--offline",
+        action="store_true",
+        help="never touch the source: installed and verified, or CHTYPES_SOURCE_UNREACHABLE",
+    )
+
+    verify = sub.add_parser(
+        "verify",
+        help="re-hash every installed line against its manifest",
+        description="Re-hash every installed line's library against its own manifest.json.",
+    )
+    dest_option(verify)
+
+    listing = sub.add_parser(
+        "list",
+        help="what is installed, and what the release offers",
+        description="What is installed in the registry, and what the release offers for it.",
+    )
+    dest_option(listing)
+    listing.add_argument("--platform", metavar="<os-arch>", choices=PLATFORMS)
+    lfrom = listing.add_mutually_exclusive_group()
+    lfrom.add_argument("--tag", metavar="<t>")
+    lfrom.add_argument("--url", metavar="<base>")
+
+    where = sub.add_parser(
+        "where",
+        help="the registry directory fetch would write to",
+        description="Print the registry directory fetch would write to (docs/fetch.md §1).",
+    )
+    where.add_argument("--platform", metavar="<os-arch>", choices=PLATFORMS)
+    return parser
+
+
+def _cmd_fetch(args: argparse.Namespace) -> int:
+    from .fetch import fetch_lines
+
+    installed = fetch_lines(
+        args.lines,
+        all_lines=args.all,
+        dest=args.dest,
+        platform=args.platform,
+        url=args.url,
+        tag=args.tag,
+        lock=args.lock,
+        frozen=args.frozen,
+        force=args.force,
+        offline=args.offline,
+        progress=_say,
+    )
+    for directory in installed:
+        sys.stdout.write(f"{directory}\n")
+    sys.stdout.flush()
+    return EXIT_OK
+
+
+def _cmd_verify(args: argparse.Namespace) -> int:
+    from .fetch import fetch_destination, verify_registry
+
+    registry = fetch_destination(args.dest)
+    report = verify_registry(registry)
+    if not report:
+        _say(f"chtypes: nothing installed under {registry}")
+        return EXIT_OK
+    failed = 0
+    for minor, directory, error in report:
+        if error is None:
+            sys.stdout.write(f"{minor:<8} ok       {directory}\n")
+        else:
+            failed += 1
+            sys.stdout.write(f"{minor:<8} FAILED   {directory}\n")
+            _say(f"  {error}")
+    sys.stdout.flush()
+    if failed:
+        _say(f"chtypes: {failed} of {len(report)} installed line(s) FAILED verification")
+        return EXIT_VERIFY_FAILED
+    _say(f"chtypes: {len(report)} installed line(s) verified under {registry}")
+    return EXIT_OK
+
+
+def _cmd_list(args: argparse.Namespace) -> int:
+    from .fetch import Fetcher, fetch_destination, host_platform, installed_lines
+
+    platform = args.platform or host_platform()
+    registry = fetch_destination(args.dest, platform=platform)
+    have = installed_lines(registry)
+    sys.stdout.write(f"installed ({registry}):\n")
+    if not have:
+        sys.stdout.write("  (nothing)\n")
+    for minor, (directory, manifest) in have.items():
+        sys.stdout.write(f"  {minor:<8} {manifest.clickhouse_version:<18} {directory}\n")
+    sys.stdout.flush()
+
+    fetcher = Fetcher(dest=registry, platform=platform, url=args.url, tag=args.tag, progress=_say)
+    release = fetcher.release()
+    offered = release.offered(platform)
+    signed = f"signed by key {release.signed_by}" if release.signed_by else "UNSIGNED"
+    sys.stdout.write(f"release ({release.source}, {signed}) offers for {platform}:\n")
+    if not offered:
+        sys.stdout.write("  (nothing)\n")
+    for entry in offered:
+        state = "installed" if _installed(have, entry) else "not installed"
+        sys.stdout.write(
+            f"  {entry.minor:<8} {entry.clickhouse_version:<18} {entry.file}  [{state}]\n"
+        )
+    sys.stdout.flush()
+    return EXIT_OK
+
+
+def _installed(have: dict[str, tuple[object, Manifest]], entry: ReleaseEntry) -> bool:
+    got = have.get(entry.minor)
+    return got is not None and got[1].clickhouse_version == entry.clickhouse_version
+
+
+def _cmd_where(args: argparse.Namespace) -> int:
+    from .fetch import fetch_destination
+
+    sys.stdout.write(f"{fetch_destination(platform=args.platform)}\n")
+    return EXIT_OK
+
+
+_COMMANDS = {
+    "fetch": _cmd_fetch,
+    "verify": _cmd_verify,
+    "list": _cmd_list,
+    "where": _cmd_where,
+}
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run the CLI; returns the exit code (docs/fetch.md §6)."""
+    parser = build_parser()
+    args = parser.parse_args(list(sys.argv[1:] if argv is None else argv))
+    # The library's loud warning reaches stderr through the progress channel
+    # here; the `warnings` copy would print it a second time.
+    warnings.simplefilter("ignore", UnsignedArtifactWarning)
+    try:
+        return _COMMANDS[args.command](args)
+    except ValueError as exc:  # a bad option combination or spelling: usage
+        _say(str(exc))
+        return EXIT_USAGE
+    except ArtifactError as exc:
+        _say(str(exc))
+        _say(f"chtypes: {exc.code}")
+        return _EXIT_FOR_CODE.get(exc.code, EXIT_VERIFY_FAILED)
+    except ChtypesError as exc:
+        _say(str(exc))
+        return EXIT_VERIFY_FAILED
+    except KeyboardInterrupt:
+        _say("chtypes: interrupted")
+        return 130
+
+
+if __name__ == "__main__":
+    sys.exit(main())