plot-io 0.12.0__py3-none-win_arm64.whl

plot_io/_native/__init__.py

@@ -0,0 +1,199 @@
+"""plot_io's bundled C library ``plio``: sources (``src/``), built libraries (``bin/``),
+``manifest.json`` and the bindings generated from ``src/plio_api.h`` (``_api.py``).
+Building is described in ``README.md``; ``python -m plot_io._native verify`` checks the
+installed library.
+
+The library is loaded on first use, from the first of: ``PLOT_IO_NATIVE_LIB`` (a sanitizer
+build, any other build), ``bin/<platform tag>.<ext>`` (a release binary), ``bin/hosted.<ext>``
+(built with the local C compiler when no release binary fits). At load the library's
+signature table (``plio_signatures``) must equal the bindings': names, argument kinds, and
+the sizes and field offsets of the parameter structures. Calls release the GIL; memory
+comes from Python's raw allocator, passed in a per-thread context that also holds a buffer
+for error messages.
+"""
+
+from __future__ import annotations
+
+import ctypes
+import os
+import platform
+import sys
+import sysconfig
+import threading
+from typing import Any
+
+from . import _api
+
+__all__ = ["NativeLibraryError", "lib", "ctx", "platform_tag", "library_path", "check", "take",
+           "ptr", "api"]
+
+api = _api
+
+
+class NativeLibraryError(RuntimeError):
+    """The bundled C library is not available on this platform or failed to load."""
+
+
+def platform_tag() -> str | None:
+    """``<os>-<arch>`` of the bundled library for this interpreter, or ``None``."""
+    if sys.maxsize <= 2 ** 32:
+        return None
+    machine = platform.machine().lower()
+    if sys.platform.startswith("win"):
+        plat = sysconfig.get_platform()
+        return {"win-amd64": "windows-x86_64", "win-arm64": "windows-arm64"}.get(plat)
+    if sys.platform == "darwin":
+        return "macos-arm64" if machine in ("arm64", "aarch64") else None
+    if sys.platform.startswith("linux"):
+        arch = {"x86_64": "x86_64", "amd64": "x86_64", "aarch64": "aarch64",
+                "arm64": "aarch64"}.get(machine)
+        return f"linux-{arch}" if arch else None
+    return None
+
+
+_EXT = {"linux": ".so", "windows": ".dll", "macos": ".dylib"}
+
+
+def _hosted_ext() -> str:
+    return ".dll" if sys.platform.startswith("win") else ".dylib" if sys.platform == "darwin" else ".so"
+
+
+def library_path() -> str | None:
+    """Path of the library that would be loaded: ``PLOT_IO_NATIVE_LIB``, else the platform's
+    binary in ``bin/``, else ``bin/hosted.<ext>``; ``None`` when there is none."""
+    env = os.environ.get("PLOT_IO_NATIVE_LIB")
+    if env:
+        return env
+    bin_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "bin")
+    tag = platform_tag()
+    if tag is not None:
+        path = os.path.join(bin_dir, tag + _EXT[tag.split("-")[0]])
+        if os.path.exists(path):
+            return path
+    hosted = os.path.join(bin_dir, "hosted" + _hosted_ext())
+    return hosted if os.path.exists(hosted) else None
+
+
+_lock = threading.Lock()
+_lib: Any = None
+_error: str | None = None
+_local = threading.local()
+_MESSAGE_SIZE = 256
+
+
+def signature_problems(handle: Any) -> list[str]:
+    """How the library's signature table differs from the bindings (empty: it matches)."""
+    text, numbers = ctypes.c_void_p(), ctypes.c_void_p()
+    n = handle.plio_signatures(None, ctypes.byref(text), ctypes.byref(numbers))
+    if not text.value or not numbers.value:
+        return ["no signature table"]
+    got = ctypes.string_at(text.value).decode("ascii").splitlines()
+    want = _api.SIGNATURE_TEXT.splitlines()
+    out = [f"library has {line!r}" for line in got if line not in want]
+    out += [f"library lacks {line!r}" for line in want if line not in got]
+    if out or n != len(got):
+        return out or [f"{n} signature lines for {len(got)} names"]
+    values = (ctypes.c_int64 * n).from_address(numbers.value)
+    for line, value in zip(got, values):
+        kind, name = line.split()[:2]
+        if kind == "s":
+            expected = ctypes.sizeof(_api.STRUCTS[name])
+        elif kind == "m":
+            struct, field = name.split(".")
+            expected = getattr(_api.STRUCTS[struct], field).offset
+        else:
+            continue
+        if value != expected:
+            out.append(f"{name}: {value} in the library, {expected} in the bindings")
+    return out
+
+
+def _load() -> None:
+    global _lib, _error
+    path = library_path()
+    if path is None:
+        _error = (f"plot_io's C library is not installed for this platform "
+                  f"({sys.platform}, {platform.machine()}): install plot_io from the sdist "
+                  f"with a C compiler (pip install --no-binary plot_io plot_io), or set "
+                  f"PLOT_IO_NATIVE_LIB to a build of it")
+        return
+    try:
+        handle = ctypes.CDLL(path)
+    except OSError as exc:
+        _error = f"cannot load plot_io's C library {path}: {exc}"
+        return
+    try:
+        for name, (restype, argtypes) in _api.FUNCTIONS.items():
+            fn = getattr(handle, name)
+            fn.restype = restype
+            fn.argtypes = argtypes
+    except AttributeError as exc:
+        _error = f"plot_io's C library {path} is incomplete: {exc}"
+        return
+    abi = handle.plio_abi_version()
+    if abi != _api.ABI_VERSION:
+        _error = f"plot_io's C library {path} has ABI {abi}, {_api.ABI_VERSION} expected"
+        return
+    problems = signature_problems(handle)
+    if problems:
+        _error = (f"plot_io's C library {path} does not match its bindings: "
+                  + "; ".join(problems[:5]))
+        return
+    _lib = handle
+
+
+def lib() -> Any:
+    """The loaded library; raises :class:`NativeLibraryError` when it is unavailable."""
+    if _lib is None:
+        with _lock:
+            if _lib is None and _error is None:
+                _load()
+        if _lib is None:
+            raise NativeLibraryError(_error)
+    return _lib
+
+
+def ctx() -> Any:
+    """The calling thread's context (pointer), the first argument of every function."""
+    c = getattr(_local, "ctx", None)
+    if c is None:
+        lib()
+        pyapi = ctypes.pythonapi
+        buf = ctypes.create_string_buffer(_MESSAGE_SIZE)
+        c = ctypes.pointer(_api.Ctx(
+            malloc=ctypes.cast(pyapi.PyMem_RawMalloc, ctypes.c_void_p).value,
+            realloc=ctypes.cast(pyapi.PyMem_RawRealloc, ctypes.c_void_p).value,
+            free=ctypes.cast(pyapi.PyMem_RawFree, ctypes.c_void_p).value,
+            message=ctypes.addressof(buf), message_size=_MESSAGE_SIZE))
+        _local.ctx, _local.buf = c, buf
+    else:
+        _local.buf[0] = 0
+    return c
+
+
+def check(rc: int, what: str = "plot_io native call", exc: type[Exception] = ValueError) -> int:
+    """``rc`` when it is not negative; otherwise raise ``exc`` (MemoryError when out of
+    memory) with the library's message, else the code's."""
+    if rc >= 0:
+        return rc
+    buf = getattr(_local, "buf", None)
+    detail = buf.value.decode("utf-8", "replace") if buf is not None and buf.value else ""
+    text = detail or _api.ERRORS.get(rc, f"error {rc}")
+    if rc == _api.E_NOMEM:
+        raise MemoryError(f"{what}: out of memory")
+    raise exc(f"{what}: {text}")
+
+
+def take(p: Any, n: int) -> bytes:
+    """Copy ``n`` bytes of a library-allocated result and release it."""
+    if not p:
+        return b""
+    try:
+        return ctypes.string_at(p, n)
+    finally:
+        _lib.plio_free(ctx(), p)
+
+
+def ptr(a: Any) -> int | None:
+    """Address of an ``array.array``'s data (``None`` when empty or ``None``)."""
+    return a.buffer_info()[0] if a is not None and len(a) else None

plot_io/_native/__main__.py

@@ -0,0 +1,197 @@
+"""``python -m plot_io._native verify``: check the C library this installation loads.
+
+Prints the platform, the file loaded, its SHA-256 and the manifest's ABI and source hash,
+then runs three steps:
+
+1. **loads**: the loader's checks (ABI version, the library's signature table against the
+   bindings);
+2. **manifest**: the file's SHA-256 is the manifest's entry of its name (a release binary,
+   or a file named like one through ``PLOT_IO_NATIVE_LIB``); a hosted build or another
+   override has no reference hash, which is not a failure;
+3. **self-test**: a density, a group summary, a sort, a colour map lookup and the library's
+   math at 16 points, each output's SHA-256 compared with the one the release binaries give
+   (every target computes identical results).
+
+Exit status 0 when every step passes, 1 otherwise (the failing steps are named).
+"""
+
+from __future__ import annotations
+
+import array
+import hashlib
+import json
+import os
+import platform
+import sys
+from typing import Callable
+
+# SHA-256 of each self-test's output, from the release binaries
+EXPECTED = {
+    "density": "c5d3b904c127c54a6ed045c800f14f6c6809bf66d0a392f8706865ad211c56db",
+    "group summary": "695de6b532cd9a9b58927bb5fb4168e9e4abd0e8e0fdc948ba4a8c27b211bc5c",
+    "sort": "c198461a79f1cd31a257c64cc9a19817a110d53c6308b655465915c6841f858e",
+    "colour map": "f39c20a0456f7c9cacefd4c632e1d9107c53b8b8ee3d4d7c8d949fa78d1d7404",
+    "math": "ada4f741f1482fe9e39f5364e179f8000b4bcc4b5c2ca25862c3167181cffbe8",
+    "linkage": "60d86a1bd7b6a55b5f82f8461a806782b3e2a8da825ed9221f16e2446c55f524",
+}
+
+
+def _values(n: int) -> array.array:
+    """Fixed values: a few clusters, ties, a NaN."""
+    out = array.array("d")
+    for i in range(n):
+        out.append(((i * 37) % 23) * 0.25 + (i % 5) * 1.5 - 3.0 + (2.75 if i % 7 == 0 else 0.0))
+    out[n // 2] = float("nan")
+    return out
+
+
+def _density() -> bytes:
+    from . import kernels
+
+    v = array.array("d", [x for x in _values(64) if x == x])
+    w = array.array("d", [1.0 + (i % 3) * 0.5 for i in range(len(v))])
+    return (kernels.kde(v, None, -4.0, 6.0, 32, 0.6, False).tobytes()
+            + kernels.kde(v, w, -4.0, 6.0, 32, 0.45, True).tobytes())
+
+
+def _group_summary() -> bytes:
+    from . import kernels
+
+    v = _values(48)                                  # 12 rows x 4 columns
+    groups = array.array("q", [i % 3 - (i == 5) for i in range(12)])
+    return b"".join(a.tobytes() for a in kernels.group_summary(v, 12, 4, groups, 3, 0.5))
+
+
+def _sort() -> bytes:
+    from . import kernels
+
+    v = _values(200)
+    v[3], v[17] = float("inf"), -0.0
+    return kernels.argsort(v, False).tobytes() + kernels.argsort(v, True).tobytes()
+
+
+def _colour_map() -> bytes:
+    from . import kernels
+
+    stops = [(0.0, (0.1, 0.2, 0.6, 1.0)), (0.4, (0.9, 0.9, 0.3, 0.8)), (1.0, (0.7, 0.1, 0.1, 1.0))]
+    ts = array.array("d", [i / 15 for i in range(16)])
+    out = b""
+    for mode in ("linear", "smooth", "spline"):
+        for space in ("srgb", "oklab"):
+            out += kernels.colormap_eval(stops, mode, space, ts).tobytes()
+    table = array.array("d", [c / 7 for c in range(4 * 8)])
+    vals = array.array("d", [-0.5, 0.0, 0.1, 0.33, 0.5, 0.77, 1.0, 1.5, float("nan")])
+    return out + kernels.colors_at(vals, table, None, (0, 0, 0, 1), (1, 1, 1, 1),
+                                   (0.5, 0.5, 0.5, 0)).tobytes()
+
+
+def _math() -> bytes:
+    import ctypes
+
+    from . import _api, ctx, lib, ptr
+
+    xs = array.array("d", [-7.5, -2.25, -1.0, -0.5, -1e-3, 0.0, 1e-3, 0.3, 0.5, 0.75, 1.0,
+                           1.5, 2.5, 3.14159, 10.0, 123.456])
+    ys = array.array("d", [0.5 + i * 0.37 for i in range(16)])
+    positive = (_api.MATH_SQRT, _api.MATH_LOG, _api.MATH_POW)
+    out = b""
+    for fn in range(_api.MATH_COS + 1):
+        res = array.array("d", bytes(8 * len(xs)))
+        # sqrt, log and pow on positive arguments: a NaN's sign bit is the compiler's
+        x = xs if fn not in positive else array.array("d", [abs(v) + 0.25 for v in xs])
+        lib().plio_math_eval(ctx(), fn, ptr(x), ptr(ys), ctypes.c_void_p(ptr(res)), len(xs))
+        out += res.tobytes()
+    return out
+
+
+def _linkage() -> bytes:
+    from . import kernels
+
+    v = array.array("d", [x for x in _values(61) if x == x])        # 20 rows x 3
+    out = b""
+    for method, metric in (("single", "cityblock"), ("average", "correlation"),
+                           ("ward", "euclidean"), ("centroid", "euclidean")):
+        out += kernels.linkage(v, 20, 3, method, metric).tobytes()
+    return out
+
+
+TESTS: dict[str, Callable[[], bytes]] = {
+    "density": _density, "group summary": _group_summary, "sort": _sort,
+    "colour map": _colour_map, "math": _math, "linkage": _linkage,
+}
+
+
+def self_test() -> dict[str, str]:
+    """The SHA-256 of each self-test's output (an exception's text when one fails)."""
+    out = {}
+    for name, fn in TESTS.items():
+        try:
+            out[name] = hashlib.sha256(fn()).hexdigest()
+        except Exception as exc:   # a failing call is a failed step
+            out[name] = f"{type(exc).__name__}: {exc}"
+    return out
+
+
+def _sha256(path: str) -> str:
+    with open(path, "rb") as f:
+        return hashlib.sha256(f.read()).hexdigest()
+
+
+def verify() -> int:
+    from . import NativeLibraryError, library_path, lib, platform_tag
+
+    here = os.path.dirname(os.path.abspath(__file__))
+    try:
+        with open(os.path.join(here, "manifest.json")) as f:
+            manifest = json.load(f)
+    except OSError:
+        manifest = {}
+    path = library_path()
+    print(f"platform  {platform_tag() or 'none'} ({sys.platform}, {platform.machine()})")
+    print(f"file      {path or 'none'}")
+    digest = _sha256(path) if path and os.path.isfile(path) else None
+    print(f"sha256    {digest or '-'}")
+    print(f"manifest  ABI {manifest.get('abi', '-')}, sources {manifest.get('source_sha256', '-')}")
+    failed = []
+    try:
+        lib()
+        print("loads     ok")
+    except NativeLibraryError as exc:
+        print(f"loads     FAIL: {exc}")
+        failed.append("loads")
+    entry = manifest.get("files", {}).get(os.path.basename(path or ""))
+    if digest is None:
+        print("manifest  FAIL: no library file")
+        failed.append("manifest")
+    elif entry is None:
+        print("manifest  no reference hash (not a release binary)")
+    elif entry.get("sha256") == digest:
+        print("manifest  ok")
+    else:
+        print(f"manifest  FAIL: the manifest says {entry.get('sha256')}")
+        failed.append("manifest")
+    if "loads" not in failed:
+        results = self_test()
+        bad = [n for n, h in results.items() if h != EXPECTED[n]]
+        for n in bad:
+            print(f"self-test FAIL: {n}: {results[n]}")
+        if bad:
+            failed.append("self-test")
+        else:
+            print(f"self-test ok ({len(results)} checks)")
+    if failed:
+        print(f"failed: {', '.join(failed)}")
+        return 1
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = sys.argv[1:] if argv is None else argv
+    if args != ["verify"]:
+        print("usage: python -m plot_io._native verify", file=sys.stderr)
+        return 2
+    return verify()
+
+
+if __name__ == "__main__":
+    sys.exit(main())