GGDLPC 0.0.1__py3-none-any.whl

GGDLPC/__init__.py

@@ -0,0 +1,28 @@
+"""GGDLPC -- lossless compression via generalized Gaussian modeling of
+linear predictive coding residuals.
+
+The primitive: one call encodes one integer tensor (values in [-N, N],
+N <= 65536 chosen at fit time; rank 1/2/3) as one self-contained
+byte-padded bitstream, truncatable at any channel boundary. Macroregions,
+pre-quantization, multi-call file layouts, and metadata are caller
+compositions.
+
+    import GGDLPC
+    channels, prov = GGDLPC.fit(loader_factory, N=31, rank=2)
+    codec = GGDLPC.Codec(GGDLPC.new_blob(31, 2, channels, prov))
+    data, ch_bits = codec.encode(z)          # (C, *S) ints -> bytes
+    z2 = codec.decode(data, z.shape)         # bit-exact
+    bits = GGDLPC.proxy_call_bits(z_noisy, codec.blob)   # differentiable
+"""
+from ._core import (A_MAX, M_MIN, PAD_BITS, TAU, n_grid, stat_cap)
+from .codec import Codec, SPEC, VERSION, new_blob
+from .fit import fit
+from .proxy import proxy_call_bits, cont_maps, ideal_bits
+from .selftest import selftest
+from .tables import build_channel, derive_split
+
+__all__ = [
+    'Codec', 'SPEC', 'VERSION', 'new_blob', 'fit', 'proxy_call_bits',
+    'cont_maps', 'ideal_bits', 'selftest', 'build_channel', 'derive_split',
+    'A_MAX', 'M_MIN', 'PAD_BITS', 'TAU', 'n_grid', 'stat_cap',
+]

GGDLPC/_core.py

@@ -0,0 +1,159 @@
+"""GGDLPC core math: derived constants, GGD machinery, Huffman lengths.
+
+Everything is parameterized by the alphabet bound N (inputs are integers in
+[-N, N], N <= 65536, chosen at fit time). Derived quantities:
+
+- NSYM = 2N + 1: the folded residual alphabet (residuals in [-2N, 2N] are
+  reduced mod NSYM by the JPEG-LS fold).
+- G = ceil(log_tau(2N / m_min)) + 1: scale-grid cells (tau = 2^{1/4},
+  m_min = 1/16). G = 41 at N = 31, 49 at N = 128, 81 at N = 32768,
+  85 at N = 65536.
+- STAT_CAP = ceil(tau^{G-1}) - 1: cap on the integer statistic S = floor(16 s)
+  at the top of the scale grid (1023 at N = 31, 4095 at N = 128,
+  2^20 - 1 at N = 32768, 2^21 - 1 at N = 65536).
+
+N = 65536 exists so that bit-depth-expanding lossless transforms of 16-bit
+sources (mid-side stereo: the side channel L - R spans [-65535, 65535]) fit
+the contract; sources above 16 bits are still pre-quantized by the caller.
+
+The Huffman-length and GGD-integration numerics are copied from the FRAPPEv3
+coder (redesign_rounds/sweep.py) so that at N = 31 the generated tables are
+bit-identical to the shipped lawcoder's.
+"""
+import heapq
+import math
+
+import numpy as np
+from scipy.optimize import brentq
+from scipy.special import gammainc, gammaln
+
+TAU = 2 ** 0.25
+M_MIN = 1.0 / 16.0
+A_MAX = 63          # windowed hi alphabet (table size <= A_MAX + 1 escape)
+N_MAX = 65536
+MIN_OCC = 10_000    # minimum bucket occupancy for a Sharifi shape estimate
+PAD_BITS = 3.5      # expected bits lost to one byte pad per call
+Q_FLOOR = 1e-12
+
+RANK_NF = {1: 2, 2: 5, 3: 9}   # FIR feature count (template M) by rank
+STAT_W = 16         # residual-window length of the pooled statistic
+
+
+def n_grid(N):
+    """Number of scale-grid cells G for alphabet bound N."""
+    return int(math.ceil(4.0 * math.log2(2 * N / M_MIN) - 1e-9)) + 1
+
+
+def stat_cap(N):
+    """Integer statistic cap: top of the scale grid, minus one LSB."""
+    G = n_grid(N)
+    return int(math.ceil(TAU ** (G - 1) - 1e-9)) - 1
+
+
+def ggd_cdf(x, beta):
+    return 0.5 + 0.5 * np.sign(x) * gammainc(1.0 / beta,
+                                             np.abs(x) ** beta + 1e-300)
+
+
+def c_of_beta(beta):
+    """Mean-absolute-deviation to GGD-scale conversion factor."""
+    return math.exp(gammaln(1.0 / beta) - gammaln(2.0 / beta))
+
+
+def sharifi_beta(rho):
+    """Sharifi--Leon-Garcia shape estimate from rho = E(x-mu)^2 / (E|x-mu|)^2."""
+    def f(b):
+        return (gammaln(1.0 / b) + gammaln(3.0 / b)
+                - 2.0 * gammaln(2.0 / b)) - math.log(rho)
+    lo, hi = 0.05, 10.0
+    if f(lo) < 0:
+        return lo
+    if f(hi) > 0:
+        return hi
+    return brentq(f, lo, hi, xtol=1e-6)
+
+
+def ggd_folded_probs(mu, beta, alpha, N):
+    """Integer-bin GGD mass over three fold periods, folded mod (2N+1).
+
+    Returns the folded pmf over w in [0, 2N] (w = (r + N) mod (2N+1)),
+    floored at Q_FLOOR and normalized. At N = 31 this reproduces the
+    FRAPPEv3 sweep.ggd_folded_probs values exactly (the v3 index
+    (r + 31) % 63 equals this w). For large N the integration is
+    restricted to the density's numerical support (|x|^beta <= 45,
+    tail mass < e^-45 ~ 3e-20, far below Q_FLOOR); outside it the bins
+    take the floor, exactly as they would from a full evaluation."""
+    nsym = 2 * N + 1
+    half = (3 * nsym - 1) // 2
+    R = int(math.ceil(alpha * 45.0 ** (1.0 / beta) + abs(mu))) + 2
+    lo_r, hi_r = max(-half, -R), min(half, R)
+    r = np.arange(lo_r, hi_r + 1)
+    hi = ggd_cdf((r + 0.5 - mu) / alpha, beta)
+    lo = ggd_cdf((r - 0.5 - mu) / alpha, beta)
+    p = np.maximum(hi - lo, 0.0)
+    w = (r + N) % nsym
+    q = np.zeros(nsym)
+    np.add.at(q, w, p)
+    q = np.maximum(q, Q_FLOOR)
+    return q / q.sum()
+
+
+def huffman_lengths(weights):
+    """Exact Huffman code lengths (float array). Copied verbatim from the
+    FRAPPEv3 sweep.huffman_lengths (same heap tie-breaking) so generated
+    tables match the shipped coder bit for bit at N = 31."""
+    w = np.asarray(weights, np.float64)
+    idx = np.nonzero(w > 0)[0]
+    if len(idx) == 0:
+        return np.zeros_like(w)
+    if len(idx) == 1:
+        L = np.zeros_like(w)
+        L[idx[0]] = 1.0
+        return L
+    heap = [(w[i], n) for n, i in enumerate(idx)]
+    heapq.heapify(heap)
+    parent = {}
+    nxt = len(idx)
+    while len(heap) > 1:
+        w1, n1 = heapq.heappop(heap)
+        w2, n2 = heapq.heappop(heap)
+        parent[n1] = parent[n2] = nxt
+        heapq.heappush(heap, (w1 + w2, nxt))
+        nxt += 1
+    L = np.zeros_like(w)
+    depths = {heap[0][1]: 0}
+    for n in range(nxt - 1, -1, -1):
+        if n in parent:
+            depths[n] = depths[parent[n]] + 1
+    for n, i in enumerate(idx):
+        L[i] = depths[n]
+    L[w <= 0] = L.max()
+    return L
+
+
+def run_rate_fixed_g(p0, L_excl, g):
+    """Model-expected bits/sample of EG_{2^g} zero-run coding (v3 formula)."""
+    if p0 >= 1.0:
+        return 1.0 / (1 << g)
+    m = 1 << g
+    pm = p0 ** m
+    cost = pm * 1.0 + (1.0 - pm) * (1.0 + g + L_excl)
+    x = p0
+    s_partial = (1 - (m + 1) * x ** m + m * x ** (m + 1)) / (1 - x) ** 2
+    samples = m * pm + (1.0 - x) * s_partial
+    return cost / samples
+
+
+def canonical_codes(lengths):
+    """Canonical codewords for integer code lengths: increasing length,
+    then increasing symbol index (matches the C table construction)."""
+    lengths = np.asarray(lengths, np.int64)
+    order = sorted(range(len(lengths)), key=lambda s: (lengths[s], s))
+    codes = np.zeros(len(lengths), np.int64)
+    code, prev = 0, int(lengths[order[0]])
+    for sym in order:
+        code <<= int(lengths[sym]) - prev
+        prev = int(lengths[sym])
+        codes[sym] = code
+        code += 1
+    return codes

GGDLPC/cbridge.py

@@ -0,0 +1,227 @@
+"""ctypes bridge to the GGDLPC C engine (lazy JIT build of libggdlpc).
+
+The C library executes the full coding loops (prediction, statistic, cell
+selection, quotient-split symbol coding, run mode); Python generates the
+tables (tables.py) and passes them in. The library is compiled once on
+first import from the vendored C sources (same pattern as the FRAPPEv3
+coder in `compressors`): gcc/cc/clang -std=c99 -O2 -shared -fPIC.
+"""
+import ctypes
+import platform
+import shutil
+import subprocess
+from pathlib import Path
+
+import numpy as np
+
+from ._core import RANK_NF
+
+_HERE = Path(__file__).resolve().parent
+
+
+def _lib_path():
+    system = platform.system()
+    if system == 'Windows':
+        return _HERE / 'libggdlpc.dll'
+    elif system == 'Darwin':
+        return _HERE / 'libggdlpc.dylib'
+    return _HERE / 'libggdlpc.so'
+
+
+def _build_lib():
+    src = [str(_HERE / 'ggdlpc.c')]
+    out = str(_lib_path())
+    system = platform.system()
+    if system == 'Windows':
+        cmd = ['cl.exe', '/O2', '/LD'] + src + [f'/Fe{out}']
+    else:
+        cc = 'gcc'
+        for try_cc in ('gcc', 'cc', 'clang'):
+            if shutil.which(try_cc):
+                cc = try_cc
+                break
+        cmd = [cc, '-std=c99', '-O2', '-shared', '-fPIC'] + src + ['-o', out]
+    try:
+        subprocess.check_call(cmd, cwd=str(_HERE))
+    except (FileNotFoundError, subprocess.CalledProcessError):
+        raise RuntimeError(
+            'GGDLPC requires a C compiler (gcc, clang, or cl.exe). '
+            'Install one and retry; the library builds automatically on '
+            'first import.')
+
+
+def _load_lib():
+    path = _lib_path()
+    src_mtime = max((_HERE / n).stat().st_mtime
+                    for n in ('ggdlpc.c', 'ggdlpc.h'))
+    if not path.exists() or path.stat().st_mtime < src_mtime:
+        _build_lib()
+    return ctypes.CDLL(str(path))
+
+
+_lib = _load_lib()
+
+_lib.glpc_channel_sizeof.restype = ctypes.c_size_t
+
+
+class _Writer(ctypes.Structure):
+    _fields_ = [('buf', ctypes.c_void_p), ('cap', ctypes.c_size_t),
+                ('acc', ctypes.c_uint64), ('nacc', ctypes.c_uint),
+                ('nbytes', ctypes.c_size_t)]
+
+
+class _Reader(ctypes.Structure):
+    _fields_ = [('buf', ctypes.c_void_p), ('nbytes', ctypes.c_size_t),
+                ('bitpos', ctypes.c_size_t)]
+
+
+for name, res, args in [
+    ('glpc_channel_config', None,
+     [ctypes.c_void_p, ctypes.c_int32, ctypes.c_int32, ctypes.c_int32,
+      ctypes.c_void_p, ctypes.c_int32, ctypes.c_void_p, ctypes.c_int32,
+      ctypes.c_void_p, ctypes.c_int32]),
+    ('glpc_cell_set', ctypes.c_int,
+     [ctypes.c_void_p, ctypes.c_int, ctypes.c_int, ctypes.c_void_p,
+      ctypes.c_int, ctypes.c_int32, ctypes.c_int32, ctypes.c_int32,
+      ctypes.c_int32, ctypes.c_int32]),
+    ('glpc_writer_init', None,
+     [ctypes.POINTER(_Writer), ctypes.c_void_p, ctypes.c_size_t]),
+    ('glpc_put_bits', ctypes.c_int,
+     [ctypes.POINTER(_Writer), ctypes.c_uint32, ctypes.c_uint]),
+    ('glpc_flush', ctypes.c_int64, [ctypes.POINTER(_Writer)]),
+    ('glpc_reader_init', None,
+     [ctypes.POINTER(_Reader), ctypes.c_void_p, ctypes.c_size_t]),
+    ('glpc_get_bits', ctypes.c_int64,
+     [ctypes.POINTER(_Reader), ctypes.c_uint]),
+    ('glpc_encode', ctypes.c_int64,
+     [ctypes.POINTER(_Writer), ctypes.c_void_p, ctypes.c_int,
+      ctypes.c_void_p, ctypes.c_void_p]),
+    ('glpc_decode', ctypes.c_int,
+     [ctypes.POINTER(_Reader), ctypes.c_void_p, ctypes.c_int,
+      ctypes.c_void_p, ctypes.c_void_p]),
+    ('glpc_maps', ctypes.c_int,
+     [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p,
+      ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]),
+]:
+    fn = getattr(_lib, name)
+    fn.restype = res
+    fn.argtypes = args
+
+_ERR = {-1: 'RANGE', -2: 'CAPACITY', -3: 'TRUNCATED', -4: 'TABLE',
+        -5: 'CONFIG'}
+
+
+def _check(ret, what=''):
+    if isinstance(ret, int) and ret < 0:
+        raise ValueError(f'GGDLPC coder error {_ERR.get(ret, ret)} {what}')
+    return ret
+
+
+class CChannel:
+    """One channel's C-side coding state, filled from a ChannelBuild."""
+
+    def __init__(self, build, rank):
+        if rank not in RANK_NF:
+            raise ValueError(
+                f'GGDLPC has no C engine for rank {rank}; supported ranks '
+                f'are {sorted(RANK_NF)} (1-D, 2-D, 3-D). Higher ranks are '
+                'not implemented -- there is no fallback.')
+        self.rank = rank
+        self.build = build
+        self._buf = ctypes.create_string_buffer(_lib.glpc_channel_sizeof())
+        edges = np.ascontiguousarray(build.edges, np.int64)
+        run_g = np.ascontiguousarray(build.run_g, np.uint8)
+        if not build.wq.size:
+            raise ValueError('channel has no fitted predictor weights')
+        if rank == 1:
+            nf = int(build.wq.size)
+            if not 1 <= nf <= 9:
+                raise ValueError(f'rank-1 FIR order {nf} outside [1, 9]')
+        else:
+            nf = RANK_NF[rank]
+            if build.wq.size != nf:
+                raise ValueError(
+                    f'predictor has {build.wq.size} weights but rank {rank} '
+                    f'needs {nf}')
+        wq = np.zeros(9, np.int32)
+        wq[:nf] = build.wq
+        wq = np.ascontiguousarray(wq)
+        _lib.glpc_channel_config(
+            self._buf, build.N, build.G, build.stat_cap,
+            edges.ctypes.data, build.run_jmax, run_g.ctypes.data,
+            nf, wq.ctypes.data, int(build.bq))
+        for j, sp in enumerate(build.splits):
+            self._set_cell(0, j, sp)
+        for j in range(build.run_jmax + 1):
+            self._set_cell(1, j, build.excl[j])
+
+    def _set_cell(self, which, j, sp):
+        lens = np.ascontiguousarray(sp.lengths, np.uint8)
+        _check(_lib.glpc_cell_set(
+            self._buf, which, j, lens.ctypes.data, lens.size,
+            sp.win_lo, sp.n_win, sp.k, sp.esc, sp.hi_bits),
+            f'cell_set({which},{j})')
+
+    def maps(self, z):
+        """Encode-side integer maps: (folded symbol, cell id, statistic)."""
+        z = np.ascontiguousarray(z, np.int32)
+        dims = np.ascontiguousarray(z.shape, np.int64)
+        w = np.empty(z.shape, np.int32)
+        cell = np.empty(z.shape, np.int32)
+        stat = np.empty(z.shape, np.int32)
+        _check(_lib.glpc_maps(self._buf, self.rank, z.ctypes.data,
+                              dims.ctypes.data, w.ctypes.data,
+                              cell.ctypes.data, stat.ctypes.data), 'maps')
+        return w, cell, stat
+
+
+class StreamWriter:
+    def __init__(self, capacity):
+        self._out = np.empty(int(capacity), dtype=np.uint8)
+        self._w = _Writer()
+        _lib.glpc_writer_init(ctypes.byref(self._w),
+                              self._out.ctypes.data, self._out.size)
+
+    def put_bits(self, value, nbits):
+        _check(_lib.glpc_put_bits(ctypes.byref(self._w), value, nbits))
+
+    @property
+    def bitpos(self):
+        return int(self._w.nbytes) * 8 + int(self._w.nacc)
+
+    def encode_channel(self, cchannel, z):
+        """Encode one channel span (raster order). Returns exact bits."""
+        z = np.ascontiguousarray(z, np.int32)
+        if z.ndim != cchannel.rank:
+            raise ValueError(
+                f'tensor rank {z.ndim} != channel rank {cchannel.rank}')
+        dims = np.ascontiguousarray(z.shape, np.int64)
+        bits = _lib.glpc_encode(ctypes.byref(self._w), cchannel._buf,
+                                cchannel.rank, z.ctypes.data,
+                                dims.ctypes.data)
+        return _check(int(bits), 'encode')
+
+    def flush(self):
+        nbytes = _check(int(_lib.glpc_flush(ctypes.byref(self._w))))
+        return self._out[:nbytes].tobytes()
+
+
+class StreamReader:
+    def __init__(self, data):
+        self._buf = np.frombuffer(data, dtype=np.uint8)
+        self._r = _Reader()
+        _lib.glpc_reader_init(ctypes.byref(self._r),
+                              self._buf.ctypes.data, self._buf.size)
+
+    def get_bits(self, nbits):
+        return _check(int(_lib.glpc_get_bits(ctypes.byref(self._r),
+                                             nbits)), 'bits')
+
+    def decode_channel(self, cchannel, shape):
+        """Sequential mirror of encode_channel; returns the int32 plane."""
+        z = np.zeros(shape, np.int32)
+        dims = np.ascontiguousarray(z.shape, np.int64)
+        _check(_lib.glpc_decode(ctypes.byref(self._r), cchannel._buf,
+                                cchannel.rank, z.ctypes.data,
+                                dims.ctypes.data), 'decode')
+        return z

GGDLPC/codec.py

@@ -0,0 +1,124 @@
+"""The GGDLPC codec: blob -> coder; encode/decode of integer tensors.
+
+The primitive: one call encodes one integer tensor (C channels, rank 1/2/3
+spatial dims, values in [-N, N]) as ONE self-contained bitstream --
+byte-padded at its end, decodable from its own bytes alone, truncatable at
+any channel boundary (channel prefixes are literal bit prefixes; recorded
+per-channel bit counts locate them). C = 1 gives the single-channel call;
+everything larger (macroregions, mixed-resolution scale groups, operating
+points, files) is caller composition via the stream-level API
+(stream_writer / encode_channel / stream_reader / decode_channel), which
+shares one byte pad across the composition.
+
+The persistent model is the blob: six scalars per channel (mu; scale law
+a, b; shape law u, v) plus fixed-point predictor weights. Edges, split
+config, canonical Huffman tables, and the run mode are generated from the
+scalars at construction (tables.py). No backward compatibility with the
+FRAPPEv3 lawcoder blob or streams.
+"""
+import numpy as np
+
+from ._core import RANK_NF
+from .cbridge import CChannel, StreamReader, StreamWriter
+from .tables import build_channel
+
+SPEC = ('GGDLPC v1: unified coder (one learned causal FIR predictor per '
+        'channel with arithmetic-mean fallback; one stateless pooled '
+        'statistic: mean of the causal template abs differences and the '
+        'last 16 coded abs residuals in scan order; two per-channel '
+        'affine laws -> analytic folded GGD '
+        'conditionals on the tau=2^(1/4) scale grid; canonical Huffman '
+        'tables under the derived quotient split, <= 63 windowed hi '
+        'values + escape at any bit depth; derived zero-run mode with '
+        'elementary Golomb codes and exclusion tables, always available); '
+        'inputs in [-N, N], N <= 65536; one call = one byte-padded '
+        'bitstream, channel prefixes are literal bit prefixes')
+VERSION = 1
+
+# Worst-case bits per coded sample: <= 63 (codeword) + 18 (escape hi)
+# + 18 (lo) < 128 -> 16 bytes/sample capacity is always safe.
+_CAP_PER_SAMPLE = 16
+
+
+def new_blob(N, rank, channels, provenance=None, K_beta=None):
+    """Assemble a JSON-able codec blob from fit results."""
+    return {'spec': SPEC, 'version': VERSION, 'N': int(N),
+            'rank': int(rank), 'K_beta': K_beta,
+            'channels': list(channels),
+            'provenance': provenance or {}}
+
+
+class Codec:
+    def __init__(self, blob):
+        assert blob.get('spec') == SPEC, \
+            'GGDLPC blob spec mismatch (no back-compat with other coders)'
+        assert blob['version'] == VERSION
+        self.blob = blob
+        self.N = int(blob['N'])
+        self.rank = int(blob['rank'])
+        if self.rank not in RANK_NF:
+            raise ValueError(
+                f'GGDLPC supports ranks {sorted(RANK_NF)}; got {self.rank}')
+        if not (1 <= self.N <= 65536):
+            raise ValueError(f'N must be in [1, 65536]; got {self.N}')
+        self.n_channels = len(blob['channels'])
+        self.builds = [build_channel(ch, self.N)
+                       for ch in blob['channels']]
+        self.cchannels = [CChannel(b, self.rank) for b in self.builds]
+
+    # ------------------------------------------------------ the primitive
+
+    def encode(self, z):
+        """(C, *S) integer tensor -> (bytes, per-channel bit counts).
+
+        One self-contained byte-padded stream; channel prefixes are
+        literal bit prefixes located by the returned counts."""
+        z = np.asarray(z)
+        if z.ndim != self.rank + 1:
+            raise ValueError(
+                f'expected a (C, *S) tensor of rank {self.rank + 1}, '
+                f'got shape {z.shape}')
+        C = z.shape[0]
+        if C > self.n_channels:
+            raise ValueError(f'{C} channels but blob has {self.n_channels}')
+        w = self.stream_writer(_CAP_PER_SAMPLE * z.size + 64)
+        ch_bits = np.zeros(C, np.int64)
+        for c in range(C):
+            ch_bits[c] = self.encode_channel(w, z[c], c)
+        data = w.flush()
+        assert len(data) == (int(ch_bits.sum()) + 7) // 8
+        return data, ch_bits
+
+    def decode(self, data, shape, n_channels=None):
+        """Decode (a prefix of) an encode() stream. shape: the full
+        (C, *S); n_channels: decode only the first n channels."""
+        C = shape[0]
+        n = C if n_channels is None else min(n_channels, C)
+        r = self.stream_reader(data)
+        out = np.zeros(shape, np.int32)
+        for c in range(n):
+            out[c] = self.decode_channel(r, shape[1:], c)
+        return out
+
+    # ------------------------------------------- composition (shared pad)
+
+    def stream_writer(self, capacity):
+        return StreamWriter(capacity)
+
+    def stream_reader(self, data):
+        return StreamReader(data)
+
+    def capacity_for(self, n_samples):
+        return _CAP_PER_SAMPLE * int(n_samples) + 64
+
+    def encode_channel(self, writer, plane, channel):
+        """Code one channel span (rank-`rank` tensor) into an open
+        writer. Returns the exact bit count."""
+        return writer.encode_channel(self.cchannels[channel], plane)
+
+    def decode_channel(self, reader, shape, channel):
+        return reader.decode_channel(self.cchannels[channel], shape)
+
+    def maps(self, plane, channel):
+        """Encode-side integer maps (folded symbol, cell, statistic)."""
+        return self.cchannels[channel].maps(plane)