tilecdc 0.1.0__tar.gz

tilecdc-0.1.0/MANIFEST.in

@@ -0,0 +1,2 @@
+include csrc/tilecdc.c csrc/tilecdc.h README.md LICENSE
+recursive-include tests *.py

tilecdc-0.1.0/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: tilecdc
+Version: 0.1.0
+Summary: Fast content-defined chunking on CPU matrix units (Intel AMX), with AVX-512 and portable fallbacks that give identical boundaries
+Author: Ashutosh Rath
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/ashutosh-rath02/tilecdc
+Keywords: deduplication,chunking,cdc,fastcdc,amx,avx512,backup
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: C
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Topic :: System :: Archiving :: Backup
+Classifier: Topic :: System :: Archiving :: Compression
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+
+# tilecdc (Python)
+
+Fast content-defined chunking on CPU matrix units. See the project README: https://github.com/ashutosh-rath02/tilecdc

tilecdc-0.1.0/README.md

@@ -0,0 +1,3 @@
+# tilecdc (Python)
+
+Fast content-defined chunking on CPU matrix units. See the project README: https://github.com/ashutosh-rath02/tilecdc

tilecdc-0.1.0/_tilecdc.c

@@ -0,0 +1,164 @@
+/* CPython binding for TileCDC. */
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include "structmember.h"
+#include "tilecdc.h"
+
+typedef struct {
+    PyObject_HEAD
+    tilecdc *t;
+    tilecdc_params p;
+} ChunkerObject;
+
+static void Chunker_dealloc(ChunkerObject *self) {
+    if (self->t) tilecdc_free(self->t);
+    Py_TYPE(self)->tp_free((PyObject *)self);
+}
+
+static int parse_impl(const char *s, tilecdc_impl *out) {
+    if (!s || !strcmp(s, "auto")) *out = TILECDC_IMPL_AUTO;
+    else if (!strcmp(s, "amx")) *out = TILECDC_IMPL_AMX;
+    else if (!strcmp(s, "avx512")) *out = TILECDC_IMPL_AVX512;
+    else if (!strcmp(s, "scalar")) *out = TILECDC_IMPL_SCALAR;
+    else return -1;
+    return 0;
+}
+
+static int Chunker_init(ChunkerObject *self, PyObject *args, PyObject *kw) {
+    static char *kwlist[] = {"min_size", "avg_size", "max_size", "seed", "norm", "impl", NULL};
+    tilecdc_params p;
+    tilecdc_default_params(&p);
+    Py_ssize_t mn = (Py_ssize_t)p.min_size, av = (Py_ssize_t)p.avg_size, mx = (Py_ssize_t)p.max_size;
+    PyObject *seed_obj = NULL;
+    int norm = p.norm;
+    const char *impl = "auto";
+    if (!PyArg_ParseTupleAndKeywords(args, kw, "|nnnOis", kwlist, &mn, &av, &mx, &seed_obj, &norm, &impl)) return -1;
+    if (mn <= 0 || av <= 0 || mx <= 0) { PyErr_SetString(PyExc_ValueError, "sizes must be positive"); return -1; }
+    unsigned long long seed = 0;
+    if (seed_obj) {
+        seed = PyLong_AsUnsignedLongLong(seed_obj);
+        if (PyErr_Occurred()) { PyErr_Clear(); PyErr_SetString(PyExc_ValueError, "seed must be an integer in [0, 2**64)"); return -1; }
+    }
+    p.min_size = (size_t)mn; p.avg_size = (size_t)av; p.max_size = (size_t)mx;
+    p.seed = seed; p.norm = norm;
+    if (parse_impl(impl, &p.impl)) { PyErr_SetString(PyExc_ValueError, "impl must be auto, amx, avx512 or scalar"); return -1; }
+    tilecdc *t = tilecdc_new(&p);   /* build the new chunker before touching the old one */
+    if (!t) {
+        if (p.impl != TILECDC_IMPL_AUTO && !tilecdc_impl_available(p.impl))
+            PyErr_Format(PyExc_RuntimeError, "implementation '%s' is not available on this CPU", impl);
+        else
+            PyErr_Format(PyExc_ValueError, "invalid parameters: need 64 <= min_size < avg_size <= max_size, avg_size <= %d, 0 <= norm <= 4", TILECDC_MAX_AVG);
+        return -1;
+    }
+    if (self->t) tilecdc_free(self->t);
+    self->t = t;
+    self->p = p;
+    return 0;
+}
+
+static int ready(ChunkerObject *self) {
+    if (self->t) return 1;
+    PyErr_SetString(PyExc_RuntimeError, "Chunker is not initialised");
+    return 0;
+}
+
+static PyObject *Chunker_cut(ChunkerObject *self, PyObject *arg) {
+    if (!ready(self)) return NULL;
+    Py_buffer b;
+    if (PyObject_GetBuffer(arg, &b, PyBUF_SIMPLE) < 0) return NULL;
+    size_t r;
+    Py_BEGIN_ALLOW_THREADS
+    r = tilecdc_cut(self->t, (const uint8_t *)b.buf, (size_t)b.len);
+    Py_END_ALLOW_THREADS
+    PyBuffer_Release(&b);
+    return PyLong_FromSize_t(r);
+}
+
+/* chunk(data) -> list of chunk lengths */
+static PyObject *Chunker_chunk(ChunkerObject *self, PyObject *arg) {
+    if (!ready(self)) return NULL;
+    Py_buffer b;
+    if (PyObject_GetBuffer(arg, &b, PyBUF_SIMPLE) < 0) return NULL;
+    size_t cap = (size_t)b.len / self->p.min_size + 2;
+    size_t *ends = PyMem_RawMalloc(cap * sizeof(size_t));
+    if (!ends) { PyBuffer_Release(&b); return PyErr_NoMemory(); }
+    size_t n;
+    Py_BEGIN_ALLOW_THREADS
+    n = tilecdc_chunk_all(self->t, (const uint8_t *)b.buf, (size_t)b.len, ends, cap);
+    Py_END_ALLOW_THREADS
+    PyBuffer_Release(&b);
+    PyObject *out = PyList_New((Py_ssize_t)n);
+    if (!out) { PyMem_RawFree(ends); return NULL; }
+    size_t s = 0;
+    for (size_t i = 0; i < n; i++) {
+        PyObject *v = PyLong_FromSize_t(ends[i] - s);
+        if (!v) { Py_DECREF(out); PyMem_RawFree(ends); return NULL; }
+        PyList_SET_ITEM(out, (Py_ssize_t)i, v);
+        s = ends[i];
+    }
+    PyMem_RawFree(ends);
+    return out;
+}
+
+static PyObject *Chunker_get_impl(ChunkerObject *self, void *c) { (void)c; if (!ready(self)) return NULL; return PyUnicode_FromString(tilecdc_impl_name(tilecdc_get_impl(self->t))); }
+static PyObject *Chunker_get_min(ChunkerObject *self, void *c) { (void)c; return PyLong_FromSize_t(self->p.min_size); }
+static PyObject *Chunker_get_avg(ChunkerObject *self, void *c) { (void)c; return PyLong_FromSize_t(self->p.avg_size); }
+static PyObject *Chunker_get_max(ChunkerObject *self, void *c) { (void)c; return PyLong_FromSize_t(self->p.max_size); }
+
+static PyGetSetDef Chunker_getset[] = {
+    {"impl", (getter)Chunker_get_impl, NULL, "implementation in use: amx, avx512 or scalar", NULL},
+    {"min_size", (getter)Chunker_get_min, NULL, NULL, NULL},
+    {"avg_size", (getter)Chunker_get_avg, NULL, NULL, NULL},
+    {"max_size", (getter)Chunker_get_max, NULL, NULL, NULL},
+    {NULL, NULL, NULL, NULL, NULL},
+};
+
+static PyMethodDef Chunker_methods[] = {
+    {"cut", (PyCFunction)Chunker_cut, METH_O, "cut(data) -> length of the first chunk (pass >= max_size bytes unless at end of stream)"},
+    {"chunk", (PyCFunction)Chunker_chunk, METH_O, "chunk(data) -> list of chunk lengths covering all of data"},
+    {NULL, NULL, 0, NULL},
+};
+
+static PyTypeObject ChunkerType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    .tp_name = "tilecdc._tilecdc.Chunker",
+    .tp_basicsize = sizeof(ChunkerObject),
+    .tp_dealloc = (destructor)Chunker_dealloc,
+    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    .tp_doc = "Chunker(min_size=2048, avg_size=8192, max_size=65536, seed=0, norm=1, impl='auto')",
+    .tp_methods = Chunker_methods,
+    .tp_getset = Chunker_getset,
+    .tp_init = (initproc)Chunker_init,
+    .tp_new = PyType_GenericNew,
+};
+
+static PyObject *py_available(PyObject *self, PyObject *unused) {
+    (void)self; (void)unused;
+    PyObject *l = PyList_New(0);
+    tilecdc_impl all[3] = {TILECDC_IMPL_AMX, TILECDC_IMPL_AVX512, TILECDC_IMPL_SCALAR};
+    for (int i = 0; i < 3; i++)
+        if (tilecdc_impl_available(all[i])) {
+            PyObject *s = PyUnicode_FromString(tilecdc_impl_name(all[i]));
+            PyList_Append(l, s);
+            Py_DECREF(s);
+        }
+    return l;
+}
+
+static PyMethodDef module_methods[] = {
+    {"available_impls", py_available, METH_NOARGS, "implementations usable on this CPU, fastest first"},
+    {NULL, NULL, 0, NULL},
+};
+
+static struct PyModuleDef moddef = {PyModuleDef_HEAD_INIT, "_tilecdc", "TileCDC native core", -1, module_methods, NULL, NULL, NULL, NULL};
+
+PyMODINIT_FUNC PyInit__tilecdc(void) {
+    if (PyType_Ready(&ChunkerType) < 0) return NULL;
+    PyObject *m = PyModule_Create(&moddef);
+    if (!m) return NULL;
+    Py_INCREF(&ChunkerType);
+    PyModule_AddObject(m, "Chunker", (PyObject *)&ChunkerType);
+    PyModule_AddStringConstant(m, "__version__", TILECDC_VERSION);
+    PyModule_AddIntConstant(m, "MAX_AVG_SIZE", TILECDC_MAX_AVG);
+    return m;
+}

tilecdc-0.1.0/csrc/tilecdc.c

@@ -0,0 +1,315 @@
+/*
+ * TileCDC: content-defined chunking on CPU matrix units (see include/tilecdc.h
+ * for the normative definition). Three implementations of one function:
+ *
+ *   scalar  reference, any CPU
+ *   avx512  AVX-512 VBMI + VNNI: vpermb builds 4-byte sliding windows, vpdpbusd
+ *           multiplies them by the key (12 of each per 16 hashes)
+ *   amx     Intel AMX: a tile load with a 16-byte stride over 64-byte rows makes
+ *           the A tile a Hankel matrix of the byte stream; one TDPBUSD with a
+ *           constant Toeplitz key tile gives 256 window hashes
+ *
+ * All three consume the same S-box-substituted block and produce bit-identical
+ * candidate bitmaps, so chunk boundaries never depend on the hardware.
+ */
+#define _GNU_SOURCE
+#include "tilecdc.h"
+
+#include <cpuid.h>
+#include <errno.h>
+#include <immintrin.h>
+#include <pthread.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+#define W TILECDC_WINDOW
+#define BLOCK 1024                 /* hashes per block */
+#define BLKBYTES (BLOCK + 64)      /* bytes needed per block (window + slack) */
+#define MIXC 0x9E3779B1u
+
+struct tilecdc {
+    size_t min, avg, max;
+    uint32_t smask, lmask;         /* top-bit masks for the strict / loose test */
+    tilecdc_impl impl;
+    int8_t key[W];
+    uint8_t sbox[256] __attribute__((aligned(64)));
+    int8_t btile[16][64] __attribute__((aligned(64)));   /* Toeplitz key tile for AMX */
+    uint8_t idx[12][64] __attribute__((aligned(64)));    /* vpermb sliding-window indices */
+    int32_t keyq[12];                                    /* KEY[4g..4g+3] packed */
+};
+
+typedef void (*block_fn)(const tilecdc *t, const uint8_t *y, uint64_t S[16], uint64_t L[16]);
+
+/* ------------------------------ constants ------------------------------ */
+static uint64_t splitmix64(uint64_t *s) {
+    uint64_t z = (*s += 0x9E3779B97F4A7C15ull);
+    z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull;
+    z = (z ^ (z >> 27)) * 0x94D049BB133111EBull;
+    return z ^ (z >> 31);
+}
+
+static void derive(tilecdc *t, uint64_t seed) {
+    uint64_t s = seed ^ 0x54696C6543444321ull;   /* "TileCDC!" */
+    for (int i = 0; i < 256; i++) t->sbox[i] = (uint8_t)i;
+    for (int i = 255; i > 0; i--) {
+        int j = (int)(splitmix64(&s) % (uint64_t)(i + 1));
+        uint8_t x = t->sbox[i]; t->sbox[i] = t->sbox[j]; t->sbox[j] = x;
+    }
+    for (int d = 0; d < W; d++) {
+        int v;
+        do { v = (int)(splitmix64(&s) & 0xFF) - 128; } while (v == 0 || v == -128);
+        t->key[d] = (int8_t)v;
+    }
+    /* AMX: C[r][j] = sum_k A[r][k] * B[k][j] with A[r][k] = y[16r + k];
+       B[k][j] = KEY[k - j] gives C[r][j] = h(16r + j). TDPBUSD wants B packed
+       as 16 rows of (16 columns x 4 consecutive k). */
+    memset(t->btile, 0, sizeof t->btile);
+    for (int k = 0; k < 64; k++)
+        for (int j = 0; j < 16; j++) {
+            int d = k - j;
+            if (d >= 0 && d < W) t->btile[k / 4][j * 4 + k % 4] = t->key[d];
+        }
+    /* AVX-512: lane k of group g holds bytes y[b + 4g + k .. +3] */
+    for (int g = 0; g < 12; g++) {
+        for (int k = 0; k < 16; k++)
+            for (int j = 0; j < 4; j++) t->idx[g][k * 4 + j] = (uint8_t)(4 * g + k + j);
+        uint32_t q = 0;
+        for (int j = 0; j < 4; j++) q |= (uint32_t)(uint8_t)t->key[4 * g + j] << (8 * j);
+        t->keyq[g] = (int32_t)q;
+    }
+}
+
+/* ------------------------------ CPU support ------------------------------ */
+static int avx512_ok, amx_ok;
+static pthread_once_t detect_once = PTHREAD_ONCE_INIT;
+
+static void detect(void) {
+    __builtin_cpu_init();
+    avx512_ok = __builtin_cpu_supports("avx512f") && __builtin_cpu_supports("avx512bw") &&
+                __builtin_cpu_supports("avx512vbmi") && __builtin_cpu_supports("avx512vnni");
+    unsigned a, b, c, d;
+    if (avx512_ok && __get_cpuid_count(7, 0, &a, &b, &c, &d) && (d & (1u << 24)) && (d & (1u << 25))) {
+        /* ARCH_REQ_XCOMP_PERM, XFEATURE_XTILEDATA: ask the kernel for tile state */
+        amx_ok = syscall(SYS_arch_prctl, 0x1023, 18) == 0;
+    }
+}
+static int has_avx512(void) { pthread_once(&detect_once, detect); return avx512_ok; }
+static int has_amx(void) { pthread_once(&detect_once, detect); return amx_ok; }
+
+int tilecdc_impl_available(tilecdc_impl impl) {
+    switch (impl) {
+    case TILECDC_IMPL_SCALAR: return 1;
+    case TILECDC_IMPL_AVX512: return has_avx512();
+    case TILECDC_IMPL_AMX: return has_amx();
+    case TILECDC_IMPL_AUTO: return 1;
+    }
+    return 0;
+}
+
+const char *tilecdc_impl_name(tilecdc_impl impl) {
+    switch (impl) {
+    case TILECDC_IMPL_SCALAR: return "scalar";
+    case TILECDC_IMPL_AVX512: return "avx512";
+    case TILECDC_IMPL_AMX: return "amx";
+    default: return "auto";
+    }
+}
+
+/* ------------------------------ S-box ------------------------------ */
+static void sbox_scalar(const tilecdc *t, const uint8_t *src, uint8_t *dst, size_t n) {
+    for (size_t i = 0; i < n; i++) dst[i] = t->sbox[src[i]];
+}
+
+__attribute__((target("avx512f,avx512bw,avx512vbmi")))
+static void sbox_avx512(const tilecdc *t, const uint8_t *src, uint8_t *dst, size_t n) {   /* n % 64 == 0 */
+    __m512i t0 = _mm512_load_si512(t->sbox), t1 = _mm512_load_si512(t->sbox + 64);
+    __m512i t2 = _mm512_load_si512(t->sbox + 128), t3 = _mm512_load_si512(t->sbox + 192);
+    for (size_t i = 0; i < n; i += 64) {
+        __m512i x = _mm512_loadu_si512(src + i);
+        __m512i lo = _mm512_permutex2var_epi8(t0, x, t1), hi = _mm512_permutex2var_epi8(t2, x, t3);
+        _mm512_storeu_si512(dst + i, _mm512_mask_blend_epi8(_mm512_movepi8_mask(x), lo, hi));
+    }
+}
+
+/* ------------------------------ hash blocks ------------------------------ */
+/* y holds BLKBYTES substituted bytes starting at hash index i0; set bit b of
+   S/L when hash i0+b passes the strict/loose test. */
+static void block_scalar(const tilecdc *t, const uint8_t *y, uint64_t S[16], uint64_t L[16]) {
+    memset(S, 0, 16 * 8); memset(L, 0, 16 * 8);
+    for (int b = 0; b < BLOCK; b++) {
+        int32_t h = 0;
+        for (int d = 0; d < W; d++) h += (int32_t)t->key[d] * (int32_t)y[b + d];
+        uint32_t m = (uint32_t)h * MIXC;
+        if (!(m & t->smask)) S[b >> 6] |= 1ull << (b & 63);
+        if (!(m & t->lmask)) L[b >> 6] |= 1ull << (b & 63);
+    }
+}
+
+__attribute__((target("avx512f,avx512bw,avx512vbmi,avx512vnni")))
+static void block_avx512(const tilecdc *t, const uint8_t *y, uint64_t S[16], uint64_t L[16]) {
+    __m512i idx[12], kq[12];
+    for (int g = 0; g < 12; g++) { idx[g] = _mm512_load_si512(t->idx[g]); kq[g] = _mm512_set1_epi32(t->keyq[g]); }
+    const __m512i mul = _mm512_set1_epi32((int)MIXC), sm = _mm512_set1_epi32((int)t->smask), lm = _mm512_set1_epi32((int)t->lmask);
+    for (int w = 0; w < 16; w++) {
+        uint64_t s64 = 0, l64 = 0;
+        for (int u = 0; u < 4; u++) {
+            int b = w * 64 + u * 16;
+            __m512i base = _mm512_loadu_si512(y + b);
+            __m512i acc0 = _mm512_setzero_si512(), acc1 = _mm512_setzero_si512();
+            for (int g = 0; g < 12; g += 2) {
+                acc0 = _mm512_dpbusd_epi32(acc0, _mm512_permutexvar_epi8(idx[g], base), kq[g]);
+                acc1 = _mm512_dpbusd_epi32(acc1, _mm512_permutexvar_epi8(idx[g + 1], base), kq[g + 1]);
+            }
+            __m512i m = _mm512_mullo_epi32(_mm512_add_epi32(acc0, acc1), mul);
+            s64 |= (uint64_t)_mm512_testn_epi32_mask(m, sm) << (16 * u);
+            l64 |= (uint64_t)_mm512_testn_epi32_mask(m, lm) << (16 * u);
+        }
+        S[w] = s64; L[w] = l64;
+    }
+}
+
+typedef struct { uint8_t palette, start; uint8_t r[14]; uint16_t colsb[16]; uint8_t rows[16]; } __attribute__((packed)) tilecfg;
+
+/* Configure tiles for TileCDC, remembering any configuration the calling thread
+   had, and put it back afterwards. Tile register contents are not preserved:
+   do not call TileCDC while holding live data in tile registers. */
+__attribute__((target("amx-tile")))
+static void amx_configure(tilecfg *saved) {
+    tilecfg c;
+    memset(saved, 0, sizeof *saved);
+    _tile_storeconfig(saved);
+    memset(&c, 0, sizeof c);
+    c.palette = 1;
+    for (int i = 0; i < 8; i++) { c.colsb[i] = 64; c.rows[i] = 16; }
+    _tile_loadconfig(&c);
+}
+__attribute__((target("amx-tile")))
+static void amx_release(const tilecfg *saved) {
+    if (saved->palette) _tile_loadconfig(saved);
+    else _tile_release();
+}
+
+__attribute__((target("amx-tile,amx-int8,avx512f,avx512bw")))
+static void block_amx(const tilecdc *t, const uint8_t *y, uint64_t S[16], uint64_t L[16]) {
+    int32_t C[4][16][16] __attribute__((aligned(64)));
+    _tile_loadd(7, t->btile, 64);
+    _tile_zero(0); _tile_zero(1); _tile_zero(2); _tile_zero(3);
+    /* stride 16 < row length 64: row r of each A tile = y[256q + 16r .. +64) */
+    _tile_loadd(4, y, 16);       _tile_dpbusd(0, 4, 7);
+    _tile_loadd(5, y + 256, 16); _tile_dpbusd(1, 5, 7);
+    _tile_loadd(6, y + 512, 16); _tile_dpbusd(2, 6, 7);
+    _tile_loadd(4, y + 768, 16); _tile_dpbusd(3, 4, 7);
+    _tile_stored(0, C[0], 64); _tile_stored(1, C[1], 64); _tile_stored(2, C[2], 64); _tile_stored(3, C[3], 64);
+    const int32_t *h = &C[0][0][0];   /* row-major: h[b] = hash i0 + b */
+    const __m512i mul = _mm512_set1_epi32((int)MIXC), sm = _mm512_set1_epi32((int)t->smask), lm = _mm512_set1_epi32((int)t->lmask);
+    for (int w = 0; w < 16; w++) {
+        uint64_t s64 = 0, l64 = 0;
+        for (int u = 0; u < 4; u++) {
+            __m512i m = _mm512_mullo_epi32(_mm512_load_si512(h + w * 64 + u * 16), mul);
+            s64 |= (uint64_t)_mm512_testn_epi32_mask(m, sm) << (16 * u);
+            l64 |= (uint64_t)_mm512_testn_epi32_mask(m, lm) << (16 * u);
+        }
+        S[w] = s64; L[w] = l64;
+    }
+}
+
+/* ------------------------------ cut search ------------------------------ */
+static size_t cut_core(tilecdc *t, const uint8_t *p, size_t n) {
+    if (n <= t->min) return n;
+    size_t end = n < t->max ? n : t->max, mid = n < t->avg ? n : t->avg;
+    block_fn blk = t->impl == TILECDC_IMPL_AMX ? block_amx : t->impl == TILECDC_IMPL_AVX512 ? block_avx512 : block_scalar;
+    int vec = t->impl != TILECDC_IMPL_SCALAR;
+    uint8_t raw[BLKBYTES] __attribute__((aligned(64)));
+    uint8_t y[BLKBYTES] __attribute__((aligned(64)));
+    uint64_t S[16], L[16];
+    for (size_t i0 = t->min + 1 - W;; i0 += BLOCK) {
+        size_t c0 = i0 + W;                        /* chunk end tested by bit 0 */
+        if (c0 > end) return end;
+        /* substitute the bytes of this block; never read past n */
+        const uint8_t *src = p + i0;
+        size_t avail = n - i0;
+        if (avail < BLKBYTES) { memcpy(raw, src, avail); memset(raw + avail, 0, BLKBYTES - avail); src = raw; }
+        if (vec) sbox_avx512(t, src, y, BLKBYTES);
+        else sbox_scalar(t, src, y, BLKBYTES);
+        blk(t, y, S, L);
+        for (int w = 0; w < 16; w++) {
+            size_t cw = c0 + (size_t)w * 64;       /* chunk end for bit 0 of this word */
+            if (cw > end) return end;
+            uint64_t sel_s = 0, sel_l = 0;
+            /* bits whose end c = cw + b lies in (min, mid] are strict, (mid, end] loose */
+            size_t ns = mid >= cw ? mid - cw + 1 : 0;               /* number of strict-region bits */
+            size_t ne = end >= cw ? end - cw + 1 : 0;               /* bits up to end */
+            uint64_t ms = ns >= 64 ? ~0ull : ((1ull << ns) - 1);
+            uint64_t me = ne >= 64 ? ~0ull : ((1ull << ne) - 1);
+            sel_s = S[w] & ms;
+            if (sel_s) return cw + (size_t)__builtin_ctzll(sel_s);
+            sel_l = L[w] & me & ~ms;
+            if (sel_l) {
+                /* a strict hit later in this word cannot precede a loose hit
+                   only if the strict region already ended: guaranteed by ~ms */
+                return cw + (size_t)__builtin_ctzll(sel_l);
+            }
+        }
+    }
+}
+
+size_t tilecdc_cut(tilecdc *t, const uint8_t *data, size_t len) {
+    if (t->impl == TILECDC_IMPL_AMX && len > t->min) {
+        tilecfg saved;
+        amx_configure(&saved);
+        size_t r = cut_core(t, data, len);
+        amx_release(&saved);
+        return r;
+    }
+    return cut_core(t, data, len);
+}
+
+size_t tilecdc_chunk_all(tilecdc *t, const uint8_t *data, size_t len, size_t *ends, size_t cap) {
+    size_t n = 0, s = 0;
+    tilecfg saved;
+    if (t->impl == TILECDC_IMPL_AMX) amx_configure(&saved);
+    while (s < len) {
+        s += cut_core(t, data + s, len - s);
+        if (n < cap) ends[n] = s;
+        n++;
+    }
+    if (t->impl == TILECDC_IMPL_AMX) amx_release(&saved);
+    return n;
+}
+
+/* ------------------------------ setup ------------------------------ */
+void tilecdc_default_params(tilecdc_params *p) {
+    p->min_size = 2048;
+    p->avg_size = 8192;
+    p->max_size = 65536;
+    p->seed = 0;
+    p->norm = 1;
+    p->impl = TILECDC_IMPL_AUTO;
+}
+
+tilecdc *tilecdc_new(const tilecdc_params *params) {
+    tilecdc_params p;
+    if (params) p = *params; else tilecdc_default_params(&p);
+    if (p.min_size < 64 || p.avg_size <= p.min_size || p.max_size < p.avg_size) { errno = EINVAL; return NULL; }
+    if (p.avg_size > TILECDC_MAX_AVG || p.norm < 0 || p.norm > 4) { errno = EINVAL; return NULL; }
+    int lg = 63 - __builtin_clzll((unsigned long long)p.avg_size);
+    int sb = lg + p.norm, lb = lg - p.norm - 1;
+    if (sb > 31 || lb < 1) { errno = EINVAL; return NULL; }
+    tilecdc_impl impl = p.impl;
+    if (impl == TILECDC_IMPL_AUTO) impl = has_amx() ? TILECDC_IMPL_AMX : has_avx512() ? TILECDC_IMPL_AVX512 : TILECDC_IMPL_SCALAR;
+    else if (!tilecdc_impl_available(impl)) { errno = ENOTSUP; return NULL; }
+    tilecdc *t = aligned_alloc(64, (sizeof(tilecdc) + 63) & ~(size_t)63);
+    if (!t) return NULL;
+    memset(t, 0, sizeof *t);
+    t->min = p.min_size; t->avg = p.avg_size; t->max = p.max_size;
+    t->smask = 0xFFFFFFFFu << (32 - sb);
+    t->lmask = 0xFFFFFFFFu << (32 - lb);
+    t->impl = impl;
+    derive(t, p.seed);
+    return t;
+}
+
+void tilecdc_free(tilecdc *t) { free(t); }
+tilecdc_impl tilecdc_get_impl(const tilecdc *t) { return t->impl; }

tilecdc-0.1.0/csrc/tilecdc.h

@@ -0,0 +1,82 @@
+/*
+ * TileCDC: content-defined chunking on CPU matrix units.
+ *
+ * TileCDC computes a windowed fingerprint at every byte offset with Intel AMX
+ * tile instructions: a tile load whose row stride (16 bytes) is shorter than
+ * its rows (64 bytes) forms a Hankel matrix of the byte stream without
+ * copying, and one TDPBUSD against a constant Toeplitz key tile yields 256
+ * window hashes. Boundaries follow FastCDC-style normalized chunking.
+ *
+ * The chunking function is defined independently of the hardware (below), and
+ * the AMX, AVX-512 and scalar implementations return identical boundaries.
+ *
+ * Definition (seed-dependent constants KEY[48] in [-127,127]\{0} and a byte
+ * permutation SBOX, both derived from the seed with splitmix64):
+ *   y[j]  = SBOX[x[j]]
+ *   h(i)  = sum_{d=0}^{47} KEY[d] * y[i+d]               (exact, int32)
+ *   m(i)  = (uint32) h(i) * 0x9E3779B1                     (mod 2^32)
+ *   a cut is allowed after byte c-1 (chunk end c) under b bits when the top b
+ *   bits of m(c-48) are zero.
+ *   For a buffer of n bytes: if n <= min the chunk is the whole buffer;
+ *   otherwise the chunk ends at the first allowed c in (min, min(n,avg)] under
+ *   the strict bits, else the first in (min(n,avg), min(n,max)] under the
+ *   loose bits, else at min(n,max).
+ *   strict bits = floor(log2(avg)) + norm, loose = floor(log2(avg)) - norm - 1.
+ */
+#ifndef TILECDC_H
+#define TILECDC_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define TILECDC_WINDOW 48
+/* Largest supported average chunk size. The window hash carries about 17 bits
+   of entropy, so masks of 17+ bits quantize: at a 64 KiB target the mean chunk
+   size varied from 55 to 74 KiB across seeds (32 KiB: 32 to 37 KiB). */
+#define TILECDC_MAX_AVG 32768
+#define TILECDC_VERSION "0.1.0"
+
+typedef enum { TILECDC_IMPL_AUTO = 0, TILECDC_IMPL_SCALAR = 1, TILECDC_IMPL_AVX512 = 2, TILECDC_IMPL_AMX = 3 } tilecdc_impl;
+
+typedef struct tilecdc tilecdc;
+
+typedef struct tilecdc_params {
+    size_t min_size;   /* default 2048; must be >= 64 (one 48-byte window plus slack) */
+    size_t avg_size;   /* default 8192; at most TILECDC_MAX_AVG */
+    size_t max_size;   /* default 65536 */
+    uint64_t seed;     /* default 0; changes KEY and SBOX */
+    int norm;          /* normalization level 0..4, default 1 */
+    tilecdc_impl impl; /* default AUTO: AMX if usable, else AVX-512, else scalar */
+} tilecdc_params;
+
+/* Fill p with defaults. */
+void tilecdc_default_params(tilecdc_params *p);
+
+/* Create a chunker; NULL params means defaults. Returns NULL with errno
+   EINVAL for invalid parameters, ENOTSUP when a forced implementation is
+   unavailable. Thread safety: a chunker holds no mutable state and may be
+   used from several threads at once. */
+tilecdc *tilecdc_new(const tilecdc_params *params);
+void tilecdc_free(tilecdc *t);
+
+/* Length of the first chunk of data[0..len). In a stream, pass at least
+   max_size bytes unless the stream has ended (the last chunk may be short). */
+size_t tilecdc_cut(tilecdc *t, const uint8_t *data, size_t len);
+
+/* Chunk a whole buffer: writes chunk end offsets into ends (capacity cap)
+   and returns the number of chunks (which may exceed cap: only cap are written). */
+size_t tilecdc_chunk_all(tilecdc *t, const uint8_t *data, size_t len, size_t *ends, size_t cap);
+
+/* Which implementation this chunker uses, and what the CPU supports. */
+tilecdc_impl tilecdc_get_impl(const tilecdc *t);
+const char *tilecdc_impl_name(tilecdc_impl impl);
+int tilecdc_impl_available(tilecdc_impl impl);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

tilecdc-0.1.0/pyproject.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["setuptools>=64", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tilecdc"
+version = "0.1.0"
+description = "Fast content-defined chunking on CPU matrix units (Intel AMX), with AVX-512 and portable fallbacks that give identical boundaries"
+readme = "README.md"
+license = { text = "Apache-2.0" }
+authors = [{ name = "Ashutosh Rath" }]
+requires-python = ">=3.9"
+keywords = ["deduplication", "chunking", "cdc", "fastcdc", "amx", "avx512", "backup"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: C",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: POSIX :: Linux",
+    "Topic :: System :: Archiving :: Backup",
+    "Topic :: System :: Archiving :: Compression",
+]
+
+[project.urls]
+Homepage = "https://github.com/ashutosh-rath02/tilecdc"
+
+[project.scripts]
+tilecdc = "tilecdc.__main__:main"
+
+[tool.setuptools]
+packages = ["tilecdc"]

tilecdc-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

tilecdc-0.1.0/setup.py

@@ -0,0 +1,30 @@
+import os
+import platform
+import shutil
+import sys
+
+from setuptools import Extension, setup
+
+if not sys.platform.startswith("linux") or platform.machine().lower() not in ("x86_64", "amd64"):
+    sys.exit("tilecdc 0.1 supports x86-64 Linux only (it uses AMX/AVX-512 intrinsics and Linux arch_prctl).")
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+ROOT = os.path.dirname(HERE)
+
+# Vendor the C core so sdists build standalone.
+VENDOR = os.path.join(HERE, "csrc")
+if os.path.exists(os.path.join(ROOT, "src", "tilecdc.c")):
+    os.makedirs(VENDOR, exist_ok=True)
+    shutil.copy(os.path.join(ROOT, "src", "tilecdc.c"), VENDOR)
+    shutil.copy(os.path.join(ROOT, "include", "tilecdc.h"), VENDOR)
+
+setup(
+    ext_modules=[
+        Extension(
+            "tilecdc._tilecdc",
+            sources=["_tilecdc.c", "csrc/tilecdc.c"],
+            include_dirs=["csrc"],
+            extra_compile_args=["-O3"],
+        )
+    ],
+)

tilecdc-0.1.0/tests/test_python.py

@@ -0,0 +1,93 @@
+"""Tests for the Python package: identical boundaries across implementations,
+streaming equals in-memory chunking, and argument validation."""
+import io
+import os
+import random
+
+import pytest
+
+import tilecdc
+
+DATA = random.Random(1).randbytes(6 << 20) + b"\x00" * 200_000 + b"abc" * 100_000
+
+
+def test_available():
+    impls = tilecdc.available_impls()
+    assert "scalar" in impls
+
+
+@pytest.mark.parametrize("impl", ["avx512", "amx"])
+def test_impls_identical(impl):
+    if impl not in tilecdc.available_impls():
+        pytest.skip(f"{impl} not available")
+    ref = tilecdc.Chunker(impl="scalar").chunk(DATA[: 1 << 20])
+    assert tilecdc.Chunker(impl=impl).chunk(DATA[: 1 << 20]) == ref
+
+
+def test_cover_and_bounds():
+    c = tilecdc.Chunker()
+    lengths = c.chunk(DATA)
+    assert sum(lengths) == len(DATA)
+    assert all(l <= c.max_size for l in lengths)
+    assert all(l > c.min_size for l in lengths[:-1])
+
+
+def test_stream_equals_memory():
+    c = tilecdc.Chunker()
+    mem = c.chunk(DATA)
+    for bs in (c.max_size, c.max_size + 1, 1 << 20, 3 << 20):
+        streamed = [l for _, l in c._iter_stream(io.BytesIO(DATA), False, bs)]
+        assert streamed == mem, bs
+
+
+def test_iter_file_with_data(tmp_path):
+    p = tmp_path / "f.bin"
+    p.write_bytes(DATA)
+    c = tilecdc.Chunker(avg_size=16384, min_size=4096, max_size=131072)
+    joined = b"".join(d for _, _, d in c.iter_file(str(p), with_data=True))
+    assert joined == DATA
+
+
+def test_seed_changes_boundaries():
+    a = tilecdc.Chunker(seed=1).chunk(DATA[: 1 << 20])
+    b = tilecdc.Chunker(seed=2).chunk(DATA[: 1 << 20])
+    assert a != b
+
+
+def test_small_inputs():
+    c = tilecdc.Chunker()
+    assert c.chunk(b"") == []
+    assert c.chunk(b"x") == [1]
+    assert c.chunk(b"y" * 2048) == [2048]
+
+
+def test_bad_args():
+    with pytest.raises(ValueError):
+        tilecdc.Chunker(min_size=10)
+    with pytest.raises(ValueError):
+        tilecdc.Chunker(impl="gpu")
+
+
+def test_uninitialised_and_failed_reinit_do_not_crash():
+    c = tilecdc.Chunker.__new__(tilecdc.Chunker)
+    with pytest.raises(RuntimeError):
+        c.chunk(b"x")
+    d = tilecdc.Chunker()
+    before = d.chunk(DATA[:100000])
+    with pytest.raises(ValueError):
+        d.__init__(min_size=10)
+    assert d.chunk(DATA[:100000]) == before   # old chunker still intact
+
+
+def test_small_buffer_size_terminates():
+    c = tilecdc.Chunker()
+    out = [l for _, l in c.iter_file(io.BytesIO(DATA), buffer_size=1000)]
+    assert out == c.chunk(DATA)
+
+
+def test_avg_cap_and_seed_validation():
+    with pytest.raises(ValueError):
+        tilecdc.Chunker(min_size=16384, avg_size=65536, max_size=524288)
+    with pytest.raises(ValueError):
+        tilecdc.Chunker(seed=-1)
+    assert tilecdc.Chunker(seed=2**64 - 1).chunk(b"a" * 5000)

tilecdc-0.1.0/tilecdc/__init__.py

@@ -0,0 +1,81 @@
+"""TileCDC: fast content-defined chunking on CPU matrix units.
+
+    import tilecdc
+
+    c = tilecdc.Chunker()                 # 2 KiB / 8 KiB / 64 KiB
+    lengths = c.chunk(data)               # chunk an in-memory buffer
+    for off, length in c.iter_file("big.tar"):
+        ...
+
+Boundaries are identical on every CPU: AMX, AVX-512 and the portable scalar
+path implement the same function, so chunks dedup across machines.
+"""
+import os
+
+from ._tilecdc import MAX_AVG_SIZE, Chunker as _Chunker, available_impls, __version__
+
+__all__ = ["Chunker", "available_impls", "chunk", "MAX_AVG_SIZE", "__version__"]
+
+
+class Chunker(_Chunker):
+    """Chunker(min_size=2048, avg_size=8192, max_size=65536, seed=0, norm=1, impl='auto')
+
+    impl is 'auto' (fastest available), 'amx', 'avx512' or 'scalar'.
+    The seed changes where boundaries fall; use the same seed everywhere
+    data should deduplicate. avg_size may be at most MAX_AVG_SIZE (32 KiB).
+    """
+
+    def iter_file(self, file, with_data=False, buffer_size=None):
+        """Yield (offset, length) or (offset, length, bytes) for each chunk of
+        a path or a binary file object, reading it in large blocks."""
+        close = False
+        if isinstance(file, (str, bytes, os.PathLike)):
+            file = open(file, "rb")
+            close = True
+        try:
+            # the buffer must hold at least two maximum-size chunks to make progress
+            size = max(buffer_size or 4 << 20, 16 * self.max_size if buffer_size is None else 2 * self.max_size)
+            yield from self._iter_stream(file, with_data, size)
+        finally:
+            if close:
+                file.close()
+
+    def _iter_stream(self, f, with_data, bufsize):
+        buf = bytearray()
+        base = 0
+        eof = False
+        while True:
+            if not eof and len(buf) < bufsize:
+                block = f.read(bufsize - len(buf))
+                if not block:
+                    eof = True
+                else:
+                    buf += block
+            if not buf:
+                return
+            view = memoryview(buf)
+            used = 0
+            # One C call chunks the whole buffer. A chunk that starts at s is
+            # final once bytes up to s + max_size are visible (or at end of file),
+            # because its cut never depends on later bytes.
+            limit = len(buf) if eof else len(buf) - self.max_size
+            for n in self.chunk(view):
+                if used > limit or (not eof and used + n > len(buf)):
+                    break
+                if not eof and used + self.max_size > len(buf):
+                    break
+                if with_data:
+                    yield base + used, n, bytes(view[used:used + n])
+                else:
+                    yield base + used, n
+                used += n
+            view.release()
+            del buf[:used]
+            base += used
+            if eof and not buf:
+                return
+
+
+def chunk(data, **kwargs):
+    """Chunk a bytes-like object; returns the list of chunk lengths."""
+    return Chunker(**kwargs).chunk(data)

tilecdc-0.1.0/tilecdc/__main__.py

@@ -0,0 +1,37 @@
+"""python -m tilecdc [-s] [--avg N] [--seed N] [--impl X] FILE...
+
+Prints "file offset length" per chunk, or a per-file summary with -s.
+"""
+import argparse
+import sys
+import time
+
+from . import Chunker, available_impls
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="tilecdc", description="Content-defined chunking on CPU matrix units")
+    ap.add_argument("files", nargs="+")
+    ap.add_argument("-s", "--summary", action="store_true", help="print a summary per file instead of every chunk")
+    ap.add_argument("--avg", type=int, default=8192, help="average chunk size (min = avg/4, max = avg*8)")
+    ap.add_argument("--seed", type=int, default=0)
+    ap.add_argument("--impl", default="auto", choices=["auto", "amx", "avx512", "scalar"])
+    a = ap.parse_args(argv)
+    c = Chunker(min_size=a.avg // 4, avg_size=a.avg, max_size=a.avg * 8, seed=a.seed, impl=a.impl)
+    out = sys.stdout
+    for path in a.files:
+        t0 = time.perf_counter()
+        n = total = 0
+        for off, length in c.iter_file(path):
+            n += 1
+            total += length
+            if not a.summary:
+                out.write(f"{path} {off} {length}\n")
+        dt = time.perf_counter() - t0
+        if a.summary:
+            avg = total / n if n else 0
+            out.write(f"{path}: {total} bytes, {n} chunks, avg {avg:.0f} B, {total / dt / 1e9:.2f} GB/s incl. I/O ({c.impl}; available: {', '.join(available_impls())})\n")
+
+
+if __name__ == "__main__":
+    main()

tilecdc-0.1.0/tilecdc.egg-info/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: tilecdc
+Version: 0.1.0
+Summary: Fast content-defined chunking on CPU matrix units (Intel AMX), with AVX-512 and portable fallbacks that give identical boundaries
+Author: Ashutosh Rath
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/ashutosh-rath02/tilecdc
+Keywords: deduplication,chunking,cdc,fastcdc,amx,avx512,backup
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: C
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Topic :: System :: Archiving :: Backup
+Classifier: Topic :: System :: Archiving :: Compression
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+
+# tilecdc (Python)
+
+Fast content-defined chunking on CPU matrix units. See the project README: https://github.com/ashutosh-rath02/tilecdc

tilecdc-0.1.0/tilecdc.egg-info/SOURCES.txt

@@ -0,0 +1,15 @@
+MANIFEST.in
+README.md
+_tilecdc.c
+pyproject.toml
+setup.py
+csrc/tilecdc.c
+csrc/tilecdc.h
+tests/test_python.py
+tilecdc/__init__.py
+tilecdc/__main__.py
+tilecdc.egg-info/PKG-INFO
+tilecdc.egg-info/SOURCES.txt
+tilecdc.egg-info/dependency_links.txt
+tilecdc.egg-info/entry_points.txt
+tilecdc.egg-info/top_level.txt

tilecdc-0.1.0/tilecdc.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

tilecdc-0.1.0/tilecdc.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+tilecdc = tilecdc.__main__:main

tilecdc-0.1.0/tilecdc.egg-info/top_level.txt

@@ -0,0 +1 @@
+tilecdc