kolmoc 1.0.0__tar.gz

kolmoc-1.0.0/MANIFEST.in

@@ -0,0 +1,7 @@
+# Ship the native core in the sdist so `pip install kolmo` can build it anywhere.
+graft native/src
+graft native/vendor
+include README.md
+include sync-native.sh
+global-exclude *.o *.a *.so *.dylib
+prune native/**/build

kolmoc-1.0.0/PKG-INFO

@@ -0,0 +1,36 @@
+Metadata-Version: 2.4
+Name: kolmoc
+Version: 1.0.0
+Summary: kolmo — structure-aware lossless compressor: best ratio across tables, logs and scientific data, with decode that scales across cores. Installs as `kolmoc`, imports as `kolmo`.
+Author: Adnane Aqartit and contributors
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/adnaneben/kolmo
+Project-URL: Repository, https://github.com/adnaneben/kolmo
+Keywords: compression,lossless,columnar,zstd,simd
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Topic :: System :: Archiving :: Compression
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+
+# kolmo (Python binding)
+
+Python bindings for [kolmo](https://github.com/adnaneben/kolmo), a structure-aware lossless
+compressor. Thin `ctypes` wrappers over the native core; the shared library is built and
+bundled at install time, so there is no runtime build dependency.
+
+```bash
+pip install kolmoc   # the distribution is "kolmoc" ("kolmo" was taken on PyPI); it imports as kolmo
+```
+
+```python
+import kolmo
+blob = kolmo.compress(open("data.csv", "rb").read(), mode="hc")  # rt|fast|eco|hc|max|ultra
+data = kolmo.decompress(blob)                                     # whole file
+part = kolmo.read_range(blob, 1_000_000, 4096)                    # decode only the overlapping blocks
+```
+
+`threads=0` (the default) uses all cores. Apache-2.0.
+
+Installing from source needs a C toolchain and `liblzma`. Distributable wheels (one per
+platform, no toolchain needed) are produced in CI with `cibuildwheel` / `python -m build`.

kolmoc-1.0.0/README.md

@@ -0,0 +1,21 @@
+# kolmo (Python binding)
+
+Python bindings for [kolmo](https://github.com/adnaneben/kolmo), a structure-aware lossless
+compressor. Thin `ctypes` wrappers over the native core; the shared library is built and
+bundled at install time, so there is no runtime build dependency.
+
+```bash
+pip install kolmoc   # the distribution is "kolmoc" ("kolmo" was taken on PyPI); it imports as kolmo
+```
+
+```python
+import kolmo
+blob = kolmo.compress(open("data.csv", "rb").read(), mode="hc")  # rt|fast|eco|hc|max|ultra
+data = kolmo.decompress(blob)                                     # whole file
+part = kolmo.read_range(blob, 1_000_000, 4096)                    # decode only the overlapping blocks
+```
+
+`threads=0` (the default) uses all cores. Apache-2.0.
+
+Installing from source needs a C toolchain and `liblzma`. Distributable wheels (one per
+platform, no toolchain needed) are produced in CI with `cibuildwheel` / `python -m build`.

kolmoc-1.0.0/kolmo.py

@@ -0,0 +1,109 @@
+"""kolmo — Python binding for the kolmo compressor (ctypes over libkolmo).
+
+    import kolmo
+    blob = kolmo.compress(open("data.csv","rb").read(), mode="hc")
+    data = kolmo.decompress(blob)                 # whole file
+    part = kolmo.read_range(blob, 1_000_000, 4096) # bytes [off, off+len) without decoding the rest
+
+Build the library first:  make lib   (produces build/libkolmo.so / .dylib).
+"""
+import ctypes, os, sys
+
+_MODES = {"rt": 0, "fast": 1, "eco": 2, "hc": 3, "max": 4, "ultra": 5}
+
+def _load():
+    here = os.path.dirname(os.path.abspath(__file__))
+    name = "libkolmo.dylib" if sys.platform == "darwin" else "libkolmo.so"
+    for p in (os.environ.get("KOLMO_LIB"), os.path.join(here, name), os.path.join(here, "..", "..", "build", name), name):
+        if not p:
+            continue
+        try:
+            return ctypes.CDLL(p)
+        except OSError:
+            continue
+    raise OSError("libkolmo not found; run `make lib` and/or set KOLMO_LIB to the shared library path")
+
+_lib = _load()
+
+class _Params(ctypes.Structure):
+    _fields_ = [("mode", ctypes.c_int), ("level", ctypes.c_int), ("threads", ctypes.c_int),
+                ("block", ctypes.c_uint32), ("transforms", ctypes.c_int)]
+
+_lib.czn_bound.restype = ctypes.c_size_t
+_lib.czn_bound.argtypes = [ctypes.c_size_t, ctypes.POINTER(_Params)]
+_lib.czn_compress.restype = ctypes.c_int64
+_lib.czn_compress.argtypes = [ctypes.c_char_p, ctypes.c_size_t, ctypes.c_char_p, ctypes.c_size_t, ctypes.POINTER(_Params)]
+_lib.czn_original_size.restype = ctypes.c_int64
+_lib.czn_original_size.argtypes = [ctypes.c_char_p, ctypes.c_size_t]
+_lib.czn_decompress.restype = ctypes.c_int64
+_lib.czn_decompress.argtypes = [ctypes.c_char_p, ctypes.c_size_t, ctypes.c_char_p, ctypes.c_size_t, ctypes.c_int]
+_lib.czn_read_range.restype = ctypes.c_int64
+_lib.czn_read_range.argtypes = [ctypes.c_char_p, ctypes.c_size_t, ctypes.c_uint64, ctypes.c_size_t, ctypes.c_char_p, ctypes.c_int]
+_lib.czn_strerror.restype = ctypes.c_char_p
+_lib.czn_strerror.argtypes = [ctypes.c_int64]
+_lib.czn_train_dict.restype = ctypes.c_int64
+_lib.czn_train_dict.argtypes = [ctypes.c_char_p, ctypes.POINTER(ctypes.c_size_t), ctypes.c_size_t,
+                                ctypes.c_char_p, ctypes.c_size_t]
+_lib.czn_compress_dict.restype = ctypes.c_int64
+_lib.czn_compress_dict.argtypes = [ctypes.c_char_p, ctypes.c_size_t, ctypes.c_char_p, ctypes.c_size_t,
+                                   ctypes.POINTER(_Params), ctypes.c_char_p, ctypes.c_size_t]
+_lib.czn_decompress_dict.restype = ctypes.c_int64
+_lib.czn_decompress_dict.argtypes = [ctypes.c_char_p, ctypes.c_size_t, ctypes.c_char_p, ctypes.c_size_t,
+                                     ctypes.c_int, ctypes.c_char_p, ctypes.c_size_t]
+
+def _check(code):
+    if code < 0:
+        raise ValueError("kolmo: " + _lib.czn_strerror(code).decode())
+    return code
+
+def original_size(blob):
+    """Original byte length recorded in a compressed blob's header."""
+    blob = bytes(blob)
+    return _check(_lib.czn_original_size(blob, len(blob)))
+
+def read_range(blob, off, length, threads=0):
+    """Return bytes [off, off+length) of the original, decoding only the blocks that overlap."""
+    blob = bytes(blob)
+    out = ctypes.create_string_buffer(length if length else 1)
+    got = _check(_lib.czn_read_range(blob, len(blob), off, length, out, threads))
+    return out.raw[:got]
+
+def train_dict(samples, dict_cap=112 * 1024):
+    """Train a dictionary from an iterable of sample bytes (small, similar files). Returns the
+    dictionary bytes. Use it with compress(..., dict=d) / decompress(..., dict=d)."""
+    samples = [bytes(s) for s in samples]
+    if not samples:
+        raise ValueError("train_dict: need at least one sample")
+    blob = b"".join(samples)
+    sizes = (ctypes.c_size_t * len(samples))(*[len(s) for s in samples])
+    out = ctypes.create_string_buffer(dict_cap)
+    n = _check(_lib.czn_train_dict(blob, sizes, len(samples), out, dict_cap))
+    return out.raw[:n]
+
+def compress(data, mode="fast", threads=0, block=0, transforms=True, level=0, dict=None):  # noqa: A002
+    """Compress bytes; returns the compressed bytes. mode: rt|fast|eco|hc|max|ultra.
+    Pass dict=<bytes from train_dict> to compress small files against a trained dictionary."""
+    if mode not in _MODES:
+        raise ValueError("mode must be one of " + ", ".join(_MODES))
+    data = bytes(data)
+    p = _Params(_MODES[mode], level, threads, block, 1 if transforms else 0)
+    cap = _lib.czn_bound(len(data), ctypes.byref(p))
+    out = ctypes.create_string_buffer(cap)
+    if dict:
+        d = bytes(dict)
+        n = _check(_lib.czn_compress_dict(data, len(data), out, cap, ctypes.byref(p), d, len(d)))
+    else:
+        n = _check(_lib.czn_compress(data, len(data), out, cap, ctypes.byref(p)))
+    return out.raw[:n]
+
+def decompress(blob, threads=0, dict=None):  # noqa: A002
+    """Decompress a whole blob; returns the original bytes. Pass the same dict used to compress."""
+    blob = bytes(blob)
+    n = original_size(blob)
+    out = ctypes.create_string_buffer(n if n else 1)
+    if dict:
+        d = bytes(dict)
+        got = _check(_lib.czn_decompress_dict(blob, len(blob), out, n, threads, d, len(d)))
+    else:
+        got = _check(_lib.czn_decompress(blob, len(blob), out, n, threads))
+    return out.raw[:got]

kolmoc-1.0.0/kolmoc.egg-info/PKG-INFO

@@ -0,0 +1,36 @@
+Metadata-Version: 2.4
+Name: kolmoc
+Version: 1.0.0
+Summary: kolmo — structure-aware lossless compressor: best ratio across tables, logs and scientific data, with decode that scales across cores. Installs as `kolmoc`, imports as `kolmo`.
+Author: Adnane Aqartit and contributors
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/adnaneben/kolmo
+Project-URL: Repository, https://github.com/adnaneben/kolmo
+Keywords: compression,lossless,columnar,zstd,simd
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Topic :: System :: Archiving :: Compression
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+
+# kolmo (Python binding)
+
+Python bindings for [kolmo](https://github.com/adnaneben/kolmo), a structure-aware lossless
+compressor. Thin `ctypes` wrappers over the native core; the shared library is built and
+bundled at install time, so there is no runtime build dependency.
+
+```bash
+pip install kolmoc   # the distribution is "kolmoc" ("kolmo" was taken on PyPI); it imports as kolmo
+```
+
+```python
+import kolmo
+blob = kolmo.compress(open("data.csv", "rb").read(), mode="hc")  # rt|fast|eco|hc|max|ultra
+data = kolmo.decompress(blob)                                     # whole file
+part = kolmo.read_range(blob, 1_000_000, 4096)                    # decode only the overlapping blocks
+```
+
+`threads=0` (the default) uses all cores. Apache-2.0.
+
+Installing from source needs a C toolchain and `liblzma`. Distributable wheels (one per
+platform, no toolchain needed) are produced in CI with `cibuildwheel` / `python -m build`.

kolmoc-1.0.0/kolmoc.egg-info/SOURCES.txt

@@ -0,0 +1,295 @@
+MANIFEST.in
+README.md
+kolmo.py
+pyproject.toml
+setup.py
+sync-native.sh
+kolmoc.egg-info/PKG-INFO
+kolmoc.egg-info/SOURCES.txt
+kolmoc.egg-info/dependency_links.txt
+kolmoc.egg-info/top_level.txt
+native/src/cli.c
+native/src/cm.inc
+native/src/czn.c
+native/src/czn.h
+native/src/log.inc
+native/src/sci.inc
+native/src/stream.inc
+native/src/tbl2.inc
+native/src/word.inc
+native/vendor/brotli/LICENSE
+native/vendor/brotli/common/constants.c
+native/vendor/brotli/common/constants.h
+native/vendor/brotli/common/context.c
+native/vendor/brotli/common/context.h
+native/vendor/brotli/common/dictionary.bin
+native/vendor/brotli/common/dictionary.bin.br
+native/vendor/brotli/common/dictionary.c
+native/vendor/brotli/common/dictionary.h
+native/vendor/brotli/common/platform.c
+native/vendor/brotli/common/platform.h
+native/vendor/brotli/common/shared_dictionary.c
+native/vendor/brotli/common/shared_dictionary_internal.h
+native/vendor/brotli/common/transform.c
+native/vendor/brotli/common/transform.h
+native/vendor/brotli/common/version.h
+native/vendor/brotli/dec/bit_reader.c
+native/vendor/brotli/dec/bit_reader.h
+native/vendor/brotli/dec/decode.c
+native/vendor/brotli/dec/huffman.c
+native/vendor/brotli/dec/huffman.h
+native/vendor/brotli/dec/prefix.h
+native/vendor/brotli/dec/state.c
+native/vendor/brotli/dec/state.h
+native/vendor/brotli/enc/backward_references.c
+native/vendor/brotli/enc/backward_references.h
+native/vendor/brotli/enc/backward_references_hq.c
+native/vendor/brotli/enc/backward_references_hq.h
+native/vendor/brotli/enc/backward_references_inc.h
+native/vendor/brotli/enc/bit_cost.c
+native/vendor/brotli/enc/bit_cost.h
+native/vendor/brotli/enc/bit_cost_inc.h
+native/vendor/brotli/enc/block_encoder_inc.h
+native/vendor/brotli/enc/block_splitter.c
+native/vendor/brotli/enc/block_splitter.h
+native/vendor/brotli/enc/block_splitter_inc.h
+native/vendor/brotli/enc/brotli_bit_stream.c
+native/vendor/brotli/enc/brotli_bit_stream.h
+native/vendor/brotli/enc/cluster.c
+native/vendor/brotli/enc/cluster.h
+native/vendor/brotli/enc/cluster_inc.h
+native/vendor/brotli/enc/command.c
+native/vendor/brotli/enc/command.h
+native/vendor/brotli/enc/compound_dictionary.c
+native/vendor/brotli/enc/compound_dictionary.h
+native/vendor/brotli/enc/compress_fragment.c
+native/vendor/brotli/enc/compress_fragment.h
+native/vendor/brotli/enc/compress_fragment_two_pass.c
+native/vendor/brotli/enc/compress_fragment_two_pass.h
+native/vendor/brotli/enc/dictionary_hash.c
+native/vendor/brotli/enc/dictionary_hash.h
+native/vendor/brotli/enc/encode.c
+native/vendor/brotli/enc/encoder_dict.c
+native/vendor/brotli/enc/encoder_dict.h
+native/vendor/brotli/enc/entropy_encode.c
+native/vendor/brotli/enc/entropy_encode.h
+native/vendor/brotli/enc/entropy_encode_static.h
+native/vendor/brotli/enc/fast_log.c
+native/vendor/brotli/enc/fast_log.h
+native/vendor/brotli/enc/find_match_length.h
+native/vendor/brotli/enc/hash.h
+native/vendor/brotli/enc/hash_composite_inc.h
+native/vendor/brotli/enc/hash_forgetful_chain_inc.h
+native/vendor/brotli/enc/hash_longest_match64_inc.h
+native/vendor/brotli/enc/hash_longest_match_inc.h
+native/vendor/brotli/enc/hash_longest_match_quickly_inc.h
+native/vendor/brotli/enc/hash_rolling_inc.h
+native/vendor/brotli/enc/hash_to_binary_tree_inc.h
+native/vendor/brotli/enc/histogram.c
+native/vendor/brotli/enc/histogram.h
+native/vendor/brotli/enc/histogram_inc.h
+native/vendor/brotli/enc/literal_cost.c
+native/vendor/brotli/enc/literal_cost.h
+native/vendor/brotli/enc/memory.c
+native/vendor/brotli/enc/memory.h
+native/vendor/brotli/enc/metablock.c
+native/vendor/brotli/enc/metablock.h
+native/vendor/brotli/enc/metablock_inc.h
+native/vendor/brotli/enc/params.h
+native/vendor/brotli/enc/prefix.h
+native/vendor/brotli/enc/quality.h
+native/vendor/brotli/enc/ringbuffer.h
+native/vendor/brotli/enc/state.h
+native/vendor/brotli/enc/static_dict.c
+native/vendor/brotli/enc/static_dict.h
+native/vendor/brotli/enc/static_dict_lut.h
+native/vendor/brotli/enc/utf8_util.c
+native/vendor/brotli/enc/utf8_util.h
+native/vendor/brotli/enc/write_bits.h
+native/vendor/brotli/include/brotli/decode.h
+native/vendor/brotli/include/brotli/encode.h
+native/vendor/brotli/include/brotli/port.h
+native/vendor/brotli/include/brotli/shared_dictionary.h
+native/vendor/brotli/include/brotli/types.h
+native/vendor/libbsc/LICENSE
+native/vendor/libbsc/VERSION
+native/vendor/libbsc/libbsc/filters.h
+native/vendor/libbsc/libbsc/libbsc.h
+native/vendor/libbsc/libbsc/adler32/adler32.cpp
+native/vendor/libbsc/libbsc/adler32/adler32.h
+native/vendor/libbsc/libbsc/bwt/bwt.cpp
+native/vendor/libbsc/libbsc/bwt/bwt.h
+native/vendor/libbsc/libbsc/bwt/libcubwt/AUTHORS
+native/vendor/libbsc/libbsc/bwt/libcubwt/CHANGES
+native/vendor/libbsc/libbsc/bwt/libcubwt/LICENSE
+native/vendor/libbsc/libbsc/bwt/libcubwt/VERSION
+native/vendor/libbsc/libbsc/bwt/libcubwt/libcubwt.cuh
+native/vendor/libbsc/libbsc/bwt/libsais/CHANGES
+native/vendor/libbsc/libbsc/bwt/libsais/LICENSE
+native/vendor/libbsc/libbsc/bwt/libsais/VERSION
+native/vendor/libbsc/libbsc/bwt/libsais/libsais.c
+native/vendor/libbsc/libbsc/bwt/libsais/libsais.h
+native/vendor/libbsc/libbsc/coder/coder.cpp
+native/vendor/libbsc/libbsc/coder/coder.h
+native/vendor/libbsc/libbsc/coder/common/predictor.h
+native/vendor/libbsc/libbsc/coder/common/rangecoder.h
+native/vendor/libbsc/libbsc/coder/common/tables.h
+native/vendor/libbsc/libbsc/coder/qlfc/qlfc.cpp
+native/vendor/libbsc/libbsc/coder/qlfc/qlfc.h
+native/vendor/libbsc/libbsc/coder/qlfc/qlfc_model.cpp
+native/vendor/libbsc/libbsc/coder/qlfc/qlfc_model.h
+native/vendor/libbsc/libbsc/filters/detectors.cpp
+native/vendor/libbsc/libbsc/filters/preprocessing.cpp
+native/vendor/libbsc/libbsc/filters/tables.h
+native/vendor/libbsc/libbsc/libbsc/libbsc.cpp
+native/vendor/libbsc/libbsc/lzp/lzp.cpp
+native/vendor/libbsc/libbsc/lzp/lzp.h
+native/vendor/libbsc/libbsc/platform/platform.cpp
+native/vendor/libbsc/libbsc/platform/platform.h
+native/vendor/libbsc/libbsc/st/st.cpp
+native/vendor/libbsc/libbsc/st/st.cuh
+native/vendor/libbsc/libbsc/st/st.h
+native/vendor/lz4/LICENSE
+native/vendor/lz4/lz4.c
+native/vendor/lz4/lz4.h
+native/vendor/lz4/lz4hc.c
+native/vendor/lz4/lz4hc.h
+native/vendor/xxhash/LICENSE
+native/vendor/xxhash/czn_xxh3.c
+native/vendor/xxhash/xxhash.h
+native/vendor/zstd/.gitignore
+native/vendor/zstd/BUCK
+native/vendor/zstd/LICENSE
+native/vendor/zstd/Makefile
+native/vendor/zstd/README.md
+native/vendor/zstd/libzstd.mk
+native/vendor/zstd/libzstd.pc.in
+native/vendor/zstd/module.modulemap
+native/vendor/zstd/zdict.h
+native/vendor/zstd/zstd.h
+native/vendor/zstd/zstd_errors.h
+native/vendor/zstd/common/allocations.h
+native/vendor/zstd/common/bits.h
+native/vendor/zstd/common/bitstream.h
+native/vendor/zstd/common/compiler.h
+native/vendor/zstd/common/cpu.h
+native/vendor/zstd/common/debug.c
+native/vendor/zstd/common/debug.h
+native/vendor/zstd/common/entropy_common.c
+native/vendor/zstd/common/error_private.c
+native/vendor/zstd/common/error_private.h
+native/vendor/zstd/common/fse.h
+native/vendor/zstd/common/fse_decompress.c
+native/vendor/zstd/common/huf.h
+native/vendor/zstd/common/mem.h
+native/vendor/zstd/common/pool.c
+native/vendor/zstd/common/pool.h
+native/vendor/zstd/common/portability_macros.h
+native/vendor/zstd/common/threading.c
+native/vendor/zstd/common/threading.h
+native/vendor/zstd/common/xxhash.c
+native/vendor/zstd/common/xxhash.h
+native/vendor/zstd/common/zstd_common.c
+native/vendor/zstd/common/zstd_deps.h
+native/vendor/zstd/common/zstd_internal.h
+native/vendor/zstd/common/zstd_trace.h
+native/vendor/zstd/compress/clevels.h
+native/vendor/zstd/compress/fse_compress.c
+native/vendor/zstd/compress/hist.c
+native/vendor/zstd/compress/hist.h
+native/vendor/zstd/compress/huf_compress.c
+native/vendor/zstd/compress/zstd_compress.c
+native/vendor/zstd/compress/zstd_compress_internal.h
+native/vendor/zstd/compress/zstd_compress_literals.c
+native/vendor/zstd/compress/zstd_compress_literals.h
+native/vendor/zstd/compress/zstd_compress_sequences.c
+native/vendor/zstd/compress/zstd_compress_sequences.h
+native/vendor/zstd/compress/zstd_compress_superblock.c
+native/vendor/zstd/compress/zstd_compress_superblock.h
+native/vendor/zstd/compress/zstd_cwksp.h
+native/vendor/zstd/compress/zstd_double_fast.c
+native/vendor/zstd/compress/zstd_double_fast.h
+native/vendor/zstd/compress/zstd_fast.c
+native/vendor/zstd/compress/zstd_fast.h
+native/vendor/zstd/compress/zstd_lazy.c
+native/vendor/zstd/compress/zstd_lazy.h
+native/vendor/zstd/compress/zstd_ldm.c
+native/vendor/zstd/compress/zstd_ldm.h
+native/vendor/zstd/compress/zstd_ldm_geartab.h
+native/vendor/zstd/compress/zstd_opt.c
+native/vendor/zstd/compress/zstd_opt.h
+native/vendor/zstd/compress/zstd_preSplit.c
+native/vendor/zstd/compress/zstd_preSplit.h
+native/vendor/zstd/compress/zstdmt_compress.c
+native/vendor/zstd/compress/zstdmt_compress.h
+native/vendor/zstd/decompress/huf_decompress.c
+native/vendor/zstd/decompress/huf_decompress_amd64.S
+native/vendor/zstd/decompress/zstd_ddict.c
+native/vendor/zstd/decompress/zstd_ddict.h
+native/vendor/zstd/decompress/zstd_decompress.c
+native/vendor/zstd/decompress/zstd_decompress_block.c
+native/vendor/zstd/decompress/zstd_decompress_block.h
+native/vendor/zstd/decompress/zstd_decompress_internal.h
+native/vendor/zstd/deprecated/zbuff.h
+native/vendor/zstd/deprecated/zbuff_common.c
+native/vendor/zstd/deprecated/zbuff_compress.c
+native/vendor/zstd/deprecated/zbuff_decompress.c
+native/vendor/zstd/dictBuilder/cover.c
+native/vendor/zstd/dictBuilder/cover.h
+native/vendor/zstd/dictBuilder/divsufsort.c
+native/vendor/zstd/dictBuilder/divsufsort.h
+native/vendor/zstd/dictBuilder/fastcover.c
+native/vendor/zstd/dictBuilder/zdict.c
+native/vendor/zstd/dll/example/Makefile
+native/vendor/zstd/dll/example/README.md
+native/vendor/zstd/dll/example/build_package.bat
+native/vendor/zstd/dll/example/fullbench-dll.sln
+native/vendor/zstd/dll/example/fullbench-dll.vcxproj
+native/vendor/zstd/legacy/zstd_legacy.h
+native/vendor/zstd/legacy/zstd_v01.c
+native/vendor/zstd/legacy/zstd_v01.h
+native/vendor/zstd/legacy/zstd_v02.c
+native/vendor/zstd/legacy/zstd_v02.h
+native/vendor/zstd/legacy/zstd_v03.c
+native/vendor/zstd/legacy/zstd_v03.h
+native/vendor/zstd/legacy/zstd_v04.c
+native/vendor/zstd/legacy/zstd_v04.h
+native/vendor/zstd/legacy/zstd_v05.c
+native/vendor/zstd/legacy/zstd_v05.h
+native/vendor/zstd/legacy/zstd_v06.c
+native/vendor/zstd/legacy/zstd_v06.h
+native/vendor/zstd/legacy/zstd_v07.c
+native/vendor/zstd/legacy/zstd_v07.h
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/cover.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/debug.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/divsufsort.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/entropy_common.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/error_private.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/fastcover.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/fse_compress.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/fse_decompress.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/hist.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/huf_compress.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/huf_decompress.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/pool.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/threading.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/xxhash.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zdict.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstd_common.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstd_compress.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstd_compress_literals.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstd_compress_sequences.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstd_compress_superblock.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstd_ddict.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstd_decompress.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstd_decompress_block.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstd_double_fast.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstd_fast.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstd_lazy.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstd_ldm.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstd_opt.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstd_preSplit.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstd_v05.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstd_v06.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstd_v07.d
+native/vendor/zstd/obj/conf_8055287114409e2ae6b110871ea321ce/static/zstdmt_compress.d

kolmoc-1.0.0/kolmoc.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

kolmoc-1.0.0/kolmoc.egg-info/top_level.txt

@@ -0,0 +1 @@
+kolmo

kolmoc-1.0.0/native/src/cli.c

@@ -0,0 +1,94 @@
+/* kolmo command line:  kolmo [-m rt|fast|eco|hc|max|ultra] [-l level] [-T threads] [-B MiB] [-n] [-D dict] in out
+ *                    kolmo -s ...                         (streaming: bounded memory, large inputs)
+ *                    kolmo -d [-T threads] [-D dict] in out  (auto-detects one-shot vs streamed)
+ *                    kolmo -r offset:length in out        (random access read)
+ *                    kolmo -t dict.bin sample1 sample2 ...   (train a dictionary from sample files) */
+#include "czn.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+static uint8_t *slurp(const char *p, size_t *n) {
+    FILE *f = fopen(p, "rb"); if (!f) { perror(p); exit(1); }
+    fseek(f, 0, SEEK_END); long s = ftell(f); fseek(f, 0, SEEK_SET);
+    uint8_t *b = malloc(s ? s : 1); if (s && fread(b, 1, s, f) != (size_t)s) { perror("read"); exit(1); }
+    fclose(f); *n = s; return b;
+}
+static void spit(const char *p, const uint8_t *b, size_t n) {
+    FILE *f = fopen(p, "wb"); if (!f || fwrite(b, 1, n, f) != n) { perror(p); exit(1); } fclose(f);
+}
+int main(int argc, char **argv) {
+    czn_params P = {CZN_FAST, 0, 0, 0, 1}; int dec = 0, stream = 0, opt, seg = 0; char *range = NULL;
+    char *dictpath = NULL, *trainpath = NULL;
+    while ((opt = getopt(argc, argv, "m:l:T:B:ndr:sS:D:t:")) != -1) switch (opt) {
+        case 'm': P.mode = !strcmp(optarg, "rt") ? CZN_RT : !strcmp(optarg, "fast") ? CZN_FAST : !strcmp(optarg, "eco") ? CZN_ECO :
+                         !strcmp(optarg, "max") ? CZN_MAX : !strcmp(optarg, "ultra") ? CZN_ULTRA : CZN_HC; break;
+        case 'l': P.level = atoi(optarg); break;
+        case 'T': P.threads = atoi(optarg); break;
+        case 'B': P.block = (uint32_t)(atof(optarg) * 1048576); break;
+        case 'n': P.transforms = 0; break;
+        case 'd': dec = 1; break;
+        case 'r': range = optarg; break;
+        case 's': stream = 1; break;
+        case 'S': stream = 1; seg = atoi(optarg); break;
+        case 'D': dictpath = optarg; break;
+        case 't': trainpath = optarg; break;
+        default: fprintf(stderr, "usage: kolmo [-m rt|fast|eco|hc|max|ultra] [-l lvl] [-T thr] [-B MiB] [-n] [-D dict] [-s|-S segMiB] | -d [-D dict] | -r off:len | -t dict samples...  in out\n"); return 2;
+    }
+
+    /* Training mode: concatenate the sample files given as positional args and write a dictionary. */
+    if (trainpath) {
+        int nf = argc - optind; if (nf < 1) { fprintf(stderr, "kolmo -t: need at least one sample file\n"); return 2; }
+        size_t *sz = malloc(nf * sizeof(size_t)), tot = 0; uint8_t **buf = malloc(nf * sizeof(uint8_t *));
+        for (int i = 0; i < nf; i++) { buf[i] = slurp(argv[optind + i], &sz[i]); tot += sz[i]; }
+        uint8_t *samp = malloc(tot ? tot : 1); size_t off = 0;
+        for (int i = 0; i < nf; i++) { memcpy(samp + off, buf[i], sz[i]); off += sz[i]; free(buf[i]); }
+        size_t dcap = 112 * 1024; uint8_t *dict = malloc(dcap);
+        int64_t dl = czn_train_dict(samp, sz, nf, dict, dcap);
+        if (dl < 0) { fprintf(stderr, "kolmo: train: %s\n", czn_strerror(dl)); return 1; }
+        spit(trainpath, dict, (size_t)dl);
+        fprintf(stderr, "kolmo: wrote %lld-byte dictionary from %d samples to %s\n", (long long)dl, nf, trainpath);
+        return 0;
+    }
+
+    if (argc - optind != 2) { fprintf(stderr, "need input and output paths\n"); return 2; }
+    const char *inp = argv[optind], *outp = argv[optind + 1];
+
+    size_t dictlen = 0; uint8_t *dict = NULL;
+    if (dictpath) dict = slurp(dictpath, &dictlen);
+
+    /* Streaming compress, or decode of a streamed (CZNS) file: bounded memory via file descriptors. */
+    int streamed_in = 0;
+    if (dec) {
+        FILE *pf = fopen(inp, "rb"); if (!pf) { perror(inp); return 1; }
+        char mg[4] = {0}; size_t g = fread(mg, 1, 4, pf); fclose(pf);
+        streamed_in = (g == 4 && !memcmp(mg, "CZNS", 4));
+    }
+    if ((stream && !dec && !range) || (dec && streamed_in)) {
+        int ifd = open(inp, O_RDONLY); if (ifd < 0) { perror(inp); return 1; }
+        int ofd = open(outp, O_WRONLY | O_CREAT | O_TRUNC, 0644); if (ofd < 0) { perror(outp); close(ifd); return 1; }
+        int64_t sr = dec ? czn_stream_decompress(ifd, ofd, P.threads)
+                         : czn_stream_compress(ifd, ofd, &P, (uint32_t)(seg > 0 ? seg : 0));
+        close(ifd); close(ofd);
+        if (sr < 0) { fprintf(stderr, "kolmo: %s\n", czn_strerror(sr)); return 1; }
+        return 0;
+    }
+
+    size_t n; uint8_t *in = slurp(inp, &n), *out; int64_t r;
+    if (range) {
+        unsigned long long off = strtoull(range, NULL, 10); char *c = strchr(range, ':'); size_t len = c ? strtoull(c + 1, NULL, 10) : 0;
+        out = malloc(len ? len : 1); r = czn_read_range(in, n, off, len, out, P.threads);
+    } else if (dec) {
+        int64_t o = czn_original_size(in, n); if (o < 0) { fprintf(stderr, "kolmo: %s\n", czn_strerror(o)); return 1; }
+        out = malloc(o ? o : 1);
+        r = dict ? czn_decompress_dict(in, n, out, o, P.threads, dict, dictlen) : czn_decompress(in, n, out, o, P.threads);
+    } else {
+        size_t cap = czn_bound(n, &P); out = malloc(cap);
+        r = dict ? czn_compress_dict(in, n, out, cap, &P, dict, dictlen) : czn_compress(in, n, out, cap, &P);
+    }
+    if (r < 0) { fprintf(stderr, "kolmo: %s\n", czn_strerror(r)); return 1; }
+    spit(outp, out, (size_t)r);
+    return 0;
+}