hyzip 0.1.0__tar.gz

hyzip-0.1.0/PKG-INFO

@@ -0,0 +1,25 @@
+Metadata-Version: 2.4
+Name: hyzip
+Version: 0.1.0
+Summary: Auto-Routing Hybrid Text Compressor (MoeZip + TikMax)
+License: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: moezip
+Requires-Dist: tiktoken>=0.5.0
+
+# hyzip
+
+Auto-Routing Hybrid Text Compression Engine combining **MoeZip** (2nd-Order Markov + LZ777) and **TikMax** (BPE + C++ LEB128 Varint).
+
+## Usage
+
+```python
+import hyzip
+
+# Compress text (Auto-selects best engine)
+compressed = hyzip.compress("Hello world!")
+
+# Decompress text
+text = hyzip.decompress(compressed)
+```

hyzip-0.1.0/README.md

@@ -0,0 +1,15 @@
+# hyzip
+
+Auto-Routing Hybrid Text Compression Engine combining **MoeZip** (2nd-Order Markov + LZ777) and **TikMax** (BPE + C++ LEB128 Varint).
+
+## Usage
+
+```python
+import hyzip
+
+# Compress text (Auto-selects best engine)
+compressed = hyzip.compress("Hello world!")
+
+# Decompress text
+text = hyzip.decompress(compressed)
+```

hyzip-0.1.0/hyzip/__init__.py

@@ -0,0 +1,62 @@
+"""
+hyzip: Auto-Routing Hybrid Text Compression Engine
+Combines installed moezip + tikmax BPE into a unified 1-byte header protocol.
+"""
+
+import tiktoken
+import moezip  # Uses already installed moezip engine!
+from hyzip import _tikmax
+
+__version__ = "0.1.0"
+
+_enc = tiktoken.get_encoding("cl100k_base")
+
+FLAG_MOEZIP = b'M'  # Header for MoeZip payload (0x4D)
+FLAG_TIKMAX = b'T'  # Header for TikMax payload (0x54)
+
+
+def _tikmax_compress(text: str) -> bytes:
+    tokens = _enc.encode(text)
+    return _tikmax.pack_tokens(tokens)
+
+
+def _tikmax_decompress(data: bytes) -> str:
+    tokens = _tikmax.unpack_tokens(data)
+    return _enc.decode(tokens)
+
+
+def compress(text: str) -> bytes:
+    """
+    Compresses text using BOTH MoeZip and TikMax engines.
+    Returns the smallest binary payload prepended with a 1-byte header flag.
+    """
+    if not text:
+        return b""
+
+    # Compress with MoeZip
+    mz_payload = FLAG_MOEZIP + moezip.compress(text)
+
+    # Compress with TikMax
+    tm_payload = FLAG_TIKMAX + _tikmax_compress(text)
+
+    # Pick the smaller winner
+    return mz_payload if len(mz_payload) <= len(tm_payload) else tm_payload
+
+
+def decompress(payload: bytes) -> str:
+    """
+    Decompresses payload by reading the 1-byte header flag
+    and routing to the correct engine.
+    """
+    if not payload:
+        return ""
+
+    header = payload[:1]
+    body = payload[1:]
+
+    if header == FLAG_MOEZIP:
+        return moezip.decompress(body)
+    elif header == FLAG_TIKMAX:
+        return _tikmax_decompress(body)
+    else:
+        raise ValueError(f"Corrupted payload or unknown header flag: {header}")

hyzip-0.1.0/hyzip/_tikmax.pyi

@@ -0,0 +1,11 @@
+# _tikmax.pyi - IDE Auto-Completion Type Stubs
+
+from typing import List
+
+def pack_tokens(tokens: List[int]) -> bytes:
+    """Packs Tiktoken BPE token IDs into an LEB128 + ZigZag Varint byte stream."""
+    ...
+
+def unpack_tokens(data: bytes) -> List[int]:
+    """Unpacks an LEB128 + ZigZag Varint byte stream back into token IDs."""
+    ...

hyzip-0.1.0/hyzip/py.typed

@@ -0,0 +1 @@
+# Marker for PEP 561

hyzip-0.1.0/hyzip.egg-info/PKG-INFO

@@ -0,0 +1,25 @@
+Metadata-Version: 2.4
+Name: hyzip
+Version: 0.1.0
+Summary: Auto-Routing Hybrid Text Compressor (MoeZip + TikMax)
+License: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: moezip
+Requires-Dist: tiktoken>=0.5.0
+
+# hyzip
+
+Auto-Routing Hybrid Text Compression Engine combining **MoeZip** (2nd-Order Markov + LZ777) and **TikMax** (BPE + C++ LEB128 Varint).
+
+## Usage
+
+```python
+import hyzip
+
+# Compress text (Auto-selects best engine)
+compressed = hyzip.compress("Hello world!")
+
+# Decompress text
+text = hyzip.decompress(compressed)
+```

hyzip-0.1.0/hyzip.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+README.md
+pyproject.toml
+setup.py
+hyzip/__init__.py
+hyzip/_tikmax.pyi
+hyzip/py.typed
+hyzip.egg-info/PKG-INFO
+hyzip.egg-info/SOURCES.txt
+hyzip.egg-info/dependency_links.txt
+hyzip.egg-info/requires.txt
+hyzip.egg-info/top_level.txt
+src/tikmax.cpp

hyzip-0.1.0/hyzip.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

hyzip-0.1.0/hyzip.egg-info/requires.txt

@@ -0,0 +1,2 @@
+moezip
+tiktoken>=0.5.0

hyzip-0.1.0/hyzip.egg-info/top_level.txt

@@ -0,0 +1 @@
+hyzip

hyzip-0.1.0/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["setuptools>=61.0", "pybind11>=2.10.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "hyzip"
+version = "0.1.0"
+description = "Auto-Routing Hybrid Text Compressor (MoeZip + TikMax)"
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+dependencies = [
+    "moezip",
+    "tiktoken>=0.5.0"
+]

hyzip-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

hyzip-0.1.0/setup.py

@@ -0,0 +1,26 @@
+import os
+from setuptools import setup, Extension, find_packages
+import pybind11
+
+cpp_args = ['/O2', '/std:c++20'] if os.name == 'nt' else ['-O3', '-std=c++20']
+
+ext_tikmax = Extension(
+    'hyzip._tikmax',
+    sources=['src/tikmax.cpp'],
+    include_dirs=[pybind11.get_include()],
+    language='c++',
+    extra_compile_args=cpp_args
+)
+
+setup(
+    name="hyzip",
+    version="0.1.0",
+    install_requires=[
+    "moezip",
+    "tiktoken>=0.5.0"
+    ],
+    packages=find_packages(),
+    ext_modules=[ext_tikmax],
+    package_data={"hyzip": ["*.pyi", "py.typed"]},
+    include_package_data=True,
+)

hyzip-0.1.0/src/tikmax.cpp

@@ -0,0 +1,58 @@
+
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <vector>
+#include <cstdint>
+#include <string>
+
+namespace py = pybind11;
+
+static py::bytes cpp_pack_tokens(const std::vector<int32_t>& tokens) {
+    std::vector<uint8_t> out;
+    out.reserve(tokens.size() * 2);
+
+    int32_t prev = 0;
+    for (int32_t token : tokens) {
+        int32_t delta = token - prev;
+        prev = token;
+        uint32_t z_val = (delta >> 31) ^ (delta << 1);
+
+        while (z_val >= 0x80) {
+            out.push_back(static_cast<uint8_t>((z_val & 0x7F) | 0x80));
+            z_val >>= 7;
+        }
+        out.push_back(static_cast<uint8_t>(z_val & 0x7F));
+    }
+    return py::bytes(reinterpret_cast<const char*>(out.data()), out.size());
+}
+
+static std::vector<int32_t> cpp_unpack_tokens(const py::bytes& data) {
+    std::string str_data = data;
+    std::vector<int32_t> tokens;
+    const uint8_t* ptr = reinterpret_cast<const uint8_t*>(str_data.data());
+    size_t size = str_data.size();
+    size_t i = 0;
+
+    int32_t prev = 0;
+    while (i < size) {
+        uint32_t z_val = 0;
+        int shift = 0;
+        while (i < size) {
+            uint8_t byte = ptr[i++];
+            z_val |= static_cast<uint32_t>(byte & 0x7F) << shift;
+            if (!(byte & 0x80)) break;
+            shift += 7;
+        }
+        int32_t delta = (z_val >> 1) ^ (-(z_val & 1));
+        int32_t token = prev + delta;
+        prev = token;
+        tokens.push_back(token);
+    }
+    return tokens;
+}
+
+PYBIND11_MODULE(_tikmax, m) {
+    m.doc() = "tikmax: Ultra-fast Tiktoken BPE + C++ Varint bit-packer";
+    m.def("pack_tokens", &cpp_pack_tokens, py::arg("tokens"), "Packs integer token IDs into LEB128 + ZigZag bytes.");
+    m.def("unpack_tokens", &cpp_unpack_tokens, py::arg("data"), "Unpacks LEB128 + ZigZag bytes into token IDs.");
+}