storetle 0.2.0__py3-none-any.whl

storetle/__init__.py

@@ -0,0 +1,111 @@
+# storetle — HTML-aware streaming compression for large document collections
+#
+# Primary API:
+#   StreamWriter   — append HTML documents to a .storetle file
+#   StreamReader   — read/iterate/random-access a .storetle file
+#   pack           — compress a folder of HTML files → .storetle
+#   unpack         — decompress .storetle → folder of HTML files
+#   benchmark      — compare storetle vs gzip on your own data
+
+from .stream import StreamWriter, StreamReader
+from .folder import pack, unpack
+
+__version__ = '0.2.0'
+__all__ = ['StreamWriter', 'StreamReader', 'pack', 'unpack', 'benchmark']
+
+
+def benchmark(folder, quiet=False):
+    """Benchmark storetle vs gzip WARC on a folder of HTML files.
+
+    Returns a dict with size comparisons. Prints a table unless quiet=True.
+
+    Example:
+        import storetle
+        results = storetle.benchmark('my_crawl_data/')
+    """
+    import gzip, os, tempfile, time
+    from pathlib import Path
+
+    files = sorted(Path(folder).glob('**/*.html'))
+    if not files:
+        raise ValueError(f'No .html files found in {folder}')
+
+    docs = [f.read_bytes() for f in files]
+    total_html = sum(len(d) for d in docs)
+
+    # gzip WARC (industry standard)
+    warc_raw = b''.join(
+        'WARC/1.0\r\nContent-Length: {}\r\n\r\n'.format(len(d)).encode()
+        + d + b'\r\n\r\n'
+        for d in docs
+    )
+    warc_gz = len(gzip.compress(warc_raw, compresslevel=9))
+
+    # gzip per-file
+    gz_pf = sum(len(gzip.compress(d, compresslevel=9)) for d in docs)
+
+    # storetle
+    with tempfile.NamedTemporaryFile(suffix='.storetle', delete=False) as tf:
+        tmp = tf.name
+    try:
+        t0 = time.time()
+        with StreamWriter(tmp) as w:
+            for d in docs:
+                w.append(d)
+        write_time = time.time() - t0
+
+        t1 = time.time()
+        with StreamReader(tmp) as r:
+            recovered = list(r)
+        read_time = time.time() - t1
+
+        cube_size = os.path.getsize(tmp)
+    finally:
+        os.unlink(tmp)
+
+    rt_ok = len(recovered) == len(docs)
+
+    result = {
+        'files':          len(files),
+        'original_bytes': total_html,
+        'gzip_warc':      warc_gz,
+        'gzip_per_file':  gz_pf,
+        'storetle':     cube_size,
+        'savings_vs_gzip_warc_pct': round((warc_gz - cube_size) / warc_gz * 100, 1),
+        'write_kbps':     int(total_html / 1024 / max(write_time, 0.001)),
+        'read_kbps':      int(total_html / 1024 / max(read_time, 0.001)),
+        'roundtrip_ok':   rt_ok,
+    }
+
+    if not quiet:
+        _print_benchmark(result)
+
+    return result
+
+
+def _print_benchmark(r):
+    def fmt(n):
+        if n < 1024: return f'{n}B'
+        if n < 1048576: return f'{n/1024:.1f}KB'
+        return f'{n/1048576:.2f}MB'
+
+    def pct(a, b):
+        return f'{100*(1-a/b):.1f}%'
+
+    orig = r['original_bytes']
+    print(f'\n  storetle benchmark — {r["files"]} files, {fmt(orig)} original\n')
+    print(f'  {"Format":<28} {"Size":>10}  {"Savings":>8}')
+    print(f'  {"─"*50}')
+    print(f'  {"Original HTML":<28} {fmt(orig):>10}')
+    print(f'  {"gzip per-file (current)":<28} {fmt(r["gzip_per_file"]):>10}  {pct(r["gzip_per_file"], orig):>8}')
+    print(f'  {"gzip WARC (Common Crawl std)":<28} {fmt(r["gzip_warc"]):>10}  {pct(r["gzip_warc"], orig):>8}')
+    print(f'  {"storetle":<28} {fmt(r["storetle"]):>10}  {pct(r["storetle"], orig):>8}')
+    print()
+
+    saved = r["gzip_warc"] - r["storetle"]
+    sign  = '+' if saved > 0 else ''
+    print(f'  vs gzip WARC:  {sign}{r["savings_vs_gzip_warc_pct"]}% smaller  ({fmt(abs(saved))} {"saved" if saved > 0 else "larger"})')
+    print(f'  Write speed:   {r["write_kbps"]:,} KB/s')
+    print(f'  Read speed:    {r["read_kbps"]:,} KB/s')
+    print(f'  Round-trip:    {"✓ all documents verified" if r["roundtrip_ok"] else "✗ FAILED"}')
+    print()

storetle/brotli_compat.py

@@ -0,0 +1,96 @@
+# brotli_compat.py — thin ctypes wrapper around the system brotli library
+#
+# Exposes compress(data, quality=11) and decompress(data) with the same
+# calling convention as zlib.compress / zlib.decompress so callers can
+# swap one for the other with a single import change.
+#
+# Falls back to zlib if the brotli dylib is not present (other machines).
+
+import ctypes, zlib, os
+
+_BROTLI_OK   = False
+_enc = None
+_dec = None
+
+def _try_load():
+    global _BROTLI_OK, _enc, _dec
+    candidates = [
+        '/usr/local/lib/libbrotlienc.dylib',
+        '/usr/local/Cellar/brotli/1.2.0/lib/libbrotlienc.dylib',
+        'libbrotlienc.so',
+        'libbrotlienc.dylib',
+    ]
+    dec_candidates = [p.replace('enc', 'dec') for p in candidates]
+
+    for ep, dp in zip(candidates, dec_candidates):
+        if os.path.exists(ep) and os.path.exists(dp):
+            try:
+                e = ctypes.CDLL(ep)
+                d = ctypes.CDLL(dp)
+                # wire up encoder
+                e.BrotliEncoderMaxCompressedSize.restype  = ctypes.c_size_t
+                e.BrotliEncoderMaxCompressedSize.argtypes = [ctypes.c_size_t]
+                e.BrotliEncoderCompress.restype  = ctypes.c_int
+                e.BrotliEncoderCompress.argtypes = [
+                    ctypes.c_int, ctypes.c_int, ctypes.c_int,
+                    ctypes.c_size_t, ctypes.c_char_p,
+                    ctypes.POINTER(ctypes.c_size_t), ctypes.c_char_p,
+                ]
+                # wire up decoder
+                d.BrotliDecoderDecompress.restype  = ctypes.c_int
+                d.BrotliDecoderDecompress.argtypes = [
+                    ctypes.c_size_t, ctypes.c_char_p,
+                    ctypes.POINTER(ctypes.c_size_t), ctypes.c_char_p,
+                ]
+                _enc = e; _dec = d; _BROTLI_OK = True
+                return
+            except Exception:
+                continue
+
+_try_load()
+
+
+def compress(data: bytes, quality: int = 11, lgwin: int = 24, mode: int = 0) -> bytes:
+    """Compress data with brotli (quality 0-11, mode 0=generic 1=text 2=font).
+    Falls back to zlib level 9."""
+    if not _BROTLI_OK:
+        return zlib.compress(data, level=9)
+
+    max_out  = _enc.BrotliEncoderMaxCompressedSize(len(data))
+    out_buf  = ctypes.create_string_buffer(max_out)
+    out_size = ctypes.c_size_t(max_out)
+
+    ok = _enc.BrotliEncoderCompress(
+        quality, lgwin, mode,
+        len(data), data,
+        ctypes.byref(out_size), out_buf,
+    )
+    if not ok:
+        raise RuntimeError('brotli compression failed')
+    return out_buf.raw[:out_size.value]
+
+
+def decompress(data: bytes) -> bytes:
+    """Decompress brotli data. Falls back to zlib if brotli unavailable."""
+    if not _BROTLI_OK:
+        return zlib.decompress(data)
+
+    # Grow output buffer until it fits (decompressed size is unknown).
+    max_out = max(len(data) * 10, 1 << 20)   # start at 10× or 1 MB
+    while True:
+        out_buf  = ctypes.create_string_buffer(max_out)
+        out_size = ctypes.c_size_t(max_out)
+        result   = _dec.BrotliDecoderDecompress(
+            len(data), data,
+            ctypes.byref(out_size), out_buf,
+        )
+        if result == 1:   # BROTLI_DECODER_RESULT_SUCCESS
+            return out_buf.raw[:out_size.value]
+        if result == 3:   # BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT
+            max_out *= 4
+            continue
+        raise RuntimeError(f'brotli decompression failed (result={result})')
+
+
+def available() -> bool:
+    return _BROTLI_OK

storetle/cli.py

@@ -0,0 +1,302 @@
+#!/usr/bin/env python3
+# cli.py — storetle command-line interface
+#
+# Commands:
+#   storetle bench  <folder>                 — benchmark your data
+#   storetle pack   <folder> <output>        — compress folder → .storetle
+#   storetle unpack <input>  <output_folder> — decompress .storetle → files
+#   storetle info   <file.storetle>        — show file stats
+#   storetle get    <file.storetle> <idx>  — extract one document by index
+
+import sys
+from pathlib import Path
+
+
+
+def cmd_bench(args):
+    if not args:
+        print('Usage: storetle bench <folder>')
+        sys.exit(1)
+    from . import benchmark
+    try:
+        benchmark(args[0])
+    except ValueError as e:
+        print(f'Error: {e}')
+        sys.exit(1)
+
+
+def cmd_pack(args):
+    if len(args) < 2:
+        print('Usage: storetle pack <folder> <output.storetle>')
+        sys.exit(1)
+    from .stream import StreamWriter
+    folder = Path(args[0])
+    output = args[1]
+    files  = sorted(folder.glob('**/*.html'))
+    if not files:
+        print(f'No .html files found in {folder}')
+        sys.exit(1)
+
+    print(f'Packing {len(files)} files...')
+    with StreamWriter(output) as w:
+        for f in files:
+            w.append(f.read_bytes())
+
+    from .stream import StreamReader
+    info = StreamReader.info(output)
+
+    def fmt(n):
+        if n < 1048576: return f'{n/1024:.1f}KB'
+        return f'{n/1048576:.2f}MB'
+
+    print(f'Done: {fmt(info["original_bytes"])} → {fmt(info["compressed_bytes"])} '
+          f'({info["ratio_pct"]}% saved, {info["docs"]} docs, {info["chunks"]} chunks)')
+    print(f'Output: {output}')
+
+
+def cmd_unpack(args):
+    if len(args) < 2:
+        print('Usage: storetle unpack <file.storetle> <output_folder>')
+        sys.exit(1)
+    from .stream import StreamReader
+    src = args[0]
+    dst = Path(args[1])
+    dst.mkdir(parents=True, exist_ok=True)
+
+    with StreamReader(src) as r:
+        print(f'Extracting {r.doc_count} documents to {dst}/')
+        for i, doc in enumerate(r):
+            out = dst / f'doc_{i:06d}.html'
+            out.write_bytes(doc)
+            if (i + 1) % 100 == 0:
+                print(f'  {i+1}/{r.doc_count}...')
+    print(f'Done: {r.doc_count} files written to {dst}/')
+
+
+def cmd_info(args):
+    if not args:
+        print('Usage: storetle info <file.storetle>')
+        sys.exit(1)
+    from .stream import StreamReader
+
+    def fmt(n):
+        if n < 1048576: return f'{n/1024:.1f}KB'
+        return f'{n/1048576:.2f}MB'
+
+    info = StreamReader.info(args[0])
+    print(f'\n  {args[0]}')
+    print(f'  Documents:    {info["docs"]:,}')
+    print(f'  Chunks:       {info["chunks"]:,}')
+    print(f'  Original:     {fmt(info["original_bytes"])}')
+    print(f'  Compressed:   {fmt(info["compressed_bytes"])}  ({info["ratio_pct"]}% saved)')
+    print()
+
+
+def cmd_get(args):
+    if len(args) < 2:
+        print('Usage: storetle get <file.storetle> <index>')
+        sys.exit(1)
+    from .stream import StreamReader
+    with StreamReader(args[0]) as r:
+        try:
+            idx = int(args[1])
+            doc = r[idx]
+            sys.stdout.buffer.write(doc)
+        except (IndexError, ValueError) as e:
+            print(f'Error: {e}')
+            sys.exit(1)
+
+
+def cmd_from_warc(args):
+    if len(args) < 2:
+        print('Usage: storetle from-warc <input.warc[.gz]> <output.storetle>')
+        sys.exit(1)
+    from .warc import from_warc
+    try:
+        from_warc(args[0], args[1], verbose=True)
+    except ValueError as e:
+        print('Error: %s' % e)
+        sys.exit(1)
+    except Exception as e:
+        print('Error: %s' % e)
+        sys.exit(1)
+
+
+def cmd_to_warc(args):
+    if len(args) < 2:
+        print('Usage: storetle to-warc <input.storetle> <output.warc[.gz]>')
+        sys.exit(1)
+    from .warc import to_warc
+    try:
+        to_warc(args[0], args[1], verbose=True)
+    except Exception as e:
+        print('Error: %s' % e)
+        sys.exit(1)
+
+
+def cmd_train(args):
+    """Train a custom zstd dictionary from a folder of HTML files.
+
+    Usage: storetle train <folder> [--output dict.bin] [--size 1024]
+
+    The trained dict can then be used with StreamWriter(path, dictionary=...).
+    Default output: storetle_dict.bin in current directory.
+    Default size: 1024KB (the same as the built-in dict).
+    """
+    if not args:
+        print('Usage: storetle train <folder> [--output dict.bin] [--size 1024]')
+        sys.exit(1)
+
+    folder  = Path(args[0])
+    outfile = 'storetle_dict.bin'
+    size_kb = 1024
+
+    i = 1
+    while i < len(args):
+        if args[i] == '--output' and i + 1 < len(args):
+            outfile = args[i + 1]; i += 2
+        elif args[i] == '--size' and i + 1 < len(args):
+            try:
+                size_kb = int(args[i + 1])
+            except ValueError:
+                print('Error: --size must be an integer (KB)')
+                sys.exit(1)
+            i += 2
+        else:
+            i += 1
+
+    from . import zstd_compat as _zs
+    if not _zs.available():
+        print('Error: zstd not available. Install libzstd first.')
+        sys.exit(1)
+
+    from .stream import _encode_doc
+
+    files = sorted(folder.glob('**/*.html'))
+    if not files:
+        print('Error: no .html files found in %s' % folder)
+        sys.exit(1)
+
+    print('Encoding %d HTML files for dictionary training...' % len(files))
+    samples = []
+    errors  = 0
+    for f in files:
+        try:
+            raw = _encode_doc(f.read_bytes())
+            samples.append(raw)
+        except Exception:
+            errors += 1
+
+    if not samples:
+        print('Error: failed to encode any files')
+        sys.exit(1)
+
+    total_kb = sum(len(s) for s in samples) // 1024
+    print('Training %dKB dictionary on %d samples (%dKB total)...' % (
+        size_kb, len(samples), total_kb))
+
+    dict_bytes = _zs.train_dictionary(samples, dict_size=size_kb * 1024)
+
+    Path(outfile).write_bytes(dict_bytes)
+    actual_kb = len(dict_bytes) // 1024
+    print('Done: %dKB dictionary saved to %s' % (actual_kb, outfile))
+    if errors:
+        print('  (%d files skipped due to encoding errors)' % errors)
+    print()
+    print('To use this dictionary:')
+    print('  import storetle')
+    print('  d = open("%s", "rb").read()' % outfile)
+    print('  with storetle.StreamWriter("out.storetle", dictionary=d) as w:')
+    print('      w.append(html)')
+    print('  with storetle.StreamReader("out.storetle") as r:')
+    print('      # reader auto-loads dict from disk; pass dictionary=d if custom')
+
+
+def cmd_warc_encode(args):
+    if len(args) < 2:
+        print('Usage: storetle warc-encode <input.warc[.gz]> <output.warc[.gz]>')
+        sys.exit(1)
+    from .warc import warc_encode
+    try:
+        warc_encode(args[0], args[1], verbose=True)
+    except Exception as e:
+        print('Error: %s' % e)
+        sys.exit(1)
+
+
+def cmd_warc_decode(args):
+    if len(args) < 2:
+        print('Usage: storetle warc-decode <encoded.warc[.gz]> <output.warc[.gz]>')
+        sys.exit(1)
+    from .warc import warc_decode
+    try:
+        warc_decode(args[0], args[1], verbose=True)
+    except Exception as e:
+        print('Error: %s' % e)
+        sys.exit(1)
+
+
+COMMANDS = {
+    'bench':       cmd_bench,
+    'pack':        cmd_pack,
+    'unpack':      cmd_unpack,
+    'info':        cmd_info,
+    'get':         cmd_get,
+    'from-warc':   cmd_from_warc,
+    'to-warc':     cmd_to_warc,
+    'warc-encode': cmd_warc_encode,
+    'warc-decode': cmd_warc_decode,
+    'train':       cmd_train,
+}
+
+HELP = """storetle — HTML-aware compression for large document collections
+
+Commands:
+  bench     <folder>                   Benchmark your HTML data vs gzip WARC
+  pack      <folder> <output>          Compress a folder → .storetle file
+  unpack    <file> <output_folder>     Extract a .storetle → HTML files
+  info      <file>                     Show file statistics
+  get       <file> <index>             Extract one document by index (0-based)
+  from-warc   <input.warc[.gz]> <out>  Convert WARC → .storetle
+  to-warc     <input.storetle> <out>  Convert .storetle → WARC (or .warc.gz)
+  warc-encode <input.warc[.gz]> <out> Encode HTML in-place → valid .warc.gz (smaller, standard format)
+  warc-decode <encoded.warc[.gz]> <out> Decode back to standard HTML WARC
+  train       <folder> [options]       Train a custom dictionary from HTML files
+            --output  dict.bin           Output path (default: storetle_dict.bin)
+            --size    1024               Dictionary size in KB (default: 1024)
+
+Examples:
+  storetle bench     my_crawl/
+  storetle pack      my_crawl/  archive.storetle
+  storetle info      archive.storetle
+  storetle get       archive.storetle 0
+  storetle from-warc CC-MAIN.warc.gz  archive.storetle
+  storetle to-warc   archive.storetle output.warc.gz
+  storetle train     my_corpus/ --output my_domain.bin --size 1024
+"""
+
+
+def main():
+    if len(sys.argv) < 2 or sys.argv[1] in ('-h', '--help', 'help'):
+        print(HELP)
+        sys.exit(0)
+
+    cmd = sys.argv[1]
+    if cmd not in COMMANDS:
+        print(f'Unknown command: {cmd}\n')
+        print(HELP)
+        sys.exit(1)
+
+    try:
+        COMMANDS[cmd](sys.argv[2:])
+    except BrokenPipeError:
+        # downstream consumer (e.g. `| head`) closed the pipe — not an error
+        import os
+        os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno())
+        sys.exit(0)
+    except KeyboardInterrupt:
+        sys.exit(130)
+
+
+if __name__ == '__main__':
+    main()