dipcompress 0.1.0__tar.gz

dipcompress-0.1.0/PKG-INFO

@@ -0,0 +1,26 @@
+Metadata-Version: 2.4
+Name: dipcompress
+Version: 0.1.0
+Summary: Lossless image compression using LZ77 + Huffman coding, with an optional C extension.
+Author-email: Subhadip Mondal <subhadipmondal789@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/16bitOni/dipcompress
+Project-URL: Repository, https://github.com/16bitOni/dipcompress
+Project-URL: Issues, https://github.com/16bitOni/dipcompress/issues
+Keywords: image,compression,lz77,huffman,lossless
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: C
+Classifier: Topic :: Multimedia :: Graphics
+Classifier: Topic :: System :: Archiving :: Compression
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: numpy>=2.0
+Requires-Dist: pillow>=10.0
+Provides-Extra: dev
+Requires-Dist: pytest>=9.0; extra == "dev"
+
+Dipcompress is my try to implement PNG style image compression from scratch

dipcompress-0.1.0/README.md

@@ -0,0 +1 @@
+Dipcompress is my try to implement PNG style image compression from scratch

dipcompress-0.1.0/dipcompress/__init__.py

@@ -0,0 +1,12 @@
+# dipcompress/__init__.py
+# This is what users see when they do: import dipcompress
+
+__version__ = "0.1.0"
+__author__ = "Subhadip"
+
+from .encoder import compress
+from .decoder import decompress
+from .image_io import load_image, save_image, pixels_to_bytes, bytes_to_pixels
+from .rle import rle_encode, rle_decode
+
+__all__ = ["compress", "decompress", "load_image", "save_image", "pixels_to_bytes", "bytes_to_pixels", "rle_encode", "rle_decode"]

dipcompress-0.1.0/dipcompress/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

dipcompress-0.1.0/dipcompress/_c_ext/build.py

@@ -0,0 +1,42 @@
+"""
+Build the lz77_cext Python extension module.
+
+Usage (run from the repo root):
+    python dipcompress/_c_ext/build.py
+
+Output: dipcompress/lz77_cext.<platform-tag>.pyd  (Windows)
+        dipcompress/lz77_cext.<platform-tag>.so   (Linux/macOS)
+
+The built module is placed in-place inside the dipcompress package so that
+`from dipcompress import lz77_cext` works immediately without installation.
+"""
+import os
+import sys
+from pathlib import Path
+
+# Run from the repo root so setuptools resolves relative source paths correctly.
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+os.chdir(REPO_ROOT)
+
+
+def build():
+    from setuptools import Extension
+    from setuptools.dist import Distribution
+    from setuptools.command.build_ext import build_ext
+
+    ext = Extension(
+        'dipcompress.lz77_cext',
+        sources=['dipcompress/_c_ext/compress.c'],
+    )
+
+    dist = Distribution(attrs={'ext_modules': [ext]})
+    cmd  = build_ext(dist)
+    cmd.inplace = True          # place .pyd/.so next to the package source
+    cmd.ensure_finalized()
+    cmd.run()
+
+    print('\nBuild complete — lz77_cext is ready.')
+
+
+if __name__ == '__main__':
+    build()

dipcompress-0.1.0/dipcompress/_c_ext/compress.c

@@ -0,0 +1,98 @@
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#define WINDOW_SIZE 4096
+#define LOOKAHEAD   18
+
+/*
+ * Core LZ77 encode — identical algorithm to lz77.py.
+ * Output: raw token stream, 4 bytes per token:
+ *   [distance_hi, distance_lo, length, next_byte]
+ * Literals have distance=0 and length=0.
+ * Returns number of bytes written.
+ */
+static int
+_lz77_encode(const uint8_t *input, int input_len,
+             uint8_t *output, int output_max)
+{
+    int pos = 0, out_pos = 0;
+
+    while (pos < input_len) {
+        int wstart = pos - WINDOW_SIZE;
+        if (wstart < 0) wstart = 0;
+
+        int lend = pos + LOOKAHEAD;
+        if (lend > input_len) lend = input_len;
+        int llen = lend - pos;
+
+        int best_dist = 0, best_len = 0;
+
+        for (int i = wstart; i < pos; i++) {
+            int ml = 0;
+            while (ml < llen && i + ml < pos && input[i + ml] == input[pos + ml])
+                ml++;
+            if (ml > best_len) {
+                best_len  = ml;
+                best_dist = pos - i;
+            }
+        }
+
+        if (best_len >= 3) {
+            int np = pos + best_len;
+            if (np >= input_len) { best_len--; np--; }
+
+            output[out_pos++] = (best_dist >> 8) & 0xFF;
+            output[out_pos++] =  best_dist       & 0xFF;
+            output[out_pos++] = (uint8_t)best_len;
+            output[out_pos++] = input[np];
+            pos += best_len + 1;
+        } else {
+            output[out_pos++] = 0;
+            output[out_pos++] = 0;
+            output[out_pos++] = 0;
+            output[out_pos++] = input[pos];
+            pos++;
+        }
+    }
+    return out_pos;
+}
+
+/* ── Python wrapper ───────────────────────────────────────────────────── */
+
+static PyObject *
+py_lz77_encode(PyObject *self, PyObject *args)
+{
+    const uint8_t *input;
+    Py_ssize_t input_len;
+
+    if (!PyArg_ParseTuple(args, "y#", &input, &input_len))
+        return NULL;
+
+    int output_max = (int)input_len * 4 + 4;
+    uint8_t *output = (uint8_t *)malloc(output_max);
+    if (!output)
+        return PyErr_NoMemory();
+
+    int n = _lz77_encode(input, (int)input_len, output, output_max);
+    PyObject *result = PyBytes_FromStringAndSize((char *)output, n);
+    free(output);
+    return result;
+}
+
+static PyMethodDef lz77_methods[] = {
+    {"lz77_encode", py_lz77_encode, METH_VARARGS,
+     "lz77_encode(data: bytes) -> bytes  —  raw 4-byte-per-token stream"},
+    {NULL, NULL, 0, NULL}
+};
+
+static struct PyModuleDef lz77_module = {
+    PyModuleDef_HEAD_INIT, "lz77_cext", NULL, -1, lz77_methods
+};
+
+PyMODINIT_FUNC
+PyInit_lz77_cext(void)
+{
+    return PyModule_Create(&lz77_module);
+}

dipcompress-0.1.0/dipcompress/cli.py

@@ -0,0 +1,108 @@
+import argparse
+import sys
+import os
+
+from . import __version__
+
+
+def _cmd_compress(args):
+    from .encoder import compress
+    if not os.path.isfile(args.input):
+        print(f"error: input file not found: {args.input}", file=sys.stderr)
+        sys.exit(1)
+
+    stats = compress(args.input, args.output, verbose=args.verbose)
+
+    if not args.verbose:
+        pct = stats['savings_pct']
+        sign = '-' if pct >= 0 else '+'
+        print(f"{stats['width']}x{stats['height']}  "
+              f"{stats['original_size']:,} B  ->  {stats['compressed_size']:,} B  "
+              f"({sign}{abs(pct):.1f}%)")
+
+
+def _cmd_decompress(args):
+    from .decoder import decompress
+    if not os.path.isfile(args.input):
+        print(f"error: input file not found: {args.input}", file=sys.stderr)
+        sys.exit(1)
+
+    decompress(args.input, args.output, verbose=args.verbose)
+
+    if not args.verbose:
+        print(f"saved -> {args.output}")
+
+
+def _cmd_info(args):
+    from .format import DipHeader, HEADER_SIZE, decode_filter_table
+    from collections import Counter
+
+    if not os.path.isfile(args.file):
+        print(f"error: file not found: {args.file}", file=sys.stderr)
+        sys.exit(1)
+
+    with open(args.file, 'rb') as f:
+        raw = f.read()
+
+    header = DipHeader.from_bytes(raw)
+    filter_bytes = raw[HEADER_SIZE: HEADER_SIZE + header.height]
+    filter_types = decode_filter_table(filter_bytes)
+    filter_names = {0: 'None', 1: 'Sub', 2: 'Up', 3: 'Average', 4: 'Paeth'}
+    filter_counts = {filter_names.get(k, k): v
+                     for k, v in Counter(filter_types).most_common()}
+
+    channels_label = {1: 'grayscale', 3: 'RGB', 4: 'RGBA'}.get(header.channels, str(header.channels))
+    raw_size = header.width * header.height * header.channels
+
+    print(f"File         : {args.file}")
+    print(f"Dimensions   : {header.width} x {header.height}  ({channels_label})")
+    print(f"Raw size     : {raw_size:,} bytes  ({raw_size/1024:.1f} KB)")
+    print(f"Stored size  : {len(raw):,} bytes  ({len(raw)/1024:.1f} KB)")
+    print(f"Ratio        : {len(raw)/raw_size:.3f}  ({(1-len(raw)/raw_size)*100:.1f}% smaller)")
+    print(f"Row filters  : {filter_counts}")
+    print(f"Format ver   : {header.version}")
+
+
+def build_parser():
+    parser = argparse.ArgumentParser(
+        prog='dipcompress',
+        description='DipCompress — lossless image compression.',
+    )
+    parser.add_argument('--version', action='version', version=f'dipcompress {__version__}')
+
+    sub = parser.add_subparsers(dest='command', metavar='command')
+    sub.required = True
+
+    # compress
+    p_c = sub.add_parser('compress', aliases=['c'],
+                         help='compress an image to .dip format')
+    p_c.add_argument('input',  help='input image (PNG, JPEG, …)')
+    p_c.add_argument('output', help='output .dip file')
+    p_c.add_argument('-v', '--verbose', action='store_true')
+    p_c.set_defaults(func=_cmd_compress)
+
+    # decompress
+    p_d = sub.add_parser('decompress', aliases=['d'],
+                         help='decompress a .dip file back to an image')
+    p_d.add_argument('input',  help='input .dip file')
+    p_d.add_argument('output', help='output image (PNG recommended)')
+    p_d.add_argument('-v', '--verbose', action='store_true')
+    p_d.set_defaults(func=_cmd_decompress)
+
+    # info
+    p_i = sub.add_parser('info', aliases=['i'],
+                         help='show metadata stored in a .dip file')
+    p_i.add_argument('file', help='.dip file to inspect')
+    p_i.set_defaults(func=_cmd_info)
+
+    return parser
+
+
+def main():
+    parser = build_parser()
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == '__main__':
+    main()

dipcompress-0.1.0/dipcompress/decoder.py

@@ -0,0 +1,70 @@
+# dipcompress/decoder.py
+
+import numpy as np
+from .image_io import save_image, bytes_to_pixels
+from .filters import remove_filters
+from .lz77 import bytes_to_tokens, lz77_decode
+from .huffman import huffman_decode
+from .format import DipHeader, HEADER_SIZE, decode_filter_table, decode_huffman_table
+
+
+def decompress(input_path: str, output_path: str, verbose: bool = False) -> None:
+    """
+    Decompress a .dip file back to an image.
+    
+    Args:
+        input_path: path to the .dip file
+        output_path: path to write the reconstructed image (PNG recommended)
+        verbose: print progress information
+    """
+    with open(input_path, 'rb') as f:
+        raw = f.read()
+    
+    # Step 1: Parse header
+    if verbose: print("Parsing header...")
+    header = DipHeader.from_bytes(raw)
+    pos = HEADER_SIZE
+    
+    width = header.width
+    height = header.height
+    channels = header.channels
+    
+    # Step 2: Read filter table (one byte per row)
+    filter_types = decode_filter_table(raw[pos:pos+height])
+    pos += height
+    
+    # Step 3: Read and parse Huffman table from payload
+    if verbose: print("Reading Huffman table...")
+    payload = raw[pos:]
+    codes, original_length, padding, table_size = decode_huffman_table(payload)
+
+    huffman_compressed = payload[table_size:]
+
+    # Step 4: Huffman decode
+    if verbose: print("Huffman decoding...")
+    huffman_meta = {
+        'codes': codes,
+        'original_length': original_length,
+        'padding': padding,
+    }
+    lz77_bytes = huffman_decode(huffman_compressed, huffman_meta)
+    
+    if verbose: print("LZ77 decoding...")
+    lz77_tokens = bytes_to_tokens(lz77_bytes)
+    filtered_data = lz77_decode(lz77_tokens)
+    
+    if verbose: print("Removing filters...")
+    row_width = width * channels
+    filtered_rows_data = [
+        (ft, filtered_data[i*row_width:(i+1)*row_width])
+        for i, ft in enumerate(filter_types)
+    ]
+    pixels = remove_filters(filtered_rows_data, width, channels)
+    
+    mode = {1: 'L', 3: 'RGB', 4: 'RGBA'}.get(channels, 'RGB')
+    save_image(pixels, output_path, mode=mode)
+    
+    if verbose:
+        print(f"\n✓ Decompression complete!")
+        print(f"  Image: {width}x{height} {mode}")
+        print(f"  Saved to: {output_path}")

dipcompress-0.1.0/dipcompress/encoder.py

@@ -0,0 +1,95 @@
+# dipcompress/encoder.py
+
+import numpy as np
+from .image_io import load_image, pixels_to_bytes
+from .filters import apply_filters
+from .lz77_fast import lz77_encode, using_c as _lz77_using_c
+from .lz77 import tokens_to_bytes
+from .huffman import huffman_encode
+from .format import DipHeader, COMPRESS_DEFLATE, encode_filter_table, encode_huffman_table
+
+
+def compress(input_path: str, output_path: str, verbose: bool = False) -> dict:
+    """
+    Compress an image file to .dip format.
+    
+    Args:
+        input_path: path to PNG/JPEG/any PIL-supported image
+        output_path: path to write the .dip file
+        verbose: print progress information
+    
+    Returns:
+        dict with compression statistics
+    """
+
+    if verbose: print(f"Loading {input_path}...")
+    pixels, meta = load_image(input_path)
+    width = meta['width']
+    height = meta['height']
+    channels = meta['channels']
+    original_size = width * height * channels
+    
+    if verbose: print("Applying filters...")
+    filtered_rows = apply_filters(pixels)
+    filter_types = [ft for ft, _ in filtered_rows]
+    filtered_data = b''.join(row for _, row in filtered_rows)
+    
+    if verbose:
+        backend = "C" if _lz77_using_c else "Python"
+        print(f"Running LZ77 ({backend})...")
+    lz77_tokens = lz77_encode(filtered_data)
+    lz77_bytes = tokens_to_bytes(lz77_tokens)
+    
+    if verbose: print("Running Huffman coding...")
+    huffman_compressed, huffman_meta = huffman_encode(lz77_bytes)
+    codes = huffman_meta['codes']
+    original_length = huffman_meta['original_length']
+    padding = huffman_meta['padding']
+
+    if verbose: print("Assembling .dip file...")
+
+    filter_table_bytes = encode_filter_table(filter_types)
+    huffman_table_bytes = encode_huffman_table(codes, original_length, padding)
+    
+    payload = huffman_table_bytes + huffman_compressed
+    
+    header = DipHeader(
+        width=width,
+        height=height,
+        channels=channels,
+        compression_mode=COMPRESS_DEFLATE,
+        compressed_length=len(payload)
+    )
+    
+    with open(output_path, 'wb') as f:
+        f.write(header.to_bytes())
+        f.write(filter_table_bytes)  
+        f.write(payload)
+    
+    compressed_size = header.to_bytes().__len__() + len(filter_table_bytes) + len(payload)
+    
+    stats = {
+        'original_size': original_size,
+        'compressed_size': compressed_size,
+        'ratio': compressed_size / original_size,
+        'savings_pct': (1 - compressed_size / original_size) * 100,
+        'width': width,
+        'height': height,
+        'channels': channels,
+        'filter_stats': _count_filters(filter_types),
+    }
+    
+    if verbose:
+        print(f"\n✓ Compression complete!")
+        print(f"  Original:   {original_size:,} bytes ({original_size/1024:.1f} KB)")
+        print(f"  Compressed: {compressed_size:,} bytes ({compressed_size/1024:.1f} KB)")
+        print(f"  Ratio:      {stats['ratio']:.3f} ({stats['savings_pct']:.1f}% smaller)")
+    
+    return stats
+
+
+def _count_filters(filter_types: list) -> dict:
+    from collections import Counter
+    filter_names = {0: 'None', 1: 'Sub', 2: 'Up', 3: 'Average', 4: 'Paeth'}
+    counts = Counter(filter_types)
+    return {filter_names.get(k, k): v for k, v in counts.items()}

dipcompress-0.1.0/dipcompress/filters.py

@@ -0,0 +1,128 @@
+import numpy as np
+
+def filter_sub(row: bytes) -> bytes:
+
+    row_arr = list(row)
+    result = [row_arr[0]]
+    for i in range(1,len(row_arr)):
+        result.append((row_arr[i] - row_arr[i-1]) % 256)
+    return bytes(result)
+
+def unfilter_sub(row: bytes) -> bytes:
+
+    row_arr = list(row)
+    result = [row_arr[0]]
+    for i in range(1,len(row_arr)):
+        result.append((row_arr[i] + result[i-1]) % 256)
+    return bytes(result)
+
+def filter_up(row: bytes, prev_row: bytes) -> bytes:
+
+    result = []
+    for curr, prev in zip(row, prev_row):
+        result.append((curr - prev) % 256)
+    return bytes(result)
+
+
+def unfilter_up(row: bytes, prev_row: bytes) -> bytes:
+
+    result = []
+    for curr, prev in zip(row, prev_row):
+        result.append((curr + prev) % 256)
+    return bytes(result)
+
+def paeth_predictor(a: int, b: int, c: int) -> int:
+
+    p = a + b - c
+    pa = abs(p - a)
+    pb = abs(p - b)
+    pc = abs(p - c)
+    if pa <= pb and pa <= pc:
+        return a
+    elif pb <= pc:
+        return b
+    else:
+        return c
+    
+def filter_paeth(row: bytes, prev_row:bytes) -> bytes:
+
+    result = []
+    for i, curr in enumerate(row):
+        a = row[i-1] if i > 0 else 0
+        b = prev_row[i]
+        c = prev_row[i-1] if i > 0 else 0
+        p = paeth_predictor(a, b, c)
+        result.append((curr - p) % 256)
+    return bytes(result)
+
+def unfilter_paeth(row: bytes, prev_row:bytes) -> bytes:
+
+    result = []
+    for i, curr in enumerate(row):
+        a = result[i-1] if i > 0 else 0
+        b = prev_row[i]
+        c = prev_row[i-1] if i > 0 else 0
+        p = paeth_predictor(a, b, c)
+        result.append((curr + p) % 256)
+    return bytes(result)
+
+def best_filter_for_row(row:bytes, prev_row:bytes) -> int:
+
+    candidates = {
+        0: row,
+        1: filter_sub(row),
+        2: filter_up(row, prev_row),
+        4: filter_paeth(row, prev_row)
+    }
+    def score(filtered:bytes) -> int:
+        return sum(min(v,256-v) for v in filtered)
+    
+    best_type = min(candidates, key=lambda t: score(candidates[t]))
+    return best_type, candidates[best_type]
+
+def apply_filters(pixels: np.array) -> list[tuple[int, bytes]]:
+    
+    if pixels.ndim ==2:
+        rows = [bytes(row) for row in pixels]
+
+    else:
+        rows = [bytes(row.flatten()) for row in pixels]
+
+    result = []
+    prev_row = bytes(len(rows[0]))
+
+    for row in rows:
+        filter_type, filtered = best_filter_for_row(row, prev_row)
+        result.append((filter_type, filtered))
+        prev_row = row
+
+    return result
+
+def remove_filters(filtered_rows: list[tuple[int, bytes]], width: int, channels: int) -> np.ndarray:
+
+    raw_rows = []
+    prev_row = bytes(width * channels)
+
+    for filter_type, row_data in filtered_rows:
+        if filter_type == 0:
+            raw_row = row_data
+        elif filter_type == 1:
+            raw_row = unfilter_sub(row_data)
+        elif filter_type == 2:
+            raw_row = unfilter_up(row_data, prev_row)
+        elif filter_type == 4:
+            raw_row = unfilter_paeth(row_data, prev_row)
+        else:
+            raise ValueError(f"Unknown filter type: {filter_type}")
+        
+        raw_rows.append(raw_row)
+        prev_row = raw_row
+
+
+    all_bytes = b''.join(raw_rows)
+    arr = np.frombuffer(all_bytes, dtype = np.uint8)
+
+    if channels == 1:
+        return arr.reshape(len(raw_rows), width)
+    else:
+        return arr.reshape(len(raw_rows), width, channels)

dipcompress-0.1.0/dipcompress/format.py

@@ -0,0 +1,99 @@
+import struct
+from dataclasses import dataclass
+from typing import Optional
+
+MAGIC = b'DIP\x01'
+VERSION = 1
+
+COMPRESS_RLE = 0X01
+COMPRESS_DEFLATE = 0X02
+
+HEADER_FORMAT = '>4sHIIBBI'
+
+HEADER_SIZE = struct.calcsize(HEADER_FORMAT)
+
+@dataclass
+class DipHeader:
+    width: int
+    height: int
+    channels: int
+    compression_mode:int
+    compressed_length: int
+    version: int = VERSION
+
+    def to_bytes(self) -> bytes:
+        return struct.pack(
+           HEADER_FORMAT,
+            MAGIC,
+            self.version,
+            self.width,
+            self.height,
+            self.channels,
+            self.compression_mode,
+            self.compressed_length
+        )
+    
+    @staticmethod
+    def from_bytes(data: bytes)->'DipHeader':
+        magic,version,width,height,channels,mode,comp_len = struct.unpack(
+            HEADER_FORMAT, data[:HEADER_SIZE]
+        )
+        if magic != MAGIC:
+            raise ValueError(f"Not a DipCompress file! Magic: {magic}")
+        return DipHeader(
+            width=width,
+            height=height,
+            channels=channels,
+            compression_mode=mode,
+            compressed_length=comp_len,
+            version=version
+        )
+
+
+def encode_filter_table(filter_types: list[int]) -> bytes:
+    return bytes(filter_types)
+
+
+def decode_filter_table(data: bytes) -> list[int]:
+    return list(data)
+
+
+def encode_huffman_table(codes: dict, original_length: int, padding: int) -> bytes:
+    entries = []
+    entries.append(struct.pack('>IB', original_length, padding))  # 4 bytes length + 1 byte padding
+    entries.append(struct.pack('>H', len(codes)))  # Number of entries
+
+    for symbol, code_str in sorted(codes.items()):
+        code_len = len(code_str)
+        padded = code_str.ljust((code_len + 7) // 8 * 8, '0')
+        code_bytes = bytes(int(padded[i:i+8], 2) for i in range(0, len(padded), 8))
+
+        entries.append(struct.pack('BB', symbol, code_len))
+        entries.append(code_bytes)
+
+    return b''.join(entries)
+
+
+def decode_huffman_table(data: bytes) -> tuple[dict, int, int, int]:
+    pos = 0
+    original_length, padding = struct.unpack('>IB', data[pos:pos+5])
+    pos += 5
+
+    num_entries = struct.unpack('>H', data[pos:pos+2])[0]
+    pos += 2
+
+    codes = {}
+    for _ in range(num_entries):
+        symbol, code_len = struct.unpack('BB', data[pos:pos+2])
+        pos += 2
+
+        num_bytes = (code_len + 7) // 8
+        code_bytes = data[pos:pos+num_bytes]
+        pos += num_bytes
+
+        full_bits = ''.join(f'{b:08b}' for b in code_bytes)
+        code_str = full_bits[:code_len]
+
+        codes[symbol] = code_str
+
+    return codes, original_length, padding, pos