cfbpy 0.0.2__tar.gz → 1.0.0__tar.gz

cfbpy-1.0.0/PKG-INFO

@@ -0,0 +1,99 @@
+Metadata-Version: 2.4
+Name: cfbpy
+Version: 1.0.0
+Summary: Python module for compressing/decompressing compound file binary format (CFB)
+Author: killerwhalee
+Author-email: killerwhalee <killerwhalee@kaist.ac.kr>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.12
+Project-URL: Homepage, https://github.com/killerwhalee/cfbpy
+Project-URL: Issues, https://github.com/killerwhalee/cfbpy/issues
+Description-Content-Type: text/markdown
+
+# cfbpy
+
+`cfbpy` is a Python package that allows you to compress and decompress compound file binary formats,
+similar to how zip/unzip works for regular zip files.
+It provides an easy-to-use interface for handling compound files through the CompoundFile class.
+
+## Features
+
+- Compress a directory into a compound file format.
+- Decompress a compound file back into a directory.
+
+## Installation
+
+cfbpy is written in pure Python and has no dependencies. You can install cfbpy via pip:
+
+```bash
+pip install cfbpy
+```
+
+Or add it to a [uv](https://docs.astral.sh/uv/) project:
+
+```bash
+uv add cfbpy
+```
+
+## Usage
+
+Here’s how you can use `cfbpy` to compress and decompress files:
+
+### Compress a Directory
+
+```python
+from cfbpy import compress
+
+compress('path/to/directory', 'path/to/output.cfb')
+```
+
+### Decompress a Compound File
+
+```python
+from cfbpy import decompress
+
+decompress('path/to/input.cfb', 'path/to/extracted_directory')
+```
+
+The same functions are also available as `CompoundFile.compress` and `CompoundFile.decompress`.
+
+## Methods
+
+- `compress(src, dest=None)`: Compresses the contents of the `src` directory into a compound file at `dest` (default: `<src>.cfb`).
+- `decompress(src, dest=None)`: Decompresses the compound file from `src` to the directory `dest` (default: `src` without its extension).
+
+Compound file names are limited by the format, so `compress` raises `ValueError` when a file or directory name
+
+- is longer than 31 UTF-16 code units,
+- contains any of `/ \ : !`, or
+- differs from a sibling only in letter case.
+
+`decompress` raises `CompoundFileError` (a subclass of `ValueError`) if the input is not a valid compound file, and `ValueError` for entry names that would escape the destination directory, such as `..`.
+
+Files are written as a major version 3 compound file (512-byte sectors). Both major version 3 and 4 (4096-byte sectors) files can be decompressed.
+
+## Development
+
+The project is managed with [uv](https://docs.astral.sh/uv/).
+
+```bash
+uv sync                             # create .venv with the package and dev dependencies
+uv run pytest                       # run tests
+uv run python benchmarks/bench.py   # measure compress/decompress time and memory
+uv build                            # build sdist and wheel into dist/
+```
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+## Contributing
+
+Pull requests are welcome! For major changes, please open an issue first to discuss what you would like to change.

cfbpy-1.0.0/README.md

@@ -0,0 +1,80 @@
+# cfbpy
+
+`cfbpy` is a Python package that allows you to compress and decompress compound file binary formats,
+similar to how zip/unzip works for regular zip files.
+It provides an easy-to-use interface for handling compound files through the CompoundFile class.
+
+## Features
+
+- Compress a directory into a compound file format.
+- Decompress a compound file back into a directory.
+
+## Installation
+
+cfbpy is written in pure Python and has no dependencies. You can install cfbpy via pip:
+
+```bash
+pip install cfbpy
+```
+
+Or add it to a [uv](https://docs.astral.sh/uv/) project:
+
+```bash
+uv add cfbpy
+```
+
+## Usage
+
+Here’s how you can use `cfbpy` to compress and decompress files:
+
+### Compress a Directory
+
+```python
+from cfbpy import compress
+
+compress('path/to/directory', 'path/to/output.cfb')
+```
+
+### Decompress a Compound File
+
+```python
+from cfbpy import decompress
+
+decompress('path/to/input.cfb', 'path/to/extracted_directory')
+```
+
+The same functions are also available as `CompoundFile.compress` and `CompoundFile.decompress`.
+
+## Methods
+
+- `compress(src, dest=None)`: Compresses the contents of the `src` directory into a compound file at `dest` (default: `<src>.cfb`).
+- `decompress(src, dest=None)`: Decompresses the compound file from `src` to the directory `dest` (default: `src` without its extension).
+
+Compound file names are limited by the format, so `compress` raises `ValueError` when a file or directory name
+
+- is longer than 31 UTF-16 code units,
+- contains any of `/ \ : !`, or
+- differs from a sibling only in letter case.
+
+`decompress` raises `CompoundFileError` (a subclass of `ValueError`) if the input is not a valid compound file, and `ValueError` for entry names that would escape the destination directory, such as `..`.
+
+Files are written as a major version 3 compound file (512-byte sectors). Both major version 3 and 4 (4096-byte sectors) files can be decompressed.
+
+## Development
+
+The project is managed with [uv](https://docs.astral.sh/uv/).
+
+```bash
+uv sync                             # create .venv with the package and dev dependencies
+uv run pytest                       # run tests
+uv run python benchmarks/bench.py   # measure compress/decompress time and memory
+uv build                            # build sdist and wheel into dist/
+```
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+## Contributing
+
+Pull requests are welcome! For major changes, please open an issue first to discuss what you would like to change.

cfbpy-1.0.0/cfbpy/cfb.py

@@ -0,0 +1,377 @@
+import array, os, secrets, shutil, sys
+
+from . import directory, reader
+from .constants import *
+from .reader import CompoundFileError
+
+SECTOR_SIZE = 1 << SECTOR_SHIFT
+MINI_SECTOR_SIZE = 1 << MINI_SECTOR_SHIFT
+ENTRIES_PER_SECTOR = SECTOR_SIZE // 4  # FAT/MINIFAT/DIFAT entries per sector
+
+# Chunk size for copying large streams
+COPY_CHUNK_SIZE = 1 << 20
+
+
+def _pack_sectors(entries) -> bytes:
+    """
+    Pack 32-bit sector numbers as little-endian, padded with FREESECT to a whole sector.
+
+    """
+
+    table = array.array(UINT32, entries)
+    table.extend([FREESECT] * (-len(table) % ENTRIES_PER_SECTOR))
+
+    if sys.byteorder == "big":
+        table.byteswap()
+
+    return table.tobytes()
+
+
+def _extend_chain(table: array.array, count: int) -> int:
+    """
+    Allocate `count` consecutive sectors at the end of `table` as one chain.
+
+    Returns first sector number of the chain, or ENDOFCHAIN if `count` is zero.
+
+    """
+
+    if not count:
+        return ENDOFCHAIN
+
+    start = len(table)
+    table.extend(range(start + 1, start + count))
+    table.append(ENDOFCHAIN)
+
+    return start
+
+
+class _MiniStream:
+    """
+    Mini stream and its MINIFAT, kept in memory as every stream is smaller than the cutoff size.
+
+    """
+
+    def __init__(self) -> None:
+        self.data = bytearray()
+        self.fat = array.array(UINT32)
+
+    def write(self, data: bytes) -> int:
+        """
+        Append data to the mini stream and return its first mini sector number.
+
+        """
+
+        count = -(-len(data) // MINI_SECTOR_SIZE)
+        self.data += data
+        self.data += bytes(count * MINI_SECTOR_SIZE - len(data))
+
+        return _extend_chain(self.fat, count)
+
+
+class _SectorWriter:
+    """
+    Write sectors sequentially to a file object while building the FAT.
+
+    Every stream is written as one run of consecutive sectors, so FAT chains are simple ranges and stream data never has to be held in memory as a whole.
+
+    """
+
+    def __init__(self, fp) -> None:
+        self.fp = fp
+        self.fat = array.array(UINT32)
+
+        # Reserve space for the header, which is written by `close()`
+        fp.write(bytes(HEADER_SIZE))
+
+    def _pad(self, size: int) -> int:
+        """
+        Pad data written so far to a whole sector and return the number of sectors used.
+
+        """
+
+        count = -(-size // SECTOR_SIZE)
+        self.fp.write(bytes(count * SECTOR_SIZE - size))
+
+        return count
+
+    def write(self, data: bytes) -> int:
+        """
+        Write data to new sectors and return its first sector number.
+
+        """
+
+        self.fp.write(data)
+
+        return _extend_chain(self.fat, self._pad(len(data)))
+
+    def copy(self, head: bytes, src) -> tuple[int, int]:
+        """
+        Write `head` followed by the rest of file object `src` to new sectors.
+
+        Returns first sector number and size of the stream.
+
+        """
+
+        self.fp.write(head)
+        size = len(head)
+
+        while chunk := src.read(COPY_CHUNK_SIZE):
+            self.fp.write(chunk)
+            size += len(chunk)
+
+        return _extend_chain(self.fat, self._pad(size)), size
+
+    def close(self, **header) -> None:
+        """
+        Write FAT and DIFAT sectors, then the header.
+
+        """
+
+        num_data_sectors = len(self.fat)
+
+        # FAT and DIFAT sectors need FAT entries themselves, so grow both until they fit
+        num_fat_sectors = num_difat_sectors = 0
+        while True:
+            total = num_data_sectors + num_fat_sectors + num_difat_sectors
+            fat_needed = -(-total // ENTRIES_PER_SECTOR)
+            difat_needed = -(
+                -max(0, fat_needed - NUM_HEADER_DIFAT) // (ENTRIES_PER_SECTOR - 1)
+            )
+
+            if (fat_needed, difat_needed) == (num_fat_sectors, num_difat_sectors):
+                break
+
+            num_fat_sectors, num_difat_sectors = fat_needed, difat_needed
+
+        first_fat_sector = num_data_sectors
+        first_difat_sector = first_fat_sector + num_fat_sectors
+        fat_sectors = list(range(first_fat_sector, first_difat_sector))
+
+        # Write FAT, marking FAT and DIFAT sectors
+        self.fat.extend([FATSECT] * num_fat_sectors)
+        self.fat.extend([DIFSECT] * num_difat_sectors)
+        self.fp.write(_pack_sectors(self.fat))
+
+        # Write DIFAT sectors for FAT sectors that do not fit in the header
+        overflow = fat_sectors[NUM_HEADER_DIFAT:]
+        per_sector = ENTRIES_PER_SECTOR - 1
+
+        for index in range(num_difat_sectors):
+            entries = overflow[index * per_sector : (index + 1) * per_sector]
+            entries.extend([FREESECT] * (per_sector - len(entries)))
+
+            if index + 1 < num_difat_sectors:
+                entries.append(first_difat_sector + index + 1)
+            else:
+                entries.append(ENDOFCHAIN)
+
+            self.fp.write(_pack_sectors(entries))
+
+        header_difat = fat_sectors[:NUM_HEADER_DIFAT]
+        header_difat.extend([FREESECT] * (NUM_HEADER_DIFAT - len(header_difat)))
+
+        # Write header
+        self.fp.seek(0)
+        self.fp.write(
+            HEADER_FORMAT.pack(
+                HEADER_SIGNATURE,
+                bytes(16),  # Header CLSID
+                MINOR_VERSION,
+                MAJOR_VERSION,
+                BYTE_ORDER,
+                SECTOR_SHIFT,
+                MINI_SECTOR_SHIFT,
+                0,  # Number of directory sectors, must be zero for major version 3
+                num_fat_sectors,
+                header["first_dir_sector"],
+                0,  # Transaction signature number
+                MINI_STREAM_CUTOFF_SIZE,
+                header["first_mini_fat_sector"],
+                header["num_mini_fat_sectors"],
+                first_difat_sector if num_difat_sectors else ENDOFCHAIN,
+                num_difat_sectors,
+                *header_difat,
+            )
+        )
+
+
+def _walk(src, skip):
+    """
+    Yield `(parent, name, path)` for every directory and file under `src`.
+
+    `parent` is the relative path of the parent directory (empty for `src` itself) and `path` is `None` for directories. Entries are sorted so output is deterministic. Files whose absolute path is in `skip` (output files inside `src`) are left out.
+
+    """
+
+    for dirpath, dirnames, filenames in os.walk(src):
+        dirnames.sort()
+        filenames.sort()
+
+        rel = os.path.relpath(dirpath, src)
+        parent = "" if rel == os.curdir else rel
+
+        if parent:
+            yield os.path.dirname(parent), os.path.basename(parent), None
+
+        for name in filenames:
+            path = os.path.join(dirpath, name)
+
+            if os.path.abspath(path) not in skip:
+                yield parent, name, path
+
+
+def compress(src, dest=None):
+    """
+    Create compound file from given directory tree.
+
+    """
+
+    src = os.path.normpath(src)
+
+    if not os.path.isdir(src):
+        raise NotADirectoryError(f"Source is not a directory: {src}")
+
+    if not dest:
+        dest = f"{src}.cfb"
+
+    dest = os.path.abspath(dest)
+
+    # Write to a temporary file next to `dest` and replace `dest` only on success, so a
+    # failure never truncates or removes an existing file
+    while True:
+        temp = os.path.join(
+            os.path.dirname(dest),
+            f".{os.path.basename(dest)}.{secrets.token_hex(4)}.tmp",
+        )
+
+        try:
+            fp = open(temp, "xb")
+            break
+
+        except FileExistsError:
+            continue
+
+    try:
+        with fp:
+            _compress(src, fp, skip={dest, temp})
+
+        # Keep permissions of a file being replaced
+        if os.path.exists(dest):
+            shutil.copymode(dest, temp)
+
+        os.replace(temp, dest)
+
+    except BaseException:
+        os.remove(temp)
+        raise
+
+
+def _compress(src, fp, skip):
+    root = directory.Entry("Root Entry", obj_type=directory.OBJTY_ROOT)
+    storages = {"": root}
+    mini_stream = _MiniStream()
+
+    writer = _SectorWriter(fp)
+
+    for parent, name, path in _walk(src, skip):
+        storage = storages[parent]
+
+        # Add storage as directory entry
+        if path is None:
+            storages[os.path.join(parent, name)] = storage.add(
+                directory.Entry(name, obj_type=directory.OBJTY_STORAGE),
+            )
+            continue
+
+        entry = storage.add(directory.Entry(name, obj_type=directory.OBJTY_STREAM))
+
+        # Write data into mini-sector if smaller than cutoff, otherwise into sector
+        with open(path, "rb") as f:
+            head = f.read(MINI_STREAM_CUTOFF_SIZE)
+
+            if len(head) < MINI_STREAM_CUTOFF_SIZE:
+                entry.sector, entry.size = mini_stream.write(head), len(head)
+
+            else:
+                entry.sector, entry.size = writer.copy(head, f)
+
+    # Write mini stream into sector; its location is stored in root entry
+    root.sector = writer.write(mini_stream.data)
+    root.size = len(mini_stream.data)
+
+    # Write mini-fat into sector
+    mini_fat_data = _pack_sectors(mini_stream.fat) if mini_stream.fat else b""
+    first_mini_fat_sector = writer.write(mini_fat_data)
+
+    # Write directory entry into sector
+    first_dir_sector = writer.write(directory.serialize(root, SECTOR_SIZE))
+
+    # Write fat, difat and header
+    writer.close(
+        first_dir_sector=first_dir_sector,
+        first_mini_fat_sector=first_mini_fat_sector,
+        num_mini_fat_sectors=len(mini_fat_data) // SECTOR_SIZE,
+    )
+
+
+def _safe_name(name):
+    """
+    Return entry name as a file name, rejecting names that would escape the destination directory.
+
+    """
+
+    if (
+        name in ("", os.curdir, os.pardir)
+        or os.sep in name
+        or (os.altsep and os.altsep in name)
+        or "\0" in name
+        or os.path.isabs(name)
+        or os.path.splitdrive(name)[0]
+    ):
+        raise ValueError(f"Unsafe entry name in compound file: {name!r}")
+
+    return name
+
+
+def decompress(src, dest=None):
+    """
+    Decompress compound file and create directory at destination.
+
+    Raises `CompoundFileError` if `src` is not a valid compound file.
+
+    """
+
+    # Set path same as `src` if dest is not set
+    if not dest:
+        dest, _ = os.path.splitext(src)
+
+    # Open before creating `dest`, so an invalid file leaves nothing behind
+    with reader.CompoundFileReader(src) as cfb:
+        os.makedirs(dest, exist_ok=True)
+
+        # Storages are yielded before their children
+        for parts, entry in cfb.walk():
+            path = os.path.join(dest, *parts[:-1], _safe_name(parts[-1]))
+
+            if entry.obj_type == directory.OBJTY_STORAGE:
+                os.makedirs(path, exist_ok=True)
+
+            else:
+                with open(path, "wb") as f:
+                    cfb.copy_stream(entry, f)
+
+    # Return True if decompressed successfully
+    return True
+
+
+class CompoundFile:
+    compress = staticmethod(compress)
+    decompress = staticmethod(decompress)
+
+    def open(self, src):
+        """
+        Open compound file and create CompoundFile instance.
+
+        """
+
+        raise NotImplementedError

cfbpy-1.0.0/cfbpy/constants.py

@@ -0,0 +1,24 @@
+import array, struct
+
+# Header constants
+HEADER_SIGNATURE = b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1"
+MINOR_VERSION = 0x003E
+MAJOR_VERSION = 0x0003
+BYTE_ORDER = 0xFFFE
+SECTOR_SHIFT = 0x0009  # 512-byte sectors for major version 3
+MINI_SECTOR_SHIFT = 0x0006
+MINI_STREAM_CUTOFF_SIZE = 0x00001000
+
+HEADER_FORMAT = struct.Struct("<8s16sHHHHH6xIIIIIIIII109I")
+HEADER_SIZE = HEADER_FORMAT.size  # 512 bytes
+NUM_HEADER_DIFAT = 109
+
+# Sector constants
+MAXREGSECT = 0xFFFFFFFA  #: (-6) maximum SECT
+DIFSECT = 0xFFFFFFFC  #: (-4) denotes a DIFAT sector in a FAT
+FATSECT = 0xFFFFFFFD  #: (-3) denotes a FAT sector in a FAT
+ENDOFCHAIN = 0xFFFFFFFE  #: (-2) end of a virtual stream chain
+FREESECT = 0xFFFFFFFF  #: (-1) unallocated sector
+
+# Array typecode of unsigned 32-bit integer, used for FAT and MINIFAT
+UINT32 = next(code for code in "IL" if array.array(code).itemsize == 4)