vcdiff-decoder 0.1.0a1__py3-none-any.whl

LICENSE

@@ -0,0 +1,204 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+   "License" shall mean the terms and conditions for use, reproduction,
+   and distribution as defined by Sections 1 through 9 of this document.
+
+   "Licensor" shall mean the copyright owner or entity granting the License.
+
+   "Legal Entity" shall mean the union of the acting entity and all
+   other entities that control, are controlled by, or are under common
+   control with that entity. For the purposes of this definition,
+   "control" means (i) the power, direct or indirect, to cause the
+   direction or management of such entity, whether by contract or
+   otherwise, or (ii) ownership of fifty percent (50%) or more of the
+   outstanding shares, or (iii) beneficial ownership of such entity.
+
+   "You" (or "Your") shall mean an individual or Legal Entity
+   exercising permissions granted by this License.
+
+   "Source" form shall mean the preferred form for making modifications,
+   including but not limited to software source code, documentation
+   source, and configuration files.
+
+   "Object" form shall mean any form resulting from mechanical
+   transformation or translation of a Source form, including but
+   not limited to compiled object code, generated documentation,
+   and conversions to other media types.
+
+   "Work" shall mean the work of authorship, whether in Source or
+   Object form, made available under the License, as indicated by a
+   copyright notice that is included in or attached to the work
+   (which shall not include the Communication Networks class of works).
+
+   "Derivative Works" shall mean any work, whether in Source or Object
+   form, that is based upon (or derived from) the Work and for which the
+   editorial revisions, annotations, elaborations, or other modifications
+   represent, as a whole, an original work of authorship. For the purposes
+   of this License, Derivative Works shall not include works that remain
+   separable from, or merely link (or bind by name) to the interfaces of,
+   the Work and derivative works thereof.
+
+   "Contribution" shall mean any work of authorship, including
+   the original version of the Work and any modifications or additions
+   to that Work or Derivative Works thereof, that is intentionally
+   submitted to Licensor for inclusion in the Work by the copyright owner
+   or by an individual or Legal Entity authorized to submit on behalf of
+   the copyright owner. For the purposes of this definition, "submitted"
+   means any form of electronic, verbal, or written communication sent
+   to the Licensor or its representatives, including but not limited to
+   communication on electronic mailing lists, source code control
+   systems, and issue tracking systems that are managed by, or on behalf
+   of, the Licensor for the purpose of discussing and improving the Work,
+   but excluding communication that is conspicuously marked or otherwise
+   designated in writing by the copyright owner as "Not a Contribution."
+
+   "Contributor" shall mean Licensor and any individual or Legal Entity
+   on behalf of whom a Contribution has been received by Licensor and
+   subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+   this License, each Contributor hereby grants to You a perpetual,
+   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+   copyright license to use, reproduce, modify, publish, distribute,
+   sublicense, and/or sell copies of the Work, and to permit persons
+   to whom the Work is furnished to do so, subject to the following
+   conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Work.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+   this License, each Contributor hereby grants to You a perpetual,
+   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+   (except as stated in this section) patent license to make, have made,
+   use, offer to sell, sell, import, and otherwise transfer the Work,
+   where such license applies only to those patent claims licensable
+   by such Contributor that are necessarily infringed by their
+   Contribution(s) alone or by combination of their Contribution(s)
+   with the Work to which such Contribution(s) was submitted. If You
+   institute patent litigation against any entity (including a
+   cross-claim or counterclaim in a lawsuit) alleging that the Work
+   or a Contribution incorporated within the Work constitutes direct
+   or contributory patent infringement, then any patent licenses
+   granted to You under this License for that Work shall terminate
+   as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+   Work or Derivative Works thereof in any medium, with or without
+   modifications, and in Source or Object form, provided that You
+   meet the following conditions:
+
+   (a) You must give any other recipients of the Work or
+       Derivative Works a copy of this License; and
+
+   (b) You must cause any modified files to carry prominent notices
+       stating that You changed the files; and
+
+   (c) You must retain, in the Source form of any Derivative Works
+       that You distribute, all copyright, trademark, patent,
+       attribution and other notices from the Source form of the Work,
+       excluding those notices that do not pertain to any part of
+       the Derivative Works; and
+
+   (d) If the Work includes a "NOTICE" text file as part of its
+       distribution, then any Derivative Works that You distribute must
+       include a readable copy of the attribution notices contained
+       within such NOTICE file, excluding those notices that do not
+       pertain to any part of the Derivative Works, in at least one
+       of the following places: within a NOTICE text file distributed
+       as part of the Derivative Works; within the Source form or
+       documentation, if provided along with the Derivative Works; or,
+       within a display generated by the Derivative Works, if and
+       wherever such third-party notices normally appear. The contents
+       of the NOTICE file are for informational purposes only and
+       do not modify the License. You may add Your own attribution
+       notices within Derivative Works that You distribute, alongside
+       or as an addendum to the NOTICE text from the Work, provided
+       that such additional attribution notices cannot be construed
+       as modifying the License.
+
+   You may add Your own copyright notice to Your modifications and
+   may provide additional or different license terms and conditions
+   for use, reproduction, or distribution of Your modifications, or
+   for any such Derivative Works as a whole, provided Your use,
+   reproduction, and distribution of the Work otherwise complies with
+   the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+   any Contribution intentionally submitted for inclusion in the Work
+   by You to the Licensor shall be under the terms and conditions of
+   this License, without any additional terms or conditions.
+   Notwithstanding the above, nothing herein shall supersede or modify
+   the terms of any separate license agreement you may have executed
+   with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+   names, trademarks, service marks, or product names of the Licensor,
+   except as required for reasonable and customary use in describing the
+   origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+   agreed to in writing, Licensor provides the Work (and each
+   Contributor provides its Contributions) on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+   implied, including, without limitation, any warranties or conditions
+   of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+   PARTICULAR PURPOSE. You are solely responsible for determining the
+   appropriateness of using or redistributing the Work and assume any
+   risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+   whether in tort (including negligence), contract, or otherwise,
+   unless required by applicable law (such as deliberate and grossly
+   negligent acts) or agreed to in writing, shall any Contributor be
+   liable to You for damages, including any direct, indirect, special,
+   incidental, or consequential damages of any character arising as a
+   result of this License or out of the use or inability to use the
+   Work (including but not limited to damages for loss of goodwill,
+   work stoppage, computer failure or malfunction, or any and all
+   other commercial damages or losses), even if such Contributor
+   has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. When redistributing
+   the Work or Derivative Works thereof, You may choose to offer,
+   and charge a fee for, acceptance of support, warranty, indemnity,
+   or other liability obligations and/or rights consistent with this
+   License. However, in accepting such obligations, You may act only
+   on Your own behalf and on Your sole responsibility, not on behalf
+   of any other Contributor, and only if You agree to indemnify,
+   defend, and hold each Contributor harmless for any liability
+   incurred by, or claims asserted against, such Contributor by reason
+   of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+   To apply the Apache License to your work, attach the following
+   boilerplate notice, with the fields enclosed by brackets "[]"
+   replaced with your own identifying information. (Don't include
+   the brackets!)  The text should be enclosed in the appropriate
+   comment syntax for the file format. We also recommend that a
+   file or class name and description of purpose be included on the
+   same page as the copyright notice for easier identification within
+   third-party archives.
+
+   Copyright 2025 Ably Realtime Limited
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,260 @@
+# VCDIFF Python Decoder
+
+A Python implementation of a VCDIFF (RFC 3284) decoder library and command-line tool for efficient binary differencing and compression.
+
+## Overview
+
+The library provides a VCDIFF decoder that can decode delta files created according to RFC 3284 - The VCDIFF Generic Differencing and Compression Data Format. VCDIFF is a format for expressing one data stream as a variant of another data stream, commonly used for binary differencing, compression, and patch applications.
+
+## Features
+
+- **Python Library**: RFC 3284 compliant VCDIFF decoding with clean, Pythonic API
+- **Command-Line Tool**: Apply deltas and inspect VCDIFF file structure
+- **Comprehensive Validation**: Support for all VCDIFF instruction types (ADD, COPY, RUN)
+- **Address Caching**: Efficient decoding with proper address cache implementation
+- **Checksum Validation**: Full Adler-32 checksum validation support
+- **Robust Error Handling**: Detailed error messages for debugging malformed files
+- **Extensive Testing**: 85 test cases with reference implementation validation
+- **Type Safety**: Full type hints for better development experience
+
+## Limitations
+
+- **Application Headers**: This implementation does not handle application header information
+- **Secondary Compression**: This decoder does not support secondary compression (e.g., gzip, bzip2)
+- **Compatibility**: Works with VCDIFF deltas created using `xdelta3 -e -S -A` (no secondary compression, no application header)
+
+## Checksum Support
+
+- **VCD_ADLER32**: This implementation detects and parses the VCD_ADLER32 extension (bit 0x04 in window indicator)
+- **Non-standard Extension**: The Adler-32 checksum is not part of RFC 3284 but is supported by some implementations
+- **Validation**: Full Adler-32 checksum validation is implemented and performed during decoding
+- **Display**: Checksums are displayed in the CLI output as `Adler32: 0x########`
+
+## Installation
+
+```bash
+pip install vcdiff
+```
+
+### Development Installation
+
+```bash
+poetry install
+```
+
+### Cloning with Test Suite
+
+This repository includes the VCDIFF test suite as a git submodule. To clone the repository with all test cases:
+
+```bash
+git clone --recursive https://github.com/ably/vcdiff-py.git
+```
+
+If you've already cloned the repository without the submodule, initialize it:
+
+```bash
+git submodule update --init --recursive
+```
+
+To update the test suite submodule to the latest version:
+
+```bash
+git submodule update --remote
+```
+
+## Quick Start
+
+### Library Usage
+
+```python
+import vcdiff
+
+# Read the source file
+with open("original.txt", "rb") as f:
+    source = f.read()
+
+# Read the VCDIFF delta file
+with open("changes.vcdiff", "rb") as f:
+    delta_data = f.read()
+
+# Apply the delta to reconstruct the target
+try:
+    result = vcdiff.decode(source, delta_data)
+    print(f"Decoded result: {result}")
+except vcdiff.VCDIFFError as e:
+    print(f"Decoding failed: {e}")
+```
+
+## API Reference
+
+### Core Functions
+
+#### `vcdiff.decode(source: bytes, delta: bytes) -> bytes`
+
+Decodes a VCDIFF delta file using the provided source data and returns the reconstructed target data.
+
+**Parameters:**
+- `source`: The original source data (may be empty for deltas that don't reference source)
+- `delta`: The VCDIFF delta file data
+
+**Returns:**
+- Decoded target data as bytes
+
+**Raises:**
+- `VCDIFFError`: If decoding fails (malformed delta, checksum validation failure, etc.)
+
+#### `vcdiff.Decoder(source: bytes)`
+
+Creates a new decoder instance with the specified source data. Useful for decoding multiple deltas against the same source.
+
+**Parameters:**
+- `source`: The source data for decoding operations
+
+**Returns:**
+- A `Decoder` instance that can be used to decode multiple deltas
+
+#### `decoder.decode(delta: bytes) -> bytes`
+
+Decodes a single VCDIFF delta using the decoder's source data.
+
+### Error Handling
+
+The decoder provides detailed error messages for various failure conditions:
+- Invalid VCDIFF format or magic bytes
+- Malformed varint encoding
+- Out-of-bounds memory access attempts
+- Checksum validation failures
+- Truncated or corrupted delta files
+
+**Exception Hierarchy:**
+- `VCDIFFError`: Base exception for all VCDIFF-related errors
+- `InvalidMagicError`: Invalid VCDIFF magic bytes
+- `InvalidVersionError`: Unsupported VCDIFF version
+- `InvalidFormatError`: Malformed delta structure
+- `CorruptedDataError`: Data corruption detected
+- `InvalidChecksumError`: Checksum validation failure
+
+## Testing
+
+### Prerequisites
+
+For comprehensive testing, this project uses xdelta3 as a reference implementation to verify the correctness of the decoder.
+
+#### Installing xdelta3
+
+##### macOS (Homebrew)
+```bash
+brew install xdelta
+```
+
+##### Linux (Ubuntu/Debian)
+```bash
+sudo apt-get install xdelta3
+```
+
+### Running Tests
+
+To run the Python unit tests:
+
+```bash
+poetry run pytest
+```
+
+To run tests with coverage:
+
+```bash
+poetry run pytest --cov=vcdiff tests/
+```
+
+To run the comprehensive test suite against the VCDIFF test cases (requires submodule):
+
+```bash
+poetry run pytest tests/test_vcdiff.py -v
+```
+
+The test suite includes:
+- **20 general positive tests**: Valid VCDIFF files that should decode successfully
+- **33 targeted negative tests**: Invalid VCDIFF files that should be rejected with appropriate errors
+- **32 targeted positive tests**: Specific feature validation tests
+- **Total**: 85 test cases with 100% pass rate
+
+### Test Results
+- **General Positive Tests**: 20/20 passed ✅
+- **Targeted Negative Tests**: 33/33 passed ✅
+- **Targeted Positive Tests**: 32/32 passed ✅
+
+## Contributing
+
+Contributions are welcomed. Please follow these guidelines:
+
+### Getting Started
+
+1. Fork the repository
+2. Clone your fork with submodules: `git clone --recursive <your-fork-url>`
+3. Create a feature branch: `git checkout -b feature/your-feature-name`
+4. Make your changes
+5. Test your changes thoroughly
+6. Submit a pull request
+
+### Development Guidelines
+
+- **Code Style**: Follow PEP 8 formatting (`black .` for automatic formatting)
+- **Type Hints**: All new code should include proper type annotations
+- **Testing**: All new features must include tests
+- **Documentation**: Update documentation for any API changes
+- **Commits**: Use clear, descriptive commit messages
+
+### Before Submitting
+
+Ensure your contribution passes all checks:
+
+```bash
+# Run all tests
+pytest
+
+# Format code
+black .
+
+# Lint code
+flake8 vcdiff/
+
+# Type checking
+mypy vcdiff/
+```
+
+### Reporting Issues
+
+When reporting bugs, please include:
+- Python version
+- Operating system
+- Minimal reproduction case
+- Expected vs actual behavior
+- Sample VCDIFF files (if applicable)
+
+### Feature Requests
+
+For new features, please:
+- Check existing issues first
+- Describe the use case
+- Provide RFC 3284 references if applicable
+- Consider backwards compatibility
+
+## Requirements
+
+- **Python**: 3.8 or higher
+- **Development Dependencies**:
+  - `pytest>=7.0.0`
+  - `pytest-cov>=4.0.0`
+  - `black>=22.0.0`
+  - `flake8>=5.0.0`
+  - `mypy>=1.0.0`
+
+## License
+
+This project is licensed under the Apache License 2.0. See the LICENSE file for details.
+
+## References
+
+- [RFC 3284: The VCDIFF Generic Differencing and Compression Data Format](https://tools.ietf.org/html/rfc3284)
+- [xdelta3: VCDIFF binary diff tool](https://github.com/jmacd/xdelta)
+- [Go VCDIFF Implementation](https://github.com/ably/vcdiff-go) - Algorithmically compatible sibling project

vcdiff_decoder/__init__.py

@@ -0,0 +1,43 @@
+"""VCDIFF Python Implementation
+
+A Python implementation of VCDIFF (RFC 3284) delta compression format.
+
+Copyright 2025 Ably Realtime Limited
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from .decoder import Decoder, decode, parse_delta
+from .exceptions import (
+    VCDIFFError,
+    InvalidMagicError,
+    InvalidVersionError,
+    InvalidFormatError,
+    CorruptedDataError,
+    InvalidChecksumError,
+)
+from .types import InstructionType
+
+__version__ = "1.0.0"
+__all__ = [
+    "Decoder",
+    "decode",
+    "parse_delta",
+    "InstructionType",
+    "VCDIFFError",
+    "InvalidMagicError",
+    "InvalidVersionError",
+    "InvalidFormatError",
+    "CorruptedDataError",
+    "InvalidChecksumError",
+]

vcdiff_decoder/addresscache.py

@@ -0,0 +1,111 @@
+"""Address cache implementation for VCDIFF COPY instructions"""
+
+import io
+from typing import BinaryIO, List
+
+from .types import SELF_MODE, HERE_MODE
+from .varint import read_varint
+from .exceptions import VCDIFFError
+
+
+class AddressCache:
+    """Manages address encoding/decoding for COPY instructions"""
+    
+    def __init__(self, near_size: int, same_size: int):
+        """Initialize address cache with specified sizes
+        
+        Args:
+            near_size: Size of the "near" address cache (typically 4)
+            same_size: Size of the "same" address cache (typically 3 * 256)
+        """
+        self.near_size = near_size
+        self.same_size = same_size
+        self.near: List[int] = [0] * near_size
+        self.next_near_slot = 0
+        self.same: List[int] = [0] * (same_size * 256)
+        self.address_stream: BinaryIO = io.BytesIO()
+    
+    def reset(self, addresses: bytes) -> None:
+        """Reset the address cache for a new window
+        
+        Args:
+            addresses: Address section data for this window
+        """
+        self.next_near_slot = 0
+        
+        # Clear near cache
+        for i in range(len(self.near)):
+            self.near[i] = 0
+        
+        # Clear same cache
+        for i in range(len(self.same)):
+            self.same[i] = 0
+        
+        self.address_stream = io.BytesIO(addresses)
+    
+    def decode_address(self, here: int, mode: int) -> int:
+        """Decode an address using the specified mode
+        
+        Args:
+            here: Current position in target stream
+            mode: Addressing mode to use
+            
+        Returns:
+            The decoded address
+            
+        Raises:
+            VCDIFFError: If the addressing mode is invalid or cache is uninitialized
+        """
+        # Validate addressing mode
+        if mode > 8:
+            raise VCDIFFError(f"invalid address cache mode {mode}: valid modes are 0-8")
+        
+        if mode == SELF_MODE:
+            addr = read_varint(self.address_stream)
+        
+        elif mode == HERE_MODE:
+            offset = read_varint(self.address_stream)
+            if offset > here:
+                raise VCDIFFError(f"HERE mode offset {offset} exceeds current position {here}")
+            addr = here - offset
+        
+        else:
+            # Near cache or same cache modes
+            if mode - 2 < self.near_size:
+                # Near cache
+                cache_index = mode - 2
+                if self.near[cache_index] == 0:
+                    raise VCDIFFError(f"near cache slot {cache_index} is uninitialized")
+                offset = read_varint(self.address_stream)
+                addr = self.near[cache_index] + offset
+            else:
+                # Same cache
+                m = mode - (2 + self.near_size)
+                if m >= self.same_size:
+                    raise VCDIFFError(
+                        f"same cache mode {mode} exceeds available slots "
+                        f"(max {2 + self.near_size + self.same_size - 1})"
+                    )
+                
+                byte_data = self.address_stream.read(1)
+                if not byte_data:
+                    raise VCDIFFError("unexpected EOF while reading same cache address")
+                
+                b = byte_data[0]
+                addr = self.same[m * 256 + b]
+        
+        self.update(addr)
+        return addr
+    
+    def update(self, address: int) -> None:
+        """Update the address cache with a new address
+        
+        Args:
+            address: Address to add to the cache
+        """
+        if self.near_size > 0:
+            self.near[self.next_near_slot] = address
+            self.next_near_slot = (self.next_near_slot + 1) % self.near_size
+        
+        if self.same_size > 0:
+            self.same[address % (self.same_size * 256)] = address

vcdiff_decoder/adler32.py

@@ -0,0 +1,43 @@
+"""Adler-32 checksum implementation for VCDIFF"""
+
+from typing import Union
+
+
+# Base for modulo arithmetic
+ADLER32_BASE = 65521
+# Number of iterations we can safely do before applying the modulo
+ADLER32_NMAX = 5552
+
+
+def compute_checksum(initial: int, data: Union[bytes, bytearray]) -> int:
+    """Compute the Adler32 checksum for the given data
+    
+    Args:
+        initial: Initial checksum value (typically 1)
+        data: Data to compute checksum for
+        
+    Returns:
+        The computed Adler32 checksum
+    """
+    if not data:
+        return initial
+    
+    s1 = initial & 0xFFFF
+    s2 = (initial >> 16) & 0xFFFF
+    
+    index = 0
+    length = len(data)
+    
+    while length > 0:
+        k = min(length, ADLER32_NMAX)
+        length -= k
+        
+        for i in range(k):
+            s1 += data[index]
+            s2 += s1
+            index += 1
+        
+        s1 %= ADLER32_BASE
+        s2 %= ADLER32_BASE
+    
+    return (s2 << 16) | s1