snappy-cython-file 1.4.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,142 @@
1
+ Metadata-Version: 2.4
2
+ Name: snappy-cython-file
3
+ Version: 1.4.7
4
+ Summary: A self-contained Snappy raw-format codec written in Cython
5
+ Author-email: Isaac Onagh <mkalafsaz@gmail.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/i4147/snappy_cython
8
+ Keywords: snappy,compression,cython
9
+ Classifier: Development Status :: 3 - Alpha
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Cython
13
+ Classifier: Topic :: System :: Archiving :: Compression
14
+ Requires-Python: >=3.9
15
+ Description-Content-Type: text/markdown
16
+ Provides-Extra: test
17
+ Requires-Dist: pytest>=7; extra == "test"
18
+
19
+ # Snappy Cython File Codec
20
+
21
+ A self-contained implementation of [Google Snappy's raw compression format](https://github.com/google/snappy/blob/main/format_description.txt), written in Cython. It provides bytes and whole-file APIs without requiring a system `libsnappy` installation.
22
+
23
+ ## Features
24
+
25
+ - Real Snappy literals and back-reference copies (not a literal-only encoder).
26
+ - Strict decoding with bounds checks, overlapping-copy support, and rejection of malformed/trailing data.
27
+ - `compress_file()` and `decompress_file()` with predictable default names, custom output paths, atomic writes, and opt-in overwriting.
28
+ - Optional decompressed-size limit for handling untrusted input.
29
+ - Pytest suite, example CLI, and a recursive folder benchmark against stdlib `gzip`, `bz2`, and `lzma` (xz).
30
+ - No native dependency beyond a C compiler and the Cython build requirements.
31
+
32
+ > **Raw vs. framed format:** this project reads/writes Snappy's raw format, with an uncompressed-length varint followed by Snappy elements. It does **not** implement the framed/streaming `.sz` container. Whole files are loaded into memory. Raw Snappy supports inputs up to 2³² − 1 bytes.
33
+
34
+ ## Install for development
35
+
36
+ Linux/macOS with Python 3.9+ and a C compiler:
37
+
38
+ ```bash
39
+ ./tools/bootstrap_dev.sh
40
+ ```
41
+
42
+ That helper creates `.venv`, installs the package in editable mode, compiles the Cython extension, and installs pytest. Equivalent manual commands are:
43
+
44
+ ```bash
45
+ python -m venv .venv
46
+ .venv/bin/python -m pip install -e '.[test]'
47
+ ```
48
+
49
+ On Windows, activate the virtual environment and run the same `pip install -e '.[test]'` command with its Python executable. A compiler supported by setuptools/Cython is required.
50
+
51
+ ## Python API
52
+
53
+ ```python
54
+ from snappy_cython import compress, compress_file, decompress, decompress_file
55
+
56
+ # Bytes-like objects in, bytes out.
57
+ encoded = compress(b"hello hello hello")
58
+ assert decompress(encoded) == b"hello hello hello"
59
+
60
+ # Default: report.txt -> report.txt.snappy
61
+ compressed_path = compress_file("report.txt")
62
+
63
+ # An explicit output name can have any extension.
64
+ compress_file("report.txt", "output/fname.ext")
65
+
66
+ # Default: report.txt.snappy -> report.txt
67
+ # Fails if report.txt already exists, unless overwrite=True is supplied.
68
+ restored_path = decompress_file(compressed_path, overwrite=True)
69
+
70
+ # Explicit destination and a pre-allocation safety limit are also supported.
71
+ decompress_file(
72
+ "output/fname.ext",
73
+ "restored.txt",
74
+ max_output_size=100_000_000,
75
+ )
76
+ ```
77
+
78
+ ### Function reference
79
+
80
+ | Function | Behavior |
81
+ | --- | --- |
82
+ | `compress(data)` | Compress a bytes-like object to raw Snappy `bytes`. |
83
+ | `decompress(data, max_output_size=None)` | Decode raw Snappy and validate the complete stream. |
84
+ | `compress_file(input_path, output_path=None, *, overwrite=False)` | Compress a file; default output appends `.snappy`; returns `pathlib.Path`. |
85
+ | `decompress_file(input_path, output_path=None, *, overwrite=False, max_output_size=None)` | Decompress a file; strips a final `.snappy`, otherwise appends `.out`; returns `Path`. |
86
+
87
+ Existing destinations are protected by default. File data is written to a temporary sibling and moved into place only after successful encoding/decoding. Parent directories must already exist.
88
+
89
+ ## Run tests and demo
90
+
91
+ ```bash
92
+ ./tools/run_checks.sh
93
+ ```
94
+
95
+ Or run them separately:
96
+
97
+ ```bash
98
+ .venv/bin/python -m pytest -q
99
+ .venv/bin/python examples/demo.py
100
+ .venv/bin/python examples/demo.py path/to/input.bin -o fname.ext
101
+ ```
102
+
103
+ The no-argument demo uses a temporary directory and verifies both the bytes and file round trips.
104
+
105
+ An optional interoperability check verifies that this codec and the independent Snappy implementation shipped in `cramjam` can decode each other's raw streams:
106
+
107
+ ```bash
108
+ .venv/bin/python -m pip install cramjam
109
+ .venv/bin/python tools/check_interoperability.py
110
+ ```
111
+
112
+ ## Folder benchmark
113
+
114
+ ```bash
115
+ # Recursively benchmark every regular file under a user-selected folder.
116
+ .venv/bin/python benchmarks/benchmark_folder.py /path/to/folder
117
+
118
+ # Only top-level logs, best of five compression runs, save detailed CSV.
119
+ .venv/bin/python benchmarks/benchmark_folder.py /path/to/folder \
120
+ --glob '*.log' --repeat 5 --csv benchmark-results.csv
121
+
122
+ # Generate a small mixed sample corpus, then benchmark it.
123
+ .venv/bin/python tools/make_benchmark_corpus.py /tmp/snappy-corpus
124
+ .venv/bin/python benchmarks/benchmark_folder.py /tmp/snappy-corpus
125
+ ```
126
+
127
+ Each file is read once. Every codec's output is decompressed and checked before timing. The reported duration is the best compression run; file I/O and verification decompression are excluded. Ratios are `compressed / original` (lower is better), and MiB/s measures original bytes processed per second (higher is better). Snappy generally targets speed rather than the smaller output typically produced by xz or bz2.
128
+
129
+ ## Project layout
130
+
131
+ ```text
132
+ src/snappy_cython/_snappy.pyx Cython raw-format encoder/decoder
133
+ src/snappy_cython/files.py Whole-file convenience functions
134
+ tests/ Pytest unit, malformed-input, and file tests
135
+ examples/demo.py Bytes and file API demonstration
136
+ benchmarks/benchmark_folder.py Snappy/gzip/bz2/xz comparison
137
+ tools/ Development and sample-corpus helper scripts
138
+ ```
139
+
140
+ ## Implementation notes
141
+
142
+ The compressor uses a hash table of four-byte sequences and emits Snappy copy commands for matches up to the format's 65,535-byte offset limit. The decoder supports all literal length encodings and all three copy-offset encodings. The implementation is independent Cython code, not a binding to Google's C++ library.
@@ -0,0 +1,124 @@
1
+ # Snappy Cython File Codec
2
+
3
+ A self-contained implementation of [Google Snappy's raw compression format](https://github.com/google/snappy/blob/main/format_description.txt), written in Cython. It provides bytes and whole-file APIs without requiring a system `libsnappy` installation.
4
+
5
+ ## Features
6
+
7
+ - Real Snappy literals and back-reference copies (not a literal-only encoder).
8
+ - Strict decoding with bounds checks, overlapping-copy support, and rejection of malformed/trailing data.
9
+ - `compress_file()` and `decompress_file()` with predictable default names, custom output paths, atomic writes, and opt-in overwriting.
10
+ - Optional decompressed-size limit for handling untrusted input.
11
+ - Pytest suite, example CLI, and a recursive folder benchmark against stdlib `gzip`, `bz2`, and `lzma` (xz).
12
+ - No native dependency beyond a C compiler and the Cython build requirements.
13
+
14
+ > **Raw vs. framed format:** this project reads/writes Snappy's raw format, with an uncompressed-length varint followed by Snappy elements. It does **not** implement the framed/streaming `.sz` container. Whole files are loaded into memory. Raw Snappy supports inputs up to 2³² − 1 bytes.
15
+
16
+ ## Install for development
17
+
18
+ Linux/macOS with Python 3.9+ and a C compiler:
19
+
20
+ ```bash
21
+ ./tools/bootstrap_dev.sh
22
+ ```
23
+
24
+ That helper creates `.venv`, installs the package in editable mode, compiles the Cython extension, and installs pytest. Equivalent manual commands are:
25
+
26
+ ```bash
27
+ python -m venv .venv
28
+ .venv/bin/python -m pip install -e '.[test]'
29
+ ```
30
+
31
+ On Windows, activate the virtual environment and run the same `pip install -e '.[test]'` command with its Python executable. A compiler supported by setuptools/Cython is required.
32
+
33
+ ## Python API
34
+
35
+ ```python
36
+ from snappy_cython import compress, compress_file, decompress, decompress_file
37
+
38
+ # Bytes-like objects in, bytes out.
39
+ encoded = compress(b"hello hello hello")
40
+ assert decompress(encoded) == b"hello hello hello"
41
+
42
+ # Default: report.txt -> report.txt.snappy
43
+ compressed_path = compress_file("report.txt")
44
+
45
+ # An explicit output name can have any extension.
46
+ compress_file("report.txt", "output/fname.ext")
47
+
48
+ # Default: report.txt.snappy -> report.txt
49
+ # Fails if report.txt already exists, unless overwrite=True is supplied.
50
+ restored_path = decompress_file(compressed_path, overwrite=True)
51
+
52
+ # Explicit destination and a pre-allocation safety limit are also supported.
53
+ decompress_file(
54
+ "output/fname.ext",
55
+ "restored.txt",
56
+ max_output_size=100_000_000,
57
+ )
58
+ ```
59
+
60
+ ### Function reference
61
+
62
+ | Function | Behavior |
63
+ | --- | --- |
64
+ | `compress(data)` | Compress a bytes-like object to raw Snappy `bytes`. |
65
+ | `decompress(data, max_output_size=None)` | Decode raw Snappy and validate the complete stream. |
66
+ | `compress_file(input_path, output_path=None, *, overwrite=False)` | Compress a file; default output appends `.snappy`; returns `pathlib.Path`. |
67
+ | `decompress_file(input_path, output_path=None, *, overwrite=False, max_output_size=None)` | Decompress a file; strips a final `.snappy`, otherwise appends `.out`; returns `Path`. |
68
+
69
+ Existing destinations are protected by default. File data is written to a temporary sibling and moved into place only after successful encoding/decoding. Parent directories must already exist.
70
+
71
+ ## Run tests and demo
72
+
73
+ ```bash
74
+ ./tools/run_checks.sh
75
+ ```
76
+
77
+ Or run them separately:
78
+
79
+ ```bash
80
+ .venv/bin/python -m pytest -q
81
+ .venv/bin/python examples/demo.py
82
+ .venv/bin/python examples/demo.py path/to/input.bin -o fname.ext
83
+ ```
84
+
85
+ The no-argument demo uses a temporary directory and verifies both the bytes and file round trips.
86
+
87
+ An optional interoperability check verifies that this codec and the independent Snappy implementation shipped in `cramjam` can decode each other's raw streams:
88
+
89
+ ```bash
90
+ .venv/bin/python -m pip install cramjam
91
+ .venv/bin/python tools/check_interoperability.py
92
+ ```
93
+
94
+ ## Folder benchmark
95
+
96
+ ```bash
97
+ # Recursively benchmark every regular file under a user-selected folder.
98
+ .venv/bin/python benchmarks/benchmark_folder.py /path/to/folder
99
+
100
+ # Only top-level logs, best of five compression runs, save detailed CSV.
101
+ .venv/bin/python benchmarks/benchmark_folder.py /path/to/folder \
102
+ --glob '*.log' --repeat 5 --csv benchmark-results.csv
103
+
104
+ # Generate a small mixed sample corpus, then benchmark it.
105
+ .venv/bin/python tools/make_benchmark_corpus.py /tmp/snappy-corpus
106
+ .venv/bin/python benchmarks/benchmark_folder.py /tmp/snappy-corpus
107
+ ```
108
+
109
+ Each file is read once. Every codec's output is decompressed and checked before timing. The reported duration is the best compression run; file I/O and verification decompression are excluded. Ratios are `compressed / original` (lower is better), and MiB/s measures original bytes processed per second (higher is better). Snappy generally targets speed rather than the smaller output typically produced by xz or bz2.
110
+
111
+ ## Project layout
112
+
113
+ ```text
114
+ src/snappy_cython/_snappy.pyx Cython raw-format encoder/decoder
115
+ src/snappy_cython/files.py Whole-file convenience functions
116
+ tests/ Pytest unit, malformed-input, and file tests
117
+ examples/demo.py Bytes and file API demonstration
118
+ benchmarks/benchmark_folder.py Snappy/gzip/bz2/xz comparison
119
+ tools/ Development and sample-corpus helper scripts
120
+ ```
121
+
122
+ ## Implementation notes
123
+
124
+ The compressor uses a hash table of four-byte sequences and emits Snappy copy commands for matches up to the format's 65,535-byte offset limit. The decoder supports all literal length encodings and all three copy-offset encodings. The implementation is independent Cython code, not a binding to Google's C++ library.
@@ -0,0 +1,34 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77", "wheel", "Cython>=3.0"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "snappy-cython-file"
7
+ version = "1.4.7"
8
+ description = "A self-contained Snappy raw-format codec written in Cython"
9
+ readme = "README.md"
10
+ requires-python = ">=3.9"
11
+ license = "MIT"
12
+ authors = [{name = "Isaac Onagh", email = "mkalafsaz@gmail.com" }]
13
+ keywords = ["snappy", "compression", "cython"]
14
+ classifiers = [
15
+ "Development Status :: 3 - Alpha",
16
+ "Intended Audience :: Developers",
17
+ "Programming Language :: Python :: 3",
18
+ "Programming Language :: Cython",
19
+ "Topic :: System :: Archiving :: Compression",
20
+ ]
21
+
22
+ [project.optional-dependencies]
23
+ test = ["pytest>=7"]
24
+ [project.urls]
25
+ Homepage = "https://github.com/i4147/snappy_cython"
26
+ [tool.setuptools]
27
+ package-dir = {"" = "src"}
28
+
29
+ [tool.setuptools.packages.find]
30
+ where = ["src"]
31
+
32
+ [tool.pytest.ini_options]
33
+ testpaths = ["tests"]
34
+ addopts = "-ra"
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,24 @@
1
+ from Cython.Build import cythonize
2
+ from setuptools import Extension, setup
3
+
4
+
5
+ extensions = [
6
+ Extension(
7
+ "snappy_cython._snappy",
8
+ sources=["src/snappy_cython/_snappy.pyx"],
9
+ define_macros=[("NDEBUG", "1")],
10
+ )
11
+ ]
12
+
13
+ setup(
14
+ ext_modules=cythonize(
15
+ extensions,
16
+ compiler_directives={
17
+ "language_level": "3str",
18
+ "boundscheck": False,
19
+ "wraparound": False,
20
+ "initializedcheck": False,
21
+ "cdivision": True,
22
+ },
23
+ )
24
+ )
@@ -0,0 +1,11 @@
1
+ """Snappy compression implemented in Cython.
2
+
3
+ The codec reads and writes the Snappy *raw* format. File helpers operate on
4
+ whole files and return the path they created.
5
+ """
6
+
7
+ from ._snappy import compress, decompress
8
+ from .files import compress_file, decompress_file
9
+
10
+ __all__ = ["compress", "decompress", "compress_file", "decompress_file"]
11
+ __version__ = "0.1.0"