snappy-cython-file 1.4.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snappy_cython_file-1.4.7/PKG-INFO +142 -0
- snappy_cython_file-1.4.7/README.md +124 -0
- snappy_cython_file-1.4.7/pyproject.toml +34 -0
- snappy_cython_file-1.4.7/setup.cfg +4 -0
- snappy_cython_file-1.4.7/setup.py +24 -0
- snappy_cython_file-1.4.7/src/snappy_cython/__init__.py +11 -0
- snappy_cython_file-1.4.7/src/snappy_cython/_snappy.c +13447 -0
- snappy_cython_file-1.4.7/src/snappy_cython/files.py +102 -0
- snappy_cython_file-1.4.7/src/snappy_cython_file.egg-info/PKG-INFO +142 -0
- snappy_cython_file-1.4.7/src/snappy_cython_file.egg-info/SOURCES.txt +13 -0
- snappy_cython_file-1.4.7/src/snappy_cython_file.egg-info/dependency_links.txt +1 -0
- snappy_cython_file-1.4.7/src/snappy_cython_file.egg-info/requires.txt +3 -0
- snappy_cython_file-1.4.7/src/snappy_cython_file.egg-info/top_level.txt +1 -0
- snappy_cython_file-1.4.7/tests/test_files.py +98 -0
- snappy_cython_file-1.4.7/tests/test_snappy.py +119 -0
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: snappy-cython-file
|
|
3
|
+
Version: 1.4.7
|
|
4
|
+
Summary: A self-contained Snappy raw-format codec written in Cython
|
|
5
|
+
Author-email: Isaac Onagh <mkalafsaz@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/i4147/snappy_cython
|
|
8
|
+
Keywords: snappy,compression,cython
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Cython
|
|
13
|
+
Classifier: Topic :: System :: Archiving :: Compression
|
|
14
|
+
Requires-Python: >=3.9
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
Provides-Extra: test
|
|
17
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
18
|
+
|
|
19
|
+
# Snappy Cython File Codec
|
|
20
|
+
|
|
21
|
+
A self-contained implementation of [Google Snappy's raw compression format](https://github.com/google/snappy/blob/main/format_description.txt), written in Cython. It provides bytes and whole-file APIs without requiring a system `libsnappy` installation.
|
|
22
|
+
|
|
23
|
+
## Features
|
|
24
|
+
|
|
25
|
+
- Real Snappy literals and back-reference copies (not a literal-only encoder).
|
|
26
|
+
- Strict decoding with bounds checks, overlapping-copy support, and rejection of malformed/trailing data.
|
|
27
|
+
- `compress_file()` and `decompress_file()` with predictable default names, custom output paths, atomic writes, and opt-in overwriting.
|
|
28
|
+
- Optional decompressed-size limit for handling untrusted input.
|
|
29
|
+
- Pytest suite, example CLI, and a recursive folder benchmark against stdlib `gzip`, `bz2`, and `lzma` (xz).
|
|
30
|
+
- No native dependency beyond a C compiler and the Cython build requirements.
|
|
31
|
+
|
|
32
|
+
> **Raw vs. framed format:** this project reads/writes Snappy's raw format, with an uncompressed-length varint followed by Snappy elements. It does **not** implement the framed/streaming `.sz` container. Whole files are loaded into memory. Raw Snappy supports inputs up to 2³² − 1 bytes.
|
|
33
|
+
|
|
34
|
+
## Install for development
|
|
35
|
+
|
|
36
|
+
Linux/macOS with Python 3.9+ and a C compiler:
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
./tools/bootstrap_dev.sh
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
That helper creates `.venv`, installs the package in editable mode, compiles the Cython extension, and installs pytest. Equivalent manual commands are:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
python -m venv .venv
|
|
46
|
+
.venv/bin/python -m pip install -e '.[test]'
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
On Windows, activate the virtual environment and run the same `pip install -e '.[test]'` command with its Python executable. A compiler supported by setuptools/Cython is required.
|
|
50
|
+
|
|
51
|
+
## Python API
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from snappy_cython import compress, compress_file, decompress, decompress_file
|
|
55
|
+
|
|
56
|
+
# Bytes-like objects in, bytes out.
|
|
57
|
+
encoded = compress(b"hello hello hello")
|
|
58
|
+
assert decompress(encoded) == b"hello hello hello"
|
|
59
|
+
|
|
60
|
+
# Default: report.txt -> report.txt.snappy
|
|
61
|
+
compressed_path = compress_file("report.txt")
|
|
62
|
+
|
|
63
|
+
# An explicit output name can have any extension.
|
|
64
|
+
compress_file("report.txt", "output/fname.ext")
|
|
65
|
+
|
|
66
|
+
# Default: report.txt.snappy -> report.txt
|
|
67
|
+
# Fails if report.txt already exists, unless overwrite=True is supplied.
|
|
68
|
+
restored_path = decompress_file(compressed_path, overwrite=True)
|
|
69
|
+
|
|
70
|
+
# Explicit destination and a pre-allocation safety limit are also supported.
|
|
71
|
+
decompress_file(
|
|
72
|
+
"output/fname.ext",
|
|
73
|
+
"restored.txt",
|
|
74
|
+
max_output_size=100_000_000,
|
|
75
|
+
)
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
### Function reference
|
|
79
|
+
|
|
80
|
+
| Function | Behavior |
|
|
81
|
+
| --- | --- |
|
|
82
|
+
| `compress(data)` | Compress a bytes-like object to raw Snappy `bytes`. |
|
|
83
|
+
| `decompress(data, max_output_size=None)` | Decode raw Snappy and validate the complete stream. |
|
|
84
|
+
| `compress_file(input_path, output_path=None, *, overwrite=False)` | Compress a file; default output appends `.snappy`; returns `pathlib.Path`. |
|
|
85
|
+
| `decompress_file(input_path, output_path=None, *, overwrite=False, max_output_size=None)` | Decompress a file; strips a final `.snappy`, otherwise appends `.out`; returns `Path`. |
|
|
86
|
+
|
|
87
|
+
Existing destinations are protected by default. File data is written to a temporary sibling and moved into place only after successful encoding/decoding. Parent directories must already exist.
|
|
88
|
+
|
|
89
|
+
## Run tests and demo
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
./tools/run_checks.sh
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Or run them separately:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
.venv/bin/python -m pytest -q
|
|
99
|
+
.venv/bin/python examples/demo.py
|
|
100
|
+
.venv/bin/python examples/demo.py path/to/input.bin -o fname.ext
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
The no-argument demo uses a temporary directory and verifies both the bytes and file round trips.
|
|
104
|
+
|
|
105
|
+
An optional interoperability check verifies that this codec and the independent Snappy implementation shipped in `cramjam` can decode each other's raw streams:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
.venv/bin/python -m pip install cramjam
|
|
109
|
+
.venv/bin/python tools/check_interoperability.py
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
## Folder benchmark
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
# Recursively benchmark every regular file under a user-selected folder.
|
|
116
|
+
.venv/bin/python benchmarks/benchmark_folder.py /path/to/folder
|
|
117
|
+
|
|
118
|
+
# Only top-level logs, best of five compression runs, save detailed CSV.
|
|
119
|
+
.venv/bin/python benchmarks/benchmark_folder.py /path/to/folder \
|
|
120
|
+
--glob '*.log' --repeat 5 --csv benchmark-results.csv
|
|
121
|
+
|
|
122
|
+
# Generate a small mixed sample corpus, then benchmark it.
|
|
123
|
+
.venv/bin/python tools/make_benchmark_corpus.py /tmp/snappy-corpus
|
|
124
|
+
.venv/bin/python benchmarks/benchmark_folder.py /tmp/snappy-corpus
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Each file is read once. Every codec's output is decompressed and checked before timing. The reported duration is the best compression run; file I/O and verification decompression are excluded. Ratios are `compressed / original` (lower is better), and MiB/s measures original bytes processed per second (higher is better). Snappy generally targets speed rather than the smaller output typically produced by xz or bz2.
|
|
128
|
+
|
|
129
|
+
## Project layout
|
|
130
|
+
|
|
131
|
+
```text
|
|
132
|
+
src/snappy_cython/_snappy.pyx Cython raw-format encoder/decoder
|
|
133
|
+
src/snappy_cython/files.py Whole-file convenience functions
|
|
134
|
+
tests/ Pytest unit, malformed-input, and file tests
|
|
135
|
+
examples/demo.py Bytes and file API demonstration
|
|
136
|
+
benchmarks/benchmark_folder.py Snappy/gzip/bz2/xz comparison
|
|
137
|
+
tools/ Development and sample-corpus helper scripts
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
## Implementation notes
|
|
141
|
+
|
|
142
|
+
The compressor uses a hash table of four-byte sequences and emits Snappy copy commands for matches up to the format's 65,535-byte offset limit. The decoder supports all literal length encodings and all three copy-offset encodings. The implementation is independent Cython code, not a binding to Google's C++ library.
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# Snappy Cython File Codec
|
|
2
|
+
|
|
3
|
+
A self-contained implementation of [Google Snappy's raw compression format](https://github.com/google/snappy/blob/main/format_description.txt), written in Cython. It provides bytes and whole-file APIs without requiring a system `libsnappy` installation.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- Real Snappy literals and back-reference copies (not a literal-only encoder).
|
|
8
|
+
- Strict decoding with bounds checks, overlapping-copy support, and rejection of malformed/trailing data.
|
|
9
|
+
- `compress_file()` and `decompress_file()` with predictable default names, custom output paths, atomic writes, and opt-in overwriting.
|
|
10
|
+
- Optional decompressed-size limit for handling untrusted input.
|
|
11
|
+
- Pytest suite, example CLI, and a recursive folder benchmark against stdlib `gzip`, `bz2`, and `lzma` (xz).
|
|
12
|
+
- No native dependency beyond a C compiler and the Cython build requirements.
|
|
13
|
+
|
|
14
|
+
> **Raw vs. framed format:** this project reads/writes Snappy's raw format, with an uncompressed-length varint followed by Snappy elements. It does **not** implement the framed/streaming `.sz` container. Whole files are loaded into memory. Raw Snappy supports inputs up to 2³² − 1 bytes.
|
|
15
|
+
|
|
16
|
+
## Install for development
|
|
17
|
+
|
|
18
|
+
Linux/macOS with Python 3.9+ and a C compiler:
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
./tools/bootstrap_dev.sh
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
That helper creates `.venv`, installs the package in editable mode, compiles the Cython extension, and installs pytest. Equivalent manual commands are:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
python -m venv .venv
|
|
28
|
+
.venv/bin/python -m pip install -e '.[test]'
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
On Windows, activate the virtual environment and run the same `pip install -e '.[test]'` command with its Python executable. A compiler supported by setuptools/Cython is required.
|
|
32
|
+
|
|
33
|
+
## Python API
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
from snappy_cython import compress, compress_file, decompress, decompress_file
|
|
37
|
+
|
|
38
|
+
# Bytes-like objects in, bytes out.
|
|
39
|
+
encoded = compress(b"hello hello hello")
|
|
40
|
+
assert decompress(encoded) == b"hello hello hello"
|
|
41
|
+
|
|
42
|
+
# Default: report.txt -> report.txt.snappy
|
|
43
|
+
compressed_path = compress_file("report.txt")
|
|
44
|
+
|
|
45
|
+
# An explicit output name can have any extension.
|
|
46
|
+
compress_file("report.txt", "output/fname.ext")
|
|
47
|
+
|
|
48
|
+
# Default: report.txt.snappy -> report.txt
|
|
49
|
+
# Fails if report.txt already exists, unless overwrite=True is supplied.
|
|
50
|
+
restored_path = decompress_file(compressed_path, overwrite=True)
|
|
51
|
+
|
|
52
|
+
# Explicit destination and a pre-allocation safety limit are also supported.
|
|
53
|
+
decompress_file(
|
|
54
|
+
"output/fname.ext",
|
|
55
|
+
"restored.txt",
|
|
56
|
+
max_output_size=100_000_000,
|
|
57
|
+
)
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
### Function reference
|
|
61
|
+
|
|
62
|
+
| Function | Behavior |
|
|
63
|
+
| --- | --- |
|
|
64
|
+
| `compress(data)` | Compress a bytes-like object to raw Snappy `bytes`. |
|
|
65
|
+
| `decompress(data, max_output_size=None)` | Decode raw Snappy and validate the complete stream. |
|
|
66
|
+
| `compress_file(input_path, output_path=None, *, overwrite=False)` | Compress a file; default output appends `.snappy`; returns `pathlib.Path`. |
|
|
67
|
+
| `decompress_file(input_path, output_path=None, *, overwrite=False, max_output_size=None)` | Decompress a file; strips a final `.snappy`, otherwise appends `.out`; returns `Path`. |
|
|
68
|
+
|
|
69
|
+
Existing destinations are protected by default. File data is written to a temporary sibling and moved into place only after successful encoding/decoding. Parent directories must already exist.
|
|
70
|
+
|
|
71
|
+
## Run tests and demo
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
./tools/run_checks.sh
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Or run them separately:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
.venv/bin/python -m pytest -q
|
|
81
|
+
.venv/bin/python examples/demo.py
|
|
82
|
+
.venv/bin/python examples/demo.py path/to/input.bin -o fname.ext
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
The no-argument demo uses a temporary directory and verifies both the bytes and file round trips.
|
|
86
|
+
|
|
87
|
+
An optional interoperability check verifies that this codec and the independent Snappy implementation shipped in `cramjam` can decode each other's raw streams:
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
.venv/bin/python -m pip install cramjam
|
|
91
|
+
.venv/bin/python tools/check_interoperability.py
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## Folder benchmark
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
# Recursively benchmark every regular file under a user-selected folder.
|
|
98
|
+
.venv/bin/python benchmarks/benchmark_folder.py /path/to/folder
|
|
99
|
+
|
|
100
|
+
# Only top-level logs, best of five compression runs, save detailed CSV.
|
|
101
|
+
.venv/bin/python benchmarks/benchmark_folder.py /path/to/folder \
|
|
102
|
+
--glob '*.log' --repeat 5 --csv benchmark-results.csv
|
|
103
|
+
|
|
104
|
+
# Generate a small mixed sample corpus, then benchmark it.
|
|
105
|
+
.venv/bin/python tools/make_benchmark_corpus.py /tmp/snappy-corpus
|
|
106
|
+
.venv/bin/python benchmarks/benchmark_folder.py /tmp/snappy-corpus
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Each file is read once. Every codec's output is decompressed and checked before timing. The reported duration is the best compression run; file I/O and verification decompression are excluded. Ratios are `compressed / original` (lower is better), and MiB/s measures original bytes processed per second (higher is better). Snappy generally targets speed rather than the smaller output typically produced by xz or bz2.
|
|
110
|
+
|
|
111
|
+
## Project layout
|
|
112
|
+
|
|
113
|
+
```text
|
|
114
|
+
src/snappy_cython/_snappy.pyx Cython raw-format encoder/decoder
|
|
115
|
+
src/snappy_cython/files.py Whole-file convenience functions
|
|
116
|
+
tests/ Pytest unit, malformed-input, and file tests
|
|
117
|
+
examples/demo.py Bytes and file API demonstration
|
|
118
|
+
benchmarks/benchmark_folder.py Snappy/gzip/bz2/xz comparison
|
|
119
|
+
tools/ Development and sample-corpus helper scripts
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## Implementation notes
|
|
123
|
+
|
|
124
|
+
The compressor uses a hash table of four-byte sequences and emits Snappy copy commands for matches up to the format's 65,535-byte offset limit. The decoder supports all literal length encodings and all three copy-offset encodings. The implementation is independent Cython code, not a binding to Google's C++ library.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77", "wheel", "Cython>=3.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "snappy-cython-file"
|
|
7
|
+
version = "1.4.7"
|
|
8
|
+
description = "A self-contained Snappy raw-format codec written in Cython"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
authors = [{name = "Isaac Onagh", email = "mkalafsaz@gmail.com" }]
|
|
13
|
+
keywords = ["snappy", "compression", "cython"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Programming Language :: Cython",
|
|
19
|
+
"Topic :: System :: Archiving :: Compression",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[project.optional-dependencies]
|
|
23
|
+
test = ["pytest>=7"]
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://github.com/i4147/snappy_cython"
|
|
26
|
+
[tool.setuptools]
|
|
27
|
+
package-dir = {"" = "src"}
|
|
28
|
+
|
|
29
|
+
[tool.setuptools.packages.find]
|
|
30
|
+
where = ["src"]
|
|
31
|
+
|
|
32
|
+
[tool.pytest.ini_options]
|
|
33
|
+
testpaths = ["tests"]
|
|
34
|
+
addopts = "-ra"
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from Cython.Build import cythonize
|
|
2
|
+
from setuptools import Extension, setup
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
extensions = [
|
|
6
|
+
Extension(
|
|
7
|
+
"snappy_cython._snappy",
|
|
8
|
+
sources=["src/snappy_cython/_snappy.pyx"],
|
|
9
|
+
define_macros=[("NDEBUG", "1")],
|
|
10
|
+
)
|
|
11
|
+
]
|
|
12
|
+
|
|
13
|
+
setup(
|
|
14
|
+
ext_modules=cythonize(
|
|
15
|
+
extensions,
|
|
16
|
+
compiler_directives={
|
|
17
|
+
"language_level": "3str",
|
|
18
|
+
"boundscheck": False,
|
|
19
|
+
"wraparound": False,
|
|
20
|
+
"initializedcheck": False,
|
|
21
|
+
"cdivision": True,
|
|
22
|
+
},
|
|
23
|
+
)
|
|
24
|
+
)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Snappy compression implemented in Cython.
|
|
2
|
+
|
|
3
|
+
The codec reads and writes the Snappy *raw* format. File helpers operate on
|
|
4
|
+
whole files and return the path they created.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from ._snappy import compress, decompress
|
|
8
|
+
from .files import compress_file, decompress_file
|
|
9
|
+
|
|
10
|
+
__all__ = ["compress", "decompress", "compress_file", "decompress_file"]
|
|
11
|
+
__version__ = "0.1.0"
|