FastWARC 0.16.0__tar.gz → 1.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fastwarc-0.16.0 → fastwarc-1.0.1}/FastWARC.egg-info/PKG-INFO +30 -8
- {fastwarc-0.16.0 → fastwarc-1.0.1}/FastWARC.egg-info/SOURCES.txt +12 -7
- {fastwarc-0.16.0 → fastwarc-1.0.1}/PKG-INFO +30 -8
- {fastwarc-0.16.0 → fastwarc-1.0.1}/README.md +28 -6
- fastwarc-1.0.1/fastwarc/__init__.py +44 -0
- fastwarc-1.0.1/fastwarc/__init__.pyi +49 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/fastwarc/cli.py +4 -4
- fastwarc-1.0.1/fastwarc/legacy/_shims.py +202 -0
- {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/tools.pyx +3 -3
- {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/warc.pxd +1 -1
- {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/warc.pyx +2 -2
- fastwarc-1.0.1/fastwarc/stream_io.pyi +187 -0
- fastwarc-1.0.1/fastwarc/warc.pyi +359 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/pyproject.toml +20 -15
- {fastwarc-0.16.0 → fastwarc-1.0.1}/setup.py +30 -17
- {fastwarc-0.16.0 → fastwarc-1.0.1}/FastWARC.egg-info/dependency_links.txt +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/FastWARC.egg-info/entry_points.txt +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/FastWARC.egg-info/requires.txt +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/FastWARC.egg-info/top_level.txt +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/MANIFEST.in +0 -0
- {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/__init__.pxd +0 -0
- {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/__init__.py +0 -0
- {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/stream_io.pxd +0 -0
- {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/stream_io.pyi +0 -0
- {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/stream_io.pyx +0 -0
- {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/tools.pyi +0 -0
- {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/warc.pyi +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/fastwarc/py.typed +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_common/__init__.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_common/string_util.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/__init__.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/algorithm.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/atomic.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/boost_regex.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/cctype.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/cstdlib.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/cstring.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/dlfcn.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/errno.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/lexbor.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/lz4frame.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/lz4hc.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/pthread.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/re2.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/stdio.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/string_view.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/time.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/uchardet.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/unistd.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/utility.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/zlib.pxd +0 -0
- {fastwarc-0.16.0 → fastwarc-1.0.1}/setup.cfg +0 -0
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: FastWARC
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 1.0.1
|
|
4
4
|
Summary: A high-performance WARC parsing library for Python written in C++/Cython.
|
|
5
5
|
Author: Janek Bevendorff
|
|
6
|
-
License: Apache
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
7
|
Project-URL: repository, https://github.com/chatnoir-eu/chatnoir-resiliparse
|
|
8
8
|
Project-URL: homepage, https://resiliparse.chatnoir.eu
|
|
9
9
|
Project-URL: documentation, https://resiliparse.chatnoir.eu
|
|
@@ -22,33 +22,51 @@ Requires-Dist: pytest>=7.2.2; extra == "test"
|
|
|
22
22
|
Requires-Dist: pytest-cov>=4.0.0; extra == "test"
|
|
23
23
|
Requires-Dist: lz4; extra == "test"
|
|
24
24
|
|
|
25
|
-
# FastWARC
|
|
25
|
+
# FastWARC (Python)
|
|
26
26
|
|
|
27
|
-
FastWARC is a high-performance WARC parsing library for Python written in
|
|
27
|
+
FastWARC is a high-performance WARC parsing library for Python written in Rust with Python bindings. FastWARC supports
|
|
28
|
+
compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are Gzip and LZ4.
|
|
28
29
|
|
|
29
|
-
|
|
30
|
+
This crate contains the Python bindings for FastWARC.
|
|
31
|
+
|
|
32
|
+
FastWARC belongs to the [ChatNoir Resiliparse toolkit](https://github.com/chatnoir-eu/chatnoir-resiliparse/) for fast
|
|
33
|
+
and robust web data processing.
|
|
30
34
|
|
|
31
35
|
## Installing FastWARC
|
|
36
|
+
|
|
32
37
|
Pre-built FastWARC binaries for most Linux platforms can be installed from PyPi:
|
|
38
|
+
|
|
33
39
|
```bash
|
|
34
40
|
pip install fastwarc
|
|
35
41
|
```
|
|
36
|
-
|
|
42
|
+
|
|
43
|
+
**However:** the Linux binaries are provided *solely for your convenience*. Since they are built on the very old
|
|
44
|
+
`manylinux` base system for better compatibility, their performance isn't optimal (though still better than WARCIO). For
|
|
45
|
+
best performance, see the next section on how to build FastWARC yourself.
|
|
37
46
|
|
|
38
47
|
## Building FastWARC From Source
|
|
39
|
-
|
|
48
|
+
|
|
49
|
+
You can compile FastWARC either from the PyPi source package or directly from this repository, though in any case, you
|
|
50
|
+
need to install all required build-time dependencies first. On Ubuntu, this is done as follows:
|
|
51
|
+
|
|
40
52
|
```bash
|
|
41
53
|
sudo apt install build-essential python3-dev zlib1g-dev liblz4-dev
|
|
42
54
|
```
|
|
55
|
+
|
|
43
56
|
To build and install FastWARC from PyPi, run
|
|
57
|
+
|
|
44
58
|
```bash
|
|
45
59
|
pip install --no-binary fastwarc fastwarc
|
|
46
60
|
```
|
|
61
|
+
|
|
47
62
|
That's it. If you prefer to build and install directly from this repository instead, run:
|
|
63
|
+
|
|
48
64
|
```bash
|
|
49
65
|
pip install -e fastwarc
|
|
50
66
|
```
|
|
67
|
+
|
|
51
68
|
To build the wheels without installing them, run:
|
|
69
|
+
|
|
52
70
|
```bash
|
|
53
71
|
pip wheel -e fastwarc
|
|
54
72
|
|
|
@@ -57,10 +75,14 @@ pip install build && python -m build --wheel fastwarc
|
|
|
57
75
|
```
|
|
58
76
|
|
|
59
77
|
## Usage Instructions
|
|
60
|
-
|
|
78
|
+
|
|
79
|
+
For detailed usage instructions, please consult
|
|
80
|
+
the [FastWARC User Manual](https://resiliparse.chatnoir.eu/en/latest/man/fastwarc.html).
|
|
61
81
|
|
|
62
82
|
## Cite Us
|
|
83
|
+
|
|
63
84
|
If you use FastWARC, please consider citing our [OSSYM 2021 abstract paper](https://arxiv.org/abs/2112.03103):
|
|
85
|
+
|
|
64
86
|
```bibtex
|
|
65
87
|
@InProceedings{bevendorff:2021,
|
|
66
88
|
author = {Janek Bevendorff and Martin Potthast and Benno Stein},
|
|
@@ -8,18 +8,23 @@ FastWARC.egg-info/dependency_links.txt
|
|
|
8
8
|
FastWARC.egg-info/entry_points.txt
|
|
9
9
|
FastWARC.egg-info/requires.txt
|
|
10
10
|
FastWARC.egg-info/top_level.txt
|
|
11
|
-
fastwarc/__init__.pxd
|
|
12
11
|
fastwarc/__init__.py
|
|
12
|
+
fastwarc/__init__.pyi
|
|
13
13
|
fastwarc/cli.py
|
|
14
14
|
fastwarc/py.typed
|
|
15
|
-
fastwarc/stream_io.pxd
|
|
16
15
|
fastwarc/stream_io.pyi
|
|
17
|
-
fastwarc/stream_io.pyx
|
|
18
|
-
fastwarc/tools.pyi
|
|
19
|
-
fastwarc/tools.pyx
|
|
20
|
-
fastwarc/warc.pxd
|
|
21
16
|
fastwarc/warc.pyi
|
|
22
|
-
fastwarc/
|
|
17
|
+
fastwarc/legacy/__init__.pxd
|
|
18
|
+
fastwarc/legacy/__init__.py
|
|
19
|
+
fastwarc/legacy/_shims.py
|
|
20
|
+
fastwarc/legacy/stream_io.pxd
|
|
21
|
+
fastwarc/legacy/stream_io.pyi
|
|
22
|
+
fastwarc/legacy/stream_io.pyx
|
|
23
|
+
fastwarc/legacy/tools.pyi
|
|
24
|
+
fastwarc/legacy/tools.pyx
|
|
25
|
+
fastwarc/legacy/warc.pxd
|
|
26
|
+
fastwarc/legacy/warc.pyi
|
|
27
|
+
fastwarc/legacy/warc.pyx
|
|
23
28
|
resiliparse_common/__init__.pxd
|
|
24
29
|
resiliparse_common/string_util.pxd
|
|
25
30
|
resiliparse_inc/__init__.pxd
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: FastWARC
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 1.0.1
|
|
4
4
|
Summary: A high-performance WARC parsing library for Python written in C++/Cython.
|
|
5
5
|
Author: Janek Bevendorff
|
|
6
|
-
License: Apache
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
7
|
Project-URL: repository, https://github.com/chatnoir-eu/chatnoir-resiliparse
|
|
8
8
|
Project-URL: homepage, https://resiliparse.chatnoir.eu
|
|
9
9
|
Project-URL: documentation, https://resiliparse.chatnoir.eu
|
|
@@ -22,33 +22,51 @@ Requires-Dist: pytest>=7.2.2; extra == "test"
|
|
|
22
22
|
Requires-Dist: pytest-cov>=4.0.0; extra == "test"
|
|
23
23
|
Requires-Dist: lz4; extra == "test"
|
|
24
24
|
|
|
25
|
-
# FastWARC
|
|
25
|
+
# FastWARC (Python)
|
|
26
26
|
|
|
27
|
-
FastWARC is a high-performance WARC parsing library for Python written in
|
|
27
|
+
FastWARC is a high-performance WARC parsing library for Python written in Rust with Python bindings. FastWARC supports
|
|
28
|
+
compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are Gzip and LZ4.
|
|
28
29
|
|
|
29
|
-
|
|
30
|
+
This crate contains the Python bindings for FastWARC.
|
|
31
|
+
|
|
32
|
+
FastWARC belongs to the [ChatNoir Resiliparse toolkit](https://github.com/chatnoir-eu/chatnoir-resiliparse/) for fast
|
|
33
|
+
and robust web data processing.
|
|
30
34
|
|
|
31
35
|
## Installing FastWARC
|
|
36
|
+
|
|
32
37
|
Pre-built FastWARC binaries for most Linux platforms can be installed from PyPi:
|
|
38
|
+
|
|
33
39
|
```bash
|
|
34
40
|
pip install fastwarc
|
|
35
41
|
```
|
|
36
|
-
|
|
42
|
+
|
|
43
|
+
**However:** the Linux binaries are provided *solely for your convenience*. Since they are built on the very old
|
|
44
|
+
`manylinux` base system for better compatibility, their performance isn't optimal (though still better than WARCIO). For
|
|
45
|
+
best performance, see the next section on how to build FastWARC yourself.
|
|
37
46
|
|
|
38
47
|
## Building FastWARC From Source
|
|
39
|
-
|
|
48
|
+
|
|
49
|
+
You can compile FastWARC either from the PyPi source package or directly from this repository, though in any case, you
|
|
50
|
+
need to install all required build-time dependencies first. On Ubuntu, this is done as follows:
|
|
51
|
+
|
|
40
52
|
```bash
|
|
41
53
|
sudo apt install build-essential python3-dev zlib1g-dev liblz4-dev
|
|
42
54
|
```
|
|
55
|
+
|
|
43
56
|
To build and install FastWARC from PyPi, run
|
|
57
|
+
|
|
44
58
|
```bash
|
|
45
59
|
pip install --no-binary fastwarc fastwarc
|
|
46
60
|
```
|
|
61
|
+
|
|
47
62
|
That's it. If you prefer to build and install directly from this repository instead, run:
|
|
63
|
+
|
|
48
64
|
```bash
|
|
49
65
|
pip install -e fastwarc
|
|
50
66
|
```
|
|
67
|
+
|
|
51
68
|
To build the wheels without installing them, run:
|
|
69
|
+
|
|
52
70
|
```bash
|
|
53
71
|
pip wheel -e fastwarc
|
|
54
72
|
|
|
@@ -57,10 +75,14 @@ pip install build && python -m build --wheel fastwarc
|
|
|
57
75
|
```
|
|
58
76
|
|
|
59
77
|
## Usage Instructions
|
|
60
|
-
|
|
78
|
+
|
|
79
|
+
For detailed usage instructions, please consult
|
|
80
|
+
the [FastWARC User Manual](https://resiliparse.chatnoir.eu/en/latest/man/fastwarc.html).
|
|
61
81
|
|
|
62
82
|
## Cite Us
|
|
83
|
+
|
|
63
84
|
If you use FastWARC, please consider citing our [OSSYM 2021 abstract paper](https://arxiv.org/abs/2112.03103):
|
|
85
|
+
|
|
64
86
|
```bibtex
|
|
65
87
|
@InProceedings{bevendorff:2021,
|
|
66
88
|
author = {Janek Bevendorff and Martin Potthast and Benno Stein},
|
|
@@ -1,30 +1,48 @@
|
|
|
1
|
-
# FastWARC
|
|
1
|
+
# FastWARC (Python)
|
|
2
2
|
|
|
3
|
-
FastWARC is a high-performance WARC parsing library for Python written in
|
|
3
|
+
FastWARC is a high-performance WARC parsing library for Python written in Rust with Python bindings. FastWARC supports
|
|
4
|
+
compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are Gzip and LZ4.
|
|
4
5
|
|
|
5
|
-
|
|
6
|
+
This crate contains the Python bindings for FastWARC.
|
|
7
|
+
|
|
8
|
+
FastWARC belongs to the [ChatNoir Resiliparse toolkit](https://github.com/chatnoir-eu/chatnoir-resiliparse/) for fast
|
|
9
|
+
and robust web data processing.
|
|
6
10
|
|
|
7
11
|
## Installing FastWARC
|
|
12
|
+
|
|
8
13
|
Pre-built FastWARC binaries for most Linux platforms can be installed from PyPi:
|
|
14
|
+
|
|
9
15
|
```bash
|
|
10
16
|
pip install fastwarc
|
|
11
17
|
```
|
|
12
|
-
|
|
18
|
+
|
|
19
|
+
**However:** the Linux binaries are provided *solely for your convenience*. Since they are built on the very old
|
|
20
|
+
`manylinux` base system for better compatibility, their performance isn't optimal (though still better than WARCIO). For
|
|
21
|
+
best performance, see the next section on how to build FastWARC yourself.
|
|
13
22
|
|
|
14
23
|
## Building FastWARC From Source
|
|
15
|
-
|
|
24
|
+
|
|
25
|
+
You can compile FastWARC either from the PyPi source package or directly from this repository, though in any case, you
|
|
26
|
+
need to install all required build-time dependencies first. On Ubuntu, this is done as follows:
|
|
27
|
+
|
|
16
28
|
```bash
|
|
17
29
|
sudo apt install build-essential python3-dev zlib1g-dev liblz4-dev
|
|
18
30
|
```
|
|
31
|
+
|
|
19
32
|
To build and install FastWARC from PyPi, run
|
|
33
|
+
|
|
20
34
|
```bash
|
|
21
35
|
pip install --no-binary fastwarc fastwarc
|
|
22
36
|
```
|
|
37
|
+
|
|
23
38
|
That's it. If you prefer to build and install directly from this repository instead, run:
|
|
39
|
+
|
|
24
40
|
```bash
|
|
25
41
|
pip install -e fastwarc
|
|
26
42
|
```
|
|
43
|
+
|
|
27
44
|
To build the wheels without installing them, run:
|
|
45
|
+
|
|
28
46
|
```bash
|
|
29
47
|
pip wheel -e fastwarc
|
|
30
48
|
|
|
@@ -33,10 +51,14 @@ pip install build && python -m build --wheel fastwarc
|
|
|
33
51
|
```
|
|
34
52
|
|
|
35
53
|
## Usage Instructions
|
|
36
|
-
|
|
54
|
+
|
|
55
|
+
For detailed usage instructions, please consult
|
|
56
|
+
the [FastWARC User Manual](https://resiliparse.chatnoir.eu/en/latest/man/fastwarc.html).
|
|
37
57
|
|
|
38
58
|
## Cite Us
|
|
59
|
+
|
|
39
60
|
If you use FastWARC, please consider citing our [OSSYM 2021 abstract paper](https://arxiv.org/abs/2112.03103):
|
|
61
|
+
|
|
40
62
|
```bibtex
|
|
41
63
|
@InProceedings{bevendorff:2021,
|
|
42
64
|
author = {Janek Bevendorff and Martin Potthast and Benno Stein},
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Copyright 2026 Janek Bevendorff
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
import importlib
|
|
16
|
+
|
|
17
|
+
_native = importlib.import_module("._fastwarc", __name__)
|
|
18
|
+
_legacy_shims = importlib.import_module(".legacy._shims", __name__)
|
|
19
|
+
|
|
20
|
+
import fastwarc.stream_io as stream_io # noqa: F401
|
|
21
|
+
import fastwarc.warc as warc # noqa: F401
|
|
22
|
+
|
|
23
|
+
# Re-export module attributes
|
|
24
|
+
globals().update({attr: getattr(_native, attr) for attr in _native.__all__})
|
|
25
|
+
|
|
26
|
+
# Patch legacy shims onto stream_io
|
|
27
|
+
for name in _legacy_shims.__all__:
|
|
28
|
+
setattr(stream_io, name, getattr(_legacy_shims, name))
|
|
29
|
+
stream_io.__all__ += getattr(_legacy_shims, "__all__", ()) # type: ignore[attr-defined]
|
|
30
|
+
|
|
31
|
+
FileStream = _legacy_shims.FileStream
|
|
32
|
+
GZipStream = _legacy_shims.GZipStream
|
|
33
|
+
LZ4Stream = _legacy_shims.LZ4Stream
|
|
34
|
+
FastWARCError = _legacy_shims.FastWARCError
|
|
35
|
+
StreamError = _legacy_shims.StreamError
|
|
36
|
+
|
|
37
|
+
__all__ = [
|
|
38
|
+
*_native.__all__,
|
|
39
|
+
'FileStream',
|
|
40
|
+
'GZipStream',
|
|
41
|
+
'LZ4Stream',
|
|
42
|
+
'FastWARCError',
|
|
43
|
+
'StreamError',
|
|
44
|
+
]
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# Copyright 2026 Janek Bevendorff
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from .warc import ArchiveIterator, WarcRecord, WarcRecordType
|
|
16
|
+
from .stream_io import (
|
|
17
|
+
GzipReader,
|
|
18
|
+
GzipWriter,
|
|
19
|
+
ZstdReader,
|
|
20
|
+
ZstdWriter,
|
|
21
|
+
Lz4Reader,
|
|
22
|
+
Lz4Writer,
|
|
23
|
+
|
|
24
|
+
# Legacy names
|
|
25
|
+
FileStream,
|
|
26
|
+
GZipStream,
|
|
27
|
+
LZ4Stream,
|
|
28
|
+
FastWARCError,
|
|
29
|
+
StreamError,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
__all__ = [
|
|
33
|
+
'ArchiveIterator',
|
|
34
|
+
'WarcRecord',
|
|
35
|
+
'WarcRecordType',
|
|
36
|
+
'GzipReader',
|
|
37
|
+
'GzipWriter',
|
|
38
|
+
'ZstdReader',
|
|
39
|
+
'ZstdWriter',
|
|
40
|
+
'Lz4Reader',
|
|
41
|
+
'Lz4Writer',
|
|
42
|
+
|
|
43
|
+
# Legacy names
|
|
44
|
+
'FileStream',
|
|
45
|
+
'GZipStream',
|
|
46
|
+
'LZ4Stream',
|
|
47
|
+
'FastWARCError',
|
|
48
|
+
'StreamError',
|
|
49
|
+
]
|
|
@@ -24,9 +24,9 @@ import urllib.request
|
|
|
24
24
|
import click
|
|
25
25
|
from tqdm import tqdm
|
|
26
26
|
|
|
27
|
-
from fastwarc.stream_io import FileStream, StreamError, FastWARCError, PythonIOStreamAdapter
|
|
28
|
-
from fastwarc.warc import ArchiveIterator, WarcRecordType
|
|
29
|
-
from fastwarc.tools import CompressionAlg, detect_compression_algorithm, wrap_warc_stream, \
|
|
27
|
+
from fastwarc.legacy.stream_io import FileStream, StreamError, FastWARCError, PythonIOStreamAdapter
|
|
28
|
+
from fastwarc.legacy.warc import ArchiveIterator, WarcRecordType
|
|
29
|
+
from fastwarc.legacy.tools import CompressionAlg, detect_compression_algorithm, wrap_warc_stream, \
|
|
30
30
|
recompress_warc_interactive, verify_digests
|
|
31
31
|
|
|
32
32
|
|
|
@@ -288,7 +288,7 @@ botocore = None
|
|
|
288
288
|
s3 = None
|
|
289
289
|
|
|
290
290
|
|
|
291
|
-
def _init_s3(endpoint_url, aws_access_key,
|
|
291
|
+
def _init_s3(endpoint_url, aws_access_key, aws_secret_key=None):
|
|
292
292
|
global boto3, botocore, s3
|
|
293
293
|
|
|
294
294
|
if s3 is not None:
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
# Copyright 2026 Janek Bevendorff
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
# Legacy shims for making the new Rust extension work with the old type names
|
|
16
|
+
|
|
17
|
+
import warnings
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
'wrap_stream',
|
|
21
|
+
'BufferedReader',
|
|
22
|
+
'IOStream',
|
|
23
|
+
'PythonIOStreamAdapter',
|
|
24
|
+
'CompressingStream',
|
|
25
|
+
'FileStream',
|
|
26
|
+
'BytesIOStream',
|
|
27
|
+
'GZipStream',
|
|
28
|
+
'LZ4Stream',
|
|
29
|
+
'BrotliStream',
|
|
30
|
+
'FastWARCError',
|
|
31
|
+
'StreamError',
|
|
32
|
+
'ReaderStaleError',
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
FastWARCError = OSError
|
|
36
|
+
|
|
37
|
+
StreamError = OSError
|
|
38
|
+
|
|
39
|
+
ReaderStaleError = OSError
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _warn_deprecated(msg):
|
|
43
|
+
warnings.warn(msg, FutureWarning, 3)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def wrap_stream(raw_stream, mode='rb', fsspec_args=None):
|
|
47
|
+
_warn_deprecated("wrap_stream is deprecated and will be removed in a future version.")
|
|
48
|
+
if isinstance(raw_stream, str):
|
|
49
|
+
if fsspec_args is not False and '://' in raw_stream:
|
|
50
|
+
try:
|
|
51
|
+
import fsspec
|
|
52
|
+
return fsspec.open(raw_stream, mode, **(fsspec_args or {})).open()
|
|
53
|
+
except ModuleNotFoundError:
|
|
54
|
+
pass
|
|
55
|
+
return open(raw_stream, mode)
|
|
56
|
+
|
|
57
|
+
return raw_stream
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class BufferedReader:
|
|
61
|
+
def __init__(self, stream, *args, **kwargs):
|
|
62
|
+
_warn_deprecated("BufferedReader is deprecated and will be removed in a future version.")
|
|
63
|
+
self._stream = stream
|
|
64
|
+
|
|
65
|
+
def close(self):
|
|
66
|
+
return self._stream.close()
|
|
67
|
+
|
|
68
|
+
def consume(self, size: int = -1):
|
|
69
|
+
return self._stream.consume(size)
|
|
70
|
+
|
|
71
|
+
def read(self, size: int = -1):
|
|
72
|
+
return self._stream.read(size)
|
|
73
|
+
|
|
74
|
+
def readline(self, crlf=True, max_line_len=8192):
|
|
75
|
+
return self._stream.readLine(max_line_len)
|
|
76
|
+
|
|
77
|
+
def tell(self):
|
|
78
|
+
return self._stream.tell()
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class IOStream:
|
|
82
|
+
def __init__(self, *_, _reader_factory=None, _writer_factory=None, **__):
|
|
83
|
+
_warn_deprecated(
|
|
84
|
+
f"{self.__class__.__name__} is deprecated and will be removed in a future version. "
|
|
85
|
+
"Use the new Reader and Writer classes from stream_io instead.")
|
|
86
|
+
|
|
87
|
+
self._reader_factory = _reader_factory
|
|
88
|
+
self._writer_factory = _writer_factory
|
|
89
|
+
self._reader = None
|
|
90
|
+
self._writer = None
|
|
91
|
+
|
|
92
|
+
def _get_stream(self):
|
|
93
|
+
if self._reader is not None:
|
|
94
|
+
return self._reader
|
|
95
|
+
if self._writer is not None:
|
|
96
|
+
return self._writer
|
|
97
|
+
stream = self._get_reader()
|
|
98
|
+
if stream is None:
|
|
99
|
+
stream = self._get_writer()
|
|
100
|
+
return stream
|
|
101
|
+
|
|
102
|
+
def _get_reader(self):
|
|
103
|
+
if self._reader is None:
|
|
104
|
+
self._reader = self._reader_factory()
|
|
105
|
+
return self._reader
|
|
106
|
+
|
|
107
|
+
def _get_writer(self):
|
|
108
|
+
if self._writer is None:
|
|
109
|
+
self._writer = self._writer_factory()
|
|
110
|
+
return self._writer
|
|
111
|
+
|
|
112
|
+
def read(self, size: int) -> bytes:
|
|
113
|
+
return self._get_reader().read(size)
|
|
114
|
+
|
|
115
|
+
def write(self, data: bytes) -> int:
|
|
116
|
+
return self._get_writer().write(data)
|
|
117
|
+
|
|
118
|
+
def close(self):
|
|
119
|
+
self._get_stream().close()
|
|
120
|
+
|
|
121
|
+
def flush(self):
|
|
122
|
+
self._get_stream().flush()
|
|
123
|
+
|
|
124
|
+
def seek(self, offset: int):
|
|
125
|
+
self._get_stream().seek(offset)
|
|
126
|
+
|
|
127
|
+
def tell(self) -> int:
|
|
128
|
+
return self._get_stream().tell()
|
|
129
|
+
|
|
130
|
+
def __enter__(self):
|
|
131
|
+
s = self._get_stream()
|
|
132
|
+
if hasattr(s, '__enter__'):
|
|
133
|
+
return self._get_stream().__enter__()
|
|
134
|
+
return self
|
|
135
|
+
|
|
136
|
+
def __exit__(self, exc_type, exc, traceback):
|
|
137
|
+
s = self._get_stream()
|
|
138
|
+
if hasattr(s, '__exit__'):
|
|
139
|
+
self._get_stream().__exit__(exc_type, exc, traceback)
|
|
140
|
+
s.close()
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class CompressingStream(IOStream):
|
|
144
|
+
def begin_member(self):
|
|
145
|
+
pass
|
|
146
|
+
|
|
147
|
+
def end_member(self):
|
|
148
|
+
self._get_writer().finish()
|
|
149
|
+
self._writer = None
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
class PythonIOStreamAdapter:
|
|
153
|
+
def __new__(cls, py_stream):
|
|
154
|
+
return IOStream(_reader_factory=lambda: py_stream,
|
|
155
|
+
_writer_factory=lambda: py_stream)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
class FileStream:
|
|
159
|
+
def __new__(cls, filename: str, mode: str = 'rb'):
|
|
160
|
+
if 'b' not in mode:
|
|
161
|
+
mode += 'b'
|
|
162
|
+
f = open(filename, mode)
|
|
163
|
+
return IOStream(_reader_factory=lambda: f, _writer_factory=lambda: f)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class BytesIOStream:
|
|
167
|
+
def __new__(cls, initial_data=None):
|
|
168
|
+
import io
|
|
169
|
+
bio = io.BytesIO(initial_data if initial_data is not None else b'')
|
|
170
|
+
return IOStream(_reader_factory=lambda: bio, _writer_factory=lambda: bio)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
class GZipStream:
|
|
174
|
+
def __new__(cls, raw_stream, compression_level=9, zlib=False, fsspec_args=None):
|
|
175
|
+
from ..stream_io import GzipReader, GzipWriter
|
|
176
|
+
stream = CompressingStream(
|
|
177
|
+
_reader_factory=lambda: GzipReader(raw_stream, zlib=zlib, fsspec_args=fsspec_args),
|
|
178
|
+
_writer_factory=lambda: GzipWriter(raw_stream, compression_level=compression_level, zlib=zlib,
|
|
179
|
+
fsspec_args=fsspec_args))
|
|
180
|
+
stream._raw_stream = raw_stream
|
|
181
|
+
return stream
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
class LZ4Stream:
|
|
185
|
+
def __new__(cls, raw_stream, mode='r', compression_level=12,
|
|
186
|
+
favor_dec_speed=True, fsspec_args=None):
|
|
187
|
+
from ..stream_io import Lz4Reader, Lz4Writer
|
|
188
|
+
stream = CompressingStream(
|
|
189
|
+
_reader_factory=lambda: Lz4Reader(raw_stream, fsspec_args=fsspec_args),
|
|
190
|
+
_writer_factory=lambda: Lz4Writer(raw_stream, fsspec_args=fsspec_args))
|
|
191
|
+
stream._raw_stream = raw_stream
|
|
192
|
+
return stream
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
class BrotliStream:
|
|
196
|
+
def __new__(cls, raw_stream, quality=11, lgwin=22, lgblock=0, fsspec_args=None):
|
|
197
|
+
from ..stream_io import BrotliReader, BrotliWriter
|
|
198
|
+
stream = CompressingStream(
|
|
199
|
+
_reader_factory=lambda: BrotliReader(raw_stream, fsspec_args=fsspec_args),
|
|
200
|
+
_writer_factory=lambda: BrotliWriter(raw_stream, fsspec_args=fsspec_args))
|
|
201
|
+
stream._raw_stream = raw_stream
|
|
202
|
+
return stream
|
|
@@ -14,9 +14,9 @@
|
|
|
14
14
|
|
|
15
15
|
# distutils: language = c++
|
|
16
16
|
|
|
17
|
-
from fastwarc.stream_io cimport IOStream, GZipStream, LZ4Stream, FileStream, PythonIOStreamAdapter
|
|
18
|
-
from fastwarc.stream_io import StreamError
|
|
19
|
-
from fastwarc.warc cimport ArchiveIterator, WarcRecordType
|
|
17
|
+
from fastwarc.legacy.stream_io cimport IOStream, GZipStream, LZ4Stream, FileStream, PythonIOStreamAdapter
|
|
18
|
+
from fastwarc.legacy.stream_io import StreamError
|
|
19
|
+
from fastwarc.legacy.warc cimport ArchiveIterator, WarcRecordType
|
|
20
20
|
|
|
21
21
|
__all__ = [
|
|
22
22
|
'CompressionAlg',
|
|
@@ -16,7 +16,7 @@ from libc.stdint cimport uint16_t
|
|
|
16
16
|
from libcpp.string cimport string
|
|
17
17
|
from libcpp.vector cimport vector
|
|
18
18
|
|
|
19
|
-
from fastwarc.stream_io cimport IOStream, BufferedReader
|
|
19
|
+
from fastwarc.legacy.stream_io cimport IOStream, BufferedReader
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
cpdef enum WarcRecordType:
|
|
@@ -36,9 +36,9 @@ from resiliparse_inc.cctype cimport isspace
|
|
|
36
36
|
from resiliparse_inc.cstdlib cimport strtol
|
|
37
37
|
from resiliparse_inc.utility cimport move
|
|
38
38
|
|
|
39
|
-
from fastwarc.stream_io cimport BufferedReader, BytesIOStream, CompressingStream, GZipStream, BrotliStream, \
|
|
39
|
+
from fastwarc.legacy.stream_io cimport BufferedReader, BytesIOStream, CompressingStream, GZipStream, BrotliStream, \
|
|
40
40
|
IOStream, wrap_stream
|
|
41
|
-
from fastwarc.stream_io import ReaderStaleError
|
|
41
|
+
from fastwarc.legacy.stream_io import ReaderStaleError
|
|
42
42
|
|
|
43
43
|
__all__ = [
|
|
44
44
|
'ArchiveIterator',
|