FastWARC 0.16.0__tar.gz → 1.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. {fastwarc-0.16.0 → fastwarc-1.0.1}/FastWARC.egg-info/PKG-INFO +30 -8
  2. {fastwarc-0.16.0 → fastwarc-1.0.1}/FastWARC.egg-info/SOURCES.txt +12 -7
  3. {fastwarc-0.16.0 → fastwarc-1.0.1}/PKG-INFO +30 -8
  4. {fastwarc-0.16.0 → fastwarc-1.0.1}/README.md +28 -6
  5. fastwarc-1.0.1/fastwarc/__init__.py +44 -0
  6. fastwarc-1.0.1/fastwarc/__init__.pyi +49 -0
  7. {fastwarc-0.16.0 → fastwarc-1.0.1}/fastwarc/cli.py +4 -4
  8. fastwarc-1.0.1/fastwarc/legacy/_shims.py +202 -0
  9. {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/tools.pyx +3 -3
  10. {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/warc.pxd +1 -1
  11. {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/warc.pyx +2 -2
  12. fastwarc-1.0.1/fastwarc/stream_io.pyi +187 -0
  13. fastwarc-1.0.1/fastwarc/warc.pyi +359 -0
  14. {fastwarc-0.16.0 → fastwarc-1.0.1}/pyproject.toml +20 -15
  15. {fastwarc-0.16.0 → fastwarc-1.0.1}/setup.py +30 -17
  16. {fastwarc-0.16.0 → fastwarc-1.0.1}/FastWARC.egg-info/dependency_links.txt +0 -0
  17. {fastwarc-0.16.0 → fastwarc-1.0.1}/FastWARC.egg-info/entry_points.txt +0 -0
  18. {fastwarc-0.16.0 → fastwarc-1.0.1}/FastWARC.egg-info/requires.txt +0 -0
  19. {fastwarc-0.16.0 → fastwarc-1.0.1}/FastWARC.egg-info/top_level.txt +0 -0
  20. {fastwarc-0.16.0 → fastwarc-1.0.1}/MANIFEST.in +0 -0
  21. {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/__init__.pxd +0 -0
  22. {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/__init__.py +0 -0
  23. {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/stream_io.pxd +0 -0
  24. {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/stream_io.pyi +0 -0
  25. {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/stream_io.pyx +0 -0
  26. {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/tools.pyi +0 -0
  27. {fastwarc-0.16.0/fastwarc → fastwarc-1.0.1/fastwarc/legacy}/warc.pyi +0 -0
  28. {fastwarc-0.16.0 → fastwarc-1.0.1}/fastwarc/py.typed +0 -0
  29. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_common/__init__.pxd +0 -0
  30. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_common/string_util.pxd +0 -0
  31. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/__init__.pxd +0 -0
  32. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/algorithm.pxd +0 -0
  33. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/atomic.pxd +0 -0
  34. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/boost_regex.pxd +0 -0
  35. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/cctype.pxd +0 -0
  36. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/cstdlib.pxd +0 -0
  37. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/cstring.pxd +0 -0
  38. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/dlfcn.pxd +0 -0
  39. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/errno.pxd +0 -0
  40. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/lexbor.pxd +0 -0
  41. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/lz4frame.pxd +0 -0
  42. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/lz4hc.pxd +0 -0
  43. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/pthread.pxd +0 -0
  44. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/re2.pxd +0 -0
  45. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/stdio.pxd +0 -0
  46. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/string_view.pxd +0 -0
  47. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/time.pxd +0 -0
  48. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/uchardet.pxd +0 -0
  49. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/unistd.pxd +0 -0
  50. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/utility.pxd +0 -0
  51. {fastwarc-0.16.0 → fastwarc-1.0.1}/resiliparse_inc/zlib.pxd +0 -0
  52. {fastwarc-0.16.0 → fastwarc-1.0.1}/setup.cfg +0 -0
@@ -1,9 +1,9 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: FastWARC
3
- Version: 0.16.0
3
+ Version: 1.0.1
4
4
  Summary: A high-performance WARC parsing library for Python written in C++/Cython.
5
5
  Author: Janek Bevendorff
6
- License: Apache License 2.0
6
+ License-Expression: Apache-2.0
7
7
  Project-URL: repository, https://github.com/chatnoir-eu/chatnoir-resiliparse
8
8
  Project-URL: homepage, https://resiliparse.chatnoir.eu
9
9
  Project-URL: documentation, https://resiliparse.chatnoir.eu
@@ -22,33 +22,51 @@ Requires-Dist: pytest>=7.2.2; extra == "test"
22
22
  Requires-Dist: pytest-cov>=4.0.0; extra == "test"
23
23
  Requires-Dist: lz4; extra == "test"
24
24
 
25
- # FastWARC
25
+ # FastWARC (Python)
26
26
 
27
- FastWARC is a high-performance WARC parsing library for Python written in C++/Cython. The API is inspired in large parts by [WARCIO](https://github.com/webrecorder/warcio), but does not aim at being a drop-in replacement. FastWARC supports compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are GZip and LZ4.
27
+ FastWARC is a high-performance WARC parsing library for Python written in Rust with Python bindings. FastWARC supports
28
+ compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are Gzip and LZ4.
28
29
 
29
- FastWARC belongs to the [ChatNoir Resiliparse toolkit](https://github.com/chatnoir-eu/chatnoir-resiliparse/) for fast and robust web data processing.
30
+ This crate contains the Python bindings for FastWARC.
31
+
32
+ FastWARC belongs to the [ChatNoir Resiliparse toolkit](https://github.com/chatnoir-eu/chatnoir-resiliparse/) for fast
33
+ and robust web data processing.
30
34
 
31
35
  ## Installing FastWARC
36
+
32
37
  Pre-built FastWARC binaries for most Linux platforms can be installed from PyPi:
38
+
33
39
  ```bash
34
40
  pip install fastwarc
35
41
  ```
36
- **However:** the Linux binaries are provided *solely for your convenience*. Since they are built on the very old `manylinux` base system for better compatibility, their performance isn't optimal (though still better than WARCIO). For best performance, see the next section on how to build FastWARC yourself.
42
+
43
+ **However:** the Linux binaries are provided *solely for your convenience*. Since they are built on the very old
44
+ `manylinux` base system for better compatibility, their performance isn't optimal (though still better than WARCIO). For
45
+ best performance, see the next section on how to build FastWARC yourself.
37
46
 
38
47
  ## Building FastWARC From Source
39
- You can compile FastWARC either from the PyPi source package or directly from this repository, though in any case, you need to install all required build-time dependencies first. On Ubuntu, this is done as follows:
48
+
49
+ You can compile FastWARC either from the PyPi source package or directly from this repository, though in any case, you
50
+ need to install all required build-time dependencies first. On Ubuntu, this is done as follows:
51
+
40
52
  ```bash
41
53
  sudo apt install build-essential python3-dev zlib1g-dev liblz4-dev
42
54
  ```
55
+
43
56
  To build and install FastWARC from PyPi, run
57
+
44
58
  ```bash
45
59
  pip install --no-binary fastwarc fastwarc
46
60
  ```
61
+
47
62
  That's it. If you prefer to build and install directly from this repository instead, run:
63
+
48
64
  ```bash
49
65
  pip install -e fastwarc
50
66
  ```
67
+
51
68
  To build the wheels without installing them, run:
69
+
52
70
  ```bash
53
71
  pip wheel -e fastwarc
54
72
 
@@ -57,10 +75,14 @@ pip install build && python -m build --wheel fastwarc
57
75
  ```
58
76
 
59
77
  ## Usage Instructions
60
- For detailed usage instructions, please consult the [FastWARC User Manual](https://resiliparse.chatnoir.eu/en/latest/man/fastwarc.html).
78
+
79
+ For detailed usage instructions, please consult
80
+ the [FastWARC User Manual](https://resiliparse.chatnoir.eu/en/latest/man/fastwarc.html).
61
81
 
62
82
  ## Cite Us
83
+
63
84
  If you use FastWARC, please consider citing our [OSSYM 2021 abstract paper](https://arxiv.org/abs/2112.03103):
85
+
64
86
  ```bibtex
65
87
  @InProceedings{bevendorff:2021,
66
88
  author = {Janek Bevendorff and Martin Potthast and Benno Stein},
@@ -8,18 +8,23 @@ FastWARC.egg-info/dependency_links.txt
8
8
  FastWARC.egg-info/entry_points.txt
9
9
  FastWARC.egg-info/requires.txt
10
10
  FastWARC.egg-info/top_level.txt
11
- fastwarc/__init__.pxd
12
11
  fastwarc/__init__.py
12
+ fastwarc/__init__.pyi
13
13
  fastwarc/cli.py
14
14
  fastwarc/py.typed
15
- fastwarc/stream_io.pxd
16
15
  fastwarc/stream_io.pyi
17
- fastwarc/stream_io.pyx
18
- fastwarc/tools.pyi
19
- fastwarc/tools.pyx
20
- fastwarc/warc.pxd
21
16
  fastwarc/warc.pyi
22
- fastwarc/warc.pyx
17
+ fastwarc/legacy/__init__.pxd
18
+ fastwarc/legacy/__init__.py
19
+ fastwarc/legacy/_shims.py
20
+ fastwarc/legacy/stream_io.pxd
21
+ fastwarc/legacy/stream_io.pyi
22
+ fastwarc/legacy/stream_io.pyx
23
+ fastwarc/legacy/tools.pyi
24
+ fastwarc/legacy/tools.pyx
25
+ fastwarc/legacy/warc.pxd
26
+ fastwarc/legacy/warc.pyi
27
+ fastwarc/legacy/warc.pyx
23
28
  resiliparse_common/__init__.pxd
24
29
  resiliparse_common/string_util.pxd
25
30
  resiliparse_inc/__init__.pxd
@@ -1,9 +1,9 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: FastWARC
3
- Version: 0.16.0
3
+ Version: 1.0.1
4
4
  Summary: A high-performance WARC parsing library for Python written in C++/Cython.
5
5
  Author: Janek Bevendorff
6
- License: Apache License 2.0
6
+ License-Expression: Apache-2.0
7
7
  Project-URL: repository, https://github.com/chatnoir-eu/chatnoir-resiliparse
8
8
  Project-URL: homepage, https://resiliparse.chatnoir.eu
9
9
  Project-URL: documentation, https://resiliparse.chatnoir.eu
@@ -22,33 +22,51 @@ Requires-Dist: pytest>=7.2.2; extra == "test"
22
22
  Requires-Dist: pytest-cov>=4.0.0; extra == "test"
23
23
  Requires-Dist: lz4; extra == "test"
24
24
 
25
- # FastWARC
25
+ # FastWARC (Python)
26
26
 
27
- FastWARC is a high-performance WARC parsing library for Python written in C++/Cython. The API is inspired in large parts by [WARCIO](https://github.com/webrecorder/warcio), but does not aim at being a drop-in replacement. FastWARC supports compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are GZip and LZ4.
27
+ FastWARC is a high-performance WARC parsing library for Python written in Rust with Python bindings. FastWARC supports
28
+ compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are Gzip and LZ4.
28
29
 
29
- FastWARC belongs to the [ChatNoir Resiliparse toolkit](https://github.com/chatnoir-eu/chatnoir-resiliparse/) for fast and robust web data processing.
30
+ This crate contains the Python bindings for FastWARC.
31
+
32
+ FastWARC belongs to the [ChatNoir Resiliparse toolkit](https://github.com/chatnoir-eu/chatnoir-resiliparse/) for fast
33
+ and robust web data processing.
30
34
 
31
35
  ## Installing FastWARC
36
+
32
37
  Pre-built FastWARC binaries for most Linux platforms can be installed from PyPi:
38
+
33
39
  ```bash
34
40
  pip install fastwarc
35
41
  ```
36
- **However:** the Linux binaries are provided *solely for your convenience*. Since they are built on the very old `manylinux` base system for better compatibility, their performance isn't optimal (though still better than WARCIO). For best performance, see the next section on how to build FastWARC yourself.
42
+
43
+ **However:** the Linux binaries are provided *solely for your convenience*. Since they are built on the very old
44
+ `manylinux` base system for better compatibility, their performance isn't optimal (though still better than WARCIO). For
45
+ best performance, see the next section on how to build FastWARC yourself.
37
46
 
38
47
  ## Building FastWARC From Source
39
- You can compile FastWARC either from the PyPi source package or directly from this repository, though in any case, you need to install all required build-time dependencies first. On Ubuntu, this is done as follows:
48
+
49
+ You can compile FastWARC either from the PyPi source package or directly from this repository, though in any case, you
50
+ need to install all required build-time dependencies first. On Ubuntu, this is done as follows:
51
+
40
52
  ```bash
41
53
  sudo apt install build-essential python3-dev zlib1g-dev liblz4-dev
42
54
  ```
55
+
43
56
  To build and install FastWARC from PyPi, run
57
+
44
58
  ```bash
45
59
  pip install --no-binary fastwarc fastwarc
46
60
  ```
61
+
47
62
  That's it. If you prefer to build and install directly from this repository instead, run:
63
+
48
64
  ```bash
49
65
  pip install -e fastwarc
50
66
  ```
67
+
51
68
  To build the wheels without installing them, run:
69
+
52
70
  ```bash
53
71
  pip wheel -e fastwarc
54
72
 
@@ -57,10 +75,14 @@ pip install build && python -m build --wheel fastwarc
57
75
  ```
58
76
 
59
77
  ## Usage Instructions
60
- For detailed usage instructions, please consult the [FastWARC User Manual](https://resiliparse.chatnoir.eu/en/latest/man/fastwarc.html).
78
+
79
+ For detailed usage instructions, please consult
80
+ the [FastWARC User Manual](https://resiliparse.chatnoir.eu/en/latest/man/fastwarc.html).
61
81
 
62
82
  ## Cite Us
83
+
63
84
  If you use FastWARC, please consider citing our [OSSYM 2021 abstract paper](https://arxiv.org/abs/2112.03103):
85
+
64
86
  ```bibtex
65
87
  @InProceedings{bevendorff:2021,
66
88
  author = {Janek Bevendorff and Martin Potthast and Benno Stein},
@@ -1,30 +1,48 @@
1
- # FastWARC
1
+ # FastWARC (Python)
2
2
 
3
- FastWARC is a high-performance WARC parsing library for Python written in C++/Cython. The API is inspired in large parts by [WARCIO](https://github.com/webrecorder/warcio), but does not aim at being a drop-in replacement. FastWARC supports compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are GZip and LZ4.
3
+ FastWARC is a high-performance WARC parsing library for Python written in Rust with Python bindings. FastWARC supports
4
+ compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are Gzip and LZ4.
4
5
 
5
- FastWARC belongs to the [ChatNoir Resiliparse toolkit](https://github.com/chatnoir-eu/chatnoir-resiliparse/) for fast and robust web data processing.
6
+ This crate contains the Python bindings for FastWARC.
7
+
8
+ FastWARC belongs to the [ChatNoir Resiliparse toolkit](https://github.com/chatnoir-eu/chatnoir-resiliparse/) for fast
9
+ and robust web data processing.
6
10
 
7
11
  ## Installing FastWARC
12
+
8
13
  Pre-built FastWARC binaries for most Linux platforms can be installed from PyPi:
14
+
9
15
  ```bash
10
16
  pip install fastwarc
11
17
  ```
12
- **However:** the Linux binaries are provided *solely for your convenience*. Since they are built on the very old `manylinux` base system for better compatibility, their performance isn't optimal (though still better than WARCIO). For best performance, see the next section on how to build FastWARC yourself.
18
+
19
+ **However:** the Linux binaries are provided *solely for your convenience*. Since they are built on the very old
20
+ `manylinux` base system for better compatibility, their performance isn't optimal (though still better than WARCIO). For
21
+ best performance, see the next section on how to build FastWARC yourself.
13
22
 
14
23
  ## Building FastWARC From Source
15
- You can compile FastWARC either from the PyPi source package or directly from this repository, though in any case, you need to install all required build-time dependencies first. On Ubuntu, this is done as follows:
24
+
25
+ You can compile FastWARC either from the PyPi source package or directly from this repository, though in any case, you
26
+ need to install all required build-time dependencies first. On Ubuntu, this is done as follows:
27
+
16
28
  ```bash
17
29
  sudo apt install build-essential python3-dev zlib1g-dev liblz4-dev
18
30
  ```
31
+
19
32
  To build and install FastWARC from PyPi, run
33
+
20
34
  ```bash
21
35
  pip install --no-binary fastwarc fastwarc
22
36
  ```
37
+
23
38
  That's it. If you prefer to build and install directly from this repository instead, run:
39
+
24
40
  ```bash
25
41
  pip install -e fastwarc
26
42
  ```
43
+
27
44
  To build the wheels without installing them, run:
45
+
28
46
  ```bash
29
47
  pip wheel -e fastwarc
30
48
 
@@ -33,10 +51,14 @@ pip install build && python -m build --wheel fastwarc
33
51
  ```
34
52
 
35
53
  ## Usage Instructions
36
- For detailed usage instructions, please consult the [FastWARC User Manual](https://resiliparse.chatnoir.eu/en/latest/man/fastwarc.html).
54
+
55
+ For detailed usage instructions, please consult
56
+ the [FastWARC User Manual](https://resiliparse.chatnoir.eu/en/latest/man/fastwarc.html).
37
57
 
38
58
  ## Cite Us
59
+
39
60
  If you use FastWARC, please consider citing our [OSSYM 2021 abstract paper](https://arxiv.org/abs/2112.03103):
61
+
40
62
  ```bibtex
41
63
  @InProceedings{bevendorff:2021,
42
64
  author = {Janek Bevendorff and Martin Potthast and Benno Stein},
@@ -0,0 +1,44 @@
1
+ # Copyright 2026 Janek Bevendorff
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import importlib
16
+
17
+ _native = importlib.import_module("._fastwarc", __name__)
18
+ _legacy_shims = importlib.import_module(".legacy._shims", __name__)
19
+
20
+ import fastwarc.stream_io as stream_io # noqa: F401
21
+ import fastwarc.warc as warc # noqa: F401
22
+
23
+ # Re-export module attributes
24
+ globals().update({attr: getattr(_native, attr) for attr in _native.__all__})
25
+
26
+ # Patch legacy shims onto stream_io
27
+ for name in _legacy_shims.__all__:
28
+ setattr(stream_io, name, getattr(_legacy_shims, name))
29
+ stream_io.__all__ += getattr(_legacy_shims, "__all__", ()) # type: ignore[attr-defined]
30
+
31
+ FileStream = _legacy_shims.FileStream
32
+ GZipStream = _legacy_shims.GZipStream
33
+ LZ4Stream = _legacy_shims.LZ4Stream
34
+ FastWARCError = _legacy_shims.FastWARCError
35
+ StreamError = _legacy_shims.StreamError
36
+
37
+ __all__ = [
38
+ *_native.__all__,
39
+ 'FileStream',
40
+ 'GZipStream',
41
+ 'LZ4Stream',
42
+ 'FastWARCError',
43
+ 'StreamError',
44
+ ]
@@ -0,0 +1,49 @@
1
+ # Copyright 2026 Janek Bevendorff
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from .warc import ArchiveIterator, WarcRecord, WarcRecordType
16
+ from .stream_io import (
17
+ GzipReader,
18
+ GzipWriter,
19
+ ZstdReader,
20
+ ZstdWriter,
21
+ Lz4Reader,
22
+ Lz4Writer,
23
+
24
+ # Legacy names
25
+ FileStream,
26
+ GZipStream,
27
+ LZ4Stream,
28
+ FastWARCError,
29
+ StreamError,
30
+ )
31
+
32
+ __all__ = [
33
+ 'ArchiveIterator',
34
+ 'WarcRecord',
35
+ 'WarcRecordType',
36
+ 'GzipReader',
37
+ 'GzipWriter',
38
+ 'ZstdReader',
39
+ 'ZstdWriter',
40
+ 'Lz4Reader',
41
+ 'Lz4Writer',
42
+
43
+ # Legacy names
44
+ 'FileStream',
45
+ 'GZipStream',
46
+ 'LZ4Stream',
47
+ 'FastWARCError',
48
+ 'StreamError',
49
+ ]
@@ -24,9 +24,9 @@ import urllib.request
24
24
  import click
25
25
  from tqdm import tqdm
26
26
 
27
- from fastwarc.stream_io import FileStream, StreamError, FastWARCError, PythonIOStreamAdapter
28
- from fastwarc.warc import ArchiveIterator, WarcRecordType
29
- from fastwarc.tools import CompressionAlg, detect_compression_algorithm, wrap_warc_stream, \
27
+ from fastwarc.legacy.stream_io import FileStream, StreamError, FastWARCError, PythonIOStreamAdapter
28
+ from fastwarc.legacy.warc import ArchiveIterator, WarcRecordType
29
+ from fastwarc.legacy.tools import CompressionAlg, detect_compression_algorithm, wrap_warc_stream, \
30
30
  recompress_warc_interactive, verify_digests
31
31
 
32
32
 
@@ -288,7 +288,7 @@ botocore = None
288
288
  s3 = None
289
289
 
290
290
 
291
- def _init_s3(endpoint_url, aws_access_key, aws_secret_key=None):
291
+ def _init_s3(endpoint_url, aws_access_key, aws_secret_key=None):
292
292
  global boto3, botocore, s3
293
293
 
294
294
  if s3 is not None:
@@ -0,0 +1,202 @@
1
+ # Copyright 2026 Janek Bevendorff
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ # Legacy shims for making the new Rust extension work with the old type names
16
+
17
+ import warnings
18
+
19
+ __all__ = [
20
+ 'wrap_stream',
21
+ 'BufferedReader',
22
+ 'IOStream',
23
+ 'PythonIOStreamAdapter',
24
+ 'CompressingStream',
25
+ 'FileStream',
26
+ 'BytesIOStream',
27
+ 'GZipStream',
28
+ 'LZ4Stream',
29
+ 'BrotliStream',
30
+ 'FastWARCError',
31
+ 'StreamError',
32
+ 'ReaderStaleError',
33
+ ]
34
+
35
+ FastWARCError = OSError
36
+
37
+ StreamError = OSError
38
+
39
+ ReaderStaleError = OSError
40
+
41
+
42
+ def _warn_deprecated(msg):
43
+ warnings.warn(msg, FutureWarning, 3)
44
+
45
+
46
+ def wrap_stream(raw_stream, mode='rb', fsspec_args=None):
47
+ _warn_deprecated("wrap_stream is deprecated and will be removed in a future version.")
48
+ if isinstance(raw_stream, str):
49
+ if fsspec_args is not False and '://' in raw_stream:
50
+ try:
51
+ import fsspec
52
+ return fsspec.open(raw_stream, mode, **(fsspec_args or {})).open()
53
+ except ModuleNotFoundError:
54
+ pass
55
+ return open(raw_stream, mode)
56
+
57
+ return raw_stream
58
+
59
+
60
+ class BufferedReader:
61
+ def __init__(self, stream, *args, **kwargs):
62
+ _warn_deprecated("BufferedReader is deprecated and will be removed in a future version.")
63
+ self._stream = stream
64
+
65
+ def close(self):
66
+ return self._stream.close()
67
+
68
+ def consume(self, size: int = -1):
69
+ return self._stream.consume(size)
70
+
71
+ def read(self, size: int = -1):
72
+ return self._stream.read(size)
73
+
74
+ def readline(self, crlf=True, max_line_len=8192):
75
+ return self._stream.readLine(max_line_len)
76
+
77
+ def tell(self):
78
+ return self._stream.tell()
79
+
80
+
81
+ class IOStream:
82
+ def __init__(self, *_, _reader_factory=None, _writer_factory=None, **__):
83
+ _warn_deprecated(
84
+ f"{self.__class__.__name__} is deprecated and will be removed in a future version. "
85
+ "Use the new Reader and Writer classes from stream_io instead.")
86
+
87
+ self._reader_factory = _reader_factory
88
+ self._writer_factory = _writer_factory
89
+ self._reader = None
90
+ self._writer = None
91
+
92
+ def _get_stream(self):
93
+ if self._reader is not None:
94
+ return self._reader
95
+ if self._writer is not None:
96
+ return self._writer
97
+ stream = self._get_reader()
98
+ if stream is None:
99
+ stream = self._get_writer()
100
+ return stream
101
+
102
+ def _get_reader(self):
103
+ if self._reader is None:
104
+ self._reader = self._reader_factory()
105
+ return self._reader
106
+
107
+ def _get_writer(self):
108
+ if self._writer is None:
109
+ self._writer = self._writer_factory()
110
+ return self._writer
111
+
112
+ def read(self, size: int) -> bytes:
113
+ return self._get_reader().read(size)
114
+
115
+ def write(self, data: bytes) -> int:
116
+ return self._get_writer().write(data)
117
+
118
+ def close(self):
119
+ self._get_stream().close()
120
+
121
+ def flush(self):
122
+ self._get_stream().flush()
123
+
124
+ def seek(self, offset: int):
125
+ self._get_stream().seek(offset)
126
+
127
+ def tell(self) -> int:
128
+ return self._get_stream().tell()
129
+
130
+ def __enter__(self):
131
+ s = self._get_stream()
132
+ if hasattr(s, '__enter__'):
133
+ return self._get_stream().__enter__()
134
+ return self
135
+
136
+ def __exit__(self, exc_type, exc, traceback):
137
+ s = self._get_stream()
138
+ if hasattr(s, '__exit__'):
139
+ self._get_stream().__exit__(exc_type, exc, traceback)
140
+ s.close()
141
+
142
+
143
+ class CompressingStream(IOStream):
144
+ def begin_member(self):
145
+ pass
146
+
147
+ def end_member(self):
148
+ self._get_writer().finish()
149
+ self._writer = None
150
+
151
+
152
+ class PythonIOStreamAdapter:
153
+ def __new__(cls, py_stream):
154
+ return IOStream(_reader_factory=lambda: py_stream,
155
+ _writer_factory=lambda: py_stream)
156
+
157
+
158
+ class FileStream:
159
+ def __new__(cls, filename: str, mode: str = 'rb'):
160
+ if 'b' not in mode:
161
+ mode += 'b'
162
+ f = open(filename, mode)
163
+ return IOStream(_reader_factory=lambda: f, _writer_factory=lambda: f)
164
+
165
+
166
+ class BytesIOStream:
167
+ def __new__(cls, initial_data=None):
168
+ import io
169
+ bio = io.BytesIO(initial_data if initial_data is not None else b'')
170
+ return IOStream(_reader_factory=lambda: bio, _writer_factory=lambda: bio)
171
+
172
+
173
+ class GZipStream:
174
+ def __new__(cls, raw_stream, compression_level=9, zlib=False, fsspec_args=None):
175
+ from ..stream_io import GzipReader, GzipWriter
176
+ stream = CompressingStream(
177
+ _reader_factory=lambda: GzipReader(raw_stream, zlib=zlib, fsspec_args=fsspec_args),
178
+ _writer_factory=lambda: GzipWriter(raw_stream, compression_level=compression_level, zlib=zlib,
179
+ fsspec_args=fsspec_args))
180
+ stream._raw_stream = raw_stream
181
+ return stream
182
+
183
+
184
+ class LZ4Stream:
185
+ def __new__(cls, raw_stream, mode='r', compression_level=12,
186
+ favor_dec_speed=True, fsspec_args=None):
187
+ from ..stream_io import Lz4Reader, Lz4Writer
188
+ stream = CompressingStream(
189
+ _reader_factory=lambda: Lz4Reader(raw_stream, fsspec_args=fsspec_args),
190
+ _writer_factory=lambda: Lz4Writer(raw_stream, fsspec_args=fsspec_args))
191
+ stream._raw_stream = raw_stream
192
+ return stream
193
+
194
+
195
+ class BrotliStream:
196
+ def __new__(cls, raw_stream, quality=11, lgwin=22, lgblock=0, fsspec_args=None):
197
+ from ..stream_io import BrotliReader, BrotliWriter
198
+ stream = CompressingStream(
199
+ _reader_factory=lambda: BrotliReader(raw_stream, fsspec_args=fsspec_args),
200
+ _writer_factory=lambda: BrotliWriter(raw_stream, fsspec_args=fsspec_args))
201
+ stream._raw_stream = raw_stream
202
+ return stream
@@ -14,9 +14,9 @@
14
14
 
15
15
  # distutils: language = c++
16
16
 
17
- from fastwarc.stream_io cimport IOStream, GZipStream, LZ4Stream, FileStream, PythonIOStreamAdapter
18
- from fastwarc.stream_io import StreamError
19
- from fastwarc.warc cimport ArchiveIterator, WarcRecordType
17
+ from fastwarc.legacy.stream_io cimport IOStream, GZipStream, LZ4Stream, FileStream, PythonIOStreamAdapter
18
+ from fastwarc.legacy.stream_io import StreamError
19
+ from fastwarc.legacy.warc cimport ArchiveIterator, WarcRecordType
20
20
 
21
21
  __all__ = [
22
22
  'CompressionAlg',
@@ -16,7 +16,7 @@ from libc.stdint cimport uint16_t
16
16
  from libcpp.string cimport string
17
17
  from libcpp.vector cimport vector
18
18
 
19
- from fastwarc.stream_io cimport IOStream, BufferedReader
19
+ from fastwarc.legacy.stream_io cimport IOStream, BufferedReader
20
20
 
21
21
 
22
22
  cpdef enum WarcRecordType:
@@ -36,9 +36,9 @@ from resiliparse_inc.cctype cimport isspace
36
36
  from resiliparse_inc.cstdlib cimport strtol
37
37
  from resiliparse_inc.utility cimport move
38
38
 
39
- from fastwarc.stream_io cimport BufferedReader, BytesIOStream, CompressingStream, GZipStream, BrotliStream, \
39
+ from fastwarc.legacy.stream_io cimport BufferedReader, BytesIOStream, CompressingStream, GZipStream, BrotliStream, \
40
40
  IOStream, wrap_stream
41
- from fastwarc.stream_io import ReaderStaleError
41
+ from fastwarc.legacy.stream_io import ReaderStaleError
42
42
 
43
43
  __all__ = [
44
44
  'ArchiveIterator',