FastWARC 1.0.0__tar.gz → 1.0.2__tar.gz

fastwarc-1.0.2/FastWARC.egg-info/PKG-INFO

@@ -0,0 +1,110 @@
+Metadata-Version: 2.4
+Name: FastWARC
+Version: 1.0.2
+Summary: A high-performance WARC parsing library written in Rust with bindings for Python.
+Author: Janek Bevendorff
+License-Expression: Apache-2.0
+Project-URL: repository, https://github.com/chatnoir-eu/chatnoir-resiliparse
+Project-URL: homepage, https://resiliparse.chatnoir.eu
+Project-URL: documentation, https://resiliparse.chatnoir.eu
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: brotli
+Requires-Dist: click
+Requires-Dist: tqdm
+Provides-Extra: all
+Requires-Dist: fastwarc[fsspec]; extra == "all"
+Provides-Extra: fsspec
+Requires-Dist: fsspec; extra == "fsspec"
+Requires-Dist: s3fs; extra == "fsspec"
+Provides-Extra: test
+Requires-Dist: fastwarc[all]; extra == "test"
+Requires-Dist: pytest>=7.2.2; extra == "test"
+Requires-Dist: pytest-cov>=4.0.0; extra == "test"
+
+# FastWARC (Python)
+
+FastWARC is a high-performance WARC parsing library written in Rust with bindings for Python.
+FastWARC supports compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are Gzip,
+Zstd, and LZ4.
+
+This package contains the Python bindings for FastWARC.
+
+FastWARC belongs to the [ChatNoir Resiliparse toolkit](https://github.com/chatnoir-eu/chatnoir-resiliparse/) for fast
+and robust web data processing.
+
+## Installing FastWARC
+
+Pre-built FastWARC binaries for Python can be installed from [PyPi](https://pypi.org/project/FastWARC/):
+
+```bash
+pip install fastwarc
+```
+
+## Building Python Bindings From Source
+
+To be able to build FastWARC, you need the Rust toolchain installed (see above). Then to build directly from the GitHub
+repository, follow these steps:
+
+```bash
+# Clone repository
+
+git clone https://github.com/chatnoir-eu/chatnoir-resiliparse.git
+cd chatnoir-resiliparse
+
+# Install Rust toolchain (if not already installed)
+
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+
+# Create a fresh venv first (recommended)
+
+python3 -m venv venv && source venv/bin/activate
+
+# Option 1: Build and install in editable mode (best for development)
+
+python3 -m pip install -e ./fastwarc-py
+
+# Option 2 (alternative): Build and install wheels in separate steps (best for redistribution)
+
+python3 -m pip wheel -w build ./fastwarc-py
+ls ./build/*.whl | xargs python3 -m pip install
+```
+
+## Usage Instructions
+
+For detailed usage instructions, please consult
+the [FastWARC User Manual](https://resiliparse.chatnoir.eu/en/latest/man/fastwarc.html).
+
+## Cite Us
+
+If you use FastWARC, please consider citing our [OSSYM 2021 abstract paper](https://arxiv.org/abs/2112.03103):
+
+```bibtex
+@InProceedings{bevendorff:2021,
+  author =                {Janek Bevendorff and Martin Potthast and Benno Stein},
+  booktitle =             {3rd International Symposium on Open Search Technology (OSSYM 2021)},
+  editor =                {Andreas Wagner and Christian Guetl and Michael Granitzer and Stefan Voigt},
+  month =                 oct,
+  publisher =             {International Open Search Symposium},
+  site =                  {CERN, Geneva, Switzerland},
+  title =                 {{FastWARC: Optimizing Large-Scale Web Archive Analytics}},
+  year =                  2021
+}
+```
+
+You can also cite our [ECIR 2018 demo paper](https://webis.de/downloads/publications/papers/bevendorff_2018.pdf):
+
+```bibtex
+@InProceedings{bevendorff:2018,
+  address =             {Berlin Heidelberg New York},
+  author =              {Janek Bevendorff and Benno Stein and Matthias Hagen and Martin Potthast},
+  booktitle =           {Advances in Information Retrieval. 40th European Conference on IR Research (ECIR 2018)},
+  editor =              {Leif Azzopardi and Allan Hanbury and Gabriella Pasi and Benjamin Piwowarski},
+  month =               mar,
+  publisher =           {Springer},
+  series =              {Lecture Notes in Computer Science},
+  site =                {Grenoble, France},
+  title =               {{Elastic ChatNoir: Search Engine for the ClueWeb and the Common Crawl}},
+  year =                2018
+}
+```

{fastwarc-1.0.0 → fastwarc-1.0.2}/FastWARC.egg-info/SOURCES.txt

@@ -10,13 +10,13 @@ FastWARC.egg-info/requires.txt
 FastWARC.egg-info/top_level.txt
 fastwarc/__init__.py
 fastwarc/__init__.pyi
-fastwarc/_fastwarc.pyi
 fastwarc/cli.py
 fastwarc/py.typed
 fastwarc/stream_io.pyi
 fastwarc/warc.pyi
 fastwarc/legacy/__init__.pxd
 fastwarc/legacy/__init__.py
+fastwarc/legacy/__init__.pyi
 fastwarc/legacy/_shims.py
 fastwarc/legacy/stream_io.pxd
 fastwarc/legacy/stream_io.pyi

{fastwarc-1.0.0 → fastwarc-1.0.2}/FastWARC.egg-info/requires.txt

@@ -7,9 +7,9 @@ fastwarc[fsspec]
 
 [fsspec]
 fsspec
+s3fs
 
 [test]
 fastwarc[all]
 pytest>=7.2.2
 pytest-cov>=4.0.0
-lz4

fastwarc-1.0.2/PKG-INFO

@@ -0,0 +1,110 @@
+Metadata-Version: 2.4
+Name: FastWARC
+Version: 1.0.2
+Summary: A high-performance WARC parsing library written in Rust with bindings for Python.
+Author: Janek Bevendorff
+License-Expression: Apache-2.0
+Project-URL: repository, https://github.com/chatnoir-eu/chatnoir-resiliparse
+Project-URL: homepage, https://resiliparse.chatnoir.eu
+Project-URL: documentation, https://resiliparse.chatnoir.eu
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: brotli
+Requires-Dist: click
+Requires-Dist: tqdm
+Provides-Extra: all
+Requires-Dist: fastwarc[fsspec]; extra == "all"
+Provides-Extra: fsspec
+Requires-Dist: fsspec; extra == "fsspec"
+Requires-Dist: s3fs; extra == "fsspec"
+Provides-Extra: test
+Requires-Dist: fastwarc[all]; extra == "test"
+Requires-Dist: pytest>=7.2.2; extra == "test"
+Requires-Dist: pytest-cov>=4.0.0; extra == "test"
+
+# FastWARC (Python)
+
+FastWARC is a high-performance WARC parsing library written in Rust with bindings for Python.
+FastWARC supports compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are Gzip,
+Zstd, and LZ4.
+
+This package contains the Python bindings for FastWARC.
+
+FastWARC belongs to the [ChatNoir Resiliparse toolkit](https://github.com/chatnoir-eu/chatnoir-resiliparse/) for fast
+and robust web data processing.
+
+## Installing FastWARC
+
+Pre-built FastWARC binaries for Python can be installed from [PyPi](https://pypi.org/project/FastWARC/):
+
+```bash
+pip install fastwarc
+```
+
+## Building Python Bindings From Source
+
+To be able to build FastWARC, you need the Rust toolchain installed (see above). Then to build directly from the GitHub
+repository, follow these steps:
+
+```bash
+# Clone repository
+
+git clone https://github.com/chatnoir-eu/chatnoir-resiliparse.git
+cd chatnoir-resiliparse
+
+# Install Rust toolchain (if not already installed)
+
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+
+# Create a fresh venv first (recommended)
+
+python3 -m venv venv && source venv/bin/activate
+
+# Option 1: Build and install in editable mode (best for development)
+
+python3 -m pip install -e ./fastwarc-py
+
+# Option 2 (alternative): Build and install wheels in separate steps (best for redistribution)
+
+python3 -m pip wheel -w build ./fastwarc-py
+ls ./build/*.whl | xargs python3 -m pip install
+```
+
+## Usage Instructions
+
+For detailed usage instructions, please consult
+the [FastWARC User Manual](https://resiliparse.chatnoir.eu/en/latest/man/fastwarc.html).
+
+## Cite Us
+
+If you use FastWARC, please consider citing our [OSSYM 2021 abstract paper](https://arxiv.org/abs/2112.03103):
+
+```bibtex
+@InProceedings{bevendorff:2021,
+  author =                {Janek Bevendorff and Martin Potthast and Benno Stein},
+  booktitle =             {3rd International Symposium on Open Search Technology (OSSYM 2021)},
+  editor =                {Andreas Wagner and Christian Guetl and Michael Granitzer and Stefan Voigt},
+  month =                 oct,
+  publisher =             {International Open Search Symposium},
+  site =                  {CERN, Geneva, Switzerland},
+  title =                 {{FastWARC: Optimizing Large-Scale Web Archive Analytics}},
+  year =                  2021
+}
+```
+
+You can also cite our [ECIR 2018 demo paper](https://webis.de/downloads/publications/papers/bevendorff_2018.pdf):
+
+```bibtex
+@InProceedings{bevendorff:2018,
+  address =             {Berlin Heidelberg New York},
+  author =              {Janek Bevendorff and Benno Stein and Matthias Hagen and Martin Potthast},
+  booktitle =           {Advances in Information Retrieval. 40th European Conference on IR Research (ECIR 2018)},
+  editor =              {Leif Azzopardi and Allan Hanbury and Gabriella Pasi and Benjamin Piwowarski},
+  month =               mar,
+  publisher =           {Springer},
+  series =              {Lecture Notes in Computer Science},
+  site =                {Grenoble, France},
+  title =               {{Elastic ChatNoir: Search Engine for the ClueWeb and the Common Crawl}},
+  year =                2018
+}
+```

fastwarc-1.0.2/README.md

@@ -0,0 +1,86 @@
+# FastWARC (Python)
+
+FastWARC is a high-performance WARC parsing library written in Rust with bindings for Python.
+FastWARC supports compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are Gzip,
+Zstd, and LZ4.
+
+This package contains the Python bindings for FastWARC.
+
+FastWARC belongs to the [ChatNoir Resiliparse toolkit](https://github.com/chatnoir-eu/chatnoir-resiliparse/) for fast
+and robust web data processing.
+
+## Installing FastWARC
+
+Pre-built FastWARC binaries for Python can be installed from [PyPi](https://pypi.org/project/FastWARC/):
+
+```bash
+pip install fastwarc
+```
+
+## Building Python Bindings From Source
+
+To be able to build FastWARC, you need the Rust toolchain installed (see above). Then to build directly from the GitHub
+repository, follow these steps:
+
+```bash
+# Clone repository
+
+git clone https://github.com/chatnoir-eu/chatnoir-resiliparse.git
+cd chatnoir-resiliparse
+
+# Install Rust toolchain (if not already installed)
+
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+
+# Create a fresh venv first (recommended)
+
+python3 -m venv venv && source venv/bin/activate
+
+# Option 1: Build and install in editable mode (best for development)
+
+python3 -m pip install -e ./fastwarc-py
+
+# Option 2 (alternative): Build and install wheels in separate steps (best for redistribution)
+
+python3 -m pip wheel -w build ./fastwarc-py
+ls ./build/*.whl | xargs python3 -m pip install
+```
+
+## Usage Instructions
+
+For detailed usage instructions, please consult
+the [FastWARC User Manual](https://resiliparse.chatnoir.eu/en/latest/man/fastwarc.html).
+
+## Cite Us
+
+If you use FastWARC, please consider citing our [OSSYM 2021 abstract paper](https://arxiv.org/abs/2112.03103):
+
+```bibtex
+@InProceedings{bevendorff:2021,
+  author =                {Janek Bevendorff and Martin Potthast and Benno Stein},
+  booktitle =             {3rd International Symposium on Open Search Technology (OSSYM 2021)},
+  editor =                {Andreas Wagner and Christian Guetl and Michael Granitzer and Stefan Voigt},
+  month =                 oct,
+  publisher =             {International Open Search Symposium},
+  site =                  {CERN, Geneva, Switzerland},
+  title =                 {{FastWARC: Optimizing Large-Scale Web Archive Analytics}},
+  year =                  2021
+}
+```
+
+You can also cite our [ECIR 2018 demo paper](https://webis.de/downloads/publications/papers/bevendorff_2018.pdf):
+
+```bibtex
+@InProceedings{bevendorff:2018,
+  address =             {Berlin Heidelberg New York},
+  author =              {Janek Bevendorff and Benno Stein and Matthias Hagen and Martin Potthast},
+  booktitle =           {Advances in Information Retrieval. 40th European Conference on IR Research (ECIR 2018)},
+  editor =              {Leif Azzopardi and Allan Hanbury and Gabriella Pasi and Benjamin Piwowarski},
+  month =               mar,
+  publisher =           {Springer},
+  series =              {Lecture Notes in Computer Science},
+  site =                {Grenoble, France},
+  title =               {{Elastic ChatNoir: Search Engine for the ClueWeb and the Common Crawl}},
+  year =                2018
+}
+```

{fastwarc-1.0.0 → fastwarc-1.0.2}/fastwarc/__init__.py

@@ -13,23 +13,20 @@
 # limitations under the License.
 
 import importlib
-import sys
 
 _native = importlib.import_module("._fastwarc", __name__)
 _legacy_shims = importlib.import_module(".legacy._shims", __name__)
 
-from ._fastwarc import *  # noqa: F401,F403
+import fastwarc.stream_io as stream_io  # noqa: F401
+import fastwarc.warc as warc  # noqa: F401
 
-stream_io = _native.stream_io
-warc = _native.warc
-
-sys.modules[__name__ + ".stream_io"] = stream_io
-sys.modules[__name__ + ".warc"] = warc
+# Re-export module attributes
+globals().update({attr: getattr(_native, attr) for attr in _native.__all__})
 
 # Patch legacy shims onto stream_io
 for name in _legacy_shims.__all__:
     setattr(stream_io, name, getattr(_legacy_shims, name))
-stream_io.__all__ += getattr(_legacy_shims, "__all__", ())
+stream_io.__all__ += getattr(_legacy_shims, "__all__", ())  # type: ignore[attr-defined]
 
 FileStream = _legacy_shims.FileStream
 GZipStream = _legacy_shims.GZipStream
@@ -39,9 +36,9 @@ StreamError = _legacy_shims.StreamError
 
 __all__ = [
     *_native.__all__,
-    "FileStream",
-    "GZipStream",
-    "LZ4Stream",
-    "FastWARCError",
-    "StreamError",
+    'FileStream',
+    'GZipStream',
+    'LZ4Stream',
+    'FastWARCError',
+    'StreamError',
 ]

{fastwarc-1.0.0 → fastwarc-1.0.2}/fastwarc/__init__.pyi

@@ -12,8 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._fastwarc import *
-from .legacy import (
+from .warc import ArchiveIterator, WarcRecord, WarcRecordType
+from .stream_io import (
+    GzipReader,
+    GzipWriter,
+    ZstdReader,
+    ZstdWriter,
+    Lz4Reader,
+    Lz4Writer,
+
+    # Legacy names
     FileStream,
     GZipStream,
     LZ4Stream,

{fastwarc-1.0.0 → fastwarc-1.0.2}/fastwarc/cli.py

@@ -14,6 +14,7 @@
 
 import getpass
 import importlib
+from enum import IntFlag
 from itertools import chain
 import json
 import os
@@ -24,14 +25,13 @@ import urllib.request
 import click
 from tqdm import tqdm
 
-from fastwarc.legacy.stream_io import FileStream, StreamError, FastWARCError, PythonIOStreamAdapter
-from fastwarc.legacy.warc import ArchiveIterator, WarcRecordType
-from fastwarc.legacy.tools import CompressionAlg, detect_compression_algorithm, wrap_warc_stream, \
-    recompress_warc_interactive, verify_digests
+from fastwarc.stream_io import *
+from fastwarc.warc import ArchiveIterator, WarcRecordType
+from fastwarc.legacy.tools import CompressionAlg, detect_compression_algorithm, wrap_warc_stream, verify_digests
 
 
 def exception_handler(exctype, value, _):
-    if exctype == FastWARCError:
+    if exctype == OSError:
         click.echo(str(value), err=True)
         sys.exit(1)
 
@@ -56,46 +56,132 @@ def _human_readable_bytes(byte_num):
         byte_num /= 1024
 
 
+class CompressionAlg(IntFlag):
+    UNCOMPRESSED = 0
+    GZIP = 1
+    ZSTD = 2
+    LZ4 = 3
+    AUTO = 65535
+
+
+def detect_compression_algorithm(infile):
+    """
+    Try to detect the used compression algorithm from the given filename.
+
+    :param infile: filename or file object
+    :return: compression algorithm
+    """
+
+    if os.fspath(infile):
+        infile = str(infile)
+    elif hasattr(infile, 'name'):
+        infile = infile.name
+    else:
+        raise ValueError('Invalid infile type.')
+
+    if type(infile) is str:
+        if infile.endswith('.gz'):
+            return CompressionAlg.GZIP
+        if infile.endswith('.zst'):
+            return CompressionAlg.ZSTD
+        if infile.endswith('.lz4'):
+            return CompressionAlg.LZ4
+        if infile.endswith('.warc'):
+            return CompressionAlg.UNCOMPRESSED
+
+    if not os.path.isfile(infile):
+        raise click.UsageError('Could not determine compression algorithm.')
+
+    with open(infile, 'rb') as infile:
+        magic_bytes = infile.read(5)
+
+    if magic_bytes[:3] == b'\x1F\x8B\x08':
+        return CompressionAlg.GZIP
+    elif magic_bytes[:4] == b'\x28\xB5\x2F\xFD' or magic_bytes[:4] == b'\x5D\x2A\x4D\x18':
+        return CompressionAlg.ZSTD
+    elif magic_bytes[:4] == b'\x04\x22\x4D\x18':
+        return CompressionAlg.LZ4
+    elif magic_bytes == b'WARC/':
+        return CompressionAlg.UNCOMPRESSED
+
+    raise click.FileError('No compression algorithm detected.')
+
+
+def get_compressor(filename, alg, level):
+    if alg == CompressionAlg.AUTO:
+        alg = detect_compression_algorithm(filename)
+    if alg == CompressionAlg.UNCOMPRESSED:
+        return open(filename, 'wb')
+    elif alg == CompressionAlg.GZIP:
+        return GzipWriter(filename, compression_level=level or 9)
+    elif alg == CompressionAlg.ZSTD:
+        return ZstdWriter(filename, compression_level=level or 3)
+    elif alg == CompressionAlg.LZ4:
+        return Lz4Writer(filename)  # compression_level currently unsupported
+    else:
+        raise ValueError(f'Unknown compression algorithm: {alg}')
+
+
+def get_decompressor(filename, decompress_alg):
+    if os.fspath(filename) and not decompress_alg:
+        # Use plain filename to avoid Python IO stack
+        return str(filename)
+
+    if decompress_alg:
+        alg = decompress_alg
+    else:
+        alg = detect_compression_algorithm(filename)
+    if alg == CompressionAlg.UNCOMPRESSED:
+        return open(filename, 'rb')
+    if alg == CompressionAlg.GZIP:
+        return GzipReader(filename)
+    if alg == CompressionAlg.ZSTD:
+        return ZstdReader(filename)
+    if alg == CompressionAlg.LZ4:
+        return Lz4Reader(filename)  # compression_level currently unsupported
+
+    raise ValueError(f'Unknown compression algorithm: {alg}')
+
+
 @main.command()
 @click.argument('infile', type=click.Path(dir_okay=False, exists=True))
 @click.argument('outfile', type=click.Path(dir_okay=False, exists=False))
-@click.option('-c', '--compress-alg', type=click.Choice(['gzip', 'lz4', 'uncompressed', 'auto']),
+@click.option('-c', '--compress-alg', type=click.Choice(['uncompressed', 'gzip', 'zstd', 'lz4', 'auto']),
               default='auto', show_default=True, help='Compression algorithm to use for output file')
-@click.option('-d', '--decompress-alg', type=click.Choice(['gzip', 'lz4', 'uncompressed', 'auto']),
+@click.option('-d', '--decompress-alg', type=click.Choice(['uncompressed', 'gzip', 'zstd', 'lz4', 'auto']),
               default='auto', show_default=True,
               help='Decompression algorithm for decoding input file (auto tries to detect based on file extension)')
-@click.option('-l', '--compress-level', type=int, default=None, help='Compression level (defaults to max)')
+@click.option('-l', '--compress-level', type=int, default=None, help='Compression level')
 @click.option('-q', '--quiet', is_flag=True, help='Do not print progress information')
 def recompress(infile, outfile, compress_alg, decompress_alg, compress_level, quiet):
     """
     Recompress a WARC file.
 
-    This command allows you to recompress a WARC file if it is uncompressed or not compressed
-    properly at the record-level if or you want to recompress a GZip WARC as LZ4 or vice versa.
+    This command allows you to recompress a WARC file if it is uncompressed, not compressed
+    properly at the record-level, or if you want to recompress a Gzip WARC as Zstd or LZ4 or vice versa.
     """
 
-    compress_alg = getattr(CompressionAlg, compress_alg)
-    decompress_alg = getattr(CompressionAlg, decompress_alg)
+    compress_alg = getattr(CompressionAlg, compress_alg.upper())
+    decompress_alg = getattr(CompressionAlg, decompress_alg.upper())
 
-    comp_args = {}
-    if compress_level is None:
-        if compress_alg == CompressionAlg.gzip:
-            comp_args['compression_level'] = 9
-        elif compress_alg == CompressionAlg.lz4:
-            comp_args['compression_level'] = 12
+    infile = get_decompressor(infile, decompress_alg if decompress_alg != CompressionAlg.AUTO else None)
+    outfile_name = outfile
+    outfile = get_compressor(outfile, compress_alg, compress_level)
 
-    bytes_written = 0
     num = 0
-    start = time.time()
+    start = time.monotonic()
     try:
-        for _, b in tqdm(recompress_warc_interactive(infile, outfile, decompress_alg, compress_alg, **comp_args),
-                         desc='Recompressing WARC file', unit=' record(s)', leave=False,
-                         disable=quiet, mininterval=0.2):
-            num += 1
-            bytes_written += b
+        with outfile as out:
+            for r in tqdm(ArchiveIterator(infile),
+                          desc='Recompressing WARC file', unit=' record(s)', leave=False,
+                          disable=quiet, mininterval=0.3):
+                num += 1
+                r.write(out)
+                if compress_alg != CompressionAlg.UNCOMPRESSED:
+                    out.finish()
 
-        if not quiet:
-            click.echo('Recompression completed.')
+            if not quiet:
+                click.echo('Recompression completed.')
     except KeyboardInterrupt:
         if not quiet:
             click.echo('Recompression aborted.')
@@ -103,8 +189,8 @@ def recompress(infile, outfile, compress_alg, decompress_alg, compress_level, qu
     finally:
         if not quiet and num > 0:
             click.echo(f'  - Records recompressed: {num}')
-            click.echo(f'  - Bytes written: {_human_readable_bytes(bytes_written)}')
-            click.echo(f'  - Completed in: {time.time() - start:.02f} seconds')
+            click.echo(f'  - Bytes written: {_human_readable_bytes(os.stat(outfile_name).st_size)}')
+            click.echo(f'  - Completed in: {time.monotonic() - start:.02f} seconds')
 
 
 @main.command()

fastwarc-1.0.2/fastwarc/legacy/__init__.pyi

@@ -0,0 +1,19 @@
+from .stream_io import (
+    FileStream,
+    GZipStream,
+    LZ4Stream,
+    FastWARCError,
+    StreamError,
+)
+from .warc import ArchiveIterator, WarcRecord, WarcRecordType
+
+__all__ = [
+    "FileStream",
+    "GZipStream",
+    "LZ4Stream",
+    "FastWARCError",
+    "StreamError",
+    "ArchiveIterator",
+    "WarcRecord",
+    "WarcRecordType",
+]

{fastwarc-1.0.0 → fastwarc-1.0.2}/fastwarc/legacy/_shims.py

@@ -172,7 +172,7 @@ class BytesIOStream:
 
 class GZipStream:
     def __new__(cls, raw_stream, compression_level=9, zlib=False, fsspec_args=None):
-        from fastwarc.stream_io import GzipReader, GzipWriter
+        from ..stream_io import GzipReader, GzipWriter
         stream = CompressingStream(
             _reader_factory=lambda: GzipReader(raw_stream, zlib=zlib, fsspec_args=fsspec_args),
             _writer_factory=lambda: GzipWriter(raw_stream, compression_level=compression_level, zlib=zlib,
@@ -184,7 +184,7 @@ class GZipStream:
 class LZ4Stream:
     def __new__(cls, raw_stream, mode='r', compression_level=12,
                 favor_dec_speed=True, fsspec_args=None):
-        from fastwarc.stream_io import Lz4Reader, Lz4Writer
+        from ..stream_io import Lz4Reader, Lz4Writer
         stream = CompressingStream(
             _reader_factory=lambda: Lz4Reader(raw_stream, fsspec_args=fsspec_args),
             _writer_factory=lambda: Lz4Writer(raw_stream, fsspec_args=fsspec_args))
@@ -194,7 +194,7 @@ class LZ4Stream:
 
 class BrotliStream:
     def __new__(cls, raw_stream, quality=11, lgwin=22, lgblock=0, fsspec_args=None):
-        from fastwarc.stream_io import BrotliReader, BrotliWriter
+        from ..stream_io import BrotliReader, BrotliWriter
         stream = CompressingStream(
             _reader_factory=lambda: BrotliReader(raw_stream, fsspec_args=fsspec_args),
             _writer_factory=lambda: BrotliWriter(raw_stream, fsspec_args=fsspec_args))