FastWARC 1.0.2__tar.gz → 1.0.4__tar.gz

{fastwarc-1.0.2 → fastwarc-1.0.4}/FastWARC.egg-info/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: FastWARC
-Version: 1.0.2
-Summary: A high-performance WARC parsing library written in Rust with bindings for Python.
+Version: 1.0.4
+Summary: The world's fastest WARC parsing library written in Rust with bindings for Python.
 Author: Janek Bevendorff
 License-Expression: Apache-2.0
 Project-URL: repository, https://github.com/chatnoir-eu/chatnoir-resiliparse
@@ -12,6 +12,7 @@ Description-Content-Type: text/markdown
 Requires-Dist: brotli
 Requires-Dist: click
 Requires-Dist: tqdm
+Requires-Dist: typing_extensions>=4.5; python_version < "3.13"
 Provides-Extra: all
 Requires-Dist: fastwarc[fsspec]; extra == "all"
 Provides-Extra: fsspec

{fastwarc-1.0.2 → fastwarc-1.0.4}/FastWARC.egg-info/requires.txt

@@ -2,6 +2,9 @@ brotli
 click
 tqdm
 
+[:python_version < "3.13"]
+typing_extensions>=4.5
+
 [all]
 fastwarc[fsspec]
 

{fastwarc-1.0.2 → fastwarc-1.0.4}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: FastWARC
-Version: 1.0.2
-Summary: A high-performance WARC parsing library written in Rust with bindings for Python.
+Version: 1.0.4
+Summary: The world's fastest WARC parsing library written in Rust with bindings for Python.
 Author: Janek Bevendorff
 License-Expression: Apache-2.0
 Project-URL: repository, https://github.com/chatnoir-eu/chatnoir-resiliparse
@@ -12,6 +12,7 @@ Description-Content-Type: text/markdown
 Requires-Dist: brotli
 Requires-Dist: click
 Requires-Dist: tqdm
+Requires-Dist: typing_extensions>=4.5; python_version < "3.13"
 Provides-Extra: all
 Requires-Dist: fastwarc[fsspec]; extra == "all"
 Provides-Extra: fsspec

{fastwarc-1.0.2 → fastwarc-1.0.4}/fastwarc/cli.py

@@ -14,7 +14,6 @@
 
 import getpass
 import importlib
-from enum import IntFlag
 from itertools import chain
 import json
 import os
@@ -25,13 +24,14 @@ import urllib.request
 import click
 from tqdm import tqdm
 
-from fastwarc.stream_io import *
+from fastwarc.stream_io import FileStream, StreamError, FastWARCError, PythonIOStreamAdapter
 from fastwarc.warc import ArchiveIterator, WarcRecordType
-from fastwarc.legacy.tools import CompressionAlg, detect_compression_algorithm, wrap_warc_stream, verify_digests
+from fastwarc.legacy.tools import CompressionAlg, detect_compression_algorithm, wrap_warc_stream, \
+    recompress_warc_interactive, verify_digests
 
 
 def exception_handler(exctype, value, _):
-    if exctype == OSError:
+    if exctype == FastWARCError:
         click.echo(str(value), err=True)
         sys.exit(1)
 
@@ -56,132 +56,46 @@ def _human_readable_bytes(byte_num):
         byte_num /= 1024
 
 
-class CompressionAlg(IntFlag):
-    UNCOMPRESSED = 0
-    GZIP = 1
-    ZSTD = 2
-    LZ4 = 3
-    AUTO = 65535
-
-
-def detect_compression_algorithm(infile):
-    """
-    Try to detect the used compression algorithm from the given filename.
-
-    :param infile: filename or file object
-    :return: compression algorithm
-    """
-
-    if os.fspath(infile):
-        infile = str(infile)
-    elif hasattr(infile, 'name'):
-        infile = infile.name
-    else:
-        raise ValueError('Invalid infile type.')
-
-    if type(infile) is str:
-        if infile.endswith('.gz'):
-            return CompressionAlg.GZIP
-        if infile.endswith('.zst'):
-            return CompressionAlg.ZSTD
-        if infile.endswith('.lz4'):
-            return CompressionAlg.LZ4
-        if infile.endswith('.warc'):
-            return CompressionAlg.UNCOMPRESSED
-
-    if not os.path.isfile(infile):
-        raise click.UsageError('Could not determine compression algorithm.')
-
-    with open(infile, 'rb') as infile:
-        magic_bytes = infile.read(5)
-
-    if magic_bytes[:3] == b'\x1F\x8B\x08':
-        return CompressionAlg.GZIP
-    elif magic_bytes[:4] == b'\x28\xB5\x2F\xFD' or magic_bytes[:4] == b'\x5D\x2A\x4D\x18':
-        return CompressionAlg.ZSTD
-    elif magic_bytes[:4] == b'\x04\x22\x4D\x18':
-        return CompressionAlg.LZ4
-    elif magic_bytes == b'WARC/':
-        return CompressionAlg.UNCOMPRESSED
-
-    raise click.FileError('No compression algorithm detected.')
-
-
-def get_compressor(filename, alg, level):
-    if alg == CompressionAlg.AUTO:
-        alg = detect_compression_algorithm(filename)
-    if alg == CompressionAlg.UNCOMPRESSED:
-        return open(filename, 'wb')
-    elif alg == CompressionAlg.GZIP:
-        return GzipWriter(filename, compression_level=level or 9)
-    elif alg == CompressionAlg.ZSTD:
-        return ZstdWriter(filename, compression_level=level or 3)
-    elif alg == CompressionAlg.LZ4:
-        return Lz4Writer(filename)  # compression_level currently unsupported
-    else:
-        raise ValueError(f'Unknown compression algorithm: {alg}')
-
-
-def get_decompressor(filename, decompress_alg):
-    if os.fspath(filename) and not decompress_alg:
-        # Use plain filename to avoid Python IO stack
-        return str(filename)
-
-    if decompress_alg:
-        alg = decompress_alg
-    else:
-        alg = detect_compression_algorithm(filename)
-    if alg == CompressionAlg.UNCOMPRESSED:
-        return open(filename, 'rb')
-    if alg == CompressionAlg.GZIP:
-        return GzipReader(filename)
-    if alg == CompressionAlg.ZSTD:
-        return ZstdReader(filename)
-    if alg == CompressionAlg.LZ4:
-        return Lz4Reader(filename)  # compression_level currently unsupported
-
-    raise ValueError(f'Unknown compression algorithm: {alg}')
-
-
 @main.command()
 @click.argument('infile', type=click.Path(dir_okay=False, exists=True))
 @click.argument('outfile', type=click.Path(dir_okay=False, exists=False))
-@click.option('-c', '--compress-alg', type=click.Choice(['uncompressed', 'gzip', 'zstd', 'lz4', 'auto']),
+@click.option('-c', '--compress-alg', type=click.Choice(['gzip', 'lz4', 'uncompressed', 'auto']),
               default='auto', show_default=True, help='Compression algorithm to use for output file')
-@click.option('-d', '--decompress-alg', type=click.Choice(['uncompressed', 'gzip', 'zstd', 'lz4', 'auto']),
+@click.option('-d', '--decompress-alg', type=click.Choice(['gzip', 'lz4', 'uncompressed', 'auto']),
               default='auto', show_default=True,
               help='Decompression algorithm for decoding input file (auto tries to detect based on file extension)')
-@click.option('-l', '--compress-level', type=int, default=None, help='Compression level')
+@click.option('-l', '--compress-level', type=int, default=None, help='Compression level (defaults to max)')
 @click.option('-q', '--quiet', is_flag=True, help='Do not print progress information')
 def recompress(infile, outfile, compress_alg, decompress_alg, compress_level, quiet):
     """
     Recompress a WARC file.
 
-    This command allows you to recompress a WARC file if it is uncompressed, not compressed
-    properly at the record-level, or if you want to recompress a Gzip WARC as Zstd or LZ4 or vice versa.
+    This command allows you to recompress a WARC file if it is uncompressed or not compressed
+    properly at the record-level if or you want to recompress a GZip WARC as LZ4 or vice versa.
     """
 
-    compress_alg = getattr(CompressionAlg, compress_alg.upper())
-    decompress_alg = getattr(CompressionAlg, decompress_alg.upper())
+    compress_alg = getattr(CompressionAlg, compress_alg)
+    decompress_alg = getattr(CompressionAlg, decompress_alg)
 
-    infile = get_decompressor(infile, decompress_alg if decompress_alg != CompressionAlg.AUTO else None)
-    outfile_name = outfile
-    outfile = get_compressor(outfile, compress_alg, compress_level)
+    comp_args = {}
+    if compress_level is None:
+        if compress_alg == CompressionAlg.gzip:
+            comp_args['compression_level'] = 9
+        elif compress_alg == CompressionAlg.lz4:
+            comp_args['compression_level'] = 12
 
+    bytes_written = 0
     num = 0
-    start = time.monotonic()
+    start = time.time()
     try:
-        with outfile as out:
-            for r in tqdm(ArchiveIterator(infile),
-                          desc='Recompressing WARC file', unit=' record(s)', leave=False,
-                          disable=quiet, mininterval=0.3):
-                num += 1
-                r.write(out)
-                if compress_alg != CompressionAlg.UNCOMPRESSED:
-                    out.finish()
+        for _, b in tqdm(recompress_warc_interactive(infile, outfile, decompress_alg, compress_alg, **comp_args),
+                         desc='Recompressing WARC file', unit=' record(s)', leave=False,
+                         disable=quiet, mininterval=0.2):
+            num += 1
+            bytes_written += b
 
-            if not quiet:
-                click.echo('Recompression completed.')
+        if not quiet:
+            click.echo('Recompression completed.')
     except KeyboardInterrupt:
         if not quiet:
             click.echo('Recompression aborted.')
@@ -189,8 +103,8 @@ def recompress(infile, outfile, compress_alg, decompress_alg, compress_level, qu
     finally:
         if not quiet and num > 0:
             click.echo(f'  - Records recompressed: {num}')
-            click.echo(f'  - Bytes written: {_human_readable_bytes(os.stat(outfile_name).st_size)}')
-            click.echo(f'  - Completed in: {time.monotonic() - start:.02f} seconds')
+            click.echo(f'  - Bytes written: {_human_readable_bytes(bytes_written)}')
+            click.echo(f'  - Completed in: {time.time() - start:.02f} seconds')
 
 
 @main.command()
@@ -525,8 +439,9 @@ def benchmark(input_url, decompress_alg, endpoint_url, aws_access_key, aws_secre
         return num, time.monotonic() - start, interrupted
 
     def _fastwarc_iterator(f):
-        s = _get_raw_stream_from_url(f, use_python_stream)
-        s = wrap_warc_stream(s, 'rb', decompress_alg)
+        # s = _get_raw_stream_from_url(f, use_python_stream)
+        # s = wrap_warc_stream(s, 'rb', decompress_alg)
+        s = '/home/roce3528/code-in-progress/code-research/web-search/chatnoir/chatnoir-resiliparse/tmpfs/CC-MAIN-20231005012006-20231005042006-00899.warc'
         return ArchiveIterator(s, rec_type_filter, parse_http=parse_http, verify_digests=verify_digests)
 
     n, t_fastwarc, interrupted = _bench(input_url, _fastwarc_iterator, 'FastWARC')

{fastwarc-1.0.2 → fastwarc-1.0.4}/fastwarc/legacy/_shims.py

@@ -14,7 +14,10 @@
 
 # Legacy shims for making the new Rust extension work with the old type names
 
-import warnings
+try:
+    from warnings import deprecated
+except ImportError:
+    from typing_extensions import deprecated
 
 __all__ = [
     'wrap_stream',
@@ -32,19 +35,21 @@ __all__ = [
     'ReaderStaleError',
 ]
 
-FastWARCError = OSError
 
-StreamError = OSError
+@deprecated("Use OSError instead.", category=FutureWarning)
+class FastWARCError(OSError): ...
 
-ReaderStaleError = OSError
 
+@deprecated("Use OSError instead.", category=FutureWarning)
+class StreamError(OSError): ...
 
-def _warn_deprecated(msg):
-    warnings.warn(msg, FutureWarning, 3)
 
+@deprecated("Use OSError instead.", category=FutureWarning)
+class ReaderStaleError(OSError): ...
 
+
+@deprecated("wrap_stream is deprecated and will be removed in a future version.", category=FutureWarning)
 def wrap_stream(raw_stream, mode='rb', fsspec_args=None):
-    _warn_deprecated("wrap_stream is deprecated and will be removed in a future version.")
     if isinstance(raw_stream, str):
         if fsspec_args is not False and '://' in raw_stream:
             try:
@@ -57,9 +62,9 @@ def wrap_stream(raw_stream, mode='rb', fsspec_args=None):
     return raw_stream
 
 
+@deprecated("BufferedReader is deprecated and will be removed in a future version.", category=FutureWarning)
 class BufferedReader:
     def __init__(self, stream, *args, **kwargs):
-        _warn_deprecated("BufferedReader is deprecated and will be removed in a future version.")
         self._stream = stream
 
     def close(self):
@@ -78,12 +83,9 @@ class BufferedReader:
         return self._stream.tell()
 
 
+@deprecated("Use the new Reader and Writer classes from stream_io instead.", category=FutureWarning)
 class IOStream:
     def __init__(self, *_, _reader_factory=None, _writer_factory=None, **__):
-        _warn_deprecated(
-            f"{self.__class__.__name__} is deprecated and will be removed in a future version. "
-            "Use the new Reader and Writer classes from stream_io instead.")
-
         self._reader_factory = _reader_factory
         self._writer_factory = _writer_factory
         self._reader = None
@@ -140,6 +142,7 @@ class IOStream:
         s.close()
 
 
+@deprecated("Use the new Reader and Writer classes from stream_io instead.", category=FutureWarning)
 class CompressingStream(IOStream):
     def begin_member(self):
         pass
@@ -149,12 +152,14 @@ class CompressingStream(IOStream):
         self._writer = None
 
 
+@deprecated("Use the plain Python stream object without any wrapper.", category=FutureWarning)
 class PythonIOStreamAdapter:
     def __new__(cls, py_stream):
         return IOStream(_reader_factory=lambda: py_stream,
                         _writer_factory=lambda: py_stream)
 
 
+@deprecated("Use plain file paths instead.", category=FutureWarning)
 class FileStream:
     def __new__(cls, filename: str, mode: str = 'rb'):
         if 'b' not in mode:
@@ -163,6 +168,7 @@ class FileStream:
         return IOStream(_reader_factory=lambda: f, _writer_factory=lambda: f)
 
 
+@deprecated("Use io.BytesIO instead.", category=FutureWarning)
 class BytesIOStream:
     def __new__(cls, initial_data=None):
         import io
@@ -170,6 +176,7 @@ class BytesIOStream:
         return IOStream(_reader_factory=lambda: bio, _writer_factory=lambda: bio)
 
 
+@deprecated("Use the new GzipReader and GzipWriter classes from stream_io instead.", category=FutureWarning)
 class GZipStream:
     def __new__(cls, raw_stream, compression_level=9, zlib=False, fsspec_args=None):
         from ..stream_io import GzipReader, GzipWriter
@@ -181,6 +188,7 @@ class GZipStream:
         return stream
 
 
+@deprecated("Use the new Lz4Reader and Lz4Writer classes from stream_io instead.", category=FutureWarning)
 class LZ4Stream:
     def __new__(cls, raw_stream, mode='r', compression_level=12,
                 favor_dec_speed=True, fsspec_args=None):
@@ -192,6 +200,7 @@ class LZ4Stream:
         return stream
 
 
+@deprecated("Use the new BrotliReader and BrotliWriter classes from stream_io instead.", category=FutureWarning)
 class BrotliStream:
     def __new__(cls, raw_stream, quality=11, lgwin=22, lgblock=0, fsspec_args=None):
         from ..stream_io import BrotliReader, BrotliWriter

{fastwarc-1.0.2 → fastwarc-1.0.4}/fastwarc/stream_io.pyi

@@ -128,18 +128,19 @@ _WriterInput = WarcWriter | BinaryIO | _GenericWriter | PathLike[str] | str
 
 @disjoint_base
 class GzipReader(WarcReader):
-    def __new__(cls, inner: _ReaderInput, buffer_size: int = 4096, zlib: bool = False, fsspec_args=None) -> Self: ...
+    def __new__(cls, inner: _ReaderInput, buffer_size: int = 64 << 10, zlib: bool = False,
+                fsspec_args=None) -> Self: ...
 
 
 @disjoint_base
 class GzipWriter(WarcWriter):
-    def __new__(cls, inner: _WriterInput, compression_level: int = 9, buffer_size=8192, zlib=False,
+    def __new__(cls, inner: _WriterInput, compression_level: int = 9, buffer_size: int = 8192, zlib=False,
                 fsspec_args=None) -> Self: ...
 
 
 @disjoint_base
 class ZstdReader(WarcReader):
-    def __new__(cls, inner: _ReaderInput, buffer_size: int = 4096, fsspec_args=None,
+    def __new__(cls, inner: _ReaderInput, buffer_size: int = 64 << 10, fsspec_args=None,
                 dictionary: bytes | None = None) -> Self: ...
 
 
@@ -151,7 +152,7 @@ class ZstdWriter(WarcWriter):
 
 @disjoint_base
 class Lz4Reader(WarcReader):
-    def __new__(cls, inner: _ReaderInput, buffer_size: int = 4096, fsspec_args=None) -> Self: ...
+    def __new__(cls, inner: _ReaderInput, buffer_size: int = 64 << 10, fsspec_args=None) -> Self: ...
 
 
 @disjoint_base
@@ -161,7 +162,7 @@ class Lz4Writer(WarcWriter):
 
 @disjoint_base
 class BrotliReader(WarcReader):
-    def __new__(cls, inner: _ReaderInput, buffer_size: int = 4096, fsspec_args=None) -> Self: ...
+    def __new__(cls, inner: _ReaderInput, buffer_size: int = 64 << 10, fsspec_args=None) -> Self: ...
 
 
 @disjoint_base
@@ -171,12 +172,12 @@ class BrotliWriter(WarcWriter):
 
 @disjoint_base
 class ChunkedReader(WarcReader):
-    def __new__(cls, inner: _ReaderInput, buffer_size: int = 1024, fsspec_args=None) -> Self: ...
+    def __new__(cls, inner: _ReaderInput, buffer_size: int = 4096, fsspec_args=None) -> Self: ...
 
 
 @disjoint_base
 class ChunkedWriter(WarcWriter):
-    def __new__(cls, inner: _WriterInput, min_chunk_size: int = 512, fsspec_args=None) -> Self: ...
+    def __new__(cls, inner: _WriterInput, min_chunk_size: int = 1024, fsspec_args=None) -> Self: ...
 
 
 def zstd_train_dictionary_from_continuous(sample_data: bytes, sample_sizes: list[int], max_size: int) -> bytes: ...

{fastwarc-1.0.2 → fastwarc-1.0.4}/fastwarc/warc.pyi

@@ -16,7 +16,7 @@ from datetime import datetime
 from collections.abc import Callable, Iterable, Iterator
 from enum import IntFlag
 from typing import BinaryIO, final, Literal, Self
-from typing_extensions import disjoint_base
+from typing_extensions import deprecated, disjoint_base
 
 from .stream_io import _GenericReader, _GenericWriter, _ReaderInput, WarcReader, WarcWriter
 
@@ -95,8 +95,7 @@ no_type = WarcRecordType.no_type
 any_type = WarcRecordType.any_type
 
 
-@final
-class HeaderMap:
+class _HeaderMap:
     def __new__(cls, encoding: str = "utf-8") -> Self: ...
 
     @property
@@ -127,12 +126,12 @@ class HeaderMap:
 
     def to_dict(self) -> dict[str, str]: ...
 
-    # deprecated
+    @deprecated("Use to_dict() instead.")
     def asdict(self) -> dict[str, str]: ...
 
     def to_tuples(self) -> tuple[tuple[str, str], ...]: ...
 
-    # deprecated
+    @deprecated("Use to_tuples() instead.")
     def astuples(self) -> tuple[tuple[str, str], ...]: ...
 
     def is_empty(self) -> bool: ...
@@ -198,8 +197,13 @@ class HeaderMap:
     def __setstate__(self, state: tuple[bytes, bool]): ...
 
 
-# Legacy name
-WarcHeaderMap = HeaderMap
+@final
+class HeaderMap(_HeaderMap): ...
+
+
+@deprecated("Use HeaderMap instead.")
+@final
+class WarcHeaderMap(_HeaderMap): ...
 
 
 @final
@@ -287,10 +291,10 @@ class WarcRecord:
 
     def consume(self, n: int | None = None) -> int: ...
 
-    def parse_warc_headers(self, quirks_mode: bool = False) -> int: ...
+    def parse_warc_headers(self, quirks_mode: bool = False, max_header_len: int = 32 << 10) -> int: ...
 
-    def parse_http(self, auto_decode: Literal['none', 'content', 'transfer', 'all'] = 'none', *,
-                   strict_mode: bool = True): ...
+    def parse_http(self, auto_decode: Literal['none', 'content', 'transfer', 'all'] = 'none',
+                   max_header_len: int = 32 << 10, quirks_mode: bool = False, *, strict_mode: bool = True): ...
 
     def verify_block_digest(self, consume: bool = False) -> bool: ...
 
@@ -332,7 +336,10 @@ class ArchiveIterator(Iterable[WarcRecord]):
             verify_digests: bool = False,
             quirks_mode: bool = False,
             auto_decode: Literal['none', 'content', 'transfer', 'all'] = 'none',
+            max_header_len: int = 32 << 10,
             stream_detect: bool = True,
+            buffer_size: int = 64 << 10,
+            inplace: bool = False,
             fsspec_args=None,
             *,
             strict_mode: bool = True

{fastwarc-1.0.2 → fastwarc-1.0.4}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "FastWARC"
-version = "1.0.2"
-description = "A high-performance WARC parsing library written in Rust with bindings for Python."
+version = "1.0.4"
+description = "The world's fastest WARC parsing library written in Rust with bindings for Python."
 authors = [
     { name = "Janek Bevendorff" }
 ]
@@ -15,7 +15,8 @@ requires-python = ">=3.10"
 dependencies = [
     "brotli",
     "click",
-    "tqdm"
+    "tqdm",
+    "typing_extensions>=4.5; python_version < '3.13'",
 ]
 
 [project.urls]