FastWARC 1.0.0__tar.gz → 1.0.1__tar.gz

{fastwarc-1.0.0 → fastwarc-1.0.1}/FastWARC.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: FastWARC
-Version: 1.0.0
+Version: 1.0.1
 Summary: A high-performance WARC parsing library for Python written in C++/Cython.
 Author: Janek Bevendorff
-License: Apache License 2.0
+License-Expression: Apache-2.0
 Project-URL: repository, https://github.com/chatnoir-eu/chatnoir-resiliparse
 Project-URL: homepage, https://resiliparse.chatnoir.eu
 Project-URL: documentation, https://resiliparse.chatnoir.eu
@@ -25,7 +25,7 @@ Requires-Dist: lz4; extra == "test"
 # FastWARC (Python)
 
 FastWARC is a high-performance WARC parsing library for Python written in Rust with Python bindings. FastWARC supports
-compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are GZip and LZ4.
+compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are Gzip and LZ4.
 
 This crate contains the Python bindings for FastWARC.
 

{fastwarc-1.0.0 → fastwarc-1.0.1}/FastWARC.egg-info/SOURCES.txt

@@ -10,7 +10,6 @@ FastWARC.egg-info/requires.txt
 FastWARC.egg-info/top_level.txt
 fastwarc/__init__.py
 fastwarc/__init__.pyi
-fastwarc/_fastwarc.pyi
 fastwarc/cli.py
 fastwarc/py.typed
 fastwarc/stream_io.pyi

{fastwarc-1.0.0 → fastwarc-1.0.1}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: FastWARC
-Version: 1.0.0
+Version: 1.0.1
 Summary: A high-performance WARC parsing library for Python written in C++/Cython.
 Author: Janek Bevendorff
-License: Apache License 2.0
+License-Expression: Apache-2.0
 Project-URL: repository, https://github.com/chatnoir-eu/chatnoir-resiliparse
 Project-URL: homepage, https://resiliparse.chatnoir.eu
 Project-URL: documentation, https://resiliparse.chatnoir.eu
@@ -25,7 +25,7 @@ Requires-Dist: lz4; extra == "test"
 # FastWARC (Python)
 
 FastWARC is a high-performance WARC parsing library for Python written in Rust with Python bindings. FastWARC supports
-compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are GZip and LZ4.
+compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are Gzip and LZ4.
 
 This crate contains the Python bindings for FastWARC.
 

{fastwarc-1.0.0 → fastwarc-1.0.1}/README.md

@@ -1,7 +1,7 @@
 # FastWARC (Python)
 
 FastWARC is a high-performance WARC parsing library for Python written in Rust with Python bindings. FastWARC supports
-compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are GZip and LZ4.
+compressed and uncompressed WARC/1.0 and WARC/1.1 streams. Supported compression algorithms are Gzip and LZ4.
 
 This crate contains the Python bindings for FastWARC.
 

{fastwarc-1.0.0 → fastwarc-1.0.1}/fastwarc/__init__.py

@@ -13,23 +13,20 @@
 # limitations under the License.
 
 import importlib
-import sys
 
 _native = importlib.import_module("._fastwarc", __name__)
 _legacy_shims = importlib.import_module(".legacy._shims", __name__)
 
-from ._fastwarc import *  # noqa: F401,F403
+import fastwarc.stream_io as stream_io  # noqa: F401
+import fastwarc.warc as warc  # noqa: F401
 
-stream_io = _native.stream_io
-warc = _native.warc
-
-sys.modules[__name__ + ".stream_io"] = stream_io
-sys.modules[__name__ + ".warc"] = warc
+# Re-export module attributes
+globals().update({attr: getattr(_native, attr) for attr in _native.__all__})
 
 # Patch legacy shims onto stream_io
 for name in _legacy_shims.__all__:
     setattr(stream_io, name, getattr(_legacy_shims, name))
-stream_io.__all__ += getattr(_legacy_shims, "__all__", ())
+stream_io.__all__ += getattr(_legacy_shims, "__all__", ())  # type: ignore[attr-defined]
 
 FileStream = _legacy_shims.FileStream
 GZipStream = _legacy_shims.GZipStream
@@ -39,9 +36,9 @@ StreamError = _legacy_shims.StreamError
 
 __all__ = [
     *_native.__all__,
-    "FileStream",
-    "GZipStream",
-    "LZ4Stream",
-    "FastWARCError",
-    "StreamError",
+    'FileStream',
+    'GZipStream',
+    'LZ4Stream',
+    'FastWARCError',
+    'StreamError',
 ]

{fastwarc-1.0.0 → fastwarc-1.0.1}/fastwarc/__init__.pyi

@@ -12,8 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._fastwarc import *
-from .legacy import (
+from .warc import ArchiveIterator, WarcRecord, WarcRecordType
+from .stream_io import (
+    GzipReader,
+    GzipWriter,
+    ZstdReader,
+    ZstdWriter,
+    Lz4Reader,
+    Lz4Writer,
+
+    # Legacy names
     FileStream,
     GZipStream,
     LZ4Stream,

{fastwarc-1.0.0 → fastwarc-1.0.1}/fastwarc/legacy/_shims.py

@@ -172,7 +172,7 @@ class BytesIOStream:
 
 class GZipStream:
     def __new__(cls, raw_stream, compression_level=9, zlib=False, fsspec_args=None):
-        from fastwarc.stream_io import GzipReader, GzipWriter
+        from ..stream_io import GzipReader, GzipWriter
         stream = CompressingStream(
             _reader_factory=lambda: GzipReader(raw_stream, zlib=zlib, fsspec_args=fsspec_args),
             _writer_factory=lambda: GzipWriter(raw_stream, compression_level=compression_level, zlib=zlib,
@@ -184,7 +184,7 @@ class GZipStream:
 class LZ4Stream:
     def __new__(cls, raw_stream, mode='r', compression_level=12,
                 favor_dec_speed=True, fsspec_args=None):
-        from fastwarc.stream_io import Lz4Reader, Lz4Writer
+        from ..stream_io import Lz4Reader, Lz4Writer
         stream = CompressingStream(
             _reader_factory=lambda: Lz4Reader(raw_stream, fsspec_args=fsspec_args),
             _writer_factory=lambda: Lz4Writer(raw_stream, fsspec_args=fsspec_args))
@@ -194,7 +194,7 @@ class LZ4Stream:
 
 class BrotliStream:
     def __new__(cls, raw_stream, quality=11, lgwin=22, lgblock=0, fsspec_args=None):
-        from fastwarc.stream_io import BrotliReader, BrotliWriter
+        from ..stream_io import BrotliReader, BrotliWriter
         stream = CompressingStream(
             _reader_factory=lambda: BrotliReader(raw_stream, fsspec_args=fsspec_args),
             _writer_factory=lambda: BrotliWriter(raw_stream, fsspec_args=fsspec_args))

fastwarc-1.0.1/fastwarc/stream_io.pyi

@@ -0,0 +1,187 @@
+# Copyright 2026 Janek Bevendorff
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from os import PathLike
+from types import TracebackType
+from typing import BinaryIO, ContextManager, Protocol, Type, Self
+from typing_extensions import disjoint_base
+
+# Legacy shims
+from .legacy._shims import *
+
+__all__ = [
+    'WarcReader',
+    'WarcWriter',
+    'GzipReader',
+    'GzipWriter',
+    'ZstdReader',
+    'ZstdWriter',
+    'Lz4Reader',
+    'Lz4Writer',
+    'BrotliReader',
+    'BrotliWriter',
+    'ChunkedReader',
+    'ChunkedWriter',
+    'zstd_train_dictionary_from_continuous',
+    'zstd_train_dictionary_from_files',
+    'zstd_train_dictionary_from_samples',
+
+    # Legacy names
+    'BrotliStream',
+    'BufferedReader',
+    'BytesIOStream',
+    'CompressingStream',
+    'FastWARCError',
+    'FileStream',
+    'GZipStream',
+    'IOStream',
+    'LZ4Stream',
+    'PythonIOStreamAdapter',
+    'ReaderStaleError',
+    'StreamError',
+    'wrap_stream'
+]
+
+
+class _GenericReader(Protocol):
+    def read(self, size: int = ...) -> bytes: ...
+
+    def seek(self, offset: int, whence: int = ...) -> int: ...
+
+    def tell(self) -> int: ...
+
+    def close(self) -> None: ...
+
+
+class _GenericWriter(Protocol):
+    def write(self, data: bytes) -> int: ...
+
+    def flush(self) -> None: ...
+
+    def close(self) -> None: ...
+
+
+@disjoint_base
+class WarcReader(ContextManager["WarcReader"]):
+    def __new__(cls) -> Self: ...
+
+    def read(self, size: int = ...) -> bytes: ...
+
+    def seek(self, offset: int, whence: int = 0) -> int: ...
+
+    def tell(self) -> int: ...
+
+    def close(self) -> None: ...
+
+    def __enter__(self) -> Self: ...
+
+    def __exit__(
+            self,
+            exc_type: Type[BaseException] | None,
+            exc: BaseException | None,
+            traceback: TracebackType | None
+    ) -> None: ...
+
+    def inner_seek(self, offset: int, whence: int = 0) -> int: ...
+
+    def inner_tell(self) -> int: ...
+
+    def frame_start_position(self) -> int | None: ...
+
+
+@disjoint_base
+class WarcWriter(ContextManager["WarcWriter"]):
+    def __new__(cls) -> Self: ...
+
+    def write(self, data: bytes) -> int: ...
+
+    def flush(self) -> None: ...
+
+    def close(self) -> None: ...
+
+    def __enter__(self) -> Self: ...
+
+    def __exit__(
+            self,
+            exc_type: Type[BaseException] | None,
+            exc: BaseException | None,
+            traceback: TracebackType | None
+    ) -> None: ...
+
+    def finish(self) -> None: ...
+
+
+_ReaderInput = WarcReader | BinaryIO | _GenericReader | PathLike[str] | str
+_WriterInput = WarcWriter | BinaryIO | _GenericWriter | PathLike[str] | str
+
+
+@disjoint_base
+class GzipReader(WarcReader):
+    def __new__(cls, inner: _ReaderInput, buffer_size=4096, zlib=False, fsspec_args=None) -> Self: ...
+
+
+@disjoint_base
+class GzipWriter(WarcWriter):
+    def __new__(cls, inner: _WriterInput, compression_level=9, buffer_size=8192, zlib=False,
+                fsspec_args=None) -> Self: ...
+
+
+@disjoint_base
+class ZstdReader(WarcReader):
+    def __new__(cls, inner: _ReaderInput, buffer_size=4096, fsspec_args=None, dictionary=None) -> Self: ...
+
+
+@disjoint_base
+class ZstdWriter(WarcWriter):
+    def __new__(cls, inner: _WriterInput, buffer_size=8192, fsspec_args=None, dictionary=None,
+                compress_dictionary_frame=False) -> Self: ...
+
+
+@disjoint_base
+class Lz4Reader(WarcReader):
+    def __new__(cls, inner: _ReaderInput, buffer_size=4096, fsspec_args=None) -> Self: ...
+
+
+@disjoint_base
+class Lz4Writer(WarcWriter):
+    def __new__(cls, inner: _WriterInput, buffer_size=8192, fsspec_args=None) -> Self: ...
+
+
+@disjoint_base
+class BrotliReader(WarcReader):
+    def __new__(cls, inner: _ReaderInput, buffer_size=4096, fsspec_args=None) -> Self: ...
+
+
+@disjoint_base
+class BrotliWriter(WarcWriter):
+    def __new__(cls, inner: _WriterInput, buffer_size=8192, fsspec_args=None) -> Self: ...
+
+
+@disjoint_base
+class ChunkedReader(WarcReader):
+    def __new__(cls, inner: _ReaderInput, buffer_size=1024, fsspec_args=None) -> Self: ...
+
+
+@disjoint_base
+class ChunkedWriter(WarcWriter):
+    def __new__(cls, inner: _WriterInput, min_chunk_size=512, fsspec_args=None) -> Self: ...
+
+
+def zstd_train_dictionary_from_continuous(sample_data: bytes, sample_sizes: list[int], max_size: int) -> bytes: ...
+
+
+def zstd_train_dictionary_from_samples(samples: list[bytes], max_size: int) -> bytes: ...
+
+
+def zstd_train_dictionary_from_files(filenames: list[str], max_size: int) -> bytes: ...

{fastwarc-1.0.0 → fastwarc-1.0.1}/fastwarc/warc.pyi

@@ -13,13 +13,45 @@
 # limitations under the License.
 
 from datetime import datetime
+from collections.abc import Callable, Iterable, Iterator
 from enum import IntFlag
-from os import PathLike
-from typing import BinaryIO, Callable, Dict, Iterable, Iterator, Literal, Optional, Self, Tuple, Union
-
-from .stream_io import _GenericReader, _GenericWriter, WarcReader, WarcWriter
-
-
+from typing import BinaryIO, final, Literal, Self
+from typing_extensions import disjoint_base
+
+from .stream_io import _GenericReader, _GenericWriter, _ReaderInput, WarcReader, WarcWriter
+
+__all__ = [
+    'WarcRecordType',
+    'WarcRecord',
+    'ArchiveIterator',
+    'HeaderMap',
+    'WarcHeaderMap',
+    'WarcRecordPayloadReader',
+
+    'warcinfo',
+    'response',
+    'resource',
+    'request',
+    'metadata',
+    'revisit',
+    'conversion',
+    'continuation',
+    'unknown',
+    'any_type',
+    'no_type',
+
+    'is_warc_10',
+    'is_warc_11',
+    'has_block_digest',
+    'has_valid_block_digest',
+    'has_payload_digest',
+    'has_valid_payload_digest',
+    'is_http',
+    'is_concurrent',
+]
+
+
+@final
 class WarcRecordType(IntFlag):
     warcinfo = 2
     response = 4
@@ -37,21 +69,17 @@ class WarcRecordType(IntFlag):
 
     def __index__(self) -> int: ...
 
-    def __and__(self, other: int) -> int: ...
-
-    def __rand__(self, other: int) -> int: ...
+    def __and__(self, other: int, /) -> Self: ...
 
-    def __or__(self, other: int) -> int: ...
+    def __or__(self, other: int, /) -> Self: ...
 
-    def __ror__(self, other: int) -> int: ...
+    def __xor__(self, other: int, /) -> Self: ...
 
-    def __xor__(self, other: int) -> int: ...
+    def __str__(self) -> str:
+        return self.name or str(self.value)
 
-    def __rxor__(self, other: int) -> int: ...
-
-    def __str__(self) -> str: ...
-
-    def __repr__(self) -> str: ...
+    def __repr__(self) -> str:
+        return self.name or str(self.value)
 
 
 warcinfo = WarcRecordType.warcinfo
@@ -67,56 +95,57 @@ no_type = WarcRecordType.no_type
 any_type = WarcRecordType.any_type
 
 
+@final
 class HeaderMap:
     def __new__(cls, encoding: str = "utf-8") -> Self: ...
 
     @property
-    def reason_phrase(self) -> Optional[str]: ...
+    def reason_phrase(self) -> str | None: ...
 
     @property
-    def status_code(self) -> Optional[int]: ...
+    def status_code(self) -> int | None: ...
 
     @property
-    def status_line(self) -> Optional[str]: ...
+    def status_line(self) -> str | None: ...
 
     @status_line.setter
     def status_line(self, status_line: str): ...
 
     @property
-    def status_line_bytes(self) -> Optional[bytes]: ...
+    def status_line_bytes(self) -> bytes | None: ...
 
     @status_line_bytes.setter
     def status_line_bytes(self, status_line: bytes): ...
 
     def encoding(self) -> str: ...
 
-    def parse(self, reader: Union[WarcReader, BinaryIO, _GenericReader], has_status_line: bool = True) -> int: ...
+    def parse(self, reader: WarcReader | BinaryIO | _GenericReader, has_status_line: bool = True) -> int: ...
 
-    def write(self, writer: Union[WarcWriter, BinaryIO, _GenericWriter]) -> int: ...
+    def write(self, writer: WarcWriter | BinaryIO | _GenericWriter) -> int: ...
 
     def append(self, key: str, value: str): ...
 
-    def to_dict(self) -> Dict[str, str]: ...
+    def to_dict(self) -> dict[str, str]: ...
 
     # deprecated
-    def asdict(self) -> Dict[str, str]: ...
+    def asdict(self) -> dict[str, str]: ...
 
-    def to_tuples(self) -> Tuple[Tuple[str, str], ...]: ...
+    def to_tuples(self) -> tuple[tuple[str, str], ...]: ...
 
     # deprecated
-    def astuples(self) -> Tuple[Tuple[str, str], ...]: ...
+    def astuples(self) -> tuple[tuple[str, str], ...]: ...
 
     def is_empty(self) -> bool: ...
 
     def clear(self): ...
 
-    def get(self, key: str, default: Optional[str] = None) -> Optional[str]: ...
+    def get(self, key: str, default: str | None = None) -> str | None: ...
 
-    def get_multiple(self, key: str) -> Tuple[str, ...]: ...
+    def get_multiple(self, key: str) -> tuple[str, ...]: ...
 
-    def get_bytes(self, key: bytes, default: Optional[bytes] = None) -> Optional[bytes]: ...
+    def get_bytes(self, key: bytes, default: bytes | None = None) -> bytes | None: ...
 
-    def get_bytes_multiple(self, key: bytes) -> Tuple[bytes, ...]: ...
+    def get_bytes_multiple(self, key: bytes) -> tuple[bytes, ...]: ...
 
     def contains_key(self, key: str) -> bool: ...
 
@@ -132,58 +161,60 @@ class HeaderMap:
 
     def remove_bytes(self, key: bytes): ...
 
-    def items(self) -> Tuple[Tuple[str, str], ...]: ...
+    def items(self) -> tuple[tuple[str, str], ...]: ...
 
-    def items_bytes(self) -> Tuple[Tuple[bytes, bytes], ...]: ...
+    def items_bytes(self) -> tuple[tuple[bytes, bytes], ...]: ...
 
-    def keys(self) -> Tuple[str, ...]: ...
+    def keys(self) -> tuple[str, ...]: ...
 
-    def keys_bytes(self) -> Tuple[bytes, ...]: ...
+    def keys_bytes(self) -> tuple[bytes, ...]: ...
 
-    def values(self) -> Tuple[str, ...]: ...
+    def values(self) -> tuple[str, ...]: ...
 
-    def values_bytes(self) -> Tuple[bytes, ...]: ...
+    def values_bytes(self) -> tuple[bytes, ...]: ...
 
-    def __getitem__(self, item: str) -> str: ...
+    def __getitem__(self, item: str, /) -> str: ...
 
-    def __iter__(self) -> Iterator[Tuple[str, str]]: ...
+    def __iter__(self) -> Iterator[tuple[str, str]]: ...
 
     def __len__(self) -> int: ...
 
-    def __setitem__(self, key: str, value: str): ...
+    def __setitem__(self, key: str, value: str, /): ...
 
-    def __contains__(self, item: str) -> bool: ...
+    def __contains__(self, item: str, /) -> bool: ...
 
-    def __eq__(self, other: HeaderMap) -> bool: ...
+    def __eq__(self, other, /) -> bool: ...
 
-    def __ne__(self, other: HeaderMap) -> bool: ...
+    def __ne__(self, other, /) -> bool: ...
 
     def __str__(self) -> str: ...
 
     def __repr__(self) -> str: ...
 
-    def __getnewargs__(self) -> Tuple[str]: ...
+    def __getnewargs__(self) -> tuple[str]: ...
 
-    def __getstate__(self) -> Tuple[bytes, bool]: ...
+    def __getstate__(self) -> tuple[bytes, bool]: ...
 
-    def __setstate__(self, state: Tuple[bytes, bool]): ...
+    def __setstate__(self, state: tuple[bytes, bool]): ...
 
 
 # Legacy name
 WarcHeaderMap = HeaderMap
 
 
+@final
 class WarcRecordPayloadReader(WarcReader):
     def readline(self, max_line_len: int = 8192) -> bytes: ...
 
-    def consume(self, size: int = -1) -> int: ...
+    def consume(self, size: int = ...) -> int: ...
 
 
+@disjoint_base
 class WarcRecord:
     def __new__(cls) -> Self: ...
 
     @property
-    def record_id(self) -> Optional[str]: ...
+    def record_id(self) -> str | None: ...
 
     @record_id.setter
     def record_id(self, record_id: str): ...
@@ -192,13 +223,13 @@ class WarcRecord:
     def record_type(self) -> WarcRecordType: ...
 
     @record_type.setter
-    def record_type(self, record_type: Union[WarcRecordType, int]): ...
+    def record_type(self, record_type: WarcRecordType | int): ...
 
     @property
     def content_length(self) -> int: ...
 
     @property
-    def record_date(self) -> Optional[datetime]: ...
+    def record_date(self) -> datetime: ...
 
     @record_date.setter
     def record_date(self, record_date: datetime): ...
@@ -216,22 +247,22 @@ class WarcRecord:
     def is_http_parsed(self) -> bool: ...
 
     @property
-    def http_headers(self) -> Optional[HeaderMap]: ...
+    def http_headers(self) -> HeaderMap: ...
 
     @property
-    def http_content_type(self) -> Optional[str]: ...
+    def http_content_type(self) -> str | None: ...
 
     @property
-    def http_charset(self) -> Optional[str]: ...
+    def http_charset(self) -> str | None: ...
 
     @property
-    def http_date(self) -> Optional[datetime]: ...
+    def http_date(self) -> datetime: ...
 
     @property
-    def http_last_modified(self) -> Optional[datetime]: ...
+    def http_last_modified(self) -> datetime: ...
 
     @property
-    def reader(self) -> Optional[WarcRecordPayloadReader]: ...
+    def reader(self) -> WarcRecordPayloadReader: ...
 
     @property
     def stream_pos(self) -> int: ...
@@ -239,9 +270,8 @@ class WarcRecord:
     @property
     def is_frozen(self) -> bool: ...
 
-    def init_headers(
-            self, content_length: int = 0, record_type: WarcRecordType = no_type, record_urn: Optional[bytes] = None
-    ): ...
+    def init_headers(self, record_type: WarcRecordType = ..., record_urn: bytes | None = None, *,
+                     content_length: int | None = None): ...
 
     def freeze(self) -> bool: ...
 
@@ -249,11 +279,12 @@ class WarcRecord:
 
     def set_bytes_payload(self, content: bytes): ...
 
-    def consume(self, n: Optional[int] = None) -> int: ...
+    def consume(self, n: int | None = None) -> int: ...
 
     def parse_warc_headers(self, quirks_mode: bool = False) -> int: ...
 
-    def parse_http(self, auto_decode: Literal['none', 'content', 'transfer', 'all'] = 'none'): ...
+    def parse_http(self, auto_decode: Literal['none', 'content', 'transfer', 'all'] = 'none', *,
+                   strict_mode: bool = True): ...
 
     def verify_block_digest(self, consume: bool = False) -> bool: ...
 
@@ -261,9 +292,9 @@ class WarcRecord:
 
     def write(
             self,
-            stream: Union[WarcWriter, BinaryIO, _GenericWriter],
+            stream: WarcWriter | BinaryIO | _GenericWriter,
             checksum_data: bool = False,
-            payload_digest: Optional[bytes] = None,
+            payload_digest: bytes | None = None,
             chunk_size: int = 16384
     ) -> int: ...
 
@@ -271,27 +302,30 @@ class WarcRecord:
 
     def __repr__(self) -> str: ...
 
-    def __getnewargs__(self) -> Tuple[()]: ...
+    def __getnewargs__(self) -> tuple[()]: ...
 
-    def __getstate__(self) -> Tuple[bytes, bool]: ...
+    def __getstate__(self) -> tuple[bytes, bool]: ...
 
-    def __setstate__(self, state: Tuple[bytes, bool]): ...
+    def __setstate__(self, state: tuple[bytes, bool]): ...
 
 
+@final
 class ArchiveIterator(Iterable[WarcRecord]):
     def __new__(
             cls,
-            stream: Union[WarcReader, BinaryIO, _GenericReader, PathLike, str],
+            stream: _ReaderInput,
             record_types: WarcRecordType = any_type,
             parse_http: bool = True,
-            min_content_length: int = -1,
-            max_content_length: int = -1,
-            func_filter: Optional[Callable[[WarcRecord], bool]] = None,
+            min_content_length: int | None = None,
+            max_content_length: int | None = None,
+            func_filter: Callable[[WarcRecord], bool] | None = None,
             verify_digests: bool = False,
             quirks_mode: bool = False,
             auto_decode: Literal['none', 'content', 'transfer', 'all'] = 'none',
             stream_detect: bool = True,
             fsspec_args=None,
+            *,
+            strict_mode: bool = True
     ) -> Self: ...
 
     def __iter__(self) -> Iterator[WarcRecord]: ...

{fastwarc-1.0.0 → fastwarc-1.0.1}/pyproject.toml

@@ -1,23 +1,21 @@
 [build-system]
-requires = ["setuptools>=75.0.0", "setuptools-rust>=1.10", "Cython>=0.29.33"]
+requires = ["setuptools>=77.0", "setuptools-rust>=1.10", "Cython>=0.29.33"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "FastWARC"
-version = "1.0.0"
+version = "1.0.1"
 description = "A high-performance WARC parsing library for Python written in C++/Cython."
 authors = [
-  { name = "Janek Bevendorff" }
+    { name = "Janek Bevendorff" }
 ]
-license = { text = "Apache License 2.0" }
-# TODO: Replace with this once we deprecate Python 3.8 and upgrade to setuptools 77:
-# license = "Apache-2.0"
+license = "Apache-2.0"
 readme = "README.md"
 requires-python = ">=3.8"
 dependencies = [
-  "brotli",
-  "click",
-  "tqdm"
+    "brotli",
+    "click",
+    "tqdm"
 ]
 
 [project.urls]
@@ -27,16 +25,16 @@ documentation = "https://resiliparse.chatnoir.eu"
 
 [project.optional-dependencies]
 all = [
-  "fastwarc[fsspec]"
+    "fastwarc[fsspec]"
 ]
 fsspec = [
-  "fsspec"
+    "fsspec"
 ]
 test = [
-  "fastwarc[all]",
-  "pytest>=7.2.2",
-  "pytest-cov>=4.0.0",
-  "lz4",
+    "fastwarc[all]",
+    "pytest>=7.2.2",
+    "pytest-cov>=4.0.0",
+    "lz4",
 ]
 
 [project.scripts]
@@ -53,3 +51,10 @@ archs = "native"
 build = "cp3*"
 build-frontend = "build"
 test-extras = ["all", "test"]
+
+[tool.mypy]
+exclude = ["fastwarc/cli\\.py", "fastwarc/legacy/.*"]
+
+[[tool.mypy.overrides]]
+module = ["fsspec.*"]
+ignore_missing_imports = true

fastwarc-1.0.0/fastwarc/_fastwarc.pyi

@@ -1,35 +0,0 @@
-# Copyright 2026 Janek Bevendorff
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from .warc import ArchiveIterator, WarcRecord, WarcRecordType
-from .stream_io import (
-    GzipReader,
-    GzipWriter,
-    ZstdReader,
-    ZstdWriter,
-    Lz4Reader,
-    Lz4Writer
-)
-
-__all__ = [
-    'ArchiveIterator',
-    'WarcRecord',
-    'WarcRecordType',
-    'GzipReader',
-    'GzipWriter',
-    'ZstdReader',
-    'ZstdWriter',
-    'Lz4Reader',
-    'Lz4Writer',
-]

fastwarc-1.0.0/fastwarc/stream_io.pyi

@@ -1,136 +0,0 @@
-# Copyright 2026 Janek Bevendorff
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from os import PathLike
-from types import TracebackType
-from typing import BinaryIO, ContextManager, Optional, Protocol, Type, Self, Union
-
-# Legacy shims
-from fastwarc.legacy.stream_io import *
-
-
-class _GenericReader(Protocol):
-    def read(self, size: int = -1) -> bytes: ...
-
-    def seek(self, offset: int, whence: int = 0) -> int: ...
-
-    def tell(self) -> int: ...
-
-    def close(self) -> None: ...
-
-
-class _GenericWriter(Protocol):
-    def write(self, data: bytes) -> int: ...
-
-    def flush(self) -> None: ...
-
-    def close(self) -> None: ...
-
-
-class WarcReader(ContextManager[WarcReader]):
-    def __new__(cls) -> Self: ...
-
-    def read(self, size: int = -1) -> bytes: ...
-
-    def seek(self, offset: int, whence: int = 0) -> int: ...
-
-    def tell(self) -> int: ...
-
-    def close(self) -> None: ...
-
-    def __enter__(self) -> Self: ...
-
-    def __exit__(
-            self,
-            exc_type: Optional[Type[BaseException]],
-            exc: Optional[BaseException],
-            traceback: Optional[TracebackType]
-    ) -> None: ...
-
-    def inner_seek(self, offset: int, whence: int = 0) -> int: ...
-
-    def inner_tell(self) -> int: ...
-
-    def frame_start_position(self) -> Optional[int]: ...
-
-
-class WarcWriter(ContextManager[WarcWriter]):
-    def __new__(cls) -> Self: ...
-
-    def write(self, data: bytes) -> int: ...
-
-    def flush(self) -> None: ...
-
-    def close(self) -> None: ...
-
-    def __enter__(self) -> Self: ...
-
-    def __exit__(
-            self,
-            exc_type: Optional[Type[BaseException]],
-            exc: Optional[BaseException],
-            traceback: Optional[TracebackType]
-    ) -> None: ...
-
-    def finish(self) -> None: ...
-
-
-class GzipReader(WarcReader):
-    def __new__(cls, inner: Union[WarcReader, BinaryIO, _GenericReader, PathLike, str],
-                buffer_size=4096, zlib=False, fsspec_args=None) -> Self: ...
-
-
-class GzipWriter(WarcWriter):
-    def __new__(cls, inner: Union[WarcWriter, BinaryIO, _GenericWriter, PathLike, str],
-                compression_level=9, buffer_size=8192, zlib=False, fsspec_args=None) -> Self: ...
-
-
-class ZstdReader(WarcReader):
-    def __new__(cls, inner: Union[WarcReader, BinaryIO, _GenericReader, PathLike, str],
-                buffer_size=4096, fsspec_args=None, dictionary=None) -> Self: ...
-
-
-class ZstdWriter(WarcWriter):
-    def __new__(cls, inner: Union[WarcWriter, BinaryIO, _GenericWriter, PathLike, str],
-                buffer_size=8192, fsspec_args=None, dictionary=None, compress_dictionary_frame=False) -> Self: ...
-
-
-class Lz4Reader(WarcReader):
-    def __new__(cls, inner: Union[WarcReader, BinaryIO, _GenericReader, PathLike, str],
-                buffer_size=4096, fsspec_args=None) -> Self: ...
-
-
-class Lz4Writer(WarcWriter):
-    def __new__(cls, inner: Union[WarcWriter, BinaryIO, _GenericWriter, PathLike, str],
-                buffer_size=8192, fsspec_args=None) -> Self: ...
-
-
-class BrotliReader(WarcReader):
-    def __new__(cls, inner: Union[WarcReader, BinaryIO, _GenericReader, PathLike, str],
-                buffer_size=4096, fsspec_args=None) -> Self: ...
-
-
-class BrotliWriter(WarcWriter):
-    def __new__(cls, inner: Union[WarcWriter, BinaryIO, _GenericWriter, PathLike, str],
-                buffer_size=8192, fsspec_args=None) -> Self: ...
-
-
-class ChunkedReader(WarcReader):
-    def __new__(cls, inner: Union[WarcReader, BinaryIO, _GenericReader, PathLike, str],
-                buffer_size=1024, fsspec_args=None) -> Self: ...
-
-
-class ChunkedWriter(WarcWriter):
-    def __new__(cls, inner: Union[WarcWriter, BinaryIO, _GenericWriter, PathLike, str],
-                min_chunk_size=512, fsspec_args=None) -> Self: ...