FastWARC 1.0.7__tar.gz → 1.0.9__tar.gz

{fastwarc-1.0.7 → fastwarc-1.0.9}/FastWARC.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FastWARC
-Version: 1.0.7
+Version: 1.0.9
 Summary: The world's fastest WARC parsing library written in Rust with bindings for Python.
 Author: Janek Bevendorff
 License-Expression: Apache-2.0
@@ -12,7 +12,7 @@ Description-Content-Type: text/markdown
 Requires-Dist: brotli
 Requires-Dist: click
 Requires-Dist: tqdm
-Requires-Dist: typing_extensions>=4.5; python_version < "3.13"
+Requires-Dist: typing_extensions>=4.15; python_version < "3.13"
 Provides-Extra: all
 Requires-Dist: fastwarc[fsspec]; extra == "all"
 Provides-Extra: fsspec

{fastwarc-1.0.7 → fastwarc-1.0.9}/FastWARC.egg-info/SOURCES.txt

@@ -8,8 +8,10 @@ FastWARC.egg-info/dependency_links.txt
 FastWARC.egg-info/entry_points.txt
 FastWARC.egg-info/requires.txt
 FastWARC.egg-info/top_level.txt
+fastwarc/Cargo.toml
 fastwarc/__init__.py
 fastwarc/__init__.pyi
+fastwarc/build.rs
 fastwarc/cli.py
 fastwarc/py.typed
 fastwarc/stream_io.pyi
@@ -26,6 +28,14 @@ fastwarc/legacy/tools.pyx
 fastwarc/legacy/warc.pxd
 fastwarc/legacy/warc.pyi
 fastwarc/legacy/warc.pyx
+fastwarc/src/lib.rs
+fastwarc/src/warc.rs
+fastwarc/src/stream_io/brotli.rs
+fastwarc/src/stream_io/chunked.rs
+fastwarc/src/stream_io/gzip.rs
+fastwarc/src/stream_io/lz4.rs
+fastwarc/src/stream_io/mod.rs
+fastwarc/src/stream_io/zstd.rs
 resiliparse_common/__init__.pxd
 resiliparse_common/string_util.pxd
 resiliparse_inc/__init__.pxd

{fastwarc-1.0.7 → fastwarc-1.0.9}/FastWARC.egg-info/requires.txt

@@ -3,7 +3,7 @@ click
 tqdm
 
 [:python_version < "3.13"]
-typing_extensions>=4.5
+typing_extensions>=4.15
 
 [all]
 fastwarc[fsspec]

{fastwarc-1.0.7 → fastwarc-1.0.9}/MANIFEST.in

@@ -1,4 +1,4 @@
-recursive-include fastwarc *.pyx *.pxd *.h
+recursive-include fastwarc *.rs Cargo.toml Cargo.lock *.pyx *.pxd *.h
 recursive-exclude fastwarc *.cpp
 graft resiliparse_common
 graft resiliparse_inc

{fastwarc-1.0.7 → fastwarc-1.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FastWARC
-Version: 1.0.7
+Version: 1.0.9
 Summary: The world's fastest WARC parsing library written in Rust with bindings for Python.
 Author: Janek Bevendorff
 License-Expression: Apache-2.0
@@ -12,7 +12,7 @@ Description-Content-Type: text/markdown
 Requires-Dist: brotli
 Requires-Dist: click
 Requires-Dist: tqdm
-Requires-Dist: typing_extensions>=4.5; python_version < "3.13"
+Requires-Dist: typing_extensions>=4.15; python_version < "3.13"
 Provides-Extra: all
 Requires-Dist: fastwarc[fsspec]; extra == "all"
 Provides-Extra: fsspec

fastwarc-1.0.9/fastwarc/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "fastwarc-py"
+edition = "2024"
+publish = false
+
+[lib]
+name = "fastwarc"
+path = "src/lib.rs"
+crate-type = [
+    "cdylib",
+]
+
+[dependencies]
+fastwarc = "1.0.9"
+pyo3 = "0.28"
+
+[dependencies.time]
+version = "0.3"
+features = [
+    "formatting",
+    "parsing",
+    "macros",
+]
+
+[build-dependencies]
+pyo3-build-config = "0.28"
+
+[workspace]

{fastwarc-1.0.7 → fastwarc-1.0.9}/fastwarc/__init__.pyi

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .warc import ArchiveIterator, WarcRecord, WarcRecordType
+from .warc import ArchiveIterator, WarcHeader, WarcRecord, WarcRecordType
 from .stream_io import (
     GzipReader,
     GzipWriter,
@@ -31,6 +31,7 @@ from .stream_io import (
 
 __all__ = [
     'ArchiveIterator',
+    'WarcHeader',
     'WarcRecord',
     'WarcRecordType',
     'GzipReader',

fastwarc-1.0.9/fastwarc/build.rs

@@ -0,0 +1,18 @@
+// Copyright 2026 Janek Bevendorff
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+fn main() {
+    // https://pyo3.rs/v0.28.3/building-and-distribution.html?utm_source=chatgpt.com#macos
+    pyo3_build_config::add_extension_module_link_args();
+}

{fastwarc-1.0.7 → fastwarc-1.0.9}/fastwarc/legacy/_shims.py

@@ -14,9 +14,11 @@
 
 # Legacy shims for making the new Rust extension work with the old type names
 
-try:
+import sys
+
+if sys.version_info >= (3, 13):
     from warnings import deprecated
-except ImportError:
+else:
     from typing_extensions import deprecated
 
 __all__ = [

fastwarc-1.0.9/fastwarc/src/lib.rs

@@ -0,0 +1,116 @@
+// Copyright 2026 Janek Bevendorff
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod stream_io;
+mod warc;
+
+use pyo3::prelude::*;
+use pyo3::types::PyModule;
+
+#[pymodule]
+pub mod _fastwarc {
+    use super::*;
+
+    #[pymodule_init]
+    pub fn __init__(m: &Bound<'_, PyModule>) -> PyResult<()> {
+        let py = m.py();
+
+        // Load submodules.
+        // Don't add with add_submodule(), otherwise they end up in __all__.
+        let stream_io = pyo3::wrap_pymodule!(_stream_io)(py);
+        let stream_io = stream_io.bind(py);
+
+        let warc = pyo3::wrap_pymodule!(_warc)(py);
+        let warc = warc.bind(py);
+
+        // Register submodules to make them importable.
+        let sys = PyModule::import(py, "sys")?;
+        let modules = sys.getattr("modules")?;
+        modules.set_item("fastwarc.stream_io", stream_io)?;
+        modules.set_item("fastwarc.warc", warc)?;
+
+        // Top-level exports
+        m.add("WarcRecordType", warc.getattr("WarcRecordType")?)?;
+        m.add("WarcHeader", warc.getattr("WarcHeader")?)?;
+        m.add("WarcRecord", warc.getattr("WarcRecord")?)?;
+        m.add("ArchiveIterator", warc.getattr("ArchiveIterator")?)?;
+
+        m.add("GzipReader", stream_io.getattr("GzipReader")?)?;
+        m.add("GzipWriter", stream_io.getattr("GzipWriter")?)?;
+        m.add("Lz4Reader", stream_io.getattr("Lz4Reader")?)?;
+        m.add("Lz4Writer", stream_io.getattr("Lz4Writer")?)?;
+        m.add("ZstdReader", stream_io.getattr("ZstdReader")?)?;
+        m.add("ZstdWriter", stream_io.getattr("ZstdWriter")?)?;
+
+        Ok(())
+    }
+}
+
+#[pymodule(name = "stream_io", module = "fastwarc")]
+pub mod _stream_io {
+    #[pymodule_export]
+    pub use crate::stream_io::{WarcReaderPy, WarcWriterPy};
+
+    #[pymodule_export]
+    pub use crate::stream_io::gzip::{GzipReaderPy, GzipWriterPy};
+
+    #[pymodule_export]
+    pub use crate::stream_io::zstd::{
+        ZstdReaderPy, ZstdWriterPy, zstd_train_dictionary_from_continuous, zstd_train_dictionary_from_files,
+        zstd_train_dictionary_from_samples,
+    };
+
+    #[pymodule_export]
+    pub use crate::stream_io::lz4::{Lz4ReaderPy, Lz4WriterPy};
+
+    #[pymodule_export]
+    pub use crate::stream_io::brotli::{BrotliReaderPy, BrotliWriterPy};
+
+    #[pymodule_export]
+    pub use crate::stream_io::chunked::{ChunkedReaderPy, ChunkedWriterPy};
+}
+
+#[pymodule(name = "warc", module = "fastwarc")]
+pub mod _warc {
+    use super::*;
+
+    #[pymodule_init]
+    fn __init__(m: &Bound<'_, PyModule>) -> PyResult<()> {
+        // Re-export enum members as individual constants.
+        m.add("warcinfo", WarcRecordTypePy::warcinfo)?;
+        m.add("response", WarcRecordTypePy::response)?;
+        m.add("resource", WarcRecordTypePy::resource)?;
+        m.add("request", WarcRecordTypePy::request)?;
+        m.add("metadata", WarcRecordTypePy::metadata)?;
+        m.add("revisit", WarcRecordTypePy::revisit)?;
+        m.add("conversion", WarcRecordTypePy::conversion)?;
+        m.add("continuation", WarcRecordTypePy::continuation)?;
+        m.add("unknown", WarcRecordTypePy::unknown)?;
+        m.add("no_type", WarcRecordTypePy::no_type)?;
+        m.add("any_type", WarcRecordTypePy::any_type)?;
+
+        // Legacy name
+        m.add("WarcHeaderMap", m.getattr("HeaderMap")?)?;
+
+        Ok(())
+    }
+
+    #[pymodule_export]
+    pub use crate::warc::{
+        ArchiveIteratorPy, HeaderMapPy, WarcHeaderPy, WarcRecordPayloadReaderPy, WarcRecordPy, WarcRecordTypePy,
+        has_block_digest_py, has_content_length_gte_py, has_content_length_lte_py, has_payload_digest_py,
+        has_record_type_py, has_valid_block_digest_py, has_valid_payload_digest_py, is_concurrent_py, is_http_py,
+        is_warc_10_py, is_warc_11_py,
+    };
+}

fastwarc-1.0.9/fastwarc/src/stream_io/brotli.rs

@@ -0,0 +1,148 @@
+// Copyright 2026 Janek Bevendorff
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::impl_macros::*;
+use crate::stream_io::{WarcReaderPy, WarcWriterPy, wrap_reader_stream, wrap_writer_stream};
+use fastwarc::stream_io::brotli::{self, BrotliWriterOptions};
+use fastwarc::stream_io::traits::{WarcRead, WarcWrite};
+use pyo3::exceptions::PyValueError;
+use pyo3::prelude::*;
+use pyo3::types::PyBytes;
+use std::io::{self, Read, Seek, Write};
+use std::sync::Mutex;
+
+/// Brotli reader.
+///
+/// :param inner: raw input stream, file-like object, file name, or URL
+/// :param buffer_size: input buffer size
+/// :param fsspec_args: arguments for :mod:`fsspec`, or ``False`` to disable it
+#[pyclass(name = "BrotliReader", module = "fastwarc.stream_io", extends = WarcReaderPy, subclass)]
+pub struct BrotliReaderPy {
+    pub(crate) inner: Mutex<Option<Box<dyn WarcRead + Send>>>,
+}
+
+// noinspection DuplicatedCode
+#[pymethods]
+impl BrotliReaderPy {
+    #[new]
+    #[pyo3(signature = (inner, buffer_size=64 << 10, fsspec_args=None))]
+    pub fn __new__(
+        py: Python<'_>,
+        inner: Py<PyAny>,
+        buffer_size: usize,
+        fsspec_args: Option<Py<PyAny>>,
+    ) -> PyResult<PyClassInitializer<Self>> {
+        let options = brotli::BrotliReaderOptions { capacity: buffer_size };
+        let inner = wrap_reader_stream(
+            py,
+            inner,
+            fsspec_args,
+            |reader| -> io::Result<Box<dyn WarcRead + Send>> {
+                Ok(Box::new(brotli::BrotliReader::with_options(reader, options)))
+            },
+            |path| Ok(Box::new(brotli::BrotliReader::from_path_with_options(path, options)?)),
+        )?;
+        Ok(PyClassInitializer::from(WarcReaderPy::__new__()).add_subclass(Self {
+            inner: Mutex::new(Some(inner)),
+        }))
+    }
+
+    #[pyo3(signature = (size=-1))]
+    pub fn read<'py>(&self, py: Python<'py>, size: i128) -> PyResult<Bound<'py, PyBytes>> {
+        impl_reader_read!(self, py, size)
+    }
+
+    #[pyo3(signature = (offset, whence=0))]
+    pub fn seek(&self, offset: i128, whence: u8) -> PyResult<u64> {
+        impl_reader_seek!(self, offset, whence, seek)
+    }
+
+    #[pyo3(signature = (offset, whence=0))]
+    pub fn inner_seek(&self, offset: i128, whence: u8) -> PyResult<u64> {
+        impl_reader_seek!(self, offset, whence, inner_seek)
+    }
+
+    pub fn tell(&self) -> PyResult<u64> {
+        forward_fn_call!(self, stream_position)
+    }
+
+    pub fn inner_tell(&self) -> PyResult<u64> {
+        forward_fn_call!(self, inner_stream_position)
+    }
+
+    #[allow(clippy::needless_question_mark)]
+    pub fn frame_start_position(&mut self) -> io::Result<Option<u64>> {
+        forward_fn_call!(self, frame_start_position)
+    }
+
+    pub fn close(&self) -> PyResult<()> {
+        impl_reader_close!(self)
+    }
+}
+
+/// Brotli writer.
+///
+/// :param inner: raw output stream, file-like object, file name, or URL
+/// :param buffer_size: compression buffer size
+/// :param fsspec_args: arguments for :mod:`fsspec`, or ``False`` to disable it
+#[pyclass(name = "BrotliWriter", module = "fastwarc.stream_io", extends = WarcWriterPy, subclass)]
+pub struct BrotliWriterPy {
+    pub(crate) inner: Mutex<Option<Box<dyn WarcWrite + Send>>>,
+}
+
+// noinspection DuplicatedCode
+#[pymethods]
+impl BrotliWriterPy {
+    #[new]
+    #[pyo3(signature = (inner, buffer_size=8192, fsspec_args=None,))]
+    pub fn __new__(
+        py: Python<'_>,
+        inner: Py<PyAny>,
+        buffer_size: usize,
+        fsspec_args: Option<Py<PyAny>>,
+    ) -> PyResult<PyClassInitializer<Self>> {
+        let options = brotli::BrotliWriterOptions {
+            capacity: buffer_size,
+            ..BrotliWriterOptions::default()
+        };
+        let inner = wrap_writer_stream(
+            py,
+            inner,
+            fsspec_args,
+            |reader| -> io::Result<Box<dyn WarcWrite + Send>> {
+                Ok(Box::new(brotli::BrotliWriter::with_options(reader, options)))
+            },
+            |path| Ok(Box::new(brotli::BrotliWriter::from_path_with_options(path, options)?)),
+        )?;
+        Ok(PyClassInitializer::from(WarcWriterPy::__new__()).add_subclass(Self {
+            inner: Mutex::new(Some(inner)),
+        }))
+    }
+
+    pub fn write(&self, data: &[u8]) -> PyResult<usize> {
+        impl_writer_write!(self, data)
+    }
+
+    pub fn flush(&self) -> PyResult<()> {
+        forward_fn_call!(self, flush)
+    }
+
+    pub fn finish(&self) -> PyResult<()> {
+        forward_fn_call!(self, finish)
+    }
+
+    pub fn close(&self) -> PyResult<()> {
+        impl_writer_close!(self)
+    }
+}

fastwarc-1.0.9/fastwarc/src/stream_io/chunked.rs

@@ -0,0 +1,140 @@
+// Copyright 2026 Janek Bevendorff
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::impl_macros::*;
+use crate::stream_io::{WarcReaderPy, WarcWriterPy, wrap_reader_stream, wrap_writer_stream};
+use fastwarc::stream_io::chunked;
+use fastwarc::stream_io::traits::{WarcRead, WarcWrite};
+use pyo3::exceptions::PyValueError;
+use pyo3::prelude::*;
+use pyo3::types::PyBytes;
+use std::io::{self, Read, Seek, Write};
+use std::sync::Mutex;
+
+/// HTTP chunked-transfer reader.
+///
+/// :param inner: raw input stream, file-like object, file name, or URL
+/// :param buffer_size: input buffer size
+/// :param fsspec_args: arguments for :mod:`fsspec`, or ``False`` to disable it
+#[pyclass(name = "ChunkedReader", module = "fastwarc.stream_io", extends = WarcReaderPy, subclass)]
+pub struct ChunkedReaderPy {
+    pub(crate) inner: Mutex<Option<Box<dyn WarcRead + Send>>>,
+}
+
+// noinspection DuplicatedCode
+#[pymethods]
+impl ChunkedReaderPy {
+    #[new]
+    #[pyo3(signature = (inner, buffer_size=4096, fsspec_args=None))]
+    pub fn __new__(
+        py: Python<'_>,
+        inner: Py<PyAny>,
+        buffer_size: usize,
+        fsspec_args: Option<Py<PyAny>>,
+    ) -> PyResult<PyClassInitializer<Self>> {
+        let options = chunked::ChunkedReaderOptions { capacity: buffer_size };
+        let inner = wrap_reader_stream(
+            py,
+            inner,
+            fsspec_args,
+            |reader| -> io::Result<Box<dyn WarcRead + Send>> {
+                Ok(Box::new(chunked::ChunkedReader::with_options(reader, options)))
+            },
+            |path| Ok(Box::new(chunked::ChunkedReader::from_path_with_options(path, options)?)),
+        )?;
+        Ok(PyClassInitializer::from(WarcReaderPy::__new__()).add_subclass(Self {
+            inner: Mutex::new(Some(inner)),
+        }))
+    }
+
+    #[pyo3(signature = (size=-1))]
+    pub fn read<'py>(&self, py: Python<'py>, size: i128) -> PyResult<Bound<'py, PyBytes>> {
+        impl_reader_read!(self, py, size)
+    }
+
+    #[pyo3(signature = (offset, whence=0))]
+    pub fn seek(&self, offset: i128, whence: u8) -> PyResult<u64> {
+        impl_reader_seek!(self, offset, whence, seek)
+    }
+
+    #[pyo3(signature = (offset, whence=0))]
+    pub fn inner_seek(&self, offset: i128, whence: u8) -> PyResult<u64> {
+        impl_reader_seek!(self, offset, whence, inner_seek)
+    }
+
+    pub fn tell(&self) -> PyResult<u64> {
+        forward_fn_call!(self, stream_position)
+    }
+
+    pub fn inner_tell(&self) -> PyResult<u64> {
+        forward_fn_call!(self, inner_stream_position)
+    }
+
+    pub fn close(&self) -> PyResult<()> {
+        impl_reader_close!(self)
+    }
+}
+
+/// HTTP chunked-transfer writer.
+///
+/// :param inner: raw output stream, file-like object, file name, or URL
+/// :param min_chunk_size: minimum chunk size
+/// :param fsspec_args: arguments for :mod:`fsspec`, or ``False`` to disable it
+#[pyclass(name = "ChunkedWriter", module = "fastwarc.stream_io", extends = WarcWriterPy, subclass)]
+pub struct ChunkedWriterPy {
+    pub(crate) inner: Mutex<Option<Box<dyn WarcWrite + Send>>>,
+}
+
+// noinspection DuplicatedCode
+#[pymethods]
+impl ChunkedWriterPy {
+    #[new]
+    #[pyo3(signature = (inner, min_chunk_size=1024, fsspec_args=None,))]
+    pub fn __new__(
+        py: Python<'_>,
+        inner: Py<PyAny>,
+        min_chunk_size: usize,
+        fsspec_args: Option<Py<PyAny>>,
+    ) -> PyResult<PyClassInitializer<Self>> {
+        let options = chunked::ChunkedWriterOptions { min_chunk_size };
+        let inner = wrap_writer_stream(
+            py,
+            inner,
+            fsspec_args,
+            |reader| -> io::Result<Box<dyn WarcWrite + Send>> {
+                Ok(Box::new(chunked::ChunkedWriter::with_options(reader, options)))
+            },
+            |path| Ok(Box::new(chunked::ChunkedWriter::from_path_with_options(path, options)?)),
+        )?;
+        Ok(PyClassInitializer::from(WarcWriterPy::__new__()).add_subclass(Self {
+            inner: Mutex::new(Some(inner)),
+        }))
+    }
+
+    pub fn write(&self, data: &[u8]) -> PyResult<usize> {
+        impl_writer_write!(self, data)
+    }
+
+    pub fn flush(&self) -> PyResult<()> {
+        forward_fn_call!(self, flush)
+    }
+
+    pub fn finish(&self) -> PyResult<()> {
+        forward_fn_call!(self, finish)
+    }
+
+    pub fn close(&self) -> PyResult<()> {
+        impl_writer_close!(self)
+    }
+}