FastWARC 1.1.0__cp315-cp315-win_amd64.whl

fastwarc/__init__.py

@@ -0,0 +1,56 @@
+# Copyright 2026 Janek Bevendorff
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# start delvewheel patch
+def _delvewheel_patch_1_13_2():
+    import os
+    if os.path.isdir(libs_dir := os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'fastwarc.libs'))):
+        os.add_dll_directory(libs_dir)
+
+
+_delvewheel_patch_1_13_2()
+del _delvewheel_patch_1_13_2
+# end delvewheel patch
+
+import importlib
+
+_native = importlib.import_module("._fastwarc", __name__)
+_legacy_shims = importlib.import_module(".legacy._shims", __name__)
+
+import fastwarc.stream_io as stream_io  # noqa: F401
+import fastwarc.warc as warc  # noqa: F401
+
+# Re-export module attributes
+globals().update({attr: getattr(_native, attr) for attr in _native.__all__})
+
+# Patch legacy shims onto stream_io
+for name in _legacy_shims.__all__:
+    setattr(stream_io, name, getattr(_legacy_shims, name))
+stream_io.__all__ += getattr(_legacy_shims, "__all__", ())  # type: ignore[attr-defined]
+
+FileStream = _legacy_shims.FileStream
+GZipStream = _legacy_shims.GZipStream
+LZ4Stream = _legacy_shims.LZ4Stream
+FastWARCError = _legacy_shims.FastWARCError
+StreamError = _legacy_shims.StreamError
+
+__all__ = [
+    *_native.__all__,
+    'FileStream',
+    'GZipStream',
+    'LZ4Stream',
+    'FastWARCError',
+    'StreamError',
+]

fastwarc/__init__.pyi

@@ -0,0 +1,50 @@
+# Copyright 2026 Janek Bevendorff
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .warc import ArchiveIterator, WarcHeader, WarcRecord, WarcRecordType
+from .stream_io import (
+    GzipReader,
+    GzipWriter,
+    ZstdReader,
+    ZstdWriter,
+    Lz4Reader,
+    Lz4Writer,
+
+    # Legacy names
+    FileStream,
+    GZipStream,
+    LZ4Stream,
+    FastWARCError,
+    StreamError,
+)
+
+__all__ = [
+    'ArchiveIterator',
+    'WarcHeader',
+    'WarcRecord',
+    'WarcRecordType',
+    'GzipReader',
+    'GzipWriter',
+    'ZstdReader',
+    'ZstdWriter',
+    'Lz4Reader',
+    'Lz4Writer',
+
+    # Legacy names
+    'FileStream',
+    'GZipStream',
+    'LZ4Stream',
+    'FastWARCError',
+    'StreamError',
+]

fastwarc/cli.py

@@ -0,0 +1,500 @@
+# Copyright 2021 Janek Bevendorff
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from contextlib import closing, nullcontext
+import getpass
+import importlib
+from itertools import chain
+import json
+import os
+import sys
+import time
+import urllib.request
+
+import click
+from tqdm import tqdm
+
+from fastwarc.stream_io import GzipReader, GzipWriter, Lz4Reader, Lz4Writer, ZstdReader, ZstdWriter
+from fastwarc.warc import ArchiveIterator, WarcRecordType
+
+
+COMPRESSION_ALGORITHMS = ['gzip', 'lz4', 'zstd', 'uncompressed', 'auto']
+_READERS = {'gzip': GzipReader, 'lz4': Lz4Reader, 'zstd': ZstdReader}
+_WRITERS = {'gzip': GzipWriter, 'lz4': Lz4Writer, 'zstd': ZstdWriter}
+
+
+class _FastWarcGroup(click.Group):
+    def invoke(self, ctx):
+        try:
+            return super().invoke(ctx)
+        except (OSError, ValueError) as error:
+            raise click.ClickException(str(error)) from error
+
+
+def _compression_from_path(path):
+    name = os.fspath(path).lower()
+    for suffix, algorithm in (('.gz', 'gzip'), ('.lz4', 'lz4'), ('.zst', 'zstd'), ('.zstd', 'zstd')):
+        if name.endswith(suffix):
+            return algorithm
+    return 'uncompressed'
+
+
+def _warc_records(source, decompress_alg='auto', **options):
+    reader = _READERS[decompress_alg](source) if decompress_alg in _READERS else nullcontext(source)
+    with reader as source:
+        yield from ArchiveIterator(source, stream_detect=decompress_alg == 'auto', **options)
+
+
+def _recompress_records(infile, outfile, decompress_alg, compress_alg, compress_level):
+    if os.path.abspath(infile) == os.path.abspath(outfile) or (
+            os.path.exists(outfile) and os.path.samefile(infile, outfile)):
+        raise click.UsageError('Input and output must be different files.')
+    if compress_alg == 'auto':
+        compress_alg = _compression_from_path(outfile)
+    if compress_level is not None and compress_alg not in ('gzip', 'zstd'):
+        raise click.BadParameter('Compression levels are supported only for gzip and zstd.',
+                                 param_hint='--compress-level')
+    with closing(_warc_records(infile, decompress_alg, parse_http=False, inplace=True)) as records, \
+            open(outfile, 'wb') as output:
+        args = {} if compress_level is None else {'compression_level': compress_level}
+        writer = _WRITERS[compress_alg](output, **args) if compress_alg in _WRITERS else nullcontext(output)
+        with writer as writer:
+            for record in records:
+                before = output.tell()
+                record.write(writer)
+                if compress_alg in _WRITERS:
+                    writer.finish()  # One independently seekable compression member per record.
+                yield output.tell() - before
+
+
+def _verify_digests(infile, verify_payloads, decompress_alg):
+    with closing(_warc_records(infile, decompress_alg, parse_http=False, inplace=True)) as records:
+        for record in records:
+            result = {
+                'record_id': record.record_id,
+                'block_digest_ok': record.verify_block_digest(not verify_payloads or not record.is_http)
+                if 'WARC-Block-Digest' in record.headers else None,
+            }
+            if verify_payloads:
+                if 'WARC-Payload-Digest' in record.headers:
+                    if record.is_http:
+                        record.parse_http()
+                    result['payload_digest_ok'] = record.verify_payload_digest(True)
+                else:
+                    result['payload_digest_ok'] = None
+            yield result
+
+
+@click.group(cls=_FastWarcGroup, context_settings=dict(help_option_names=['-h', '--help']))
+def main():
+    """FastWARC Command Line Interface."""
+    return 0
+
+
+def _human_readable_bytes(byte_num):
+    for unit in ['bytes', 'KiB', 'MiB', 'GiB', 'TiB']:
+        if byte_num <= 1024 or unit == 'TiB':
+            if int(byte_num) == byte_num:
+                return f'{int(byte_num)} {unit}'
+            return f'{byte_num:.2f} {unit}'
+        byte_num /= 1024
+
+
+@main.command()
+@click.argument('infile', type=click.Path(dir_okay=False, exists=True))
+@click.argument('outfile', type=click.Path(dir_okay=False, exists=False))
+@click.option('-c', '--compress-alg', type=click.Choice(COMPRESSION_ALGORITHMS),
+              default='auto', show_default=True, help='Compression algorithm to use for output file')
+@click.option('-d', '--decompress-alg', type=click.Choice(COMPRESSION_ALGORITHMS),
+              default='auto', show_default=True,
+              help='Decompression algorithm for decoding input file (auto detects input compression)')
+@click.option('-l', '--compress-level', type=int, default=None,
+              help='Compression level for gzip or zstd (defaults: gzip 9, zstd 3)')
+@click.option('-q', '--quiet', is_flag=True, help='Do not print progress information')
+def recompress(infile, outfile, compress_alg, decompress_alg, compress_level, quiet):
+    """
+    Recompress a WARC file.
+
+    This command allows you to recompress a WARC file if it is uncompressed or not compressed
+    properly at the record level, or to change between GZip, LZ4, and Zstandard.
+    """
+
+    bytes_written = 0
+    num = 0
+    start = time.time()
+    try:
+        with closing(_recompress_records(infile, outfile, decompress_alg, compress_alg, compress_level)) as records:
+            for b in tqdm(records, desc='Recompressing WARC file', unit=' record(s)', leave=False,
+                          disable=quiet, mininterval=0.2):
+                num += 1
+                bytes_written += b
+        if not quiet:
+            click.echo('Recompression completed.')
+    finally:
+        if not quiet and num > 0:
+            click.echo(f'  - Records recompressed: {num}')
+            click.echo(f'  - Bytes written: {_human_readable_bytes(bytes_written)}')
+            click.echo(f'  - Completed in: {time.time() - start:.02f} seconds')
+
+
+@main.command()
+@click.argument('infile', type=click.Path(dir_okay=False, exists=True))
+@click.option('-d', '--decompress-alg', type=click.Choice(COMPRESSION_ALGORITHMS),
+              default='auto', show_default=True, help='Decompression algorithm')
+@click.option('-p', '--verify-payloads', is_flag=True, help='Also verify payload digests')
+@click.option('-q', '--quiet', is_flag=True, help='Do not print progress information')
+@click.option('-o', '--output', type=click.File('w'), help='Output file with verification details')
+def check(infile, decompress_alg, verify_payloads, quiet, output):
+    """
+    Verify WARC record consistency.
+
+    Check digests of all records in a WARC file and print a summary. You can verify all block and
+    payload digests in the given WARC file and print a summary of all corrupted and (optionally)
+    all intact records.
+
+    The command will exit with a non-zero exit code if at least one record fails verification.
+    """
+
+    failed_digests = []
+    num_total = 0
+    num_no_digest = 0
+    try:
+        desc_tpl = 'Verifying digests ({} failed)'
+        pbar = tqdm(_verify_digests(infile, verify_payloads, decompress_alg),
+                    desc=desc_tpl.format(0), unit=' record(s)', leave=False, disable=quiet, mininterval=0.2)
+
+        for v in pbar:
+            num_total += 1
+
+            status = 'OK'
+            if v['block_digest_ok'] is False:
+                status = 'FAIL'
+            elif v['block_digest_ok'] is None:
+                status = 'NO_DIGEST'
+
+            if 'payload_digest_ok' in v:
+                if v['payload_digest_ok'] is True:
+                    status += ', PAYLOAD_OK'
+                elif v['payload_digest_ok'] is False:
+                    status += ', PAYLOAD_FAIL'
+                elif v['payload_digest_ok'] is None:
+                    status += ', PAYLOAD_NO_DIGEST'
+
+            if 'FAIL' in status:
+                failed_digests.append(v['record_id'])
+                pbar.set_description(desc_tpl.format(len(failed_digests)))
+            elif 'NO_DIGEST' in status and 'OK' not in status:
+                num_no_digest += 1
+
+            if output:
+                output.write(f'{v["record_id"]}: {status}\n')
+
+    finally:
+        if not quiet and num_total > 0:
+            if not failed_digests:
+                click.echo(f'{num_total - num_no_digest} records were verified successfully.')
+                if num_no_digest:
+                    click.echo(f'{num_no_digest} records were skipped without digest.')
+            else:
+                click.echo('Failed records:')
+                click.echo('===============')
+                for rec in failed_digests:
+                    click.echo(rec)
+
+    if failed_digests:
+        raise click.exceptions.Exit(1)
+
+
+@main.command()
+@click.argument('infile', type=click.Path(dir_okay=False, exists=True))
+@click.argument('offset', type=click.IntRange(min=0))
+@click.option('-o', '--output', type=click.File('wb'), default='-',
+              help='Output file, default is stdout')
+@click.option('--payload', is_flag=True,
+              help='Output only record payload (transfer and/or content encoding are preserved')
+@click.option('--headers', is_flag=True, help='Output only record (and HTTP) headers')
+def extract(infile, offset, output, payload, headers):
+    """
+    Extract WARC record by offset.
+
+    You can extract individual records at a given byte offset with either just
+    headers, payload, or both.
+    """
+
+    compl_record = not (payload or headers)
+    with open(infile, 'rb') as stream:
+        stream.seek(offset)
+        try:
+            iterator = ArchiveIterator(stream)
+            record = next(iterator)
+            if compl_record:
+                record.write(output)
+            elif headers:
+                record.headers.write(output)
+                if record.is_http:
+                    record.http_headers.write(output)
+            elif payload:
+                buf = record.reader.read(4096)
+                while buf:
+                    output.write(buf)
+                    buf = record.reader.read(4096)
+        except StopIteration:
+            return
+        except (OSError, ValueError) as e:
+            raise click.ClickException(f'Failed to extract WARC record at offset {offset}: {e}') from e
+
+
+def _index_record(output, fields, preserve_multi_header, record, next_record_offset, file_name):
+    idx = dict()
+    for f in fields:
+        f = f.strip().lower()
+
+        if f == 'offset':
+            idx[f] = str(record.stream_pos)
+        elif f == 'length':
+            idx[f] = str(next_record_offset - record.stream_pos)
+        elif f == 'filename':
+            idx[f] = file_name
+        elif f == 'http:status' and record.is_http:
+            idx[f] = str(record.http_headers.status_code)
+        elif f.startswith('http:') and record.is_http:
+            if preserve_multi_header:
+                values = record.http_headers.get_multiple(f[5:])
+                if len(values) == 1:
+                    idx[f] = values[0]
+                elif values:
+                    idx[f] = list(values)
+            else:
+                idx[f] = record.http_headers.get(f[5:])
+        elif f in record.headers:
+            idx[f] = record.headers.get(f)
+
+    output.write(json.dumps(idx) + '\n')
+
+
+@main.command()
+@click.argument('infiles', type=click.Path(dir_okay=False, exists=True), nargs=-1)
+@click.option('-o', '--output', type=click.File('w'), default='-',
+              help='Output file, default is stdout')
+@click.option('-f', '--fields', type=str, metavar='FIELDS',
+              default='offset,warc-type,warc-target-uri', show_default=True,
+              help='Comma-separated list of indexed fields, eg. "offset", "length", "filename", '
+                   '"http:status", "http:<http-header>", or "<warc-record-header>"')
+@click.option('--preserve-multi-header', is_flag=True,
+              help='Preserve multiple values of HTTP headers as JSON list')
+def index(infiles, output, fields, preserve_multi_header):
+    """
+    Index WARC records as CDXJ.
+
+    WARC files can be indexed to the `CDXJ <https://github.com/webrecorder/cdxj-indexer>`_
+    format with a configurable set of fields.
+    """
+    fields = fields.split(',')
+    for infile in infiles:
+        prev_record = None
+        for record in ArchiveIterator(infile):
+            if prev_record is not None:
+                _index_record(output, fields, preserve_multi_header, prev_record, record.stream_pos, infile)
+            prev_record = record
+
+        if prev_record is not None:
+            _index_record(output, fields, preserve_multi_header, prev_record, os.path.getsize(infile), infile)
+
+
+boto3 = None
+botocore = None
+s3 = None
+
+
+def _init_s3(endpoint_url, aws_access_key, aws_secret_key=None):
+    global boto3, botocore, s3
+
+    if s3 is not None:
+        return
+
+    try:
+        boto3 = importlib.import_module('boto3')
+        botocore = importlib.import_module('botocore')
+    except ModuleNotFoundError:
+        raise click.UsageError('Boto3 needs to be installed for S3 benchmarking.')
+
+    if not aws_access_key:
+        raise click.UsageError("s3:// URLs require '--aws-access-key' to be set.")
+    if not aws_secret_key:
+        aws_secret_key = getpass.getpass()
+
+    s3 = boto3.resource('s3', endpoint_url=endpoint_url, aws_access_key_id=aws_access_key,
+                        aws_secret_access_key=aws_secret_key)
+
+
+# noinspection PyProtectedMember, HttpUrlsUsage
+def _get_raw_stream_from_url(input_url, use_python_stream=False):
+    if input_url.startswith('s3://'):
+        assert s3 is not None
+        try:
+            s3_bucket, s3_object = input_url[5:].split('/', 1)
+            stream = s3.Object(s3_bucket, s3_object).get()['Body']._raw_stream
+        except botocore.exceptions.ClientError as e:
+            click.echo(f'Error connecting to S3. Message: {e}.', err=True)
+            sys.exit(1)
+
+    elif input_url.startswith('http://') or input_url.startswith('https://'):
+        stream = urllib.request.urlopen(input_url)
+
+    else:
+        if input_url.startswith('file://'):
+            input_url = urllib.request.url2pathname(input_url[7:])
+        if not os.path.isfile(input_url):
+            click.echo(f"No such file or directory: '{input_url}", err=True)
+            sys.exit(1)
+
+        stream = open(input_url, 'rb') if use_python_stream else input_url
+
+    return stream
+
+
+def _expand_s3_prefixes(input_urls, endpoint_url, aws_access_key, aws_secret_key):
+    urls_expanded = []
+    for url in input_urls:
+        if url.startswith('s3://'):
+            _init_s3(endpoint_url, aws_access_key, aws_secret_key)
+            s3_bucket, s3_object_prefix = url[5:].split('/', 1)
+            urls_expanded.extend('/'.join(('s3:/', s3_bucket, o.key))
+                                 for o in s3.Bucket(s3_bucket).objects.filter(Prefix=s3_object_prefix))
+        else:
+            urls_expanded.append(url)
+    return urls_expanded
+
+
+# noinspection PyPackageRequirements
+@main.command()
+@click.argument('input_url', nargs=-1)
+@click.option('-d', '--decompress-alg', type=click.Choice(COMPRESSION_ALGORITHMS),
+              default='auto', show_default=True, help='Decompression algorithm')
+@click.option('-e', '--endpoint-url', help='S3 endpoint URL', default='https://s3.amazonaws.com', show_default=True)
+@click.option('-a', '--aws-access-key', help='AWS access key for s3:// URLs')
+@click.option('-s', '--aws-secret-key', help='AWS secret key for s3:// URLs (leave empty to read from STDIN)')
+@click.option('--is-prefix', is_flag=True, help='Treat input URL as prefix (only for S3)')
+@click.option('-p', '--use-python-stream', is_flag=True,
+              help='Use Python I/O instead of native file readers for local files')
+@click.option('-H', '--parse-http', is_flag=True, help='Parse HTTP headers')
+@click.option('-v', '--verify-digests', is_flag=True, help='Verify record block digests')
+@click.option('-f', '--filter-type', type=click.Choice(['warcinfo', 'response', 'resource', 'request', 'metadata',
+                                                        'revisit', 'conversion', 'continuation', 'unknown', 'any_type']),
+              default=['any_type'], multiple=True, show_default=True, help='Filter for specific WARC record types')
+@click.option('-w', '--bench-warcio', is_flag=True, help='Compare FastWARC performance with WARCIO')
+def benchmark(input_url, decompress_alg, endpoint_url, aws_access_key, aws_secret_key, is_prefix, use_python_stream,
+              verify_digests, parse_http, filter_type, bench_warcio):
+    """
+    Benchmark FastWARC read performance.
+
+    The FastWARC CLI comes with a benchmarking tool for measuring WARC record decompression and parsing
+    performance on your own machine. The benchmarking results can be compared directly with WARCIO.
+
+    Supported WARC sources are local files, S3 and HTTP(s) URLs. Supported compression algorithms
+    are GZip, LZ4, Zstandard, or uncompressed.
+
+    The read benchmarking tool has additional options, such as reading WARCs directly from a remote S3 data source
+    using `Boto3 <https://boto3.amazonaws.com/v1/documentation/api/latest/index.html>`_.
+
+    See :ref:`FastWARC Benchmarks <fastwarc-benchmarks>` for more information and example benchmarking results.
+    """
+
+    if not input_url:
+        raise click.UsageError('Need at least one input path.')
+
+    if is_prefix:
+        input_url = _expand_s3_prefixes(input_url, endpoint_url, aws_access_key, aws_secret_key)
+    else:
+        for u in input_url:
+            if u.startswith('s3://'):
+                _init_s3(endpoint_url, aws_access_key, aws_secret_key)
+                break
+
+    if bench_warcio:
+        algorithms = {_compression_from_path(url) for url in input_url} if decompress_alg == 'auto' else {decompress_alg}
+        if algorithms & {'lz4', 'zstd'}:
+            raise click.UsageError('WARCIO does not support LZ4 or Zstandard.')
+
+        try:
+            import warcio
+        except ModuleNotFoundError:
+            click.echo("--bench-warcio requires 'warcio' to be installed.", err=True)
+            sys.exit(1)
+
+    click.echo(f'Benchmarking read performance from {len(input_url)} input path(s)...')
+
+    rec_type_filter = WarcRecordType.any_type
+    if 'any_type' not in filter_type:
+        rec_type_filter = WarcRecordType.no_type
+        for t in filter_type:
+            rec_type_filter |= getattr(WarcRecordType, t)
+
+    def _bench(urls, stream_loader, lib):
+        interrupted = False
+        start = time.perf_counter()
+        num = 0
+
+        try:
+            def _load_lazy(u, l):
+                yield from l(u)
+
+            for _ in tqdm(chain(*(_load_lazy(u, stream_loader) for u in urls)),
+                          desc=f'Benchmarking {lib}', unit=' records', leave=False, mininterval=0.5):
+                num += 1
+        except (OSError, ValueError) as e:
+            raise click.ClickException(f'Error reading input: {e}') from e
+        except KeyboardInterrupt:
+            click.echo(f'Benchmark interrupted.', err=True)
+            interrupted = True
+
+        # Short or empty inputs can finish within one clock tick.
+        elapsed = max(time.perf_counter() - start, time.get_clock_info('perf_counter').resolution)
+        return num, elapsed, interrupted
+
+    def _fastwarc_iterator(f):
+        source = _get_raw_stream_from_url(f, use_python_stream)
+        try:
+            yield from _warc_records(source, decompress_alg, record_types=rec_type_filter,
+                                     parse_http=parse_http, verify_digests=verify_digests, inplace=True)
+        finally:
+            if not isinstance(source, str):
+                source.close()
+
+    n, t_fastwarc, interrupted = _bench(input_url, _fastwarc_iterator, 'FastWARC')
+    click.echo(f'FastWARC: {n:,} records read in {t_fastwarc:.02f} seconds ({n / t_fastwarc:,.02f} records/s).')
+
+    if interrupted:
+        sys.exit(1)
+
+    if bench_warcio:
+        def _warcio_iterator(f):
+            source = _get_raw_stream_from_url(f, True)
+            try:
+                for rec in warcio.ArchiveIterator(source, no_record_parse=not parse_http,
+                                                 check_digests=verify_digests):
+                    if rec_type_filter & getattr(WarcRecordType, rec.rec_type):
+                        yield rec
+            finally:
+                source.close()
+
+        n, t_warcio, interrupted = _bench(input_url, _warcio_iterator, 'WARCIO')
+        click.echo(f'WARCIO:   {n:,} records read in {t_warcio:.02f} seconds ({n / t_warcio:,.02f} records/s).')
+        if interrupted:
+            raise click.exceptions.Exit(1)
+        click.echo(f'Time difference: {t_fastwarc - t_warcio:.02f} seconds, speedup: {t_warcio / t_fastwarc:,.02f}')
+
+
+if __name__ == '__main__':
+    main()

fastwarc/legacy/__init__.py

@@ -0,0 +1,42 @@
+# Copyright 2021 Janek Bevendorff
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+
+# Load Cython exports only when requested. Importing the Rust-backed package's
+# compatibility shims must not require the legacy extensions.
+
+__all__ = [
+    "FileStream",
+    "GZipStream",
+    "LZ4Stream",
+    "FastWARCError",
+    "StreamError",
+    "ArchiveIterator",
+    "WarcRecord",
+    "WarcRecordType"
+]
+
+
+def __getattr__(name):
+    if name not in __all__:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    module = '.warc' if name in ('ArchiveIterator', 'WarcRecord', 'WarcRecordType') else '.stream_io'
+    value = getattr(importlib.import_module(module, __name__), name)
+    globals()[name] = value
+    return value
+
+
+def __dir__():
+    return sorted(set(globals()) | set(__all__))

fastwarc/legacy/__init__.pyi

@@ -0,0 +1,19 @@
+from .stream_io import (
+    FileStream,
+    GZipStream,
+    LZ4Stream,
+    FastWARCError,
+    StreamError,
+)
+from .warc import ArchiveIterator, WarcRecord, WarcRecordType
+
+__all__ = [
+    "FileStream",
+    "GZipStream",
+    "LZ4Stream",
+    "FastWARCError",
+    "StreamError",
+    "ArchiveIterator",
+    "WarcRecord",
+    "WarcRecordType",
+]