pbz2 0.1.0__py3-none-any.whl

pbz2/__init__.py

@@ -0,0 +1,12 @@
+"""pbz2: stream and parallel-process `.bz2` files via pbzip2."""
+
+from .parallel import process_parallel
+from .reader import iter_chunks, iter_jsonl, iter_lines, open_decompress
+
+__all__ = [
+    "iter_chunks",
+    "iter_jsonl",
+    "iter_lines",
+    "open_decompress",
+    "process_parallel",
+]

pbz2/cli.py

@@ -0,0 +1,27 @@
+"""pbz2 CLI."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+
+from .reader import iter_lines
+
+app = typer.Typer(help="Stream `.bz2` files via pbzip2.")
+
+
+@app.command()
+def count(path: Path) -> None:
+    """Count lines in a `.bz2` file."""
+    n = sum(1 for _ in iter_lines(path))
+    typer.echo(n)
+
+
+@app.command()
+def head(path: Path, n: int = typer.Option(10, "-n", help="Number of lines.")) -> None:
+    """Print the first N lines of a `.bz2` file."""
+    for i, line in enumerate(iter_lines(path)):
+        if i >= n:
+            break
+        typer.echo(line)

pbz2/parallel.py

@@ -0,0 +1,77 @@
+"""Parallel processing of `.bz2` streams via a process pool.
+
+Streams chunks of newline-terminated records from a `.bz2` file through a
+worker function in a `ProcessPoolExecutor`, dispatching each result to an
+optional handler in the main process.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from collections.abc import Callable, Sequence
+from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, wait
+from typing import Any
+
+from .reader import (
+    DEFAULT_BUFSIZE_MB,
+    DEFAULT_STREAM_BUFFER_MB,
+    iter_chunks,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def process_parallel(
+    path: str | os.PathLike,
+    worker_fn: Callable[..., Any],
+    *,
+    on_result: Callable[[Any], None] | None = None,
+    worker_args: Sequence[Any] = (),
+    num_processes: int | None = None,
+    max_pending: int | None = None,
+    bufsize_mb: int = DEFAULT_BUFSIZE_MB,
+    stream_buffer_mb: int = DEFAULT_STREAM_BUFFER_MB,
+) -> None:
+    """Process chunks of `path` in parallel.
+
+    `worker_fn(chunk, *worker_args)` runs in worker processes; each chunk is a
+    `str` of complete newline-terminated records. Results are dispatched to
+    `on_result` in the main process as they complete.
+
+    When `max_pending` is set, the producer pauses while that many futures are
+    in-flight to bound memory.
+    """
+    nproc = num_processes or max(1, (os.cpu_count() or 2) // 2)
+    cap = max_pending if max_pending is not None else nproc * 2
+
+    def drain_done(pending: list, *, block: bool = False) -> list:
+        if block and pending:
+            wait(pending, return_when=FIRST_COMPLETED)
+        still_pending = []
+        for fut in pending:
+            if fut.done():
+                result = fut.result()
+                if on_result is not None:
+                    on_result(result)
+            else:
+                still_pending.append(fut)
+        return still_pending
+
+    with ProcessPoolExecutor(max_workers=nproc) as executor:
+        pending: list = []
+        for chunk in iter_chunks(
+            path,
+            num_processors=nproc,
+            bufsize_mb=bufsize_mb,
+            stream_buffer_mb=stream_buffer_mb,
+        ):
+            if not chunk.strip():
+                continue
+            pending.append(executor.submit(worker_fn, chunk, *worker_args))
+            pending = drain_done(pending)
+            while len(pending) >= cap:
+                pending = drain_done(pending, block=True)
+
+        while pending:
+            pending = drain_done(pending, block=True)

pbz2/reader.py

@@ -0,0 +1,136 @@
+"""Streaming readers for `.bz2` files using pbzip2 for parallel decompression.
+
+Falls back to the stdlib `bz2` module when the `pbzip2` binary is unavailable.
+"""
+
+from __future__ import annotations
+
+import bz2
+import codecs
+import logging
+import os
+import shutil
+import subprocess
+from collections.abc import Callable, Iterator
+from typing import IO, Any, cast
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_BUFSIZE_MB = 32  # OS pipe buffer between pbzip2 and Python
+DEFAULT_STREAM_BUFFER_MB = 4  # Python-side read chunk size
+
+
+def _has_pbzip2() -> bool:
+    return shutil.which("pbzip2") is not None
+
+
+def open_decompress(
+    path: str | os.PathLike,
+    *,
+    num_processors: int | None = None,
+    bufsize_mb: int = DEFAULT_BUFSIZE_MB,
+) -> tuple[IO[bytes], subprocess.Popen | None]:
+    """Open a binary stream that yields decompressed bytes from `path`.
+
+    Returns `(stream, process)`. `process` is the pbzip2 subprocess when
+    pbzip2 is available, else `None` (stdlib `bz2.open` fallback).
+    """
+    if _has_pbzip2():
+        nproc = num_processors or max(1, (os.cpu_count() or 2) - 1)
+        cmd = [
+            "pbzip2",
+            "-dc",
+            f"-p{int(nproc)}",
+            os.fspath(path),
+        ]
+        logger.info("decompress cmd: %s", " ".join(cmd))
+        proc = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            bufsize=bufsize_mb * 1024 * 1024,
+        )
+        assert proc.stdout is not None
+        return proc.stdout, proc
+
+    logger.info("pbzip2 not found; falling back to stdlib bz2 for %s", path)
+    return cast(IO[bytes], bz2.open(os.fspath(path), "rb")), None
+
+
+def iter_chunks(
+    path: str | os.PathLike,
+    *,
+    num_processors: int | None = None,
+    bufsize_mb: int = DEFAULT_BUFSIZE_MB,
+    stream_buffer_mb: int = DEFAULT_STREAM_BUFFER_MB,
+) -> Iterator[str]:
+    """Yield UTF-8 text chunks of complete newline-terminated records.
+
+    Each yielded chunk ends on a newline boundary, so callers can `splitlines()`
+    safely. Handles UTF-8 multibyte characters split across read boundaries.
+    """
+    stream, proc = open_decompress(
+        path,
+        num_processors=num_processors,
+        bufsize_mb=bufsize_mb,
+    )
+    read_size = stream_buffer_mb * 1024 * 1024
+    decoder = codecs.getincrementaldecoder("utf-8")()
+    buffer = ""
+
+    try:
+        while True:
+            data = stream.read(read_size)
+            if not data:
+                # Flush any final bytes through the incremental decoder
+                buffer += decoder.decode(b"", final=True)
+                if buffer:
+                    yield buffer
+                return
+
+            buffer += decoder.decode(data)
+            last_nl = buffer.rfind("\n")
+            if last_nl == -1:
+                continue
+            yield buffer[: last_nl + 1]
+            buffer = buffer[last_nl + 1 :]
+    finally:
+        stream.close()
+        if proc is not None:
+            proc.wait()
+
+
+def iter_lines(
+    path: str | os.PathLike,
+    **kwargs: Any,
+) -> Iterator[str]:
+    """Yield UTF-8 lines (without trailing newline) from a `.bz2` file."""
+    for chunk in iter_chunks(path, **kwargs):
+        for line in chunk.splitlines():
+            if line:
+                yield line
+
+
+def iter_jsonl(
+    path: str | os.PathLike,
+    *,
+    loads: Callable[[str | bytes], Any] | None = None,
+    **kwargs: Any,
+) -> Iterator[Any]:
+    """Yield parsed JSON objects from a `.json.bz2` file (one object per line).
+
+    Uses `orjson.loads` when available, else stdlib `json.loads`. Pass `loads=`
+    to override.
+    """
+    if loads is None:
+        try:
+            import orjson
+
+            loads = orjson.loads
+        except ImportError:
+            import json
+
+            loads = json.loads
+
+    for line in iter_lines(path, **kwargs):
+        yield loads(line)

pbz2-0.1.0.dist-info/METADATA

@@ -0,0 +1,94 @@
+Metadata-Version: 2.4
+Name: pbz2
+Version: 0.1.0
+Summary: Stream and parallel-process .bz2 files via pbzip2.
+Project-URL: repository, https://github.com/gitronald/pbz2
+Author-email: gitronald <gitronald@users.noreply.github.com>
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: orjson>=3.11.9
+Requires-Dist: typer
+Description-Content-Type: text/markdown
+
+# pbz2
+
+Stream and parallel-process `.bz2` files via [pbzip2](http://compression.great-site.net/pbzip2/) (parallel bzip2). Falls back to the stdlib `bz2` module when the `pbzip2` binary is unavailable.
+
+## Install
+
+```bash
+uv add pbz2
+```
+
+Install `pbzip2` for parallel decompression:
+
+```bash
+sudo apt install pbzip2     # Debian/Ubuntu
+brew install pbzip2         # macOS
+```
+
+## Usage
+
+### Iterate
+
+```python
+import pbz2
+
+# Parsed JSON objects from a .json.bz2 file
+for obj in pbz2.iter_jsonl("data.json.bz2"):
+    ...
+
+# Raw UTF-8 lines
+for line in pbz2.iter_lines("data.txt.bz2"):
+    ...
+
+# Newline-aligned text chunks (useful for batched processing)
+for chunk in pbz2.iter_chunks("data.txt.bz2"):
+    ...
+```
+
+### Parallel processing
+
+`process_parallel` streams chunks of newline-terminated records through a worker pool. The worker function receives raw text chunks (so parsing happens in the worker, not the main process), and `on_result` runs in the main process to handle each result as it completes.
+
+```python
+import json
+import pbz2
+
+def parse_chunk(chunk: str) -> list[dict]:
+    return [json.loads(line) for line in chunk.splitlines() if line]
+
+def save(records: list[dict]) -> None:
+    ...  # write to db, file, etc.
+
+pbz2.process_parallel(
+    "data.json.bz2",
+    worker_fn=parse_chunk,
+    on_result=save,
+    num_processes=8,
+)
+```
+
+### CLI
+
+```bash
+pbz2 count data.json.bz2
+pbz2 head data.json.bz2 -n 5
+```
+
+## API
+
+| Function | Description |
+| --- | --- |
+| `iter_chunks(path, **opts)` | Yield UTF-8 text chunks ending on a newline boundary. |
+| `iter_lines(path, **opts)` | Yield non-empty UTF-8 lines (no trailing newline). |
+| `iter_jsonl(path, *, loads=None, **opts)` | Yield parsed JSON objects (uses `orjson` if installed). |
+| `process_parallel(path, worker_fn, *, on_result=None, worker_args=(), num_processes=None, max_pending=None, ...)` | Run `worker_fn(chunk, *worker_args)` in a process pool, dispatching results to `on_result`. |
+| `open_decompress(path, **opts)` | Low-level: open a binary stream of decompressed bytes. |
+
+### Common options
+
+- `num_processors` — pbzip2 worker count (default: cpu_count - 1)
+- `bufsize_mb` — OS pipe buffer between pbzip2 and Python (default: 32 MB)
+- `stream_buffer_mb` — Python-side read chunk size (default: 4 MB)

pbz2-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+pbz2/__init__.py,sha256=iUiLFKg8eYEHID2xQ83-X9WygPjnr_im9HgIppET3o8,295
+pbz2/cli.py,sha256=_cqQkeDAdnOLGGdcE5uN3MeudSVenrfDixwWggqg7Pw,597
+pbz2/parallel.py,sha256=kLJUr1nWET-ol6_spyXnq9T4AHVIoAvGrh4wzO3Vk3s,2521
+pbz2/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pbz2/reader.py,sha256=VmLSnUJFiYzJvEFCFgiPI6VFrxBnh_kf9iqu0qRfDlg,3876
+pbz2-0.1.0.dist-info/METADATA,sha256=0-ChnaBE5GNZSaBEPAEvsEaBLFRahwgEcV1dTi7hMlg,2686
+pbz2-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+pbz2-0.1.0.dist-info/entry_points.txt,sha256=ov0ld90XMBPEvkBaCVnPD0lJgM278YBjZ9hvuutRL-Y,38
+pbz2-0.1.0.dist-info/licenses/LICENSE,sha256=a1bWEq8sJGCTdJ_BtJVrfK44EUtx2ToP2sLCcKl7xkU,1077
+pbz2-0.1.0.dist-info/RECORD,,

pbz2-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

pbz2-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+pbz2 = pbz2.cli:app

pbz2-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2026 Ronald E. Robertson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+