pbz2 0.1.0__tar.gz

pbz2-0.1.0/.claude/settings.local.json

@@ -0,0 +1,69 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(date:*)",
+      "Bash(diff:*)",
+      "Bash(du:*)",
+      "Bash(file:*)",
+      "Bash(find:*)",
+      "Bash(gh api:*)",
+      "Bash(gh issue:*)",
+      "Bash(gh pr create:*)",
+      "Bash(gh pr diff:*)",
+      "Bash(gh pr list:*)",
+      "Bash(gh pr merge:*)",
+      "Bash(gh pr view:*)",
+      "Bash(gh repo:*)",
+      "Bash(git add:*)",
+      "Bash(git branch:*)",
+      "Bash(git checkout:*)",
+      "Bash(git commit:*)",
+      "Bash(git config:*)",
+      "Bash(git diff:*)",
+      "Bash(git fetch:*)",
+      "Bash(git log:*)",
+      "Bash(git merge:*)",
+      "Bash(git mv:*)",
+      "Bash(git pull:*)",
+      "Bash(git remote:*)",
+      "Bash(git show:*)",
+      "Bash(git stash:*)",
+      "Bash(git status:*)",
+      "Bash(git switch:*)",
+      "Bash(git tag:*)",
+      "Bash(grep:*)",
+      "Bash(head:*)",
+      "Bash(jq:*)",
+      "Bash(ls:*)",
+      "Bash(sqlite3:*)",
+      "Bash(stanza:*)",
+      "Bash(test:*)",
+      "Bash(tree:*)",
+      "Bash(uv add:*)",
+      "Bash(uv build:*)",
+      "Bash(uv lock:*)",
+      "Bash(uv pip:*)",
+      "Bash(uv remove:*)",
+      "Bash(uv run python:*)",
+      "Bash(uv run:*)",
+      "Bash(uv sync:*)",
+      "Bash(wc:*)",
+      "Bash(which:*)",
+      "Bash(xxd:*)",
+      "Edit(.claude/**)"
+    ],
+    "deny": [
+      "Bash(git clean:*)",
+      "Bash(git push --force:*)",
+      "Bash(git reset --hard:*)",
+      "Bash(rm -rf:*)"
+    ],
+    "ask": [
+      "Bash(git checkout .:*)",
+      "Bash(git push:*)",
+      "Bash(git rebase:*)",
+      "Bash(git restore:*)",
+      "Bash(rm:*)"
+    ]
+  }
+}

pbz2-0.1.0/.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: uv
+    directory: /
+    schedule:
+      interval: weekly

pbz2-0.1.0/.github/pull_request_template.md

@@ -0,0 +1,5 @@
+## Summary
+<!-- What does this PR do and why? -->
+
+## Test plan
+<!-- How was this tested? -->

pbz2-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,31 @@
+name: Publish
+
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v8.1.0
+      - run: uv build
+      - uses: actions/upload-artifact@v7
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

pbz2-0.1.0/.github/workflows/test.yml

@@ -0,0 +1,26 @@
+name: Tests
+
+on:
+  push:
+    branches: [dev, main]
+  pull_request:
+    branches: [dev, main]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13", "3.14"]
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v8.1.0
+      - run: uv python install ${{ matrix.python-version }}
+      - run: uv sync --all-groups --python ${{ matrix.python-version }}
+      - run: uv run ruff check .
+      - run: uv run ruff format --check .
+      - run: uv run pyrefly check
+      - run: uv run pytest --cov --cov-report=term-missing

pbz2-0.1.0/.gitignore

@@ -0,0 +1,13 @@
+.archive/
+.pytest_cache/
+.ruff_cache/
+.venv/
+__pycache__/
+
+.DS_Store
+
+.env
+.env.*
+!.env.example
+
+!.claude/settings.local.json

pbz2-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.5
+    hooks:
+      - id: ruff-format
+      - id: ruff
+        args: [--fix]
+  - repo: local
+    hooks:
+      - id: pyrefly-check
+        name: pyrefly check
+        entry: uv run pyrefly check
+        language: system
+        types_or: [python, pyi]
+        pass_filenames: false
+        require_serial: true

pbz2-0.1.0/.python-version

@@ -0,0 +1 @@
+3.14

pbz2-0.1.0/CHANGELOG.md

@@ -0,0 +1,34 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/),
+and this project adheres to [Semantic Versioning](https://semver.org/).
+
+## [Unreleased]
+
+### Added
+
+### Changed
+
+### Deprecated
+
+### Removed
+
+### Fixed
+
+### Security
+
+## [0.1.0] - 2026-05-10
+
+Initial release.
+
+### Added
+
+- `open_decompress` — file-like reader that streams a `.bz2` file through `pbzip2 -dc`.
+- `iter_chunks` — iterate over fixed-size byte chunks from a decompressed stream.
+- `iter_lines` — iterate over decoded text lines from a decompressed stream.
+- `iter_jsonl` — iterate over parsed JSON records from a decompressed JSONL stream, backed by `orjson`.
+- `process_parallel` — fan out decompressed records across worker processes.
+- `pbz2` CLI entry point (Typer-based) for streaming and processing `.bz2` files from the shell.
+- Python 3.11–3.14 support.

pbz2-0.1.0/CLAUDE.md

@@ -0,0 +1,28 @@
+# Claude Settings
+
+This file provides guidance to [Claude Code](claude.ai/code).
+
+## Package Structure
+
+```
+pbz2/
+├── cli.py              # Typer CLI entry point
+└── __init__.py
+```
+
+## Development
+
+- Install: `uv sync --all-groups`
+- Tests: `uv run pytest`
+- Linting: pre-commit hooks run ruff format + lint on commit
+- Type checking: pre-commit hooks run pyrefly on commit
+- CI: GitHub Actions runs lint + type check + test matrix (Python 3.11–3.14) on push/PR to dev/main
+
+## Release Automation
+
+Use [stanza](https://github.com/gitronald/stanza) for release workflows:
+
+```bash
+stanza release [patch|minor|major|prerelease]
+stanza init
+```

pbz2-0.1.0/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2026 Ronald E. Robertson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

pbz2-0.1.0/PKG-INFO

@@ -0,0 +1,94 @@
+Metadata-Version: 2.4
+Name: pbz2
+Version: 0.1.0
+Summary: Stream and parallel-process .bz2 files via pbzip2.
+Project-URL: repository, https://github.com/gitronald/pbz2
+Author-email: gitronald <gitronald@users.noreply.github.com>
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: orjson>=3.11.9
+Requires-Dist: typer
+Description-Content-Type: text/markdown
+
+# pbz2
+
+Stream and parallel-process `.bz2` files via [pbzip2](http://compression.great-site.net/pbzip2/) (parallel bzip2). Falls back to the stdlib `bz2` module when the `pbzip2` binary is unavailable.
+
+## Install
+
+```bash
+uv add pbz2
+```
+
+Install `pbzip2` for parallel decompression:
+
+```bash
+sudo apt install pbzip2     # Debian/Ubuntu
+brew install pbzip2         # macOS
+```
+
+## Usage
+
+### Iterate
+
+```python
+import pbz2
+
+# Parsed JSON objects from a .json.bz2 file
+for obj in pbz2.iter_jsonl("data.json.bz2"):
+    ...
+
+# Raw UTF-8 lines
+for line in pbz2.iter_lines("data.txt.bz2"):
+    ...
+
+# Newline-aligned text chunks (useful for batched processing)
+for chunk in pbz2.iter_chunks("data.txt.bz2"):
+    ...
+```
+
+### Parallel processing
+
+`process_parallel` streams chunks of newline-terminated records through a worker pool. The worker function receives raw text chunks (so parsing happens in the worker, not the main process), and `on_result` runs in the main process to handle each result as it completes.
+
+```python
+import json
+import pbz2
+
+def parse_chunk(chunk: str) -> list[dict]:
+    return [json.loads(line) for line in chunk.splitlines() if line]
+
+def save(records: list[dict]) -> None:
+    ...  # write to db, file, etc.
+
+pbz2.process_parallel(
+    "data.json.bz2",
+    worker_fn=parse_chunk,
+    on_result=save,
+    num_processes=8,
+)
+```
+
+### CLI
+
+```bash
+pbz2 count data.json.bz2
+pbz2 head data.json.bz2 -n 5
+```
+
+## API
+
+| Function | Description |
+| --- | --- |
+| `iter_chunks(path, **opts)` | Yield UTF-8 text chunks ending on a newline boundary. |
+| `iter_lines(path, **opts)` | Yield non-empty UTF-8 lines (no trailing newline). |
+| `iter_jsonl(path, *, loads=None, **opts)` | Yield parsed JSON objects (uses `orjson` if installed). |
+| `process_parallel(path, worker_fn, *, on_result=None, worker_args=(), num_processes=None, max_pending=None, ...)` | Run `worker_fn(chunk, *worker_args)` in a process pool, dispatching results to `on_result`. |
+| `open_decompress(path, **opts)` | Low-level: open a binary stream of decompressed bytes. |
+
+### Common options
+
+- `num_processors` — pbzip2 worker count (default: cpu_count - 1)
+- `bufsize_mb` — OS pipe buffer between pbzip2 and Python (default: 32 MB)
+- `stream_buffer_mb` — Python-side read chunk size (default: 4 MB)

pbz2-0.1.0/README.md

@@ -0,0 +1,81 @@
+# pbz2
+
+Stream and parallel-process `.bz2` files via [pbzip2](http://compression.great-site.net/pbzip2/) (parallel bzip2). Falls back to the stdlib `bz2` module when the `pbzip2` binary is unavailable.
+
+## Install
+
+```bash
+uv add pbz2
+```
+
+Install `pbzip2` for parallel decompression:
+
+```bash
+sudo apt install pbzip2     # Debian/Ubuntu
+brew install pbzip2         # macOS
+```
+
+## Usage
+
+### Iterate
+
+```python
+import pbz2
+
+# Parsed JSON objects from a .json.bz2 file
+for obj in pbz2.iter_jsonl("data.json.bz2"):
+    ...
+
+# Raw UTF-8 lines
+for line in pbz2.iter_lines("data.txt.bz2"):
+    ...
+
+# Newline-aligned text chunks (useful for batched processing)
+for chunk in pbz2.iter_chunks("data.txt.bz2"):
+    ...
+```
+
+### Parallel processing
+
+`process_parallel` streams chunks of newline-terminated records through a worker pool. The worker function receives raw text chunks (so parsing happens in the worker, not the main process), and `on_result` runs in the main process to handle each result as it completes.
+
+```python
+import json
+import pbz2
+
+def parse_chunk(chunk: str) -> list[dict]:
+    return [json.loads(line) for line in chunk.splitlines() if line]
+
+def save(records: list[dict]) -> None:
+    ...  # write to db, file, etc.
+
+pbz2.process_parallel(
+    "data.json.bz2",
+    worker_fn=parse_chunk,
+    on_result=save,
+    num_processes=8,
+)
+```
+
+### CLI
+
+```bash
+pbz2 count data.json.bz2
+pbz2 head data.json.bz2 -n 5
+```
+
+## API
+
+| Function | Description |
+| --- | --- |
+| `iter_chunks(path, **opts)` | Yield UTF-8 text chunks ending on a newline boundary. |
+| `iter_lines(path, **opts)` | Yield non-empty UTF-8 lines (no trailing newline). |
+| `iter_jsonl(path, *, loads=None, **opts)` | Yield parsed JSON objects (uses `orjson` if installed). |
+| `process_parallel(path, worker_fn, *, on_result=None, worker_args=(), num_processes=None, max_pending=None, ...)` | Run `worker_fn(chunk, *worker_args)` in a process pool, dispatching results to `on_result`. |
+| `open_decompress(path, **opts)` | Low-level: open a binary stream of decompressed bytes. |
+
+### Common options
+
+- `num_processors` — pbzip2 worker count (default: cpu_count - 1)
+- `bufsize_mb` — OS pipe buffer between pbzip2 and Python (default: 32 MB)
+- `stream_buffer_mb` — Python-side read chunk size (default: 4 MB)

pbz2-0.1.0/TODO.md

@@ -0,0 +1 @@
+# TODO

pbz2-0.1.0/docs/README.md

@@ -0,0 +1,13 @@
+# Documentation
+
+## Plans
+
+Implementation plans for project features and improvements. Each plan has YAML frontmatter tracking status, branch, and timestamps. Plans are linked from `TODO.md` in the project root — open tasks reference their plan file, and completed tasks retain the link as a historical record.
+
+See [`plans/`](plans/).
+
+## Guides
+
+Reference guides for project tasks or setup.
+
+See [`guides/`](guides/).

pbz2-0.1.0/pbz2/__init__.py

@@ -0,0 +1,12 @@
+"""pbz2: stream and parallel-process `.bz2` files via pbzip2."""
+
+from .parallel import process_parallel
+from .reader import iter_chunks, iter_jsonl, iter_lines, open_decompress
+
+__all__ = [
+    "iter_chunks",
+    "iter_jsonl",
+    "iter_lines",
+    "open_decompress",
+    "process_parallel",
+]

pbz2-0.1.0/pbz2/cli.py

@@ -0,0 +1,27 @@
+"""pbz2 CLI."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+
+from .reader import iter_lines
+
+app = typer.Typer(help="Stream `.bz2` files via pbzip2.")
+
+
+@app.command()
+def count(path: Path) -> None:
+    """Count lines in a `.bz2` file."""
+    n = sum(1 for _ in iter_lines(path))
+    typer.echo(n)
+
+
+@app.command()
+def head(path: Path, n: int = typer.Option(10, "-n", help="Number of lines.")) -> None:
+    """Print the first N lines of a `.bz2` file."""
+    for i, line in enumerate(iter_lines(path)):
+        if i >= n:
+            break
+        typer.echo(line)

pbz2-0.1.0/pbz2/parallel.py

@@ -0,0 +1,77 @@
+"""Parallel processing of `.bz2` streams via a process pool.
+
+Streams chunks of newline-terminated records from a `.bz2` file through a
+worker function in a `ProcessPoolExecutor`, dispatching each result to an
+optional handler in the main process.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from collections.abc import Callable, Sequence
+from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, wait
+from typing import Any
+
+from .reader import (
+    DEFAULT_BUFSIZE_MB,
+    DEFAULT_STREAM_BUFFER_MB,
+    iter_chunks,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def process_parallel(
+    path: str | os.PathLike,
+    worker_fn: Callable[..., Any],
+    *,
+    on_result: Callable[[Any], None] | None = None,
+    worker_args: Sequence[Any] = (),
+    num_processes: int | None = None,
+    max_pending: int | None = None,
+    bufsize_mb: int = DEFAULT_BUFSIZE_MB,
+    stream_buffer_mb: int = DEFAULT_STREAM_BUFFER_MB,
+) -> None:
+    """Process chunks of `path` in parallel.
+
+    `worker_fn(chunk, *worker_args)` runs in worker processes; each chunk is a
+    `str` of complete newline-terminated records. Results are dispatched to
+    `on_result` in the main process as they complete.
+
+    When `max_pending` is set, the producer pauses while that many futures are
+    in-flight to bound memory.
+    """
+    nproc = num_processes or max(1, (os.cpu_count() or 2) // 2)
+    cap = max_pending if max_pending is not None else nproc * 2
+
+    def drain_done(pending: list, *, block: bool = False) -> list:
+        if block and pending:
+            wait(pending, return_when=FIRST_COMPLETED)
+        still_pending = []
+        for fut in pending:
+            if fut.done():
+                result = fut.result()
+                if on_result is not None:
+                    on_result(result)
+            else:
+                still_pending.append(fut)
+        return still_pending
+
+    with ProcessPoolExecutor(max_workers=nproc) as executor:
+        pending: list = []
+        for chunk in iter_chunks(
+            path,
+            num_processors=nproc,
+            bufsize_mb=bufsize_mb,
+            stream_buffer_mb=stream_buffer_mb,
+        ):
+            if not chunk.strip():
+                continue
+            pending.append(executor.submit(worker_fn, chunk, *worker_args))
+            pending = drain_done(pending)
+            while len(pending) >= cap:
+                pending = drain_done(pending, block=True)
+
+        while pending:
+            pending = drain_done(pending, block=True)