gharc 0.1.0__py3-none-any.whl

gharc/__init__.py

@@ -0,0 +1,19 @@
+__version__ = "0.1.0"
+
+from .filters import passes_filters, fast_string_check
+from .storage import DataWriter, jsonl_to_parquet
+from .streamer import process_range
+from .utils import parse_date, date_range, get_url_for_time, setup_logging
+
+__all__ = [
+    "__version__",
+    "passes_filters",
+    "fast_string_check",
+    "DataWriter",
+    "jsonl_to_parquet",
+    "process_range",
+    "parse_date",
+    "date_range",
+    "get_url_for_time",
+    "setup_logging",
+]

gharc/cli.py

@@ -0,0 +1,48 @@
+# src/gharc/cli.py
+import click
+import sys
+from .utils import parse_date, logger, setup_logging
+from .streamer import process_range
+from .storage import jsonl_to_parquet
+
+@click.group()
+def main():
+    """gharc: Stream-filter GitHub Archive data."""
+    setup_logging()
+
+@main.command()
+@click.option('--start', required=True, help='Start date (YYYY-MM-DD-HH)')
+@click.option('--end', required=True, help='End date (YYYY-MM-DD-HH)')
+@click.option('--repos', help='Comma-separated repos (e.g. apache/spark)')
+@click.option('--event-types', help='Comma-separated events (e.g. PushEvent)')
+@click.option('--output', default='filtered.jsonl', help='Output file')
+@click.option('--workers', default=4, help='Parallel downloads')
+def download(start, end, repos, event_types, output, workers):
+    try:
+        s_dt = parse_date(start)
+        e_dt = parse_date(end)
+        repo_list = [r.strip() for r in repos.split(',')] if repos else None
+        type_list = [t.strip() for t in event_types.split(',')] if event_types else None
+        
+        process_range(s_dt, e_dt, repo_list, type_list, output, workers)
+        
+    except Exception as e:
+        logger.error(str(e))
+        sys.exit(1)
+
+
+@main.command()
+@click.argument('input_path', type=click.Path(exists=True, dir_okay=False))
+@click.argument('output_path', type=click.Path(dir_okay=False))
+@click.option('--batch-size', default=10000, help='Rows per Parquet row group')
+def convert(input_path, output_path, batch_size):
+    """Convert a JSONL output from `gharc download` into a single Parquet file."""
+    try:
+        jsonl_to_parquet(input_path, output_path, batch_size=batch_size)
+    except Exception as e:
+        logger.error(str(e))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

gharc/filters.py

@@ -0,0 +1,29 @@
+# src/gharc/filters.py
+import json
+
+def passes_filters(event_dict: dict, repos: list = None, event_types: list = None) -> bool:
+    """
+    Returns True if the event matches ANY of the provided filters.
+    If a filter list is None, it is ignored (pass-through).
+    """
+    # 1. Filter by Repo Name
+    if repos:
+        repo_name = event_dict.get('repo', {}).get('name')
+        if repo_name not in repos:
+            return False
+            
+    # 2. Filter by Event Type
+    if event_types:
+        if event_dict.get('type') not in event_types:
+            return False
+            
+    return True
+
+def fast_string_check(line: str, tokens: list) -> bool:
+    """
+    Optimization: Returns False if NONE of the tokens appear in the string.
+    Avoids expensive JSON parsing for lines that definitely don't match.
+    """
+    if not tokens:
+        return True
+    return any(t in line for t in tokens)

gharc/storage.py

@@ -0,0 +1,136 @@
+# src/gharc/storage.py
+import json
+import os
+import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
+from .utils import logger
+
+
+class DataWriter:
+    def __init__(self, filename: str, append: bool = False):
+        self.filename = filename
+        self.is_parquet = filename.endswith('.parquet')
+        self.buffer = []
+        self.buffer_size = 10000
+        self._pq_writer = None
+
+        if append and self.is_parquet and os.path.exists(self.filename):
+            # ParquetWriter cannot append to a closed Parquet file. For long
+            # crash-safe runs use JSONL; convert to Parquet at the end.
+            raise ValueError(
+                f"Cannot resume into existing Parquet file {filename}. "
+                f"Use JSONL output for resumable runs and convert at the end."
+            )
+
+        if not append and os.path.exists(self.filename):
+            os.remove(self.filename)
+
+    def write(self, record: dict):
+        self.buffer.append(record)
+        if len(self.buffer) >= self.buffer_size:
+            self.flush()
+
+    def flush(self):
+        if not self.buffer:
+            return
+
+        if self.is_parquet:
+            rows = [_flatten_event(e) for e in self.buffer]
+            df = pd.DataFrame(rows)
+            table = pa.Table.from_pandas(df, preserve_index=False)
+
+            if self._pq_writer is None:
+                self._pq_writer = pq.ParquetWriter(
+                    self.filename,
+                    schema=table.schema,
+                    compression='snappy',
+                )
+            else:
+                # Cast to the schema we opened with; event payloads vary in shape.
+                table = table.cast(self._pq_writer.schema, safe=False)
+
+            self._pq_writer.write_table(table)
+        else:
+            with open(self.filename, 'a', encoding='utf-8') as f:
+                for rec in self.buffer:
+                    f.write(json.dumps(rec) + '\n')
+
+        self.buffer = []
+
+    def close(self):
+        self.flush()
+        if self._pq_writer is not None:
+            self._pq_writer.close()
+            self._pq_writer = None
+        logger.info(f"Wrote output to {self.filename}")
+
+
+def _flatten_event(event: dict) -> dict:
+    # JSON-stringify nested fields so Parquet sees a stable flat schema.
+    out = {}
+    for key, value in event.items():
+        if isinstance(value, (dict, list)):
+            out[key] = json.dumps(value, ensure_ascii=False)
+        else:
+            out[key] = value
+    return out
+
+
+def jsonl_to_parquet(input_path: str, output_path: str, batch_size: int = 10000) -> int:
+    """Stream a JSONL file into a single Parquet file.
+
+    Reads `input_path` line by line, batches into Parquet row groups of up to
+    `batch_size` rows, and writes to `output_path`. Returns the number of rows
+    written. Designed to handle multi-GB inputs without loading the whole file
+    into memory.
+    """
+    if os.path.exists(output_path):
+        os.remove(output_path)
+
+    writer = None
+    buffer = []
+    rows_written = 0
+
+    def flush():
+        nonlocal writer
+        if not buffer:
+            return
+        rows = [_flatten_event(e) for e in buffer]
+        df = pd.DataFrame(rows)
+        table = pa.Table.from_pandas(df, preserve_index=False)
+        if writer is None:
+            writer = pq.ParquetWriter(
+                output_path,
+                schema=table.schema,
+                compression='snappy',
+            )
+        else:
+            table = table.cast(writer.schema, safe=False)
+        writer.write_table(table)
+
+    try:
+        with open(input_path, 'r', encoding='utf-8') as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    buffer.append(json.loads(line))
+                except json.JSONDecodeError:
+                    logger.warning(f"Skipping malformed JSON line in {input_path}")
+                    continue
+                if len(buffer) >= batch_size:
+                    flush()
+                    rows_written += len(buffer)
+                    buffer.clear()
+
+        if buffer:
+            flush()
+            rows_written += len(buffer)
+    finally:
+        if writer is not None:
+            writer.close()
+
+    logger.info(f"Converted {rows_written:,} rows from {input_path} to {output_path}")
+    return rows_written

gharc/streamer.py

@@ -0,0 +1,249 @@
+# src/gharc/streamer.py
+import requests
+import gzip
+import json
+import concurrent.futures
+import threading
+import time
+import os
+import tempfile
+from datetime import datetime
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+from tqdm import tqdm
+from .utils import get_url_for_time, date_range, logger
+from .filters import passes_filters, fast_string_check
+from .storage import DataWriter
+
+# Use orjson if available for 3-5x faster parsing
+try:
+    import orjson
+    HAS_ORJSON = True
+except ImportError:
+    HAS_ORJSON = False
+
+_thread_local = threading.local()
+
+def get_robust_session():
+    """Creates a requests session with retry logic."""
+    session = requests.Session()
+    retry_strategy = Retry(
+        total=5,
+        backoff_factor=1,
+        status_forcelist=[429, 500, 502, 503, 504],
+        allowed_methods=["HEAD", "GET", "OPTIONS"]
+    )
+    adapter = HTTPAdapter(max_retries=retry_strategy)
+    session.mount("https://", adapter)
+    session.mount("http://", adapter)
+    return session
+
+def _session_for_thread() -> requests.Session:
+    # One session per worker thread so connection pooling actually kicks in.
+    session = getattr(_thread_local, "session", None)
+    if session is None:
+        session = get_robust_session()
+        _thread_local.session = session
+    return session
+
+def download_resumable(url: str, temp_path: str, session: requests.Session) -> bool:
+    """
+    Downloads a file with resume capability.
+    """
+    resume_header = {}
+    mode = 'wb'
+    if os.path.exists(temp_path):
+        current_size = os.path.getsize(temp_path)
+        if current_size > 0:
+            resume_header = {'Range': f'bytes={current_size}-'}
+            mode = 'ab'
+            # Only log resume if it's significant to keep bar clean
+            if current_size > 1024 * 1024: 
+                tqdm.write(f"   ↳ Resuming from {current_size/(1024*1024):.1f} MB")
+
+    try:
+        with session.get(url, headers=resume_header, stream=True, timeout=(30, 120)) as r:
+            if r.status_code == 416: # Range not satisfiable (file done)
+                return True
+            if r.status_code not in [200, 206]:
+                logger.debug(f"HTTP {r.status_code} for {url}")
+                return False
+
+            if r.status_code == 200 and mode == 'ab':
+                mode = 'wb'
+
+            with open(temp_path, mode) as f:
+                for chunk in r.iter_content(chunk_size=65536):
+                    if chunk:
+                        f.write(chunk)
+        return True
+    except Exception as e:
+        logger.debug(f"Download attempt failed for {url}: {e}")
+        return False
+
+def process_single_hour(dt: datetime, repos: list, event_types: list) -> list:
+    """
+    Downloads with resume -> Process -> Delete.
+    """
+    url = get_url_for_time(dt)
+    results = []
+    # Convert filters to bytes if using orjson for speed
+    if HAS_ORJSON:
+        fast_tokens = [t.encode('utf-8') for t in ((repos if repos else []) + (event_types if event_types else []))]
+    else:
+        fast_tokens = (repos if repos else []) + (event_types if event_types else [])
+        
+    session = _session_for_thread()
+
+    fd, temp_path = tempfile.mkstemp(suffix=".json.gz")
+    os.close(fd)
+    
+    download_success = False
+    
+    try:
+        for attempt in range(10):
+            if download_resumable(url, temp_path, session):
+                download_success = True
+                break
+            time.sleep(2)
+            
+        if not download_success:
+            tqdm.write(f"Failed to download {url} after 10 attempts (run with debug logging for details)")
+            return []
+
+        try:
+            with gzip.open(temp_path, 'rb') as f:
+                for line in f:
+                    try:
+                        # OPTIMIZATION: Check tokens before full parse
+                        # orjson returns bytes, so we don't even need to decode to utf-8 yet
+                        if HAS_ORJSON:
+                            if fast_tokens:
+                                # Simple byte-level check (very fast)
+                                if not any(t in line for t in fast_tokens):
+                                    continue
+                            event = orjson.loads(line)
+                        else:
+                            # Standard Fallback
+                            decoded = line.decode('utf-8')
+                            if fast_tokens and not fast_string_check(decoded, fast_tokens):
+                                continue
+                            event = json.loads(decoded)
+
+                        if passes_filters(event, repos, event_types):
+                            results.append(event)
+                    except Exception:
+                        # GHArchive occasionally has malformed lines; expected at low rates.
+                        continue
+        except Exception as e:
+             tqdm.write(f"Error reading gzip for {url}: {e}")
+
+        return results
+
+    finally:
+        if os.path.exists(temp_path):
+            os.remove(temp_path)
+
+class _RunState:
+    """Tracks which hours have been completed so a crashed run can resume.
+
+    State lives next to the output file as <output>.state.json. The fingerprint
+    of the run (window + filters) is stored alongside the done-hour list, so
+    re-running with different filters against the same output triggers a clear
+    error rather than silently mixing data.
+    """
+
+    def __init__(self, output_path, fingerprint):
+        self._path = str(output_path) + ".state.json"
+        self._fingerprint = fingerprint
+        self._done = set()
+        if os.path.exists(self._path):
+            try:
+                with open(self._path, "r", encoding="utf-8") as f:
+                    payload = json.load(f)
+            except (json.JSONDecodeError, OSError):
+                logger.warning(f"State file {self._path} unreadable, starting fresh")
+                return
+            if payload.get("fingerprint") != fingerprint:
+                raise ValueError(
+                    f"State file {self._path} was written for a different run "
+                    f"(window or filters changed). Remove it or use a new --output."
+                )
+            self._done = set(payload.get("done_hours", []))
+
+    def is_done(self, ts):
+        return ts.isoformat() in self._done
+
+    def mark_done(self, ts):
+        self._done.add(ts.isoformat())
+        with open(self._path, "w", encoding="utf-8") as f:
+            json.dump(
+                {"fingerprint": self._fingerprint, "done_hours": sorted(self._done)},
+                f,
+            )
+
+    def clear(self):
+        if os.path.exists(self._path):
+            os.remove(self._path)
+
+
+def _run_fingerprint(start, end, repos, event_types):
+    return {
+        "start": start.isoformat(),
+        "end": end.isoformat(),
+        "repos": sorted(repos) if repos else None,
+        "event_types": sorted(event_types) if event_types else None,
+    }
+
+
+def process_range(start, end, repos, event_types, output, workers):
+    fingerprint = _run_fingerprint(start, end, repos, event_types)
+    state = _RunState(output, fingerprint)
+
+    resuming = bool(state._done)
+    writer = DataWriter(output, append=resuming)
+    all_timestamps = list(date_range(start, end))
+    todo = [t for t in all_timestamps if not state.is_done(t)]
+    skipped = len(all_timestamps) - len(todo)
+    if skipped:
+        logger.info(f"Resuming: skipping {skipped} hours already in state file")
+
+    if not todo:
+        writer.close()
+        logger.info(f"Nothing to do; output already complete at {output}")
+        return
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
+        future_to_time = {
+            executor.submit(process_single_hour, ts, repos, event_types): ts
+            for ts in todo
+        }
+
+        with tqdm(
+            total=len(todo),
+            desc="Processing",
+            unit="hr",
+            smoothing=0,
+            dynamic_ncols=True,
+        ) as pbar:
+
+            for future in concurrent.futures.as_completed(future_to_time):
+                ts = future_to_time[future]
+                try:
+                    data = future.result()
+                    if data:
+                        for record in data:
+                            writer.write(record)
+                    # Flush so this hour is durable on disk before we mark it
+                    # done. If the process crashes after mark_done, restart
+                    # skips this hour and the data is already written.
+                    writer.flush()
+                    state.mark_done(ts)
+                except Exception as exc:
+                    tqdm.write(f"Worker exception for {ts}: {exc}")
+                finally:
+                    pbar.update(1)
+
+    writer.close()
+    state.clear()
+    logger.info(f"Done! Data written to {output}")

gharc/utils.py

@@ -0,0 +1,53 @@
+# src/gharc/utils.py
+import logging
+from datetime import datetime, timedelta
+from typing import Iterator
+
+logger = logging.getLogger("gharc")
+
+
+def setup_logging(level: int = logging.INFO) -> None:
+    """Attach a console handler to the gharc logger. Safe to call twice."""
+    if logger.handlers:
+        return
+    handler = logging.StreamHandler()
+    handler.setFormatter(logging.Formatter(
+        '[%(asctime)s] %(levelname)s: %(message)s',
+        datefmt='%H:%M:%S',
+    ))
+    logger.addHandler(handler)
+    logger.setLevel(level)
+
+def parse_date(date_str: str) -> datetime:
+    """Parses YYYY-MM-DD or YYYY-MM-DD-HH"""
+    try:
+        if len(date_str.split('-')) == 3:
+            return datetime.strptime(date_str, "%Y-%m-%d")
+        else:
+            return datetime.strptime(date_str, "%Y-%m-%d-%H")
+    except ValueError:
+        raise ValueError(f"Invalid date format: {date_str}. Use YYYY-MM-DD or YYYY-MM-DD-HH")
+
+def date_range(start: datetime, end: datetime) -> Iterator[datetime]:
+    """Yields hourly datetimes in [start, end). End is exclusive.
+
+    Use --end 2024-02-01 to cover all of January 2024.
+    """
+    current = start.replace(minute=0, second=0, microsecond=0)
+    end_rounded = end.replace(minute=0, second=0, microsecond=0)
+
+    while current < end_rounded:
+        yield current
+        current += timedelta(hours=1)
+
+def get_url_for_time(dt: datetime) -> str:
+    """
+    Constructs the GHArchive URL for a specific hour.
+    Example: 2024-01-01 15:00 -> https://data.gharchive.org/2024-01-01-15.json.gz
+    """
+    # GHArchive uses 24-hour format without leading zeros for hours 0-9? 
+    # Actually checking standard GHArchive urls: 2024-01-01-1.json.gz or 01.json.gz?
+    # GHArchive documentation says: {year}-{month}-{day}-{hour}.json.gz
+    # Hour is usually 0-23 (no leading zero required by spec, but usually provided).
+    # Let's use simple integer formatting which works for their redirects.
+    return f"https://data.gharchive.org/{dt.year}-{dt.month:02d}-{dt.day:02d}-{dt.hour}.json.gz"

gharc-0.1.0.dist-info/METADATA

@@ -0,0 +1,261 @@
+Metadata-Version: 2.4
+Name: gharc
+Version: 0.1.0
+Summary: A stream-processing tool for GitHub Archive data filtering.
+Author-email: Arav Panwar <aravpanwar@outlook.com>
+Project-URL: Homepage, https://github.com/aravpanwar/gharc
+Project-URL: Bug Tracker, https://github.com/aravpanwar/gharc/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.28.0
+Requires-Dist: click>=8.0.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: pyarrow>=12.0.0
+Requires-Dist: tqdm>=4.65.0
+Provides-Extra: fast
+Requires-Dist: orjson>=3.9.0; extra == "fast"
+Provides-Extra: test
+Requires-Dist: pytest>=7.0; extra == "test"
+Dynamic: license-file
+
+# gharc: GitHub Archive Stream-Processor
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Tests](https://github.com/aravpanwar/gharc/actions/workflows/test.yml/badge.svg)](https://github.com/aravpanwar/gharc/actions)
+[![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![DOI](https://zenodo.org/badge/1112791047.svg)](https://doi.org/10.5281/zenodo.19814232)
+
+**Mine the GitHub Archive on a standard laptop.**
+
+`gharc` is a command-line tool and Python library that filters the [GitHub Archive](https://www.gharchive.org/) dataset on consumer hardware. Each hourly archive is streamed through memory, filtered against your criteria, and written out as Parquet or JSONL. Peak local storage stays bounded by a single in-flight download (about 150 MB) regardless of how long a window you process.
+
+---
+
+## Why gharc?
+
+The full GitHub Archive dataset exceeds petabytes in size. Traditional analysis requires either massive local storage or expensive cloud warehousing (BigQuery).
+
+`gharc` solves this by implementing a **Stream-and-Filter** architecture:
+1.  **Streaming:** Downloads each hourly archive (~60 to 150 MB compressed in 2024) to a temporary file.
+2.  **Filtering:** Extracts only events matching your criteria (e.g., specific repos or event types).
+3.  **Writing:** Streams matching events into a single **Parquet** or **JSONL** file via `pyarrow.ParquetWriter` for true append.
+4.  **Cleanup:** Deletes the temporary download immediately after, so disk usage never accumulates.
+
+**Ideal for:**
+- Academic research on Open Source Software (OSS).
+- Large scale data mining on consumer hardware.
+- Creating custom datasets for specific organizations or ecosystems.
+
+![Architecture: GHArchive HTTPS to thread pool to resumable download to temp file to streaming decode and filter to DataWriter to output file.](paper/figures/architecture.png)
+
+---
+
+## Key Features
+
+* **Zero-Storage Overhead:** Processes terabytes of data with a constant disk footprint of <100MB.
+* **Resumable Downloads:** Smart handling of network interruptions (common with residential internet) using HTTP Range requests.
+* **High Performance:**
+    * Parallel processing with thread pools.
+    * Optimized "Fast String Check" (zero-copy filtering) to skip irrelevant data.
+    * Optional `orjson` support for 3-5x faster parsing.
+* **Parquet Native:** Outputs columnar data ready for Pandas, Spark, or Polars, often reducing file size by 90% compared to JSON.
+
+---
+
+## Performance
+
+Measured on a Windows 11 laptop (12 logical cores, 15 GB RAM) over a typical residential connection. Reproducible scripts in [`benchmarks/`](benchmarks/).
+
+A six-hour window of GHArchive (2024-01-01 00:00 to 06:00 UTC), filtered to `apache/spark`:
+
+| Workers | Wall-clock | Hours/sec | Spark events | Peak RSS |
+|---|---|---|---|---|
+| 1 | 76.0 s | 0.079 | 14 | 94.2 MB |
+| 4 | 58.1 s | 0.103 | 14 | 106.7 MB |
+
+Both runs recovered the same events, so concurrency does not affect output. Peak RSS stays below 110 MB. The bottleneck on residential links is HTTPS download throughput rather than CPU; additional workers help up to a point and then saturate the connection.
+
+The same six-hour window comprises about 1.2 GB of compressed source on the GHArchive side, while the filtered Parquet output is 53 KB. That is a storage saving of roughly 22,000 to 1, and at no point does peak local disk exceed the size of a single in-flight temporary file (about 150 MB).
+
+---
+
+## Installation
+
+### Prerequisites
+- Python 3.8 or higher
+- `pip`
+
+### Install from Source
+```bash
+git clone https://github.com/aravpanwar/gharc.git
+cd gharc
+python3 -m venv venv
+source venv/bin/activate
+pip install -e .
+```
+
+### Optional Performance Boost
+
+For maximum speed, install with the `fast` extra. `gharc` detects and uses `orjson` automatically when available.
+
+```bash
+pip install -e ".[fast]"
+```
+
+---
+
+## Usage
+
+### Basic Command
+
+Download all activity for a specific repository over a one-day window.
+Note that `--end` is exclusive, so this covers all 24 hours of 2024-01-01.
+
+```bash
+gharc download \
+    --start 2024-01-01 \
+    --end 2024-01-02 \
+    --repos "apache/spark" \
+    --output spark_data.parquet
+
+```
+
+### Advanced Filtering
+
+Filter for multiple repositories and specific event types (e.g., only Pull Requests and Pushes).
+This covers all of June 2023 (June 1 inclusive through July 1 exclusive).
+
+```bash
+gharc download \
+    --start 2023-06-01 \
+    --end 2023-07-01 \
+    --repos "apache/spark, pandas-dev/pandas, pytorch/pytorch" \
+    --event-types "PullRequestEvent, PushEvent" \
+    --output oss_summer_2023.parquet \
+    --workers 4
+
+```
+
+### Arguments
+
+| Argument | Description | Example |
+| --- | --- | --- |
+| `--start` | Start date, inclusive (YYYY-MM-DD or YYYY-MM-DD-HH) | `2024-01-01` |
+| `--end` | End date, exclusive (YYYY-MM-DD or YYYY-MM-DD-HH) | `2024-02-01` |
+| `--repos` | Comma-separated list of repositories to keep | `apache/spark,tensorflow/tensorflow` |
+| `--event-types` | Comma-separated list of GHArchive event types | `WatchEvent,ForkEvent` |
+| `--output` | Output filename (`.parquet` or `.jsonl`) | `data.parquet` |
+| `--workers` | Number of parallel download threads (default: 4) | `8` |
+
+---
+
+## Resumable runs
+
+For long jobs, `gharc` keeps a small `<output>.state.json` next to the output file listing which hours it has already processed. If the run crashes, restarting the same command picks up where it left off rather than redoing completed hours. The state file is removed automatically when the run finishes cleanly.
+
+Resume support requires JSONL output. Parquet writers cannot append to a closed file, so for multi-hour runs use `--output run.jsonl` and convert to Parquet at the end:
+
+```bash
+gharc convert run.jsonl run.parquet
+```
+
+---
+
+## Python API
+
+The CLI is a thin wrapper around `gharc.process_range`, which you can call directly:
+
+```python
+from datetime import datetime
+import gharc
+
+gharc.setup_logging()
+gharc.process_range(
+    start=datetime(2024, 1, 1),
+    end=datetime(2024, 1, 2),
+    repos=["apache/spark"],
+    event_types=None,
+    output="spark_one_day.jsonl",
+    workers=4,
+)
+
+gharc.jsonl_to_parquet("spark_one_day.jsonl", "spark_one_day.parquet")
+```
+
+`__all__` in `gharc/__init__.py` lists the public surface (`process_range`, `jsonl_to_parquet`, `DataWriter`, `parse_date`, `date_range`, `get_url_for_time`, `setup_logging`, plus the filter helpers).
+
+---
+
+##  Automating Bulk Downloads
+
+For long date ranges, the included [`examples/orchestrator.py`](examples/orchestrator.py) script runs `gharc` month by month so each year produces one Parquet file per month rather than one giant output:
+
+```bash
+python examples/orchestrator.py \
+    --start 2023-01-01 \
+    --end 2024-01-01 \
+    --repos "apache/spark,pandas-dev/pandas" \
+    --output-dir ./gharc_out \
+    --workers 4
+```
+
+---
+
+## Repository Layout
+
+```
+gharc/
+├── src/gharc/        # Library + CLI entry point
+├── tests/            # pytest test suite
+├── benchmarks/       # Reproducible runs that back the performance claims
+├── examples/         # Driver scripts (e.g. month-by-month orchestrator)
+├── paper/            # paper.md, paper.bib, figures (the JOSS submission)
+└── CITATION.cff      # GitHub-detectable citation metadata
+```
+
+---
+
+## Contributing
+
+Contributions are welcome. Please read [CONTRIBUTING.md](CONTRIBUTING.md) for details on the process for submitting pull requests.
+
+**Running Tests:**
+
+```bash
+pip install -e ".[test]"
+pytest tests/
+```
+
+---
+
+## Citation
+
+The accompanying paper is at [`paper/paper.pdf`](paper/paper.pdf) and is rebuilt automatically on every push by the [Paper CI workflow](.github/workflows/paper.yml).
+
+If you use `gharc` in your research, please cite it using the metadata in `CITATION.cff` or as follows:
+
+```bibtex
+@software{gharc2026,
+  author = {Panwar, Arav},
+  title = {gharc: A stream-and-filter tool for the GitHub Archive on consumer hardware},
+  year = {2026},
+  url = {https://github.com/aravpanwar/gharc}
+}
+
+```
+
+---
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+Created by Arav Panwar
+[aravpanwar.com](https://www.aravpanwar.com)
+

gharc-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+gharc/__init__.py,sha256=ueXnxVXSj8uUDs896_Xy2WC7zRJS-7vvBgWHgqNsq8c,464
+gharc/cli.py,sha256=VOwHaP8GW43Wsasj5VaW7ScxIJsgYHAKwIFHGnAqaMA,1808
+gharc/filters.py,sha256=lZus0uhC_4v6wct6GpEtAX8QZGlJyROIBD2tbGcj2k8,934
+gharc/storage.py,sha256=ExJe9NtzZxCqlkZ0f7czjjt_rF3MtdVSN-cz8rVI3mU,4398
+gharc/streamer.py,sha256=QxpUf0jv6hPalhdhYF_CUYY40iVhd2l3IgZIPdjOVmk,9170
+gharc/utils.py,sha256=HZjOC_HS3NbTPIspxOVulMkxKP-P1XocLMOvzelEAdg,2090
+gharc-0.1.0.dist-info/licenses/LICENSE,sha256=hLV7YOq2UYYw8ediyrCxcRSUos0HsY5SgDchmEhFWwk,1089
+gharc-0.1.0.dist-info/METADATA,sha256=DP_5BWZaIukrLG3kRvEg9dwxGRBoTHjTi5Ho4uSPudU,9572
+gharc-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+gharc-0.1.0.dist-info/entry_points.txt,sha256=Zlj3sVYsSFLRTsk1zFdzeWcMmeFP2KCJHhLQK6KpcTw,41
+gharc-0.1.0.dist-info/top_level.txt,sha256=DvL9ErtXeThzfYDkvOo2dscEIKt7UDXH1WMZmYEjSxw,6
+gharc-0.1.0.dist-info/RECORD,,

gharc-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

gharc-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+gharc = gharc.cli:main

gharc-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Arav Panwar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

gharc-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+gharc