ergane 0.1.0__tar.gz

ergane-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,70 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build twine
+
+      - name: Build package
+        run: python -m build
+
+      - name: Check package
+        run: twine check dist/*
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish-testpypi:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: testpypi
+
+    steps:
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+  publish-pypi:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+
+    steps:
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

ergane-0.1.0/.github/workflows/test.yml

@@ -0,0 +1,47 @@
+name: Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .[dev]
+
+      - name: Run ruff linter
+        run: ruff check src/ tests/
+
+      - name: Run ruff formatter check
+        run: ruff format --check src/ tests/
+
+      - name: Run mypy
+        run: mypy src/
+
+      - name: Run tests with coverage
+        run: pytest tests/ -v --cov=src --cov-report=term-missing --cov-report=xml
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        if: matrix.python-version == '3.12'
+        with:
+          files: ./coverage.xml
+          fail_ci_if_error: false

ergane-0.1.0/.gitignore

@@ -0,0 +1,11 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.eggs/
+*.parquet
+*.egg
+.venv/
+venv/
+.cache/

ergane-0.1.0/CHANGELOG.md

@@ -0,0 +1,26 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.0] - 2025-01-24
+
+### Added
+
+- Initial release of Ergane web crawler
+- Async HTTP/2 client using httpx for fast concurrent connections
+- Per-domain rate limiting with token bucket algorithm
+- Exponential backoff retry logic (max 3 attempts)
+- robots.txt compliance (enabled by default)
+- Fast HTML parsing with selectolax (16x faster than BeautifulSoup)
+- Priority queue scheduler with URL deduplication
+- Parquet output format via polars for efficient storage
+- Graceful shutdown handling for SIGINT/SIGTERM
+- CLI interface with configurable options:
+  - Multiple start URLs support
+  - Configurable concurrency and rate limits
+  - Depth limiting
+  - Same-domain or cross-domain crawling
+  - Custom output paths

ergane-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 pyamin1878
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ergane-0.1.0/PKG-INFO

@@ -0,0 +1,143 @@
+Metadata-Version: 2.4
+Name: ergane
+Version: 0.1.0
+Summary: High-performance async web scraper with selectolax parsing
+Project-URL: Homepage, https://github.com/pyamin1878/ergane
+Project-URL: Repository, https://github.com/pyamin1878/ergane
+Project-URL: Issues, https://github.com/pyamin1878/ergane/issues
+Author: pyamin1878
+License: MIT
+License-File: LICENSE
+Keywords: async,crawler,httpx,selectolax,spider,web-scraper
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: AsyncIO
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP
+Requires-Python: >=3.10
+Requires-Dist: click>=8.1.0
+Requires-Dist: httpx[http2]>=0.27.0
+Requires-Dist: polars>=1.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: selectolax>=0.3.21
+Provides-Extra: dev
+Requires-Dist: beautifulsoup4>=4.12.0; extra == 'dev'
+Requires-Dist: lxml>=5.0.0; extra == 'dev'
+Requires-Dist: mypy>=1.8.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.1.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.2.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Ergane
+
+[![PyPI version](https://badge.fury.io/py/ergane.svg)](https://badge.fury.io/py/ergane)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+
+High-performance async web scraper with HTTP/2 support, built with Python.
+
+*Named after Ergane, Athena's title as goddess of crafts and weaving in Greek mythology.*
+
+## Features
+
+- **HTTP/2 Support** - Fast concurrent connections via httpx
+- **Rate Limiting** - Per-domain token bucket throttling
+- **Retry Logic** - Exponential backoff (max 3 attempts)
+- **robots.txt Compliance** - Respects crawler directives by default
+- **Fast HTML Parsing** - Selectolax with CSS selector extraction (16x faster than BeautifulSoup)
+- **Smart Scheduling** - Priority queue with URL deduplication
+- **Parquet Output** - Efficient columnar storage via polars
+- **Graceful Shutdown** - Clean termination on SIGINT/SIGTERM
+
+## Installation
+
+```bash
+pip install ergane
+```
+
+For development:
+
+```bash
+pip install ergane[dev]
+```
+
+## Quick Start
+
+```bash
+# Crawl a single site
+ergane -u https://example.com -n 100
+
+# Crawl multiple start URLs
+ergane -u https://site1.com -u https://site2.com -n 500
+
+# Custom output and settings
+ergane -u https://docs.python.org -n 50 -c 20 -r 5 -o python_docs.parquet
+```
+
+## CLI Options
+
+| Option | Short | Default | Description |
+|--------|-------|---------|-------------|
+| `--url` | `-u` | required | Start URL(s), can specify multiple |
+| `--output` | `-o` | `output.parquet` | Output file path |
+| `--max-pages` | `-n` | `100` | Maximum pages to crawl |
+| `--max-depth` | `-d` | `3` | Maximum crawl depth from start URLs |
+| `--concurrency` | `-c` | `10` | Concurrent requests |
+| `--rate-limit` | `-r` | `10.0` | Requests per second per domain |
+| `--timeout` | `-t` | `30.0` | Request timeout in seconds |
+| `--same-domain` | | `true` | Stay on same domain as start URLs |
+| `--any-domain` | | `false` | Follow links to any domain |
+| `--ignore-robots` | | `false` | Ignore robots.txt restrictions |
+
+## Output Format
+
+Results are saved as a Parquet file with the following schema:
+
+| Column | Type | Description |
+|--------|------|-------------|
+| `url` | string | Page URL |
+| `title` | string | Page title |
+| `text` | string | Extracted text content (max 10k chars) |
+| `links` | string | JSON array of extracted links |
+| `extracted_data` | string | JSON object of custom extractions |
+| `crawled_at` | string | ISO timestamp |
+
+Read results with polars:
+
+```python
+import polars as pl
+
+df = pl.read_parquet("output.parquet")
+print(df.head())
+```
+
+## Benchmarks
+
+Ergane uses selectolax for HTML parsing, which is significantly faster than BeautifulSoup:
+
+| Operation         | Selectolax | BS4 + lxml | Speedup |
+|-------------------|------------|------------|---------|
+| Parse (small)     | 0.05ms     | 0.11ms     | 2.0x    |
+| Parse (large)     | 0.19ms     | 6.05ms     | 31.1x   |
+| Extract title     | 0.20ms     | 6.06ms     | 30.7x   |
+| Extract links     | 0.25ms     | 6.73ms     | 27.3x   |
+| Extract text      | 0.29ms     | 7.03ms     | 24.5x   |
+| CSS selector      | 0.20ms     | 7.25ms     | 35.7x   |
+
+**Average: 16x faster** (1000 iterations, 34KB HTML)
+
+Run the benchmark:
+```bash
+pip install beautifulsoup4 lxml
+python benchmarks/parse_benchmark.py
+```
+
+## License
+
+MIT

ergane-0.1.0/README.md

@@ -0,0 +1,107 @@
+# Ergane
+
+[![PyPI version](https://badge.fury.io/py/ergane.svg)](https://badge.fury.io/py/ergane)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+
+High-performance async web scraper with HTTP/2 support, built with Python.
+
+*Named after Ergane, Athena's title as goddess of crafts and weaving in Greek mythology.*
+
+## Features
+
+- **HTTP/2 Support** - Fast concurrent connections via httpx
+- **Rate Limiting** - Per-domain token bucket throttling
+- **Retry Logic** - Exponential backoff (max 3 attempts)
+- **robots.txt Compliance** - Respects crawler directives by default
+- **Fast HTML Parsing** - Selectolax with CSS selector extraction (16x faster than BeautifulSoup)
+- **Smart Scheduling** - Priority queue with URL deduplication
+- **Parquet Output** - Efficient columnar storage via polars
+- **Graceful Shutdown** - Clean termination on SIGINT/SIGTERM
+
+## Installation
+
+```bash
+pip install ergane
+```
+
+For development:
+
+```bash
+pip install ergane[dev]
+```
+
+## Quick Start
+
+```bash
+# Crawl a single site
+ergane -u https://example.com -n 100
+
+# Crawl multiple start URLs
+ergane -u https://site1.com -u https://site2.com -n 500
+
+# Custom output and settings
+ergane -u https://docs.python.org -n 50 -c 20 -r 5 -o python_docs.parquet
+```
+
+## CLI Options
+
+| Option | Short | Default | Description |
+|--------|-------|---------|-------------|
+| `--url` | `-u` | required | Start URL(s), can specify multiple |
+| `--output` | `-o` | `output.parquet` | Output file path |
+| `--max-pages` | `-n` | `100` | Maximum pages to crawl |
+| `--max-depth` | `-d` | `3` | Maximum crawl depth from start URLs |
+| `--concurrency` | `-c` | `10` | Concurrent requests |
+| `--rate-limit` | `-r` | `10.0` | Requests per second per domain |
+| `--timeout` | `-t` | `30.0` | Request timeout in seconds |
+| `--same-domain` | | `true` | Stay on same domain as start URLs |
+| `--any-domain` | | `false` | Follow links to any domain |
+| `--ignore-robots` | | `false` | Ignore robots.txt restrictions |
+
+## Output Format
+
+Results are saved as a Parquet file with the following schema:
+
+| Column | Type | Description |
+|--------|------|-------------|
+| `url` | string | Page URL |
+| `title` | string | Page title |
+| `text` | string | Extracted text content (max 10k chars) |
+| `links` | string | JSON array of extracted links |
+| `extracted_data` | string | JSON object of custom extractions |
+| `crawled_at` | string | ISO timestamp |
+
+Read results with polars:
+
+```python
+import polars as pl
+
+df = pl.read_parquet("output.parquet")
+print(df.head())
+```
+
+## Benchmarks
+
+Ergane uses selectolax for HTML parsing, which is significantly faster than BeautifulSoup:
+
+| Operation         | Selectolax | BS4 + lxml | Speedup |
+|-------------------|------------|------------|---------|
+| Parse (small)     | 0.05ms     | 0.11ms     | 2.0x    |
+| Parse (large)     | 0.19ms     | 6.05ms     | 31.1x   |
+| Extract title     | 0.20ms     | 6.06ms     | 30.7x   |
+| Extract links     | 0.25ms     | 6.73ms     | 27.3x   |
+| Extract text      | 0.29ms     | 7.03ms     | 24.5x   |
+| CSS selector      | 0.20ms     | 7.25ms     | 35.7x   |
+
+**Average: 16x faster** (1000 iterations, 34KB HTML)
+
+Run the benchmark:
+```bash
+pip install beautifulsoup4 lxml
+python benchmarks/parse_benchmark.py
+```
+
+## License
+
+MIT