onyxweb 0.2.0__tar.gz

onyxweb-0.2.0/.cargo/config.toml

@@ -0,0 +1,3 @@
+# No V8 anymore. Chromium is a subprocess now, not a linked library.
+# This file is empty/unused in the CDP engine era. Kept as a placeholder so
+# anyone consulting .cargo/ knows no special env vars are required.

onyxweb-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,142 @@
+# Cross-platform wheel matrix. Wheels are uploaded as artifacts — no auto-
+# publish. Trigger: push to master, PRs, and manual workflow_dispatch.
+#
+# Platforms target what `onyxweb-download-chrome` can stage a binary for:
+# linux x86_64 / aarch64, darwin x86_64 / arm64, windows x86_64. Adding a
+# platform requires both a matrix entry here AND a PLATFORMS row in
+# python/onyxweb/_download_chrome.py.
+
+name: CI
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    # Static gates: ruff, mypy strict, cargo fmt, cargo clippy. Fast — runs
+    # on every PR / push to master alongside the wheel-build matrix.
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
+      - name: Sync dev deps (builds Rust ext via cache-keys)
+        run: uv sync --group dev
+      - name: Ruff
+        run: uv run ruff check python/ tests/
+      - name: Mypy (strict)
+        run: uv run mypy python/onyxweb/ tests/
+      - name: Cargo fmt
+        run: cargo fmt --all -- --check
+      - name: Cargo clippy
+        run: cargo clippy --all-targets -- -D warnings
+
+  linux:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64, aarch64]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Build wheel (manylinux_2_28)
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist --find-interpreter
+          manylinux: "2_28"
+          sccache: true
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-linux-${{ matrix.target }}
+          path: dist
+
+  macos:
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: macos-latest
+            target: aarch64
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Build wheel
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist --find-interpreter
+          sccache: true
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-macos-${{ matrix.target }}
+          path: dist
+
+  windows:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          architecture: x64
+      - name: Build wheel
+        uses: PyO3/maturin-action@v1
+        with:
+          target: x64
+          args: --release --out dist --find-interpreter
+          sccache: true
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-windows-x86_64
+          path: dist
+
+  sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: dist
+
+  test:
+    # Smoke-test the linux x86_64 wheel end-to-end: install into a fresh venv,
+    # fetch chrome-headless-shell via the new `onyxweb --install` path, run
+    # the default pytest suite. Skips the benchmark tests (-m 'not benchmark'
+    # from pyproject addopts).
+    needs: [linux]
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - uses: actions/download-artifact@v4
+        with:
+          name: wheels-linux-x86_64
+          path: dist
+      - name: Install wheel + dev deps
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install onyxweb --find-links dist --force-reinstall
+          python -m pip install --group dev
+      - name: Fetch chrome-headless-shell
+        run: onyxweb --install
+      - name: Run default test suite
+        run: pytest tests/

onyxweb-0.2.0/.github/workflows/publish.yml

@@ -0,0 +1,235 @@
+# Manual publish to (Test)PyPI via Trusted Publishing (OIDC). Builds the full
+# wheel matrix + sdist, smoke-tests the release wheel, then publishes to the
+# chosen index. Run it from the Actions tab ("Run workflow" -> pick testpypi
+# (default) or pypi). Requires a matching GitHub Environment (testpypi / pypi)
+# and a PyPI trusted publisher.
+#
+# testpypi runs publish a unique dev version (BASE.dev<run_number>) so retries
+# never collide with PyPI's immutable filenames; pypi runs publish the
+# pyproject version as-is and tag the commit.
+name: Publish
+
+on:
+  workflow_dispatch:
+    inputs:
+      repository:
+        description: "Index to publish to"
+        type: choice
+        options: [testpypi, pypi]
+        default: testpypi
+
+# Never run two publishes to the same index at once, and never cancel one
+# mid-upload (a half-finished upload is exactly what we want to avoid).
+concurrency:
+  group: publish-${{ inputs.repository }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  version:
+    # Single source of the published version. For testpypi it is made unique per
+    # run so re-publishing never hits PyPI's immutable-filename wall. Only
+    # pyproject.toml carries it (maturin reads the wheel version from [project]);
+    # Cargo.toml stays valid semver. Hand the file to every build job as an
+    # artifact so the rewrite happens once, with no per-platform shell.
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.calc.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: calc
+        run: |
+          BASE=$(python3 -c "import re;print(re.search(r'(?m)^version = \"([^\"]+)\"', open('pyproject.toml').read()).group(1))")
+          if [ "${{ inputs.repository }}" = "testpypi" ]; then
+            VERSION="${BASE}.dev${{ github.run_number }}"
+          else
+            VERSION="$BASE"
+          fi
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "Publishing $VERSION to ${{ inputs.repository }}"
+      - name: Pin version into pyproject.toml
+        env:
+          V: ${{ steps.calc.outputs.version }}
+        run: |
+          python3 - <<'EOF'
+          import os, re
+          v = os.environ["V"]
+          s = open("pyproject.toml").read()
+          s = re.sub(r'(?m)^version = "[^"]+"', f'version = "{v}"', s, count=1)
+          open("pyproject.toml", "w").write(s)
+          EOF
+      - uses: actions/upload-artifact@v4
+        with:
+          name: version-files
+          path: pyproject.toml
+
+  linux:
+    needs: version
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64, aarch64]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: version-files
+          path: .
+      - name: Build wheel (manylinux_2_28)
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          manylinux: "2_28"
+          args: --release --out dist -i 3.10 3.11 3.12 3.13
+          sccache: true
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-linux-${{ matrix.target }}
+          path: dist
+
+  macos:
+    # One job + --find-interpreter builds all versions. A python-version matrix
+    # would duplicate wheel filenames across artifacts and corrupt the publish
+    # merge (PyPI then rejects them as "Trailing data").
+    needs: version
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: version-files
+          path: .
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+      - uses: PyO3/maturin-action@v1
+        with:
+          target: aarch64
+          args: --release --out dist --find-interpreter
+          sccache: true
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-macos-aarch64
+          path: dist
+
+  windows:
+    # One job (see macos): a python-version matrix duplicates wheels across
+    # artifacts and corrupts the publish merge.
+    needs: version
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: version-files
+          path: .
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+          architecture: x64
+      - uses: PyO3/maturin-action@v1
+        with:
+          target: x64
+          args: --release --out dist --find-interpreter
+          sccache: true
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-windows-x86_64
+          path: dist
+
+  sdist:
+    needs: version
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: version-files
+          path: .
+      - uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: dist
+
+  smoke:
+    # Gate the publish on the ACTUAL release wheel importing and fetching a real
+    # page (catches ABI / packaging breakage a unit suite on a dev build won't).
+    needs: [version, linux]
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: actions/download-artifact@v4
+        with:
+          name: wheels-linux-x86_64
+          path: dist
+      - name: Install the built wheel
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --pre onyxweb --find-links dist --force-reinstall
+      - name: Fetch chrome-headless-shell
+        run: onyxweb --install
+      - name: Smoke test
+        run: |
+          python -c "import onyxweb; r = onyxweb.fetch('https://example.com'); assert r.status_code == 200, r.status_code; assert 'Example Domain' in r; print('smoke OK', r.status_code)"
+
+  publish:
+    name: Publish to ${{ inputs.repository }}
+    needs: [version, linux, macos, windows, sdist, smoke]
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ inputs.repository }}
+      url: ${{ inputs.repository == 'testpypi' && 'https://test.pypi.org/p/onyxweb' || 'https://pypi.org/p/onyxweb' }}
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      # Only wheel + sdist artifacts (not version-files) into dist.
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: wheels-*
+          path: dist
+          merge-multiple: true
+      - uses: actions/download-artifact@v4
+        with:
+          name: sdist
+          path: dist
+      - uses: pypa/gh-action-pypi-publish@release/v1.14
+        with:
+          packages-dir: dist
+          # Tolerate files already uploaded (e.g. a retry after a partial run).
+          skip-existing: true
+          # Empty string → real PyPI; only set for TestPyPI.
+          repository-url: ${{ inputs.repository == 'testpypi' && 'https://test.pypi.org/legacy/' || '' }}
+
+  tag:
+    # Tag real releases (pypi only) with v<version> for traceability.
+    needs: [version, publish]
+    if: inputs.repository == 'pypi'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Tag the release commit
+        env:
+          V: v${{ needs.version.outputs.version }}
+        run: |
+          git config user.name "github-actions"
+          git config user.email "github-actions@github.com"
+          if git rev-parse "$V" >/dev/null 2>&1; then
+            echo "Tag $V already exists, skipping."
+          else
+            git tag -a "$V" -m "Release $V"
+            git push origin "$V"
+          fi

onyxweb-0.2.0/.gitignore

@@ -0,0 +1,33 @@
+# Rust
+/target/
+
+# Python
+__pycache__/
+*.pyc
+*.pyo
+*.egg-info/
+dist/
+*.so
+*.pyd
+
+# IDE
+.idea/
+
+# Env
+.venv/
+.env
+
+# Maturin
+*.whl
+
+# OS
+.DS_Store
+
+# Ruff
+.ruff_cache/
+
+CLAUDE.md
+
+# Bundled chromium binaries — populated at wheel build time (`uv run onyxweb-download-chrome`).
+# Not committed to git (too large); maturin `include` picks them up from disk.
+python/onyxweb/_binaries/

onyxweb-0.2.0/.python-version

@@ -0,0 +1 @@
+3.11

onyxweb-0.2.0/BENCHMARKS.md

@@ -0,0 +1,106 @@
+# Screenshot engine benchmarks
+
+BBOT-scale subdomain screenshot pumping. One workload, multiple engines, honest numbers.
+
+## Test setup
+
+- **Host**: Linux 6.19.10-203.nobara.fc43 (Nobara 43 / Fedora 43), 16 CPU cores, 24 GB free RAM
+- **URLs**: 48–50 URLs from `servo_spike/urls_bench_big.txt` (mixed public sites; `gnu.org` and `google.com` excluded from `/tmp/urls_stable.txt` for the CDP/CEF head-to-head — the first due to network flakiness, the second due to headless anti-bot variance)
+- **Viewport**: 1200×800
+- **Timeout**: 30–45s per URL
+- **Build**: all release-mode with `lto = "thin"`, `codegen-units = 1`
+- **Measurement**: `/usr/bin/time` for wall + peak RSS
+- **Metric**: URL/second sustained (ok_count / wall_time)
+
+Important caveat on RSS: `/usr/bin/time` captures peak RSS of the wrapped process only — for chromiumoxide it UNDERCOUNTS (Chromium is a subprocess not counted); for cef_spike warm mode it's accurate (everything in-process).
+
+## Headline table — fully rendered URL → PNG
+
+| Engine | Best Config | **URL/s** | Peak per-proc RAM | Binary + runtime | Compat |
+|---|---|---|---|---|---|
+| Servo 0.1.0 (in-process) | cold P=8 | 1.13 | 1.6 GB | 121 MB (self-contained) | 45–46/50 |
+| Chromium CLI (fork per URL) | cold P=16 | 4.51 | 310 MB | system chromium | 50/50 |
+| Chromium via Python CDP | warm P=16 | 5.46 | ~500 MB | system chromium + Python | 50/50 |
+| **CEF via tauri-apps/cef-rs (Option A)** | **cold P=16** | **6.21** | 331 MB | 4 MB bin + 1.3 GB libcef.so | 48/48 |
+| **Chromium via chromiumoxide (CDP)** | warm P=32 | **9.30** | ~430 MB | 6 MB + system chromium | 50/50 |
+| **Chromium via chromiumoxide + chrome-headless-shell** | **warm P=16** | **8.66** | ~400 MB | 6 MB + 190 MB shell | 47–48/48 |
+
+**Winner for BBOT-scale**: chromiumoxide (Rust-native CDP client). With bundled chrome-headless-shell it's self-contained, 8.66 URL/s, ~350 MB wheel. The "bundle CEF" path (Option A) is ~30% slower AND ~4× larger to distribute.
+
+## Deep data: Option A vs Option C head-to-head (48 URLs stable list)
+
+### Option A — CEF via tauri-apps/cef-rs (in-process)
+
+| Config | Wall (s) | OK/48 | URL/s | Peak MB |
+|---|---|---|---|---|
+| cef cold P=1 | 63.63 | 48 | 0.75 | 330 |
+| cef cold P=4 | 17.08 | 48 | 2.81 | 328 |
+| cef cold P=8 | 10.08 | 48 | 4.76 | 330 |
+| **cef cold P=16** | **7.73** | 48 | **6.21** | 331 |
+| cef warm P=1 | 47.91 | 48 | 1.00 | 1023 |
+| cef warm P=4 | 16.36 | 48 | 2.93 | 1025 |
+| cef warm P=8 | 15.30 | 48 | 3.14 | 1024 |
+| cef warm P=16 | 10.76 | 48 | 4.46 | 999 |
+
+**Counterintuitive finding**: warm mode (one CEF init, N browsers in flight) is SLOWER than cold (N separate CEF processes) at P=16. Reason: CEF's single main thread serializes paint/event dispatch for all browsers in one instance. Cold-parallel gives each browser its own main thread. For BBOT-scale, **always use cold-process parallelism with CEF, not warm-batch**.
+
+### Option C — chromiumoxide (CDP via WebSocket)
+
+| Config | Wall (median, 3 runs) | OK/48 | URL/s |
+|---|---|---|---|
+| sys chromium P=16 PNG | 6.22 | 48/48 | 7.72 |
+| **headless-shell 148 P=16 PNG** | **5.43** | 47/48 | **8.66** |
+| headless-shell 148 P=16 PNG+HTML | 5.60 | 48/48 | 8.57 |
+| sys chromium P=8 PNG | 20.49 | 47/48 | 2.29 (straggler-dominated) |
+| headless-shell 148 P=8 PNG | 7.81 | 47/48 | 6.02 |
+| headless-shell 148 P=8 PNG+HTML | 7.45 | 48/48 | 6.44 |
+
+**chrome-headless-shell 148** (Google's automation-optimized chromium variant) is ~12% faster than the full system chromium 146 at the same P. HTML extraction via `page.content()` (post-JS `document.documentElement.outerHTML`) is essentially free — same P, same wall time.
+
+## Decision-rubric recap
+
+| Concern | Option A (CEF) | Option C (chromiumoxide) |
+|---|---|---|
+| Peak throughput | 6.21 URL/s | **8.66 URL/s** (40% higher) |
+| Distribution size | ~1.5 GB (libcef.so + resources + bin) | ~350 MB (wheel + bundled headless-shell) |
+| Bundle chromium with wheel | ✓ (that's the point) | ✓ also possible (ship headless-shell alongside) |
+| Headless server (no X) | ✗ (needs Xvfb) | **✓** (headless-first) |
+| Build complexity | High (wrap macros, multi-process dispatch, sandbox, cache-path collision handling) | Low (one async function) |
+| Rust code lines for basic use | ~500 | ~200 |
+| Deeper integration possible | ✓ in-process API | ✗ via CDP only |
+| Version pinning | ✓ (link libcef version) | ✓ (pin headless-shell download) |
+| Time to working spike | ~1 day | ~2 hours |
+| External deps at runtime | libcef + libX11 + lots | chrome binary + glibc |
+
+Option A (CEF) has NO win that matters for BBOT-scale headless screenshot pumping. Its unique strength — in-process API — is unused by a fetch-load-screenshot-exit workflow. Its unique cost — 4× larger distribution, X11 requirement, multi-process dispatch complexity — is paid every day.
+
+## Conclusion
+
+**Build onyxweb on chromiumoxide, not CEF.** Ship chrome-headless-shell 148 alongside the wheel (~350 MB total). Target:
+
+- ~9 URL/s per process at sweet-spot concurrency (P=16–32)
+- HTML extraction free-of-charge via `page.content()`
+- ~400 MB RSS per process; 4 parallel processes handle ~35 URL/s, network-bound beyond
+- Wheel distribution: unpacks `chrome-headless-shell` into site-packages
+- Python API: `onyxweb.Client()`, `client.screenshot(url)`, `client.batch(urls, concurrency=16)`
+
+10,000 URLs → ~18 min single process, ~5 min with 4 parallel processes.
+
+## Reproduction
+
+```bash
+# Option A — CEF
+cd experiments/cef_spike
+cargo build --release                        # first build downloads CEF 146 binaries (~300MB)
+DISPLAY=:0 LD_LIBRARY_PATH=$(find target -name libcef.so | xargs dirname) ./bench.sh
+
+# Option C — chromiumoxide + chrome-headless-shell
+cd experiments/chromiumoxide_spike
+cargo build --release                        # 90s
+curl -sSL -o /tmp/hs.zip https://storage.googleapis.com/chrome-for-testing-public/148.0.7778.56/linux64/chrome-headless-shell-linux64.zip
+unzip -q /tmp/hs.zip -d /tmp
+HS=/tmp/chrome-headless-shell-linux64/chrome-headless-shell
+./target/release/chromiumoxide_spike --chrome $HS --out-dir /tmp/shots --concurrency 16 --mode both < urls.txt
+```
+
+All raw benchmark artifacts under `experiments/{cef_spike,chromiumoxide_spike,servo_spike}/bench_*/`.