webskrap 0.1.0__tar.gz

webskrap-0.1.0/.github/workflows/workflow.yml

@@ -0,0 +1,28 @@
+name: Publish
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+      - name: Install build
+        run: python -m pip install --upgrade build
+      - name: Build package
+        run: python -m build
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

webskrap-0.1.0/.gitignore

@@ -0,0 +1,13 @@
+.venv/
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.coverage
+htmlcov/
+dist/
+build/
+*.egg-info/
+.webskrap/
+test.py

webskrap-0.1.0/PKG-INFO

@@ -0,0 +1,278 @@
+Metadata-Version: 2.4
+Name: webskrap
+Version: 0.1.0
+Summary: A Playwright-based Python scraping framework with coherent browser profiles and session controls.
+Project-URL: Homepage, https://github.com/kacigaya/webskrap
+Project-URL: Repository, https://github.com/kacigaya/webskrap
+Project-URL: Issues, https://github.com/kacigaya/webskrap/issues
+Author: WebSkrap contributors
+License-Expression: MIT
+Keywords: browser-automation,crawler,playwright,scraping
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: playwright>=1.49
+Requires-Dist: pydantic>=2.8
+Requires-Dist: rich>=13.9
+Requires-Dist: typer>=0.15
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
+Requires-Dist: pytest>=8.3; extra == 'dev'
+Requires-Dist: ruff>=0.8; extra == 'dev'
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <img src="assets/webskrap-logo.png" alt="WebSkrap logo" width="200">
+</p>
+
+<h1 align="center">WebSkrap</h1>
+
+<p align="center">
+   <strong>Async-first Python scraping framework built on Playwright.</strong><br>
+   <em>It provides coherent browser profiles, persistent sessions, resource routing, and configurable browser hardening for data collection workflows that need realistic browser behavior.</em>
+</p>
+
+WebSkrap does not include CAPTCHA solving, login-wall bypassing, credential bypassing, or access-control circumvention. Use it only on targets you are allowed to access.
+
+## Install
+
+```bash
+pip install webskrap
+python -m playwright install chromium
+```
+
+## Quick Start
+
+```python
+import asyncio
+
+from webskrap import WebSkrapClient
+
+
+async def main() -> None:
+    async with WebSkrapClient() as client:
+        result = await client.fetch("https://example.com")
+        print(result.status)
+        print(result.title)
+        print(result.text[:200])
+
+
+asyncio.run(main())
+```
+
+## Persistent Session
+
+```python
+import asyncio
+from pathlib import Path
+
+from webskrap import SessionConfig, WebSkrapClient
+
+
+async def main() -> None:
+    config = SessionConfig(
+        user_data_dir=Path(".webskrap/sessions/shop"),
+        headless=True,
+    )
+
+    async with WebSkrapClient() as client:
+        session = await client.session("shop", config=config, profile="desktop-chrome")
+        first = await session.fetch("https://example.com")
+        second = await session.fetch("https://example.com/account")
+        print(first.final_url, second.final_url)
+
+
+asyncio.run(main())
+```
+
+## Headed Browser
+
+Use a persistent session when you want the browser to stay open.
+
+```python
+import asyncio
+from pathlib import Path
+
+from webskrap import SessionConfig, WebSkrapClient
+
+
+async def main() -> None:
+    config = SessionConfig(
+        headless=False,
+        user_data_dir=Path(".webskrap/dev-session"),
+    )
+
+    async with WebSkrapClient() as client:
+        session = await client.session("dev", config=config)
+        page = await session.context.new_page()
+        await page.goto("https://example.com", wait_until="domcontentloaded")
+
+        input("Press Enter to close browser...")
+
+
+asyncio.run(main())
+```
+
+Example for a headed Chrome session with a French desktop profile:
+
+```python
+import asyncio
+from pathlib import Path
+
+from webskrap import BrowserProfile, SessionConfig, Viewport, WebSkrapClient
+
+
+async def main() -> None:
+    config = SessionConfig(
+        headless=False,
+        channel="chrome",
+        user_data_dir=Path(".webskrap/gmf"),
+        navigation_timeout_ms=90_000,
+        default_timeout_ms=90_000,
+        slow_mo_ms=50,
+        launch_args=[
+            "--start-maximized",
+            "--disable-blink-features=AutomationControlled",
+            "--no-first-run",
+            "--no-default-browser-check",
+        ],
+    )
+    profile = BrowserProfile(
+        name="fr-desktop",
+        viewport=Viewport(width=1440, height=900),
+        screen=Viewport(width=1440, height=900),
+        locale="fr-FR",
+        timezone_id="Europe/Paris",
+        navigator_languages=["fr-FR", "fr", "en-US", "en"],
+    )
+
+    async with WebSkrapClient() as client:
+        session = await client.session("gmf", config=config, profile=profile)
+        page = await session.context.new_page()
+        await page.goto("https://www.gmf.fr/habitation/devis", wait_until="domcontentloaded")
+
+        input("Press Enter to close browser...")
+
+
+asyncio.run(main())
+```
+
+## Custom Profile
+
+```python
+from webskrap import BrowserProfile, Viewport
+
+profile = BrowserProfile(
+    name="workstation",
+    viewport=Viewport(width=1440, height=900),
+    screen=Viewport(width=1440, height=900),
+    locale="en-US",
+    timezone_id="Europe/Paris",
+)
+```
+
+## Session Options
+
+```python
+from pathlib import Path
+
+from webskrap import ProxyConfig, ResourcePolicy, SessionConfig
+
+config = SessionConfig(
+    browser="chromium",
+    channel="chrome",
+    headless=False,
+    user_data_dir=Path(".webskrap/session"),
+    storage_state=None,
+    proxy=ProxyConfig(server="http://127.0.0.1:8080"),
+    resource_policy=ResourcePolicy.LITE,
+    ignore_https_errors=True,
+    java_script_enabled=True,
+    service_workers="allow",
+    timeout_ms=30_000,
+    navigation_timeout_ms=90_000,
+    default_timeout_ms=90_000,
+    slow_mo_ms=50,
+    launch_args=[
+        "--start-maximized",
+        "--disable-blink-features=AutomationControlled",
+        "--disable-dev-shm-usage",
+        "--no-first-run",
+        "--no-default-browser-check",
+    ],
+)
+```
+
+`resource_policy` values:
+
+- `ResourcePolicy.ALL`: allow all resources.
+- `ResourcePolicy.LITE`: block images, fonts, and media.
+- `ResourcePolicy.DOCUMENTS`: block images, fonts, media, and stylesheets.
+
+## Profile Options
+
+```python
+from webskrap import BrowserProfile, Viewport
+
+profile = BrowserProfile(
+    name="fr-desktop",
+    user_agent=None,
+    viewport=Viewport(width=1440, height=900),
+    screen=Viewport(width=1440, height=900),
+    locale="fr-FR",
+    timezone_id="Europe/Paris",
+    device_scale_factor=1,
+    is_mobile=False,
+    has_touch=False,
+    color_scheme="light",
+    reduced_motion="no-preference",
+    extra_http_headers={},
+    navigator_languages=["fr-FR", "fr", "en-US", "en"],
+    hardware_concurrency=8,
+    device_memory=8,
+    webgl_vendor="Google Inc. (Intel)",
+    webgl_renderer="ANGLE (Intel, Intel(R) Iris(TM) Plus Graphics, OpenGL 4.1)",
+)
+```
+
+## Stealth Options
+
+```python
+from webskrap import SessionConfig, StealthConfig
+
+config = SessionConfig(
+    stealth=StealthConfig(
+        enabled=True,
+        patch_webdriver=True,
+        patch_chrome_runtime=True,
+        patch_permissions=True,
+        patch_plugins=True,
+        patch_webgl=True,
+        patch_canvas=True,
+        patch_hardware=True,
+    )
+)
+```
+
+## CLI
+
+```bash
+webskrap profiles
+webskrap doctor
+webskrap fetch https://example.com --profile desktop-chrome
+webskrap fetch https://example.com --headed --screenshot example.png
+```
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+ruff check .
+```

webskrap-0.1.0/README.md

@@ -0,0 +1,249 @@
+<p align="center">
+  <img src="assets/webskrap-logo.png" alt="WebSkrap logo" width="200">
+</p>
+
+<h1 align="center">WebSkrap</h1>
+
+<p align="center">
+   <strong>Async-first Python scraping framework built on Playwright.</strong><br>
+   <em>It provides coherent browser profiles, persistent sessions, resource routing, and configurable browser hardening for data collection workflows that need realistic browser behavior.</em>
+</p>
+
+WebSkrap does not include CAPTCHA solving, login-wall bypassing, credential bypassing, or access-control circumvention. Use it only on targets you are allowed to access.
+
+## Install
+
+```bash
+pip install webskrap
+python -m playwright install chromium
+```
+
+## Quick Start
+
+```python
+import asyncio
+
+from webskrap import WebSkrapClient
+
+
+async def main() -> None:
+    async with WebSkrapClient() as client:
+        result = await client.fetch("https://example.com")
+        print(result.status)
+        print(result.title)
+        print(result.text[:200])
+
+
+asyncio.run(main())
+```
+
+## Persistent Session
+
+```python
+import asyncio
+from pathlib import Path
+
+from webskrap import SessionConfig, WebSkrapClient
+
+
+async def main() -> None:
+    config = SessionConfig(
+        user_data_dir=Path(".webskrap/sessions/shop"),
+        headless=True,
+    )
+
+    async with WebSkrapClient() as client:
+        session = await client.session("shop", config=config, profile="desktop-chrome")
+        first = await session.fetch("https://example.com")
+        second = await session.fetch("https://example.com/account")
+        print(first.final_url, second.final_url)
+
+
+asyncio.run(main())
+```
+
+## Headed Browser
+
+Use a persistent session when you want the browser to stay open.
+
+```python
+import asyncio
+from pathlib import Path
+
+from webskrap import SessionConfig, WebSkrapClient
+
+
+async def main() -> None:
+    config = SessionConfig(
+        headless=False,
+        user_data_dir=Path(".webskrap/dev-session"),
+    )
+
+    async with WebSkrapClient() as client:
+        session = await client.session("dev", config=config)
+        page = await session.context.new_page()
+        await page.goto("https://example.com", wait_until="domcontentloaded")
+
+        input("Press Enter to close browser...")
+
+
+asyncio.run(main())
+```
+
+Example for a headed Chrome session with a French desktop profile:
+
+```python
+import asyncio
+from pathlib import Path
+
+from webskrap import BrowserProfile, SessionConfig, Viewport, WebSkrapClient
+
+
+async def main() -> None:
+    config = SessionConfig(
+        headless=False,
+        channel="chrome",
+        user_data_dir=Path(".webskrap/gmf"),
+        navigation_timeout_ms=90_000,
+        default_timeout_ms=90_000,
+        slow_mo_ms=50,
+        launch_args=[
+            "--start-maximized",
+            "--disable-blink-features=AutomationControlled",
+            "--no-first-run",
+            "--no-default-browser-check",
+        ],
+    )
+    profile = BrowserProfile(
+        name="fr-desktop",
+        viewport=Viewport(width=1440, height=900),
+        screen=Viewport(width=1440, height=900),
+        locale="fr-FR",
+        timezone_id="Europe/Paris",
+        navigator_languages=["fr-FR", "fr", "en-US", "en"],
+    )
+
+    async with WebSkrapClient() as client:
+        session = await client.session("gmf", config=config, profile=profile)
+        page = await session.context.new_page()
+        await page.goto("https://www.gmf.fr/habitation/devis", wait_until="domcontentloaded")
+
+        input("Press Enter to close browser...")
+
+
+asyncio.run(main())
+```
+
+## Custom Profile
+
+```python
+from webskrap import BrowserProfile, Viewport
+
+profile = BrowserProfile(
+    name="workstation",
+    viewport=Viewport(width=1440, height=900),
+    screen=Viewport(width=1440, height=900),
+    locale="en-US",
+    timezone_id="Europe/Paris",
+)
+```
+
+## Session Options
+
+```python
+from pathlib import Path
+
+from webskrap import ProxyConfig, ResourcePolicy, SessionConfig
+
+config = SessionConfig(
+    browser="chromium",
+    channel="chrome",
+    headless=False,
+    user_data_dir=Path(".webskrap/session"),
+    storage_state=None,
+    proxy=ProxyConfig(server="http://127.0.0.1:8080"),
+    resource_policy=ResourcePolicy.LITE,
+    ignore_https_errors=True,
+    java_script_enabled=True,
+    service_workers="allow",
+    timeout_ms=30_000,
+    navigation_timeout_ms=90_000,
+    default_timeout_ms=90_000,
+    slow_mo_ms=50,
+    launch_args=[
+        "--start-maximized",
+        "--disable-blink-features=AutomationControlled",
+        "--disable-dev-shm-usage",
+        "--no-first-run",
+        "--no-default-browser-check",
+    ],
+)
+```
+
+`resource_policy` values:
+
+- `ResourcePolicy.ALL`: allow all resources.
+- `ResourcePolicy.LITE`: block images, fonts, and media.
+- `ResourcePolicy.DOCUMENTS`: block images, fonts, media, and stylesheets.
+
+## Profile Options
+
+```python
+from webskrap import BrowserProfile, Viewport
+
+profile = BrowserProfile(
+    name="fr-desktop",
+    user_agent=None,
+    viewport=Viewport(width=1440, height=900),
+    screen=Viewport(width=1440, height=900),
+    locale="fr-FR",
+    timezone_id="Europe/Paris",
+    device_scale_factor=1,
+    is_mobile=False,
+    has_touch=False,
+    color_scheme="light",
+    reduced_motion="no-preference",
+    extra_http_headers={},
+    navigator_languages=["fr-FR", "fr", "en-US", "en"],
+    hardware_concurrency=8,
+    device_memory=8,
+    webgl_vendor="Google Inc. (Intel)",
+    webgl_renderer="ANGLE (Intel, Intel(R) Iris(TM) Plus Graphics, OpenGL 4.1)",
+)
+```
+
+## Stealth Options
+
+```python
+from webskrap import SessionConfig, StealthConfig
+
+config = SessionConfig(
+    stealth=StealthConfig(
+        enabled=True,
+        patch_webdriver=True,
+        patch_chrome_runtime=True,
+        patch_permissions=True,
+        patch_plugins=True,
+        patch_webgl=True,
+        patch_canvas=True,
+        patch_hardware=True,
+    )
+)
+```
+
+## CLI
+
+```bash
+webskrap profiles
+webskrap doctor
+webskrap fetch https://example.com --profile desktop-chrome
+webskrap fetch https://example.com --headed --screenshot example.png
+```
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+ruff check .
+```

webskrap-0.1.0/pyproject.toml

@@ -0,0 +1,66 @@
+[build-system]
+requires = ["hatchling>=1.26"]
+build-backend = "hatchling.build"
+
+[project]
+name = "webskrap"
+version = "0.1.0"
+description = "A Playwright-based Python scraping framework with coherent browser profiles and session controls."
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+authors = [
+  { name = "WebSkrap contributors" }
+]
+keywords = ["browser-automation", "playwright", "scraping", "crawler"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Typing :: Typed",
+]
+dependencies = [
+  "playwright>=1.49",
+  "pydantic>=2.8",
+  "rich>=13.9",
+  "typer>=0.15",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.3",
+  "pytest-asyncio>=0.24",
+  "ruff>=0.8",
+]
+
+[project.scripts]
+webskrap = "webskrap.cli:app"
+
+[project.urls]
+Homepage = "https://github.com/kacigaya/webskrap"
+Repository = "https://github.com/kacigaya/webskrap"
+Issues = "https://github.com/kacigaya/webskrap/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/webskrap"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+markers = [
+  "browser: requires installed Playwright browsers",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM"]
+
+[tool.ruff.format]
+quote-style = "double"

webskrap-0.1.0/src/webskrap/__init__.py

@@ -0,0 +1,26 @@
+from webskrap.client import WebSkrapClient, WebSkrapError, WebSkrapSession
+from webskrap.models import (
+    BrowserProfile,
+    FetchResult,
+    ProxyConfig,
+    ResourcePolicy,
+    SessionConfig,
+    StealthConfig,
+    Viewport,
+)
+from webskrap.profiles import get_profile, list_profiles
+
+__all__ = [
+    "BrowserProfile",
+    "FetchResult",
+    "ProxyConfig",
+    "ResourcePolicy",
+    "SessionConfig",
+    "StealthConfig",
+    "Viewport",
+    "WebSkrapClient",
+    "WebSkrapError",
+    "WebSkrapSession",
+    "get_profile",
+    "list_profiles",
+]