lncrawl-scraper 0.1.0__tar.gz

lncrawl_scraper-0.1.0/.github/workflows/bump.yml

@@ -0,0 +1,39 @@
+name: Bump Version
+
+on:
+  workflow_dispatch:
+    inputs:
+      bump:
+        description: Version part to bump
+        required: true
+        default: patch
+        type: choice
+        options: [patch, minor, major]
+
+jobs:
+  ci:
+    uses: ./.github/workflows/ci.yml
+
+  bump:
+    needs: ci
+    name: Bump version and push tag
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v7
+
+      - name: Bump version
+        run: uv version --bump '${{ inputs.bump }}'
+
+      - name: Read new version
+        id: version
+        run: echo "value=$(uv version | awk '{print $NF}')" >> $GITHUB_OUTPUT
+
+      - name: Commit and tag
+        uses: EndBug/add-and-commit@v9
+        with:
+          add: "pyproject.toml uv.lock"
+          tag: "v${{ steps.version.outputs.value }}"
+          message: "Bump version to v${{ steps.version.outputs.value }}"

lncrawl_scraper-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,78 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_call:
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.9"
+      - run: uv sync --all-groups --all-extras
+      - run: uv run ruff check
+      - run: uv run ruff format --check
+      - run: uv run pyright
+
+  coverage:
+    name: Coverage
+    runs-on: ubuntu-latest
+    needs: lint
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.12"
+      - run: uv sync --all-groups --all-extras
+
+      - name: Run tests with coverage
+        run: uv run pytest --cov --cov-report=xml --cov-report=term-missing
+
+      - name: Write coverage to job summary
+        if: always()
+        run: |
+          {
+            echo '## Coverage'
+            uv run coverage report --format=markdown
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Coverage PR comment + badge
+        uses: py-cov-action/python-coverage-comment-action@v3
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload coverage.xml
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-xml
+          path: coverage.xml
+
+  build:
+    name: Build (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    needs: lint
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: uv sync --all-groups --all-extras
+      - run: uv run pytest
+      - run: uv build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist-${{ matrix.python-version }}
+          path: dist/

lncrawl_scraper-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,28 @@
+name: Publish
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.9"
+
+      - name: Build distributions
+        run: uv build
+
+      - name: Check metadata renders on PyPI
+        run: uvx twine check dist/*
+
+      - name: Publish to PyPI (trusted publishing)
+        uses: pypa/gh-action-pypi-publish@release/v1

lncrawl_scraper-0.1.0/.gitignore

@@ -0,0 +1,218 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+# Temporary file for partial code execution
+tempCodeRunnerFile.py
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml

lncrawl_scraper-0.1.0/.python-version

@@ -0,0 +1 @@
+3.12

lncrawl_scraper-0.1.0/CHANGELOG.md

@@ -0,0 +1,33 @@
+# Changelog
+
+All notable changes to this project are documented here. The format is based on
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres
+to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.0] - 2026-06-04
+
+Initial public release of `lncrawl-scraper`, extracted from
+[lightnovel-crawler](https://github.com/lncrawl/lightnovel-crawler).
+
+### Added
+
+- `Scraper` — a `requests.Session` subclass with transparent Cloudflare
+  challenge handling (v1, v2, v3, Turnstile) and helpers: `get_soup`,
+  `post_soup`, `get_json`, `post_json`, `get_file`, `get_image`, `submit_form`,
+  `ping`.
+- `PageSoup` — a null-safe BeautifulSoup wrapper; selection methods never return
+  `None` and text/HTML accessors always return `str`.
+- Typed configuration: `ScraperConfig`, `StealthConfig`, `ProxyConfig`,
+  `BrowserConfig`, plus the `default_config()` factory.
+- **Browser fingerprint impersonation** (`impersonate` extra): route requests
+  through `curl_cffi` for a real Chrome/Firefox TLS (JA3/JA4) and HTTP/2
+  fingerprint, with the spoofed User-Agent family aligned to the target.
+- **Browser-assisted clearance**: `apply_browser_clearance()` to reuse a
+  `cf_clearance` cookie + User-Agent solved by an external real browser.
+- **Accurate Client Hints**: `sec-ch-ua` / platform / mobile derived from the
+  chosen User-Agent (Chromium only) instead of hardcoded values.
+- Stealth mode, proxy rotation with Tor identity refresh, TLS cipher rotation,
+  rate limiting, and cooperative `abort()`.
+- `py.typed` marker (PEP 561) and full type coverage.
+
+[0.1.0]: https://github.com/lncrawl/scraper/releases/tag/v0.1.0

lncrawl_scraper-0.1.0/CLAUDE.md

@@ -0,0 +1,172 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## What this is
+
+`lncrawl-scraper` (import name `scraper`) is a standalone HTTP scraping library
+extracted from [lightnovel-crawler](https://github.com/lncrawl/lightnovel-crawler).
+It is a `requests.Session` subclass that transparently handles Cloudflare
+challenges, plus a null-safe BeautifulSoup wrapper and a set of HTTP helpers.
+
+Published to PyPI as `lncrawl-scraper`; imported as `scraper`. Targets Python
+**3.9+**.
+
+## Commands
+
+Tooling is driven by [uv](https://docs.astral.sh/uv/) + [poethepoet](https://poethepoet.natn.io/).
+
+```bash
+uv sync                 # install deps + editable package into .venv
+uv run poe lint         # ruff check + ruff format --check + pyright
+uv run poe lint-fix     # ruff check --fix + ruff format
+uv run poe test         # pytest
+uv run poe cov          # pytest with coverage (term-missing + html + xml)
+uv run poe build        # lint + test + uv build (wheel/sdist)
+uv run poe publish      # build + uv publish
+```
+
+Always run `uv run poe lint` before considering a change done. CI
+(`.github/workflows/ci.yml`) runs three jobs: `lint` (ruff + pyright), a
+`build` matrix testing on Python 3.9–3.14, and `coverage` (which posts a PR
+comment + badge via `python-coverage-comment-action` and a job-summary table).
+
+## Architecture
+
+The package is a thin, ergonomic layer over an in-house Cloudflare-bypass engine.
+
+```text
+src/scraper/
+├── __init__.py         # public API + __version__ (via importlib.metadata)
+├── session.py          # Scraper — the main class (subclasses ScraperEngine)
+├── soup.py             # PageSoup — null-safe BeautifulSoup wrapper
+├── config.py           # public config surface + default_config() factory
+├── py.typed            # PEP 561 marker
+├── _utils/             # internal helpers (event_lock, url_tools, file_tools)
+└── _engine/            # internal Cloudflare-bypass engine (private)
+```
+
+### Layers
+
+- **`Scraper`** ([session.py](src/scraper/session.py)) — the public entry point.
+  Adds Origin/Referer injection, default timeouts, and helpers: `get_soup`,
+  `post_soup`, `get_json`, `post_json`, `get_image` (returns a PIL Image),
+  `get_file` (streamed, abortable), `submit_form`, `ping`. Subclasses
+  `ScraperEngine`, so all of `requests.Session` is available too.
+- **`PageSoup`** ([soup.py](src/scraper/soup.py)) — wraps a BeautifulSoup `Tag`.
+  Selection methods (`select`, `select_one`, `find`, `xpath`, `closest`, …)
+  always return `PageSoup`/`list`, never `None`; text/HTML accessors always
+  return `str`. An empty `PageSoup` is falsy. Reach the raw tag via `.tag`.
+- **`_engine/`** — the private engine: `ScraperEngine` (the `requests.Session`
+  subclass with the full request pipeline) in `_engine/__init__.py`, plus CF
+  challenge handlers v1/v2/v3 + Turnstile, TLS cipher rotation, stealth mode,
+  proxy/Tor manager, and UA selection. It is implementation detail — nothing
+  here is part of the public API except what `config.py`/`__init__.py`
+  re-export.
+
+### Cloudflare-bypass surface
+
+The realistic ceiling of a `requests`-based engine is its TLS (JA3/JA4) and
+HTTP/1.1 fingerprint — `set_ciphers()` in [tls.py](src/scraper/_engine/tls.py)
+only reorders ciphers, so the ClientHello still reads as Python. Three features
+push past that:
+
+- **Impersonation transport** ([_engine/impersonate.py](src/scraper/_engine/impersonate.py)):
+  when `ScraperConfig.impersonate` is set (e.g. `"chrome"`), `ScraperEngine.perform_request`
+  routes through `curl_cffi` (curl-impersonate) for a real browser TLS + HTTP/2
+  fingerprint, and adapts the result back into a `requests.Response`. The
+  curl_cffi session is the cookie authority and is mirrored into `self.cookies`
+  after each request (`_mirror_transport_cookies`). Cipher rotation is skipped
+  while impersonating. Requires the `impersonate` extra (`curl_cffi`).
+- **Client Hints** are derived from the actual UA in
+  `UserAgent._client_hints` (Chromium only; Firefox sends none) so `sec-ch-ua`
+  version/platform always match the User-Agent. `stealth.py` no longer hardcodes
+  them — it only defaults the non-version-specific `Sec-Fetch-*` nav hints.
+- **`apply_browser_clearance(domain, cf_clearance=, user_agent=, cookies=)`**
+  injects a clearance solved by an external real browser; the UA must match the
+  one that obtained it. `put_cookie` keeps the requests jar and the impersonation
+  jar in sync.
+
+### Configuration
+
+All config flows through `ScraperConfig` (a dataclass with nested
+`StealthConfig`, `ProxyConfig`, `BrowserConfig`). The public surface is
+[config.py](src/scraper/config.py), which re-exports the dataclasses from
+`_engine.config` and adds the `default_config()` factory:
+
+```python
+from scraper import Scraper, default_config
+from scraper.config import BrowserConfig, StealthConfig
+
+cfg = default_config()                 # fresh, fully-populated defaults
+cfg.browser = BrowserConfig(browser="chrome", platform="darwin")
+s = Scraper(origin="https://site.com", config=cfg)
+```
+
+- **`default_config()` returns a fresh instance every call.** Never reintroduce
+  a shared module-level config singleton — `ScraperEngine` hands the nested
+  `proxy`/`stealth` objects to managers that may mutate them, so sharing would
+  leak state across `Scraper` instances.
+- `ScraperConfig.browser` accepts `BrowserConfig | dict | None`; the dict form
+  is accepted as a convenience and normalized via `asdict` in `UserAgent.load`.
+
+## Conventions
+
+- **Python 3.9 compatibility is mandatory.** Bare `X | Y` unions must not be
+  *evaluated at runtime* — only use them in files that have
+  `from __future__ import annotations`, or in pure annotations. Prefer
+  `typing.Optional/Union` in new non-future-annotated modules. `importlib`,
+  dataclasses, etc. must all work on 3.9.
+- **Keep the public surface in public modules.** `_engine/` and `_utils/` are
+  private; user-facing names live in `__init__.py`/`config.py` and are listed
+  in `__all__`. Update `__all__` and the README when changing that surface.
+- **`ruff`**: line-length 100, double quotes, `force-sort-within-sections`,
+  combine-as-imports. **`pyright`** runs in `standard` mode over `src` + `tests`
+  — keep it clean (use real `isinstance` narrowing rather than `is_dataclass`,
+  which pyright doesn't narrow on).
+- **Dependencies**: core runtime deps live in `[project.dependencies]`. Optional
+  extras: `image` (`Pillow`, for `get_image`) and `impersonate` (`curl_cffi`,
+  for `ScraperConfig.impersonate`) — both imported lazily so the package works
+  without them. Add deps via `uv add` / `uv add --dev`.
+- **Public API** is whatever `src/scraper/__init__.py` exports in `__all__`.
+  Update it (and the README) when adding user-facing surface.
+
+## Commit messages
+
+Match the existing history (`git log`):
+
+- **No type prefix.** Do NOT use Conventional Commits (`feat:`, `fix:`,
+  `docs:`, …) — subjects are plain capitalized text.
+- **Imperative mood**, capitalized first word, no trailing period, subject
+  ≤ ~60 chars (e.g. `Add coverage reporting to CI`, `Restructure into src layout`).
+- **Body only for non-trivial changes**: a blank line, then a short rationale
+  paragraph and/or `-` bullets covering *what* changed and *why* (wrap at ~72
+  chars). Small changes are subject-only.
+- **Do NOT append a `Co-Authored-By` trailer** — this overrides the default
+  Claude Code behaviour; the maintainer's commits never carry it.
+
+## Testing
+
+`pytest` under [tests/](tests/). The src/ layout means tests import the
+*installed* package, so run them via `uv run poe test` / `uv run poe cov` (which
+use the editable install).
+
+- **Tests must be offline and fast.** [conftest.py](tests/conftest.py) provides
+  an autouse fixture that stubs `scraper._engine.user_agent._load_ua_data` to
+  `None` (forces the deterministic embedded UA generator, no network), plus
+  `fast_config` / `make_fast_config()` which disable stealth delays, throttling,
+  and session refresh. Use these in any test that constructs a `Scraper`.
+- **Mock HTTP with `responses`** (`responses.RequestsMock()`), never real
+  requests. It patches `HTTPAdapter.send`, so it intercepts the mounted TLS
+  adapter too. Note: a set abort signal trips the pre-send check, so the request
+  never fires — use `assert_all_requests_are_fired=False` in that case.
+- **UA-family gotcha**: the offline generator can pick iOS, where Chrome's UA is
+  `CriOS/…` and Firefox's is `FxiOS/…` (neither contains `Chrome/` / `Firefox/`).
+  When asserting on UA family, pin a desktop platform
+  (`BrowserConfig(platform="windows", mobile=False)`).
+- `curl_cffi`-dependent tests use `pytest.importorskip("curl_cffi")`.
+- **Coverage** config is in `pyproject.toml` (`[tool.coverage]`, `source =
+  ["scraper"]`, `relative_files = true`). `uv run poe cov` writes `htmlcov/`,
+  `coverage.xml`, and a terminal report (all coverage artifacts are gitignored).
+  The deep CF challenge solvers (`cloudflare_v1/v2/v3`, `interpreter`) are
+  integration-only and stay low-coverage without live Cloudflare traffic.