dataapp_crawler 0.1.0__tar.gz

dataapp_crawler-0.1.0/PKG-INFO

@@ -0,0 +1,211 @@
+Metadata-Version: 2.5
+Name: dataapp_crawler
+Version: 0.1.0
+Summary: Composable HTTP, browser and proxy acquisition SDK for DataOS DataApps
+Requires-Python: >=3.11
+Requires-Dist: httpx<1,>=0.28
+Provides-Extra: browser
+Requires-Dist: playwright<2,>=1.64; extra == 'browser'
+Provides-Extra: fingerprint
+Requires-Dist: camoufox<0.6,>=0.5.9; extra == 'fingerprint'
+Provides-Extra: parse
+Requires-Dist: parsel<2,>=1.9; extra == 'parse'
+Provides-Extra: resource-proxy
+Requires-Dist: dataapp-resource-proxy<0.2,>=0.1.0; extra == 'resource-proxy'
+Description-Content-Type: text/markdown
+
+# Data App Crawler core
+
+`dataapp-crawler` (imported as `dataapp_crawler`) is the provider-neutral website
+acquisition library for Python 3.11+. It can run independently of the DataApp
+Runtime. Core includes generic CDP/Playwright providers and an optional Camoufox
+adapter, but no Relay- or AWS-specific provider.
+
+> Dependency versions are defined in each package's `pyproject.toml`; an
+> application uses its own `uv.lock`. Local source changes are not delivered to
+> applications until matching packages and browser images are released and the
+> application's lock is updated.
+
+## Basic acquisition
+
+```python
+import asyncio
+
+from dataapp_crawler import Crawler
+
+
+async def main() -> None:
+    async with Crawler() as crawler:
+        capture = await crawler.fetch("https://example.com")
+        print(capture.status)
+
+
+asyncio.run(main())
+```
+
+Use an ordinary synchronous or asynchronous `process(capture, context)` function
+for extraction, OCR/ASR calls, model extraction, or custom logic. The SDK does
+not own a processing engine or Dataset:
+
+```python
+async def process(capture, context):
+    heading = capture.page.locator("h1").first
+    await context.run(heading.wait_for(state="visible"), timeout=10)
+    return {"title": (await context.run(heading.inner_text(), timeout=5)).strip()}
+```
+
+An operation timeout raises `TimeoutError`; exhausting `Limits.max_duration`
+raises `LimitReached`. Diagnostics are bounded and omit raw credentials, headers,
+connection URLs, response bodies, and arbitrary upstream exception text.
+
+## Page-driven flows
+
+`Crawler.run_browser(collect)` hands one native Playwright `Page` and a
+`CrawlContext` to ordinary application code. A single collection function can
+submit a search, traverse list pages, open detail pages, and stop on business
+conditions. It does not require one URL per `Crawler` object.
+
+```python
+from dataapp_crawler import Crawler, Limits
+from dataapp_crawler.browser_tools import open_url, text_of
+
+
+async def collect(page, context):
+    await open_url(page, "https://example.com", context)
+    return {"title": await text_of(page.locator("h1"), context, required=True)}
+
+
+async with Crawler(browser=browser_provider, limits=Limits(max_duration=120)) as crawler:
+    record = await crawler.run_browser(collect)
+```
+
+`browser_tools` includes bounded navigation and extraction helpers such as
+`open_url`, `new_page`, `popup_page`, `same_tab`, `wait_until`, `text_of`,
+`attribute_of`, and `iterate_list`. Page and Locator remain native Playwright
+objects. Raw Playwright actions remain available, but they do not automatically
+gain Crawler accounting, scheduling, diagnostics, or retry semantics.
+
+The application defines selectors, stable business keys, pagination, readiness,
+and side-effect boundaries. Do not automatically retry clicks or submissions
+that can create duplicate actions.
+
+Failure summaries include a fixed `exception_type` without its message. Browser
+flow failures also report `root_host`, the root Page's hostname only. This may be
+the list page rather than a failing detail tab. The original exception still
+propagates, with cancellation and cleanup unchanged. A failure before the root
+Page exists reports `phase=page_creation`.
+
+To inspect a failing detail page, explicitly capture a bounded screenshot inside
+that page's scope before it closes. Keep artifacts private, outside uploaded
+source packages and logs. `diagnostics_dir` applies to request-driven captures;
+it does not automatically snapshot every native Page action in `run_browser`.
+An exception type or screenshot does not by itself establish whether the target
+site blocked access, returned a different layout, or had not finished loading.
+
+## Read-only CAPTCHA observations
+
+`detect_captchas(html, ...)` and `await detect_captchas_on_page(page, ...)` classify
+observable reCAPTCHA v2/Enterprise, Turnstile, Cloudflare full-page Challenge,
+BotDetect, and image-CAPTCHA
+integrations. Results contain fixed evidence codes, component matches, and missing
+inputs. Script-only or unsupported evidence remains `unknown`; a form widget is
+distinct from an access challenge. `CaptchaOverride` accepts explicit site knowledge.
+
+Detection does not navigate, click, capture images, submit forms, or contact paid
+services. A solver match is structural readiness, not authorization or proof of
+website acceptance. `capture_image(locator)` separately captures the currently
+rendered challenge for an explicitly chosen resource call. Keep submission and
+site acceptance checks in ordinary application code.
+
+The candidate `CloudflareChallengeCapture` API observes dynamic full-page
+render parameters when installed before navigation, and applies a managed
+token only to the same active challenge. See the [token workflow](docs/cloudflare-challenge.md).
+The local contract and browser fixture have passed; real-site managed solving
+and deployment acceptance remain pending. Ordinary Turnstile and full-page
+Challenge tasks have different required fields. Detection alone never enables
+a paid call, and the `cf_clearance` cookie mode is not exposed.
+
+See the source template's [API reference](../../examples/crawler/relay_template/help/docs/API.zh-CN.md)
+for parameters and the Runtime/resource boundary.
+
+## Provider extensions
+
+Supplier-specific provider implementations are separate distributions:
+
+| Distribution | Import | Public provider types |
+|---|---|---|
+| `dataapp-crawler-octoparse-browser` | `dataapp_crawler_octoparse` | `BrowserEnvRelay`, `RelayBrowser` |
+| `dataapp-crawler-agentcore` | `dataapp_crawler_agentcore` | `AgentCoreBrowser`, `AWSBrowser` |
+
+The core wheel retains `CDPBrowser`, `PlaywrightBrowser`, and optional
+`CamoufoxBrowser` as generic integrations. It does not contain either supplier
+implementation and does not depend on Relay or boto3. Applications install one
+supplier extension deliberately and import its provider from that extension.
+This makes an accidental supplier-provider import fail at project-install time
+instead of silently exposing an unsupported runtime path.
+
+Source packages may share this repository; that does not merge their wheel
+contents, dependency graphs, project templates, or base images.
+
+## Proxy boundary
+
+Browser and proxy selection are independent. Core provides the common proxy
+contract and supports:
+
+- direct access with `proxy=None`;
+- a developer-provided `StaticProxy`;
+- a managed `DataOSProxy` through the Resource Proxy client;
+- a custom `ProxyProvider.acquire()` returning a `ProxyHandle`.
+
+`DataOSProxy.proxy_group` accepts `auto`, `domestic_basic`,
+`overseas_basic`, or `overseas_enhanced`. `auto` uses the basic managed
+service. The SDK does not select or name proxy suppliers: Resource Proxy owns
+the published supplier order and validates location and duration support.
+
+If managed proxy acquisition fails, `DataOSProxy` raises `ConfigurationError`
+with bounded `code` and `status_code` attributes. These diagnostics contain only
+the documented resource error class and HTTP status; supplier responses, proxy
+URLs, and credentials are never retained in the exception.
+
+A proxy handle is tied to a session lifecycle. It is not a promise to rotate the
+exit IP on every request. Credentials must remain outside inputs, source files,
+logs, and build arguments.
+
+## DataApp integration
+
+Within a Data App, use `DataAppCrawler` from
+`bazhuayu_dataapp.plugins.crawler`:
+
+```python
+from bazhuayu_dataapp import DataApp
+from bazhuayu_dataapp.plugins.crawler import DataAppCrawler
+
+
+async with DataApp:
+    async with DataAppCrawler(browser=browser_provider) as crawler:
+        records = await crawler.run_browser(collect)
+    await DataApp.push_data(records)
+```
+
+Runtime owns Run input, Dataset writes, progress, cancellation, and final state.
+`DataAppCrawler` only joins Crawler cancellation and cleanup to that lifecycle.
+
+## Logging
+
+The standalone Crawler writes INFO stage logs to stderr by default. The DataApp
+adapter uses `DataApp.log`, so the platform receives the same bounded stages.
+Pass a standard logger to change verbosity:
+
+```python
+import logging
+
+logger = logging.getLogger("my_app.crawler")
+logger.setLevel(logging.WARNING)
+
+async with Crawler(logger=logger) as crawler:
+    capture = await crawler.fetch(target_url)
+```
+
+The SDK does not configure the root logger and never logs the value returned by a
+process callback. Acquisition success is not Dataset success; the application
+must push records explicitly and validate the Runtime outcome separately.

dataapp_crawler-0.1.0/README.md

@@ -0,0 +1,195 @@
+# Data App Crawler core
+
+`dataapp-crawler` (imported as `dataapp_crawler`) is the provider-neutral website
+acquisition library for Python 3.11+. It can run independently of the DataApp
+Runtime. Core includes generic CDP/Playwright providers and an optional Camoufox
+adapter, but no Relay- or AWS-specific provider.
+
+> Dependency versions are defined in each package's `pyproject.toml`; an
+> application uses its own `uv.lock`. Local source changes are not delivered to
+> applications until matching packages and browser images are released and the
+> application's lock is updated.
+
+## Basic acquisition
+
+```python
+import asyncio
+
+from dataapp_crawler import Crawler
+
+
+async def main() -> None:
+    async with Crawler() as crawler:
+        capture = await crawler.fetch("https://example.com")
+        print(capture.status)
+
+
+asyncio.run(main())
+```
+
+Use an ordinary synchronous or asynchronous `process(capture, context)` function
+for extraction, OCR/ASR calls, model extraction, or custom logic. The SDK does
+not own a processing engine or Dataset:
+
+```python
+async def process(capture, context):
+    heading = capture.page.locator("h1").first
+    await context.run(heading.wait_for(state="visible"), timeout=10)
+    return {"title": (await context.run(heading.inner_text(), timeout=5)).strip()}
+```
+
+An operation timeout raises `TimeoutError`; exhausting `Limits.max_duration`
+raises `LimitReached`. Diagnostics are bounded and omit raw credentials, headers,
+connection URLs, response bodies, and arbitrary upstream exception text.
+
+## Page-driven flows
+
+`Crawler.run_browser(collect)` hands one native Playwright `Page` and a
+`CrawlContext` to ordinary application code. A single collection function can
+submit a search, traverse list pages, open detail pages, and stop on business
+conditions. It does not require one URL per `Crawler` object.
+
+```python
+from dataapp_crawler import Crawler, Limits
+from dataapp_crawler.browser_tools import open_url, text_of
+
+
+async def collect(page, context):
+    await open_url(page, "https://example.com", context)
+    return {"title": await text_of(page.locator("h1"), context, required=True)}
+
+
+async with Crawler(browser=browser_provider, limits=Limits(max_duration=120)) as crawler:
+    record = await crawler.run_browser(collect)
+```
+
+`browser_tools` includes bounded navigation and extraction helpers such as
+`open_url`, `new_page`, `popup_page`, `same_tab`, `wait_until`, `text_of`,
+`attribute_of`, and `iterate_list`. Page and Locator remain native Playwright
+objects. Raw Playwright actions remain available, but they do not automatically
+gain Crawler accounting, scheduling, diagnostics, or retry semantics.
+
+The application defines selectors, stable business keys, pagination, readiness,
+and side-effect boundaries. Do not automatically retry clicks or submissions
+that can create duplicate actions.
+
+Failure summaries include a fixed `exception_type` without its message. Browser
+flow failures also report `root_host`, the root Page's hostname only. This may be
+the list page rather than a failing detail tab. The original exception still
+propagates, with cancellation and cleanup unchanged. A failure before the root
+Page exists reports `phase=page_creation`.
+
+To inspect a failing detail page, explicitly capture a bounded screenshot inside
+that page's scope before it closes. Keep artifacts private, outside uploaded
+source packages and logs. `diagnostics_dir` applies to request-driven captures;
+it does not automatically snapshot every native Page action in `run_browser`.
+An exception type or screenshot does not by itself establish whether the target
+site blocked access, returned a different layout, or had not finished loading.
+
+## Read-only CAPTCHA observations
+
+`detect_captchas(html, ...)` and `await detect_captchas_on_page(page, ...)` classify
+observable reCAPTCHA v2/Enterprise, Turnstile, Cloudflare full-page Challenge,
+BotDetect, and image-CAPTCHA
+integrations. Results contain fixed evidence codes, component matches, and missing
+inputs. Script-only or unsupported evidence remains `unknown`; a form widget is
+distinct from an access challenge. `CaptchaOverride` accepts explicit site knowledge.
+
+Detection does not navigate, click, capture images, submit forms, or contact paid
+services. A solver match is structural readiness, not authorization or proof of
+website acceptance. `capture_image(locator)` separately captures the currently
+rendered challenge for an explicitly chosen resource call. Keep submission and
+site acceptance checks in ordinary application code.
+
+The candidate `CloudflareChallengeCapture` API observes dynamic full-page
+render parameters when installed before navigation, and applies a managed
+token only to the same active challenge. See the [token workflow](docs/cloudflare-challenge.md).
+The local contract and browser fixture have passed; real-site managed solving
+and deployment acceptance remain pending. Ordinary Turnstile and full-page
+Challenge tasks have different required fields. Detection alone never enables
+a paid call, and the `cf_clearance` cookie mode is not exposed.
+
+See the source template's [API reference](../../examples/crawler/relay_template/help/docs/API.zh-CN.md)
+for parameters and the Runtime/resource boundary.
+
+## Provider extensions
+
+Supplier-specific provider implementations are separate distributions:
+
+| Distribution | Import | Public provider types |
+|---|---|---|
+| `dataapp-crawler-octoparse-browser` | `dataapp_crawler_octoparse` | `BrowserEnvRelay`, `RelayBrowser` |
+| `dataapp-crawler-agentcore` | `dataapp_crawler_agentcore` | `AgentCoreBrowser`, `AWSBrowser` |
+
+The core wheel retains `CDPBrowser`, `PlaywrightBrowser`, and optional
+`CamoufoxBrowser` as generic integrations. It does not contain either supplier
+implementation and does not depend on Relay or boto3. Applications install one
+supplier extension deliberately and import its provider from that extension.
+This makes an accidental supplier-provider import fail at project-install time
+instead of silently exposing an unsupported runtime path.
+
+Source packages may share this repository; that does not merge their wheel
+contents, dependency graphs, project templates, or base images.
+
+## Proxy boundary
+
+Browser and proxy selection are independent. Core provides the common proxy
+contract and supports:
+
+- direct access with `proxy=None`;
+- a developer-provided `StaticProxy`;
+- a managed `DataOSProxy` through the Resource Proxy client;
+- a custom `ProxyProvider.acquire()` returning a `ProxyHandle`.
+
+`DataOSProxy.proxy_group` accepts `auto`, `domestic_basic`,
+`overseas_basic`, or `overseas_enhanced`. `auto` uses the basic managed
+service. The SDK does not select or name proxy suppliers: Resource Proxy owns
+the published supplier order and validates location and duration support.
+
+If managed proxy acquisition fails, `DataOSProxy` raises `ConfigurationError`
+with bounded `code` and `status_code` attributes. These diagnostics contain only
+the documented resource error class and HTTP status; supplier responses, proxy
+URLs, and credentials are never retained in the exception.
+
+A proxy handle is tied to a session lifecycle. It is not a promise to rotate the
+exit IP on every request. Credentials must remain outside inputs, source files,
+logs, and build arguments.
+
+## DataApp integration
+
+Within a Data App, use `DataAppCrawler` from
+`bazhuayu_dataapp.plugins.crawler`:
+
+```python
+from bazhuayu_dataapp import DataApp
+from bazhuayu_dataapp.plugins.crawler import DataAppCrawler
+
+
+async with DataApp:
+    async with DataAppCrawler(browser=browser_provider) as crawler:
+        records = await crawler.run_browser(collect)
+    await DataApp.push_data(records)
+```
+
+Runtime owns Run input, Dataset writes, progress, cancellation, and final state.
+`DataAppCrawler` only joins Crawler cancellation and cleanup to that lifecycle.
+
+## Logging
+
+The standalone Crawler writes INFO stage logs to stderr by default. The DataApp
+adapter uses `DataApp.log`, so the platform receives the same bounded stages.
+Pass a standard logger to change verbosity:
+
+```python
+import logging
+
+logger = logging.getLogger("my_app.crawler")
+logger.setLevel(logging.WARNING)
+
+async with Crawler(logger=logger) as crawler:
+    capture = await crawler.fetch(target_url)
+```
+
+The SDK does not configure the root logger and never logs the value returned by a
+process callback. Acquisition success is not Dataset success; the application
+must push records explicitly and validate the Runtime outcome separately.

dataapp_crawler-0.1.0/pyproject.toml

@@ -0,0 +1,52 @@
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[project]
+name = "dataapp_crawler"
+version = "0.1.0"
+description = "Composable HTTP, browser and proxy acquisition SDK for DataOS DataApps"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = ["httpx>=0.28,<1"]
+
+[project.optional-dependencies]
+parse = ["parsel>=1.9,<2"]
+browser = ["playwright>=1.64,<2"]
+resource-proxy = ["dataapp-resource-proxy>=0.1.0,<0.2"]
+fingerprint = ["camoufox>=0.5.9,<0.6"]
+
+[tool.uv]
+# Camoufox owns a Firefox build and currently requires Playwright <1.63.
+# Select it in a separate environment from native Chromium/Relay extras.
+conflicts = [
+  [{extra = "fingerprint"}, {extra = "browser"}],
+]
+
+[dependency-groups]
+dev = ["pytest>=8,<10", "pytest-asyncio>=0.24,<2", "ruff>=0.9", "mypy>=1.13"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dataapp_crawler"]
+
+[tool.hatch.build.targets.sdist]
+only-include = ["src", "README.md", "pyproject.toml"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
+testpaths = ["tests"]
+addopts = "-ra"
+pythonpath = ["src", "../crawler-octoparse/src", "../crawler-agentcore/src"]
+
+[tool.ruff]
+target-version = "py311"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B"]
+
+[tool.mypy]
+python_version = "3.11"
+check_untyped_defs = true
+ignore_missing_imports = true

dataapp_crawler-0.1.0/src/dataapp_crawler/__init__.py

@@ -0,0 +1,87 @@
+"""Developer-friendly acquisition, without a dependency on the DataOS Runtime."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from .captcha import (
+    CaptchaConfidence,
+    CaptchaEvidence,
+    CaptchaFamily,
+    CaptchaObservation,
+    CaptchaOverride,
+    CaptchaPresence,
+    SolverComponent,
+    SolverMatch,
+    detect_captchas,
+    detect_captchas_on_page,
+)
+from .checkpoint import CrawlCheckpoint
+from .cloudflare import CloudflareChallengeCapture, CloudflareChallengeParameters
+from .context import CrawlContext
+from .crawler import Crawler
+from .errors import (
+    ConfigurationError,
+    CrawlCancelled,
+    CrawlerError,
+    FetchError,
+    LimitReached,
+    ProcessingError,
+    QueueFull,
+    ResponseTooLarge,
+)
+from .local_auth import local_resource_environment, read_api_key_file, run_local_app
+from .models import CaptureResult, CrawlItem, Failure, Identity, Limits, Request, Stats
+from .pagination import Pagination
+from .routing import RouteAlreadyRegistered, RouteNotFound, Router
+from .scheduling import AdaptivePolicy, AutoConcurrency, ResourceSample
+from .sessions import Session, SessionInfo, SessionPool
+
+__all__ = [
+    "AdaptivePolicy",
+    "AutoConcurrency",
+    "CaptchaConfidence",
+    "CaptchaEvidence",
+    "CaptchaFamily",
+    "CaptchaObservation",
+    "CaptchaOverride",
+    "CaptchaPresence",
+    "CaptureResult",
+    "CrawlCheckpoint",
+    "CloudflareChallengeCapture",
+    "CloudflareChallengeParameters",
+    "ResourceSample",
+    "ConfigurationError",
+    "CrawlCancelled",
+    "CrawlContext",
+    "Crawler",
+    "CrawlerError",
+    "CrawlItem",
+    "Failure",
+    "FetchError",
+    "Identity",
+    "LimitReached",
+    "local_resource_environment",
+    "Limits",
+    "ProcessingError",
+    "QueueFull",
+    "Request",
+    "ResponseTooLarge",
+    "read_api_key_file",
+    "RouteAlreadyRegistered",
+    "RouteNotFound",
+    "run_local_app",
+    "Router",
+    "Pagination",
+    "Stats",
+    "Session",
+    "SessionInfo",
+    "SessionPool",
+    "SolverComponent",
+    "SolverMatch",
+    "detect_captchas",
+    "detect_captchas_on_page",
+]
+
+try:
+    __version__ = version("dataapp_crawler")
+except PackageNotFoundError:
+    __version__ = "0.0.0"

dataapp_crawler-0.1.0/src/dataapp_crawler/__main__.py

@@ -0,0 +1,35 @@
+"""Local Crawler SDK command helpers."""
+
+from __future__ import annotations
+
+import argparse
+from collections.abc import Sequence
+
+from .errors import ConfigurationError
+from .local_auth import run_local_app
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="python -m dataapp_crawler")
+    commands = parser.add_subparsers(dest="command", required=True)
+    run = commands.add_parser(
+        "run",
+        help="run a local DataApp with a file-backed platform API Key",
+    )
+    run.add_argument("--api-key-file", required=True, help="regular file containing one API Key")
+    run.add_argument("app_command", nargs=argparse.REMAINDER, help="application command after --")
+    options = parser.parse_args(argv)
+    if options.command != "run":
+        return 2
+    command = list(options.app_command)
+    if command[:1] == ["--"]:
+        command = command[1:]
+    try:
+        return run_local_app(options.api_key_file, command)
+    except ConfigurationError as error:
+        print(f"配置错误：{error}")
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())