scrapebadger-cli 0.2.1__py3-none-any.whl

scrapebadger_cli/commands/web.py

@@ -0,0 +1,117 @@
+"""Web scraping commands."""
+
+import typer
+
+from scrapebadger_cli.client import api_get, api_post
+from scrapebadger_cli.output import render
+
+app = typer.Typer(no_args_is_help=True)
+
+FMT = typer.Option("json", "--output", "-o", help="Output format: json, csv, table, markdown")
+FIELDS = typer.Option(None, "--fields", "-f", help="Comma-separated fields to include")
+
+
+@app.command()
+def scrape(
+    url: str = typer.Argument(..., help="URL to scrape"),
+    render_js: bool = typer.Option(False, "--js", help="Enable JavaScript rendering"),
+    wait_for: str | None = typer.Option(None, "--wait-for", help="CSS selector to wait for"),
+    timeout_ms: int | None = typer.Option(None, "--timeout", help="Timeout in milliseconds"),
+    proxy_country: str | None = typer.Option(None, "--proxy", help="Proxy country code"),
+    return_format: str | None = typer.Option(
+        None, "--format", help="Response format: html, markdown, text"
+    ),
+    fmt: str = FMT,
+    fields: str | None = FIELDS,
+) -> None:
+    """Scrape a URL and return its content."""
+    body: dict = {"url": url}
+    if render_js:
+        body["render_js"] = True
+    if wait_for:
+        body["wait_for"] = wait_for
+    if timeout_ms:
+        body["timeout_ms"] = timeout_ms
+    if proxy_country:
+        body["proxy_country"] = proxy_country
+    if return_format:
+        body["return_format"] = return_format
+    data = api_post("/v1/web/scrape", body)
+    render(data, fmt, fields)
+
+
+@app.command()
+def detect(
+    url: str = typer.Argument(..., help="URL to analyze"),
+    fmt: str = FMT,
+    fields: str | None = FIELDS,
+) -> None:
+    """Detect anti-bot and CAPTCHA protection on a URL."""
+    data = api_post("/v1/web/detect", {"url": url})
+    render(data, fmt, fields)
+
+
+@app.command()
+def screenshot(
+    url: str = typer.Argument(..., help="URL to screenshot"),
+    full_page: bool = typer.Option(False, "--full-page", help="Capture full scrollable page"),
+    width: int | None = typer.Option(None, "--width", help="Viewport width"),
+    height: int | None = typer.Option(None, "--height", help="Viewport height"),
+    fmt: str = FMT,
+    fields: str | None = FIELDS,
+) -> None:
+    """Take a screenshot of a URL."""
+    body: dict = {"url": url}
+    if full_page:
+        body["full_page"] = True
+    if width:
+        body["width"] = width
+    if height:
+        body["height"] = height
+    data = api_post("/v1/web/screenshot", body)
+    render(data, fmt, fields)
+
+
+@app.command()
+def extract(
+    url: str = typer.Argument(..., help="URL to extract data from"),
+    rules: str | None = typer.Option(None, "--rules", help="CSS/XPath rules as JSON"),
+    ai_rules: str | None = typer.Option(None, "--ai-rules", help="AI extraction rules as JSON"),
+    ai_query: str | None = typer.Option(None, "--ai-query", help="Freeform AI query"),
+    fmt: str = FMT,
+    fields: str | None = FIELDS,
+) -> None:
+    """Extract structured data from a URL."""
+    import json as json_mod
+
+    body: dict = {"url": url}
+    if rules:
+        body["extract_rules"] = json_mod.loads(rules)
+    if ai_rules:
+        body["ai_extract_rules"] = json_mod.loads(ai_rules)
+    if ai_query:
+        body["ai_query"] = ai_query
+    data = api_post("/v1/web/extract", body)
+    render(data, fmt, fields)
+
+
+@app.command(name="batch")
+def batch_submit(
+    urls: list[str] = typer.Argument(..., help="URLs to scrape (space-separated)"),
+    fmt: str = FMT,
+    fields: str | None = FIELDS,
+) -> None:
+    """Submit a batch of URLs for scraping."""
+    data = api_post("/v1/web/batch", {"urls": urls})
+    render(data, fmt, fields)
+
+
+@app.command(name="batch-status")
+def batch_status(
+    job_id: str = typer.Argument(..., help="Batch job ID"),
+    fmt: str = FMT,
+    fields: str | None = FIELDS,
+) -> None:
+    """Get the status of a batch scraping job."""
+    data = api_get(f"/v1/web/batch/{job_id}")
+    render(data, fmt, fields)