scrapebadger-cli 0.4.2__tar.gz → 0.5.0__tar.gz

{scrapebadger_cli-0.4.2 → scrapebadger_cli-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapebadger-cli
-Version: 0.4.2
+Version: 0.5.0
 Summary: ScrapeBadger CLI — Twitter, Vinted, Google, and Web Scraping from the command line
 Author: ScrapeBadger Team
 License: MIT

{scrapebadger_cli-0.4.2 → scrapebadger_cli-0.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapebadger-cli"
-version = "0.4.2"
+version = "0.5.0"
 description = "ScrapeBadger CLI — Twitter, Vinted, Google, and Web Scraping from the command line"
 requires-python = ">=3.10"
 license = {text = "MIT"}

{scrapebadger_cli-0.4.2 → scrapebadger_cli-0.5.0}/src/scrapebadger_cli/client.py

@@ -43,6 +43,19 @@ def api_post(path: str, json_body: dict[str, Any] | None = None) -> dict | list:
     return _handle_response(resp)
 
 
+def api_post_raw(path: str, json_body: dict[str, Any] | None = None) -> tuple[bytes, str]:
+    """POST and return ``(body_bytes, content_type)`` without JSON-decoding.
+
+    For `/v1/web/scrape` with `raw_content`, whose response is the scraped body
+    itself. Returns bytes, not text: the payload may be an image or a PDF and
+    decoding it would corrupt it.
+    """
+    resp = _get_client().post(path, json=json_body)
+    if resp.status_code >= 400:
+        _handle_response(resp)
+    return resp.content, resp.headers.get("content-type", "")
+
+
 def _handle_response(resp: httpx.Response) -> dict | list:
     """Handle API response, exit on errors."""
     if resp.status_code >= 400:

{scrapebadger_cli-0.4.2 → scrapebadger_cli-0.5.0}/src/scrapebadger_cli/commands/web.py

@@ -2,7 +2,7 @@
 
 import typer
 
-from scrapebadger_cli.client import api_get, api_post
+from scrapebadger_cli.client import api_get, api_post, api_post_raw
 from scrapebadger_cli.output import render
 
 app = typer.Typer(no_args_is_help=True)
@@ -21,22 +21,58 @@ def scrape(
     return_format: str | None = typer.Option(
         None, "--format", help="Response format: html, markdown, text"
     ),
+    save: str | None = typer.Option(
+        None,
+        "--save",
+        help="Write the response body to this file. Required for binary targets "
+        "(images, PDFs, archives) — use it to download a file.",
+    ),
     fmt: str = FMT,
     fields: str | None = FIELDS,
 ) -> None:
     """Scrape a URL and return its content."""
+    # These option names are the CLI's contract; the wire names are
+    # ScrapeRequest's. Three of them differ, and ScrapeRequest sets no
+    # model_config, so pydantic SILENTLY DROPS unknown keys — sending
+    # timeout_ms/proxy_country/return_format was a no-op that looked like it
+    # worked. Map them explicitly.
     body: dict = {"url": url}
     if render_js:
         body["render_js"] = True
     if wait_for:
         body["wait_for"] = wait_for
     if timeout_ms:
-        body["timeout_ms"] = timeout_ms
+        body["wait_timeout"] = timeout_ms
     if proxy_country:
-        body["proxy_country"] = proxy_country
+        body["country"] = proxy_country
     if return_format:
-        body["return_format"] = return_format
+        body["format"] = return_format
+
+    # --save streams the body straight to disk instead of JSON-wrapping it: no
+    # base64 expansion, and binary bytes are never decoded.
+    if save:
+        body["raw_content"] = True
+        payload, content_type = api_post_raw("/v1/web/scrape", body)
+        with open(save, "wb") as handle:
+            handle.write(payload)
+        typer.echo(f"Wrote {len(payload)} bytes to {save} ({content_type or 'unknown type'})")
+        return
+
     data = api_post("/v1/web/scrape", body)
+
+    # A binary target returns base64; dumping megabytes of it into a terminal
+    # helps nobody. Say what it is and how to get it.
+    if isinstance(data, dict) and data.get("is_binary"):
+        encoded = data.get("content_base64")
+        size = data.get("content_length", 0)
+        kind = data.get("content_type") or "binary"
+        if encoded:
+            data = {**data, "content_base64": f"<{size} bytes of {kind}, omitted>"}
+        typer.echo(
+            f"Binary response ({kind}, {size} bytes). Re-run with --save PATH to download it.",
+            err=True,
+        )
+
     render(data, fmt, fields)