scrapingbee-cli 1.2.3__tar.gz → 1.3.1__tar.gz

{scrapingbee_cli-1.2.3/src/scrapingbee_cli.egg-info → scrapingbee_cli-1.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapingbee-cli
-Version: 1.2.3
+Version: 1.3.1
 Summary: Command-line client for the ScrapingBee API: scrape pages (single or batch), crawl sites, check usage/credits, and use Google Search, Fast Search, Amazon, Walmart, YouTube, and ChatGPT from the terminal.
 Author: ScrapingBee
 License-Expression: MIT
@@ -81,7 +81,9 @@ scrapingbee [command] [arguments] [options]
 - **`scrapingbee --help`** – List all commands.
 - **`scrapingbee [command] --help`** – Options and parameters for that command.
 
-**Options are per-command.** Each command has its own set of options — run `scrapingbee [command] --help` to see them. Common options across batch-capable commands include `--output-file`, `--output-dir`, `--input-file`, `--input-column`, `--concurrency`, `--output-format`, `--retries`, `--backoff`, `--resume`, `--update-csv`, `--no-progress`, `--extract-field`, `--fields`, `--deduplicate`, `--sample`, `--post-process`, `--on-complete`, and `--verbose`. For details, see the [documentation](https://www.scrapingbee.com/documentation/).
+**Options are per-command.** Each command has its own set of options — run `scrapingbee [command] --help` to see them. Common options across batch-capable commands include `--output-file`, `--output-dir`, `--input-file`, `--input-column`, `--concurrency`, `--output-format`, `--retries`, `--backoff`, `--resume`, `--update-csv`, `--no-progress`, `--extract-field`, `--fields`, `--deduplicate`, `--sample`, `--post-process`, `--on-complete`, `--scraping-config`, and `--verbose`. For details, see the [documentation](https://www.scrapingbee.com/documentation/).
+
+**Parameter values:** Choice parameters accept both hyphens and underscores interchangeably (e.g. `--sort-by price-low` and `--sort-by price_low` both work).
 
 ### Commands
 
@@ -117,6 +119,7 @@ scrapingbee [command] [arguments] [options]
 - **Scheduling:** `scrapingbee schedule --every 1d --name prices scrape --input-file products.csv --update-csv` registers a cron job. Use `--list`, `--stop NAME`, or `--stop all`.
 - **Deduplication & sampling:** `--deduplicate` removes duplicate URLs; `--sample 100` processes only 100 random items.
 - **RAG chunking:** `scrape --chunk-size 500 --chunk-overlap 50 --return-page-markdown true` outputs NDJSON chunks ready for vector DB ingestion.
+- **Scraping configurations:** `--scraping-config "My-Config"` applies a pre-saved configuration from your ScrapingBee dashboard. Inline options override config settings. Create configurations in the [request builder](https://app.scrapingbee.com/).
 
 ### Examples
 
@@ -133,6 +136,14 @@ scrapingbee schedule --every 1d --name price-tracker scrape --input-file product
 scrapingbee schedule --list
 ```
 
+## Security
+
+The `--post-process`, `--on-complete`, and `schedule` commands execute arbitrary shell commands on your machine. These features are **disabled by default** and require explicit human setup to enable.
+
+For advanced features setup, see the Security section in our [CLI documentation](https://www.scrapingbee.com/documentation/cli/).
+
+**Do not enable these features in AI agent environments** where commands may be constructed from scraped web content. ScrapingBee is not responsible for any damages caused by shell execution features. Use at your own discretion.
+
 ## More information
 
 - **[CLI Documentation](https://www.scrapingbee.com/documentation/cli/)** – Full CLI reference with pipelines, parameters, and examples.

{scrapingbee_cli-1.2.3 → scrapingbee_cli-1.3.1}/README.md

@@ -44,7 +44,9 @@ scrapingbee [command] [arguments] [options]
 - **`scrapingbee --help`** – List all commands.
 - **`scrapingbee [command] --help`** – Options and parameters for that command.
 
-**Options are per-command.** Each command has its own set of options — run `scrapingbee [command] --help` to see them. Common options across batch-capable commands include `--output-file`, `--output-dir`, `--input-file`, `--input-column`, `--concurrency`, `--output-format`, `--retries`, `--backoff`, `--resume`, `--update-csv`, `--no-progress`, `--extract-field`, `--fields`, `--deduplicate`, `--sample`, `--post-process`, `--on-complete`, and `--verbose`. For details, see the [documentation](https://www.scrapingbee.com/documentation/).
+**Options are per-command.** Each command has its own set of options — run `scrapingbee [command] --help` to see them. Common options across batch-capable commands include `--output-file`, `--output-dir`, `--input-file`, `--input-column`, `--concurrency`, `--output-format`, `--retries`, `--backoff`, `--resume`, `--update-csv`, `--no-progress`, `--extract-field`, `--fields`, `--deduplicate`, `--sample`, `--post-process`, `--on-complete`, `--scraping-config`, and `--verbose`. For details, see the [documentation](https://www.scrapingbee.com/documentation/).
+
+**Parameter values:** Choice parameters accept both hyphens and underscores interchangeably (e.g. `--sort-by price-low` and `--sort-by price_low` both work).
 
 ### Commands
 
@@ -80,6 +82,7 @@ scrapingbee [command] [arguments] [options]
 - **Scheduling:** `scrapingbee schedule --every 1d --name prices scrape --input-file products.csv --update-csv` registers a cron job. Use `--list`, `--stop NAME`, or `--stop all`.
 - **Deduplication & sampling:** `--deduplicate` removes duplicate URLs; `--sample 100` processes only 100 random items.
 - **RAG chunking:** `scrape --chunk-size 500 --chunk-overlap 50 --return-page-markdown true` outputs NDJSON chunks ready for vector DB ingestion.
+- **Scraping configurations:** `--scraping-config "My-Config"` applies a pre-saved configuration from your ScrapingBee dashboard. Inline options override config settings. Create configurations in the [request builder](https://app.scrapingbee.com/).
 
 ### Examples
 
@@ -96,6 +99,14 @@ scrapingbee schedule --every 1d --name price-tracker scrape --input-file product
 scrapingbee schedule --list
 ```
 
+## Security
+
+The `--post-process`, `--on-complete`, and `schedule` commands execute arbitrary shell commands on your machine. These features are **disabled by default** and require explicit human setup to enable.
+
+For advanced features setup, see the Security section in our [CLI documentation](https://www.scrapingbee.com/documentation/cli/).
+
+**Do not enable these features in AI agent environments** where commands may be constructed from scraped web content. ScrapingBee is not responsible for any damages caused by shell execution features. Use at your own discretion.
+
 ## More information
 
 - **[CLI Documentation](https://www.scrapingbee.com/documentation/cli/)** – Full CLI reference with pipelines, parameters, and examples.

{scrapingbee_cli-1.2.3 → scrapingbee_cli-1.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapingbee-cli"
-version = "1.2.3"
+version = "1.3.1"
 description = "Command-line client for the ScrapingBee API: scrape pages (single or batch), crawl sites, check usage/credits, and use Google Search, Fast Search, Amazon, Walmart, YouTube, and ChatGPT from the terminal."
 readme = "README.md"
 license = "MIT"

{scrapingbee_cli-1.2.3 → scrapingbee_cli-1.3.1}/src/scrapingbee_cli/__init__.py

@@ -3,7 +3,7 @@
 import platform
 import sys
 
-__version__ = "1.2.3"
+__version__ = "1.3.1"
 
 
 def user_agent() -> str:

scrapingbee_cli-1.3.1/src/scrapingbee_cli/audit.py

@@ -0,0 +1,60 @@
+"""Audit logging for exec features (--post-process, --on-complete, schedule).
+
+Logs every shell command execution to a fixed location for forensics
+and guard skill monitoring.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+
+AUDIT_LOG_PATH = Path.home() / ".config" / "scrapingbee-cli" / "audit.log"
+MAX_LINES = 10_000
+
+
+def log_exec(
+    feature: str,
+    command: str,
+    *,
+    input_source: str = "",
+    output_dir: str = "",
+) -> None:
+    """Append an entry to the audit log.
+
+    Format: ISO_TIMESTAMP | FEATURE | COMMAND | INPUT | OUTPUT_DIR
+    """
+    AUDIT_LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    timestamp = datetime.now(timezone.utc).isoformat()
+    entry = f"{timestamp} | {feature} | {command} | {input_source} | {output_dir}\n"
+    try:
+        with open(AUDIT_LOG_PATH, "a", encoding="utf-8") as f:
+            f.write(entry)
+        _rotate_if_needed()
+    except OSError:
+        pass
+
+
+def read_audit_log(n: int = 50) -> str:
+    """Read the last N lines of the audit log."""
+    if not AUDIT_LOG_PATH.is_file():
+        return "No audit log found."
+    try:
+        with open(AUDIT_LOG_PATH, encoding="utf-8") as f:
+            lines = f.readlines()
+        recent = lines[-n:] if len(lines) > n else lines
+        return "".join(recent)
+    except OSError:
+        return "Could not read audit log."
+
+
+def _rotate_if_needed() -> None:
+    """Keep only the last MAX_LINES entries."""
+    try:
+        with open(AUDIT_LOG_PATH, encoding="utf-8") as f:
+            lines = f.readlines()
+        if len(lines) > MAX_LINES:
+            with open(AUDIT_LOG_PATH, "w", encoding="utf-8") as f:
+                f.writelines(lines[-MAX_LINES:])
+    except OSError:
+        pass

{scrapingbee_cli-1.2.3 → scrapingbee_cli-1.3.1}/src/scrapingbee_cli/batch.py

@@ -705,6 +705,13 @@ def apply_post_process(body: bytes, cmd: str) -> bytes:
     """Run shell command with body as stdin, return stdout. On failure, return original body."""
     import subprocess
 
+    from .audit import log_exec
+    from .exec_gate import require_exec
+
+    require_exec("--post-process", cmd)
+    log_exec("post-process", cmd)
+    click.echo(f"⚠ Executing: {cmd.split()[0] if cmd.split() else cmd} (whitelisted)", err=True)
+
     try:
         result = subprocess.run(
             cmd,

{scrapingbee_cli-1.2.3 → scrapingbee_cli-1.3.1}/src/scrapingbee_cli/cli_utils.py

@@ -9,6 +9,23 @@ from typing import Any
 import click
 
 
+class NormalizedChoice(click.Choice):
+    """Choice type that accepts both hyphens and underscores.
+
+    Automatically converts underscores to hyphens before validation,
+    allowing users to use either format interchangeably.
+    Example: both --sort-by price-low and --sort-by price_low work.
+    """
+
+    def convert(self, value: str, param: Any, ctx: Any) -> str:
+        """Convert underscores to hyphens before validation."""
+        if value is not None:
+            normalized = value.replace("_", "-")
+        else:
+            normalized = value
+        return super().convert(normalized, param, ctx)
+
+
 def _output_options(f: Any) -> Any:
     """Output + Retry options (for commands without batch support)."""
     f = click.option(
@@ -105,7 +122,7 @@ def _batch_options(f: Any) -> Any:
         "post_process",
         type=str,
         default=None,
-        help="Batch: pipe each result through a shell command (e.g. 'jq .title').",
+        help="[Advanced] Batch: pipe each result through a shell command (e.g. 'jq .title'). Requires unsafe mode.",
     )(f)
     f = click.option(
         "--update-csv",
@@ -132,7 +149,7 @@ def _batch_options(f: Any) -> Any:
         "on_complete",
         type=str,
         default=None,
-        help="Batch: shell command to run after completion.",
+        help="[Advanced] Batch: shell command to run after completion. Requires unsafe mode.",
     )(f)
     f = click.option("--retries", type=int, default=3, help="Retry on errors (default: 3).")(f)
     f = click.option(
@@ -385,6 +402,7 @@ def build_scrape_kwargs(
     custom_google: str | None = None,
     transparent_status_code: str | None = None,
     body: str | None = None,
+    scraping_config: str | None = None,
 ) -> dict[str, Any]:
     """Build kwargs for Client.scrape() from scrape command options.
     Single source of parse_bool for bool-like opts."""
@@ -424,6 +442,7 @@ def build_scrape_kwargs(
         "custom_google": parse_bool(custom_google),
         "transparent_status_code": parse_bool(transparent_status_code),
         "body": body,
+        "scraping_config": scraping_config,
     }
 
 
@@ -595,11 +614,17 @@ def run_on_complete(
     import os
     import subprocess
 
+    from .audit import log_exec
+    from .exec_gate import require_exec
+
+    require_exec("--on-complete", cmd)
+    log_exec("on-complete", cmd, output_dir=output_dir)
+    click.echo(f"⚠ Executing: {cmd.split()[0] if cmd.split() else cmd} (whitelisted)", err=True)
+
     env = os.environ.copy()
     env["SCRAPINGBEE_OUTPUT_DIR"] = output_dir
     env["SCRAPINGBEE_SUCCEEDED"] = str(succeeded)
     env["SCRAPINGBEE_FAILED"] = str(failed)
-    click.echo(f"[on-complete] Running: {cmd}", err=True)
     result = subprocess.run(cmd, shell=True, env=env)  # noqa: S602
     if result.returncode != 0:
         click.echo(f"[on-complete] Exit code: {result.returncode}", err=True)

{scrapingbee_cli-1.2.3 → scrapingbee_cli-1.3.1}/src/scrapingbee_cli/client.py

@@ -177,6 +177,7 @@ class Client:
         custom_google: bool | None = None,
         transparent_status_code: bool | None = None,
         body: str | None = None,
+        scraping_config: str | None = None,
         retries: int = 3,
         backoff: float = 2.0,
         **kwargs: Any,
@@ -217,6 +218,7 @@ class Client:
             ("device", device),
             ("custom_google", self._bool(custom_google)),
             ("transparent_status_code", self._bool(transparent_status_code)),
+            ("scraping_config", scraping_config),
         ]:
             if v is not None:
                 params[k] = str(v) if not isinstance(v, str) else v
@@ -415,6 +417,7 @@ class Client:
     async def walmart_search(
         self,
         query: str,
+        start_page: int | None = None,
         min_price: int | None = None,
         max_price: int | None = None,
         sort_by: str | None = None,
@@ -432,6 +435,7 @@ class Client:
     ) -> tuple[bytes, dict, int]:
         params = {
             "query": query,
+            "start_page": start_page if start_page is not None else None,
             "min_price": min_price if min_price is not None else None,
             "max_price": max_price if max_price is not None else None,
             "sort_by": sort_by,
@@ -455,6 +459,7 @@ class Client:
     async def walmart_product(
         self,
         product_id: str,
+        device: str | None = None,
         domain: str | None = None,
         delivery_zip: str | None = None,
         store_id: str | None = None,
@@ -466,6 +471,7 @@ class Client:
     ) -> tuple[bytes, dict, int]:
         params = {
             "product_id": product_id,
+            "device": device,
             "domain": domain,
             "delivery_zip": delivery_zip,
             "store_id": store_id,
@@ -497,6 +503,7 @@ class Client:
         hdr: bool | None = None,
         location: bool | None = None,
         vr180: bool | None = None,
+        purchased: bool | None = None,
         retries: int = 3,
         backoff: float = 2.0,
     ) -> tuple[bytes, dict, int]:
@@ -516,6 +523,7 @@ class Client:
             "hdr": self._bool(hdr),
             "location": self._bool(location),
             "vr180": self._bool(vr180),
+            "purchased": self._bool(purchased),
         }
         return await self._get_with_retry(
             "/youtube/search",

{scrapingbee_cli-1.2.3 → scrapingbee_cli-1.3.1}/src/scrapingbee_cli/commands/__init__.py

@@ -35,3 +35,6 @@ def register_commands(cli: click.Group) -> None:
     chatgpt.register(cli)
     export.register(cli)
     schedule.register(cli)
+    from . import unsafe
+
+    unsafe.register(cli)

{scrapingbee_cli-1.2.3 → scrapingbee_cli-1.3.1}/src/scrapingbee_cli/commands/amazon.py

@@ -17,6 +17,7 @@ from ..batch import (
 )
 from ..cli_utils import (
     DEVICE_DESKTOP_MOBILE_TABLET,
+    NormalizedChoice,
     _batch_options,
     _validate_page,
     check_api_response,
@@ -191,7 +192,7 @@ def amazon_product_cmd(
 @optgroup.option("--pages", type=int, default=None, help="Number of pages to fetch.")
 @optgroup.option(
     "--sort-by",
-    type=click.Choice(AMAZON_SORT_BY, case_sensitive=False),
+    type=NormalizedChoice(AMAZON_SORT_BY, case_sensitive=False),
     default=None,
     help="Sort order.",
 )

scrapingbee_cli-1.3.1/src/scrapingbee_cli/commands/auth.py

@@ -0,0 +1,264 @@
+"""Auth, docs, and logout commands."""
+
+from __future__ import annotations
+
+import asyncio
+import getpass
+
+import click
+
+from ..client import Client
+from ..config import (
+    BASE_URL,
+    auth_config_path,
+    get_api_key_if_set,
+    remove_api_key_from_dotenv,
+    save_api_key_to_dotenv,
+)
+
+DOCS_URL = "https://www.scrapingbee.com/documentation/"
+
+
+def _validate_api_key(key: str) -> bool:
+    """Validate API key by calling the usage endpoint. Returns True if valid."""
+
+    async def _check() -> int:
+        async with Client(key, BASE_URL) as client:
+            _, _, status_code = await client.usage(retries=1, backoff=1.0)
+            return status_code
+
+    try:
+        status = asyncio.run(_check())
+        return status == 200
+    except Exception:
+        return False
+
+
+_UNSAFE_DISCLAIMER = """
+════════════════════════════════════════════════════════════════
+⚠  WARNING: UNSAFE MODE
+════════════════════════════════════════════════════════════════
+
+You are enabling shell execution features (--post-process,
+--on-complete, and the schedule command). These execute ARBITRARY SHELL COMMANDS
+on your machine.
+
+RISKS:
+  • Data exfiltration (SSH keys, credentials, files)
+  • Arbitrary code execution
+  • Persistent backdoors via cron scheduling
+
+DO NOT enable this in AI agent environments where commands
+may be constructed from scraped web content.
+
+ScrapingBee is NOT responsible for any damages caused by
+these features. Use at your own discretion.
+
+════════════════════════════════════════════════════════════════
+"""
+
+
+def _wipe_api_key_everywhere() -> None:
+    """Remove the API key from config .env, cwd .env, and os.environ."""
+    import os
+    from pathlib import Path
+
+    from ..config import ENV_API_KEY
+
+    # Remove from config .env
+    remove_api_key_from_dotenv()
+
+    # Remove from cwd .env if present
+    cwd_env = Path.cwd() / ".env"
+    if cwd_env.is_file():
+        try:
+            lines = []
+            with open(cwd_env, encoding="utf-8") as f:
+                for line in f:
+                    if ENV_API_KEY not in line:
+                        lines.append(line)
+            with open(cwd_env, "w", encoding="utf-8") as f:
+                f.writelines(lines)
+        except OSError:
+            pass
+
+    # Remove from current process env
+    os.environ.pop(ENV_API_KEY, None)
+
+
+@click.command()
+@click.option(
+    "--api-key",
+    "auth_api_key",
+    default=None,
+    help="API key to save (non-interactive); otherwise uses env/.env or prompt.",
+)
+@click.option(
+    "--show",
+    "show_path_only",
+    is_flag=True,
+    default=False,
+    help="Only show the path where the API key is or would be stored; do not save.",
+)
+@click.option(
+    "--unsafe",
+    "unsafe_mode",
+    is_flag=True,
+    default=False,
+    hidden=True,
+    help="Enable advanced shell execution features.",
+)
+@click.pass_obj
+def auth_cmd(obj: dict, auth_api_key: str | None, show_path_only: bool, unsafe_mode: bool) -> None:
+    """Save API key to ~/.config/scrapingbee-cli/.env (from --api-key, env/.env, or prompt)."""
+    from ..exec_gate import is_exec_enabled, require_auth_unsafe, set_unsafe_verified
+
+    path = auth_config_path()
+
+    if show_path_only:
+        click.echo(str(path))
+        return
+
+    if unsafe_mode:
+        # Gate: check env vars are set (vague error if not)
+        if not require_auth_unsafe():
+            raise SystemExit(1)
+
+        # Gate: reject --api-key (must be interactive only)
+        if auth_api_key:
+            click.echo("Something went wrong. Please try again later.", err=True)
+            raise SystemExit(1)
+
+        # Wipe API key from everywhere
+        _wipe_api_key_everywhere()
+        click.echo("API key removed for security re-authentication.", err=True)
+
+        # Show disclaimer
+        click.echo(_UNSAFE_DISCLAIMER, err=True)
+
+        # Require acceptance
+        try:
+            answer = input("Do you accept the risks? (yes/no): ").strip().lower()
+        except (EOFError, KeyboardInterrupt):
+            click.echo("\nAborted.", err=True)
+            raise SystemExit(1)
+        if answer != "yes":
+            click.echo("Aborted. Unsafe mode not enabled.", err=True)
+            raise SystemExit(1)
+
+        # Prompt for API key (interactive only)
+        try:
+            raw = getpass.getpass("ScrapingBee API key: ")
+        except (EOFError, KeyboardInterrupt):
+            click.echo("\nAborted.", err=True)
+            raise SystemExit(1)
+        key = raw.strip()
+        if not key:
+            click.echo("No API key entered.", err=True)
+            raise SystemExit(1)
+
+        click.echo("Validating API key...", err=True)
+        if not _validate_api_key(key):
+            click.echo("Invalid API key.", err=True)
+            raise SystemExit(1)
+
+        # Save key and set unsafe verified
+        save_api_key_to_dotenv(key)
+        set_unsafe_verified()
+        click.echo("API key saved. Unsafe mode enabled.", err=True)
+        return
+
+    # Normal auth flow (show warning if unsafe is enabled)
+    if is_exec_enabled():
+        click.echo("⚠ Unsafe mode is active. Shell execution features are enabled.", err=True)
+
+    key = auth_api_key or get_api_key_if_set(None)
+    if not key:
+        try:
+            raw = getpass.getpass("ScrapingBee API key: ")
+        except (EOFError, KeyboardInterrupt):
+            click.echo(
+                "Cannot read API key (non-interactive). Use --api-key KEY or set SCRAPINGBEE_API_KEY.",
+                err=True,
+            )
+            raise SystemExit(1)
+        key = raw.strip()
+        if not key:
+            click.echo("No API key entered.", err=True)
+            raise SystemExit(1)
+    click.echo("Validating API key...", err=True)
+    if not _validate_api_key(key):
+        click.echo("Invalid API key. Please check your key and try again.", err=True)
+        raise SystemExit(1)
+    path = save_api_key_to_dotenv(key)
+    click.echo(f"API key saved to {path}. You can now run scrapingbee commands.")
+
+
+@click.command()
+@click.option(
+    "--open/--no-open",
+    "open_browser",
+    default=False,
+    help="Open the documentation URL in the default browser.",
+)
+def docs_cmd(open_browser: bool) -> None:
+    """Print or open the ScrapingBee API documentation URL."""
+    click.echo(DOCS_URL)
+    if open_browser:
+        import webbrowser
+
+        webbrowser.open(DOCS_URL)
+
+
+@click.command()
+@click.pass_obj
+def logout_cmd(obj: dict) -> None:
+    """Remove stored API key from ~/.config/scrapingbee-cli/.env."""
+    import json
+    from pathlib import Path
+
+    # Check for active schedules
+    registry_path = Path.home() / ".config" / "scrapingbee-cli" / "schedules.json"
+    try:
+        registry = json.loads(registry_path.read_text(encoding="utf-8"))
+    except Exception:
+        registry = {}
+
+    if registry:
+        names = ", ".join(registry)
+        click.echo(
+            f"Warning: you have {len(registry)} active schedule(s): {names}.\n"
+            "These will fail without an API key.",
+            err=True,
+        )
+        if not click.confirm("Stop all schedules and logout?"):
+            click.echo("Logout cancelled.", err=True)
+            return
+        # Stop all schedules
+        from .schedule import _remove_cron_entry, _save_registry
+
+        for name in list(registry):
+            _remove_cron_entry(name)
+            click.echo(f"  Stopped schedule '{name}'.", err=True)
+        _save_registry({})
+
+    removed = remove_api_key_from_dotenv()
+
+    # Also remove unsafe verified flag
+    from ..exec_gate import remove_unsafe_verified
+
+    remove_unsafe_verified()
+
+    if removed:
+        click.echo(f"API key removed from {auth_config_path()}.")
+    else:
+        click.echo(f"No stored API key found in {auth_config_path()}.")
+    click.echo(
+        "If you set SCRAPINGBEE_API_KEY in your shell, unset it with: unset SCRAPINGBEE_API_KEY"
+    )
+
+
+def register(cli: click.Group) -> None:
+    cli.add_command(auth_cmd, "auth")
+    cli.add_command(docs_cmd, "docs")
+    cli.add_command(logout_cmd, "logout")

{scrapingbee_cli-1.2.3 → scrapingbee_cli-1.3.1}/src/scrapingbee_cli/commands/crawl.py

@@ -59,6 +59,7 @@ def _crawl_build_params(
     device: str | None,
     custom_google: str | None,
     transparent_status_code: str | None,
+    scraping_config: str | None = None,
 ) -> dict[str, str]:
     """Build ScrapingBee API params dict from crawl options (quick-crawl URL mode)."""
     kwargs = build_scrape_kwargs(
@@ -97,6 +98,7 @@ def _crawl_build_params(
         custom_google=custom_google,
         transparent_status_code=transparent_status_code,
         body=None,
+        scraping_config=scraping_config,
     )
     return scrape_kwargs_to_api_params(kwargs)
 
@@ -117,6 +119,12 @@ def _crawl_build_params(
     default=None,
     help="Path to Scrapy project. Spider mode only.",
 )
+@click.option(
+    "--scraping-config",
+    type=str,
+    default=None,
+    help="Apply a pre-saved scraping configuration by name. Create configs in the ScrapingBee dashboard. Inline options override config settings.",
+)
 @optgroup.group("Rendering", help="JavaScript rendering and viewport options")
 @optgroup.option(
     "--render-js",
@@ -314,7 +322,7 @@ def _crawl_build_params(
     "on_complete",
     type=str,
     default=None,
-    help="Shell command to run after crawl completes.",
+    help="[Advanced] Shell command to run after crawl completes. Requires unsafe mode.",
 )
 @_output_options
 @click.pass_obj
@@ -323,6 +331,7 @@ def crawl_cmd(
     target: tuple[str, ...],
     from_sitemap: str | None,
     project: str | None,
+    scraping_config: str | None,
     render_js: str | None,
     js_scenario: str | None,
     wait: int | None,
@@ -467,6 +476,7 @@ def crawl_cmd(
                 device=device,
                 custom_google=custom_google,
                 transparent_status_code=transparent_status_code,
+                scraping_config=scraping_config,
             )
         except ValueError as e:
             click.echo(str(e), err=True)

{scrapingbee_cli-1.2.3 → scrapingbee_cli-1.3.1}/src/scrapingbee_cli/commands/google.py

@@ -17,6 +17,7 @@ from ..batch import (
 )
 from ..cli_utils import (
     DEVICE_DESKTOP_MOBILE,
+    NormalizedChoice,
     _batch_options,
     _validate_page,
     check_api_response,
@@ -56,7 +57,7 @@ def _warn_empty_organic(data: bytes, search_type: str | None) -> None:
 @optgroup.group("Search", help="Search type, locale, and pagination")
 @optgroup.option(
     "--search-type",
-    type=click.Choice(
+    type=NormalizedChoice(
         ["classic", "news", "maps", "lens", "shopping", "images", "ai-mode"],
         case_sensitive=False,
     ),