geonode-scraper-cli 0.1.0__py3-none-any.whl

geonode_scraper_cli/config.py

@@ -0,0 +1,104 @@
+"""Configuration resolution: flag > env > TOML file > default."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+try:  # Python 3.11+
+    import tomllib
+except ModuleNotFoundError:  # Python 3.10
+    import tomli as tomllib  # type: ignore[no-redef]
+
+from geonode_scraper_tools_core import ScraperToolSettings
+
+ENV_PREFIX = "GEONODE_SCRAPER_"
+DEFAULT_PROFILE = "default"
+DEFAULT_HOST = "http://localhost"
+
+
+class ConfigError(Exception):
+    """Raised when required configuration is missing or a config file is invalid."""
+
+
+def config_path() -> Path:
+    """Location of the TOML config file, honouring ``XDG_CONFIG_HOME``."""
+    override = os.environ.get(f"{ENV_PREFIX}CONFIG")
+    if override:
+        return Path(override).expanduser()
+    base = os.environ.get("XDG_CONFIG_HOME") or os.path.join(os.path.expanduser("~"), ".config")
+    return Path(base) / "geonode-scraper" / "config.toml"
+
+
+def load_profiles() -> dict[str, Any]:
+    path = config_path()
+    if not path.exists():
+        return {}
+    try:
+        return tomllib.loads(path.read_text(encoding="utf-8"))
+    except (tomllib.TOMLDecodeError, OSError) as exc:
+        raise ConfigError(f"Could not read config file {path}: {exc}") from exc
+
+
+def _env(name: str) -> str | None:
+    return os.environ.get(f"{ENV_PREFIX}{name}")
+
+
+def _as_bool(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    return str(value).strip().lower() in {"1", "true", "yes", "on"}
+
+
+def resolve_profile_name(profile: str | None) -> str:
+    return profile or _env("PROFILE") or DEFAULT_PROFILE
+
+
+def resolve_settings(
+    *,
+    api_key: str | None = None,
+    host: str | None = None,
+    profile: str | None = None,
+    verify_ssl: bool | None = None,
+    timeout: float | None = None,
+) -> ScraperToolSettings:
+    """Merge CLI flags, environment, and the config-file profile into settings.
+
+    Precedence (highest first): explicit flag, environment variable, config file,
+    built-in default.
+    """
+    profile_name = resolve_profile_name(profile)
+    profiles = load_profiles()
+    file_cfg = profiles.get(profile_name) or {}
+    if profile is not None and profile_name not in profiles:
+        raise ConfigError(f"Profile '{profile_name}' not found in {config_path()}.")
+
+    def pick(flag: Any, env_name: str, file_key: str, default: Any = None) -> Any:
+        if flag is not None:
+            return flag
+        env_value = _env(env_name)
+        if env_value is not None:
+            return env_value
+        if file_key in file_cfg:
+            return file_cfg[file_key]
+        return default
+
+    resolved_host = pick(host, "HOST", "host", DEFAULT_HOST)
+    resolved_key = pick(api_key, "API_KEY", "api_key")
+    resolved_verify = pick(verify_ssl, "VERIFY_SSL", "verify_ssl", True)
+    resolved_timeout = pick(timeout, "TIMEOUT", "timeout")
+
+    if not resolved_key:
+        raise ConfigError(
+            "No API key configured. Pass --api-key, set "
+            f"{ENV_PREFIX}API_KEY, or add 'api_key' to your config profile "
+            f"({config_path()})."
+        )
+
+    return ScraperToolSettings(
+        host=str(resolved_host),
+        api_key=str(resolved_key),
+        verify_ssl=_as_bool(resolved_verify),
+        request_timeout=float(resolved_timeout) if resolved_timeout is not None else None,
+    )

geonode_scraper_cli/context.py

@@ -0,0 +1,59 @@
+"""Per-invocation CLI state and lazy service construction."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import typer
+from geonode_scraper_tools_core import ScraperToolService, ScraperToolSettings
+
+from .config import ConfigError, resolve_settings
+from .errors import ExitCode
+from .output import OutputFormat
+
+
+@dataclass
+class CLIContext:
+    """Holds resolved global options; builds the service on demand.
+
+    Credentials are only required when a command actually calls the API, so
+    ``--help``/``--version`` and the ``config`` commands never fail on a missing
+    API key.
+    """
+
+    api_key: str | None
+    host: str | None
+    profile: str | None
+    verify_ssl: bool | None
+    timeout: float | None
+    output: OutputFormat
+    verbose: bool
+
+    _settings: ScraperToolSettings | None = None
+    _service: ScraperToolService | None = None
+
+    def settings(self) -> ScraperToolSettings:
+        if self._settings is None:
+            try:
+                self._settings = resolve_settings(
+                    api_key=self.api_key,
+                    host=self.host,
+                    profile=self.profile,
+                    verify_ssl=self.verify_ssl,
+                    timeout=self.timeout,
+                )
+            except ConfigError as exc:
+                typer.echo(str(exc), err=True)
+                raise typer.Exit(int(ExitCode.USAGE)) from exc
+        return self._settings
+
+    def service(self) -> ScraperToolService:
+        if self._service is None:
+            self._service = ScraperToolService(self.settings())
+        return self._service
+
+
+def get_cli_context(ctx: typer.Context) -> CLIContext:
+    obj = ctx.obj
+    assert isinstance(obj, CLIContext)  # set by the root callback
+    return obj

geonode_scraper_cli/errors.py

@@ -0,0 +1,50 @@
+"""Map service error envelopes and exceptions to user-facing messages + exit codes."""
+
+from __future__ import annotations
+
+from enum import IntEnum
+from typing import Any, Mapping
+
+
+class ExitCode(IntEnum):
+    OK = 0
+    GENERIC = 1
+    USAGE = 2
+    AUTH = 4
+    NOT_FOUND = 5
+    VALIDATION = 6
+    NETWORK = 7
+    TIMEOUT = 8
+
+
+def exit_code_for_error(error: Mapping[str, Any]) -> ExitCode:
+    """Choose a POSIX exit code from a service error dict."""
+    code = error.get("code")
+    if code == "POLL_TIMEOUT":
+        return ExitCode.TIMEOUT
+
+    status = error.get("status")
+    if status in (401, 403):
+        return ExitCode.AUTH
+    if status == 404:
+        return ExitCode.NOT_FOUND
+    if status == 422:
+        return ExitCode.VALIDATION
+    if status is None:
+        # No HTTP status reached the server: local/connection problem.
+        return ExitCode.NETWORK
+    return ExitCode.GENERIC
+
+
+def format_error(error: Mapping[str, Any]) -> str:
+    """Build a single-line human message from a service error dict."""
+    message = error.get("message") or error.get("reason") or "Request failed."
+    status = error.get("status")
+    code = error.get("code")
+    prefix_parts = []
+    if status is not None:
+        prefix_parts.append(f"HTTP {status}")
+    if code:
+        prefix_parts.append(str(code))
+    prefix = f"[{' '.join(prefix_parts)}] " if prefix_parts else ""
+    return f"{prefix}{message}"

geonode_scraper_cli/invoke.py

@@ -0,0 +1,56 @@
+"""Shared command plumbing: validate flags via tools-core schemas, call, emit."""
+
+from __future__ import annotations
+
+from typing import Any, NoReturn
+
+import typer
+from pydantic import BaseModel, ValidationError
+from rich.console import Console
+
+from .context import get_cli_context
+from .errors import ExitCode
+from .renderers import emit
+
+_stderr = Console(stderr=True)
+
+
+def validated(schema_cls: type[BaseModel], /, **kwargs: Any) -> dict[str, Any]:
+    """Validate user-supplied flags through a tools-core input schema.
+
+    Only pass keys the user actually provided; the schema fills defaults. Keeps
+    validation rules (country code, ranges, enums) defined once in tools-core.
+    """
+    try:
+        model = schema_cls(**kwargs)
+    except ValidationError as exc:
+        _stderr.print("[red]Invalid arguments:[/red]")
+        for err in exc.errors():
+            loc = ".".join(str(part) for part in err["loc"]) or "(value)"
+            _stderr.print(f"  {loc}: {err['msg']}")
+        raise typer.Exit(int(ExitCode.USAGE)) from exc
+    return model.model_dump()
+
+
+def call(ctx: typer.Context, method_name: str, payload: dict[str, Any]) -> NoReturn:
+    """Invoke a service method and emit the result envelope.
+
+    Unexpected (non-``ApiException``) failures — e.g. connection errors — are
+    wrapped into an error envelope so output stays uniform; ``--verbose`` lets
+    the original traceback through.
+    """
+    cli = get_cli_context(ctx)
+    service = cli.service()
+    try:
+        result = getattr(service, method_name)(**payload)
+    except typer.Exit:
+        raise
+    except Exception as exc:  # noqa: BLE001 - normalise to an error envelope
+        if cli.verbose:
+            raise
+        result = {
+            "ok": False,
+            "operation": method_name,
+            "error": {"status": None, "code": "CLIENT_ERROR", "message": str(exc)},
+        }
+    emit(cli.output, result)

geonode_scraper_cli/main.py

@@ -0,0 +1,83 @@
+"""gscraper — command-line interface for the Geonode Scraper API."""
+
+from __future__ import annotations
+
+from typing import Optional
+
+import typer
+
+from . import __version__
+from .commands.batch import batch_app
+from .commands.config import config_app
+from .commands.crawl import crawl_app
+from .commands.extract import extract
+from .commands.jobs import jobs_app
+from .commands.map import map_app
+from .commands.stats import stats
+from .commands.system import health
+from .context import CLIContext
+from .output import OutputFormat
+
+app = typer.Typer(
+    name="gscraper",
+    help="Command-line interface for the Geonode Scraper API.",
+    no_args_is_help=True,
+    add_completion=True,
+)
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        typer.echo(f"gscraper {__version__}")
+        raise typer.Exit()
+
+
+def _resolve_output(as_json: bool, as_yaml: bool) -> OutputFormat:
+    if as_json and as_yaml:
+        raise typer.BadParameter("Use only one of --json or --yaml.")
+    if as_json:
+        return OutputFormat.JSON
+    if as_yaml:
+        return OutputFormat.YAML
+    return OutputFormat.TABLE
+
+
+@app.callback()
+def main_callback(
+    ctx: typer.Context,
+    api_key: Optional[str] = typer.Option(None, "--api-key", help="API key (prefer env/config over the CLI flag)."),
+    host: Optional[str] = typer.Option(None, "--host", help="API base URL."),
+    profile: Optional[str] = typer.Option(None, "--profile", help="Config profile to use."),
+    as_json: bool = typer.Option(False, "--json", help="Output the raw JSON envelope."),
+    as_yaml: bool = typer.Option(False, "--yaml", help="Output the raw YAML envelope."),
+    verify_ssl: Optional[bool] = typer.Option(None, "--verify-ssl/--no-verify-ssl", help="Verify TLS certificates."),
+    timeout: Optional[float] = typer.Option(None, "--timeout", help="Request timeout in seconds."),
+    verbose: bool = typer.Option(False, "--verbose", "-v", help="Show full tracebacks on unexpected errors."),
+    _version: bool = typer.Option(
+        False, "--version", callback=_version_callback, is_eager=True, help="Show the version and exit."
+    ),
+) -> None:
+    """Resolve global options into the shared CLI context."""
+    ctx.obj = CLIContext(
+        api_key=api_key,
+        host=host,
+        profile=profile,
+        verify_ssl=verify_ssl,
+        timeout=timeout,
+        output=_resolve_output(as_json, as_yaml),
+        verbose=verbose,
+    )
+
+
+app.command("extract")(extract)
+app.command("stats")(stats)
+app.command("health")(health)
+app.add_typer(jobs_app, name="jobs")
+app.add_typer(batch_app, name="batch")
+app.add_typer(crawl_app, name="crawl")
+app.add_typer(map_app, name="map")
+app.add_typer(config_app, name="config")
+
+
+if __name__ == "__main__":
+    app()

geonode_scraper_cli/output.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from enum import Enum
+
+
+class OutputFormat(str, Enum):
+    TABLE = "table"
+    JSON = "json"
+    YAML = "yaml"

geonode_scraper_cli/renderers/__init__.py

@@ -0,0 +1,49 @@
+"""Output dispatch: render a service result envelope and exit with a code."""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Mapping, NoReturn
+
+import typer
+import yaml
+from rich.console import Console
+
+from ..errors import ExitCode, exit_code_for_error, format_error
+from ..output import OutputFormat
+from .tables import render_human
+
+# rich auto-disables styling when stdout is not a TTY and honours NO_COLOR.
+_stdout = Console()
+_stderr = Console(stderr=True)
+
+
+def console() -> Console:
+    return _stdout
+
+
+def _to_jsonable(value: Any) -> Any:
+    return json.loads(json.dumps(value, default=str))
+
+
+def emit(output: OutputFormat, result: Mapping[str, Any]) -> NoReturn:
+    """Render a service envelope (``{ok, operation, result|error, ...}``) and exit.
+
+    The JSON/YAML formats print the full envelope so both success and error are
+    machine-parseable; the table format prints a human summary or a friendly
+    error line on stderr.
+    """
+    ok = bool(result.get("ok"))
+    code = ExitCode.OK if ok else exit_code_for_error(result.get("error") or {})
+
+    if output is OutputFormat.JSON:
+        _stdout.print_json(json.dumps(_to_jsonable(result)))
+    elif output is OutputFormat.YAML:
+        _stdout.print(yaml.safe_dump(_to_jsonable(result), sort_keys=False).rstrip())
+    else:
+        if ok:
+            render_human(result, _stdout)
+        else:
+            _stderr.print(f"[red]{format_error(result.get('error') or {})}[/red]")
+
+    raise typer.Exit(int(code))

geonode_scraper_cli/renderers/tables.py

@@ -0,0 +1,118 @@
+"""Human-friendly rendering of service result payloads.
+
+Designed to be defensive: the service returns plain dicts whose shape tracks the
+OpenAPI spec and can change between versions. Renderers use safe access and a
+generic fallback, and ``render_human`` ultimately falls back to pretty JSON for
+any shape it cannot format.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Mapping, Sequence
+
+from rich.console import Console
+from rich.table import Table
+
+# Long text fields (markdown/html/raw content) are summarised by length rather
+# than dumped into the terminal.
+_LONG_TEXT_THRESHOLD = 200
+
+# Result keys that commonly hold a list of records worth rendering as a table.
+_COLLECTION_KEYS = ("jobs", "items", "links", "results", "data")
+
+
+def _is_scalar(value: Any) -> bool:
+    return value is None or isinstance(value, (str, int, float, bool))
+
+
+def _summarize_scalar(value: Any) -> str:
+    if value is None:
+        return "-"
+    if isinstance(value, str) and len(value) > _LONG_TEXT_THRESHOLD:
+        return f"<{len(value)} chars>"
+    return str(value)
+
+
+def _kv_table(data: Mapping[str, Any], console: Console, *, title: str | None = None) -> None:
+    table = Table(show_header=False, box=None, title=title, title_justify="left", pad_edge=False)
+    table.add_column("field", style="bold cyan", no_wrap=True)
+    table.add_column("value", overflow="fold")
+    for key, value in data.items():
+        if _is_scalar(value):
+            table.add_row(str(key), _summarize_scalar(value))
+        elif isinstance(value, Mapping):
+            table.add_row(str(key), _summarize_mapping(value))
+        elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+            table.add_row(str(key), f"<{len(value)} item(s)>")
+        else:
+            table.add_row(str(key), str(value))
+    console.print(table)
+
+
+def _summarize_mapping(data: Mapping[str, Any]) -> str:
+    parts = []
+    for key, value in data.items():
+        if _is_scalar(value):
+            parts.append(f"{key}={_summarize_scalar(value)}")
+        else:
+            parts.append(f"{key}=…")
+    return ", ".join(parts) if parts else "{}"
+
+
+def _records_table(records: Sequence[Mapping[str, Any]], console: Console, *, title: str | None = None) -> None:
+    # Union of keys across records, preserving first-seen order; scalar columns only.
+    columns: list[str] = []
+    for record in records:
+        if not isinstance(record, Mapping):
+            # Heterogeneous / non-dict list: bail to the generic path.
+            console.print_json(data=list(records))
+            return
+        for key, value in record.items():
+            if key not in columns and _is_scalar(value):
+                columns.append(key)
+
+    if not columns:
+        console.print_json(data=list(records))
+        return
+
+    table = Table(title=title, title_justify="left", header_style="bold cyan")
+    for column in columns:
+        table.add_column(column, overflow="fold")
+    for record in records:
+        table.add_row(*[_summarize_scalar(record.get(column)) for column in columns])
+    console.print(table)
+
+
+def _render_generic(data: Any, console: Console) -> None:
+    if isinstance(data, Mapping):
+        scalars = {k: v for k, v in data.items() if _is_scalar(v) or isinstance(v, Mapping)}
+        if scalars:
+            _kv_table(scalars, console)
+        # Render the first meaningful collection field as its own table.
+        for key, value in data.items():
+            if (
+                isinstance(value, Sequence)
+                and not isinstance(value, (str, bytes, bytearray))
+                and value
+                and (key in _COLLECTION_KEYS or all(isinstance(item, Mapping) for item in value))
+            ):
+                console.print()
+                _records_table(value, console, title=str(key))
+                break
+    elif isinstance(data, Sequence) and not isinstance(data, (str, bytes, bytearray)):
+        _records_table(data, console)
+    else:
+        console.print(str(data))
+
+
+def render_human(result: Mapping[str, Any], console: Console) -> None:
+    """Render a successful service envelope's ``result`` for human consumption."""
+    data = result.get("result")
+    if data is None:
+        operation = result.get("operation", "operation")
+        console.print(f"[green]{operation}: ok[/green]")
+        return
+    try:
+        _render_generic(data, console)
+    except Exception:  # noqa: BLE001 - unfamiliar shape: never crash on rendering
+        console.print_json(data=data)

geonode_scraper_cli-0.1.0.dist-info/METADATA

@@ -0,0 +1,159 @@
+Metadata-Version: 2.4
+Name: geonode-scraper-cli
+Version: 0.1.0
+Summary: Command-line interface for the Geonode Scraper API
+Author: Geonode Team
+License-Expression: MIT
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: geonode-scraper-tools-core>=0.3.1
+Requires-Dist: typer>=0.12
+Requires-Dist: rich>=13.0
+Requires-Dist: PyYAML>=6.0
+Requires-Dist: tomli>=2.0; python_version < "3.11"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0; extra == "dev"
+Requires-Dist: ruff>=0.12.11; extra == "dev"
+
+# Geonode Scraper CLI
+
+`gscraper` is the command-line interface for the Geonode Scraper API. It is a thin
+presentation layer over [`geonode-scraper-tools-core`](../python-agent-tools-core):
+commands parse flags, resolve configuration, call a stable service method, and
+render the result. All domain logic (validation, polling, retries) lives in the
+service layer, not in the CLI.
+
+## Requirements
+
+- Python 3.10+
+- Works on **Linux**, **macOS**, and **Windows**
+
+## Installation
+
+**Recommended — install as a standalone tool with [pipx](https://pipx.pypa.io):**
+
+```sh
+pipx install geonode-scraper-cli
+```
+
+`pipx` installs `gscraper` into its own isolated virtual environment and puts it
+on your `PATH`, so it never conflicts with other Python projects. This is the
+preferred way to install CLI tools globally.
+
+**Alternative — install with pip:**
+
+```sh
+pip install geonode-scraper-cli
+```
+
+> **Windows note:** on Windows the `gscraper` command is placed in the Python
+> Scripts folder (e.g. `%APPDATA%\Python\Python3xx\Scripts`). If the command is
+> not found after installation, add that folder to your `PATH`, or use
+> `python -m geonode_scraper_cli` as a fallback. `pipx` handles this
+> automatically and is the simpler choice on Windows.
+
+## Configuration
+
+Configuration is resolved with the following precedence (highest first):
+
+1. Command-line flags (`--api-key`, `--host`, ...)
+2. Environment variables (`GEONODE_SCRAPER_API_KEY`, `GEONODE_SCRAPER_HOST`,
+   `GEONODE_SCRAPER_VERIFY_SSL`, `GEONODE_SCRAPER_TIMEOUT`, `GEONODE_SCRAPER_PROFILE`)
+3. A TOML config file at `~/.config/geonode-scraper/config.toml`
+4. Built-in defaults
+
+Prefer environment variables or the config file for your API key — passing
+`--api-key` on the command line can leak it into your shell history.
+
+Example `~/.config/geonode-scraper/config.toml`:
+
+```toml
+[default]
+host = "https://api.example.com"
+api_key = "your-api-key"
+verify_ssl = true
+
+[staging]
+host = "https://staging.example.com"
+api_key = "your-staging-key"
+```
+
+Select a non-default profile with `--profile staging` or
+`GEONODE_SCRAPER_PROFILE=staging`. Inspect the active configuration with:
+
+```sh
+gscraper config path     # print the config file location
+gscraper config show     # show profiles (API keys masked)
+```
+
+## Output
+
+Commands print a human-readable summary by default. Use `--json` or `--yaml`
+to print the raw result envelope for scripting. These flags can appear either
+before the subcommand (global position) or after it (per-command position) —
+both work:
+
+```sh
+gscraper extract https://example.com --json | jq -r .result.data.markdown
+gscraper --json extract https://example.com | jq -r .result.data.markdown
+```
+
+The JSON/YAML envelope has the shape `{ "ok": bool, "operation": str, "result": {...} }`
+on success, or `{ "ok": false, "operation": str, "error": {...} }` on failure.
+
+## Commands
+
+```text
+gscraper extract URL [--format markdown|html] [--render-js] [--async] \
+                     [--proxy-country US] [--proxy-type residential] \
+                     [--header "K: V"] [--output out.md]
+
+gscraper jobs get JOB_ID
+gscraper jobs list [--status completed] [--url ...] [--page N]
+gscraper jobs wait JOB_ID [--timeout S] [--interval S]
+
+gscraper batch create URL [URL ...] [--format markdown]
+gscraper batch status JOB_ID
+gscraper batch wait JOB_ID [--timeout S] [--interval S]
+gscraper batch list [--status ...]
+gscraper batch cancel JOB_ID
+
+gscraper crawl create URL [--depth 2] [--limit 50] [--include-subdomains]
+gscraper crawl status JOB_ID
+gscraper crawl wait JOB_ID
+gscraper crawl list [--url ...]
+gscraper crawl cancel JOB_ID
+
+gscraper map run URL [--search term] [--no-subdomains]   # primary action
+gscraper map jobs list                                   # inspect past map jobs
+gscraper map jobs get JOB_ID
+
+gscraper stats [--start-date ISO] [--end-date ISO]
+gscraper health
+```
+
+Run `gscraper --help` or `gscraper <command> --help` for full details.
+
+## Exit codes
+
+| Code | Meaning |
+| ---- | ------- |
+| 0 | Success |
+| 1 | Generic error |
+| 2 | Usage / invalid arguments |
+| 4 | Authentication / authorization (401, 403) |
+| 5 | Not found (404) |
+| 6 | Validation error (422) |
+| 7 | Network / connection error |
+| 8 | Polling timeout (`wait` commands) |
+
+## Shell completion
+
+```sh
+gscraper --install-completion
+```