geonode-scraper-cli 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. geonode_scraper_cli-0.1.0/PKG-INFO +159 -0
  2. geonode_scraper_cli-0.1.0/README.md +137 -0
  3. geonode_scraper_cli-0.1.0/geonode_scraper_cli/__init__.py +3 -0
  4. geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/__init__.py +1 -0
  5. geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/_common.py +42 -0
  6. geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/batch.py +114 -0
  7. geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/config.py +48 -0
  8. geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/crawl.py +119 -0
  9. geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/extract.py +78 -0
  10. geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/jobs.py +77 -0
  11. geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/map.py +75 -0
  12. geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/stats.py +24 -0
  13. geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/system.py +18 -0
  14. geonode_scraper_cli-0.1.0/geonode_scraper_cli/config.py +104 -0
  15. geonode_scraper_cli-0.1.0/geonode_scraper_cli/context.py +59 -0
  16. geonode_scraper_cli-0.1.0/geonode_scraper_cli/errors.py +50 -0
  17. geonode_scraper_cli-0.1.0/geonode_scraper_cli/invoke.py +56 -0
  18. geonode_scraper_cli-0.1.0/geonode_scraper_cli/main.py +83 -0
  19. geonode_scraper_cli-0.1.0/geonode_scraper_cli/output.py +9 -0
  20. geonode_scraper_cli-0.1.0/geonode_scraper_cli/py.typed +0 -0
  21. geonode_scraper_cli-0.1.0/geonode_scraper_cli/renderers/__init__.py +49 -0
  22. geonode_scraper_cli-0.1.0/geonode_scraper_cli/renderers/tables.py +118 -0
  23. geonode_scraper_cli-0.1.0/geonode_scraper_cli.egg-info/PKG-INFO +159 -0
  24. geonode_scraper_cli-0.1.0/geonode_scraper_cli.egg-info/SOURCES.txt +29 -0
  25. geonode_scraper_cli-0.1.0/geonode_scraper_cli.egg-info/dependency_links.txt +1 -0
  26. geonode_scraper_cli-0.1.0/geonode_scraper_cli.egg-info/entry_points.txt +2 -0
  27. geonode_scraper_cli-0.1.0/geonode_scraper_cli.egg-info/requires.txt +12 -0
  28. geonode_scraper_cli-0.1.0/geonode_scraper_cli.egg-info/top_level.txt +1 -0
  29. geonode_scraper_cli-0.1.0/pyproject.toml +58 -0
  30. geonode_scraper_cli-0.1.0/setup.cfg +4 -0
  31. geonode_scraper_cli-0.1.0/tests/test_cli.py +137 -0
@@ -0,0 +1,159 @@
1
+ Metadata-Version: 2.4
2
+ Name: geonode-scraper-cli
3
+ Version: 0.1.0
4
+ Summary: Command-line interface for the Geonode Scraper API
5
+ Author: Geonode Team
6
+ License-Expression: MIT
7
+ Classifier: Programming Language :: Python :: 3.10
8
+ Classifier: Programming Language :: Python :: 3.11
9
+ Classifier: Programming Language :: Python :: 3.12
10
+ Classifier: Typing :: Typed
11
+ Requires-Python: >=3.10
12
+ Description-Content-Type: text/markdown
13
+ Requires-Dist: geonode-scraper-tools-core>=0.3.1
14
+ Requires-Dist: typer>=0.12
15
+ Requires-Dist: rich>=13.0
16
+ Requires-Dist: PyYAML>=6.0
17
+ Requires-Dist: tomli>=2.0; python_version < "3.11"
18
+ Provides-Extra: dev
19
+ Requires-Dist: pytest>=8.0; extra == "dev"
20
+ Requires-Dist: pytest-cov>=5.0; extra == "dev"
21
+ Requires-Dist: ruff>=0.12.11; extra == "dev"
22
+
23
+ # Geonode Scraper CLI
24
+
25
+ `gscraper` is the command-line interface for the Geonode Scraper API. It is a thin
26
+ presentation layer over [`geonode-scraper-tools-core`](../python-agent-tools-core):
27
+ commands parse flags, resolve configuration, call a stable service method, and
28
+ render the result. All domain logic (validation, polling, retries) lives in the
29
+ service layer, not in the CLI.
30
+
31
+ ## Requirements
32
+
33
+ - Python 3.10+
34
+ - Works on **Linux**, **macOS**, and **Windows**
35
+
36
+ ## Installation
37
+
38
+ **Recommended — install as a standalone tool with [pipx](https://pipx.pypa.io):**
39
+
40
+ ```sh
41
+ pipx install geonode-scraper-cli
42
+ ```
43
+
44
+ `pipx` installs `gscraper` into its own isolated virtual environment and puts it
45
+ on your `PATH`, so it never conflicts with other Python projects. This is the
46
+ preferred way to install CLI tools globally.
47
+
48
+ **Alternative — install with pip:**
49
+
50
+ ```sh
51
+ pip install geonode-scraper-cli
52
+ ```
53
+
54
+ > **Windows note:** on Windows the `gscraper` command is placed in the Python
55
+ > Scripts folder (e.g. `%APPDATA%\Python\Python3xx\Scripts`). If the command is
56
+ > not found after installation, add that folder to your `PATH`, or use
57
+ > `python -m geonode_scraper_cli` as a fallback. `pipx` handles this
58
+ > automatically and is the simpler choice on Windows.
59
+
60
+ ## Configuration
61
+
62
+ Configuration is resolved with the following precedence (highest first):
63
+
64
+ 1. Command-line flags (`--api-key`, `--host`, ...)
65
+ 2. Environment variables (`GEONODE_SCRAPER_API_KEY`, `GEONODE_SCRAPER_HOST`,
66
+ `GEONODE_SCRAPER_VERIFY_SSL`, `GEONODE_SCRAPER_TIMEOUT`, `GEONODE_SCRAPER_PROFILE`)
67
+ 3. A TOML config file at `~/.config/geonode-scraper/config.toml`
68
+ 4. Built-in defaults
69
+
70
+ Prefer environment variables or the config file for your API key — passing
71
+ `--api-key` on the command line can leak it into your shell history.
72
+
73
+ Example `~/.config/geonode-scraper/config.toml`:
74
+
75
+ ```toml
76
+ [default]
77
+ host = "https://api.example.com"
78
+ api_key = "your-api-key"
79
+ verify_ssl = true
80
+
81
+ [staging]
82
+ host = "https://staging.example.com"
83
+ api_key = "your-staging-key"
84
+ ```
85
+
86
+ Select a non-default profile with `--profile staging` or
87
+ `GEONODE_SCRAPER_PROFILE=staging`. Inspect the active configuration with:
88
+
89
+ ```sh
90
+ gscraper config path # print the config file location
91
+ gscraper config show # show profiles (API keys masked)
92
+ ```
93
+
94
+ ## Output
95
+
96
+ Commands print a human-readable summary by default. Use `--json` or `--yaml`
97
+ to print the raw result envelope for scripting. These flags can appear either
98
+ before the subcommand (global position) or after it (per-command position) —
99
+ both work:
100
+
101
+ ```sh
102
+ gscraper extract https://example.com --json | jq -r .result.data.markdown
103
+ gscraper --json extract https://example.com | jq -r .result.data.markdown
104
+ ```
105
+
106
+ The JSON/YAML envelope has the shape `{ "ok": bool, "operation": str, "result": {...} }`
107
+ on success, or `{ "ok": false, "operation": str, "error": {...} }` on failure.
108
+
109
+ ## Commands
110
+
111
+ ```text
112
+ gscraper extract URL [--format markdown|html] [--render-js] [--async] \
113
+ [--proxy-country US] [--proxy-type residential] \
114
+ [--header "K: V"] [--output out.md]
115
+
116
+ gscraper jobs get JOB_ID
117
+ gscraper jobs list [--status completed] [--url ...] [--page N]
118
+ gscraper jobs wait JOB_ID [--timeout S] [--interval S]
119
+
120
+ gscraper batch create URL [URL ...] [--format markdown]
121
+ gscraper batch status JOB_ID
122
+ gscraper batch wait JOB_ID [--timeout S] [--interval S]
123
+ gscraper batch list [--status ...]
124
+ gscraper batch cancel JOB_ID
125
+
126
+ gscraper crawl create URL [--depth 2] [--limit 50] [--include-subdomains]
127
+ gscraper crawl status JOB_ID
128
+ gscraper crawl wait JOB_ID
129
+ gscraper crawl list [--url ...]
130
+ gscraper crawl cancel JOB_ID
131
+
132
+ gscraper map run URL [--search term] [--no-subdomains] # primary action
133
+ gscraper map jobs list # inspect past map jobs
134
+ gscraper map jobs get JOB_ID
135
+
136
+ gscraper stats [--start-date ISO] [--end-date ISO]
137
+ gscraper health
138
+ ```
139
+
140
+ Run `gscraper --help` or `gscraper <command> --help` for full details.
141
+
142
+ ## Exit codes
143
+
144
+ | Code | Meaning |
145
+ | ---- | ------- |
146
+ | 0 | Success |
147
+ | 1 | Generic error |
148
+ | 2 | Usage / invalid arguments |
149
+ | 4 | Authentication / authorization (401, 403) |
150
+ | 5 | Not found (404) |
151
+ | 6 | Validation error (422) |
152
+ | 7 | Network / connection error |
153
+ | 8 | Polling timeout (`wait` commands) |
154
+
155
+ ## Shell completion
156
+
157
+ ```sh
158
+ gscraper --install-completion
159
+ ```
@@ -0,0 +1,137 @@
1
+ # Geonode Scraper CLI
2
+
3
+ `gscraper` is the command-line interface for the Geonode Scraper API. It is a thin
4
+ presentation layer over [`geonode-scraper-tools-core`](../python-agent-tools-core):
5
+ commands parse flags, resolve configuration, call a stable service method, and
6
+ render the result. All domain logic (validation, polling, retries) lives in the
7
+ service layer, not in the CLI.
8
+
9
+ ## Requirements
10
+
11
+ - Python 3.10+
12
+ - Works on **Linux**, **macOS**, and **Windows**
13
+
14
+ ## Installation
15
+
16
+ **Recommended — install as a standalone tool with [pipx](https://pipx.pypa.io):**
17
+
18
+ ```sh
19
+ pipx install geonode-scraper-cli
20
+ ```
21
+
22
+ `pipx` installs `gscraper` into its own isolated virtual environment and puts it
23
+ on your `PATH`, so it never conflicts with other Python projects. This is the
24
+ preferred way to install CLI tools globally.
25
+
26
+ **Alternative — install with pip:**
27
+
28
+ ```sh
29
+ pip install geonode-scraper-cli
30
+ ```
31
+
32
+ > **Windows note:** on Windows the `gscraper` command is placed in the Python
33
+ > Scripts folder (e.g. `%APPDATA%\Python\Python3xx\Scripts`). If the command is
34
+ > not found after installation, add that folder to your `PATH`, or use
35
+ > `python -m geonode_scraper_cli` as a fallback. `pipx` handles this
36
+ > automatically and is the simpler choice on Windows.
37
+
38
+ ## Configuration
39
+
40
+ Configuration is resolved with the following precedence (highest first):
41
+
42
+ 1. Command-line flags (`--api-key`, `--host`, ...)
43
+ 2. Environment variables (`GEONODE_SCRAPER_API_KEY`, `GEONODE_SCRAPER_HOST`,
44
+ `GEONODE_SCRAPER_VERIFY_SSL`, `GEONODE_SCRAPER_TIMEOUT`, `GEONODE_SCRAPER_PROFILE`)
45
+ 3. A TOML config file at `~/.config/geonode-scraper/config.toml`
46
+ 4. Built-in defaults
47
+
48
+ Prefer environment variables or the config file for your API key — passing
49
+ `--api-key` on the command line can leak it into your shell history.
50
+
51
+ Example `~/.config/geonode-scraper/config.toml`:
52
+
53
+ ```toml
54
+ [default]
55
+ host = "https://api.example.com"
56
+ api_key = "your-api-key"
57
+ verify_ssl = true
58
+
59
+ [staging]
60
+ host = "https://staging.example.com"
61
+ api_key = "your-staging-key"
62
+ ```
63
+
64
+ Select a non-default profile with `--profile staging` or
65
+ `GEONODE_SCRAPER_PROFILE=staging`. Inspect the active configuration with:
66
+
67
+ ```sh
68
+ gscraper config path # print the config file location
69
+ gscraper config show # show profiles (API keys masked)
70
+ ```
71
+
72
+ ## Output
73
+
74
+ Commands print a human-readable summary by default. Use `--json` or `--yaml`
75
+ to print the raw result envelope for scripting. These flags can appear either
76
+ before the subcommand (global position) or after it (per-command position) —
77
+ both work:
78
+
79
+ ```sh
80
+ gscraper extract https://example.com --json | jq -r .result.data.markdown
81
+ gscraper --json extract https://example.com | jq -r .result.data.markdown
82
+ ```
83
+
84
+ The JSON/YAML envelope has the shape `{ "ok": bool, "operation": str, "result": {...} }`
85
+ on success, or `{ "ok": false, "operation": str, "error": {...} }` on failure.
86
+
87
+ ## Commands
88
+
89
+ ```text
90
+ gscraper extract URL [--format markdown|html] [--render-js] [--async] \
91
+ [--proxy-country US] [--proxy-type residential] \
92
+ [--header "K: V"] [--output out.md]
93
+
94
+ gscraper jobs get JOB_ID
95
+ gscraper jobs list [--status completed] [--url ...] [--page N]
96
+ gscraper jobs wait JOB_ID [--timeout S] [--interval S]
97
+
98
+ gscraper batch create URL [URL ...] [--format markdown]
99
+ gscraper batch status JOB_ID
100
+ gscraper batch wait JOB_ID [--timeout S] [--interval S]
101
+ gscraper batch list [--status ...]
102
+ gscraper batch cancel JOB_ID
103
+
104
+ gscraper crawl create URL [--depth 2] [--limit 50] [--include-subdomains]
105
+ gscraper crawl status JOB_ID
106
+ gscraper crawl wait JOB_ID
107
+ gscraper crawl list [--url ...]
108
+ gscraper crawl cancel JOB_ID
109
+
110
+ gscraper map run URL [--search term] [--no-subdomains] # primary action
111
+ gscraper map jobs list # inspect past map jobs
112
+ gscraper map jobs get JOB_ID
113
+
114
+ gscraper stats [--start-date ISO] [--end-date ISO]
115
+ gscraper health
116
+ ```
117
+
118
+ Run `gscraper --help` or `gscraper <command> --help` for full details.
119
+
120
+ ## Exit codes
121
+
122
+ | Code | Meaning |
123
+ | ---- | ------- |
124
+ | 0 | Success |
125
+ | 1 | Generic error |
126
+ | 2 | Usage / invalid arguments |
127
+ | 4 | Authentication / authorization (401, 403) |
128
+ | 5 | Not found (404) |
129
+ | 6 | Validation error (422) |
130
+ | 7 | Network / connection error |
131
+ | 8 | Polling timeout (`wait` commands) |
132
+
133
+ ## Shell completion
134
+
135
+ ```sh
136
+ gscraper --install-completion
137
+ ```
@@ -0,0 +1,3 @@
1
+ __version__ = "0.1.0"
2
+
3
+ __all__ = ["__version__"]
@@ -0,0 +1 @@
1
+ """Typer command modules for the gscraper CLI."""
@@ -0,0 +1,42 @@
1
+ """Helpers shared across command modules."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Iterable
6
+
7
+ import typer
8
+
9
+ from ..errors import ExitCode
10
+ from ..output import OutputFormat
11
+
12
+
13
+ def apply_output_override(ctx: typer.Context, as_json: bool, as_yaml: bool) -> None:
14
+ """Override the output format set by the global --json/--yaml flags.
15
+
16
+ Allows per-command placement: ``gscraper extract https://example.com --json``.
17
+ """
18
+ if as_json and as_yaml:
19
+ raise typer.BadParameter("Use only one of --json or --yaml.")
20
+ if as_json:
21
+ ctx.obj.output = OutputFormat.JSON
22
+ elif as_yaml:
23
+ ctx.obj.output = OutputFormat.YAML
24
+
25
+
26
+ def parse_headers(raw_headers: Iterable[str] | None) -> dict[str, str] | None:
27
+ """Parse repeated ``--header "Key: Value"`` flags into a dict."""
28
+ if not raw_headers:
29
+ return None
30
+ headers: dict[str, str] = {}
31
+ for item in raw_headers:
32
+ if ":" not in item:
33
+ typer.echo(f"Invalid header '{item}'. Expected 'Key: Value'.", err=True)
34
+ raise typer.Exit(int(ExitCode.USAGE))
35
+ key, value = item.split(":", 1)
36
+ headers[key.strip()] = value.strip()
37
+ return headers
38
+
39
+
40
+ def optional(**kwargs: object) -> dict[str, object]:
41
+ """Drop keys whose value is ``None`` so schema defaults apply."""
42
+ return {key: value for key, value in kwargs.items() if value is not None}
@@ -0,0 +1,114 @@
1
+ """`gscraper batch` — submit and manage batch extraction jobs."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import List, Optional
6
+
7
+ import typer
8
+ from geonode_scraper_tools_core.schemas import (
9
+ CreateBatchInput,
10
+ GetBatchStatusInput,
11
+ ListBatchJobsInput,
12
+ WaitForBatchInput,
13
+ )
14
+
15
+ from ..invoke import call, validated
16
+ from ._common import apply_output_override, optional, parse_headers
17
+
18
+ batch_app = typer.Typer(help="Submit and manage batch extraction jobs.", no_args_is_help=True)
19
+
20
+
21
+ @batch_app.command("create")
22
+ def create(
23
+ ctx: typer.Context,
24
+ urls: List[str] = typer.Argument(..., help="One or more URLs to extract."),
25
+ formats: Optional[List[str]] = typer.Option(None, "--format", "-f", help="Output format(s). Repeatable."),
26
+ render_js: bool = typer.Option(False, "--render-js", help="Render JavaScript with a headless browser."),
27
+ proxy_country: Optional[str] = typer.Option(None, "--proxy-country", help="ISO 3166-1 alpha-2 country code."),
28
+ proxy_type: Optional[str] = typer.Option(None, "--proxy-type", help="Proxy type: datacenter, residential, mix."),
29
+ header: Optional[List[str]] = typer.Option(None, "--header", "-H", help="Request header 'Key: Value'. Repeatable."),
30
+ as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
31
+ as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
32
+ ) -> None:
33
+ """Submit multiple URLs in a single batch job."""
34
+ apply_output_override(ctx, as_json, as_yaml)
35
+ payload = validated(
36
+ CreateBatchInput,
37
+ urls=urls,
38
+ render_js=render_js,
39
+ **optional(
40
+ formats=formats,
41
+ proxy_country=proxy_country,
42
+ proxy_type=proxy_type,
43
+ headers=parse_headers(header),
44
+ ),
45
+ )
46
+ call(ctx, "create_batch", payload)
47
+
48
+
49
+ @batch_app.command("status")
50
+ def status(
51
+ ctx: typer.Context,
52
+ job_id: str = typer.Argument(..., help="Batch job ID."),
53
+ page: int = typer.Option(1, "--page", help="Page number."),
54
+ page_size: int = typer.Option(10, "--page-size", help="Results per page (max 50)."),
55
+ as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
56
+ as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
57
+ ) -> None:
58
+ """Show the status and per-URL results of a batch job."""
59
+ apply_output_override(ctx, as_json, as_yaml)
60
+ payload = validated(GetBatchStatusInput, job_id=job_id, page=page, page_size=page_size)
61
+ call(ctx, "get_batch_status", payload)
62
+
63
+
64
+ @batch_app.command("wait")
65
+ def wait(
66
+ ctx: typer.Context,
67
+ job_id: str = typer.Argument(..., help="Batch job ID."),
68
+ timeout_seconds: Optional[float] = typer.Option(None, "--timeout", help="Polling timeout override (seconds)."),
69
+ poll_interval_seconds: Optional[float] = typer.Option(None, "--interval", help="Polling interval override (s)."),
70
+ as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
71
+ as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
72
+ ) -> None:
73
+ """Poll a batch job until it reaches a terminal state."""
74
+ apply_output_override(ctx, as_json, as_yaml)
75
+ payload = validated(
76
+ WaitForBatchInput,
77
+ job_id=job_id,
78
+ **optional(timeout_seconds=timeout_seconds, poll_interval_seconds=poll_interval_seconds),
79
+ )
80
+ call(ctx, "wait_for_batch", payload)
81
+
82
+
83
+ @batch_app.command("list")
84
+ def list_jobs(
85
+ ctx: typer.Context,
86
+ status: Optional[str] = typer.Option(None, "--status", help="queued|processing|completed|failed|cancelled."),
87
+ start_date: Optional[str] = typer.Option(None, "--start-date", help="ISO 8601 lower bound."),
88
+ end_date: Optional[str] = typer.Option(None, "--end-date", help="ISO 8601 upper bound."),
89
+ page: int = typer.Option(1, "--page", help="Page number."),
90
+ page_size: int = typer.Option(100, "--page-size", help="Results per page (max 100)."),
91
+ as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
92
+ as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
93
+ ) -> None:
94
+ """List and filter batch jobs."""
95
+ apply_output_override(ctx, as_json, as_yaml)
96
+ payload = validated(
97
+ ListBatchJobsInput,
98
+ page=page,
99
+ page_size=page_size,
100
+ **optional(status=status, start_date=start_date, end_date=end_date),
101
+ )
102
+ call(ctx, "list_batch_jobs", payload)
103
+
104
+
105
+ @batch_app.command("cancel")
106
+ def cancel(
107
+ ctx: typer.Context,
108
+ job_id: str = typer.Argument(..., help="Batch job ID to cancel."),
109
+ as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
110
+ as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
111
+ ) -> None:
112
+ """Cancel a running batch job."""
113
+ apply_output_override(ctx, as_json, as_yaml)
114
+ call(ctx, "cancel_batch", {"job_id": job_id})
@@ -0,0 +1,48 @@
1
+ """`gscraper config` — inspect CLI configuration (read-only)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import typer
6
+
7
+ from ..config import ConfigError, config_path, load_profiles, resolve_profile_name
8
+ from ..context import get_cli_context
9
+ from ..errors import ExitCode
10
+ from ..renderers import console
11
+
12
+ config_app = typer.Typer(help="Inspect CLI configuration.", no_args_is_help=True)
13
+
14
+
15
+ @config_app.command("path")
16
+ def path() -> None:
17
+ """Print the location of the TOML config file."""
18
+ typer.echo(str(config_path()))
19
+
20
+
21
+ @config_app.command("show")
22
+ def show(ctx: typer.Context) -> None:
23
+ """Show the active profile and config file contents (API keys masked)."""
24
+ cli = get_cli_context(ctx)
25
+ profile = resolve_profile_name(cli.profile)
26
+ try:
27
+ profiles = load_profiles()
28
+ except ConfigError as exc:
29
+ typer.echo(str(exc), err=True)
30
+ raise typer.Exit(int(ExitCode.USAGE)) from exc
31
+
32
+ out = console()
33
+ out.print(f"config file: {config_path()}")
34
+ out.print(f"active profile: {profile}")
35
+ if not profiles:
36
+ out.print("[yellow]No config file found.[/yellow]")
37
+ return
38
+ masked = {name: _mask(section) for name, section in profiles.items()}
39
+ out.print_json(data=masked)
40
+
41
+
42
+ def _mask(section: object) -> object:
43
+ if not isinstance(section, dict):
44
+ return section
45
+ redacted = dict(section)
46
+ if "api_key" in redacted and redacted["api_key"]:
47
+ redacted["api_key"] = "***"
48
+ return redacted
@@ -0,0 +1,119 @@
1
+ """`gscraper crawl` — submit and manage site crawl jobs."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import List, Optional
6
+
7
+ import typer
8
+ from geonode_scraper_tools_core.schemas import (
9
+ CreateCrawlInput,
10
+ GetCrawlStatusInput,
11
+ ListCrawlJobsInput,
12
+ WaitForCrawlInput,
13
+ )
14
+
15
+ from ..invoke import call, validated
16
+ from ._common import apply_output_override, optional
17
+
18
+ crawl_app = typer.Typer(help="Submit and manage site crawl jobs.", no_args_is_help=True)
19
+
20
+
21
+ @crawl_app.command("create")
22
+ def create(
23
+ ctx: typer.Context,
24
+ url: str = typer.Argument(..., help="Seed URL to start crawling from."),
25
+ depth: int = typer.Option(2, "--depth", help="Maximum crawl depth."),
26
+ limit: int = typer.Option(50, "--limit", help="Maximum number of pages."),
27
+ formats: Optional[List[str]] = typer.Option(None, "--format", "-f", help="Output format(s). Repeatable."),
28
+ render_js: bool = typer.Option(False, "--render-js", help="Render JavaScript with a headless browser."),
29
+ same_domain_only: bool = typer.Option(
30
+ True, "--same-domain-only/--all-domains", help="Restrict to the seed domain."
31
+ ),
32
+ include_subdomains: bool = typer.Option(False, "--include-subdomains", help="Include subdomains when crawling."),
33
+ proxy_country: Optional[str] = typer.Option(None, "--proxy-country", help="ISO 3166-1 alpha-2 country code."),
34
+ proxy_type: Optional[str] = typer.Option(None, "--proxy-type", help="Proxy type: datacenter, residential, mix."),
35
+ as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
36
+ as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
37
+ ) -> None:
38
+ """Crawl a website from a seed URL up to a depth and page limit."""
39
+ apply_output_override(ctx, as_json, as_yaml)
40
+ payload = validated(
41
+ CreateCrawlInput,
42
+ url=url,
43
+ depth=depth,
44
+ limit=limit,
45
+ render_js=render_js,
46
+ same_domain_only=same_domain_only,
47
+ include_subdomains=include_subdomains,
48
+ **optional(formats=formats, proxy_country=proxy_country, proxy_type=proxy_type),
49
+ )
50
+ call(ctx, "create_crawl", payload)
51
+
52
+
53
+ @crawl_app.command("status")
54
+ def status(
55
+ ctx: typer.Context,
56
+ job_id: str = typer.Argument(..., help="Crawl job ID."),
57
+ page: int = typer.Option(1, "--page", help="Page number."),
58
+ page_size: int = typer.Option(10, "--page-size", help="Results per page (max 50)."),
59
+ as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
60
+ as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
61
+ ) -> None:
62
+ """Show the status and per-page results of a crawl job."""
63
+ apply_output_override(ctx, as_json, as_yaml)
64
+ payload = validated(GetCrawlStatusInput, job_id=job_id, page=page, page_size=page_size)
65
+ call(ctx, "get_crawl_status", payload)
66
+
67
+
68
+ @crawl_app.command("wait")
69
+ def wait(
70
+ ctx: typer.Context,
71
+ job_id: str = typer.Argument(..., help="Crawl job ID."),
72
+ timeout_seconds: Optional[float] = typer.Option(None, "--timeout", help="Polling timeout override (seconds)."),
73
+ poll_interval_seconds: Optional[float] = typer.Option(None, "--interval", help="Polling interval override (s)."),
74
+ as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
75
+ as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
76
+ ) -> None:
77
+ """Poll a crawl job until it reaches a terminal state."""
78
+ apply_output_override(ctx, as_json, as_yaml)
79
+ payload = validated(
80
+ WaitForCrawlInput,
81
+ job_id=job_id,
82
+ **optional(timeout_seconds=timeout_seconds, poll_interval_seconds=poll_interval_seconds),
83
+ )
84
+ call(ctx, "wait_for_crawl", payload)
85
+
86
+
87
+ @crawl_app.command("list")
88
+ def list_jobs(
89
+ ctx: typer.Context,
90
+ url: Optional[str] = typer.Option(None, "--url", help="Filter by seed URL."),
91
+ status: Optional[str] = typer.Option(None, "--status", help="queued|processing|completed|failed|cancelled."),
92
+ start_date: Optional[str] = typer.Option(None, "--start-date", help="ISO 8601 lower bound."),
93
+ end_date: Optional[str] = typer.Option(None, "--end-date", help="ISO 8601 upper bound."),
94
+ page: int = typer.Option(1, "--page", help="Page number."),
95
+ page_size: int = typer.Option(100, "--page-size", help="Results per page (max 100)."),
96
+ as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
97
+ as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
98
+ ) -> None:
99
+ """List and filter crawl jobs."""
100
+ apply_output_override(ctx, as_json, as_yaml)
101
+ payload = validated(
102
+ ListCrawlJobsInput,
103
+ page=page,
104
+ page_size=page_size,
105
+ **optional(url=url, status=status, start_date=start_date, end_date=end_date),
106
+ )
107
+ call(ctx, "list_crawl_jobs", payload)
108
+
109
+
110
+ @crawl_app.command("cancel")
111
+ def cancel(
112
+ ctx: typer.Context,
113
+ job_id: str = typer.Argument(..., help="Crawl job ID to cancel."),
114
+ as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
115
+ as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
116
+ ) -> None:
117
+ """Cancel a running crawl job."""
118
+ apply_output_override(ctx, as_json, as_yaml)
119
+ call(ctx, "cancel_crawl", {"job_id": job_id})