geonode-scraper-cli 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- geonode_scraper_cli-0.1.0/PKG-INFO +159 -0
- geonode_scraper_cli-0.1.0/README.md +137 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/__init__.py +3 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/__init__.py +1 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/_common.py +42 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/batch.py +114 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/config.py +48 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/crawl.py +119 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/extract.py +78 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/jobs.py +77 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/map.py +75 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/stats.py +24 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/commands/system.py +18 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/config.py +104 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/context.py +59 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/errors.py +50 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/invoke.py +56 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/main.py +83 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/output.py +9 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/py.typed +0 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/renderers/__init__.py +49 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli/renderers/tables.py +118 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli.egg-info/PKG-INFO +159 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli.egg-info/SOURCES.txt +29 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli.egg-info/dependency_links.txt +1 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli.egg-info/entry_points.txt +2 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli.egg-info/requires.txt +12 -0
- geonode_scraper_cli-0.1.0/geonode_scraper_cli.egg-info/top_level.txt +1 -0
- geonode_scraper_cli-0.1.0/pyproject.toml +58 -0
- geonode_scraper_cli-0.1.0/setup.cfg +4 -0
- geonode_scraper_cli-0.1.0/tests/test_cli.py +137 -0
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: geonode-scraper-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Command-line interface for the Geonode Scraper API
|
|
5
|
+
Author: Geonode Team
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
10
|
+
Classifier: Typing :: Typed
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
Requires-Dist: geonode-scraper-tools-core>=0.3.1
|
|
14
|
+
Requires-Dist: typer>=0.12
|
|
15
|
+
Requires-Dist: rich>=13.0
|
|
16
|
+
Requires-Dist: PyYAML>=6.0
|
|
17
|
+
Requires-Dist: tomli>=2.0; python_version < "3.11"
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
20
|
+
Requires-Dist: pytest-cov>=5.0; extra == "dev"
|
|
21
|
+
Requires-Dist: ruff>=0.12.11; extra == "dev"
|
|
22
|
+
|
|
23
|
+
# Geonode Scraper CLI
|
|
24
|
+
|
|
25
|
+
`gscraper` is the command-line interface for the Geonode Scraper API. It is a thin
|
|
26
|
+
presentation layer over [`geonode-scraper-tools-core`](../python-agent-tools-core):
|
|
27
|
+
commands parse flags, resolve configuration, call a stable service method, and
|
|
28
|
+
render the result. All domain logic (validation, polling, retries) lives in the
|
|
29
|
+
service layer, not in the CLI.
|
|
30
|
+
|
|
31
|
+
## Requirements
|
|
32
|
+
|
|
33
|
+
- Python 3.10+
|
|
34
|
+
- Works on **Linux**, **macOS**, and **Windows**
|
|
35
|
+
|
|
36
|
+
## Installation
|
|
37
|
+
|
|
38
|
+
**Recommended — install as a standalone tool with [pipx](https://pipx.pypa.io):**
|
|
39
|
+
|
|
40
|
+
```sh
|
|
41
|
+
pipx install geonode-scraper-cli
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
`pipx` installs `gscraper` into its own isolated virtual environment and puts it
|
|
45
|
+
on your `PATH`, so it never conflicts with other Python projects. This is the
|
|
46
|
+
preferred way to install CLI tools globally.
|
|
47
|
+
|
|
48
|
+
**Alternative — install with pip:**
|
|
49
|
+
|
|
50
|
+
```sh
|
|
51
|
+
pip install geonode-scraper-cli
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
> **Windows note:** on Windows the `gscraper` command is placed in the Python
|
|
55
|
+
> Scripts folder (e.g. `%APPDATA%\Python\Python3xx\Scripts`). If the command is
|
|
56
|
+
> not found after installation, add that folder to your `PATH`, or use
|
|
57
|
+
> `python -m geonode_scraper_cli` as a fallback. `pipx` handles this
|
|
58
|
+
> automatically and is the simpler choice on Windows.
|
|
59
|
+
|
|
60
|
+
## Configuration
|
|
61
|
+
|
|
62
|
+
Configuration is resolved with the following precedence (highest first):
|
|
63
|
+
|
|
64
|
+
1. Command-line flags (`--api-key`, `--host`, ...)
|
|
65
|
+
2. Environment variables (`GEONODE_SCRAPER_API_KEY`, `GEONODE_SCRAPER_HOST`,
|
|
66
|
+
`GEONODE_SCRAPER_VERIFY_SSL`, `GEONODE_SCRAPER_TIMEOUT`, `GEONODE_SCRAPER_PROFILE`)
|
|
67
|
+
3. A TOML config file at `~/.config/geonode-scraper/config.toml`
|
|
68
|
+
4. Built-in defaults
|
|
69
|
+
|
|
70
|
+
Prefer environment variables or the config file for your API key — passing
|
|
71
|
+
`--api-key` on the command line can leak it into your shell history.
|
|
72
|
+
|
|
73
|
+
Example `~/.config/geonode-scraper/config.toml`:
|
|
74
|
+
|
|
75
|
+
```toml
|
|
76
|
+
[default]
|
|
77
|
+
host = "https://api.example.com"
|
|
78
|
+
api_key = "your-api-key"
|
|
79
|
+
verify_ssl = true
|
|
80
|
+
|
|
81
|
+
[staging]
|
|
82
|
+
host = "https://staging.example.com"
|
|
83
|
+
api_key = "your-staging-key"
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Select a non-default profile with `--profile staging` or
|
|
87
|
+
`GEONODE_SCRAPER_PROFILE=staging`. Inspect the active configuration with:
|
|
88
|
+
|
|
89
|
+
```sh
|
|
90
|
+
gscraper config path # print the config file location
|
|
91
|
+
gscraper config show # show profiles (API keys masked)
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## Output
|
|
95
|
+
|
|
96
|
+
Commands print a human-readable summary by default. Use `--json` or `--yaml`
|
|
97
|
+
to print the raw result envelope for scripting. These flags can appear either
|
|
98
|
+
before the subcommand (global position) or after it (per-command position) —
|
|
99
|
+
both work:
|
|
100
|
+
|
|
101
|
+
```sh
|
|
102
|
+
gscraper extract https://example.com --json | jq -r .result.data.markdown
|
|
103
|
+
gscraper --json extract https://example.com | jq -r .result.data.markdown
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
The JSON/YAML envelope has the shape `{ "ok": bool, "operation": str, "result": {...} }`
|
|
107
|
+
on success, or `{ "ok": false, "operation": str, "error": {...} }` on failure.
|
|
108
|
+
|
|
109
|
+
## Commands
|
|
110
|
+
|
|
111
|
+
```text
|
|
112
|
+
gscraper extract URL [--format markdown|html] [--render-js] [--async] \
|
|
113
|
+
[--proxy-country US] [--proxy-type residential] \
|
|
114
|
+
[--header "K: V"] [--output out.md]
|
|
115
|
+
|
|
116
|
+
gscraper jobs get JOB_ID
|
|
117
|
+
gscraper jobs list [--status completed] [--url ...] [--page N]
|
|
118
|
+
gscraper jobs wait JOB_ID [--timeout S] [--interval S]
|
|
119
|
+
|
|
120
|
+
gscraper batch create URL [URL ...] [--format markdown]
|
|
121
|
+
gscraper batch status JOB_ID
|
|
122
|
+
gscraper batch wait JOB_ID [--timeout S] [--interval S]
|
|
123
|
+
gscraper batch list [--status ...]
|
|
124
|
+
gscraper batch cancel JOB_ID
|
|
125
|
+
|
|
126
|
+
gscraper crawl create URL [--depth 2] [--limit 50] [--include-subdomains]
|
|
127
|
+
gscraper crawl status JOB_ID
|
|
128
|
+
gscraper crawl wait JOB_ID
|
|
129
|
+
gscraper crawl list [--url ...]
|
|
130
|
+
gscraper crawl cancel JOB_ID
|
|
131
|
+
|
|
132
|
+
gscraper map run URL [--search term] [--no-subdomains] # primary action
|
|
133
|
+
gscraper map jobs list # inspect past map jobs
|
|
134
|
+
gscraper map jobs get JOB_ID
|
|
135
|
+
|
|
136
|
+
gscraper stats [--start-date ISO] [--end-date ISO]
|
|
137
|
+
gscraper health
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Run `gscraper --help` or `gscraper <command> --help` for full details.
|
|
141
|
+
|
|
142
|
+
## Exit codes
|
|
143
|
+
|
|
144
|
+
| Code | Meaning |
|
|
145
|
+
| ---- | ------- |
|
|
146
|
+
| 0 | Success |
|
|
147
|
+
| 1 | Generic error |
|
|
148
|
+
| 2 | Usage / invalid arguments |
|
|
149
|
+
| 4 | Authentication / authorization (401, 403) |
|
|
150
|
+
| 5 | Not found (404) |
|
|
151
|
+
| 6 | Validation error (422) |
|
|
152
|
+
| 7 | Network / connection error |
|
|
153
|
+
| 8 | Polling timeout (`wait` commands) |
|
|
154
|
+
|
|
155
|
+
## Shell completion
|
|
156
|
+
|
|
157
|
+
```sh
|
|
158
|
+
gscraper --install-completion
|
|
159
|
+
```
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# Geonode Scraper CLI
|
|
2
|
+
|
|
3
|
+
`gscraper` is the command-line interface for the Geonode Scraper API. It is a thin
|
|
4
|
+
presentation layer over [`geonode-scraper-tools-core`](../python-agent-tools-core):
|
|
5
|
+
commands parse flags, resolve configuration, call a stable service method, and
|
|
6
|
+
render the result. All domain logic (validation, polling, retries) lives in the
|
|
7
|
+
service layer, not in the CLI.
|
|
8
|
+
|
|
9
|
+
## Requirements
|
|
10
|
+
|
|
11
|
+
- Python 3.10+
|
|
12
|
+
- Works on **Linux**, **macOS**, and **Windows**
|
|
13
|
+
|
|
14
|
+
## Installation
|
|
15
|
+
|
|
16
|
+
**Recommended — install as a standalone tool with [pipx](https://pipx.pypa.io):**
|
|
17
|
+
|
|
18
|
+
```sh
|
|
19
|
+
pipx install geonode-scraper-cli
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
`pipx` installs `gscraper` into its own isolated virtual environment and puts it
|
|
23
|
+
on your `PATH`, so it never conflicts with other Python projects. This is the
|
|
24
|
+
preferred way to install CLI tools globally.
|
|
25
|
+
|
|
26
|
+
**Alternative — install with pip:**
|
|
27
|
+
|
|
28
|
+
```sh
|
|
29
|
+
pip install geonode-scraper-cli
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
> **Windows note:** on Windows the `gscraper` command is placed in the Python
|
|
33
|
+
> Scripts folder (e.g. `%APPDATA%\Python\Python3xx\Scripts`). If the command is
|
|
34
|
+
> not found after installation, add that folder to your `PATH`, or use
|
|
35
|
+
> `python -m geonode_scraper_cli` as a fallback. `pipx` handles this
|
|
36
|
+
> automatically and is the simpler choice on Windows.
|
|
37
|
+
|
|
38
|
+
## Configuration
|
|
39
|
+
|
|
40
|
+
Configuration is resolved with the following precedence (highest first):
|
|
41
|
+
|
|
42
|
+
1. Command-line flags (`--api-key`, `--host`, ...)
|
|
43
|
+
2. Environment variables (`GEONODE_SCRAPER_API_KEY`, `GEONODE_SCRAPER_HOST`,
|
|
44
|
+
`GEONODE_SCRAPER_VERIFY_SSL`, `GEONODE_SCRAPER_TIMEOUT`, `GEONODE_SCRAPER_PROFILE`)
|
|
45
|
+
3. A TOML config file at `~/.config/geonode-scraper/config.toml`
|
|
46
|
+
4. Built-in defaults
|
|
47
|
+
|
|
48
|
+
Prefer environment variables or the config file for your API key — passing
|
|
49
|
+
`--api-key` on the command line can leak it into your shell history.
|
|
50
|
+
|
|
51
|
+
Example `~/.config/geonode-scraper/config.toml`:
|
|
52
|
+
|
|
53
|
+
```toml
|
|
54
|
+
[default]
|
|
55
|
+
host = "https://api.example.com"
|
|
56
|
+
api_key = "your-api-key"
|
|
57
|
+
verify_ssl = true
|
|
58
|
+
|
|
59
|
+
[staging]
|
|
60
|
+
host = "https://staging.example.com"
|
|
61
|
+
api_key = "your-staging-key"
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Select a non-default profile with `--profile staging` or
|
|
65
|
+
`GEONODE_SCRAPER_PROFILE=staging`. Inspect the active configuration with:
|
|
66
|
+
|
|
67
|
+
```sh
|
|
68
|
+
gscraper config path # print the config file location
|
|
69
|
+
gscraper config show # show profiles (API keys masked)
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Output
|
|
73
|
+
|
|
74
|
+
Commands print a human-readable summary by default. Use `--json` or `--yaml`
|
|
75
|
+
to print the raw result envelope for scripting. These flags can appear either
|
|
76
|
+
before the subcommand (global position) or after it (per-command position) —
|
|
77
|
+
both work:
|
|
78
|
+
|
|
79
|
+
```sh
|
|
80
|
+
gscraper extract https://example.com --json | jq -r .result.data.markdown
|
|
81
|
+
gscraper --json extract https://example.com | jq -r .result.data.markdown
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
The JSON/YAML envelope has the shape `{ "ok": bool, "operation": str, "result": {...} }`
|
|
85
|
+
on success, or `{ "ok": false, "operation": str, "error": {...} }` on failure.
|
|
86
|
+
|
|
87
|
+
## Commands
|
|
88
|
+
|
|
89
|
+
```text
|
|
90
|
+
gscraper extract URL [--format markdown|html] [--render-js] [--async] \
|
|
91
|
+
[--proxy-country US] [--proxy-type residential] \
|
|
92
|
+
[--header "K: V"] [--output out.md]
|
|
93
|
+
|
|
94
|
+
gscraper jobs get JOB_ID
|
|
95
|
+
gscraper jobs list [--status completed] [--url ...] [--page N]
|
|
96
|
+
gscraper jobs wait JOB_ID [--timeout S] [--interval S]
|
|
97
|
+
|
|
98
|
+
gscraper batch create URL [URL ...] [--format markdown]
|
|
99
|
+
gscraper batch status JOB_ID
|
|
100
|
+
gscraper batch wait JOB_ID [--timeout S] [--interval S]
|
|
101
|
+
gscraper batch list [--status ...]
|
|
102
|
+
gscraper batch cancel JOB_ID
|
|
103
|
+
|
|
104
|
+
gscraper crawl create URL [--depth 2] [--limit 50] [--include-subdomains]
|
|
105
|
+
gscraper crawl status JOB_ID
|
|
106
|
+
gscraper crawl wait JOB_ID
|
|
107
|
+
gscraper crawl list [--url ...]
|
|
108
|
+
gscraper crawl cancel JOB_ID
|
|
109
|
+
|
|
110
|
+
gscraper map run URL [--search term] [--no-subdomains] # primary action
|
|
111
|
+
gscraper map jobs list # inspect past map jobs
|
|
112
|
+
gscraper map jobs get JOB_ID
|
|
113
|
+
|
|
114
|
+
gscraper stats [--start-date ISO] [--end-date ISO]
|
|
115
|
+
gscraper health
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Run `gscraper --help` or `gscraper <command> --help` for full details.
|
|
119
|
+
|
|
120
|
+
## Exit codes
|
|
121
|
+
|
|
122
|
+
| Code | Meaning |
|
|
123
|
+
| ---- | ------- |
|
|
124
|
+
| 0 | Success |
|
|
125
|
+
| 1 | Generic error |
|
|
126
|
+
| 2 | Usage / invalid arguments |
|
|
127
|
+
| 4 | Authentication / authorization (401, 403) |
|
|
128
|
+
| 5 | Not found (404) |
|
|
129
|
+
| 6 | Validation error (422) |
|
|
130
|
+
| 7 | Network / connection error |
|
|
131
|
+
| 8 | Polling timeout (`wait` commands) |
|
|
132
|
+
|
|
133
|
+
## Shell completion
|
|
134
|
+
|
|
135
|
+
```sh
|
|
136
|
+
gscraper --install-completion
|
|
137
|
+
```
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Typer command modules for the gscraper CLI."""
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Helpers shared across command modules."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Iterable
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
from ..errors import ExitCode
|
|
10
|
+
from ..output import OutputFormat
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def apply_output_override(ctx: typer.Context, as_json: bool, as_yaml: bool) -> None:
|
|
14
|
+
"""Override the output format set by the global --json/--yaml flags.
|
|
15
|
+
|
|
16
|
+
Allows per-command placement: ``gscraper extract https://example.com --json``.
|
|
17
|
+
"""
|
|
18
|
+
if as_json and as_yaml:
|
|
19
|
+
raise typer.BadParameter("Use only one of --json or --yaml.")
|
|
20
|
+
if as_json:
|
|
21
|
+
ctx.obj.output = OutputFormat.JSON
|
|
22
|
+
elif as_yaml:
|
|
23
|
+
ctx.obj.output = OutputFormat.YAML
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def parse_headers(raw_headers: Iterable[str] | None) -> dict[str, str] | None:
|
|
27
|
+
"""Parse repeated ``--header "Key: Value"`` flags into a dict."""
|
|
28
|
+
if not raw_headers:
|
|
29
|
+
return None
|
|
30
|
+
headers: dict[str, str] = {}
|
|
31
|
+
for item in raw_headers:
|
|
32
|
+
if ":" not in item:
|
|
33
|
+
typer.echo(f"Invalid header '{item}'. Expected 'Key: Value'.", err=True)
|
|
34
|
+
raise typer.Exit(int(ExitCode.USAGE))
|
|
35
|
+
key, value = item.split(":", 1)
|
|
36
|
+
headers[key.strip()] = value.strip()
|
|
37
|
+
return headers
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def optional(**kwargs: object) -> dict[str, object]:
|
|
41
|
+
"""Drop keys whose value is ``None`` so schema defaults apply."""
|
|
42
|
+
return {key: value for key, value in kwargs.items() if value is not None}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""`gscraper batch` — submit and manage batch extraction jobs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import List, Optional
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
from geonode_scraper_tools_core.schemas import (
|
|
9
|
+
CreateBatchInput,
|
|
10
|
+
GetBatchStatusInput,
|
|
11
|
+
ListBatchJobsInput,
|
|
12
|
+
WaitForBatchInput,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
from ..invoke import call, validated
|
|
16
|
+
from ._common import apply_output_override, optional, parse_headers
|
|
17
|
+
|
|
18
|
+
batch_app = typer.Typer(help="Submit and manage batch extraction jobs.", no_args_is_help=True)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@batch_app.command("create")
|
|
22
|
+
def create(
|
|
23
|
+
ctx: typer.Context,
|
|
24
|
+
urls: List[str] = typer.Argument(..., help="One or more URLs to extract."),
|
|
25
|
+
formats: Optional[List[str]] = typer.Option(None, "--format", "-f", help="Output format(s). Repeatable."),
|
|
26
|
+
render_js: bool = typer.Option(False, "--render-js", help="Render JavaScript with a headless browser."),
|
|
27
|
+
proxy_country: Optional[str] = typer.Option(None, "--proxy-country", help="ISO 3166-1 alpha-2 country code."),
|
|
28
|
+
proxy_type: Optional[str] = typer.Option(None, "--proxy-type", help="Proxy type: datacenter, residential, mix."),
|
|
29
|
+
header: Optional[List[str]] = typer.Option(None, "--header", "-H", help="Request header 'Key: Value'. Repeatable."),
|
|
30
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
31
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
32
|
+
) -> None:
|
|
33
|
+
"""Submit multiple URLs in a single batch job."""
|
|
34
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
35
|
+
payload = validated(
|
|
36
|
+
CreateBatchInput,
|
|
37
|
+
urls=urls,
|
|
38
|
+
render_js=render_js,
|
|
39
|
+
**optional(
|
|
40
|
+
formats=formats,
|
|
41
|
+
proxy_country=proxy_country,
|
|
42
|
+
proxy_type=proxy_type,
|
|
43
|
+
headers=parse_headers(header),
|
|
44
|
+
),
|
|
45
|
+
)
|
|
46
|
+
call(ctx, "create_batch", payload)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@batch_app.command("status")
|
|
50
|
+
def status(
|
|
51
|
+
ctx: typer.Context,
|
|
52
|
+
job_id: str = typer.Argument(..., help="Batch job ID."),
|
|
53
|
+
page: int = typer.Option(1, "--page", help="Page number."),
|
|
54
|
+
page_size: int = typer.Option(10, "--page-size", help="Results per page (max 50)."),
|
|
55
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
56
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
57
|
+
) -> None:
|
|
58
|
+
"""Show the status and per-URL results of a batch job."""
|
|
59
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
60
|
+
payload = validated(GetBatchStatusInput, job_id=job_id, page=page, page_size=page_size)
|
|
61
|
+
call(ctx, "get_batch_status", payload)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@batch_app.command("wait")
|
|
65
|
+
def wait(
|
|
66
|
+
ctx: typer.Context,
|
|
67
|
+
job_id: str = typer.Argument(..., help="Batch job ID."),
|
|
68
|
+
timeout_seconds: Optional[float] = typer.Option(None, "--timeout", help="Polling timeout override (seconds)."),
|
|
69
|
+
poll_interval_seconds: Optional[float] = typer.Option(None, "--interval", help="Polling interval override (s)."),
|
|
70
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
71
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
72
|
+
) -> None:
|
|
73
|
+
"""Poll a batch job until it reaches a terminal state."""
|
|
74
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
75
|
+
payload = validated(
|
|
76
|
+
WaitForBatchInput,
|
|
77
|
+
job_id=job_id,
|
|
78
|
+
**optional(timeout_seconds=timeout_seconds, poll_interval_seconds=poll_interval_seconds),
|
|
79
|
+
)
|
|
80
|
+
call(ctx, "wait_for_batch", payload)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@batch_app.command("list")
|
|
84
|
+
def list_jobs(
|
|
85
|
+
ctx: typer.Context,
|
|
86
|
+
status: Optional[str] = typer.Option(None, "--status", help="queued|processing|completed|failed|cancelled."),
|
|
87
|
+
start_date: Optional[str] = typer.Option(None, "--start-date", help="ISO 8601 lower bound."),
|
|
88
|
+
end_date: Optional[str] = typer.Option(None, "--end-date", help="ISO 8601 upper bound."),
|
|
89
|
+
page: int = typer.Option(1, "--page", help="Page number."),
|
|
90
|
+
page_size: int = typer.Option(100, "--page-size", help="Results per page (max 100)."),
|
|
91
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
92
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
93
|
+
) -> None:
|
|
94
|
+
"""List and filter batch jobs."""
|
|
95
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
96
|
+
payload = validated(
|
|
97
|
+
ListBatchJobsInput,
|
|
98
|
+
page=page,
|
|
99
|
+
page_size=page_size,
|
|
100
|
+
**optional(status=status, start_date=start_date, end_date=end_date),
|
|
101
|
+
)
|
|
102
|
+
call(ctx, "list_batch_jobs", payload)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@batch_app.command("cancel")
|
|
106
|
+
def cancel(
|
|
107
|
+
ctx: typer.Context,
|
|
108
|
+
job_id: str = typer.Argument(..., help="Batch job ID to cancel."),
|
|
109
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
110
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
111
|
+
) -> None:
|
|
112
|
+
"""Cancel a running batch job."""
|
|
113
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
114
|
+
call(ctx, "cancel_batch", {"job_id": job_id})
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""`gscraper config` — inspect CLI configuration (read-only)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import typer
|
|
6
|
+
|
|
7
|
+
from ..config import ConfigError, config_path, load_profiles, resolve_profile_name
|
|
8
|
+
from ..context import get_cli_context
|
|
9
|
+
from ..errors import ExitCode
|
|
10
|
+
from ..renderers import console
|
|
11
|
+
|
|
12
|
+
config_app = typer.Typer(help="Inspect CLI configuration.", no_args_is_help=True)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@config_app.command("path")
|
|
16
|
+
def path() -> None:
|
|
17
|
+
"""Print the location of the TOML config file."""
|
|
18
|
+
typer.echo(str(config_path()))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@config_app.command("show")
|
|
22
|
+
def show(ctx: typer.Context) -> None:
|
|
23
|
+
"""Show the active profile and config file contents (API keys masked)."""
|
|
24
|
+
cli = get_cli_context(ctx)
|
|
25
|
+
profile = resolve_profile_name(cli.profile)
|
|
26
|
+
try:
|
|
27
|
+
profiles = load_profiles()
|
|
28
|
+
except ConfigError as exc:
|
|
29
|
+
typer.echo(str(exc), err=True)
|
|
30
|
+
raise typer.Exit(int(ExitCode.USAGE)) from exc
|
|
31
|
+
|
|
32
|
+
out = console()
|
|
33
|
+
out.print(f"config file: {config_path()}")
|
|
34
|
+
out.print(f"active profile: {profile}")
|
|
35
|
+
if not profiles:
|
|
36
|
+
out.print("[yellow]No config file found.[/yellow]")
|
|
37
|
+
return
|
|
38
|
+
masked = {name: _mask(section) for name, section in profiles.items()}
|
|
39
|
+
out.print_json(data=masked)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _mask(section: object) -> object:
|
|
43
|
+
if not isinstance(section, dict):
|
|
44
|
+
return section
|
|
45
|
+
redacted = dict(section)
|
|
46
|
+
if "api_key" in redacted and redacted["api_key"]:
|
|
47
|
+
redacted["api_key"] = "***"
|
|
48
|
+
return redacted
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""`gscraper crawl` — submit and manage site crawl jobs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import List, Optional
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
from geonode_scraper_tools_core.schemas import (
|
|
9
|
+
CreateCrawlInput,
|
|
10
|
+
GetCrawlStatusInput,
|
|
11
|
+
ListCrawlJobsInput,
|
|
12
|
+
WaitForCrawlInput,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
from ..invoke import call, validated
|
|
16
|
+
from ._common import apply_output_override, optional
|
|
17
|
+
|
|
18
|
+
crawl_app = typer.Typer(help="Submit and manage site crawl jobs.", no_args_is_help=True)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@crawl_app.command("create")
|
|
22
|
+
def create(
|
|
23
|
+
ctx: typer.Context,
|
|
24
|
+
url: str = typer.Argument(..., help="Seed URL to start crawling from."),
|
|
25
|
+
depth: int = typer.Option(2, "--depth", help="Maximum crawl depth."),
|
|
26
|
+
limit: int = typer.Option(50, "--limit", help="Maximum number of pages."),
|
|
27
|
+
formats: Optional[List[str]] = typer.Option(None, "--format", "-f", help="Output format(s). Repeatable."),
|
|
28
|
+
render_js: bool = typer.Option(False, "--render-js", help="Render JavaScript with a headless browser."),
|
|
29
|
+
same_domain_only: bool = typer.Option(
|
|
30
|
+
True, "--same-domain-only/--all-domains", help="Restrict to the seed domain."
|
|
31
|
+
),
|
|
32
|
+
include_subdomains: bool = typer.Option(False, "--include-subdomains", help="Include subdomains when crawling."),
|
|
33
|
+
proxy_country: Optional[str] = typer.Option(None, "--proxy-country", help="ISO 3166-1 alpha-2 country code."),
|
|
34
|
+
proxy_type: Optional[str] = typer.Option(None, "--proxy-type", help="Proxy type: datacenter, residential, mix."),
|
|
35
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
36
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
37
|
+
) -> None:
|
|
38
|
+
"""Crawl a website from a seed URL up to a depth and page limit."""
|
|
39
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
40
|
+
payload = validated(
|
|
41
|
+
CreateCrawlInput,
|
|
42
|
+
url=url,
|
|
43
|
+
depth=depth,
|
|
44
|
+
limit=limit,
|
|
45
|
+
render_js=render_js,
|
|
46
|
+
same_domain_only=same_domain_only,
|
|
47
|
+
include_subdomains=include_subdomains,
|
|
48
|
+
**optional(formats=formats, proxy_country=proxy_country, proxy_type=proxy_type),
|
|
49
|
+
)
|
|
50
|
+
call(ctx, "create_crawl", payload)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@crawl_app.command("status")
|
|
54
|
+
def status(
|
|
55
|
+
ctx: typer.Context,
|
|
56
|
+
job_id: str = typer.Argument(..., help="Crawl job ID."),
|
|
57
|
+
page: int = typer.Option(1, "--page", help="Page number."),
|
|
58
|
+
page_size: int = typer.Option(10, "--page-size", help="Results per page (max 50)."),
|
|
59
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
60
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
61
|
+
) -> None:
|
|
62
|
+
"""Show the status and per-page results of a crawl job."""
|
|
63
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
64
|
+
payload = validated(GetCrawlStatusInput, job_id=job_id, page=page, page_size=page_size)
|
|
65
|
+
call(ctx, "get_crawl_status", payload)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@crawl_app.command("wait")
|
|
69
|
+
def wait(
|
|
70
|
+
ctx: typer.Context,
|
|
71
|
+
job_id: str = typer.Argument(..., help="Crawl job ID."),
|
|
72
|
+
timeout_seconds: Optional[float] = typer.Option(None, "--timeout", help="Polling timeout override (seconds)."),
|
|
73
|
+
poll_interval_seconds: Optional[float] = typer.Option(None, "--interval", help="Polling interval override (s)."),
|
|
74
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
75
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
76
|
+
) -> None:
|
|
77
|
+
"""Poll a crawl job until it reaches a terminal state."""
|
|
78
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
79
|
+
payload = validated(
|
|
80
|
+
WaitForCrawlInput,
|
|
81
|
+
job_id=job_id,
|
|
82
|
+
**optional(timeout_seconds=timeout_seconds, poll_interval_seconds=poll_interval_seconds),
|
|
83
|
+
)
|
|
84
|
+
call(ctx, "wait_for_crawl", payload)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@crawl_app.command("list")
|
|
88
|
+
def list_jobs(
|
|
89
|
+
ctx: typer.Context,
|
|
90
|
+
url: Optional[str] = typer.Option(None, "--url", help="Filter by seed URL."),
|
|
91
|
+
status: Optional[str] = typer.Option(None, "--status", help="queued|processing|completed|failed|cancelled."),
|
|
92
|
+
start_date: Optional[str] = typer.Option(None, "--start-date", help="ISO 8601 lower bound."),
|
|
93
|
+
end_date: Optional[str] = typer.Option(None, "--end-date", help="ISO 8601 upper bound."),
|
|
94
|
+
page: int = typer.Option(1, "--page", help="Page number."),
|
|
95
|
+
page_size: int = typer.Option(100, "--page-size", help="Results per page (max 100)."),
|
|
96
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
97
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
98
|
+
) -> None:
|
|
99
|
+
"""List and filter crawl jobs."""
|
|
100
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
101
|
+
payload = validated(
|
|
102
|
+
ListCrawlJobsInput,
|
|
103
|
+
page=page,
|
|
104
|
+
page_size=page_size,
|
|
105
|
+
**optional(url=url, status=status, start_date=start_date, end_date=end_date),
|
|
106
|
+
)
|
|
107
|
+
call(ctx, "list_crawl_jobs", payload)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@crawl_app.command("cancel")
|
|
111
|
+
def cancel(
|
|
112
|
+
ctx: typer.Context,
|
|
113
|
+
job_id: str = typer.Argument(..., help="Crawl job ID to cancel."),
|
|
114
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
115
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
116
|
+
) -> None:
|
|
117
|
+
"""Cancel a running crawl job."""
|
|
118
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
119
|
+
call(ctx, "cancel_crawl", {"job_id": job_id})
|