geonode-scraper-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- geonode_scraper_cli/__init__.py +3 -0
- geonode_scraper_cli/commands/__init__.py +1 -0
- geonode_scraper_cli/commands/_common.py +42 -0
- geonode_scraper_cli/commands/batch.py +114 -0
- geonode_scraper_cli/commands/config.py +48 -0
- geonode_scraper_cli/commands/crawl.py +119 -0
- geonode_scraper_cli/commands/extract.py +78 -0
- geonode_scraper_cli/commands/jobs.py +77 -0
- geonode_scraper_cli/commands/map.py +75 -0
- geonode_scraper_cli/commands/stats.py +24 -0
- geonode_scraper_cli/commands/system.py +18 -0
- geonode_scraper_cli/config.py +104 -0
- geonode_scraper_cli/context.py +59 -0
- geonode_scraper_cli/errors.py +50 -0
- geonode_scraper_cli/invoke.py +56 -0
- geonode_scraper_cli/main.py +83 -0
- geonode_scraper_cli/output.py +9 -0
- geonode_scraper_cli/py.typed +0 -0
- geonode_scraper_cli/renderers/__init__.py +49 -0
- geonode_scraper_cli/renderers/tables.py +118 -0
- geonode_scraper_cli-0.1.0.dist-info/METADATA +159 -0
- geonode_scraper_cli-0.1.0.dist-info/RECORD +25 -0
- geonode_scraper_cli-0.1.0.dist-info/WHEEL +5 -0
- geonode_scraper_cli-0.1.0.dist-info/entry_points.txt +2 -0
- geonode_scraper_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Typer command modules for the gscraper CLI."""
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Helpers shared across command modules."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Iterable
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
from ..errors import ExitCode
|
|
10
|
+
from ..output import OutputFormat
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def apply_output_override(ctx: typer.Context, as_json: bool, as_yaml: bool) -> None:
|
|
14
|
+
"""Override the output format set by the global --json/--yaml flags.
|
|
15
|
+
|
|
16
|
+
Allows per-command placement: ``gscraper extract https://example.com --json``.
|
|
17
|
+
"""
|
|
18
|
+
if as_json and as_yaml:
|
|
19
|
+
raise typer.BadParameter("Use only one of --json or --yaml.")
|
|
20
|
+
if as_json:
|
|
21
|
+
ctx.obj.output = OutputFormat.JSON
|
|
22
|
+
elif as_yaml:
|
|
23
|
+
ctx.obj.output = OutputFormat.YAML
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def parse_headers(raw_headers: Iterable[str] | None) -> dict[str, str] | None:
|
|
27
|
+
"""Parse repeated ``--header "Key: Value"`` flags into a dict."""
|
|
28
|
+
if not raw_headers:
|
|
29
|
+
return None
|
|
30
|
+
headers: dict[str, str] = {}
|
|
31
|
+
for item in raw_headers:
|
|
32
|
+
if ":" not in item:
|
|
33
|
+
typer.echo(f"Invalid header '{item}'. Expected 'Key: Value'.", err=True)
|
|
34
|
+
raise typer.Exit(int(ExitCode.USAGE))
|
|
35
|
+
key, value = item.split(":", 1)
|
|
36
|
+
headers[key.strip()] = value.strip()
|
|
37
|
+
return headers
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def optional(**kwargs: object) -> dict[str, object]:
|
|
41
|
+
"""Drop keys whose value is ``None`` so schema defaults apply."""
|
|
42
|
+
return {key: value for key, value in kwargs.items() if value is not None}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""`gscraper batch` — submit and manage batch extraction jobs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import List, Optional
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
from geonode_scraper_tools_core.schemas import (
|
|
9
|
+
CreateBatchInput,
|
|
10
|
+
GetBatchStatusInput,
|
|
11
|
+
ListBatchJobsInput,
|
|
12
|
+
WaitForBatchInput,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
from ..invoke import call, validated
|
|
16
|
+
from ._common import apply_output_override, optional, parse_headers
|
|
17
|
+
|
|
18
|
+
batch_app = typer.Typer(help="Submit and manage batch extraction jobs.", no_args_is_help=True)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@batch_app.command("create")
|
|
22
|
+
def create(
|
|
23
|
+
ctx: typer.Context,
|
|
24
|
+
urls: List[str] = typer.Argument(..., help="One or more URLs to extract."),
|
|
25
|
+
formats: Optional[List[str]] = typer.Option(None, "--format", "-f", help="Output format(s). Repeatable."),
|
|
26
|
+
render_js: bool = typer.Option(False, "--render-js", help="Render JavaScript with a headless browser."),
|
|
27
|
+
proxy_country: Optional[str] = typer.Option(None, "--proxy-country", help="ISO 3166-1 alpha-2 country code."),
|
|
28
|
+
proxy_type: Optional[str] = typer.Option(None, "--proxy-type", help="Proxy type: datacenter, residential, mix."),
|
|
29
|
+
header: Optional[List[str]] = typer.Option(None, "--header", "-H", help="Request header 'Key: Value'. Repeatable."),
|
|
30
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
31
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
32
|
+
) -> None:
|
|
33
|
+
"""Submit multiple URLs in a single batch job."""
|
|
34
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
35
|
+
payload = validated(
|
|
36
|
+
CreateBatchInput,
|
|
37
|
+
urls=urls,
|
|
38
|
+
render_js=render_js,
|
|
39
|
+
**optional(
|
|
40
|
+
formats=formats,
|
|
41
|
+
proxy_country=proxy_country,
|
|
42
|
+
proxy_type=proxy_type,
|
|
43
|
+
headers=parse_headers(header),
|
|
44
|
+
),
|
|
45
|
+
)
|
|
46
|
+
call(ctx, "create_batch", payload)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@batch_app.command("status")
|
|
50
|
+
def status(
|
|
51
|
+
ctx: typer.Context,
|
|
52
|
+
job_id: str = typer.Argument(..., help="Batch job ID."),
|
|
53
|
+
page: int = typer.Option(1, "--page", help="Page number."),
|
|
54
|
+
page_size: int = typer.Option(10, "--page-size", help="Results per page (max 50)."),
|
|
55
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
56
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
57
|
+
) -> None:
|
|
58
|
+
"""Show the status and per-URL results of a batch job."""
|
|
59
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
60
|
+
payload = validated(GetBatchStatusInput, job_id=job_id, page=page, page_size=page_size)
|
|
61
|
+
call(ctx, "get_batch_status", payload)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@batch_app.command("wait")
|
|
65
|
+
def wait(
|
|
66
|
+
ctx: typer.Context,
|
|
67
|
+
job_id: str = typer.Argument(..., help="Batch job ID."),
|
|
68
|
+
timeout_seconds: Optional[float] = typer.Option(None, "--timeout", help="Polling timeout override (seconds)."),
|
|
69
|
+
poll_interval_seconds: Optional[float] = typer.Option(None, "--interval", help="Polling interval override (s)."),
|
|
70
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
71
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
72
|
+
) -> None:
|
|
73
|
+
"""Poll a batch job until it reaches a terminal state."""
|
|
74
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
75
|
+
payload = validated(
|
|
76
|
+
WaitForBatchInput,
|
|
77
|
+
job_id=job_id,
|
|
78
|
+
**optional(timeout_seconds=timeout_seconds, poll_interval_seconds=poll_interval_seconds),
|
|
79
|
+
)
|
|
80
|
+
call(ctx, "wait_for_batch", payload)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@batch_app.command("list")
|
|
84
|
+
def list_jobs(
|
|
85
|
+
ctx: typer.Context,
|
|
86
|
+
status: Optional[str] = typer.Option(None, "--status", help="queued|processing|completed|failed|cancelled."),
|
|
87
|
+
start_date: Optional[str] = typer.Option(None, "--start-date", help="ISO 8601 lower bound."),
|
|
88
|
+
end_date: Optional[str] = typer.Option(None, "--end-date", help="ISO 8601 upper bound."),
|
|
89
|
+
page: int = typer.Option(1, "--page", help="Page number."),
|
|
90
|
+
page_size: int = typer.Option(100, "--page-size", help="Results per page (max 100)."),
|
|
91
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
92
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
93
|
+
) -> None:
|
|
94
|
+
"""List and filter batch jobs."""
|
|
95
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
96
|
+
payload = validated(
|
|
97
|
+
ListBatchJobsInput,
|
|
98
|
+
page=page,
|
|
99
|
+
page_size=page_size,
|
|
100
|
+
**optional(status=status, start_date=start_date, end_date=end_date),
|
|
101
|
+
)
|
|
102
|
+
call(ctx, "list_batch_jobs", payload)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@batch_app.command("cancel")
|
|
106
|
+
def cancel(
|
|
107
|
+
ctx: typer.Context,
|
|
108
|
+
job_id: str = typer.Argument(..., help="Batch job ID to cancel."),
|
|
109
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
110
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
111
|
+
) -> None:
|
|
112
|
+
"""Cancel a running batch job."""
|
|
113
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
114
|
+
call(ctx, "cancel_batch", {"job_id": job_id})
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""`gscraper config` — inspect CLI configuration (read-only)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import typer
|
|
6
|
+
|
|
7
|
+
from ..config import ConfigError, config_path, load_profiles, resolve_profile_name
|
|
8
|
+
from ..context import get_cli_context
|
|
9
|
+
from ..errors import ExitCode
|
|
10
|
+
from ..renderers import console
|
|
11
|
+
|
|
12
|
+
config_app = typer.Typer(help="Inspect CLI configuration.", no_args_is_help=True)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@config_app.command("path")
|
|
16
|
+
def path() -> None:
|
|
17
|
+
"""Print the location of the TOML config file."""
|
|
18
|
+
typer.echo(str(config_path()))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@config_app.command("show")
|
|
22
|
+
def show(ctx: typer.Context) -> None:
|
|
23
|
+
"""Show the active profile and config file contents (API keys masked)."""
|
|
24
|
+
cli = get_cli_context(ctx)
|
|
25
|
+
profile = resolve_profile_name(cli.profile)
|
|
26
|
+
try:
|
|
27
|
+
profiles = load_profiles()
|
|
28
|
+
except ConfigError as exc:
|
|
29
|
+
typer.echo(str(exc), err=True)
|
|
30
|
+
raise typer.Exit(int(ExitCode.USAGE)) from exc
|
|
31
|
+
|
|
32
|
+
out = console()
|
|
33
|
+
out.print(f"config file: {config_path()}")
|
|
34
|
+
out.print(f"active profile: {profile}")
|
|
35
|
+
if not profiles:
|
|
36
|
+
out.print("[yellow]No config file found.[/yellow]")
|
|
37
|
+
return
|
|
38
|
+
masked = {name: _mask(section) for name, section in profiles.items()}
|
|
39
|
+
out.print_json(data=masked)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _mask(section: object) -> object:
|
|
43
|
+
if not isinstance(section, dict):
|
|
44
|
+
return section
|
|
45
|
+
redacted = dict(section)
|
|
46
|
+
if "api_key" in redacted and redacted["api_key"]:
|
|
47
|
+
redacted["api_key"] = "***"
|
|
48
|
+
return redacted
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""`gscraper crawl` — submit and manage site crawl jobs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import List, Optional
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
from geonode_scraper_tools_core.schemas import (
|
|
9
|
+
CreateCrawlInput,
|
|
10
|
+
GetCrawlStatusInput,
|
|
11
|
+
ListCrawlJobsInput,
|
|
12
|
+
WaitForCrawlInput,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
from ..invoke import call, validated
|
|
16
|
+
from ._common import apply_output_override, optional
|
|
17
|
+
|
|
18
|
+
crawl_app = typer.Typer(help="Submit and manage site crawl jobs.", no_args_is_help=True)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@crawl_app.command("create")
|
|
22
|
+
def create(
|
|
23
|
+
ctx: typer.Context,
|
|
24
|
+
url: str = typer.Argument(..., help="Seed URL to start crawling from."),
|
|
25
|
+
depth: int = typer.Option(2, "--depth", help="Maximum crawl depth."),
|
|
26
|
+
limit: int = typer.Option(50, "--limit", help="Maximum number of pages."),
|
|
27
|
+
formats: Optional[List[str]] = typer.Option(None, "--format", "-f", help="Output format(s). Repeatable."),
|
|
28
|
+
render_js: bool = typer.Option(False, "--render-js", help="Render JavaScript with a headless browser."),
|
|
29
|
+
same_domain_only: bool = typer.Option(
|
|
30
|
+
True, "--same-domain-only/--all-domains", help="Restrict to the seed domain."
|
|
31
|
+
),
|
|
32
|
+
include_subdomains: bool = typer.Option(False, "--include-subdomains", help="Include subdomains when crawling."),
|
|
33
|
+
proxy_country: Optional[str] = typer.Option(None, "--proxy-country", help="ISO 3166-1 alpha-2 country code."),
|
|
34
|
+
proxy_type: Optional[str] = typer.Option(None, "--proxy-type", help="Proxy type: datacenter, residential, mix."),
|
|
35
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
36
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
37
|
+
) -> None:
|
|
38
|
+
"""Crawl a website from a seed URL up to a depth and page limit."""
|
|
39
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
40
|
+
payload = validated(
|
|
41
|
+
CreateCrawlInput,
|
|
42
|
+
url=url,
|
|
43
|
+
depth=depth,
|
|
44
|
+
limit=limit,
|
|
45
|
+
render_js=render_js,
|
|
46
|
+
same_domain_only=same_domain_only,
|
|
47
|
+
include_subdomains=include_subdomains,
|
|
48
|
+
**optional(formats=formats, proxy_country=proxy_country, proxy_type=proxy_type),
|
|
49
|
+
)
|
|
50
|
+
call(ctx, "create_crawl", payload)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@crawl_app.command("status")
|
|
54
|
+
def status(
|
|
55
|
+
ctx: typer.Context,
|
|
56
|
+
job_id: str = typer.Argument(..., help="Crawl job ID."),
|
|
57
|
+
page: int = typer.Option(1, "--page", help="Page number."),
|
|
58
|
+
page_size: int = typer.Option(10, "--page-size", help="Results per page (max 50)."),
|
|
59
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
60
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
61
|
+
) -> None:
|
|
62
|
+
"""Show the status and per-page results of a crawl job."""
|
|
63
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
64
|
+
payload = validated(GetCrawlStatusInput, job_id=job_id, page=page, page_size=page_size)
|
|
65
|
+
call(ctx, "get_crawl_status", payload)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@crawl_app.command("wait")
|
|
69
|
+
def wait(
|
|
70
|
+
ctx: typer.Context,
|
|
71
|
+
job_id: str = typer.Argument(..., help="Crawl job ID."),
|
|
72
|
+
timeout_seconds: Optional[float] = typer.Option(None, "--timeout", help="Polling timeout override (seconds)."),
|
|
73
|
+
poll_interval_seconds: Optional[float] = typer.Option(None, "--interval", help="Polling interval override (s)."),
|
|
74
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
75
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
76
|
+
) -> None:
|
|
77
|
+
"""Poll a crawl job until it reaches a terminal state."""
|
|
78
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
79
|
+
payload = validated(
|
|
80
|
+
WaitForCrawlInput,
|
|
81
|
+
job_id=job_id,
|
|
82
|
+
**optional(timeout_seconds=timeout_seconds, poll_interval_seconds=poll_interval_seconds),
|
|
83
|
+
)
|
|
84
|
+
call(ctx, "wait_for_crawl", payload)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@crawl_app.command("list")
|
|
88
|
+
def list_jobs(
|
|
89
|
+
ctx: typer.Context,
|
|
90
|
+
url: Optional[str] = typer.Option(None, "--url", help="Filter by seed URL."),
|
|
91
|
+
status: Optional[str] = typer.Option(None, "--status", help="queued|processing|completed|failed|cancelled."),
|
|
92
|
+
start_date: Optional[str] = typer.Option(None, "--start-date", help="ISO 8601 lower bound."),
|
|
93
|
+
end_date: Optional[str] = typer.Option(None, "--end-date", help="ISO 8601 upper bound."),
|
|
94
|
+
page: int = typer.Option(1, "--page", help="Page number."),
|
|
95
|
+
page_size: int = typer.Option(100, "--page-size", help="Results per page (max 100)."),
|
|
96
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
97
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
98
|
+
) -> None:
|
|
99
|
+
"""List and filter crawl jobs."""
|
|
100
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
101
|
+
payload = validated(
|
|
102
|
+
ListCrawlJobsInput,
|
|
103
|
+
page=page,
|
|
104
|
+
page_size=page_size,
|
|
105
|
+
**optional(url=url, status=status, start_date=start_date, end_date=end_date),
|
|
106
|
+
)
|
|
107
|
+
call(ctx, "list_crawl_jobs", payload)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@crawl_app.command("cancel")
|
|
111
|
+
def cancel(
|
|
112
|
+
ctx: typer.Context,
|
|
113
|
+
job_id: str = typer.Argument(..., help="Crawl job ID to cancel."),
|
|
114
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
115
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
116
|
+
) -> None:
|
|
117
|
+
"""Cancel a running crawl job."""
|
|
118
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
119
|
+
call(ctx, "cancel_crawl", {"job_id": job_id})
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""`gscraper extract` — extract content from a single URL."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import List, Optional
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
from geonode_scraper_tools_core.schemas import ExtractInput
|
|
10
|
+
|
|
11
|
+
from ..context import get_cli_context
|
|
12
|
+
from ..errors import ExitCode
|
|
13
|
+
from ..invoke import call, validated
|
|
14
|
+
from ..renderers import emit
|
|
15
|
+
from ._common import apply_output_override, optional, parse_headers
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def extract(
|
|
19
|
+
ctx: typer.Context,
|
|
20
|
+
url: str = typer.Argument(..., help="URL to extract content from."),
|
|
21
|
+
formats: Optional[List[str]] = typer.Option(
|
|
22
|
+
None, "--format", "-f", help="Output format(s): markdown, html. Repeatable."
|
|
23
|
+
),
|
|
24
|
+
render_js: bool = typer.Option(False, "--render-js", help="Render JavaScript with a headless browser."),
|
|
25
|
+
is_async: bool = typer.Option(False, "--async", "-a", help="Submit as an async job instead of waiting inline."),
|
|
26
|
+
proxy_country: Optional[str] = typer.Option(None, "--proxy-country", help="ISO 3166-1 alpha-2 country code."),
|
|
27
|
+
proxy_type: Optional[str] = typer.Option(None, "--proxy-type", help="Proxy type: datacenter, residential, mix."),
|
|
28
|
+
header: Optional[List[str]] = typer.Option(None, "--header", "-H", help="Request header 'Key: Value'. Repeatable."),
|
|
29
|
+
wait_for: Optional[str] = typer.Option(
|
|
30
|
+
None, "--wait-for", help="CSS/XPath selector to wait for (browser only, requires --render-js)."
|
|
31
|
+
),
|
|
32
|
+
wait_timeout: Optional[int] = typer.Option(
|
|
33
|
+
None, "--wait-timeout", help="Pause in ms after wait_for (0-30000, requires --render-js)."
|
|
34
|
+
),
|
|
35
|
+
wait_until: Optional[str] = typer.Option(
|
|
36
|
+
None, "--wait-until", help="Lifecycle event: load, networkidle, ... (requires --render-js)."
|
|
37
|
+
),
|
|
38
|
+
output_file: Optional[Path] = typer.Option(
|
|
39
|
+
None, "--output", "-o", help="Write the extracted content (markdown/html) to this file."
|
|
40
|
+
),
|
|
41
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
42
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
43
|
+
) -> None:
|
|
44
|
+
"""Extract content from URL. Sync by default; use --async to queue a job."""
|
|
45
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
46
|
+
|
|
47
|
+
if (wait_for is not None or wait_timeout is not None or wait_until is not None) and not render_js:
|
|
48
|
+
typer.echo("--wait-for, --wait-until, and --wait-timeout require --render-js.", err=True)
|
|
49
|
+
raise typer.Exit(int(ExitCode.USAGE))
|
|
50
|
+
|
|
51
|
+
payload = validated(
|
|
52
|
+
ExtractInput,
|
|
53
|
+
url=url,
|
|
54
|
+
render_js=render_js,
|
|
55
|
+
processing_mode="async" if is_async else "sync",
|
|
56
|
+
**optional(
|
|
57
|
+
formats=formats,
|
|
58
|
+
proxy_country=proxy_country,
|
|
59
|
+
proxy_type=proxy_type,
|
|
60
|
+
headers=parse_headers(header),
|
|
61
|
+
wait_for=wait_for,
|
|
62
|
+
wait_timeout=wait_timeout,
|
|
63
|
+
wait_until=wait_until,
|
|
64
|
+
),
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
if output_file is None:
|
|
68
|
+
call(ctx, "extract", payload)
|
|
69
|
+
|
|
70
|
+
# --output: call once, write the content body, then emit the same envelope.
|
|
71
|
+
cli = get_cli_context(ctx)
|
|
72
|
+
result = cli.service().extract(**payload)
|
|
73
|
+
if result.get("ok"):
|
|
74
|
+
data = (result.get("result") or {}).get("data") or {}
|
|
75
|
+
content = data.get("markdown") or data.get("html") or ""
|
|
76
|
+
output_file.write_text(content, encoding="utf-8")
|
|
77
|
+
typer.echo(f"Wrote {len(content)} chars to {output_file}", err=True)
|
|
78
|
+
emit(cli.output, result)
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""`gscraper jobs` — inspect and wait on extraction jobs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
from geonode_scraper_tools_core.schemas import GetJobResultInput, ListJobsInput, WaitForJobInput
|
|
9
|
+
|
|
10
|
+
from ..invoke import call, validated
|
|
11
|
+
from ._common import apply_output_override, optional
|
|
12
|
+
|
|
13
|
+
jobs_app = typer.Typer(help="Inspect and wait on extraction jobs.", no_args_is_help=True)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@jobs_app.command("get")
|
|
17
|
+
def get(
|
|
18
|
+
ctx: typer.Context,
|
|
19
|
+
job_id: str = typer.Argument(..., help="Extraction job ID."),
|
|
20
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
21
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
22
|
+
) -> None:
|
|
23
|
+
"""Fetch the current state or final result of an extraction job."""
|
|
24
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
25
|
+
payload = validated(GetJobResultInput, job_id=job_id)
|
|
26
|
+
call(ctx, "get_job_result", payload)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@jobs_app.command("list")
|
|
30
|
+
def list_jobs(
|
|
31
|
+
ctx: typer.Context,
|
|
32
|
+
job_id: Optional[str] = typer.Option(None, "--job-id", help="Filter by job ID."),
|
|
33
|
+
url: Optional[str] = typer.Option(None, "--url", help="Filter by target URL."),
|
|
34
|
+
status: Optional[str] = typer.Option(None, "--status", help="queued|processing|completed|failed|cancelled."),
|
|
35
|
+
output: Optional[str] = typer.Option(None, "--output-format", help="Filter by requested output: markdown|html."),
|
|
36
|
+
start_date: Optional[str] = typer.Option(None, "--start-date", help="ISO 8601 lower bound."),
|
|
37
|
+
end_date: Optional[str] = typer.Option(None, "--end-date", help="ISO 8601 upper bound."),
|
|
38
|
+
page: int = typer.Option(1, "--page", help="Page number."),
|
|
39
|
+
page_size: int = typer.Option(100, "--page-size", help="Results per page (max 100)."),
|
|
40
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
41
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
42
|
+
) -> None:
|
|
43
|
+
"""List and filter extraction jobs."""
|
|
44
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
45
|
+
payload = validated(
|
|
46
|
+
ListJobsInput,
|
|
47
|
+
page=page,
|
|
48
|
+
page_size=page_size,
|
|
49
|
+
**optional(
|
|
50
|
+
job_id=job_id,
|
|
51
|
+
url=url,
|
|
52
|
+
status=status,
|
|
53
|
+
output=output,
|
|
54
|
+
start_date=start_date,
|
|
55
|
+
end_date=end_date,
|
|
56
|
+
),
|
|
57
|
+
)
|
|
58
|
+
call(ctx, "list_jobs", payload)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@jobs_app.command("wait")
|
|
62
|
+
def wait(
|
|
63
|
+
ctx: typer.Context,
|
|
64
|
+
job_id: str = typer.Argument(..., help="Extraction job ID."),
|
|
65
|
+
timeout_seconds: Optional[float] = typer.Option(None, "--timeout", help="Polling timeout override (seconds)."),
|
|
66
|
+
poll_interval_seconds: Optional[float] = typer.Option(None, "--interval", help="Polling interval override (s)."),
|
|
67
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
68
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
69
|
+
) -> None:
|
|
70
|
+
"""Poll an extraction job until it reaches a terminal state."""
|
|
71
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
72
|
+
payload = validated(
|
|
73
|
+
WaitForJobInput,
|
|
74
|
+
job_id=job_id,
|
|
75
|
+
**optional(timeout_seconds=timeout_seconds, poll_interval_seconds=poll_interval_seconds),
|
|
76
|
+
)
|
|
77
|
+
call(ctx, "wait_for_job", payload)
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""`gscraper map` — discover URLs under a base URL."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
from geonode_scraper_tools_core.schemas import GetMapJobInput, ListMapJobsInput, MapUrlsInput
|
|
9
|
+
|
|
10
|
+
from ..invoke import call, validated
|
|
11
|
+
from ._common import apply_output_override, optional
|
|
12
|
+
|
|
13
|
+
map_app = typer.Typer(help="Discover URLs under a base URL.", no_args_is_help=True)
|
|
14
|
+
map_jobs_app = typer.Typer(help="Inspect map jobs.", no_args_is_help=True)
|
|
15
|
+
map_app.add_typer(map_jobs_app, name="jobs")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@map_app.command("run")
|
|
19
|
+
def run(
|
|
20
|
+
ctx: typer.Context,
|
|
21
|
+
url: str = typer.Argument(..., help="Base URL to discover links from."),
|
|
22
|
+
search: Optional[str] = typer.Option(None, "--search", help="Filter discovered links by substring."),
|
|
23
|
+
include_subdomains: bool = typer.Option(True, "--include-subdomains/--no-subdomains", help="Include subdomains."),
|
|
24
|
+
ignore_query_parameters: bool = typer.Option(
|
|
25
|
+
True, "--ignore-query/--keep-query", help="Ignore query parameters when de-duplicating links."
|
|
26
|
+
),
|
|
27
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
28
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
29
|
+
) -> None:
|
|
30
|
+
"""Discover all URLs under a base URL (returns synchronously)."""
|
|
31
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
32
|
+
payload = validated(
|
|
33
|
+
MapUrlsInput,
|
|
34
|
+
url=url,
|
|
35
|
+
include_subdomains=include_subdomains,
|
|
36
|
+
ignore_query_parameters=ignore_query_parameters,
|
|
37
|
+
**optional(search=search),
|
|
38
|
+
)
|
|
39
|
+
call(ctx, "map_urls", payload)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@map_jobs_app.command("list")
|
|
43
|
+
def list_jobs(
|
|
44
|
+
ctx: typer.Context,
|
|
45
|
+
url: Optional[str] = typer.Option(None, "--url", help="Filter by base URL."),
|
|
46
|
+
status: Optional[str] = typer.Option(None, "--status", help="queued|processing|completed|failed|cancelled."),
|
|
47
|
+
start_date: Optional[str] = typer.Option(None, "--start-date", help="ISO 8601 lower bound."),
|
|
48
|
+
end_date: Optional[str] = typer.Option(None, "--end-date", help="ISO 8601 upper bound."),
|
|
49
|
+
page: int = typer.Option(1, "--page", help="Page number."),
|
|
50
|
+
page_size: int = typer.Option(100, "--page-size", help="Results per page (max 100)."),
|
|
51
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
52
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
53
|
+
) -> None:
|
|
54
|
+
"""List and filter map jobs."""
|
|
55
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
56
|
+
payload = validated(
|
|
57
|
+
ListMapJobsInput,
|
|
58
|
+
page=page,
|
|
59
|
+
page_size=page_size,
|
|
60
|
+
**optional(url=url, status=status, start_date=start_date, end_date=end_date),
|
|
61
|
+
)
|
|
62
|
+
call(ctx, "list_map_jobs", payload)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@map_jobs_app.command("get")
|
|
66
|
+
def get(
|
|
67
|
+
ctx: typer.Context,
|
|
68
|
+
job_id: str = typer.Argument(..., help="Map job ID."),
|
|
69
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
70
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
71
|
+
) -> None:
|
|
72
|
+
"""Fetch a single map job by ID."""
|
|
73
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
74
|
+
payload = validated(GetMapJobInput, job_id=job_id)
|
|
75
|
+
call(ctx, "get_map_job", payload)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""`gscraper stats` — usage statistics."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
from geonode_scraper_tools_core.schemas import GetStatisticsInput
|
|
9
|
+
|
|
10
|
+
from ..invoke import call, validated
|
|
11
|
+
from ._common import apply_output_override, optional
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def stats(
|
|
15
|
+
ctx: typer.Context,
|
|
16
|
+
start_date: Optional[str] = typer.Option(None, "--start-date", help="ISO 8601 lower bound."),
|
|
17
|
+
end_date: Optional[str] = typer.Option(None, "--end-date", help="ISO 8601 upper bound."),
|
|
18
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
19
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
20
|
+
) -> None:
|
|
21
|
+
"""Show usage statistics for an optional date range."""
|
|
22
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
23
|
+
payload = validated(GetStatisticsInput, **optional(start_date=start_date, end_date=end_date))
|
|
24
|
+
call(ctx, "get_statistics", payload)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""`gscraper health` — API health check."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import typer
|
|
6
|
+
|
|
7
|
+
from ..invoke import call
|
|
8
|
+
from ._common import apply_output_override
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def health(
|
|
12
|
+
ctx: typer.Context,
|
|
13
|
+
as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
|
|
14
|
+
as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
|
|
15
|
+
) -> None:
|
|
16
|
+
"""Check the Scraper API health endpoint."""
|
|
17
|
+
apply_output_override(ctx, as_json, as_yaml)
|
|
18
|
+
call(ctx, "health_check", {})
|