geonode-scraper-cli 0.1.0__tar.gz → 0.2.1__tar.gz

{geonode_scraper_cli-0.1.0 → geonode_scraper_cli-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: geonode-scraper-cli
-Version: 0.1.0
+Version: 0.2.1
 Summary: Command-line interface for the Geonode Scraper API
 Author: Geonode Team
 License-Expression: MIT
@@ -10,7 +10,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
-Requires-Dist: geonode-scraper-tools-core>=0.3.1
+Requires-Dist: geonode-scraper-tools-core>=0.5.0
 Requires-Dist: typer>=0.12
 Requires-Dist: rich>=13.0
 Requires-Dist: PyYAML>=6.0
@@ -129,11 +129,16 @@ gscraper crawl wait JOB_ID
 gscraper crawl list [--url ...]
 gscraper crawl cancel JOB_ID
 
-gscraper map run URL [--search term] [--no-subdomains]   # primary action
-gscraper map jobs list                                   # inspect past map jobs
+gscraper map run URL [--search term] [--include-subdomains]   # primary action
+gscraper map jobs list                                        # inspect past map jobs
 gscraper map jobs get JOB_ID
 
+gscraper search run QUERY [--locale en] [--page 1] [--safe off] [--time-range week]
+gscraper search jobs list [--query ...] [--status completed]
+gscraper search jobs get JOB_ID
+
 gscraper stats [--start-date ISO] [--end-date ISO]
+gscraper usage concurrency
 gscraper health
 ```
 

{geonode_scraper_cli-0.1.0 → geonode_scraper_cli-0.2.1}/README.md

@@ -107,11 +107,16 @@ gscraper crawl wait JOB_ID
 gscraper crawl list [--url ...]
 gscraper crawl cancel JOB_ID
 
-gscraper map run URL [--search term] [--no-subdomains]   # primary action
-gscraper map jobs list                                   # inspect past map jobs
+gscraper map run URL [--search term] [--include-subdomains]   # primary action
+gscraper map jobs list                                        # inspect past map jobs
 gscraper map jobs get JOB_ID
 
+gscraper search run QUERY [--locale en] [--page 1] [--safe off] [--time-range week]
+gscraper search jobs list [--query ...] [--status completed]
+gscraper search jobs get JOB_ID
+
 gscraper stats [--start-date ISO] [--end-date ISO]
+gscraper usage concurrency
 gscraper health
 ```
 

{geonode_scraper_cli-0.1.0 → geonode_scraper_cli-0.2.1}/geonode_scraper_cli/__init__.py

@@ -1,3 +1,3 @@
-__version__ = "0.1.0"
+__version__ = "0.2.1"
 
 __all__ = ["__version__"]

{geonode_scraper_cli-0.1.0 → geonode_scraper_cli-0.2.1}/geonode_scraper_cli/commands/map.py

@@ -20,7 +20,7 @@ def run(
     ctx: typer.Context,
     url: str = typer.Argument(..., help="Base URL to discover links from."),
     search: Optional[str] = typer.Option(None, "--search", help="Filter discovered links by substring."),
-    include_subdomains: bool = typer.Option(True, "--include-subdomains/--no-subdomains", help="Include subdomains."),
+    include_subdomains: bool = typer.Option(False, "--include-subdomains/--no-subdomains", help="Include subdomains."),
     ignore_query_parameters: bool = typer.Option(
         True, "--ignore-query/--keep-query", help="Ignore query parameters when de-duplicating links."
     ),

geonode_scraper_cli-0.2.1/geonode_scraper_cli/commands/search.py

@@ -0,0 +1,74 @@
+"""`gscraper search` — run a search query and inspect search jobs."""
+
+from __future__ import annotations
+
+from typing import Optional
+
+import typer
+from geonode_scraper_tools_core.schemas import GetSearchJobInput, ListSearchJobsInput, SearchInput
+
+from ..invoke import call, validated
+from ._common import apply_output_override, optional
+
+search_app = typer.Typer(help="Run search queries and inspect search jobs.", no_args_is_help=True)
+search_jobs_app = typer.Typer(help="Inspect search jobs.", no_args_is_help=True)
+search_app.add_typer(search_jobs_app, name="jobs")
+
+
+@search_app.command("run")
+def run(
+    ctx: typer.Context,
+    query: str = typer.Argument(..., help="Search query."),
+    locale: Optional[str] = typer.Option(None, "--locale", help="Locale tag, e.g. 'en' or 'en-US'."),
+    page: int = typer.Option(1, "--page", help="Result page to fetch (1-20)."),
+    safe: str = typer.Option("off", "--safe", help="Safe-search level: off|moderate|strict."),
+    time_range: Optional[str] = typer.Option(None, "--time-range", help="day|week|month|year."),
+    as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
+    as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
+) -> None:
+    """Run a search query (returns synchronously)."""
+    apply_output_override(ctx, as_json, as_yaml)
+    payload = validated(
+        SearchInput,
+        query=query,
+        page=page,
+        safe=safe,
+        **optional(locale=locale, time_range=time_range),
+    )
+    call(ctx, "search", payload)
+
+
+@search_jobs_app.command("list")
+def list_jobs(
+    ctx: typer.Context,
+    query: Optional[str] = typer.Option(None, "--query", help="Filter by search query."),
+    status: Optional[str] = typer.Option(None, "--status", help="queued|processing|completed|failed|cancelled."),
+    start_date: Optional[str] = typer.Option(None, "--start-date", help="ISO 8601 lower bound."),
+    end_date: Optional[str] = typer.Option(None, "--end-date", help="ISO 8601 upper bound."),
+    page: int = typer.Option(1, "--page", help="Page number."),
+    page_size: int = typer.Option(100, "--page-size", help="Results per page (max 100)."),
+    as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
+    as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
+) -> None:
+    """List and filter search jobs."""
+    apply_output_override(ctx, as_json, as_yaml)
+    payload = validated(
+        ListSearchJobsInput,
+        page=page,
+        page_size=page_size,
+        **optional(query=query, status=status, start_date=start_date, end_date=end_date),
+    )
+    call(ctx, "list_search_jobs", payload)
+
+
+@search_jobs_app.command("get")
+def get(
+    ctx: typer.Context,
+    job_id: str = typer.Argument(..., help="Search job ID."),
+    as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
+    as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
+) -> None:
+    """Fetch a single search job by ID."""
+    apply_output_override(ctx, as_json, as_yaml)
+    payload = validated(GetSearchJobInput, job_id=job_id)
+    call(ctx, "get_search_job", payload)

geonode_scraper_cli-0.2.1/geonode_scraper_cli/commands/usage.py

@@ -0,0 +1,21 @@
+"""`gscraper usage` — plan usage and limits."""
+
+from __future__ import annotations
+
+import typer
+
+from ..invoke import call
+from ._common import apply_output_override
+
+usage_app = typer.Typer(help="Usage and limits.", no_args_is_help=True)
+
+
+@usage_app.command("concurrency")
+def concurrency(
+    ctx: typer.Context,
+    as_json: bool = typer.Option(False, "--json", help="Output raw JSON envelope."),
+    as_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML envelope."),
+) -> None:
+    """Check the caller's live work-concurrency slot usage against their plan limit."""
+    apply_output_override(ctx, as_json, as_yaml)
+    call(ctx, "get_concurrency_usage", {})

{geonode_scraper_cli-0.1.0 → geonode_scraper_cli-0.2.1}/geonode_scraper_cli/config.py

@@ -15,7 +15,13 @@ from geonode_scraper_tools_core import ScraperToolSettings
 
 ENV_PREFIX = "GEONODE_SCRAPER_"
 DEFAULT_PROFILE = "default"
-DEFAULT_HOST = "http://localhost"
+# The production API. It used to be `http://localhost`, which is the OpenAPI
+# generator's fallback for a spec with no `servers` block - harmless in a generated
+# client that documents it, and wrong in a CLI a person installs and runs, where it
+# means every command fails to connect until a host is configured. A flag, the
+# environment or the config file still override it; this is only what happens when
+# nobody said anything.
+DEFAULT_HOST = "https://scraper.geonode.io"
 
 
 class ConfigError(Exception):

{geonode_scraper_cli-0.1.0 → geonode_scraper_cli-0.2.1}/geonode_scraper_cli/context.py

@@ -7,6 +7,7 @@ from dataclasses import dataclass
 import typer
 from geonode_scraper_tools_core import ScraperToolService, ScraperToolSettings
 
+from . import __version__
 from .config import ConfigError, resolve_settings
 from .errors import ExitCode
 from .output import OutputFormat
@@ -49,7 +50,8 @@ class CLIContext:
 
     def service(self) -> ScraperToolService:
         if self._service is None:
-            self._service = ScraperToolService(self.settings())
+            # The API labels requests from this package `cli` (scraper-api#674).
+            self._service = ScraperToolService(self.settings(), client=f"geonode-cli/{__version__}")
         return self._service
 
 

{geonode_scraper_cli-0.1.0 → geonode_scraper_cli-0.2.1}/geonode_scraper_cli/main.py

@@ -13,8 +13,10 @@ from .commands.crawl import crawl_app
 from .commands.extract import extract
 from .commands.jobs import jobs_app
 from .commands.map import map_app
+from .commands.search import search_app
 from .commands.stats import stats
 from .commands.system import health
+from .commands.usage import usage_app
 from .context import CLIContext
 from .output import OutputFormat
 
@@ -76,6 +78,8 @@ app.add_typer(jobs_app, name="jobs")
 app.add_typer(batch_app, name="batch")
 app.add_typer(crawl_app, name="crawl")
 app.add_typer(map_app, name="map")
+app.add_typer(search_app, name="search")
+app.add_typer(usage_app, name="usage")
 app.add_typer(config_app, name="config")
 
 

{geonode_scraper_cli-0.1.0 → geonode_scraper_cli-0.2.1}/geonode_scraper_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: geonode-scraper-cli
-Version: 0.1.0
+Version: 0.2.1
 Summary: Command-line interface for the Geonode Scraper API
 Author: Geonode Team
 License-Expression: MIT
@@ -10,7 +10,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
-Requires-Dist: geonode-scraper-tools-core>=0.3.1
+Requires-Dist: geonode-scraper-tools-core>=0.5.0
 Requires-Dist: typer>=0.12
 Requires-Dist: rich>=13.0
 Requires-Dist: PyYAML>=6.0
@@ -129,11 +129,16 @@ gscraper crawl wait JOB_ID
 gscraper crawl list [--url ...]
 gscraper crawl cancel JOB_ID
 
-gscraper map run URL [--search term] [--no-subdomains]   # primary action
-gscraper map jobs list                                   # inspect past map jobs
+gscraper map run URL [--search term] [--include-subdomains]   # primary action
+gscraper map jobs list                                        # inspect past map jobs
 gscraper map jobs get JOB_ID
 
+gscraper search run QUERY [--locale en] [--page 1] [--safe off] [--time-range week]
+gscraper search jobs list [--query ...] [--status completed]
+gscraper search jobs get JOB_ID
+
 gscraper stats [--start-date ISO] [--end-date ISO]
+gscraper usage concurrency
 gscraper health
 ```
 

{geonode_scraper_cli-0.1.0 → geonode_scraper_cli-0.2.1}/geonode_scraper_cli.egg-info/SOURCES.txt

@@ -22,8 +22,11 @@ geonode_scraper_cli/commands/crawl.py
 geonode_scraper_cli/commands/extract.py
 geonode_scraper_cli/commands/jobs.py
 geonode_scraper_cli/commands/map.py
+geonode_scraper_cli/commands/search.py
 geonode_scraper_cli/commands/stats.py
 geonode_scraper_cli/commands/system.py
+geonode_scraper_cli/commands/usage.py
 geonode_scraper_cli/renderers/__init__.py
 geonode_scraper_cli/renderers/tables.py
-tests/test_cli.py
+tests/test_cli.py
+tests/test_user_agent.py

{geonode_scraper_cli-0.1.0 → geonode_scraper_cli-0.2.1}/geonode_scraper_cli.egg-info/requires.txt

@@ -1,4 +1,4 @@
-geonode-scraper-tools-core>=0.3.1
+geonode-scraper-tools-core>=0.5.0
 typer>=0.12
 rich>=13.0
 PyYAML>=6.0

{geonode_scraper_cli-0.1.0 → geonode_scraper_cli-0.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "geonode-scraper-cli"
-version = "0.1.0"
+version = "0.2.1"
 description = "Command-line interface for the Geonode Scraper API"
 authors = [
   {name = "Geonode Team"},
@@ -15,7 +15,7 @@ classifiers = [
   "Typing :: Typed",
 ]
 dependencies = [
-  "geonode-scraper-tools-core>=0.3.1",
+  "geonode-scraper-tools-core>=0.5.0",
   "typer>=0.12",
   "rich>=13.0",
   "PyYAML>=6.0",

{geonode_scraper_cli-0.1.0 → geonode_scraper_cli-0.2.1}/tests/test_cli.py

@@ -97,7 +97,7 @@ def test_missing_api_key_is_usage_error(invoke, fake_service):
 def test_flag_overrides_env_host(invoke, fake_service, monkeypatch):
     captured: dict[str, object] = {}
 
-    def _capture(settings):
+    def _capture(settings, **_):
         captured["host"] = settings.host
         return fake_service
 
@@ -131,6 +131,55 @@ def test_map_run_renders_links(invoke, fake_service):
     assert "https://a.com" in result.stdout
 
 
+def test_map_run_defaults_include_subdomains_false(invoke, fake_service):
+    invoke(["map", "run", "https://example.com"])
+    _, kwargs = fake_service.calls[-1]
+    assert kwargs["include_subdomains"] is False
+
+
+def test_search_run_passes_validated_payload(invoke, fake_service):
+    fake_service.responses["search"] = {
+        "ok": True,
+        "operation": "search",
+        "result": {"job_id": "00000000-0000-0000-0000-000000000007", "results": []},
+    }
+    result = invoke(["search", "run", "geonode scraper api", "--page", "2", "--safe", "moderate"])
+    assert result.exit_code == 0
+    name, kwargs = fake_service.calls[-1]
+    assert name == "search"
+    assert kwargs["query"] == "geonode scraper api"
+    assert kwargs["page"] == 2
+    assert kwargs["safe"] == "moderate"
+
+
+def test_search_jobs_list_calls_service(invoke, fake_service):
+    invoke(["search", "jobs", "list", "--query", "geonode", "--status", "completed"])
+    name, kwargs = fake_service.calls[-1]
+    assert name == "list_search_jobs"
+    assert kwargs["query"] == "geonode"
+    assert kwargs["status"] == "completed"
+
+
+def test_search_jobs_get_calls_service(invoke, fake_service):
+    invoke(["search", "jobs", "get", "00000000-0000-0000-0000-000000000008"])
+    name, kwargs = fake_service.calls[-1]
+    assert name == "get_search_job"
+    assert kwargs == {"job_id": "00000000-0000-0000-0000-000000000008"}
+
+
+def test_usage_concurrency_calls_service(invoke, fake_service):
+    fake_service.responses["get_concurrency_usage"] = {
+        "ok": True,
+        "operation": "get_concurrency_usage",
+        "result": {"work_concurrency_in_use": 2, "work_concurrency_limit": 10},
+    }
+    result = invoke(["usage", "concurrency"])
+    assert result.exit_code == 0
+    name, kwargs = fake_service.calls[-1]
+    assert name == "get_concurrency_usage"
+    assert kwargs == {}
+
+
 def test_config_path(invoke):
     result = invoke(["config", "path"], with_key=False)
     assert result.exit_code == 0

geonode_scraper_cli-0.2.1/tests/test_user_agent.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+import json
+from unittest.mock import MagicMock, patch
+
+import urllib3
+
+HEALTH = {"version": "0.1.0", "service": "scraper-api", "timestamp": "2026-01-01T00:00:00Z", "status": "ok"}
+
+
+def test_requests_name_the_cli_first(runner, tmp_path):
+    # The API labels a request by the first User-Agent token (scraper-api#674).
+    from geonode_scraper_cli import __version__
+    from geonode_scraper_cli.main import app
+
+    response = MagicMock(status=200, reason="OK", data=json.dumps(HEALTH).encode())
+    response.headers = {"content-type": "application/json"}
+    env = {
+        "GEONODE_SCRAPER_CONFIG": str(tmp_path / "config.toml"),
+        "GEONODE_SCRAPER_API_KEY": "test-key",
+        "GEONODE_SCRAPER_HOST": "http://api.mock.local",
+    }
+
+    with patch.object(urllib3.PoolManager, "request", return_value=response) as request:
+        result = runner.invoke(app, ["--json", "health"], env=env)
+
+    assert result.exit_code == 0, result.output
+    assert request.call_args.kwargs["headers"]["User-Agent"].split()[0] == f"geonode-cli/{__version__}"