geonode-scraper-tools-core 0.1.0__tar.gz

geonode_scraper_tools_core-0.1.0/MANIFEST.in

@@ -0,0 +1,2 @@
+include README.md
+recursive-include geonode_scraper_tools_core py.typed

geonode_scraper_tools_core-0.1.0/PKG-INFO

@@ -0,0 +1,56 @@
+Metadata-Version: 2.4
+Name: geonode-scraper-tools-core
+Version: 0.1.0
+Summary: Shared runtime and schemas for Geonode Scraper framework tools
+Author: Geonode Team
+License-Expression: MIT
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: geonode-scraper-sdk>=0.1.0
+Requires-Dist: pydantic>=2.11
+Requires-Dist: typing-extensions>=4.7.1
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0; extra == "dev"
+Requires-Dist: ruff>=0.12.11; extra == "dev"
+
+# Geonode Scraper Tools Core
+
+Shared runtime, schemas, and operation registry for Geonode Scraper tool
+integrations.
+
+Most users should install one of the framework packages instead:
+
+- `geonode-scraper-langchain`
+- `geonode-scraper-crewai`
+
+Install the core package directly only if you are building your own wrapper layer
+on top of the shared service.
+
+## Installation
+
+```sh
+pip install geonode-scraper-tools-core
+```
+
+## Public API
+
+- `ScraperToolSettings`
+- `ScraperToolService`
+- `OperationSpec`
+- `OPERATIONS`
+- `get_operations()`
+
+The shared service normalizes SDK responses into JSON-friendly dictionaries and
+exposes the following operations:
+
+- `extract`
+- `get_job_result`
+- `wait_for_job`
+- `list_jobs`
+- `get_statistics`
+- `health_check`

geonode_scraper_tools_core-0.1.0/README.md

@@ -0,0 +1,36 @@
+# Geonode Scraper Tools Core
+
+Shared runtime, schemas, and operation registry for Geonode Scraper tool
+integrations.
+
+Most users should install one of the framework packages instead:
+
+- `geonode-scraper-langchain`
+- `geonode-scraper-crewai`
+
+Install the core package directly only if you are building your own wrapper layer
+on top of the shared service.
+
+## Installation
+
+```sh
+pip install geonode-scraper-tools-core
+```
+
+## Public API
+
+- `ScraperToolSettings`
+- `ScraperToolService`
+- `OperationSpec`
+- `OPERATIONS`
+- `get_operations()`
+
+The shared service normalizes SDK responses into JSON-friendly dictionaries and
+exposes the following operations:
+
+- `extract`
+- `get_job_result`
+- `wait_for_job`
+- `list_jobs`
+- `get_statistics`
+- `health_check`

geonode_scraper_tools_core-0.1.0/geonode_scraper_tools_core/__init__.py

@@ -0,0 +1,26 @@
+from .registry import OPERATIONS, OperationSpec, get_operations
+from .schemas import (
+    ExtractInput,
+    GetJobResultInput,
+    GetStatisticsInput,
+    HealthCheckInput,
+    ListJobsInput,
+    WaitForJobInput,
+)
+from .service import ScraperToolService, ScraperToolSettings
+
+__all__ = [
+    "OPERATIONS",
+    "ExtractInput",
+    "GetJobResultInput",
+    "GetStatisticsInput",
+    "HealthCheckInput",
+    "ListJobsInput",
+    "OperationSpec",
+    "ScraperToolService",
+    "ScraperToolSettings",
+    "WaitForJobInput",
+    "get_operations",
+]
+
+__version__ = "0.1.0"

geonode_scraper_tools_core-0.1.0/geonode_scraper_tools_core/registry.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Sequence
+
+from pydantic import BaseModel
+
+from .schemas import (
+    ExtractInput,
+    GetJobResultInput,
+    GetStatisticsInput,
+    HealthCheckInput,
+    ListJobsInput,
+    WaitForJobInput,
+)
+
+if TYPE_CHECKING:
+    from .service import ScraperToolService
+
+
+@dataclass(frozen=True)
+class OperationSpec:
+    key: str
+    tool_name: str
+    description: str
+    args_schema: type[BaseModel]
+    service_method: str
+
+    def invoke(self, service: ScraperToolService, **kwargs: Any) -> dict[str, Any]:
+        return getattr(service, self.service_method)(**kwargs)
+
+
+OPERATIONS: tuple[OperationSpec, ...] = (
+    OperationSpec(
+        key="extract",
+        tool_name="scraper_extract_content",
+        description="Extract structured content from a URL.",
+        args_schema=ExtractInput,
+        service_method="extract",
+    ),
+    OperationSpec(
+        key="get_job_result",
+        tool_name="scraper_get_job_result",
+        description="Fetch the current state or final result for an async extraction job.",
+        args_schema=GetJobResultInput,
+        service_method="get_job_result",
+    ),
+    OperationSpec(
+        key="wait_for_job",
+        tool_name="scraper_wait_for_job",
+        description="Poll an async extraction job until it reaches a terminal state or a timeout expires.",
+        args_schema=WaitForJobInput,
+        service_method="wait_for_job",
+    ),
+    OperationSpec(
+        key="list_jobs",
+        tool_name="scraper_list_jobs",
+        description="List previously submitted extraction jobs with optional filters.",
+        args_schema=ListJobsInput,
+        service_method="list_jobs",
+    ),
+    OperationSpec(
+        key="get_statistics",
+        tool_name="scraper_get_statistics",
+        description="Retrieve aggregated extraction statistics for an optional date range.",
+        args_schema=GetStatisticsInput,
+        service_method="get_statistics",
+    ),
+    OperationSpec(
+        key="health_check",
+        tool_name="scraper_check_health",
+        description="Check the scraper service health and version metadata.",
+        args_schema=HealthCheckInput,
+        service_method="health_check",
+    ),
+)
+
+_OPERATIONS_BY_KEY = {operation.key: operation for operation in OPERATIONS}
+
+
+def get_operations(keys: Sequence[str] | None = None) -> tuple[OperationSpec, ...]:
+    if keys is None:
+        return OPERATIONS
+
+    selected: list[OperationSpec] = []
+    missing: list[str] = []
+    for key in keys:
+        operation = _OPERATIONS_BY_KEY.get(key)
+        if operation is None:
+            missing.append(key)
+            continue
+        selected.append(operation)
+
+    if missing:
+        available = ", ".join(sorted(_OPERATIONS_BY_KEY))
+        missing_text = ", ".join(sorted(missing))
+        raise ValueError(f"Unknown operations: {missing_text}. Available operations: {available}")
+
+    return tuple(selected)

geonode_scraper_tools_core-0.1.0/geonode_scraper_tools_core/schemas.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ToolInputModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class HealthCheckInput(ToolInputModel):
+    pass
+
+
+class ExtractInput(ToolInputModel):
+    url: str = Field(
+        min_length=1,
+        max_length=2083,
+        description="URL to extract content from.",
+    )
+    formats: list[Literal["markdown", "html"]] = Field(
+        default_factory=lambda: ["html"],
+        description="Output formats to return.",
+    )
+    render_js: bool = Field(
+        default=False,
+        description="If true, uses a headless browser to render JavaScript before extraction.",
+    )
+    processing_mode: Literal["sync", "async"] = Field(
+        default="sync",
+        description="sync returns the extraction inline; async starts a job and returns a job ID.",
+    )
+    proxy_country: str | None = Field(
+        default=None,
+        pattern="^[A-Z]{2}$",
+        description="Optional ISO 3166-1 alpha-2 country code for proxy geo-targeting.",
+    )
+    proxy_type: Literal["datacenter", "residential", "mix"] | None = Field(
+        default=None,
+        description="Optional proxy type.",
+    )
+    headers: dict[str, str] | None = Field(
+        default=None,
+        description="Optional HTTP headers forwarded to the target URL.",
+    )
+
+
+class GetJobResultInput(ToolInputModel):
+    job_id: str = Field(description="Extraction job ID returned by an async extract request.")
+
+
+class WaitForJobInput(ToolInputModel):
+    job_id: str = Field(description="Extraction job ID returned by an async extract request.")
+    timeout_seconds: float | None = Field(
+        default=None,
+        gt=0,
+        description="Override the polling timeout in seconds.",
+    )
+    poll_interval_seconds: float | None = Field(
+        default=None,
+        gt=0,
+        description="Override the polling interval in seconds.",
+    )
+
+
+class ListJobsInput(ToolInputModel):
+    job_id: str | None = Field(default=None, description="Filter by job ID.")
+    url: str | None = Field(default=None, description="Filter by target URL.")
+    status: Literal["queued", "processing", "completed", "failed", "cancelled"] | None = Field(
+        default=None,
+        description="Filter by job status.",
+    )
+    output: Literal["markdown", "html"] | None = Field(
+        default=None,
+        description="Filter by requested output format.",
+    )
+    start_date: datetime | None = Field(
+        default=None,
+        description="Filter jobs created on or after this ISO 8601 datetime.",
+    )
+    end_date: datetime | None = Field(
+        default=None,
+        description="Filter jobs created on or before this ISO 8601 datetime.",
+    )
+    page: int = Field(default=1, ge=1, description="Page number.")
+    page_size: int = Field(default=100, ge=1, le=100, description="Number of results per page.")
+
+
+class GetStatisticsInput(ToolInputModel):
+    start_date: datetime | None = Field(
+        default=None,
+        description="Filter by start date as an ISO 8601 datetime.",
+    )
+    end_date: datetime | None = Field(
+        default=None,
+        description="Filter by end date as an ISO 8601 datetime.",
+    )

geonode_scraper_tools_core-0.1.0/geonode_scraper_tools_core/service.py

@@ -0,0 +1,326 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import datetime
+from enum import Enum
+from time import monotonic, sleep
+from typing import Any, Callable, ContextManager, Iterator, Mapping, Sequence
+from uuid import UUID
+
+from geonode_scraper_sdk import (
+    ApiClient,
+    Configuration,
+    ExtractionApi,
+    ExtractRequest,
+    OutputFormat,
+    ProcessingMode,
+    ProxySettings,
+    ProxyType,
+    StatisticsApi,
+    SystemApi,
+)
+from geonode_scraper_sdk.api_client import ApiClient as GeneratedApiClient
+from geonode_scraper_sdk.exceptions import ApiException
+from pydantic import BaseModel
+
+RequestTimeout = float | tuple[float, float] | None
+ApiClientFactory = Callable[[], ContextManager[GeneratedApiClient]]
+
+
+@dataclass(frozen=True)
+class ScraperToolSettings:
+    host: str
+    api_key: str
+    verify_ssl: bool = True
+    request_timeout: RequestTimeout = None
+    max_retries: int = 0
+    retry_backoff_seconds: float = 1.0
+    poll_interval_seconds: float = 3.0
+    poll_timeout_seconds: float = 60.0
+
+
+class ScraperToolService:
+    def __init__(
+        self,
+        settings: ScraperToolSettings,
+        *,
+        api_client_factory: ApiClientFactory | None = None,
+        sleep_fn: Callable[[float], None] = sleep,
+    ) -> None:
+        self.settings = settings
+        self._api_client_factory = api_client_factory
+        self._sleep = sleep_fn
+
+    def extract(
+        self,
+        *,
+        url: str,
+        formats: Sequence[str] | None = None,
+        render_js: bool = False,
+        processing_mode: str = "sync",
+        proxy_country: str | None = None,
+        proxy_type: str | None = None,
+        headers: Mapping[str, str] | None = None,
+    ) -> dict[str, Any]:
+        request = ExtractRequest(
+            url=url,
+            formats=self._build_output_formats(formats),
+            render_js=render_js,
+            processing_mode=ProcessingMode(processing_mode),
+            proxy=self._build_proxy(proxy_country=proxy_country, proxy_type=proxy_type),
+            headers=dict(headers) if headers is not None else None,
+        )
+
+        return self._execute_api_call(
+            operation="extract",
+            api_cls=ExtractionApi,
+            method_name="extract_v1_extract_post",
+            extract_request=request,
+            _request_timeout=self.settings.request_timeout,
+        )
+
+    def get_job_result(self, *, job_id: str) -> dict[str, Any]:
+        return self._execute_api_call(
+            operation="get_job_result",
+            api_cls=ExtractionApi,
+            method_name="get_job_result_v1_extract_job_id_get",
+            job_id=job_id,
+            _request_timeout=self.settings.request_timeout,
+        )
+
+    def wait_for_job(
+        self,
+        *,
+        job_id: str,
+        timeout_seconds: float | None = None,
+        poll_interval_seconds: float | None = None,
+    ) -> dict[str, Any]:
+        timeout = timeout_seconds if timeout_seconds is not None else self.settings.poll_timeout_seconds
+        interval = poll_interval_seconds if poll_interval_seconds is not None else self.settings.poll_interval_seconds
+        deadline = monotonic() + timeout
+        attempts = 0
+
+        while True:
+            attempts += 1
+            result = self.get_job_result(job_id=job_id)
+            result = {
+                **result,
+                "operation": "wait_for_job",
+                "poll_attempts": attempts,
+            }
+
+            if not result["ok"]:
+                return result
+
+            status = result["result"]["status"]
+            if status in {"completed", "failed", "cancelled"}:
+                return result
+
+            if monotonic() >= deadline:
+                return {
+                    "ok": False,
+                    "operation": "wait_for_job",
+                    "poll_attempts": attempts,
+                    "error": {
+                        "status": None,
+                        "code": "POLL_TIMEOUT",
+                        "message": f"Job {job_id} did not reach a terminal state before the polling timeout expired.",
+                        "retryable": True,
+                        "data": {"job_id": job_id, "last_status": status},
+                    },
+                }
+
+            self._sleep(interval)
+
+    def list_jobs(
+        self,
+        *,
+        job_id: str | None = None,
+        url: str | None = None,
+        status: str | None = None,
+        output: str | None = None,
+        start_date: datetime | None = None,
+        end_date: datetime | None = None,
+        page: int = 1,
+        page_size: int = 100,
+    ) -> dict[str, Any]:
+        return self._execute_api_call(
+            operation="list_jobs",
+            api_cls=ExtractionApi,
+            method_name="list_jobs_v1_extract_jobs_get",
+            job_id=job_id,
+            url=url,
+            status=self._enum_or_none(status, enum_type=self._job_status_enum()),
+            output=self._enum_or_none(output, enum_type=OutputFormat),
+            start_date=start_date,
+            end_date=end_date,
+            page=page,
+            page_size=page_size,
+            _request_timeout=self.settings.request_timeout,
+        )
+
+    def get_statistics(
+        self,
+        *,
+        start_date: datetime | None = None,
+        end_date: datetime | None = None,
+    ) -> dict[str, Any]:
+        return self._execute_api_call(
+            operation="get_statistics",
+            api_cls=StatisticsApi,
+            method_name="get_statistics_v1_statistics_get",
+            start_date=start_date,
+            end_date=end_date,
+            _request_timeout=self.settings.request_timeout,
+        )
+
+    def health_check(self) -> dict[str, Any]:
+        return self._execute_api_call(
+            operation="health_check",
+            api_cls=SystemApi,
+            method_name="health_check_health_get",
+            _request_timeout=self.settings.request_timeout,
+        )
+
+    def _execute_api_call(
+        self,
+        *,
+        operation: str,
+        api_cls: type[Any],
+        method_name: str,
+        **call_kwargs: Any,
+    ) -> dict[str, Any]:
+        return self._execute(
+            operation,
+            self._invoke_api_method,
+            api_cls,
+            method_name,
+            call_kwargs,
+        )
+
+    def _execute(self, operation: str, callback: Callable[..., Any], *args: Any) -> dict[str, Any]:
+        attempts = 0
+        while True:
+            try:
+                result = callback(*args)
+                payload = {
+                    "ok": True,
+                    "operation": operation,
+                    "attempts": attempts + 1,
+                    "result": self._normalize_value(result),
+                }
+                response_type = self._extract_response_type(operation, result)
+                if response_type is not None:
+                    payload["response_type"] = response_type
+                return payload
+            except ApiException as exc:
+                if self._should_retry(exc=exc, attempts=attempts):
+                    attempts += 1
+                    self._sleep(self.settings.retry_backoff_seconds * attempts)
+                    continue
+
+                return {
+                    "ok": False,
+                    "operation": operation,
+                    "attempts": attempts + 1,
+                    "error": self._normalize_exception(exc),
+                }
+
+    def _should_retry(self, *, exc: ApiException, attempts: int) -> bool:
+        if attempts >= self.settings.max_retries:
+            return False
+
+        payload = exc.data.error if getattr(exc.data, "error", None) is not None else exc.data
+        retryable = getattr(payload, "retryable", None)
+        if retryable is not None:
+            return bool(retryable)
+
+        return exc.status in {429, 500, 503}
+
+    def _invoke_api_method(self, api_cls: type[Any], method_name: str, call_kwargs: Mapping[str, Any]) -> Any:
+        with self._api_client_context() as api_client:
+            api = api_cls(api_client)
+            api_method = getattr(api, method_name)
+            return api_method(**call_kwargs)
+
+    @contextmanager
+    def _api_client_context(self) -> Iterator[GeneratedApiClient]:
+        if self._api_client_factory is not None:
+            with self._api_client_factory() as api_client:
+                yield api_client
+            return
+
+        configuration = Configuration(host=self.settings.host)
+        configuration.api_key["APIKeyHeader"] = self.settings.api_key
+        configuration.verify_ssl = self.settings.verify_ssl
+        with ApiClient(configuration) as api_client:
+            yield api_client
+
+    @staticmethod
+    def _build_output_formats(formats: Sequence[str] | None) -> list[OutputFormat] | None:
+        if formats is None:
+            return None
+        return [OutputFormat(format_name) for format_name in formats]
+
+    @staticmethod
+    def _build_proxy(*, proxy_country: str | None, proxy_type: str | None) -> ProxySettings | None:
+        if proxy_country is None and proxy_type is None:
+            return None
+
+        proxy_kwargs: dict[str, Any] = {}
+        if proxy_country is not None:
+            proxy_kwargs["country"] = proxy_country
+        if proxy_type is not None:
+            proxy_kwargs["type"] = ProxyType(proxy_type)
+
+        return ProxySettings(**proxy_kwargs)
+
+    @staticmethod
+    def _job_status_enum() -> type[Enum]:
+        from geonode_scraper_sdk import JobStatus
+
+        return JobStatus
+
+    @staticmethod
+    def _enum_or_none(value: str | None, *, enum_type: type[Enum]) -> Enum | None:
+        if value is None:
+            return None
+        return enum_type(value)
+
+    def _extract_response_type(self, operation: str, result: Any) -> str | None:
+        if operation != "extract":
+            return None
+
+        if getattr(result, "job_id", None) is not None and getattr(result, "status_url", None) is not None:
+            return "async"
+        return "sync"
+
+    def _normalize_exception(self, exc: ApiException) -> dict[str, Any]:
+        payload = exc.data.error if getattr(exc.data, "error", None) is not None else exc.data
+        return {
+            "status": exc.status,
+            "reason": exc.reason,
+            "code": self._normalize_value(getattr(payload, "code", None)),
+            "message": getattr(payload, "message", None) or exc.reason,
+            "retryable": getattr(payload, "retryable", None),
+            "body": exc.body,
+            "data": self._normalize_value(exc.data),
+        }
+
+    def _normalize_value(self, value: Any) -> Any:
+        if value is None:
+            return None
+        if isinstance(value, BaseModel):
+            return value.model_dump(mode="json", exclude_none=True)
+        if isinstance(value, Enum):
+            return value.value
+        if isinstance(value, datetime):
+            return value.isoformat()
+        if isinstance(value, UUID):
+            return str(value)
+        if isinstance(value, Mapping):
+            return {str(key): self._normalize_value(item) for key, item in value.items()}
+        if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+            return [self._normalize_value(item) for item in value]
+        return value

geonode_scraper_tools_core-0.1.0/geonode_scraper_tools_core.egg-info/PKG-INFO

@@ -0,0 +1,56 @@
+Metadata-Version: 2.4
+Name: geonode-scraper-tools-core
+Version: 0.1.0
+Summary: Shared runtime and schemas for Geonode Scraper framework tools
+Author: Geonode Team
+License-Expression: MIT
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: geonode-scraper-sdk>=0.1.0
+Requires-Dist: pydantic>=2.11
+Requires-Dist: typing-extensions>=4.7.1
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0; extra == "dev"
+Requires-Dist: ruff>=0.12.11; extra == "dev"
+
+# Geonode Scraper Tools Core
+
+Shared runtime, schemas, and operation registry for Geonode Scraper tool
+integrations.
+
+Most users should install one of the framework packages instead:
+
+- `geonode-scraper-langchain`
+- `geonode-scraper-crewai`
+
+Install the core package directly only if you are building your own wrapper layer
+on top of the shared service.
+
+## Installation
+
+```sh
+pip install geonode-scraper-tools-core
+```
+
+## Public API
+
+- `ScraperToolSettings`
+- `ScraperToolService`
+- `OperationSpec`
+- `OPERATIONS`
+- `get_operations()`
+
+The shared service normalizes SDK responses into JSON-friendly dictionaries and
+exposes the following operations:
+
+- `extract`
+- `get_job_result`
+- `wait_for_job`
+- `list_jobs`
+- `get_statistics`
+- `health_check`

geonode_scraper_tools_core-0.1.0/geonode_scraper_tools_core.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+MANIFEST.in
+README.md
+pyproject.toml
+geonode_scraper_tools_core/__init__.py
+geonode_scraper_tools_core/py.typed
+geonode_scraper_tools_core/registry.py
+geonode_scraper_tools_core/schemas.py
+geonode_scraper_tools_core/service.py
+geonode_scraper_tools_core.egg-info/PKG-INFO
+geonode_scraper_tools_core.egg-info/SOURCES.txt
+geonode_scraper_tools_core.egg-info/dependency_links.txt
+geonode_scraper_tools_core.egg-info/requires.txt
+geonode_scraper_tools_core.egg-info/top_level.txt
+tests/test_service.py

geonode_scraper_tools_core-0.1.0/geonode_scraper_tools_core.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

geonode_scraper_tools_core-0.1.0/geonode_scraper_tools_core.egg-info/requires.txt

@@ -0,0 +1,8 @@
+geonode-scraper-sdk>=0.1.0
+pydantic>=2.11
+typing-extensions>=4.7.1
+
+[dev]
+pytest>=8.0
+pytest-cov>=5.0
+ruff>=0.12.11

geonode_scraper_tools_core-0.1.0/geonode_scraper_tools_core.egg-info/top_level.txt

@@ -0,0 +1 @@
+geonode_scraper_tools_core

geonode_scraper_tools_core-0.1.0/pyproject.toml

@@ -0,0 +1,48 @@
+[project]
+name = "geonode-scraper-tools-core"
+version = "0.1.0"
+description = "Shared runtime and schemas for Geonode Scraper framework tools"
+authors = [
+  {name = "Geonode Team"},
+]
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+classifiers = [
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Typing :: Typed",
+]
+dependencies = [
+  "geonode-scraper-sdk>=0.1.0",
+  "pydantic>=2.11",
+  "typing-extensions>=4.7.1",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.0",
+  "pytest-cov>=5.0",
+  "ruff>=0.12.11",
+]
+
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+include = ["geonode_scraper_tools_core*"]
+
+[tool.setuptools.package-data]
+geonode_scraper_tools_core = ["py.typed"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 120
+
+[tool.ruff.lint]
+select = ["E", "W", "F", "I", "B"]

geonode_scraper_tools_core-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

geonode_scraper_tools_core-0.1.0/tests/test_service.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from unittest.mock import patch
+
+from geonode_scraper_tools_core.service import ScraperToolService, ScraperToolSettings
+
+
+def test_extract_sync_normalizes_response(service, mock_api_client, make_mock_http_response, sync_response_payload):
+    mock_response = make_mock_http_response(200, sync_response_payload)
+    with patch.object(mock_api_client.rest_client.pool_manager, "request", return_value=mock_response):
+        result = service.extract(url="https://example.com")
+
+    assert result["ok"] is True
+    assert result["operation"] == "extract"
+    assert result["response_type"] == "sync"
+    assert result["result"]["tokens_charged"] == 10
+    assert result["result"]["data"]["markdown"] == "# Example"
+
+
+def test_extract_async_normalizes_union_response(
+    service,
+    mock_api_client,
+    make_mock_http_response,
+    async_response_payload,
+):
+    mock_response = make_mock_http_response(202, async_response_payload)
+    with patch.object(mock_api_client.rest_client.pool_manager, "request", return_value=mock_response):
+        result = service.extract(url="https://example.com", processing_mode="async")
+
+    assert result["ok"] is True
+    assert result["response_type"] == "async"
+    assert result["result"]["job_id"] == "test-job-uuid"
+
+
+def test_wait_for_job_is_exposed_as_its_own_operation(service):
+    queued = {
+        "ok": True,
+        "operation": "get_job_result",
+        "attempts": 1,
+        "result": {"status": "processing", "job_id": "test-job-uuid"},
+    }
+    completed = {
+        "ok": True,
+        "operation": "get_job_result",
+        "attempts": 1,
+        "result": {"status": "completed", "job_id": "test-job-uuid", "data": {"markdown": "# Done"}},
+    }
+
+    with patch.object(service, "get_job_result", side_effect=[queued, completed]) as mock_get_job_result:
+        result = service.wait_for_job(job_id="test-job-uuid")
+
+    assert result["ok"] is True
+    assert result["operation"] == "wait_for_job"
+    assert result["poll_attempts"] == 2
+    assert result["result"]["status"] == "completed"
+    assert mock_get_job_result.call_count == 2
+
+
+def test_extract_retries_retryable_errors(mock_api_client, make_mock_http_response):
+    @patch.object(mock_api_client.rest_client.pool_manager, "request")
+    def _run(mock_request):
+        retry_response = make_mock_http_response(
+            429,
+            {
+                "code": "RATE_LIMITED",
+                "message": "Too many requests",
+                "retryable": True,
+            },
+        )
+        success_response = make_mock_http_response(
+            200,
+            {
+                "data": {"html": "<html/>", "markdown": "# Example"},
+                "metadata": {
+                    "url": "https://example.com",
+                    "render_js": False,
+                    "formats": ["html"],
+                    "processing_mode": "sync",
+                },
+                "tokens_charged": 10,
+            },
+        )
+        mock_request.side_effect = [retry_response, success_response]
+
+        settings = ScraperToolSettings(
+            host="http://api.mock.local",
+            api_key="test-mock-api-key",
+            max_retries=1,
+            retry_backoff_seconds=0,
+        )
+
+        from contextlib import contextmanager
+
+        @contextmanager
+        def _factory():
+            yield mock_api_client
+
+        service = ScraperToolService(settings, api_client_factory=_factory, sleep_fn=lambda _seconds: None)
+        result = service.extract(url="https://example.com")
+
+        assert result["ok"] is True
+        assert result["attempts"] == 2
+        assert mock_request.call_count == 2
+
+    _run()
+
+
+def test_list_jobs_serializes_filters(service, mock_api_client, make_mock_http_response, get_request_details):
+    mock_response = make_mock_http_response(
+        200,
+        {
+            "jobs": [
+                {
+                    "job_id": "00000000-0000-0000-0000-000000000001",
+                    "status": "completed",
+                    "url": "https://example.com",
+                    "created_at": datetime(2026, 1, 1, tzinfo=timezone.utc).isoformat(),
+                }
+            ],
+            "page": 1,
+            "page_size": 100,
+            "page_count": 1,
+        },
+    )
+
+    start_date = datetime(2026, 1, 1, tzinfo=timezone.utc)
+    end_date = datetime(2026, 1, 7, tzinfo=timezone.utc)
+    with patch.object(mock_api_client.rest_client.pool_manager, "request", return_value=mock_response) as mock_request:
+        result = service.list_jobs(status="completed", output="markdown", start_date=start_date, end_date=end_date)
+
+    request_details = get_request_details(mock_request)
+    assert result["ok"] is True
+    assert request_details["query"]["status"] == "completed"
+    assert request_details["query"]["output"] == "markdown"
+    assert request_details["query"]["start_date"] == start_date.strftime(mock_api_client.configuration.datetime_format)
+    assert request_details["query"]["end_date"] == end_date.strftime(mock_api_client.configuration.datetime_format)
+
+
+def test_health_check_preserves_error_payload(service, mock_api_client, make_mock_http_response, health_payload):
+    mock_response = make_mock_http_response(503, {**health_payload, "status": "degraded"})
+    with patch.object(mock_api_client.rest_client.pool_manager, "request", return_value=mock_response):
+        result = service.health_check()
+
+    assert result["ok"] is False
+    assert result["error"]["status"] == 503
+    assert result["error"]["data"]["status"] == "degraded"