gofetch-client 0.1.0__py3-none-any.whl

gofetch/__init__.py

@@ -0,0 +1,94 @@
+"""
+GoFetch Python Client - Social Media Scraping API
+
+A drop-in replacement for apify-client that uses the GoFetch.io API.
+
+Usage:
+    from gofetch import GoFetchClient
+
+    client = GoFetchClient(api_key="sk_scr_...")
+
+    # Sync execution (blocks until complete)
+    actor = client.actor("instagram")
+    run = actor.call(run_input={"directUrls": ["https://instagram.com/nike"]})
+
+    # Fetch results
+    dataset = client.dataset(run["defaultDatasetId"])
+    items = list(dataset.iterate_items())
+
+    # Async execution (returns immediately, uses webhooks)
+    run = actor.start(
+        run_input={"directUrls": ["https://instagram.com/nike"]},
+        webhooks=[{"request_url": "https://...", "event_types": ["ACTOR.RUN.SUCCEEDED"]}]
+    )
+"""
+
+from gofetch.actor import ActorClient, AsyncActorClient
+from gofetch.client import AsyncGoFetchClient, GoFetchClient
+from gofetch.dataset import AsyncDatasetClient, DatasetClient
+from gofetch.exceptions import (
+    APIError,
+    AuthenticationError,
+    GoFetchError,
+    JobError,
+    RateLimitError,
+    TimeoutError,
+    ValidationError,
+)
+from gofetch.log import AsyncLogClient, LogClient
+from gofetch.run import AsyncRunClient, RunClient
+from gofetch.types import (
+    JobStatus,
+    RunStatus,
+    ScraperType,
+)
+from gofetch.webhook import (
+    WebhookEventType,
+    generate_webhook_config,
+    transform_webhook_payload,
+    verify_webhook_signature,
+)
+from gofetch.webhook_client import (
+    AsyncWebhookClient,
+    AsyncWebhookCollectionClient,
+    WebhookClient,
+    WebhookCollectionClient,
+)
+
+# Apify compatibility alias
+ApifyClient = GoFetchClient
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "APIError",
+    "ActorClient",
+    "ApifyClient",
+    "AsyncActorClient",
+    "AsyncDatasetClient",
+    "AsyncGoFetchClient",
+    "AsyncLogClient",
+    "AsyncRunClient",
+    "AsyncWebhookClient",
+    "AsyncWebhookCollectionClient",
+    "AuthenticationError",
+    "DatasetClient",
+    "GoFetchClient",
+    "GoFetchError",
+    "JobError",
+    "JobStatus",
+    "LogClient",
+    "RateLimitError",
+    "RunClient",
+    "RunStatus",
+    "ScraperType",
+    "TimeoutError",
+    "ValidationError",
+    "WebhookClient",
+    "WebhookCollectionClient",
+    "WebhookEventType",
+    "__version__",
+    "generate_webhook_config",
+    "transform_webhook_payload",
+    "verify_webhook_signature",
+]

gofetch/actor.py

@@ -0,0 +1,312 @@
+"""
+Actor client for GoFetch API.
+
+Provides Apify-compatible interface for running scraper jobs.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+import warnings
+from typing import TYPE_CHECKING, Any
+
+from gofetch.constants import (
+    DEFAULT_POLL_INTERVAL,
+    GOFETCH_TO_APIFY_STATUS,
+    MAX_POLL_INTERVAL,
+    POLL_BACKOFF_FACTOR,
+)
+from gofetch.webhook import APIFY_TO_GOFETCH_EVENTS
+
+if TYPE_CHECKING:
+    from gofetch.http import AsyncHTTPClient, HTTPClient
+
+logger = logging.getLogger(__name__)
+
+TERMINAL_STATUSES = frozenset({"completed", "failed", "timed_out", "cancelled"})
+
+
+def _is_terminal(status: str) -> bool:
+    return status in TERMINAL_STATUSES
+
+
+def _next_poll_interval(current: float) -> float:
+    return min(current * POLL_BACKOFF_FACTOR, MAX_POLL_INTERVAL)
+
+
+def _format_job_as_apify_run(
+    job: dict[str, Any],
+    scraper_type: str | None = None,
+    extra_fields: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Convert GoFetch job dict to Apify run format.
+
+    IMPORTANT: Any extra fields on the job dict (e.g., scraper_metadata)
+    are preserved on the returned run dict.
+    """
+    scraper_type = scraper_type or job.get("scraper_type", "unknown")
+    status = GOFETCH_TO_APIFY_STATUS.get(job.get("status", ""), "RUNNING")
+
+    run: dict[str, Any] = {
+        "id": job["id"],
+        "actId": f"gofetch/{scraper_type}",
+        "status": status,
+        "defaultDatasetId": job["id"],
+        "startedAt": job.get("started_at"),
+        "finishedAt": job.get("completed_at"),
+        "buildId": None,
+        "buildNumber": None,
+        "exitCode": 0 if status == "SUCCEEDED" else None,
+        "defaultKeyValueStoreId": None,
+        "defaultRequestQueueId": None,
+        "_gofetch_job": job,
+    }
+    if extra_fields:
+        run.update(extra_fields)
+    return run
+
+
+def _translate_webhooks(webhooks: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Translate Apify-format webhooks to GoFetch format.
+
+    Apify format:
+        {"request_url": "https://...", "event_types": ["ACTOR.RUN.SUCCEEDED"]}
+    or camelCase:
+        {"requestUrl": "https://...", "eventTypes": ["ACTOR.RUN.SUCCEEDED"]}
+
+    GoFetch format:
+        {"url": "https://...", "events": ["job.completed"]}
+    """
+    translated = []
+    for wh in webhooks:
+        url = wh.get("request_url") or wh.get("requestUrl")
+        event_types = wh.get("event_types") or wh.get("eventTypes", [])
+        translated.append({
+            "url": url,
+            "events": [
+                APIFY_TO_GOFETCH_EVENTS.get(et, et)
+                for et in event_types
+            ],
+        })
+    return translated
+
+
+class ActorClient:
+    """
+    Actor client that wraps GoFetch job API with Apify-compatible interface.
+
+    Provides the same methods as Apify's ActorClient:
+    - call() for synchronous execution
+    - start() for asynchronous execution with webhooks
+    """
+
+    def __init__(
+        self,
+        http: HTTPClient,
+        scraper_type: str,
+    ) -> None:
+        self._http = http
+        self._scraper_type = scraper_type
+
+    def call(
+        self,
+        run_input: dict[str, Any],
+        *,
+        wait_secs: int | None = None,
+        timeout_secs: int | None = None,
+        memory_mbytes: int | None = None,
+        build: str | None = None,
+        webhooks: list[dict[str, Any]] | None = None,
+    ) -> dict[str, Any]:
+        """Run actor synchronously (blocking).
+
+        Matches Apify's behavioral contract:
+        - Returns run dict regardless of final status (never raises on failure/timeout)
+        - On timeout, returns the current run state (status may be "RUNNING")
+        - Default wait_secs=None means wait indefinitely
+
+        Args:
+            run_input: Scraper configuration parameters
+            wait_secs: Maximum wait time in seconds (None = indefinite)
+            timeout_secs: Deprecated alias for wait_secs
+            memory_mbytes: Ignored (Apify compatibility)
+            build: Ignored (Apify compatibility)
+            webhooks: Per-run webhooks to register
+
+        Returns:
+            Dict in Apify run format
+        """
+        _ = memory_mbytes, build
+
+        effective_wait = self._resolve_wait_secs(wait_secs, timeout_secs)
+
+        job = self._create_job(run_input, webhooks=webhooks)
+        job_id = job["id"]
+
+        return self._wait_for_completion(job_id, effective_wait)
+
+    def start(
+        self,
+        run_input: dict[str, Any],
+        *,
+        webhooks: list[dict[str, Any]] | None = None,
+        wait_secs: int | None = None,
+        timeout_secs: int | None = None,
+        memory_mbytes: int | None = None,
+        build: str | None = None,
+    ) -> dict[str, Any]:
+        """Start actor asynchronously (non-blocking).
+
+        Returns immediately after creating the job.
+        """
+        _ = wait_secs, timeout_secs, memory_mbytes, build
+
+        job = self._create_job(run_input, webhooks=webhooks)
+        return _format_job_as_apify_run(job, scraper_type=self._scraper_type)
+
+    def _create_job(
+        self,
+        run_input: dict[str, Any],
+        webhooks: list[dict[str, Any]] | None = None,
+    ) -> dict[str, Any]:
+        config = self._transform_input(run_input)
+        payload: dict[str, Any] = {
+            "scraper_type": self._scraper_type,
+            "config": config,
+        }
+        if webhooks:
+            payload["webhooks"] = _translate_webhooks(webhooks)
+
+        return self._http.post("/api/v1/jobs/create/", json=payload)
+
+    def _transform_input(self, run_input: dict[str, Any]) -> dict[str, Any]:
+        return run_input.copy()
+
+    def _wait_for_completion(
+        self,
+        job_id: str,
+        wait_secs: int | None,
+    ) -> dict[str, Any]:
+        """Poll job status until terminal or timeout.
+
+        Returns Apify-format run dict in all cases (never raises).
+        On timeout, returns current run state. On 404, returns minimal dict.
+        """
+        start_time = time.monotonic()
+        poll_interval = DEFAULT_POLL_INTERVAL
+
+        while True:
+            job = self._http.get(f"/api/v1/jobs/{job_id}/")
+
+            if _is_terminal(job.get("status", "")):
+                return _format_job_as_apify_run(job, scraper_type=self._scraper_type)
+
+            if wait_secs is not None and (time.monotonic() - start_time) >= wait_secs:
+                return _format_job_as_apify_run(job, scraper_type=self._scraper_type)
+
+            time.sleep(poll_interval)
+            poll_interval = _next_poll_interval(poll_interval)
+
+    @staticmethod
+    def _resolve_wait_secs(
+        wait_secs: int | None,
+        timeout_secs: int | None,
+    ) -> int | None:
+        if timeout_secs is not None and wait_secs is None:
+            warnings.warn(
+                "timeout_secs is deprecated, use wait_secs instead",
+                DeprecationWarning,
+                stacklevel=3,
+            )
+            return timeout_secs
+        return wait_secs
+
+
+class AsyncActorClient:
+    """Async actor client for GoFetch API."""
+
+    def __init__(
+        self,
+        http: AsyncHTTPClient,
+        scraper_type: str,
+    ) -> None:
+        self._http = http
+        self._scraper_type = scraper_type
+
+    async def call(
+        self,
+        run_input: dict[str, Any],
+        *,
+        wait_secs: int | None = None,
+        timeout_secs: int | None = None,
+        memory_mbytes: int | None = None,
+        build: str | None = None,
+        webhooks: list[dict[str, Any]] | None = None,
+    ) -> dict[str, Any]:
+        """Run actor synchronously (blocking). Async version."""
+        _ = memory_mbytes, build
+
+        effective_wait = ActorClient._resolve_wait_secs(wait_secs, timeout_secs)
+
+        job = await self._create_job(run_input, webhooks=webhooks)
+        job_id = job["id"]
+
+        return await self._wait_for_completion(job_id, effective_wait)
+
+    async def start(
+        self,
+        run_input: dict[str, Any],
+        *,
+        webhooks: list[dict[str, Any]] | None = None,
+        wait_secs: int | None = None,
+        timeout_secs: int | None = None,
+        memory_mbytes: int | None = None,
+        build: str | None = None,
+    ) -> dict[str, Any]:
+        """Start actor asynchronously (non-blocking). Async version."""
+        _ = wait_secs, timeout_secs, memory_mbytes, build
+
+        job = await self._create_job(run_input, webhooks=webhooks)
+        return _format_job_as_apify_run(job, scraper_type=self._scraper_type)
+
+    async def _create_job(
+        self,
+        run_input: dict[str, Any],
+        webhooks: list[dict[str, Any]] | None = None,
+    ) -> dict[str, Any]:
+        config = self._transform_input(run_input)
+        payload: dict[str, Any] = {
+            "scraper_type": self._scraper_type,
+            "config": config,
+        }
+        if webhooks:
+            payload["webhooks"] = _translate_webhooks(webhooks)
+
+        return await self._http.post("/api/v1/jobs/create/", json=payload)
+
+    def _transform_input(self, run_input: dict[str, Any]) -> dict[str, Any]:
+        return run_input.copy()
+
+    async def _wait_for_completion(
+        self,
+        job_id: str,
+        wait_secs: int | None,
+    ) -> dict[str, Any]:
+        """Poll job status until terminal or timeout. Async version."""
+        import asyncio
+
+        start_time = time.monotonic()
+        poll_interval = DEFAULT_POLL_INTERVAL
+
+        while True:
+            job = await self._http.get(f"/api/v1/jobs/{job_id}/")
+
+            if _is_terminal(job.get("status", "")):
+                return _format_job_as_apify_run(job, scraper_type=self._scraper_type)
+
+            if wait_secs is not None and (time.monotonic() - start_time) >= wait_secs:
+                return _format_job_as_apify_run(job, scraper_type=self._scraper_type)
+
+            await asyncio.sleep(poll_interval)
+            poll_interval = _next_poll_interval(poll_interval)

gofetch/client.py

@@ -0,0 +1,305 @@
+"""
+Main GoFetch client.
+
+Provides the primary entry point for interacting with the GoFetch API.
+Designed as a drop-in replacement for ApifyClient.
+"""
+
+from __future__ import annotations
+
+from typing import NoReturn
+
+from gofetch.actor import ActorClient, AsyncActorClient
+from gofetch.constants import DEFAULT_BASE_URL, DEFAULT_TIMEOUT
+from gofetch.dataset import AsyncDatasetClient, DatasetClient
+from gofetch.http import AsyncHTTPClient, HTTPClient
+from gofetch.run import AsyncRunClient, RunClient
+from gofetch.types import resolve_actor_url
+from gofetch.webhook_client import (
+    AsyncWebhookClient,
+    AsyncWebhookCollectionClient,
+    WebhookClient,
+    WebhookCollectionClient,
+)
+
+
+class GoFetchClient:
+    """
+    GoFetch API client - drop-in replacement for ApifyClient.
+
+    This client provides the same interface as Apify's SDK, allowing
+    you to switch from Apify to GoFetch with minimal code changes.
+
+    Usage:
+        # Initialize client
+        client = GoFetchClient(api_key="sk_scr_myorg_xxxx")
+
+        # Get actor client (same as Apify)
+        actor = client.actor("apify/instagram-scraper")  # Apify URL works!
+        # Or use GoFetch scraper type directly
+        actor = client.actor("instagram")
+
+        # Run synchronously (blocking)
+        run = actor.call(run_input={"directUrls": ["https://instagram.com/nike"]})
+        print(f"Job completed: {run['id']}")
+
+        # Fetch results
+        dataset = client.dataset(run["defaultDatasetId"])
+        for item in dataset.iterate_items():
+            print(item)
+
+        # Run asynchronously (with webhooks)
+        run = actor.start(
+            run_input={"directUrls": ["https://instagram.com/nike"]},
+            webhooks=[{
+                "request_url": "https://myapp.com/webhook",
+                "event_types": ["ACTOR.RUN.SUCCEEDED"]
+            }]
+        )
+        print(f"Job started: {run['id']}")
+
+    Attributes:
+        base_url: The API base URL
+        timeout: Request timeout in seconds
+    """
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+        token: str | None = None,  # Alias for api_key (Apify compat)
+        base_url: str = DEFAULT_BASE_URL,
+        timeout: float = DEFAULT_TIMEOUT,
+        max_retries: int = 3,
+    ) -> None:
+        """
+        Initialize the GoFetch client.
+
+        Args:
+            api_key: GoFetch API key (format: sk_scr_...)
+            token: Alias for api_key (for Apify compatibility)
+            base_url: API base URL (default: https://api.go-fetch.io)
+            timeout: Request timeout in seconds (default: 30)
+            max_retries: Maximum retries for failed requests (default: 3)
+
+        Raises:
+            ValueError: If neither api_key nor token is provided
+        """
+        # Support both api_key and token for Apify compatibility
+        self._api_key = api_key or token
+        if not self._api_key:
+            raise ValueError("Either 'api_key' or 'token' must be provided")
+
+        self._base_url = base_url.rstrip("/")
+        self._timeout = timeout
+        self._max_retries = max_retries
+
+        self._http = HTTPClient(
+            api_key=self._api_key,
+            base_url=self._base_url,
+            timeout=self._timeout,
+            max_retries=self._max_retries,
+        )
+
+    @property
+    def base_url(self) -> str:
+        """Get the API base URL."""
+        return self._base_url
+
+    @property
+    def timeout(self) -> float:
+        """Get the request timeout."""
+        return self._timeout
+
+    def actor(self, actor_url: str) -> ActorClient:
+        """
+        Get an actor client for a specific scraper.
+
+        Args:
+            actor_url: Apify-style actor URL or GoFetch scraper type.
+                Supported values:
+                - "apify/instagram-scraper" -> instagram
+                - "apify/instagram-profile-scraper" -> instagram_profile
+                - "clockworks/tiktok-profile-scraper" -> tiktok
+                - "streamers/youtube-scraper" -> youtube
+                - Or direct: "instagram", "tiktok", "youtube"
+
+        Returns:
+            ActorClient instance for the specified scraper
+
+        Raises:
+            ValueError: If actor_url is empty
+
+        Example:
+            # Using Apify URL (for compatibility)
+            actor = client.actor("apify/instagram-scraper")
+
+            # Using GoFetch type directly
+            actor = client.actor("instagram")
+        """
+        if not actor_url:
+            raise ValueError("actor_id must not be empty")
+        scraper_type = resolve_actor_url(actor_url)
+        return ActorClient(
+            http=self._http,
+            scraper_type=scraper_type,
+        )
+
+    def dataset(self, dataset_id: str) -> DatasetClient:
+        """
+        Get a dataset client to fetch results.
+
+        Args:
+            dataset_id: Dataset/Job ID. In GoFetch, the job ID serves
+                as the dataset ID (they are equivalent).
+
+        Returns:
+            DatasetClient instance for fetching results
+
+        Example:
+            run = actor.call(run_input={...})
+            dataset = client.dataset(run["defaultDatasetId"])
+            items = list(dataset.iterate_items())
+        """
+        return DatasetClient(
+            http=self._http,
+            job_id=dataset_id,
+        )
+
+    def run(self, run_id: str) -> RunClient:
+        """Get a run client for a specific job."""
+        return RunClient(http=self._http, run_id=run_id)
+
+    def webhook(self, webhook_id: str) -> WebhookClient:
+        """Get a webhook client for a specific webhook."""
+        return WebhookClient(http=self._http, webhook_id=webhook_id)
+
+    def webhooks(self) -> WebhookCollectionClient:
+        """Get webhook collection client for listing/creating webhooks."""
+        return WebhookCollectionClient(http=self._http)
+
+    def key_value_store(self, store_id: str) -> NoReturn:
+        """Not supported in GoFetch.
+
+        Apify's key-value store has no GoFetch equivalent.
+        """
+        raise NotImplementedError(
+            f"GoFetch does not have a key-value store. "
+            f"key_value_store('{store_id}') cannot be used. "
+            f"Media uploaded by GoFetch scrapers is accessible via direct URLs "
+            f"and does not require manual cleanup."
+        )
+
+    def close(self) -> None:
+        """Close the client and release resources."""
+        self._http.close()
+
+    def __enter__(self) -> GoFetchClient:
+        """Context manager entry."""
+        return self
+
+    def __exit__(self, *args: object) -> None:
+        """Context manager exit."""
+        self.close()
+
+
+class AsyncGoFetchClient:
+    """
+    Async GoFetch API client.
+
+    Same interface as GoFetchClient but uses async/await.
+
+    Usage:
+        async with AsyncGoFetchClient(api_key="...") as client:
+            actor = client.actor("instagram")
+            run = await actor.call(run_input={...})
+            dataset = client.dataset(run["defaultDatasetId"])
+            items = await dataset.list_items()
+    """
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+        token: str | None = None,
+        base_url: str = DEFAULT_BASE_URL,
+        timeout: float = DEFAULT_TIMEOUT,
+        max_retries: int = 3,
+    ) -> None:
+        """Initialize the async GoFetch client."""
+        self._api_key = api_key or token
+        if not self._api_key:
+            raise ValueError("Either 'api_key' or 'token' must be provided")
+
+        self._base_url = base_url.rstrip("/")
+        self._timeout = timeout
+        self._max_retries = max_retries
+
+        self._http = AsyncHTTPClient(
+            api_key=self._api_key,
+            base_url=self._base_url,
+            timeout=self._timeout,
+            max_retries=self._max_retries,
+        )
+
+    @property
+    def base_url(self) -> str:
+        """Get the API base URL."""
+        return self._base_url
+
+    @property
+    def timeout(self) -> float:
+        """Get the request timeout."""
+        return self._timeout
+
+    def actor(self, actor_url: str) -> AsyncActorClient:
+        """Get an async actor client for a specific scraper."""
+        if not actor_url:
+            raise ValueError("actor_id must not be empty")
+        scraper_type = resolve_actor_url(actor_url)
+        return AsyncActorClient(
+            http=self._http,
+            scraper_type=scraper_type,
+        )
+
+    def dataset(self, dataset_id: str) -> AsyncDatasetClient:
+        """Get an async dataset client to fetch results."""
+        return AsyncDatasetClient(
+            http=self._http,
+            job_id=dataset_id,
+        )
+
+    def run(self, run_id: str) -> AsyncRunClient:
+        """Get an async run client for a specific job."""
+        return AsyncRunClient(http=self._http, run_id=run_id)
+
+    def webhook(self, webhook_id: str) -> AsyncWebhookClient:
+        """Get an async webhook client for a specific webhook."""
+        return AsyncWebhookClient(http=self._http, webhook_id=webhook_id)
+
+    def webhooks(self) -> AsyncWebhookCollectionClient:
+        """Get async webhook collection client for listing/creating webhooks."""
+        return AsyncWebhookCollectionClient(http=self._http)
+
+    def key_value_store(self, store_id: str) -> NoReturn:
+        """Not supported in GoFetch."""
+        raise NotImplementedError(
+            f"GoFetch does not have a key-value store. "
+            f"key_value_store('{store_id}') cannot be used. "
+            f"Media uploaded by GoFetch scrapers is accessible via direct URLs "
+            f"and does not require manual cleanup."
+        )
+
+    async def close(self) -> None:
+        """Close the client and release resources."""
+        await self._http.close()
+
+    async def __aenter__(self) -> AsyncGoFetchClient:
+        """Async context manager entry."""
+        return self
+
+    async def __aexit__(self, *args: object) -> None:
+        """Async context manager exit."""
+        await self.close()
+
+
+# Alias for Apify compatibility
+ApifyClient = GoFetchClient