webcrawlerapi 2.1.5__tar.gz → 2.1.7__tar.gz

{webcrawlerapi-2.1.5 → webcrawlerapi-2.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webcrawlerapi
-Version: 2.1.5
+Version: 2.1.7
 Summary: Python SDK for WebCrawler API
 Home-page: https://github.com/webcrawlerapi/webcrawlerapi-python-sdk
 Author: Andrew

{webcrawlerapi-2.1.5 → webcrawlerapi-2.1.7}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="webcrawlerapi",
-    version="2.1.5",
+    version="2.1.7",
     packages=find_packages(),
     install_requires=[
         "requests>=2.25.0",

{webcrawlerapi-2.1.5 → webcrawlerapi-2.1.7}/tests/test_client.py

@@ -351,31 +351,64 @@ class TestWebCrawlerAPI:
         assert result.success is False
 
     @responses.activate
-    def test_scrape_with_polling_success(self, client):
-        """Test scrape method returns done result from sync endpoint."""
+    def test_scrape_starts_async_and_polls_until_done(self, client):
+        """scrape() starts an async scrape and polls get_scrape() until done."""
         responses.add(
             responses.POST,
-            "https://api.test.com/v2/scrape",
+            "https://api.test.com/v2/scrape?async=true",
+            json={"id": "scrape-123"},
+            status=200,
+        )
+        responses.add(
+            responses.GET,
+            "https://api.test.com/v2/scrape/scrape-123",
+            json={
+                "status": "in_progress",
+                "success": False,
+                "recommended_pull_delay_ms": 1000,
+            },
+            status=200,
+        )
+        responses.add(
+            responses.GET,
+            "https://api.test.com/v2/scrape/scrape-123",
             json={
                 "status": "done",
                 "success": True,
                 "markdown": "# Scraped Content",
+                "page_status_code": 200,
+                "page_title": "Test Page",
             },
             status=200,
         )
 
-        result = client.scrape(url="https://example.com")
+        with patch("webcrawlerapi.client.time.sleep") as mock_sleep:
+            result = client.scrape(url="https://example.com")
 
         assert isinstance(result, ScrapeResponse)
         assert result.status == "done"
         assert result.markdown == "# Scraped Content"
+        assert result.page_status_code == 200
+        assert result.page_title == "Test Page"
+
+        assert len(responses.calls) == 3
+        assert responses.calls[0].request.url == (
+            "https://api.test.com/v2/scrape?async=true"
+        )
+        mock_sleep.assert_called_once_with(1.0)  # recommended_pull_delay_ms / 1000
 
     @responses.activate
-    def test_scrape_with_polling_error(self, client):
-        """Test scrape method that gets error from sync endpoint."""
+    def test_scrape_returns_error_from_poll(self, client):
+        """scrape() returns ScrapeResponseError when polling yields an error."""
         responses.add(
             responses.POST,
-            "https://api.test.com/v2/scrape",
+            "https://api.test.com/v2/scrape?async=true",
+            json={"id": "scrape-123"},
+            status=200,
+        )
+        responses.add(
+            responses.GET,
+            "https://api.test.com/v2/scrape/scrape-123",
             json={
                 "status": "error",
                 "success": False,
@@ -385,30 +418,46 @@ class TestWebCrawlerAPI:
             status=200,
         )
 
-        result = client.scrape(url="https://example.com")
+        with patch("webcrawlerapi.client.time.sleep") as mock_sleep:
+            result = client.scrape(url="https://example.com")
 
         assert isinstance(result, ScrapeResponseError)
         assert result.error_code == "FETCH_ERROR"
+        mock_sleep.assert_not_called()
 
     @responses.activate
-    def test_scrape_with_polling_max_polls(self, client):
-        """Test scrape method returns full response data from sync endpoint."""
+    def test_scrape_times_out_after_400_seconds(self, client):
+        """scrape() stops polling after 400 seconds and returns the last state."""
         responses.add(
             responses.POST,
-            "https://api.test.com/v2/scrape",
+            "https://api.test.com/v2/scrape?async=true",
+            json={"id": "scrape-123"},
+            status=200,
+        )
+        responses.add(
+            responses.GET,
+            "https://api.test.com/v2/scrape/scrape-123",
             json={
-                "status": "done",
-                "success": True,
-                "markdown": "# Content",
-                "page_status_code": 200,
-                "page_title": "Test Page",
+                "status": "in_progress",
+                "success": False,
+                "recommended_pull_delay_ms": 1000,
             },
             status=200,
         )
 
-        result = client.scrape(url="https://example.com")
+        clock = {"now": 1000.0}
+
+        def fake_sleep(seconds):
+            clock["now"] += seconds
+
+        with patch("webcrawlerapi.client.time.monotonic", lambda: clock["now"]), patch(
+            "webcrawlerapi.client.time.sleep", side_effect=fake_sleep
+        ) as mock_sleep:
+            result = client.scrape(url="https://example.com")
 
         assert isinstance(result, ScrapeResponse)
-        assert result.status == "done"
-        assert result.page_status_code == 200
-        assert result.page_title == "Test Page"
+        assert result.status == "in_progress"
+        assert mock_sleep.call_count == 400
+        assert sum(c.args[0] for c in mock_sleep.call_args_list) == pytest.approx(400.0)
+        # 1 initial poll + 400 polls after each 1s sleep
+        assert len(responses.calls) == 402  # 1 POST + 401 GETs

{webcrawlerapi-2.1.5 → webcrawlerapi-2.1.7}/tests/test_models.py

@@ -428,4 +428,4 @@ class TestJob:
 
     def test_job_terminal_statuses_constant(self):
         """Test that TERMINAL_STATUSES constant is correctly defined."""
-        assert Job.TERMINAL_STATUSES == {"done", "error", "cancelled"}
+        assert Job.TERMINAL_STATUSES == {"done", "error", "cancelled", "canceled"}

{webcrawlerapi-2.1.5 → webcrawlerapi-2.1.7}/webcrawlerapi/__init__.py

@@ -14,7 +14,7 @@ Basic usage:
     >>> # Or asynchronous crawling
     >>> response = crawler.crawl_async(url="https://example.com")
     >>> job = crawler.get_job(response.id)
-    >>> # Single page scraping (synchronous)
+    >>> # Single page scraping (starts the scrape and waits for the result)
     >>> result = crawler.scrape(url="https://example.com", output_format="markdown")
     >>> if result.success:
     ...     print(result.markdown)  # Access the scraped content
@@ -38,7 +38,7 @@ from .models import (
     WebCrawlerApiError,
 )
 
-__version__ = "1.0.0"
+__version__ = "2.1.7"
 __all__ = [
     "WebCrawlerAPI",
     "WebCrawlerApiError",

{webcrawlerapi-2.1.5 → webcrawlerapi-2.1.7}/webcrawlerapi/client.py

@@ -26,6 +26,8 @@ class WebCrawlerAPI:
     """Python SDK for WebCrawler API."""
 
     DEFAULT_POLL_DELAY_SECONDS = 5
+    SCRAPE_DEFAULT_POLL_DELAY_SECONDS = 2
+    SCRAPE_TIMEOUT_SECONDS = 400
 
     def __init__(self, api_key: str, base_url: str = "https://api.webcrawlerapi.com"):
         """
@@ -609,6 +611,9 @@ class WebCrawlerAPI:
                 page_title=response_data.get("page_title"),
                 structured_data=response_data.get("structured_data"),
                 links=response_data.get("links"),
+                recommended_pull_delay_ms=response_data.get(
+                    "recommended_pull_delay_ms"
+                ),
             )
         elif status == "error":
             return ScrapeResponseError(
@@ -616,9 +621,19 @@ class WebCrawlerAPI:
                 error_code=response_data.get("error_code", "unknown"),
                 error_message=response_data.get("error_message", "Scraping failed"),
                 status=status,
+                recommended_pull_delay_ms=response_data.get(
+                    "recommended_pull_delay_ms"
+                ),
             )
         else:  # in_progress or any other status
-            return ScrapeResponse(success=False, status=status, page_status_code=0)
+            return ScrapeResponse(
+                success=False,
+                status=status,
+                page_status_code=0,
+                recommended_pull_delay_ms=response_data.get(
+                    "recommended_pull_delay_ms"
+                ),
+            )
 
     def scrape(
         self,
@@ -636,10 +651,13 @@ class WebCrawlerAPI:
         keep_query_params: Optional[bool] = None,
     ) -> Union[ScrapeResponse, ScrapeResponseError]:
         """
-        Scrape a single URL synchronously and return the result.
+        Scrape a single URL and wait for the result.
 
-        Calls the synchronous scrape endpoint which blocks until the scrape is complete.
-        For fire-and-poll behaviour use scrape_async() + get_scrape() instead.
+        Starts the scrape with scrape_async() and polls get_scrape() until the status is
+        "done" or an error is returned, waiting the server-recommended delay between polls
+        (falls back to 2 seconds). Polling stops after 400 seconds in total; on timeout
+        the last known state is returned.
+        For manual control use scrape_async() + get_scrape() instead.
 
         Args:
             url (str): The URL to scrape
@@ -656,66 +674,42 @@ class WebCrawlerAPI:
             keep_query_params (bool, optional): Keep URL query params when storing the URL (default: True).
 
         Returns:
-            Union[ScrapeResponse, ScrapeResponseError]: The scrape result
+            Union[ScrapeResponse, ScrapeResponseError]: The scrape result, or the last known
+            state (status "in_progress") if the 400 second timeout is reached
 
         Raises:
             WebCrawlerApiError: If the API returns an error response
             requests.exceptions.RequestException: If the HTTP request fails
         """
-        payload: Dict[str, Any] = {"url": url}
-
-        if output_formats is not None:
-            payload["output_formats"] = output_formats
-        elif output_format is not None:
-            payload["output_format"] = output_format
-
-        if webhook_url:
-            payload["webhook_url"] = webhook_url
-        if clean_selectors:
-            payload["clean_selectors"] = clean_selectors
-        if prompt:
-            payload["prompt"] = prompt
-        if response_schema is not None:
-            payload["response_schema"] = response_schema
-        if max_age is not None:
-            payload["max_age"] = max_age
-        if main_content_only:
-            payload["main_content_only"] = main_content_only
-        if respect_robots_txt:
-            payload["respect_robots_txt"] = respect_robots_txt
-        if keep_query_params is not None:
-            payload["keep_query_params"] = keep_query_params
-        if actions:
-            action_list = [actions] if not isinstance(actions, list) else actions
-            payload["actions"] = [vars(action) for action in action_list]
-
-        response = self.session.post(
-            urljoin(self.base_url, f"/{SCRAPER_VERSION}/scrape"),
-            json=payload,
-        )
+        scrape_id = self.scrape_async(
+            url,
+            output_formats=output_formats,
+            output_format=output_format,
+            webhook_url=webhook_url,
+            clean_selectors=clean_selectors,
+            prompt=prompt,
+            response_schema=response_schema,
+            actions=actions,
+            main_content_only=main_content_only,
+            max_age=max_age,
+            respect_robots_txt=respect_robots_txt,
+            keep_query_params=keep_query_params,
+        ).id
 
-        if not response.ok:
-            self._raise_for_error(response)
+        deadline = time.monotonic() + self.SCRAPE_TIMEOUT_SECONDS
+        result = self.get_scrape(scrape_id)
 
-        response_data = response.json()
-        status = response_data.get("status")
+        while not isinstance(result, ScrapeResponseError) and result.status != "done":
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                return result
 
-        if status == "error":
-            return ScrapeResponseError(
-                success=False,
-                error_code=response_data.get("error_code", "unknown"),
-                error_message=response_data.get("error_message", "Scraping failed"),
-                status=status,
+            delay_ms = result.recommended_pull_delay_ms
+            delay = (
+                delay_ms / 1000 if delay_ms else self.SCRAPE_DEFAULT_POLL_DELAY_SECONDS
             )
+            time.sleep(min(delay, remaining))
 
-        return ScrapeResponse(
-            success=response_data.get("success", True),
-            status=status,
-            markdown=response_data.get("markdown"),
-            cleaned_content=response_data.get("cleaned_content"),
-            raw_content=response_data.get("raw_content"),
-            page_status_code=response_data.get("page_status_code", 0),
-            page_title=response_data.get("page_title"),
-            structured_data=response_data.get("structured_data"),
-            links=response_data.get("links"),
-        )
+            result = self.get_scrape(scrape_id)
+
+        return result

{webcrawlerapi-2.1.5 → webcrawlerapi-2.1.7}/webcrawlerapi/models.py

@@ -90,6 +90,7 @@ class ScrapeResponse:
     page_title: Optional[str] = None
     structured_data: Optional[Dict[str, Any]] = None
     links: Optional[List[str]] = None
+    recommended_pull_delay_ms: Optional[int] = None
 
 
 @dataclass
@@ -100,6 +101,7 @@ class ScrapeResponseError:
     error_code: str
     error_message: str
     status: Optional[str] = None
+    recommended_pull_delay_ms: Optional[int] = None
 
 
 class AgentRun:
@@ -284,7 +286,7 @@ class JobItem:
 class Job:
     """Represents a crawling job."""
 
-    TERMINAL_STATUSES = {"done", "error", "cancelled"}
+    TERMINAL_STATUSES = {"done", "error", "cancelled", "canceled"}
 
     def __init__(self, data: Dict[str, Any]):
         self.id: str = data["id"]
@@ -317,5 +319,5 @@ class Job:
 
     @property
     def is_terminal(self) -> bool:
-        """Check if the job is in a terminal state (done, error, or cancelled)."""
+        """Check if the job is in a terminal state (done, error, or cancelled/canceled)."""
         return self.status in self.TERMINAL_STATUSES

{webcrawlerapi-2.1.5 → webcrawlerapi-2.1.7}/webcrawlerapi.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webcrawlerapi
-Version: 2.1.5
+Version: 2.1.7
 Summary: Python SDK for WebCrawler API
 Home-page: https://github.com/webcrawlerapi/webcrawlerapi-python-sdk
 Author: Andrew