webcrawlerapi 2.1.0__tar.gz → 2.1.3__tar.gz

{webcrawlerapi-2.1.0 → webcrawlerapi-2.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webcrawlerapi
-Version: 2.1.0
+Version: 2.1.3
 Summary: Python SDK for WebCrawler API
 Home-page: https://github.com/webcrawlerapi/webcrawlerapi-python-sdk
 Author: Andrew

{webcrawlerapi-2.1.0 → webcrawlerapi-2.1.3}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="webcrawlerapi",
-    version="2.1.0",
+    version="2.1.3",
     packages=find_packages(),
     install_requires=[
         "requests>=2.25.0",

{webcrawlerapi-2.1.0 → webcrawlerapi-2.1.3}/tests/test_client.py

@@ -13,7 +13,6 @@ from webcrawlerapi.models import (
     ScrapeId,
     ScrapeResponse,
     ScrapeResponseError,
-    UploadS3Action,
     WebCrawlerApiError,
 )
 
@@ -99,37 +98,6 @@ class TestWebCrawlerAPI:
         assert payload["items_limit"] == 5
         assert payload["respect_robots_txt"] is True
 
-    @responses.activate
-    def test_crawl_async_with_actions(self, client):
-        """Test crawl_async with S3 upload action."""
-        responses.add(
-            responses.POST,
-            "https://api.test.com/v1/crawl",
-            json={"id": "crawl-456"},
-            status=200,
-        )
-
-        s3_action = UploadS3Action(
-            path="crawl-results/",
-            access_key_id="AKIAEXAMPLE",
-            secret_access_key="secret123",
-            bucket="my-bucket",
-        )
-
-        result = client.crawl_async(url="https://example.com", actions=[s3_action])
-
-        assert result.id == "crawl-456"
-
-        # Verify action in payload
-        request = responses.calls[0].request
-        import json
-
-        payload = json.loads(request.body)
-        assert "actions" in payload
-        assert len(payload["actions"]) == 1
-        assert payload["actions"][0]["type"] == "upload_s3"
-        assert payload["actions"][0]["bucket"] == "my-bucket"
-
     @responses.activate
     def test_crawl_async_with_max_age(self, client):
         """Test crawl_async includes max_age when provided."""

{webcrawlerapi-2.1.0 → webcrawlerapi-2.1.3}/webcrawlerapi/__init__.py

@@ -25,6 +25,7 @@ Basic usage:
 from .client import WebCrawlerAPI
 from .models import (
     Action,
+    AgentRun,
     CrawlResponse,
     Job,
     JobItem,
@@ -49,4 +50,5 @@ __all__ = [
     "ScrapeResponseError",
     "Action",
     "UploadS3Action",
+    "AgentRun",
 ]

{webcrawlerapi-2.1.0 → webcrawlerapi-2.1.3}/webcrawlerapi/client.py

@@ -6,6 +6,7 @@ import requests
 
 from .models import (
     Action,
+    AgentRun,
     CrawlResponse,
     Job,
     JobMarkdownResponse,
@@ -17,6 +18,7 @@ from .models import (
 
 CRAWLER_VERSION = "v1"
 SCRAPER_VERSION = "v2"
+AGENT_TERMINAL_STATUSES = {"done", "error", "canceled"}
 
 
 class WebCrawlerAPI:
@@ -66,7 +68,6 @@ class WebCrawlerAPI:
         webhook_url: Optional[str] = None,
         whitelist_regexp: Optional[str] = None,
         blacklist_regexp: Optional[str] = None,
-        actions: Optional[Union[Action, List[Action]]] = None,
         respect_robots_txt: bool = False,
         main_content_only: bool = False,
         max_depth: Optional[int] = None,
@@ -85,7 +86,6 @@ class WebCrawlerAPI:
             webhook_url (str, optional): URL for webhook notifications
             whitelist_regexp (str, optional): Regex pattern for URL whitelist
             blacklist_regexp (str, optional): Regex pattern for URL blacklist
-            actions (Action or List[Action], optional): Actions to perform during crawling
             respect_robots_txt (bool): Whether to respect robots.txt (default: False)
             main_content_only (bool): Whether to extract only main content (default: False)
             max_depth (int, optional): Maximum crawl depth (0 = seed only, 1 = seed + direct links)
@@ -129,9 +129,6 @@ class WebCrawlerAPI:
             payload["main_content_only"] = main_content_only
         if keep_query_params is not None:
             payload["keep_query_params"] = keep_query_params
-        if actions:
-            action_list = [actions] if not isinstance(actions, list) else actions
-            payload["actions"] = [vars(action) for action in action_list]
 
         response = self.session.post(
             urljoin(self.base_url, f"/{CRAWLER_VERSION}/crawl"), json=payload
@@ -235,7 +232,6 @@ class WebCrawlerAPI:
         webhook_url: Optional[str] = None,
         whitelist_regexp: Optional[str] = None,
         blacklist_regexp: Optional[str] = None,
-        actions: Optional[Union[Action, List[Action]]] = None,
         respect_robots_txt: bool = False,
         main_content_only: bool = False,
         max_depth: Optional[int] = None,
@@ -254,7 +250,6 @@ class WebCrawlerAPI:
             webhook_url (str, optional): URL for webhook notifications
             whitelist_regexp (str, optional): Regex pattern for URL whitelist
             blacklist_regexp (str, optional): Regex pattern for URL blacklist
-            actions (Action or List[Action], optional): Actions to perform during crawling
             respect_robots_txt (bool): Whether to respect robots.txt (default: False)
             main_content_only (bool): Whether to extract only main content (default: False)
             max_depth (int, optional): Maximum crawl depth
@@ -277,7 +272,6 @@ class WebCrawlerAPI:
             webhook_url=webhook_url,
             whitelist_regexp=whitelist_regexp,
             blacklist_regexp=blacklist_regexp,
-            actions=actions,
             respect_robots_txt=respect_robots_txt,
             main_content_only=main_content_only,
             max_depth=max_depth,
@@ -313,7 +307,6 @@ class WebCrawlerAPI:
         webhook_url: Optional[str] = None,
         whitelist_regexp: Optional[str] = None,
         blacklist_regexp: Optional[str] = None,
-        actions: Optional[Union[Action, List[Action]]] = None,
         respect_robots_txt: bool = False,
         main_content_only: bool = False,
         max_depth: Optional[int] = None,
@@ -338,7 +331,6 @@ class WebCrawlerAPI:
             webhook_url=webhook_url,
             whitelist_regexp=whitelist_regexp,
             blacklist_regexp=blacklist_regexp,
-            actions=actions,
             respect_robots_txt=respect_robots_txt,
             main_content_only=main_content_only,
             max_depth=max_depth,
@@ -355,6 +347,130 @@ class WebCrawlerAPI:
 
         return self.get_job_markdown_content(job.id)
 
+    def run_agent_async(
+        self,
+        prompt: str,
+        max_spend_usd: float,
+        urls: Optional[List[str]] = None,
+        seed_urls_only: Optional[bool] = None,
+        output_schema: Optional[Dict[str, Any]] = None,
+        model: Optional[str] = None,
+    ) -> AgentRun:
+        """
+        Start a new agent run asynchronously.
+
+        Args:
+            prompt (str): Natural-language task for the agent to complete
+            max_spend_usd (float): Maximum spend for this run in USD
+            urls (list, optional): Optional seed URLs the agent can use while running the task
+            seed_urls_only (bool, optional): If True, the agent only uses the provided seed URLs
+            output_schema (dict, optional): Optional JSON schema for structured agent output
+            model (str, optional): Model to use for the run. Omit to use the API default.
+
+        Returns:
+            AgentRun: The created agent run
+
+        Raises:
+            WebCrawlerApiError: If the API returns an error response
+            requests.exceptions.RequestException: If the HTTP request fails
+        """
+        payload: Dict[str, Any] = {"prompt": prompt, "max_spend_usd": max_spend_usd}
+
+        if urls is not None:
+            payload["urls"] = urls
+        if seed_urls_only is not None:
+            payload["seed_urls_only"] = seed_urls_only
+        if output_schema is not None:
+            payload["output_schema"] = output_schema
+        if model is not None:
+            payload["model"] = model
+
+        response = self.session.post(
+            urljoin(self.base_url, f"/{CRAWLER_VERSION}/agent"), json=payload
+        )
+        if not response.ok:
+            self._raise_for_error(response)
+        return AgentRun(response.json())
+
+    def get_agent_job(self, job_id: str) -> AgentRun:
+        """
+        Get the status and result of a specific agent run.
+
+        Args:
+            job_id (str): The unique identifier of the agent run
+
+        Returns:
+            AgentRun: The agent run status and result
+
+        Raises:
+            WebCrawlerApiError: If the API returns an error response
+            requests.exceptions.RequestException: If the HTTP request fails
+        """
+        response = self.session.get(
+            urljoin(self.base_url, f"/{CRAWLER_VERSION}/agent/job/{job_id}"),
+            headers={"Cache-Control": "no-cache, no-store, must-revalidate"},
+        )
+        if not response.ok:
+            self._raise_for_error(response)
+        return AgentRun(response.json())
+
+    def run_agent(
+        self,
+        prompt: str,
+        max_spend_usd: float,
+        urls: Optional[List[str]] = None,
+        seed_urls_only: Optional[bool] = None,
+        output_schema: Optional[Dict[str, Any]] = None,
+        model: Optional[str] = None,
+        max_polls: int = 100,
+    ) -> AgentRun:
+        """
+        Start a new agent run and wait for its completion.
+
+        Args:
+            prompt (str): Natural-language task for the agent to complete
+            max_spend_usd (float): Maximum spend for this run in USD
+            urls (list, optional): Optional seed URLs the agent can use while running the task
+            seed_urls_only (bool, optional): If True, the agent only uses the provided seed URLs
+            output_schema (dict, optional): Optional JSON schema for structured agent output
+            model (str, optional): Model to use for the run. Omit to use the API default.
+            max_polls (int): Maximum number of status checks before raising a timeout (default: 100)
+
+        Returns:
+            AgentRun: The final agent run state after completion
+
+        Raises:
+            WebCrawlerApiError: If the API returns an error response or the run times out
+            requests.exceptions.RequestException: If any API request fails
+        """
+        run = self.run_agent_async(
+            prompt=prompt,
+            max_spend_usd=max_spend_usd,
+            urls=urls,
+            seed_urls_only=seed_urls_only,
+            output_schema=output_schema,
+            model=model,
+        )
+
+        if not run.id:
+            raise WebCrawlerApiError(
+                "invalid_response", "Failed to fetch agent job status"
+            )
+
+        if run.status in AGENT_TERMINAL_STATUSES:
+            return run
+
+        for _ in range(max_polls):
+            time.sleep(2)
+            agent_job = self.get_agent_job(run.id)
+            if agent_job.status in AGENT_TERMINAL_STATUSES:
+                return agent_job
+
+        raise WebCrawlerApiError(
+            "timeout",
+            "Agent run took too long, please retry or increase the number of polling retries",
+        )
+
     def scrape_async(
         self,
         url: str,

{webcrawlerapi-2.1.0 → webcrawlerapi-2.1.3}/webcrawlerapi/models.py

@@ -94,6 +94,31 @@ class ScrapeResponseError:
     status: Optional[str] = None
 
 
+class AgentRun:
+    """Represents an agent run job."""
+
+    TERMINAL_STATUSES = {"done", "error", "canceled"}
+
+    def __init__(self, data: Dict[str, Any]):
+        self.id: str = data["id"]
+        self.status: str = data["status"]
+        self.prompt: str = data["prompt"]
+        self.model: str = data["model"]
+        self.data: Optional[Any] = data.get("data")
+        self.error_reason: Optional[str] = data.get("error_reason")
+        self.balance_used_usd: float = data.get("balance_used_usd", 0)
+        self.max_spend_usd: Optional[float] = data.get("max_spend_usd")
+        self.urls: Optional[List[str]] = data.get("urls")
+        self.created_at: datetime = parse_datetime(data["created_at"])
+        self.updated_at: datetime = parse_datetime(data["updated_at"])
+        self.success: bool = data.get("success", False)
+
+    @property
+    def is_terminal(self) -> bool:
+        """Check if the agent run is in a terminal state (done, error, or canceled)."""
+        return self.status in self.TERMINAL_STATUSES
+
+
 @dataclass
 class Action:
     """Base class for actions that can be performed during crawling."""

{webcrawlerapi-2.1.0 → webcrawlerapi-2.1.3}/webcrawlerapi.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webcrawlerapi
-Version: 2.1.0
+Version: 2.1.3
 Summary: Python SDK for WebCrawler API
 Home-page: https://github.com/webcrawlerapi/webcrawlerapi-python-sdk
 Author: Andrew