webcrawlerapi 2.1.2__tar.gz → 2.1.5__tar.gz

{webcrawlerapi-2.1.2 → webcrawlerapi-2.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webcrawlerapi
-Version: 2.1.2
+Version: 2.1.5
 Summary: Python SDK for WebCrawler API
 Home-page: https://github.com/webcrawlerapi/webcrawlerapi-python-sdk
 Author: Andrew

{webcrawlerapi-2.1.2 → webcrawlerapi-2.1.5}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="webcrawlerapi",
-    version="2.1.2",
+    version="2.1.5",
     packages=find_packages(),
     install_requires=[
         "requests>=2.25.0",

{webcrawlerapi-2.1.2 → webcrawlerapi-2.1.5}/tests/test_client.py

@@ -13,7 +13,6 @@ from webcrawlerapi.models import (
     ScrapeId,
     ScrapeResponse,
     ScrapeResponseError,
-    UploadS3Action,
     WebCrawlerApiError,
 )
 
@@ -99,37 +98,6 @@ class TestWebCrawlerAPI:
         assert payload["items_limit"] == 5
         assert payload["respect_robots_txt"] is True
 
-    @responses.activate
-    def test_crawl_async_with_actions(self, client):
-        """Test crawl_async with S3 upload action."""
-        responses.add(
-            responses.POST,
-            "https://api.test.com/v1/crawl",
-            json={"id": "crawl-456"},
-            status=200,
-        )
-
-        s3_action = UploadS3Action(
-            path="crawl-results/",
-            access_key_id="AKIAEXAMPLE",
-            secret_access_key="secret123",
-            bucket="my-bucket",
-        )
-
-        result = client.crawl_async(url="https://example.com", actions=[s3_action])
-
-        assert result.id == "crawl-456"
-
-        # Verify action in payload
-        request = responses.calls[0].request
-        import json
-
-        payload = json.loads(request.body)
-        assert "actions" in payload
-        assert len(payload["actions"]) == 1
-        assert payload["actions"][0]["type"] == "upload_s3"
-        assert payload["actions"][0]["bucket"] == "my-bucket"
-
     @responses.activate
     def test_crawl_async_with_max_age(self, client):
         """Test crawl_async includes max_age when provided."""

{webcrawlerapi-2.1.2 → webcrawlerapi-2.1.5}/webcrawlerapi/__init__.py

@@ -30,6 +30,7 @@ from .models import (
     Job,
     JobItem,
     JobMarkdownResponse,
+    MarkdownResponse,
     ScrapeId,
     ScrapeResponse,
     ScrapeResponseError,
@@ -44,6 +45,7 @@ __all__ = [
     "Job",
     "JobItem",
     "JobMarkdownResponse",
+    "MarkdownResponse",
     "CrawlResponse",
     "ScrapeId",
     "ScrapeResponse",

{webcrawlerapi-2.1.2 → webcrawlerapi-2.1.5}/webcrawlerapi/client.py

@@ -10,6 +10,7 @@ from .models import (
     CrawlResponse,
     Job,
     JobMarkdownResponse,
+    MarkdownResponse,
     ScrapeId,
     ScrapeResponse,
     ScrapeResponseError,
@@ -68,7 +69,6 @@ class WebCrawlerAPI:
         webhook_url: Optional[str] = None,
         whitelist_regexp: Optional[str] = None,
         blacklist_regexp: Optional[str] = None,
-        actions: Optional[Union[Action, List[Action]]] = None,
         respect_robots_txt: bool = False,
         main_content_only: bool = False,
         max_depth: Optional[int] = None,
@@ -87,7 +87,6 @@ class WebCrawlerAPI:
             webhook_url (str, optional): URL for webhook notifications
             whitelist_regexp (str, optional): Regex pattern for URL whitelist
             blacklist_regexp (str, optional): Regex pattern for URL blacklist
-            actions (Action or List[Action], optional): Actions to perform during crawling
             respect_robots_txt (bool): Whether to respect robots.txt (default: False)
             main_content_only (bool): Whether to extract only main content (default: False)
             max_depth (int, optional): Maximum crawl depth (0 = seed only, 1 = seed + direct links)
@@ -131,9 +130,6 @@ class WebCrawlerAPI:
             payload["main_content_only"] = main_content_only
         if keep_query_params is not None:
             payload["keep_query_params"] = keep_query_params
-        if actions:
-            action_list = [actions] if not isinstance(actions, list) else actions
-            payload["actions"] = [vars(action) for action in action_list]
 
         response = self.session.post(
             urljoin(self.base_url, f"/{CRAWLER_VERSION}/crawl"), json=payload
@@ -206,6 +202,31 @@ class WebCrawlerAPI:
             self._raise_for_error(response)
         return response.text
 
+    def markdown(self, url: str) -> MarkdownResponse:
+        """
+        Extract cleaned article markdown (main content only) from a webpage.
+
+        Args:
+            url (str): The URL of the webpage to extract the article markdown from
+
+        Returns:
+            MarkdownResponse: Response containing the extracted markdown
+
+        Raises:
+            WebCrawlerApiError: If the API returns an error response
+            requests.exceptions.RequestException: If the HTTP request fails
+        """
+        response = self.session.post(
+            urljoin(self.base_url, "/markdown"),
+            json={"url": url},
+        )
+        if not response.ok:
+            self._raise_for_error(response)
+        data = response.json()
+        return MarkdownResponse(
+            success=data.get("success", True), markdown=data.get("markdown")
+        )
+
     def cancel_job(self, job_id: str) -> Dict[str, str]:
         """
         Cancel a running job. All items that are not in progress and not done
@@ -237,7 +258,6 @@ class WebCrawlerAPI:
         webhook_url: Optional[str] = None,
         whitelist_regexp: Optional[str] = None,
         blacklist_regexp: Optional[str] = None,
-        actions: Optional[Union[Action, List[Action]]] = None,
         respect_robots_txt: bool = False,
         main_content_only: bool = False,
         max_depth: Optional[int] = None,
@@ -256,7 +276,6 @@ class WebCrawlerAPI:
             webhook_url (str, optional): URL for webhook notifications
             whitelist_regexp (str, optional): Regex pattern for URL whitelist
             blacklist_regexp (str, optional): Regex pattern for URL blacklist
-            actions (Action or List[Action], optional): Actions to perform during crawling
             respect_robots_txt (bool): Whether to respect robots.txt (default: False)
             main_content_only (bool): Whether to extract only main content (default: False)
             max_depth (int, optional): Maximum crawl depth
@@ -279,7 +298,6 @@ class WebCrawlerAPI:
             webhook_url=webhook_url,
             whitelist_regexp=whitelist_regexp,
             blacklist_regexp=blacklist_regexp,
-            actions=actions,
             respect_robots_txt=respect_robots_txt,
             main_content_only=main_content_only,
             max_depth=max_depth,
@@ -315,7 +333,6 @@ class WebCrawlerAPI:
         webhook_url: Optional[str] = None,
         whitelist_regexp: Optional[str] = None,
         blacklist_regexp: Optional[str] = None,
-        actions: Optional[Union[Action, List[Action]]] = None,
         respect_robots_txt: bool = False,
         main_content_only: bool = False,
         max_depth: Optional[int] = None,
@@ -340,7 +357,6 @@ class WebCrawlerAPI:
             webhook_url=webhook_url,
             whitelist_regexp=whitelist_regexp,
             blacklist_regexp=blacklist_regexp,
-            actions=actions,
             respect_robots_txt=respect_robots_txt,
             main_content_only=main_content_only,
             max_depth=max_depth,

{webcrawlerapi-2.1.2 → webcrawlerapi-2.1.5}/webcrawlerapi/models.py

@@ -62,6 +62,14 @@ class JobMarkdownResponse:
     content_url: str
 
 
+@dataclass
+class MarkdownResponse:
+    """Response from the markdown endpoint."""
+
+    success: bool
+    markdown: Optional[str] = None
+
+
 @dataclass
 class ScrapeId:
     """Response from an asynchronous scrape request."""

{webcrawlerapi-2.1.2 → webcrawlerapi-2.1.5}/webcrawlerapi.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webcrawlerapi
-Version: 2.1.2
+Version: 2.1.5
 Summary: Python SDK for WebCrawler API
 Home-page: https://github.com/webcrawlerapi/webcrawlerapi-python-sdk
 Author: Andrew