webcrawlerapi 2.1.3__tar.gz → 2.1.6__tar.gz

{webcrawlerapi-2.1.3 → webcrawlerapi-2.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webcrawlerapi
-Version: 2.1.3
+Version: 2.1.6
 Summary: Python SDK for WebCrawler API
 Home-page: https://github.com/webcrawlerapi/webcrawlerapi-python-sdk
 Author: Andrew

{webcrawlerapi-2.1.3 → webcrawlerapi-2.1.6}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="webcrawlerapi",
-    version="2.1.3",
+    version="2.1.6",
     packages=find_packages(),
     install_requires=[
         "requests>=2.25.0",

{webcrawlerapi-2.1.3 → webcrawlerapi-2.1.6}/tests/test_models.py

@@ -428,4 +428,4 @@ class TestJob:
 
     def test_job_terminal_statuses_constant(self):
         """Test that TERMINAL_STATUSES constant is correctly defined."""
-        assert Job.TERMINAL_STATUSES == {"done", "error", "cancelled"}
+        assert Job.TERMINAL_STATUSES == {"done", "error", "cancelled", "canceled"}

{webcrawlerapi-2.1.3 → webcrawlerapi-2.1.6}/webcrawlerapi/__init__.py

@@ -30,6 +30,7 @@ from .models import (
     Job,
     JobItem,
     JobMarkdownResponse,
+    MarkdownResponse,
     ScrapeId,
     ScrapeResponse,
     ScrapeResponseError,
@@ -44,6 +45,7 @@ __all__ = [
     "Job",
     "JobItem",
     "JobMarkdownResponse",
+    "MarkdownResponse",
     "CrawlResponse",
     "ScrapeId",
     "ScrapeResponse",

{webcrawlerapi-2.1.3 → webcrawlerapi-2.1.6}/webcrawlerapi/client.py

@@ -10,6 +10,7 @@ from .models import (
     CrawlResponse,
     Job,
     JobMarkdownResponse,
+    MarkdownResponse,
     ScrapeId,
     ScrapeResponse,
     ScrapeResponseError,
@@ -201,6 +202,31 @@ class WebCrawlerAPI:
             self._raise_for_error(response)
         return response.text
 
+    def markdown(self, url: str) -> MarkdownResponse:
+        """
+        Extract cleaned article markdown (main content only) from a webpage.
+
+        Args:
+            url (str): The URL of the webpage to extract the article markdown from
+
+        Returns:
+            MarkdownResponse: Response containing the extracted markdown
+
+        Raises:
+            WebCrawlerApiError: If the API returns an error response
+            requests.exceptions.RequestException: If the HTTP request fails
+        """
+        response = self.session.post(
+            urljoin(self.base_url, "/markdown"),
+            json={"url": url},
+        )
+        if not response.ok:
+            self._raise_for_error(response)
+        data = response.json()
+        return MarkdownResponse(
+            success=data.get("success", True), markdown=data.get("markdown")
+        )
+
     def cancel_job(self, job_id: str) -> Dict[str, str]:
         """
         Cancel a running job. All items that are not in progress and not done

{webcrawlerapi-2.1.3 → webcrawlerapi-2.1.6}/webcrawlerapi/models.py

@@ -62,6 +62,14 @@ class JobMarkdownResponse:
     content_url: str
 
 
+@dataclass
+class MarkdownResponse:
+    """Response from the markdown endpoint."""
+
+    success: bool
+    markdown: Optional[str] = None
+
+
 @dataclass
 class ScrapeId:
     """Response from an asynchronous scrape request."""
@@ -276,7 +284,7 @@ class JobItem:
 class Job:
     """Represents a crawling job."""
 
-    TERMINAL_STATUSES = {"done", "error", "cancelled"}
+    TERMINAL_STATUSES = {"done", "error", "cancelled", "canceled"}
 
     def __init__(self, data: Dict[str, Any]):
         self.id: str = data["id"]
@@ -309,5 +317,5 @@ class Job:
 
     @property
     def is_terminal(self) -> bool:
-        """Check if the job is in a terminal state (done, error, or cancelled)."""
+        """Check if the job is in a terminal state (done, error, or cancelled/canceled)."""
         return self.status in self.TERMINAL_STATUSES

{webcrawlerapi-2.1.3 → webcrawlerapi-2.1.6}/webcrawlerapi.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webcrawlerapi
-Version: 2.1.3
+Version: 2.1.6
 Summary: Python SDK for WebCrawler API
 Home-page: https://github.com/webcrawlerapi/webcrawlerapi-python-sdk
 Author: Andrew