webcrawlerapi 2.0.7__tar.gz → 2.0.9__tar.gz

{webcrawlerapi-2.0.7 → webcrawlerapi-2.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webcrawlerapi
-Version: 2.0.7
+Version: 2.0.9
 Summary: Python SDK for WebCrawler API
 Home-page: https://github.com/webcrawlerapi/webcrawlerapi-python-sdk
 Author: Andrew
@@ -124,6 +124,8 @@ Starts a new crawling job and waits for its completion. This method will continu
 ### crawl_async()
 Starts a new crawling job and returns immediately with a job ID. Use this when you want to handle polling and status checks yourself, or when using webhooks.
 
+Use `crawl_raw_markdown()` when you need the combined `/job/{id}/markdown` output after a crawl finishes.
+
 ### get_job()
 Retrieves the current status and details of a specific job.
 

{webcrawlerapi-2.0.7 → webcrawlerapi-2.0.9}/README.md

@@ -103,6 +103,8 @@ Starts a new crawling job and waits for its completion. This method will continu
 ### crawl_async()
 Starts a new crawling job and returns immediately with a job ID. Use this when you want to handle polling and status checks yourself, or when using webhooks.
 
+Use `crawl_raw_markdown()` when you need the combined `/job/{id}/markdown` output after a crawl finishes.
+
 ### get_job()
 Retrieves the current status and details of a specific job.
 
@@ -181,4 +183,4 @@ Each JobItem object represents a crawled page and contains:
 
 ## License
 
-MIT License 
+MIT License 

{webcrawlerapi-2.0.7 → webcrawlerapi-2.0.9}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="webcrawlerapi",
-    version="2.0.7",
+    version="2.0.9",
     packages=find_packages(),
     install_requires=[
         "requests>=2.25.0",

{webcrawlerapi-2.0.7 → webcrawlerapi-2.0.9}/webcrawlerapi/client.py

@@ -50,6 +50,7 @@ class WebCrawlerAPI:
         actions: Optional[Union[Action, List[Action]]] = None,
         respect_robots_txt: bool = False,
         main_content_only: bool = False,
+        max_depth: Optional[int] = None,
     ) -> CrawlResponse:
         """
         Start a new crawling job asynchronously.
@@ -65,6 +66,7 @@ class WebCrawlerAPI:
             actions (Action or List[Action], optional): Actions to perform during crawling
             respect_robots_txt (bool): Whether to respect robots.txt file (default: False)
             main_content_only (bool): Whether to extract only main content (default: False)
+            max_depth (int, optional): Maximum depth of crawl (0 for seed URL only, 1 for one level deep, etc.)
 
         Returns:
             CrawlResponse: Response containing the job ID
@@ -87,6 +89,8 @@ class WebCrawlerAPI:
             payload["whitelist_regexp"] = whitelist_regexp
         if blacklist_regexp:
             payload["blacklist_regexp"] = blacklist_regexp
+        if max_depth is not None:
+            payload["max_depth"] = max_depth
         if actions:
             # Convert single action to list if needed
             action_list = [actions] if not isinstance(actions, list) else actions
@@ -118,6 +122,42 @@ class WebCrawlerAPI:
         response.raise_for_status()
         return Job(response.json())
 
+    def get_job_markdown(self, job_id: str) -> str:
+        """
+        Get combined markdown content for a completed markdown job.
+
+        Args:
+            job_id (str): The unique identifier of the job
+
+        Returns:
+            str: Combined markdown content
+
+        Raises:
+            requests.exceptions.RequestException: If the API request fails
+        """
+        response = self.session.get(
+            urljoin(self.base_url, f"/{CRAWLER_VERSION}/job/{job_id}/markdown")
+        )
+
+        if not response.ok:
+            try:
+                error_payload = response.json()
+                detail = (
+                    error_payload.get("message")
+                    or error_payload.get("error_message")
+                    or error_payload.get("error")
+                )
+            except ValueError:
+                detail = response.text
+
+            try:
+                response.raise_for_status()
+            except requests.exceptions.HTTPError as exc:
+                raise requests.exceptions.HTTPError(f"{exc}: {detail}") from exc
+
+        response.raise_for_status()
+        return response.text
+
     def cancel_job(self, job_id: str) -> Dict[str, str]:
         """
         Cancel a running job. All items that are not in progress and not done
@@ -150,6 +190,7 @@ class WebCrawlerAPI:
         actions: Optional[Union[Action, List[Action]]] = None,
         respect_robots_txt: bool = False,
         main_content_only: bool = False,
+        max_depth: Optional[int] = None,
         max_polls: int = 100,
     ) -> Job:
         """
@@ -170,6 +211,7 @@ class WebCrawlerAPI:
             actions (Action or List[Action], optional): Actions to perform during crawling
             respect_robots_txt (bool): Whether to respect robots.txt file (default: False)
             main_content_only (bool): Whether to extract only main content (default: False)
+            max_depth (int, optional): Maximum depth of crawl (0 for seed URL only, 1 for one level deep, etc.)
             max_polls (int): Maximum number of status checks before returning (default: 100)
 
         Returns:
@@ -190,6 +232,7 @@ class WebCrawlerAPI:
             actions=actions,
             respect_robots_txt=respect_robots_txt,
             main_content_only=main_content_only,
+            max_depth=max_depth,
         )
 
         job_id = response.id
@@ -215,6 +258,54 @@ class WebCrawlerAPI:
         # Return the last known state if max_polls is reached
         return job
 
+    def crawl_raw_markdown(
+        self,
+        url: str,
+        scrape_type: str = "markdown",
+        items_limit: int = 10,
+        webhook_url: Optional[str] = None,
+        allow_subdomains: bool = False,
+        whitelist_regexp: Optional[str] = None,
+        blacklist_regexp: Optional[str] = None,
+        actions: Optional[Union[Action, List[Action]]] = None,
+        respect_robots_txt: bool = False,
+        main_content_only: bool = False,
+        max_depth: Optional[int] = None,
+        max_polls: int = 100,
+    ) -> str:
+        """
+        Run a crawl job and return the combined markdown output when finished.
+
+        Raises:
+            requests.exceptions.RequestException: If any API request fails
+        """
+        job = self.crawl(
+            url=url,
+            scrape_type=scrape_type,
+            items_limit=items_limit,
+            webhook_url=webhook_url,
+            allow_subdomains=allow_subdomains,
+            whitelist_regexp=whitelist_regexp,
+            blacklist_regexp=blacklist_regexp,
+            actions=actions,
+            respect_robots_txt=respect_robots_txt,
+            main_content_only=main_content_only,
+            max_depth=max_depth,
+            max_polls=max_polls,
+        )
+
+        if job.scrape_type != "markdown":
+            raise requests.exceptions.HTTPError(
+                "crawl_raw_markdown requires scrape_type to be markdown"
+            )
+
+        if job.status != "done":
+            raise requests.exceptions.HTTPError(
+                f"Job finished with status {job.status}"
+            )
+
+        return self.get_job_markdown(job.id)
+
     def scrape_async(
         self,
         url: str,

{webcrawlerapi-2.0.7 → webcrawlerapi-2.0.9}/webcrawlerapi/models.py

@@ -125,6 +125,7 @@ class JobItem:
         self.referred_url: Optional[str] = data.get("referred_url")
         self.last_error: Optional[str] = data.get("last_error")
         self.error_code: Optional[str] = data.get("error_code")
+        self.depth: Optional[int] = data.get("depth")
 
         # Optional content URLs based on scrape_type
         self.raw_content_url: Optional[str] = data.get("raw_content_url")
@@ -201,6 +202,7 @@ class Job:
         self.blacklist_regexp: Optional[str] = data.get("blacklist_regexp")
         self.allow_subdomains: bool = data.get("allow_subdomains", False)
         self.items_limit: int = data["items_limit"]
+        self.max_depth: Optional[int] = data.get("max_depth")
         self.created_at: datetime = parse_datetime(data["created_at"])
         self.updated_at: datetime = parse_datetime(data["updated_at"])
         self.webhook_url: Optional[str] = data.get("webhook_url")

{webcrawlerapi-2.0.7 → webcrawlerapi-2.0.9}/webcrawlerapi.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webcrawlerapi
-Version: 2.0.7
+Version: 2.0.9
 Summary: Python SDK for WebCrawler API
 Home-page: https://github.com/webcrawlerapi/webcrawlerapi-python-sdk
 Author: Andrew
@@ -124,6 +124,8 @@ Starts a new crawling job and waits for its completion. This method will continu
 ### crawl_async()
 Starts a new crawling job and returns immediately with a job ID. Use this when you want to handle polling and status checks yourself, or when using webhooks.
 
+Use `crawl_raw_markdown()` when you need the combined `/job/{id}/markdown` output after a crawl finishes.
+
 ### get_job()
 Retrieves the current status and details of a specific job.