hyperbrowser 0.5.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

hyperbrowser/client/async_client.py

@@ -1,5 +1,11 @@
 from typing import Optional
 
+from hyperbrowser.models.crawl import (
+    CrawlJobResponse,
+    GetCrawlJobParams,
+    StartCrawlJobParams,
+    StartCrawlJobResponse,
+)
 from hyperbrowser.models.scrape import (
     ScrapeJobResponse,
     StartScrapeJobParams,
@@ -61,9 +67,26 @@ class AsyncHyperbrowser(HyperbrowserBase):
         return StartScrapeJobResponse(**response.data)
 
     async def get_scrape_job(self, job_id: str) -> ScrapeJobResponse:
-        response = await self.transport.get(self._build_url(f"/api/scrape/{job_id}"))
+        response = await self.transport.get(self._build_url(f"/scrape/{job_id}"))
         return ScrapeJobResponse(**response.data)
 
+    async def start_crawl_job(
+        self, params: StartCrawlJobParams
+    ) -> StartCrawlJobResponse:
+        response = await self.transport.post(
+            self._build_url("/crawl"),
+            data=params.model_dump(exclude_none=True, by_alias=True),
+        )
+        return StartCrawlJobResponse(**response.data)
+
+    async def get_crawl_job(
+        self, job_id: str, params: GetCrawlJobParams = GetCrawlJobParams()
+    ) -> CrawlJobResponse:
+        response = await self.transport.get(
+            self._build_url(f"/crawl/{job_id}"), params=params.__dict__
+        )
+        return CrawlJobResponse(**response.data)
+
     async def close(self) -> None:
         await self.transport.close()
 

hyperbrowser/client/sync.py

@@ -1,5 +1,11 @@
 from typing import Optional
 
+from hyperbrowser.models.crawl import (
+    CrawlJobResponse,
+    GetCrawlJobParams,
+    StartCrawlJobParams,
+    StartCrawlJobResponse,
+)
 from hyperbrowser.models.scrape import (
     ScrapeJobResponse,
     StartScrapeJobParams,
@@ -57,8 +63,23 @@ class Hyperbrowser(HyperbrowserBase):
         return StartScrapeJobResponse(**response.data)
 
     def get_scrape_job(self, job_id: str) -> ScrapeJobResponse:
-        response = self.transport.get(self._build_url(f"/api/scrape/{job_id}"))
+        response = self.transport.get(self._build_url(f"/scrape/{job_id}"))
         return ScrapeJobResponse(**response.data)
 
+    def start_crawl_job(self, params: StartCrawlJobParams) -> StartCrawlJobResponse:
+        response = self.transport.post(
+            self._build_url("/crawl"),
+            data=params.model_dump(exclude_none=True, by_alias=True),
+        )
+        return StartCrawlJobResponse(**response.data)
+
+    def get_crawl_job(
+        self, job_id: str, params: GetCrawlJobParams = GetCrawlJobParams()
+    ) -> CrawlJobResponse:
+        response = self.transport.get(
+            self._build_url(f"/crawl/{job_id}"), params=params.__dict__
+        )
+        return CrawlJobResponse(**response.data)
+
     def close(self) -> None:
         self.transport.close()

hyperbrowser/models/crawl.py

@@ -0,0 +1,98 @@
+from typing import List, Literal, Optional
+from pydantic import BaseModel, ConfigDict, Field
+
+CrawlJobStatus = Literal["pending", "running", "completed", "failed"]
+
+
+class StartCrawlJobParams(BaseModel):
+    """
+    Parameters for creating a new crawl job.
+    """
+
+    model_config = ConfigDict(
+        populate_by_alias=True,
+    )
+
+    url: str
+    max_pages: int = Field(default=10, ge=1, le=50, serialization_alias="maxPages")
+    follow_links: bool = Field(default=True, serialization_alias="followLinks")
+    exclude_patterns: List[str] = Field(
+        default=[], serialization_alias="excludePatterns"
+    )
+    include_patterns: List[str] = Field(
+        default=[], serialization_alias="includePatterns"
+    )
+    use_proxy: bool = Field(default=False, serialization_alias="useProxy")
+    solve_captchas: bool = Field(default=False, serialization_alias="solveCaptchas")
+
+
+class StartCrawlJobResponse(BaseModel):
+    """
+    Response from creating a crawl job.
+    """
+
+    model_config = ConfigDict(
+        populate_by_alias=True,
+    )
+
+    job_id: str = Field(alias="jobId")
+
+
+class CrawledPageMetadata(BaseModel):
+    """
+    Metadata for the crawled page.
+    """
+
+    model_config = ConfigDict(
+        populate_by_alias=True,
+    )
+
+    title: str
+    description: str
+    robots: str
+    og_title: str = Field(alias="ogTitle")
+    og_description: str = Field(alias="ogDescription")
+    og_url: str = Field(alias="ogUrl")
+    og_image: str = Field(alias="ogImage")
+    og_locale_alternate: List[str] = Field(alias="ogLocaleAlternate")
+    og_site_name: str = Field(alias="ogSiteName")
+    source_url: str = Field(alias="sourceURL")
+
+
+class CrawledPage(BaseModel):
+    """
+    Data from a crawled page.
+    """
+
+    metadata: CrawledPageMetadata
+    markdown: str
+    url: str
+
+
+class GetCrawlJobParams(BaseModel):
+    """
+    Parameters for getting a crawl job.
+    """
+
+    page: Optional[int] = Field(default=None, serialization_alias="page")
+    batch_size: Optional[int] = Field(
+        default=20, ge=1, le=50, serialization_alias="batchSize"
+    )
+
+
+class CrawlJobResponse(BaseModel):
+    """
+    Response from getting a crawl job.
+    """
+
+    model_config = ConfigDict(
+        populate_by_alias=True,
+    )
+
+    status: CrawlJobStatus
+    error: Optional[str] = None
+    data: List[CrawledPage] = Field(alias="data")
+    total_crawled_pages: int = Field(alias="totalCrawledPages")
+    total_page_batches: int = Field(alias="totalPageBatches")
+    current_page_batch: int = Field(alias="currentPageBatch")
+    batch_size: int = Field(alias="batchSize")

hyperbrowser/models/scrape.py

@@ -1,4 +1,4 @@
-from typing import Literal, Optional
+from typing import List, Literal, Optional
 from pydantic import BaseModel, ConfigDict, Field
 
 ScrapeJobStatus = Literal["pending", "running", "completed", "failed"]
@@ -14,6 +14,8 @@ class StartScrapeJobParams(BaseModel):
     )
 
     url: str
+    use_proxy: bool = Field(default=False, serialization_alias="useProxy")
+    solve_captchas: bool = Field(default=False, serialization_alias="solveCaptchas")
 
 
 class StartScrapeJobResponse(BaseModel):
@@ -44,7 +46,7 @@ class ScrapeJobMetadata(BaseModel):
     og_description: str = Field(alias="ogDescription")
     og_url: str = Field(alias="ogUrl")
     og_image: str = Field(alias="ogImage")
-    og_locale_alternate: list[str] = Field(alias="ogLocaleAlternate")
+    og_locale_alternate: List[str] = Field(alias="ogLocaleAlternate")
     og_site_name: str = Field(alias="ogSiteName")
     source_url: str = Field(alias="sourceURL")
 

{hyperbrowser-0.5.0.dist-info → hyperbrowser-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hyperbrowser
-Version: 0.5.0
+Version: 0.7.0
 Summary: Python SDK for hyperbrowser
 Home-page: https://github.com/hyperbrowserai/python-sdk
 License: MIT

{hyperbrowser-0.5.0.dist-info → hyperbrowser-0.7.0.dist-info}/RECORD

@@ -1,17 +1,18 @@
 LICENSE,sha256=6rUGKlyKb_1ZAH7h7YITYAAUNFN3MNGGKCyfrw49NLE,1071
 hyperbrowser/__init__.py,sha256=zWGcLhqhvWy6BTwuNpzWK1-0LpIn311ks-4U9nrsb7Y,187
-hyperbrowser/client/async_client.py,sha256=Elja054YwFyAgc43aQ1R21Y8c_adsIJ4ueg6gWYYIME,2457
+hyperbrowser/client/async_client.py,sha256=nmbbqaIEOWk01-mqBCYkg8hU01_sQLrAJNiPaWfINEI,3225
 hyperbrowser/client/base.py,sha256=9gFma7RdvJBUlDCqr8tZd315UPrjn4ldU4B0-Y-L4O4,1268
-hyperbrowser/client/sync.py,sha256=47WzDbNAHV6WxaA7Ph7FVoI-kbmolFKv30Dha1C4fF8,2165
+hyperbrowser/client/sync.py,sha256=2ZembYQu7h0ph57jYFUH6ytXA0ebohceu39PevwSWaM,2895
 hyperbrowser/config.py,sha256=2J6GYNR_83vzJZ6jEV-LXO1U-q6DHIrfyAU0WrUPhw8,625
 hyperbrowser/exceptions.py,sha256=SUUkptK2OL36xDORYmSicaTYR7pMbxeWAjAgz35xnM8,1171
 hyperbrowser/models/consts.py,sha256=VmtqbXqK6WTvlD5XExL3e2JE3WaFTi_iniEAQlRSQgs,4917
-hyperbrowser/models/scrape.py,sha256=PoqdmF2EIBSJuyNcrt3NtZ0jSnjVB_EqLBXYC2RQ2_Q,1512
+hyperbrowser/models/crawl.py,sha256=-u0pJ28sNjyycfbuLHjuA5bftDtkV60ZFvek7Z510ao,2582
+hyperbrowser/models/scrape.py,sha256=JIS6zbHlpv-U1hc9qVYeCazXYHBiRzjQX6y_TXsl4js,1678
 hyperbrowser/models/session.py,sha256=N05NLI0NFul7uQPkLihOv82-JCjXkWW8hlMbQsPZMvo,4173
 hyperbrowser/transport/async_transport.py,sha256=P-nX9iczGVYJyvqtqlGAOFQ3PghRC2_bE6Lruiiecn0,3511
 hyperbrowser/transport/base.py,sha256=9l7k-qTX4Q2KaZIR_fwsNlxDgOzsmc8zgucZ9tfHgkw,1622
 hyperbrowser/transport/sync.py,sha256=DFDPYqF-_WQSZkRbWDRFTPowQMzz-B3N869r2vvocPc,2829
-hyperbrowser-0.5.0.dist-info/LICENSE,sha256=6rUGKlyKb_1ZAH7h7YITYAAUNFN3MNGGKCyfrw49NLE,1071
-hyperbrowser-0.5.0.dist-info/METADATA,sha256=X8aMEPn5yli9jf8a3Gv2rI7zL4VsYdGNSsnvyT0_Jpg,3289
-hyperbrowser-0.5.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-hyperbrowser-0.5.0.dist-info/RECORD,,
+hyperbrowser-0.7.0.dist-info/LICENSE,sha256=6rUGKlyKb_1ZAH7h7YITYAAUNFN3MNGGKCyfrw49NLE,1071
+hyperbrowser-0.7.0.dist-info/METADATA,sha256=tgrKlcUojlBC7D1cClvMIMiA7_krPD7NGpxyuHLGLA0,3289
+hyperbrowser-0.7.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+hyperbrowser-0.7.0.dist-info/RECORD,,