hyperbrowser 0.32.0__py3-none-any.whl → 0.34.0__py3-none-any.whl

hyperbrowser/client/async_client.py

@@ -1,15 +1,15 @@
 from typing import Optional
 
+from ..config import ClientConfig
+from ..transport.async_transport import AsyncTransport
+from .base import HyperbrowserBase
+from .managers.async_manager.agents import Agents
+from .managers.async_manager.crawl import CrawlManager
+from .managers.async_manager.extension import ExtensionManager
 from .managers.async_manager.extract import ExtractManager
 from .managers.async_manager.profile import ProfileManager
-from .managers.async_manager.session import SessionManager
 from .managers.async_manager.scrape import ScrapeManager
-from .managers.async_manager.crawl import CrawlManager
-from .managers.async_manager.extension import ExtensionManager
-from .managers.async_manager.beta import Beta
-from .base import HyperbrowserBase
-from ..transport.async_transport import AsyncTransport
-from ..config import ClientConfig
+from .managers.async_manager.session import SessionManager
 
 
 class AsyncHyperbrowser(HyperbrowserBase):
@@ -30,7 +30,7 @@ class AsyncHyperbrowser(HyperbrowserBase):
         self.extract = ExtractManager(self)
         self.profiles = ProfileManager(self)
         self.extensions = ExtensionManager(self)
-        self.beta = Beta(self)
+        self.agents = Agents(self)
 
     async def close(self) -> None:
         await self.transport.close()

hyperbrowser/client/managers/async_manager/{beta/agents → agents}/__init__.py

@@ -1,5 +1,6 @@
+from .browser_use import BrowserUseManager
+
+
 class Agents:
     def __init__(self, client):
-        from .browser_use import BrowserUseManager
-
         self.browser_use = BrowserUseManager(client)

hyperbrowser/client/managers/async_manager/{beta/agents → agents}/browser_use.py

@@ -1,12 +1,14 @@
 import asyncio
+
 from hyperbrowser.exceptions import HyperbrowserError
-from ......models import (
+
+from .....models import (
     POLLING_ATTEMPTS,
     BasicResponse,
+    BrowserUseTaskResponse,
+    BrowserUseTaskStatusResponse,
     StartBrowserUseTaskParams,
     StartBrowserUseTaskResponse,
-    BrowserUseTaskStatusResponse,
-    BrowserUseTaskResponse,
 )
 
 

hyperbrowser/client/managers/async_manager/crawl.py

@@ -3,6 +3,8 @@ import asyncio
 from hyperbrowser.models.consts import POLLING_ATTEMPTS
 from ....models.crawl import (
     CrawlJobResponse,
+    CrawlJobStatus,
+    CrawlJobStatusResponse,
     GetCrawlJobParams,
     StartCrawlJobParams,
     StartCrawlJobResponse,
@@ -21,11 +23,18 @@ class CrawlManager:
         )
         return StartCrawlJobResponse(**response.data)
 
+    async def get_status(self, job_id: str) -> CrawlJobStatusResponse:
+        response = await self._client.transport.get(
+            self._client._build_url(f"/crawl/{job_id}/status")
+        )
+        return CrawlJobStatusResponse(**response.data)
+
     async def get(
         self, job_id: str, params: GetCrawlJobParams = GetCrawlJobParams()
     ) -> CrawlJobResponse:
         response = await self._client.transport.get(
-            self._client._build_url(f"/crawl/{job_id}"), params=params.__dict__
+            self._client._build_url(f"/crawl/{job_id}"),
+            params=params.model_dump(exclude_none=True, by_alias=True),
         )
         return CrawlJobResponse(**response.data)
 
@@ -37,18 +46,13 @@ class CrawlManager:
         if not job_id:
             raise HyperbrowserError("Failed to start crawl job")
 
-        job_response: CrawlJobResponse
+        job_status: CrawlJobStatus = "pending"
         failures = 0
         while True:
             try:
-                job_response = await self.get(
-                    job_id,
-                    params=GetCrawlJobParams(batch_size=1),
-                )
-                if (
-                    job_response.status == "completed"
-                    or job_response.status == "failed"
-                ):
+                job_status_resp = await self.get_status(job_id)
+                job_status = job_status_resp.status
+                if job_status == "completed" or job_status == "failed":
                     break
             except Exception as e:
                 failures += 1
@@ -62,8 +66,7 @@ class CrawlManager:
         if not return_all_pages:
             while True:
                 try:
-                    job_response = await self.get(job_id)
-                    return job_response
+                    return await self.get(job_id)
                 except Exception as e:
                     failures += 1
                     if failures >= POLLING_ATTEMPTS:
@@ -73,9 +76,20 @@ class CrawlManager:
                 await asyncio.sleep(0.5)
 
         failures = 0
-        job_response.current_page_batch = 0
-        job_response.data = []
-        while job_response.current_page_batch < job_response.total_page_batches:
+        job_response = CrawlJobResponse(
+            jobId=job_id,
+            status=job_status,
+            data=[],
+            currentPageBatch=0,
+            totalPageBatches=0,
+            totalCrawledPages=0,
+            batchSize=100,
+        )
+        first_check = True
+        while (
+            first_check
+            or job_response.current_page_batch < job_response.total_page_batches
+        ):
             try:
                 tmp_job_response = await self.get(
                     job_start_resp.job_id,
@@ -90,6 +104,7 @@ class CrawlManager:
                 job_response.total_page_batches = tmp_job_response.total_page_batches
                 job_response.batch_size = tmp_job_response.batch_size
                 failures = 0
+                first_check = False
             except Exception as e:
                 failures += 1
                 if failures >= POLLING_ATTEMPTS:

hyperbrowser/client/managers/async_manager/extract.py

@@ -3,9 +3,11 @@ from hyperbrowser.exceptions import HyperbrowserError
 from hyperbrowser.models.consts import POLLING_ATTEMPTS
 from hyperbrowser.models.extract import (
     ExtractJobResponse,
+    ExtractJobStatusResponse,
     StartExtractJobParams,
     StartExtractJobResponse,
 )
+import jsonref
 
 
 class ExtractManager:
@@ -17,7 +19,9 @@ class ExtractManager:
             raise HyperbrowserError("Either schema or prompt must be provided")
         if params.schema_:
             if hasattr(params.schema_, "model_json_schema"):
-                params.schema_ = params.schema_.model_json_schema()
+                params.schema_ = jsonref.replace_refs(
+                    params.schema_.model_json_schema(), proxies=False, lazy_load=False
+                )
 
         response = await self._client.transport.post(
             self._client._build_url("/extract"),
@@ -25,6 +29,12 @@ class ExtractManager:
         )
         return StartExtractJobResponse(**response.data)
 
+    async def get_status(self, job_id: str) -> ExtractJobStatusResponse:
+        response = await self._client.transport.get(
+            self._client._build_url(f"/extract/{job_id}/status")
+        )
+        return ExtractJobStatusResponse(**response.data)
+
     async def get(self, job_id: str) -> ExtractJobResponse:
         response = await self._client.transport.get(
             self._client._build_url(f"/extract/{job_id}")
@@ -40,12 +50,10 @@ class ExtractManager:
         failures = 0
         while True:
             try:
-                job_response = await self.get(job_id)
-                if (
-                    job_response.status == "completed"
-                    or job_response.status == "failed"
-                ):
-                    return job_response
+                job_status_resp = await self.get_status(job_id)
+                job_status = job_status_resp.status
+                if job_status == "completed" or job_status == "failed":
+                    return await self.get(job_id)
                 failures = 0
             except Exception as e:
                 failures += 1

hyperbrowser/client/managers/async_manager/profile.py

@@ -33,6 +33,7 @@ class ProfileManager:
         self, params: ProfileListParams = ProfileListParams()
     ) -> ProfileListResponse:
         response = await self._client.transport.get(
-            self._client._build_url("/profiles"), params=params.__dict__
+            self._client._build_url("/profiles"),
+            params=params.model_dump(exclude_none=True, by_alias=True),
         )
         return ProfileListResponse(**response.data)

hyperbrowser/client/managers/async_manager/scrape.py

@@ -4,8 +4,11 @@ from typing import Optional
 from hyperbrowser.models.consts import POLLING_ATTEMPTS
 from ....models.scrape import (
     BatchScrapeJobResponse,
+    BatchScrapeJobStatusResponse,
     GetBatchScrapeJobParams,
     ScrapeJobResponse,
+    ScrapeJobStatus,
+    ScrapeJobStatusResponse,
     StartBatchScrapeJobParams,
     StartBatchScrapeJobResponse,
     StartScrapeJobParams,
@@ -27,11 +30,18 @@ class BatchScrapeManager:
         )
         return StartBatchScrapeJobResponse(**response.data)
 
+    async def get_status(self, job_id: str) -> BatchScrapeJobStatusResponse:
+        response = await self._client.transport.get(
+            self._client._build_url(f"/scrape/batch/{job_id}/status")
+        )
+        return BatchScrapeJobStatusResponse(**response.data)
+
     async def get(
         self, job_id: str, params: GetBatchScrapeJobParams = GetBatchScrapeJobParams()
     ) -> BatchScrapeJobResponse:
         response = await self._client.transport.get(
-            self._client._build_url(f"/scrape/batch/{job_id}"), params=params.__dict__
+            self._client._build_url(f"/scrape/batch/{job_id}"),
+            params=params.model_dump(exclude_none=True, by_alias=True),
         )
         return BatchScrapeJobResponse(**response.data)
 
@@ -43,19 +53,14 @@ class BatchScrapeManager:
         if not job_id:
             raise HyperbrowserError("Failed to start batch scrape job")
 
-        job_response: BatchScrapeJobResponse
+        job_status: ScrapeJobStatus = "pending"
         failures = 0
         while True:
             try:
-                job_response = await self.get(
-                    job_id, params=GetBatchScrapeJobParams(batch_size=1)
-                )
-                if (
-                    job_response.status == "completed"
-                    or job_response.status == "failed"
-                ):
+                job_status_resp = await self.get_status(job_id)
+                job_status = job_status_resp.status
+                if job_status == "completed" or job_status == "failed":
                     break
-                failures = 0
             except Exception as e:
                 failures += 1
                 if failures >= POLLING_ATTEMPTS:
@@ -68,8 +73,7 @@ class BatchScrapeManager:
         if not return_all_pages:
             while True:
                 try:
-                    job_response = await self.get(job_id)
-                    return job_response
+                    return await self.get(job_id)
                 except Exception as e:
                     failures += 1
                     if failures >= POLLING_ATTEMPTS:
@@ -79,9 +83,21 @@ class BatchScrapeManager:
                 await asyncio.sleep(0.5)
 
         failures = 0
-        job_response.current_page_batch = 0
-        job_response.data = []
-        while job_response.current_page_batch < job_response.total_page_batches:
+        job_response = BatchScrapeJobResponse(
+            jobId=job_id,
+            status=job_status,
+            data=[],
+            currentPageBatch=0,
+            totalPageBatches=0,
+            totalScrapedPages=0,
+            batchSize=100,
+        )
+        first_check = True
+
+        while (
+            first_check
+            or job_response.current_page_batch < job_response.total_page_batches
+        ):
             try:
                 tmp_job_response = await self.get(
                     job_id,
@@ -96,6 +112,7 @@ class BatchScrapeManager:
                 job_response.total_page_batches = tmp_job_response.total_page_batches
                 job_response.batch_size = tmp_job_response.batch_size
                 failures = 0
+                first_check = False
             except Exception as e:
                 failures += 1
                 if failures >= POLLING_ATTEMPTS:
@@ -119,6 +136,12 @@ class ScrapeManager:
         )
         return StartScrapeJobResponse(**response.data)
 
+    async def get_status(self, job_id: str) -> ScrapeJobStatusResponse:
+        response = await self._client.transport.get(
+            self._client._build_url(f"/scrape/{job_id}/status")
+        )
+        return ScrapeJobStatusResponse(**response.data)
+
     async def get(self, job_id: str) -> ScrapeJobResponse:
         response = await self._client.transport.get(
             self._client._build_url(f"/scrape/{job_id}")
@@ -134,12 +157,10 @@ class ScrapeManager:
         failures = 0
         while True:
             try:
-                job_response = await self.get(job_id)
-                if (
-                    job_response.status == "completed"
-                    or job_response.status == "failed"
-                ):
-                    return job_response
+                job_status_resp = await self.get_status(job_id)
+                job_status = job_status_resp.status
+                if job_status == "completed" or job_status == "failed":
+                    return await self.get(job_id)
                 failures = 0
             except Exception as e:
                 failures += 1

hyperbrowser/client/managers/async_manager/session.py

@@ -42,7 +42,8 @@ class SessionManager:
         self, params: SessionListParams = SessionListParams()
     ) -> SessionListResponse:
         response = await self._client.transport.get(
-            self._client._build_url("/sessions"), params=params.__dict__
+            self._client._build_url("/sessions"),
+            params=params.model_dump(exclude_none=True, by_alias=True),
         )
         return SessionListResponse(**response.data)
 

hyperbrowser/client/managers/sync_manager/{beta/agents → agents}/__init__.py

@@ -1,5 +1,6 @@
+from .browser_use import BrowserUseManager
+
+
 class Agents:
     def __init__(self, client):
-        from .browser_use import BrowserUseManager
-
         self.browser_use = BrowserUseManager(client)

hyperbrowser/client/managers/sync_manager/{beta/agents → agents}/browser_use.py

@@ -1,12 +1,14 @@
 import time
+
 from hyperbrowser.exceptions import HyperbrowserError
-from ......models import (
+
+from .....models import (
     POLLING_ATTEMPTS,
     BasicResponse,
+    BrowserUseTaskResponse,
+    BrowserUseTaskStatusResponse,
     StartBrowserUseTaskParams,
     StartBrowserUseTaskResponse,
-    BrowserUseTaskStatusResponse,
-    BrowserUseTaskResponse,
 )
 
 

hyperbrowser/client/managers/sync_manager/crawl.py

@@ -4,6 +4,8 @@ from typing import Optional
 from hyperbrowser.models.consts import POLLING_ATTEMPTS
 from ....models.crawl import (
     CrawlJobResponse,
+    CrawlJobStatus,
+    CrawlJobStatusResponse,
     GetCrawlJobParams,
     StartCrawlJobParams,
     StartCrawlJobResponse,
@@ -22,11 +24,18 @@ class CrawlManager:
         )
         return StartCrawlJobResponse(**response.data)
 
+    def get_status(self, job_id: str) -> CrawlJobStatusResponse:
+        response = self._client.transport.get(
+            self._client._build_url(f"/crawl/{job_id}/status")
+        )
+        return CrawlJobStatusResponse(**response.data)
+
     def get(
         self, job_id: str, params: GetCrawlJobParams = GetCrawlJobParams()
     ) -> CrawlJobResponse:
         response = self._client.transport.get(
-            self._client._build_url(f"/crawl/{job_id}"), params=params.__dict__
+            self._client._build_url(f"/crawl/{job_id}"),
+            params=params.model_dump(exclude_none=True, by_alias=True),
         )
         return CrawlJobResponse(**response.data)
 
@@ -38,18 +47,13 @@ class CrawlManager:
         if not job_id:
             raise HyperbrowserError("Failed to start crawl job")
 
-        job_response: CrawlJobResponse
+        job_status: CrawlJobStatus = "pending"
         failures = 0
         while True:
             try:
-                job_response = self.get(
-                    job_id,
-                    params=GetCrawlJobParams(batch_size=1),
-                )
-                if (
-                    job_response.status == "completed"
-                    or job_response.status == "failed"
-                ):
+                job_status_resp = self.get_status(job_id)
+                job_status = job_status_resp.status
+                if job_status == "completed" or job_status == "failed":
                     break
             except Exception as e:
                 failures += 1
@@ -63,8 +67,7 @@ class CrawlManager:
         if not return_all_pages:
             while True:
                 try:
-                    job_response = self.get(job_id)
-                    return job_response
+                    return self.get(job_id)
                 except Exception as e:
                     failures += 1
                     if failures >= POLLING_ATTEMPTS:
@@ -74,12 +77,23 @@ class CrawlManager:
                 time.sleep(0.5)
 
         failures = 0
-        job_response.current_page_batch = 0
-        job_response.data = []
-        while job_response.current_page_batch < job_response.total_page_batches:
+        job_response = CrawlJobResponse(
+            jobId=job_id,
+            status=job_status,
+            data=[],
+            currentPageBatch=0,
+            totalPageBatches=0,
+            totalCrawledPages=0,
+            batchSize=100,
+        )
+        first_check = True
+        while (
+            first_check
+            or job_response.current_page_batch < job_response.total_page_batches
+        ):
             try:
                 tmp_job_response = self.get(
-                    job_id,
+                    job_start_resp.job_id,
                     GetCrawlJobParams(
                         page=job_response.current_page_batch + 1, batch_size=100
                     ),
@@ -91,6 +105,7 @@ class CrawlManager:
                 job_response.total_page_batches = tmp_job_response.total_page_batches
                 job_response.batch_size = tmp_job_response.batch_size
                 failures = 0
+                first_check = False
             except Exception as e:
                 failures += 1
                 if failures >= POLLING_ATTEMPTS:

hyperbrowser/client/managers/sync_manager/extract.py

@@ -3,9 +3,11 @@ from hyperbrowser.exceptions import HyperbrowserError
 from hyperbrowser.models.consts import POLLING_ATTEMPTS
 from hyperbrowser.models.extract import (
     ExtractJobResponse,
+    ExtractJobStatusResponse,
     StartExtractJobParams,
     StartExtractJobResponse,
 )
+import jsonref
 
 
 class ExtractManager:
@@ -17,7 +19,9 @@ class ExtractManager:
             raise HyperbrowserError("Either schema or prompt must be provided")
         if params.schema_:
             if hasattr(params.schema_, "model_json_schema"):
-                params.schema_ = params.schema_.model_json_schema()
+                params.schema_ = jsonref.replace_refs(
+                    params.schema_.model_json_schema(), proxies=False, lazy_load=False
+                )
 
         response = self._client.transport.post(
             self._client._build_url("/extract"),
@@ -25,6 +29,12 @@ class ExtractManager:
         )
         return StartExtractJobResponse(**response.data)
 
+    def get_status(self, job_id: str) -> ExtractJobStatusResponse:
+        response = self._client.transport.get(
+            self._client._build_url(f"/extract/{job_id}/status")
+        )
+        return ExtractJobStatusResponse(**response.data)
+
     def get(self, job_id: str) -> ExtractJobResponse:
         response = self._client.transport.get(
             self._client._build_url(f"/extract/{job_id}")
@@ -40,12 +50,10 @@ class ExtractManager:
         failures = 0
         while True:
             try:
-                job_response = self.get(job_start_resp.job_id)
-                if (
-                    job_response.status == "completed"
-                    or job_response.status == "failed"
-                ):
-                    return job_response
+                job_status_resp = self.get_status(job_id)
+                job_status = job_status_resp.status
+                if job_status == "completed" or job_status == "failed":
+                    return self.get(job_id)
             except Exception as e:
                 failures += 1
                 if failures >= POLLING_ATTEMPTS:

hyperbrowser/client/managers/sync_manager/profile.py

@@ -33,6 +33,7 @@ class ProfileManager:
         self, params: ProfileListParams = ProfileListParams()
     ) -> ProfileListResponse:
         response = self._client.transport.get(
-            self._client._build_url("/profiles"), params=params.__dict__
+            self._client._build_url("/profiles"),
+            params=params.model_dump(exclude_none=True, by_alias=True),
         )
         return ProfileListResponse(**response.data)

hyperbrowser/client/managers/sync_manager/scrape.py

@@ -4,8 +4,11 @@ from typing import Optional
 from hyperbrowser.models.consts import POLLING_ATTEMPTS
 from ....models.scrape import (
     BatchScrapeJobResponse,
+    BatchScrapeJobStatusResponse,
     GetBatchScrapeJobParams,
     ScrapeJobResponse,
+    ScrapeJobStatus,
+    ScrapeJobStatusResponse,
     StartBatchScrapeJobParams,
     StartBatchScrapeJobResponse,
     StartScrapeJobParams,
@@ -25,11 +28,18 @@ class BatchScrapeManager:
         )
         return StartBatchScrapeJobResponse(**response.data)
 
+    def get_status(self, job_id: str) -> BatchScrapeJobStatusResponse:
+        response = self._client.transport.get(
+            self._client._build_url(f"/scrape/batch/{job_id}/status")
+        )
+        return BatchScrapeJobStatusResponse(**response.data)
+
     def get(
         self, job_id: str, params: GetBatchScrapeJobParams = GetBatchScrapeJobParams()
     ) -> BatchScrapeJobResponse:
         response = self._client.transport.get(
-            self._client._build_url(f"/scrape/batch/{job_id}"), params=params.__dict__
+            self._client._build_url(f"/scrape/batch/{job_id}"),
+            params=params.model_dump(exclude_none=True, by_alias=True),
         )
         return BatchScrapeJobResponse(**response.data)
 
@@ -41,19 +51,14 @@ class BatchScrapeManager:
         if not job_id:
             raise HyperbrowserError("Failed to start batch scrape job")
 
-        job_response: BatchScrapeJobResponse
+        job_status: ScrapeJobStatus = "pending"
         failures = 0
         while True:
             try:
-                job_response = self.get(
-                    job_id, params=GetBatchScrapeJobParams(batch_size=1)
-                )
-                if (
-                    job_response.status == "completed"
-                    or job_response.status == "failed"
-                ):
+                job_status_resp = self.get_status(job_id)
+                job_status = job_status_resp.status
+                if job_status == "completed" or job_status == "failed":
                     break
-                failures = 0
             except Exception as e:
                 failures += 1
                 if failures >= POLLING_ATTEMPTS:
@@ -66,8 +71,7 @@ class BatchScrapeManager:
         if not return_all_pages:
             while True:
                 try:
-                    job_response = self.get(job_id)
-                    return job_response
+                    return self.get(job_id)
                 except Exception as e:
                     failures += 1
                     if failures >= POLLING_ATTEMPTS:
@@ -77,13 +81,25 @@ class BatchScrapeManager:
                 time.sleep(0.5)
 
         failures = 0
-        job_response.current_page_batch = 0
-        job_response.data = []
-        while job_response.current_page_batch < job_response.total_page_batches:
+        job_response = BatchScrapeJobResponse(
+            jobId=job_id,
+            status=job_status,
+            data=[],
+            currentPageBatch=0,
+            totalPageBatches=0,
+            totalScrapedPages=0,
+            batchSize=100,
+        )
+        first_check = True
+
+        while (
+            first_check
+            or job_response.current_page_batch < job_response.total_page_batches
+        ):
             try:
                 tmp_job_response = self.get(
-                    job_start_resp.job_id,
-                    GetBatchScrapeJobParams(
+                    job_id,
+                    params=GetBatchScrapeJobParams(
                         page=job_response.current_page_batch + 1, batch_size=100
                     ),
                 )
@@ -94,6 +110,7 @@ class BatchScrapeManager:
                 job_response.total_page_batches = tmp_job_response.total_page_batches
                 job_response.batch_size = tmp_job_response.batch_size
                 failures = 0
+                first_check = False
             except Exception as e:
                 failures += 1
                 if failures >= POLLING_ATTEMPTS:
@@ -117,6 +134,12 @@ class ScrapeManager:
         )
         return StartScrapeJobResponse(**response.data)
 
+    def get_status(self, job_id: str) -> ScrapeJobStatusResponse:
+        response = self._client.transport.get(
+            self._client._build_url(f"/scrape/{job_id}/status")
+        )
+        return ScrapeJobStatusResponse(**response.data)
+
     def get(self, job_id: str) -> ScrapeJobResponse:
         response = self._client.transport.get(
             self._client._build_url(f"/scrape/{job_id}")
@@ -132,12 +155,10 @@ class ScrapeManager:
         failures = 0
         while True:
             try:
-                job_response = self.get(job_id)
-                if (
-                    job_response.status == "completed"
-                    or job_response.status == "failed"
-                ):
-                    return job_response
+                job_status_resp = self.get_status(job_id)
+                job_status = job_status_resp.status
+                if job_status == "completed" or job_status == "failed":
+                    return self.get(job_id)
                 failures = 0
             except Exception as e:
                 failures += 1