firecrawl-py 3.3.1__py3-none-any.whl → 3.3.3__py3-none-any.whl

firecrawl/__init__.py

@@ -17,7 +17,7 @@ from .v1 import (
     V1ChangeTrackingOptions,
 )
 
-__version__ = "3.3.1"
+__version__ = "3.3.3"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")

firecrawl/__tests__/e2e/v2/test_scrape.py

@@ -151,4 +151,40 @@ class TestScrapeE2E:
             max_age=0,
             store_in_cache=False,
         )
-        assert isinstance(doc, Document)
+        assert isinstance(doc, Document)
+
+    def test_scrape_images_format(self):
+        """Test images format extraction."""
+        doc = self.client.scrape(
+            "https://firecrawl.dev",
+            formats=["images"]
+        )
+        assert isinstance(doc, Document)
+        assert doc.images is not None
+        assert isinstance(doc.images, list)
+        assert len(doc.images) > 0
+        # Should find firecrawl logo/branding images
+        assert any("firecrawl" in img.lower() or "logo" in img.lower() for img in doc.images)
+
+    def test_scrape_images_with_multiple_formats(self):
+        """Test images format works with other formats."""
+        doc = self.client.scrape(
+            "https://github.com",
+            formats=["markdown", "links", "images"]
+        )
+        assert isinstance(doc, Document)
+        assert doc.markdown is not None
+        assert doc.links is not None  
+        assert doc.images is not None
+        assert isinstance(doc.images, list)
+        assert len(doc.images) > 0
+        
+        # Images should find content not available in links format
+        image_extensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico']
+        link_images = [
+            link for link in (doc.links or [])
+            if any(ext in link.lower() for ext in image_extensions)
+        ]
+        
+        # Should discover additional images beyond those with obvious extensions
+        assert len(doc.images) >= len(link_images)

firecrawl/client.py

@@ -56,7 +56,6 @@ class V2Proxy:
         self._client = client_instance
 
         if client_instance:
-            # self.scrape = client_instance.scrape
             self.search = client_instance.search
             self.crawl = client_instance.crawl
             self.get_crawl_status = client_instance.get_crawl_status
@@ -168,14 +167,17 @@ class Firecrawl:
         self.v1 = V1Proxy(self._v1_client) if self._v1_client else None
         self.v2 = V2Proxy(self._v2_client)
         
-        
         self.scrape = self._v2_client.scrape
+        self.search = self._v2_client.search
+        self.map = self._v2_client.map
+
         self.crawl = self._v2_client.crawl
         self.start_crawl = self._v2_client.start_crawl
         self.crawl_params_preview = self._v2_client.crawl_params_preview
         self.get_crawl_status = self._v2_client.get_crawl_status
         self.cancel_crawl = self._v2_client.cancel_crawl
         self.get_crawl_errors = self._v2_client.get_crawl_errors
+        self.get_active_crawls = self._v2_client.get_active_crawls
         self.active_crawls = self._v2_client.active_crawls
 
         self.start_batch_scrape = self._v2_client.start_batch_scrape
@@ -183,13 +185,15 @@ class Firecrawl:
         self.cancel_batch_scrape = self._v2_client.cancel_batch_scrape
         self.batch_scrape = self._v2_client.batch_scrape
         self.get_batch_scrape_errors = self._v2_client.get_batch_scrape_errors
+
+        self.start_extract = self._v2_client.start_extract
         self.get_extract_status = self._v2_client.get_extract_status
-        self.map = self._v2_client.map
-        self.search = self._v2_client.search
         self.extract = self._v2_client.extract
+
         self.get_concurrency = self._v2_client.get_concurrency
         self.get_credit_usage = self._v2_client.get_credit_usage
         self.get_token_usage = self._v2_client.get_token_usage
+
         self.watcher = self._v2_client.watcher
         
 class AsyncFirecrawl:

firecrawl/v2/types.py

@@ -114,6 +114,12 @@ class DocumentMetadata(BaseModel):
     def coerce_status_code_to_int(cls, v):
         return cls._coerce_string_to_int(v)
 
+class AttributeResult(BaseModel):
+    """Result of attribute extraction."""
+    selector: str
+    attribute: str
+    values: List[str]
+
 class Document(BaseModel):
     """A scraped document."""
     markdown: Optional[str] = None
@@ -123,6 +129,7 @@ class Document(BaseModel):
     summary: Optional[str] = None
     metadata: Optional[DocumentMetadata] = None
     links: Optional[List[str]] = None
+    images: Optional[List[str]] = None
     screenshot: Optional[str] = None
     actions: Optional[Dict[str, Any]] = None
     warning: Optional[str] = None
@@ -182,7 +189,7 @@ CategoryOption = Union[str, Category]
 
 FormatString = Literal[
     # camelCase versions (API format)
-    "markdown", "html", "rawHtml", "links", "screenshot", "summary", "changeTracking", "json",
+    "markdown", "html", "rawHtml", "links",  "images", "screenshot", "summary", "changeTracking", "json", "attributes",
     # snake_case versions (user-friendly)
     "raw_html", "change_tracking"
 ]
@@ -214,9 +221,18 @@ class ScreenshotFormat(BaseModel):
     full_page: Optional[bool] = None
     quality: Optional[int] = None
     viewport: Optional[Union[Dict[str, int], Viewport]] = None
+    
+class AttributeSelector(BaseModel):
+    """Selector and attribute pair for attribute extraction."""
+    selector: str
+    attribute: str
 
-FormatOption = Union[Dict[str, Any], FormatString, JsonFormat, ChangeTrackingFormat, ScreenshotFormat, Format]
+class AttributesFormat(Format):
+    """Configuration for attribute extraction."""
+    type: Literal["attributes"] = "attributes"
+    selectors: List[AttributeSelector]
 
+FormatOption = Union[Dict[str, Any], FormatString, JsonFormat, ChangeTrackingFormat, ScreenshotFormat, AttributesFormat, Format]
 # Scrape types
 class ScrapeFormats(BaseModel):
     """Output formats for scraping."""
@@ -226,6 +242,7 @@ class ScrapeFormats(BaseModel):
     raw_html: bool = False
     summary: bool = False
     links: bool = False
+    images: bool = False
     screenshot: bool = False
     change_tracking: bool = False
     json: bool = False

{firecrawl_py-3.3.1.dist-info → firecrawl_py-3.3.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: firecrawl-py
-Version: 3.3.1
+Version: 3.3.3
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/firecrawl/firecrawl
 Author: Mendable.ai
@@ -38,8 +38,12 @@ Requires-Dist: httpx
 Requires-Dist: python-dotenv
 Requires-Dist: websockets
 Requires-Dist: nest-asyncio
-Requires-Dist: pydantic (>=2.0)
+Requires-Dist: pydantic>=2.0
 Requires-Dist: aiohttp
+Dynamic: author
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
 
 # Firecrawl Python SDK
 

firecrawl_py-3.3.3.dist-info/RECORD

@@ -0,0 +1,79 @@
+firecrawl/__init__.py,sha256=1MYT5_7-p8sfruL_5y1m1n9AoWG_6aNduWGW4NId86M,2192
+firecrawl/client.py,sha256=tp3mUo_3aGPuZ53kpU4bhM-5EtwD_IUWrJ7wm0GMuCc,11159
+firecrawl/firecrawl.backup.py,sha256=v1FEN3jR4g5Aupg4xp6SLkuFvYMQuUKND2YELbYjE6c,200430
+firecrawl/types.py,sha256=W9N2pqQuevEIIjYHN9rbDf31E-nwdCECqIn11Foz2T8,2836
+firecrawl/__tests__/e2e/v2/conftest.py,sha256=I28TUpN5j0-9gM79NlbrDS8Jlsheao657od2f-2xK0Y,2587
+firecrawl/__tests__/e2e/v2/test_async.py,sha256=ZXpf1FVOJgNclITglrxIyFwP4cOiqzWLicGaxIm70BQ,2526
+firecrawl/__tests__/e2e/v2/test_batch_scrape.py,sha256=H9GtuwHIFdOQ958SOVThi_kvDDxcXAK_ECRh95ogonQ,3265
+firecrawl/__tests__/e2e/v2/test_crawl.py,sha256=cOssZvIwtghAtLiM1QdNLhPEwAxZ9j9umTrBUPtJjpU,9951
+firecrawl/__tests__/e2e/v2/test_extract.py,sha256=HgvGiDlyWtFygiPo5EP44Dem1oWrwgRF-hfc1LfeVSU,1670
+firecrawl/__tests__/e2e/v2/test_map.py,sha256=9sT-Yq8V_8c9esl_bv5hnTA9WXb2Dg81kj6M-s0484c,1618
+firecrawl/__tests__/e2e/v2/test_scrape.py,sha256=oyroF_WaEdxgD8t_SHkLBBfDRv1_6xZ_7vSTQpwlmA8,7198
+firecrawl/__tests__/e2e/v2/test_search.py,sha256=tvU9_eg_3H5em0fhIwPPjuYe9BRAQ5St-BLM0l_FfVs,9079
+firecrawl/__tests__/e2e/v2/test_usage.py,sha256=JlBkYblhThua5qF2crRjsPpq4Ja0cBsdzxZ5zxXnQ_Y,805
+firecrawl/__tests__/e2e/v2/test_watcher.py,sha256=OPTKLhVAKWqXl2Tieo6zCN1xpEwZDsz-B977CVJgLMA,1932
+firecrawl/__tests__/e2e/v2/aio/test_aio_batch_scrape.py,sha256=gJv_mLzzoAYftETB2TLkrpSfB5c04kaYgkD4hQTYsIg,2639
+firecrawl/__tests__/e2e/v2/aio/test_aio_crawl.py,sha256=X-nk5tkYUYIkM6kTYl7GDjvxh2JT9GxJqk2KlO8xpWw,7282
+firecrawl/__tests__/e2e/v2/aio/test_aio_extract.py,sha256=3CNRIFzgBMcOYOLhnKcK1k5a3Gy--u08EGDkL31uieM,1199
+firecrawl/__tests__/e2e/v2/aio/test_aio_map.py,sha256=nckl1kbiEaaTdu5lm__tOoTDG-txTYwwSH3KZEvyKzc,1199
+firecrawl/__tests__/e2e/v2/aio/test_aio_scrape.py,sha256=b17A7advBEjxrjdait2w8GHztZeKy_P3zZ3ixm5H7xw,4453
+firecrawl/__tests__/e2e/v2/aio/test_aio_search.py,sha256=ehV0Ai_hknAkaoE551j2lbktV4bi_J0h3FKzC7G15Iw,8246
+firecrawl/__tests__/e2e/v2/aio/test_aio_usage.py,sha256=Dh9BVo48NKSZOKgLbO7n8fpMjvYmeMXDFzbIhnCTMhE,1014
+firecrawl/__tests__/e2e/v2/aio/test_aio_watcher.py,sha256=hwES4Nu5c0hniZ9heIPDfvh_2JmJ2wPoX9ULTZ0Asjs,1471
+firecrawl/__tests__/unit/v2/methods/test_batch_request_preparation.py,sha256=HeOxN-sPYSssytcIRAEicJSZsFt_Oa5qGXAtdumR54c,4040
+firecrawl/__tests__/unit/v2/methods/test_crawl_params.py,sha256=p9hzg14uAs1iHKXPDSXhGU6hEzPBF_Ae34RAf5XYa10,2387
+firecrawl/__tests__/unit/v2/methods/test_crawl_request_preparation.py,sha256=PEKbooNXfQwPpvcPHXABJnveztgAA-RFBhtlSs8uPro,8780
+firecrawl/__tests__/unit/v2/methods/test_crawl_validation.py,sha256=kErOmHSD01eMjXiMd4rgsMVGd_aU2G9uVymBjbAFoGw,3918
+firecrawl/__tests__/unit/v2/methods/test_map_request_preparation.py,sha256=toVcgnMp_cFeYsIUuyKGEWZGp0nAAkzaeFGUbY0zY0o,1868
+firecrawl/__tests__/unit/v2/methods/test_scrape_request_preparation.py,sha256=wDOslsA5BN4kyezlaT5GeMv_Ifn8f461EaA7i5ujnaQ,3482
+firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py,sha256=14lUgFpQsiosgMKjDustBRVE0zXnHujBI76F8BC5PZ4,6072
+firecrawl/__tests__/unit/v2/methods/test_search_validation.py,sha256=7UGcNHpQzCpZbAPYjthfdPFWmAPcoApY-ED-khtuANs,9498
+firecrawl/__tests__/unit/v2/methods/test_usage_types.py,sha256=cCHHfa6agSjD0brQ9rcAcw2kaI9riUH5C0dXV-fqktg,591
+firecrawl/__tests__/unit/v2/methods/test_webhook.py,sha256=AvvW-bKpUA--Lvtif2bmUIp-AxiaMJ29ie1i9dk8WbI,4586
+firecrawl/__tests__/unit/v2/methods/aio/test_aio_crawl_params.py,sha256=9azJxVvDOBqUevLp-wBF9gF7Ptj-7nN6LOkPQncFX2M,456
+firecrawl/__tests__/unit/v2/methods/aio/test_aio_crawl_request_preparation.py,sha256=RkIKt7uxBzVhAkrLQwXYjmC-9sj32SUNQrJZgF2WEMs,2565
+firecrawl/__tests__/unit/v2/methods/aio/test_aio_crawl_validation.py,sha256=WMgltdrrT2HOflqGyahC4v-Wb29_8sypN0hwS9lYXe8,403
+firecrawl/__tests__/unit/v2/methods/aio/test_aio_map_request_preparation.py,sha256=PdUJrR0JLWqrithAnRXwuRrnsIN2h_DTu6-xvTOn_UU,725
+firecrawl/__tests__/unit/v2/methods/aio/test_aio_scrape_request_preparation.py,sha256=A5DT4wpH4vrIPvFxKVHrtDH5A3bgJ_ad4fmVQ8LN1t0,1993
+firecrawl/__tests__/unit/v2/methods/aio/test_aio_search_request_preparation.py,sha256=hFk4XgqF3aFPGFJe0ikB1uwf_0FsppNGA088OrWUXvg,2091
+firecrawl/__tests__/unit/v2/methods/aio/test_batch_request_preparation_async.py,sha256=E26UnUhpbjG-EG0ab4WRD94AxA5IBWmIHq8ZLBOWoAA,1202
+firecrawl/__tests__/unit/v2/methods/aio/test_ensure_async.py,sha256=pUwuWhRbVUTbgsZn4hgZesMkTMesTv_NPmvFW--ls-Y,3815
+firecrawl/__tests__/unit/v2/utils/test_validation.py,sha256=E4n4jpBhH_W7E0ikI5r8KMAKiOhbfGD3i_B8-dv3PlI,10803
+firecrawl/__tests__/unit/v2/watcher/test_ws_watcher.py,sha256=87w47n0iOihtu4jTR4-4rw1-xVKWmLg2BOBGxjQPnUk,9517
+firecrawl/v1/__init__.py,sha256=aP1oisPeZVGGZynvENc07JySMOZfv_4zAlxQ0ecMJXA,481
+firecrawl/v1/client.py,sha256=sydurfEFTsXyowyaGryA1lkPxN_r9Nf6iQpM43OwJyM,201672
+firecrawl/v2/__init__.py,sha256=Jc6a8tBjYG5OPkjDM5pl-notyys-7DEj7PLEfepv3fc,137
+firecrawl/v2/client.py,sha256=_DZFZO1aWvODzznK0g2Svcd2-xxXgWGR0d9vniNlk1w,30621
+firecrawl/v2/client_async.py,sha256=zwxHis1bSh0tSF1480ze-4XDQEDJ5yDur1ZqtL94dwc,10127
+firecrawl/v2/types.py,sha256=F-RCADQFdpAmF5t8LUabLOgyIV02Ol34yNa9y3S3ZMg,22667
+firecrawl/v2/watcher.py,sha256=FOU71tqSKxgeuGycu4ye0SLc2dw7clIcoQjPsi-4Csc,14229
+firecrawl/v2/watcher_async.py,sha256=AVjW2mgABniolSsauK4u0FW8ya6WzRUdyEg2R-8vGCw,10278
+firecrawl/v2/methods/batch.py,sha256=us7zUGl7u9ZDIEk2J3rNqj87bkaNjXU27SMFW_fdcg8,11932
+firecrawl/v2/methods/crawl.py,sha256=4ZUmanHNuNtq9wbKMAZ3lenuPcNdOaV0kYXqMI5XJJ8,15485
+firecrawl/v2/methods/extract.py,sha256=-Jr4BtraU3b7hd3JIY73V-S69rUclxyXyUpoQb6DCQk,4274
+firecrawl/v2/methods/map.py,sha256=4SADb0-lkbdOWDmO6k8_TzK0yRti5xsN40N45nUl9uA,2592
+firecrawl/v2/methods/scrape.py,sha256=CSHBwC-P91UfrW3zHirjNAs2h899FKcWvd1DY_4fJdo,1921
+firecrawl/v2/methods/search.py,sha256=6BKiQ1aKJjWBKm9BBtKxFKGD74kCKBeMIp_OgjcDFAw,7673
+firecrawl/v2/methods/usage.py,sha256=OJlkxwaB-AAtgO3WLr9QiqBRmjdh6GVhroCgleegupQ,1460
+firecrawl/v2/methods/aio/__init__.py,sha256=RocMJnGwnLIvGu3G8ZvY8INkipC7WHZiu2bE31eSyJs,35
+firecrawl/v2/methods/aio/batch.py,sha256=GS_xsd_Uib1fxFITBK1sH88VGzFMrIcqJVQqOvMQ540,3735
+firecrawl/v2/methods/aio/crawl.py,sha256=pC6bHVk30Hj1EJdAChxpMOg0Xx_GVqq4tIlvU2e5RQ4,6688
+firecrawl/v2/methods/aio/extract.py,sha256=IfNr2ETqt4dR73JFzrEYI4kk5vpKnJOG0BmPEjGEoO4,4217
+firecrawl/v2/methods/aio/map.py,sha256=EuT-5A0cQr_e5SBfEZ6pnl8u0JUwEEvSwhyT2N-QoKU,2326
+firecrawl/v2/methods/aio/scrape.py,sha256=ilA9qco8YGwCFpE0PN1XBQUyuHPQwH2QioZ-xsfxhgU,1386
+firecrawl/v2/methods/aio/search.py,sha256=_TqTFGQLlOCCLNdWcOvakTqPGD2r9AOlBg8RasOgmvw,6177
+firecrawl/v2/methods/aio/usage.py,sha256=OtBi6X-aT09MMR2dpm3vBCm9JrJZIJLCQ8jJ3L7vie4,1606
+firecrawl/v2/utils/__init__.py,sha256=i1GgxySmqEXpWSBQCu3iZBPIJG7fXj0QXCDWGwerWNs,338
+firecrawl/v2/utils/error_handler.py,sha256=Iuf916dHphDY8ObNNlWy75628DFeJ0Rv8ljRp4LttLE,4199
+firecrawl/v2/utils/get_version.py,sha256=0CxW_41q2hlzIxEWOivUCaYw3GFiSIH32RPUMcIgwAY,492
+firecrawl/v2/utils/http_client.py,sha256=_n8mp4xi6GGihg662Lsv6TSlvw9zykyADwEk0fg8mYA,4873
+firecrawl/v2/utils/http_client_async.py,sha256=iy89_bk2HS3afSRHZ8016eMCa9Fk-5MFTntcOHfbPgE,1936
+firecrawl/v2/utils/normalize.py,sha256=nlTU6QRghT1YKZzNZlIQj4STSRuSUGrS9cCErZIcY5w,3636
+firecrawl/v2/utils/validation.py,sha256=L8by7z-t6GuMGIYkK7il1BM8d-4_-sAdG9hDMF_LeG4,14518
+firecrawl_py-3.3.3.dist-info/licenses/LICENSE,sha256=nPCunEDwjRGHlmjvsiDUyIWbkqqyj3Ej84ntnh0g0zA,1084
+tests/test_change_tracking.py,sha256=_IJ5ShLcoj2fHDBaw-nE4I4lHdmDB617ocK_XMHhXps,4177
+tests/test_timeout_conversion.py,sha256=PWlIEMASQNhu4cp1OW_ebklnE9NCiigPnEFCtI5N3w0,3996
+firecrawl_py-3.3.3.dist-info/METADATA,sha256=_5tGMWJrCEIJy1UCLKbPAgV6iczF2_T4aRpLhpoL0F4,7395
+firecrawl_py-3.3.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+firecrawl_py-3.3.3.dist-info/top_level.txt,sha256=8T3jOaSN5mtLghO-R3MQ8KO290gIX8hmfxQmglBPdLE,16
+firecrawl_py-3.3.3.dist-info/RECORD,,

{firecrawl_py-3.3.1.dist-info → firecrawl_py-3.3.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.38.4)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{firecrawl_py-3.3.1.dist-info → firecrawl_py-3.3.3.dist-info}/top_level.txt

@@ -1,4 +1,2 @@
-build
-dist
 firecrawl
 tests

build/lib/firecrawl/__init__.py

@@ -1,87 +0,0 @@
-"""
-Firecrawl Python SDK
-
-"""
-
-import logging
-import os
-
-from .client import Firecrawl, AsyncFirecrawl, FirecrawlApp, AsyncFirecrawlApp
-from .v2.watcher import Watcher
-from .v2.watcher_async import AsyncWatcher
-from .v1 import (
-    V1FirecrawlApp,
-    AsyncV1FirecrawlApp,
-    V1JsonConfig,
-    V1ScrapeOptions,
-    V1ChangeTrackingOptions,
-)
-
-__version__ = "3.3.1"
-
-# Define the logger for the Firecrawl project
-logger: logging.Logger = logging.getLogger("firecrawl")
-
-
-def _configure_logger() -> None:
-    """
-    Configure the firecrawl logger for console output.
-
-    The function attaches a handler for console output with a specific format and date
-    format to the firecrawl logger.
-    """
-    try:
-        formatter = logging.Formatter(
-            "[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s",
-            datefmt="%Y-%m-%d %H:%M:%S",
-        )
-
-        console_handler = logging.StreamHandler()
-        console_handler.setFormatter(formatter)
-
-        logger.addHandler(console_handler)
-    except Exception as e:
-        logger.error("Failed to configure logging: %s", e)
-
-
-def setup_logging() -> None:
-    """Set up logging based on the FIRECRAWL_LOGGING_LEVEL environment variable."""
-    if logger.hasHandlers():
-        return
-
-    if not (env := os.getenv("FIRECRAWL_LOGGING_LEVEL", "").upper()):
-        logger.addHandler(logging.NullHandler()) 
-        return
-
-    _configure_logger()
-
-    if env == "DEBUG":
-        logger.setLevel(logging.DEBUG)
-    elif env == "INFO":
-        logger.setLevel(logging.INFO)
-    elif env == "WARNING":
-        logger.setLevel(logging.WARNING)
-    elif env == "ERROR":
-        logger.setLevel(logging.ERROR)
-    elif env == "CRITICAL":
-        logger.setLevel(logging.CRITICAL)
-    else:
-        logger.setLevel(logging.INFO)
-        logger.warning("Unknown logging level: %s, defaulting to INFO", env)
-
-setup_logging()
-logger.debug("Debugging logger setup")
-
-__all__ = [
-    'Firecrawl',
-    'AsyncFirecrawl',
-    'FirecrawlApp',
-    'AsyncFirecrawlApp',
-    'Watcher',
-    'AsyncWatcher',
-    'V1FirecrawlApp',
-    'AsyncV1FirecrawlApp',
-    'V1JsonConfig',
-    'V1ScrapeOptions',
-    'V1ChangeTrackingOptions',
-]

build/lib/firecrawl/__tests__/e2e/v2/aio/test_aio_batch_scrape.py

@@ -1,79 +0,0 @@
-import os
-import asyncio
-import pytest
-from dotenv import load_dotenv
-from firecrawl import AsyncFirecrawl
-
-
-load_dotenv()
-
-if not os.getenv("API_KEY"):
-    raise ValueError("API_KEY is not set")
-
-if not os.getenv("API_URL"):
-    raise ValueError("API_URL is not set")
-
-
-@pytest.mark.asyncio
-async def test_async_batch_start_and_status():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_batch_scrape([
-        "https://docs.firecrawl.dev",
-        "https://firecrawl.dev",
-    ], formats=["markdown"], max_concurrency=1)
-    job_id = start.id
-
-    deadline = asyncio.get_event_loop().time() + 240
-    status = await client.get_batch_scrape_status(job_id)
-    while status.status not in ("completed", "failed", "cancelled") and asyncio.get_event_loop().time() < deadline:
-        await asyncio.sleep(2)
-        status = await client.get_batch_scrape_status(job_id)
-
-    assert status.status in ("completed", "failed", "cancelled")
-
-
-@pytest.mark.asyncio
-async def test_async_batch_wait_minimal():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    job = await client.batch_scrape([
-        "https://docs.firecrawl.dev",
-        "https://firecrawl.dev",
-    ], formats=["markdown"], poll_interval=1, timeout=120)
-    assert job.status in ("completed", "failed")
-
-
-@pytest.mark.asyncio
-async def test_async_batch_wait_with_all_params():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
-    job = await client.batch_scrape(
-        [
-            "https://docs.firecrawl.dev",
-            "https://firecrawl.dev",
-        ],
-        formats=[
-            "markdown",
-            {"type": "json", "prompt": "Extract page title", "schema": json_schema},
-            {"type": "changeTracking", "prompt": "Track changes", "modes": ["json"]},
-        ],
-        only_main_content=True,
-        mobile=False,
-        ignore_invalid_urls=True,
-        max_concurrency=2,
-        zero_data_retention=False,
-        poll_interval=1,
-        timeout=180,
-    )
-    assert job.status in ("completed", "failed")
-
-
-@pytest.mark.asyncio
-async def test_async_cancel_batch():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_batch_scrape([
-        "https://docs.firecrawl.dev",
-        "https://firecrawl.dev",
-    ], formats=["markdown"], max_concurrency=1)
-    ok = await client.cancel_batch_scrape(start.id)
-    assert ok is True
-

build/lib/firecrawl/__tests__/e2e/v2/aio/test_aio_crawl.py

@@ -1,188 +0,0 @@
-import os
-import asyncio
-import pytest
-from dotenv import load_dotenv
-from firecrawl import AsyncFirecrawl
-from firecrawl.v2.types import ScrapeOptions
-
-
-load_dotenv()
-
-if not os.getenv("API_KEY"):
-    raise ValueError("API_KEY is not set")
-
-if not os.getenv("API_URL"):
-    raise ValueError("API_URL is not set")
-
-
-@pytest.mark.asyncio
-async def test_async_crawl_start_and_status():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2)
-    job_id = start.id
-
-    deadline = asyncio.get_event_loop().time() + 180
-    status = await client.get_crawl_status(job_id)
-    while status.status not in ("completed", "failed") and asyncio.get_event_loop().time() < deadline:
-        await asyncio.sleep(2)
-        status = await client.get_crawl_status(job_id)
-
-    assert status.status in ("completed", "failed")
-
-
-@pytest.mark.asyncio
-async def test_async_crawl_with_all_params():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    # rich scrape options including json format
-    json_schema = {
-        "type": "object",
-        "properties": {"title": {"type": "string"}},
-        "required": ["title"],
-    }
-    status = await client.crawl(
-        url="https://docs.firecrawl.dev",
-        prompt="Extract docs and blog",
-        include_paths=["/docs/*", "/blog/*"],
-        exclude_paths=["/admin/*"],
-        max_discovery_depth=2,
-        ignore_sitemap=False,
-        ignore_query_parameters=True,
-        limit=5,
-        crawl_entire_domain=False,
-        allow_external_links=True,
-        allow_subdomains=True,
-        delay=1,
-        max_concurrency=2,
-        webhook="https://example.com/hook",
-        scrape_options=ScrapeOptions(
-            formats=[
-                "markdown",
-                "rawHtml",
-                {"type": "json", "prompt": "Extract title", "schema": json_schema},
-            ],
-            only_main_content=True,
-            mobile=False,
-            timeout=20000,
-            wait_for=500,
-            skip_tls_verification=False,
-            remove_base64_images=False,
-        ),
-        zero_data_retention=False,
-        poll_interval=2,
-        timeout=180,
-    )
-    assert status.status in ("completed", "failed")
-
-
-@pytest.mark.asyncio
-async def test_async_start_crawl_with_options():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=5, max_discovery_depth=2)
-    assert start.id is not None and start.url is not None
-
-
-@pytest.mark.asyncio
-async def test_async_start_crawl_with_prompt():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_crawl("https://firecrawl.dev", prompt="Extract all blog posts", limit=3)
-    assert start.id is not None and start.url is not None
-
-
-@pytest.mark.asyncio
-async def test_async_get_crawl_status_shape():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=3)
-    status = await client.get_crawl_status(start.id)
-    assert status.status in ("scraping", "completed", "failed")
-    assert status.completed >= 0
-    assert status.expires_at is not None
-    assert isinstance(status.data, list)
-
-
-@pytest.mark.asyncio
-async def test_async_crawl_with_wait():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    job = await client.crawl(url="https://docs.firecrawl.dev", limit=3, max_discovery_depth=2, poll_interval=1, timeout=120)
-    assert job.status in ("completed", "failed")
-    assert job.completed >= 0 and job.total >= 0 and isinstance(job.data, list)
-
-
-@pytest.mark.asyncio
-async def test_async_crawl_with_prompt_and_wait():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    job = await client.crawl(url="https://docs.firecrawl.dev", prompt="Extract all blog posts", limit=3, poll_interval=1, timeout=120)
-    assert job.status in ("completed", "failed")
-    assert job.completed >= 0 and job.total >= 0 and isinstance(job.data, list)
-
-
-@pytest.mark.asyncio
-async def test_async_crawl_with_scrape_options():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    scrape_opts = ScrapeOptions(formats=["markdown", "links"], only_main_content=False, mobile=True)
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2, scrape_options=scrape_opts)
-    assert start.id is not None
-
-
-@pytest.mark.asyncio
-async def test_async_crawl_with_json_format_object():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
-    scrape_opts = ScrapeOptions(formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}])
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2, scrape_options=scrape_opts)
-    assert start.id is not None
-
-
-@pytest.mark.asyncio
-async def test_async_cancel_crawl():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=3)
-    cancelled = await client.cancel_crawl(start.id)
-    assert cancelled is True
-
-
-@pytest.mark.asyncio
-async def test_async_get_crawl_errors_and_invalid_job():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2)
-    errs = await client.get_crawl_errors(start.id)
-    assert hasattr(errs, "errors") and hasattr(errs, "robots_blocked")
-    with pytest.raises(Exception):
-        await client.get_crawl_errors("invalid-job-id-12345")
-
-
-@pytest.mark.asyncio
-async def test_async_active_crawls():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    resp = await client.active_crawls()
-    assert hasattr(resp, "success") and hasattr(resp, "crawls")
-
-
-@pytest.mark.asyncio
-async def test_async_active_crawls_with_running_crawl():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=3)
-    # fetch active crawls and assert our ID is listed
-    active = await client.active_crawls()
-    ids = [c.id for c in active.crawls]
-    assert start.id in ids
-    # cleanup
-    await client.cancel_crawl(start.id)
-
-
-@pytest.mark.asyncio
-async def test_async_crawl_params_preview():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    params = await client.crawl_params_preview(
-        url="https://docs.firecrawl.dev",
-        prompt="Extract all blog posts and documentation",
-    )
-    assert params is not None
-    # basic sanity: at least one field should be suggested
-    has_any = any([
-        getattr(params, "limit", None) is not None,
-        getattr(params, "include_paths", None) is not None,
-        getattr(params, "max_discovery_depth", None) is not None,
-    ])
-    assert has_any
-
-

build/lib/firecrawl/__tests__/e2e/v2/aio/test_aio_extract.py

@@ -1,38 +0,0 @@
-import os
-import pytest
-from dotenv import load_dotenv
-from firecrawl import AsyncFirecrawl
-
-
-load_dotenv()
-
-if not os.getenv("API_KEY"):
-    raise ValueError("API_KEY is not set")
-
-if not os.getenv("API_URL"):
-    raise ValueError("API_URL is not set")
-
-
-@pytest.mark.asyncio
-async def test_async_extract_minimal():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    res = await client.extract(urls=["https://docs.firecrawl.dev"], prompt="Extract title")
-    assert res is not None
-
-
-@pytest.mark.asyncio
-async def test_async_extract_with_schema_and_options():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
-    res = await client.extract(
-        urls=["https://docs.firecrawl.dev"],
-        prompt="Extract title",
-        schema=schema,
-        system_prompt="You are a helpful extractor",
-        allow_external_links=False,
-        enable_web_search=False,
-        show_sources=False,
-        # agent={"model": "FIRE-1", "prompt": "Extract title"}, # Skipping agent test in CI
-    )
-    assert res is not None
-