firecrawl-py 3.3.0__py3-none-any.whl → 3.3.2__py3-none-any.whl

firecrawl/__init__.py

@@ -17,7 +17,7 @@ from .v1 import (
     V1ChangeTrackingOptions,
 )
 
-__version__ = "3.3.0"
+__version__ = "3.3.2"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")

firecrawl/v2/client.py

@@ -13,6 +13,7 @@ from .types import (
     SearchRequest,
     SearchData,
     SourceOption,
+    CategoryOption,
     CrawlRequest,
     CrawlResponse,
     CrawlJob,
@@ -171,6 +172,7 @@ class FirecrawlClient:
         query: str,
         *,
         sources: Optional[List[SourceOption]] = None,
+        categories: Optional[List[CategoryOption]] = None,
         limit: Optional[int] = None,
         tbs: Optional[str] = None,
         location: Optional[str] = None,
@@ -195,6 +197,7 @@ class FirecrawlClient:
         request = SearchRequest(
             query=query,
             sources=sources,
+            categories=categories,
             limit=limit,
             tbs=tbs,
             location=location,

{firecrawl_py-3.3.0.dist-info → firecrawl_py-3.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: firecrawl-py
-Version: 3.3.0
+Version: 3.3.2
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/firecrawl/firecrawl
 Author: Mendable.ai

firecrawl_py-3.3.2.dist-info/RECORD

@@ -0,0 +1,79 @@
+firecrawl/__init__.py,sha256=l_9wjt7rVPqg4ovtOFC46I_8LghOX_jDFoEXuSavJJ0,2192
+firecrawl/client.py,sha256=2BGIRTiW2eR6q3wu_g2s3VTQtrHYauoDeNF1YklQpHo,11089
+firecrawl/firecrawl.backup.py,sha256=v1FEN3jR4g5Aupg4xp6SLkuFvYMQuUKND2YELbYjE6c,200430
+firecrawl/types.py,sha256=W9N2pqQuevEIIjYHN9rbDf31E-nwdCECqIn11Foz2T8,2836
+firecrawl/__tests__/e2e/v2/conftest.py,sha256=I28TUpN5j0-9gM79NlbrDS8Jlsheao657od2f-2xK0Y,2587
+firecrawl/__tests__/e2e/v2/test_async.py,sha256=ZXpf1FVOJgNclITglrxIyFwP4cOiqzWLicGaxIm70BQ,2526
+firecrawl/__tests__/e2e/v2/test_batch_scrape.py,sha256=H9GtuwHIFdOQ958SOVThi_kvDDxcXAK_ECRh95ogonQ,3265
+firecrawl/__tests__/e2e/v2/test_crawl.py,sha256=cOssZvIwtghAtLiM1QdNLhPEwAxZ9j9umTrBUPtJjpU,9951
+firecrawl/__tests__/e2e/v2/test_extract.py,sha256=HgvGiDlyWtFygiPo5EP44Dem1oWrwgRF-hfc1LfeVSU,1670
+firecrawl/__tests__/e2e/v2/test_map.py,sha256=9sT-Yq8V_8c9esl_bv5hnTA9WXb2Dg81kj6M-s0484c,1618
+firecrawl/__tests__/e2e/v2/test_scrape.py,sha256=psW2nfcA_hMFpZ4msL_VJWJTMa3Sidp11ubhftbm52g,5759
+firecrawl/__tests__/e2e/v2/test_search.py,sha256=tvU9_eg_3H5em0fhIwPPjuYe9BRAQ5St-BLM0l_FfVs,9079
+firecrawl/__tests__/e2e/v2/test_usage.py,sha256=JlBkYblhThua5qF2crRjsPpq4Ja0cBsdzxZ5zxXnQ_Y,805
+firecrawl/__tests__/e2e/v2/test_watcher.py,sha256=OPTKLhVAKWqXl2Tieo6zCN1xpEwZDsz-B977CVJgLMA,1932
+firecrawl/__tests__/e2e/v2/aio/test_aio_batch_scrape.py,sha256=gJv_mLzzoAYftETB2TLkrpSfB5c04kaYgkD4hQTYsIg,2639
+firecrawl/__tests__/e2e/v2/aio/test_aio_crawl.py,sha256=X-nk5tkYUYIkM6kTYl7GDjvxh2JT9GxJqk2KlO8xpWw,7282
+firecrawl/__tests__/e2e/v2/aio/test_aio_extract.py,sha256=3CNRIFzgBMcOYOLhnKcK1k5a3Gy--u08EGDkL31uieM,1199
+firecrawl/__tests__/e2e/v2/aio/test_aio_map.py,sha256=nckl1kbiEaaTdu5lm__tOoTDG-txTYwwSH3KZEvyKzc,1199
+firecrawl/__tests__/e2e/v2/aio/test_aio_scrape.py,sha256=b17A7advBEjxrjdait2w8GHztZeKy_P3zZ3ixm5H7xw,4453
+firecrawl/__tests__/e2e/v2/aio/test_aio_search.py,sha256=ehV0Ai_hknAkaoE551j2lbktV4bi_J0h3FKzC7G15Iw,8246
+firecrawl/__tests__/e2e/v2/aio/test_aio_usage.py,sha256=Dh9BVo48NKSZOKgLbO7n8fpMjvYmeMXDFzbIhnCTMhE,1014
+firecrawl/__tests__/e2e/v2/aio/test_aio_watcher.py,sha256=hwES4Nu5c0hniZ9heIPDfvh_2JmJ2wPoX9ULTZ0Asjs,1471
+firecrawl/__tests__/unit/v2/methods/test_batch_request_preparation.py,sha256=HeOxN-sPYSssytcIRAEicJSZsFt_Oa5qGXAtdumR54c,4040
+firecrawl/__tests__/unit/v2/methods/test_crawl_params.py,sha256=p9hzg14uAs1iHKXPDSXhGU6hEzPBF_Ae34RAf5XYa10,2387
+firecrawl/__tests__/unit/v2/methods/test_crawl_request_preparation.py,sha256=PEKbooNXfQwPpvcPHXABJnveztgAA-RFBhtlSs8uPro,8780
+firecrawl/__tests__/unit/v2/methods/test_crawl_validation.py,sha256=kErOmHSD01eMjXiMd4rgsMVGd_aU2G9uVymBjbAFoGw,3918
+firecrawl/__tests__/unit/v2/methods/test_map_request_preparation.py,sha256=toVcgnMp_cFeYsIUuyKGEWZGp0nAAkzaeFGUbY0zY0o,1868
+firecrawl/__tests__/unit/v2/methods/test_scrape_request_preparation.py,sha256=wDOslsA5BN4kyezlaT5GeMv_Ifn8f461EaA7i5ujnaQ,3482
+firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py,sha256=14lUgFpQsiosgMKjDustBRVE0zXnHujBI76F8BC5PZ4,6072
+firecrawl/__tests__/unit/v2/methods/test_search_validation.py,sha256=7UGcNHpQzCpZbAPYjthfdPFWmAPcoApY-ED-khtuANs,9498
+firecrawl/__tests__/unit/v2/methods/test_usage_types.py,sha256=cCHHfa6agSjD0brQ9rcAcw2kaI9riUH5C0dXV-fqktg,591
+firecrawl/__tests__/unit/v2/methods/test_webhook.py,sha256=AvvW-bKpUA--Lvtif2bmUIp-AxiaMJ29ie1i9dk8WbI,4586
+firecrawl/__tests__/unit/v2/methods/aio/test_aio_crawl_params.py,sha256=9azJxVvDOBqUevLp-wBF9gF7Ptj-7nN6LOkPQncFX2M,456
+firecrawl/__tests__/unit/v2/methods/aio/test_aio_crawl_request_preparation.py,sha256=RkIKt7uxBzVhAkrLQwXYjmC-9sj32SUNQrJZgF2WEMs,2565
+firecrawl/__tests__/unit/v2/methods/aio/test_aio_crawl_validation.py,sha256=WMgltdrrT2HOflqGyahC4v-Wb29_8sypN0hwS9lYXe8,403
+firecrawl/__tests__/unit/v2/methods/aio/test_aio_map_request_preparation.py,sha256=PdUJrR0JLWqrithAnRXwuRrnsIN2h_DTu6-xvTOn_UU,725
+firecrawl/__tests__/unit/v2/methods/aio/test_aio_scrape_request_preparation.py,sha256=A5DT4wpH4vrIPvFxKVHrtDH5A3bgJ_ad4fmVQ8LN1t0,1993
+firecrawl/__tests__/unit/v2/methods/aio/test_aio_search_request_preparation.py,sha256=hFk4XgqF3aFPGFJe0ikB1uwf_0FsppNGA088OrWUXvg,2091
+firecrawl/__tests__/unit/v2/methods/aio/test_batch_request_preparation_async.py,sha256=E26UnUhpbjG-EG0ab4WRD94AxA5IBWmIHq8ZLBOWoAA,1202
+firecrawl/__tests__/unit/v2/methods/aio/test_ensure_async.py,sha256=pUwuWhRbVUTbgsZn4hgZesMkTMesTv_NPmvFW--ls-Y,3815
+firecrawl/__tests__/unit/v2/utils/test_validation.py,sha256=E4n4jpBhH_W7E0ikI5r8KMAKiOhbfGD3i_B8-dv3PlI,10803
+firecrawl/__tests__/unit/v2/watcher/test_ws_watcher.py,sha256=87w47n0iOihtu4jTR4-4rw1-xVKWmLg2BOBGxjQPnUk,9517
+firecrawl/v1/__init__.py,sha256=aP1oisPeZVGGZynvENc07JySMOZfv_4zAlxQ0ecMJXA,481
+firecrawl/v1/client.py,sha256=sydurfEFTsXyowyaGryA1lkPxN_r9Nf6iQpM43OwJyM,201672
+firecrawl/v2/__init__.py,sha256=Jc6a8tBjYG5OPkjDM5pl-notyys-7DEj7PLEfepv3fc,137
+firecrawl/v2/client.py,sha256=_DZFZO1aWvODzznK0g2Svcd2-xxXgWGR0d9vniNlk1w,30621
+firecrawl/v2/client_async.py,sha256=zwxHis1bSh0tSF1480ze-4XDQEDJ5yDur1ZqtL94dwc,10127
+firecrawl/v2/types.py,sha256=rBdTaTQmb1SmdR8O0GvA_gCfBG-QCtBOrMsFgA9Usms,22114
+firecrawl/v2/watcher.py,sha256=FOU71tqSKxgeuGycu4ye0SLc2dw7clIcoQjPsi-4Csc,14229
+firecrawl/v2/watcher_async.py,sha256=AVjW2mgABniolSsauK4u0FW8ya6WzRUdyEg2R-8vGCw,10278
+firecrawl/v2/methods/batch.py,sha256=us7zUGl7u9ZDIEk2J3rNqj87bkaNjXU27SMFW_fdcg8,11932
+firecrawl/v2/methods/crawl.py,sha256=4ZUmanHNuNtq9wbKMAZ3lenuPcNdOaV0kYXqMI5XJJ8,15485
+firecrawl/v2/methods/extract.py,sha256=-Jr4BtraU3b7hd3JIY73V-S69rUclxyXyUpoQb6DCQk,4274
+firecrawl/v2/methods/map.py,sha256=4SADb0-lkbdOWDmO6k8_TzK0yRti5xsN40N45nUl9uA,2592
+firecrawl/v2/methods/scrape.py,sha256=CSHBwC-P91UfrW3zHirjNAs2h899FKcWvd1DY_4fJdo,1921
+firecrawl/v2/methods/search.py,sha256=6BKiQ1aKJjWBKm9BBtKxFKGD74kCKBeMIp_OgjcDFAw,7673
+firecrawl/v2/methods/usage.py,sha256=OJlkxwaB-AAtgO3WLr9QiqBRmjdh6GVhroCgleegupQ,1460
+firecrawl/v2/methods/aio/__init__.py,sha256=RocMJnGwnLIvGu3G8ZvY8INkipC7WHZiu2bE31eSyJs,35
+firecrawl/v2/methods/aio/batch.py,sha256=GS_xsd_Uib1fxFITBK1sH88VGzFMrIcqJVQqOvMQ540,3735
+firecrawl/v2/methods/aio/crawl.py,sha256=pC6bHVk30Hj1EJdAChxpMOg0Xx_GVqq4tIlvU2e5RQ4,6688
+firecrawl/v2/methods/aio/extract.py,sha256=IfNr2ETqt4dR73JFzrEYI4kk5vpKnJOG0BmPEjGEoO4,4217
+firecrawl/v2/methods/aio/map.py,sha256=EuT-5A0cQr_e5SBfEZ6pnl8u0JUwEEvSwhyT2N-QoKU,2326
+firecrawl/v2/methods/aio/scrape.py,sha256=ilA9qco8YGwCFpE0PN1XBQUyuHPQwH2QioZ-xsfxhgU,1386
+firecrawl/v2/methods/aio/search.py,sha256=_TqTFGQLlOCCLNdWcOvakTqPGD2r9AOlBg8RasOgmvw,6177
+firecrawl/v2/methods/aio/usage.py,sha256=OtBi6X-aT09MMR2dpm3vBCm9JrJZIJLCQ8jJ3L7vie4,1606
+firecrawl/v2/utils/__init__.py,sha256=i1GgxySmqEXpWSBQCu3iZBPIJG7fXj0QXCDWGwerWNs,338
+firecrawl/v2/utils/error_handler.py,sha256=Iuf916dHphDY8ObNNlWy75628DFeJ0Rv8ljRp4LttLE,4199
+firecrawl/v2/utils/get_version.py,sha256=0CxW_41q2hlzIxEWOivUCaYw3GFiSIH32RPUMcIgwAY,492
+firecrawl/v2/utils/http_client.py,sha256=_n8mp4xi6GGihg662Lsv6TSlvw9zykyADwEk0fg8mYA,4873
+firecrawl/v2/utils/http_client_async.py,sha256=iy89_bk2HS3afSRHZ8016eMCa9Fk-5MFTntcOHfbPgE,1936
+firecrawl/v2/utils/normalize.py,sha256=nlTU6QRghT1YKZzNZlIQj4STSRuSUGrS9cCErZIcY5w,3636
+firecrawl/v2/utils/validation.py,sha256=L8by7z-t6GuMGIYkK7il1BM8d-4_-sAdG9hDMF_LeG4,14518
+tests/test_change_tracking.py,sha256=_IJ5ShLcoj2fHDBaw-nE4I4lHdmDB617ocK_XMHhXps,4177
+tests/test_timeout_conversion.py,sha256=PWlIEMASQNhu4cp1OW_ebklnE9NCiigPnEFCtI5N3w0,3996
+firecrawl_py-3.3.2.dist-info/LICENSE,sha256=nPCunEDwjRGHlmjvsiDUyIWbkqqyj3Ej84ntnh0g0zA,1084
+firecrawl_py-3.3.2.dist-info/METADATA,sha256=xGcVdw1lhmAlArE2riSVeSnQV_ht0EalkT_QGm8HaNM,7316
+firecrawl_py-3.3.2.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+firecrawl_py-3.3.2.dist-info/top_level.txt,sha256=8T3jOaSN5mtLghO-R3MQ8KO290gIX8hmfxQmglBPdLE,16
+firecrawl_py-3.3.2.dist-info/RECORD,,

{firecrawl_py-3.3.0.dist-info → firecrawl_py-3.3.2.dist-info}/top_level.txt

@@ -1,4 +1,2 @@
-build
-dist
 firecrawl
 tests

build/lib/firecrawl/__init__.py

@@ -1,87 +0,0 @@
-"""
-Firecrawl Python SDK
-
-"""
-
-import logging
-import os
-
-from .client import Firecrawl, AsyncFirecrawl, FirecrawlApp, AsyncFirecrawlApp
-from .v2.watcher import Watcher
-from .v2.watcher_async import AsyncWatcher
-from .v1 import (
-    V1FirecrawlApp,
-    AsyncV1FirecrawlApp,
-    V1JsonConfig,
-    V1ScrapeOptions,
-    V1ChangeTrackingOptions,
-)
-
-__version__ = "3.3.0"
-
-# Define the logger for the Firecrawl project
-logger: logging.Logger = logging.getLogger("firecrawl")
-
-
-def _configure_logger() -> None:
-    """
-    Configure the firecrawl logger for console output.
-
-    The function attaches a handler for console output with a specific format and date
-    format to the firecrawl logger.
-    """
-    try:
-        formatter = logging.Formatter(
-            "[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s",
-            datefmt="%Y-%m-%d %H:%M:%S",
-        )
-
-        console_handler = logging.StreamHandler()
-        console_handler.setFormatter(formatter)
-
-        logger.addHandler(console_handler)
-    except Exception as e:
-        logger.error("Failed to configure logging: %s", e)
-
-
-def setup_logging() -> None:
-    """Set up logging based on the FIRECRAWL_LOGGING_LEVEL environment variable."""
-    if logger.hasHandlers():
-        return
-
-    if not (env := os.getenv("FIRECRAWL_LOGGING_LEVEL", "").upper()):
-        logger.addHandler(logging.NullHandler()) 
-        return
-
-    _configure_logger()
-
-    if env == "DEBUG":
-        logger.setLevel(logging.DEBUG)
-    elif env == "INFO":
-        logger.setLevel(logging.INFO)
-    elif env == "WARNING":
-        logger.setLevel(logging.WARNING)
-    elif env == "ERROR":
-        logger.setLevel(logging.ERROR)
-    elif env == "CRITICAL":
-        logger.setLevel(logging.CRITICAL)
-    else:
-        logger.setLevel(logging.INFO)
-        logger.warning("Unknown logging level: %s, defaulting to INFO", env)
-
-setup_logging()
-logger.debug("Debugging logger setup")
-
-__all__ = [
-    'Firecrawl',
-    'AsyncFirecrawl',
-    'FirecrawlApp',
-    'AsyncFirecrawlApp',
-    'Watcher',
-    'AsyncWatcher',
-    'V1FirecrawlApp',
-    'AsyncV1FirecrawlApp',
-    'V1JsonConfig',
-    'V1ScrapeOptions',
-    'V1ChangeTrackingOptions',
-]

build/lib/firecrawl/__tests__/e2e/v2/aio/test_aio_batch_scrape.py

@@ -1,79 +0,0 @@
-import os
-import asyncio
-import pytest
-from dotenv import load_dotenv
-from firecrawl import AsyncFirecrawl
-
-
-load_dotenv()
-
-if not os.getenv("API_KEY"):
-    raise ValueError("API_KEY is not set")
-
-if not os.getenv("API_URL"):
-    raise ValueError("API_URL is not set")
-
-
-@pytest.mark.asyncio
-async def test_async_batch_start_and_status():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_batch_scrape([
-        "https://docs.firecrawl.dev",
-        "https://firecrawl.dev",
-    ], formats=["markdown"], max_concurrency=1)
-    job_id = start.id
-
-    deadline = asyncio.get_event_loop().time() + 240
-    status = await client.get_batch_scrape_status(job_id)
-    while status.status not in ("completed", "failed", "cancelled") and asyncio.get_event_loop().time() < deadline:
-        await asyncio.sleep(2)
-        status = await client.get_batch_scrape_status(job_id)
-
-    assert status.status in ("completed", "failed", "cancelled")
-
-
-@pytest.mark.asyncio
-async def test_async_batch_wait_minimal():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    job = await client.batch_scrape([
-        "https://docs.firecrawl.dev",
-        "https://firecrawl.dev",
-    ], formats=["markdown"], poll_interval=1, timeout=120)
-    assert job.status in ("completed", "failed")
-
-
-@pytest.mark.asyncio
-async def test_async_batch_wait_with_all_params():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
-    job = await client.batch_scrape(
-        [
-            "https://docs.firecrawl.dev",
-            "https://firecrawl.dev",
-        ],
-        formats=[
-            "markdown",
-            {"type": "json", "prompt": "Extract page title", "schema": json_schema},
-            {"type": "changeTracking", "prompt": "Track changes", "modes": ["json"]},
-        ],
-        only_main_content=True,
-        mobile=False,
-        ignore_invalid_urls=True,
-        max_concurrency=2,
-        zero_data_retention=False,
-        poll_interval=1,
-        timeout=180,
-    )
-    assert job.status in ("completed", "failed")
-
-
-@pytest.mark.asyncio
-async def test_async_cancel_batch():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_batch_scrape([
-        "https://docs.firecrawl.dev",
-        "https://firecrawl.dev",
-    ], formats=["markdown"], max_concurrency=1)
-    ok = await client.cancel_batch_scrape(start.id)
-    assert ok is True
-

build/lib/firecrawl/__tests__/e2e/v2/aio/test_aio_crawl.py

@@ -1,188 +0,0 @@
-import os
-import asyncio
-import pytest
-from dotenv import load_dotenv
-from firecrawl import AsyncFirecrawl
-from firecrawl.v2.types import ScrapeOptions
-
-
-load_dotenv()
-
-if not os.getenv("API_KEY"):
-    raise ValueError("API_KEY is not set")
-
-if not os.getenv("API_URL"):
-    raise ValueError("API_URL is not set")
-
-
-@pytest.mark.asyncio
-async def test_async_crawl_start_and_status():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2)
-    job_id = start.id
-
-    deadline = asyncio.get_event_loop().time() + 180
-    status = await client.get_crawl_status(job_id)
-    while status.status not in ("completed", "failed") and asyncio.get_event_loop().time() < deadline:
-        await asyncio.sleep(2)
-        status = await client.get_crawl_status(job_id)
-
-    assert status.status in ("completed", "failed")
-
-
-@pytest.mark.asyncio
-async def test_async_crawl_with_all_params():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    # rich scrape options including json format
-    json_schema = {
-        "type": "object",
-        "properties": {"title": {"type": "string"}},
-        "required": ["title"],
-    }
-    status = await client.crawl(
-        url="https://docs.firecrawl.dev",
-        prompt="Extract docs and blog",
-        include_paths=["/docs/*", "/blog/*"],
-        exclude_paths=["/admin/*"],
-        max_discovery_depth=2,
-        ignore_sitemap=False,
-        ignore_query_parameters=True,
-        limit=5,
-        crawl_entire_domain=False,
-        allow_external_links=True,
-        allow_subdomains=True,
-        delay=1,
-        max_concurrency=2,
-        webhook="https://example.com/hook",
-        scrape_options=ScrapeOptions(
-            formats=[
-                "markdown",
-                "rawHtml",
-                {"type": "json", "prompt": "Extract title", "schema": json_schema},
-            ],
-            only_main_content=True,
-            mobile=False,
-            timeout=20000,
-            wait_for=500,
-            skip_tls_verification=False,
-            remove_base64_images=False,
-        ),
-        zero_data_retention=False,
-        poll_interval=2,
-        timeout=180,
-    )
-    assert status.status in ("completed", "failed")
-
-
-@pytest.mark.asyncio
-async def test_async_start_crawl_with_options():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=5, max_discovery_depth=2)
-    assert start.id is not None and start.url is not None
-
-
-@pytest.mark.asyncio
-async def test_async_start_crawl_with_prompt():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_crawl("https://firecrawl.dev", prompt="Extract all blog posts", limit=3)
-    assert start.id is not None and start.url is not None
-
-
-@pytest.mark.asyncio
-async def test_async_get_crawl_status_shape():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=3)
-    status = await client.get_crawl_status(start.id)
-    assert status.status in ("scraping", "completed", "failed")
-    assert status.completed >= 0
-    assert status.expires_at is not None
-    assert isinstance(status.data, list)
-
-
-@pytest.mark.asyncio
-async def test_async_crawl_with_wait():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    job = await client.crawl(url="https://docs.firecrawl.dev", limit=3, max_discovery_depth=2, poll_interval=1, timeout=120)
-    assert job.status in ("completed", "failed")
-    assert job.completed >= 0 and job.total >= 0 and isinstance(job.data, list)
-
-
-@pytest.mark.asyncio
-async def test_async_crawl_with_prompt_and_wait():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    job = await client.crawl(url="https://docs.firecrawl.dev", prompt="Extract all blog posts", limit=3, poll_interval=1, timeout=120)
-    assert job.status in ("completed", "failed")
-    assert job.completed >= 0 and job.total >= 0 and isinstance(job.data, list)
-
-
-@pytest.mark.asyncio
-async def test_async_crawl_with_scrape_options():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    scrape_opts = ScrapeOptions(formats=["markdown", "links"], only_main_content=False, mobile=True)
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2, scrape_options=scrape_opts)
-    assert start.id is not None
-
-
-@pytest.mark.asyncio
-async def test_async_crawl_with_json_format_object():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
-    scrape_opts = ScrapeOptions(formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}])
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2, scrape_options=scrape_opts)
-    assert start.id is not None
-
-
-@pytest.mark.asyncio
-async def test_async_cancel_crawl():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=3)
-    cancelled = await client.cancel_crawl(start.id)
-    assert cancelled is True
-
-
-@pytest.mark.asyncio
-async def test_async_get_crawl_errors_and_invalid_job():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=2)
-    errs = await client.get_crawl_errors(start.id)
-    assert hasattr(errs, "errors") and hasattr(errs, "robots_blocked")
-    with pytest.raises(Exception):
-        await client.get_crawl_errors("invalid-job-id-12345")
-
-
-@pytest.mark.asyncio
-async def test_async_active_crawls():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    resp = await client.active_crawls()
-    assert hasattr(resp, "success") and hasattr(resp, "crawls")
-
-
-@pytest.mark.asyncio
-async def test_async_active_crawls_with_running_crawl():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    start = await client.start_crawl("https://docs.firecrawl.dev", limit=3)
-    # fetch active crawls and assert our ID is listed
-    active = await client.active_crawls()
-    ids = [c.id for c in active.crawls]
-    assert start.id in ids
-    # cleanup
-    await client.cancel_crawl(start.id)
-
-
-@pytest.mark.asyncio
-async def test_async_crawl_params_preview():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    params = await client.crawl_params_preview(
-        url="https://docs.firecrawl.dev",
-        prompt="Extract all blog posts and documentation",
-    )
-    assert params is not None
-    # basic sanity: at least one field should be suggested
-    has_any = any([
-        getattr(params, "limit", None) is not None,
-        getattr(params, "include_paths", None) is not None,
-        getattr(params, "max_discovery_depth", None) is not None,
-    ])
-    assert has_any
-
-

build/lib/firecrawl/__tests__/e2e/v2/aio/test_aio_extract.py

@@ -1,38 +0,0 @@
-import os
-import pytest
-from dotenv import load_dotenv
-from firecrawl import AsyncFirecrawl
-
-
-load_dotenv()
-
-if not os.getenv("API_KEY"):
-    raise ValueError("API_KEY is not set")
-
-if not os.getenv("API_URL"):
-    raise ValueError("API_URL is not set")
-
-
-@pytest.mark.asyncio
-async def test_async_extract_minimal():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    res = await client.extract(urls=["https://docs.firecrawl.dev"], prompt="Extract title")
-    assert res is not None
-
-
-@pytest.mark.asyncio
-async def test_async_extract_with_schema_and_options():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
-    res = await client.extract(
-        urls=["https://docs.firecrawl.dev"],
-        prompt="Extract title",
-        schema=schema,
-        system_prompt="You are a helpful extractor",
-        allow_external_links=False,
-        enable_web_search=False,
-        show_sources=False,
-        # agent={"model": "FIRE-1", "prompt": "Extract title"}, # Skipping agent test in CI
-    )
-    assert res is not None
-

build/lib/firecrawl/__tests__/e2e/v2/aio/test_aio_map.py

@@ -1,40 +0,0 @@
-import os
-import pytest
-from dotenv import load_dotenv
-from firecrawl import AsyncFirecrawl
-
-
-load_dotenv()
-
-if not os.getenv("API_KEY"):
-    raise ValueError("API_KEY is not set")
-
-if not os.getenv("API_URL"):
-    raise ValueError("API_URL is not set")
-
-
-@pytest.mark.asyncio
-async def test_async_map_minimal():
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    resp = await client.map("https://docs.firecrawl.dev")
-    assert hasattr(resp, "links") and isinstance(resp.links, list)
-    if resp.links:
-        first = resp.links[0]
-        assert hasattr(first, "url") and isinstance(first.url, str) and first.url.startswith("http")
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize("sitemap", ["only", "include", "skip"])
-async def test_async_map_with_all_params(sitemap):
-    client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-    resp = await client.map(
-        "https://docs.firecrawl.dev",
-        search="docs",
-        include_subdomains=True,
-        limit=10,
-        sitemap=sitemap,
-        timeout=15000,
-    )
-    assert hasattr(resp, "links") and isinstance(resp.links, list)
-    assert len(resp.links) <= 10
-