firecrawl-py 3.3.1__py3-none-any.whl → 3.3.3__py3-none-any.whl

build/lib/firecrawl/__tests__/e2e/v2/test_search.py

@@ -1,269 +0,0 @@
-from firecrawl import Firecrawl
-import os
-from dotenv import load_dotenv
-from firecrawl.types import SearchData, Document, ScrapeOptions, SearchResultWeb, SearchResultNews, SearchResultImages
-
-load_dotenv()
-
-firecrawl = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-
-def _collect_texts(entries):
-    texts = []
-    for r in entries or []:
-        title = getattr(r, 'title', None) if hasattr(r, 'title') else None
-        desc = getattr(r, 'description', None) if hasattr(r, 'description') else None
-        if title:
-            texts.append(str(title).lower())
-        if desc:
-            texts.append(str(desc).lower())
-    return texts
-
-def _is_document(entry) -> bool:
-    try:
-        from firecrawl.v2.types import Document
-        return isinstance(entry, Document) or \
-               hasattr(entry, 'markdown') or \
-               hasattr(entry, 'html') or \
-               hasattr(entry, 'raw_html') or \
-               hasattr(entry, 'json') or \
-               hasattr(entry, 'screenshot') or \
-               hasattr(entry, 'change_tracking') or \
-               hasattr(entry, 'summary')
-    except Exception:
-        return hasattr(entry, 'markdown') or \
-               hasattr(entry, 'html') or \
-               hasattr(entry, 'raw_html') or \
-               hasattr(entry, 'json') or \
-               hasattr(entry, 'screenshot') or \
-               hasattr(entry, 'change_tracking') or \
-               hasattr(entry, 'summary')
-
-def test_search_minimal_request():
-    results = firecrawl.search(
-        query="What is the capital of France?"
-    )
-    
-    assert isinstance(results, SearchData)
-    assert hasattr(results, 'web')
-    assert results.web is not None
-    assert len(results.web) > 0
-    assert hasattr(results, 'news')
-    assert results.news is None
-    assert hasattr(results, 'images')
-    assert results.images is None
-    
-    for result in results.web:
-        assert isinstance(result, SearchResultWeb)
-        assert hasattr(result, 'url')
-        assert hasattr(result, 'title')
-        assert hasattr(result, 'description')
-        assert result.url.startswith('http')
-        assert result.title is not None
-        assert result.description is not None
-
-    all_text = ' '.join(_collect_texts(results.web))
-    
-    assert 'paris' in all_text
-    
-    assert results.news is None
-    assert results.images is None
-
-
-def test_search_with_sources():
-    """Test search with specific sources."""
-    results = firecrawl.search(
-        query="firecrawl",
-        sources=["web", "news", "images"],
-        limit=3
-    )
-    
-    assert isinstance(results, SearchData)
-    
-    assert results.web is not None
-    assert len(results.web) <= 3
-    assert isinstance(results.web[0], SearchResultWeb)
-    
-    if results.news is not None:
-        assert len(results.news) <= 3
-        assert isinstance(results.news[0], SearchResultNews)
-    
-    if results.images is not None:
-        assert len(results.images) <= 3
-        assert isinstance(results.images[0], SearchResultImages)
-    
-    web_titles = [result.title.lower() for result in results.web]
-    web_descriptions = [result.description.lower() for result in results.web]
-    all_web_text = ' '.join(web_titles + web_descriptions)
-    
-    assert 'firecrawl' in all_web_text
-
-def test_search_result_structure():
-    """Test that SearchResult objects have the correct structure."""
-    results = firecrawl.search(
-        query="test query",
-        limit=1
-    )
-    
-    if results.web and len(results.web) > 0:
-        result = results.web[0]
-        
-        assert hasattr(result, 'url')
-        assert hasattr(result, 'title')
-        assert hasattr(result, 'description')
-        
-        assert isinstance(result.url, str)
-        assert isinstance(result.title, str) or result.title is None
-        assert isinstance(result.description, str) or result.description is None
-        
-        # Test URL format
-        assert result.url.startswith('http')
-
-def test_search_all_parameters():
-    """Test search with all available parameters (comprehensive e2e test)."""
-    from firecrawl.types import ScrapeOptions, JsonFormat, Location, WaitAction
-    
-    # Define a schema for JSON extraction
-    schema = {
-        "type": "object",
-        "properties": {
-            "title": {"type": "string"},
-            "description": {"type": "string"},
-            "url": {"type": "string"}
-        },
-        "required": ["title", "description"]
-    }
-    
-    results = firecrawl.search(
-        query="artificial intelligence",
-        sources=[
-            {"type": "web"},
-            {"type": "news"}
-        ],
-        limit=3,
-        tbs="qdr:m",  # Last month
-        location="US",
-        ignore_invalid_urls=True,
-        timeout=60000,
-        scrape_options=ScrapeOptions(
-            formats=[
-                "markdown",
-                "html",
-                {
-                    "type": "json",
-                    "prompt": "Extract the title and description from the page",
-                    "schema": schema
-                },
-                {"type": "summary"}
-            ],
-            headers={"User-Agent": "Firecrawl-Test/1.0"},
-            include_tags=["h1", "h2", "p"],
-            exclude_tags=["nav", "footer"],
-            only_main_content=True,
-            wait_for=2000,
-            mobile=False,
-            skip_tls_verification=False,
-            remove_base64_images=True,
-            block_ads=True,
-            proxy="basic",
-            max_age=3600000,  # 1 hour cache
-            store_in_cache=True,
-            location=Location(
-                country="US",
-                languages=["en"]
-            ),
-            actions=[
-                WaitAction(milliseconds=1000)
-            ]
-            # Note: raw_html and screenshot_full_page are not supported by v2 API yet
-        )
-    )
-    
-    # Test structure
-    assert isinstance(results, SearchData)
-    assert hasattr(results, 'web')
-    assert hasattr(results, 'news')
-    assert hasattr(results, 'images')
-    
-    # Test that web results exist
-    assert results.web is not None
-    assert len(results.web) <= 3  # Should respect limit
-    
-    # Test that results contain expected content for non-document entries only
-    non_doc_entries = [r for r in (results.web or []) if not _is_document(r)]
-    if non_doc_entries:
-        all_web_text = ' '.join(_collect_texts(non_doc_entries))
-        ai_terms = ['artificial', 'intelligence', 'ai', 'machine', 'learning']
-        assert any(term in all_web_text for term in ai_terms)
-    
-    # Test that each result has proper structure
-    for result in results.web:
-        assert isinstance(result, (SearchResultWeb, Document))
-        if isinstance(result, Document):
-            # Document path: ensure content present
-            assert (result.markdown is not None) or (result.html is not None)
-        else:
-            # LinkResult path
-            assert hasattr(result, 'url')
-            assert isinstance(result.url, str) and result.url.startswith('http')
-    
-    # Test that news results exist (if API supports it)
-    if results.news is not None:
-        assert len(results.news) <= 3
-        for result in results.news:
-            assert isinstance(result, (SearchResultNews, Document))
-            if isinstance(result, Document):
-                assert (result.markdown is not None) or (result.html is not None)
-            else:
-                assert hasattr(result, 'url')
-                assert isinstance(result.url, str) and result.url.startswith('http')
-    
-    # Test that unspecified sources are None
-    assert results.images is None
-
-
-def test_search_formats_flexibility():
-    """Test that both list and ScrapeFormats work for formats."""
-    from firecrawl.types import ScrapeFormats
-    
-    # Test with list format
-    results1 = firecrawl.search(
-        query="python programming",
-        limit=1,
-        scrape_options=ScrapeOptions(
-            formats=["markdown"]
-        )
-    )
-    
-    # Test with ScrapeFormats object
-    results2 = firecrawl.search(
-        query="python programming", 
-        limit=1,
-        scrape_options=ScrapeOptions(
-            formats=ScrapeFormats(markdown=True)
-        )
-    )
-    
-    # Both should work without errors
-    assert isinstance(results1, SearchData)
-    assert isinstance(results2, SearchData)
-    assert results1.web is not None
-    assert results2.web is not None
-
-def test_search_with_json_format_object():
-    """Search with scrape_options including a JSON format object (prompt + schema)."""
-    json_schema = {
-        "type": "object",
-        "properties": {
-            "title": {"type": "string"}
-        },
-        "required": ["title"],
-    }
-    results = firecrawl.search(
-        query="site:docs.firecrawl.dev",
-        limit=1,
-        scrape_options=ScrapeOptions(
-            formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}]
-        ),
-    )
-    assert isinstance(results, SearchData)
-    assert results.web is not None and len(results.web) >= 0

build/lib/firecrawl/__tests__/e2e/v2/test_usage.py

@@ -1,26 +0,0 @@
-import os
-from dotenv import load_dotenv
-from firecrawl import Firecrawl
-
-load_dotenv()
-
-
-class TestUsageE2E:
-    def setup_method(self):
-        # Environment is exported by conftest at import time
-        self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-
-    def test_get_concurrency(self):
-        resp = self.client.get_concurrency()
-        # Shape assertions (endpoint not live yet, but types are defined)
-        assert hasattr(resp, "concurrency")
-        assert hasattr(resp, "max_concurrency")
-
-    def test_get_credit_usage(self):
-        resp = self.client.get_credit_usage()
-        assert hasattr(resp, "remaining_credits")
-
-    def test_get_token_usage(self):
-        resp = self.client.get_token_usage()
-        assert hasattr(resp, "remaining_tokens")
-

build/lib/firecrawl/__tests__/e2e/v2/test_watcher.py

@@ -1,65 +0,0 @@
-import os
-import time
-from dotenv import load_dotenv
-from firecrawl import Firecrawl
-
-load_dotenv()
-
-if not os.getenv("API_KEY"):
-    raise ValueError("API_KEY is not set")
-
-if not os.getenv("API_URL"):
-    raise ValueError("API_URL is not set")
-
-
-class TestWatcherE2E:
-    def setup_method(self):
-        from firecrawl import Firecrawl
-        self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
-
-    def test_crawl_watcher(self):
-        # Start a small crawl job
-        start_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=2)
-        job_id = start_job.id
-
-        statuses = []
-        w = self.client.watcher(job_id, kind="crawl", poll_interval=1, timeout=120)
-        w.add_listener(lambda s: statuses.append(s.status))
-        w.start()
-
-        # Wait for terminal state up to 180 seconds
-        deadline = time.time() + 180
-        while time.time() < deadline:
-            if statuses and statuses[-1] in ["completed", "failed"]:
-                break
-            time.sleep(1)
-
-        w.stop()
-
-        assert len(statuses) > 0
-        assert statuses[-1] in ["completed", "failed"]
-
-    def test_batch_watcher(self):
-        urls = [
-            "https://docs.firecrawl.dev",
-            "https://firecrawl.dev",
-        ]
-        start_resp = self.client.start_batch_scrape(urls, formats=["markdown"], max_concurrency=1)
-        job_id = start_resp.id
-
-        statuses = []
-        w = self.client.watcher(job_id, kind="batch", poll_interval=1, timeout=180)
-        w.add_listener(lambda s: statuses.append(s.status))
-        w.start()
-
-        deadline = time.time() + 240
-        while time.time() < deadline:
-            if statuses and statuses[-1] in ["completed", "failed", "cancelled"]:
-                break
-            time.sleep(1)
-
-        w.stop()
-
-        assert len(statuses) > 0
-        assert statuses[-1] in ["completed", "failed", "cancelled"]
-

build/lib/firecrawl/__tests__/unit/v2/methods/aio/test_aio_crawl_params.py

@@ -1,12 +0,0 @@
-import pytest
-from firecrawl.v2.types import CrawlParamsRequest
-from firecrawl.v2.methods.aio import crawl as aio_crawl
-
-
-@pytest.mark.asyncio
-async def test_crawl_params_request_validation():
-    with pytest.raises(ValueError):
-        await aio_crawl.crawl_params_preview(None, CrawlParamsRequest(url="", prompt="x"))
-    with pytest.raises(ValueError):
-        await aio_crawl.crawl_params_preview(None, CrawlParamsRequest(url="https://x", prompt=""))
-

build/lib/firecrawl/__tests__/unit/v2/methods/aio/test_aio_crawl_request_preparation.py

@@ -1,61 +0,0 @@
-from firecrawl.v2.types import CrawlRequest, ScrapeOptions, WebhookConfig
-from firecrawl.v2.methods.aio.crawl import _prepare_crawl_request
-
-
-class TestAsyncCrawlRequestPreparation:
-    def test_basic_request(self):
-        req = CrawlRequest(url="https://example.com")
-        payload = _prepare_crawl_request(req)
-        assert payload["url"] == "https://example.com"
-
-    def test_field_mappings(self):
-        req = CrawlRequest(
-            url="https://example.com",
-            include_paths=["/docs/*"],
-            exclude_paths=["/admin/*"],
-            max_discovery_depth=2,
-            sitemap="skip",
-            ignore_query_parameters=True,
-            crawl_entire_domain=True,
-            allow_external_links=False,
-            allow_subdomains=True,
-            max_concurrency=5,
-            zero_data_retention=True,
-        )
-        payload = _prepare_crawl_request(req)
-        assert payload["includePaths"] == ["/docs/*"]
-        assert payload["excludePaths"] == ["/admin/*"]
-        assert payload["maxDiscoveryDepth"] == 2
-        assert payload["sitemap"] == "skip"
-        assert payload["ignoreQueryParameters"] is True
-        assert payload["crawlEntireDomain"] is True
-        assert payload["allowExternalLinks"] is False
-        assert payload["allowSubdomains"] is True
-        assert payload["maxConcurrency"] == 5
-        assert payload["zeroDataRetention"] is True
-
-    def test_webhook_preparation(self):
-        # string webhook
-        req = CrawlRequest(url="https://example.com", webhook="https://example.com/hook")
-        payload = _prepare_crawl_request(req)
-        assert payload["webhook"] == "https://example.com/hook"
-
-        # object webhook
-        req2 = CrawlRequest(url="https://example.com", webhook=WebhookConfig(url="https://x/h", headers={"X": "1"}, events=["completed"]))
-        payload2 = _prepare_crawl_request(req2)
-        assert isinstance(payload2["webhook"], dict)
-        assert payload2["webhook"]["url"] == "https://x/h"
-        assert payload2["webhook"]["headers"] == {"X": "1"}
-
-    def test_webhook_none_values_excluded(self):
-        req = CrawlRequest(
-            url="https://example.com",
-            webhook=WebhookConfig(url="https://example.com/webhook", headers=None, metadata=None, events=None),
-        )
-        payload = _prepare_crawl_request(req)
-        webhook = payload["webhook"]
-        assert webhook["url"] == "https://example.com/webhook"
-        assert "headers" not in webhook
-        assert "metadata" not in webhook
-        assert "events" not in webhook
-

build/lib/firecrawl/__tests__/unit/v2/methods/aio/test_aio_crawl_validation.py

@@ -1,12 +0,0 @@
-from firecrawl.v2.types import CrawlRequest, ScrapeOptions
-from firecrawl.v2.methods.aio.crawl import _prepare_crawl_request
-import pytest
-
-
-class TestAsyncCrawlValidation:
-    def test_invalid_url(self):
-        with pytest.raises(ValueError):
-            _prepare_crawl_request(CrawlRequest(url=""))
-        with pytest.raises(ValueError):
-            _prepare_crawl_request(CrawlRequest(url="   "))
-

build/lib/firecrawl/__tests__/unit/v2/methods/aio/test_aio_map_request_preparation.py

@@ -1,19 +0,0 @@
-import pytest
-from firecrawl.v2.types import MapOptions
-from firecrawl.v2.methods.aio.map import _prepare_map_request
-
-
-class TestAsyncMapRequestPreparation:
-    def test_basic(self):
-        payload = _prepare_map_request("https://example.com")
-        assert payload["url"] == "https://example.com"
-
-    def test_fields(self):
-        opts = MapOptions(search="docs", include_subdomains=True, limit=10, sitemap="only", timeout=15000)
-        payload = _prepare_map_request("https://example.com", opts)
-        assert payload["search"] == "docs"
-        assert payload["includeSubdomains"] is True
-        assert payload["limit"] == 10
-        assert payload["sitemap"] == "only"
-        assert payload["timeout"] == 15000
-

build/lib/firecrawl/__tests__/unit/v2/methods/aio/test_aio_scrape_request_preparation.py

@@ -1,50 +0,0 @@
-import pytest
-from firecrawl.v2.types import ScrapeOptions, Location
-from firecrawl.v2.methods.aio.scrape import _prepare_scrape_request
-
-
-class TestAsyncScrapeRequestPreparation:
-    @pytest.mark.asyncio
-    async def test_basic_request_preparation(self):
-        payload = await _prepare_scrape_request("https://example.com", None)
-        assert payload["url"] == "https://example.com"
-
-    @pytest.mark.asyncio
-    async def test_options_conversion(self):
-        opts = ScrapeOptions(
-            formats=["markdown", {"type": "screenshot", "full_page": True, "quality": 80}],
-            include_tags=["main"],
-            exclude_tags=["nav"],
-            only_main_content=True,
-            wait_for=500,
-            timeout=30000,
-            mobile=True,
-            parsers=["pdf"],
-            location=Location(country="us", languages=["en"]),
-            skip_tls_verification=False,
-            remove_base64_images=False,
-            fast_mode=True,
-            use_mock="test",
-            block_ads=False,
-            proxy="basic",
-            max_age=1000,
-            store_in_cache=False,
-        )
-        payload = await _prepare_scrape_request("https://example.com", opts)
-        assert payload["url"] == "https://example.com"
-        assert isinstance(payload.get("formats"), list) and "markdown" in payload["formats"]
-        assert payload["includeTags"] == ["main"]
-        assert payload["excludeTags"] == ["nav"]
-        assert payload["onlyMainContent"] is True
-        assert payload["waitFor"] == 500
-        assert payload["timeout"] == 30000
-        assert payload["mobile"] is True
-        assert payload["skipTlsVerification"] is False
-        assert payload["removeBase64Images"] is False
-        assert payload["fastMode"] is True
-        assert payload["useMock"] == "test"
-        assert payload["blockAds"] is False
-        assert payload["proxy"] == "basic"
-        assert payload["maxAge"] == 1000
-        assert payload["storeInCache"] is False
-

build/lib/firecrawl/__tests__/unit/v2/methods/aio/test_aio_search_request_preparation.py

@@ -1,63 +0,0 @@
-import pytest
-from firecrawl.v2.types import SearchRequest, ScrapeOptions
-from firecrawl.v2.methods.aio.search import _prepare_search_request
-
-
-class TestAsyncSearchRequestPreparation:
-    def test_basic_request_preparation(self):
-        request = SearchRequest(query="test query")
-        data = _prepare_search_request(request)
-        assert data["query"] == "test query"
-        assert "ignore_invalid_urls" not in data
-        assert "scrape_options" not in data
-
-    def test_all_fields_conversion(self):
-        scrape_opts = ScrapeOptions(
-            formats=["markdown"],
-            headers={"User-Agent": "Test"},
-            include_tags=["h1", "h2"],
-            exclude_tags=["nav"],
-            only_main_content=False,
-            timeout=15000,
-            wait_for=2000,
-            mobile=True,
-            skip_tls_verification=True,
-            remove_base64_images=False,
-        )
-        request = SearchRequest(
-            query="test query",
-            sources=["web", "news"],
-            limit=10,
-            tbs="qdr:w",
-            location="US",
-            ignore_invalid_urls=False,
-            timeout=30000,
-            scrape_options=scrape_opts,
-        )
-        data = _prepare_search_request(request)
-        assert data["ignoreInvalidURLs"] is False
-        assert "scrapeOptions" in data
-
-    def test_exclude_none_behavior(self):
-        request = SearchRequest(
-            query="test",
-            sources=None,
-            limit=None,
-            tbs=None,
-            location=None,
-            ignore_invalid_urls=None,
-            timeout=None,
-            scrape_options=None,
-        )
-        data = _prepare_search_request(request)
-        assert "query" in data
-        assert len(data) == 1
-
-    def test_empty_scrape_options(self):
-        request = SearchRequest(query="test", scrape_options=ScrapeOptions())
-        data = _prepare_search_request(request)
-        assert "scrapeOptions" in data
-        scrape_data = data["scrapeOptions"]
-        assert "onlyMainContent" in scrape_data
-        assert "mobile" in scrape_data
-

build/lib/firecrawl/__tests__/unit/v2/methods/aio/test_batch_request_preparation_async.py

@@ -1,28 +0,0 @@
-from firecrawl.v2.types import ScrapeOptions, Location
-from firecrawl.v2.methods.aio.batch import _prepare as _prepare_batch
-
-
-class TestAsyncBatchRequestPreparation:
-    def test_urls_validation_and_conversion(self):
-        payload = _prepare_batch(["https://example.com", "http://foo.bar"], options=None)
-        assert payload["urls"] == ["https://example.com", "http://foo.bar"]
-
-    def test_options_and_batch_fields(self):
-        opts = ScrapeOptions(formats=["markdown"], only_main_content=True)
-        payload = _prepare_batch(
-            ["https://example.com"],
-            options=opts,
-            webhook="https://hook.example",
-            append_to_id="00000000-0000-0000-0000-000000000000",
-            ignore_invalid_urls=True,
-            max_concurrency=3,
-            zero_data_retention=True,
-            integration="zapier",
-        )
-        assert payload["webhook"] == "https://hook.example"
-        assert payload["appendToId"] == "00000000-0000-0000-0000-000000000000"
-        assert payload["ignoreInvalidURLs"] is True
-        assert payload["maxConcurrency"] == 3
-        assert payload["zeroDataRetention"] is True
-        assert payload["integration"] == "zapier"
-