lfx-nightly 0.2.0.dev25__py3-none-any.whl

lfx/components/firecrawl/__init__.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from .firecrawl_crawl_api import FirecrawlCrawlApi
+    from .firecrawl_extract_api import FirecrawlExtractApi
+    from .firecrawl_map_api import FirecrawlMapApi
+    from .firecrawl_scrape_api import FirecrawlScrapeApi
+
+_dynamic_imports = {
+    "FirecrawlCrawlApi": "firecrawl_crawl_api",
+    "FirecrawlExtractApi": "firecrawl_extract_api",
+    "FirecrawlMapApi": "firecrawl_map_api",
+    "FirecrawlScrapeApi": "firecrawl_scrape_api",
+}
+
+__all__ = [
+    "FirecrawlCrawlApi",
+    "FirecrawlExtractApi",
+    "FirecrawlMapApi",
+    "FirecrawlScrapeApi",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import firecrawl components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

lfx/components/firecrawl/firecrawl_crawl_api.py

@@ -0,0 +1,88 @@
+import uuid
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import DataInput, IntInput, MultilineInput, Output, SecretStrInput, StrInput
+from lfx.schema.data import Data
+
+
+class FirecrawlCrawlApi(Component):
+    display_name: str = "Firecrawl Crawl API"
+    description: str = "Crawls a URL and returns the results."
+    name = "FirecrawlCrawlApi"
+
+    documentation: str = "https://docs.firecrawl.dev/v1/api-reference/endpoint/crawl-post"
+
+    inputs = [
+        SecretStrInput(
+            name="api_key",
+            display_name="Firecrawl API Key",
+            required=True,
+            password=True,
+            info="The API key to use Firecrawl API.",
+        ),
+        MultilineInput(
+            name="url",
+            display_name="URL",
+            required=True,
+            info="The URL to scrape.",
+            tool_mode=True,
+        ),
+        IntInput(
+            name="timeout",
+            display_name="Timeout",
+            info="Timeout in milliseconds for the request.",
+        ),
+        StrInput(
+            name="idempotency_key",
+            display_name="Idempotency Key",
+            info="Optional idempotency key to ensure unique requests.",
+        ),
+        DataInput(
+            name="crawlerOptions",
+            display_name="Crawler Options",
+            info="The crawler options to send with the request.",
+        ),
+        DataInput(
+            name="scrapeOptions",
+            display_name="Scrape Options",
+            info="The page options to send with the request.",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Data", name="data", method="crawl"),
+    ]
+    idempotency_key: str | None = None
+
+    def crawl(self) -> Data:
+        try:
+            from firecrawl import FirecrawlApp
+        except ImportError as e:
+            msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`."
+            raise ImportError(msg) from e
+
+        params = self.crawlerOptions.__dict__["data"] if self.crawlerOptions else {}
+        scrape_options_dict = self.scrapeOptions.__dict__["data"] if self.scrapeOptions else {}
+        if scrape_options_dict:
+            params["scrapeOptions"] = scrape_options_dict
+
+        # Set default values for new parameters in v1
+        params.setdefault("maxDepth", 2)
+        params.setdefault("limit", 10000)
+        params.setdefault("allowExternalLinks", False)
+        params.setdefault("allowBackwardLinks", False)
+        params.setdefault("ignoreSitemap", False)
+        params.setdefault("ignoreQueryParameters", False)
+
+        # Ensure onlyMainContent is explicitly set if not provided
+        if "scrapeOptions" in params:
+            params["scrapeOptions"].setdefault("onlyMainContent", True)
+        else:
+            params["scrapeOptions"] = {"onlyMainContent": True}
+
+        if not self.idempotency_key:
+            self.idempotency_key = str(uuid.uuid4())
+
+        app = FirecrawlApp(api_key=self.api_key)
+        crawl_result = app.crawl_url(self.url, params=params, idempotency_key=self.idempotency_key)
+        return Data(data={"results": crawl_result})

lfx/components/firecrawl/firecrawl_extract_api.py

@@ -0,0 +1,136 @@
+from lfx.custom.custom_component.component import Component
+from lfx.io import BoolInput, DataInput, MultilineInput, Output, SecretStrInput
+from lfx.log.logger import logger
+from lfx.schema.data import Data
+
+
+class FirecrawlExtractApi(Component):
+    display_name: str = "Firecrawl Extract API"
+    description: str = "Extracts data from a URL."
+    name = "FirecrawlExtractApi"
+
+    documentation: str = "https://docs.firecrawl.dev/api-reference/endpoint/extract"
+
+    inputs = [
+        SecretStrInput(
+            name="api_key",
+            display_name="Firecrawl API Key",
+            required=True,
+            password=True,
+            info="The API key to use Firecrawl API.",
+        ),
+        MultilineInput(
+            name="urls",
+            display_name="URLs",
+            required=True,
+            info="List of URLs to extract data from (separated by commas or new lines).",
+            tool_mode=True,
+        ),
+        MultilineInput(
+            name="prompt",
+            display_name="Prompt",
+            required=True,
+            info="Prompt to guide the extraction process.",
+            tool_mode=True,
+        ),
+        DataInput(
+            name="schema",
+            display_name="Schema",
+            required=False,
+            info="Schema to define the structure of the extracted data.",
+        ),
+        BoolInput(
+            name="enable_web_search",
+            display_name="Enable Web Search",
+            info="When true, the extraction will use web search to find additional data.",
+        ),
+        # # Optional: Not essential for basic extraction
+        # BoolInput(
+        #     name="ignore_sitemap",
+        #     display_name="Ignore Sitemap",
+        #     info="When true, sitemap.xml files will be ignored during website scanning.",
+        # ),
+        # # Optional: Not essential for basic extraction
+        # BoolInput(
+        #     name="include_subdomains",
+        #     display_name="Include Subdomains",
+        #     info="When true, subdomains of the provided URLs will also be scanned.",
+        # ),
+        # # Optional: Not essential for basic extraction
+        # BoolInput(
+        #     name="show_sources",
+        #     display_name="Show Sources",
+        #     info="When true, the sources used to extract the data will be included in the response.",
+        # ),
+    ]
+
+    outputs = [
+        Output(display_name="Data", name="data", method="extract"),
+    ]
+
+    def extract(self) -> Data:
+        try:
+            from firecrawl import FirecrawlApp
+        except ImportError as e:
+            msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`."
+            raise ImportError(msg) from e
+
+        # Validate API key
+        if not self.api_key:
+            msg = "API key is required"
+            raise ValueError(msg)
+
+        # Validate URLs
+        if not self.urls:
+            msg = "URLs are required"
+            raise ValueError(msg)
+
+        # Split and validate URLs (handle both commas and newlines)
+        urls = [url.strip() for url in self.urls.replace("\n", ",").split(",") if url.strip()]
+        if not urls:
+            msg = "No valid URLs provided"
+            raise ValueError(msg)
+
+        # Validate and process prompt
+        if not self.prompt:
+            msg = "Prompt is required"
+            raise ValueError(msg)
+
+        # Get the prompt text (handling both string and multiline input)
+        prompt_text = self.prompt.strip()
+
+        # Enhance the prompt to encourage comprehensive extraction
+        enhanced_prompt = prompt_text
+        if "schema" not in prompt_text.lower():
+            enhanced_prompt = f"{prompt_text}. Please extract all instances in a comprehensive, structured format."
+
+        params = {
+            "prompt": enhanced_prompt,
+            "enableWebSearch": self.enable_web_search,
+            # Optional parameters - not essential for basic extraction
+            "ignoreSitemap": self.ignore_sitemap,
+            "includeSubdomains": self.include_subdomains,
+            "showSources": self.show_sources,
+            "timeout": 300,
+        }
+
+        # Only add schema to params if it's provided and is a valid schema structure
+        if self.schema:
+            try:
+                if isinstance(self.schema, dict) and "type" in self.schema:
+                    params["schema"] = self.schema
+                elif hasattr(self.schema, "dict") and "type" in self.schema.dict():
+                    params["schema"] = self.schema.dict()
+                else:
+                    # Skip invalid schema without raising an error
+                    pass
+            except Exception as e:  # noqa: BLE001
+                logger.error(f"Invalid schema: {e!s}")
+
+        try:
+            app = FirecrawlApp(api_key=self.api_key)
+            extract_result = app.extract(urls, params=params)
+            return Data(data=extract_result)
+        except Exception as e:
+            msg = f"Error during extraction: {e!s}"
+            raise ValueError(msg) from e

lfx/components/firecrawl/firecrawl_map_api.py

@@ -0,0 +1,89 @@
+from lfx.custom.custom_component.component import Component
+from lfx.io import (
+    BoolInput,
+    MultilineInput,
+    Output,
+    SecretStrInput,
+)
+from lfx.schema.data import Data
+
+
+class FirecrawlMapApi(Component):
+    display_name: str = "Firecrawl Map API"
+    description: str = "Maps a URL and returns the results."
+    name = "FirecrawlMapApi"
+
+    documentation: str = "https://docs.firecrawl.dev/api-reference/endpoint/map"
+
+    inputs = [
+        SecretStrInput(
+            name="api_key",
+            display_name="Firecrawl API Key",
+            required=True,
+            password=True,
+            info="The API key to use Firecrawl API.",
+        ),
+        MultilineInput(
+            name="urls",
+            display_name="URLs",
+            required=True,
+            info="List of URLs to create maps from (separated by commas or new lines).",
+            tool_mode=True,
+        ),
+        BoolInput(
+            name="ignore_sitemap",
+            display_name="Ignore Sitemap",
+            info="When true, the sitemap.xml file will be ignored during crawling.",
+        ),
+        BoolInput(
+            name="sitemap_only",
+            display_name="Sitemap Only",
+            info="When true, only links found in the sitemap will be returned.",
+        ),
+        BoolInput(
+            name="include_subdomains",
+            display_name="Include Subdomains",
+            info="When true, subdomains of the provided URL will also be scanned.",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Data", name="data", method="map"),
+    ]
+
+    def map(self) -> Data:
+        try:
+            from firecrawl import FirecrawlApp
+        except ImportError as e:
+            msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`."
+            raise ImportError(msg) from e
+
+        # Validate URLs
+        if not self.urls:
+            msg = "URLs are required"
+            raise ValueError(msg)
+
+        # Split and validate URLs (handle both commas and newlines)
+        urls = [url.strip() for url in self.urls.replace("\n", ",").split(",") if url.strip()]
+        if not urls:
+            msg = "No valid URLs provided"
+            raise ValueError(msg)
+
+        params = {
+            "ignoreSitemap": self.ignore_sitemap,
+            "sitemapOnly": self.sitemap_only,
+            "includeSubdomains": self.include_subdomains,
+        }
+
+        app = FirecrawlApp(api_key=self.api_key)
+
+        # Map all provided URLs and combine results
+        combined_links = []
+        for url in urls:
+            result = app.map_url(url, params=params)
+            if isinstance(result, dict) and "links" in result:
+                combined_links.extend(result["links"])
+
+        map_result = {"success": True, "links": combined_links}
+
+        return Data(data=map_result)

lfx/components/firecrawl/firecrawl_scrape_api.py

@@ -0,0 +1,73 @@
+from lfx.custom.custom_component.component import Component
+from lfx.io import (
+    DataInput,
+    IntInput,
+    MultilineInput,
+    Output,
+    SecretStrInput,
+)
+from lfx.schema.data import Data
+
+
+class FirecrawlScrapeApi(Component):
+    display_name: str = "Firecrawl Scrape API"
+    description: str = "Scrapes a URL and returns the results."
+    name = "FirecrawlScrapeApi"
+
+    documentation: str = "https://docs.firecrawl.dev/api-reference/endpoint/scrape"
+
+    inputs = [
+        SecretStrInput(
+            name="api_key",
+            display_name="Firecrawl API Key",
+            required=True,
+            password=True,
+            info="The API key to use Firecrawl API.",
+        ),
+        MultilineInput(
+            name="url",
+            display_name="URL",
+            required=True,
+            info="The URL to scrape.",
+            tool_mode=True,
+        ),
+        IntInput(
+            name="timeout",
+            display_name="Timeout",
+            info="Timeout in milliseconds for the request.",
+        ),
+        DataInput(
+            name="scrapeOptions",
+            display_name="Scrape Options",
+            info="The page options to send with the request.",
+        ),
+        DataInput(
+            name="extractorOptions",
+            display_name="Extractor Options",
+            info="The extractor options to send with the request.",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Data", name="data", method="scrape"),
+    ]
+
+    def scrape(self) -> Data:
+        try:
+            from firecrawl import FirecrawlApp
+        except ImportError as e:
+            msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`."
+            raise ImportError(msg) from e
+
+        params = self.scrapeOptions.__dict__.get("data", {}) if self.scrapeOptions else {}
+        extractor_options_dict = self.extractorOptions.__dict__.get("data", {}) if self.extractorOptions else {}
+        if extractor_options_dict:
+            params["extract"] = extractor_options_dict
+
+        # Set default values for parameters
+        params.setdefault("formats", ["markdown"])  # Default output format
+        params.setdefault("onlyMainContent", True)  # Default to only main content
+
+        app = FirecrawlApp(api_key=self.api_key)
+        results = app.scrape_url(self.url, params=params)
+        return Data(data=results)

lfx/components/flow_controls/__init__.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from lfx.components.flow_controls.conditional_router import ConditionalRouterComponent
+    from lfx.components.flow_controls.data_conditional_router import DataConditionalRouterComponent
+    from lfx.components.flow_controls.flow_tool import FlowToolComponent
+    from lfx.components.flow_controls.listen import ListenComponent
+    from lfx.components.flow_controls.loop import LoopComponent
+    from lfx.components.flow_controls.notify import NotifyComponent
+    from lfx.components.flow_controls.pass_message import PassMessageComponent
+    from lfx.components.flow_controls.run_flow import RunFlowComponent
+    from lfx.components.flow_controls.sub_flow import SubFlowComponent
+
+_dynamic_imports = {
+    "ConditionalRouterComponent": "conditional_router",
+    "DataConditionalRouterComponent": "data_conditional_router",
+    "FlowToolComponent": "flow_tool",
+    "ListenComponent": "listen",
+    "LoopComponent": "loop",
+    "NotifyComponent": "notify",
+    "PassMessageComponent": "pass_message",
+    "RunFlowComponent": "run_flow",
+    "SubFlowComponent": "sub_flow",
+}
+
+__all__ = [
+    "ConditionalRouterComponent",
+    "DataConditionalRouterComponent",
+    "FlowToolComponent",
+    "ListenComponent",
+    "LoopComponent",
+    "NotifyComponent",
+    "PassMessageComponent",
+    "RunFlowComponent",
+    "SubFlowComponent",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import flow control components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)