lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/firecrawl/firecrawl_extract_api.py

@@ -0,0 +1,136 @@
+from lfx.custom.custom_component.component import Component
+from lfx.io import BoolInput, DataInput, MultilineInput, Output, SecretStrInput
+from lfx.log.logger import logger
+from lfx.schema.data import Data
+
+
+class FirecrawlExtractApi(Component):
+    display_name: str = "Firecrawl Extract API"
+    description: str = "Extracts data from a URL."
+    name = "FirecrawlExtractApi"
+
+    documentation: str = "https://docs.firecrawl.dev/api-reference/endpoint/extract"
+
+    inputs = [
+        SecretStrInput(
+            name="api_key",
+            display_name="API Key",
+            required=True,
+            password=True,
+            info="The API key to use Firecrawl API.",
+        ),
+        MultilineInput(
+            name="urls",
+            display_name="URLs",
+            required=True,
+            info="List of URLs to extract data from (separated by commas or new lines).",
+            tool_mode=True,
+        ),
+        MultilineInput(
+            name="prompt",
+            display_name="Prompt",
+            required=True,
+            info="Prompt to guide the extraction process.",
+            tool_mode=True,
+        ),
+        DataInput(
+            name="schema",
+            display_name="Schema",
+            required=False,
+            info="Schema to define the structure of the extracted data.",
+        ),
+        BoolInput(
+            name="enable_web_search",
+            display_name="Enable Web Search",
+            info="When true, the extraction will use web search to find additional data.",
+        ),
+        # # Optional: Not essential for basic extraction
+        # BoolInput(
+        #     name="ignore_sitemap",
+        #     display_name="Ignore Sitemap",
+        #     info="When true, sitemap.xml files will be ignored during website scanning.",
+        # ),
+        # # Optional: Not essential for basic extraction
+        # BoolInput(
+        #     name="include_subdomains",
+        #     display_name="Include Subdomains",
+        #     info="When true, subdomains of the provided URLs will also be scanned.",
+        # ),
+        # # Optional: Not essential for basic extraction
+        # BoolInput(
+        #     name="show_sources",
+        #     display_name="Show Sources",
+        #     info="When true, the sources used to extract the data will be included in the response.",
+        # ),
+    ]
+
+    outputs = [
+        Output(display_name="Data", name="data", method="extract"),
+    ]
+
+    def extract(self) -> Data:
+        try:
+            from firecrawl import FirecrawlApp
+        except ImportError as e:
+            msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`."
+            raise ImportError(msg) from e
+
+        # Validate API key
+        if not self.api_key:
+            msg = "API key is required"
+            raise ValueError(msg)
+
+        # Validate URLs
+        if not self.urls:
+            msg = "URLs are required"
+            raise ValueError(msg)
+
+        # Split and validate URLs (handle both commas and newlines)
+        urls = [url.strip() for url in self.urls.replace("\n", ",").split(",") if url.strip()]
+        if not urls:
+            msg = "No valid URLs provided"
+            raise ValueError(msg)
+
+        # Validate and process prompt
+        if not self.prompt:
+            msg = "Prompt is required"
+            raise ValueError(msg)
+
+        # Get the prompt text (handling both string and multiline input)
+        prompt_text = self.prompt.strip()
+
+        # Enhance the prompt to encourage comprehensive extraction
+        enhanced_prompt = prompt_text
+        if "schema" not in prompt_text.lower():
+            enhanced_prompt = f"{prompt_text}. Please extract all instances in a comprehensive, structured format."
+
+        params = {
+            "prompt": enhanced_prompt,
+            "enableWebSearch": self.enable_web_search,
+            # Optional parameters - not essential for basic extraction
+            "ignoreSitemap": self.ignore_sitemap,
+            "includeSubdomains": self.include_subdomains,
+            "showSources": self.show_sources,
+            "timeout": 300,
+        }
+
+        # Only add schema to params if it's provided and is a valid schema structure
+        if self.schema:
+            try:
+                if isinstance(self.schema, dict) and "type" in self.schema:
+                    params["schema"] = self.schema
+                elif hasattr(self.schema, "dict") and "type" in self.schema.dict():
+                    params["schema"] = self.schema.dict()
+                else:
+                    # Skip invalid schema without raising an error
+                    pass
+            except Exception as e:  # noqa: BLE001
+                logger.error(f"Invalid schema: {e!s}")
+
+        try:
+            app = FirecrawlApp(api_key=self.api_key)
+            extract_result = app.extract(urls, params=params)
+            return Data(data=extract_result)
+        except Exception as e:
+            msg = f"Error during extraction: {e!s}"
+            raise ValueError(msg) from e

lfx/components/firecrawl/firecrawl_map_api.py

@@ -0,0 +1,89 @@
+from lfx.custom.custom_component.component import Component
+from lfx.io import (
+    BoolInput,
+    MultilineInput,
+    Output,
+    SecretStrInput,
+)
+from lfx.schema.data import Data
+
+
+class FirecrawlMapApi(Component):
+    display_name: str = "Firecrawl Map API"
+    description: str = "Maps a URL and returns the results."
+    name = "FirecrawlMapApi"
+
+    documentation: str = "https://docs.firecrawl.dev/api-reference/endpoint/map"
+
+    inputs = [
+        SecretStrInput(
+            name="api_key",
+            display_name="API Key",
+            required=True,
+            password=True,
+            info="The API key to use Firecrawl API.",
+        ),
+        MultilineInput(
+            name="urls",
+            display_name="URLs",
+            required=True,
+            info="List of URLs to create maps from (separated by commas or new lines).",
+            tool_mode=True,
+        ),
+        BoolInput(
+            name="ignore_sitemap",
+            display_name="Ignore Sitemap",
+            info="When true, the sitemap.xml file will be ignored during crawling.",
+        ),
+        BoolInput(
+            name="sitemap_only",
+            display_name="Sitemap Only",
+            info="When true, only links found in the sitemap will be returned.",
+        ),
+        BoolInput(
+            name="include_subdomains",
+            display_name="Include Subdomains",
+            info="When true, subdomains of the provided URL will also be scanned.",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Data", name="data", method="map"),
+    ]
+
+    def map(self) -> Data:
+        try:
+            from firecrawl import FirecrawlApp
+        except ImportError as e:
+            msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`."
+            raise ImportError(msg) from e
+
+        # Validate URLs
+        if not self.urls:
+            msg = "URLs are required"
+            raise ValueError(msg)
+
+        # Split and validate URLs (handle both commas and newlines)
+        urls = [url.strip() for url in self.urls.replace("\n", ",").split(",") if url.strip()]
+        if not urls:
+            msg = "No valid URLs provided"
+            raise ValueError(msg)
+
+        params = {
+            "ignoreSitemap": self.ignore_sitemap,
+            "sitemapOnly": self.sitemap_only,
+            "includeSubdomains": self.include_subdomains,
+        }
+
+        app = FirecrawlApp(api_key=self.api_key)
+
+        # Map all provided URLs and combine results
+        combined_links = []
+        for url in urls:
+            result = app.map_url(url, params=params)
+            if isinstance(result, dict) and "links" in result:
+                combined_links.extend(result["links"])
+
+        map_result = {"success": True, "links": combined_links}
+
+        return Data(data=map_result)

lfx/components/firecrawl/firecrawl_scrape_api.py

@@ -0,0 +1,73 @@
+from lfx.custom.custom_component.component import Component
+from lfx.io import (
+    DataInput,
+    IntInput,
+    MultilineInput,
+    Output,
+    SecretStrInput,
+)
+from lfx.schema.data import Data
+
+
+class FirecrawlScrapeApi(Component):
+    display_name: str = "Firecrawl Scrape API"
+    description: str = "Scrapes a URL and returns the results."
+    name = "FirecrawlScrapeApi"
+
+    documentation: str = "https://docs.firecrawl.dev/api-reference/endpoint/scrape"
+
+    inputs = [
+        SecretStrInput(
+            name="api_key",
+            display_name="API Key",
+            required=True,
+            password=True,
+            info="The API key to use Firecrawl API.",
+        ),
+        MultilineInput(
+            name="url",
+            display_name="URL",
+            required=True,
+            info="The URL to scrape.",
+            tool_mode=True,
+        ),
+        IntInput(
+            name="timeout",
+            display_name="Timeout",
+            info="Timeout in milliseconds for the request.",
+        ),
+        DataInput(
+            name="scrapeOptions",
+            display_name="Scrape Options",
+            info="The page options to send with the request.",
+        ),
+        DataInput(
+            name="extractorOptions",
+            display_name="Extractor Options",
+            info="The extractor options to send with the request.",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Data", name="data", method="scrape"),
+    ]
+
+    def scrape(self) -> Data:
+        try:
+            from firecrawl import FirecrawlApp
+        except ImportError as e:
+            msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`."
+            raise ImportError(msg) from e
+
+        params = self.scrapeOptions.__dict__.get("data", {}) if self.scrapeOptions else {}
+        extractor_options_dict = self.extractorOptions.__dict__.get("data", {}) if self.extractorOptions else {}
+        if extractor_options_dict:
+            params["extract"] = extractor_options_dict
+
+        # Set default values for parameters
+        params.setdefault("formats", ["markdown"])  # Default output format
+        params.setdefault("onlyMainContent", True)  # Default to only main content
+
+        app = FirecrawlApp(api_key=self.api_key)
+        results = app.scrape_url(self.url, params=params)
+        return Data(data=results)

lfx/components/git/__init__.py

@@ -0,0 +1,4 @@
+from .git import GitLoaderComponent
+from .gitextractor import GitExtractorComponent
+
+__all__ = ["GitExtractorComponent", "GitLoaderComponent"]

lfx/components/git/git.py

@@ -0,0 +1,262 @@
+import re
+import tempfile
+from contextlib import asynccontextmanager
+from fnmatch import fnmatch
+from pathlib import Path
+
+import anyio
+from langchain_community.document_loaders.git import GitLoader
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import DropdownInput, MessageTextInput, Output
+from lfx.schema.data import Data
+
+
+class GitLoaderComponent(Component):
+    display_name = "Git"
+    description = (
+        "Load and filter documents from a local or remote Git repository. "
+        "Use a local repo path or clone from a remote URL."
+    )
+    trace_type = "tool"
+    icon = "GitLoader"
+
+    inputs = [
+        DropdownInput(
+            name="repo_source",
+            display_name="Repository Source",
+            options=["Local", "Remote"],
+            required=True,
+            info="Select whether to use a local repo path or clone from a remote URL.",
+            real_time_refresh=True,
+        ),
+        MessageTextInput(
+            name="repo_path",
+            display_name="Local Repository Path",
+            required=False,
+            info="The local path to the existing Git repository (used if 'Local' is selected).",
+            dynamic=True,
+            show=False,
+        ),
+        MessageTextInput(
+            name="clone_url",
+            display_name="Clone URL",
+            required=False,
+            info="The URL of the Git repository to clone (used if 'Clone' is selected).",
+            dynamic=True,
+            show=False,
+        ),
+        MessageTextInput(
+            name="branch",
+            display_name="Branch",
+            required=False,
+            value="main",
+            info="The branch to load files from. Defaults to 'main'.",
+        ),
+        MessageTextInput(
+            name="file_filter",
+            display_name="File Filter",
+            required=False,
+            advanced=True,
+            info=(
+                "Patterns to filter files. For example:\n"
+                "Include only .py files: '*.py'\n"
+                "Exclude .py files: '!*.py'\n"
+                "Multiple patterns can be separated by commas."
+            ),
+        ),
+        MessageTextInput(
+            name="content_filter",
+            display_name="Content Filter",
+            required=False,
+            advanced=True,
+            info="A regex pattern to filter files based on their content.",
+        ),
+    ]
+
+    outputs = [
+        Output(name="data", display_name="Data", method="load_documents"),
+    ]
+
+    @staticmethod
+    def is_binary(file_path: str | Path) -> bool:
+        """Check if a file is binary by looking for null bytes."""
+        try:
+            with Path(file_path).open("rb") as file:
+                content = file.read(1024)
+                return b"\x00" in content
+        except Exception:  # noqa: BLE001
+            return True
+
+    @staticmethod
+    def check_file_patterns(file_path: str | Path, patterns: str) -> bool:
+        """Check if a file matches the given patterns.
+
+        Args:
+            file_path: Path to the file to check
+            patterns: Comma-separated list of glob patterns
+
+        Returns:
+            bool: True if file should be included, False if excluded
+        """
+        # Handle empty or whitespace-only patterns
+        if not patterns or patterns.isspace():
+            return True
+
+        path_str = str(file_path)
+        file_name = Path(path_str).name
+        pattern_list: list[str] = [pattern.strip() for pattern in patterns.split(",") if pattern.strip()]
+
+        # If no valid patterns after stripping, treat as include all
+        if not pattern_list:
+            return True
+
+        # Process exclusion patterns first
+        for pattern in pattern_list:
+            if pattern.startswith("!"):
+                # For exclusions, match against both full path and filename
+                exclude_pattern = pattern[1:]
+                if fnmatch(path_str, exclude_pattern) or fnmatch(file_name, exclude_pattern):
+                    return False
+
+        # Then check inclusion patterns
+        include_patterns = [p for p in pattern_list if not p.startswith("!")]
+        # If no include patterns, treat as include all
+        if not include_patterns:
+            return True
+
+        # For inclusions, match against both full path and filename
+        return any(fnmatch(path_str, pattern) or fnmatch(file_name, pattern) for pattern in include_patterns)
+
+    @staticmethod
+    def check_content_pattern(file_path: str | Path, pattern: str) -> bool:
+        """Check if file content matches the given regex pattern.
+
+        Args:
+            file_path: Path to the file to check
+            pattern: Regex pattern to match against content
+
+        Returns:
+            bool: True if content matches, False otherwise
+        """
+        try:
+            # Check if file is binary
+            with Path(file_path).open("rb") as file:
+                content = file.read(1024)
+                if b"\x00" in content:
+                    return False
+
+            # Try to compile the regex pattern first
+            try:
+                # Use the MULTILINE flag to better handle text content
+                content_regex = re.compile(pattern, re.MULTILINE)
+                # Test the pattern with a simple string to catch syntax errors
+                test_str = "test\nstring"
+                if not content_regex.search(test_str):
+                    # Pattern is valid but doesn't match test string
+                    pass
+            except (re.error, TypeError, ValueError):
+                return False
+
+            # If not binary and regex is valid, check content
+            with Path(file_path).open(encoding="utf-8") as file:
+                file_content = file.read()
+            return bool(content_regex.search(file_content))
+        except (OSError, UnicodeDecodeError):
+            return False
+
+    def build_combined_filter(self, file_filter_patterns: str | None = None, content_filter_pattern: str | None = None):
+        """Build a combined filter function from file and content patterns.
+
+        Args:
+            file_filter_patterns: Comma-separated glob patterns
+            content_filter_pattern: Regex pattern for content
+
+        Returns:
+            callable: Filter function that takes a file path and returns bool
+        """
+
+        def combined_filter(file_path: str) -> bool:
+            try:
+                path = Path(file_path)
+
+                # Check if file exists and is readable
+                if not path.exists():
+                    return False
+
+                # Check if file is binary
+                if self.is_binary(path):
+                    return False
+
+                # Apply file pattern filters
+                if file_filter_patterns and not self.check_file_patterns(path, file_filter_patterns):
+                    return False
+
+                # Apply content filter
+                return not (content_filter_pattern and not self.check_content_pattern(path, content_filter_pattern))
+            except Exception:  # noqa: BLE001
+                return False
+
+        return combined_filter
+
+    @asynccontextmanager
+    async def temp_clone_dir(self):
+        """Context manager for handling temporary clone directory."""
+        temp_dir = None
+        try:
+            temp_dir = tempfile.mkdtemp(prefix="langflow_clone_")
+            yield temp_dir
+        finally:
+            if temp_dir:
+                await anyio.Path(temp_dir).rmdir()
+
+    def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:
+        # Hide fields by default
+        build_config["repo_path"]["show"] = False
+        build_config["clone_url"]["show"] = False
+
+        if field_name == "repo_source":
+            if field_value == "Local":
+                build_config["repo_path"]["show"] = True
+                build_config["repo_path"]["required"] = True
+                build_config["clone_url"]["required"] = False
+            elif field_value == "Remote":
+                build_config["clone_url"]["show"] = True
+                build_config["clone_url"]["required"] = True
+                build_config["repo_path"]["required"] = False
+
+        return build_config
+
+    async def build_gitloader(self) -> GitLoader:
+        file_filter_patterns = getattr(self, "file_filter", None)
+        content_filter_pattern = getattr(self, "content_filter", None)
+
+        combined_filter = self.build_combined_filter(file_filter_patterns, content_filter_pattern)
+
+        repo_source = getattr(self, "repo_source", None)
+        if repo_source == "Local":
+            repo_path = self.repo_path
+            clone_url = None
+        else:
+            # Clone source
+            clone_url = self.clone_url
+            async with self.temp_clone_dir() as temp_dir:
+                repo_path = temp_dir
+
+        # Only pass branch if it's explicitly set
+        branch = getattr(self, "branch", None)
+        if not branch:
+            branch = None
+
+        return GitLoader(
+            repo_path=repo_path,
+            clone_url=clone_url if repo_source == "Remote" else None,
+            branch=branch,
+            file_filter=combined_filter,
+        )
+
+    async def load_documents(self) -> list[Data]:
+        gitloader = await self.build_gitloader()
+        data = [Data.from_document(doc) async for doc in gitloader.alazy_load()]
+        self.status = data
+        return data