lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/data/url.py

@@ -0,0 +1,311 @@
+import importlib
+import re
+
+import requests
+from bs4 import BeautifulSoup
+from langchain_community.document_loaders import RecursiveUrlLoader
+
+from lfx.custom.custom_component.component import Component
+from lfx.field_typing.range_spec import RangeSpec
+from lfx.helpers.data import safe_convert
+from lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput
+from lfx.log.logger import logger
+from lfx.schema.dataframe import DataFrame
+from lfx.schema.message import Message
+from lfx.utils.request_utils import get_user_agent
+
+# Constants
+DEFAULT_TIMEOUT = 30
+DEFAULT_MAX_DEPTH = 1
+DEFAULT_FORMAT = "Text"
+
+
+URL_REGEX = re.compile(
+    r"^(https?:\/\/)?" r"(www\.)?" r"([a-zA-Z0-9.-]+)" r"(\.[a-zA-Z]{2,})?" r"(:\d+)?" r"(\/[^\s]*)?$",
+    re.IGNORECASE,
+)
+
+USER_AGENT = None
+# Check if langflow is installed using importlib.util.find_spec(name))
+if importlib.util.find_spec("langflow"):
+    langflow_installed = True
+    USER_AGENT = get_user_agent()
+else:
+    langflow_installed = False
+    USER_AGENT = "lfx"
+
+
+class URLComponent(Component):
+    """A component that loads and parses content from web pages recursively.
+
+    This component allows fetching content from one or more URLs, with options to:
+    - Control crawl depth
+    - Prevent crawling outside the root domain
+    - Use async loading for better performance
+    - Extract either raw HTML or clean text
+    - Configure request headers and timeouts
+    """
+
+    display_name = "URL"
+    description = "Fetch content from one or more web pages, following links recursively."
+    documentation: str = "https://docs.langflow.org/components-data#url"
+    icon = "layout-template"
+    name = "URLComponent"
+
+    inputs = [
+        MessageTextInput(
+            name="urls",
+            display_name="URLs",
+            info="Enter one or more URLs to crawl recursively, by clicking the '+' button.",
+            is_list=True,
+            tool_mode=True,
+            placeholder="Enter a URL...",
+            list_add_label="Add URL",
+            input_types=[],
+        ),
+        SliderInput(
+            name="max_depth",
+            display_name="Depth",
+            info=(
+                "Controls how many 'clicks' away from the initial page the crawler will go:\n"
+                "- depth 1: only the initial page\n"
+                "- depth 2: initial page + all pages linked directly from it\n"
+                "- depth 3: initial page + direct links + links found on those direct link pages\n"
+                "Note: This is about link traversal, not URL path depth."
+            ),
+            value=DEFAULT_MAX_DEPTH,
+            range_spec=RangeSpec(min=1, max=5, step=1),
+            required=False,
+            min_label=" ",
+            max_label=" ",
+            min_label_icon="None",
+            max_label_icon="None",
+            # slider_input=True
+        ),
+        BoolInput(
+            name="prevent_outside",
+            display_name="Prevent Outside",
+            info=(
+                "If enabled, only crawls URLs within the same domain as the root URL. "
+                "This helps prevent the crawler from going to external websites."
+            ),
+            value=True,
+            required=False,
+            advanced=True,
+        ),
+        BoolInput(
+            name="use_async",
+            display_name="Use Async",
+            info=(
+                "If enabled, uses asynchronous loading which can be significantly faster "
+                "but might use more system resources."
+            ),
+            value=True,
+            required=False,
+            advanced=True,
+        ),
+        DropdownInput(
+            name="format",
+            display_name="Output Format",
+            info="Output Format. Use 'Text' to extract the text from the HTML or 'HTML' for the raw HTML content.",
+            options=["Text", "HTML"],
+            value=DEFAULT_FORMAT,
+            advanced=True,
+        ),
+        IntInput(
+            name="timeout",
+            display_name="Timeout",
+            info="Timeout for the request in seconds.",
+            value=DEFAULT_TIMEOUT,
+            required=False,
+            advanced=True,
+        ),
+        TableInput(
+            name="headers",
+            display_name="Headers",
+            info="The headers to send with the request",
+            table_schema=[
+                {
+                    "name": "key",
+                    "display_name": "Header",
+                    "type": "str",
+                    "description": "Header name",
+                },
+                {
+                    "name": "value",
+                    "display_name": "Value",
+                    "type": "str",
+                    "description": "Header value",
+                },
+            ],
+            value=[{"key": "User-Agent", "value": USER_AGENT}],
+            advanced=True,
+            input_types=["DataFrame"],
+        ),
+        BoolInput(
+            name="filter_text_html",
+            display_name="Filter Text/HTML",
+            info="If enabled, filters out text/css content type from the results.",
+            value=True,
+            required=False,
+            advanced=True,
+        ),
+        BoolInput(
+            name="continue_on_failure",
+            display_name="Continue on Failure",
+            info="If enabled, continues crawling even if some requests fail.",
+            value=True,
+            required=False,
+            advanced=True,
+        ),
+        BoolInput(
+            name="check_response_status",
+            display_name="Check Response Status",
+            info="If enabled, checks the response status of the request.",
+            value=False,
+            required=False,
+            advanced=True,
+        ),
+        BoolInput(
+            name="autoset_encoding",
+            display_name="Autoset Encoding",
+            info="If enabled, automatically sets the encoding of the request.",
+            value=True,
+            required=False,
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Extracted Pages", name="page_results", method="fetch_content"),
+        Output(display_name="Raw Content", name="raw_results", method="fetch_content_as_message", tool_mode=False),
+    ]
+
+    @staticmethod
+    def validate_url(url: str) -> bool:
+        """Validates if the given string matches URL pattern.
+
+        Args:
+            url: The URL string to validate
+
+        Returns:
+            bool: True if the URL is valid, False otherwise
+        """
+        return bool(URL_REGEX.match(url))
+
+    def ensure_url(self, url: str) -> str:
+        """Ensures the given string is a valid URL.
+
+        Args:
+            url: The URL string to validate and normalize
+
+        Returns:
+            str: The normalized URL
+
+        Raises:
+            ValueError: If the URL is invalid
+        """
+        url = url.strip()
+        if not url.startswith(("http://", "https://")):
+            url = "https://" + url
+
+        if not self.validate_url(url):
+            msg = f"Invalid URL: {url}"
+            raise ValueError(msg)
+
+        return url
+
+    def _create_loader(self, url: str) -> RecursiveUrlLoader:
+        """Creates a RecursiveUrlLoader instance with the configured settings.
+
+        Args:
+            url: The URL to load
+
+        Returns:
+            RecursiveUrlLoader: Configured loader instance
+        """
+        headers_dict = {header["key"]: header["value"] for header in self.headers if header["value"] is not None}
+        extractor = (lambda x: x) if self.format == "HTML" else (lambda x: BeautifulSoup(x, "lxml").get_text())
+
+        return RecursiveUrlLoader(
+            url=url,
+            max_depth=self.max_depth,
+            prevent_outside=self.prevent_outside,
+            use_async=self.use_async,
+            extractor=extractor,
+            timeout=self.timeout,
+            headers=headers_dict,
+            check_response_status=self.check_response_status,
+            continue_on_failure=self.continue_on_failure,
+            base_url=url,  # Add base_url to ensure consistent domain crawling
+            autoset_encoding=self.autoset_encoding,  # Enable automatic encoding detection
+            exclude_dirs=[],  # Allow customization of excluded directories
+            link_regex=None,  # Allow customization of link filtering
+        )
+
+    def fetch_url_contents(self) -> list[dict]:
+        """Load documents from the configured URLs.
+
+        Returns:
+            List[Data]: List of Data objects containing the fetched content
+
+        Raises:
+            ValueError: If no valid URLs are provided or if there's an error loading documents
+        """
+        try:
+            urls = list({self.ensure_url(url) for url in self.urls if url.strip()})
+            logger.debug(f"URLs: {urls}")
+            if not urls:
+                msg = "No valid URLs provided."
+                raise ValueError(msg)
+
+            all_docs = []
+            for url in urls:
+                logger.debug(f"Loading documents from {url}")
+
+                try:
+                    loader = self._create_loader(url)
+                    docs = loader.load()
+
+                    if not docs:
+                        logger.warning(f"No documents found for {url}")
+                        continue
+
+                    logger.debug(f"Found {len(docs)} documents from {url}")
+                    all_docs.extend(docs)
+
+                except requests.exceptions.RequestException as e:
+                    logger.exception(f"Error loading documents from {url}: {e}")
+                    continue
+
+            if not all_docs:
+                msg = "No documents were successfully loaded from any URL"
+                raise ValueError(msg)
+
+            # data = [Data(text=doc.page_content, **doc.metadata) for doc in all_docs]
+            data = [
+                {
+                    "text": safe_convert(doc.page_content, clean_data=True),
+                    "url": doc.metadata.get("source", ""),
+                    "title": doc.metadata.get("title", ""),
+                    "description": doc.metadata.get("description", ""),
+                    "content_type": doc.metadata.get("content_type", ""),
+                    "language": doc.metadata.get("language", ""),
+                }
+                for doc in all_docs
+            ]
+        except Exception as e:
+            error_msg = e.message if hasattr(e, "message") else e
+            msg = f"Error loading documents: {error_msg!s}"
+            logger.exception(msg)
+            raise ValueError(msg) from e
+        return data
+
+    def fetch_content(self) -> DataFrame:
+        """Convert the documents to a DataFrame."""
+        return DataFrame(data=self.fetch_url_contents())
+
+    def fetch_content_as_message(self) -> Message:
+        """Convert the documents to a Message."""
+        url_contents = self.fetch_url_contents()
+        return Message(text="\n\n".join([x["text"] for x in url_contents]), data={"data": url_contents})

lfx/components/data/web_search.py

@@ -0,0 +1,112 @@
+import re
+from urllib.parse import parse_qs, unquote, urlparse
+
+import pandas as pd
+import requests
+from bs4 import BeautifulSoup
+
+from lfx.custom import Component
+from lfx.io import IntInput, MessageTextInput, Output
+from lfx.schema import DataFrame
+from lfx.utils.request_utils import get_user_agent
+
+
+class WebSearchComponent(Component):
+    display_name = "Web Search"
+    description = "Performs a basic DuckDuckGo search (HTML scraping). May be subject to rate limits."
+    documentation: str = "https://docs.langflow.org/components-data#web-search"
+    icon = "search"
+    name = "WebSearchNoAPI"
+
+    inputs = [
+        MessageTextInput(
+            name="query",
+            display_name="Search Query",
+            info="Keywords to search for.",
+            tool_mode=True,
+            required=True,
+        ),
+        IntInput(
+            name="timeout",
+            display_name="Timeout",
+            info="Timeout for the web search request.",
+            value=5,
+            advanced=True,
+        ),
+    ]
+
+    outputs = [Output(name="results", display_name="Search Results", method="perform_search")]
+
+    def validate_url(self, string: str) -> bool:
+        url_regex = re.compile(
+            r"^(https?:\/\/)?" r"(www\.)?" r"([a-zA-Z0-9.-]+)" r"(\.[a-zA-Z]{2,})?" r"(:\d+)?" r"(\/[^\s]*)?$",
+            re.IGNORECASE,
+        )
+        return bool(url_regex.match(string))
+
+    def ensure_url(self, url: str) -> str:
+        if not url.startswith(("http://", "https://")):
+            url = "https://" + url
+        if not self.validate_url(url):
+            msg = f"Invalid URL: {url}"
+            raise ValueError(msg)
+        return url
+
+    def _sanitize_query(self, query: str) -> str:
+        """Sanitize search query."""
+        # Remove potentially dangerous characters
+        return re.sub(r'[<>"\']', "", query.strip())
+
+    def perform_search(self) -> DataFrame:
+        query = self._sanitize_query(self.query)
+        if not query:
+            msg = "Empty search query"
+            raise ValueError(msg)
+        headers = {"User-Agent": get_user_agent()}
+        params = {"q": query, "kl": "us-en"}
+        url = "https://html.duckduckgo.com/html/"
+
+        try:
+            response = requests.get(url, params=params, headers=headers, timeout=self.timeout)
+            response.raise_for_status()
+        except requests.RequestException as e:
+            self.status = f"Failed request: {e!s}"
+            return DataFrame(pd.DataFrame([{"title": "Error", "link": "", "snippet": str(e), "content": ""}]))
+
+        if not response.text or "text/html" not in response.headers.get("content-type", "").lower():
+            self.status = "No results found"
+            return DataFrame(
+                pd.DataFrame([{"title": "Error", "link": "", "snippet": "No results found", "content": ""}])
+            )
+        soup = BeautifulSoup(response.text, "html.parser")
+        results = []
+
+        for result in soup.select("div.result"):
+            title_tag = result.select_one("a.result__a")
+            snippet_tag = result.select_one("a.result__snippet")
+            if title_tag:
+                raw_link = title_tag.get("href", "")
+                parsed = urlparse(raw_link)
+                uddg = parse_qs(parsed.query).get("uddg", [""])[0]
+                decoded_link = unquote(uddg) if uddg else raw_link
+
+                try:
+                    final_url = self.ensure_url(decoded_link)
+                    page = requests.get(final_url, headers=headers, timeout=self.timeout)
+                    page.raise_for_status()
+                    content = BeautifulSoup(page.text, "lxml").get_text(separator=" ", strip=True)
+                except requests.RequestException as e:
+                    final_url = decoded_link
+                    content = f"(Failed to fetch: {e!s}"
+
+                results.append(
+                    {
+                        "title": title_tag.get_text(strip=True),
+                        "link": final_url,
+                        "snippet": snippet_tag.get_text(strip=True) if snippet_tag else "",
+                        "content": content,
+                    }
+                )
+
+        df_results = pd.DataFrame(results)
+        return DataFrame(df_results)

lfx/components/data/webhook.py

@@ -0,0 +1,56 @@
+import json
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import MultilineInput, Output
+from lfx.schema.data import Data
+
+
+class WebhookComponent(Component):
+    display_name = "Webhook"
+    documentation: str = "https://docs.langflow.org/components-data#webhook"
+    name = "Webhook"
+    icon = "webhook"
+
+    inputs = [
+        MultilineInput(
+            name="data",
+            display_name="Payload",
+            info="Receives a payload from external systems via HTTP POST.",
+            advanced=True,
+        ),
+        MultilineInput(
+            name="curl",
+            display_name="cURL",
+            value="CURL_WEBHOOK",
+            advanced=True,
+            input_types=[],
+        ),
+        MultilineInput(
+            name="endpoint",
+            display_name="Endpoint",
+            value="BACKEND_URL",
+            advanced=False,
+            copy_field=True,
+            input_types=[],
+        ),
+    ]
+    outputs = [
+        Output(display_name="Data", name="output_data", method="build_data"),
+    ]
+
+    def build_data(self) -> Data:
+        message: str | Data = ""
+        if not self.data:
+            self.status = "No data provided."
+            return Data(data={})
+        try:
+            my_data = self.data.replace('"\n"', '"\\n"')
+            body = json.loads(my_data or "{}")
+        except json.JSONDecodeError:
+            body = {"payload": self.data}
+            message = f"Invalid JSON payload. Please check the format.\n\n{self.data}"
+        data = Data(data=body)
+        if not message:
+            message = data
+        self.status = message
+        return data

lfx/components/datastax/__init__.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from .astra_assistant_manager import AstraAssistantManager
+    from .astra_db import AstraDBChatMemory
+    from .astra_vectorize import AstraVectorizeComponent
+    from .astradb_cql import AstraDBCQLToolComponent
+    from .astradb_tool import AstraDBToolComponent
+    from .astradb_vectorstore import AstraDBVectorStoreComponent
+    from .create_assistant import AssistantsCreateAssistant
+    from .create_thread import AssistantsCreateThread
+    from .dotenv import Dotenv
+    from .get_assistant import AssistantsGetAssistantName
+    from .getenvvar import GetEnvVar
+    from .list_assistants import AssistantsListAssistants
+    from .run import AssistantsRun
+
+_dynamic_imports = {
+    "AssistantsCreateAssistant": "create_assistant",
+    "AssistantsCreateThread": "create_thread",
+    "AssistantsGetAssistantName": "get_assistant",
+    "AssistantsListAssistants": "list_assistants",
+    "AssistantsRun": "run",
+    "AstraAssistantManager": "astra_assistant_manager",
+    "AstraDBCQLToolComponent": "astradb_cql",
+    "AstraDBChatMemory": "astra_db",
+    "AstraDBToolComponent": "astradb_tool",
+    "AstraDBVectorStoreComponent": "astradb_vectorstore",
+    "AstraVectorizeComponent": "astra_vectorize",
+    "Dotenv": "dotenv",
+    "GetEnvVar": "getenvvar",
+}
+
+__all__ = [
+    "AssistantsCreateAssistant",
+    "AssistantsCreateThread",
+    "AssistantsGetAssistantName",
+    "AssistantsListAssistants",
+    "AssistantsRun",
+    "AstraAssistantManager",
+    "AstraDBCQLToolComponent",
+    "AstraDBChatMemory",
+    "AstraDBToolComponent",
+    "AstraDBVectorStoreComponent",
+    "AstraVectorizeComponent",
+    "Dotenv",
+    "GetEnvVar",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import datastax components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)