langflow-base-nightly 0.5.1.dev3__py3-none-any.whl → 0.5.1.dev4__py3-none-any.whl

langflow/components/data/news_search.py

@@ -1,164 +0,0 @@
-from urllib.parse import quote_plus
-
-import pandas as pd
-import requests
-from bs4 import BeautifulSoup
-
-from langflow.custom import Component
-from langflow.io import IntInput, MessageTextInput, Output
-from langflow.schema import DataFrame
-
-
-class NewsSearchComponent(Component):
-    display_name = "News Search"
-    description = "Searches Google News via RSS. Returns clean article data."
-    documentation: str = "https://docs.langflow.org/components-data#news-search"
-    icon = "newspaper"
-    name = "NewsSearch"
-
-    inputs = [
-        MessageTextInput(
-            name="query",
-            display_name="Search Query",
-            info="Search keywords for news articles.",
-            tool_mode=True,
-            required=True,
-        ),
-        MessageTextInput(
-            name="hl",
-            display_name="Language (hl)",
-            info="Language code, e.g. en-US, fr, de. Default: en-US.",
-            tool_mode=False,
-            input_types=[],
-            required=False,
-            advanced=True,
-        ),
-        MessageTextInput(
-            name="gl",
-            display_name="Country (gl)",
-            info="Country code, e.g. US, FR, DE. Default: US.",
-            tool_mode=False,
-            input_types=[],
-            required=False,
-            advanced=True,
-        ),
-        MessageTextInput(
-            name="ceid",
-            display_name="Country:Language (ceid)",
-            info="e.g. US:en, FR:fr. Default: US:en.",
-            tool_mode=False,
-            value="US:en",
-            input_types=[],
-            required=False,
-            advanced=True,
-        ),
-        MessageTextInput(
-            name="topic",
-            display_name="Topic",
-            info="One of: WORLD, NATION, BUSINESS, TECHNOLOGY, ENTERTAINMENT, SCIENCE, SPORTS, HEALTH.",
-            tool_mode=False,
-            input_types=[],
-            required=False,
-            advanced=True,
-        ),
-        MessageTextInput(
-            name="location",
-            display_name="Location (Geo)",
-            info="City, state, or country for location-based news. Leave blank for keyword search.",
-            tool_mode=False,
-            input_types=[],
-            required=False,
-            advanced=True,
-        ),
-        IntInput(
-            name="timeout",
-            display_name="Timeout",
-            info="Timeout for the request in seconds.",
-            value=5,
-            required=False,
-            advanced=True,
-        ),
-    ]
-
-    outputs = [Output(name="articles", display_name="News Articles", method="search_news")]
-
-    def search_news(self) -> DataFrame:
-        # Defaults
-        hl = getattr(self, "hl", None) or "en-US"
-        gl = getattr(self, "gl", None) or "US"
-        ceid = getattr(self, "ceid", None) or f"{gl}:{hl.split('-')[0]}"
-        topic = getattr(self, "topic", None)
-        location = getattr(self, "location", None)
-        query = getattr(self, "query", None)
-
-        # Build base URL
-        if topic:
-            # Topic-based feed
-            base_url = f"https://news.google.com/rss/headlines/section/topic/{quote_plus(topic.upper())}"
-            params = f"?hl={hl}&gl={gl}&ceid={ceid}"
-            rss_url = base_url + params
-        elif location:
-            # Location-based feed
-            base_url = f"https://news.google.com/rss/headlines/section/geo/{quote_plus(location)}"
-            params = f"?hl={hl}&gl={gl}&ceid={ceid}"
-            rss_url = base_url + params
-        elif query:
-            # Keyword search feed
-            base_url = "https://news.google.com/rss/search?q="
-            query_parts = [query]
-            query_encoded = quote_plus(" ".join(query_parts))
-            params = f"&hl={hl}&gl={gl}&ceid={ceid}"
-            rss_url = f"{base_url}{query_encoded}{params}"
-        else:
-            self.status = "No search query, topic, or location provided."
-            self.log(self.status)
-            return DataFrame(
-                pd.DataFrame(
-                    [
-                        {
-                            "title": "Error",
-                            "link": "",
-                            "published": "",
-                            "summary": "No search query, topic, or location provided.",
-                        }
-                    ]
-                )
-            )
-
-        try:
-            response = requests.get(rss_url, timeout=self.timeout)
-            response.raise_for_status()
-            soup = BeautifulSoup(response.content, "xml")
-            items = soup.find_all("item")
-        except requests.RequestException as e:
-            self.status = f"Failed to fetch news: {e}"
-            self.log(self.status)
-            return DataFrame(pd.DataFrame([{"title": "Error", "link": "", "published": "", "summary": str(e)}]))
-        except (AttributeError, ValueError, TypeError) as e:
-            self.status = f"Unexpected error: {e!s}"
-            self.log(self.status)
-            return DataFrame(pd.DataFrame([{"title": "Error", "link": "", "published": "", "summary": str(e)}]))
-
-        if not items:
-            self.status = "No news articles found."
-            self.log(self.status)
-            return DataFrame(pd.DataFrame([{"title": "No articles found", "link": "", "published": "", "summary": ""}]))
-
-        articles = []
-        for item in items:
-            try:
-                title = self.clean_html(item.title.text if item.title else "")
-                link = item.link.text if item.link else ""
-                published = item.pubDate.text if item.pubDate else ""
-                summary = self.clean_html(item.description.text if item.description else "")
-                articles.append({"title": title, "link": link, "published": published, "summary": summary})
-            except (AttributeError, ValueError, TypeError) as e:
-                self.log(f"Error parsing article: {e!s}")
-                continue
-
-        df_articles = pd.DataFrame(articles)
-        self.log(f"Found {len(df_articles)} articles.")
-        return DataFrame(df_articles)
-
-    def clean_html(self, html_string: str) -> str:
-        return BeautifulSoup(html_string, "html.parser").get_text(separator=" ", strip=True)

langflow/components/data/rss.py

@@ -1,69 +0,0 @@
-import pandas as pd
-import requests
-from bs4 import BeautifulSoup
-
-from langflow.custom import Component
-from langflow.io import IntInput, MessageTextInput, Output
-from langflow.logging import logger
-from langflow.schema import DataFrame
-
-
-class RSSReaderComponent(Component):
-    display_name = "RSS Reader"
-    description = "Fetches and parses an RSS feed."
-    documentation: str = "https://docs.langflow.org/components-data#rss-reader"
-    icon = "rss"
-    name = "RSSReaderSimple"
-
-    inputs = [
-        MessageTextInput(
-            name="rss_url",
-            display_name="RSS Feed URL",
-            info="URL of the RSS feed to parse.",
-            tool_mode=True,
-            required=True,
-        ),
-        IntInput(
-            name="timeout",
-            display_name="Timeout",
-            info="Timeout for the RSS feed request.",
-            value=5,
-            advanced=True,
-        ),
-    ]
-
-    outputs = [Output(name="articles", display_name="Articles", method="read_rss")]
-
-    def read_rss(self) -> DataFrame:
-        try:
-            response = requests.get(self.rss_url, timeout=self.timeout)
-            response.raise_for_status()
-            if not response.content.strip():
-                msg = "Empty response received"
-                raise ValueError(msg)
-            # Check if the response is valid XML
-            try:
-                BeautifulSoup(response.content, "xml")
-            except Exception as e:
-                msg = f"Invalid XML response: {e}"
-                raise ValueError(msg) from e
-            soup = BeautifulSoup(response.content, "xml")
-            items = soup.find_all("item")
-        except (requests.RequestException, ValueError) as e:
-            self.status = f"Failed to fetch RSS: {e}"
-            return DataFrame(pd.DataFrame([{"title": "Error", "link": "", "published": "", "summary": str(e)}]))
-
-        articles = [
-            {
-                "title": item.title.text if item.title else "",
-                "link": item.link.text if item.link else "",
-                "published": item.pubDate.text if item.pubDate else "",
-                "summary": item.description.text if item.description else "",
-            }
-            for item in items
-        ]
-
-        # Ensure the DataFrame has the correct columns even if empty
-        df_articles = pd.DataFrame(articles, columns=["title", "link", "published", "summary"])
-        logger.info(f"Fetched {len(df_articles)} articles.")
-        return DataFrame(df_articles)

langflow/components/data/sql_executor.py

@@ -1,99 +0,0 @@
-from typing import TYPE_CHECKING, Any
-
-from langchain_community.utilities import SQLDatabase
-from sqlalchemy.exc import SQLAlchemyError
-
-from langflow.custom.custom_component.component_with_cache import ComponentWithCache
-from langflow.io import BoolInput, MessageTextInput, MultilineInput, Output
-from langflow.schema.dataframe import DataFrame
-from langflow.schema.message import Message
-from langflow.services.cache.utils import CacheMiss
-
-if TYPE_CHECKING:
-    from sqlalchemy.engine import Result
-
-
-class SQLComponent(ComponentWithCache):
-    """A sql component."""
-
-    display_name = "SQL Database"
-    description = "Executes SQL queries on SQLAlchemy-compatible databases."
-    documentation: str = "https://docs.langflow.org/components-data#sql-database"
-    icon = "database"
-    name = "SQLComponent"
-    metadata = {"keywords": ["sql", "database", "query", "db", "fetch"]}
-
-    def __init__(self, **kwargs) -> None:
-        super().__init__(**kwargs)
-        self.db: SQLDatabase = None
-
-    def maybe_create_db(self):
-        if self.database_url != "":
-            cached_db = self._shared_component_cache.get(self.database_url)
-            if not isinstance(cached_db, CacheMiss):
-                self.db = cached_db
-                return
-            self.log("Connecting to database")
-            try:
-                self.db = SQLDatabase.from_uri(self.database_url)
-            except Exception as e:
-                msg = f"An error occurred while connecting to the database: {e}"
-                raise ValueError(msg) from e
-            self._shared_component_cache.set(self.database_url, self.db)
-
-    inputs = [
-        MessageTextInput(name="database_url", display_name="Database URL", required=True),
-        MultilineInput(name="query", display_name="SQL Query", tool_mode=True, required=True),
-        BoolInput(name="include_columns", display_name="Include Columns", value=True, tool_mode=True, advanced=True),
-        BoolInput(
-            name="add_error",
-            display_name="Add Error",
-            value=False,
-            tool_mode=True,
-            info="If True, the error will be added to the result",
-            advanced=True,
-        ),
-    ]
-
-    outputs = [
-        Output(display_name="Result Table", name="run_sql_query", method="run_sql_query"),
-    ]
-
-    def build_component(
-        self,
-    ) -> Message:
-        error = None
-        self.maybe_create_db()
-        try:
-            result = self.db.run(self.query, include_columns=self.include_columns)
-            self.status = result
-        except SQLAlchemyError as e:
-            msg = f"An error occurred while running the SQL Query: {e}"
-            self.log(msg)
-            result = str(e)
-            self.status = result
-            error = repr(e)
-
-        if self.add_error and error is not None:
-            result = f"{result}\n\nError: {error}\n\nQuery: {self.query}"
-        elif error is not None:
-            # Then we won't add the error to the result
-            result = self.query
-
-        return Message(text=result)
-
-    def __execute_query(self) -> list[dict[str, Any]]:
-        self.maybe_create_db()
-        try:
-            cursor: Result[Any] = self.db.run(self.query, fetch="cursor")
-            return [x._asdict() for x in cursor.fetchall()]
-        except SQLAlchemyError as e:
-            msg = f"An error occurred while running the SQL Query: {e}"
-            self.log(msg)
-            raise ValueError(msg) from e
-
-    def run_sql_query(self) -> DataFrame:
-        result = self.__execute_query()
-        df_result = DataFrame(result)
-        self.status = df_result
-        return df_result

langflow/components/data/url.py

@@ -1,299 +0,0 @@
-import re
-
-import requests
-from bs4 import BeautifulSoup
-from langchain_community.document_loaders import RecursiveUrlLoader
-
-from langflow.custom.custom_component.component import Component
-from langflow.field_typing.range_spec import RangeSpec
-from langflow.helpers.data import safe_convert
-from langflow.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput
-from langflow.logging.logger import logger
-from langflow.schema.dataframe import DataFrame
-from langflow.schema.message import Message
-from langflow.services.deps import get_settings_service
-
-# Constants
-DEFAULT_TIMEOUT = 30
-DEFAULT_MAX_DEPTH = 1
-DEFAULT_FORMAT = "Text"
-URL_REGEX = re.compile(
-    r"^(https?:\/\/)?" r"(www\.)?" r"([a-zA-Z0-9.-]+)" r"(\.[a-zA-Z]{2,})?" r"(:\d+)?" r"(\/[^\s]*)?$",
-    re.IGNORECASE,
-)
-
-
-class URLComponent(Component):
-    """A component that loads and parses content from web pages recursively.
-
-    This component allows fetching content from one or more URLs, with options to:
-    - Control crawl depth
-    - Prevent crawling outside the root domain
-    - Use async loading for better performance
-    - Extract either raw HTML or clean text
-    - Configure request headers and timeouts
-    """
-
-    display_name = "URL"
-    description = "Fetch content from one or more web pages, following links recursively."
-    documentation: str = "https://docs.langflow.org/components-data#url"
-    icon = "layout-template"
-    name = "URLComponent"
-
-    inputs = [
-        MessageTextInput(
-            name="urls",
-            display_name="URLs",
-            info="Enter one or more URLs to crawl recursively, by clicking the '+' button.",
-            is_list=True,
-            tool_mode=True,
-            placeholder="Enter a URL...",
-            list_add_label="Add URL",
-            input_types=[],
-        ),
-        SliderInput(
-            name="max_depth",
-            display_name="Depth",
-            info=(
-                "Controls how many 'clicks' away from the initial page the crawler will go:\n"
-                "- depth 1: only the initial page\n"
-                "- depth 2: initial page + all pages linked directly from it\n"
-                "- depth 3: initial page + direct links + links found on those direct link pages\n"
-                "Note: This is about link traversal, not URL path depth."
-            ),
-            value=DEFAULT_MAX_DEPTH,
-            range_spec=RangeSpec(min=1, max=5, step=1),
-            required=False,
-            min_label=" ",
-            max_label=" ",
-            min_label_icon="None",
-            max_label_icon="None",
-            # slider_input=True
-        ),
-        BoolInput(
-            name="prevent_outside",
-            display_name="Prevent Outside",
-            info=(
-                "If enabled, only crawls URLs within the same domain as the root URL. "
-                "This helps prevent the crawler from going to external websites."
-            ),
-            value=True,
-            required=False,
-            advanced=True,
-        ),
-        BoolInput(
-            name="use_async",
-            display_name="Use Async",
-            info=(
-                "If enabled, uses asynchronous loading which can be significantly faster "
-                "but might use more system resources."
-            ),
-            value=True,
-            required=False,
-            advanced=True,
-        ),
-        DropdownInput(
-            name="format",
-            display_name="Output Format",
-            info="Output Format. Use 'Text' to extract the text from the HTML or 'HTML' for the raw HTML content.",
-            options=["Text", "HTML"],
-            value=DEFAULT_FORMAT,
-            advanced=True,
-        ),
-        IntInput(
-            name="timeout",
-            display_name="Timeout",
-            info="Timeout for the request in seconds.",
-            value=DEFAULT_TIMEOUT,
-            required=False,
-            advanced=True,
-        ),
-        TableInput(
-            name="headers",
-            display_name="Headers",
-            info="The headers to send with the request",
-            table_schema=[
-                {
-                    "name": "key",
-                    "display_name": "Header",
-                    "type": "str",
-                    "description": "Header name",
-                },
-                {
-                    "name": "value",
-                    "display_name": "Value",
-                    "type": "str",
-                    "description": "Header value",
-                },
-            ],
-            value=[{"key": "User-Agent", "value": get_settings_service().settings.user_agent}],
-            advanced=True,
-            input_types=["DataFrame"],
-        ),
-        BoolInput(
-            name="filter_text_html",
-            display_name="Filter Text/HTML",
-            info="If enabled, filters out text/css content type from the results.",
-            value=True,
-            required=False,
-            advanced=True,
-        ),
-        BoolInput(
-            name="continue_on_failure",
-            display_name="Continue on Failure",
-            info="If enabled, continues crawling even if some requests fail.",
-            value=True,
-            required=False,
-            advanced=True,
-        ),
-        BoolInput(
-            name="check_response_status",
-            display_name="Check Response Status",
-            info="If enabled, checks the response status of the request.",
-            value=False,
-            required=False,
-            advanced=True,
-        ),
-        BoolInput(
-            name="autoset_encoding",
-            display_name="Autoset Encoding",
-            info="If enabled, automatically sets the encoding of the request.",
-            value=True,
-            required=False,
-            advanced=True,
-        ),
-    ]
-
-    outputs = [
-        Output(display_name="Extracted Pages", name="page_results", method="fetch_content"),
-        Output(display_name="Raw Content", name="raw_results", method="fetch_content_as_message", tool_mode=False),
-    ]
-
-    @staticmethod
-    def validate_url(url: str) -> bool:
-        """Validates if the given string matches URL pattern.
-
-        Args:
-            url: The URL string to validate
-
-        Returns:
-            bool: True if the URL is valid, False otherwise
-        """
-        return bool(URL_REGEX.match(url))
-
-    def ensure_url(self, url: str) -> str:
-        """Ensures the given string is a valid URL.
-
-        Args:
-            url: The URL string to validate and normalize
-
-        Returns:
-            str: The normalized URL
-
-        Raises:
-            ValueError: If the URL is invalid
-        """
-        url = url.strip()
-        if not url.startswith(("http://", "https://")):
-            url = "https://" + url
-
-        if not self.validate_url(url):
-            msg = f"Invalid URL: {url}"
-            raise ValueError(msg)
-
-        return url
-
-    def _create_loader(self, url: str) -> RecursiveUrlLoader:
-        """Creates a RecursiveUrlLoader instance with the configured settings.
-
-        Args:
-            url: The URL to load
-
-        Returns:
-            RecursiveUrlLoader: Configured loader instance
-        """
-        headers_dict = {header["key"]: header["value"] for header in self.headers}
-        extractor = (lambda x: x) if self.format == "HTML" else (lambda x: BeautifulSoup(x, "lxml").get_text())
-
-        return RecursiveUrlLoader(
-            url=url,
-            max_depth=self.max_depth,
-            prevent_outside=self.prevent_outside,
-            use_async=self.use_async,
-            extractor=extractor,
-            timeout=self.timeout,
-            headers=headers_dict,
-            check_response_status=self.check_response_status,
-            continue_on_failure=self.continue_on_failure,
-            base_url=url,  # Add base_url to ensure consistent domain crawling
-            autoset_encoding=self.autoset_encoding,  # Enable automatic encoding detection
-            exclude_dirs=[],  # Allow customization of excluded directories
-            link_regex=None,  # Allow customization of link filtering
-        )
-
-    def fetch_url_contents(self) -> list[dict]:
-        """Load documents from the configured URLs.
-
-        Returns:
-            List[Data]: List of Data objects containing the fetched content
-
-        Raises:
-            ValueError: If no valid URLs are provided or if there's an error loading documents
-        """
-        try:
-            urls = list({self.ensure_url(url) for url in self.urls if url.strip()})
-            logger.debug(f"URLs: {urls}")
-            if not urls:
-                msg = "No valid URLs provided."
-                raise ValueError(msg)
-
-            all_docs = []
-            for url in urls:
-                logger.debug(f"Loading documents from {url}")
-
-                try:
-                    loader = self._create_loader(url)
-                    docs = loader.load()
-
-                    if not docs:
-                        logger.warning(f"No documents found for {url}")
-                        continue
-
-                    logger.debug(f"Found {len(docs)} documents from {url}")
-                    all_docs.extend(docs)
-
-                except requests.exceptions.RequestException as e:
-                    logger.exception(f"Error loading documents from {url}: {e}")
-                    continue
-
-            if not all_docs:
-                msg = "No documents were successfully loaded from any URL"
-                raise ValueError(msg)
-
-            # data = [Data(text=doc.page_content, **doc.metadata) for doc in all_docs]
-            data = [
-                {
-                    "text": safe_convert(doc.page_content, clean_data=True),
-                    "url": doc.metadata.get("source", ""),
-                    "title": doc.metadata.get("title", ""),
-                    "description": doc.metadata.get("description", ""),
-                    "content_type": doc.metadata.get("content_type", ""),
-                    "language": doc.metadata.get("language", ""),
-                }
-                for doc in all_docs
-            ]
-        except Exception as e:
-            error_msg = e.message if hasattr(e, "message") else e
-            msg = f"Error loading documents: {error_msg!s}"
-            logger.exception(msg)
-            raise ValueError(msg) from e
-        return data
-
-    def fetch_content(self) -> DataFrame:
-        """Convert the documents to a DataFrame."""
-        return DataFrame(data=self.fetch_url_contents())
-
-    def fetch_content_as_message(self) -> Message:
-        """Convert the documents to a Message."""
-        url_contents = self.fetch_url_contents()
-        return Message(text="\n\n".join([x["text"] for x in url_contents]), data={"data": url_contents})