langflow-base-nightly 0.0.95.dev8__tar.gz → 0.0.96.dev0__tar.gz

langflow_base_nightly-0.0.96.dev0/PKG-INFO

@@ -0,0 +1,79 @@
+Metadata-Version: 2.1
+Name: langflow-base-nightly
+Version: 0.0.96.dev0
+Summary: A Python package with a built-in web application
+Home-page: https://github.com/langflow-ai/langflow
+License: MIT
+Keywords: nlp,langchain,openai,gpt,gui
+Author: Langflow
+Author-email: contact@langflow.org
+Maintainer: Carlos Coelho
+Maintainer-email: carlos@langflow.org
+Requires-Python: >=3.10,<3.13
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: all
+Provides-Extra: deploy
+Provides-Extra: local
+Requires-Dist: aiofiles (>=24.1.0,<25.0.0)
+Requires-Dist: alembic (>=1.13.0,<2.0.0)
+Requires-Dist: asyncer (>=0.0.5,<0.0.6)
+Requires-Dist: bcrypt (==4.0.1)
+Requires-Dist: cachetools (>=5.3.1,<6.0.0)
+Requires-Dist: chardet (>=5.2.0,<6.0.0)
+Requires-Dist: clickhouse-connect (==0.7.19)
+Requires-Dist: crewai (>=0.36.0,<0.37.0)
+Requires-Dist: cryptography (>=42.0.5,<44.0.0)
+Requires-Dist: diskcache (>=5.6.3,<6.0.0)
+Requires-Dist: docstring-parser (>=0.16,<0.17)
+Requires-Dist: duckdb (>=1.0.0,<2.0.0)
+Requires-Dist: emoji (>=2.12.0,<3.0.0)
+Requires-Dist: fastapi (>=0.111.0,<0.112.0)
+Requires-Dist: filelock (>=3.15.4,<4.0.0)
+Requires-Dist: firecrawl-py (>=0.0.16,<0.0.17)
+Requires-Dist: grandalf (>=0.8.0,<0.9.0)
+Requires-Dist: gunicorn (>=22.0.0,<23.0.0)
+Requires-Dist: httpx
+Requires-Dist: jq (>=1.7.0,<2.0.0) ; sys_platform != "win32"
+Requires-Dist: langchain (>=0.2.0,<0.3.0)
+Requires-Dist: langchain-core (>=0.2.32,<0.3.0)
+Requires-Dist: langchain-experimental (>=0.0.61,<0.0.62)
+Requires-Dist: langchainhub (>=0.1.15,<0.2.0)
+Requires-Dist: loguru (>=0.7.1,<0.8.0)
+Requires-Dist: multiprocess (>=0.70.14,<0.71.0)
+Requires-Dist: nanoid (>=2.0.0,<3.0.0)
+Requires-Dist: nest-asyncio (>=1.6.0,<2.0.0)
+Requires-Dist: opentelemetry-api (>=1.25.0,<2.0.0)
+Requires-Dist: opentelemetry-exporter-prometheus (>=0.46b0,<0.47)
+Requires-Dist: opentelemetry-instrumentation-fastapi (>=0.46b0,<0.47)
+Requires-Dist: opentelemetry-sdk (>=1.25.0,<2.0.0)
+Requires-Dist: orjson (==3.10.0)
+Requires-Dist: pandas (==2.2.2)
+Requires-Dist: passlib (>=1.7.4,<2.0.0)
+Requires-Dist: pillow (>=10.2.0,<11.0.0)
+Requires-Dist: platformdirs (>=4.2.0,<5.0.0)
+Requires-Dist: prometheus-client (>=0.20.0,<0.21.0)
+Requires-Dist: pydantic (>=2.7.0,<3.0.0)
+Requires-Dist: pydantic-settings (>=2.2.0,<3.0.0)
+Requires-Dist: pypdf (>=4.2.0,<5.0.0)
+Requires-Dist: pyperclip (>=1.8.2,<2.0.0)
+Requires-Dist: python-docx (>=1.1.0,<2.0.0)
+Requires-Dist: python-jose (>=3.3.0,<4.0.0)
+Requires-Dist: python-multipart (>=0.0.7,<0.0.8)
+Requires-Dist: rich (>=13.7.0,<14.0.0)
+Requires-Dist: sentry-sdk[fastapi,loguru] (>=2.5.1,<3.0.0)
+Requires-Dist: setuptools (>=70)
+Requires-Dist: spider-client (>=0.0.27,<0.0.28)
+Requires-Dist: sqlmodel (>=0.0.18,<0.0.19)
+Requires-Dist: typer (>=0.12.0,<0.13.0)
+Requires-Dist: uncurl (>=0.0.11,<0.0.12)
+Requires-Dist: uvicorn (>=0.30.0,<0.31.0)
+Requires-Dist: websockets
+Project-URL: Documentation, https://docs.langflow.org
+Project-URL: Repository, https://github.com/langflow-ai/langflow
+Description-Content-Type: text/markdown
+
+

langflow_base_nightly-0.0.96.dev0/langflow/api/utils.py

@@ -0,0 +1,243 @@
+import uuid
+import warnings
+from typing import TYPE_CHECKING, Any
+
+from fastapi import HTTPException
+from sqlmodel import Session
+
+from langflow.graph.graph.base import Graph
+from langflow.services.chat.service import ChatService
+from langflow.services.database.models.flow import Flow
+from langflow.services.store.schema import StoreComponentCreate
+from langflow.services.store.utils import get_lf_version_from_pypi
+
+if TYPE_CHECKING:
+    from langflow.services.database.models.flow.model import Flow
+
+
+API_WORDS = ["api", "key", "token"]
+
+
+def has_api_terms(word: str):
+    return "api" in word and ("key" in word or ("token" in word and "tokens" not in word))
+
+
+def remove_api_keys(flow: dict):
+    """Remove api keys from flow data."""
+    if flow.get("data") and flow["data"].get("nodes"):
+        for node in flow["data"]["nodes"]:
+            node_data = node.get("data").get("node")
+            template = node_data.get("template")
+            for value in template.values():
+                if isinstance(value, dict) and has_api_terms(value["name"]) and value.get("password"):
+                    value["value"] = None
+
+    return flow
+
+
+def build_input_keys_response(langchain_object, artifacts):
+    """Build the input keys response."""
+
+    input_keys_response = {
+        "input_keys": {key: "" for key in langchain_object.input_keys},
+        "memory_keys": [],
+        "handle_keys": artifacts.get("handle_keys", []),
+    }
+
+    # Set the input keys values from artifacts
+    for key, value in artifacts.items():
+        if key in input_keys_response["input_keys"]:
+            input_keys_response["input_keys"][key] = value
+    # If the object has memory, that memory will have a memory_variables attribute
+    # memory variables should be removed from the input keys
+    if hasattr(langchain_object, "memory") and hasattr(langchain_object.memory, "memory_variables"):
+        # Remove memory variables from input keys
+        input_keys_response["input_keys"] = {
+            key: value
+            for key, value in input_keys_response["input_keys"].items()
+            if key not in langchain_object.memory.memory_variables
+        }
+        # Add memory variables to memory_keys
+        input_keys_response["memory_keys"] = langchain_object.memory.memory_variables
+
+    if hasattr(langchain_object, "prompt") and hasattr(langchain_object.prompt, "template"):
+        input_keys_response["template"] = langchain_object.prompt.template
+
+    return input_keys_response
+
+
+def validate_is_component(flows: list["Flow"]):
+    for flow in flows:
+        if not flow.data or flow.is_component is not None:
+            continue
+
+        is_component = get_is_component_from_data(flow.data)
+        if is_component is not None:
+            flow.is_component = is_component
+        else:
+            flow.is_component = len(flow.data.get("nodes", [])) == 1
+    return flows
+
+
+def get_is_component_from_data(data: dict):
+    """Returns True if the data is a component."""
+    return data.get("is_component")
+
+
+async def check_langflow_version(component: StoreComponentCreate):
+    from langflow.utils.version import get_version_info
+
+    __version__ = get_version_info()["version"]
+
+    if not component.last_tested_version:
+        component.last_tested_version = __version__
+
+    langflow_version = get_lf_version_from_pypi()
+    if langflow_version is None:
+        raise HTTPException(status_code=500, detail="Unable to verify the latest version of Langflow")
+    elif langflow_version != component.last_tested_version:
+        warnings.warn(
+            f"Your version of Langflow ({component.last_tested_version}) is outdated. "
+            f"Please update to the latest version ({langflow_version}) and try again."
+        )
+
+
+def format_elapsed_time(elapsed_time: float) -> str:
+    """Format elapsed time to a human-readable format coming from perf_counter().
+
+    - Less than 1 second: returns milliseconds
+    - Less than 1 minute: returns seconds rounded to 2 decimals
+    - 1 minute or more: returns minutes and seconds
+    """
+    if elapsed_time < 1:
+        milliseconds = int(round(elapsed_time * 1000))
+        return f"{milliseconds} ms"
+    elif elapsed_time < 60:
+        seconds = round(elapsed_time, 2)
+        unit = "second" if seconds == 1 else "seconds"
+        return f"{seconds} {unit}"
+    else:
+        minutes = int(elapsed_time // 60)
+        seconds = round(elapsed_time % 60, 2)
+        minutes_unit = "minute" if minutes == 1 else "minutes"
+        seconds_unit = "second" if seconds == 1 else "seconds"
+        return f"{minutes} {minutes_unit}, {seconds} {seconds_unit}"
+
+
+async def build_graph_from_data(flow_id: str, payload: dict, **kwargs):
+    """Build and cache the graph."""
+    graph = Graph.from_payload(payload, flow_id, **kwargs)
+    for vertex_id in graph._has_session_id_vertices:
+        vertex = graph.get_vertex(vertex_id)
+        if vertex is None:
+            raise ValueError(f"Vertex {vertex_id} not found")
+        if not vertex._raw_params.get("session_id"):
+            vertex.update_raw_params({"session_id": flow_id}, overwrite=True)
+
+    run_id = uuid.uuid4()
+    graph.set_run_id(run_id)
+    graph.set_run_name()
+    await graph.initialize_run()
+    return graph
+
+
+async def build_graph_from_db_no_cache(flow_id: str, session: Session):
+    """Build and cache the graph."""
+    flow: Flow | None = session.get(Flow, flow_id)
+    if not flow or not flow.data:
+        raise ValueError("Invalid flow ID")
+    return await build_graph_from_data(flow_id, flow.data, flow_name=flow.name, user_id=str(flow.user_id))
+
+
+async def build_graph_from_db(flow_id: str, session: Session, chat_service: "ChatService"):
+    graph = await build_graph_from_db_no_cache(flow_id, session)
+    await chat_service.set_cache(flow_id, graph)
+    return graph
+
+
+async def build_and_cache_graph_from_data(
+    flow_id: str,
+    chat_service: "ChatService",
+    graph_data: dict,
+):  # -> Graph | Any:
+    """Build and cache the graph."""
+    graph = Graph.from_payload(graph_data, flow_id)
+    await chat_service.set_cache(flow_id, graph)
+    return graph
+
+
+def format_syntax_error_message(exc: SyntaxError) -> str:
+    """Format a SyntaxError message for returning to the frontend."""
+    if exc.text is None:
+        return f"Syntax error in code. Error on line {exc.lineno}"
+    return f"Syntax error in code. Error on line {exc.lineno}: {exc.text.strip()}"
+
+
+def get_causing_exception(exc: BaseException) -> BaseException:
+    """Get the causing exception from an exception."""
+    if hasattr(exc, "__cause__") and exc.__cause__:
+        return get_causing_exception(exc.__cause__)
+    return exc
+
+
+def format_exception_message(exc: Exception) -> str:
+    """Format an exception message for returning to the frontend."""
+    # We need to check if the __cause__ is a SyntaxError
+    # If it is, we need to return the message of the SyntaxError
+    causing_exception = get_causing_exception(exc)
+    if isinstance(causing_exception, SyntaxError):
+        return format_syntax_error_message(causing_exception)
+    return str(exc)
+
+
+def get_top_level_vertices(graph, vertices_ids):
+    """
+    Retrieves the top-level vertices from the given graph based on the provided vertex IDs.
+
+    Args:
+        graph (Graph): The graph object containing the vertices.
+        vertices_ids (list): A list of vertex IDs.
+
+    Returns:
+        list: A list of top-level vertex IDs.
+
+    """
+    top_level_vertices = []
+    for vertex_id in vertices_ids:
+        vertex = graph.get_vertex(vertex_id)
+        if vertex.parent_is_top_level:
+            top_level_vertices.append(vertex.parent_node_id)
+        else:
+            top_level_vertices.append(vertex_id)
+    return top_level_vertices
+
+
+def parse_exception(exc):
+    """Parse the exception message."""
+    if hasattr(exc, "body"):
+        return exc.body["message"]
+    return str(exc)
+
+
+def get_suggestion_message(outdated_components: list[str]) -> str:
+    """Get the suggestion message for the outdated components."""
+    count = len(outdated_components)
+    if count == 0:
+        return "The flow contains no outdated components."
+    elif count == 1:
+        return f"The flow contains 1 outdated component. We recommend updating the following component: {outdated_components[0]}."
+    else:
+        components = ", ".join(outdated_components)
+        return f"The flow contains {count} outdated components. We recommend updating the following components: {components}."
+
+
+def parse_value(value: Any, input_type: str) -> Any:
+    """Helper function to parse the value based on input type."""
+    if value == "":
+        return value
+    elif input_type == "IntInput":
+        return int(value) if value is not None else None
+    elif input_type == "FloatInput":
+        return float(value) if value is not None else None
+    else:
+        return value

langflow_base_nightly-0.0.96.dev0/langflow/base/agents/agent.py

@@ -0,0 +1,159 @@
+from abc import abstractmethod
+from typing import List, Optional, Union, cast
+
+from langchain.agents import AgentExecutor, BaseMultiActionAgent, BaseSingleActionAgent
+from langchain.agents.agent import RunnableAgent
+from langchain_core.messages import BaseMessage
+from langchain_core.runnables import Runnable
+
+from langflow.base.agents.callback import AgentAsyncHandler
+from langflow.base.agents.utils import data_to_messages
+from langflow.custom import Component
+from langflow.field_typing import Text
+from langflow.inputs.inputs import InputTypes
+from langflow.io import BoolInput, HandleInput, IntInput, MessageTextInput
+from langflow.schema import Data
+from langflow.schema.message import Message
+from langflow.template import Output
+from langflow.utils.constants import MESSAGE_SENDER_AI
+
+
+class LCAgentComponent(Component):
+    trace_type = "agent"
+    _base_inputs: List[InputTypes] = [
+        MessageTextInput(name="input_value", display_name="Input"),
+        BoolInput(
+            name="handle_parsing_errors",
+            display_name="Handle Parse Errors",
+            value=True,
+            advanced=True,
+        ),
+        BoolInput(
+            name="verbose",
+            display_name="Verbose",
+            value=True,
+            advanced=True,
+        ),
+        IntInput(
+            name="max_iterations",
+            display_name="Max Iterations",
+            value=15,
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Agent", name="agent", method="build_agent"),
+        Output(display_name="Response", name="response", method="message_response"),
+    ]
+
+    @abstractmethod
+    def build_agent(self) -> AgentExecutor:
+        """Create the agent."""
+        pass
+
+    async def message_response(self) -> Message:
+        """Run the agent and return the response."""
+        agent = self.build_agent()
+        result = await self.run_agent(agent=agent)
+
+        if isinstance(result, list):
+            result = "\n".join([result_dict["text"] for result_dict in result])
+        message = Message(text=result, sender=MESSAGE_SENDER_AI)
+        self.status = message
+        return message
+
+    def _validate_outputs(self):
+        required_output_methods = ["build_agent"]
+        output_names = [output.name for output in self.outputs]
+        for method_name in required_output_methods:
+            if method_name not in output_names:
+                raise ValueError(f"Output with name '{method_name}' must be defined.")
+            elif not hasattr(self, method_name):
+                raise ValueError(f"Method '{method_name}' must be defined.")
+
+    def get_agent_kwargs(self, flatten: bool = False) -> dict:
+        base = {
+            "handle_parsing_errors": self.handle_parsing_errors,
+            "verbose": self.verbose,
+            "allow_dangerous_code": True,
+        }
+        agent_kwargs = {
+            "handle_parsing_errors": self.handle_parsing_errors,
+            "max_iterations": self.max_iterations,
+        }
+        if flatten:
+            return {
+                **base,
+                **agent_kwargs,
+            }
+        return {**base, "agent_executor_kwargs": agent_kwargs}
+
+    def get_chat_history_data(self) -> Optional[List[Data]]:
+        # might be overridden in subclasses
+        return None
+
+    async def run_agent(self, agent: AgentExecutor) -> Text:
+        input_dict: dict[str, str | list[BaseMessage]] = {"input": self.input_value}
+        self.chat_history = self.get_chat_history_data()
+        if self.chat_history:
+            input_dict["chat_history"] = data_to_messages(self.chat_history)
+        result = await agent.ainvoke(
+            input_dict, config={"callbacks": [AgentAsyncHandler(self.log)] + self.get_langchain_callbacks()}
+        )
+        self.status = result
+        if "output" not in result:
+            raise ValueError("Output key not found in result. Tried 'output'.")
+
+        return cast(str, result.get("output"))
+
+
+class LCToolsAgentComponent(LCAgentComponent):
+    _base_inputs = LCAgentComponent._base_inputs + [
+        HandleInput(
+            name="tools",
+            display_name="Tools",
+            input_types=["Tool", "BaseTool"],
+            is_list=True,
+        ),
+    ]
+
+    def build_agent(self) -> AgentExecutor:
+        agent = self.create_agent_runnable()
+        return AgentExecutor.from_agent_and_tools(
+            agent=RunnableAgent(runnable=agent, input_keys_arg=["input"], return_keys_arg=["output"]),
+            tools=self.tools,
+            **self.get_agent_kwargs(flatten=True),
+        )
+
+    async def run_agent(
+        self,
+        agent: Union[Runnable, BaseSingleActionAgent, BaseMultiActionAgent, AgentExecutor],
+    ) -> Text:
+        if isinstance(agent, AgentExecutor):
+            runnable = agent
+        else:
+            runnable = AgentExecutor.from_agent_and_tools(
+                agent=agent,  # type: ignore
+                tools=self.tools,
+                handle_parsing_errors=self.handle_parsing_errors,
+                verbose=self.verbose,
+                max_iterations=self.max_iterations,
+            )
+        input_dict: dict[str, str | list[BaseMessage]] = {"input": self.input_value}
+        if self.chat_history:
+            input_dict["chat_history"] = data_to_messages(self.chat_history)
+
+        result = runnable.invoke(
+            input_dict, config={"callbacks": [AgentAsyncHandler(self.log)] + self.get_langchain_callbacks()}
+        )
+        self.status = result
+        if "output" not in result:
+            raise ValueError("Output key not found in result. Tried 'output'.")
+
+        return cast(str, result.get("output"))
+
+    @abstractmethod
+    def create_agent_runnable(self) -> Runnable:
+        """Create the agent."""
+        pass

langflow_base_nightly-0.0.96.dev0/langflow/base/document_transformers/model.py

@@ -0,0 +1,49 @@
+from abc import abstractmethod
+from typing import Any
+
+from langchain_core.documents import BaseDocumentTransformer
+
+from langflow.custom import Component
+from langflow.io import Output
+from langflow.schema import Data
+from langflow.utils.util import build_loader_repr_from_data
+
+
+class LCDocumentTransformerComponent(Component):
+    trace_type = "document_transformer"
+    outputs = [
+        Output(display_name="Data", name="data", method="transform_data"),
+    ]
+
+    def transform_data(self) -> list[Data]:
+        data_input = self.get_data_input()
+        documents = []
+
+        if not isinstance(data_input, list):
+            data_input = [data_input]
+
+        for _input in data_input:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        transformer = self.build_document_transformer()
+        docs = transformer.transform_documents(documents)
+        data = self.to_data(docs)
+        self.repr_value = build_loader_repr_from_data(data)
+        return data
+
+    @abstractmethod
+    def get_data_input(self) -> Any:
+        """
+        Get the data input.
+        """
+        pass
+
+    @abstractmethod
+    def build_document_transformer(self) -> BaseDocumentTransformer:
+        """
+        Build the text splitter.
+        """
+        pass

langflow_base_nightly-0.0.96.dev0/langflow/base/textsplitters/model.py

@@ -0,0 +1,37 @@
+from abc import abstractmethod
+
+from langchain_core.documents import BaseDocumentTransformer
+from langchain_text_splitters import TextSplitter
+
+from langflow.base.document_transformers.model import LCDocumentTransformerComponent
+from langflow.io import Output
+from langflow.schema import Data
+
+
+class LCTextSplitterComponent(LCDocumentTransformerComponent):
+    trace_type = "text_splitter"
+    outputs = [
+        Output(display_name="Data", name="data", method="split_data"),
+    ]
+
+    def _validate_outputs(self):
+        required_output_methods = ["text_splitter"]
+        output_names = [output.name for output in self.outputs]
+        for method_name in required_output_methods:
+            if method_name not in output_names:
+                raise ValueError(f"Output with name '{method_name}' must be defined.")
+            elif not hasattr(self, method_name):
+                raise ValueError(f"Method '{method_name}' must be defined.")
+
+    def split_data(self) -> list[Data]:
+        return self.transform_data()
+
+    def build_document_transformer(self) -> BaseDocumentTransformer:
+        return self.build_text_splitter()
+
+    @abstractmethod
+    def build_text_splitter(self) -> TextSplitter:
+        """
+        Build the text splitter.
+        """
+        pass

langflow_base_nightly-0.0.96.dev0/langflow/components/__init__.py

@@ -0,0 +1,39 @@
+from . import (
+    agents,
+    chains,
+    documentloaders,
+    embeddings,
+    helpers,
+    inputs,
+    link_extractors,
+    memories,
+    models,
+    outputs,
+    prompts,
+    prototypes,
+    retrievers,
+    textsplitters,
+    toolkits,
+    tools,
+    vectorstores,
+)
+
+__all__ = [
+    "agents",
+    "chains",
+    "documentloaders",
+    "embeddings",
+    "prompts",
+    "prototypes",
+    "models",
+    "helpers",
+    "inputs",
+    "link_extractors",
+    "memories",
+    "outputs",
+    "retrievers",
+    "textsplitters",
+    "toolkits",
+    "tools",
+    "vectorstores",
+]

langflow_base_nightly-0.0.96.dev0/langflow/components/data/URL.py

@@ -0,0 +1,78 @@
+import re
+
+from langchain_community.document_loaders.web_base import WebBaseLoader
+
+from langflow.helpers.data import data_to_text
+from langflow.custom import Component
+from langflow.io import MessageTextInput, Output
+from langflow.schema import Data
+from langflow.schema.message import Message
+
+
+class URLComponent(Component):
+    display_name = "URL"
+    description = "Fetch content from one or more URLs."
+    icon = "layout-template"
+    name = "URL"
+
+    inputs = [
+        MessageTextInput(
+            name="urls",
+            display_name="URLs",
+            info="Enter one or more URLs, by clicking the '+' button.",
+            is_list=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Data", name="data", method="fetch_content"),
+        Output(display_name="Text", name="text", method="fetch_content_text"),
+    ]
+
+    def ensure_url(self, string: str) -> str:
+        """
+        Ensures the given string is a URL by adding 'http://' if it doesn't start with 'http://' or 'https://'.
+        Raises an error if the string is not a valid URL.
+
+        Parameters:
+            string (str): The string to be checked and possibly modified.
+
+        Returns:
+            str: The modified string that is ensured to be a URL.
+
+        Raises:
+            ValueError: If the string is not a valid URL.
+        """
+        if not string.startswith(("http://", "https://")):
+            string = "http://" + string
+
+        # Basic URL validation regex
+        url_regex = re.compile(
+            r"^(https?:\/\/)?"  # optional protocol
+            r"(www\.)?"  # optional www
+            r"([a-zA-Z0-9.-]+)"  # domain
+            r"(\.[a-zA-Z]{2,})?"  # top-level domain
+            r"(:\d+)?"  # optional port
+            r"(\/[^\s]*)?$",  # optional path
+            re.IGNORECASE,
+        )
+
+        if not url_regex.match(string):
+            raise ValueError(f"Invalid URL: {string}")
+
+        return string
+
+    def fetch_content(self) -> list[Data]:
+        urls = [self.ensure_url(url.strip()) for url in self.urls if url.strip()]
+        loader = WebBaseLoader(web_paths=urls, encoding="utf-8")
+        docs = loader.load()
+        data = [Data(text=doc.page_content, **doc.metadata) for doc in docs]
+        self.status = data
+        return data
+
+    def fetch_content_text(self) -> Message:
+        data = self.fetch_content()
+
+        result_string = data_to_text("{text}", data)
+        self.status = result_string
+        return Message(text=result_string)