lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/langchain_utilities/spider.py

@@ -0,0 +1,142 @@
+from spider.spider import Spider
+
+from lfx.base.langchain_utilities.spider_constants import MODES
+from lfx.custom.custom_component.component import Component
+from lfx.io import (
+    BoolInput,
+    DictInput,
+    DropdownInput,
+    IntInput,
+    Output,
+    SecretStrInput,
+    StrInput,
+)
+from lfx.schema.data import Data
+
+
+class SpiderTool(Component):
+    display_name: str = "Spider Web Crawler & Scraper"
+    description: str = "Spider API for web crawling and scraping."
+    output_types: list[str] = ["Document"]
+    documentation: str = "https://spider.cloud/docs/api"
+
+    inputs = [
+        SecretStrInput(
+            name="spider_api_key",
+            display_name="Spider API Key",
+            required=True,
+            password=True,
+            info="The Spider API Key, get it from https://spider.cloud",
+        ),
+        StrInput(
+            name="url",
+            display_name="URL",
+            required=True,
+            info="The URL to scrape or crawl",
+        ),
+        DropdownInput(
+            name="mode",
+            display_name="Mode",
+            required=True,
+            options=MODES,
+            value=MODES[0],
+            info="The mode of operation: scrape or crawl",
+        ),
+        IntInput(
+            name="limit",
+            display_name="Limit",
+            info="The maximum amount of pages allowed to crawl per website. Set to 0 to crawl all pages.",
+            advanced=True,
+        ),
+        IntInput(
+            name="depth",
+            display_name="Depth",
+            info="The crawl limit for maximum depth. If 0, no limit will be applied.",
+            advanced=True,
+        ),
+        StrInput(
+            name="blacklist",
+            display_name="Blacklist",
+            info="Blacklist paths that you do not want to crawl. Use Regex patterns.",
+            advanced=True,
+        ),
+        StrInput(
+            name="whitelist",
+            display_name="Whitelist",
+            info="Whitelist paths that you want to crawl, ignoring all other routes. Use Regex patterns.",
+            advanced=True,
+        ),
+        BoolInput(
+            name="readability",
+            display_name="Use Readability",
+            info="Use readability to pre-process the content for reading.",
+            advanced=True,
+        ),
+        IntInput(
+            name="request_timeout",
+            display_name="Request Timeout",
+            info="Timeout for the request in seconds.",
+            advanced=True,
+        ),
+        BoolInput(
+            name="metadata",
+            display_name="Metadata",
+            info="Include metadata in the response.",
+            advanced=True,
+        ),
+        DictInput(
+            name="params",
+            display_name="Additional Parameters",
+            info="Additional parameters to pass to the API. If provided, other inputs will be ignored.",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Markdown", name="content", method="crawl"),
+    ]
+
+    def crawl(self) -> list[Data]:
+        if self.params:
+            parameters = self.params["data"]
+        else:
+            parameters = {
+                "limit": self.limit or None,
+                "depth": self.depth or None,
+                "blacklist": self.blacklist or None,
+                "whitelist": self.whitelist or None,
+                "readability": self.readability,
+                "request_timeout": self.request_timeout or None,
+                "metadata": self.metadata,
+                "return_format": "markdown",
+            }
+
+        app = Spider(api_key=self.spider_api_key)
+        if self.mode == "scrape":
+            parameters["limit"] = 1
+            result = app.scrape_url(self.url, parameters)
+        elif self.mode == "crawl":
+            result = app.crawl_url(self.url, parameters)
+        else:
+            msg = f"Invalid mode: {self.mode}. Must be 'scrape' or 'crawl'."
+            raise ValueError(msg)
+
+        records = []
+
+        for record in result:
+            if self.metadata:
+                records.append(
+                    Data(
+                        data={
+                            "content": record["content"],
+                            "url": record["url"],
+                            "metadata": record["metadata"],
+                        }
+                    )
+                )
+            else:
+                records.append(Data(data={"content": record["content"], "url": record["url"]}))
+        return records
+
+
+class SpiderToolError(Exception):
+    """SpiderTool error."""

lfx/components/langchain_utilities/sql.py

@@ -0,0 +1,40 @@
+from langchain.agents import AgentExecutor
+from langchain_community.agent_toolkits import SQLDatabaseToolkit
+from langchain_community.agent_toolkits.sql.base import create_sql_agent
+from langchain_community.utilities import SQLDatabase
+
+from lfx.base.agents.agent import LCAgentComponent
+from lfx.inputs.inputs import HandleInput, MessageTextInput
+from lfx.io import Output
+
+
+class SQLAgentComponent(LCAgentComponent):
+    display_name = "SQLAgent"
+    description = "Construct an SQL agent from an LLM and tools."
+    name = "SQLAgent"
+    icon = "LangChain"
+    inputs = [
+        *LCAgentComponent.get_base_inputs(),
+        HandleInput(name="llm", display_name="Language Model", input_types=["LanguageModel"], required=True),
+        MessageTextInput(name="database_uri", display_name="Database URI", required=True),
+        HandleInput(
+            name="extra_tools",
+            display_name="Extra Tools",
+            input_types=["Tool"],
+            is_list=True,
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Response", name="response", method="message_response"),
+        Output(display_name="Agent", name="agent", method="build_agent", tool_mode=False),
+    ]
+
+    def build_agent(self) -> AgentExecutor:
+        db = SQLDatabase.from_uri(self.database_uri)
+        toolkit = SQLDatabaseToolkit(db=db, llm=self.llm)
+        agent_args = self.get_agent_kwargs()
+        agent_args["max_iterations"] = agent_args["agent_executor_kwargs"]["max_iterations"]
+        del agent_args["agent_executor_kwargs"]["max_iterations"]
+        return create_sql_agent(llm=self.llm, toolkit=toolkit, extra_tools=self.extra_tools or [], **agent_args)

lfx/components/langchain_utilities/sql_database.py

@@ -0,0 +1,35 @@
+from langchain_community.utilities.sql_database import SQLDatabase
+from sqlalchemy import create_engine
+from sqlalchemy.pool import StaticPool
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import (
+    Output,
+    StrInput,
+)
+
+
+class SQLDatabaseComponent(Component):
+    display_name = "SQLDatabase"
+    description = "SQL Database"
+    name = "SQLDatabase"
+    icon = "LangChain"
+
+    inputs = [
+        StrInput(name="uri", display_name="URI", info="URI to the database.", required=True),
+    ]
+
+    outputs = [
+        Output(display_name="SQLDatabase", name="SQLDatabase", method="build_sqldatabase"),
+    ]
+
+    def clean_up_uri(self, uri: str) -> str:
+        if uri.startswith("postgres://"):
+            uri = uri.replace("postgres://", "postgresql://")
+        return uri.strip()
+
+    def build_sqldatabase(self) -> SQLDatabase:
+        uri = self.clean_up_uri(self.uri)
+        # Create an engine using SQLAlchemy with StaticPool
+        engine = create_engine(uri, poolclass=StaticPool)
+        return SQLDatabase(engine)

lfx/components/langchain_utilities/sql_generator.py

@@ -0,0 +1,78 @@
+from typing import TYPE_CHECKING
+
+from langchain.chains import create_sql_query_chain
+from langchain_core.prompts import PromptTemplate
+
+from lfx.base.chains.model import LCChainComponent
+from lfx.inputs.inputs import HandleInput, IntInput, MultilineInput
+from lfx.schema import Message
+from lfx.template.field.base import Output
+
+if TYPE_CHECKING:
+    from langchain_core.runnables import Runnable
+
+
+class SQLGeneratorComponent(LCChainComponent):
+    display_name = "Natural Language to SQL"
+    description = "Generate SQL from natural language."
+    name = "SQLGenerator"
+    legacy: bool = True
+    icon = "LangChain"
+
+    inputs = [
+        MultilineInput(
+            name="input_value",
+            display_name="Input",
+            info="The input value to pass to the chain.",
+            required=True,
+        ),
+        HandleInput(
+            name="llm",
+            display_name="Language Model",
+            input_types=["LanguageModel"],
+            required=True,
+        ),
+        HandleInput(
+            name="db",
+            display_name="SQLDatabase",
+            input_types=["SQLDatabase"],
+            required=True,
+        ),
+        IntInput(
+            name="top_k",
+            display_name="Top K",
+            info="The number of results per select statement to return.",
+            value=5,
+        ),
+        MultilineInput(
+            name="prompt",
+            display_name="Prompt",
+            info="The prompt must contain `{question}`.",
+        ),
+    ]
+
+    outputs = [Output(display_name="Message", name="text", method="invoke_chain")]
+
+    def invoke_chain(self) -> Message:
+        prompt_template = PromptTemplate.from_template(template=self.prompt) if self.prompt else None
+
+        if self.top_k < 1:
+            msg = "Top K must be greater than 0."
+            raise ValueError(msg)
+
+        if not prompt_template:
+            sql_query_chain = create_sql_query_chain(llm=self.llm, db=self.db, k=self.top_k)
+        else:
+            # Check if {question} is in the prompt
+            if "{question}" not in prompt_template.template or "question" not in prompt_template.input_variables:
+                msg = "Prompt must contain `{question}` to be used with Natural Language to SQL."
+                raise ValueError(msg)
+            sql_query_chain = create_sql_query_chain(llm=self.llm, db=self.db, prompt=prompt_template, k=self.top_k)
+        query_writer: Runnable = sql_query_chain | {"query": lambda x: x.replace("SQLQuery:", "").strip()}
+        response = query_writer.invoke(
+            {"question": self.input_value},
+            config={"callbacks": self.get_langchain_callbacks()},
+        )
+        query = response.get("query")
+        self.status = query
+        return query

lfx/components/langchain_utilities/tool_calling.py

@@ -0,0 +1,59 @@
+from langchain.agents import create_tool_calling_agent
+from langchain_core.prompts import ChatPromptTemplate
+
+from lfx.base.agents.agent import LCToolsAgentComponent
+from lfx.inputs.inputs import (
+    DataInput,
+    HandleInput,
+    MessageTextInput,
+)
+from lfx.schema.data import Data
+
+
+class ToolCallingAgentComponent(LCToolsAgentComponent):
+    display_name: str = "Tool Calling Agent"
+    description: str = "An agent designed to utilize various tools seamlessly within workflows."
+    icon = "LangChain"
+    name = "ToolCallingAgent"
+
+    inputs = [
+        *LCToolsAgentComponent.get_base_inputs(),
+        HandleInput(
+            name="llm",
+            display_name="Language Model",
+            input_types=["LanguageModel"],
+            required=True,
+            info="Language model that the agent utilizes to perform tasks effectively.",
+        ),
+        MessageTextInput(
+            name="system_prompt",
+            display_name="System Prompt",
+            info="System prompt to guide the agent's behavior.",
+            value="You are a helpful assistant that can use tools to answer questions and perform tasks.",
+        ),
+        DataInput(
+            name="chat_history",
+            display_name="Chat Memory",
+            is_list=True,
+            advanced=True,
+            info="This input stores the chat history, allowing the agent to remember previous conversations.",
+        ),
+    ]
+
+    def get_chat_history_data(self) -> list[Data] | None:
+        return self.chat_history
+
+    def create_agent_runnable(self):
+        messages = [
+            ("system", "{system_prompt}"),
+            ("placeholder", "{chat_history}"),
+            ("human", "{input}"),
+            ("placeholder", "{agent_scratchpad}"),
+        ]
+        prompt = ChatPromptTemplate.from_messages(messages)
+        self.validate_tool_names()
+        try:
+            return create_tool_calling_agent(self.llm, self.tools or [], prompt)
+        except NotImplementedError as e:
+            message = f"{self.display_name} does not support tool calling. Please try using a compatible model."
+            raise NotImplementedError(message) from e

lfx/components/langchain_utilities/vector_store_info.py

@@ -0,0 +1,49 @@
+from langchain.agents.agent_toolkits.vectorstore.toolkit import VectorStoreInfo
+
+from lfx.custom.custom_component.component import Component
+from lfx.inputs.inputs import HandleInput, MessageTextInput, MultilineInput
+from lfx.template.field.base import Output
+
+
+class VectorStoreInfoComponent(Component):
+    display_name = "VectorStoreInfo"
+    description = "Information about a VectorStore"
+    name = "VectorStoreInfo"
+    legacy: bool = True
+    icon = "LangChain"
+
+    inputs = [
+        MessageTextInput(
+            name="vectorstore_name",
+            display_name="Name",
+            info="Name of the VectorStore",
+            required=True,
+        ),
+        MultilineInput(
+            name="vectorstore_description",
+            display_name="Description",
+            info="Description of the VectorStore",
+            required=True,
+        ),
+        HandleInput(
+            name="input_vectorstore",
+            display_name="Vector Store",
+            input_types=["VectorStore"],
+            required=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Vector Store Info", name="info", method="build_info"),
+    ]
+
+    def build_info(self) -> VectorStoreInfo:
+        self.status = {
+            "name": self.vectorstore_name,
+            "description": self.vectorstore_description,
+        }
+        return VectorStoreInfo(
+            vectorstore=self.input_vectorstore,
+            description=self.vectorstore_description,
+            name=self.vectorstore_name,
+        )

lfx/components/langchain_utilities/vector_store_router.py

@@ -0,0 +1,33 @@
+from langchain.agents import AgentExecutor, create_vectorstore_router_agent
+from langchain.agents.agent_toolkits.vectorstore.toolkit import VectorStoreRouterToolkit
+
+from lfx.base.agents.agent import LCAgentComponent
+from lfx.inputs.inputs import HandleInput
+
+
+class VectorStoreRouterAgentComponent(LCAgentComponent):
+    display_name = "VectorStoreRouterAgent"
+    description = "Construct an agent from a Vector Store Router."
+    name = "VectorStoreRouterAgent"
+    legacy: bool = True
+
+    inputs = [
+        *LCAgentComponent.get_base_inputs(),
+        HandleInput(
+            name="llm",
+            display_name="Language Model",
+            input_types=["LanguageModel"],
+            required=True,
+        ),
+        HandleInput(
+            name="vectorstores",
+            display_name="Vector Stores",
+            input_types=["VectorStoreInfo"],
+            is_list=True,
+            required=True,
+        ),
+    ]
+
+    def build_agent(self) -> AgentExecutor:
+        toolkit = VectorStoreRouterToolkit(vectorstores=self.vectorstores, llm=self.llm)
+        return create_vectorstore_router_agent(llm=self.llm, toolkit=toolkit, **self.get_agent_kwargs())

lfx/components/langchain_utilities/xml_agent.py

@@ -0,0 +1,71 @@
+from langchain.agents import create_xml_agent
+from langchain_core.prompts import ChatPromptTemplate, HumanMessagePromptTemplate, PromptTemplate
+
+from lfx.base.agents.agent import LCToolsAgentComponent
+from lfx.inputs.inputs import (
+    DataInput,
+    HandleInput,
+    MultilineInput,
+)
+from lfx.schema.data import Data
+
+
+class XMLAgentComponent(LCToolsAgentComponent):
+    display_name: str = "XML Agent"
+    description: str = "Agent that uses tools formatting instructions as xml to the Language Model."
+    icon = "LangChain"
+    beta = True
+    name = "XMLAgent"
+    inputs = [
+        *LCToolsAgentComponent.get_base_inputs(),
+        HandleInput(name="llm", display_name="Language Model", input_types=["LanguageModel"], required=True),
+        DataInput(name="chat_history", display_name="Chat History", is_list=True, advanced=True),
+        MultilineInput(
+            name="system_prompt",
+            display_name="System Prompt",
+            info="System prompt for the agent.",
+            value="""You are a helpful assistant. Help the user answer any questions.
+
+You have access to the following tools:
+
+{tools}
+
+In order to use a tool, you can use <tool></tool> and <tool_input></tool_input> tags. You will then get back a response in the form <observation></observation>
+
+For example, if you have a tool called 'search' that could run a google search, in order to search for the weather in SF you would respond:
+
+<tool>search</tool><tool_input>weather in SF</tool_input>
+
+<observation>64 degrees</observation>
+
+When you are done, respond with a final answer between <final_answer></final_answer>. For example:
+
+<final_answer>The weather in SF is 64 degrees</final_answer>
+
+Begin!
+
+Question: {input}
+
+{agent_scratchpad}
+            """,  # noqa: E501
+        ),
+        MultilineInput(
+            name="user_prompt", display_name="Prompt", info="This prompt must contain 'input' key.", value="{input}"
+        ),
+    ]
+
+    def get_chat_history_data(self) -> list[Data] | None:
+        return self.chat_history
+
+    def create_agent_runnable(self):
+        if "input" not in self.user_prompt:
+            msg = "Prompt must contain 'input' key."
+            raise ValueError(msg)
+        messages = [
+            ("system", self.system_prompt),
+            ("placeholder", "{chat_history}"),
+            HumanMessagePromptTemplate(prompt=PromptTemplate(input_variables=["input"], template=self.user_prompt)),
+            ("ai", "{agent_scratchpad}"),
+        ]
+        prompt = ChatPromptTemplate.from_messages(messages)
+        return create_xml_agent(self.llm, self.tools, prompt)

lfx/components/langwatch/__init__.py

@@ -0,0 +1,3 @@
+from .langwatch import LangWatchComponent
+
+__all__ = ["LangWatchComponent"]