lfx-nightly 0.2.0.dev25__py3-none-any.whl

lfx/components/langchain_utilities/language_semantic.py

@@ -0,0 +1,138 @@
+from langchain.docstore.document import Document
+from langchain_experimental.text_splitter import SemanticChunker
+
+from lfx.base.textsplitters.model import LCTextSplitterComponent
+from lfx.io import (
+    DropdownInput,
+    FloatInput,
+    HandleInput,
+    IntInput,
+    MessageTextInput,
+    Output,
+)
+from lfx.schema.data import Data
+
+
+class SemanticTextSplitterComponent(LCTextSplitterComponent):
+    """Split text into semantically meaningful chunks using semantic similarity."""
+
+    display_name: str = "Semantic Text Splitter"
+    name: str = "SemanticTextSplitter"
+    description: str = "Split text into semantically meaningful chunks using semantic similarity."
+    documentation = "https://python.langchain.com/docs/how_to/semantic-chunker/"
+    beta = True  # this component is beta because it is imported from langchain_experimental
+    icon = "LangChain"
+
+    inputs = [
+        HandleInput(
+            name="data_inputs",
+            display_name="Data Inputs",
+            info="List of Data objects containing text and metadata to split.",
+            input_types=["Data"],
+            is_list=True,
+            required=True,
+        ),
+        HandleInput(
+            name="embeddings",
+            display_name="Embeddings",
+            info="Embeddings model to use for semantic similarity. Required.",
+            input_types=["Embeddings"],
+            is_list=False,
+            required=True,
+        ),
+        DropdownInput(
+            name="breakpoint_threshold_type",
+            display_name="Breakpoint Threshold Type",
+            info=(
+                "Method to determine breakpoints. Options: 'percentile', "
+                "'standard_deviation', 'interquartile'. Defaults to 'percentile'."
+            ),
+            value="percentile",
+            options=["percentile", "standard_deviation", "interquartile"],
+        ),
+        FloatInput(
+            name="breakpoint_threshold_amount",
+            display_name="Breakpoint Threshold Amount",
+            info="Numerical amount for the breakpoint threshold.",
+            value=0.5,
+        ),
+        IntInput(
+            name="number_of_chunks",
+            display_name="Number of Chunks",
+            info="Number of chunks to split the text into.",
+            value=5,
+        ),
+        MessageTextInput(
+            name="sentence_split_regex",
+            display_name="Sentence Split Regex",
+            info="Regular expression to split sentences. Optional.",
+            value="",
+            advanced=True,
+        ),
+        IntInput(
+            name="buffer_size",
+            display_name="Buffer Size",
+            info="Size of the buffer.",
+            value=0,
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Chunks", name="chunks", method="split_text"),
+    ]
+
+    def _docs_to_data(self, docs: list[Document]) -> list[Data]:
+        """Convert a list of Document objects to Data objects."""
+        return [Data(text=doc.page_content, data=doc.metadata) for doc in docs]
+
+    def split_text(self) -> list[Data]:
+        """Split the input data into semantically meaningful chunks."""
+        try:
+            embeddings = getattr(self, "embeddings", None)
+            if embeddings is None:
+                error_msg = "An embeddings model is required for SemanticTextSplitter."
+                raise ValueError(error_msg)
+
+            if not self.data_inputs:
+                error_msg = "Data inputs cannot be empty."
+                raise ValueError(error_msg)
+
+            documents = []
+            for _input in self.data_inputs:
+                if isinstance(_input, Data):
+                    documents.append(_input.to_lc_document())
+                else:
+                    error_msg = f"Invalid data input type: {_input}"
+                    raise TypeError(error_msg)
+
+            if not documents:
+                error_msg = "No valid Data objects found in data_inputs."
+                raise ValueError(error_msg)
+
+            texts = [doc.page_content for doc in documents]
+            metadatas = [doc.metadata for doc in documents]
+
+            splitter_params = {
+                "embeddings": embeddings,
+                "breakpoint_threshold_type": self.breakpoint_threshold_type or "percentile",
+                "breakpoint_threshold_amount": self.breakpoint_threshold_amount,
+                "number_of_chunks": self.number_of_chunks,
+                "buffer_size": self.buffer_size,
+            }
+
+            if self.sentence_split_regex:
+                splitter_params["sentence_split_regex"] = self.sentence_split_regex
+
+            splitter = SemanticChunker(**splitter_params)
+            docs = splitter.create_documents(texts, metadatas=metadatas)
+
+            data = self._docs_to_data(docs)
+            self.status = data
+
+        except Exception as e:
+            error_msg = f"An error occurred during semantic splitting: {e}"
+            raise RuntimeError(error_msg) from e
+
+        else:
+            return data

lfx/components/langchain_utilities/llm_checker.py

@@ -0,0 +1,39 @@
+from langchain.chains import LLMCheckerChain
+
+from lfx.base.chains.model import LCChainComponent
+from lfx.inputs.inputs import HandleInput, MultilineInput
+from lfx.schema import Message
+
+
+class LLMCheckerChainComponent(LCChainComponent):
+    display_name = "LLMCheckerChain"
+    description = "Chain for question-answering with self-verification."
+    documentation = "https://python.langchain.com/docs/modules/chains/additional/llm_checker"
+    name = "LLMCheckerChain"
+    legacy: bool = True
+    icon = "LangChain"
+    inputs = [
+        MultilineInput(
+            name="input_value",
+            display_name="Input",
+            info="The input value to pass to the chain.",
+            required=True,
+        ),
+        HandleInput(
+            name="llm",
+            display_name="Language Model",
+            input_types=["LanguageModel"],
+            required=True,
+        ),
+    ]
+
+    def invoke_chain(self) -> Message:
+        chain = LLMCheckerChain.from_llm(llm=self.llm)
+        response = chain.invoke(
+            {chain.input_key: self.input_value},
+            config={"callbacks": self.get_langchain_callbacks()},
+        )
+        result = response.get(chain.output_key, "")
+        result = str(result)
+        self.status = result
+        return Message(text=result)

lfx/components/langchain_utilities/llm_math.py

@@ -0,0 +1,42 @@
+from langchain.chains import LLMMathChain
+
+from lfx.base.chains.model import LCChainComponent
+from lfx.inputs.inputs import HandleInput, MultilineInput
+from lfx.schema import Message
+from lfx.template.field.base import Output
+
+
+class LLMMathChainComponent(LCChainComponent):
+    display_name = "LLMMathChain"
+    description = "Chain that interprets a prompt and executes python code to do math."
+    documentation = "https://python.langchain.com/docs/modules/chains/additional/llm_math"
+    name = "LLMMathChain"
+    legacy: bool = True
+    icon = "LangChain"
+    inputs = [
+        MultilineInput(
+            name="input_value",
+            display_name="Input",
+            info="The input value to pass to the chain.",
+            required=True,
+        ),
+        HandleInput(
+            name="llm",
+            display_name="Language Model",
+            input_types=["LanguageModel"],
+            required=True,
+        ),
+    ]
+
+    outputs = [Output(display_name="Message", name="text", method="invoke_chain")]
+
+    def invoke_chain(self) -> Message:
+        chain = LLMMathChain.from_llm(llm=self.llm)
+        response = chain.invoke(
+            {chain.input_key: self.input_value},
+            config={"callbacks": self.get_langchain_callbacks()},
+        )
+        result = response.get(chain.output_key, "")
+        result = str(result)
+        self.status = result
+        return Message(text=result)

lfx/components/langchain_utilities/natural_language.py

@@ -0,0 +1,61 @@
+from typing import Any
+
+from langchain_text_splitters import NLTKTextSplitter, TextSplitter
+
+from lfx.base.textsplitters.model import LCTextSplitterComponent
+from lfx.inputs.inputs import DataInput, IntInput, MessageTextInput
+from lfx.utils.util import unescape_string
+
+
+class NaturalLanguageTextSplitterComponent(LCTextSplitterComponent):
+    display_name = "Natural Language Text Splitter"
+    description = "Split text based on natural language boundaries, optimized for a specified language."
+    documentation = (
+        "https://python.langchain.com/v0.1/docs/modules/data_connection/document_transformers/split_by_token/#nltk"
+    )
+    name = "NaturalLanguageTextSplitter"
+    icon = "LangChain"
+    inputs = [
+        IntInput(
+            name="chunk_size",
+            display_name="Chunk Size",
+            info="The maximum number of characters in each chunk after splitting.",
+            value=1000,
+        ),
+        IntInput(
+            name="chunk_overlap",
+            display_name="Chunk Overlap",
+            info="The number of characters that overlap between consecutive chunks.",
+            value=200,
+        ),
+        DataInput(
+            name="data_input",
+            display_name="Input",
+            info="The text data to be split.",
+            input_types=["Document", "Data"],
+            required=True,
+        ),
+        MessageTextInput(
+            name="separator",
+            display_name="Separator",
+            info='The character(s) to use as a delimiter when splitting text.\nDefaults to "\\n\\n" if left empty.',
+        ),
+        MessageTextInput(
+            name="language",
+            display_name="Language",
+            info='The language of the text. Default is "English". '
+            "Supports multiple languages for better text boundary recognition.",
+        ),
+    ]
+
+    def get_data_input(self) -> Any:
+        return self.data_input
+
+    def build_text_splitter(self) -> TextSplitter:
+        separator = unescape_string(self.separator) if self.separator else "\n\n"
+        return NLTKTextSplitter(
+            language=self.language.lower() if self.language else "english",
+            separator=separator,
+            chunk_size=self.chunk_size,
+            chunk_overlap=self.chunk_overlap,
+        )

lfx/components/langchain_utilities/openai_tools.py

@@ -0,0 +1,53 @@
+from langchain.agents import create_openai_tools_agent
+from langchain_core.prompts import ChatPromptTemplate, HumanMessagePromptTemplate, PromptTemplate
+
+from lfx.base.agents.agent import LCToolsAgentComponent
+from lfx.inputs.inputs import (
+    DataInput,
+    HandleInput,
+    MultilineInput,
+)
+from lfx.schema.data import Data
+
+
+class OpenAIToolsAgentComponent(LCToolsAgentComponent):
+    display_name: str = "OpenAI Tools Agent"
+    description: str = "Agent that uses tools via openai-tools."
+    icon = "LangChain"
+    name = "OpenAIToolsAgent"
+
+    inputs = [
+        *LCToolsAgentComponent.get_base_inputs(),
+        HandleInput(
+            name="llm",
+            display_name="Language Model",
+            input_types=["LanguageModel", "ToolEnabledLanguageModel"],
+            required=True,
+        ),
+        MultilineInput(
+            name="system_prompt",
+            display_name="System Prompt",
+            info="System prompt for the agent.",
+            value="You are a helpful assistant",
+        ),
+        MultilineInput(
+            name="user_prompt", display_name="Prompt", info="This prompt must contain 'input' key.", value="{input}"
+        ),
+        DataInput(name="chat_history", display_name="Chat History", is_list=True, advanced=True),
+    ]
+
+    def get_chat_history_data(self) -> list[Data] | None:
+        return self.chat_history
+
+    def create_agent_runnable(self):
+        if "input" not in self.user_prompt:
+            msg = "Prompt must contain 'input' key."
+            raise ValueError(msg)
+        messages = [
+            ("system", self.system_prompt),
+            ("placeholder", "{chat_history}"),
+            HumanMessagePromptTemplate(prompt=PromptTemplate(input_variables=["input"], template=self.user_prompt)),
+            ("placeholder", "{agent_scratchpad}"),
+        ]
+        prompt = ChatPromptTemplate.from_messages(messages)
+        return create_openai_tools_agent(self.llm, self.tools, prompt)

lfx/components/langchain_utilities/openapi.py

@@ -0,0 +1,48 @@
+from pathlib import Path
+
+import yaml
+from langchain.agents import AgentExecutor
+from langchain_community.agent_toolkits import create_openapi_agent
+from langchain_community.agent_toolkits.openapi.toolkit import OpenAPIToolkit
+from langchain_community.tools.json.tool import JsonSpec
+from langchain_community.utilities.requests import TextRequestsWrapper
+
+from lfx.base.agents.agent import LCAgentComponent
+from lfx.inputs.inputs import BoolInput, FileInput, HandleInput
+
+
+class OpenAPIAgentComponent(LCAgentComponent):
+    display_name = "OpenAPI Agent"
+    description = "Agent to interact with OpenAPI API."
+    name = "OpenAPIAgent"
+    icon = "LangChain"
+    inputs = [
+        *LCAgentComponent.get_base_inputs(),
+        HandleInput(name="llm", display_name="Language Model", input_types=["LanguageModel"], required=True),
+        FileInput(name="path", display_name="File Path", file_types=["json", "yaml", "yml"], required=True),
+        BoolInput(name="allow_dangerous_requests", display_name="Allow Dangerous Requests", value=False, required=True),
+    ]
+
+    def build_agent(self) -> AgentExecutor:
+        path = Path(self.path)
+        if path.suffix in {"yaml", "yml"}:
+            with path.open(encoding="utf-8") as file:
+                yaml_dict = yaml.safe_load(file)
+            spec = JsonSpec(dict_=yaml_dict)
+        else:
+            spec = JsonSpec.from_file(path)
+        requests_wrapper = TextRequestsWrapper()
+        toolkit = OpenAPIToolkit.from_llm(
+            llm=self.llm,
+            json_spec=spec,
+            requests_wrapper=requests_wrapper,
+            allow_dangerous_requests=self.allow_dangerous_requests,
+        )
+
+        agent_args = self.get_agent_kwargs()
+
+        # This is bit weird - generally other create_*_agent functions have max_iterations in the
+        # `agent_executor_kwargs`, but openai has this parameter passed directly.
+        agent_args["max_iterations"] = agent_args["agent_executor_kwargs"]["max_iterations"]
+        del agent_args["agent_executor_kwargs"]["max_iterations"]
+        return create_openapi_agent(llm=self.llm, toolkit=toolkit, **agent_args)

lfx/components/langchain_utilities/recursive_character.py

@@ -0,0 +1,60 @@
+from typing import Any
+
+from langchain_text_splitters import RecursiveCharacterTextSplitter, TextSplitter
+
+from lfx.base.textsplitters.model import LCTextSplitterComponent
+from lfx.inputs.inputs import DataInput, IntInput, MessageTextInput
+from lfx.utils.util import unescape_string
+
+
+class RecursiveCharacterTextSplitterComponent(LCTextSplitterComponent):
+    display_name: str = "Recursive Character Text Splitter"
+    description: str = "Split text trying to keep all related text together."
+    documentation: str = "https://docs.langflow.org/components-processing"
+    name = "RecursiveCharacterTextSplitter"
+    icon = "LangChain"
+
+    inputs = [
+        IntInput(
+            name="chunk_size",
+            display_name="Chunk Size",
+            info="The maximum length of each chunk.",
+            value=1000,
+        ),
+        IntInput(
+            name="chunk_overlap",
+            display_name="Chunk Overlap",
+            info="The amount of overlap between chunks.",
+            value=200,
+        ),
+        DataInput(
+            name="data_input",
+            display_name="Input",
+            info="The texts to split.",
+            input_types=["Document", "Data"],
+            required=True,
+        ),
+        MessageTextInput(
+            name="separators",
+            display_name="Separators",
+            info='The characters to split on.\nIf left empty defaults to ["\\n\\n", "\\n", " ", ""].',
+            is_list=True,
+        ),
+    ]
+
+    def get_data_input(self) -> Any:
+        return self.data_input
+
+    def build_text_splitter(self) -> TextSplitter:
+        if not self.separators:
+            separators: list[str] | None = None
+        else:
+            # check if the separators list has escaped characters
+            # if there are escaped characters, unescape them
+            separators = [unescape_string(x) for x in self.separators]
+
+        return RecursiveCharacterTextSplitter(
+            separators=separators,
+            chunk_size=self.chunk_size,
+            chunk_overlap=self.chunk_overlap,
+        )

lfx/components/langchain_utilities/retrieval_qa.py

@@ -0,0 +1,83 @@
+from typing import cast
+
+from langchain.chains import RetrievalQA
+
+from lfx.base.chains.model import LCChainComponent
+from lfx.inputs.inputs import BoolInput, DropdownInput, HandleInput, MultilineInput
+from lfx.schema import Message
+
+
+class RetrievalQAComponent(LCChainComponent):
+    display_name = "Retrieval QA"
+    description = "Chain for question-answering querying sources from a retriever."
+    name = "RetrievalQA"
+    legacy: bool = True
+    icon = "LangChain"
+    inputs = [
+        MultilineInput(
+            name="input_value",
+            display_name="Input",
+            info="The input value to pass to the chain.",
+            required=True,
+        ),
+        DropdownInput(
+            name="chain_type",
+            display_name="Chain Type",
+            info="Chain type to use.",
+            options=["Stuff", "Map Reduce", "Refine", "Map Rerank"],
+            value="Stuff",
+            advanced=True,
+        ),
+        HandleInput(
+            name="llm",
+            display_name="Language Model",
+            input_types=["LanguageModel"],
+            required=True,
+        ),
+        HandleInput(
+            name="retriever",
+            display_name="Retriever",
+            input_types=["Retriever"],
+            required=True,
+        ),
+        HandleInput(
+            name="memory",
+            display_name="Memory",
+            input_types=["BaseChatMemory"],
+        ),
+        BoolInput(
+            name="return_source_documents",
+            display_name="Return Source Documents",
+            value=False,
+        ),
+    ]
+
+    def invoke_chain(self) -> Message:
+        chain_type = self.chain_type.lower().replace(" ", "_")
+        if self.memory:
+            self.memory.input_key = "query"
+            self.memory.output_key = "result"
+
+        runnable = RetrievalQA.from_chain_type(
+            llm=self.llm,
+            chain_type=chain_type,
+            retriever=self.retriever,
+            memory=self.memory,
+            # always include to help debugging
+            #
+            return_source_documents=True,
+        )
+
+        result = runnable.invoke(
+            {"query": self.input_value},
+            config={"callbacks": self.get_langchain_callbacks()},
+        )
+
+        source_docs = self.to_data(result.get("source_documents", keys=[]))
+        result_str = str(result.get("result", ""))
+        if self.return_source_documents and len(source_docs):
+            references_str = self.create_references_from_data(source_docs)
+            result_str = f"{result_str}\n{references_str}"
+        # put the entire result to debug history, query and content
+        self.status = {**result, "source_documents": source_docs, "output": result_str}
+        return cast("Message", result_str)

lfx/components/langchain_utilities/runnable_executor.py

@@ -0,0 +1,137 @@
+from langchain.agents import AgentExecutor
+
+from lfx.custom.custom_component.component import Component
+from lfx.inputs.inputs import BoolInput, HandleInput, MessageTextInput
+from lfx.schema.message import Message
+from lfx.template.field.base import Output
+
+
+class RunnableExecComponent(Component):
+    description = "Execute a runnable. It will try to guess the input and output keys."
+    display_name = "Runnable Executor"
+    name = "RunnableExecutor"
+    beta: bool = True
+    icon = "LangChain"
+
+    inputs = [
+        MessageTextInput(name="input_value", display_name="Input", required=True),
+        HandleInput(
+            name="runnable",
+            display_name="Agent Executor",
+            input_types=["Chain", "AgentExecutor", "Agent", "Runnable"],
+            required=True,
+        ),
+        MessageTextInput(
+            name="input_key",
+            display_name="Input Key",
+            value="input",
+            advanced=True,
+        ),
+        MessageTextInput(
+            name="output_key",
+            display_name="Output Key",
+            value="output",
+            advanced=True,
+        ),
+        BoolInput(
+            name="use_stream",
+            display_name="Stream",
+            value=False,
+        ),
+    ]
+
+    outputs = [
+        Output(
+            display_name="Message",
+            name="text",
+            method="build_executor",
+        ),
+    ]
+
+    def get_output(self, result, input_key, output_key):
+        """Retrieves the output value from the given result dictionary based on the specified input and output keys.
+
+        Args:
+            result (dict): The result dictionary containing the output value.
+            input_key (str): The key used to retrieve the input value from the result dictionary.
+            output_key (str): The key used to retrieve the output value from the result dictionary.
+
+        Returns:
+            tuple: A tuple containing the output value and the status message.
+
+        """
+        possible_output_keys = ["answer", "response", "output", "result", "text"]
+        status = ""
+        result_value = None
+
+        if output_key in result:
+            result_value = result.get(output_key)
+        elif len(result) == 2 and input_key in result:  # noqa: PLR2004
+            # get the other key from the result dict
+            other_key = next(k for k in result if k != input_key)
+            if other_key == output_key:
+                result_value = result.get(output_key)
+            else:
+                status += f"Warning: The output key is not '{output_key}'. The output key is '{other_key}'."
+                result_value = result.get(other_key)
+        elif len(result) == 1:
+            result_value = next(iter(result.values()))
+        elif any(k in result for k in possible_output_keys):
+            for key in possible_output_keys:
+                if key in result:
+                    result_value = result.get(key)
+                    status += f"Output key: '{key}'."
+                    break
+            if result_value is None:
+                result_value = result
+                status += f"Warning: The output key is not '{output_key}'."
+        else:
+            result_value = result
+            status += f"Warning: The output key is not '{output_key}'."
+
+        return result_value, status
+
+    def get_input_dict(self, runnable, input_key, input_value):
+        """Returns a dictionary containing the input key-value pair for the given runnable.
+
+        Args:
+            runnable: The runnable object.
+            input_key: The key for the input value.
+            input_value: The value for the input key.
+
+        Returns:
+            input_dict: A dictionary containing the input key-value pair.
+            status: A status message indicating if the input key is not in the runnable's input keys.
+        """
+        input_dict = {}
+        status = ""
+        if hasattr(runnable, "input_keys"):
+            # Check if input_key is in the runnable's input_keys
+            if input_key in runnable.input_keys:
+                input_dict[input_key] = input_value
+            else:
+                input_dict = dict.fromkeys(runnable.input_keys, input_value)
+                status = f"Warning: The input key is not '{input_key}'. The input key is '{runnable.input_keys}'."
+        return input_dict, status
+
+    async def build_executor(self) -> Message:
+        input_dict, status = self.get_input_dict(self.runnable, self.input_key, self.input_value)
+        if not isinstance(self.runnable, AgentExecutor):
+            msg = "The runnable must be an AgentExecutor"
+            raise TypeError(msg)
+
+        if self.use_stream:
+            return self.astream_events(input_dict)
+        result = await self.runnable.ainvoke(input_dict)
+        result_value, status_ = self.get_output(result, self.input_key, self.output_key)
+        status += status_
+        status += f"\n\nOutput: {result_value}\n\nRaw Output: {result}"
+        self.status = status
+        return result_value
+
+    async def astream_events(self, runnable_input):
+        async for event in self.runnable.astream_events(runnable_input, version="v1"):
+            if event.get("event") != "on_chat_model_stream":
+                continue
+
+            yield event.get("data").get("chunk")