lfx-nightly 0.2.0.dev25__py3-none-any.whl

lfx/components/models_and_agents/memory.py

@@ -0,0 +1,268 @@
+from typing import Any, cast
+
+from lfx.custom.custom_component.component import Component
+from lfx.helpers.data import data_to_text
+from lfx.inputs.inputs import DropdownInput, HandleInput, IntInput, MessageTextInput, MultilineInput, TabInput
+from lfx.memory import aget_messages, astore_message
+from lfx.schema.data import Data
+from lfx.schema.dataframe import DataFrame
+from lfx.schema.dotdict import dotdict
+from lfx.schema.message import Message
+from lfx.template.field.base import Output
+from lfx.utils.component_utils import set_current_fields, set_field_display
+from lfx.utils.constants import MESSAGE_SENDER_AI, MESSAGE_SENDER_NAME_AI, MESSAGE_SENDER_USER
+
+
+class MemoryComponent(Component):
+    display_name = "Message History"
+    description = "Stores or retrieves stored chat messages from Langflow tables or an external memory."
+    documentation: str = "https://docs.langflow.org/message-history"
+    icon = "message-square-more"
+    name = "Memory"
+    default_keys = ["mode", "memory", "session_id", "context_id"]
+    mode_config = {
+        "Store": ["message", "memory", "sender", "sender_name", "session_id", "context_id"],
+        "Retrieve": ["n_messages", "order", "template", "memory", "session_id", "context_id"],
+    }
+
+    inputs = [
+        TabInput(
+            name="mode",
+            display_name="Mode",
+            options=["Retrieve", "Store"],
+            value="Retrieve",
+            info="Operation mode: Store messages or Retrieve messages.",
+            real_time_refresh=True,
+        ),
+        MessageTextInput(
+            name="message",
+            display_name="Message",
+            info="The chat message to be stored.",
+            tool_mode=True,
+            dynamic=True,
+            show=False,
+        ),
+        HandleInput(
+            name="memory",
+            display_name="External Memory",
+            input_types=["Memory"],
+            info="Retrieve messages from an external memory. If empty, it will use the Langflow tables.",
+            advanced=True,
+        ),
+        DropdownInput(
+            name="sender_type",
+            display_name="Sender Type",
+            options=[MESSAGE_SENDER_AI, MESSAGE_SENDER_USER, "Machine and User"],
+            value="Machine and User",
+            info="Filter by sender type.",
+            advanced=True,
+        ),
+        MessageTextInput(
+            name="sender",
+            display_name="Sender",
+            info="The sender of the message. Might be Machine or User. "
+            "If empty, the current sender parameter will be used.",
+            advanced=True,
+        ),
+        MessageTextInput(
+            name="sender_name",
+            display_name="Sender Name",
+            info="Filter by sender name.",
+            advanced=True,
+            show=False,
+        ),
+        IntInput(
+            name="n_messages",
+            display_name="Number of Messages",
+            value=100,
+            info="Number of messages to retrieve.",
+            advanced=True,
+            show=True,
+        ),
+        MessageTextInput(
+            name="session_id",
+            display_name="Session ID",
+            info="The session ID of the chat. If empty, the current session ID parameter will be used.",
+            value="",
+            advanced=True,
+        ),
+        MessageTextInput(
+            name="context_id",
+            display_name="Context ID",
+            info="The context ID of the chat. Adds an extra layer to the local memory.",
+            value="",
+            advanced=True,
+        ),
+        DropdownInput(
+            name="order",
+            display_name="Order",
+            options=["Ascending", "Descending"],
+            value="Ascending",
+            info="Order of the messages.",
+            advanced=True,
+            tool_mode=True,
+            required=True,
+        ),
+        MultilineInput(
+            name="template",
+            display_name="Template",
+            info="The template to use for formatting the data. "
+            "It can contain the keys {text}, {sender} or any other key in the message data.",
+            value="{sender_name}: {text}",
+            advanced=True,
+            show=False,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Message", name="messages_text", method="retrieve_messages_as_text", dynamic=True),
+        Output(display_name="Dataframe", name="dataframe", method="retrieve_messages_dataframe", dynamic=True),
+    ]
+
+    def update_outputs(self, frontend_node: dict, field_name: str, field_value: Any) -> dict:
+        """Dynamically show only the relevant output based on the selected output type."""
+        if field_name == "mode":
+            # Start with empty outputs
+            frontend_node["outputs"] = []
+            if field_value == "Store":
+                frontend_node["outputs"] = [
+                    Output(
+                        display_name="Stored Messages",
+                        name="stored_messages",
+                        method="store_message",
+                        hidden=True,
+                        dynamic=True,
+                    )
+                ]
+            if field_value == "Retrieve":
+                frontend_node["outputs"] = [
+                    Output(
+                        display_name="Messages", name="messages_text", method="retrieve_messages_as_text", dynamic=True
+                    ),
+                    Output(
+                        display_name="Dataframe", name="dataframe", method="retrieve_messages_dataframe", dynamic=True
+                    ),
+                ]
+        return frontend_node
+
+    async def store_message(self) -> Message:
+        message = Message(text=self.message) if isinstance(self.message, str) else self.message
+
+        message.context_id = self.context_id or message.context_id
+        message.session_id = self.session_id or message.session_id
+        message.sender = self.sender or message.sender or MESSAGE_SENDER_AI
+        message.sender_name = self.sender_name or message.sender_name or MESSAGE_SENDER_NAME_AI
+
+        stored_messages: list[Message] = []
+
+        if self.memory:
+            self.memory.context_id = message.context_id
+            self.memory.session_id = message.session_id
+            lc_message = message.to_lc_message()
+            await self.memory.aadd_messages([lc_message])
+
+            stored_messages = await self.memory.aget_messages() or []
+
+            stored_messages = [Message.from_lc_message(m) for m in stored_messages] if stored_messages else []
+
+            if message.sender:
+                stored_messages = [m for m in stored_messages if m.sender == message.sender]
+        else:
+            await astore_message(message, flow_id=self.graph.flow_id)
+            stored_messages = (
+                await aget_messages(
+                    session_id=message.session_id,
+                    context_id=message.context_id,
+                    sender_name=message.sender_name,
+                    sender=message.sender,
+                )
+                or []
+            )
+
+        if not stored_messages:
+            msg = "No messages were stored. Please ensure that the session ID and sender are properly set."
+            raise ValueError(msg)
+
+        stored_message = stored_messages[0]
+        self.status = stored_message
+        return stored_message
+
+    async def retrieve_messages(self) -> Data:
+        sender_type = self.sender_type
+        sender_name = self.sender_name
+        session_id = self.session_id
+        context_id = self.context_id
+        n_messages = self.n_messages
+        order = "DESC" if self.order == "Descending" else "ASC"
+
+        if sender_type == "Machine and User":
+            sender_type = None
+
+        if self.memory and not hasattr(self.memory, "aget_messages"):
+            memory_name = type(self.memory).__name__
+            err_msg = f"External Memory object ({memory_name}) must have 'aget_messages' method."
+            raise AttributeError(err_msg)
+        # Check if n_messages is None or 0
+        if n_messages == 0:
+            stored = []
+        elif self.memory:
+            # override session_id
+            self.memory.session_id = session_id
+            self.memory.context_id = context_id
+
+            stored = await self.memory.aget_messages()
+            # langchain memories are supposed to return messages in ascending order
+
+            if n_messages:
+                stored = stored[-n_messages:]  # Get last N messages first
+
+            if order == "DESC":
+                stored = stored[::-1]  # Then reverse if needed
+
+            stored = [Message.from_lc_message(m) for m in stored]
+            if sender_type:
+                expected_type = MESSAGE_SENDER_AI if sender_type == MESSAGE_SENDER_AI else MESSAGE_SENDER_USER
+                stored = [m for m in stored if m.type == expected_type]
+        else:
+            # For internal memory, we always fetch the last N messages by ordering by DESC
+            stored = await aget_messages(
+                sender=sender_type,
+                sender_name=sender_name,
+                session_id=session_id,
+                context_id=context_id,
+                limit=10000,
+                order=order,
+            )
+            if n_messages:
+                stored = stored[-n_messages:]  # Get last N messages
+
+        # self.status = stored
+        return cast("Data", stored)
+
+    async def retrieve_messages_as_text(self) -> Message:
+        stored_text = data_to_text(self.template, await self.retrieve_messages())
+        # self.status = stored_text
+        return Message(text=stored_text)
+
+    async def retrieve_messages_dataframe(self) -> DataFrame:
+        """Convert the retrieved messages into a DataFrame.
+
+        Returns:
+            DataFrame: A DataFrame containing the message data.
+        """
+        messages = await self.retrieve_messages()
+        return DataFrame(messages)
+
+    def update_build_config(
+        self,
+        build_config: dotdict,
+        field_value: Any,  # noqa: ARG002
+        field_name: str | None = None,  # noqa: ARG002
+    ) -> dotdict:
+        return set_current_fields(
+            build_config=build_config,
+            action_fields=self.mode_config,
+            selected_action=build_config["mode"]["value"],
+            default_fields=self.default_keys,
+            func=set_field_display,
+        )

lfx/components/models_and_agents/prompt.py

@@ -0,0 +1,67 @@
+from lfx.base.prompts.api_utils import process_prompt_template
+from lfx.custom.custom_component.component import Component
+from lfx.inputs.inputs import DefaultPromptField
+from lfx.io import MessageTextInput, Output, PromptInput
+from lfx.schema.message import Message
+from lfx.template.utils import update_template_values
+
+
+class PromptComponent(Component):
+    display_name: str = "Prompt Template"
+    description: str = "Create a prompt template with dynamic variables."
+    documentation: str = "https://docs.langflow.org/components-prompts"
+    icon = "braces"
+    trace_type = "prompt"
+    name = "Prompt Template"
+    priority = 0  # Set priority to 0 to make it appear first
+
+    inputs = [
+        PromptInput(name="template", display_name="Template"),
+        MessageTextInput(
+            name="tool_placeholder",
+            display_name="Tool Placeholder",
+            tool_mode=True,
+            advanced=True,
+            info="A placeholder input for tool mode.",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Prompt", name="prompt", method="build_prompt"),
+    ]
+
+    async def build_prompt(self) -> Message:
+        prompt = Message.from_template(**self._attributes)
+        self.status = prompt.text
+        return prompt
+
+    def _update_template(self, frontend_node: dict):
+        prompt_template = frontend_node["template"]["template"]["value"]
+        custom_fields = frontend_node["custom_fields"]
+        frontend_node_template = frontend_node["template"]
+        _ = process_prompt_template(
+            template=prompt_template,
+            name="template",
+            custom_fields=custom_fields,
+            frontend_node_template=frontend_node_template,
+        )
+        return frontend_node
+
+    async def update_frontend_node(self, new_frontend_node: dict, current_frontend_node: dict):
+        """This function is called after the code validation is done."""
+        frontend_node = await super().update_frontend_node(new_frontend_node, current_frontend_node)
+        template = frontend_node["template"]["template"]["value"]
+        # Kept it duplicated for backwards compatibility
+        _ = process_prompt_template(
+            template=template,
+            name="template",
+            custom_fields=frontend_node["custom_fields"],
+            frontend_node_template=frontend_node["template"],
+        )
+        # Now that template is updated, we need to grab any values that were set in the current_frontend_node
+        # and update the frontend_node with those values
+        update_template_values(new_template=frontend_node, previous_template=current_frontend_node["template"])
+        return frontend_node
+
+    def _get_fallback_input(self, **kwargs):
+        return DefaultPromptField(**kwargs)

lfx/components/mongodb/__init__.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from .mongodb_atlas import MongoVectorStoreComponent
+
+_dynamic_imports = {
+    "MongoVectorStoreComponent": "mongodb_atlas",
+}
+
+__all__ = [
+    "MongoVectorStoreComponent",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import MongoDB components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

lfx/components/mongodb/mongodb_atlas.py

@@ -0,0 +1,213 @@
+import tempfile
+import time
+
+import certifi
+from langchain_community.vectorstores import MongoDBAtlasVectorSearch
+from pymongo.collection import Collection
+from pymongo.operations import SearchIndexModel
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.io import BoolInput, DropdownInput, HandleInput, IntInput, SecretStrInput, StrInput
+from lfx.schema.data import Data
+
+
+class MongoVectorStoreComponent(LCVectorStoreComponent):
+    display_name = "MongoDB Atlas"
+    description = "MongoDB Atlas Vector Store with search capabilities"
+    name = "MongoDBAtlasVector"
+    icon = "MongoDB"
+    INSERT_MODES = ["append", "overwrite"]
+    SIMILARITY_OPTIONS = ["cosine", "euclidean", "dotProduct"]
+    QUANTIZATION_OPTIONS = ["scalar", "binary"]
+    inputs = [
+        SecretStrInput(name="mongodb_atlas_cluster_uri", display_name="MongoDB Atlas Cluster URI", required=True),
+        BoolInput(name="enable_mtls", display_name="Enable mTLS", value=False, advanced=True, required=True),
+        SecretStrInput(
+            name="mongodb_atlas_client_cert",
+            display_name="MongoDB Atlas Combined Client Certificate",
+            required=False,
+            info="Client Certificate combined with the private key in the following format:\n "
+            "-----BEGIN PRIVATE KEY-----\n...\n -----END PRIVATE KEY-----\n-----BEGIN CERTIFICATE-----\n"
+            "...\n-----END CERTIFICATE-----\n",
+        ),
+        StrInput(name="db_name", display_name="Database Name", required=True),
+        StrInput(name="collection_name", display_name="Collection Name", required=True),
+        StrInput(
+            name="index_name",
+            display_name="Index Name",
+            required=True,
+            info="The name of Atlas Search index, it should be a Vector Search.",
+        ),
+        *LCVectorStoreComponent.inputs,
+        DropdownInput(
+            name="insert_mode",
+            display_name="Insert Mode",
+            options=INSERT_MODES,
+            value=INSERT_MODES[0],
+            info="How to insert new documents into the collection.",
+            advanced=True,
+        ),
+        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            value=4,
+            advanced=True,
+        ),
+        StrInput(
+            name="index_field",
+            display_name="Index Field",
+            advanced=True,
+            required=True,
+            info="The field to index.",
+            value="embedding",
+        ),
+        StrInput(
+            name="filter_field", display_name="Filter Field", advanced=True, info="The field to filter the index."
+        ),
+        IntInput(
+            name="number_dimensions",
+            display_name="Number of Dimensions",
+            info="Embedding Context Length.",
+            value=1536,
+            advanced=True,
+            required=True,
+        ),
+        DropdownInput(
+            name="similarity",
+            display_name="Similarity",
+            options=SIMILARITY_OPTIONS,
+            value=SIMILARITY_OPTIONS[0],
+            info="The method used to measure the similarity between vectors.",
+            advanced=True,
+        ),
+        DropdownInput(
+            name="quantization",
+            display_name="Quantization",
+            options=QUANTIZATION_OPTIONS,
+            value=None,
+            info="Quantization reduces memory costs converting 32-bit floats to smaller data types",
+            advanced=True,
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> MongoDBAtlasVectorSearch:
+        try:
+            from pymongo import MongoClient
+        except ImportError as e:
+            msg = "Please install pymongo to use MongoDB Atlas Vector Store"
+            raise ImportError(msg) from e
+
+        # Create temporary files for the client certificate
+        if self.enable_mtls:
+            client_cert_path = None
+            try:
+                client_cert = self.mongodb_atlas_client_cert.replace(" ", "\n")
+                client_cert = client_cert.replace("-----BEGIN\nPRIVATE\nKEY-----", "-----BEGIN PRIVATE KEY-----")
+                client_cert = client_cert.replace(
+                    "-----END\nPRIVATE\nKEY-----\n-----BEGIN\nCERTIFICATE-----",
+                    "-----END PRIVATE KEY-----\n-----BEGIN CERTIFICATE-----",
+                )
+                client_cert = client_cert.replace("-----END\nCERTIFICATE-----", "-----END CERTIFICATE-----")
+                with tempfile.NamedTemporaryFile(delete=False) as client_cert_file:
+                    client_cert_file.write(client_cert.encode("utf-8"))
+                    client_cert_path = client_cert_file.name
+
+            except Exception as e:
+                msg = f"Failed to write certificate to temporary file: {e}"
+                raise ValueError(msg) from e
+
+        try:
+            mongo_client: MongoClient = (
+                MongoClient(
+                    self.mongodb_atlas_cluster_uri,
+                    tls=True,
+                    tlsCertificateKeyFile=client_cert_path,
+                    tlsCAFile=certifi.where(),
+                )
+                if self.enable_mtls
+                else MongoClient(self.mongodb_atlas_cluster_uri)
+            )
+
+            collection = mongo_client[self.db_name][self.collection_name]
+
+        except Exception as e:
+            msg = f"Failed to connect to MongoDB Atlas: {e}"
+            raise ValueError(msg) from e
+
+        # Convert DataFrame to Data if needed using parent's method
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in self.ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        if documents:
+            self.__insert_mode(collection)
+
+            return MongoDBAtlasVectorSearch.from_documents(
+                documents=documents, embedding=self.embedding, collection=collection, index_name=self.index_name
+            )
+        return MongoDBAtlasVectorSearch(embedding=self.embedding, collection=collection, index_name=self.index_name)
+
+    def search_documents(self) -> list[Data]:
+        from bson.objectid import ObjectId
+
+        vector_store = self.build_vector_store()
+
+        self.verify_search_index(vector_store._collection)
+
+        if self.search_query and isinstance(self.search_query, str):
+            docs = vector_store.similarity_search(
+                query=self.search_query,
+                k=self.number_of_results,
+            )
+            for doc in docs:
+                doc.metadata = {
+                    key: str(value) if isinstance(value, ObjectId) else value for key, value in doc.metadata.items()
+                }
+
+            data = docs_to_data(docs)
+            self.status = data
+            return data
+        return []
+
+    def __insert_mode(self, collection: Collection) -> None:
+        if self.insert_mode == "overwrite":
+            collection.delete_many({})  # Delete all documents while preserving collection structure
+
+    def verify_search_index(self, collection: Collection) -> None:
+        """Verify if the search index exists, if not, create it.
+
+        Args:
+            collection (Collection): The collection to verify the search index on.
+        """
+        indexes = collection.list_search_indexes()
+
+        index_names_types = {idx["name"]: idx["type"] for idx in indexes}
+        index_names = list(index_names_types.keys())
+        index_type = index_names_types.get(self.index_name)
+        if self.index_name not in index_names and index_type != "vectorSearch":
+            collection.create_search_index(self.__create_index_definition())
+
+            time.sleep(20)  # Give some time for index to be ready
+
+    def __create_index_definition(self) -> SearchIndexModel:
+        fields = [
+            {
+                "type": "vector",
+                "path": self.index_field,
+                "numDimensions": self.number_dimensions,
+                "similarity": self.similarity,
+                "quantization": self.quantization,
+            }
+        ]
+        if self.filter_field:
+            fields.append({"type": "filter", "path": self.filter_field})
+        return SearchIndexModel(definition={"fields": fields}, name=self.index_name, type="vectorSearch")

lfx/components/needle/__init__.py

@@ -0,0 +1,3 @@
+from .needle import NeedleComponent
+
+__all__ = ["NeedleComponent"]

lfx/components/needle/needle.py

@@ -0,0 +1,104 @@
+from langchain_community.retrievers.needle import NeedleRetriever
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import IntInput, MessageTextInput, Output, SecretStrInput
+from lfx.schema.message import Message
+from lfx.utils.constants import MESSAGE_SENDER_AI
+
+
+class NeedleComponent(Component):
+    display_name = "Needle Retriever"
+    description = "A retriever that uses the Needle API to search collections."
+    documentation = "https://docs.needle-ai.com"
+    icon = "Needle"
+    name = "needle"
+
+    inputs = [
+        SecretStrInput(
+            name="needle_api_key",
+            display_name="Needle API Key",
+            info="Your Needle API key.",
+            required=True,
+        ),
+        MessageTextInput(
+            name="collection_id",
+            display_name="Collection ID",
+            info="The ID of the Needle collection.",
+            required=True,
+        ),
+        MessageTextInput(
+            name="query",
+            display_name="User Query",
+            info="Enter your question here. In tool mode, you can also specify top_k parameter (min: 20).",
+            required=True,
+            tool_mode=True,
+        ),
+        IntInput(
+            name="top_k",
+            display_name="Top K Results",
+            info="Number of search results to return (min: 20).",
+            value=20,
+            required=True,
+        ),
+    ]
+
+    outputs = [Output(display_name="Result", name="result", type_="Message", method="run")]
+
+    def run(self) -> Message:
+        # Extract query and top_k
+        query_input = self.query
+        actual_query = query_input.get("query", "") if isinstance(query_input, dict) else query_input
+
+        # Parse top_k from tool input or use default, always enforcing minimum of 20
+        try:
+            if isinstance(query_input, dict) and "top_k" in query_input:
+                agent_top_k = query_input.get("top_k")
+                # Check if agent_top_k is not None before converting to int
+                top_k = max(20, int(agent_top_k)) if agent_top_k is not None else max(20, self.top_k)
+            else:
+                top_k = max(20, self.top_k)
+        except (ValueError, TypeError):
+            top_k = max(20, self.top_k)
+
+        # Validate required inputs
+        if not self.needle_api_key or not self.needle_api_key.strip():
+            error_msg = "The Needle API key cannot be empty."
+            raise ValueError(error_msg)
+        if not self.collection_id or not self.collection_id.strip():
+            error_msg = "The Collection ID cannot be empty."
+            raise ValueError(error_msg)
+        if not actual_query or not actual_query.strip():
+            error_msg = "The query cannot be empty."
+            raise ValueError(error_msg)
+
+        try:
+            # Initialize the retriever and get documents
+            retriever = NeedleRetriever(
+                needle_api_key=self.needle_api_key,
+                collection_id=self.collection_id,
+                top_k=top_k,
+            )
+
+            docs = retriever.get_relevant_documents(actual_query)
+
+            # Format the response
+            if not docs:
+                text_content = "No relevant documents found for the query."
+            else:
+                context = "\n\n".join([f"Document {i + 1}:\n{doc.page_content}" for i, doc in enumerate(docs)])
+                text_content = f"Question: {actual_query}\n\nContext:\n{context}"
+
+            # Return formatted message
+            return Message(
+                text=text_content,
+                type="assistant",
+                sender=MESSAGE_SENDER_AI,
+                additional_kwargs={
+                    "source_documents": [{"page_content": doc.page_content, "metadata": doc.metadata} for doc in docs],
+                    "top_k_used": top_k,
+                },
+            )
+
+        except Exception as e:
+            error_msg = f"Error processing query: {e!s}"
+            raise ValueError(error_msg) from e