lfx-nightly 0.1.12.dev13__py3-none-any.whl → 0.1.12.dev15__py3-none-any.whl

lfx/components/knowledge_bases/retrieval.py

@@ -0,0 +1,256 @@
+import json
+from pathlib import Path
+from typing import Any
+
+from cryptography.fernet import InvalidToken
+from langchain_chroma import Chroma
+from langflow.services.auth.utils import decrypt_api_key
+from langflow.services.database.models.user.crud import get_user_by_id
+from pydantic import SecretStr
+
+from lfx.base.knowledge_bases.knowledge_base_utils import get_knowledge_bases
+from lfx.custom import Component
+from lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SecretStrInput
+from lfx.log.logger import logger
+from lfx.schema.data import Data
+from lfx.schema.dataframe import DataFrame
+from lfx.services.deps import get_settings_service, session_scope
+
+settings = get_settings_service().settings
+knowledge_directory = settings.knowledge_bases_dir
+if not knowledge_directory:
+    msg = "Knowledge bases directory is not set in the settings."
+    raise ValueError(msg)
+KNOWLEDGE_BASES_ROOT_PATH = Path(knowledge_directory).expanduser()
+
+
+class KnowledgeRetrievalComponent(Component):
+    display_name = "Knowledge Retrieval"
+    description = "Search and retrieve data from knowledge."
+    icon = "download"
+    name = "KnowledgeRetrieval"
+
+    inputs = [
+        DropdownInput(
+            name="knowledge_base",
+            display_name="Knowledge",
+            info="Select the knowledge to load data from.",
+            required=True,
+            options=[],
+            refresh_button=True,
+            real_time_refresh=True,
+        ),
+        SecretStrInput(
+            name="api_key",
+            display_name="Embedding Provider API Key",
+            info="API key for the embedding provider to generate embeddings.",
+            advanced=True,
+            required=False,
+        ),
+        MessageTextInput(
+            name="search_query",
+            display_name="Search Query",
+            info="Optional search query to filter knowledge base data.",
+            tool_mode=True,
+        ),
+        IntInput(
+            name="top_k",
+            display_name="Top K Results",
+            info="Number of top results to return from the knowledge base.",
+            value=5,
+            advanced=True,
+            required=False,
+        ),
+        BoolInput(
+            name="include_metadata",
+            display_name="Include Metadata",
+            info="Whether to include all metadata in the output. If false, only content is returned.",
+            value=True,
+            advanced=False,
+        ),
+        BoolInput(
+            name="include_embeddings",
+            display_name="Include Embeddings",
+            info="Whether to include embeddings in the output. Only applicable if 'Include Metadata' is enabled.",
+            value=False,
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(
+            name="retrieve_data",
+            display_name="Results",
+            method="retrieve_data",
+            info="Returns the data from the selected knowledge base.",
+        ),
+    ]
+
+    async def update_build_config(self, build_config, field_value, field_name=None):  # noqa: ARG002
+        if field_name == "knowledge_base":
+            # Update the knowledge base options dynamically
+            build_config["knowledge_base"]["options"] = await get_knowledge_bases(
+                KNOWLEDGE_BASES_ROOT_PATH,
+                user_id=self.user_id,  # Use the user_id from the component context
+            )
+
+            # If the selected knowledge base is not available, reset it
+            if build_config["knowledge_base"]["value"] not in build_config["knowledge_base"]["options"]:
+                build_config["knowledge_base"]["value"] = None
+
+        return build_config
+
+    def _get_kb_metadata(self, kb_path: Path) -> dict:
+        """Load and process knowledge base metadata."""
+        metadata: dict[str, Any] = {}
+        metadata_file = kb_path / "embedding_metadata.json"
+        if not metadata_file.exists():
+            logger.warning(f"Embedding metadata file not found at {metadata_file}")
+            return metadata
+
+        try:
+            with metadata_file.open("r", encoding="utf-8") as f:
+                metadata = json.load(f)
+        except json.JSONDecodeError:
+            logger.error(f"Error decoding JSON from {metadata_file}")
+            return {}
+
+        # Decrypt API key if it exists
+        if "api_key" in metadata and metadata.get("api_key"):
+            settings_service = get_settings_service()
+            try:
+                decrypted_key = decrypt_api_key(metadata["api_key"], settings_service)
+                metadata["api_key"] = decrypted_key
+            except (InvalidToken, TypeError, ValueError) as e:
+                logger.error(f"Could not decrypt API key. Please provide it manually. Error: {e}")
+                metadata["api_key"] = None
+        return metadata
+
+    def _build_embeddings(self, metadata: dict):
+        """Build embedding model from metadata."""
+        runtime_api_key = self.api_key.get_secret_value() if isinstance(self.api_key, SecretStr) else self.api_key
+        provider = metadata.get("embedding_provider")
+        model = metadata.get("embedding_model")
+        api_key = runtime_api_key or metadata.get("api_key")
+        chunk_size = metadata.get("chunk_size")
+
+        # Handle various providers
+        if provider == "OpenAI":
+            from langchain_openai import OpenAIEmbeddings
+
+            if not api_key:
+                msg = "OpenAI API key is required. Provide it in the component's advanced settings."
+                raise ValueError(msg)
+            return OpenAIEmbeddings(
+                model=model,
+                api_key=api_key,
+                chunk_size=chunk_size,
+            )
+        if provider == "HuggingFace":
+            from langchain_huggingface import HuggingFaceEmbeddings
+
+            return HuggingFaceEmbeddings(
+                model=model,
+            )
+        if provider == "Cohere":
+            from langchain_cohere import CohereEmbeddings
+
+            if not api_key:
+                msg = "Cohere API key is required when using Cohere provider"
+                raise ValueError(msg)
+            return CohereEmbeddings(
+                model=model,
+                cohere_api_key=api_key,
+            )
+        if provider == "Custom":
+            # For custom embedding models, we would need additional configuration
+            msg = "Custom embedding models not yet supported"
+            raise NotImplementedError(msg)
+        # Add other providers here if they become supported in ingest
+        msg = f"Embedding provider '{provider}' is not supported for retrieval."
+        raise NotImplementedError(msg)
+
+    async def retrieve_data(self) -> DataFrame:
+        """Retrieve data from the selected knowledge base by reading the Chroma collection.
+
+        Returns:
+            A DataFrame containing the data rows from the knowledge base.
+        """
+        # Get the current user
+        async with session_scope() as db:
+            if not self.user_id:
+                msg = "User ID is required for fetching Knowledge Base data."
+                raise ValueError(msg)
+            current_user = await get_user_by_id(db, self.user_id)
+            if not current_user:
+                msg = f"User with ID {self.user_id} not found."
+                raise ValueError(msg)
+            kb_user = current_user.username
+        kb_path = KNOWLEDGE_BASES_ROOT_PATH / kb_user / self.knowledge_base
+
+        metadata = self._get_kb_metadata(kb_path)
+        if not metadata:
+            msg = f"Metadata not found for knowledge base: {self.knowledge_base}. Ensure it has been indexed."
+            raise ValueError(msg)
+
+        # Build the embedder for the knowledge base
+        embedding_function = self._build_embeddings(metadata)
+
+        # Load vector store
+        chroma = Chroma(
+            persist_directory=str(kb_path),
+            embedding_function=embedding_function,
+            collection_name=self.knowledge_base,
+        )
+
+        # If a search query is provided, perform a similarity search
+        if self.search_query:
+            # Use the search query to perform a similarity search
+            logger.info(f"Performing similarity search with query: {self.search_query}")
+            results = chroma.similarity_search_with_score(
+                query=self.search_query or "",
+                k=self.top_k,
+            )
+        else:
+            results = chroma.similarity_search(
+                query=self.search_query or "",
+                k=self.top_k,
+            )
+
+            # For each result, make it a tuple to match the expected output format
+            results = [(doc, 0) for doc in results]  # Assign a dummy score of 0
+
+        # If include_embeddings is enabled, get embeddings for the results
+        id_to_embedding = {}
+        if self.include_embeddings and results:
+            doc_ids = [doc[0].metadata.get("_id") for doc in results if doc[0].metadata.get("_id")]
+
+            # Only proceed if we have valid document IDs
+            if doc_ids:
+                # Access underlying client to get embeddings
+                collection = chroma._client.get_collection(name=self.knowledge_base)
+                embeddings_result = collection.get(where={"_id": {"$in": doc_ids}}, include=["metadatas", "embeddings"])
+
+                # Create a mapping from document ID to embedding
+                for i, metadata in enumerate(embeddings_result.get("metadatas", [])):
+                    if metadata and "_id" in metadata:
+                        id_to_embedding[metadata["_id"]] = embeddings_result["embeddings"][i]
+
+        # Build output data based on include_metadata setting
+        data_list = []
+        for doc in results:
+            kwargs = {
+                "content": doc[0].page_content,
+            }
+            if self.search_query:
+                kwargs["_score"] = -1 * doc[1]
+            if self.include_metadata:
+                # Include all metadata, embeddings, and content
+                kwargs.update(doc[0].metadata)
+            if self.include_embeddings:
+                kwargs["_embeddings"] = id_to_embedding.get(doc[0].metadata.get("_id"))
+
+            data_list.append(Data(**kwargs))
+
+        # Return the DataFrame containing the data
+        return DataFrame(data=data_list)

lfx/components/langchain_utilities/langchain_hub.py

@@ -19,7 +19,7 @@ class LangChainHubPromptComponent(Component):
     inputs = [
         SecretStrInput(
             name="langchain_api_key",
-            display_name="Your LangChain API Key",
+            display_name="LangChain API Key",
             info="The LangChain API Key to use.",
             required=True,
         ),

lfx/components/langwatch/langwatch.py

@@ -41,7 +41,7 @@ class LangWatchComponent(Component):
         ),
         SecretStrInput(
             name="api_key",
-            display_name="API Key",
+            display_name="LangWatch API Key",
             required=True,
             info="Enter your LangWatch API key.",
         ),

lfx/components/logic/conditional_router.py

@@ -129,20 +129,55 @@ class ConditionalRouterComponent(Component):
         return False
 
     def iterate_and_stop_once(self, route_to_stop: str):
+        """Handles cycle iteration counting and branch exclusion.
+
+        Uses two complementary mechanisms:
+        1. stop() - ACTIVE/INACTIVE state for cycle management (gets reset each iteration)
+        2. exclude_branch_conditionally() - Persistent exclusion for conditional routing
+
+        When max_iterations is reached, breaks the cycle by allowing the default_route to execute.
+        """
         if not self.__iteration_updated:
             self.update_ctx({f"{self._id}_iteration": self.ctx.get(f"{self._id}_iteration", 0) + 1})
             self.__iteration_updated = True
-            if self.ctx.get(f"{self._id}_iteration", 0) >= self.max_iterations and route_to_stop == self.default_route:
+            current_iteration = self.ctx.get(f"{self._id}_iteration", 0)
+
+            # Check if max iterations reached and we're trying to stop the default route
+            if current_iteration >= self.max_iterations and route_to_stop == self.default_route:
+                # Clear ALL conditional exclusions to allow default route to execute
+                if self._id in self.graph.conditional_exclusion_sources:
+                    previous_exclusions = self.graph.conditional_exclusion_sources[self._id]
+                    self.graph.conditionally_excluded_vertices -= previous_exclusions
+                    del self.graph.conditional_exclusion_sources[self._id]
+
+                # Switch which route to stop - stop the NON-default route to break the cycle
                 route_to_stop = "true_result" if route_to_stop == "false_result" else "false_result"
+
+                # Call stop to break the cycle
+                self.stop(route_to_stop)
+                # Don't apply conditional exclusion when breaking cycle
+                return
+
+            # Normal case: Use BOTH mechanisms
+            # 1. stop() for cycle management (marks INACTIVE, updates run manager, gets reset)
             self.stop(route_to_stop)
 
+            # 2. Conditional exclusion for persistent routing (doesn't get reset except by this router)
+            self.graph.exclude_branch_conditionally(self._id, output_name=route_to_stop)
+
     def true_response(self) -> Message:
         result = self.evaluate_condition(
             self.input_text, self.match_text, self.operator, case_sensitive=self.case_sensitive
         )
-        if result:
+
+        # Check if we should force output due to max_iterations on default route
+        current_iteration = self.ctx.get(f"{self._id}_iteration", 0)
+        force_output = current_iteration >= self.max_iterations and self.default_route == "true_result"
+
+        if result or force_output:
             self.status = self.true_case_message
-            self.iterate_and_stop_once("false_result")
+            if not force_output:  # Only stop the other branch if not forcing due to max iterations
+                self.iterate_and_stop_once("false_result")
             return self.true_case_message
         self.iterate_and_stop_once("true_result")
         return Message(content="")
@@ -151,10 +186,12 @@ class ConditionalRouterComponent(Component):
         result = self.evaluate_condition(
             self.input_text, self.match_text, self.operator, case_sensitive=self.case_sensitive
         )
+
         if not result:
             self.status = self.false_case_message
             self.iterate_and_stop_once("true_result")
             return self.false_case_message
+
         self.iterate_and_stop_once("false_result")
         return Message(content="")
 

lfx/components/logic/data_conditional_router.py

@@ -12,6 +12,7 @@ class DataConditionalRouterComponent(Component):
     icon = "split"
     name = "DataConditionalRouter"
     legacy = True
+    replacement = ["logic.ConditionalRouter"]
 
     inputs = [
         DataInput(

lfx/components/logic/flow_tool.py

@@ -14,12 +14,13 @@ from lfx.schema.dotdict import dotdict
 
 
 class FlowToolComponent(LCToolComponent):
-    display_name = "Flow as Tool [Deprecated]"
+    display_name = "Flow as Tool"
     description = "Construct a Tool from a function that runs the loaded Flow."
     field_order = ["flow_name", "name", "description", "return_direct"]
     trace_type = "tool"
     name = "FlowTool"
     legacy: bool = True
+    replacement = ["logic.RunFlow"]
     icon = "hammer"
 
     async def get_flow_names(self) -> list[str]:

lfx/components/logic/pass_message.py

@@ -10,6 +10,7 @@ class PassMessageComponent(Component):
     name = "Pass"
     icon = "arrow-right"
     legacy: bool = True
+    replacement = ["logic.ConditionalRouter"]
 
     inputs = [
         MessageInput(

lfx/components/logic/sub_flow.py

@@ -12,10 +12,11 @@ from lfx.schema.dotdict import dotdict
 
 
 class SubFlowComponent(Component):
-    display_name = "Sub Flow [Deprecated]"
+    display_name = "Sub Flow"
     description = "Generates a Component from a Flow, with all of its inputs, and "
     name = "SubFlow"
     legacy: bool = True
+    replacement = ["logic.RunFlow"]
     icon = "Workflow"
 
     async def get_flow_names(self) -> list[str]:

lfx/components/milvus/milvus.py

@@ -31,7 +31,7 @@ class MilvusVectorStoreComponent(LCVectorStoreComponent):
         ),
         SecretStrInput(
             name="password",
-            display_name="Token",
+            display_name="Milvus Token",
             value="",
             info="Ignore this field if no token is required to make connection.",
         ),

lfx/components/olivya/olivya.py

@@ -18,7 +18,7 @@ class OlivyaComponent(Component):
     inputs = [
         MessageTextInput(
             name="api_key",
-            display_name="API Key",
+            display_name="Olivya API Key",
             info="Your API key for authentication",
             value="",
             required=True,

lfx/components/processing/alter_metadata.py

@@ -11,6 +11,7 @@ class AlterMetadataComponent(Component):
     icon = "merge"
     name = "AlterMetadata"
     legacy = True
+    replacement = ["processing.DataOperations"]
 
     inputs = [
         HandleInput(

lfx/components/processing/combine_text.py

@@ -9,6 +9,7 @@ class CombineTextComponent(Component):
     icon = "merge"
     name = "CombineText"
     legacy: bool = True
+    replacement = ["processing.DataOperations"]
 
     inputs = [
         MessageTextInput(

lfx/components/processing/create_data.py

@@ -14,6 +14,7 @@ class CreateDataComponent(Component):
     name: str = "CreateData"
     MAX_FIELDS = 15  # Define a constant for maximum number of fields
     legacy = True
+    replacement = ["processing.DataOperations"]
     icon = "ListFilter"
 
     inputs = [

lfx/components/processing/data_to_dataframe.py

@@ -14,6 +14,7 @@ class DataToDataFrameComponent(Component):
     icon = "table"
     name = "DataToDataFrame"
     legacy = True
+    replacement = ["processing.DataOperations", "processing.TypeConverterComponent"]
 
     inputs = [
         DataInput(

lfx/components/processing/extract_key.py

@@ -12,6 +12,7 @@ class ExtractDataKeyComponent(Component):
     icon = "key"
     name = "ExtractaKey"
     legacy = True
+    replacement = ["processing.DataOperations"]
 
     inputs = [
         DataInput(

lfx/components/processing/filter_data.py

@@ -10,6 +10,7 @@ class FilterDataComponent(Component):
     beta = True
     name = "FilterData"
     legacy = True
+    replacement = ["processing.DataOperations"]
 
     inputs = [
         DataInput(

lfx/components/processing/filter_data_values.py

@@ -15,6 +15,7 @@ class DataFilterComponent(Component):
     beta = True
     name = "FilterDataValues"
     legacy = True
+    replacement = ["processing.DataOperations"]
 
     inputs = [
         DataInput(name="input_data", display_name="Input Data", info="The list of data items to filter.", is_list=True),

lfx/components/processing/json_cleaner.py

@@ -15,6 +15,7 @@ class JSONCleaner(Component):
         "so that they are fully compliant with the JSON spec."
     )
     legacy = True
+    replacement = ["processing.ParserComponent"]
     inputs = [
         MessageTextInput(
             name="json_str", display_name="JSON String", info="The JSON string to be cleaned.", required=True

lfx/components/processing/merge_data.py

@@ -20,6 +20,7 @@ class MergeDataComponent(Component):
     icon = "merge"
     MIN_INPUTS_REQUIRED = 2
     legacy = True
+    replacement = ["processing.DataOperations"]
 
     inputs = [
         DataInput(name="data_inputs", display_name="Data Inputs", info="Data to combine", is_list=True, required=True),

lfx/components/processing/message_to_data.py

@@ -11,6 +11,7 @@ class MessageToDataComponent(Component):
     beta = True
     name = "MessagetoData"
     legacy = True
+    replacement = ["processing.TypeConverterComponent"]
 
     inputs = [
         MessageInput(

lfx/components/processing/parse_data.py

@@ -11,6 +11,7 @@ class ParseDataComponent(Component):
     icon = "message-square"
     name = "ParseData"
     legacy = True
+    replacement = ["processing.DataOperations", "processing.TypeConverterComponent"]
     metadata = {
         "legacy_name": "Parse Data",
     }

lfx/components/processing/parse_dataframe.py

@@ -12,6 +12,7 @@ class ParseDataFrameComponent(Component):
     icon = "braces"
     name = "ParseDataFrame"
     legacy = True
+    replacement = ["processing.DataFrameOperations", "processing.TypeConverterComponent"]
 
     inputs = [
         DataFrameInput(name="df", display_name="DataFrame", info="The DataFrame to convert to text rows."),

lfx/components/processing/parse_json_data.py

@@ -18,6 +18,7 @@ class ParseJSONDataComponent(Component):
     icon = "braces"
     name = "ParseJSONData"
     legacy: bool = True
+    replacement = ["processing.ParserComponent"]
 
     inputs = [
         HandleInput(

lfx/components/processing/regex.py

@@ -11,6 +11,7 @@ class RegexExtractorComponent(Component):
     description = "Extract patterns from text using regular expressions."
     icon = "regex"
     legacy = True
+    replacement = ["processing.ParserComponent"]
 
     inputs = [
         MessageTextInput(

lfx/components/processing/select_data.py

@@ -11,6 +11,7 @@ class SelectDataComponent(Component):
     name: str = "SelectData"
     icon = "prototypes"
     legacy = True
+    replacement = ["processing.DataOperations"]
 
     inputs = [
         DataInput(

lfx/components/processing/structured_output.py

@@ -197,6 +197,10 @@ class StructuredOutputComponent(Component):
             # handle empty or unexpected type case
             msg = "No structured output returned"
             raise ValueError(msg)
-        data_list = [Data(data=output[0])] if len(output) == 1 else [Data(data=item) for item in output]
-
-        return DataFrame(data_list)
+        if len(output) == 1:
+            # For single dictionary, wrap in a list to create DataFrame with one row
+            return DataFrame([output[0]])
+        if len(output) > 1:
+            # Multiple outputs - convert to DataFrame directly
+            return DataFrame(output)
+        return DataFrame()

lfx/components/processing/update_data.py

@@ -21,6 +21,7 @@ class UpdateDataComponent(Component):
     MAX_FIELDS = 15  # Define a constant for maximum number of fields
     icon = "FolderSync"
     legacy = True
+    replacement = ["processing.DataOperations"]
 
     inputs = [
         DataInput(

lfx/components/prototypes/__init__.py

@@ -1,23 +1,24 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 from lfx.components._importing import import_mod
 
-if TYPE_CHECKING:
-    from .python_function import PythonFunctionComponent
+# _dynamic_imports = {
+#     "KnowledgeIngestionComponent": "ingestion",
+#     "KnowledgeRetrievalComponent": "retrieval",
 
+# }
 _dynamic_imports = {
     "PythonFunctionComponent": "python_function",
 }
 
-__all__ = [
-    "PythonFunctionComponent",
-]
+# __all__ = ["KnowledgeIngestionComponent", "KnowledgeRetrievalComponent"]
+__all__ = ["PythonFunctionComponent"]
 
 
 def __getattr__(attr_name: str) -> Any:
-    """Lazily import prototype components on attribute access."""
+    """Lazily import input/output components on attribute access."""
     if attr_name not in _dynamic_imports:
         msg = f"module '{__name__}' has no attribute '{attr_name}'"
         raise AttributeError(msg)

lfx/components/qdrant/qdrant.py

@@ -23,7 +23,7 @@ class QdrantVectorStoreComponent(LCVectorStoreComponent):
         StrInput(name="host", display_name="Host", value="localhost", advanced=True),
         IntInput(name="port", display_name="Port", value=6333, advanced=True),
         IntInput(name="grpc_port", display_name="gRPC Port", value=6334, advanced=True),
-        SecretStrInput(name="api_key", display_name="API Key", advanced=True),
+        SecretStrInput(name="api_key", display_name="Qdrant API Key", advanced=True),
         StrInput(name="prefix", display_name="Prefix", advanced=True),
         IntInput(name="timeout", display_name="Timeout", advanced=True),
         StrInput(name="path", display_name="Path", advanced=True),

lfx/components/redis/redis_chat.py

@@ -23,7 +23,7 @@ class RedisIndexChatMemory(LCChatMemoryComponent):
             name="username", display_name="Username", value="", info="The Redis user name.", advanced=True
         ),
         SecretStrInput(
-            name="password", display_name="Password", value="", info="The password for username.", advanced=True
+            name="password", display_name="Redis Password", value="", info="The password for username.", advanced=True
         ),
         StrInput(name="key_prefix", display_name="Key prefix", info="Key prefix.", advanced=True),
         MessageTextInput(

lfx/components/tools/__init__.py

@@ -9,8 +9,6 @@ from lfx.components._importing import import_mod
 
 if TYPE_CHECKING:
     from .calculator import CalculatorToolComponent
-    from .google_search_api import GoogleSearchAPIComponent
-    from .google_serper_api import GoogleSerperAPIComponent
     from .python_code_structured_tool import PythonCodeStructuredTool
     from .python_repl import PythonREPLToolComponent
     from .search_api import SearchAPIComponent
@@ -23,8 +21,6 @@ if TYPE_CHECKING:
 
 _dynamic_imports = {
     "CalculatorToolComponent": "calculator",
-    "GoogleSearchAPIComponent": "google_search_api",
-    "GoogleSerperAPIComponent": "google_serper_api",
     "PythonCodeStructuredTool": "python_code_structured_tool",
     "PythonREPLToolComponent": "python_repl",
     "SearchAPIComponent": "search_api",
@@ -38,8 +34,6 @@ _dynamic_imports = {
 
 __all__ = [
     "CalculatorToolComponent",
-    "GoogleSearchAPIComponent",
-    "GoogleSerperAPIComponent",
     "PythonCodeStructuredTool",
     "PythonREPLToolComponent",
     "SearXNGToolComponent",