langflow-base-nightly 0.5.0.dev36__py3-none-any.whl → 0.5.0.dev38__py3-none-any.whl

langflow/components/data/kb_ingest.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import asyncio
+import contextlib
 import hashlib
 import json
 import re
@@ -14,6 +16,7 @@ from cryptography.fernet import InvalidToken
 from langchain_chroma import Chroma
 from loguru import logger
 
+from langflow.base.data.kb_utils import get_knowledge_bases
 from langflow.base.models.openai_constants import OPENAI_EMBEDDING_MODEL_NAMES
 from langflow.custom import Component
 from langflow.io import BoolInput, DataFrameInput, DropdownInput, IntInput, Output, SecretStrInput, StrInput, TableInput
@@ -21,7 +24,8 @@ from langflow.schema.data import Data
 from langflow.schema.dotdict import dotdict  # noqa: TC001
 from langflow.schema.table import EditMode
 from langflow.services.auth.utils import decrypt_api_key, encrypt_api_key
-from langflow.services.deps import get_settings_service
+from langflow.services.database.models.user.crud import get_user_by_id
+from langflow.services.deps import get_settings_service, get_variable_service, session_scope
 
 HUGGINGFACE_MODEL_NAMES = ["sentence-transformers/all-MiniLM-L6-v2", "sentence-transformers/all-mpnet-base-v2"]
 COHERE_MODEL_NAMES = ["embed-english-v3.0", "embed-multilingual-v3.0"]
@@ -43,6 +47,10 @@ class KBIngestionComponent(Component):
     icon = "database"
     name = "KBIngestion"
 
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self._cached_kb_path: Path | None = None
+
     @dataclass
     class NewKnowledgeBaseInput:
         functionality: str = "create"
@@ -76,7 +84,7 @@ class KBIngestionComponent(Component):
                                 display_name="API Key",
                                 info="Provider API key for embedding model",
                                 required=True,
-                                load_from_db=True,
+                                load_from_db=False,
                             ),
                         },
                     },
@@ -91,11 +99,7 @@ class KBIngestionComponent(Component):
             display_name="Knowledge",
             info="Select the knowledge to load data from.",
             required=True,
-            options=[
-                str(d.name) for d in KNOWLEDGE_BASES_ROOT_PATH.iterdir() if not d.name.startswith(".") and d.is_dir()
-            ]
-            if KNOWLEDGE_BASES_ROOT_PATH.exists()
-            else [],
+            options=[],
             refresh_button=True,
             dialog_inputs=asdict(NewKnowledgeBaseInput()),
         ),
@@ -329,22 +333,23 @@ class KBIngestionComponent(Component):
 
         return metadata
 
-    def _create_vector_store(
+    async def _create_vector_store(
         self, df_source: pd.DataFrame, config_list: list[dict[str, Any]], embedding_model: str, api_key: str
     ) -> None:
         """Create vector store following Local DB component pattern."""
         try:
             # Set up vector store directory
-            base_dir = self._get_kb_root()
-
-            vector_store_dir = base_dir / self.knowledge_base
+            vector_store_dir = await self._kb_path()
+            if not vector_store_dir:
+                msg = "Knowledge base path is not set. Please create a new knowledge base first."
+                raise ValueError(msg)
             vector_store_dir.mkdir(parents=True, exist_ok=True)
 
             # Create embeddings model
             embedding_function = self._build_embeddings(embedding_model, api_key)
 
             # Convert DataFrame to Data objects (following Local DB pattern)
-            data_objects = self._convert_df_to_data_objects(df_source, config_list)
+            data_objects = await self._convert_df_to_data_objects(df_source, config_list)
 
             # Create vector store
             chroma = Chroma(
@@ -367,16 +372,18 @@ class KBIngestionComponent(Component):
         except (OSError, ValueError, RuntimeError) as e:
             self.log(f"Error creating vector store: {e}")
 
-    def _convert_df_to_data_objects(self, df_source: pd.DataFrame, config_list: list[dict[str, Any]]) -> list[Data]:
+    async def _convert_df_to_data_objects(
+        self, df_source: pd.DataFrame, config_list: list[dict[str, Any]]
+    ) -> list[Data]:
         """Convert DataFrame to Data objects for vector store."""
         data_objects: list[Data] = []
 
         # Set up vector store directory
-        base_dir = self._get_kb_root()
+        kb_path = await self._kb_path()
 
         # If we don't allow duplicates, we need to get the existing hashes
         chroma = Chroma(
-            persist_directory=str(base_dir / self.knowledge_base),
+            persist_directory=str(kb_path),
             collection_name=self.knowledge_base,
         )
 
@@ -466,10 +473,34 @@ class KBIngestionComponent(Component):
         # Check allowed characters (condition 3)
         return re.match(r"^[a-zA-Z0-9_-]+$", name) is not None
 
+    async def _kb_path(self) -> Path | None:
+        # Check if we already have the path cached
+        cached_path = getattr(self, "_cached_kb_path", None)
+        if cached_path is not None:
+            return cached_path
+
+        # If not cached, compute it
+        async with session_scope() as db:
+            if not self.user_id:
+                msg = "User ID is required for fetching knowledge base path."
+                raise ValueError(msg)
+            current_user = await get_user_by_id(db, self.user_id)
+            if not current_user:
+                msg = f"User with ID {self.user_id} not found."
+                raise ValueError(msg)
+            kb_user = current_user.username
+
+        kb_root = self._get_kb_root()
+
+        # Cache the result
+        self._cached_kb_path = kb_root / kb_user / self.knowledge_base
+
+        return self._cached_kb_path
+
     # ---------------------------------------------------------------------
     #                         OUTPUT METHODS
     # ---------------------------------------------------------------------
-    def build_kb_info(self) -> Data:
+    async def build_kb_info(self) -> Data:
         """Main ingestion routine → returns a dict with KB metadata."""
         try:
             # Get source DataFrame
@@ -479,11 +510,11 @@ class KBIngestionComponent(Component):
             config_list = self._validate_column_config(df_source)
             column_metadata = self._build_column_metadata(config_list, df_source)
 
-            # Prepare KB folder (using File Component patterns)
-            kb_root = self._get_kb_root()
-            kb_path = kb_root / self.knowledge_base
-
             # Read the embedding info from the knowledge base folder
+            kb_path = await self._kb_path()
+            if not kb_path:
+                msg = "Knowledge base path is not set. Please create a new knowledge base first."
+                raise ValueError(msg)
             metadata_path = kb_path / "embedding_metadata.json"
 
             # If the API key is not provided, try to read it from the metadata file
@@ -506,7 +537,7 @@ class KBIngestionComponent(Component):
                 )
 
             # Create vector store following Local DB component pattern
-            self._create_vector_store(df_source, config_list, embedding_model=embedding_model, api_key=api_key)
+            await self._create_vector_store(df_source, config_list, embedding_model=embedding_model, api_key=api_key)
 
             # Save KB files (using File Component storage patterns)
             self._save_kb_files(kb_path, config_list)
@@ -532,40 +563,77 @@ class KBIngestionComponent(Component):
             self.status = f"❌ KB ingestion failed: {e}"
             return Data(data={"error": str(e), "kb_name": self.knowledge_base})
 
-    def _get_knowledge_bases(self) -> list[str]:
-        """Retrieve a list of available knowledge bases.
-
-        Returns:
-            A list of knowledge base names.
-        """
-        # Return the list of directories in the knowledge base root path
-        kb_root_path = self._get_kb_root()
-
-        if not kb_root_path.exists():
-            return []
-
-        return [str(d.name) for d in kb_root_path.iterdir() if not d.name.startswith(".") and d.is_dir()]
+    async def _get_api_key_variable(self, field_value: dict[str, Any]):
+        async with session_scope() as db:
+            if not self.user_id:
+                msg = "User ID is required for fetching global variables."
+                raise ValueError(msg)
+            current_user = await get_user_by_id(db, self.user_id)
+            if not current_user:
+                msg = f"User with ID {self.user_id} not found."
+                raise ValueError(msg)
+            variable_service = get_variable_service()
+
+            # Process the api_key field variable
+            return await variable_service.get_variable(
+                user_id=current_user.id,
+                name=field_value["03_api_key"],
+                field="",
+                session=db,
+            )
 
-    def update_build_config(self, build_config: dotdict, field_value: Any, field_name: str | None = None) -> dotdict:
+    async def update_build_config(
+        self,
+        build_config: dotdict,
+        field_value: Any,
+        field_name: str | None = None,
+    ) -> dotdict:
         """Update build configuration based on provider selection."""
         # Create a new knowledge base
         if field_name == "knowledge_base":
+            async with session_scope() as db:
+                if not self.user_id:
+                    msg = "User ID is required for fetching knowledge base list."
+                    raise ValueError(msg)
+                current_user = await get_user_by_id(db, self.user_id)
+                if not current_user:
+                    msg = f"User with ID {self.user_id} not found."
+                    raise ValueError(msg)
+                kb_user = current_user.username
             if isinstance(field_value, dict) and "01_new_kb_name" in field_value:
                 # Validate the knowledge base name - Make sure it follows these rules:
                 if not self.is_valid_collection_name(field_value["01_new_kb_name"]):
                     msg = f"Invalid knowledge base name: {field_value['01_new_kb_name']}"
                     raise ValueError(msg)
 
+                api_key = field_value.get("03_api_key", None)
+                with contextlib.suppress(Exception):
+                    # If the API key is a variable, resolve it
+                    api_key = await self._get_api_key_variable(field_value)
+
+                # Make sure api_key is a string
+                if not isinstance(api_key, str):
+                    msg = "API key must be a string."
+                    raise ValueError(msg)
+
                 # We need to test the API Key one time against the embedding model
-                embed_model = self._build_embeddings(
-                    embedding_model=field_value["02_embedding_model"], api_key=field_value["03_api_key"]
-                )
+                embed_model = self._build_embeddings(embedding_model=field_value["02_embedding_model"], api_key=api_key)
 
-                # Try to generate a dummy embedding to validate the API key
-                embed_model.embed_query("test")
+                # Try to generate a dummy embedding to validate the API key without blocking the event loop
+                try:
+                    await asyncio.wait_for(
+                        asyncio.to_thread(embed_model.embed_query, "test"),
+                        timeout=10,
+                    )
+                except TimeoutError as e:
+                    msg = "Embedding validation timed out. Please verify network connectivity and key."
+                    raise ValueError(msg) from e
+                except Exception as e:
+                    msg = f"Embedding validation failed: {e!s}"
+                    raise ValueError(msg) from e
 
                 # Create the new knowledge base directory
-                kb_path = KNOWLEDGE_BASES_ROOT_PATH / field_value["01_new_kb_name"]
+                kb_path = KNOWLEDGE_BASES_ROOT_PATH / kb_user / field_value["01_new_kb_name"]
                 kb_path.mkdir(parents=True, exist_ok=True)
 
                 # Save the embedding metadata
@@ -573,11 +641,16 @@ class KBIngestionComponent(Component):
                 self._save_embedding_metadata(
                     kb_path=kb_path,
                     embedding_model=field_value["02_embedding_model"],
-                    api_key=field_value["03_api_key"],
+                    api_key=api_key,
                 )
 
             # Update the knowledge base options dynamically
-            build_config["knowledge_base"]["options"] = self._get_knowledge_bases()
+            build_config["knowledge_base"]["options"] = await get_knowledge_bases(
+                KNOWLEDGE_BASES_ROOT_PATH,
+                user_id=self.user_id,
+            )
+
+            # If the selected knowledge base is not available, reset it
             if build_config["knowledge_base"]["value"] not in build_config["knowledge_base"]["options"]:
                 build_config["knowledge_base"]["value"] = None
 

langflow/components/data/kb_retrieval.py

@@ -5,13 +5,16 @@ from typing import Any
 from cryptography.fernet import InvalidToken
 from langchain_chroma import Chroma
 from loguru import logger
+from pydantic import SecretStr
 
+from langflow.base.data.kb_utils import get_knowledge_bases
 from langflow.custom import Component
 from langflow.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SecretStrInput
 from langflow.schema.data import Data
 from langflow.schema.dataframe import DataFrame
 from langflow.services.auth.utils import decrypt_api_key
-from langflow.services.deps import get_settings_service
+from langflow.services.database.models.user.crud import get_user_by_id
+from langflow.services.deps import get_settings_service, session_scope
 
 settings = get_settings_service().settings
 knowledge_directory = settings.knowledge_bases_dir
@@ -33,11 +36,7 @@ class KBRetrievalComponent(Component):
             display_name="Knowledge",
             info="Select the knowledge to load data from.",
             required=True,
-            options=[
-                str(d.name) for d in KNOWLEDGE_BASES_ROOT_PATH.iterdir() if not d.name.startswith(".") and d.is_dir()
-            ]
-            if KNOWLEDGE_BASES_ROOT_PATH.exists()
-            else [],
+            options=[],
             refresh_button=True,
             real_time_refresh=True,
         ),
@@ -79,21 +78,13 @@ class KBRetrievalComponent(Component):
         ),
     ]
 
-    def _get_knowledge_bases(self) -> list[str]:
-        """Retrieve a list of available knowledge bases.
-
-        Returns:
-            A list of knowledge base names.
-        """
-        if not KNOWLEDGE_BASES_ROOT_PATH.exists():
-            return []
-
-        return [str(d.name) for d in KNOWLEDGE_BASES_ROOT_PATH.iterdir() if not d.name.startswith(".") and d.is_dir()]
-
-    def update_build_config(self, build_config, field_value, field_name=None):  # noqa: ARG002
+    async def update_build_config(self, build_config, field_value, field_name=None):  # noqa: ARG002
         if field_name == "knowledge_base":
             # Update the knowledge base options dynamically
-            build_config["knowledge_base"]["options"] = self._get_knowledge_bases()
+            build_config["knowledge_base"]["options"] = await get_knowledge_bases(
+                KNOWLEDGE_BASES_ROOT_PATH,
+                user_id=self.user_id,  # Use the user_id from the component context
+            )
 
             # If the selected knowledge base is not available, reset it
             if build_config["knowledge_base"]["value"] not in build_config["knowledge_base"]["options"]:
@@ -129,15 +120,12 @@ class KBRetrievalComponent(Component):
 
     def _build_embeddings(self, metadata: dict):
         """Build embedding model from metadata."""
+        runtime_api_key = self.api_key.get_secret_value() if isinstance(self.api_key, SecretStr) else self.api_key
         provider = metadata.get("embedding_provider")
         model = metadata.get("embedding_model")
-        api_key = metadata.get("api_key")
+        api_key = runtime_api_key or metadata.get("api_key")
         chunk_size = metadata.get("chunk_size")
 
-        # If user provided a key in the input, it overrides the stored one.
-        if self.api_key and self.api_key.get_secret_value():
-            api_key = self.api_key.get_secret_value()
-
         # Handle various providers
         if provider == "OpenAI":
             from langchain_openai import OpenAIEmbeddings
@@ -174,13 +162,23 @@ class KBRetrievalComponent(Component):
         msg = f"Embedding provider '{provider}' is not supported for retrieval."
         raise NotImplementedError(msg)
 
-    def get_chroma_kb_data(self) -> DataFrame:
+    async def get_chroma_kb_data(self) -> DataFrame:
         """Retrieve data from the selected knowledge base by reading the Chroma collection.
 
         Returns:
             A DataFrame containing the data rows from the knowledge base.
         """
-        kb_path = KNOWLEDGE_BASES_ROOT_PATH / self.knowledge_base
+        # Get the current user
+        async with session_scope() as db:
+            if not self.user_id:
+                msg = "User ID is required for fetching Knowledge Base data."
+                raise ValueError(msg)
+            current_user = await get_user_by_id(db, self.user_id)
+            if not current_user:
+                msg = f"User with ID {self.user_id} not found."
+                raise ValueError(msg)
+            kb_user = current_user.username
+        kb_path = KNOWLEDGE_BASES_ROOT_PATH / kb_user / self.knowledge_base
 
         metadata = self._get_kb_metadata(kb_path)
         if not metadata:

langflow/components/data/url.py

@@ -3,12 +3,12 @@ import re
 import requests
 from bs4 import BeautifulSoup
 from langchain_community.document_loaders import RecursiveUrlLoader
-from loguru import logger
 
 from langflow.custom.custom_component.component import Component
 from langflow.field_typing.range_spec import RangeSpec
 from langflow.helpers.data import safe_convert
 from langflow.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput
+from langflow.logging.logger import logger
 from langflow.schema.dataframe import DataFrame
 from langflow.schema.message import Message
 from langflow.services.deps import get_settings_service

langflow/components/datastax/astra_assistant_manager.py

@@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Any, cast
 
 from astra_assistants.astra_assistants_manager import AssistantManager
 from langchain_core.agents import AgentFinish
-from loguru import logger
 
 from langflow.base.agents.events import ExceptionWithMessageError, process_agent_events
 from langflow.base.astra_assistants.util import (
@@ -15,6 +14,7 @@ from langflow.base.astra_assistants.util import (
 )
 from langflow.custom.custom_component.component_with_cache import ComponentWithCache
 from langflow.inputs.inputs import DropdownInput, FileInput, HandleInput, MultilineInput
+from langflow.logging.logger import logger
 from langflow.memory import delete_message
 from langflow.schema.content_block import ContentBlock
 from langflow.schema.message import Message
@@ -186,8 +186,8 @@ class AstraAssistantManager(ComponentWithCache):
                 self.initialized = True
 
     async def process_inputs(self) -> None:
-        logger.info(f"env_set is {self.env_set}")
-        logger.info(self.input_tools)
+        await logger.ainfo(f"env_set is {self.env_set}")
+        await logger.ainfo(self.input_tools)
         tools = []
         tool_obj = None
         if self.input_tools is None:

langflow/components/datastax/create_assistant.py

@@ -1,8 +1,7 @@
-from loguru import logger
-
 from langflow.base.astra_assistants.util import get_patched_openai_client
 from langflow.custom.custom_component.component_with_cache import ComponentWithCache
 from langflow.inputs.inputs import MultilineInput, StrInput
+from langflow.logging.logger import logger
 from langflow.schema.message import Message
 from langflow.template.field.base import Output
 

langflow/components/deactivated/merge_data.py

@@ -1,7 +1,6 @@
-from loguru import logger
-
 from langflow.custom.custom_component.component import Component
 from langflow.io import DataInput, Output
+from langflow.logging.logger import logger
 from langflow.schema.data import Data
 
 

langflow/components/deactivated/sub_flow.py

@@ -1,12 +1,11 @@
 from typing import TYPE_CHECKING, Any
 
-from loguru import logger
-
 from langflow.base.flow_processing.utils import build_data_from_result_data
 from langflow.custom.custom_component.custom_component import CustomComponent
 from langflow.graph.graph.base import Graph
 from langflow.graph.vertex.base import Vertex
 from langflow.helpers.flow import get_flow_inputs
+from langflow.logging.logger import logger
 from langflow.schema.data import Data
 from langflow.schema.dotdict import dotdict
 from langflow.template.field.base import Input
@@ -36,7 +35,7 @@ class SubFlowComponent(CustomComponent):
         return None
 
     async def update_build_config(self, build_config: dotdict, field_value: Any, field_name: str | None = None):
-        logger.debug(f"Updating build config with field value {field_value} and field name {field_name}")
+        await logger.adebug(f"Updating build config with field value {field_value} and field name {field_name}")
         if field_name == "flow_name":
             build_config["flow_name"]["options"] = await self.get_flow_names()
         # Clean up the build config
@@ -47,11 +46,11 @@ class SubFlowComponent(CustomComponent):
             try:
                 flow_data = await self.get_flow(field_value)
             except Exception:  # noqa: BLE001
-                logger.exception(f"Error getting flow {field_value}")
+                await logger.aexception(f"Error getting flow {field_value}")
             else:
                 if not flow_data:
                     msg = f"Flow {field_value} not found."
-                    logger.error(msg)
+                    await logger.aerror(msg)
                 else:
                     try:
                         graph = Graph.from_payload(flow_data.data["data"])
@@ -60,7 +59,7 @@ class SubFlowComponent(CustomComponent):
                         # Add inputs to the build config
                         build_config = self.add_inputs_to_build_config(inputs, build_config)
                     except Exception:  # noqa: BLE001
-                        logger.exception(f"Error building graph for flow {field_value}")
+                        await logger.aexception(f"Error building graph for flow {field_value}")
 
         return build_config
 
@@ -121,5 +120,5 @@ class SubFlowComponent(CustomComponent):
                     data.extend(build_data_from_result_data(output))
 
         self.status = data
-        logger.debug(data)
+        await logger.adebug(data)
         return data

langflow/components/deactivated/vectara_self_query.py

@@ -68,9 +68,9 @@ class VectaraSelfQueryRetriverComponent(CustomComponent):
             metadata_field_obj.append(attribute_info)
 
         return SelfQueryRetriever.from_llm(
-            self.llm,  # noqa: ignore[attr-defined]
-            self.vectorstore,  # noqa: ignore[attr-defined]
-            self.document_content_description,  # noqa: ignore[attr-defined]
+            self.llm,  # type: ignore[attr-defined]
+            self.vectorstore,  # type: ignore[attr-defined]
+            self.document_content_description,  # type: ignore[attr-defined]
             metadata_field_obj,
             verbose=True,
         )