langflow-base-nightly 0.5.0.dev30__py3-none-any.whl → 0.5.0.dev31__py3-none-any.whl

langflow/api/router.py

@@ -8,6 +8,7 @@ from langflow.api.v1 import (
     files_router,
     flows_router,
     folders_router,
+    knowledge_bases_router,
     login_router,
     mcp_projects_router,
     mcp_router,
@@ -45,6 +46,7 @@ router_v1.include_router(monitor_router)
 router_v1.include_router(folders_router)
 router_v1.include_router(projects_router)
 router_v1.include_router(starter_projects_router)
+router_v1.include_router(knowledge_bases_router)
 router_v1.include_router(mcp_router)
 router_v1.include_router(voice_mode_router)
 router_v1.include_router(mcp_projects_router)

langflow/api/v1/__init__.py

@@ -4,6 +4,7 @@ from langflow.api.v1.endpoints import router as endpoints_router
 from langflow.api.v1.files import router as files_router
 from langflow.api.v1.flows import router as flows_router
 from langflow.api.v1.folders import router as folders_router
+from langflow.api.v1.knowledge_bases import router as knowledge_bases_router
 from langflow.api.v1.login import router as login_router
 from langflow.api.v1.mcp import router as mcp_router
 from langflow.api.v1.mcp_projects import router as mcp_projects_router
@@ -23,6 +24,7 @@ __all__ = [
     "files_router",
     "flows_router",
     "folders_router",
+    "knowledge_bases_router",
     "login_router",
     "mcp_projects_router",
     "mcp_router",

langflow/api/v1/knowledge_bases.py

@@ -0,0 +1,437 @@
+import json
+import shutil
+from http import HTTPStatus
+from pathlib import Path
+
+import pandas as pd
+from fastapi import APIRouter, HTTPException
+from langchain_chroma import Chroma
+from loguru import logger
+from pydantic import BaseModel
+
+from langflow.services.deps import get_settings_service
+
+router = APIRouter(tags=["Knowledge Bases"], prefix="/knowledge_bases")
+
+
+settings = get_settings_service().settings
+knowledge_directory = settings.knowledge_bases_dir
+if not knowledge_directory:
+    msg = "Knowledge bases directory is not set in the settings."
+    raise ValueError(msg)
+KNOWLEDGE_BASES_DIR = Path(knowledge_directory).expanduser()
+
+
+class KnowledgeBaseInfo(BaseModel):
+    id: str
+    name: str
+    embedding_provider: str | None = "Unknown"
+    embedding_model: str | None = "Unknown"
+    size: int = 0
+    words: int = 0
+    characters: int = 0
+    chunks: int = 0
+    avg_chunk_size: float = 0.0
+
+
+class BulkDeleteRequest(BaseModel):
+    kb_names: list[str]
+
+
+def get_kb_root_path() -> Path:
+    """Get the knowledge bases root path."""
+    return KNOWLEDGE_BASES_DIR
+
+
+def get_directory_size(path: Path) -> int:
+    """Calculate the total size of all files in a directory."""
+    total_size = 0
+    try:
+        for file_path in path.rglob("*"):
+            if file_path.is_file():
+                total_size += file_path.stat().st_size
+    except (OSError, PermissionError):
+        pass
+    return total_size
+
+
+def detect_embedding_provider(kb_path: Path) -> str:
+    """Detect the embedding provider from config files and directory structure."""
+    # Provider patterns to check for
+    provider_patterns = {
+        "OpenAI": ["openai", "text-embedding-ada", "text-embedding-3"],
+        "HuggingFace": ["sentence-transformers", "huggingface", "bert-"],
+        "Cohere": ["cohere", "embed-english", "embed-multilingual"],
+        "Google": ["palm", "gecko", "google"],
+        "Chroma": ["chroma"],
+    }
+
+    # Check JSON config files for provider information
+    for config_file in kb_path.glob("*.json"):
+        try:
+            with config_file.open("r", encoding="utf-8") as f:
+                config_data = json.load(f)
+                if not isinstance(config_data, dict):
+                    continue
+
+                config_str = json.dumps(config_data).lower()
+
+                # Check for explicit provider fields first
+                provider_fields = ["embedding_provider", "provider", "embedding_model_provider"]
+                for field in provider_fields:
+                    if field in config_data:
+                        provider_value = str(config_data[field]).lower()
+                        for provider, patterns in provider_patterns.items():
+                            if any(pattern in provider_value for pattern in patterns):
+                                return provider
+
+                # Check for model name patterns
+                for provider, patterns in provider_patterns.items():
+                    if any(pattern in config_str for pattern in patterns):
+                        return provider
+
+        except (OSError, json.JSONDecodeError) as _:
+            logger.exception("Error reading config file '%s'", config_file)
+            continue
+
+    # Fallback to directory structure
+    if (kb_path / "chroma").exists():
+        return "Chroma"
+    if (kb_path / "vectors.npy").exists():
+        return "Local"
+
+    return "Unknown"
+
+
+def detect_embedding_model(kb_path: Path) -> str:
+    """Detect the embedding model from config files."""
+    # First check the embedding metadata file (most accurate)
+    metadata_file = kb_path / "embedding_metadata.json"
+    if metadata_file.exists():
+        try:
+            with metadata_file.open("r", encoding="utf-8") as f:
+                metadata = json.load(f)
+                if isinstance(metadata, dict) and "embedding_model" in metadata:
+                    # Check for embedding model field
+                    model_value = str(metadata.get("embedding_model", "unknown"))
+                    if model_value and model_value.lower() != "unknown":
+                        return model_value
+        except (OSError, json.JSONDecodeError) as _:
+            logger.exception("Error reading embedding metadata file '%s'", metadata_file)
+
+    # Check other JSON config files for model information
+    for config_file in kb_path.glob("*.json"):
+        # Skip the embedding metadata file since we already checked it
+        if config_file.name == "embedding_metadata.json":
+            continue
+
+        try:
+            with config_file.open("r", encoding="utf-8") as f:
+                config_data = json.load(f)
+                if not isinstance(config_data, dict):
+                    continue
+
+                # Check for explicit model fields first and return the actual model name
+                model_fields = ["embedding_model", "model", "embedding_model_name", "model_name"]
+                for field in model_fields:
+                    if field in config_data:
+                        model_value = str(config_data[field])
+                        if model_value and model_value.lower() != "unknown":
+                            return model_value
+
+                # Check for OpenAI specific model names
+                if "openai" in json.dumps(config_data).lower():
+                    openai_models = ["text-embedding-ada-002", "text-embedding-3-small", "text-embedding-3-large"]
+                    config_str = json.dumps(config_data).lower()
+                    for model in openai_models:
+                        if model in config_str:
+                            return model
+
+                # Check for HuggingFace model names (usually in model field)
+                if "model" in config_data:
+                    model_name = str(config_data["model"])
+                    # Common HuggingFace embedding models
+                    hf_patterns = ["sentence-transformers", "all-MiniLM", "all-mpnet", "multi-qa"]
+                    if any(pattern in model_name for pattern in hf_patterns):
+                        return model_name
+
+        except (OSError, json.JSONDecodeError) as _:
+            logger.exception("Error reading config file '%s'", config_file)
+            continue
+
+    return "Unknown"
+
+
+def get_text_columns(df: pd.DataFrame, schema_data: list | None = None) -> list[str]:
+    """Get the text columns to analyze for word/character counts."""
+    # First try schema-defined text columns
+    if schema_data:
+        text_columns = [
+            col["column_name"]
+            for col in schema_data
+            if col.get("vectorize", False) and col.get("data_type") == "string"
+        ]
+        if text_columns:
+            return [col for col in text_columns if col in df.columns]
+
+    # Fallback to common text column names
+    common_names = ["text", "content", "document", "chunk"]
+    text_columns = [col for col in df.columns if col.lower() in common_names]
+    if text_columns:
+        return text_columns
+
+    # Last resort: all string columns
+    return [col for col in df.columns if df[col].dtype == "object"]
+
+
+def calculate_text_metrics(df: pd.DataFrame, text_columns: list[str]) -> tuple[int, int]:
+    """Calculate total words and characters from text columns."""
+    total_words = 0
+    total_characters = 0
+
+    for col in text_columns:
+        if col not in df.columns:
+            continue
+
+        text_series = df[col].astype(str).fillna("")
+        total_characters += text_series.str.len().sum()
+        total_words += text_series.str.split().str.len().sum()
+
+    return int(total_words), int(total_characters)
+
+
+def get_kb_metadata(kb_path: Path) -> dict:
+    """Extract metadata from a knowledge base directory."""
+    metadata: dict[str, float | int | str] = {
+        "chunks": 0,
+        "words": 0,
+        "characters": 0,
+        "avg_chunk_size": 0.0,
+        "embedding_provider": "Unknown",
+        "embedding_model": "Unknown",
+    }
+
+    try:
+        # First check embedding metadata file for accurate provider and model info
+        metadata_file = kb_path / "embedding_metadata.json"
+        if metadata_file.exists():
+            try:
+                with metadata_file.open("r", encoding="utf-8") as f:
+                    embedding_metadata = json.load(f)
+                    if isinstance(embedding_metadata, dict):
+                        if "embedding_provider" in embedding_metadata:
+                            metadata["embedding_provider"] = embedding_metadata["embedding_provider"]
+                        if "embedding_model" in embedding_metadata:
+                            metadata["embedding_model"] = embedding_metadata["embedding_model"]
+            except (OSError, json.JSONDecodeError) as _:
+                logger.exception("Error reading embedding metadata file '%s'", metadata_file)
+
+        # Fallback to detection if not found in metadata file
+        if metadata["embedding_provider"] == "Unknown":
+            metadata["embedding_provider"] = detect_embedding_provider(kb_path)
+        if metadata["embedding_model"] == "Unknown":
+            metadata["embedding_model"] = detect_embedding_model(kb_path)
+
+        # Read schema for text column information
+        schema_data = None
+        schema_file = kb_path / "schema.json"
+        if schema_file.exists():
+            try:
+                with schema_file.open("r", encoding="utf-8") as f:
+                    schema_data = json.load(f)
+                    if not isinstance(schema_data, list):
+                        schema_data = None
+            except (ValueError, TypeError, OSError) as _:
+                logger.exception("Error reading schema file '%s'", schema_file)
+
+        # Create vector store
+        chroma = Chroma(
+            persist_directory=str(kb_path),
+            collection_name=kb_path.name,
+        )
+
+        # Access the raw collection
+        collection = chroma._collection
+
+        # Fetch all documents and metadata
+        results = collection.get(include=["documents", "metadatas"])
+
+        # Convert to pandas DataFrame
+        source_chunks = pd.DataFrame(
+            {
+                "document": results["documents"],
+                "metadata": results["metadatas"],
+            }
+        )
+
+        # Process the source data for metadata
+        try:
+            metadata["chunks"] = len(source_chunks)
+
+            # Get text columns and calculate metrics
+            text_columns = get_text_columns(source_chunks, schema_data)
+            if text_columns:
+                words, characters = calculate_text_metrics(source_chunks, text_columns)
+                metadata["words"] = words
+                metadata["characters"] = characters
+
+                # Calculate average chunk size
+                if int(metadata["chunks"]) > 0:
+                    metadata["avg_chunk_size"] = round(int(characters) / int(metadata["chunks"]), 1)
+
+        except (OSError, ValueError, TypeError) as _:
+            logger.exception("Error processing Chroma DB '%s'", kb_path.name)
+
+    except (OSError, ValueError, TypeError) as _:
+        logger.exception("Error processing knowledge base directory '%s'", kb_path)
+
+    return metadata
+
+
+@router.get("", status_code=HTTPStatus.OK)
+@router.get("/", status_code=HTTPStatus.OK)
+async def list_knowledge_bases() -> list[KnowledgeBaseInfo]:
+    """List all available knowledge bases."""
+    try:
+        kb_root_path = get_kb_root_path()
+
+        if not kb_root_path.exists():
+            return []
+
+        knowledge_bases = []
+
+        for kb_dir in kb_root_path.iterdir():
+            if not kb_dir.is_dir() or kb_dir.name.startswith("."):
+                continue
+
+            try:
+                # Get size of the directory
+                size = get_directory_size(kb_dir)
+
+                # Get metadata from KB files
+                metadata = get_kb_metadata(kb_dir)
+
+                kb_info = KnowledgeBaseInfo(
+                    id=kb_dir.name,
+                    name=kb_dir.name.replace("_", " ").replace("-", " ").title(),
+                    embedding_provider=metadata["embedding_provider"],
+                    embedding_model=metadata["embedding_model"],
+                    size=size,
+                    words=metadata["words"],
+                    characters=metadata["characters"],
+                    chunks=metadata["chunks"],
+                    avg_chunk_size=metadata["avg_chunk_size"],
+                )
+
+                knowledge_bases.append(kb_info)
+
+            except OSError as _:
+                # Log the exception and skip directories that can't be read
+                logger.exception("Error reading knowledge base directory '%s'", kb_dir)
+                continue
+
+        # Sort by name alphabetically
+        knowledge_bases.sort(key=lambda x: x.name)
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error listing knowledge bases: {e!s}") from e
+    else:
+        return knowledge_bases
+
+
+@router.get("/{kb_name}", status_code=HTTPStatus.OK)
+async def get_knowledge_base(kb_name: str) -> KnowledgeBaseInfo:
+    """Get detailed information about a specific knowledge base."""
+    try:
+        kb_root_path = get_kb_root_path()
+        kb_path = kb_root_path / kb_name
+
+        if not kb_path.exists() or not kb_path.is_dir():
+            raise HTTPException(status_code=404, detail=f"Knowledge base '{kb_name}' not found")
+
+        # Get size of the directory
+        size = get_directory_size(kb_path)
+
+        # Get metadata from KB files
+        metadata = get_kb_metadata(kb_path)
+
+        return KnowledgeBaseInfo(
+            id=kb_name,
+            name=kb_name.replace("_", " ").replace("-", " ").title(),
+            embedding_provider=metadata["embedding_provider"],
+            embedding_model=metadata["embedding_model"],
+            size=size,
+            words=metadata["words"],
+            characters=metadata["characters"],
+            chunks=metadata["chunks"],
+            avg_chunk_size=metadata["avg_chunk_size"],
+        )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error getting knowledge base '{kb_name}': {e!s}") from e
+
+
+@router.delete("/{kb_name}", status_code=HTTPStatus.OK)
+async def delete_knowledge_base(kb_name: str) -> dict[str, str]:
+    """Delete a specific knowledge base."""
+    try:
+        kb_root_path = get_kb_root_path()
+        kb_path = kb_root_path / kb_name
+
+        if not kb_path.exists() or not kb_path.is_dir():
+            raise HTTPException(status_code=404, detail=f"Knowledge base '{kb_name}' not found")
+
+        # Delete the entire knowledge base directory
+        shutil.rmtree(kb_path)
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error deleting knowledge base '{kb_name}': {e!s}") from e
+    else:
+        return {"message": f"Knowledge base '{kb_name}' deleted successfully"}
+
+
+@router.delete("", status_code=HTTPStatus.OK)
+@router.delete("/", status_code=HTTPStatus.OK)
+async def delete_knowledge_bases_bulk(request: BulkDeleteRequest) -> dict[str, object]:
+    """Delete multiple knowledge bases."""
+    try:
+        kb_root_path = get_kb_root_path()
+        deleted_count = 0
+        not_found_kbs = []
+
+        for kb_name in request.kb_names:
+            kb_path = kb_root_path / kb_name
+
+            if not kb_path.exists() or not kb_path.is_dir():
+                not_found_kbs.append(kb_name)
+                continue
+
+            try:
+                # Delete the entire knowledge base directory
+                shutil.rmtree(kb_path)
+                deleted_count += 1
+            except (OSError, PermissionError) as e:
+                logger.exception("Error deleting knowledge base '%s': %s", kb_name, e)
+                # Continue with other deletions even if one fails
+
+        if not_found_kbs and deleted_count == 0:
+            raise HTTPException(status_code=404, detail=f"Knowledge bases not found: {', '.join(not_found_kbs)}")
+
+        result = {
+            "message": f"Successfully deleted {deleted_count} knowledge base(s)",
+            "deleted_count": deleted_count,
+        }
+
+        if not_found_kbs:
+            result["not_found"] = ", ".join(not_found_kbs)
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error deleting knowledge bases: {e!s}") from e
+    else:
+        return result

langflow/base/data/kb_utils.py

@@ -0,0 +1,104 @@
+import math
+from collections import Counter
+
+
+def compute_tfidf(documents: list[str], query_terms: list[str]) -> list[float]:
+    """Compute TF-IDF scores for query terms across a collection of documents.
+
+    Args:
+        documents: List of document strings
+        query_terms: List of query terms to score
+
+    Returns:
+        List of TF-IDF scores for each document
+    """
+    # Tokenize documents (simple whitespace splitting)
+    tokenized_docs = [doc.lower().split() for doc in documents]
+    n_docs = len(documents)
+
+    # Calculate document frequency for each term
+    document_frequencies = {}
+    for term in query_terms:
+        document_frequencies[term] = sum(1 for doc in tokenized_docs if term.lower() in doc)
+
+    scores = []
+
+    for doc_tokens in tokenized_docs:
+        doc_score = 0.0
+        doc_length = len(doc_tokens)
+        term_counts = Counter(doc_tokens)
+
+        for term in query_terms:
+            term_lower = term.lower()
+
+            # Term frequency (TF)
+            tf = term_counts[term_lower] / doc_length if doc_length > 0 else 0
+
+            # Inverse document frequency (IDF)
+            idf = math.log(n_docs / document_frequencies[term]) if document_frequencies[term] > 0 else 0
+
+            # TF-IDF score
+            doc_score += tf * idf
+
+        scores.append(doc_score)
+
+    return scores
+
+
+def compute_bm25(documents: list[str], query_terms: list[str], k1: float = 1.2, b: float = 0.75) -> list[float]:
+    """Compute BM25 scores for query terms across a collection of documents.
+
+    Args:
+        documents: List of document strings
+        query_terms: List of query terms to score
+        k1: Controls term frequency scaling (default: 1.2)
+        b: Controls document length normalization (default: 0.75)
+
+    Returns:
+        List of BM25 scores for each document
+    """
+    # Tokenize documents
+    tokenized_docs = [doc.lower().split() for doc in documents]
+    n_docs = len(documents)
+
+    # Calculate average document length
+    avg_doc_length = sum(len(doc) for doc in tokenized_docs) / n_docs if n_docs > 0 else 0
+
+    # Handle edge case where all documents are empty
+    if avg_doc_length == 0:
+        return [0.0] * n_docs
+
+    # Calculate document frequency for each term
+    document_frequencies = {}
+    for term in query_terms:
+        document_frequencies[term] = sum(1 for doc in tokenized_docs if term.lower() in doc)
+
+    scores = []
+
+    for doc_tokens in tokenized_docs:
+        doc_score = 0.0
+        doc_length = len(doc_tokens)
+        term_counts = Counter(doc_tokens)
+
+        for term in query_terms:
+            term_lower = term.lower()
+
+            # Term frequency in document
+            tf = term_counts[term_lower]
+
+            # Inverse document frequency (IDF)
+            # Use standard BM25 IDF formula that ensures non-negative values
+            idf = math.log(n_docs / document_frequencies[term]) if document_frequencies[term] > 0 else 0
+
+            # BM25 score calculation
+            numerator = tf * (k1 + 1)
+            denominator = tf + k1 * (1 - b + b * (doc_length / avg_doc_length))
+
+            # Handle division by zero when tf=0 and k1=0
+            term_score = 0 if denominator == 0 else idf * (numerator / denominator)
+
+            doc_score += term_score
+
+        scores.append(doc_score)
+
+    return scores

langflow/components/data/__init__.py

@@ -3,6 +3,8 @@ from .csv_to_data import CSVToDataComponent
 from .directory import DirectoryComponent
 from .file import FileComponent
 from .json_to_data import JSONToDataComponent
+from .kb_ingest import KBIngestionComponent
+from .kb_retrieval import KBRetrievalComponent
 from .news_search import NewsSearchComponent
 from .rss import RSSReaderComponent
 from .sql_executor import SQLComponent
@@ -16,6 +18,8 @@ __all__ = [
     "DirectoryComponent",
     "FileComponent",
     "JSONToDataComponent",
+    "KBIngestionComponent",
+    "KBRetrievalComponent",
     "NewsSearchComponent",
     "RSSReaderComponent",
     "SQLComponent",