lfx-nightly 0.2.0.dev25__py3-none-any.whl

lfx/base/document_transformers/model.py

@@ -0,0 +1,43 @@
+from abc import abstractmethod
+from typing import Any
+
+from langchain_core.documents import BaseDocumentTransformer
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import Output
+from lfx.schema.data import Data
+from lfx.utils.util import build_loader_repr_from_data
+
+
+class LCDocumentTransformerComponent(Component):
+    trace_type = "document_transformer"
+    outputs = [
+        Output(display_name="Data", name="data", method="transform_data"),
+    ]
+
+    def transform_data(self) -> list[Data]:
+        data_input = self.get_data_input()
+        documents = []
+
+        if not isinstance(data_input, list):
+            data_input = [data_input]
+
+        for _input in data_input:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        transformer = self.build_document_transformer()
+        docs = transformer.transform_documents(documents)
+        data = self.to_data(docs)
+        self.repr_value = build_loader_repr_from_data(data)
+        return data
+
+    @abstractmethod
+    def get_data_input(self) -> Any:
+        """Get the data input."""
+
+    @abstractmethod
+    def build_document_transformer(self) -> BaseDocumentTransformer:
+        """Build the text splitter."""

lfx/base/embeddings/aiml_embeddings.py

@@ -0,0 +1,62 @@
+import concurrent.futures
+import json
+
+import httpx
+from pydantic import BaseModel, SecretStr
+
+from lfx.field_typing import Embeddings
+from lfx.log.logger import logger
+
+
+class AIMLEmbeddingsImpl(BaseModel, Embeddings):
+    embeddings_completion_url: str = "https://api.aimlapi.com/v1/embeddings"
+
+    api_key: SecretStr
+    model: str
+
+    def embed_documents(self, texts: list[str]) -> list[list[float]]:
+        embeddings = [None] * len(texts)
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.api_key.get_secret_value()}",
+        }
+
+        with httpx.Client() as client, concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = []
+            for i, text in enumerate(texts):
+                futures.append((i, executor.submit(self._embed_text, client, headers, text)))
+
+            for index, future in futures:
+                try:
+                    result_data = future.result()
+                    if len(result_data["data"]) != 1:
+                        msg = f"Expected one embedding, got {len(result_data['data'])}"
+                        raise ValueError(msg)
+                    embeddings[index] = result_data["data"][0]["embedding"]
+                except (
+                    httpx.HTTPStatusError,
+                    httpx.RequestError,
+                    json.JSONDecodeError,
+                    KeyError,
+                    ValueError,
+                ):
+                    logger.exception("Error occurred")
+                    raise
+
+        return embeddings  # type: ignore[return-value]
+
+    def _embed_text(self, client: httpx.Client, headers: dict, text: str) -> dict:
+        payload = {
+            "model": self.model,
+            "input": text,
+        }
+        response = client.post(
+            self.embeddings_completion_url,
+            headers=headers,
+            json=payload,
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def embed_query(self, text: str) -> list[float]:
+        return self.embed_documents([text])[0]

lfx/base/embeddings/embeddings_class.py

@@ -0,0 +1,113 @@
+"""Extended embeddings class with available models metadata."""
+
+from langchain_core.embeddings import Embeddings
+
+
+class EmbeddingsWithModels(Embeddings):
+    """Extended Embeddings class that includes available models with dedicated instances.
+
+    This class inherits from LangChain Embeddings and provides a mapping of model names
+    to their dedicated embedding instances, enabling multi-model support without the need
+    for dynamic model switching.
+
+    Attributes:
+        embeddings: The primary LangChain Embeddings instance (used as fallback).
+        available_models: Dict mapping model names to their dedicated Embeddings instances.
+                         Each model has its own pre-configured instance with specific parameters.
+    """
+
+    def __init__(
+        self,
+        embeddings: Embeddings,
+        available_models: dict[str, Embeddings] | None = None,
+    ):
+        """Initialize the EmbeddingsWithModels wrapper.
+
+        Args:
+            embeddings: The primary LangChain Embeddings instance (used as default/fallback).
+            available_models: Dict mapping model names to dedicated Embeddings instances.
+                            Each value should be a fully configured Embeddings object ready to use.
+                            Defaults to empty dict if not provided.
+        """
+        super().__init__()
+        self.embeddings = embeddings
+        self.available_models = available_models if available_models is not None else {}
+
+    def embed_documents(self, texts: list[str]) -> list[list[float]]:
+        """Embed search docs by delegating to the underlying embeddings instance.
+
+        Args:
+            texts: List of text to embed.
+
+        Returns:
+            List of embeddings.
+        """
+        return self.embeddings.embed_documents(texts)
+
+    def embed_query(self, text: str) -> list[float]:
+        """Embed query text by delegating to the underlying embeddings instance.
+
+        Args:
+            text: Text to embed.
+
+        Returns:
+            Embedding.
+        """
+        return self.embeddings.embed_query(text)
+
+    async def aembed_documents(self, texts: list[str]) -> list[list[float]]:
+        """Asynchronously embed search docs.
+
+        Args:
+            texts: List of text to embed.
+
+        Returns:
+            List of embeddings.
+        """
+        return await self.embeddings.aembed_documents(texts)
+
+    async def aembed_query(self, text: str) -> list[float]:
+        """Asynchronously embed query text.
+
+        Args:
+            text: Text to embed.
+
+        Returns:
+            Embedding.
+        """
+        return await self.embeddings.aembed_query(text)
+
+    def __call__(self, *args, **kwargs):
+        """Make the class callable by delegating to the underlying embeddings instance.
+
+        This handles cases where the embeddings object is used as a callable.
+
+        Args:
+            *args: Positional arguments to pass to the underlying embeddings instance.
+            **kwargs: Keyword arguments to pass to the underlying embeddings instance.
+
+        Returns:
+            The result of calling the underlying embeddings instance.
+        """
+        if callable(self.embeddings):
+            return self.embeddings(*args, **kwargs)
+        msg = f"'{type(self.embeddings).__name__}' object is not callable"
+        raise TypeError(msg)
+
+    def __getattr__(self, name: str):
+        """Delegate attribute access to the underlying embeddings instance.
+
+        This ensures full compatibility with any additional methods or attributes
+        that the underlying embeddings instance might have.
+
+        Args:
+            name: The attribute name to access.
+
+        Returns:
+            The attribute from the underlying embeddings instance.
+        """
+        return getattr(self.embeddings, name)
+
+    def __repr__(self) -> str:
+        """Return string representation of the wrapper."""
+        return f"EmbeddingsWithModels(embeddings={self.embeddings!r}, available_models={self.available_models!r})"

lfx/base/embeddings/model.py

@@ -0,0 +1,26 @@
+from lfx.custom.custom_component.component import Component
+from lfx.field_typing import Embeddings
+from lfx.io import Output
+
+
+class LCEmbeddingsModel(Component):
+    trace_type = "embedding"
+
+    outputs = [
+        Output(display_name="Embedding Model", name="embeddings", method="build_embeddings"),
+    ]
+
+    def _validate_outputs(self) -> None:
+        required_output_methods = ["build_embeddings"]
+        output_names = [output.name for output in self.outputs]
+        for method_name in required_output_methods:
+            if method_name not in output_names:
+                msg = f"Output with name '{method_name}' must be defined."
+                raise ValueError(msg)
+            if not hasattr(self, method_name):
+                msg = f"Method '{method_name}' must be defined."
+                raise ValueError(msg)
+
+    def build_embeddings(self) -> Embeddings:
+        msg = "You must implement the build_embeddings method in your class."
+        raise NotImplementedError(msg)

lfx/base/flow_processing/utils.py

@@ -0,0 +1,86 @@
+from lfx.graph.schema import ResultData, RunOutputs
+from lfx.log.logger import logger
+from lfx.schema.data import Data
+from lfx.schema.message import Message
+
+
+def build_data_from_run_outputs(run_outputs: RunOutputs) -> list[Data]:
+    """Build a list of data from the given RunOutputs.
+
+    Args:
+        run_outputs (RunOutputs): The RunOutputs object containing the output data.
+
+    Returns:
+        List[Data]: A list of data built from the RunOutputs.
+
+    """
+    if not run_outputs:
+        return []
+    data = []
+    for result_data in run_outputs.outputs:
+        if result_data:
+            data.extend(build_data_from_result_data(result_data))
+    return data
+
+
+def build_data_from_result_data(result_data: ResultData) -> list[Data]:
+    """Build a list of data from the given ResultData.
+
+    Args:
+        result_data (ResultData): The ResultData object containing the result data.
+
+    Returns:
+        List[Data]: A list of data built from the ResultData.
+
+    """
+    messages = result_data.messages
+
+    if not messages:
+        return []
+    data = []
+
+    # Handle results without chat messages (calling flow)
+    if not messages:
+        # Result with a single record
+        if isinstance(result_data.artifacts, dict):
+            data.append(Data(data=result_data.artifacts))
+        # List of artifacts
+        elif isinstance(result_data.artifacts, list):
+            for artifact in result_data.artifacts:
+                # If multiple records are found as artifacts, return as-is
+                if isinstance(artifact, Data):
+                    data.append(artifact)
+                else:
+                    # Warn about unknown output type
+                    logger.warning(f"Unable to build record output from unknown ResultData.artifact: {artifact}")
+        # Chat or text output
+        elif result_data.results:
+            data.append(Data(data={"result": result_data.results}, text_key="result"))
+            return data
+        else:
+            return []
+
+    if isinstance(result_data.results, dict):
+        for name, result in result_data.results.items():
+            dataobj: Data | Message | None
+            dataobj = result if isinstance(result, Message) else Data(data=result, text_key=name)
+
+            data.append(dataobj)
+    else:
+        data.append(Data(data=result_data.results))
+    return data
+
+
+def format_flow_output_data(data: list[Data]) -> str:
+    """Format the flow output data into a string.
+
+    Args:
+        data (List[Data]): The list of data to format.
+
+    Returns:
+        str: The formatted flow output data.
+
+    """
+    result = "Flow run output:\n"
+    results = "\n".join([value.get_text() if hasattr(value, "get_text") else str(value) for value in data])
+    return result + results

lfx/base/huggingface/model_bridge.py

@@ -0,0 +1,133 @@
+# Import LangChain base
+from langchain_core.language_models.chat_models import BaseChatModel
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolCall
+from langchain_core.tools import BaseTool
+
+# Import Hugging Face Model base
+from smolagents import Model, Tool
+from smolagents.models import ChatMessage, ChatMessageToolCall, ChatMessageToolCallDefinition
+
+
+def _lc_tool_call_to_hf_tool_call(tool_call: ToolCall) -> ChatMessageToolCall:
+    """Convert a LangChain ToolCall to a Hugging Face ChatMessageToolCall.
+
+    Args:
+        tool_call (ToolCall): LangChain tool call to convert
+
+    Returns:
+        ChatMessageToolCall: Equivalent Hugging Face tool call
+    """
+    return ChatMessageToolCall(
+        function=ChatMessageToolCallDefinition(name=tool_call.name, arguments=tool_call.args),
+        id=tool_call.id,
+    )
+
+
+def _hf_tool_to_lc_tool(tool) -> BaseTool:
+    """Convert a Hugging Face Tool to a LangChain BaseTool.
+
+    Args:
+        tool (Tool): Hugging Face tool to convert
+
+    Returns:
+        BaseTool: Equivalent LangChain tool
+    """
+    if not hasattr(tool, "langchain_tool"):
+        msg = "Hugging Face Tool does not have a langchain_tool attribute"
+        raise ValueError(msg)
+    return tool.langchain_tool
+
+
+class LangChainHFModel(Model):
+    """A class bridging Hugging Face's `Model` interface with a LangChain `BaseChatModel`.
+
+    This adapter allows using any LangChain chat model with the Hugging Face interface.
+    It handles conversion of message formats and tool calls between the two frameworks.
+
+    Usage:
+       >>> lc_model = LangChainChatModel(...)  # any BaseChatModel
+       >>> hf_model = LangChainHFModel(lc_model)
+       >>> hf_model(messages=[{"role": "user", "content": "Hello!"}])
+    """
+
+    def __init__(self, chat_model: BaseChatModel, **kwargs):
+        """Initialize the bridge model.
+
+        Args:
+            chat_model (BaseChatModel): LangChain chat model to wrap
+            **kwargs: Additional arguments passed to Model.__init__
+        """
+        super().__init__(**kwargs)
+        self.chat_model = chat_model
+
+    def __call__(
+        self,
+        messages: list[dict[str, str]],
+        stop_sequences: list[str] | None = None,
+        grammar: str | None = None,
+        tools_to_call_from: list[Tool] | None = None,
+        **kwargs,
+    ) -> ChatMessage:
+        """Process messages through the LangChain model and return Hugging Face format.
+
+        Args:
+            messages: List of message dictionaries with 'role' and 'content' keys
+            stop_sequences: Optional list of strings to stop generation
+            grammar: Optional grammar specification (not used)
+            tools_to_call_from: Optional list of available tools (not used)
+            **kwargs: Additional arguments passed to the LangChain model
+
+        Returns:
+            ChatMessage: Response in Hugging Face format
+        """
+        if grammar:
+            msg = "Grammar is not yet supported."
+            raise ValueError(msg)
+
+        # Convert HF messages to LangChain messages
+        lc_messages = []
+        for m in messages:
+            role = m["role"]
+            content = m["content"]
+            if role == "system":
+                lc_messages.append(SystemMessage(content=content))
+            elif role == "assistant":
+                lc_messages.append(AIMessage(content=content))
+            else:
+                # Default any unknown role to "user"
+                lc_messages.append(HumanMessage(content=content))
+
+        # Convert tools to LangChain tools
+        if tools_to_call_from:
+            tools_to_call_from = [_hf_tool_to_lc_tool(tool) for tool in tools_to_call_from]
+
+        model = self.chat_model.bind_tools(tools_to_call_from) if tools_to_call_from else self.chat_model
+
+        # Call the LangChain model
+        result_msg: AIMessage = model.invoke(lc_messages, stop=stop_sequences, **kwargs)
+
+        # Convert the AIMessage into an HF ChatMessage
+        return ChatMessage(
+            role="assistant",
+            content=result_msg.content or "",
+            tool_calls=[_lc_tool_call_to_hf_tool_call(tool_call) for tool_call in result_msg.tool_calls],
+        )
+
+
+# How to use
+# if __name__ == "__main__":
+#     from langchain_community.tools import DuckDuckGoSearchRun
+#     from langchain_openai import ChatOpenAI
+#     from rich import rprint
+#     from smolagents import CodeAgent
+
+#     # Example usage
+#     model = LangChainHFModel(chat_model=ChatOpenAI(model="gpt-4o-mini"))
+#     search_tool = DuckDuckGoSearchRun()
+#     hf_tool = Tool.from_langchain(search_tool)
+
+#     code_agent = CodeAgent(
+#         model=model,
+#         tools=[hf_tool],
+#     )
+#     rprint(code_agent.run("Search for Langflow on DuckDuckGo and return the first result"))

lfx/base/io/chat.py

@@ -0,0 +1,21 @@
+from lfx.custom.custom_component.component import Component
+
+
+class ChatComponent(Component):
+    display_name = "Chat Component"
+    description = "Use as base for chat components."
+
+    def get_properties_from_source_component(self):
+        vertex = self.get_vertex()
+        if vertex and hasattr(vertex, "incoming_edges") and vertex.incoming_edges:
+            source_id = vertex.incoming_edges[0].source_id
+            source_vertex = self.graph.get_vertex(source_id)
+            component = source_vertex.custom_component
+            source = component.display_name
+            icon = component.icon
+            possible_attributes = ["model_name", "model_id", "model"]
+            for attribute in possible_attributes:
+                if hasattr(component, attribute) and getattr(component, attribute):
+                    return getattr(component, attribute), icon, source, component.get_id()
+            return source, icon, component.display_name, component.get_id()
+        return None, None, None, None

lfx/base/io/text.py

@@ -0,0 +1,22 @@
+from lfx.custom.custom_component.component import Component
+
+
+class TextComponent(Component):
+    display_name = "Text Component"
+    description = "Used to pass text to the next component."
+
+    def build_config(self):
+        return {
+            "input_value": {
+                "display_name": "Value",
+                "input_types": ["Message", "Data"],
+                "info": "Text or Data to be passed.",
+            },
+            "data_template": {
+                "display_name": "Data Template",
+                "multiline": True,
+                "info": "Template to convert Data to Text. "
+                "If left empty, it will be dynamically set to the Data's text key.",
+                "advanced": True,
+            },
+        }

lfx/base/knowledge_bases/__init__.py

@@ -0,0 +1,3 @@
+from .knowledge_base_utils import compute_bm25, compute_tfidf, get_knowledge_bases
+
+__all__ = ["compute_bm25", "compute_tfidf", "get_knowledge_bases"]

lfx/base/knowledge_bases/knowledge_base_utils.py

@@ -0,0 +1,137 @@
+import math
+from collections import Counter
+from pathlib import Path
+from uuid import UUID
+
+from langflow.services.database.models.user.crud import get_user_by_id
+from langflow.services.deps import session_scope
+
+
+def compute_tfidf(documents: list[str], query_terms: list[str]) -> list[float]:
+    """Compute TF-IDF scores for query terms across a collection of documents.
+
+    Args:
+        documents: List of document strings
+        query_terms: List of query terms to score
+
+    Returns:
+        List of TF-IDF scores for each document
+    """
+    # Tokenize documents (simple whitespace splitting)
+    tokenized_docs = [doc.lower().split() for doc in documents]
+    n_docs = len(documents)
+
+    # Calculate document frequency for each term
+    document_frequencies = {}
+    for term in query_terms:
+        document_frequencies[term] = sum(1 for doc in tokenized_docs if term.lower() in doc)
+
+    scores = []
+
+    for doc_tokens in tokenized_docs:
+        doc_score = 0.0
+        doc_length = len(doc_tokens)
+        term_counts = Counter(doc_tokens)
+
+        for term in query_terms:
+            term_lower = term.lower()
+
+            # Term frequency (TF)
+            tf = term_counts[term_lower] / doc_length if doc_length > 0 else 0
+
+            # Inverse document frequency (IDF)
+            idf = math.log(n_docs / document_frequencies[term]) if document_frequencies[term] > 0 else 0
+
+            # TF-IDF score
+            doc_score += tf * idf
+
+        scores.append(doc_score)
+
+    return scores
+
+
+def compute_bm25(documents: list[str], query_terms: list[str], k1: float = 1.2, b: float = 0.75) -> list[float]:
+    """Compute BM25 scores for query terms across a collection of documents.
+
+    Args:
+        documents: List of document strings
+        query_terms: List of query terms to score
+        k1: Controls term frequency scaling (default: 1.2)
+        b: Controls document length normalization (default: 0.75)
+
+    Returns:
+        List of BM25 scores for each document
+    """
+    # Tokenize documents
+    tokenized_docs = [doc.lower().split() for doc in documents]
+    n_docs = len(documents)
+
+    # Calculate average document length
+    avg_doc_length = sum(len(doc) for doc in tokenized_docs) / n_docs if n_docs > 0 else 0
+
+    # Handle edge case where all documents are empty
+    if avg_doc_length == 0:
+        return [0.0] * n_docs
+
+    # Calculate document frequency for each term
+    document_frequencies = {}
+    for term in query_terms:
+        document_frequencies[term] = sum(1 for doc in tokenized_docs if term.lower() in doc)
+
+    scores = []
+
+    for doc_tokens in tokenized_docs:
+        doc_score = 0.0
+        doc_length = len(doc_tokens)
+        term_counts = Counter(doc_tokens)
+
+        for term in query_terms:
+            term_lower = term.lower()
+
+            # Term frequency in document
+            tf = term_counts[term_lower]
+
+            # Inverse document frequency (IDF)
+            # Use standard BM25 IDF formula that ensures non-negative values
+            idf = math.log(n_docs / document_frequencies[term]) if document_frequencies[term] > 0 else 0
+
+            # BM25 score calculation
+            numerator = tf * (k1 + 1)
+            denominator = tf + k1 * (1 - b + b * (doc_length / avg_doc_length))
+
+            # Handle division by zero when tf=0 and k1=0
+            term_score = 0 if denominator == 0 else idf * (numerator / denominator)
+
+            doc_score += term_score
+
+        scores.append(doc_score)
+
+    return scores
+
+
+async def get_knowledge_bases(kb_root: Path, user_id: UUID | str) -> list[str]:
+    """Retrieve a list of available knowledge bases.
+
+    Returns:
+        A list of knowledge base names.
+    """
+    if not kb_root.exists():
+        return []
+
+    # Get the current user
+    async with session_scope() as db:
+        if not user_id:
+            msg = "User ID is required for fetching knowledge bases."
+            raise ValueError(msg)
+        user_id = UUID(user_id) if isinstance(user_id, str) else user_id
+        current_user = await get_user_by_id(db, user_id)
+        if not current_user:
+            msg = f"User with ID {user_id} not found."
+            raise ValueError(msg)
+        kb_user = current_user.username
+    kb_path = kb_root / kb_user
+
+    if not kb_path.exists():
+        return []
+
+    return [str(d.name) for d in kb_path.iterdir() if not d.name.startswith(".") and d.is_dir()]