lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/elastic/opensearch.py

@@ -0,0 +1,243 @@
+import json
+from typing import Any
+
+from langchain_community.vectorstores import OpenSearchVectorSearch
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection
+from lfx.io import (
+    BoolInput,
+    DropdownInput,
+    FloatInput,
+    HandleInput,
+    IntInput,
+    MultilineInput,
+    SecretStrInput,
+    StrInput,
+)
+from lfx.schema.data import Data
+
+
+@vector_store_connection
+class OpenSearchVectorStoreComponent(LCVectorStoreComponent):
+    """OpenSearch Vector Store with advanced, customizable search capabilities."""
+
+    display_name: str = "OpenSearch"
+    description: str = "OpenSearch Vector Store with advanced, customizable search capabilities."
+    name = "OpenSearch"
+    icon = "OpenSearch"
+
+    inputs = [
+        StrInput(
+            name="opensearch_url",
+            display_name="OpenSearch URL",
+            value="http://localhost:9200",
+            info="URL for OpenSearch cluster (e.g. https://192.168.1.1:9200).",
+        ),
+        StrInput(
+            name="index_name",
+            display_name="Index Name",
+            value="langflow",
+            info="The index name where the vectors will be stored in OpenSearch cluster.",
+        ),
+        *LCVectorStoreComponent.inputs,
+        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
+        DropdownInput(
+            name="search_type",
+            display_name="Search Type",
+            options=["similarity", "similarity_score_threshold", "mmr"],
+            value="similarity",
+            advanced=True,
+        ),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            advanced=True,
+            value=4,
+        ),
+        FloatInput(
+            name="search_score_threshold",
+            display_name="Search Score Threshold",
+            info="Minimum similarity score threshold for search results.",
+            value=0.0,
+            advanced=True,
+        ),
+        StrInput(
+            name="username",
+            display_name="Username",
+            value="admin",
+            advanced=True,
+        ),
+        SecretStrInput(
+            name="password",
+            display_name="Password",
+            value="admin",
+            advanced=True,
+        ),
+        BoolInput(
+            name="use_ssl",
+            display_name="Use SSL",
+            value=True,
+            advanced=True,
+        ),
+        BoolInput(
+            name="verify_certs",
+            display_name="Verify Certificates",
+            value=False,
+            advanced=True,
+        ),
+        MultilineInput(
+            name="hybrid_search_query",
+            display_name="Hybrid Search Query",
+            value="",
+            advanced=True,
+            info=(
+                "Provide a custom hybrid search query in JSON format. This allows you to combine "
+                "vector similarity and keyword matching."
+            ),
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> OpenSearchVectorSearch:
+        """Builds the OpenSearch Vector Store object."""
+        try:
+            from langchain_community.vectorstores import OpenSearchVectorSearch
+        except ImportError as e:
+            error_message = f"Failed to import required modules: {e}"
+            self.log(error_message)
+            raise ImportError(error_message) from e
+
+        try:
+            opensearch = OpenSearchVectorSearch(
+                index_name=self.index_name,
+                embedding_function=self.embedding,
+                opensearch_url=self.opensearch_url,
+                http_auth=(self.username, self.password),
+                use_ssl=self.use_ssl,
+                verify_certs=self.verify_certs,
+                ssl_assert_hostname=False,
+                ssl_show_warn=False,
+            )
+        except Exception as e:
+            error_message = f"Failed to create OpenSearchVectorSearch instance: {e}"
+            self.log(error_message)
+            raise RuntimeError(error_message) from e
+
+        if self.ingest_data:
+            self._add_documents_to_vector_store(opensearch)
+
+        return opensearch
+
+    def _add_documents_to_vector_store(self, vector_store: "OpenSearchVectorSearch") -> None:
+        """Adds documents to the Vector Store."""
+        # Convert DataFrame to Data if needed using parent's method
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in self.ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                error_message = f"Expected Data object, got {type(_input)}"
+                self.log(error_message)
+                raise TypeError(error_message)
+
+        if documents and self.embedding is not None:
+            self.log(f"Adding {len(documents)} documents to the Vector Store.")
+            try:
+                vector_store.add_documents(documents)
+            except Exception as e:
+                error_message = f"Error adding documents to Vector Store: {e}"
+                self.log(error_message)
+                raise RuntimeError(error_message) from e
+        else:
+            self.log("No documents to add to the Vector Store.")
+
+    def search(self, query: str | None = None) -> list[dict[str, Any]]:
+        """Search for similar documents in the vector store or retrieve all documents if no query is provided."""
+        try:
+            vector_store = self.build_vector_store()
+
+            query = query or ""
+
+            if self.hybrid_search_query.strip():
+                try:
+                    hybrid_query = json.loads(self.hybrid_search_query)
+                except json.JSONDecodeError as e:
+                    error_message = f"Invalid hybrid search query JSON: {e}"
+                    self.log(error_message)
+                    raise ValueError(error_message) from e
+
+                results = vector_store.client.search(index=self.index_name, body=hybrid_query)
+
+                processed_results = []
+                for hit in results.get("hits", {}).get("hits", []):
+                    source = hit.get("_source", {})
+                    text = source.get("text", "")
+                    metadata = source.get("metadata", {})
+
+                    if isinstance(text, dict):
+                        text = text.get("text", "")
+
+                    processed_results.append(
+                        {
+                            "page_content": text,
+                            "metadata": metadata,
+                        }
+                    )
+                return processed_results
+
+            search_kwargs = {"k": self.number_of_results}
+            search_type = self.search_type.lower()
+
+            if search_type == "similarity":
+                results = vector_store.similarity_search(query, **search_kwargs)
+                return [{"page_content": doc.page_content, "metadata": doc.metadata} for doc in results]
+            if search_type == "similarity_score_threshold":
+                search_kwargs["score_threshold"] = self.search_score_threshold
+                results = vector_store.similarity_search_with_relevance_scores(query, **search_kwargs)
+                return [
+                    {
+                        "page_content": doc.page_content,
+                        "metadata": doc.metadata,
+                        "score": score,
+                    }
+                    for doc, score in results
+                ]
+            if search_type == "mmr":
+                results = vector_store.max_marginal_relevance_search(query, **search_kwargs)
+                return [{"page_content": doc.page_content, "metadata": doc.metadata} for doc in results]
+
+        except Exception as e:
+            error_message = f"Error during search: {e}"
+            self.log(error_message)
+            raise RuntimeError(error_message) from e
+
+        error_message = f"Error during search. Invalid search type: {self.search_type}"
+        self.log(error_message)
+        raise ValueError(error_message)
+
+    def search_documents(self) -> list[Data]:
+        """Search for documents in the vector store based on the search input.
+
+        If no search input is provided, retrieve all documents.
+        """
+        try:
+            query = self.search_query.strip() if self.search_query else None
+            results = self.search(query)
+            retrieved_data = [
+                Data(
+                    file_path=result["metadata"].get("file_path", ""),
+                    text=result["page_content"],
+                )
+                for result in results
+            ]
+        except Exception as e:
+            error_message = f"Error during document search: {e}"
+            self.log(error_message)
+            raise RuntimeError(error_message) from e
+
+        self.status = retrieved_data
+        return retrieved_data

lfx/components/embeddings/__init__.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from lfx.components.embeddings.similarity import EmbeddingSimilarityComponent
+    from lfx.components.embeddings.text_embedder import TextEmbedderComponent
+
+_dynamic_imports = {
+    "EmbeddingSimilarityComponent": "similarity",
+    "TextEmbedderComponent": "text_embedder",
+}
+
+__all__ = [
+    "EmbeddingSimilarityComponent",
+    "TextEmbedderComponent",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import embedding components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

lfx/components/embeddings/similarity.py

@@ -0,0 +1,76 @@
+from typing import Any
+
+import numpy as np
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import DataInput, DropdownInput, Output
+from lfx.schema.data import Data
+
+
+class EmbeddingSimilarityComponent(Component):
+    display_name: str = "Embedding Similarity"
+    description: str = "Compute selected form of similarity between two embedding vectors."
+    icon = "equal"
+    legacy: bool = True
+
+    inputs = [
+        DataInput(
+            name="embedding_vectors",
+            display_name="Embedding Vectors",
+            info="A list containing exactly two data objects with embedding vectors to compare.",
+            is_list=True,
+            required=True,
+        ),
+        DropdownInput(
+            name="similarity_metric",
+            display_name="Similarity Metric",
+            info="Select the similarity metric to use.",
+            options=["Cosine Similarity", "Euclidean Distance", "Manhattan Distance"],
+            value="Cosine Similarity",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Similarity Data", name="similarity_data", method="compute_similarity"),
+    ]
+
+    def compute_similarity(self) -> Data:
+        embedding_vectors: list[Data] = self.embedding_vectors
+
+        # Assert that the list contains exactly two Data objects
+        if len(embedding_vectors) != 2:  # noqa: PLR2004
+            msg = "Exactly two embedding vectors are required."
+            raise ValueError(msg)
+
+        embedding_1 = np.array(embedding_vectors[0].data["embeddings"])
+        embedding_2 = np.array(embedding_vectors[1].data["embeddings"])
+
+        if embedding_1.shape != embedding_2.shape:
+            similarity_score: dict[str, Any] = {"error": "Embeddings must have the same dimensions."}
+        else:
+            similarity_metric = self.similarity_metric
+
+            if similarity_metric == "Cosine Similarity":
+                score = np.dot(embedding_1, embedding_2) / (np.linalg.norm(embedding_1) * np.linalg.norm(embedding_2))
+                similarity_score = {"cosine_similarity": score}
+
+            elif similarity_metric == "Euclidean Distance":
+                score = np.linalg.norm(embedding_1 - embedding_2)
+                similarity_score = {"euclidean_distance": score}
+
+            elif similarity_metric == "Manhattan Distance":
+                score = np.sum(np.abs(embedding_1 - embedding_2))
+                similarity_score = {"manhattan_distance": score}
+
+        # Create a Data object to encapsulate the similarity score and additional information
+        similarity_data = Data(
+            data={
+                "embedding_1": embedding_vectors[0].data["embeddings"],
+                "embedding_2": embedding_vectors[1].data["embeddings"],
+                "similarity_score": similarity_score,
+            },
+            text_key="similarity_score",
+        )
+
+        self.status = similarity_data
+        return similarity_data

lfx/components/embeddings/text_embedder.py

@@ -0,0 +1,64 @@
+from typing import TYPE_CHECKING
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import HandleInput, MessageInput, Output
+from lfx.log.logger import logger
+from lfx.schema.data import Data
+
+if TYPE_CHECKING:
+    from lfx.field_typing import Embeddings
+    from lfx.schema.message import Message
+
+
+class TextEmbedderComponent(Component):
+    display_name: str = "Text Embedder"
+    description: str = "Generate embeddings for a given message using the specified embedding model."
+    icon = "binary"
+    legacy: bool = True
+    inputs = [
+        HandleInput(
+            name="embedding_model",
+            display_name="Embedding Model",
+            info="The embedding model to use for generating embeddings.",
+            input_types=["Embeddings"],
+            required=True,
+        ),
+        MessageInput(
+            name="message",
+            display_name="Message",
+            info="The message to generate embeddings for.",
+            required=True,
+        ),
+    ]
+    outputs = [
+        Output(display_name="Embedding Data", name="embeddings", method="generate_embeddings"),
+    ]
+
+    def generate_embeddings(self) -> Data:
+        try:
+            embedding_model: Embeddings = self.embedding_model
+            message: Message = self.message
+
+            # Combine validation checks to reduce nesting
+            if not embedding_model or not hasattr(embedding_model, "embed_documents"):
+                msg = "Invalid or incompatible embedding model"
+                raise ValueError(msg)
+
+            text_content = message.text if message and message.text else ""
+            if not text_content:
+                msg = "No text content found in message"
+                raise ValueError(msg)
+
+            embeddings = embedding_model.embed_documents([text_content])
+            if not embeddings or not isinstance(embeddings, list):
+                msg = "Invalid embeddings generated"
+                raise ValueError(msg)
+
+            embedding_vector = embeddings[0]
+            self.status = {"text": text_content, "embeddings": embedding_vector}
+            return Data(data={"text": text_content, "embeddings": embedding_vector})
+        except Exception as e:  # noqa: BLE001
+            logger.exception("Error generating embeddings")
+            error_data = Data(data={"text": "", "embeddings": [], "error": str(e)})
+            self.status = {"error": str(e)}
+            return error_data

lfx/components/exa/__init__.py

@@ -0,0 +1,3 @@
+from .exa_search import ExaSearchToolkit
+
+__all__ = ["ExaSearchToolkit"]

lfx/components/exa/exa_search.py

@@ -0,0 +1,68 @@
+from langchain_core.tools import tool
+from metaphor_python import Metaphor
+
+from lfx.custom.custom_component.component import Component
+from lfx.field_typing import Tool
+from lfx.io import BoolInput, IntInput, Output, SecretStrInput
+
+
+class ExaSearchToolkit(Component):
+    display_name = "Exa Search"
+    description = "Exa Search toolkit for search and content retrieval"
+    documentation = "https://python.langchain.com/docs/integrations/tools/metaphor_search"
+    beta = True
+    name = "ExaSearch"
+    icon = "ExaSearch"
+
+    inputs = [
+        SecretStrInput(
+            name="metaphor_api_key",
+            display_name="Exa Search API Key",
+            password=True,
+        ),
+        BoolInput(
+            name="use_autoprompt",
+            display_name="Use Autoprompt",
+            value=True,
+        ),
+        IntInput(
+            name="search_num_results",
+            display_name="Search Number of Results",
+            value=5,
+        ),
+        IntInput(
+            name="similar_num_results",
+            display_name="Similar Number of Results",
+            value=5,
+        ),
+    ]
+
+    outputs = [
+        Output(name="tools", display_name="Tools", method="build_toolkit"),
+    ]
+
+    def build_toolkit(self) -> Tool:
+        client = Metaphor(api_key=self.metaphor_api_key)
+
+        @tool
+        def search(query: str):
+            """Call search engine with a query."""
+            return client.search(query, use_autoprompt=self.use_autoprompt, num_results=self.search_num_results)
+
+        @tool
+        def get_contents(ids: list[str]):
+            """Get contents of a webpage.
+
+            The ids passed in should be a list of ids as fetched from `search`.
+            """
+            return client.get_contents(ids)
+
+        @tool
+        def find_similar(url: str):
+            """Get search results similar to a given URL.
+
+            The url passed in should be a URL returned from `search`
+            """
+            return client.find_similar(url, num_results=self.similar_num_results)
+
+        return [search, get_contents, find_similar]

lfx/components/firecrawl/__init__.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from .firecrawl_crawl_api import FirecrawlCrawlApi
+    from .firecrawl_extract_api import FirecrawlExtractApi
+    from .firecrawl_map_api import FirecrawlMapApi
+    from .firecrawl_scrape_api import FirecrawlScrapeApi
+
+_dynamic_imports = {
+    "FirecrawlCrawlApi": "firecrawl_crawl_api",
+    "FirecrawlExtractApi": "firecrawl_extract_api",
+    "FirecrawlMapApi": "firecrawl_map_api",
+    "FirecrawlScrapeApi": "firecrawl_scrape_api",
+}
+
+__all__ = [
+    "FirecrawlCrawlApi",
+    "FirecrawlExtractApi",
+    "FirecrawlMapApi",
+    "FirecrawlScrapeApi",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import firecrawl components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

lfx/components/firecrawl/firecrawl_crawl_api.py

@@ -0,0 +1,88 @@
+import uuid
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import DataInput, IntInput, MultilineInput, Output, SecretStrInput, StrInput
+from lfx.schema.data import Data
+
+
+class FirecrawlCrawlApi(Component):
+    display_name: str = "Firecrawl Crawl API"
+    description: str = "Crawls a URL and returns the results."
+    name = "FirecrawlCrawlApi"
+
+    documentation: str = "https://docs.firecrawl.dev/v1/api-reference/endpoint/crawl-post"
+
+    inputs = [
+        SecretStrInput(
+            name="api_key",
+            display_name="API Key",
+            required=True,
+            password=True,
+            info="The API key to use Firecrawl API.",
+        ),
+        MultilineInput(
+            name="url",
+            display_name="URL",
+            required=True,
+            info="The URL to scrape.",
+            tool_mode=True,
+        ),
+        IntInput(
+            name="timeout",
+            display_name="Timeout",
+            info="Timeout in milliseconds for the request.",
+        ),
+        StrInput(
+            name="idempotency_key",
+            display_name="Idempotency Key",
+            info="Optional idempotency key to ensure unique requests.",
+        ),
+        DataInput(
+            name="crawlerOptions",
+            display_name="Crawler Options",
+            info="The crawler options to send with the request.",
+        ),
+        DataInput(
+            name="scrapeOptions",
+            display_name="Scrape Options",
+            info="The page options to send with the request.",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Data", name="data", method="crawl"),
+    ]
+    idempotency_key: str | None = None
+
+    def crawl(self) -> Data:
+        try:
+            from firecrawl import FirecrawlApp
+        except ImportError as e:
+            msg = "Could not import firecrawl integration package. Please install it with `pip install firecrawl-py`."
+            raise ImportError(msg) from e
+
+        params = self.crawlerOptions.__dict__["data"] if self.crawlerOptions else {}
+        scrape_options_dict = self.scrapeOptions.__dict__["data"] if self.scrapeOptions else {}
+        if scrape_options_dict:
+            params["scrapeOptions"] = scrape_options_dict
+
+        # Set default values for new parameters in v1
+        params.setdefault("maxDepth", 2)
+        params.setdefault("limit", 10000)
+        params.setdefault("allowExternalLinks", False)
+        params.setdefault("allowBackwardLinks", False)
+        params.setdefault("ignoreSitemap", False)
+        params.setdefault("ignoreQueryParameters", False)
+
+        # Ensure onlyMainContent is explicitly set if not provided
+        if "scrapeOptions" in params:
+            params["scrapeOptions"].setdefault("onlyMainContent", True)
+        else:
+            params["scrapeOptions"] = {"onlyMainContent": True}
+
+        if not self.idempotency_key:
+            self.idempotency_key = str(uuid.uuid4())
+
+        app = FirecrawlApp(api_key=self.api_key)
+        crawl_result = app.crawl_url(self.url, params=params, idempotency_key=self.idempotency_key)
+        return Data(data={"results": crawl_result})