lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/vectorstores/qdrant.py

@@ -0,0 +1,109 @@
+from langchain.embeddings.base import Embeddings
+from langchain_community.vectorstores import Qdrant
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.io import (
+    DropdownInput,
+    HandleInput,
+    IntInput,
+    SecretStrInput,
+    StrInput,
+)
+from lfx.schema.data import Data
+
+
+class QdrantVectorStoreComponent(LCVectorStoreComponent):
+    display_name = "Qdrant"
+    description = "Qdrant Vector Store with search capabilities"
+    icon = "Qdrant"
+
+    inputs = [
+        StrInput(name="collection_name", display_name="Collection Name", required=True),
+        StrInput(name="host", display_name="Host", value="localhost", advanced=True),
+        IntInput(name="port", display_name="Port", value=6333, advanced=True),
+        IntInput(name="grpc_port", display_name="gRPC Port", value=6334, advanced=True),
+        SecretStrInput(name="api_key", display_name="API Key", advanced=True),
+        StrInput(name="prefix", display_name="Prefix", advanced=True),
+        IntInput(name="timeout", display_name="Timeout", advanced=True),
+        StrInput(name="path", display_name="Path", advanced=True),
+        StrInput(name="url", display_name="URL", advanced=True),
+        DropdownInput(
+            name="distance_func",
+            display_name="Distance Function",
+            options=["Cosine", "Euclidean", "Dot Product"],
+            value="Cosine",
+            advanced=True,
+        ),
+        StrInput(name="content_payload_key", display_name="Content Payload Key", value="page_content", advanced=True),
+        StrInput(name="metadata_payload_key", display_name="Metadata Payload Key", value="metadata", advanced=True),
+        *LCVectorStoreComponent.inputs,
+        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            value=4,
+            advanced=True,
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> Qdrant:
+        qdrant_kwargs = {
+            "collection_name": self.collection_name,
+            "content_payload_key": self.content_payload_key,
+            "metadata_payload_key": self.metadata_payload_key,
+        }
+
+        server_kwargs = {
+            "host": self.host or None,
+            "port": int(self.port),  # Ensure port is an integer
+            "grpc_port": int(self.grpc_port),  # Ensure grpc_port is an integer
+            "api_key": self.api_key,
+            "prefix": self.prefix,
+            # Ensure timeout is an integer
+            "timeout": int(self.timeout) if self.timeout else None,
+            "path": self.path or None,
+            "url": self.url or None,
+        }
+
+        server_kwargs = {k: v for k, v in server_kwargs.items() if v is not None}
+
+        # Convert DataFrame to Data if needed using parent's method
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in self.ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        if not isinstance(self.embedding, Embeddings):
+            msg = "Invalid embedding object"
+            raise TypeError(msg)
+
+        if documents:
+            qdrant = Qdrant.from_documents(documents, embedding=self.embedding, **qdrant_kwargs, **server_kwargs)
+        else:
+            from qdrant_client import QdrantClient
+
+            client = QdrantClient(**server_kwargs)
+            qdrant = Qdrant(embeddings=self.embedding, client=client, **qdrant_kwargs)
+
+        return qdrant
+
+    def search_documents(self) -> list[Data]:
+        vector_store = self.build_vector_store()
+
+        if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
+            docs = vector_store.similarity_search(
+                query=self.search_query,
+                k=self.number_of_results,
+            )
+
+            data = docs_to_data(docs)
+            self.status = data
+            return data
+        return []

lfx/components/vectorstores/supabase.py

@@ -0,0 +1,76 @@
+from langchain_community.vectorstores import SupabaseVectorStore
+from supabase.client import Client, create_client
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.io import HandleInput, IntInput, SecretStrInput, StrInput
+from lfx.schema.data import Data
+
+
+class SupabaseVectorStoreComponent(LCVectorStoreComponent):
+    display_name = "Supabase"
+    description = "Supabase Vector Store with search capabilities"
+    name = "SupabaseVectorStore"
+    icon = "Supabase"
+
+    inputs = [
+        StrInput(name="supabase_url", display_name="Supabase URL", required=True),
+        SecretStrInput(name="supabase_service_key", display_name="Supabase Service Key", required=True),
+        StrInput(name="table_name", display_name="Table Name", advanced=True),
+        StrInput(name="query_name", display_name="Query Name"),
+        *LCVectorStoreComponent.inputs,
+        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            value=4,
+            advanced=True,
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> SupabaseVectorStore:
+        supabase: Client = create_client(self.supabase_url, supabase_key=self.supabase_service_key)
+
+        # Convert DataFrame to Data if needed using parent's method
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in self.ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        if documents:
+            supabase_vs = SupabaseVectorStore.from_documents(
+                documents=documents,
+                embedding=self.embedding,
+                query_name=self.query_name,
+                client=supabase,
+                table_name=self.table_name,
+            )
+        else:
+            supabase_vs = SupabaseVectorStore(
+                client=supabase,
+                embedding=self.embedding,
+                table_name=self.table_name,
+                query_name=self.query_name,
+            )
+
+        return supabase_vs
+
+    def search_documents(self) -> list[Data]:
+        vector_store = self.build_vector_store()
+
+        if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
+            docs = vector_store.similarity_search(
+                query=self.search_query,
+                k=self.number_of_results,
+            )
+
+            data = docs_to_data(docs)
+            self.status = data
+            return data
+        return []

lfx/components/vectorstores/upstash.py

@@ -0,0 +1,124 @@
+from langchain_community.vectorstores import UpstashVectorStore
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.io import (
+    HandleInput,
+    IntInput,
+    MultilineInput,
+    SecretStrInput,
+    StrInput,
+)
+from lfx.schema.data import Data
+
+
+class UpstashVectorStoreComponent(LCVectorStoreComponent):
+    display_name = "Upstash"
+    description = "Upstash Vector Store with search capabilities"
+    name = "Upstash"
+    icon = "Upstash"
+
+    inputs = [
+        StrInput(
+            name="index_url",
+            display_name="Index URL",
+            info="The URL of the Upstash index.",
+            required=True,
+        ),
+        SecretStrInput(
+            name="index_token",
+            display_name="Index Token",
+            info="The token for the Upstash index.",
+            required=True,
+        ),
+        StrInput(
+            name="text_key",
+            display_name="Text Key",
+            info="The key in the record to use as text.",
+            value="text",
+            advanced=True,
+        ),
+        StrInput(
+            name="namespace",
+            display_name="Namespace",
+            info="Leave empty for default namespace.",
+        ),
+        *LCVectorStoreComponent.inputs,
+        MultilineInput(
+            name="metadata_filter",
+            display_name="Metadata Filter",
+            info="Filters documents by metadata. Look at the documentation for more information.",
+        ),
+        HandleInput(
+            name="embedding",
+            display_name="Embedding",
+            input_types=["Embeddings"],
+            info="To use Upstash's embeddings, don't provide an embedding.",
+        ),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            value=4,
+            advanced=True,
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> UpstashVectorStore:
+        use_upstash_embedding = self.embedding is None
+
+        # Convert DataFrame to Data if needed using parent's method
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in self.ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        if documents:
+            if use_upstash_embedding:
+                upstash_vs = UpstashVectorStore(
+                    embedding=use_upstash_embedding,
+                    text_key=self.text_key,
+                    index_url=self.index_url,
+                    index_token=self.index_token,
+                    namespace=self.namespace,
+                )
+                upstash_vs.add_documents(documents)
+            else:
+                upstash_vs = UpstashVectorStore.from_documents(
+                    documents=documents,
+                    embedding=self.embedding,
+                    text_key=self.text_key,
+                    index_url=self.index_url,
+                    index_token=self.index_token,
+                    namespace=self.namespace,
+                )
+        else:
+            upstash_vs = UpstashVectorStore(
+                embedding=self.embedding or use_upstash_embedding,
+                text_key=self.text_key,
+                index_url=self.index_url,
+                index_token=self.index_token,
+                namespace=self.namespace,
+            )
+
+        return upstash_vs
+
+    def search_documents(self) -> list[Data]:
+        vector_store = self.build_vector_store()
+
+        if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
+            docs = vector_store.similarity_search(
+                query=self.search_query,
+                k=self.number_of_results,
+                filter=self.metadata_filter,
+            )
+
+            data = docs_to_data(docs)
+            self.status = data
+            return data
+        return []

lfx/components/vectorstores/vectara.py

@@ -0,0 +1,97 @@
+from typing import TYPE_CHECKING
+
+from langchain_community.vectorstores import Vectara
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.io import HandleInput, IntInput, SecretStrInput, StrInput
+from lfx.schema.data import Data
+
+if TYPE_CHECKING:
+    from lfx.schema.dataframe import DataFrame
+
+
+class VectaraVectorStoreComponent(LCVectorStoreComponent):
+    """Vectara Vector Store with search capabilities."""
+
+    display_name: str = "Vectara"
+    description: str = "Vectara Vector Store with search capabilities"
+    name = "Vectara"
+    icon = "Vectara"
+
+    inputs = [
+        StrInput(name="vectara_customer_id", display_name="Vectara Customer ID", required=True),
+        StrInput(name="vectara_corpus_id", display_name="Vectara Corpus ID", required=True),
+        SecretStrInput(name="vectara_api_key", display_name="Vectara API Key", required=True),
+        HandleInput(
+            name="embedding",
+            display_name="Embedding",
+            input_types=["Embeddings"],
+        ),
+        *LCVectorStoreComponent.inputs,
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            value=4,
+            advanced=True,
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> Vectara:
+        """Builds the Vectara object."""
+        try:
+            from langchain_community.vectorstores import Vectara
+        except ImportError as e:
+            msg = "Could not import Vectara. Please install it with `pip install langchain-community`."
+            raise ImportError(msg) from e
+
+        vectara = Vectara(
+            vectara_customer_id=self.vectara_customer_id,
+            vectara_corpus_id=self.vectara_corpus_id,
+            vectara_api_key=self.vectara_api_key,
+        )
+
+        self._add_documents_to_vector_store(vectara)
+        return vectara
+
+    def _add_documents_to_vector_store(self, vector_store: Vectara) -> None:
+        """Adds documents to the Vector Store."""
+        ingest_data: list | Data | DataFrame = self.ingest_data
+        if not ingest_data:
+            self.status = "No documents to add to Vectara"
+            return
+
+        # Convert DataFrame to Data if needed using parent's method
+        ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        if documents:
+            self.log(f"Adding {len(documents)} documents to Vectara.")
+            vector_store.add_documents(documents)
+            self.status = f"Added {len(documents)} documents to Vectara"
+        else:
+            self.log("No documents to add to Vectara.")
+            self.status = "No valid documents to add to Vectara"
+
+    def search_documents(self) -> list[Data]:
+        vector_store = self.build_vector_store()
+
+        if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
+            docs = vector_store.similarity_search(
+                query=self.search_query,
+                k=self.number_of_results,
+            )
+
+            data = docs_to_data(docs)
+            self.status = f"Found {len(data)} results for the query: {self.search_query}"
+            return data
+        self.status = "No search query provided"
+        return []

lfx/components/vectorstores/vectara_rag.py

@@ -0,0 +1,164 @@
+from lfx.custom.custom_component.component import Component
+from lfx.field_typing.range_spec import RangeSpec
+from lfx.io import DropdownInput, FloatInput, IntInput, MessageTextInput, Output, SecretStrInput, StrInput
+from lfx.schema.message import Message
+
+
+class VectaraRagComponent(Component):
+    display_name = "Vectara RAG"
+    description = "Vectara's full end to end RAG"
+    documentation = "https://docs.vectara.com/docs"
+    icon = "Vectara"
+    name = "VectaraRAG"
+    SUMMARIZER_PROMPTS = [
+        "vectara-summary-ext-24-05-sml",
+        "vectara-summary-ext-24-05-med-omni",
+        "vectara-summary-ext-24-05-large",
+        "vectara-summary-ext-24-05-med",
+        "vectara-summary-ext-v1.3.0",
+    ]
+
+    RERANKER_TYPES = ["mmr", "rerank_multilingual_v1", "none"]
+
+    RESPONSE_LANGUAGES = [
+        "auto",
+        "eng",
+        "spa",
+        "fra",
+        "zho",
+        "deu",
+        "hin",
+        "ara",
+        "por",
+        "ita",
+        "jpn",
+        "kor",
+        "rus",
+        "tur",
+        "fas",
+        "vie",
+        "tha",
+        "heb",
+        "nld",
+        "ind",
+        "pol",
+        "ukr",
+        "ron",
+        "swe",
+        "ces",
+        "ell",
+        "ben",
+        "msa",
+        "urd",
+    ]
+
+    field_order = ["vectara_customer_id", "vectara_corpus_id", "vectara_api_key", "search_query", "reranker"]
+
+    inputs = [
+        StrInput(name="vectara_customer_id", display_name="Vectara Customer ID", required=True),
+        StrInput(name="vectara_corpus_id", display_name="Vectara Corpus ID", required=True),
+        SecretStrInput(name="vectara_api_key", display_name="Vectara API Key", required=True),
+        MessageTextInput(
+            name="search_query",
+            display_name="Search Query",
+            info="The query to receive an answer on.",
+            tool_mode=True,
+        ),
+        FloatInput(
+            name="lexical_interpolation",
+            display_name="Hybrid Search Factor",
+            range_spec=RangeSpec(min=0.005, max=0.1, step=0.005),
+            value=0.005,
+            advanced=True,
+            info="How much to weigh lexical scores compared to the embedding score. "
+            "0 means lexical search is not used at all, and 1 means only lexical search is used.",
+        ),
+        MessageTextInput(
+            name="filter",
+            display_name="Metadata Filters",
+            value="",
+            advanced=True,
+            info="The filter string to narrow the search to according to metadata attributes.",
+        ),
+        DropdownInput(
+            name="reranker",
+            display_name="Reranker Type",
+            options=RERANKER_TYPES,
+            value=RERANKER_TYPES[0],
+            info="How to rerank the retrieved search results.",
+        ),
+        IntInput(
+            name="reranker_k",
+            display_name="Number of Results to Rerank",
+            value=50,
+            range_spec=RangeSpec(min=1, max=100, step=1),
+            advanced=True,
+        ),
+        FloatInput(
+            name="diversity_bias",
+            display_name="Diversity Bias",
+            value=0.2,
+            range_spec=RangeSpec(min=0, max=1, step=0.01),
+            advanced=True,
+            info="Ranges from 0 to 1, with higher values indicating greater diversity (only applies to MMR reranker).",
+        ),
+        IntInput(
+            name="max_results",
+            display_name="Max Results to Summarize",
+            value=7,
+            range_spec=RangeSpec(min=1, max=100, step=1),
+            advanced=True,
+            info="The maximum number of search results to be available to the prompt.",
+        ),
+        DropdownInput(
+            name="response_lang",
+            display_name="Response Language",
+            options=RESPONSE_LANGUAGES,
+            value="eng",
+            advanced=True,
+            info="Use the ISO 639-1 or 639-3 language code or auto to automatically detect the language.",
+        ),
+        DropdownInput(
+            name="prompt",
+            display_name="Prompt Name",
+            options=SUMMARIZER_PROMPTS,
+            value=SUMMARIZER_PROMPTS[0],
+            advanced=True,
+            info="Only vectara-summary-ext-24-05-sml is for Growth customers; "
+            "all other prompts are for Scale customers only.",
+        ),
+    ]
+
+    outputs = [
+        Output(name="answer", display_name="Answer", method="generate_response"),
+    ]
+
+    def generate_response(
+        self,
+    ) -> Message:
+        text_output = ""
+
+        try:
+            from langchain_community.vectorstores import Vectara
+            from langchain_community.vectorstores.vectara import RerankConfig, SummaryConfig, VectaraQueryConfig
+        except ImportError as e:
+            msg = "Could not import Vectara. Please install it with `pip install langchain-community`."
+            raise ImportError(msg) from e
+
+        vectara = Vectara(self.vectara_customer_id, self.vectara_corpus_id, self.vectara_api_key)
+        rerank_config = RerankConfig(self.reranker, self.reranker_k, self.diversity_bias)
+        summary_config = SummaryConfig(
+            is_enabled=True, max_results=self.max_results, response_lang=self.response_lang, prompt_name=self.prompt
+        )
+        config = VectaraQueryConfig(
+            lambda_val=self.lexical_interpolation,
+            filter=self.filter,
+            summary_config=summary_config,
+            rerank_config=rerank_config,
+        )
+        rag = vectara.as_rag(config)
+        response = rag.invoke(self.search_query, config={"callbacks": self.get_langchain_callbacks()})
+
+        text_output = response["answer"]
+
+        return Message(text=text_output)

lfx/components/vectorstores/weaviate.py

@@ -0,0 +1,89 @@
+import weaviate
+from langchain_community.vectorstores import Weaviate
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.io import BoolInput, HandleInput, IntInput, SecretStrInput, StrInput
+from lfx.schema.data import Data
+
+
+class WeaviateVectorStoreComponent(LCVectorStoreComponent):
+    display_name = "Weaviate"
+    description = "Weaviate Vector Store with search capabilities"
+    name = "Weaviate"
+    icon = "Weaviate"
+
+    inputs = [
+        StrInput(name="url", display_name="Weaviate URL", value="http://localhost:8080", required=True),
+        SecretStrInput(name="api_key", display_name="API Key", required=False),
+        StrInput(
+            name="index_name",
+            display_name="Index Name",
+            required=True,
+            info="Requires capitalized index name.",
+        ),
+        StrInput(name="text_key", display_name="Text Key", value="text", advanced=True),
+        *LCVectorStoreComponent.inputs,
+        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            value=4,
+            advanced=True,
+        ),
+        BoolInput(name="search_by_text", display_name="Search By Text", advanced=True),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> Weaviate:
+        if self.api_key:
+            auth_config = weaviate.AuthApiKey(api_key=self.api_key)
+            client = weaviate.Client(url=self.url, auth_client_secret=auth_config)
+        else:
+            client = weaviate.Client(url=self.url)
+
+        if self.index_name != self.index_name.capitalize():
+            msg = f"Weaviate requires the index name to be capitalized. Use: {self.index_name.capitalize()}"
+            raise ValueError(msg)
+
+        # Convert DataFrame to Data if needed using parent's method
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in self.ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        if documents and self.embedding:
+            return Weaviate.from_documents(
+                client=client,
+                index_name=self.index_name,
+                documents=documents,
+                embedding=self.embedding,
+                by_text=self.search_by_text,
+            )
+
+        return Weaviate(
+            client=client,
+            index_name=self.index_name,
+            text_key=self.text_key,
+            embedding=self.embedding,
+            by_text=self.search_by_text,
+        )
+
+    def search_documents(self) -> list[Data]:
+        vector_store = self.build_vector_store()
+
+        if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
+            docs = vector_store.similarity_search(
+                query=self.search_query,
+                k=self.number_of_results,
+            )
+
+            data = docs_to_data(docs)
+            self.status = data
+            return data
+        return []