lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/datastax/hcd.py

@@ -0,0 +1,314 @@
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.inputs.inputs import DictInput, FloatInput
+from lfx.io import (
+    BoolInput,
+    DropdownInput,
+    HandleInput,
+    IntInput,
+    MultilineInput,
+    SecretStrInput,
+    StrInput,
+)
+from lfx.schema.data import Data
+
+
+class HCDVectorStoreComponent(LCVectorStoreComponent):
+    display_name: str = "Hyper-Converged Database"
+    description: str = "Implementation of Vector Store using Hyper-Converged Database (HCD) with search capabilities"
+    name = "HCD"
+    icon: str = "HCD"
+
+    inputs = [
+        StrInput(
+            name="collection_name",
+            display_name="Collection Name",
+            info="The name of the collection within HCD where the vectors will be stored.",
+            required=True,
+        ),
+        StrInput(
+            name="username",
+            display_name="HCD Username",
+            info="Authentication username for accessing HCD.",
+            value="hcd-superuser",
+            required=True,
+        ),
+        SecretStrInput(
+            name="password",
+            display_name="HCD Password",
+            info="Authentication password for accessing HCD.",
+            value="HCD_PASSWORD",
+            required=True,
+        ),
+        SecretStrInput(
+            name="api_endpoint",
+            display_name="HCD API Endpoint",
+            info="API endpoint URL for the HCD service.",
+            value="HCD_API_ENDPOINT",
+            required=True,
+        ),
+        *LCVectorStoreComponent.inputs,
+        StrInput(
+            name="namespace",
+            display_name="Namespace",
+            info="Optional namespace within HCD to use for the collection.",
+            value="default_namespace",
+            advanced=True,
+        ),
+        MultilineInput(
+            name="ca_certificate",
+            display_name="CA Certificate",
+            info="Optional CA certificate for TLS connections to HCD.",
+            advanced=True,
+        ),
+        DropdownInput(
+            name="metric",
+            display_name="Metric",
+            info="Optional distance metric for vector comparisons in the vector store.",
+            options=["cosine", "dot_product", "euclidean"],
+            advanced=True,
+        ),
+        IntInput(
+            name="batch_size",
+            display_name="Batch Size",
+            info="Optional number of data to process in a single batch.",
+            advanced=True,
+        ),
+        IntInput(
+            name="bulk_insert_batch_concurrency",
+            display_name="Bulk Insert Batch Concurrency",
+            info="Optional concurrency level for bulk insert operations.",
+            advanced=True,
+        ),
+        IntInput(
+            name="bulk_insert_overwrite_concurrency",
+            display_name="Bulk Insert Overwrite Concurrency",
+            info="Optional concurrency level for bulk insert operations that overwrite existing data.",
+            advanced=True,
+        ),
+        IntInput(
+            name="bulk_delete_concurrency",
+            display_name="Bulk Delete Concurrency",
+            info="Optional concurrency level for bulk delete operations.",
+            advanced=True,
+        ),
+        DropdownInput(
+            name="setup_mode",
+            display_name="Setup Mode",
+            info="Configuration mode for setting up the vector store, with options like 'Sync', 'Async', or 'Off'.",
+            options=["Sync", "Async", "Off"],
+            advanced=True,
+            value="Sync",
+        ),
+        BoolInput(
+            name="pre_delete_collection",
+            display_name="Pre Delete Collection",
+            info="Boolean flag to determine whether to delete the collection before creating a new one.",
+            advanced=True,
+        ),
+        StrInput(
+            name="metadata_indexing_include",
+            display_name="Metadata Indexing Include",
+            info="Optional list of metadata fields to include in the indexing.",
+            advanced=True,
+        ),
+        HandleInput(
+            name="embedding",
+            display_name="Embedding or Astra Vectorize",
+            input_types=["Embeddings", "dict"],
+            # TODO: This should be optional, but need to refactor langchain-astradb first.
+            info="Allows either an embedding model or an Astra Vectorize configuration.",
+        ),
+        StrInput(
+            name="metadata_indexing_exclude",
+            display_name="Metadata Indexing Exclude",
+            info="Optional list of metadata fields to exclude from the indexing.",
+            advanced=True,
+        ),
+        StrInput(
+            name="collection_indexing_policy",
+            display_name="Collection Indexing Policy",
+            info="Optional dictionary defining the indexing policy for the collection.",
+            advanced=True,
+        ),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            advanced=True,
+            value=4,
+        ),
+        DropdownInput(
+            name="search_type",
+            display_name="Search Type",
+            info="Search type to use",
+            options=["Similarity", "Similarity with score threshold", "MMR (Max Marginal Relevance)"],
+            value="Similarity",
+            advanced=True,
+        ),
+        FloatInput(
+            name="search_score_threshold",
+            display_name="Search Score Threshold",
+            info="Minimum similarity score threshold for search results. "
+            "(when using 'Similarity with score threshold')",
+            value=0,
+            advanced=True,
+        ),
+        DictInput(
+            name="search_filter",
+            display_name="Search Metadata Filter",
+            info="Optional dictionary of filters to apply to the search query.",
+            advanced=True,
+            is_list=True,
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self):
+        try:
+            from langchain_astradb import AstraDBVectorStore
+            from langchain_astradb.utils.astradb import SetupMode
+        except ImportError as e:
+            msg = (
+                "Could not import langchain Astra DB integration package. "
+                "Please install it with `pip install langchain-astradb`."
+            )
+            raise ImportError(msg) from e
+
+        try:
+            from astrapy.authentication import UsernamePasswordTokenProvider
+            from astrapy.constants import Environment
+        except ImportError as e:
+            msg = "Could not import astrapy integration package. Please install it with `pip install astrapy`."
+            raise ImportError(msg) from e
+
+        try:
+            if not self.setup_mode:
+                self.setup_mode = self._inputs["setup_mode"].options[0]
+
+            setup_mode_value = SetupMode[self.setup_mode.upper()]
+        except KeyError as e:
+            msg = f"Invalid setup mode: {self.setup_mode}"
+            raise ValueError(msg) from e
+
+        if not isinstance(self.embedding, dict):
+            embedding_dict = {"embedding": self.embedding}
+        else:
+            from astrapy.info import VectorServiceOptions
+
+            dict_options = self.embedding.get("collection_vector_service_options", {})
+            dict_options["authentication"] = {
+                k: v for k, v in dict_options.get("authentication", {}).items() if k and v
+            }
+            dict_options["parameters"] = {k: v for k, v in dict_options.get("parameters", {}).items() if k and v}
+            embedding_dict = {"collection_vector_service_options": VectorServiceOptions.from_dict(dict_options)}
+            collection_embedding_api_key = self.embedding.get("collection_embedding_api_key")
+            if collection_embedding_api_key:
+                embedding_dict["collection_embedding_api_key"] = collection_embedding_api_key
+
+        token_provider = UsernamePasswordTokenProvider(self.username, self.password)
+        vector_store_kwargs = {
+            **embedding_dict,
+            "collection_name": self.collection_name,
+            "token": token_provider,
+            "api_endpoint": self.api_endpoint,
+            "namespace": self.namespace,
+            "metric": self.metric or None,
+            "batch_size": self.batch_size or None,
+            "bulk_insert_batch_concurrency": self.bulk_insert_batch_concurrency or None,
+            "bulk_insert_overwrite_concurrency": self.bulk_insert_overwrite_concurrency or None,
+            "bulk_delete_concurrency": self.bulk_delete_concurrency or None,
+            "setup_mode": setup_mode_value,
+            "pre_delete_collection": self.pre_delete_collection or False,
+            "environment": Environment.HCD,
+        }
+
+        if self.metadata_indexing_include:
+            vector_store_kwargs["metadata_indexing_include"] = self.metadata_indexing_include
+        elif self.metadata_indexing_exclude:
+            vector_store_kwargs["metadata_indexing_exclude"] = self.metadata_indexing_exclude
+        elif self.collection_indexing_policy:
+            vector_store_kwargs["collection_indexing_policy"] = self.collection_indexing_policy
+
+        try:
+            vector_store = AstraDBVectorStore(**vector_store_kwargs)
+        except Exception as e:
+            msg = f"Error initializing AstraDBVectorStore: {e}"
+            raise ValueError(msg) from e
+
+        self._add_documents_to_vector_store(vector_store)
+        return vector_store
+
+    def _add_documents_to_vector_store(self, vector_store) -> None:
+        # Convert DataFrame to Data if needed using parent's method
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in self.ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                msg = "Vector Store Inputs must be Data objects."
+                raise TypeError(msg)
+
+        if documents:
+            self.log(f"Adding {len(documents)} documents to the Vector Store.")
+            try:
+                vector_store.add_documents(documents)
+            except Exception as e:
+                msg = f"Error adding documents to AstraDBVectorStore: {e}"
+                raise ValueError(msg) from e
+        else:
+            self.log("No documents to add to the Vector Store.")
+
+    def _map_search_type(self) -> str:
+        if self.search_type == "Similarity with score threshold":
+            return "similarity_score_threshold"
+        if self.search_type == "MMR (Max Marginal Relevance)":
+            return "mmr"
+        return "similarity"
+
+    def _build_search_args(self):
+        args = {
+            "k": self.number_of_results,
+            "score_threshold": self.search_score_threshold,
+        }
+
+        if self.search_filter:
+            clean_filter = {k: v for k, v in self.search_filter.items() if k and v}
+            if len(clean_filter) > 0:
+                args["filter"] = clean_filter
+        return args
+
+    def search_documents(self) -> list[Data]:
+        vector_store = self.build_vector_store()
+
+        self.log(f"Search query: {self.search_query}")
+        self.log(f"Search type: {self.search_type}")
+        self.log(f"Number of results: {self.number_of_results}")
+
+        if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
+            try:
+                search_type = self._map_search_type()
+                search_args = self._build_search_args()
+
+                docs = vector_store.search(query=self.search_query, search_type=search_type, **search_args)
+            except Exception as e:
+                msg = f"Error performing search in AstraDBVectorStore: {e}"
+                raise ValueError(msg) from e
+
+            self.log(f"Retrieved documents: {len(docs)}")
+
+            data = docs_to_data(docs)
+            self.log(f"Converted documents to data: {len(data)}")
+            self.status = data
+            return data
+        self.log("No search input provided. Skipping search.")
+        return []
+
+    def get_retriever_kwargs(self):
+        search_args = self._build_search_args()
+        return {
+            "search_type": self._map_search_type(),
+            "search_kwargs": search_args,
+        }

lfx/components/datastax/list_assistants.py

@@ -0,0 +1,25 @@
+from lfx.base.astra_assistants.util import get_patched_openai_client
+from lfx.custom.custom_component.component_with_cache import ComponentWithCache
+from lfx.schema.message import Message
+from lfx.template.field.base import Output
+
+
+class AssistantsListAssistants(ComponentWithCache):
+    display_name = "List Assistants"
+    description = "Returns a list of assistant id's"
+    icon = "AstraDB"
+    outputs = [
+        Output(display_name="Assistants", name="assistants", method="process_inputs"),
+    ]
+
+    def __init__(self, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self.client = get_patched_openai_client(self._shared_component_cache)
+
+    def process_inputs(self) -> Message:
+        assistants = self.client.beta.assistants.list().data
+        id_list = [assistant.id for assistant in assistants]
+        return Message(
+            # get text from list
+            text="\n".join(id_list)
+        )

lfx/components/datastax/run.py

@@ -0,0 +1,89 @@
+from typing import Any
+
+from openai.lib.streaming import AssistantEventHandler
+
+from lfx.base.astra_assistants.util import get_patched_openai_client
+from lfx.custom.custom_component.component_with_cache import ComponentWithCache
+from lfx.inputs.inputs import MultilineInput
+from lfx.schema.dotdict import dotdict
+from lfx.schema.message import Message
+from lfx.template.field.base import Output
+
+
+class AssistantsRun(ComponentWithCache):
+    display_name = "Run Assistant"
+    description = "Executes an Assistant Run against a thread"
+    icon = "AstraDB"
+
+    def __init__(self, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self.client = get_patched_openai_client(self._shared_component_cache)
+        self.thread_id = None
+
+    def update_build_config(
+        self,
+        build_config: dotdict,
+        field_value: Any,
+        field_name: str | None = None,
+    ) -> None:
+        if field_name == "thread_id":
+            if field_value is None:
+                thread = self.client.beta.threads.create()
+                self.thread_id = thread.id
+            build_config["thread_id"] = field_value
+
+    inputs = [
+        MultilineInput(
+            name="assistant_id",
+            display_name="Assistant ID",
+            info=(
+                "The ID of the assistant to run. \n\n"
+                "Can be retrieved using the List Assistants component or created with the Create Assistant component."
+            ),
+        ),
+        MultilineInput(
+            name="user_message",
+            display_name="User Message",
+            info="User message to pass to the run.",
+        ),
+        MultilineInput(
+            name="thread_id",
+            display_name="Thread ID",
+            required=False,
+            info="Thread ID to use with the run. If not provided, a new thread will be created.",
+        ),
+        MultilineInput(
+            name="env_set",
+            display_name="Environment Set",
+            info="Dummy input to allow chaining with Dotenv Component.",
+        ),
+    ]
+
+    outputs = [Output(display_name="Assistant Response", name="assistant_response", method="process_inputs")]
+
+    def process_inputs(self) -> Message:
+        text = ""
+
+        if self.thread_id is None:
+            thread = self.client.beta.threads.create()
+            self.thread_id = thread.id
+
+        # add the user message
+        self.client.beta.threads.messages.create(thread_id=self.thread_id, role="user", content=self.user_message)
+
+        class EventHandler(AssistantEventHandler):
+            def __init__(self) -> None:
+                super().__init__()
+
+            def on_exception(self, exception: Exception) -> None:
+                raise exception
+
+        event_handler = EventHandler()
+        with self.client.beta.threads.runs.create_and_stream(
+            thread_id=self.thread_id,
+            assistant_id=self.assistant_id,
+            event_handler=event_handler,
+        ) as stream:
+            for part in stream.text_deltas:
+                text += part
+        return Message(text=text)

lfx/components/deactivated/__init__.py

@@ -0,0 +1,15 @@
+from .extract_key_from_data import ExtractKeyFromDataComponent
+from .list_flows import ListFlowsComponent
+from .merge_data import MergeDataComponent
+from .selective_passthrough import SelectivePassThroughComponent
+from .split_text import SplitTextComponent
+from .sub_flow import SubFlowComponent
+
+__all__ = [
+    "ExtractKeyFromDataComponent",
+    "ListFlowsComponent",
+    "MergeDataComponent",
+    "SelectivePassThroughComponent",
+    "SplitTextComponent",
+    "SubFlowComponent",
+]

lfx/components/deactivated/amazon_kendra.py

@@ -0,0 +1,66 @@
+# mypy: disable-error-code="attr-defined"
+from langchain_community.retrievers import AmazonKendraRetriever
+
+from lfx.base.vectorstores.model import check_cached_vector_store
+from lfx.custom.custom_component.custom_component import CustomComponent
+from lfx.io import DictInput, IntInput, StrInput
+
+
+class AmazonKendraRetrieverComponent(CustomComponent):
+    display_name: str = "Amazon Kendra Retriever"
+    description: str = "Retriever that uses the Amazon Kendra API."
+    name = "AmazonKendra"
+    icon = "Amazon"
+    legacy = True
+
+    inputs = [
+        StrInput(
+            name="index_id",
+            display_name="Index ID",
+        ),
+        StrInput(
+            name="region_name",
+            display_name="Region Name",
+        ),
+        StrInput(
+            name="credentials_profile_name",
+            display_name="Credentials Profile Name",
+        ),
+        DictInput(
+            name="attribute_filter",
+            display_name="Attribute Filter",
+        ),
+        IntInput(
+            name="top_k",
+            display_name="Top K",
+            value=3,
+        ),
+        DictInput(
+            name="user_context",
+            display_name="User Context",
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> AmazonKendraRetriever:
+        """Builds the Amazon Kendra Retriever."""
+        try:
+            from langchain_community.retrievers import AmazonKendraRetriever
+        except ImportError as e:
+            msg = "Could not import AmazonKendraRetriever. Please install it with `pip install langchain-community`."
+            raise ImportError(msg) from e
+
+        try:
+            output = AmazonKendraRetriever(
+                index_id=self.index_id,
+                top_k=self.top_k,
+                region_name=self.region_name,
+                credentials_profile_name=self.credentials_profile_name,
+                attribute_filter=self.attribute_filter,
+                user_context=self.user_context,
+            )
+        except Exception as e:
+            msg = "Could not connect to AmazonKendra API."
+            raise ValueError(msg) from e
+
+        return output

lfx/components/deactivated/chat_litellm_model.py

@@ -0,0 +1,158 @@
+from langchain_community.chat_models.litellm import ChatLiteLLM, ChatLiteLLMException
+
+from lfx.base.constants import STREAM_INFO_TEXT
+from lfx.base.models.model import LCModelComponent
+from lfx.field_typing import LanguageModel
+from lfx.io import (
+    BoolInput,
+    DictInput,
+    DropdownInput,
+    FloatInput,
+    IntInput,
+    MessageInput,
+    SecretStrInput,
+    StrInput,
+)
+
+
+class ChatLiteLLMModelComponent(LCModelComponent):
+    display_name = "LiteLLM"
+    description = "`LiteLLM` collection of large language models."
+    documentation = "https://python.langchain.com/docs/integrations/chat/litellm"
+    icon = "🚄"
+
+    inputs = [
+        MessageInput(name="input_value", display_name="Input"),
+        StrInput(
+            name="model",
+            display_name="Model name",
+            advanced=False,
+            required=True,
+            info="The name of the model to use. For example, `gpt-3.5-turbo`.",
+        ),
+        SecretStrInput(
+            name="api_key",
+            display_name="API Key",
+            advanced=False,
+            required=False,
+        ),
+        DropdownInput(
+            name="provider",
+            display_name="Provider",
+            info="The provider of the API key.",
+            options=[
+                "OpenAI",
+                "Azure",
+                "Anthropic",
+                "Replicate",
+                "Cohere",
+                "OpenRouter",
+            ],
+        ),
+        FloatInput(
+            name="temperature",
+            display_name="Temperature",
+            advanced=False,
+            required=False,
+            value=0.7,
+        ),
+        DictInput(
+            name="kwargs",
+            display_name="Kwargs",
+            advanced=True,
+            required=False,
+            is_list=True,
+            value={},
+        ),
+        DictInput(
+            name="model_kwargs",
+            display_name="Model kwargs",
+            advanced=True,
+            required=False,
+            is_list=True,
+            value={},
+        ),
+        FloatInput(name="top_p", display_name="Top p", advanced=True, required=False, value=0.5),
+        IntInput(name="top_k", display_name="Top k", advanced=True, required=False, value=35),
+        IntInput(
+            name="n",
+            display_name="N",
+            advanced=True,
+            required=False,
+            info="Number of chat completions to generate for each prompt. "
+            "Note that the API may not return the full n completions if duplicates are generated.",
+            value=1,
+        ),
+        IntInput(
+            name="max_tokens",
+            display_name="Max tokens",
+            advanced=False,
+            value=256,
+            info="The maximum number of tokens to generate for each chat completion.",
+        ),
+        IntInput(
+            name="max_retries",
+            display_name="Max retries",
+            advanced=True,
+            required=False,
+            value=6,
+        ),
+        BoolInput(
+            name="verbose",
+            display_name="Verbose",
+            advanced=True,
+            required=False,
+            value=False,
+        ),
+        BoolInput(
+            name="stream",
+            display_name="Stream",
+            info=STREAM_INFO_TEXT,
+            advanced=True,
+        ),
+        StrInput(
+            name="system_message",
+            display_name="System Message",
+            info="System message to pass to the model.",
+            advanced=True,
+        ),
+    ]
+
+    def build_model(self) -> LanguageModel:  # type: ignore[type-var]
+        try:
+            import litellm
+
+            litellm.drop_params = True
+            litellm.set_verbose = self.verbose
+        except ImportError as e:
+            msg = "Could not import litellm python package. Please install it with `pip install litellm`"
+            raise ChatLiteLLMException(msg) from e
+        # Remove empty keys
+        if "" in self.kwargs:
+            del self.kwargs[""]
+        if "" in self.model_kwargs:
+            del self.model_kwargs[""]
+        # Report missing fields for Azure provider
+        if self.provider == "Azure":
+            if "api_base" not in self.kwargs:
+                msg = "Missing api_base on kwargs"
+                raise ValueError(msg)
+            if "api_version" not in self.model_kwargs:
+                msg = "Missing api_version on model_kwargs"
+                raise ValueError(msg)
+        output = ChatLiteLLM(
+            model=f"{self.provider.lower()}/{self.model}",
+            client=None,
+            streaming=self.stream,
+            temperature=self.temperature,
+            model_kwargs=self.model_kwargs if self.model_kwargs is not None else {},
+            top_p=self.top_p,
+            top_k=self.top_k,
+            n=self.n,
+            max_tokens=self.max_tokens,
+            max_retries=self.max_retries,
+            **self.kwargs,
+        )
+        output.client.api_key = self.api_key
+
+        return output

lfx/components/deactivated/code_block_extractor.py

@@ -0,0 +1,26 @@
+import re
+
+from lfx.custom.custom_component.component import Component
+from lfx.field_typing import Input, Output, Text
+
+
+class CodeBlockExtractor(Component):
+    display_name = "Code Block Extractor"
+    description = "Extracts code block from text."
+    name = "CodeBlockExtractor"
+
+    inputs = [Input(name="text", field_type=Text, description="Text to extract code blocks from.")]
+
+    outputs = [Output(name="code_block", display_name="Code Block", method="get_code_block")]
+
+    def get_code_block(self) -> Text:
+        text = self.text.strip()
+        # Extract code block
+        # It may start with ``` or ```language
+        # It may end with ```
+        pattern = r"^```(?:\w+)?\s*\n(.*?)(?=^```)```"
+        match = re.search(pattern, text, re.MULTILINE)
+        code_block = ""
+        if match:
+            code_block = match.group(1)
+        return code_block