lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/base/tools/run_flow.py

@@ -0,0 +1,224 @@
+from abc import abstractmethod
+from typing import TYPE_CHECKING
+
+from lfx.custom.custom_component.component import Component, get_component_toolkit
+from lfx.field_typing import Tool
+from lfx.graph.graph.base import Graph
+from lfx.graph.vertex.base import Vertex
+from lfx.helpers.flow import get_flow_inputs
+from lfx.inputs.inputs import DropdownInput, InputTypes, MessageInput
+from lfx.log.logger import logger
+from lfx.schema.data import Data
+from lfx.schema.dataframe import DataFrame
+from lfx.schema.dotdict import dotdict
+from lfx.schema.message import Message
+from lfx.template.field.base import Output
+
+if TYPE_CHECKING:
+    from lfx.base.tools.component_tool import ComponentToolkit
+
+
+class RunFlowBaseComponent(Component):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.add_tool_output = True
+
+    _base_inputs: list[InputTypes] = [
+        DropdownInput(
+            name="flow_name_selected",
+            display_name="Flow Name",
+            info="The name of the flow to run.",
+            options=[],
+            real_time_refresh=True,
+            value=None,
+        ),
+        MessageInput(
+            name="session_id",
+            display_name="Session ID",
+            info="The session ID to run the flow in.",
+            value="",
+            advanced=True,
+        ),
+    ]
+    _base_outputs: list[Output] = [
+        Output(
+            name="flow_outputs_data",
+            display_name="Flow Data Output",
+            method="data_output",
+            hidden=True,
+            group_outputs=True,
+            tool_mode=False,  # This output is not intended to be used as a tool, so tool_mode is disabled.
+        ),
+        Output(
+            name="flow_outputs_dataframe",
+            display_name="Flow Dataframe Output",
+            method="dataframe_output",
+            hidden=True,
+            group_outputs=True,
+            tool_mode=False,  # This output is not intended to be used as a tool, so tool_mode is disabled.
+        ),
+        Output(
+            name="flow_outputs_message", group_outputs=True, display_name="Flow Message Output", method="message_output"
+        ),
+    ]
+    default_keys = ["code", "_type", "flow_name_selected", "session_id"]
+    FLOW_INPUTS: list[dotdict] = []
+    flow_tweak_data: dict = {}
+
+    @abstractmethod
+    async def run_flow_with_tweaks(self) -> list[Data]:
+        """Run the flow with tweaks."""
+
+    async def data_output(self) -> Data:
+        """Return the data output."""
+        run_outputs = await self.run_flow_with_tweaks()
+        first_output = run_outputs[0]
+
+        if isinstance(first_output, Data):
+            return first_output
+
+        # just adaptive output Message
+        _, message_result = next(iter(run_outputs[0].outputs[0].results.items()))
+        message_data = message_result.data
+        return Data(data=message_data)
+
+    async def dataframe_output(self) -> DataFrame:
+        """Return the dataframe output."""
+        run_outputs = await self.run_flow_with_tweaks()
+        first_output = run_outputs[0]
+
+        if isinstance(first_output, DataFrame):
+            return first_output
+
+        # just adaptive output Message
+        _, message_result = next(iter(run_outputs[0].outputs[0].results.items()))
+        message_data = message_result.data
+        return DataFrame(data=message_data if isinstance(message_data, list) else [message_data])
+
+    async def message_output(self) -> Message:
+        """Return the message output."""
+        run_outputs = await self.run_flow_with_tweaks()
+        _, message_result = next(iter(run_outputs[0].outputs[0].results.items()))
+        if isinstance(message_result, Message):
+            return message_result
+        if isinstance(message_result, str):
+            return Message(text=message_result)
+        return Message(text=message_result.data["text"])
+
+    async def get_flow_names(self) -> list[str]:
+        # TODO: get flfow ID with flow name
+        flow_data = await self.alist_flows()
+        return [flow_data.data["name"] for flow_data in flow_data]
+
+    async def get_flow(self, flow_name_selected: str) -> Data | None:
+        # get flow from flow id
+        flow_datas = await self.alist_flows()
+        for flow_data in flow_datas:
+            if flow_data.data["name"] == flow_name_selected:
+                return flow_data
+        return None
+
+    async def get_graph(self, flow_name_selected: str | None = None) -> Graph:
+        if flow_name_selected:
+            flow_data = await self.get_flow(flow_name_selected)
+            if flow_data:
+                return Graph.from_payload(flow_data.data["data"])
+            msg = "Flow not found"
+            raise ValueError(msg)
+        # Ensure a Graph is always returned or an exception is raised
+        msg = "No valid flow JSON or flow name selected."
+        raise ValueError(msg)
+
+    def get_new_fields_from_graph(self, graph: Graph) -> list[dotdict]:
+        inputs = get_flow_inputs(graph)
+        return self.get_new_fields(inputs)
+
+    def update_build_config_from_graph(self, build_config: dotdict, graph: Graph):
+        try:
+            # Get all inputs from the graph
+            new_fields = self.get_new_fields_from_graph(graph)
+            old_fields = self.get_old_fields(build_config, new_fields)
+            self.delete_fields(build_config, old_fields)
+            build_config = self.add_new_fields(build_config, new_fields)
+
+        except Exception as e:
+            msg = "Error updating build config from graph"
+            logger.exception(msg)
+            raise RuntimeError(msg) from e
+
+    def get_new_fields(self, inputs_vertex: list[Vertex]) -> list[dotdict]:
+        new_fields: list[dotdict] = []
+
+        for vertex in inputs_vertex:
+            field_template = vertex.data.get("node", {}).get("template", {})
+            field_order = vertex.data.get("node", {}).get("field_order", [])
+            if field_order and field_template:
+                new_vertex_inputs = [
+                    dotdict(
+                        {
+                            **field_template[input_name],
+                            "display_name": vertex.display_name + " - " + field_template[input_name]["display_name"],
+                            "name": f"{vertex.id}~{input_name}",
+                            "tool_mode": not (field_template[input_name].get("advanced", False)),
+                        }
+                    )
+                    for input_name in field_order
+                    if input_name in field_template
+                ]
+                new_fields += new_vertex_inputs
+        return new_fields
+
+    def add_new_fields(self, build_config: dotdict, new_fields: list[dotdict]) -> dotdict:
+        """Add new fields to the build_config."""
+        for field in new_fields:
+            build_config[field["name"]] = field
+        return build_config
+
+    def delete_fields(self, build_config: dotdict, fields: dict | list[str]) -> None:
+        """Delete specified fields from build_config."""
+        if isinstance(fields, dict):
+            fields = list(fields.keys())
+        for field in fields:
+            build_config.pop(field, None)
+
+    def get_old_fields(self, build_config: dotdict, new_fields: list[dotdict]) -> list[str]:
+        """Get fields that are in build_config but not in new_fields."""
+        return [
+            field
+            for field in build_config
+            if field not in [new_field["name"] for new_field in new_fields] + self.default_keys
+        ]
+
+    async def get_required_data(self, flow_name_selected):
+        self.flow_data = await self.alist_flows()
+        for flow_data in self.flow_data:
+            if flow_data.data["name"] == flow_name_selected:
+                graph = Graph.from_payload(flow_data.data["data"])
+                new_fields = self.get_new_fields_from_graph(graph)
+                new_fields = self.update_input_types(new_fields)
+
+                return flow_data.data["description"], [field for field in new_fields if field.get("tool_mode") is True]
+        return None
+
+    def update_input_types(self, fields: list[dotdict]) -> list[dotdict]:
+        for field in fields:
+            if isinstance(field, dict):
+                if field.get("input_types") is None:
+                    field["input_types"] = []
+            elif hasattr(field, "input_types") and field.input_types is None:
+                field.input_types = []
+        return fields
+
+    async def _get_tools(self) -> list[Tool]:
+        component_toolkit: type[ComponentToolkit] = get_component_toolkit()
+        flow_description, tool_mode_inputs = await self.get_required_data(self.flow_name_selected)
+        # # convert list of dicts to list of dotdicts
+        tool_mode_inputs = [dotdict(field) for field in tool_mode_inputs]
+        return component_toolkit(component=self).get_tools(
+            tool_name=f"{self.flow_name_selected}_tool",
+            tool_description=(
+                f"Tool designed to execute the flow '{self.flow_name_selected}'. Flow details: {flow_description}."
+            ),
+            callbacks=self.get_langchain_callbacks(),
+            flow_mode_inputs=tool_mode_inputs,
+        )

lfx/base/vectorstores/model.py

@@ -0,0 +1,193 @@
+from abc import abstractmethod
+from functools import wraps
+from typing import TYPE_CHECKING, Any
+
+from lfx.custom.custom_component.component import Component
+from lfx.field_typing import Text, VectorStore
+from lfx.helpers.data import docs_to_data
+from lfx.inputs.inputs import BoolInput
+from lfx.io import HandleInput, Output, QueryInput
+from lfx.schema.data import Data
+from lfx.schema.dataframe import DataFrame
+
+if TYPE_CHECKING:
+    from langchain_core.documents import Document
+
+
+def check_cached_vector_store(f):
+    """Decorator to check for cached vector stores, and returns them if they exist.
+
+    Note: caching only occurs during the execution of a component - they do not persist
+    across separate invocations of the component. This method exists so that components with
+    multiple output methods share the same vector store during the same invocation of the
+    component.
+    """
+
+    @wraps(f)
+    def check_cached(self, *args, **kwargs):
+        should_cache = getattr(self, "should_cache_vector_store", True)
+
+        if should_cache and self._cached_vector_store is not None:
+            return self._cached_vector_store
+
+        result = f(self, *args, **kwargs)
+        self._cached_vector_store = result
+        return result
+
+    check_cached.is_cached_vector_store_checked = True
+    return check_cached
+
+
+class LCVectorStoreComponent(Component):
+    # Used to ensure a single vector store is built for each run of the flow
+    _cached_vector_store: VectorStore | None = None
+
+    def __init_subclass__(cls, **kwargs):
+        """Enforces the check cached decorator on all subclasses."""
+        super().__init_subclass__(**kwargs)
+        if hasattr(cls, "build_vector_store"):
+            method = cls.build_vector_store
+            if not hasattr(method, "is_cached_vector_store_checked"):
+                msg = (
+                    f"The method 'build_vector_store' in class {cls.__name__} "
+                    "must be decorated with @check_cached_vector_store"
+                )
+                raise TypeError(msg)
+
+    trace_type = "retriever"
+
+    inputs = [
+        HandleInput(
+            name="ingest_data",
+            display_name="Ingest Data",
+            input_types=["Data", "DataFrame"],
+            is_list=True,
+        ),
+        QueryInput(
+            name="search_query",
+            display_name="Search Query",
+            info="Enter a query to run a similarity search.",
+            placeholder="Enter a query...",
+            tool_mode=True,
+        ),
+        BoolInput(
+            name="should_cache_vector_store",
+            display_name="Cache Vector Store",
+            value=True,
+            advanced=True,
+            info="If True, the vector store will be cached for the current build of the component. "
+            "This is useful for components that have multiple output methods and want to share the same vector store.",
+        ),
+    ]
+
+    outputs = [
+        Output(
+            display_name="Search Results",
+            name="search_results",
+            method="search_documents",
+        ),
+        Output(display_name="DataFrame", name="dataframe", method="as_dataframe"),
+    ]
+
+    def _validate_outputs(self) -> None:
+        # At least these three outputs must be defined
+        required_output_methods = [
+            "search_documents",
+            "build_vector_store",
+        ]
+        output_names = [output.name for output in self.outputs]
+        for method_name in required_output_methods:
+            if method_name not in output_names:
+                msg = f"Output with name '{method_name}' must be defined."
+                raise ValueError(msg)
+            if not hasattr(self, method_name):
+                msg = f"Method '{method_name}' must be defined."
+                raise ValueError(msg)
+
+    def _prepare_ingest_data(self) -> list[Any]:
+        """Prepares ingest_data by converting DataFrame to Data if needed."""
+        ingest_data: list | Data | DataFrame = self.ingest_data
+        if not ingest_data:
+            return []
+
+        if not isinstance(ingest_data, list):
+            ingest_data = [ingest_data]
+
+        result = []
+
+        for _input in ingest_data:
+            if isinstance(_input, DataFrame):
+                result.extend(_input.to_data_list())
+            else:
+                result.append(_input)
+        return result
+
+    def search_with_vector_store(
+        self,
+        input_value: Text,
+        search_type: str,
+        vector_store: VectorStore,
+        k=10,
+        **kwargs,
+    ) -> list[Data]:
+        """Search for data in the vector store based on the input value and search type.
+
+        Args:
+            input_value (Text): The input value to search for.
+            search_type (str): The type of search to perform.
+            vector_store (VectorStore): The vector store to search in.
+            k (int): The number of results to return.
+            **kwargs: Additional keyword arguments to pass to the vector store search method.
+
+        Returns:
+            List[Data]: A list of data matching the search criteria.
+
+        Raises:
+            ValueError: If invalid inputs are provided.
+        """
+        docs: list[Document] = []
+        if input_value and isinstance(input_value, str) and hasattr(vector_store, "search"):
+            docs = vector_store.search(query=input_value, search_type=search_type.lower(), k=k, **kwargs)
+        else:
+            msg = "Invalid inputs provided."
+            raise ValueError(msg)
+        data = docs_to_data(docs)
+        self.status = data
+        return data
+
+    def search_documents(self) -> list[Data]:
+        """Search for documents in the vector store."""
+        if self._cached_vector_store is not None:
+            vector_store = self._cached_vector_store
+        else:
+            vector_store = self.build_vector_store()
+            self._cached_vector_store = vector_store
+
+        search_query: str = self.search_query
+        if not search_query:
+            self.status = ""
+            return []
+
+        self.log(f"Search input: {search_query}")
+        self.log(f"Search type: {self.search_type}")
+        self.log(f"Number of results: {self.number_of_results}")
+
+        search_results = self.search_with_vector_store(
+            search_query, self.search_type, vector_store, k=self.number_of_results
+        )
+        self.status = search_results
+        return search_results
+
+    def as_dataframe(self) -> DataFrame:
+        return DataFrame(self.search_documents())
+
+    def get_retriever_kwargs(self):
+        """Get the retriever kwargs. Implementations can override this method to provide custom retriever kwargs."""
+        return {}
+
+    @abstractmethod
+    @check_cached_vector_store
+    def build_vector_store(self) -> VectorStore:
+        """Builds the Vector Store object."""
+        msg = "build_vector_store method must be implemented."
+        raise NotImplementedError(msg)

lfx/base/vectorstores/utils.py

@@ -0,0 +1,22 @@
+from lfx.schema.data import Data
+
+
+def chroma_collection_to_data(collection_dict: dict):
+    """Converts a collection of chroma vectors into a list of data.
+
+    Args:
+        collection_dict (dict): A dictionary containing the collection of chroma vectors.
+
+    Returns:
+        list: A list of data, where each record represents a document in the collection.
+    """
+    data = []
+    for i, doc in enumerate(collection_dict["documents"]):
+        data_dict = {
+            "id": collection_dict["ids"][i],
+            "text": doc,
+        }
+        if ("metadatas" in collection_dict) and collection_dict["metadatas"][i]:
+            data_dict.update(collection_dict["metadatas"][i].items())
+        data.append(Data(**data_dict))
+    return data

lfx/base/vectorstores/vector_store_connection_decorator.py

@@ -0,0 +1,52 @@
+from langchain_core.vectorstores import VectorStore
+
+from lfx.io import Output
+
+
+def vector_store_connection(cls):
+    """A decorator that adds vector store connection functionality to a class.
+
+    This decorator performs the following actions:
+    1. Adds a `decorated` attribute to the class and sets it to True.
+    2. Checks if the class has an `outputs` attribute:
+        - If it does, it makes a copy of the `outputs` attribute to avoid modifying the base class attribute.
+        - It then checks if "vectorstoreconnection" is not already in the output names.
+        - If not, it adds a new `Output` entry for "Vector Store Connection" to the `outputs` attribute.
+    3. Adds an `as_vector_store` method to the class, which returns the result of the `build_vector_store` method.
+
+    Args:
+        cls (type): The class to be decorated.
+
+    Returns:
+        type: The decorated class with added vector store connection functionality.
+    """
+    cls.decorated = True  # Add an attribute to the class
+
+    if hasattr(cls, "outputs"):
+        cls.outputs = cls.outputs.copy()  # Make a copy to avoid modifying the base class attribute
+        output_names = [output.name for output in cls.outputs]
+
+        if "vectorstoreconnection" not in output_names:
+            cls.outputs.extend(
+                [
+                    Output(
+                        display_name="Vector Store Connection",
+                        hidden=True,
+                        name="vectorstoreconnection",
+                        method="as_vector_store",
+                        group_outputs=False,
+                    )
+                ]
+            )
+
+    def as_vector_store(self) -> VectorStore:
+        """Converts the current instance to a VectorStore object.
+
+        Returns:
+            VectorStore: The resulting VectorStore object.
+        """
+        return self.build_vector_store()
+
+    cls.as_vector_store = as_vector_store  # Ensures that the method is added to the class
+
+    return cls

lfx/cli/__init__.py

@@ -0,0 +1,5 @@
+"""LFX CLI module for serving flows."""
+
+from lfx.cli.commands import serve_command
+
+__all__ = ["serve_command"]