lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/processing/structured_output.py

@@ -0,0 +1,202 @@
+from pydantic import BaseModel, Field, create_model
+from trustcall import create_extractor
+
+from lfx.base.models.chat_result import get_chat_result
+from lfx.custom.custom_component.component import Component
+from lfx.helpers.base_model import build_model_from_schema
+from lfx.io import (
+    HandleInput,
+    MessageTextInput,
+    MultilineInput,
+    Output,
+    TableInput,
+)
+from lfx.schema.data import Data
+from lfx.schema.dataframe import DataFrame
+from lfx.schema.table import EditMode
+
+
+class StructuredOutputComponent(Component):
+    display_name = "Structured Output"
+    description = "Uses an LLM to generate structured data. Ideal for extraction and consistency."
+    documentation: str = "https://docs.langflow.org/components-processing#structured-output"
+    name = "StructuredOutput"
+    icon = "braces"
+
+    inputs = [
+        HandleInput(
+            name="llm",
+            display_name="Language Model",
+            info="The language model to use to generate the structured output.",
+            input_types=["LanguageModel"],
+            required=True,
+        ),
+        MultilineInput(
+            name="input_value",
+            display_name="Input Message",
+            info="The input message to the language model.",
+            tool_mode=True,
+            required=True,
+        ),
+        MultilineInput(
+            name="system_prompt",
+            display_name="Format Instructions",
+            info="The instructions to the language model for formatting the output.",
+            value=(
+                "You are an AI that extracts structured JSON objects from unstructured text. "
+                "Use a predefined schema with expected types (str, int, float, bool, dict). "
+                "Extract ALL relevant instances that match the schema - if multiple patterns exist, capture them all. "
+                "Fill missing or ambiguous values with defaults: null for missing values. "
+                "Remove exact duplicates but keep variations that have different field values. "
+                "Always return valid JSON in the expected format, never throw errors. "
+                "If multiple objects can be extracted, return them all in the structured format."
+            ),
+            required=True,
+            advanced=True,
+        ),
+        MessageTextInput(
+            name="schema_name",
+            display_name="Schema Name",
+            info="Provide a name for the output data schema.",
+            advanced=True,
+        ),
+        TableInput(
+            name="output_schema",
+            display_name="Output Schema",
+            info="Define the structure and data types for the model's output.",
+            required=True,
+            # TODO: remove deault value
+            table_schema=[
+                {
+                    "name": "name",
+                    "display_name": "Name",
+                    "type": "str",
+                    "description": "Specify the name of the output field.",
+                    "default": "field",
+                    "edit_mode": EditMode.INLINE,
+                },
+                {
+                    "name": "description",
+                    "display_name": "Description",
+                    "type": "str",
+                    "description": "Describe the purpose of the output field.",
+                    "default": "description of field",
+                    "edit_mode": EditMode.POPOVER,
+                },
+                {
+                    "name": "type",
+                    "display_name": "Type",
+                    "type": "str",
+                    "edit_mode": EditMode.INLINE,
+                    "description": ("Indicate the data type of the output field (e.g., str, int, float, bool, dict)."),
+                    "options": ["str", "int", "float", "bool", "dict"],
+                    "default": "str",
+                },
+                {
+                    "name": "multiple",
+                    "display_name": "As List",
+                    "type": "boolean",
+                    "description": "Set to True if this output field should be a list of the specified type.",
+                    "default": "False",
+                    "edit_mode": EditMode.INLINE,
+                },
+            ],
+            value=[
+                {
+                    "name": "field",
+                    "description": "description of field",
+                    "type": "str",
+                    "multiple": "False",
+                }
+            ],
+        ),
+    ]
+
+    outputs = [
+        Output(
+            name="structured_output",
+            display_name="Structured Output",
+            method="build_structured_output",
+        ),
+        Output(
+            name="dataframe_output",
+            display_name="Structured Output",
+            method="build_structured_dataframe",
+        ),
+    ]
+
+    def build_structured_output_base(self):
+        schema_name = self.schema_name or "OutputModel"
+
+        if not hasattr(self.llm, "with_structured_output"):
+            msg = "Language model does not support structured output."
+            raise TypeError(msg)
+        if not self.output_schema:
+            msg = "Output schema cannot be empty"
+            raise ValueError(msg)
+
+        output_model_ = build_model_from_schema(self.output_schema)
+
+        output_model = create_model(
+            schema_name,
+            __doc__=f"A list of {schema_name}.",
+            objects=(list[output_model_], Field(description=f"A list of {schema_name}.")),  # type: ignore[valid-type]
+        )
+
+        try:
+            llm_with_structured_output = create_extractor(self.llm, tools=[output_model])
+        except NotImplementedError as exc:
+            msg = f"{self.llm.__class__.__name__} does not support structured output."
+            raise TypeError(msg) from exc
+
+        config_dict = {
+            "run_name": self.display_name,
+            "project_name": self.get_project_name(),
+            "callbacks": self.get_langchain_callbacks(),
+        }
+        result = get_chat_result(
+            runnable=llm_with_structured_output,
+            system_message=self.system_prompt,
+            input_value=self.input_value,
+            config=config_dict,
+        )
+
+        # OPTIMIZATION NOTE: Simplified processing based on trustcall response structure
+        # Handle non-dict responses (shouldn't happen with trustcall, but defensive)
+        if not isinstance(result, dict):
+            return result
+
+        # Extract first response and convert BaseModel to dict
+        responses = result.get("responses", [])
+        if not responses:
+            return result
+
+        # Convert BaseModel to dict (creates the "objects" key)
+        first_response = responses[0]
+        structured_data = first_response.model_dump() if isinstance(first_response, BaseModel) else first_response
+
+        # Extract the objects array (guaranteed to exist due to our Pydantic model structure)
+        return structured_data.get("objects", structured_data)
+
+    def build_structured_output(self) -> Data:
+        output = self.build_structured_output_base()
+        if not isinstance(output, list) or not output:
+            # handle empty or unexpected type case
+            msg = "No structured output returned"
+            raise ValueError(msg)
+        if len(output) == 1:
+            return Data(data=output[0])
+        if len(output) > 1:
+            # Multiple outputs - wrap them in a results container
+            return Data(data={"results": output})
+        return Data()
+
+    def build_structured_dataframe(self) -> DataFrame:
+        output = self.build_structured_output_base()
+        if not isinstance(output, list) or not output:
+            # handle empty or unexpected type case
+            msg = "No structured output returned"
+            raise ValueError(msg)
+        data_list = [Data(data=output[0])] if len(output) == 1 else [Data(data=item) for item in output]
+
+        return DataFrame(data_list)

lfx/components/processing/update_data.py

@@ -0,0 +1,160 @@
+from typing import Any
+
+from lfx.custom.custom_component.component import Component
+from lfx.field_typing.range_spec import RangeSpec
+from lfx.inputs.inputs import (
+    BoolInput,
+    DataInput,
+    DictInput,
+    IntInput,
+    MessageTextInput,
+)
+from lfx.io import Output
+from lfx.schema.data import Data
+from lfx.schema.dotdict import dotdict
+
+
+class UpdateDataComponent(Component):
+    display_name: str = "Update Data"
+    description: str = "Dynamically update or append data with the specified fields."
+    name: str = "UpdateData"
+    MAX_FIELDS = 15  # Define a constant for maximum number of fields
+    icon = "FolderSync"
+    legacy = True
+
+    inputs = [
+        DataInput(
+            name="old_data",
+            display_name="Data",
+            info="The record to update.",
+            is_list=True,  # Changed to True to handle list of Data objects
+            required=True,
+        ),
+        IntInput(
+            name="number_of_fields",
+            display_name="Number of Fields",
+            info="Number of fields to be added to the record.",
+            real_time_refresh=True,
+            value=0,
+            range_spec=RangeSpec(min=1, max=MAX_FIELDS, step=1, step_type="int"),
+        ),
+        MessageTextInput(
+            name="text_key",
+            display_name="Text Key",
+            info="Key that identifies the field to be used as the text content.",
+            advanced=True,
+        ),
+        BoolInput(
+            name="text_key_validator",
+            display_name="Text Key Validator",
+            advanced=True,
+            info="If enabled, checks if the given 'Text Key' is present in the given 'Data'.",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Data", name="data", method="build_data"),
+    ]
+
+    def update_build_config(self, build_config: dotdict, field_value: Any, field_name: str | None = None):
+        """Update the build configuration when the number of fields changes.
+
+        Args:
+            build_config (dotdict): The current build configuration.
+            field_value (Any): The new value for the field.
+            field_name (Optional[str]): The name of the field being updated.
+        """
+        if field_name == "number_of_fields":
+            default_keys = {
+                "code",
+                "_type",
+                "number_of_fields",
+                "text_key",
+                "old_data",
+                "text_key_validator",
+            }
+            try:
+                field_value_int = int(field_value)
+            except ValueError:
+                return build_config
+
+            if field_value_int > self.MAX_FIELDS:
+                build_config["number_of_fields"]["value"] = self.MAX_FIELDS
+                msg = f"Number of fields cannot exceed {self.MAX_FIELDS}. Try using a Component to combine two Data."
+                raise ValueError(msg)
+
+            existing_fields = {}
+            # Back up the existing template fields
+            for key in list(build_config.keys()):
+                if key not in default_keys:
+                    existing_fields[key] = build_config.pop(key)
+
+            for i in range(1, field_value_int + 1):
+                key = f"field_{i}_key"
+                if key in existing_fields:
+                    field = existing_fields[key]
+                    build_config[key] = field
+                else:
+                    field = DictInput(
+                        display_name=f"Field {i}",
+                        name=key,
+                        info=f"Key for field {i}.",
+                        input_types=["Message", "Data"],
+                    )
+                    build_config[field.name] = field.to_dict()
+
+            build_config["number_of_fields"]["value"] = field_value_int
+        return build_config
+
+    async def build_data(self) -> Data | list[Data]:
+        """Build the updated data by combining the old data with new fields."""
+        new_data = self.get_data()
+        if isinstance(self.old_data, list):
+            for data_item in self.old_data:
+                if not isinstance(data_item, Data):
+                    continue  # Skip invalid items
+                data_item.data.update(new_data)
+                if self.text_key:
+                    data_item.text_key = self.text_key
+                self.validate_text_key(data_item)
+            self.status = self.old_data
+            return self.old_data  # Returns List[Data]
+        if isinstance(self.old_data, Data):
+            self.old_data.data.update(new_data)
+            if self.text_key:
+                self.old_data.text_key = self.text_key
+            self.status = self.old_data
+            self.validate_text_key(self.old_data)
+            return self.old_data  # Returns Data
+        msg = "old_data is not a Data object or list of Data objects."
+        raise ValueError(msg)
+
+    def get_data(self):
+        """Function to get the Data from the attributes."""
+        data = {}
+        default_keys = {
+            "code",
+            "_type",
+            "number_of_fields",
+            "text_key",
+            "old_data",
+            "text_key_validator",
+        }
+        for attr_name, attr_value in self._attributes.items():
+            if attr_name in default_keys:
+                continue  # Skip default attributes
+            if isinstance(attr_value, dict):
+                for key, value in attr_value.items():
+                    data[key] = value.get_text() if isinstance(value, Data) else value
+            elif isinstance(attr_value, Data):
+                data[attr_name] = attr_value.get_text()
+            else:
+                data[attr_name] = attr_value
+        return data
+
+    def validate_text_key(self, data: Data) -> None:
+        """This function validates that the Text Key is one of the keys in the Data."""
+        data_keys = data.data.keys()
+        if self.text_key and self.text_key not in data_keys:
+            msg = f"Text Key: '{self.text_key}' not found in the Data keys: {', '.join(data_keys)}"
+            raise ValueError(msg)

lfx/components/prototypes/__init__.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from .python_function import PythonFunctionComponent
+
+_dynamic_imports = {
+    "PythonFunctionComponent": "python_function",
+}
+
+__all__ = [
+    "PythonFunctionComponent",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import prototype components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

lfx/components/prototypes/python_function.py

@@ -0,0 +1,73 @@
+from collections.abc import Callable
+
+from lfx.custom.custom_component.component import Component
+from lfx.custom.utils import get_function
+from lfx.io import CodeInput, Output
+from lfx.log.logger import logger
+from lfx.schema.data import Data
+from lfx.schema.dotdict import dotdict
+from lfx.schema.message import Message
+
+
+class PythonFunctionComponent(Component):
+    display_name = "Python Function"
+    description = "Define and execute a Python function that returns a Data object or a Message."
+    icon = "Python"
+    name = "PythonFunction"
+    legacy = True
+
+    inputs = [
+        CodeInput(
+            name="function_code",
+            display_name="Function Code",
+            info="The code for the function.",
+        ),
+    ]
+
+    outputs = [
+        Output(
+            name="function_output",
+            display_name="Function Callable",
+            method="get_function_callable",
+        ),
+        Output(
+            name="function_output_data",
+            display_name="Function Output (Data)",
+            method="execute_function_data",
+        ),
+        Output(
+            name="function_output_str",
+            display_name="Function Output (Message)",
+            method="execute_function_message",
+        ),
+    ]
+
+    def get_function_callable(self) -> Callable:
+        function_code = self.function_code
+        self.status = function_code
+        return get_function(function_code)
+
+    def execute_function(self) -> list[dotdict | str] | dotdict | str:
+        function_code = self.function_code
+
+        if not function_code:
+            return "No function code provided."
+
+        try:
+            func = get_function(function_code)
+            return func()
+        except Exception as e:  # noqa: BLE001
+            logger.debug("Error executing function", exc_info=True)
+            return f"Error executing function: {e}"
+
+    def execute_function_data(self) -> list[Data]:
+        results = self.execute_function()
+        results = results if isinstance(results, list) else [results]
+        return [(Data(text=x) if isinstance(x, str) else Data(**x)) for x in results]
+
+    def execute_function_message(self) -> Message:
+        results = self.execute_function()
+        results = results if isinstance(results, list) else [results]
+        results_list = [str(x) for x in results]
+        results_str = "\n".join(results_list)
+        return Message(text=results_str)

lfx/components/qdrant/__init__.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from .qdrant import QdrantVectorStoreComponent
+
+_dynamic_imports = {
+    "QdrantVectorStoreComponent": "qdrant",
+}
+
+__all__ = [
+    "QdrantVectorStoreComponent",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import Qdrant components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

lfx/components/qdrant/qdrant.py

@@ -0,0 +1,109 @@
+from langchain.embeddings.base import Embeddings
+from langchain_community.vectorstores import Qdrant
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.io import (
+    DropdownInput,
+    HandleInput,
+    IntInput,
+    SecretStrInput,
+    StrInput,
+)
+from lfx.schema.data import Data
+
+
+class QdrantVectorStoreComponent(LCVectorStoreComponent):
+    display_name = "Qdrant"
+    description = "Qdrant Vector Store with search capabilities"
+    icon = "Qdrant"
+
+    inputs = [
+        StrInput(name="collection_name", display_name="Collection Name", required=True),
+        StrInput(name="host", display_name="Host", value="localhost", advanced=True),
+        IntInput(name="port", display_name="Port", value=6333, advanced=True),
+        IntInput(name="grpc_port", display_name="gRPC Port", value=6334, advanced=True),
+        SecretStrInput(name="api_key", display_name="API Key", advanced=True),
+        StrInput(name="prefix", display_name="Prefix", advanced=True),
+        IntInput(name="timeout", display_name="Timeout", advanced=True),
+        StrInput(name="path", display_name="Path", advanced=True),
+        StrInput(name="url", display_name="URL", advanced=True),
+        DropdownInput(
+            name="distance_func",
+            display_name="Distance Function",
+            options=["Cosine", "Euclidean", "Dot Product"],
+            value="Cosine",
+            advanced=True,
+        ),
+        StrInput(name="content_payload_key", display_name="Content Payload Key", value="page_content", advanced=True),
+        StrInput(name="metadata_payload_key", display_name="Metadata Payload Key", value="metadata", advanced=True),
+        *LCVectorStoreComponent.inputs,
+        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            value=4,
+            advanced=True,
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> Qdrant:
+        qdrant_kwargs = {
+            "collection_name": self.collection_name,
+            "content_payload_key": self.content_payload_key,
+            "metadata_payload_key": self.metadata_payload_key,
+        }
+
+        server_kwargs = {
+            "host": self.host or None,
+            "port": int(self.port),  # Ensure port is an integer
+            "grpc_port": int(self.grpc_port),  # Ensure grpc_port is an integer
+            "api_key": self.api_key,
+            "prefix": self.prefix,
+            # Ensure timeout is an integer
+            "timeout": int(self.timeout) if self.timeout else None,
+            "path": self.path or None,
+            "url": self.url or None,
+        }
+
+        server_kwargs = {k: v for k, v in server_kwargs.items() if v is not None}
+
+        # Convert DataFrame to Data if needed using parent's method
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in self.ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        if not isinstance(self.embedding, Embeddings):
+            msg = "Invalid embedding object"
+            raise TypeError(msg)
+
+        if documents:
+            qdrant = Qdrant.from_documents(documents, embedding=self.embedding, **qdrant_kwargs, **server_kwargs)
+        else:
+            from qdrant_client import QdrantClient
+
+            client = QdrantClient(**server_kwargs)
+            qdrant = Qdrant(embeddings=self.embedding, client=client, **qdrant_kwargs)
+
+        return qdrant
+
+    def search_documents(self) -> list[Data]:
+        vector_store = self.build_vector_store()
+
+        if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
+            docs = vector_store.similarity_search(
+                query=self.search_query,
+                k=self.number_of_results,
+            )
+
+            data = docs_to_data(docs)
+            self.status = data
+            return data
+        return []

lfx/components/redis/__init__.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from .redis import RedisVectorStoreComponent
+    from .redis_chat import RedisIndexChatMemory
+
+_dynamic_imports = {
+    "RedisVectorStoreComponent": "redis",
+    "RedisIndexChatMemory": "redis_chat",
+}
+
+__all__ = [
+    "RedisIndexChatMemory",
+    "RedisVectorStoreComponent",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import Redis components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)