lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/pinecone/pinecone.py

@@ -0,0 +1,134 @@
+import numpy as np
+from langchain_core.vectorstores import VectorStore
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.io import DropdownInput, HandleInput, IntInput, SecretStrInput, StrInput
+from lfx.schema.data import Data
+
+
+class PineconeVectorStoreComponent(LCVectorStoreComponent):
+    display_name = "Pinecone"
+    description = "Pinecone Vector Store with search capabilities"
+    name = "Pinecone"
+    icon = "Pinecone"
+    inputs = [
+        StrInput(name="index_name", display_name="Index Name", required=True),
+        StrInput(name="namespace", display_name="Namespace", info="Namespace for the index."),
+        DropdownInput(
+            name="distance_strategy",
+            display_name="Distance Strategy",
+            options=["Cosine", "Euclidean", "Dot Product"],
+            value="Cosine",
+            advanced=True,
+        ),
+        SecretStrInput(name="pinecone_api_key", display_name="Pinecone API Key", required=True),
+        StrInput(
+            name="text_key",
+            display_name="Text Key",
+            info="Key in the record to use as text.",
+            value="text",
+            advanced=True,
+        ),
+        *LCVectorStoreComponent.inputs,
+        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            value=4,
+            advanced=True,
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> VectorStore:
+        """Build and return a Pinecone vector store instance."""
+        try:
+            from langchain_pinecone import PineconeVectorStore
+        except ImportError as e:
+            msg = "langchain-pinecone is not installed. Please install it with `pip install langchain-pinecone`."
+            raise ValueError(msg) from e
+
+        try:
+            from langchain_pinecone._utilities import DistanceStrategy
+
+            # Wrap the embedding model to ensure float32 output
+            wrapped_embeddings = Float32Embeddings(self.embedding)
+
+            # Convert distance strategy
+            distance_strategy = self.distance_strategy.replace(" ", "_").upper()
+            distance_strategy = DistanceStrategy[distance_strategy]
+
+            # Initialize Pinecone instance with wrapped embeddings
+            pinecone = PineconeVectorStore(
+                index_name=self.index_name,
+                embedding=wrapped_embeddings,  # Use wrapped embeddings
+                text_key=self.text_key,
+                namespace=self.namespace,
+                distance_strategy=distance_strategy,
+                pinecone_api_key=self.pinecone_api_key,
+            )
+        except Exception as e:
+            error_msg = "Error building Pinecone vector store"
+            raise ValueError(error_msg) from e
+        else:
+            self.ingest_data = self._prepare_ingest_data()
+
+            # Process documents if any
+            documents = []
+            if self.ingest_data:
+                # Convert DataFrame to Data if needed using parent's method
+
+                for doc in self.ingest_data:
+                    if isinstance(doc, Data):
+                        documents.append(doc.to_lc_document())
+                    else:
+                        documents.append(doc)
+
+                if documents:
+                    pinecone.add_documents(documents)
+
+            return pinecone
+
+    def search_documents(self) -> list[Data]:
+        """Search documents in the vector store."""
+        try:
+            if not self.search_query or not isinstance(self.search_query, str) or not self.search_query.strip():
+                return []
+
+            vector_store = self.build_vector_store()
+            docs = vector_store.similarity_search(
+                query=self.search_query,
+                k=self.number_of_results,
+            )
+        except Exception as e:
+            error_msg = "Error searching documents"
+            raise ValueError(error_msg) from e
+        else:
+            data = docs_to_data(docs)
+            self.status = data
+            return data
+
+
+class Float32Embeddings:
+    """Wrapper class to ensure float32 embeddings."""
+
+    def __init__(self, base_embeddings):
+        self.base_embeddings = base_embeddings
+
+    def embed_documents(self, texts):
+        embeddings = self.base_embeddings.embed_documents(texts)
+        if isinstance(embeddings, np.ndarray):
+            return [[self._force_float32(x) for x in vec] for vec in embeddings]
+        return [[self._force_float32(x) for x in vec] for vec in embeddings]
+
+    def embed_query(self, text):
+        embedding = self.base_embeddings.embed_query(text)
+        if isinstance(embedding, np.ndarray):
+            return [self._force_float32(x) for x in embedding]
+        return [self._force_float32(x) for x in embedding]
+
+    def _force_float32(self, value):
+        """Convert any numeric type to Python float."""
+        return float(np.float32(value))

lfx/components/processing/__init__.py

@@ -0,0 +1,117 @@
+"""Processing components for LangFlow."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from lfx.components.processing.alter_metadata import AlterMetadataComponent
+    from lfx.components.processing.batch_run import BatchRunComponent
+    from lfx.components.processing.combine_text import CombineTextComponent
+    from lfx.components.processing.converter import TypeConverterComponent
+    from lfx.components.processing.create_data import CreateDataComponent
+    from lfx.components.processing.data_operations import DataOperationsComponent
+    from lfx.components.processing.data_to_dataframe import DataToDataFrameComponent
+    from lfx.components.processing.dataframe_operations import DataFrameOperationsComponent
+    from lfx.components.processing.extract_key import ExtractDataKeyComponent
+    from lfx.components.processing.filter_data import FilterDataComponent
+    from lfx.components.processing.filter_data_values import DataFilterComponent
+    from lfx.components.processing.json_cleaner import JSONCleaner
+    from lfx.components.processing.lambda_filter import LambdaFilterComponent
+    from lfx.components.processing.llm_router import LLMRouterComponent
+    from lfx.components.processing.merge_data import MergeDataComponent
+    from lfx.components.processing.message_to_data import MessageToDataComponent
+    from lfx.components.processing.parse_data import ParseDataComponent
+    from lfx.components.processing.parse_dataframe import ParseDataFrameComponent
+    from lfx.components.processing.parse_json_data import ParseJSONDataComponent
+    from lfx.components.processing.parser import ParserComponent
+    from lfx.components.processing.prompt import PromptComponent
+    from lfx.components.processing.python_repl_core import PythonREPLComponent
+    from lfx.components.processing.regex import RegexExtractorComponent
+    from lfx.components.processing.save_file import SaveToFileComponent
+    from lfx.components.processing.select_data import SelectDataComponent
+    from lfx.components.processing.split_text import SplitTextComponent
+    from lfx.components.processing.structured_output import StructuredOutputComponent
+    from lfx.components.processing.update_data import UpdateDataComponent
+
+_dynamic_imports = {
+    "AlterMetadataComponent": "alter_metadata",
+    "BatchRunComponent": "batch_run",
+    "CombineTextComponent": "combine_text",
+    "TypeConverterComponent": "converter",
+    "CreateDataComponent": "create_data",
+    "DataOperationsComponent": "data_operations",
+    "DataToDataFrameComponent": "data_to_dataframe",
+    "DataFrameOperationsComponent": "dataframe_operations",
+    "ExtractDataKeyComponent": "extract_key",
+    "FilterDataComponent": "filter_data",
+    "DataFilterComponent": "filter_data_values",
+    "JSONCleaner": "json_cleaner",
+    "LambdaFilterComponent": "lambda_filter",
+    "LLMRouterComponent": "llm_router",
+    "MergeDataComponent": "merge_data",
+    "MessageToDataComponent": "message_to_data",
+    "ParseDataComponent": "parse_data",
+    "ParseDataFrameComponent": "parse_dataframe",
+    "ParseJSONDataComponent": "parse_json_data",
+    "ParserComponent": "parser",
+    "PromptComponent": "prompt",
+    "PythonREPLComponent": "python_repl_core",
+    "RegexExtractorComponent": "regex",
+    "SaveToFileComponent": "save_file",
+    "SelectDataComponent": "select_data",
+    "SplitTextComponent": "split_text",
+    "StructuredOutputComponent": "structured_output",
+    "UpdateDataComponent": "update_data",
+}
+
+__all__ = [
+    "AlterMetadataComponent",
+    "BatchRunComponent",
+    "CombineTextComponent",
+    "CreateDataComponent",
+    "DataFilterComponent",
+    "DataFrameOperationsComponent",
+    "DataOperationsComponent",
+    "DataToDataFrameComponent",
+    "ExtractDataKeyComponent",
+    "FilterDataComponent",
+    "JSONCleaner",
+    "LLMRouterComponent",
+    "LambdaFilterComponent",
+    "MergeDataComponent",
+    "MessageToDataComponent",
+    "ParseDataComponent",
+    "ParseDataFrameComponent",
+    "ParseJSONDataComponent",
+    "ParserComponent",
+    "PromptComponent",
+    "PythonREPLComponent",
+    "RegexExtractorComponent",
+    "SaveToFileComponent",
+    "SelectDataComponent",
+    "SplitTextComponent",
+    "StructuredOutputComponent",
+    "TypeConverterComponent",
+    "UpdateDataComponent",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import processing components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

lfx/components/processing/alter_metadata.py

@@ -0,0 +1,108 @@
+from lfx.custom.custom_component.component import Component
+from lfx.inputs.inputs import MessageTextInput
+from lfx.io import HandleInput, NestedDictInput, Output, StrInput
+from lfx.schema.data import Data
+from lfx.schema.dataframe import DataFrame
+
+
+class AlterMetadataComponent(Component):
+    display_name = "Alter Metadata"
+    description = "Adds/Removes Metadata Dictionary on inputs"
+    icon = "merge"
+    name = "AlterMetadata"
+    legacy = True
+
+    inputs = [
+        HandleInput(
+            name="input_value",
+            display_name="Input",
+            info="Object(s) to which Metadata should be added",
+            required=False,
+            input_types=["Message", "Data"],
+            is_list=True,
+        ),
+        StrInput(
+            name="text_in",
+            display_name="User Text",
+            info="Text input; value will be in 'text' attribute of Data object. Empty text entries are ignored.",
+            required=False,
+        ),
+        NestedDictInput(
+            name="metadata",
+            display_name="Metadata",
+            info="Metadata to add to each object",
+            input_types=["Data"],
+            required=True,
+        ),
+        MessageTextInput(
+            name="remove_fields",
+            display_name="Fields to Remove",
+            info="Metadata Fields to Remove",
+            required=False,
+            is_list=True,
+        ),
+    ]
+
+    outputs = [
+        Output(
+            name="data",
+            display_name="Data",
+            info="List of Input objects each with added Metadata",
+            method="process_output",
+        ),
+        Output(
+            display_name="DataFrame",
+            name="dataframe",
+            info="Data objects as a DataFrame, with metadata as columns",
+            method="as_dataframe",
+        ),
+    ]
+
+    def _as_clean_dict(self, obj):
+        """Convert a Data object or a standard dictionary to a standard dictionary."""
+        if isinstance(obj, dict):
+            as_dict = obj
+        elif isinstance(obj, Data):
+            as_dict = obj.data
+        else:
+            msg = f"Expected a Data object or a dictionary but got {type(obj)}."
+            raise TypeError(msg)
+
+        return {k: v for k, v in (as_dict or {}).items() if k and k.strip()}
+
+    def process_output(self) -> list[Data]:
+        # Ensure metadata is a dictionary, filtering out any empty keys
+        metadata = self._as_clean_dict(self.metadata)
+
+        # Convert text_in to a Data object if it exists, and initialize our list of Data objects
+        data_objects = [Data(text=self.text_in)] if self.text_in else []
+
+        # Append existing Data objects from input_value, if any
+        if self.input_value:
+            data_objects.extend(self.input_value)
+
+        # Update each Data object with the new metadata, preserving existing fields
+        for data in data_objects:
+            data.data.update(metadata)
+
+        # Handle removal of fields specified in remove_fields
+        if self.remove_fields:
+            fields_to_remove = {field.strip() for field in self.remove_fields if field.strip()}
+
+            # Remove specified fields from each Data object's metadata
+            for data in data_objects:
+                data.data = {k: v for k, v in data.data.items() if k not in fields_to_remove}
+
+        # Set the status for tracking/debugging purposes
+        self.status = data_objects
+        return data_objects
+
+    def as_dataframe(self) -> DataFrame:
+        """Convert the processed data objects into a DataFrame.
+
+        Returns:
+            DataFrame: A DataFrame where each row corresponds to a Data object,
+                    with metadata fields as columns.
+        """
+        data_list = self.process_output()
+        return DataFrame(data_list)

lfx/components/processing/batch_run.py

@@ -0,0 +1,205 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, cast
+
+import toml  # type: ignore[import-untyped]
+
+from lfx.custom.custom_component.component import Component
+from lfx.io import BoolInput, DataFrameInput, HandleInput, MessageTextInput, MultilineInput, Output
+from lfx.log.logger import logger
+from lfx.schema.dataframe import DataFrame
+
+if TYPE_CHECKING:
+    from langchain_core.runnables import Runnable
+
+
+class BatchRunComponent(Component):
+    display_name = "Batch Run"
+    description = "Runs an LLM on each row of a DataFrame column. If no column is specified, all columns are used."
+    documentation: str = "https://docs.langflow.org/components-processing#batch-run"
+    icon = "List"
+
+    inputs = [
+        HandleInput(
+            name="model",
+            display_name="Language Model",
+            info="Connect the 'Language Model' output from your LLM component here.",
+            input_types=["LanguageModel"],
+            required=True,
+        ),
+        MultilineInput(
+            name="system_message",
+            display_name="Instructions",
+            info="Multi-line system instruction for all rows in the DataFrame.",
+            required=False,
+        ),
+        DataFrameInput(
+            name="df",
+            display_name="DataFrame",
+            info="The DataFrame whose column (specified by 'column_name') we'll treat as text messages.",
+            required=True,
+        ),
+        MessageTextInput(
+            name="column_name",
+            display_name="Column Name",
+            info=(
+                "The name of the DataFrame column to treat as text messages. "
+                "If empty, all columns will be formatted in TOML."
+            ),
+            required=False,
+            advanced=False,
+        ),
+        MessageTextInput(
+            name="output_column_name",
+            display_name="Output Column Name",
+            info="Name of the column where the model's response will be stored.",
+            value="model_response",
+            required=False,
+            advanced=True,
+        ),
+        BoolInput(
+            name="enable_metadata",
+            display_name="Enable Metadata",
+            info="If True, add metadata to the output DataFrame.",
+            value=False,
+            required=False,
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(
+            display_name="LLM Results",
+            name="batch_results",
+            method="run_batch",
+            info="A DataFrame with all original columns plus the model's response column.",
+        ),
+    ]
+
+    def _format_row_as_toml(self, row: dict[str, Any]) -> str:
+        """Convert a dictionary (row) into a TOML-formatted string."""
+        formatted_dict = {str(col): {"value": str(val)} for col, val in row.items()}
+        return toml.dumps(formatted_dict)
+
+    def _create_base_row(
+        self, original_row: dict[str, Any], model_response: str = "", batch_index: int = -1
+    ) -> dict[str, Any]:
+        """Create a base row with original columns and additional metadata."""
+        row = original_row.copy()
+        row[self.output_column_name] = model_response
+        row["batch_index"] = batch_index
+        return row
+
+    def _add_metadata(
+        self, row: dict[str, Any], *, success: bool = True, system_msg: str = "", error: str | None = None
+    ) -> None:
+        """Add metadata to a row if enabled."""
+        if not self.enable_metadata:
+            return
+
+        if success:
+            row["metadata"] = {
+                "has_system_message": bool(system_msg),
+                "input_length": len(row.get("text_input", "")),
+                "response_length": len(row[self.output_column_name]),
+                "processing_status": "success",
+            }
+        else:
+            row["metadata"] = {
+                "error": error,
+                "processing_status": "failed",
+            }
+
+    async def run_batch(self) -> DataFrame:
+        """Process each row in df[column_name] with the language model asynchronously.
+
+        Returns:
+            DataFrame: A new DataFrame containing:
+                - All original columns
+                - The model's response column (customizable name)
+                - 'batch_index' column for processing order
+                - 'metadata' (optional)
+
+        Raises:
+            ValueError: If the specified column is not found in the DataFrame
+            TypeError: If the model is not compatible or input types are wrong
+        """
+        model: Runnable = self.model
+        system_msg = self.system_message or ""
+        df: DataFrame = self.df
+        col_name = self.column_name or ""
+
+        # Validate inputs first
+        if not isinstance(df, DataFrame):
+            msg = f"Expected DataFrame input, got {type(df)}"
+            raise TypeError(msg)
+
+        if col_name and col_name not in df.columns:
+            msg = f"Column '{col_name}' not found in the DataFrame. Available columns: {', '.join(df.columns)}"
+            raise ValueError(msg)
+
+        try:
+            # Determine text input for each row
+            if col_name:
+                user_texts = df[col_name].astype(str).tolist()
+            else:
+                user_texts = [
+                    self._format_row_as_toml(cast("dict[str, Any]", row)) for row in df.to_dict(orient="records")
+                ]
+
+            total_rows = len(user_texts)
+            await logger.ainfo(f"Processing {total_rows} rows with batch run")
+
+            # Prepare the batch of conversations
+            conversations = [
+                [{"role": "system", "content": system_msg}, {"role": "user", "content": text}]
+                if system_msg
+                else [{"role": "user", "content": text}]
+                for text in user_texts
+            ]
+
+            # Configure the model with project info and callbacks
+            model = model.with_config(
+                {
+                    "run_name": self.display_name,
+                    "project_name": self.get_project_name(),
+                    "callbacks": self.get_langchain_callbacks(),
+                }
+            )
+            # Process batches and track progress
+            responses_with_idx = list(
+                zip(
+                    range(len(conversations)),
+                    await model.abatch(list(conversations)),
+                    strict=True,
+                )
+            )
+
+            # Sort by index to maintain order
+            responses_with_idx.sort(key=lambda x: x[0])
+
+            # Build the final data with enhanced metadata
+            rows: list[dict[str, Any]] = []
+            for idx, (original_row, response) in enumerate(
+                zip(df.to_dict(orient="records"), responses_with_idx, strict=False)
+            ):
+                response_text = response[1].content if hasattr(response[1], "content") else str(response[1])
+                row = self._create_base_row(
+                    cast("dict[str, Any]", original_row), model_response=response_text, batch_index=idx
+                )
+                self._add_metadata(row, success=True, system_msg=system_msg)
+                rows.append(row)
+
+                # Log progress
+                if (idx + 1) % max(1, total_rows // 10) == 0:
+                    await logger.ainfo(f"Processed {idx + 1}/{total_rows} rows")
+
+            await logger.ainfo("Batch processing completed successfully")
+            return DataFrame(rows)
+
+        except (KeyError, AttributeError) as e:
+            # Handle data structure and attribute access errors
+            await logger.aerror(f"Data processing error: {e!s}")
+            error_row = self._create_base_row(dict.fromkeys(df.columns, ""), model_response="", batch_index=-1)
+            self._add_metadata(error_row, success=False, error=str(e))
+            return DataFrame([error_row])

lfx/components/processing/combine_text.py

@@ -0,0 +1,39 @@
+from lfx.custom.custom_component.component import Component
+from lfx.io import MessageTextInput, Output
+from lfx.schema.message import Message
+
+
+class CombineTextComponent(Component):
+    display_name = "Combine Text"
+    description = "Concatenate two text sources into a single text chunk using a specified delimiter."
+    icon = "merge"
+    name = "CombineText"
+    legacy: bool = True
+
+    inputs = [
+        MessageTextInput(
+            name="text1",
+            display_name="First Text",
+            info="The first text input to concatenate.",
+        ),
+        MessageTextInput(
+            name="text2",
+            display_name="Second Text",
+            info="The second text input to concatenate.",
+        ),
+        MessageTextInput(
+            name="delimiter",
+            display_name="Delimiter",
+            info="A string used to separate the two text inputs. Defaults to a whitespace.",
+            value=" ",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Combined Text", name="combined_text", method="combine_texts"),
+    ]
+
+    def combine_texts(self) -> Message:
+        combined = self.delimiter.join([self.text1, self.text2])
+        self.status = combined
+        return Message(text=combined)