lfx-nightly 0.2.0.dev25__py3-none-any.whl

lfx/components/pinecone/pinecone.py

@@ -0,0 +1,134 @@
+import numpy as np
+from langchain_core.vectorstores import VectorStore
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.io import DropdownInput, HandleInput, IntInput, SecretStrInput, StrInput
+from lfx.schema.data import Data
+
+
+class PineconeVectorStoreComponent(LCVectorStoreComponent):
+    display_name = "Pinecone"
+    description = "Pinecone Vector Store with search capabilities"
+    name = "Pinecone"
+    icon = "Pinecone"
+    inputs = [
+        StrInput(name="index_name", display_name="Index Name", required=True),
+        StrInput(name="namespace", display_name="Namespace", info="Namespace for the index."),
+        DropdownInput(
+            name="distance_strategy",
+            display_name="Distance Strategy",
+            options=["Cosine", "Euclidean", "Dot Product"],
+            value="Cosine",
+            advanced=True,
+        ),
+        SecretStrInput(name="pinecone_api_key", display_name="Pinecone API Key", required=True),
+        StrInput(
+            name="text_key",
+            display_name="Text Key",
+            info="Key in the record to use as text.",
+            value="text",
+            advanced=True,
+        ),
+        *LCVectorStoreComponent.inputs,
+        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            value=4,
+            advanced=True,
+        ),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> VectorStore:
+        """Build and return a Pinecone vector store instance."""
+        try:
+            from langchain_pinecone import PineconeVectorStore
+        except ImportError as e:
+            msg = "langchain-pinecone is not installed. Please install it with `pip install langchain-pinecone`."
+            raise ValueError(msg) from e
+
+        try:
+            from langchain_pinecone._utilities import DistanceStrategy
+
+            # Wrap the embedding model to ensure float32 output
+            wrapped_embeddings = Float32Embeddings(self.embedding)
+
+            # Convert distance strategy
+            distance_strategy = self.distance_strategy.replace(" ", "_").upper()
+            distance_strategy = DistanceStrategy[distance_strategy]
+
+            # Initialize Pinecone instance with wrapped embeddings
+            pinecone = PineconeVectorStore(
+                index_name=self.index_name,
+                embedding=wrapped_embeddings,  # Use wrapped embeddings
+                text_key=self.text_key,
+                namespace=self.namespace,
+                distance_strategy=distance_strategy,
+                pinecone_api_key=self.pinecone_api_key,
+            )
+        except Exception as e:
+            error_msg = "Error building Pinecone vector store"
+            raise ValueError(error_msg) from e
+        else:
+            self.ingest_data = self._prepare_ingest_data()
+
+            # Process documents if any
+            documents = []
+            if self.ingest_data:
+                # Convert DataFrame to Data if needed using parent's method
+
+                for doc in self.ingest_data:
+                    if isinstance(doc, Data):
+                        documents.append(doc.to_lc_document())
+                    else:
+                        documents.append(doc)
+
+                if documents:
+                    pinecone.add_documents(documents)
+
+            return pinecone
+
+    def search_documents(self) -> list[Data]:
+        """Search documents in the vector store."""
+        try:
+            if not self.search_query or not isinstance(self.search_query, str) or not self.search_query.strip():
+                return []
+
+            vector_store = self.build_vector_store()
+            docs = vector_store.similarity_search(
+                query=self.search_query,
+                k=self.number_of_results,
+            )
+        except Exception as e:
+            error_msg = "Error searching documents"
+            raise ValueError(error_msg) from e
+        else:
+            data = docs_to_data(docs)
+            self.status = data
+            return data
+
+
+class Float32Embeddings:
+    """Wrapper class to ensure float32 embeddings."""
+
+    def __init__(self, base_embeddings):
+        self.base_embeddings = base_embeddings
+
+    def embed_documents(self, texts):
+        embeddings = self.base_embeddings.embed_documents(texts)
+        if isinstance(embeddings, np.ndarray):
+            return [[self._force_float32(x) for x in vec] for vec in embeddings]
+        return [[self._force_float32(x) for x in vec] for vec in embeddings]
+
+    def embed_query(self, text):
+        embedding = self.base_embeddings.embed_query(text)
+        if isinstance(embedding, np.ndarray):
+            return [self._force_float32(x) for x in embedding]
+        return [self._force_float32(x) for x in embedding]
+
+    def _force_float32(self, value):
+        """Convert any numeric type to Python float."""
+        return float(np.float32(value))

lfx/components/processing/__init__.py

@@ -0,0 +1,72 @@
+"""Processing components for LangFlow."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from lfx.components.processing.combine_text import CombineTextComponent
+    from lfx.components.processing.converter import TypeConverterComponent
+    from lfx.components.processing.create_list import CreateListComponent
+    from lfx.components.processing.data_operations import DataOperationsComponent
+    from lfx.components.processing.dataframe_operations import DataFrameOperationsComponent
+    from lfx.components.processing.dataframe_to_toolset import DataFrameToToolsetComponent
+    from lfx.components.processing.json_cleaner import JSONCleaner
+    from lfx.components.processing.output_parser import OutputParserComponent
+    from lfx.components.processing.parse_data import ParseDataComponent
+    from lfx.components.processing.parser import ParserComponent
+    from lfx.components.processing.regex import RegexExtractorComponent
+    from lfx.components.processing.split_text import SplitTextComponent
+    from lfx.components.processing.store_message import MessageStoreComponent
+
+_dynamic_imports = {
+    "CombineTextComponent": "combine_text",
+    "TypeConverterComponent": "converter",
+    "CreateListComponent": "create_list",
+    "DataOperationsComponent": "data_operations",
+    "DataFrameOperationsComponent": "dataframe_operations",
+    "DataFrameToToolsetComponent": "dataframe_to_toolset",
+    "JSONCleaner": "json_cleaner",
+    "OutputParserComponent": "output_parser",
+    "ParseDataComponent": "parse_data",
+    "ParserComponent": "parser",
+    "RegexExtractorComponent": "regex",
+    "SplitTextComponent": "split_text",
+    "MessageStoreComponent": "store_message",
+}
+
+__all__ = [
+    "CombineTextComponent",
+    "CreateListComponent",
+    "DataFrameOperationsComponent",
+    "DataFrameToToolsetComponent",
+    "DataOperationsComponent",
+    "JSONCleaner",
+    "MessageStoreComponent",
+    "OutputParserComponent",
+    "ParseDataComponent",
+    "ParserComponent",
+    "RegexExtractorComponent",
+    "SplitTextComponent",
+    "TypeConverterComponent",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import processing components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

lfx/components/processing/alter_metadata.py

@@ -0,0 +1,109 @@
+from lfx.custom.custom_component.component import Component
+from lfx.inputs.inputs import MessageTextInput
+from lfx.io import HandleInput, NestedDictInput, Output, StrInput
+from lfx.schema.data import Data
+from lfx.schema.dataframe import DataFrame
+
+
+class AlterMetadataComponent(Component):
+    display_name = "Alter Metadata"
+    description = "Adds/Removes Metadata Dictionary on inputs"
+    icon = "merge"
+    name = "AlterMetadata"
+    legacy = True
+    replacement = ["processing.DataOperations"]
+
+    inputs = [
+        HandleInput(
+            name="input_value",
+            display_name="Input",
+            info="Object(s) to which Metadata should be added",
+            required=False,
+            input_types=["Message", "Data"],
+            is_list=True,
+        ),
+        StrInput(
+            name="text_in",
+            display_name="User Text",
+            info="Text input; value will be in 'text' attribute of Data object. Empty text entries are ignored.",
+            required=False,
+        ),
+        NestedDictInput(
+            name="metadata",
+            display_name="Metadata",
+            info="Metadata to add to each object",
+            input_types=["Data"],
+            required=True,
+        ),
+        MessageTextInput(
+            name="remove_fields",
+            display_name="Fields to Remove",
+            info="Metadata Fields to Remove",
+            required=False,
+            is_list=True,
+        ),
+    ]
+
+    outputs = [
+        Output(
+            name="data",
+            display_name="Data",
+            info="List of Input objects each with added Metadata",
+            method="process_output",
+        ),
+        Output(
+            display_name="DataFrame",
+            name="dataframe",
+            info="Data objects as a DataFrame, with metadata as columns",
+            method="as_dataframe",
+        ),
+    ]
+
+    def _as_clean_dict(self, obj):
+        """Convert a Data object or a standard dictionary to a standard dictionary."""
+        if isinstance(obj, dict):
+            as_dict = obj
+        elif isinstance(obj, Data):
+            as_dict = obj.data
+        else:
+            msg = f"Expected a Data object or a dictionary but got {type(obj)}."
+            raise TypeError(msg)
+
+        return {k: v for k, v in (as_dict or {}).items() if k and k.strip()}
+
+    def process_output(self) -> list[Data]:
+        # Ensure metadata is a dictionary, filtering out any empty keys
+        metadata = self._as_clean_dict(self.metadata)
+
+        # Convert text_in to a Data object if it exists, and initialize our list of Data objects
+        data_objects = [Data(text=self.text_in)] if self.text_in else []
+
+        # Append existing Data objects from input_value, if any
+        if self.input_value:
+            data_objects.extend(self.input_value)
+
+        # Update each Data object with the new metadata, preserving existing fields
+        for data in data_objects:
+            data.data.update(metadata)
+
+        # Handle removal of fields specified in remove_fields
+        if self.remove_fields:
+            fields_to_remove = {field.strip() for field in self.remove_fields if field.strip()}
+
+            # Remove specified fields from each Data object's metadata
+            for data in data_objects:
+                data.data = {k: v for k, v in data.data.items() if k not in fields_to_remove}
+
+        # Set the status for tracking/debugging purposes
+        self.status = data_objects
+        return data_objects
+
+    def as_dataframe(self) -> DataFrame:
+        """Convert the processed data objects into a DataFrame.
+
+        Returns:
+            DataFrame: A DataFrame where each row corresponds to a Data object,
+                    with metadata fields as columns.
+        """
+        data_list = self.process_output()
+        return DataFrame(data_list)

lfx/components/processing/combine_text.py

@@ -0,0 +1,40 @@
+from lfx.custom.custom_component.component import Component
+from lfx.io import MessageTextInput, Output
+from lfx.schema.message import Message
+
+
+class CombineTextComponent(Component):
+    display_name = "Combine Text"
+    description = "Concatenate two text sources into a single text chunk using a specified delimiter."
+    icon = "merge"
+    name = "CombineText"
+    legacy: bool = True
+    replacement = ["processing.DataOperations"]
+
+    inputs = [
+        MessageTextInput(
+            name="text1",
+            display_name="First Text",
+            info="The first text input to concatenate.",
+        ),
+        MessageTextInput(
+            name="text2",
+            display_name="Second Text",
+            info="The second text input to concatenate.",
+        ),
+        MessageTextInput(
+            name="delimiter",
+            display_name="Delimiter",
+            info="A string used to separate the two text inputs. Defaults to a whitespace.",
+            value=" ",
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Combined Text", name="combined_text", method="combine_texts"),
+    ]
+
+    def combine_texts(self) -> Message:
+        combined = self.delimiter.join([self.text1, self.text2])
+        self.status = combined
+        return Message(text=combined)

lfx/components/processing/converter.py

@@ -0,0 +1,248 @@
+import json
+from typing import Any
+
+from lfx.custom import Component
+from lfx.io import BoolInput, HandleInput, Output, TabInput
+from lfx.schema import Data, DataFrame, Message
+
+MIN_CSV_LINES = 2
+
+
+def convert_to_message(v) -> Message:
+    """Convert input to Message type.
+
+    Args:
+        v: Input to convert (Message, Data, DataFrame, or dict)
+
+    Returns:
+        Message: Converted Message object
+    """
+    return v if isinstance(v, Message) else v.to_message()
+
+
+def convert_to_data(v: DataFrame | Data | Message | dict, *, auto_parse: bool) -> Data:
+    """Convert input to Data type.
+
+    Args:
+        v: Input to convert (Message, Data, DataFrame, or dict)
+        auto_parse: Enable automatic parsing of structured data (JSON/CSV)
+
+    Returns:
+        Data: Converted Data object
+    """
+    if isinstance(v, dict):
+        return Data(v)
+    if isinstance(v, Message):
+        data = Data(data={"text": v.data["text"]})
+        return parse_structured_data(data) if auto_parse else data
+
+    return v if isinstance(v, Data) else v.to_data()
+
+
+def convert_to_dataframe(v: DataFrame | Data | Message | dict, *, auto_parse: bool) -> DataFrame:
+    """Convert input to DataFrame type.
+
+    Args:
+        v: Input to convert (Message, Data, DataFrame, or dict)
+        auto_parse: Enable automatic parsing of structured data (JSON/CSV)
+
+    Returns:
+        DataFrame: Converted DataFrame object
+    """
+    import pandas as pd
+
+    if isinstance(v, dict):
+        return DataFrame([v])
+    if isinstance(v, DataFrame):
+        return v
+    # Handle pandas DataFrame
+    if isinstance(v, pd.DataFrame):
+        # Convert pandas DataFrame to our DataFrame by creating Data objects
+        return DataFrame(data=v)
+
+    if isinstance(v, Message):
+        data = Data(data={"text": v.data["text"]})
+        return parse_structured_data(data).to_dataframe() if auto_parse else data.to_dataframe()
+    # For other types, call to_dataframe method
+    return v.to_dataframe()
+
+
+def parse_structured_data(data: Data) -> Data:
+    """Parse structured data (JSON, CSV) from Data's text field.
+
+    Args:
+        data: Data object with text content to parse
+
+    Returns:
+        Data: Modified Data object with parsed content or original if parsing fails
+    """
+    raw_text = data.get_text() or ""
+    text = raw_text.lstrip("\ufeff").strip()
+
+    # Try JSON parsing first
+    parsed_json = _try_parse_json(text)
+    if parsed_json is not None:
+        return parsed_json
+
+    # Try CSV parsing
+    if _looks_like_csv(text):
+        try:
+            return _parse_csv_to_data(text)
+        except Exception:  # noqa: BLE001
+            # Heuristic misfire or malformed CSV — keep original data
+            return data
+
+    # Return original data if no parsing succeeded
+    return data
+
+
+def _try_parse_json(text: str) -> Data | None:
+    """Try to parse text as JSON and return Data object."""
+    try:
+        parsed = json.loads(text)
+
+        if isinstance(parsed, dict):
+            # Single JSON object
+            return Data(data=parsed)
+        if isinstance(parsed, list) and all(isinstance(item, dict) for item in parsed):
+            # Array of JSON objects - create Data with the list
+            return Data(data={"records": parsed})
+
+    except (json.JSONDecodeError, ValueError):
+        pass
+
+    return None
+
+
+def _looks_like_csv(text: str) -> bool:
+    """Simple heuristic to detect CSV content."""
+    lines = text.strip().split("\n")
+    if len(lines) < MIN_CSV_LINES:
+        return False
+
+    header_line = lines[0]
+    return "," in header_line and len(lines) > 1
+
+
+def _parse_csv_to_data(text: str) -> Data:
+    """Parse CSV text and return Data object."""
+    from io import StringIO
+
+    import pandas as pd
+
+    # Parse CSV to DataFrame, then convert to list of dicts
+    parsed_df = pd.read_csv(StringIO(text))
+    records = parsed_df.to_dict(orient="records")
+
+    return Data(data={"records": records})
+
+
+class TypeConverterComponent(Component):
+    display_name = "Type Convert"
+    description = "Convert between different types (Message, Data, DataFrame)"
+    documentation: str = "https://docs.langflow.org/type-convert"
+    icon = "repeat"
+
+    inputs = [
+        HandleInput(
+            name="input_data",
+            display_name="Input",
+            input_types=["Message", "Data", "DataFrame"],
+            info="Accept Message, Data or DataFrame as input",
+            required=True,
+        ),
+        BoolInput(
+            name="auto_parse",
+            display_name="Auto Parse",
+            info="Detect and convert JSON/CSV strings automatically.",
+            advanced=True,
+            value=False,
+            required=False,
+        ),
+        TabInput(
+            name="output_type",
+            display_name="Output Type",
+            options=["Message", "Data", "DataFrame"],
+            info="Select the desired output data type",
+            real_time_refresh=True,
+            value="Message",
+        ),
+    ]
+
+    outputs = [
+        Output(
+            display_name="Message Output",
+            name="message_output",
+            method="convert_to_message",
+        )
+    ]
+
+    def update_outputs(self, frontend_node: dict, field_name: str, field_value: Any) -> dict:
+        """Dynamically show only the relevant output based on the selected output type."""
+        if field_name == "output_type":
+            # Start with empty outputs
+            frontend_node["outputs"] = []
+
+            # Add only the selected output type
+            if field_value == "Message":
+                frontend_node["outputs"].append(
+                    Output(
+                        display_name="Message Output",
+                        name="message_output",
+                        method="convert_to_message",
+                    ).to_dict()
+                )
+            elif field_value == "Data":
+                frontend_node["outputs"].append(
+                    Output(
+                        display_name="Data Output",
+                        name="data_output",
+                        method="convert_to_data",
+                    ).to_dict()
+                )
+            elif field_value == "DataFrame":
+                frontend_node["outputs"].append(
+                    Output(
+                        display_name="DataFrame Output",
+                        name="dataframe_output",
+                        method="convert_to_dataframe",
+                    ).to_dict()
+                )
+
+        return frontend_node
+
+    def convert_to_message(self) -> Message:
+        """Convert input to Message type."""
+        input_value = self.input_data[0] if isinstance(self.input_data, list) else self.input_data
+
+        # Handle string input by converting to Message first
+        if isinstance(input_value, str):
+            input_value = Message(text=input_value)
+
+        result = convert_to_message(input_value)
+        self.status = result
+        return result
+
+    def convert_to_data(self) -> Data:
+        """Convert input to Data type."""
+        input_value = self.input_data[0] if isinstance(self.input_data, list) else self.input_data
+
+        # Handle string input by converting to Message first
+        if isinstance(input_value, str):
+            input_value = Message(text=input_value)
+
+        result = convert_to_data(input_value, auto_parse=self.auto_parse)
+        self.status = result
+        return result
+
+    def convert_to_dataframe(self) -> DataFrame:
+        """Convert input to DataFrame type."""
+        input_value = self.input_data[0] if isinstance(self.input_data, list) else self.input_data
+
+        # Handle string input by converting to Message first
+        if isinstance(input_value, str):
+            input_value = Message(text=input_value)
+
+        result = convert_to_dataframe(input_value, auto_parse=self.auto_parse)
+        self.status = result
+        return result