lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/schema/content_block.py

@@ -0,0 +1,62 @@
+from typing import Annotated
+
+from pydantic import BaseModel, Discriminator, Field, Tag, field_serializer, field_validator
+from typing_extensions import TypedDict
+
+from .content_types import CodeContent, ErrorContent, JSONContent, MediaContent, TextContent, ToolContent
+
+
+def _get_type(d: dict | BaseModel) -> str | None:
+    if isinstance(d, dict):
+        return d.get("type")
+    return getattr(d, "type", None)
+
+
+# Create a union type of all content types
+ContentType = Annotated[
+    Annotated[ToolContent, Tag("tool_use")]
+    | Annotated[ErrorContent, Tag("error")]
+    | Annotated[TextContent, Tag("text")]
+    | Annotated[MediaContent, Tag("media")]
+    | Annotated[CodeContent, Tag("code")]
+    | Annotated[JSONContent, Tag("json")],
+    Discriminator(_get_type),
+]
+
+
+class ContentBlock(BaseModel):
+    """A block of content that can contain different types of content."""
+
+    title: str
+    contents: list[ContentType]
+    allow_markdown: bool = Field(default=True)
+    media_url: list[str] | None = None
+
+    def __init__(self, **data) -> None:
+        super().__init__(**data)
+        schema_dict = self.__pydantic_core_schema__["schema"]
+        if "fields" in schema_dict:
+            fields = schema_dict["fields"]
+        elif "schema" in schema_dict:
+            fields = schema_dict["schema"]["fields"]
+        fields_with_default = (f for f, d in fields.items() if "default" in d["schema"])
+        self.model_fields_set.update(fields_with_default)
+
+    @field_validator("contents", mode="before")
+    @classmethod
+    def validate_contents(cls, v) -> list[ContentType]:
+        if isinstance(v, dict):
+            msg = "Contents must be a list of ContentTypes"
+            raise TypeError(msg)
+        return [v] if isinstance(v, BaseModel) else v
+
+    @field_serializer("contents")
+    def serialize_contents(self, value) -> list[dict]:
+        return [v.model_dump() for v in value]
+
+
+class ContentBlockDict(TypedDict):
+    title: str
+    contents: list[dict]
+    allow_markdown: bool
+    media_url: list[str] | None

lfx/schema/content_types.py

@@ -0,0 +1,91 @@
+from typing import Any, Literal
+
+from fastapi.encoders import jsonable_encoder
+from pydantic import BaseModel, ConfigDict, Field, model_serializer
+from typing_extensions import TypedDict
+
+from lfx.schema.encoders import CUSTOM_ENCODERS
+
+
+class HeaderDict(TypedDict, total=False):
+    title: str | None
+    icon: str | None
+
+
+class BaseContent(BaseModel):
+    """Base class for all content types."""
+
+    type: str = Field(..., description="Type of the content")
+    duration: int | None = None
+    header: HeaderDict | None = Field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return self.model_dump()
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "BaseContent":
+        return cls(**data)
+
+    @model_serializer(mode="wrap")
+    def serialize_model(self, nxt) -> dict[str, Any]:
+        try:
+            dump = nxt(self)
+            return jsonable_encoder(dump, custom_encoder=CUSTOM_ENCODERS)
+        except Exception:  # noqa: BLE001
+            return nxt(self)
+
+
+class ErrorContent(BaseContent):
+    """Content type for error messages."""
+
+    type: Literal["error"] = Field(default="error")
+    component: str | None = None
+    field: str | None = None
+    reason: str | None = None
+    solution: str | None = None
+    traceback: str | None = None
+
+
+class TextContent(BaseContent):
+    """Content type for simple text content."""
+
+    type: Literal["text"] = Field(default="text")
+    text: str
+    duration: int | None = None
+
+
+class MediaContent(BaseContent):
+    """Content type for media content."""
+
+    type: Literal["media"] = Field(default="media")
+    urls: list[str]
+    caption: str | None = None
+
+
+class JSONContent(BaseContent):
+    """Content type for JSON content."""
+
+    type: Literal["json"] = Field(default="json")
+    data: dict[str, Any]
+
+
+class CodeContent(BaseContent):
+    """Content type for code snippets."""
+
+    type: Literal["code"] = Field(default="code")
+    code: str
+    language: str
+    title: str | None = None
+
+
+class ToolContent(BaseContent):
+    """Content type for tool start content."""
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    type: Literal["tool_use"] = Field(default="tool_use")
+    name: str | None = None
+    tool_input: dict[str, Any] = Field(default_factory=dict, alias="input")
+    output: Any | None = None
+    error: Any | None = None
+    duration: int | None = None

lfx/schema/data.py

@@ -0,0 +1,308 @@
+"""Lightweight Data class for lfx package - contains only methods with no langflow dependencies."""
+
+from __future__ import annotations
+
+import copy
+import json
+from datetime import datetime, timezone
+from decimal import Decimal
+from typing import TYPE_CHECKING, cast
+from uuid import UUID
+
+from langchain_core.documents import Document
+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage
+from pydantic import BaseModel, ConfigDict, model_serializer, model_validator
+
+from lfx.log.logger import logger
+from lfx.utils.constants import MESSAGE_SENDER_AI, MESSAGE_SENDER_USER
+from lfx.utils.image import create_image_content_dict
+
+if TYPE_CHECKING:
+    from lfx.schema.dataframe import DataFrame
+    from lfx.schema.message import Message
+
+
+class Data(BaseModel):
+    """Represents a record with text and optional data.
+
+    Attributes:
+        data (dict, optional): Additional data associated with the record.
+    """
+
+    model_config = ConfigDict(validate_assignment=True)
+
+    text_key: str = "text"
+    data: dict = {}
+    default_value: str | None = ""
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_data(cls, values):
+        if not isinstance(values, dict):
+            msg = "Data must be a dictionary"
+            raise ValueError(msg)  # noqa: TRY004
+        if "data" not in values or values["data"] is None:
+            values["data"] = {}
+        if not isinstance(values["data"], dict):
+            msg = (
+                f"Invalid data format: expected dictionary but got {type(values).__name__}."
+                " This will raise an error in version langflow==1.3.0."
+            )
+            logger.warning(msg)
+        # Any other keyword should be added to the data dictionary
+        for key in values:
+            if key not in values["data"] and key not in {"text_key", "data", "default_value"}:
+                values["data"][key] = values[key]
+        return values
+
+    @model_serializer(mode="plain", when_used="json")
+    def serialize_model(self):
+        return {k: v.to_json() if hasattr(v, "to_json") else v for k, v in self.data.items()}
+
+    def get_text(self):
+        """Retrieves the text value from the data dictionary.
+
+        If the text key is present in the data dictionary, the corresponding value is returned.
+        Otherwise, the default value is returned.
+
+        Returns:
+            The text value from the data dictionary or the default value.
+        """
+        return self.data.get(self.text_key, self.default_value)
+
+    def set_text(self, text: str | None) -> str:
+        r"""Sets the text value in the data dictionary.
+
+        The object's `text` value is set to `text parameter as given, with the following modifications:
+
+         - `text` value of `None` is converted to an empty string.
+         - `text` value is converted to `str` type.
+
+        Args:
+            text (str): The text to be set in the data dictionary.
+
+        Returns:
+            str: The text value that was set in the data dictionary.
+        """
+        new_text = "" if text is None else str(text)
+        self.data[self.text_key] = new_text
+        return new_text
+
+    @classmethod
+    def from_document(cls, document: Document) -> Data:
+        """Converts a Document to a Data.
+
+        Args:
+            document (Document): The Document to convert.
+
+        Returns:
+            Data: The converted Data.
+        """
+        data = document.metadata
+        data["text"] = document.page_content
+        return cls(data=data, text_key="text")
+
+    @classmethod
+    def from_lc_message(cls, message: BaseMessage) -> Data:
+        """Converts a BaseMessage to a Data.
+
+        Args:
+            message (BaseMessage): The BaseMessage to convert.
+
+        Returns:
+            Data: The converted Data.
+        """
+        data: dict = {"text": message.content}
+        data["metadata"] = cast("dict", message.to_json())
+        return cls(data=data, text_key="text")
+
+    def __add__(self, other: Data) -> Data:
+        """Combines the data of two data by attempting to add values for overlapping keys.
+
+        Combines the data of two data by attempting to add values for overlapping keys
+        for all types that support the addition operation. Falls back to the value from 'other'
+        record when addition is not supported.
+        """
+        combined_data = self.data.copy()
+        for key, value in other.data.items():
+            # If the key exists in both data and both values support the addition operation
+            if key in combined_data:
+                try:
+                    combined_data[key] += value
+                except TypeError:
+                    # Fallback: Use the value from 'other' record if addition is not supported
+                    combined_data[key] = value
+            else:
+                # If the key is not in the first record, simply add it
+                combined_data[key] = value
+
+        return Data(data=combined_data)
+
+    def to_lc_document(self) -> Document:
+        """Converts the Data to a Document.
+
+        Returns:
+            Document: The converted Document.
+        """
+        data_copy = self.data.copy()
+        text = data_copy.pop(self.text_key, self.default_value)
+        if isinstance(text, str):
+            return Document(page_content=text, metadata=data_copy)
+        return Document(page_content=str(text), metadata=data_copy)
+
+    def to_lc_message(
+        self,
+    ) -> BaseMessage:
+        """Converts the Data to a BaseMessage.
+
+        Returns:
+            BaseMessage: The converted BaseMessage.
+        """
+        # The idea of this function is to be a helper to convert a Data to a BaseMessage
+        # It will use the "sender" key to determine if the message is Human or AI
+        # If the key is not present, it will default to AI
+        # But first we check if all required keys are present in the data dictionary
+        # they are: "text", "sender"
+        if not all(key in self.data for key in ["text", "sender"]):
+            msg = f"Missing required keys ('text', 'sender') in Data: {self.data}"
+            raise ValueError(msg)
+        sender = self.data.get("sender", MESSAGE_SENDER_AI)
+        text = self.data.get("text", "")
+        files = self.data.get("files", [])
+        if sender == MESSAGE_SENDER_USER:
+            if files:
+                from lfx.schema.image import get_file_paths
+
+                resolved_file_paths = get_file_paths(files)
+                contents = [create_image_content_dict(file_path) for file_path in resolved_file_paths]
+                # add to the beginning of the list
+                contents.insert(0, {"type": "text", "text": text})
+                human_message = HumanMessage(content=contents)
+            else:
+                human_message = HumanMessage(
+                    content=[{"type": "text", "text": text}],
+                )
+
+            return human_message
+
+        return AIMessage(content=text)
+
+    def __getattr__(self, key):
+        """Allows attribute-like access to the data dictionary."""
+        try:
+            if key.startswith("__"):
+                return self.__getattribute__(key)
+            if key in {"data", "text_key"} or key.startswith("_"):
+                return super().__getattr__(key)
+            return self.data[key]
+        except KeyError as e:
+            # Fallback to default behavior to raise AttributeError for undefined attributes
+            msg = f"'{type(self).__name__}' object has no attribute '{key}'"
+            raise AttributeError(msg) from e
+
+    def __setattr__(self, key, value) -> None:
+        """Set attribute-like values in the data dictionary.
+
+        Allows attribute-like setting of values in the data dictionary.
+        while still allowing direct assignment to class attributes.
+        """
+        if key in {"data", "text_key"} or key.startswith("_"):
+            super().__setattr__(key, value)
+        elif key in self.model_fields:
+            self.data[key] = value
+            super().__setattr__(key, value)
+        else:
+            self.data[key] = value
+
+    def __delattr__(self, key) -> None:
+        """Allows attribute-like deletion from the data dictionary."""
+        if key in {"data", "text_key"} or key.startswith("_"):
+            super().__delattr__(key)
+        else:
+            del self.data[key]
+
+    def __deepcopy__(self, memo):
+        """Custom deepcopy implementation to handle copying of the Data object."""
+        # Create a new Data object with a deep copy of the data dictionary
+        return Data(data=copy.deepcopy(self.data, memo), text_key=self.text_key, default_value=self.default_value)
+
+    # check which attributes the Data has by checking the keys in the data dictionary
+    def __dir__(self):
+        return super().__dir__() + list(self.data.keys())
+
+    def __str__(self) -> str:
+        # return a JSON string representation of the Data atributes
+        try:
+            data = {k: v.to_json() if hasattr(v, "to_json") else v for k, v in self.data.items()}
+            return serialize_data(data)  # use the custom serializer
+        except Exception:  # noqa: BLE001
+            logger.debug("Error converting Data to JSON", exc_info=True)
+            return str(self.data)
+
+    def __contains__(self, key) -> bool:
+        return key in self.data
+
+    def __eq__(self, /, other):
+        return isinstance(other, Data) and self.data == other.data
+
+    def filter_data(self, filter_str: str) -> Data:
+        """Filters the data dictionary based on the filter string.
+
+        Args:
+            filter_str (str): The filter string to apply to the data dictionary.
+
+        Returns:
+            Data: The filtered Data.
+        """
+        from lfx.template.utils import apply_json_filter
+
+        return apply_json_filter(self.data, filter_str)
+
+    def to_message(self) -> Message:
+        from lfx.schema.message import Message  # Local import to avoid circular import
+
+        if self.text_key in self.data:
+            return Message(text=self.get_text())
+        return Message(text=str(self.data))
+
+    def to_dataframe(self) -> DataFrame:
+        from lfx.schema.dataframe import DataFrame  # Local import to avoid circular import
+
+        data_dict = self.data
+        # If data contains only one key and the value is a list of dictionaries, convert to DataFrame
+        if (
+            len(data_dict) == 1
+            and isinstance(next(iter(data_dict.values())), list)
+            and all(isinstance(item, dict) for item in next(iter(data_dict.values())))
+        ):
+            return DataFrame(data=next(iter(data_dict.values())))
+        return DataFrame(data=[self])
+
+    def __repr__(self) -> str:
+        """Return string representation of the Data object."""
+        return f"Data(text_key={self.text_key!r}, data={self.data!r}, default_value={self.default_value!r})"
+
+    def __hash__(self) -> int:
+        """Return hash of the Data object based on its string representation."""
+        return hash(self.__repr__())
+
+
+def custom_serializer(obj):
+    if isinstance(obj, datetime):
+        utc_date = obj.replace(tzinfo=timezone.utc)
+        return utc_date.strftime("%Y-%m-%d %H:%M:%S %Z")
+    if isinstance(obj, Decimal):
+        return float(obj)
+    if isinstance(obj, UUID):
+        return str(obj)
+    if isinstance(obj, BaseModel):
+        return obj.model_dump()
+    if isinstance(obj, bytes):
+        return obj.decode("utf-8", errors="replace")
+    # Add more custom serialization rules as needed
+    msg = f"Type {type(obj)} not serializable"
+    raise TypeError(msg)
+
+
+def serialize_data(data):
+    return json.dumps(data, indent=4, default=custom_serializer)

lfx/schema/dataframe.py

@@ -0,0 +1,210 @@
+from typing import TYPE_CHECKING, cast
+
+import pandas as pd
+from langchain_core.documents import Document
+from pandas import DataFrame as pandas_DataFrame
+
+from lfx.schema.data import Data
+
+if TYPE_CHECKING:
+    from lfx.schema.message import Message
+
+
+class DataFrame(pandas_DataFrame):
+    """A pandas DataFrame subclass specialized for handling collections of Data objects.
+
+    This class extends pandas.DataFrame to provide seamless integration between
+    Langflow's Data objects and pandas' powerful data manipulation capabilities.
+
+    Args:
+        data: Input data in various formats:
+            - List[Data]: List of Data objects
+            - List[Dict]: List of dictionaries
+            - Dict: Dictionary of arrays/lists
+            - pandas.DataFrame: Existing DataFrame
+            - Any format supported by pandas.DataFrame
+        **kwargs: Additional arguments passed to pandas.DataFrame constructor
+
+    Examples:
+        >>> # From Data objects
+        >>> dataset = DataFrame([Data(data={"name": "John"}), Data(data={"name": "Jane"})])
+
+        >>> # From dictionaries
+        >>> dataset = DataFrame([{"name": "John"}, {"name": "Jane"}])
+
+        >>> # From dictionary of lists
+        >>> dataset = DataFrame({"name": ["John", "Jane"], "age": [30, 25]})
+    """
+
+    def __init__(
+        self,
+        data: list[dict] | list[Data] | pd.DataFrame | None = None,
+        text_key: str = "text",
+        default_value: str = "",
+        **kwargs,
+    ):
+        # Initialize pandas DataFrame first without data
+        super().__init__(**kwargs)  # Removed data parameter
+
+        # Store attributes as private members to avoid conflicts with pandas
+        self._text_key = text_key
+        self._default_value = default_value
+
+        if data is None:
+            return
+
+        if isinstance(data, list):
+            if all(isinstance(x, Data) for x in data):
+                data = [d.data for d in data if hasattr(d, "data")]
+            elif not all(isinstance(x, dict) for x in data):
+                msg = "List items must be either all Data objects or all dictionaries"
+                raise ValueError(msg)
+            self._update(data, **kwargs)
+        elif isinstance(data, dict | pd.DataFrame):  # Fixed type check syntax
+            self._update(data, **kwargs)
+
+    def _update(self, data, **kwargs):
+        """Helper method to update DataFrame with new data."""
+        new_df = pd.DataFrame(data, **kwargs)
+        self._update_inplace(new_df)
+
+    # Update property accessors
+    @property
+    def text_key(self) -> str:
+        return self._text_key
+
+    @text_key.setter
+    def text_key(self, value: str) -> None:
+        if value not in self.columns:
+            msg = f"Text key '{value}' not found in DataFrame columns"
+            raise ValueError(msg)
+        self._text_key = value
+
+    @property
+    def default_value(self) -> str:
+        return self._default_value
+
+    @default_value.setter
+    def default_value(self, value: str) -> None:
+        self._default_value = value
+
+    def to_data_list(self) -> list[Data]:
+        """Converts the DataFrame back to a list of Data objects."""
+        list_of_dicts = self.to_dict(orient="records")
+        # suggested change: [Data(**row) for row in list_of_dicts]
+        return [Data(data=row) for row in list_of_dicts]
+
+    def add_row(self, data: dict | Data) -> "DataFrame":
+        """Adds a single row to the dataset.
+
+        Args:
+            data: Either a Data object or a dictionary to add as a new row
+
+        Returns:
+            DataFrame: A new DataFrame with the added row
+
+        Example:
+            >>> dataset = DataFrame([{"name": "John"}])
+            >>> dataset = dataset.add_row({"name": "Jane"})
+        """
+        if isinstance(data, Data):
+            data = data.data
+        new_df = self._constructor([data])
+        return cast("DataFrame", pd.concat([self, new_df], ignore_index=True))
+
+    def add_rows(self, data: list[dict | Data]) -> "DataFrame":
+        """Adds multiple rows to the dataset.
+
+        Args:
+            data: List of Data objects or dictionaries to add as new rows
+
+        Returns:
+            DataFrame: A new DataFrame with the added rows
+        """
+        processed_data = []
+        for item in data:
+            if isinstance(item, Data):
+                processed_data.append(item.data)
+            else:
+                processed_data.append(item)
+        new_df = self._constructor(processed_data)
+        return cast("DataFrame", pd.concat([self, new_df], ignore_index=True))
+
+    @property
+    def _constructor(self):
+        def _c(*args, **kwargs):
+            return DataFrame(*args, **kwargs).__finalize__(self)
+
+        return _c
+
+    def __bool__(self):
+        """Truth value testing for the DataFrame.
+
+        Returns True if the DataFrame has at least one row, False otherwise.
+        """
+        return not self.empty
+
+    __hash__ = None  # DataFrames are mutable and shouldn't be hashable
+
+    def to_lc_documents(self) -> list[Document]:
+        """Converts the DataFrame to a list of Documents.
+
+        Returns:
+            list[Document]: The converted list of Documents.
+        """
+        list_of_dicts = self.to_dict(orient="records")
+        documents = []
+        for row in list_of_dicts:
+            data_copy = row.copy()
+            text = data_copy.pop(self._text_key, self._default_value)
+            if isinstance(text, str):
+                documents.append(Document(page_content=text, metadata=data_copy))
+            else:
+                documents.append(Document(page_content=str(text), metadata=data_copy))
+        return documents
+
+    def _docs_to_dataframe(self, docs):
+        """Converts a list of Documents to a DataFrame.
+
+        Args:
+            docs: List of Document objects
+
+        Returns:
+            DataFrame: A new DataFrame with the converted Documents
+        """
+        return DataFrame(docs)
+
+    def __eq__(self, other):
+        """Override equality to handle comparison with empty DataFrames and non-DataFrame objects."""
+        if self.empty:
+            return False
+        if isinstance(other, list) and not other:  # Empty list case
+            return False
+        if not isinstance(other, DataFrame | pd.DataFrame):  # Non-DataFrame case
+            return False
+        return super().__eq__(other)
+
+    def to_data(self) -> Data:
+        """Convert this DataFrame to a Data object.
+
+        Returns:
+            Data: A Data object containing the DataFrame records under 'results' key.
+        """
+        dict_list = self.to_dict(orient="records")
+        return Data(data={"results": dict_list})
+
+    def to_message(self) -> "Message":
+        from lfx.schema.message import Message
+
+        # Process DataFrame similar to the _safe_convert method
+        # Remove empty rows
+        processed_df = self.dropna(how="all")
+        # Remove empty lines in each cell
+        processed_df = processed_df.replace(r"^\s*$", "", regex=True)
+        # Replace multiple newlines with a single newline
+        processed_df = processed_df.replace(r"\n+", "\n", regex=True)
+        # Replace pipe characters to avoid markdown table issues
+        processed_df = processed_df.replace(r"\|", r"\\|", regex=True)
+        processed_df = processed_df.map(lambda x: str(x).replace("\n", "<br/>") if isinstance(x, str) else x)
+        # Convert to markdown and wrap in a Message
+        return Message(text=processed_df.to_markdown(index=False))