lfx-nightly 0.2.0.dev25__py3-none-any.whl

lfx/utils/concurrency.py

@@ -0,0 +1,60 @@
+import re
+import threading
+from contextlib import contextmanager
+from pathlib import Path
+
+from filelock import FileLock
+from platformdirs import user_cache_dir
+
+
+class KeyedMemoryLockManager:
+    """A manager for acquiring and releasing memory locks based on a key."""
+
+    def __init__(self) -> None:
+        self.locks: dict[str, threading.Lock] = {}
+        self.global_lock = threading.Lock()
+
+    def _get_lock(self, key: str):
+        with self.global_lock:
+            if key not in self.locks:
+                self.locks[key] = threading.Lock()
+            return self.locks[key]
+
+    @contextmanager
+    def lock(self, key: str):
+        lock = self._get_lock(key)
+        lock.acquire()
+        try:
+            yield
+        finally:
+            lock.release()
+
+
+class KeyedWorkerLockManager:
+    """A manager for acquiring locks between workers based on a key."""
+
+    def __init__(self) -> None:
+        self.locks_dir = Path(user_cache_dir("langflow"), ensure_exists=True) / "worker_locks"
+
+    @staticmethod
+    def _validate_key(key: str) -> bool:
+        """Validate that the string only contains alphanumeric characters and underscores.
+
+        Parameters:
+        s (str): The string to validate.
+
+        Returns:
+        bool: True if the string is valid, False otherwise.
+        """
+        pattern = re.compile(r"^\w+$")
+        return bool(pattern.match(key))
+
+    @contextmanager
+    def lock(self, key: str):
+        if not self._validate_key(key):
+            msg = f"Invalid key: {key}"
+            raise ValueError(msg)
+
+        lock = FileLock(self.locks_dir / key)
+        with lock:
+            yield

lfx/utils/connection_string_parser.py

@@ -0,0 +1,11 @@
+"""Connection string parser utilities for lfx package."""
+
+from urllib.parse import quote
+
+
+def transform_connection_string(connection_string) -> str:
+    """Transform connection string by encoding the password part."""
+    auth_part, db_url_name = connection_string.rsplit("@", 1)
+    protocol_user, password_string = auth_part.rsplit(":", 1)
+    encoded_password = quote(password_string)
+    return f"{protocol_user}:{encoded_password}@{db_url_name}"

lfx/utils/constants.py

@@ -0,0 +1,233 @@
+from typing import Any
+
+OPENAI_MODELS = [
+    "text-davinci-003",
+    "text-davinci-002",
+    "text-curie-001",
+    "text-babbage-001",
+    "text-ada-001",
+]
+CHAT_OPENAI_MODELS = [
+    "gpt-4o",
+    "gpt-4o-mini",
+    "gpt-4-turbo-preview",
+    "gpt-4-0125-preview",
+    "gpt-4-1106-preview",
+    "gpt-4-vision-preview",
+    "gpt-3.5-turbo-0125",
+    "gpt-3.5-turbo-1106",
+]
+
+REASONING_OPENAI_MODELS = [
+    "o1",
+    "o1-mini",
+    "o1-pro",
+    "o3-mini",
+    "o3",
+    "o3-pro",
+    "o4-mini",
+    "o4-mini-high",
+]
+
+ANTHROPIC_MODELS = [
+    # largest model, ideal for a wide range of more complex tasks.
+    "claude-v1",
+    # An enhanced version of claude-v1 with a 100,000 token (roughly 75,000 word) context window.
+    "claude-v1-100k",
+    # A smaller model with far lower latency, sampling at roughly 40 words/sec!
+    "claude-instant-v1",
+    # Like claude-instant-v1 with a 100,000 token context window but retains its performance.
+    "claude-instant-v1-100k",
+    # Specific sub-versions of the above models:
+    # Vs claude-v1.2: better instruction-following, code, and non-English dialogue and writing.
+    "claude-v1.3",
+    # An enhanced version of claude-v1.3 with a 100,000 token (roughly 75,000 word) context window.
+    "claude-v1.3-100k",
+    # Vs claude-v1.1: small adv in general helpfulness, instruction following, coding, and other tasks.
+    "claude-v1.2",
+    # An earlier version of claude-v1.
+    "claude-v1.0",
+    # Latest version of claude-instant-v1. Better than claude-instant-v1.0 at most tasks.
+    "claude-instant-v1.1",
+    # Version of claude-instant-v1.1 with a 100K token context window.
+    "claude-instant-v1.1-100k",
+    # An earlier version of claude-instant-v1.
+    "claude-instant-v1.0",
+]
+
+DEFAULT_PYTHON_FUNCTION = """
+def python_function(text: str) -> str:
+    \"\"\"This is a default python function that returns the input text\"\"\"
+    return text
+"""
+
+
+PYTHON_BASIC_TYPES = [str, bool, int, float, tuple, list, dict, set]
+DIRECT_TYPES = [
+    "str",
+    "bool",
+    "dict",
+    "int",
+    "float",
+    "Any",
+    "prompt",
+    "code",
+    "NestedDict",
+    "table",
+    "slider",
+    "tab",
+    "sortableList",
+    "auth",
+    "connect",
+    "query",
+    "tools",
+    "mcp",
+]
+
+
+LOADERS_INFO: list[dict[str, Any]] = [
+    {
+        "loader": "AirbyteJSONLoader",
+        "name": "Airbyte JSON (.jsonl)",
+        "import": "langchain_community.document_loaders.AirbyteJSONLoader",
+        "defaultFor": ["jsonl"],
+        "allowdTypes": ["jsonl"],
+    },
+    {
+        "loader": "JSONLoader",
+        "name": "JSON (.json)",
+        "import": "langchain_community.document_loaders.JSONLoader",
+        "defaultFor": ["json"],
+        "allowdTypes": ["json"],
+    },
+    {
+        "loader": "BSHTMLLoader",
+        "name": "BeautifulSoup4 HTML (.html, .htm)",
+        "import": "langchain_community.document_loaders.BSHTMLLoader",
+        "allowdTypes": ["html", "htm"],
+    },
+    {
+        "loader": "CSVLoader",
+        "name": "CSV (.csv)",
+        "import": "langchain_community.document_loaders.CSVLoader",
+        "defaultFor": ["csv"],
+        "allowdTypes": ["csv"],
+    },
+    {
+        "loader": "CoNLLULoader",
+        "name": "CoNLL-U (.conllu)",
+        "import": "langchain_community.document_loaders.CoNLLULoader",
+        "defaultFor": ["conllu"],
+        "allowdTypes": ["conllu"],
+    },
+    {
+        "loader": "EverNoteLoader",
+        "name": "EverNote (.enex)",
+        "import": "langchain_community.document_loaders.EverNoteLoader",
+        "defaultFor": ["enex"],
+        "allowdTypes": ["enex"],
+    },
+    {
+        "loader": "FacebookChatLoader",
+        "name": "Facebook Chat (.json)",
+        "import": "langchain_community.document_loaders.FacebookChatLoader",
+        "allowdTypes": ["json"],
+    },
+    {
+        "loader": "OutlookMessageLoader",
+        "name": "Outlook Message (.msg)",
+        "import": "langchain_community.document_loaders.OutlookMessageLoader",
+        "defaultFor": ["msg"],
+        "allowdTypes": ["msg"],
+    },
+    {
+        "loader": "PyPDFLoader",
+        "name": "PyPDF (.pdf)",
+        "import": "langchain_community.document_loaders.PyPDFLoader",
+        "defaultFor": ["pdf"],
+        "allowdTypes": ["pdf"],
+    },
+    {
+        "loader": "STRLoader",
+        "name": "Subtitle (.str)",
+        "import": "langchain_community.document_loaders.STRLoader",
+        "defaultFor": ["str"],
+        "allowdTypes": ["str"],
+    },
+    {
+        "loader": "TextLoader",
+        "name": "Text (.txt)",
+        "import": "langchain_community.document_loaders.TextLoader",
+        "defaultFor": ["txt"],
+        "allowdTypes": ["txt"],
+    },
+    {
+        "loader": "UnstructuredEmailLoader",
+        "name": "Unstructured Email (.eml)",
+        "import": "langchain_community.document_loaders.UnstructuredEmailLoader",
+        "defaultFor": ["eml"],
+        "allowdTypes": ["eml"],
+    },
+    {
+        "loader": "UnstructuredHTMLLoader",
+        "name": "Unstructured HTML (.html, .htm)",
+        "import": "langchain_community.document_loaders.UnstructuredHTMLLoader",
+        "defaultFor": ["html", "htm"],
+        "allowdTypes": ["html", "htm"],
+    },
+    {
+        "loader": "UnstructuredMarkdownLoader",
+        "name": "Unstructured Markdown (.md)",
+        "import": "langchain_community.document_loaders.UnstructuredMarkdownLoader",
+        "defaultFor": ["md", "mdx"],
+        "allowdTypes": ["md", "mdx"],
+    },
+    {
+        "loader": "UnstructuredPowerPointLoader",
+        "name": "Unstructured PowerPoint (.pptx)",
+        "import": "langchain_community.document_loaders.UnstructuredPowerPointLoader",
+        "defaultFor": ["pptx"],
+        "allowdTypes": ["pptx"],
+    },
+    {
+        "loader": "UnstructuredWordLoader",
+        "name": "Unstructured Word (.docx)",
+        "import": "langchain_community.document_loaders.UnstructuredWordLoader",
+        "defaultFor": ["docx"],
+        "allowdTypes": ["docx"],
+    },
+]
+
+
+MESSAGE_SENDER_AI = "Machine"
+MESSAGE_SENDER_USER = "User"
+MESSAGE_SENDER_NAME_AI = "AI"
+MESSAGE_SENDER_NAME_USER = "User"
+EXTENSION_TO_CONTENT_TYPE = {
+    "json": "application/json",
+    "txt": "text/plain",
+    "csv": "text/csv",
+    "html": "text/html",
+    "pdf": "application/pdf",
+    "png": "image/png",
+    "jpg": "image/jpeg",
+    "jpeg": "image/jpeg",
+    "gif": "image/gif",
+    "svg": "image/svg+xml",
+    "mp3": "audio/mpeg",
+    "wav": "audio/wav",
+    "mp4": "video/mp4",
+    "webm": "video/webm",
+    "zip": "application/zip",
+    "tar": "application/x-tar",
+    "gz": "application/gzip",
+    "doc": "application/msword",
+    "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "xls": "application/vnd.ms-excel",
+    "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "ppt": "application/vnd.ms-powerpoint",
+    "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+    "xml": "application/xml",
+    "yaml": "application/x-yaml",
+    "yml": "application/x-yaml",
+}

lfx/utils/data_structure.py

@@ -0,0 +1,212 @@
+import json
+from collections import Counter
+from typing import Any
+
+from lfx.schema.data import Data
+
+
+def infer_list_type(items: list, max_samples: int = 5) -> str:
+    """Infer the type of a list by sampling its items.
+
+    Handles mixed types and provides more detailed type information.
+    """
+    if not items:
+        return "list(unknown)"
+
+    # Sample items (use all if less than max_samples)
+    samples = items[:max_samples]
+    types = [get_type_str(item) for item in samples]
+
+    # Count type occurrences
+    type_counter = Counter(types)
+
+    if len(type_counter) == 1:
+        # Single type
+        return f"list({types[0]})"
+    # Mixed types - show all found types
+    type_str = "|".join(sorted(type_counter.keys()))
+    return f"list({type_str})"
+
+
+def get_type_str(value: Any) -> str:
+    """Get a detailed string representation of the type of a value.
+
+    Handles special cases and provides more specific type information.
+    """
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return "bool"
+    if isinstance(value, int):
+        return "int"
+    if isinstance(value, float):
+        return "float"
+    if isinstance(value, str):
+        # Check if string is actually a date/datetime
+        if any(date_pattern in value.lower() for date_pattern in ["date", "time", "yyyy", "mm/dd", "dd/mm", "yyyy-mm"]):
+            return "str(possible_date)"
+        # Check if it's a JSON string
+        try:
+            json.loads(value)
+            return "str(json)"
+        except (json.JSONDecodeError, TypeError):
+            pass
+        else:
+            return "str"
+    if isinstance(value, list | tuple | set):
+        return infer_list_type(list(value))
+    if isinstance(value, dict):
+        return "dict"
+    # Handle custom objects
+    return type(value).__name__
+
+
+def analyze_value(
+    value: Any,
+    max_depth: int = 10,
+    current_depth: int = 0,
+    path: str = "",
+    *,
+    size_hints: bool = True,
+    include_samples: bool = True,
+) -> str | dict:
+    """Analyze a value and return its structure with additional metadata.
+
+    Args:
+        value: The value to analyze
+        max_depth: Maximum recursion depth
+        current_depth: Current recursion depth
+        path: Current path in the structure
+        size_hints: Whether to include size information for collections
+        include_samples: Whether to include sample structure for lists
+    """
+    if current_depth >= max_depth:
+        return f"max_depth_reached(depth={max_depth})"
+
+    try:
+        if isinstance(value, list | tuple | set):
+            length = len(value)
+            if length == 0:
+                return "list(unknown)"
+
+            type_info = infer_list_type(list(value))
+            size_info = f"[size={length}]" if size_hints else ""
+
+            # For lists of complex objects, include a sample of the structure
+            if (
+                include_samples
+                and length > 0
+                and isinstance(value, list | tuple)
+                and isinstance(value[0], dict | list)
+                and current_depth < max_depth - 1
+            ):
+                sample = analyze_value(
+                    value[0],
+                    max_depth,
+                    current_depth + 1,
+                    f"{path}[0]",
+                    size_hints=size_hints,
+                    include_samples=include_samples,
+                )
+                return f"{type_info}{size_info}, sample: {json.dumps(sample)}"
+
+            return f"{type_info}{size_info}"
+
+        if isinstance(value, dict):
+            result = {}
+            for k, v in value.items():
+                new_path = f"{path}.{k}" if path else k
+                try:
+                    result[k] = analyze_value(
+                        v,
+                        max_depth,
+                        current_depth + 1,
+                        new_path,
+                        size_hints=size_hints,
+                        include_samples=include_samples,
+                    )
+                except Exception as e:  # noqa: BLE001
+                    result[k] = f"error({e!s})"
+            return result
+
+        return get_type_str(value)
+
+    except Exception as e:  # noqa: BLE001
+        return f"error({e!s})"
+
+
+def get_data_structure(
+    data_obj: Data | dict,
+    max_depth: int = 10,
+    max_sample_size: int = 3,
+    *,
+    size_hints: bool = True,
+    include_sample_values: bool = False,
+    include_sample_structure: bool = True,
+) -> dict:
+    """Convert a Data object or dictionary into a detailed schema representation.
+
+    Args:
+        data_obj: The Data object or dictionary to analyze
+        max_depth: Maximum depth for nested structures
+        size_hints: Include size information for collections
+        include_sample_values: Whether to include sample values in the output
+        include_sample_structure: Whether to include sample structure for lists
+        max_sample_size: Maximum number of sample values to include
+
+    Returns:
+        dict: A dictionary containing:
+            - structure: The structure of the data
+            - samples: (optional) Sample values from the data
+
+    Example:
+        >>> data = {
+        ...     "name": "John",
+        ...     "scores": [1, 2, 3, 4, 5],
+        ...     "details": {
+        ...         "age": 30,
+        ...         "cities": ["NY", "LA", "SF", "CHI"],
+        ...         "metadata": {
+        ...             "created": "2023-01-01",
+        ...             "tags": ["user", "admin", 123]
+        ...         }
+        ...     }
+        ... }
+        >>> result = get_data_structure(data)
+        {
+            "structure": {
+                "name": "str",
+                "scores": "list(int)[size=5]",
+                "details": {
+                    "age": "int",
+                    "cities": "list(str)[size=4]",
+                    "metadata": {
+                        "created": "str(possible_date)",
+                        "tags": "list(str|int)[size=3]"
+                    }
+                }
+            }
+        }
+    """
+    # Handle both Data objects and dictionaries
+    data = data_obj.data if isinstance(data_obj, Data) else data_obj
+
+    result = {
+        "structure": analyze_value(
+            data, max_depth=max_depth, size_hints=size_hints, include_samples=include_sample_structure
+        )
+    }
+
+    if include_sample_values:
+        result["samples"] = get_sample_values(data, max_items=max_sample_size)
+
+    return result
+
+
+def get_sample_values(data: Any, max_items: int = 3) -> Any:
+    """Get sample values from a data structure, handling nested structures."""
+    if isinstance(data, list | tuple | set):
+        return [get_sample_values(item) for item in list(data)[:max_items]]
+    if isinstance(data, dict):
+        return {k: get_sample_values(v, max_items) for k, v in data.items()}
+    return data

lfx/utils/exceptions.py

@@ -0,0 +1,22 @@
+def format_syntax_error_message(exc: SyntaxError) -> str:
+    """Format a SyntaxError message for returning to the frontend."""
+    if exc.text is None:
+        return f"Syntax error in code. Error on line {exc.lineno}"
+    return f"Syntax error in code. Error on line {exc.lineno}: {exc.text.strip()}"
+
+
+def get_causing_exception(exc: BaseException) -> BaseException:
+    """Get the causing exception from an exception."""
+    if hasattr(exc, "__cause__") and exc.__cause__:
+        return get_causing_exception(exc.__cause__)
+    return exc
+
+
+def format_exception_message(exc: Exception) -> str:
+    """Format an exception message for returning to the frontend."""
+    # We need to check if the __cause__ is a SyntaxError
+    # If it is, we need to return the message of the SyntaxError
+    causing_exception = get_causing_exception(exc)
+    if isinstance(causing_exception, SyntaxError):
+        return format_syntax_error_message(causing_exception)
+    return str(exc)

lfx/utils/helpers.py

@@ -0,0 +1,34 @@
+"""Helper utility functions for lfx package."""
+
+from __future__ import annotations
+
+import mimetypes
+from typing import TYPE_CHECKING
+
+from lfx.utils.constants import EXTENSION_TO_CONTENT_TYPE
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+def get_mime_type(file_path: str | Path) -> str:
+    """Get the MIME type of a file based on its extension.
+
+    Args:
+        file_path: Path to the file
+
+    Returns:
+        MIME type string (e.g., 'image/jpeg', 'image/png')
+
+    Raises:
+        ValueError: If MIME type cannot be determined
+    """
+    mime_type, _ = mimetypes.guess_type(str(file_path))
+    if mime_type is None:
+        msg = f"Could not determine MIME type for: {file_path}"
+        raise ValueError(msg)
+    return mime_type
+
+
+def build_content_type_from_extension(extension: str):
+    return EXTENSION_TO_CONTENT_TYPE.get(extension.lower(), "application/octet-stream")

lfx/utils/image.py

@@ -0,0 +1,79 @@
+"""Image utility functions for lfx package."""
+
+from __future__ import annotations
+
+import base64
+from functools import lru_cache
+from pathlib import Path
+
+from lfx.utils.helpers import get_mime_type
+
+
+def convert_image_to_base64(image_path: str | Path) -> str:
+    """Convert an image file to a base64 encoded string.
+
+    Args:
+        image_path: Path to the image file
+
+    Returns:
+        Base64 encoded string of the image
+
+    Raises:
+        FileNotFoundError: If the image file doesn't exist
+    """
+    image_path = Path(image_path)
+    if not image_path.exists():
+        msg = f"Image file not found: {image_path}"
+        raise FileNotFoundError(msg)
+
+    with image_path.open("rb") as image_file:
+        return base64.b64encode(image_file.read()).decode("utf-8")
+
+
+def create_data_url(image_path: str | Path, mime_type: str | None = None) -> str:
+    """Create a data URL from an image file.
+
+    Args:
+        image_path: Path to the image file
+        mime_type: MIME type of the image. If None, will be auto-detected
+
+    Returns:
+        Data URL string in format: data:mime/type;base64,{base64_data}
+
+    Raises:
+        FileNotFoundError: If the image file doesn't exist
+    """
+    image_path = Path(image_path)
+    if not image_path.exists():
+        msg = f"Image file not found: {image_path}"
+        raise FileNotFoundError(msg)
+
+    if mime_type is None:
+        mime_type = get_mime_type(image_path)
+
+    base64_data = convert_image_to_base64(image_path)
+    return f"data:{mime_type};base64,{base64_data}"
+
+
+@lru_cache(maxsize=50)
+def create_image_content_dict(
+    image_path: str | Path, mime_type: str | None = None, model_name: str | None = None
+) -> dict:
+    """Create a content dictionary for multimodal inputs from an image file.
+
+    Args:
+        image_path: Path to the image file
+        mime_type: MIME type of the image. If None, will be auto-detected
+        model_name: Optional model parameter to determine content dict structure
+
+    Returns:
+        Content dictionary with type and image_url fields
+
+    Raises:
+        FileNotFoundError: If the image file doesn't exist
+    """
+    data_url = create_data_url(image_path, mime_type)
+
+    if model_name == "OllamaModel":
+        return {"type": "image_url", "source_type": "url", "image_url": data_url}
+    return {"type": "image", "source_type": "url", "url": data_url}