lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/type_extraction/__init__.py

@@ -0,0 +1,19 @@
+"""Type extraction module for lfx package."""
+
+from lfx.type_extraction.type_extraction import (
+    extract_inner_type,
+    extract_inner_type_from_generic_alias,
+    extract_union_types,
+    extract_union_types_from_generic_alias,
+    extract_uniont_types_from_generic_alias,
+    post_process_type,
+)
+
+__all__ = [
+    "extract_inner_type",
+    "extract_inner_type_from_generic_alias",
+    "extract_union_types",
+    "extract_union_types_from_generic_alias",
+    "extract_uniont_types_from_generic_alias",
+    "post_process_type",
+]

lfx/type_extraction/type_extraction.py

@@ -0,0 +1,75 @@
+import re
+from collections.abc import Sequence as SequenceABC
+from itertools import chain
+from types import GenericAlias
+from typing import Any, Union
+
+
+def extract_inner_type_from_generic_alias(return_type: GenericAlias) -> Any:
+    """Extracts the inner type from a type hint that is a list or a Optional."""
+    if return_type.__origin__ in {list, SequenceABC}:
+        return list(return_type.__args__)
+    return return_type
+
+
+def extract_inner_type(return_type: str) -> str:
+    """Extracts the inner type from a type hint that is a list."""
+    if match := re.match(r"list\[(.*)\]", return_type, re.IGNORECASE):
+        return match[1]
+    return return_type
+
+
+def extract_union_types(return_type: str) -> list[str]:
+    """Extracts the inner type from a type hint that is a list."""
+    # If the return type is a Union, then we need to parse it
+    return_type = return_type.replace("Union", "").replace("[", "").replace("]", "")
+    return_types = return_type.split(",")
+    return [item.strip() for item in return_types]
+
+
+def extract_uniont_types_from_generic_alias(return_type: GenericAlias) -> list:
+    """Extracts the inner type from a type hint that is a Union."""
+    if isinstance(return_type, list):
+        return [
+            _inner_arg
+            for _type in return_type
+            for _inner_arg in _type.__args__
+            if _inner_arg not in {Any, type(None), type(Any)}
+        ]
+    return list(return_type.__args__)
+
+
+def post_process_type(type_):
+    """Process the return type of a function.
+
+    Args:
+        type_ (Any): The return type of the function.
+
+    Returns:
+        Union[List[Any], Any]: The processed return type.
+    """
+    if hasattr(type_, "__origin__") and type_.__origin__ in {list, list, SequenceABC}:
+        type_ = extract_inner_type_from_generic_alias(type_)
+
+    # If the return type is not a Union, then we just return it as a list
+    inner_type = type_[0] if isinstance(type_, list) else type_
+    if (not hasattr(inner_type, "__origin__") or inner_type.__origin__ != Union) and (
+        not hasattr(inner_type, "__class__") or inner_type.__class__.__name__ != "UnionType"
+    ):
+        return type_ if isinstance(type_, list) else [type_]
+    # If the return type is a Union, then we need to parse it
+    type_ = extract_union_types_from_generic_alias(type_)
+    type_ = set(chain.from_iterable([post_process_type(t) for t in type_]))
+    return list(type_)
+
+
+def extract_union_types_from_generic_alias(return_type: GenericAlias) -> list:
+    """Extracts the inner type from a type hint that is a Union."""
+    if isinstance(return_type, list):
+        return [
+            _inner_arg
+            for _type in return_type
+            for _inner_arg in _type.__args__
+            if _inner_arg not in {Any, type(None), type(Any)}
+        ]
+    return list(return_type.__args__)

lfx/type_extraction.py

@@ -0,0 +1,80 @@
+"""Type extraction utilities copied from langflow for lfx package."""
+
+import re
+from collections.abc import Sequence as SequenceABC
+from itertools import chain
+from types import GenericAlias
+from typing import Any, Union
+
+
+def extract_inner_type_from_generic_alias(return_type: GenericAlias) -> Any:
+    """Extracts the inner type from a type hint that is a list or a Optional."""
+    if return_type.__origin__ in {list, SequenceABC}:
+        return list(return_type.__args__)
+    return return_type
+
+
+def extract_inner_type(return_type: str) -> str:
+    """Extracts the inner type from a type hint that is a list."""
+    if match := re.match(r"list\[(.*)\]", return_type, re.IGNORECASE):
+        return match[1]
+    return return_type
+
+
+def extract_union_types(return_type: str) -> list[str]:
+    """Extracts the inner type from a type hint that is a list."""
+    # If the return type is a Union, then we need to parse it
+    return_type = return_type.replace("Union", "").replace("[", "").replace("]", "")
+    return_types = return_type.split(",")
+    return [item.strip() for item in return_types]
+
+
+def extract_uniont_types_from_generic_alias(return_type: GenericAlias) -> list:
+    """Extracts the inner type from a type hint that is a Union."""
+    if isinstance(return_type, list):
+        return [
+            _inner_arg
+            for _type in return_type
+            for _inner_arg in _type.__args__
+            if _inner_arg not in {Any, type(None), type(Any)}
+        ]
+
+    return list(return_type.__args__)
+
+
+def post_process_type(type_):
+    """Process the return type of a function.
+
+    Args:
+        type_ (Any): The return type of the function.
+
+    Returns:
+        Union[List[Any], Any]: The processed return type.
+
+    """
+    if hasattr(type_, "__origin__") and type_.__origin__ in {list, list, SequenceABC}:
+        type_ = extract_inner_type_from_generic_alias(type_)
+
+    # If the return type is not a Union, then we just return it as a list
+    inner_type = type_[0] if isinstance(type_, list) else type_
+    if (not hasattr(inner_type, "__origin__") or inner_type.__origin__ != Union) and (
+        not hasattr(inner_type, "__class__") or inner_type.__class__.__name__ != "UnionType"
+    ):
+        return type_ if isinstance(type_, list) else [type_]
+    # If the return type is a Union, then we need to parse it
+    type_ = extract_union_types_from_generic_alias(type_)
+    type_ = set(chain.from_iterable([post_process_type(t) for t in type_]))
+    return list(type_)
+
+
+def extract_union_types_from_generic_alias(return_type: GenericAlias) -> list:
+    """Extracts the inner type from a type hint that is a Union."""
+    if isinstance(return_type, list):
+        return [
+            _inner_arg
+            for _type in return_type
+            for _inner_arg in _type.__args__
+            if _inner_arg not in {Any, type(None), type(Any)}
+        ]
+
+    return list(return_type.__args__)

lfx/utils/__init__.py

@@ -0,0 +1 @@
+"""Utilities for lfx package."""

lfx/utils/async_helpers.py

@@ -0,0 +1,42 @@
+import asyncio
+from contextlib import asynccontextmanager
+
+if hasattr(asyncio, "timeout"):
+
+    @asynccontextmanager
+    async def timeout_context(timeout_seconds):
+        with asyncio.timeout(timeout_seconds) as ctx:
+            yield ctx
+
+else:
+
+    @asynccontextmanager
+    async def timeout_context(timeout_seconds):
+        try:
+            yield await asyncio.wait_for(asyncio.Future(), timeout=timeout_seconds)
+        except asyncio.TimeoutError as e:
+            msg = f"Operation timed out after {timeout_seconds} seconds"
+            raise TimeoutError(msg) from e
+
+
+def run_until_complete(coro):
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        # If there's no event loop, create a new one and run the coroutine
+        return asyncio.run(coro)
+    # If there's already a running event loop, we can't call run_until_complete on it
+    # Instead, we need to run the coroutine in a new thread with a new event loop
+    import concurrent.futures
+
+    def run_in_new_loop():
+        new_loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(new_loop)
+        try:
+            return new_loop.run_until_complete(coro)
+        finally:
+            new_loop.close()
+
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        future = executor.submit(run_in_new_loop)
+        return future.result()

lfx/utils/component_utils.py

@@ -0,0 +1,154 @@
+from collections.abc import Callable
+from typing import Any
+
+from lfx.schema.dotdict import dotdict
+
+DEFAULT_FIELDS = ["code", "_type"]
+
+
+def update_fields(build_config: dotdict, fields: dict[str, Any]) -> dotdict:
+    """Update specified fields in build_config with new values."""
+    for key, value in fields.items():
+        if key in build_config:
+            build_config[key] = value
+    return build_config
+
+
+def add_fields(build_config: dotdict, fields: dict[str, Any]) -> dotdict:
+    """Add new fields to build_config."""
+    build_config.update(fields)
+    return build_config
+
+
+def delete_fields(build_config: dotdict, fields: dict[str, Any] | list[str]) -> dotdict:
+    """Delete specified fields from build_config."""
+    if isinstance(fields, dict):
+        fields = list(fields.keys())
+
+    for field in fields:
+        build_config.pop(field, None)
+    return build_config
+
+
+def get_fields(build_config: dotdict, fields: list[str] | None = None) -> dict[str, Any]:
+    """Get fields from build_config.If fields is None, return all fields."""
+    if fields is None:
+        return dict(build_config)
+
+    result = {}
+    for field in fields:
+        if field in build_config:
+            result[field] = build_config[field]
+    return result
+
+
+def update_input_types(build_config: dotdict) -> dotdict:
+    """Update input types for all fields in build_config."""
+    for key, value in build_config.items():
+        if isinstance(value, dict):
+            if value.get("input_types") is None:
+                build_config[key]["input_types"] = []
+        elif hasattr(value, "input_types") and value.input_types is None:
+            value.input_types = []
+    return build_config
+
+
+def set_field_display(build_config: dotdict, field: str, value: bool | None = None) -> dotdict:  # noqa: FBT001
+    """Set whether a field should be displayed in the UI."""
+    if field in build_config and isinstance(build_config[field], dict) and "show" in build_config[field]:
+        build_config[field]["show"] = value
+    return build_config
+
+
+def set_multiple_field_display(
+    build_config: dotdict,
+    *,
+    fields: dict[str, bool] | None = None,
+    value: bool | None = None,
+    field_list: list[str] | None = None,
+) -> dotdict:
+    """Set display property for multiple fields at once."""
+    if fields is not None:
+        for field, visibility in fields.items():
+            build_config = set_field_display(build_config, field, value=visibility)
+    elif field_list is not None:
+        for field in field_list:
+            build_config = set_field_display(build_config, field, value=value)
+    return build_config
+
+
+def set_field_advanced(build_config: dotdict, field: str, *, value: bool | None = None) -> dotdict:
+    """Set whether a field is considered 'advanced' in the UI."""
+    if value is None:
+        value = False
+    if field in build_config and isinstance(build_config[field], dict):
+        build_config[field]["advanced"] = value
+    return build_config
+
+
+def set_multiple_field_advanced(
+    build_config: dotdict,
+    *,
+    fields: dict[str, bool] | None = None,
+    value: bool | None = None,
+    field_list: list[str] | None = None,
+) -> dotdict:
+    """Set advanced property for multiple fields at once."""
+    if fields is not None:
+        for field, advanced in fields.items():
+            build_config = set_field_advanced(build_config, field, value=advanced)
+    elif field_list is not None:
+        for field in field_list:
+            build_config = set_field_advanced(build_config, field, value=value)
+    return build_config
+
+
+def merge_build_configs(base_config: dotdict, override_config: dotdict) -> dotdict:
+    """Merge two build configurations, with override_config taking precedence."""
+    result = dotdict(base_config.copy())
+    for key, value in override_config.items():
+        if key in result and isinstance(value, dict) and isinstance(result[key], dict):
+            # Recursively merge nested dictionaries
+            for sub_key, sub_value in value.items():
+                result[key][sub_key] = sub_value
+        else:
+            result[key] = value
+    return result
+
+
+def set_current_fields(
+    build_config: dotdict,
+    action_fields: dict[str, list[str]],
+    *,
+    selected_action: str | None = None,
+    default_fields: list[str] = DEFAULT_FIELDS,
+    func: Callable = set_field_display,
+    default_value: bool | None = None,
+) -> dotdict:
+    """Set the current fields for a selected action."""
+    # action_fields = {action1: [field1, field2], action2: [field3, field4]}
+    # we need to show action of one field and disable the rest
+    if default_value is None:
+        default_value = False
+
+    def _call_func(build_config: dotdict, field: str, *, value: bool) -> dotdict:
+        """Helper to call the function with appropriate signature."""
+        if func == set_field_advanced:
+            return func(build_config, field, value=value)
+        return func(build_config, field, value)
+
+    if selected_action in action_fields:
+        for field in action_fields[selected_action]:
+            build_config = _call_func(build_config, field, value=not default_value)
+        for key, value in action_fields.items():
+            if key != selected_action:
+                for field in value:
+                    build_config = _call_func(build_config, field, value=default_value)
+    if selected_action is None:
+        for value in action_fields.values():
+            for field in value:
+                build_config = _call_func(build_config, field, value=default_value)
+    if default_fields is not None:
+        for field in default_fields:
+            build_config = _call_func(build_config, field, value=not default_value)
+    return build_config

lfx/utils/concurrency.py

@@ -0,0 +1,60 @@
+import re
+import threading
+from contextlib import contextmanager
+from pathlib import Path
+
+from filelock import FileLock
+from platformdirs import user_cache_dir
+
+
+class KeyedMemoryLockManager:
+    """A manager for acquiring and releasing memory locks based on a key."""
+
+    def __init__(self) -> None:
+        self.locks: dict[str, threading.Lock] = {}
+        self.global_lock = threading.Lock()
+
+    def _get_lock(self, key: str):
+        with self.global_lock:
+            if key not in self.locks:
+                self.locks[key] = threading.Lock()
+            return self.locks[key]
+
+    @contextmanager
+    def lock(self, key: str):
+        lock = self._get_lock(key)
+        lock.acquire()
+        try:
+            yield
+        finally:
+            lock.release()
+
+
+class KeyedWorkerLockManager:
+    """A manager for acquiring locks between workers based on a key."""
+
+    def __init__(self) -> None:
+        self.locks_dir = Path(user_cache_dir("langflow"), ensure_exists=True) / "worker_locks"
+
+    @staticmethod
+    def _validate_key(key: str) -> bool:
+        """Validate that the string only contains alphanumeric characters and underscores.
+
+        Parameters:
+        s (str): The string to validate.
+
+        Returns:
+        bool: True if the string is valid, False otherwise.
+        """
+        pattern = re.compile(r"^\w+$")
+        return bool(pattern.match(key))
+
+    @contextmanager
+    def lock(self, key: str):
+        if not self._validate_key(key):
+            msg = f"Invalid key: {key}"
+            raise ValueError(msg)
+
+        lock = FileLock(self.locks_dir / key)
+        with lock:
+            yield

lfx/utils/connection_string_parser.py

@@ -0,0 +1,11 @@
+"""Connection string parser utilities for lfx package."""
+
+from urllib.parse import quote
+
+
+def transform_connection_string(connection_string) -> str:
+    """Transform connection string by encoding the password part."""
+    auth_part, db_url_name = connection_string.rsplit("@", 1)
+    protocol_user, password_string = auth_part.rsplit(":", 1)
+    encoded_password = quote(password_string)
+    return f"{protocol_user}:{encoded_password}@{db_url_name}"

lfx/utils/constants.py

@@ -0,0 +1,205 @@
+from typing import Any
+
+OPENAI_MODELS = [
+    "text-davinci-003",
+    "text-davinci-002",
+    "text-curie-001",
+    "text-babbage-001",
+    "text-ada-001",
+]
+CHAT_OPENAI_MODELS = [
+    "gpt-4o",
+    "gpt-4o-mini",
+    "gpt-4-turbo-preview",
+    "gpt-4-0125-preview",
+    "gpt-4-1106-preview",
+    "gpt-4-vision-preview",
+    "gpt-3.5-turbo-0125",
+    "gpt-3.5-turbo-1106",
+]
+
+REASONING_OPENAI_MODELS = [
+    "o1",
+    "o1-mini",
+    "o1-pro",
+    "o3-mini",
+    "o3",
+    "o3-pro",
+    "o4-mini",
+    "o4-mini-high",
+]
+
+ANTHROPIC_MODELS = [
+    # largest model, ideal for a wide range of more complex tasks.
+    "claude-v1",
+    # An enhanced version of claude-v1 with a 100,000 token (roughly 75,000 word) context window.
+    "claude-v1-100k",
+    # A smaller model with far lower latency, sampling at roughly 40 words/sec!
+    "claude-instant-v1",
+    # Like claude-instant-v1 with a 100,000 token context window but retains its performance.
+    "claude-instant-v1-100k",
+    # Specific sub-versions of the above models:
+    # Vs claude-v1.2: better instruction-following, code, and non-English dialogue and writing.
+    "claude-v1.3",
+    # An enhanced version of claude-v1.3 with a 100,000 token (roughly 75,000 word) context window.
+    "claude-v1.3-100k",
+    # Vs claude-v1.1: small adv in general helpfulness, instruction following, coding, and other tasks.
+    "claude-v1.2",
+    # An earlier version of claude-v1.
+    "claude-v1.0",
+    # Latest version of claude-instant-v1. Better than claude-instant-v1.0 at most tasks.
+    "claude-instant-v1.1",
+    # Version of claude-instant-v1.1 with a 100K token context window.
+    "claude-instant-v1.1-100k",
+    # An earlier version of claude-instant-v1.
+    "claude-instant-v1.0",
+]
+
+DEFAULT_PYTHON_FUNCTION = """
+def python_function(text: str) -> str:
+    \"\"\"This is a default python function that returns the input text\"\"\"
+    return text
+"""
+
+
+PYTHON_BASIC_TYPES = [str, bool, int, float, tuple, list, dict, set]
+DIRECT_TYPES = [
+    "str",
+    "bool",
+    "dict",
+    "int",
+    "float",
+    "Any",
+    "prompt",
+    "code",
+    "NestedDict",
+    "table",
+    "slider",
+    "tab",
+    "sortableList",
+    "auth",
+    "connect",
+    "query",
+    "tools",
+    "mcp",
+]
+
+
+LOADERS_INFO: list[dict[str, Any]] = [
+    {
+        "loader": "AirbyteJSONLoader",
+        "name": "Airbyte JSON (.jsonl)",
+        "import": "langchain_community.document_loaders.AirbyteJSONLoader",
+        "defaultFor": ["jsonl"],
+        "allowdTypes": ["jsonl"],
+    },
+    {
+        "loader": "JSONLoader",
+        "name": "JSON (.json)",
+        "import": "langchain_community.document_loaders.JSONLoader",
+        "defaultFor": ["json"],
+        "allowdTypes": ["json"],
+    },
+    {
+        "loader": "BSHTMLLoader",
+        "name": "BeautifulSoup4 HTML (.html, .htm)",
+        "import": "langchain_community.document_loaders.BSHTMLLoader",
+        "allowdTypes": ["html", "htm"],
+    },
+    {
+        "loader": "CSVLoader",
+        "name": "CSV (.csv)",
+        "import": "langchain_community.document_loaders.CSVLoader",
+        "defaultFor": ["csv"],
+        "allowdTypes": ["csv"],
+    },
+    {
+        "loader": "CoNLLULoader",
+        "name": "CoNLL-U (.conllu)",
+        "import": "langchain_community.document_loaders.CoNLLULoader",
+        "defaultFor": ["conllu"],
+        "allowdTypes": ["conllu"],
+    },
+    {
+        "loader": "EverNoteLoader",
+        "name": "EverNote (.enex)",
+        "import": "langchain_community.document_loaders.EverNoteLoader",
+        "defaultFor": ["enex"],
+        "allowdTypes": ["enex"],
+    },
+    {
+        "loader": "FacebookChatLoader",
+        "name": "Facebook Chat (.json)",
+        "import": "langchain_community.document_loaders.FacebookChatLoader",
+        "allowdTypes": ["json"],
+    },
+    {
+        "loader": "OutlookMessageLoader",
+        "name": "Outlook Message (.msg)",
+        "import": "langchain_community.document_loaders.OutlookMessageLoader",
+        "defaultFor": ["msg"],
+        "allowdTypes": ["msg"],
+    },
+    {
+        "loader": "PyPDFLoader",
+        "name": "PyPDF (.pdf)",
+        "import": "langchain_community.document_loaders.PyPDFLoader",
+        "defaultFor": ["pdf"],
+        "allowdTypes": ["pdf"],
+    },
+    {
+        "loader": "STRLoader",
+        "name": "Subtitle (.str)",
+        "import": "langchain_community.document_loaders.STRLoader",
+        "defaultFor": ["str"],
+        "allowdTypes": ["str"],
+    },
+    {
+        "loader": "TextLoader",
+        "name": "Text (.txt)",
+        "import": "langchain_community.document_loaders.TextLoader",
+        "defaultFor": ["txt"],
+        "allowdTypes": ["txt"],
+    },
+    {
+        "loader": "UnstructuredEmailLoader",
+        "name": "Unstructured Email (.eml)",
+        "import": "langchain_community.document_loaders.UnstructuredEmailLoader",
+        "defaultFor": ["eml"],
+        "allowdTypes": ["eml"],
+    },
+    {
+        "loader": "UnstructuredHTMLLoader",
+        "name": "Unstructured HTML (.html, .htm)",
+        "import": "langchain_community.document_loaders.UnstructuredHTMLLoader",
+        "defaultFor": ["html", "htm"],
+        "allowdTypes": ["html", "htm"],
+    },
+    {
+        "loader": "UnstructuredMarkdownLoader",
+        "name": "Unstructured Markdown (.md)",
+        "import": "langchain_community.document_loaders.UnstructuredMarkdownLoader",
+        "defaultFor": ["md", "mdx"],
+        "allowdTypes": ["md", "mdx"],
+    },
+    {
+        "loader": "UnstructuredPowerPointLoader",
+        "name": "Unstructured PowerPoint (.pptx)",
+        "import": "langchain_community.document_loaders.UnstructuredPowerPointLoader",
+        "defaultFor": ["pptx"],
+        "allowdTypes": ["pptx"],
+    },
+    {
+        "loader": "UnstructuredWordLoader",
+        "name": "Unstructured Word (.docx)",
+        "import": "langchain_community.document_loaders.UnstructuredWordLoader",
+        "defaultFor": ["docx"],
+        "allowdTypes": ["docx"],
+    },
+]
+
+
+MESSAGE_SENDER_AI = "Machine"
+MESSAGE_SENDER_USER = "User"
+MESSAGE_SENDER_NAME_AI = "AI"
+MESSAGE_SENDER_NAME_USER = "User"