lfx-nightly 0.2.0.dev0__py3-none-any.whl → 0.2.0.dev41__py3-none-any.whl

lfx/base/data/utils.py

@@ -1,14 +1,21 @@
+import contextlib
+import tempfile
 import unicodedata
 from collections.abc import Callable
 from concurrent import futures
+from io import BytesIO
 from pathlib import Path
 
 import chardet
 import orjson
 import yaml
 from defusedxml import ElementTree
+from pypdf import PdfReader
 
+from lfx.base.data.storage_utils import read_file_bytes
 from lfx.schema.data import Data
+from lfx.services.deps import get_settings_service
+from lfx.utils.async_helpers import run_until_complete
 
 # Types of files that can be read simply by file.read()
 # and have 100% to be completely readable
@@ -36,6 +43,34 @@ TEXT_FILE_TYPES = [
 IMG_FILE_TYPES = ["jpg", "jpeg", "png", "bmp", "image"]
 
 
+def parse_structured_text(text: str, file_path: str) -> str | dict | list:
+    """Parse structured text formats (JSON, YAML, XML) and normalize text.
+
+    Args:
+        text: The text content to parse
+        file_path: The file path (used to determine format)
+
+    Returns:
+        Parsed content (dict/list for JSON, dict for YAML, str for XML)
+    """
+    if file_path.endswith(".json"):
+        loaded_json = orjson.loads(text)
+        if isinstance(loaded_json, dict):
+            loaded_json = {k: normalize_text(v) if isinstance(v, str) else v for k, v in loaded_json.items()}
+        elif isinstance(loaded_json, list):
+            loaded_json = [normalize_text(item) if isinstance(item, str) else item for item in loaded_json]
+        return orjson.dumps(loaded_json).decode("utf-8")
+
+    if file_path.endswith((".yaml", ".yml")):
+        return yaml.safe_load(text)
+
+    if file_path.endswith(".xml"):
+        xml_element = ElementTree.fromstring(text)
+        return ElementTree.tostring(xml_element, encoding="unicode")
+
+    return text
+
+
 def normalize_text(text):
     return unicodedata.normalize("NFKD", text)
 
@@ -109,6 +144,14 @@ def partition_file_to_data(file_path: str, *, silent_errors: bool) -> Data | Non
 
 
 def read_text_file(file_path: str) -> str:
+    """Read a text file with automatic encoding detection.
+
+    Args:
+        file_path: Path to the file (local path only, not storage service path)
+
+    Returns:
+        str: The file content as text
+    """
     file_path_ = Path(file_path)
     raw_data = file_path_.read_bytes()
     result = chardet.detect(raw_data)
@@ -120,13 +163,90 @@ def read_text_file(file_path: str) -> str:
     return file_path_.read_text(encoding=encoding)
 
 
+async def read_text_file_async(file_path: str) -> str:
+    """Read a text file with automatic encoding detection (async, storage-aware).
+
+    Args:
+        file_path: Path to the file (S3 key format "flow_id/filename" or local path)
+
+    Returns:
+        str: The file content as text
+    """
+    from .storage_utils import read_file_bytes
+
+    # Use storage-aware read to get bytes
+    raw_data = await read_file_bytes(file_path)
+
+    # Auto-detect encoding
+    result = chardet.detect(raw_data)
+    encoding = result.get("encoding")
+
+    # If encoding detection fails (e.g., binary file), default to utf-8
+    if not encoding or encoding in {"Windows-1252", "Windows-1254", "MacRoman"}:
+        encoding = "utf-8"
+
+    return raw_data.decode(encoding, errors="replace")
+
+
 def read_docx_file(file_path: str) -> str:
+    """Read a DOCX file and extract text.
+
+    ote: python-docx requires a file path, so this only works with local files.
+    For storage service files, use read_docx_file_async which downloads to temp.
+
+    Args:
+        file_path: Path to the DOCX file (local path only)
+
+    Returns:
+        str: Extracted text from the document
+    """
     from docx import Document
 
     doc = Document(file_path)
     return "\n\n".join([p.text for p in doc.paragraphs])
 
 
+async def read_docx_file_async(file_path: str) -> str:
+    """Read a DOCX file and extract text (async, storage-aware).
+
+    For S3 storage, downloads to temp file (python-docx requires file path).
+    For local storage, reads directly.
+
+    Args:
+        file_path: Path to the DOCX file (S3 key format "flow_id/filename" or local path)
+
+    Returns:
+        str: Extracted text from the document
+    """
+    from docx import Document
+
+    from .storage_utils import read_file_bytes
+
+    settings = get_settings_service().settings
+
+    if settings.storage_type == "local":
+        # Local storage - read directly
+        doc = Document(file_path)
+        return "\n\n".join([p.text for p in doc.paragraphs])
+
+    # S3 storage - need temp file for python-docx (doesn't support BytesIO)
+    content = await read_file_bytes(file_path)
+
+    # Create temp file with .docx extension
+    # Extract filename from path for suffix
+    suffix = Path(file_path.split("/")[-1]).suffix
+    with tempfile.NamedTemporaryFile(mode="wb", suffix=suffix, delete=False) as tmp_file:
+        tmp_file.write(content)
+        temp_path = tmp_file.name
+
+    try:
+        doc = Document(temp_path)
+        return "\n\n".join([p.text for p in doc.paragraphs])
+    finally:
+        with contextlib.suppress(Exception):
+            Path(temp_path).unlink()
+
+
 def parse_pdf_to_text(file_path: str) -> str:
     from pypdf import PdfReader
 
@@ -134,7 +254,35 @@ def parse_pdf_to_text(file_path: str) -> str:
         return "\n\n".join([page.extract_text() for page in reader.pages])
 
 
+async def parse_pdf_to_text_async(file_path: str) -> str:
+    """Parse a PDF file to extract text (async, storage-aware).
+
+    Uses storage-aware file reading to support both local and S3 storage.
+
+    Args:
+        file_path: Path to the PDF file (S3 key format "flow_id/filename" or local path)
+
+    Returns:
+        str: Extracted text from all pages
+    """
+    content = await read_file_bytes(file_path)
+    with BytesIO(content) as f, PdfReader(f) as reader:
+        return "\n\n".join([page.extract_text() for page in reader.pages])
+
+
 def parse_text_file_to_data(file_path: str, *, silent_errors: bool) -> Data | None:
+    """Parse a text file to Data (sync version).
+
+    For S3 storage, this will use async operations to fetch the file.
+    For local storage, reads directly from filesystem.
+    """
+    settings = get_settings_service().settings
+
+    # If using S3 storage, we need to use async operations
+    if settings.storage_type == "s3":
+        # Run the async version safely (handles existing event loops)
+        return run_until_complete(parse_text_file_to_data_async(file_path, silent_errors=silent_errors))
+
     try:
         if file_path.endswith(".pdf"):
             text = parse_pdf_to_text(file_path)
@@ -143,20 +291,7 @@ def parse_text_file_to_data(file_path: str, *, silent_errors: bool) -> Data | No
         else:
             text = read_text_file(file_path)
 
-        # if file is json, yaml, or xml, we can parse it
-        if file_path.endswith(".json"):
-            loaded_json = orjson.loads(text)
-            if isinstance(loaded_json, dict):
-                loaded_json = {k: normalize_text(v) if isinstance(v, str) else v for k, v in loaded_json.items()}
-            elif isinstance(loaded_json, list):
-                loaded_json = [normalize_text(item) if isinstance(item, str) else item for item in loaded_json]
-            text = orjson.dumps(loaded_json).decode("utf-8")
-
-        elif file_path.endswith((".yaml", ".yml")):
-            text = yaml.safe_load(text)
-        elif file_path.endswith(".xml"):
-            xml_element = ElementTree.fromstring(text)
-            text = ElementTree.tostring(xml_element, encoding="unicode")
+        text = parse_structured_text(text, file_path)
     except Exception as e:
         if not silent_errors:
             msg = f"Error loading file {file_path}: {e}"
@@ -166,6 +301,35 @@ def parse_text_file_to_data(file_path: str, *, silent_errors: bool) -> Data | No
     return Data(data={"file_path": file_path, "text": text})
 
 
+async def parse_text_file_to_data_async(file_path: str, *, silent_errors: bool) -> Data | None:
+    """Parse a text file to Data (async version, supports storage service).
+
+    This version properly handles storage service files:
+    - For text/JSON/YAML/XML: reads bytes directly (no temp file)
+    - For PDF: reads bytes directly via BytesIO (no temp file)
+    - For DOCX: downloads to temp file (python-docx requires file path)
+    """
+    try:
+        if file_path.endswith(".pdf"):
+            text = await parse_pdf_to_text_async(file_path)
+        elif file_path.endswith(".docx"):
+            text = await read_docx_file_async(file_path)
+        else:
+            # Text files - read directly, no temp file needed
+            text = await read_text_file_async(file_path)
+
+        # Parse structured formats (JSON, YAML, XML)
+        text = parse_structured_text(text, file_path)
+
+        return Data(data={"file_path": file_path, "text": text})
+
+    except Exception as e:
+        if not silent_errors:
+            msg = f"Error loading file {file_path}: {e}"
+            raise ValueError(msg) from e
+        return None
+
+
 # ! Removing unstructured dependency until
 # ! 3.12 is supported
 # def get_elements(

lfx/base/mcp/util.py

@@ -23,6 +23,7 @@ from pydantic import BaseModel
 from lfx.log.logger import logger
 from lfx.schema.json_schema import create_input_schema_from_json_schema
 from lfx.services.deps import get_settings_service
+from lfx.utils.async_helpers import run_until_complete
 
 HTTP_ERROR_STATUS_CODE = httpx_codes.BAD_REQUEST  # HTTP status code for client errors
 
@@ -351,8 +352,7 @@ def create_tool_func(tool_name: str, arg_schema: type[BaseModel], client) -> Cal
             _handle_tool_validation_error(e, tool_name, provided_args, arg_schema)
 
         try:
-            loop = asyncio.get_event_loop()
-            return loop.run_until_complete(client.run_tool(tool_name, arguments=validated.model_dump()))
+            return run_until_complete(client.run_tool(tool_name, arguments=validated.model_dump()))
         except Exception as e:
             logger.error(f"Tool '{tool_name}' execution failed: {e}")
             # Re-raise with more context

lfx/base/models/anthropic_constants.py

@@ -2,32 +2,41 @@ from .model_metadata import create_model_metadata
 
 ANTHROPIC_MODELS_DETAILED = [
     # Tool calling supported models
+    create_model_metadata(provider="Anthropic", name="claude-opus-4-5-20251101", icon="Anthropic", tool_calling=True),
+    create_model_metadata(provider="Anthropic", name="claude-haiku-4-5-20251001", icon="Anthropic", tool_calling=True),
     create_model_metadata(provider="Anthropic", name="claude-sonnet-4-5-20250929", icon="Anthropic", tool_calling=True),
     create_model_metadata(provider="Anthropic", name="claude-opus-4-1-20250805", icon="Anthropic", tool_calling=True),
     create_model_metadata(provider="Anthropic", name="claude-opus-4-20250514", icon="Anthropic", tool_calling=True),
     create_model_metadata(provider="Anthropic", name="claude-sonnet-4-20250514", icon="Anthropic", tool_calling=True),
-    create_model_metadata(provider="Anthropic", name="claude-3-7-sonnet-latest", icon="Anthropic", tool_calling=True),
-    create_model_metadata(provider="Anthropic", name="claude-3-5-sonnet-latest", icon="Anthropic", tool_calling=True),
-    create_model_metadata(provider="Anthropic", name="claude-3-5-haiku-latest", icon="Anthropic", tool_calling=True),
-    create_model_metadata(provider="Anthropic", name="claude-3-opus-latest", icon="Anthropic", tool_calling=True),
+    create_model_metadata(provider="Anthropic", name="claude-3-5-haiku-20241022", icon="Anthropic", tool_calling=True),
+    create_model_metadata(provider="Anthropic", name="claude-3-haiku-20240307", icon="Anthropic", tool_calling=True),
+    # Deprecated models
+    create_model_metadata(
+        provider="Anthropic", name="claude-3-7-sonnet-latest", icon="Anthropic", tool_calling=True, deprecated=True
+    ),
+    create_model_metadata(
+        provider="Anthropic", name="claude-3-5-sonnet-latest", icon="Anthropic", tool_calling=True, deprecated=True
+    ),
+    create_model_metadata(
+        provider="Anthropic", name="claude-3-5-haiku-latest", icon="Anthropic", tool_calling=True, deprecated=True
+    ),
+    create_model_metadata(
+        provider="Anthropic", name="claude-3-opus-latest", icon="Anthropic", tool_calling=True, deprecated=True
+    ),
     create_model_metadata(
         provider="Anthropic", name="claude-3-sonnet-20240229", icon="Anthropic", tool_calling=True, deprecated=True
     ),
-    # Tool calling unsupported models
-    create_model_metadata(provider="Anthropic", name="claude-2.1", icon="Anthropic", tool_calling=False),
-    create_model_metadata(provider="Anthropic", name="claude-2.0", icon="Anthropic", tool_calling=False),
-    # Deprecated models
     create_model_metadata(
-        provider="Anthropic", name="claude-3-5-sonnet-20240620", icon="Anthropic", tool_calling=True, deprecated=True
+        provider="Anthropic", name="claude-2.1", icon="Anthropic", tool_calling=False, deprecated=True
     ),
     create_model_metadata(
-        provider="Anthropic", name="claude-3-5-sonnet-20241022", icon="Anthropic", tool_calling=True, deprecated=True
+        provider="Anthropic", name="claude-2.0", icon="Anthropic", tool_calling=False, deprecated=True
     ),
     create_model_metadata(
-        provider="Anthropic", name="claude-3-5-haiku-20241022", icon="Anthropic", tool_calling=True, deprecated=True
+        provider="Anthropic", name="claude-3-5-sonnet-20240620", icon="Anthropic", tool_calling=True, deprecated=True
     ),
     create_model_metadata(
-        provider="Anthropic", name="claude-3-haiku-20240307", icon="Anthropic", tool_calling=True, deprecated=True
+        provider="Anthropic", name="claude-3-5-sonnet-20241022", icon="Anthropic", tool_calling=True, deprecated=True
     ),
 ]
 

lfx/base/models/groq_constants.py

@@ -1,87 +1,91 @@
 from .model_metadata import create_model_metadata
 
-# Unified model metadata - single source of truth
+# Unified model metadata
+#
+# NOTE: This file serves as a FALLBACK when the dynamic model discovery system
+# (groq_model_discovery.py) cannot fetch fresh data from the Groq API.
+#
+# The dynamic system is the PRIMARY source and will:
+# - Fetch available models directly from Groq API
+# - Test each model for tool calling support automatically
+# - Cache results for 24 hours
+# - Always provide up-to-date model lists
+#
+# This fallback list should contain:
+# - Minimal set of stable production models
+# - Deprecated models for backwards compatibility
+# - Non-LLM models (audio, TTS) marked as not_supported
+#
+# Last manually updated: 2025-01-06
+#
 GROQ_MODELS_DETAILED = [
-    # Production Models - Stable and reliable for production use
-    create_model_metadata(  # Google
-        provider="Groq", name="gemma2-9b-it", icon="Groq", tool_calling=True
-    ),
-    create_model_metadata(  # Meta
-        provider="Groq", name="llama-3.3-70b-versatile", icon="Groq", tool_calling=True
-    ),
-    create_model_metadata(  # Meta
-        provider="Groq", name="llama-3.1-8b-instant", icon="Groq"
-    ),
-    create_model_metadata(  # Meta
-        provider="Groq", name="llama-guard-3-8b", icon="Groq"
+    # ===== FALLBACK PRODUCTION MODELS =====
+    # These are stable models that are very unlikely to be removed
+    create_model_metadata(provider="Groq", name="llama-3.1-8b-instant", icon="Groq", tool_calling=True),
+    create_model_metadata(provider="Groq", name="llama-3.3-70b-versatile", icon="Groq", tool_calling=True),
+    # ===== DEPRECATED MODELS =====
+    # Keep these for backwards compatibility - users may have flows using them
+    # These will appear in the list but show as deprecated in the UI
+    create_model_metadata(  # Google - Removed
+        provider="Groq", name="gemma2-9b-it", icon="Groq", deprecated=True
     ),
-    create_model_metadata(  # Meta
-        provider="Groq", name="llama3-70b-8192", icon="Groq"
-    ),
-    create_model_metadata(  # Meta
-        provider="Groq", name="llama3-8b-8192", icon="Groq"
-    ),
-    # Preview Models - For evaluation purposes only
-    create_model_metadata(  # Meta
-        provider="Groq", name="meta-llama/llama-4-scout-17b-16e-instruct", icon="Groq", tool_calling=True, preview=True
+    create_model_metadata(  # Google
+        provider="Groq", name="gemma-7b-it", icon="Groq", deprecated=True
     ),
-    create_model_metadata(  # Meta
-        provider="Groq",
-        name="meta-llama/llama-4-maverick-17b-128e-instruct",
-        icon="Groq",
-        tool_calling=True,
-        preview=True,
+    create_model_metadata(  # Meta - Removed
+        provider="Groq", name="llama3-70b-8192", icon="Groq", deprecated=True
     ),
-    create_model_metadata(  # Alibaba Cloud
-        provider="Groq", name="qwen-qwq-32b", icon="Groq", tool_calling=True, preview=True
+    create_model_metadata(  # Meta - Removed
+        provider="Groq", name="llama3-8b-8192", icon="Groq", deprecated=True
     ),
-    create_model_metadata(  # Alibaba Cloud
-        provider="Groq", name="qwen-2.5-coder-32b", icon="Groq", tool_calling=True, preview=True
+    create_model_metadata(  # Meta - Removed, replaced by llama-guard-4-12b
+        provider="Groq", name="llama-guard-3-8b", icon="Groq", deprecated=True
     ),
-    create_model_metadata(  # Alibaba Cloud
-        provider="Groq", name="qwen-2.5-32b", icon="Groq", tool_calling=True, preview=True
+    create_model_metadata(  # Meta - Removed
+        provider="Groq", name="llama-3.2-1b-preview", icon="Groq", deprecated=True
     ),
-    create_model_metadata(  # DeepSeek
-        provider="Groq", name="deepseek-r1-distill-qwen-32b", icon="Groq", tool_calling=True, preview=True
+    create_model_metadata(  # Meta - Removed
+        provider="Groq", name="llama-3.2-3b-preview", icon="Groq", deprecated=True
     ),
-    create_model_metadata(  # DeepSeek
-        provider="Groq", name="deepseek-r1-distill-llama-70b", icon="Groq", preview=True
+    create_model_metadata(  # Meta - Removed
+        provider="Groq", name="llama-3.2-11b-vision-preview", icon="Groq", deprecated=True
     ),
-    create_model_metadata(  # Meta
-        provider="Groq", name="llama-3.3-70b-specdec", icon="Groq", tool_calling=True, preview=True
+    create_model_metadata(  # Meta - Removed
+        provider="Groq", name="llama-3.2-90b-vision-preview", icon="Groq", deprecated=True
     ),
-    create_model_metadata(  # Meta
-        provider="Groq", name="llama-3.2-1b-preview", icon="Groq", tool_calling=True, preview=True
+    create_model_metadata(  # Meta - Removed
+        provider="Groq", name="llama-3.3-70b-specdec", icon="Groq", deprecated=True
     ),
-    create_model_metadata(  # Meta
-        provider="Groq", name="llama-3.2-3b-preview", icon="Groq", tool_calling=True, preview=True
+    create_model_metadata(  # Alibaba - Removed, replaced by qwen/qwen3-32b
+        provider="Groq", name="qwen-qwq-32b", icon="Groq", deprecated=True
     ),
-    create_model_metadata(  # Meta
-        provider="Groq", name="llama-3.2-11b-vision-preview", icon="Groq", preview=True
+    create_model_metadata(  # Alibaba - Removed
+        provider="Groq", name="qwen-2.5-coder-32b", icon="Groq", deprecated=True
     ),
-    create_model_metadata(  # Meta
-        provider="Groq", name="llama-3.2-90b-vision-preview", icon="Groq", tool_calling=True, preview=True
+    create_model_metadata(  # Alibaba - Removed
+        provider="Groq", name="qwen-2.5-32b", icon="Groq", deprecated=True
     ),
-    create_model_metadata(  # Saudi Data and AI Authority (SDAIA)
-        provider="Groq", name="allam-2-7b", icon="Groq", preview=True
+    create_model_metadata(  # DeepSeek - Removed
+        provider="Groq", name="deepseek-r1-distill-qwen-32b", icon="Groq", deprecated=True
     ),
-    # Deprecated Models - Previously available but now removed
-    create_model_metadata(  # Google
-        provider="Groq", name="gemma-7b-it", icon="Groq", tool_calling=True, deprecated=True
+    create_model_metadata(  # DeepSeek - Removed
+        provider="Groq", name="deepseek-r1-distill-llama-70b", icon="Groq", deprecated=True
     ),
     create_model_metadata(  # Groq
-        provider="Groq", name="llama3-groq-70b-8192-tool-use-preview", icon="Groq", tool_calling=True, deprecated=True
+        provider="Groq", name="llama3-groq-70b-8192-tool-use-preview", icon="Groq", deprecated=True
     ),
     create_model_metadata(  # Groq
-        provider="Groq", name="llama3-groq-8b-8192-tool-use-preview", icon="Groq", tool_calling=True, deprecated=True
+        provider="Groq", name="llama3-groq-8b-8192-tool-use-preview", icon="Groq", deprecated=True
     ),
     create_model_metadata(  # Meta
-        provider="Groq", name="llama-3.1-70b-versatile", icon="Groq", tool_calling=True, deprecated=True
+        provider="Groq", name="llama-3.1-70b-versatile", icon="Groq", deprecated=True
     ),
     create_model_metadata(  # Mistral
-        provider="Groq", name="mixtral-8x7b-32768", icon="Groq", tool_calling=True, deprecated=True
+        provider="Groq", name="mixtral-8x7b-32768", icon="Groq", deprecated=True
     ),
-    # Unsupported Models
+    # ===== UNSUPPORTED MODELS =====
+    # Audio/TTS/Guard models that should not appear in LLM model lists
+    # The dynamic system automatically filters these out
     create_model_metadata(  # Mistral
         provider="Groq", name="mistral-saba-24b", icon="Groq", not_supported=True
     ),
@@ -100,6 +104,18 @@ GROQ_MODELS_DETAILED = [
     create_model_metadata(  # Hugging Face
         provider="Groq", name="distil-whisper-large-v3-en", icon="Groq", not_supported=True
     ),
+    create_model_metadata(  # Meta
+        provider="Groq", name="meta-llama/llama-guard-4-12b", icon="Groq", not_supported=True
+    ),
+    create_model_metadata(  # Meta
+        provider="Groq", name="meta-llama/llama-prompt-guard-2-86m", icon="Groq", not_supported=True
+    ),
+    create_model_metadata(  # Meta
+        provider="Groq", name="meta-llama/llama-prompt-guard-2-22m", icon="Groq", not_supported=True
+    ),
+    create_model_metadata(  # OpenAI
+        provider="Groq", name="openai/gpt-oss-safeguard-20b", icon="Groq", not_supported=True
+    ),
 ]
 
 # Generate backwards-compatible lists from the metadata