lfx-nightly 0.2.0.dev0__py3-none-any.whl → 0.2.0.dev41__py3-none-any.whl

lfx/services/storage/service.py

@@ -1,54 +1,196 @@
-"""Base storage service for lfx package."""
+from __future__ import annotations
 
-from abc import ABC, abstractmethod
-from pathlib import Path
+from abc import abstractmethod
+from typing import TYPE_CHECKING
 
+import anyio
 
-class StorageService(ABC):
-    """Abstract base class for storage services."""
+from lfx.services.base import Service
 
-    def __init__(self, data_dir: str | Path | None = None) -> None:
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
+    from lfx.services.settings.service import SettingsService
+
+
+class StorageService(Service):
+    """Abstract base class for file storage services.
+
+    This class defines the interface for file storage operations that can be
+    implemented by different backends (local filesystem, S3, etc.).
+
+    All file operations are namespaced by flow_id to isolate files between
+    different flows or users.
+    """
+
+    name = "storage_service"
+
+    def __init__(self, session_service, settings_service: SettingsService):
         """Initialize the storage service.
 
         Args:
-            data_dir: Directory path for storing data. Defaults to ~/.lfx/data
+            session_service: The session service instance
+            settings_service: The settings service instance containing configuration
+        """
+        self.settings_service = settings_service
+        self.session_service = session_service
+        self.data_dir: anyio.Path = anyio.Path(settings_service.settings.config_dir)
+        self.set_ready()
+
+    @abstractmethod
+    def build_full_path(self, flow_id: str, file_name: str) -> str:
+        """Build the full path/key for a file.
+
+        Args:
+            flow_id: The flow/user identifier for namespacing
+            file_name: The name of the file
+
+        Returns:
+            str: The full path or key for the file
         """
-        if data_dir is None:
-            data_dir = Path.home() / ".lfx" / "data"
-        self.data_dir = Path(data_dir)
-        self._ready = False
+        raise NotImplementedError
+
+    @abstractmethod
+    def parse_file_path(self, full_path: str) -> tuple[str, str]:
+        """Parse a full storage path to extract flow_id and file_name.
+
+        This reverses the build_full_path operation.
+
+        Args:
+            full_path: Full path as returned by build_full_path
+
+        Returns:
+            tuple[str, str]: A tuple of (flow_id, file_name)
+
+        Raises:
+            ValueError: If the path format is invalid or doesn't match expected structure
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def resolve_component_path(self, logical_path: str) -> str:
+        """Convert a logical path to a format that components can use directly.
+
+        Logical paths are in the format "{flow_id}/{filename}" as stored in the database.
+        This method converts them to a format appropriate for the storage backend:
+        - Local storage: Absolute filesystem path (/data_dir/flow_id/filename)
+        - S3 storage: Logical path as-is (flow_id/filename)
+
+        Components receive this resolved path and can use it without knowing the
+        storage implementation details.
+
+        Args:
+            logical_path: Path in the format "flow_id/filename"
+
+        Returns:
+            str: A path that components can use directly
+        """
+        raise NotImplementedError
 
     def set_ready(self) -> None:
         """Mark the service as ready."""
         self._ready = True
-        # Ensure the data directory exists
-        self.data_dir.mkdir(parents=True, exist_ok=True)
-
-    @property
-    def ready(self) -> bool:
-        """Check if the service is ready."""
-        return self._ready
 
     @abstractmethod
-    def build_full_path(self, flow_id: str, file_name: str) -> str:
-        """Build the full path for a file."""
+    async def save_file(self, flow_id: str, file_name: str, data: bytes, *, append: bool = False) -> None:
+        """Save a file to storage.
 
-    @abstractmethod
-    async def save_file(self, flow_id: str, file_name: str, data: bytes) -> None:
-        """Save a file."""
+        Args:
+            flow_id: The flow/user identifier for namespacing
+            file_name: The name of the file to save
+            data: The file content as bytes
+            append: If True, append to existing file instead of overwriting.
+
+        Raises:
+            Exception: If the file cannot be saved
+        """
+        raise NotImplementedError
 
     @abstractmethod
     async def get_file(self, flow_id: str, file_name: str) -> bytes:
-        """Retrieve a file."""
+        """Retrieve a file from storage.
+
+        Args:
+            flow_id: The flow/user identifier for namespacing
+            file_name: The name of the file to retrieve
+
+        Returns:
+            bytes: The file content
+
+        Raises:
+            FileNotFoundError: If the file does not exist
+        """
+        raise NotImplementedError
+
+    async def get_file_stream(self, flow_id: str, file_name: str, chunk_size: int = 8192) -> AsyncIterator[bytes]:
+        """Retrieve a file from storage as a stream.
+
+        Default implementation loads the entire file and yields it in chunks.
+        Subclasses can override this for more efficient streaming.
+
+        Args:
+            flow_id: The flow/user identifier for namespacing
+            file_name: The name of the file to retrieve
+            chunk_size: Size of chunks to yield (default: 8192 bytes)
+
+        Yields:
+            bytes: Chunks of the file content
+
+        Raises:
+            FileNotFoundError: If the file does not exist
+        """
+        # Default implementation - subclasses can override for true streaming
+        content = await self.get_file(flow_id, file_name)
+        for i in range(0, len(content), chunk_size):
+            yield content[i : i + chunk_size]
 
     @abstractmethod
     async def list_files(self, flow_id: str) -> list[str]:
-        """List files in a flow."""
+        """List all files in a flow's storage namespace.
 
-    @abstractmethod
-    async def delete_file(self, flow_id: str, file_name: str) -> None:
-        """Delete a file."""
+        Args:
+            flow_id: The flow/user identifier for namespacing
+
+        Returns:
+            list[str]: List of file names in the namespace
+
+        Raises:
+            FileNotFoundError: If the namespace directory does not exist
+        """
+        raise NotImplementedError
 
     @abstractmethod
     async def get_file_size(self, flow_id: str, file_name: str) -> int:
-        """Get the size of a file."""
+        """Get the size of a file in bytes.
+
+        Args:
+            flow_id: The flow/user identifier for namespacing
+            file_name: The name of the file
+
+        Returns:
+            int: Size of the file in bytes
+
+        Raises:
+            FileNotFoundError: If the file does not exist
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    async def delete_file(self, flow_id: str, file_name: str) -> None:
+        """Delete a file from storage.
+
+        Args:
+            flow_id: The flow/user identifier for namespacing
+            file_name: The name of the file to delete
+
+        Note:
+            Should not raise an error if the file doesn't exist
+        """
+        raise NotImplementedError
+
+    async def teardown(self) -> None:
+        """Perform cleanup operations when the service is being shut down.
+
+        Subclasses should override this to clean up any resources (connections, etc.)
+        """
+        raise NotImplementedError

lfx/template/field/base.py

@@ -208,6 +208,9 @@ class Output(BaseModel):
     allows_loop: bool = Field(default=False)
     """Specifies if the output allows looping."""
 
+    loop_types: list[str] | None = Field(default=None)
+    """List of additional types to include for loop inputs when allows_loop is True."""
+
     group_outputs: bool = Field(default=False)
     """Specifies if all outputs should be grouped and shown without dropdowns."""
 

lfx/utils/image.py

@@ -6,14 +6,19 @@ import base64
 from functools import lru_cache
 from pathlib import Path
 
+from lfx.log import logger
+from lfx.services.deps import get_storage_service
+from lfx.utils.async_helpers import run_until_complete
 from lfx.utils.helpers import get_mime_type
 
 
 def convert_image_to_base64(image_path: str | Path) -> str:
     """Convert an image file to a base64 encoded string.
 
+    Handles both local files and S3 storage paths.
+
     Args:
-        image_path: Path to the image file
+        image_path: Path to the image file (local or S3 path like "flow_id/filename")
 
     Returns:
         Base64 encoded string of the image
@@ -22,6 +27,20 @@ def convert_image_to_base64(image_path: str | Path) -> str:
         FileNotFoundError: If the image file doesn't exist
     """
     image_path = Path(image_path)
+
+    storage_service = get_storage_service()
+    if storage_service:
+        flow_id, file_name = storage_service.parse_file_path(str(image_path))
+        try:
+            file_content = run_until_complete(
+                storage_service.get_file(flow_id=flow_id, file_name=file_name)  # type: ignore[call-arg]
+            )
+            return base64.b64encode(file_content).decode("utf-8")
+        except Exception as e:
+            logger.error(f"Error reading image file: {e}")
+            raise
+
+    # Fall back to local file access
     if not image_path.exists():
         msg = f"Image file not found: {image_path}"
         raise FileNotFoundError(msg)
@@ -34,7 +53,7 @@ def create_data_url(image_path: str | Path, mime_type: str | None = None) -> str
     """Create a data URL from an image file.
 
     Args:
-        image_path: Path to the image file
+        image_path: Path to the image file (local or S3 path like "flow_id/filename")
         mime_type: MIME type of the image. If None, will be auto-detected
 
     Returns:
@@ -44,9 +63,6 @@ def create_data_url(image_path: str | Path, mime_type: str | None = None) -> str
         FileNotFoundError: If the image file doesn't exist
     """
     image_path = Path(image_path)
-    if not image_path.exists():
-        msg = f"Image file not found: {image_path}"
-        raise FileNotFoundError(msg)
 
     if mime_type is None:
         mime_type = get_mime_type(image_path)
@@ -57,14 +73,16 @@ def create_data_url(image_path: str | Path, mime_type: str | None = None) -> str
 
 @lru_cache(maxsize=50)
 def create_image_content_dict(
-    image_path: str | Path, mime_type: str | None = None, model_name: str | None = None
+    image_path: str | Path,
+    mime_type: str | None = None,
+    model_name: str | None = None,  # noqa: ARG001
 ) -> dict:
     """Create a content dictionary for multimodal inputs from an image file.
 
     Args:
-        image_path: Path to the image file
+        image_path: Path to the image file (local or S3 path like "flow_id/filename")
         mime_type: MIME type of the image. If None, will be auto-detected
-        model_name: Optional model parameter to determine content dict structure
+        model_name: Optional model parameter (kept for backward compatibility, no longer used)
 
     Returns:
         Content dictionary with type and image_url fields
@@ -74,6 +92,6 @@ def create_image_content_dict(
     """
     data_url = create_data_url(image_path, mime_type)
 
-    if model_name == "OllamaModel":
-        return {"type": "image_url", "source_type": "url", "image_url": data_url}
-    return {"type": "image", "source_type": "url", "url": data_url}
+    # Standard format for OpenAI, Anthropic, Gemini, and most providers
+    # Format: {"type": "image_url", "image_url": {"url": "data:..."}}
+    return {"type": "image_url", "image_url": {"url": data_url}}