Flowfile 0.3.9__py3-none-any.whl → 0.3.10__py3-none-any.whl

flowfile_core/fileExplorer/funcs.py

@@ -19,13 +19,30 @@ class FileInfo(BaseModel):
     exists: bool = True
 
     @classmethod
-    def from_path(cls, path: Path) -> 'FileInfo':
-        """Create FileInfo instance from a path."""
+    def from_path(cls, path: Path, sandbox_root: Optional[Path] = None,
+                  use_relative_paths: bool = False) -> 'FileInfo':
+        """Create FileInfo instance from a path.
+
+        Args:
+            path: The path to create FileInfo from
+            sandbox_root: The root directory for sandboxing (for relative path calculation)
+            use_relative_paths: If True, store relative paths; if False, store absolute paths
+        """
         try:
             stats = path.stat()
+
+            # Decide whether to use relative or absolute path
+            if use_relative_paths and sandbox_root:
+                try:
+                    display_path = str(path.relative_to(sandbox_root))
+                except ValueError:
+                    display_path = str(path.absolute())
+            else:
+                display_path = str(path.absolute())
+
             return cls(
                 name=path.name,
-                path=str(path.absolute()),
+                path=display_path,
                 is_directory=path.is_dir(),
                 size=stats.st_size,
                 file_type=path.suffix[1:] if path.suffix else "",
@@ -37,9 +54,18 @@ class FileInfo(BaseModel):
                 exists=True
             )
         except (PermissionError, OSError):
+            # Handle error case
+            if use_relative_paths and sandbox_root:
+                try:
+                    display_path = str(path.relative_to(sandbox_root))
+                except ValueError:
+                    display_path = str(path.absolute())
+            else:
+                display_path = str(path.absolute())
+
             return cls(
                 name=path.name,
-                path=str(path.absolute()),
+                path=display_path,
                 is_directory=False,
                 size=0,
                 file_type="",
@@ -50,30 +76,108 @@ class FileInfo(BaseModel):
             )
 
 
-class FileExplorer:
-    def __init__(self, start_path: Optional[str|Path] = None):
-        """Initialize FileExplorer with user's home directory or specified path."""
-        if start_path is None:
-            self.current_path = Path.home()
+class SecureFileExplorer:
+    """File explorer with sandbox enforcement to prevent directory traversal."""
+
+    def __init__(self, start_path: Union[str, Path],
+                 sandbox_root: Optional[Union[str, Path]] = None,
+                 use_relative_paths: bool = False):
+        """Initialize SecureFileExplorer with sandboxing.
+
+        Args:
+            start_path: Initial directory to start in
+            sandbox_root: Root directory that user cannot escape from.
+                         If None, no sandbox enforcement.
+            use_relative_paths: If True, FileInfo will contain relative paths;
+                               if False (default), contains absolute paths
+        """
+        self.use_relative_paths = use_relative_paths
+
+        # Set up the sandbox root
+        if sandbox_root is not None:
+            self.sandbox_root = Path(sandbox_root).expanduser().resolve()
+        else:
+            self.sandbox_root = None
+
+        # Set initial current path
+        initial_path = Path(start_path).expanduser().resolve()
+
+        # If sandbox is set and initial path is outside it, use sandbox root
+        if self.sandbox_root and not self._is_path_safe(initial_path):
+            self.current_path = self.sandbox_root
         else:
-            self.current_path = Path(start_path).expanduser().resolve()
+            self.current_path = initial_path
 
-        if not self.current_path.exists():
-            raise ValueError(f"Path does not exist: {self.current_path}")
+    def _is_path_safe(self, path: Path) -> bool:
+        """Check if a path is within the sandbox root.
 
-        if not self.current_path.is_dir():
-            raise ValueError(f"Path is not a directory: {self.current_path}")
+        Uses resolve() to handle symlinks and relative paths securely.
+        Returns True if no sandbox is set (no restrictions).
+        """
+        if self.sandbox_root is None:
+            return True  # No sandbox = no restrictions
+
+        try:
+            resolved_path = path.resolve()
+            resolved_sandbox = self.sandbox_root.resolve()
+            # Check if the resolved path is within sandbox
+            resolved_path.relative_to(resolved_sandbox)
+            return True
+        except (ValueError, RuntimeError):
+            return False
+
+    def _sanitize_path(self, path: Union[str, Path]) -> Optional[Path]:
+        """Sanitize and validate a path, ensuring it stays within sandbox.
+
+        Returns None if path would escape sandbox.
+        """
+        try:
+            # Handle relative paths from current directory
+            if isinstance(path, str):
+                # Remove any suspicious patterns
+                if '..' in Path(path).parts or path.startswith('/'):
+                    # For absolute paths or parent references, resolve from sandbox root
+                    test_path = Path(path).expanduser()
+                else:
+                    # For simple relative paths, resolve from current directory
+                    test_path = self.current_path / path
+            else:
+                test_path = path
+
+            # Resolve to absolute path
+            resolved = test_path.resolve()
+
+            # Check if within sandbox
+            if self._is_path_safe(resolved):
+                return resolved
+            else:
+                return None
+        except (ValueError, RuntimeError, OSError):
+            return None
 
     @property
     def current_directory(self) -> str:
-        """Get the current directory path."""
-        return str(self.current_path.absolute())
+        """Get the current directory path relative to sandbox root."""
+        if self.sandbox_root is None:
+            return str(self.current_path)
+
+        try:
+            relative = self.current_path.relative_to(self.sandbox_root)
+            return str(relative) if str(relative) != "." else "/"
+        except ValueError:
+            return "/"
 
     @property
     def parent_directory(self) -> Optional[str]:
-        """Get the parent directory path if it exists."""
+        """Get the parent directory path if it exists and is within sandbox."""
         parent = self.current_path.parent
-        return str(parent.absolute()) if parent != self.current_path else None
+        if self._is_path_safe(parent) and parent != self.current_path:
+            try:
+                relative = parent.relative_to(self.sandbox_root)
+                return str(relative) if str(relative) != "." else "/"
+            except ValueError:
+                return None
+        return None
 
     def list_contents(
             self,
@@ -85,34 +189,23 @@ class FileExplorer:
             max_size: Optional[int] = None,
             sort_by: Literal['name', 'date', 'size', 'type'] = 'name',
             reverse: bool = False,
-            exclude_patterns: Optional[List[str]] = None
+            exclude_patterns: Optional[List[str]] = None,
+            max_depth: int = 5  # Add depth limit for recursive operations
     ) -> List[FileInfo]:
-        """
-        List contents of the current directory with advanced filtering and sorting.
-
-        Args:
-            show_hidden: Whether to show hidden files and directories
-            file_types: List of file extensions to include (without dots)
-            recursive: Whether to scan subdirectories
-            min_size: Minimum file size in bytes
-            max_size: Maximum file size in bytes
-            sort_by: Field to sort results by
-            reverse: Whether to reverse sort order
-            exclude_patterns: Glob patterns to exclude
-
-        Returns:
-            List of FileInfo objects sorted according to parameters
-        """
+        """List contents with security-conscious filtering."""
         contents: List[FileInfo] = []
         excluded_paths: Set[str] = set()
 
         if exclude_patterns:
             for pattern in exclude_patterns:
-                excluded_paths.update(str(p) for p in self.current_path.glob(pattern))
+                # Ensure patterns don't escape sandbox
+                safe_pattern = pattern.replace('../', '').replace('..\\', '')
+                excluded_paths.update(str(p) for p in self.current_path.glob(safe_pattern))
 
         def should_include(info: FileInfo) -> bool:
             """Determine if a file should be included based on filters."""
-            if str(info.path) in excluded_paths:
+            full_path = self.current_path / info.path
+            if str(full_path) in excluded_paths:
                 return False
             if not show_hidden and info.is_hidden:
                 return False
@@ -125,20 +218,55 @@ class FileExplorer:
             return True
 
         try:
-            # Define the scan pattern based on recursion
-            pattern = '**/*' if recursive else '*'
+            if recursive:
+                # Use iterative approach with depth limit for safety
+                dirs_to_process = [(self.current_path, 0)]
+                processed = set()
 
-            # Scan directory
-            for item in self.current_path.glob(pattern):
-                try:
-                    file_info = FileInfo.from_path(item)
-                    if should_include(file_info):
-                        contents.append(file_info)
-                except (PermissionError, OSError):
-                    continue
+                while dirs_to_process:
+                    current_dir, depth = dirs_to_process.pop(0)
+
+                    # Skip if already processed or exceeds depth
+                    if current_dir in processed or depth > max_depth:
+                        continue
+
+                    processed.add(current_dir)
+
+                    try:
+                        for item in current_dir.iterdir():
+                            # Security check for each item
+                            if not self._is_path_safe(item):
+                                continue
+
+                            try:
+                                file_info = FileInfo.from_path(item, self.sandbox_root,
+                                                               self.use_relative_paths)
+                                if should_include(file_info):
+                                    contents.append(file_info)
+
+                                if item.is_dir() and depth < max_depth:
+                                    dirs_to_process.append((item, depth + 1))
+                            except (PermissionError, OSError):
+                                continue
+                    except (PermissionError, OSError):
+                        continue
+            else:
+                # Non-recursive listing
+                for item in self.current_path.iterdir():
+                    # Security check
+                    if not self._is_path_safe(item):
+                        continue
+
+                    try:
+                        file_info = FileInfo.from_path(item, self.sandbox_root,
+                                                       self.use_relative_paths)
+                        if should_include(file_info):
+                            contents.append(file_info)
+                    except (PermissionError, OSError):
+                        continue
 
         except PermissionError:
-            raise PermissionError(f"Permission denied to access directory: {self.current_path}")
+            raise PermissionError(f"Permission denied to access directory: {self.current_directory}")
 
         # Sort results
         sort_key = {
@@ -151,109 +279,99 @@ class FileExplorer:
         return sorted(contents, key=sort_key, reverse=reverse)
 
     def navigate_to(self, path: str) -> bool:
-        """
-        Navigate to a new directory path.
-        Returns True if navigation was successful, False otherwise.
-        """
-        new_path = None
-        try:
-            new_path = Path(path).expanduser().resolve()
+        """Navigate to a new directory path within sandbox."""
+        sanitized = self._sanitize_path(path)
 
-            if not new_path.exists() or not new_path.is_dir():
-                return False
+        if sanitized is None:
+            return False
 
+        if not sanitized.exists() or not sanitized.is_dir():
+            return False
+
+        try:
             # Test if we can actually read the directory
-            next(new_path.iterdir(), None)
-            self.current_path = new_path
+            next(sanitized.iterdir(), None)
+            self.current_path = sanitized
             return True
-
-        except PermissionError:
-            if new_path:
-                self.current_path = new_path
+        except (PermissionError, OSError):
+            # Still navigate if we have permission issues (user will see empty dir)
+            self.current_path = sanitized
             return True
-        except OSError:
-            return False
 
     def navigate_up(self) -> bool:
-        """
-        Navigate up to the parent directory.
-        Returns True if navigation was successful, False otherwise.
-        """
-        parent = self.parent_directory
-        if parent is None:
+        """Navigate up to the parent directory, respecting sandbox."""
+        parent = self.current_path.parent
+
+        # Check if parent is within sandbox
+        if not self._is_path_safe(parent):
             return False
-        return self.navigate_to(parent)
+
+        # Don't navigate if we're already at sandbox root
+        if parent == self.current_path:
+            return False
+
+        self.current_path = parent
+        return True
 
     def navigate_into(self, directory_name: str) -> bool:
-        """
-        Navigate into a subdirectory of the current directory.
-        Returns True if navigation was successful, False otherwise.
-        """
+        """Navigate into a subdirectory, with path sanitization."""
+        # Sanitize directory name
+        if '/' in directory_name or '\\' in directory_name or '..' in directory_name:
+            return False
+
         new_path = self.current_path / directory_name
         return self.navigate_to(str(new_path))
 
+    def get_absolute_path(self, relative_path: str) -> Optional[Path]:
+        """Get absolute path for a file within sandbox.
+
+        Returns None if the path would escape sandbox.
+        """
+        sanitized = self._sanitize_path(relative_path)
+        return sanitized if sanitized else None
+
 
 def get_files_from_directory(
         dir_name: Union[str, Path],
         types: Optional[List[str]] = None,
         *,
         include_hidden: bool = False,
-        recursive: bool = False
+        recursive: bool = False,
+        sandbox_root: Optional[Union[str, Path]] = None
 ) -> Optional[List[FileInfo]]:
     """
-    Get list of files from a directory with optional type filtering.
+    Get list of files from a directory with sandbox enforcement.
 
     Args:
         dir_name: Directory path to scan
-        types: List of file extensions to include (without dots). None means all types
+        types: List of file extensions to include
         include_hidden: Whether to include hidden files
         recursive: Whether to scan subdirectories
+        sandbox_root: Root directory to enforce as sandbox boundary
 
     Returns:
-        List of FileInfo objects or None if directory doesn't exist
-
-    Example:
-        >>> files = get_files_from_directory("/path/to/dir", types=["pdf", "txt"])
-        >>> for file in files:
-        ...     print(f"{file.name} - {file.size} bytes")
+        List of FileInfo objects or None if directory doesn't exist or is outside sandbox
     """
     try:
-        dir_path = Path(dir_name).resolve()
-        if not dir_path.exists():
-            return None
-        if not dir_path.is_dir():
-            raise ValueError(f"Path is not a directory: {dir_path}")
-
-        # Normalize file types
-        if types:
-            types = [t.lower().lstrip('.') for t in types]
-
-        files = []
-        pattern = '**/*' if recursive else '*'
-
-        for item in dir_path.glob(pattern):
-            try:
-                # Skip hidden files unless specifically requested
-                if not include_hidden and item.name.startswith('.'):
-                    continue
-
-                # Skip directories unless recursive is True
-                if item.is_dir() and not recursive:
-                    continue
-
-                # Check file type if types are specified
-                if types and not item.is_dir():
-                    if item.suffix[1:].lower() not in types:
-                        continue
-
-                file_info = FileInfo.from_path(item)
-                files.append(file_info)
-
-            except (PermissionError, OSError):
-                continue
-
-        return sorted(files, key=lambda x: (not x.is_directory, x.name.lower()))
-
+        # Create a secure explorer with sandbox
+        if sandbox_root:
+            explorer = SecureFileExplorer(start_path=dir_name, sandbox_root=sandbox_root)
+        else:
+            explorer = SecureFileExplorer(start_path=dir_name)
+
+        # Use the explorer's list_contents method
+        return explorer.list_contents(
+            show_hidden=include_hidden,
+            file_types=types,
+            recursive=recursive
+        )
+
+    except (ValueError, PermissionError) as e:
+        # Return None for invalid/inaccessible directories
+        return None
     except Exception as e:
         raise type(e)(f"Error scanning directory {dir_name}: {str(e)}") from e
 
+
+# Alias for backward compatibility
+FileExplorer = SecureFileExplorer

flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py

@@ -39,7 +39,7 @@ def trigger_df_operation(flow_id: int, node_id: int | str, lf: pl.LazyFrame, fil
              'flowfile_flow_id': flow_id, 'flowfile_node_id': node_id}
     v = requests.post(url=f'{WORKER_URL}/submit_query/', json=_json)
     if not v.ok:
-        raise Exception(f'Could not cache the data, {v.text}')
+        raise Exception(f'trigger_df_operation: Could not cache the data, {v.text}')
     return Status(**v.json())
 
 
@@ -49,7 +49,7 @@ def trigger_sample_operation(lf: pl.LazyFrame, file_ref: str, flow_id: int, node
              'sample_size': sample_size, 'flowfile_flow_id': flow_id, 'flowfile_node_id': node_id}
     v = requests.post(url=f'{WORKER_URL}/store_sample/', json=_json)
     if not v.ok:
-        raise Exception(f'Could not cache the data, {v.text}')
+        raise Exception(f'trigger_sample_operation: Could not cache the data, {v.text}')
     return Status(**v.json())
 
 
@@ -67,9 +67,10 @@ def trigger_fuzzy_match_operation(left_df: pl.LazyFrame, right_df: pl.LazyFrame,
                                       flowfile_flow_id=flow_id,
                                       flowfile_node_id=node_id
                                       )
+    print("fuzzy join input", fuzzy_join_input)
     v = requests.post(f'{WORKER_URL}/add_fuzzy_join', data=fuzzy_join_input.model_dump_json())
     if not v.ok:
-        raise Exception(f'Could not cache the data, {v.text}')
+        raise Exception(f'trigger_fuzzy_match_operation: Could not cache the data, {v.text}')
     return Status(**v.json())
 
 
@@ -78,7 +79,7 @@ def trigger_create_operation(flow_id: int, node_id: int | str, received_table: R
     f = requests.post(url=f'{WORKER_URL}/create_table/{file_type}', data=received_table.model_dump_json(),
                       params={'flowfile_flow_id': flow_id, 'flowfile_node_id': node_id})
     if not f.ok:
-        raise Exception(f'Could not cache the data, {f.text}')
+        raise Exception(f'trigger_create_operation: Could not cache the data, {f.text}')
     return Status(**f.json())
 
 
@@ -86,7 +87,7 @@ def trigger_database_read_collector(database_external_read_settings: DatabaseExt
     f = requests.post(url=f'{WORKER_URL}/store_database_read_result',
                       data=database_external_read_settings.model_dump_json())
     if not f.ok:
-        raise Exception(f'Could not cache the data, {f.text}')
+        raise Exception(f'trigger_database_read_collector: Could not cache the data, {f.text}')
     return Status(**f.json())
 
 
@@ -94,7 +95,7 @@ def trigger_database_write(database_external_write_settings: DatabaseExternalWri
     f = requests.post(url=f'{WORKER_URL}/store_database_write_result',
                       data=database_external_write_settings.model_dump_json())
     if not f.ok:
-        raise Exception(f'Could not cache the data, {f.text}')
+        raise Exception(f'trigger_database_write: Could not cache the data, {f.text}')
     return Status(**f.json())
 
 
@@ -102,7 +103,7 @@ def trigger_cloud_storage_write(database_external_write_settings: CloudStorageWr
     f = requests.post(url=f'{WORKER_URL}/write_data_to_cloud',
                       data=database_external_write_settings.model_dump_json())
     if not f.ok:
-        raise Exception(f'Could not cache the data, {f.text}')
+        raise Exception(f'trigger_cloud_storage_write: Could not cache the data, {f.text}')
     return Status(**f.json())
 
 
@@ -111,7 +112,7 @@ def get_results(file_ref: str) -> Status | None:
     if f.status_code == 200:
         return Status(**f.json())
     else:
-        raise Exception(f'Could not fetch the data, {f.text}')
+        raise Exception(f'get_results: Could not fetch the data, {f.text}')
 
 
 def results_exists(file_ref: str):
@@ -128,6 +129,25 @@ def results_exists(file_ref: str):
         return False
 
 
+def clear_task_from_worker(file_ref: str) -> bool:
+    """
+    Clears a task from the worker service by making a DELETE request. It also removes associated cached files.
+    Args:
+        file_ref (str): The unique identifier of the task to clear.
+
+    Returns:
+        bool: True if the task was successfully cleared, False otherwise.
+    """
+    try:
+        f = requests.delete(f'{WORKER_URL}/clear_task/{file_ref}')
+        if f.status_code == 200:
+            return True
+        return False
+    except requests.RequestException as e:
+        logger.error(f"Failed to remove results: {str(e)}")
+        return False
+
+
 def get_df_result(encoded_df: str) -> pl.LazyFrame:
     r = decodebytes(encoded_df.encode())
     return pl.LazyFrame.deserialize(io.BytesIO(r))