Flowfile 0.3.9__py3-none-any.whl → 0.4.0__py3-none-any.whl

flowfile_core/fileExplorer/funcs.py

@@ -19,13 +19,30 @@ class FileInfo(BaseModel):
     exists: bool = True
 
     @classmethod
-    def from_path(cls, path: Path) -> 'FileInfo':
-        """Create FileInfo instance from a path."""
+    def from_path(cls, path: Path, sandbox_root: Optional[Path] = None,
+                  use_relative_paths: bool = False) -> 'FileInfo':
+        """Create FileInfo instance from a path.
+
+        Args:
+            path: The path to create FileInfo from
+            sandbox_root: The root directory for sandboxing (for relative path calculation)
+            use_relative_paths: If True, store relative paths; if False, store absolute paths
+        """
         try:
             stats = path.stat()
+
+            # Decide whether to use relative or absolute path
+            if use_relative_paths and sandbox_root:
+                try:
+                    display_path = str(path.relative_to(sandbox_root))
+                except ValueError:
+                    display_path = str(path.absolute())
+            else:
+                display_path = str(path.absolute())
+
             return cls(
                 name=path.name,
-                path=str(path.absolute()),
+                path=display_path,
                 is_directory=path.is_dir(),
                 size=stats.st_size,
                 file_type=path.suffix[1:] if path.suffix else "",
@@ -37,9 +54,18 @@ class FileInfo(BaseModel):
                 exists=True
             )
         except (PermissionError, OSError):
+            # Handle error case
+            if use_relative_paths and sandbox_root:
+                try:
+                    display_path = str(path.relative_to(sandbox_root))
+                except ValueError:
+                    display_path = str(path.absolute())
+            else:
+                display_path = str(path.absolute())
+
             return cls(
                 name=path.name,
-                path=str(path.absolute()),
+                path=display_path,
                 is_directory=False,
                 size=0,
                 file_type="",
@@ -50,30 +76,108 @@ class FileInfo(BaseModel):
             )
 
 
-class FileExplorer:
-    def __init__(self, start_path: Optional[str|Path] = None):
-        """Initialize FileExplorer with user's home directory or specified path."""
-        if start_path is None:
-            self.current_path = Path.home()
+class SecureFileExplorer:
+    """File explorer with sandbox enforcement to prevent directory traversal."""
+
+    def __init__(self, start_path: Union[str, Path],
+                 sandbox_root: Optional[Union[str, Path]] = None,
+                 use_relative_paths: bool = False):
+        """Initialize SecureFileExplorer with sandboxing.
+
+        Args:
+            start_path: Initial directory to start in
+            sandbox_root: Root directory that user cannot escape from.
+                         If None, no sandbox enforcement.
+            use_relative_paths: If True, FileInfo will contain relative paths;
+                               if False (default), contains absolute paths
+        """
+        self.use_relative_paths = use_relative_paths
+
+        # Set up the sandbox root
+        if sandbox_root is not None:
+            self.sandbox_root = Path(sandbox_root).expanduser().resolve()
+        else:
+            self.sandbox_root = None
+
+        # Set initial current path
+        initial_path = Path(start_path).expanduser().resolve()
+
+        # If sandbox is set and initial path is outside it, use sandbox root
+        if self.sandbox_root and not self._is_path_safe(initial_path):
+            self.current_path = self.sandbox_root
         else:
-            self.current_path = Path(start_path).expanduser().resolve()
+            self.current_path = initial_path
 
-        if not self.current_path.exists():
-            raise ValueError(f"Path does not exist: {self.current_path}")
+    def _is_path_safe(self, path: Path) -> bool:
+        """Check if a path is within the sandbox root.
 
-        if not self.current_path.is_dir():
-            raise ValueError(f"Path is not a directory: {self.current_path}")
+        Uses resolve() to handle symlinks and relative paths securely.
+        Returns True if no sandbox is set (no restrictions).
+        """
+        if self.sandbox_root is None:
+            return True  # No sandbox = no restrictions
+
+        try:
+            resolved_path = path.resolve()
+            resolved_sandbox = self.sandbox_root.resolve()
+            # Check if the resolved path is within sandbox
+            resolved_path.relative_to(resolved_sandbox)
+            return True
+        except (ValueError, RuntimeError):
+            return False
+
+    def _sanitize_path(self, path: Union[str, Path]) -> Optional[Path]:
+        """Sanitize and validate a path, ensuring it stays within sandbox.
+
+        Returns None if path would escape sandbox.
+        """
+        try:
+            # Handle relative paths from current directory
+            if isinstance(path, str):
+                # Remove any suspicious patterns
+                if '..' in Path(path).parts or path.startswith('/'):
+                    # For absolute paths or parent references, resolve from sandbox root
+                    test_path = Path(path).expanduser()
+                else:
+                    # For simple relative paths, resolve from current directory
+                    test_path = self.current_path / path
+            else:
+                test_path = path
+
+            # Resolve to absolute path
+            resolved = test_path.resolve()
+
+            # Check if within sandbox
+            if self._is_path_safe(resolved):
+                return resolved
+            else:
+                return None
+        except (ValueError, RuntimeError, OSError):
+            return None
 
     @property
     def current_directory(self) -> str:
-        """Get the current directory path."""
-        return str(self.current_path.absolute())
+        """Get the current directory path relative to sandbox root."""
+        if self.sandbox_root is None:
+            return str(self.current_path)
+
+        try:
+            relative = self.current_path.relative_to(self.sandbox_root)
+            return str(relative) if str(relative) != "." else "/"
+        except ValueError:
+            return "/"
 
     @property
     def parent_directory(self) -> Optional[str]:
-        """Get the parent directory path if it exists."""
+        """Get the parent directory path if it exists and is within sandbox."""
         parent = self.current_path.parent
-        return str(parent.absolute()) if parent != self.current_path else None
+        if self._is_path_safe(parent) and parent != self.current_path:
+            try:
+                relative = parent.relative_to(self.sandbox_root)
+                return str(relative) if str(relative) != "." else "/"
+            except ValueError:
+                return None
+        return None
 
     def list_contents(
             self,
@@ -85,34 +189,23 @@ class FileExplorer:
             max_size: Optional[int] = None,
             sort_by: Literal['name', 'date', 'size', 'type'] = 'name',
             reverse: bool = False,
-            exclude_patterns: Optional[List[str]] = None
+            exclude_patterns: Optional[List[str]] = None,
+            max_depth: int = 5  # Add depth limit for recursive operations
     ) -> List[FileInfo]:
-        """
-        List contents of the current directory with advanced filtering and sorting.
-
-        Args:
-            show_hidden: Whether to show hidden files and directories
-            file_types: List of file extensions to include (without dots)
-            recursive: Whether to scan subdirectories
-            min_size: Minimum file size in bytes
-            max_size: Maximum file size in bytes
-            sort_by: Field to sort results by
-            reverse: Whether to reverse sort order
-            exclude_patterns: Glob patterns to exclude
-
-        Returns:
-            List of FileInfo objects sorted according to parameters
-        """
+        """List contents with security-conscious filtering."""
         contents: List[FileInfo] = []
         excluded_paths: Set[str] = set()
 
         if exclude_patterns:
             for pattern in exclude_patterns:
-                excluded_paths.update(str(p) for p in self.current_path.glob(pattern))
+                # Ensure patterns don't escape sandbox
+                safe_pattern = pattern.replace('../', '').replace('..\\', '')
+                excluded_paths.update(str(p) for p in self.current_path.glob(safe_pattern))
 
         def should_include(info: FileInfo) -> bool:
             """Determine if a file should be included based on filters."""
-            if str(info.path) in excluded_paths:
+            full_path = self.current_path / info.path
+            if str(full_path) in excluded_paths:
                 return False
             if not show_hidden and info.is_hidden:
                 return False
@@ -125,20 +218,55 @@ class FileExplorer:
             return True
 
         try:
-            # Define the scan pattern based on recursion
-            pattern = '**/*' if recursive else '*'
+            if recursive:
+                # Use iterative approach with depth limit for safety
+                dirs_to_process = [(self.current_path, 0)]
+                processed = set()
 
-            # Scan directory
-            for item in self.current_path.glob(pattern):
-                try:
-                    file_info = FileInfo.from_path(item)
-                    if should_include(file_info):
-                        contents.append(file_info)
-                except (PermissionError, OSError):
-                    continue
+                while dirs_to_process:
+                    current_dir, depth = dirs_to_process.pop(0)
+
+                    # Skip if already processed or exceeds depth
+                    if current_dir in processed or depth > max_depth:
+                        continue
+
+                    processed.add(current_dir)
+
+                    try:
+                        for item in current_dir.iterdir():
+                            # Security check for each item
+                            if not self._is_path_safe(item):
+                                continue
+
+                            try:
+                                file_info = FileInfo.from_path(item, self.sandbox_root,
+                                                               self.use_relative_paths)
+                                if should_include(file_info):
+                                    contents.append(file_info)
+
+                                if item.is_dir() and depth < max_depth:
+                                    dirs_to_process.append((item, depth + 1))
+                            except (PermissionError, OSError):
+                                continue
+                    except (PermissionError, OSError):
+                        continue
+            else:
+                # Non-recursive listing
+                for item in self.current_path.iterdir():
+                    # Security check
+                    if not self._is_path_safe(item):
+                        continue
+
+                    try:
+                        file_info = FileInfo.from_path(item, self.sandbox_root,
+                                                       self.use_relative_paths)
+                        if should_include(file_info):
+                            contents.append(file_info)
+                    except (PermissionError, OSError):
+                        continue
 
         except PermissionError:
-            raise PermissionError(f"Permission denied to access directory: {self.current_path}")
+            raise PermissionError(f"Permission denied to access directory: {self.current_directory}")
 
         # Sort results
         sort_key = {
@@ -151,109 +279,99 @@ class FileExplorer:
         return sorted(contents, key=sort_key, reverse=reverse)
 
     def navigate_to(self, path: str) -> bool:
-        """
-        Navigate to a new directory path.
-        Returns True if navigation was successful, False otherwise.
-        """
-        new_path = None
-        try:
-            new_path = Path(path).expanduser().resolve()
+        """Navigate to a new directory path within sandbox."""
+        sanitized = self._sanitize_path(path)
 
-            if not new_path.exists() or not new_path.is_dir():
-                return False
+        if sanitized is None:
+            return False
 
+        if not sanitized.exists() or not sanitized.is_dir():
+            return False
+
+        try:
             # Test if we can actually read the directory
-            next(new_path.iterdir(), None)
-            self.current_path = new_path
+            next(sanitized.iterdir(), None)
+            self.current_path = sanitized
             return True
-
-        except PermissionError:
-            if new_path:
-                self.current_path = new_path
+        except (PermissionError, OSError):
+            # Still navigate if we have permission issues (user will see empty dir)
+            self.current_path = sanitized
             return True
-        except OSError:
-            return False
 
     def navigate_up(self) -> bool:
-        """
-        Navigate up to the parent directory.
-        Returns True if navigation was successful, False otherwise.
-        """
-        parent = self.parent_directory
-        if parent is None:
+        """Navigate up to the parent directory, respecting sandbox."""
+        parent = self.current_path.parent
+
+        # Check if parent is within sandbox
+        if not self._is_path_safe(parent):
             return False
-        return self.navigate_to(parent)
+
+        # Don't navigate if we're already at sandbox root
+        if parent == self.current_path:
+            return False
+
+        self.current_path = parent
+        return True
 
     def navigate_into(self, directory_name: str) -> bool:
-        """
-        Navigate into a subdirectory of the current directory.
-        Returns True if navigation was successful, False otherwise.
-        """
+        """Navigate into a subdirectory, with path sanitization."""
+        # Sanitize directory name
+        if '/' in directory_name or '\\' in directory_name or '..' in directory_name:
+            return False
+
         new_path = self.current_path / directory_name
         return self.navigate_to(str(new_path))
 
+    def get_absolute_path(self, relative_path: str) -> Optional[Path]:
+        """Get absolute path for a file within sandbox.
+
+        Returns None if the path would escape sandbox.
+        """
+        sanitized = self._sanitize_path(relative_path)
+        return sanitized if sanitized else None
+
 
 def get_files_from_directory(
         dir_name: Union[str, Path],
         types: Optional[List[str]] = None,
         *,
         include_hidden: bool = False,
-        recursive: bool = False
+        recursive: bool = False,
+        sandbox_root: Optional[Union[str, Path]] = None
 ) -> Optional[List[FileInfo]]:
     """
-    Get list of files from a directory with optional type filtering.
+    Get list of files from a directory with sandbox enforcement.
 
     Args:
         dir_name: Directory path to scan
-        types: List of file extensions to include (without dots). None means all types
+        types: List of file extensions to include
         include_hidden: Whether to include hidden files
         recursive: Whether to scan subdirectories
+        sandbox_root: Root directory to enforce as sandbox boundary
 
     Returns:
-        List of FileInfo objects or None if directory doesn't exist
-
-    Example:
-        >>> files = get_files_from_directory("/path/to/dir", types=["pdf", "txt"])
-        >>> for file in files:
-        ...     print(f"{file.name} - {file.size} bytes")
+        List of FileInfo objects or None if directory doesn't exist or is outside sandbox
     """
     try:
-        dir_path = Path(dir_name).resolve()
-        if not dir_path.exists():
-            return None
-        if not dir_path.is_dir():
-            raise ValueError(f"Path is not a directory: {dir_path}")
-
-        # Normalize file types
-        if types:
-            types = [t.lower().lstrip('.') for t in types]
-
-        files = []
-        pattern = '**/*' if recursive else '*'
-
-        for item in dir_path.glob(pattern):
-            try:
-                # Skip hidden files unless specifically requested
-                if not include_hidden and item.name.startswith('.'):
-                    continue
-
-                # Skip directories unless recursive is True
-                if item.is_dir() and not recursive:
-                    continue
-
-                # Check file type if types are specified
-                if types and not item.is_dir():
-                    if item.suffix[1:].lower() not in types:
-                        continue
-
-                file_info = FileInfo.from_path(item)
-                files.append(file_info)
-
-            except (PermissionError, OSError):
-                continue
-
-        return sorted(files, key=lambda x: (not x.is_directory, x.name.lower()))
-
+        # Create a secure explorer with sandbox
+        if sandbox_root:
+            explorer = SecureFileExplorer(start_path=dir_name, sandbox_root=sandbox_root)
+        else:
+            explorer = SecureFileExplorer(start_path=dir_name)
+
+        # Use the explorer's list_contents method
+        return explorer.list_contents(
+            show_hidden=include_hidden,
+            file_types=types,
+            recursive=recursive
+        )
+
+    except (ValueError, PermissionError) as e:
+        # Return None for invalid/inaccessible directories
+        return None
     except Exception as e:
         raise type(e)(f"Error scanning directory {dir_name}: {str(e)}") from e
 
+
+# Alias for backward compatibility
+FileExplorer = SecureFileExplorer

flowfile_core/flowfile/flow_data_engine/flow_file_column/interface.py

@@ -0,0 +1,4 @@
+from typing import Literal
+
+DataTypeGroup = Literal['numeric', 'str', 'date']
+ReadableDataTypeGroup = Literal['Numeric', 'String', 'Date', 'Other', 'Boolean', 'Binary', 'Complex']

flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py

@@ -1,44 +1,13 @@
 
 from dataclasses import dataclass
-from typing import Optional, Any, List, Dict, Literal, Iterable
+from typing import Optional, Any, List, Dict, Iterable
 
 from flowfile_core.schemas import input_schema
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import cast_str_to_polars_type
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.polars_type import PlType
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.interface import ReadableDataTypeGroup, DataTypeGroup
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.type_registry import convert_pl_type_to_string
 import polars as pl
-# TODO: rename flow_file_column to flowfile_column
-DataTypeGroup = Literal['numeric', 'str', 'date']
-
-
-def convert_pl_type_to_string(pl_type: pl.DataType, inner: bool = False) -> str:
-    if isinstance(pl_type, pl.List):
-        inner_str = convert_pl_type_to_string(pl_type.inner, inner=True)
-        return f"pl.List({inner_str})"
-    elif isinstance(pl_type, pl.Array):
-        inner_str = convert_pl_type_to_string(pl_type.inner, inner=True)
-        return f"pl.Array({inner_str})"
-    elif isinstance(pl_type, pl.Decimal):
-        precision = pl_type.precision if hasattr(pl_type, 'precision') else None
-        scale = pl_type.scale if hasattr(pl_type, 'scale') else None
-        if precision is not None and scale is not None:
-            return f"pl.Decimal({precision}, {scale})"
-        elif precision is not None:
-            return f"pl.Decimal({precision})"
-        else:
-            return "pl.Decimal()"
-    elif isinstance(pl_type, pl.Struct):
-        # Handle Struct with field definitions
-        fields = []
-        if hasattr(pl_type, 'fields'):
-            for field in pl_type.fields:
-                field_name = field.name
-                field_type = convert_pl_type_to_string(field.dtype, inner=True)
-                fields.append(f'pl.Field("{field_name}", {field_type})')
-        field_str = ", ".join(fields)
-        return f"pl.Struct([{field_str}])"
-    else:
-        # For base types, we want the full pl.TypeName format
-        return str(pl_type.base_type()) if not inner else f"pl.{pl_type}"
 
 
 @dataclass
@@ -52,6 +21,7 @@ class FlowfileColumn:
     number_of_empty_values: int
     number_of_unique_values: int
     example_values: str
+    data_type_group: ReadableDataTypeGroup
     __sql_type: Optional[Any]
     __is_unique: Optional[bool]
     __nullable: Optional[bool]
@@ -75,6 +45,7 @@ class FlowfileColumn:
         self.__is_unique = None
         self.__sql_type = None
         self.__perc_unique = None
+        self.data_type_group = self.get_readable_datatype_group()
 
     def __repr__(self):
         """
@@ -220,6 +191,20 @@ class FlowfileColumn:
             return 'numeric'
         elif self.data_type in ('datetime', 'date', 'Date', 'Datetime', 'Time'):
             return 'date'
+        else:
+            return 'str'
+
+    def get_readable_datatype_group(self) -> ReadableDataTypeGroup:
+        if self.data_type in ('Utf8', 'VARCHAR', 'CHAR', 'NVARCHAR', 'String'):
+            return 'String'
+        elif self.data_type in ('fixed_decimal', 'decimal', 'float', 'integer', 'boolean', 'double', 'Int16', 'Int32',
+                                'Int64', 'Float32', 'Float64', 'Decimal', 'Binary', 'Boolean', 'Uint8', 'Uint16',
+                                'Uint32', 'Uint64'):
+            return 'Numeric'
+        elif self.data_type in ('datetime', 'date', 'Date', 'Datetime', 'Time'):
+            return 'Date'
+        else:
+            return 'Other'
 
     def get_polars_type(self) -> PlType:
         pl_datatype = cast_str_to_polars_type(self.data_type)

flowfile_core/flowfile/flow_data_engine/flow_file_column/type_registry.py

@@ -0,0 +1,36 @@
+
+from typing import Type, Literal, List, Dict, Union, Tuple
+import polars as pl
+DataTypeGroup = Literal['numeric', 'string', 'datetime', 'boolean', 'binary', 'complex', 'unknown']
+
+
+def convert_pl_type_to_string(pl_type: pl.DataType, inner: bool = False) -> str:
+    if isinstance(pl_type, pl.List):
+        inner_str = convert_pl_type_to_string(pl_type.inner, inner=True)
+        return f"pl.List({inner_str})"
+    elif isinstance(pl_type, pl.Array):
+        inner_str = convert_pl_type_to_string(pl_type.inner, inner=True)
+        return f"pl.Array({inner_str})"
+    elif isinstance(pl_type, pl.Decimal):
+        precision = pl_type.precision if hasattr(pl_type, 'precision') else None
+        scale = pl_type.scale if hasattr(pl_type, 'scale') else None
+        if precision is not None and scale is not None:
+            return f"pl.Decimal({precision}, {scale})"
+        elif precision is not None:
+            return f"pl.Decimal({precision})"
+        else:
+            return "pl.Decimal()"
+    elif isinstance(pl_type, pl.Struct):
+        # Handle Struct with field definitions
+        fields = []
+        if hasattr(pl_type, 'fields'):
+            for field in pl_type.fields:
+                field_name = field.name
+                field_type = convert_pl_type_to_string(field.dtype, inner=True)
+                fields.append(f'pl.Field("{field_name}", {field_type})')
+        field_str = ", ".join(fields)
+        return f"pl.Struct([{field_str}])"
+    else:
+        # For base types, we want the full pl.TypeName format
+        return str(pl_type.base_type()) if not inner else f"pl.{pl_type}"
+