Flowfile 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

flowfile_core/fileExplorer/funcs.py

@@ -1,13 +1,17 @@
-from pathlib import Path
-from typing import List, Optional, Set, Union
+import os
 from datetime import datetime
+from pathlib import Path
+from typing import Literal, Optional
+
+from fastapi import HTTPException
 from pydantic import BaseModel
-from typing_extensions import Literal
-import os
+
+from shared.storage_config import storage
 
 
 class FileInfo(BaseModel):
     """Comprehensive information about a file or directory."""
+
     name: str
     path: str
     is_directory: bool
@@ -19,8 +23,7 @@ class FileInfo(BaseModel):
     exists: bool = True
 
     @classmethod
-    def from_path(cls, path: Path, sandbox_root: Optional[Path] = None,
-                  use_relative_paths: bool = False) -> 'FileInfo':
+    def from_path(cls, path: Path, sandbox_root: Path | None = None, use_relative_paths: bool = False) -> "FileInfo":
         """Create FileInfo instance from a path.
 
         Args:
@@ -48,10 +51,8 @@ class FileInfo(BaseModel):
                 file_type=path.suffix[1:] if path.suffix else "",
                 last_modified=datetime.fromtimestamp(stats.st_mtime),
                 created_date=datetime.fromtimestamp(stats.st_ctime),
-                is_hidden=path.name.startswith('.') or (
-                        os.name == 'nt' and bool(stats.st_file_attributes & 0x2)
-                ),
-                exists=True
+                is_hidden=path.name.startswith(".") or (os.name == "nt" and bool(stats.st_file_attributes & 0x2)),
+                exists=True,
             )
         except (PermissionError, OSError):
             # Handle error case
@@ -72,16 +73,16 @@ class FileInfo(BaseModel):
                 last_modified=datetime.fromtimestamp(0),
                 created_date=datetime.fromtimestamp(0),
                 is_hidden=False,
-                exists=False
+                exists=False,
             )
 
 
 class SecureFileExplorer:
     """File explorer with sandbox enforcement to prevent directory traversal."""
 
-    def __init__(self, start_path: Union[str, Path],
-                 sandbox_root: Optional[Union[str, Path]] = None,
-                 use_relative_paths: bool = False):
+    def __init__(
+        self, start_path: str | Path, sandbox_root: str | Path | None = None, use_relative_paths: bool = False
+    ):
         """Initialize SecureFileExplorer with sandboxing.
 
         Args:
@@ -126,16 +127,17 @@ class SecureFileExplorer:
         except (ValueError, RuntimeError):
             return False
 
-    def _sanitize_path(self, path: Union[str, Path]) -> Optional[Path]:
+    def _sanitize_path(self, path: str | Path) -> Path | None:
         """Sanitize and validate a path, ensuring it stays within sandbox.
 
         Returns None if path would escape sandbox.
         """
         try:
-            # Handle relative paths from current directory
+            # Handle relative paths from current directoryb
             if isinstance(path, str):
+
                 # Remove any suspicious patterns
-                if '..' in Path(path).parts or path.startswith('/'):
+                if path.startswith("/"):
                     # For absolute paths or parent references, resolve from sandbox root
                     test_path = Path(path).expanduser()
                 else:
@@ -168,7 +170,7 @@ class SecureFileExplorer:
             return "/"
 
     @property
-    def parent_directory(self) -> Optional[str]:
+    def parent_directory(self) -> str | None:
         """Get the parent directory path if it exists and is within sandbox."""
         parent = self.current_path.parent
         if self._is_path_safe(parent) and parent != self.current_path:
@@ -180,26 +182,26 @@ class SecureFileExplorer:
         return None
 
     def list_contents(
-            self,
-            *,
-            show_hidden: bool = False,
-            file_types: Optional[List[str]] = None,
-            recursive: bool = False,
-            min_size: Optional[int] = None,
-            max_size: Optional[int] = None,
-            sort_by: Literal['name', 'date', 'size', 'type'] = 'name',
-            reverse: bool = False,
-            exclude_patterns: Optional[List[str]] = None,
-            max_depth: int = 5  # Add depth limit for recursive operations
-    ) -> List[FileInfo]:
+        self,
+        *,
+        show_hidden: bool = False,
+        file_types: list[str] | None = None,
+        recursive: bool = False,
+        min_size: int | None = None,
+        max_size: int | None = None,
+        sort_by: Literal["name", "date", "size", "type"] = "name",
+        reverse: bool = False,
+        exclude_patterns: list[str] | None = None,
+        max_depth: int = 5,  # Add depth limit for recursive operations
+    ) -> list[FileInfo]:
         """List contents with security-conscious filtering."""
-        contents: List[FileInfo] = []
-        excluded_paths: Set[str] = set()
+        contents: list[FileInfo] = []
+        excluded_paths: set[str] = set()
 
         if exclude_patterns:
             for pattern in exclude_patterns:
                 # Ensure patterns don't escape sandbox
-                safe_pattern = pattern.replace('../', '').replace('..\\', '')
+                safe_pattern = pattern.replace("../", "").replace("..\\", "")
                 excluded_paths.update(str(p) for p in self.current_path.glob(safe_pattern))
 
         def should_include(info: FileInfo) -> bool:
@@ -239,8 +241,7 @@ class SecureFileExplorer:
                                 continue
 
                             try:
-                                file_info = FileInfo.from_path(item, self.sandbox_root,
-                                                               self.use_relative_paths)
+                                file_info = FileInfo.from_path(item, self.sandbox_root, self.use_relative_paths)
                                 if should_include(file_info):
                                     contents.append(file_info)
 
@@ -258,8 +259,7 @@ class SecureFileExplorer:
                         continue
 
                     try:
-                        file_info = FileInfo.from_path(item, self.sandbox_root,
-                                                       self.use_relative_paths)
+                        file_info = FileInfo.from_path(item, self.sandbox_root, self.use_relative_paths)
                         if should_include(file_info):
                             contents.append(file_info)
                     except (PermissionError, OSError):
@@ -270,10 +270,10 @@ class SecureFileExplorer:
 
         # Sort results
         sort_key = {
-            'name': lambda x: (not x.is_directory, x.name.lower()),
-            'date': lambda x: (not x.is_directory, x.last_modified),
-            'size': lambda x: (not x.is_directory, x.size),
-            'type': lambda x: (not x.is_directory, x.file_type.lower(), x.name.lower())
+            "name": lambda x: (not x.is_directory, x.name.lower()),
+            "date": lambda x: (not x.is_directory, x.last_modified),
+            "size": lambda x: (not x.is_directory, x.size),
+            "type": lambda x: (not x.is_directory, x.file_type.lower(), x.name.lower()),
         }[sort_by]
 
         return sorted(contents, key=sort_key, reverse=reverse)
@@ -316,13 +316,13 @@ class SecureFileExplorer:
     def navigate_into(self, directory_name: str) -> bool:
         """Navigate into a subdirectory, with path sanitization."""
         # Sanitize directory name
-        if '/' in directory_name or '\\' in directory_name or '..' in directory_name:
+        if "/" in directory_name or "\\" in directory_name or ".." in directory_name:
             return False
 
         new_path = self.current_path / directory_name
         return self.navigate_to(str(new_path))
 
-    def get_absolute_path(self, relative_path: str) -> Optional[Path]:
+    def get_absolute_path(self, relative_path: str) -> Path | None:
         """Get absolute path for a file within sandbox.
 
         Returns None if the path would escape sandbox.
@@ -332,13 +332,13 @@ class SecureFileExplorer:
 
 
 def get_files_from_directory(
-        dir_name: Union[str, Path],
-        types: Optional[List[str]] = None,
-        *,
-        include_hidden: bool = False,
-        recursive: bool = False,
-        sandbox_root: Optional[Union[str, Path]] = None
-) -> Optional[List[FileInfo]]:
+    dir_name: str | Path,
+    types: list[str] | None = None,
+    *,
+    include_hidden: bool = False,
+    recursive: bool = False,
+    sandbox_root: str | Path | None = None,
+) -> list[FileInfo] | None:
     """
     Get list of files from a directory with sandbox enforcement.
 
@@ -360,18 +360,84 @@ def get_files_from_directory(
             explorer = SecureFileExplorer(start_path=dir_name)
 
         # Use the explorer's list_contents method
-        return explorer.list_contents(
-            show_hidden=include_hidden,
-            file_types=types,
-            recursive=recursive
-        )
+        return explorer.list_contents(show_hidden=include_hidden, file_types=types, recursive=recursive)
 
-    except (ValueError, PermissionError) as e:
+    except (ValueError, PermissionError):
         # Return None for invalid/inaccessible directories
         return None
     except Exception as e:
         raise type(e)(f"Error scanning directory {dir_name}: {str(e)}") from e
 
 
+def validate_file_path(user_path: str, allowed_base: Path) -> Optional[Path]:
+    """Validate a file path is safe and within allowed_base.
+
+    Uses os.path.realpath + startswith pattern recognized by CodeQL as safe.
+
+    Args:
+        user_path: The user-provided path string
+        allowed_base: The base directory that the path must be within
+
+    Returns:
+        The validated Path object, or None if validation fails
+    """
+    try:
+        # Block obvious path traversal patterns early
+        if '..' in user_path:
+            return None
+
+        # Get the base path as a normalized, real path string
+        base_path = os.path.realpath(str(allowed_base.resolve()))
+
+        # Always resolve the user path relative to the allowed base directory
+        candidate_path = os.path.join(base_path, user_path)
+        fullpath = os.path.realpath(candidate_path)
+
+        # Ensure the resolved path stays within the allowed base directory
+        if not fullpath.startswith(base_path + os.sep) and fullpath != base_path:
+            return None
+
+        return Path(fullpath)
+
+    except (ValueError, RuntimeError, OSError):
+        return None
+
+
+def validate_path_under_cwd(user_path: str) -> str:
+    """Validate that a user-provided path resolves to within allowed directories.
+
+    Uses the exact pattern from CodeQL documentation for py/path-injection:
+    - os.path.normpath for path normalization
+    - os.path.join to combine base with user input
+    - startswith check to ensure path stays within base
+
+    Allowed directories:
+    - Current working directory (for development/testing)
+    - Flowfile storage directory (~/.flowfile)
+
+    Args:
+        user_path: The user-provided path string
+
+    Returns:
+        The validated, normalized full path as a string
+
+    Raises:
+        HTTPException: 403 if path escapes the allowed directories
+    """
+    # Try current working directory first
+    base_path = os.path.normpath(os.getcwd())
+    fullpath = os.path.normpath(os.path.join(base_path, user_path))
+    if fullpath.startswith(base_path):
+        return fullpath
+
+    # Try flowfile storage directory (~/.flowfile)
+    base_path = os.path.normpath(str(storage.base_directory))
+    fullpath = os.path.normpath(os.path.join(base_path, user_path))
+    if fullpath.startswith(base_path):
+        return fullpath
+
+    raise HTTPException(403, 'Access denied')
+
+
 # Alias for backward compatibility
-FileExplorer = SecureFileExplorer
+FileExplorer = SecureFileExplorer

flowfile_core/fileExplorer/utils.py

@@ -1,8 +1,8 @@
 import os
 import re
 
-video_types = ['mp4', "webm", "opgg"]
-audio_types = ['mp3', "wav", "ogg", "mpeg", "aac", "3gpp", "3gpp2", "aiff", "x-aiff", "amr", "mpga"]
+video_types = ["mp4", "webm", "opgg"]
+audio_types = ["mp3", "wav", "ogg", "mpeg", "aac", "3gpp", "3gpp2", "aiff", "x-aiff", "amr", "mpga"]
 
 
 def get_chunk(start_byte=None, end_byte=None, full_path=None):
@@ -11,7 +11,7 @@ def get_chunk(start_byte=None, end_byte=None, full_path=None):
         length = end_byte + 1 - start_byte
     else:
         length = file_size - start_byte
-    with open(full_path, 'rb') as f:
+    with open(full_path, "rb") as f:
         f.seek(start_byte)
         chunk = f.read(length)
     return chunk, start_byte, length, file_size
@@ -19,10 +19,10 @@ def get_chunk(start_byte=None, end_byte=None, full_path=None):
 
 def get_file(file_path, mimetype):
     f = ""
-    range_header = f.headers.get('Range', None)
+    range_header = f.headers.get("Range", None)
     start_byte, end_byte = 0, None
     if range_header:
-        match = re.search(r'(\d+)-(\d*)', range_header)
+        match = re.search(r"(\d+)-(\d*)", range_header)
         groups = match.groups()
         if groups[0]:
             start_byte = int(groups[0])
@@ -30,15 +30,14 @@ def get_file(file_path, mimetype):
             end_byte = int(groups[1])
 
     chunk, start, length, file_size = get_chunk(start_byte, end_byte, file_path)
-    resp = Response(chunk, 206, mimetype=f'video/{mimetype}',
-                    content_type=mimetype, direct_passthrough=True)
+    resp = Response(chunk, 206, mimetype=f"video/{mimetype}", content_type=mimetype, direct_passthrough=True)
     print(length)
-    resp.headers.add('Content-Range', 'bytes {0}-{1}/{2}'.format(start, start + length - 1, file_size))
+    resp.headers.add("Content-Range", f"bytes {start}-{start + length - 1}/{file_size}")
     return resp
 
 
 def is_media(filepath):
-    found_media = re.search("\.mp4$|\.mp3$", filepath, re.IGNORECASE)
+    found_media = re.search(r"\.mp4$|\.mp3$", filepath, re.IGNORECASE)
     if found_media:
         extension = found_media[0].lower()[1:]
         if found_media in video_types:
@@ -48,6 +47,6 @@ def is_media(filepath):
 
 
 def get_file_extension(fname):
-    found_extension = re.search("\.[A-Za-z0-9]*$", fname, re.IGNORECASE)
+    found_extension = re.search(r"\.[A-Za-z0-9]*$", fname, re.IGNORECASE)
     if found_extension:
-        return found_extension[0][1:].lower()
+        return found_extension[0][1:].lower()

flowfile_core/flowfile/_extensions/real_time_interface.py

@@ -1,8 +1,9 @@
+from dataclasses import dataclass
+
 import polars as pl
 from polars_expr_transformer import simple_function_to_expr
+
 from flowfile_core.configs import logger
-from dataclasses import dataclass
-from typing import Optional
 
 
 @dataclass
@@ -10,16 +11,16 @@ class RealTimeResult:
     result_df: pl.DataFrame
     data_type: pl.DataType
     readable_result: str
-    success: Optional[bool] = None
+    success: bool | None = None
 
     def __init__(self, result_value: pl.DataFrame, data_type: pl.DataType):
         self.result_df = result_value
         self.data_type = data_type
-        if len(result_value)>0:
-            self.readable_result = str(result_value.item(0,0))
+        if len(result_value) > 0:
+            self.readable_result = str(result_value.item(0, 0))
             self.success = True
         else:
-            self.readable_result = ''
+            self.readable_result = ""
             self.success = None
 
     def is_filterable_result(self):
@@ -44,8 +45,9 @@ def get_realtime_func_results(df: pl.DataFrame | pl.LazyFrame, func_string: str,
     print(get_first_result_of_function('year(today())', df))
     """
     if isinstance(df, pl.LazyFrame):
-        logger.warning('Performance in this case can be '
-                       'improved by using polars.DataFrame to ensure it returns instantly')
+        logger.warning(
+            "Performance in this case can be " "improved by using polars.DataFrame to ensure it returns instantly"
+        )
         df = df.head(sample).collect()
     result = df.head(1).select(simple_function_to_expr(func_string))
     return RealTimeResult(result, result.dtypes[0])

flowfile_core/flowfile/analytics/analytics_processor.py

@@ -1,19 +1,20 @@
-from typing import Dict, Set, List
-
-from flowfile_core.flowfile.analytics.graphic_walker import (get_initial_gf_data_from_ff,
-                                                             convert_ff_columns_to_gw_fields)
+from flowfile_core.configs import logger
+from flowfile_core.flowfile.analytics.graphic_walker import convert_ff_columns_to_gw_fields, get_initial_gf_data_from_ff
 from flowfile_core.flowfile.flow_node.flow_node import FlowNode
+from flowfile_core.schemas.analysis_schemas.graphic_walker_schemas import (
+    DataModel,
+    GraphicWalkerInput,
+    MutField,
+    ViewField,
+)
 from flowfile_core.schemas.input_schema import NodeExploreData
-from flowfile_core.schemas.analysis_schemas.graphic_walker_schemas import GraphicWalkerInput, DataModel, MutField, ViewField
-from flowfile_core.configs import logger
 
 
 class AnalyticsProcessor:
-
     @staticmethod
     def process_graphic_walker_input(node_step: FlowNode) -> NodeExploreData:
         node_explore_data: NodeExploreData = node_step.setting_input
-        if hasattr(node_explore_data, 'graphic_walker_input'):
+        if hasattr(node_explore_data, "graphic_walker_input"):
             graphic_walker_input = node_explore_data.graphic_walker_input
         else:
             logger.error("NodeExploreData is not an instance of GraphicWalkerInput.")
@@ -24,8 +25,9 @@ class AnalyticsProcessor:
         return node_explore_data
 
     @staticmethod
-    def create_graphic_walker_input(node_step: FlowNode,
-                                    graphic_walker_input: GraphicWalkerInput = None) -> GraphicWalkerInput:
+    def create_graphic_walker_input(
+        node_step: FlowNode, graphic_walker_input: GraphicWalkerInput = None
+    ) -> GraphicWalkerInput:
         if not node_step.results.analysis_data_generator:
             node_step.get_predicted_schema()
             fields = convert_ff_columns_to_gw_fields(node_step.get_predicted_schema())
@@ -42,17 +44,17 @@ class AnalyticsProcessor:
         return graphic_walker_input
 
 
-def check_if_field_in_spec_list_encodings(spec_list: Dict, field_name: str) -> bool:
-    for encoding in spec_list['encodings']['dimensions'].values():
-        if field_name in encoding['fid']:
+def check_if_field_in_spec_list_encodings(spec_list: dict, field_name: str) -> bool:
+    for encoding in spec_list["encodings"]["dimensions"].values():
+        if field_name in encoding["fid"]:
             return True
-    for encoding in spec_list['encodings']['measures'].values():
-        if field_name in encoding['fid']:
+    for encoding in spec_list["encodings"]["measures"].values():
+        if field_name in encoding["fid"]:
             return True
     return False
 
 
-def get_existing_encoding_fields(spec_list: Dict) -> Set[str]:
+def get_existing_encoding_fields(spec_list: dict) -> set[str]:
     """
     Get the existing encoding fields from the spec_list.
 
@@ -62,8 +64,8 @@ def get_existing_encoding_fields(spec_list: Dict) -> Set[str]:
     Returns:
         Set[str]: A set of existing encoding fields.
     """
-    dimensions = {encoding['fid'] for encoding in spec_list['encodings']['dimensions']}
-    measures = {encoding['fid'] for encoding in spec_list['encodings']['measures']}
+    dimensions = {encoding["fid"] for encoding in spec_list["encodings"]["dimensions"]}
+    measures = {encoding["fid"] for encoding in spec_list["encodings"]["measures"]}
     return dimensions.union(measures)
 
 
@@ -72,7 +74,7 @@ def transform_mut_field_to_view_field(mut_field: MutField) -> ViewField:
     return view_field
 
 
-def add_field_to_spec_list(spec_list: Dict, mut_field: MutField) -> None:
+def add_field_to_spec_list(spec_list: dict, mut_field: MutField) -> None:
     """
     Add a field to the spec_list.
 
@@ -84,13 +86,13 @@ def add_field_to_spec_list(spec_list: Dict, mut_field: MutField) -> None:
         None
     """
     view_field = transform_mut_field_to_view_field(mut_field)
-    if mut_field.analyticType == 'measure':
-        spec_list['encodings']['measures'].append(view_field.model_dump_dict())
+    if mut_field.analyticType == "measure":
+        spec_list["encodings"]["measures"].append(view_field.model_dump_dict())
     else:
-        spec_list['encodings']['dimensions'].append(view_field.model_dump_dict())
+        spec_list["encodings"]["dimensions"].append(view_field.model_dump_dict())
 
 
-def validate_spec_list_with_data_model_types(spec_list: Dict, data_model: DataModel) -> None:
+def validate_spec_list_with_data_model_types(spec_list: dict, data_model: DataModel) -> None:
     """
     Validate the spec_list with the data model types.
 
@@ -109,7 +111,7 @@ def validate_spec_list_with_data_model_types(spec_list: Dict, data_model: DataMo
             add_field_to_spec_list(spec_list, field)
 
 
-def validate_spec_lists_with_data_model(spec_lists: List[Dict], data_model: DataModel) -> None:
+def validate_spec_lists_with_data_model(spec_lists: list[dict], data_model: DataModel) -> None:
     """
     Validate the spec lists with the data model.
 

flowfile_core/flowfile/analytics/graphic_walker.py

@@ -1,23 +1,22 @@
 from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine, FlowfileColumn
 from flowfile_core.schemas.analysis_schemas import graphic_walker_schemas as gw_schema
-from typing import List
 
 
 def get_semantic_type(data_type: str) -> str:
     """Determine the semanticType based on the data_type."""
-    if data_type in ['Utf8', 'VARCHAR', 'CHAR', 'NVARCHAR', 'String']:
-        return 'nominal'
-    elif data_type in ['Int64', 'Float64', 'Int32', 'Float32', 'Int16', 'Float16', 'Decimal']:
-        return 'quantitative'
-    elif data_type in ['Datetime', 'Date']:
-        return 'temporal'
+    if data_type in ["Utf8", "VARCHAR", "CHAR", "NVARCHAR", "String"]:
+        return "nominal"
+    elif data_type in ["Int64", "Float64", "Int32", "Float32", "Int16", "Float16", "Decimal"]:
+        return "quantitative"
+    elif data_type in ["Datetime", "Date"]:
+        return "temporal"
     else:
-        return 'nominal'  # Default case; adjust as necessary
+        return "nominal"  # Default case; adjust as necessary
 
 
 def get_analytic_type(semantic_type: str) -> gw_schema.AnalyticTypeLit:
     """Determine the analyticType based on the semanticType."""
-    return 'measure' if semantic_type == 'quantitative' else 'dimension'
+    return "measure" if semantic_type == "quantitative" else "dimension"
 
 
 def convert_ff_column_to_gw_field(flow_file_column: FlowfileColumn) -> gw_schema.MutField:
@@ -42,11 +41,11 @@ def convert_ff_column_to_gw_field(flow_file_column: FlowfileColumn) -> gw_schema
         basename=flow_file_column.name,
         key=flow_file_column.name,
         semanticType=semantic_type,
-        analyticType=analytic_type
+        analyticType=analytic_type,
     )
 
 
-def convert_ff_columns_to_gw_fields(ff_columns: List[FlowfileColumn]) -> [gw_schema.MutField]:
+def convert_ff_columns_to_gw_fields(ff_columns: list[FlowfileColumn]) -> [gw_schema.MutField]:
     return [convert_ff_column_to_gw_field(ff_column) for ff_column in ff_columns]
 
 
@@ -55,6 +54,6 @@ def get_initial_gf_data_from_ff(flow_file: FlowDataEngine) -> gw_schema.DataMode
     return gw_schema.DataModel(fields=fields, data=[])
 
 
-def get_gf_data_from_ff(flow_file: FlowDataEngine, fields: List[gw_schema.MutField]) -> gw_schema.DataModel:
+def get_gf_data_from_ff(flow_file: FlowDataEngine, fields: list[gw_schema.MutField]) -> gw_schema.DataModel:
     data = flow_file.to_pylist()
     return gw_schema.DataModel(fields=fields, data=data)

flowfile_core/flowfile/analytics/utils.py

@@ -1,5 +1,5 @@
-from flowfile_core.schemas.input_schema import NodeExploreData, NodePromise
 from flowfile_core.schemas.analysis_schemas.graphic_walker_schemas import GraphicWalkerInput
+from flowfile_core.schemas.input_schema import NodeExploreData, NodePromise
 
 
 def create_graphic_walker_node_from_node_promise(node_promise: NodePromise) -> NodeExploreData: