Flowfile 0.2.2__py3-none-any.whl

flowfile_core/configs/node_store/nodes.py

@@ -0,0 +1,98 @@
+from typing import List
+from flowfile_core.schemas.schemas import NodeTemplate, NodeDefault, TransformTypeLiteral, NodeTypeLiteral
+
+nodes_list: List[NodeTemplate] = [
+    NodeTemplate(name='Read Airbyte', item='airbyte_reader', input=0, output=1, image='airbyte.png',
+                 node_group='input'),
+    NodeTemplate(name='Google sheets', item='google_sheet', input=0, output=1,
+                 image='google_sheet.png', node_group='input', prod_ready=False),
+    NodeTemplate(name='External source', item='external_source', input=0, output=1,
+                 image='external_source.png', node_group='input', prod_ready=False),
+    NodeTemplate(name='Manual input', item='manual_input', input=0, output=1,
+                 image='manual_input.png', node_group='input'),
+    NodeTemplate(name='Read data', item='read', input=0, output=1, image='input_data.png', node_group='input'),
+    NodeTemplate(name='Join', color='#49494970', item='join', input=2, output=1, image='join.png',
+                 node_group='combine'),
+    NodeTemplate(name='Formula', color='blue', item='formula', input=1, output=1, image='formula.png',
+                 node_group='transform'),
+    # NodeTemplate(name='Sql editor', color='blue', item='sql_editor', input=1, output=1, image='sql.png'),
+    NodeTemplate(name='Write data', item='output', input=1, output=0, image='output.png', node_group='output'),
+    NodeTemplate(name='Select data', item='select', input=1, output=1, image='select.png', node_group='transform'),
+    NodeTemplate(name='Filter data', item='filter', input=1, output=1, image='filter.png', node_group='transform'),
+    NodeTemplate(name='Group by', item='group_by', input=1, output=1, image='group_by.png', node_group='aggregate'),
+    NodeTemplate(name='Fuzzy match', item='fuzzy_match', input=2, output=1, image='fuzzy_match.png',
+                 node_group='combine'),
+    NodeTemplate(name='Sort data', item='sort', input=1, output=1, image='sort.png', node_group='transform'),
+    NodeTemplate(name='Add record Id', item='record_id', input=1, output=1, image='record_id.png',
+                 node_group='transform'),
+    NodeTemplate(name='Take Sample', item='sample', input=1, output=1, image='sample.png', node_group='transform'),
+    NodeTemplate(name='Explore data', item='explore_data', input=1, output=0,
+                 image='explore_data.png', node_group='output'),
+    NodeTemplate(name='Pivot data', item='pivot', input=1, output=1, image='pivot.png', node_group='aggregate'),
+    NodeTemplate(name='Unpivot data', item='unpivot', input=1, output=1, image='unpivot.png', node_group='aggregate'),
+    NodeTemplate(name='Union data', item='union', input=10, output=1, image='union.png', multi=True,
+                 node_group='combine'),
+    NodeTemplate(name='Drop duplicates', item='unique', input=1, output=1, image='unique.png', node_group='transform'),
+    NodeTemplate(name='Graph solver', item='graph_solver', input=1, output=1, image='graph_solver.png',
+                 node_group='combine'),
+    NodeTemplate(name='Count records', item='record_count', input=1, output=1, image='record_count.png',
+                 node_group='aggregate'),
+    NodeTemplate(name='Cross join', item='cross_join', input=2, output=1, image='cross_join.png', node_group='combine'),
+    NodeTemplate(name='Text to rows', item='text_to_rows', input=1, output=1, image='text_to_rows.png',
+                 node_group='transform'),
+    NodeTemplate(name="Polars code", item="polars_code", input=10, output=1, image='polars_code.png',
+                 node_group='transform', multi=True, can_be_start=True),
+    NodeTemplate(name="Read from Database", item="database_reader", input=0, output=1, image='database_reader.svg',
+                 node_group='input'),
+    NodeTemplate(name='Write to Database', item='database_writer', input=1, output=0, image='database_writer.svg',
+                 node_group='output'),
+]
+
+nodes_list.sort(key=lambda x: x.name)
+
+output = ['Explore data', 'Write data', 'Write to Database']
+_input = ['Read Airbyte', 'Google sheets', 'Manual input', 'Read data', 'External source', 'Read from Database']
+transform = ['Join', 'Formula', 'Select data', 'Filter data', 'Group by', 'Fuzzy match', 'Sort data', 'Add record Id',
+             'Take Sample', 'Pivot data', 'Unpivot data', 'Union data', 'Drop duplicates', 'Graph solver',
+             'Count records', 'Cross join', 'Text to rows', 'Polars code']
+narrow = ['Select data', 'Filter data', 'Take Sample', 'Formula', 'Read data', 'Union data', 'Polars code']
+wide = ['Join', 'Group by', 'Fuzzy match', 'Sort data', 'Pivot data', 'Unpivot data', 'Add record Id',
+        'Graph solver', 'Drop duplicates', 'Count records', 'Cross join', 'Text to rows']
+other = ['Explore data', 'Write data', 'Read Airbyte', 'Google sheets', 'Manual input', 'Read data', 'External source',
+         'Read from Database', 'Write to Database']
+nodes_with_defaults = {'sample', 'sort', 'union', 'select', 'record_count'}
+
+
+def get_node_type(node_name: str) -> NodeTypeLiteral:
+    if node_name in output:
+        return 'output'
+    if node_name in _input:
+        return 'input'
+    if node_name in transform:
+        return 'process'
+    else:
+        raise ValueError(f'Node name {node_name} not found in any of the node types')
+
+
+def check_if_has_default_setting(node_item: str):
+    return node_item in nodes_with_defaults
+
+
+def get_transform_type(node_name: str) -> TransformTypeLiteral:
+    if node_name in narrow:
+        return 'narrow'
+    if node_name in wide:
+        return 'wide'
+    if node_name in other:
+        return 'other'
+    else:
+        raise ValueError(f'Node name {node_name} not found in any of the transform types')
+
+
+node_defaults = {node.item: NodeDefault(node_name=node.name,
+                                        node_type=get_node_type(node.name),
+                                        transform_type=get_transform_type(node.name),
+                                        has_default_settings=check_if_has_default_setting(node.item)
+                                        ) for node in nodes_list}
+
+node_dict = {n.item: n for n in nodes_list}

flowfile_core/configs/settings.py

@@ -0,0 +1,120 @@
+
+# flowfile_core/flowfile_core/configs/settings.py
+import platform
+import os
+import tempfile
+import argparse
+
+from databases import DatabaseURL
+from passlib.context import CryptContext
+from starlette.config import Config
+from starlette.datastructures import Secret
+
+
+# Constants for server and worker configuration
+DEFAULT_SERVER_HOST = "0.0.0.0"
+DEFAULT_SERVER_PORT = 63578
+DEFAULT_WORKER_PORT = 63579
+
+
+def parse_args():
+    """Parse command line arguments"""
+    parser = argparse.ArgumentParser(description="Flowfile Backend Server")
+    parser.add_argument(
+        "--host", type=str, default=DEFAULT_SERVER_HOST, help="Host to bind to"
+    )
+    parser.add_argument(
+        "--port", type=int, default=DEFAULT_SERVER_PORT, help="Port to bind to"
+    )
+    parser.add_argument(
+        "--worker-port",
+        type=int,
+        default=DEFAULT_WORKER_PORT,
+        help="Port for the worker process",
+    )
+
+    # Use known_args to handle PyInstaller's extra args
+    args = parser.parse_known_args()[0]
+
+    # Validate arguments
+    if args.port < 1 or args.port > 65535:
+        raise ValueError(
+            f"Invalid port number: {args.port}. Port must be between 1 and 65535."
+        )
+
+    if args.worker_port < 1 or args.worker_port > 65535:
+        raise ValueError(
+            f"Invalid worker port number: {args.worker_port}. Port must be between 1 and 65535."
+        )
+
+    # Check if host is valid (basic check)
+    if not args.host:
+        raise ValueError("Host cannot be empty")
+
+    # Check that server and worker ports are different
+    if args.port == args.worker_port:
+        raise ValueError(
+            f"Server port ({args.port}) and worker port ({args.worker_port}) must be different"
+        )
+
+    return args
+
+
+def get_temp_dir() -> str:
+    """Get the appropriate temp directory path based on environment"""
+    # Check for Docker environment variable first
+    docker_temp = os.getenv("TEMP_DIR")
+    if docker_temp:
+        return docker_temp
+
+    return tempfile.gettempdir()
+
+
+def get_default_worker_url(worker_port=None):
+    """
+    Get the default worker URL based on environment and settings
+
+    Args:
+        worker_port: Optional port override (used when passed as command line arg)
+    """
+    # Check for Docker environment first
+    worker_host = os.getenv("WORKER_HOST", None)
+
+    # Use the provided port if available, otherwise get from env or default
+    if worker_port is None:
+        worker_port = os.getenv("WORKER_PORT", DEFAULT_WORKER_PORT)
+
+    # Convert to int if it's a string
+    worker_port = int(worker_port) if isinstance(worker_port, str) else worker_port
+
+    if worker_host:
+        return f"http://{worker_host}:{worker_port}"
+
+    if platform.system() == "Windows":
+        return f"http://127.0.0.1:{worker_port}"
+    else:
+        return f"http://0.0.0.0:{worker_port}"
+
+
+# Get server arguments
+args = parse_args()
+
+SERVER_HOST = args.host if args.host is not None else DEFAULT_SERVER_HOST
+SERVER_PORT = args.port if args.port is not None else DEFAULT_SERVER_PORT
+WORKER_PORT = args.worker_port if args.worker_port is not None else int(os.getenv("WORKER_PORT", DEFAULT_WORKER_PORT))
+
+
+# Worker configuration
+WORKER_HOST = os.getenv("WORKER_HOST", "0.0.0.0" if platform.system() != "Windows" else "127.0.0.1")
+
+config = Config(".env")
+DEBUG: bool = config("DEBUG", cast=bool, default=False)
+FILE_LOCATION = config("FILE_LOCATION", cast=str, default=".\\files\\")
+AVAILABLE_RAM = config("AVAILABLE_RAM", cast=int, default=8)
+WORKER_URL = config("WORKER_URL", cast=str, default=get_default_worker_url(WORKER_PORT))
+IS_RUNNING_IN_DOCKER = os.getenv('RUNNING_IN_DOCKER', 'false').lower() == 'true'
+TEMP_DIR = get_temp_dir()
+
+ALGORITHM = "HS256"
+ACCESS_TOKEN_EXPIRE_MINUTES = 120
+PWD_CONTEXT = CryptContext(schemes=["bcrypt"], deprecated="auto")

flowfile_core/database/connection.py

@@ -0,0 +1,51 @@
+
+from sqlalchemy import create_engine
+from contextlib import contextmanager
+from sqlalchemy.orm import sessionmaker
+import os
+
+
+def get_database_url():
+    if os.environ.get("TESTING") == "True":
+        # Use a file-based test database instead of in-memory
+        test_db_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../test_flowfile.db")
+        return f"sqlite:///{test_db_path}"
+    # elif os.environ.get("FLOWFILE_MODE") == "electron":
+
+    return "sqlite:///./flowfile.db"
+
+    # else:
+    #     # Use PostgreSQL for Docker mode
+    #     host = os.environ.get("DB_HOST", "localhost")
+    #     port = os.environ.get("DB_PORT", "5432")
+    #     user = os.environ.get("DB_USER", "postgres")
+    #     password = os.environ.get("DB_PASSWORD", "postgres")
+    #     db = os.environ.get("DB_NAME", "flowfile")
+    #     return f"postgresql://{user}:{password}@{host}:{port}/{db}"
+    #
+
+# Create database engine
+engine = create_engine(
+    get_database_url(),
+    connect_args={"check_same_thread": False} if os.environ.get("FLOWFILE_MODE") == "electron" else {}
+)
+
+# Create session factory
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()
+
+
+@contextmanager
+def get_db_context():
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()

flowfile_core/database/init_db.py

@@ -0,0 +1,45 @@
+# Generate a random secure password and hash it
+import secrets
+import string
+from sqlalchemy.orm import Session
+from flowfile_core.database import models as db_models
+from flowfile_core.database.connection import engine, SessionLocal
+
+from passlib.context import CryptContext
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+db_models.Base.metadata.create_all(bind=engine)
+
+
+def create_default_local_user(db: Session):
+    local_user = db.query(db_models.User).filter(db_models.User.username == "local_user").first()
+
+    if not local_user:
+        random_password = ''.join(secrets.choice(string.ascii_letters + string.digits) for _ in range(32))
+        hashed_password = pwd_context.hash(random_password)
+
+        local_user = db_models.User(
+            username="local_user",
+            email="local@flowfile.app",
+            full_name="Local User",
+            hashed_password=hashed_password
+        )
+        db.add(local_user)
+        db.commit()
+        return True
+    else:
+        return False
+
+
+def init_db():
+    db = SessionLocal()
+    try:
+        create_default_local_user(db)
+    finally:
+        db.close()
+
+
+if __name__ == "__main__":
+    init_db()
+    print("Local user created successfully")
+

flowfile_core/database/models.py

@@ -0,0 +1,41 @@
+
+from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, Text
+from sqlalchemy.ext.declarative import declarative_base
+
+Base = declarative_base()
+
+
+class User(Base):
+    __tablename__ = "users"
+
+    id = Column(Integer, primary_key=True, index=True)
+    username = Column(String, unique=True, index=True)
+    email = Column(String, unique=True, index=True)
+    full_name = Column(String)
+    hashed_password = Column(String)
+    disabled = Column(Boolean, default=False)
+
+
+class Secret(Base):
+    __tablename__ = "secrets"
+
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String, index=True)
+    encrypted_value = Column(Text)
+    iv = Column(String)
+    user_id = Column(Integer, ForeignKey("users.id"))
+
+
+class DatabaseConnection(Base):
+    __tablename__ = "database_connections"
+
+    id = Column(Integer, primary_key=True, index=True)
+    connection_name = Column(String, index=True)
+    database_type = Column(String)
+    username = Column(String)
+    host = Column(String)
+    port = Column(Integer)
+    database = Column(String, default=None)
+    ssl_enabled = Column(Boolean, default=False)
+    password_id = Column(Integer, ForeignKey("secrets.id"))
+    user_id = Column(Integer, ForeignKey("users.id"))

flowfile_core/fileExplorer/funcs.py

@@ -0,0 +1,259 @@
+from pathlib import Path
+from typing import List, Optional, Set, Union
+from datetime import datetime
+from pydantic import BaseModel
+from typing_extensions import Literal
+import os
+
+
+class FileInfo(BaseModel):
+    """Comprehensive information about a file or directory."""
+    name: str
+    path: str
+    is_directory: bool
+    size: int
+    file_type: str
+    last_modified: datetime
+    created_date: datetime
+    is_hidden: bool
+    exists: bool = True
+
+    @classmethod
+    def from_path(cls, path: Path) -> 'FileInfo':
+        """Create FileInfo instance from a path."""
+        try:
+            stats = path.stat()
+            return cls(
+                name=path.name,
+                path=str(path.absolute()),
+                is_directory=path.is_dir(),
+                size=stats.st_size,
+                file_type=path.suffix[1:] if path.suffix else "",
+                last_modified=datetime.fromtimestamp(stats.st_mtime),
+                created_date=datetime.fromtimestamp(stats.st_ctime),
+                is_hidden=path.name.startswith('.') or (
+                        os.name == 'nt' and bool(stats.st_file_attributes & 0x2)
+                ),
+                exists=True
+            )
+        except (PermissionError, OSError):
+            return cls(
+                name=path.name,
+                path=str(path.absolute()),
+                is_directory=False,
+                size=0,
+                file_type="",
+                last_modified=datetime.fromtimestamp(0),
+                created_date=datetime.fromtimestamp(0),
+                is_hidden=False,
+                exists=False
+            )
+
+
+class FileExplorer:
+    def __init__(self, start_path: Optional[str|Path] = None):
+        """Initialize FileExplorer with user's home directory or specified path."""
+        if start_path is None:
+            self.current_path = Path.home()
+        else:
+            self.current_path = Path(start_path).expanduser().resolve()
+
+        if not self.current_path.exists():
+            raise ValueError(f"Path does not exist: {self.current_path}")
+
+        if not self.current_path.is_dir():
+            raise ValueError(f"Path is not a directory: {self.current_path}")
+
+    @property
+    def current_directory(self) -> str:
+        """Get the current directory path."""
+        return str(self.current_path.absolute())
+
+    @property
+    def parent_directory(self) -> Optional[str]:
+        """Get the parent directory path if it exists."""
+        parent = self.current_path.parent
+        return str(parent.absolute()) if parent != self.current_path else None
+
+    def list_contents(
+            self,
+            *,
+            show_hidden: bool = False,
+            file_types: Optional[List[str]] = None,
+            recursive: bool = False,
+            min_size: Optional[int] = None,
+            max_size: Optional[int] = None,
+            sort_by: Literal['name', 'date', 'size', 'type'] = 'name',
+            reverse: bool = False,
+            exclude_patterns: Optional[List[str]] = None
+    ) -> List[FileInfo]:
+        """
+        List contents of the current directory with advanced filtering and sorting.
+
+        Args:
+            show_hidden: Whether to show hidden files and directories
+            file_types: List of file extensions to include (without dots)
+            recursive: Whether to scan subdirectories
+            min_size: Minimum file size in bytes
+            max_size: Maximum file size in bytes
+            sort_by: Field to sort results by
+            reverse: Whether to reverse sort order
+            exclude_patterns: Glob patterns to exclude
+
+        Returns:
+            List of FileInfo objects sorted according to parameters
+        """
+        contents: List[FileInfo] = []
+        excluded_paths: Set[str] = set()
+
+        if exclude_patterns:
+            for pattern in exclude_patterns:
+                excluded_paths.update(str(p) for p in self.current_path.glob(pattern))
+
+        def should_include(info: FileInfo) -> bool:
+            """Determine if a file should be included based on filters."""
+            if str(info.path) in excluded_paths:
+                return False
+            if not show_hidden and info.is_hidden:
+                return False
+            if min_size is not None and info.size < min_size:
+                return False
+            if max_size is not None and info.size > max_size:
+                return False
+            if file_types and not info.is_directory:
+                return info.file_type.lower() in (t.lower() for t in file_types)
+            return True
+
+        try:
+            # Define the scan pattern based on recursion
+            pattern = '**/*' if recursive else '*'
+
+            # Scan directory
+            for item in self.current_path.glob(pattern):
+                try:
+                    file_info = FileInfo.from_path(item)
+                    if should_include(file_info):
+                        contents.append(file_info)
+                except (PermissionError, OSError):
+                    continue
+
+        except PermissionError:
+            raise PermissionError(f"Permission denied to access directory: {self.current_path}")
+
+        # Sort results
+        sort_key = {
+            'name': lambda x: (not x.is_directory, x.name.lower()),
+            'date': lambda x: (not x.is_directory, x.last_modified),
+            'size': lambda x: (not x.is_directory, x.size),
+            'type': lambda x: (not x.is_directory, x.file_type.lower(), x.name.lower())
+        }[sort_by]
+
+        return sorted(contents, key=sort_key, reverse=reverse)
+
+    def navigate_to(self, path: str) -> bool:
+        """
+        Navigate to a new directory path.
+        Returns True if navigation was successful, False otherwise.
+        """
+        new_path = None
+        try:
+            new_path = Path(path).expanduser().resolve()
+
+            if not new_path.exists() or not new_path.is_dir():
+                return False
+
+            # Test if we can actually read the directory
+            next(new_path.iterdir(), None)
+            self.current_path = new_path
+            return True
+
+        except PermissionError:
+            if new_path:
+                self.current_path = new_path
+            return True
+        except OSError:
+            return False
+
+    def navigate_up(self) -> bool:
+        """
+        Navigate up to the parent directory.
+        Returns True if navigation was successful, False otherwise.
+        """
+        parent = self.parent_directory
+        if parent is None:
+            return False
+        return self.navigate_to(parent)
+
+    def navigate_into(self, directory_name: str) -> bool:
+        """
+        Navigate into a subdirectory of the current directory.
+        Returns True if navigation was successful, False otherwise.
+        """
+        new_path = self.current_path / directory_name
+        return self.navigate_to(str(new_path))
+
+
+def get_files_from_directory(
+        dir_name: Union[str, Path],
+        types: Optional[List[str]] = None,
+        *,
+        include_hidden: bool = False,
+        recursive: bool = False
+) -> Optional[List[FileInfo]]:
+    """
+    Get list of files from a directory with optional type filtering.
+
+    Args:
+        dir_name: Directory path to scan
+        types: List of file extensions to include (without dots). None means all types
+        include_hidden: Whether to include hidden files
+        recursive: Whether to scan subdirectories
+
+    Returns:
+        List of FileInfo objects or None if directory doesn't exist
+
+    Example:
+        >>> files = get_files_from_directory("/path/to/dir", types=["pdf", "txt"])
+        >>> for file in files:
+        ...     print(f"{file.name} - {file.size} bytes")
+    """
+    try:
+        dir_path = Path(dir_name).resolve()
+        if not dir_path.exists():
+            return None
+        if not dir_path.is_dir():
+            raise ValueError(f"Path is not a directory: {dir_path}")
+
+        # Normalize file types
+        if types:
+            types = [t.lower().lstrip('.') for t in types]
+
+        files = []
+        pattern = '**/*' if recursive else '*'
+
+        for item in dir_path.glob(pattern):
+            try:
+                # Skip hidden files unless specifically requested
+                if not include_hidden and item.name.startswith('.'):
+                    continue
+
+                # Skip directories unless recursive is True
+                if item.is_dir() and not recursive:
+                    continue
+
+                # Check file type if types are specified
+                if types and not item.is_dir():
+                    if item.suffix[1:].lower() not in types:
+                        continue
+
+                file_info = FileInfo.from_path(item)
+                files.append(file_info)
+
+            except (PermissionError, OSError):
+                continue
+
+        return sorted(files, key=lambda x: (not x.is_directory, x.name.lower()))
+
+    except Exception as e:
+        raise type(e)(f"Error scanning directory {dir_name}: {str(e)}") from e
+

flowfile_core/fileExplorer/utils.py

@@ -0,0 +1,53 @@
+import os
+import re
+
+video_types = ['mp4', "webm", "opgg"]
+audio_types = ['mp3', "wav", "ogg", "mpeg", "aac", "3gpp", "3gpp2", "aiff", "x-aiff", "amr", "mpga"]
+
+
+def get_chunk(start_byte=None, end_byte=None, full_path=None):
+    file_size = os.stat(full_path).st_size
+    if end_byte:
+        length = end_byte + 1 - start_byte
+    else:
+        length = file_size - start_byte
+    with open(full_path, 'rb') as f:
+        f.seek(start_byte)
+        chunk = f.read(length)
+    return chunk, start_byte, length, file_size
+
+
+def get_file(file_path, mimetype):
+    f = ""
+    range_header = f.headers.get('Range', None)
+    start_byte, end_byte = 0, None
+    if range_header:
+        match = re.search(r'(\d+)-(\d*)', range_header)
+        groups = match.groups()
+        if groups[0]:
+            start_byte = int(groups[0])
+        if groups[1]:
+            end_byte = int(groups[1])
+
+    chunk, start, length, file_size = get_chunk(start_byte, end_byte, file_path)
+    resp = Response(chunk, 206, mimetype=f'video/{mimetype}',
+                    content_type=mimetype, direct_passthrough=True)
+    print(length)
+    resp.headers.add('Content-Range', 'bytes {0}-{1}/{2}'.format(start, start + length - 1, file_size))
+    return resp
+
+
+def is_media(filepath):
+    found_media = re.search("\.mp4$|\.mp3$", filepath, re.IGNORECASE)
+    if found_media:
+        extension = found_media[0].lower()[1:]
+        if found_media in video_types:
+            return f"video/{extension}"
+        return f"audio/{extension}"
+    return False
+
+
+def get_file_extension(fname):
+    found_extension = re.search("\.[A-Za-z0-9]*$", fname, re.IGNORECASE)
+    if found_extension:
+        return found_extension[0][1:].lower()