Flowfile 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

flowfile_core/configs/settings.py

@@ -15,10 +15,10 @@ from flowfile_core.configs.utils import MutableBool
 DEFAULT_SERVER_HOST = "0.0.0.0"
 DEFAULT_SERVER_PORT = 63578
 DEFAULT_WORKER_PORT = 63579
-SINGLE_FILE_MODE: bool = os.environ.get("SINGLE_FILE_MODE", "0") == "1"
+SINGLE_FILE_MODE: bool = os.environ.get("FLOWFILE_SINGLE_FILE_MODE", "0") == "1"
 
 
-OFFLOAD_TO_WORKER = MutableBool(True)
+OFFLOAD_TO_WORKER = MutableBool(not SINGLE_FILE_MODE)
 
 
 def parse_args():
@@ -61,7 +61,7 @@ def get_default_worker_url(worker_port=None):
     worker_host = os.getenv("WORKER_HOST", None)
 
     if worker_port is None:
-        worker_port = os.getenv("WORKER_PORT", DEFAULT_WORKER_PORT)
+        worker_port = os.getenv("FLOWFILE_WORKER_PORT", DEFAULT_WORKER_PORT)
 
     # Convert to int if it's a string
     worker_port = int(worker_port) if isinstance(worker_port, str) else worker_port
@@ -81,14 +81,15 @@ args = parse_args()
 
 SERVER_HOST = args.host if args.host is not None else DEFAULT_SERVER_HOST
 SERVER_PORT = args.port if args.port is not None else DEFAULT_SERVER_PORT
-WORKER_PORT = args.worker_port if args.worker_port is not None else int(os.getenv("WORKER_PORT", DEFAULT_WORKER_PORT))
+WORKER_PORT = args.worker_port if args.worker_port is not None else int(os.getenv("FLOWFILE_WORKER_PORT",
+                                                                                  DEFAULT_WORKER_PORT))
 WORKER_HOST = os.getenv("WORKER_HOST", "0.0.0.0" if platform.system() != "Windows" else "127.0.0.1")
 
 config = Config(".env")
 DEBUG: bool = config("DEBUG", cast=bool, default=False)
 FILE_LOCATION = config("FILE_LOCATION", cast=str, default=".\\files\\")
 AVAILABLE_RAM = config("AVAILABLE_RAM", cast=int, default=8)
-WORKER_URL = config("WORKER_URL", cast=str, default=get_default_worker_url(WORKER_PORT))
+WORKER_URL = config("FLOWFILE_WORKER_URL", cast=str, default=get_default_worker_url(WORKER_PORT))
 IS_RUNNING_IN_DOCKER = os.getenv('RUNNING_IN_DOCKER', 'false').lower() == 'true'
 TEMP_DIR = get_temp_dir()
 

flowfile_core/database/connection.py

@@ -1,33 +1,69 @@
-
 from sqlalchemy import create_engine
 from contextlib import contextmanager
 from sqlalchemy.orm import sessionmaker
 import os
+import sys
+from pathlib import Path
+from flowfile_core.configs import logger
+
+
+def get_app_data_dir() -> Path:
+    """Get the appropriate application data directory for the current platform."""
+    app_name = "Flowfile"
+
+    if sys.platform == "win32":
+        # Windows: C:\Users\{username}\AppData\Local\flowfile
+        base_dir = os.environ.get("LOCALAPPDATA")
+        if not base_dir:
+            base_dir = os.path.join(os.path.expanduser("~"), "AppData", "Local")
+    elif sys.platform == "darwin":
+        # macOS: ~/Library/Application Support/flowfile
+        base_dir = os.path.join(os.path.expanduser("~"), "Library", "Application Support")
+    else:
+        # Linux: ~/.local/share/flowfile or use XDG_DATA_HOME
+        base_dir = os.environ.get("XDG_DATA_HOME")
+        if not base_dir:
+            base_dir = os.path.join(os.path.expanduser("~"), ".local", "share")
+
+    app_dir = Path(base_dir) / app_name
+
+    print(f"Using application data directory: {app_dir}")
+    app_dir.mkdir(parents=True, exist_ok=True)
+
+    return app_dir
 
 
 def get_database_url():
+    """Get the database URL based on the current environment."""
     if os.environ.get("TESTING") == "True":
-        # Use a file-based test database instead of in-memory
+        # Use a temporary test database
         test_db_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../test_flowfile.db")
         return f"sqlite:///{test_db_path}"
-    # elif os.environ.get("FLOWFILE_MODE") == "electron":
 
-    return "sqlite:///./flowfile.db"
+    custom_db_path = os.environ.get("FLOWFILE_DB_PATH")
+    if custom_db_path:
+        # logger.error("Using database URL:", os.environ.get("FLOWFILE_DB_URL"))
+        return f"sqlite:///{custom_db_path}"
+    # Use centralized location
+    app_dir = get_app_data_dir()
+
+    db_path = app_dir / "flowfile.db"
+    logger.info(f"Using database URL: sqlite:///{db_path}")
+    return f"sqlite:///{db_path}"
+
+
+def get_database_path() -> Path:
+    """Get the actual path to the database file (useful for backup/info purposes)."""
+    url = get_database_url()
+    if url.startswith("sqlite:///"):
+        return Path(url.replace("sqlite:///", ""))
+    return None
 
-    # else:
-    #     # Use PostgreSQL for Docker mode
-    #     host = os.environ.get("DB_HOST", "localhost")
-    #     port = os.environ.get("DB_PORT", "5432")
-    #     user = os.environ.get("DB_USER", "postgres")
-    #     password = os.environ.get("DB_PASSWORD", "postgres")
-    #     db = os.environ.get("DB_NAME", "flowfile")
-    #     return f"postgresql://{user}:{password}@{host}:{port}/{db}"
-    #
 
 # Create database engine
 engine = create_engine(
     get_database_url(),
-    connect_args={"check_same_thread": False} if os.environ.get("FLOWFILE_MODE") == "electron" else {}
+    connect_args={"check_same_thread": False} if "sqlite" in get_database_url() else {}
 )
 
 # Create session factory
@@ -35,6 +71,7 @@ SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
 
 def get_db():
+    """Dependency for FastAPI to get database session."""
     db = SessionLocal()
     try:
         yield db
@@ -44,8 +81,19 @@ def get_db():
 
 @contextmanager
 def get_db_context():
+    """Context manager for getting database session."""
     db = SessionLocal()
     try:
         yield db
     finally:
-        db.close()
+        db.close()
+
+
+def get_database_info():
+    """Get information about the current database configuration."""
+    return {
+        "url": get_database_url(),
+        "path": str(get_database_path()) if get_database_path() else None,
+        "app_data_dir": str(get_app_data_dir()),
+        "platform": sys.platform
+    }

flowfile_core/database/init_db.py

@@ -13,7 +13,6 @@ db_models.Base.metadata.create_all(bind=engine)
 
 def create_default_local_user(db: Session):
     local_user = db.query(db_models.User).filter(db_models.User.username == "local_user").first()
-
     if not local_user:
         random_password = ''.join(secrets.choice(string.ascii_letters + string.digits) for _ in range(32))
         hashed_password = pwd_context.hash(random_password)

flowfile_core/database/models.py

@@ -1,6 +1,6 @@
-
-from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, Text
+from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, Text, DateTime
 from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.sql import func
 
 Base = declarative_base()
 
@@ -39,3 +39,50 @@ class DatabaseConnection(Base):
     ssl_enabled = Column(Boolean, default=False)
     password_id = Column(Integer, ForeignKey("secrets.id"))
     user_id = Column(Integer, ForeignKey("users.id"))
+
+
+class CloudStorageConnection(Base):
+    __tablename__ = "cloud_storage_connections"
+
+    id = Column(Integer, primary_key=True, index=True)
+    connection_name = Column(String, index=True, nullable=False)
+    storage_type = Column(String, nullable=False)  # 's3', 'adls', 'gcs'
+    auth_method = Column(String, nullable=False)  # 'access_key', 'iam_role', etc.
+
+    # AWS S3 fields
+    aws_region = Column(String, nullable=True)
+    aws_access_key_id = Column(String, nullable=True)
+    aws_secret_access_key_id = Column(Integer, ForeignKey("secrets.id"), nullable=True)
+    aws_session_token_id = Column(Integer, ForeignKey("secrets.id"), nullable=True)
+    aws_role_arn = Column(String, nullable=True)
+    aws_allow_unsafe_html = Column(Boolean, nullable=True)
+
+    # Azure ADLS fields
+    azure_account_name = Column(String, nullable=True)
+    azure_account_key_id = Column(Integer, ForeignKey("secrets.id"), nullable=True)
+    azure_tenant_id = Column(String, nullable=True)
+    azure_client_id = Column(String, nullable=True)
+    azure_client_secret_id = Column(Integer, ForeignKey("secrets.id"), nullable=True)
+    azure_sas_token_id = Column(Integer, ForeignKey("secrets.id"), nullable=True)
+
+    # Common fields
+    endpoint_url = Column(String, nullable=True)
+    extra_config = Column(Text, nullable=True)  # JSON field for additional config
+    verify_ssl = Column(Boolean, default=True)
+
+    # Metadata
+    user_id = Column(Integer, ForeignKey("users.id"), nullable=False)
+    created_at = Column(DateTime, default=func.now(), nullable=False)
+    updated_at = Column(DateTime, default=func.now(), onupdate=func.now(), nullable=False)
+
+
+class CloudStoragePermission(Base):
+    __tablename__ = "cloud_storage_permissions"
+
+    id = Column(Integer, primary_key=True, index=True)
+    connection_id = Column(Integer, ForeignKey("cloud_storage_connections.id"), nullable=False)
+    resource_path = Column(String, nullable=False)  # e.g., "s3://bucket-name"
+    can_read = Column(Boolean, default=True)
+    can_write = Column(Boolean, default=False)
+    can_delete = Column(Boolean, default=False)
+    can_list = Column(Boolean, default=True)