Flowfile 0.3.9__py3-none-any.whl → 0.3.10__py3-none-any.whl

flowfile_core/configs/node_store/nodes.py

@@ -1,51 +1,310 @@
 from typing import List
 from flowfile_core.schemas.schemas import NodeTemplate, NodeDefault, TransformTypeLiteral, NodeTypeLiteral
 
+from typing import List
+from flowfile_core.schemas.schemas import NodeTemplate, NodeDefault, TransformTypeLiteral, NodeTypeLiteral
+
 nodes_list: List[NodeTemplate] = [
-    NodeTemplate(name='External source', item='external_source', input=0, output=1,
-                 image='external_source.png', node_group='input', prod_ready=False),
-    NodeTemplate(name='Manual input', item='manual_input', input=0, output=1,
-                 image='manual_input.png', node_group='input'),
-    NodeTemplate(name='Read data', item='read', input=0, output=1, image='input_data.png', node_group='input'),
-    NodeTemplate(name='Join', color='#49494970', item='join', input=2, output=1, image='join.png',
-                 node_group='combine'),
-    NodeTemplate(name='Formula', color='blue', item='formula', input=1, output=1, image='formula.png',
-                 node_group='transform'),
-    # NodeTemplate(name='Sql editor', color='blue', item='sql_editor', input=1, output=1, image='sql.png'),
-    NodeTemplate(name='Write data', item='output', input=1, output=0, image='output.png', node_group='output'),
-    NodeTemplate(name='Select data', item='select', input=1, output=1, image='select.png', node_group='transform'),
-    NodeTemplate(name='Filter data', item='filter', input=1, output=1, image='filter.png', node_group='transform'),
-    NodeTemplate(name='Group by', item='group_by', input=1, output=1, image='group_by.png', node_group='aggregate'),
-    NodeTemplate(name='Fuzzy match', item='fuzzy_match', input=2, output=1, image='fuzzy_match.png',
-                 node_group='combine'),
-    NodeTemplate(name='Sort data', item='sort', input=1, output=1, image='sort.png', node_group='transform'),
-    NodeTemplate(name='Add record Id', item='record_id', input=1, output=1, image='record_id.png',
-                 node_group='transform'),
-    NodeTemplate(name='Take Sample', item='sample', input=1, output=1, image='sample.png', node_group='transform'),
-    NodeTemplate(name='Explore data', item='explore_data', input=1, output=0,
-                 image='explore_data.png', node_group='output'),
-    NodeTemplate(name='Pivot data', item='pivot', input=1, output=1, image='pivot.png', node_group='aggregate'),
-    NodeTemplate(name='Unpivot data', item='unpivot', input=1, output=1, image='unpivot.png', node_group='aggregate'),
-    NodeTemplate(name='Union data', item='union', input=10, output=1, image='union.png', multi=True,
-                 node_group='combine'),
-    NodeTemplate(name='Drop duplicates', item='unique', input=1, output=1, image='unique.png', node_group='transform'),
-    NodeTemplate(name='Graph solver', item='graph_solver', input=1, output=1, image='graph_solver.png',
-                 node_group='combine'),
-    NodeTemplate(name='Count records', item='record_count', input=1, output=1, image='record_count.png',
-                 node_group='aggregate'),
-    NodeTemplate(name='Cross join', item='cross_join', input=2, output=1, image='cross_join.png', node_group='combine'),
-    NodeTemplate(name='Text to rows', item='text_to_rows', input=1, output=1, image='text_to_rows.png',
-                 node_group='transform'),
-    NodeTemplate(name="Polars code", item="polars_code", input=10, output=1, image='polars_code.png',
-                 node_group='transform', multi=True, can_be_start=True),
-    NodeTemplate(name="Read from Database", item="database_reader", input=0, output=1, image='database_reader.svg',
-                 node_group='input'),
-    NodeTemplate(name='Write to Database', item='database_writer', input=1, output=0, image='database_writer.svg',
-                 node_group='output'),
-    NodeTemplate(name='Read from cloud provider', item='cloud_storage_reader', input=0, output=1,
-                 image='cloud_storage_reader.png', node_group='input'),
-    NodeTemplate(name='Write to cloud provider', item='cloud_storage_writer', input=1, output=0,
-                image='cloud_storage_writer.png', node_group='output'),
+    NodeTemplate(
+        name='External source',
+        item='external_source',
+        input=0,
+        output=1,
+        image='external_source.png',
+        node_group='input',
+        prod_ready=False,
+        drawer_title="External Source",
+        drawer_intro="Connect to external data sources and APIs"
+    ),
+
+    NodeTemplate(
+        name='Manual input',
+        item='manual_input',
+        input=0,
+        output=1,
+        image='manual_input.png',
+        node_group='input',
+        drawer_title="Manual Input",
+        drawer_intro="Create data directly"
+    ),
+
+    NodeTemplate(
+        name='Read data',
+        item='read',
+        input=0,
+        output=1,
+        image='input_data.png',
+        node_group='input',
+        drawer_title="Read Data",
+        drawer_intro="Load data from CSV, Excel, or Parquet files"
+    ),
+
+    NodeTemplate(
+        name='Join',
+        item='join',
+        input=2,
+        output=1,
+        image='join.png',
+        node_group='combine',
+        drawer_title="Join Datasets",
+        drawer_intro="Merge two datasets based on matching column values"
+    ),
+
+    NodeTemplate(
+        name='Formula',
+        item='formula',
+        input=1,
+        output=1,
+        image='formula.png',
+        node_group='transform',
+        drawer_title="Formula Editor",
+        drawer_intro="Create or modify columns using custom expressions"
+    ),
+
+    NodeTemplate(
+        name='Write data',
+        item='output',
+        input=1,
+        output=0,
+        image='output.png',
+        node_group='output',
+        drawer_title="Write Data",
+        drawer_intro="Save your data as CSV, Excel, or Parquet files"
+    ),
+
+    NodeTemplate(
+        name='Select data',
+        item='select',
+        input=1,
+        output=1,
+        image='select.png',
+        node_group='transform',
+        drawer_title="Select Columns",
+        drawer_intro="Choose, rename, and reorder columns to keep"
+    ),
+
+    NodeTemplate(
+        name='Filter data',
+        item='filter',
+        input=1,
+        output=1,
+        image='filter.png',
+        node_group='transform',
+        drawer_title="Filter Rows",
+        drawer_intro="Keep only rows that match your conditions"
+    ),
+
+    NodeTemplate(
+        name='Group by',
+        item='group_by',
+        input=1,
+        output=1,
+        image='group_by.png',
+        node_group='aggregate',
+        drawer_title="Group By",
+        drawer_intro="Aggregate data by grouping and calculating statistics"
+    ),
+
+    NodeTemplate(
+        name='Fuzzy match',
+        item='fuzzy_match',
+        input=2,
+        output=1,
+        image='fuzzy_match.png',
+        node_group='combine',
+        drawer_title="Fuzzy Match",
+        drawer_intro="Join datasets based on similar values instead of exact matches"
+    ),
+
+    NodeTemplate(
+        name='Sort data',
+        item='sort',
+        input=1,
+        output=1,
+        image='sort.png',
+        node_group='transform',
+        drawer_title="Sort Data",
+        drawer_intro="Order your data by one or more columns"
+    ),
+
+    NodeTemplate(
+        name='Add record Id',
+        item='record_id',
+        input=1,
+        output=1,
+        image='record_id.png',
+        node_group='transform',
+        drawer_title="Add Record ID",
+        drawer_intro="Generate unique identifiers for each row"
+    ),
+
+    NodeTemplate(
+        name='Take Sample',
+        item='sample',
+        input=1,
+        output=1,
+        image='sample.png',
+        node_group='transform',
+        drawer_title="Take Sample",
+        drawer_intro="Work with a subset of your data"
+    ),
+
+    NodeTemplate(
+        name='Explore data',
+        item='explore_data',
+        input=1,
+        output=0,
+        image='explore_data.png',
+        node_group='output',
+        drawer_title="Explore Data",
+        drawer_intro="Interactive data exploration and analysis"
+    ),
+
+    NodeTemplate(
+        name='Pivot data',
+        item='pivot',
+        input=1,
+        output=1,
+        image='pivot.png',
+        node_group='aggregate',
+        drawer_title="Pivot Data",
+        drawer_intro="Convert data from long format to wide format"
+    ),
+
+    NodeTemplate(
+        name='Unpivot data',
+        item='unpivot',
+        input=1,
+        output=1,
+        image='unpivot.png',
+        node_group='aggregate',
+        drawer_title="Unpivot Data",
+        drawer_intro="Transform data from wide format to long format"
+    ),
+
+    NodeTemplate(
+        name='Union data',
+        item='union',
+        input=10,
+        output=1,
+        image='union.png',
+        multi=True,
+        node_group='combine',
+        drawer_title="Union Data",
+        drawer_intro="Stack multiple datasets by combining rows"
+    ),
+
+    NodeTemplate(
+        name='Drop duplicates',
+        item='unique',
+        input=1,
+        output=1,
+        image='unique.png',
+        node_group='transform',
+        drawer_title="Drop Duplicates",
+        drawer_intro="Remove duplicate rows based on selected columns"
+    ),
+
+    NodeTemplate(
+        name='Graph solver',
+        item='graph_solver',
+        input=1,
+        output=1,
+        image='graph_solver.png',
+        node_group='combine',
+        drawer_title="Graph Solver",
+        drawer_intro="Group related records in graph-structured data"
+    ),
+
+    NodeTemplate(
+        name='Count records',
+        item='record_count',
+        input=1,
+        output=1,
+        image='record_count.png',
+        node_group='aggregate',
+        drawer_title="Count Records",
+        drawer_intro="Calculate the total number of rows"
+    ),
+
+    NodeTemplate(
+        name='Cross join',
+        item='cross_join',
+        input=2,
+        output=1,
+        image='cross_join.png',
+        node_group='combine',
+        drawer_title="Cross Join",
+        drawer_intro="Create all possible combinations between two datasets"
+    ),
+
+    NodeTemplate(
+        name='Text to rows',
+        item='text_to_rows',
+        input=1,
+        output=1,
+        image='text_to_rows.png',
+        node_group='transform',
+        drawer_title="Text to Rows",
+        drawer_intro="Split text into multiple rows based on a delimiter"
+    ),
+
+    NodeTemplate(
+        name="Polars code",
+        item="polars_code",
+        input=10,
+        output=1,
+        image='polars_code.png',
+        node_group='transform',
+        multi=True,
+        can_be_start=True,
+        drawer_title="Polars Code",
+        drawer_intro="Write custom Polars DataFrame transformations"
+    ),
+
+    NodeTemplate(
+        name="Read from Database",
+        item="database_reader",
+        input=0,
+        output=1,
+        image='database_reader.svg',
+        node_group='input',
+        drawer_title="Database Reader",
+        drawer_intro="Load data from database tables or queries"
+    ),
+
+    NodeTemplate(
+        name='Write to Database',
+        item='database_writer',
+        input=1,
+        output=0,
+        image='database_writer.svg',
+        node_group='output',
+        drawer_title="Database Writer",
+        drawer_intro="Save data to database tables"
+    ),
+
+    NodeTemplate(
+        name='Read from cloud provider',
+        item='cloud_storage_reader',
+        input=0,
+        output=1,
+        image='cloud_storage_reader.png',
+        node_group='input',
+        drawer_title="Cloud Storage Reader",
+        drawer_intro="Read data from AWS S3 and other cloud storage"
+    ),
+
+    NodeTemplate(
+        name='Write to cloud provider',
+        item='cloud_storage_writer',
+        input=1,
+        output=0,
+        image='cloud_storage_writer.png',
+        node_group='output',
+        drawer_title="Cloud Storage Writer",
+        drawer_intro="Save data to AWS S3 and other cloud storage"
+    ),
 ]
 
 nodes_list.sort(key=lambda x: x.name)

flowfile_core/configs/settings.py

@@ -7,6 +7,7 @@ import argparse
 
 from passlib.context import CryptContext
 from starlette.config import Config
+from shared.storage_config import storage
 
 from flowfile_core.configs.utils import MutableBool
 
@@ -93,7 +94,7 @@ FILE_LOCATION = config("FILE_LOCATION", cast=str, default=".\\files\\")
 AVAILABLE_RAM = config("AVAILABLE_RAM", cast=int, default=8)
 WORKER_URL = config("FLOWFILE_WORKER_URL", cast=str, default=get_default_worker_url(WORKER_PORT))
 IS_RUNNING_IN_DOCKER = os.getenv('RUNNING_IN_DOCKER', 'false').lower() == 'true'
-TEMP_DIR = get_temp_dir()
+TEMP_DIR = storage.temp_directory
 
 ALGORITHM = "HS256"
 ACCESS_TOKEN_EXPIRE_MINUTES = 120

flowfile_core/database/connection.py

@@ -5,37 +5,21 @@ import os
 import sys
 from pathlib import Path
 from flowfile_core.configs import logger
+from shared.storage_config import storage
 
 
 def get_app_data_dir() -> Path:
     """Get the appropriate application data directory for the current platform."""
-    app_name = "Flowfile"
 
-    if sys.platform == "win32":
-        # Windows: C:\Users\{username}\AppData\Local\flowfile
-        base_dir = os.environ.get("LOCALAPPDATA")
-        if not base_dir:
-            base_dir = os.path.join(os.path.expanduser("~"), "AppData", "Local")
-    elif sys.platform == "darwin":
-        # macOS: ~/Library/Application Support/flowfile
-        base_dir = os.path.join(os.path.expanduser("~"), "Library", "Application Support")
-    else:
-        # Linux: ~/.local/share/flowfile or use XDG_DATA_HOME
-        base_dir = os.environ.get("XDG_DATA_HOME")
-        if not base_dir:
-            base_dir = os.path.join(os.path.expanduser("~"), ".local", "share")
-
-    app_dir = Path(base_dir) / app_name
-    app_dir.mkdir(parents=True, exist_ok=True)
-
-    return app_dir
+    return storage.database_directory
 
 
 def get_database_url():
     """Get the database URL based on the current environment."""
     if os.environ.get("TESTING") == "True":
         # Use a temporary test database
-        test_db_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../test_flowfile.db")
+        test_db_path = storage.temp_directory / "test_flowfile.db"
+        logger.debug(f"Using TESTING database URL: sqlite:///{test_db_path}")
         return f"sqlite:///{test_db_path}"
 
     custom_db_path = os.environ.get("FLOWFILE_DB_PATH")
@@ -94,4 +78,4 @@ def get_database_info():
         "path": str(get_database_path()) if get_database_path() else None,
         "app_data_dir": str(get_app_data_dir()),
         "platform": sys.platform
-    }
+    }