Flowfile 0.3.8__py3-none-any.whl → 0.3.10__py3-none-any.whl

flowfile_worker/routes.py

@@ -17,13 +17,21 @@ from flowfile_worker.external_sources.sql_source.main import read_sql_source, wr
 router = APIRouter()
 
 
+def create_and_get_default_cache_dir(flowfile_flow_id: int) -> str:
+    default_cache_dir = CACHE_DIR / str(flowfile_flow_id)
+    default_cache_dir.mkdir(parents=True, exist_ok=True)
+    return str(default_cache_dir)
+
+
 @router.post("/submit_query/")
 def submit_query(polars_script: models.PolarsScript, background_tasks: BackgroundTasks) -> models.Status:
     logger.info(f"Processing query with operation: {polars_script.operation_type}")
 
     try:
         polars_script.task_id = str(uuid.uuid4()) if polars_script.task_id is None else polars_script.task_id
-        polars_script.cache_dir = polars_script.cache_dir if polars_script.cache_dir is not None else CACHE_DIR.name
+        default_cache_dir = create_and_get_default_cache_dir(polars_script.flowfile_flow_id)
+
+        polars_script.cache_dir = polars_script.cache_dir if polars_script.cache_dir is not None else default_cache_dir
         polars_serializable_object = polars_script.polars_serializable_object()
         file_path = os.path.join(polars_script.cache_dir, f"{polars_script.task_id}.arrow")
         result_type = "polars" if polars_script.operation_type == "store" else "other"
@@ -49,8 +57,9 @@ def store_sample(polars_script: models.PolarsScriptSample, background_tasks: Bac
     logger.info(f"Processing sample storage with size: {polars_script.sample_size}")
 
     try:
+        default_cache_dir = create_and_get_default_cache_dir(polars_script.flowfile_flow_id)
         polars_script.task_id = str(uuid.uuid4()) if polars_script.task_id is None else polars_script.task_id
-        polars_script.cache_dir = polars_script.cache_dir if polars_script.cache_dir is not None else CACHE_DIR.name
+        polars_script.cache_dir = polars_script.cache_dir if polars_script.cache_dir is not None else default_cache_dir
         polars_serializable_object = polars_script.polars_serializable_object()
 
         file_path = os.path.join(polars_script.cache_dir, f"{polars_script.task_id}.arrow")
@@ -210,7 +219,8 @@ def store_sql_db_result(database_read_settings: DatabaseReadSettings, background
 
     try:
         task_id = str(uuid.uuid4())
-        file_path = os.path.join(CACHE_DIR.name, f"{task_id}.arrow")
+        file_path = os.path.join(create_and_get_default_cache_dir(database_read_settings.flowfile_flow_id),
+                                 f"{task_id}.arrow")
         status = models.Status(background_task_id=task_id, status="Starting", file_ref=file_path,
                                result_type="polars")
         status_dict[task_id] = status
@@ -246,7 +256,7 @@ def create_table(file_type: FileType, received_table: Dict, background_tasks: Ba
 
     try:
         task_id = str(uuid.uuid4())
-        file_ref = os.path.join(CACHE_DIR.name, f"{task_id}.arrow")
+        file_ref = os.path.join(create_and_get_default_cache_dir(flowfile_flow_id), f"{task_id}.arrow")
 
         status = models.Status(background_task_id=task_id, status="Starting", file_ref=file_ref,
                                result_type="polars")
@@ -382,8 +392,9 @@ async def add_fuzzy_join(polars_script: models.FuzzyJoinInput, background_tasks:
     """
     logger.info("Starting fuzzy join operation")
     try:
+        default_cache_dir = create_and_get_default_cache_dir(polars_script.flowfile_flow_id)
         polars_script.task_id = str(uuid.uuid4()) if polars_script.task_id is None else polars_script.task_id
-        polars_script.cache_dir = polars_script.cache_dir if polars_script.cache_dir is not None else CACHE_DIR.name
+        polars_script.cache_dir = polars_script.cache_dir if polars_script.cache_dir is not None else default_cache_dir
         left_serializable_object = polars_script.left_df_operation.polars_serializable_object()
         right_serializable_object = polars_script.right_df_operation.polars_serializable_object()
 
@@ -405,6 +416,37 @@ async def add_fuzzy_join(polars_script: models.FuzzyJoinInput, background_tasks:
         raise HTTPException(status_code=500, detail=str(e))
 
 
+@router.delete("/clear_task/{task_id}")
+def clear_task(task_id: str):
+    """
+    Clear task data and status by ID.
+
+    Args:
+        task_id: Unique identifier of the task to clear
+    Returns:
+        dict: Success message
+    Raises:
+        HTTPException: If task not found
+    """
+
+    logger.info(f"Clearing task: {task_id}")
+    status = status_dict.get(task_id)
+    if not status:
+        logger.warning(f"Task not found for clearing: {task_id}")
+        raise HTTPException(status_code=404, detail="Task not found")
+    try:
+        if os.path.exists(status.file_ref):
+            os.remove(status.file_ref)
+            logger.debug(f"Removed file: {status.file_ref}")
+    except Exception as e:
+        logger.error(f"Error removing file {status.file_ref}: {str(e)}", exc_info=True)
+    with status_dict_lock:
+        status_dict.pop(task_id, None)
+        PROCESS_MEMORY_USAGE.pop(task_id, None)
+        logger.info(f"Successfully cleared task: {task_id}")
+    return {"message": f"Task {task_id} has been cleared."}
+
+
 @router.post("/cancel_task/{task_id}")
 def cancel_task(task_id: str):
     """Cancel a running task by ID.

shared/__init__.py

@@ -0,0 +1,15 @@
+"""
+Shared utilities for Flowfile services.
+This package contains common functionality that can be used across
+flowfile_core, flowfile_worker, and other components without creating
+circular dependencies.
+"""
+
+from .storage_config import storage, get_cache_directory, get_temp_directory, get_flows_directory
+
+__all__ = [
+    'storage',
+    'get_cache_directory', 
+    'get_temp_directory',
+    'get_flows_directory'
+]

shared/storage_config.py

@@ -0,0 +1,243 @@
+# shared/storage_config.py - Updated for Option 3
+"""
+Centralized storage configuration for Flowfile.
+This module can be imported by both core and worker without creating dependencies.
+"""
+import os
+from pathlib import Path
+from typing import Optional, Literal
+
+DirectoryOptions = Literal["temp_directory", "logs_directory",
+                            "system_logs_directory", "database_directory",
+                            "cache_directory", "flows_directory"]
+
+
+class FlowfileStorage:
+    """Centralized storage manager for Flowfile applications."""
+
+    def __init__(self):
+        self._base_dir: Optional[Path] = None
+        self._user_data_dir: Optional[Path] = None
+        self._ensure_directories()
+
+    @property
+    def base_directory(self) -> Path:
+        """Get the base Flowfile storage directory (for internal container communication)."""
+        if self._base_dir is None:
+            if os.environ.get("RUNNING_IN_DOCKER") == "true":
+                # In Docker, internal storage stays inside /app
+                base_path = os.environ.get("FLOWFILE_STORAGE_DIR", "/app/internal_storage")
+            else:
+                # Local development
+                base_path = os.environ.get("FLOWFILE_STORAGE_DIR")
+                if not base_path:
+                    home_dir = Path.home()
+                    base_path = home_dir / ".flowfile"
+
+            self._base_dir = Path(base_path)
+        return self._base_dir
+
+    @property
+    def user_data_directory(self) -> Path:
+        """Get the user data directory (completely separate from application code)."""
+        if self._user_data_dir is None:
+            if os.environ.get("RUNNING_IN_DOCKER") == "true":
+                # In Docker, user data is at /data/user (completely outside /app)
+                user_data_path = os.environ.get("FLOWFILE_USER_DATA_DIR", "/data/user")
+            else:
+                # Local development - use user's home directory
+                user_data_path = Path.home()
+
+            self._user_data_dir = Path(user_data_path)
+        return self._user_data_dir
+
+    @property
+    def cache_directory(self) -> Path:
+        """Cache directory for worker-core communication (internal)."""
+        return self.base_directory / "cache"
+
+    def get_flow_cache_directory(self, flow_id: int) -> Path:
+        """Get or create a cache directory for a specific flow (internal)."""
+        flow_cache_dir = self.cache_directory / str(flow_id)
+        flow_cache_dir.mkdir(parents=True, exist_ok=True)
+        return flow_cache_dir
+
+    @property
+    def system_logs_directory(self) -> Path:
+        """Directory for system logs (internal)."""
+        return self.base_directory / "system_logs"
+
+    @property
+    def flows_directory(self) -> Path:
+        """Directory for flow storage (user-accessible)."""
+        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            # In Docker, flows are in separate user data area
+            return self.user_data_directory / "flows"
+        else:
+            # Local development - flows in ~/.flowfile/flows
+            return self.base_directory / "flows"
+
+    @property
+    def uploads_directory(self) -> Path:
+        """Directory for user uploads (user-accessible)."""
+        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            # In Docker, uploads are in separate user data area
+            return self.user_data_directory / "uploads"
+        else:
+            # Local development - uploads in ~/.flowfile/uploads
+            return self.base_directory / "uploads"
+
+    @property
+    def outputs_directory(self) -> Path:
+        """Directory for user outputs (user-accessible)."""
+        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            # In Docker, outputs are in separate user data area
+            return self.user_data_directory / "outputs"
+        else:
+            # Local development - outputs in ~/.flowfile/outputs
+            return self.base_directory / "outputs"
+
+    @property
+    def database_directory(self) -> Path:
+        """Directory for local database files (internal)."""
+        return self.base_directory / "database"
+
+    @property
+    def logs_directory(self) -> Path:
+        """Directory for application logs (internal)."""
+        return self.base_directory / "logs"
+
+    @property
+    def temp_directory(self) -> Path:
+        """Directory for temporary files (internal)."""
+        return self.base_directory / "temp"
+
+    @property
+    def temp_directory_for_flows(self) -> Path:
+        """Directory for temporary files specific to flows (internal)."""
+        return self.temp_directory / "flows"
+
+    def _ensure_directories(self) -> None:
+        """Create all necessary directories if they don't exist."""
+        # Internal directories (always created in base_directory)
+        internal_directories = [
+            self.cache_directory,
+            self.database_directory,
+            self.logs_directory,
+            self.temp_directory,
+            self.system_logs_directory,
+            self.temp_directory_for_flows,
+        ]
+
+        # User-accessible directories (location depends on environment)
+        user_directories = [
+            self.flows_directory,
+            self.uploads_directory,
+            self.outputs_directory,
+        ]
+
+        for directory in internal_directories + user_directories:
+            directory.mkdir(parents=True, exist_ok=True)
+
+    def get_cache_file_path(self, filename: str) -> Path:
+        """Get full path for a cache file (internal)."""
+        return self.cache_directory / filename
+
+    def get_flow_file_path(self, filename: str) -> Path:
+        """Get full path for a flow file (user-accessible)."""
+        return self.flows_directory / filename
+
+    def get_upload_file_path(self, filename: str) -> Path:
+        """Get full path for an uploaded file (user-accessible)."""
+        return self.uploads_directory / filename
+
+    def get_output_file_path(self, filename: str) -> Path:
+        """Get full path for an output file (user-accessible)."""
+        return self.outputs_directory / filename
+
+    def get_log_file_path(self, filename: str) -> Path:
+        """Get full path for an application log file (internal)."""
+        return self.logs_directory / filename
+
+    def get_system_log_file_path(self, filename: str) -> Path:
+        """Get full path for a system log file (internal)."""
+        return self.system_logs_directory / filename
+
+    def get_temp_file_path(self, filename: str) -> Path:
+        """Get full path for a temporary file (internal)."""
+        return self.temp_directory / filename
+
+    def cleanup_directory(self, directory_option: DirectoryOptions, storage_duration_hours: int = 24) -> None:
+        """Clean up any directory of the folder"""
+        import time
+        import shutil
+
+        if not hasattr(self, directory_option):
+            raise Exception(f"Directory does not exist in {self.base_directory}")
+
+        directory = getattr(self, directory_option)
+        if not isinstance(directory, Path):
+            raise Exception(f"Directory attribute {directory_option} is not a Path object")
+
+        if not directory.exists():
+            return
+
+        current_time = time.time()
+        cutoff_time = current_time - (storage_duration_hours * 60 * 60)
+
+        for item in directory.iterdir():
+            try:
+                if item.stat().st_mtime < cutoff_time:
+                    if item.is_file():
+                        item.unlink()
+                    elif item.is_dir():
+                        shutil.rmtree(item)
+            except (OSError, FileNotFoundError):
+                # Handle permission errors or files that disappeared
+                continue
+
+    def cleanup_directories(self) -> None:
+        """Clean up temporary files older than specified hours."""
+        self.cleanup_directory("temp_directory", storage_duration_hours=24)
+        self.cleanup_directory("cache_directory", storage_duration_hours=1)
+        self.cleanup_directory("logs_directory", storage_duration_hours=168)
+        self.cleanup_directory("system_logs_directory", storage_duration_hours=168)
+
+
+storage = FlowfileStorage()
+
+
+# Convenience functions for backward compatibility
+def get_cache_directory() -> str:
+    """Get cache directory path as string."""
+    return str(storage.cache_directory)
+
+
+def get_temp_directory() -> str:
+    """Get temp directory path as string."""
+    return str(storage.temp_directory)
+
+
+def get_flows_directory() -> str:
+    """Get flows directory path as string."""
+    return str(storage.flows_directory)
+
+
+def get_uploads_directory() -> str:
+    """Get uploads directory path as string."""
+    return str(storage.uploads_directory)
+
+
+def get_outputs_directory() -> str:
+    """Get outputs directory path as string."""
+    return str(storage.outputs_directory)
+
+
+def get_logs_directory() -> str:
+    """Get application logs directory path as string."""
+    return str(storage.logs_directory)
+
+
+def get_system_logs_directory() -> str:
+    """Get system logs directory path as string."""
+    return str(storage.system_logs_directory)

flowfile/web/static/assets/GraphSolver-17fd26db.css

@@ -1,68 +0,0 @@
-
-.context-menu[data-v-d0286fd2] {
-  position: fixed;
-  z-index: 1000;
-  border: 1px solid #ccc;
-  background-color: white;
-  box-shadow: 0 2px 10px rgba(0, 0, 0, 0.2);
-  border-radius: 4px;
-  user-select: none;
-}
-.context-menu ul[data-v-d0286fd2] {
-  list-style: none;
-  padding: 0;
-  margin: 0;
-}
-.context-menu li[data-v-d0286fd2] {
-  padding: 8px 16px;
-  cursor: pointer;
-}
-.context-menu li.disabled[data-v-d0286fd2] {
-  color: #ccc;
-  cursor: not-allowed;
-}
-.context-menu li[data-v-d0286fd2]:hover:not(.disabled) {
-  background-color: #f0f0f0;
-}
-
-.items-container[data-v-43acf78a] {
-  display: flex;
-  gap: 10px; /* Space between items */
-}
-.item-box[data-v-43acf78a] {
-  display: flex;
-  align-items: center;
-  padding: 5px 10px;
-  background-color: #f0f0f0;
-  border-radius: 4px;
-  font-size: 12px; /* Font size set to 12px */
-  position: relative;
-}
-.remove-btn[data-v-43acf78a] {
-  margin-left: 8px;
-  cursor: pointer;
-  color: #100f0f72;
-  font-weight: bold;
-}
-
-.context-menu[data-v-5a467b52] {
-  position: fixed;
-  z-index: 1000;
-  border: 1px solid #ccc;
-  background-color: white;
-  padding: 8px;
-  box-shadow: 0 2px 10px rgba(0, 0, 0, 0.2);
-  border-radius: 4px;
-}
-.context-menu ul[data-v-5a467b52] {
-  list-style: none;
-  padding: 0;
-  margin: 0;
-}
-.context-menu li[data-v-5a467b52] {
-  padding: 8px 16px;
-  cursor: pointer;
-}
-.context-menu li[data-v-5a467b52]:hover {
-  background-color: #f0f0f0;
-}

flowfile/web/static/assets/Pivot-f415e85f.css

@@ -1,35 +0,0 @@
-
-.validation-wrapper[data-v-3eb585b2] {
-  background-color: #ffffff;
-}
-.error-message[data-v-3eb585b2] {
-  color: #991b1b;
-  margin: 5px 0;
-  font-size: 12px;
-}
-.warning-icon[data-v-3eb585b2] {
-  cursor: pointer;
-  font-size: 24px;
-}
-
-.context-menu[data-v-d0f14439] {
-  position: fixed;
-  z-index: 1000;
-  border: 1px solid #ccc;
-  background-color: white;
-  padding: 8px;
-  box-shadow: 0 2px 10px rgba(0, 0, 0, 0.2);
-  border-radius: 4px;
-}
-.context-menu ul[data-v-d0f14439] {
-  list-style: none;
-  padding: 0;
-  margin: 0;
-}
-.context-menu li[data-v-d0f14439] {
-  padding: 8px 16px;
-  cursor: pointer;
-}
-.context-menu li[data-v-d0f14439]:hover {
-  background-color: #f0f0f0;
-}

flowfile/web/static/assets/Read-80dc1675.css

@@ -1,197 +0,0 @@
-
-.selectors[data-v-e7395814] {
-  display: flex;
-  flex-direction: column;
-  gap: 12px;
-}
-.input-wrapper[data-v-e7395814] {
-  display: flex;
-  flex-direction: column;
-  gap: 4px;
-  flex: 1;
-}
-label[data-v-e7395814] {
-  font-weight: 500;
-  color: #333;
-  font-size: 14px;
-}
-input[data-v-e7395814] {
-  padding: 6px;
-  border: 1px solid #ccc;
-  border-radius: 4px;
-  font-size: 14px;
-  width: 99%;
-}
-.row[data-v-e7395814] {
-  display: flex;
-  align-items: center;
-  justify-content: space-between;
-  gap: 16px;
-}
-.compact-input[data-v-e7395814] {
-  width: 96%;
-  padding: 6px;
-  font-size: 14px;
-  border: 1px solid #ccc;
-  border-radius: 4px;
-}
-.button-container[data-v-e7395814] {
-  display: flex;
-  justify-content: center;
-  margin: 16px 0;
-}
-.optional-section[data-v-e7395814] {
-  margin-top: 20px;
-}
-.section-divider[data-v-e7395814] {
-  margin: 16px 0;
-  border: none;
-  border-top: 1px solid #ddd;
-}
-.table-sizes[data-v-e7395814] {
-  font-weight: bold;
-  margin-bottom: 10px;
-}
-.warning-sign[data-v-e7395814] {
-  color: #e74c3c;
-  font-size: 16px;
-  margin-left: 8px;
-}
-@media (max-width: 600px) {
-.row[data-v-e7395814] {
-    flex-direction: column;
-}
-}
-
-[data-v-d0b76f7b] {
-  box-sizing: border-box;
-}
-.csv-table-settings[data-v-d0b76f7b] {
-  background: #fff;
-  border-radius: 6px;
-  padding: 16px;
-  box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
-  margin: 20px auto;
-  max-width: 600px;
-  overflow-x: hidden; /* Prevent horizontal scroll */
-}
-.row[data-v-d0b76f7b] {
-  display: flex;
-  align-items: center;
-  justify-content: space-between;
-  margin-bottom: 12px;
-  gap: 20px;
-  max-width: 100%; /* Prevent overflow */
-}
-label[data-v-d0b76f7b] {
-  font-weight: 500;
-  color: #333;
-  font-size: 14px;
-  flex: 1;
-  max-width: 100%;
-}
-.el-select[data-v-d0b76f7b],
-.el-slider[data-v-d0b76f7b],
-.el-checkbox[data-v-d0b76f7b] {
-  flex: 2;
-  max-width: 100%; /* Prevent overflow */
-}
-.el-slider[data-v-d0b76f7b] {
-  padding-top: 10px;
-}
-.el-checkbox[data-v-d0b76f7b] {
-  display: flex;
-  align-items: center;
-}
-@media (max-width: 600px) {
-.row[data-v-d0b76f7b] {
-    flex-direction: column;
-    align-items: flex-start;
-}
-.el-select[data-v-d0b76f7b],
-  .el-slider[data-v-d0b76f7b],
-  .el-checkbox[data-v-d0b76f7b] {
-    width: 100%;
-}
-}
-
-.parquet-table-settings[data-v-0faf0508] {
-  background: #f9fafb;
-  border-radius: 8px;
-  padding: 20px;
-  box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
-  max-width: 600px;
-  margin: 20px auto;
-  text-align: center; /* Center the text */
-}
-.message h2[data-v-0faf0508] {
-  font-size: 24px;
-  color: #3498db;
-  margin-bottom: 10px;
-}
-.message p[data-v-0faf0508] {
-  font-size: 16px;
-  color: #333;
-}
-
-.context-menu[data-v-221ad292] {
-  position: fixed;
-  z-index: 1000;
-  border: 1px solid #ccc;
-  background-color: white;
-  padding: 8px;
-  box-shadow: 0 2px 10px rgba(0, 0, 0, 0.2);
-  border-radius: 4px;
-}
-.context-menu button[data-v-221ad292] {
-  display: block;
-  background: none;
-  border: none;
-  padding: 4px 8px;
-  text-align: left;
-  width: 100%;
-  cursor: pointer;
-}
-.context-menu button[data-v-221ad292]:hover {
-  background-color: #f0f0f0;
-}
-.file-upload-wrapper[data-v-221ad292] {
-  position: relative;
-  width: 100%;
-}
-.file-upload-label[data-v-221ad292] {
-  display: flex;
-  align-items: center;
-  background-color: #f5f5f5;
-  border: 1px solid #ddd;
-  border-radius: 4px;
-  padding: 10px 15px;
-  color: #333;
-  font-size: 16px;
-  font-weight: 500;
-  cursor: pointer;
-  transition: background-color 0.3s ease;
-}
-.file-upload-label[data-v-221ad292]:hover {
-  background-color: #e4e4e4;
-}
-.file-icon[data-v-221ad292] {
-  margin-right: 10px;
-  font-size: 20px;
-}
-.file-label-text[data-v-221ad292] {
-  flex-grow: 1;
-  margin-left: 10px;
-}
-input[type="text"][data-v-221ad292] {
-  width: 100%;
-  padding: 8px;
-  border: 1px solid #ddd;
-  border-radius: 4px;
-  box-shadow: inset 0 1px 3px rgba(0, 0, 0, 0.1);
-  transition: border-color 0.2s ease;
-}
-input[type="text"][data-v-221ad292]:focus {
-  border-color: #3498db;
-  outline: none;
-}