Flowfile 0.3.8__py3-none-any.whl → 0.3.10__py3-none-any.whl

flowfile_core/flowfile/setting_generator/settings.py

@@ -4,6 +4,7 @@ from typing import Callable, Iterable
 from functools import wraps
 from flowfile_core.schemas.output_model import NodeData
 from flowfile_core.flowfile.setting_generator.setting_generator import SettingGenerator, SettingUpdator
+from pl_fuzzy_frame_match.models import FuzzyMapping
 
 setting_generator = SettingGenerator()
 setting_updator = SettingUpdator()
@@ -55,7 +56,6 @@ def cross_join(node_data: "NodeData") -> NodeData:
         ji = transform_schema.CrossJoinInput(left_select=node_data.main_input.columns,
                                              right_select=node_data.right_input.columns)
         ji.auto_rename()
-        print(ji)
         node_data.setting_input = input_schema.NodeCrossJoin(flow_id=node_data.flow_id,
                                                              node_id=node_data.node_id,
                                                              cross_join_input=ji)
@@ -135,7 +135,7 @@ def cross_join(node_data: NodeData):
 
 
 def check_if_fuzzy_match_is_valid(left_columns: Iterable[str], right_columns: Iterable[str],
-                                  fuzzy_map: transform_schema.FuzzyMap) -> bool:
+                                  fuzzy_map: FuzzyMapping) -> bool:
     if fuzzy_map.left_col not in left_columns:
         return False
     if fuzzy_map.right_col not in right_columns:

flowfile_core/flowfile/util/execution_orderer.py

@@ -2,6 +2,15 @@ from typing import List, Dict, Set
 from flowfile_core.flowfile.flow_node.flow_node import FlowNode
 from flowfile_core.configs import logger
 from collections import deque, defaultdict
+from flowfile_core.flowfile.util.node_skipper import determine_nodes_to_skip
+
+def compute_execution_plan(nodes: List[FlowNode], flow_starts: List[FlowNode] = None):
+    """ Computes the execution order after finding the nodes to skip on the execution step."""
+    skip_nodes = determine_nodes_to_skip(nodes=nodes)
+    computed_execution_order = determine_execution_order(all_nodes=[node for node in nodes if node not in skip_nodes],
+                                                        flow_starts=flow_starts)
+    return skip_nodes, computed_execution_order
+
 
 
 def determine_execution_order(all_nodes: List[FlowNode], flow_starts: List[FlowNode] = None) -> List[FlowNode]:

flowfile_core/flowfile/util/node_skipper.py

@@ -0,0 +1,8 @@
+from typing import List
+from flowfile_core.flowfile.flow_node.flow_node import FlowNode
+
+def determine_nodes_to_skip(nodes : List[FlowNode]) -> List[FlowNode]:
+    """ Finds nodes to skip on the execution step. """
+    skip_nodes = [node for node in nodes if not node.is_correct]
+    skip_nodes.extend([lead_to_node for node in skip_nodes for lead_to_node in node.leads_to_nodes])
+    return skip_nodes

flowfile_core/main.py

@@ -7,6 +7,8 @@ import uvicorn
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
+from shared.storage_config import storage
+
 from flowfile_core import ServerRun
 from flowfile_core.configs.settings import (SERVER_HOST, SERVER_PORT, WORKER_HOST, WORKER_PORT, WORKER_URL,)
 
@@ -18,13 +20,13 @@ from flowfile_core.routes.logs import router as logs_router
 from flowfile_core.routes.cloud_connections import router as cloud_connections_router
 
 from flowfile_core.configs.flow_logger import clear_all_flow_logs
+storage.cleanup_directories()
 
 os.environ["FLOWFILE_MODE"] = "electron"
 
 should_exit = False
 server_instance = None
 
-
 @asynccontextmanager
 async def shutdown_handler(app: FastAPI):
     """Handles the graceful startup and shutdown of the FastAPI application.
@@ -79,6 +81,7 @@ app.include_router(secrets_router, prefix="/secrets", tags=["secrets"])
 app.include_router(cloud_connections_router, prefix="/cloud_connections", tags=["cloud_connections"])
 
 
+
 @app.post("/shutdown")
 async def shutdown():
     """An API endpoint to gracefully shut down the server.

flowfile_core/routes/routes.py

@@ -26,7 +26,7 @@ from flowfile_core.configs.node_store import nodes
 from flowfile_core.configs.settings import IS_RUNNING_IN_DOCKER
 # File handling
 from flowfile_core.fileExplorer.funcs import (
-    FileExplorer,
+    SecureFileExplorer,
     FileInfo,
     get_files_from_directory
 )
@@ -39,9 +39,11 @@ from flowfile_core.flowfile.extensions import get_instant_func_results
 from flowfile_core.flowfile.sources.external_sources.sql_source.sql_source import create_sql_source_from_db_settings
 from flowfile_core.run_lock import get_flow_run_lock
 # Schema and models
+
+from shared.storage_config import storage
 from flowfile_core.schemas import input_schema, schemas, output_model
 from flowfile_core.utils import excel_file_manager
-from flowfile_core.utils.fileManager import create_dir, remove_paths
+from flowfile_core.utils.fileManager import create_dir
 from flowfile_core.utils.utils import camel_case_to_snake_case
 from flowfile_core import flow_file_handler
 from flowfile_core.flowfile.database_connection_manager.db_connections import (store_database_connection,
@@ -54,7 +56,10 @@ from flowfile_core.database.connection import get_db
 router = APIRouter(dependencies=[Depends(get_current_active_user)])
 
 # Initialize services
-file_explorer = FileExplorer('/app/shared' if IS_RUNNING_IN_DOCKER else None)
+file_explorer = SecureFileExplorer(
+    start_path=storage.user_data_directory,
+    sandbox_root=storage.user_data_directory
+)
 
 
 def get_node_model(setting_name_ref: str):
@@ -148,7 +153,7 @@ async def get_directory_contents(directory: str, file_types: List[str] = None,
     Returns:
         A list of `FileInfo` objects representing the directory's contents.
     """
-    directory_explorer = FileExplorer(directory)
+    directory_explorer = SecureFileExplorer(directory, storage.user_data_directory)
     try:
         return directory_explorer.list_contents(show_hidden=include_hidden, file_types=file_types)
     except Exception as e:
@@ -198,6 +203,24 @@ async def get_active_flow_file_sessions() -> List[schemas.FlowSettings]:
     return [flf.flow_settings for flf in flow_file_handler.flowfile_flows]
 
 
+@router.post("/node/trigger_fetch_data", tags=['editor'])
+async def trigger_fetch_node_data(flow_id: int, node_id: int, background_tasks: BackgroundTasks):
+    """Fetches and refreshes the data for a specific node."""
+    flow = flow_file_handler.get_flow(flow_id)
+    lock = get_flow_run_lock(flow_id)
+    async with lock:
+        if flow.flow_settings.is_running:
+            raise HTTPException(422, 'Flow is already running')
+        try:
+            flow.validate_if_node_can_be_fetched(node_id)
+        except Exception as e:
+            raise HTTPException(422, str(e))
+        background_tasks.add_task(flow.trigger_fetch_node, node_id)
+    return JSONResponse(content={"message": "Data started",
+                                 "flow_id": flow_id,
+                                 "node_id": node_id}, status_code=status.HTTP_200_OK)
+
+
 @router.post('/flow/run/', tags=['editor'])
 async def run_flow(flow_id: int, background_tasks: BackgroundTasks) -> JSONResponse:
     """Executes a flow in a background task.
@@ -228,6 +251,16 @@ def cancel_flow(flow_id: int):
     flow.cancel()
 
 
+@router.post("/flow/apply_standard_layout/", tags=["editor"])
+def apply_standard_layout(flow_id: int):
+    flow = flow_file_handler.get_flow(flow_id)
+    if not flow:
+        raise HTTPException(status_code=404, detail="Flow not found")
+    if flow.flow_settings.is_running:
+        raise HTTPException(422, "Flow is running")
+    flow.apply_layout()
+
+
 @router.get('/flow/run_status/', tags=['editor'],
             response_model=output_model.RunInformation)
 def get_run_status(flow_id: int, response: Response):
@@ -238,10 +271,12 @@ def get_run_status(flow_id: int, response: Response):
     flow = flow_file_handler.get_flow(flow_id)
     if not flow:
         raise HTTPException(status_code=404, detail="Flow not found")
+    if flow.latest_run_info is None:
+        raise HTTPException(status_code=404, detail="No run information available")
     if flow.flow_settings.is_running:
         response.status_code = status.HTTP_202_ACCEPTED
-        return flow.get_run_info()
-    response.status_code = status.HTTP_200_OK
+    else:
+        response.status_code = status.HTTP_200_OK
     return flow.get_run_info()
 
 
@@ -439,11 +474,24 @@ def get_generated_code(flow_id: int) -> str:
 
 
 @router.post('/editor/create_flow/', tags=['editor'])
-def create_flow(flow_path: str):
+def create_flow(flow_path: str = None, name: str = None):
     """Creates a new, empty flow file at the specified path and registers a session for it."""
-    flow_path = Path(flow_path)
-    logger.info('Creating flow')
-    return flow_file_handler.add_flow(name=flow_path.stem, flow_path=str(flow_path))
+    if flow_path is not None and name is None:
+        name = Path(flow_path).stem
+    elif flow_path is not None and name is not None:
+        if name not in flow_path and flow_path.endswith(".flowfile"):
+            raise HTTPException(422, 'The name must be part of the flow path when a full path is provided')
+        elif name in flow_path and not flow_path.endswith(".flowfile"):
+            flow_path = str(Path(flow_path) / (name + ".flowfile"))
+        elif name not in flow_path and name.endswith(".flowfile"):
+            flow_path = str(Path(flow_path) / name)
+        elif name not in flow_path and not name.endswith(".flowfile"):
+            flow_path = str(Path(flow_path) / (name + ".flowfile"))
+    if flow_path is not None:
+        flow_path_ref = Path(flow_path)
+        if not flow_path_ref.parent.exists():
+            raise HTTPException(422, 'The directory does not exist')
+    return flow_file_handler.add_flow(name=name, flow_path=flow_path)
 
 
 @router.post('/editor/close_flow/', tags=['editor'])
@@ -471,6 +519,7 @@ def add_generic_settings(input_data: Dict[str, Any], node_type: str, current_use
     add_func = getattr(flow, 'add_' + node_type)
     parsed_input = None
     setting_name_ref = 'node' + node_type.replace('_', '')
+
     if add_func is None:
         raise HTTPException(404, 'could not find the function')
     try:

flowfile_core/schemas/input_schema.py

@@ -196,7 +196,6 @@ class NodeBase(BaseModel):
     user_id: Optional[int] = None
     is_flow_output: Optional[bool] = False
 
-
 class NodeSingleInput(NodeBase):
     """A base model for any node that takes a single data input."""
     depending_on_id: Optional[int] = -1

flowfile_core/schemas/output_model.py

@@ -1,4 +1,4 @@
-from typing import List, Dict, Optional, Any
+from typing import List, Dict, Optional, Any, Literal
 from pydantic import BaseModel, Field
 from datetime import datetime
 import time
@@ -21,10 +21,11 @@ class RunInformation(BaseModel):
     flow_id: int
     start_time: Optional[datetime] = Field(default_factory=datetime.now)
     end_time: Optional[datetime] = None
-    success: bool
+    success: Optional[bool] = None
     nodes_completed: int = 0
     number_of_nodes: int = 0
     node_step_result: List[NodeResult]
+    run_type: Literal["fetch_one", "full_run"]
 
 
 class BaseItem(BaseModel):
@@ -61,6 +62,8 @@ class TableExample(BaseModel):
     table_schema: List[FileColumn]
     columns: List[str]
     data: Optional[List[Dict]] = {}
+    has_example_data: bool = False
+    has_run_with_current_setup: bool = False
 
 
 class NodeData(BaseModel):

flowfile_core/schemas/schemas.py

@@ -1,8 +1,35 @@
 from typing import Optional, List, Dict, Tuple, Any, Literal, Annotated
 from pydantic import BaseModel, field_validator, ConfigDict, Field, StringConstraints
 from flowfile_core.flowfile.utils import create_unique_id
+from flowfile_core.configs.settings import OFFLOAD_TO_WORKER
 ExecutionModeLiteral = Literal['Development', 'Performance']
-ExecutionLocationsLiteral = Literal['auto', 'local', 'remote']
+ExecutionLocationsLiteral = Literal['local', 'remote']
+
+
+def get_global_execution_location() -> ExecutionLocationsLiteral:
+    """
+    Calculates the default execution location based on the global settings
+    Returns
+    -------
+    ExecutionLocationsLiteral where the current
+    """
+    if OFFLOAD_TO_WORKER:
+        return "remote"
+    return "local"
+
+
+def is_valid_execution_location_in_current_global_settings(execution_location: ExecutionLocationsLiteral) -> bool:
+    return not (get_global_execution_location() == "local" and execution_location == "remote")
+
+
+def get_prio_execution_location(local_execution_location: ExecutionLocationsLiteral,
+                                global_execution_location: ExecutionLocationsLiteral) -> ExecutionLocationsLiteral:
+    if local_execution_location == global_execution_location:
+        return local_execution_location
+    elif global_execution_location == "local" and local_execution_location == "remote":
+        return "local"
+    else:
+        return local_execution_location
 
 
 class FlowGraphConfig(BaseModel):
@@ -16,7 +43,7 @@ class FlowGraphConfig(BaseModel):
         name (str): The name of the flow.
         path (str): The file path associated with the flow.
         execution_mode (ExecutionModeLiteral): The mode of execution ('Development' or 'Performance').
-        execution_location (ExecutionLocationsLiteral): The location for execution ('auto', 'local', 'remote').
+        execution_location (ExecutionLocationsLiteral): The location for execution ('local', 'remote').
     """
     flow_id: int = Field(default_factory=create_unique_id, description="Unique identifier for the flow.")
     description: Optional[str] = None
@@ -24,7 +51,23 @@ class FlowGraphConfig(BaseModel):
     name: str = ''
     path: str = ''
     execution_mode: ExecutionModeLiteral = 'Performance'
-    execution_location: ExecutionLocationsLiteral = "auto"
+    execution_location: ExecutionLocationsLiteral = Field(default_factory=get_global_execution_location)
+
+    @field_validator('execution_location', mode='before')
+    def validate_and_set_execution_location(cls, v: Optional[ExecutionLocationsLiteral]) -> ExecutionLocationsLiteral:
+        """
+        Validates and sets the execution location.
+        1.  **If `None` is provided**: It defaults to the location determined by global settings.
+        2.  **If a value is provided**: It checks if the value is compatible with the global
+            settings. If not (e.g., requesting 'remote' when only 'local' is possible),
+            it corrects the value to a compatible one.
+        """
+        if v is None:
+            return get_global_execution_location()
+        if v == "auto":
+            return get_global_execution_location()
+
+        return get_prio_execution_location(v, get_global_execution_location())
 
 
 class FlowSettings(FlowGraphConfig):
@@ -95,6 +138,8 @@ class NodeTemplate(BaseModel):
     node_group: str
     prod_ready: bool = True
     can_be_start: bool = False
+    drawer_title: str = "Node title"
+    drawer_intro: str = "Drawer into"
 
 
 class NodeInformation(BaseModel):

flowfile_core/schemas/transform_schema.py

@@ -6,6 +6,9 @@ from copy import deepcopy
 
 from typing import NamedTuple
 
+from pl_fuzzy_frame_match.models import FuzzyMapping
+
+FuzzyMap = FuzzyMapping  # For backwards compatibility
 
 def get_func_type_mapping(func: str):
     """Infers the output data type of common aggregation functions."""
@@ -158,6 +161,19 @@ class SelectInputs:
         """Gets a list of original column names to select from the source DataFrame."""
         return [v.old_name for v in self.renames if v.keep or (v.join_key and include_join_key)]
 
+    def has_drop_cols(self) -> bool:
+        """Checks if any column is marked to be dropped from the selection."""
+        return any(not v.keep for v in self.renames)
+
+    @property
+    def drop_columns(self) -> List[SelectInput]:
+        """Returns a list of column names that are marked to be dropped from the selection."""
+        return [v for v in self.renames if not v.keep and v.is_available]
+
+    @property
+    def non_jk_drop_columns(self) -> List[SelectInput]:
+        return [v for v in self.renames if not v.keep and v.is_available and not v.join_key]
+
     def __add__(self, other: "SelectInput"):
         """Allows adding a SelectInput using the '+' operator."""
         self.renames.append(other)
@@ -225,32 +241,6 @@ class JoinMap:
     right_col: str
 
 
-@dataclass
-class FuzzyMap(JoinMap):
-    """Extends `JoinMap` with settings for fuzzy string matching, such as the algorithm and similarity threshold."""
-    threshold_score: Optional[float] = 80.0
-    fuzzy_type: Optional[FuzzyTypeLiteral] = 'levenshtein'
-    perc_unique: Optional[float] = 0.0
-    output_column_name: Optional[str] = None
-    valid: Optional[bool] = True
-
-    def __init__(self, left_col: str, right_col: str = None, threshold_score: float = 80.0,
-                 fuzzy_type: FuzzyTypeLiteral = 'levenshtein', perc_unique: float = 0, output_column_name: str = None,
-                 _output_col_name: str = None, valid: bool = True, output_col_name: str = None):
-        if right_col is None:
-            right_col = left_col
-        self.valid = valid
-        self.left_col = left_col
-        self.right_col = right_col
-        self.threshold_score = threshold_score
-        self.fuzzy_type = fuzzy_type
-        self.perc_unique = perc_unique
-        self.output_column_name = output_column_name if output_column_name is not None else _output_col_name
-        self.output_column_name = self.output_column_name if self.output_column_name is not None else output_col_name
-        if self.output_column_name is None:
-            self.output_column_name = f'fuzzy_score_{self.left_col}_{self.right_col}'
-
-
 class JoinSelectMixin:
     """A mixin providing common methods for join-like operations that involve left and right inputs."""
     left_select: JoinInputs = None
@@ -430,32 +420,32 @@ class JoinInput(JoinSelectMixin):
 @dataclass
 class FuzzyMatchInput(JoinInput):
     """Extends `JoinInput` with settings specific to fuzzy matching, such as the matching algorithm and threshold."""
-    join_mapping: List[FuzzyMap]
+    join_mapping: List[FuzzyMapping]
     aggregate_output: bool = False
 
     @staticmethod
-    def parse_fuzz_mapping(fuzz_mapping: List[FuzzyMap] | Tuple[str, str] | str) -> List[FuzzyMap]:
+    def parse_fuzz_mapping(fuzz_mapping: List[FuzzyMapping] | Tuple[str, str] | str) -> List[FuzzyMapping]:
         if isinstance(fuzz_mapping, (tuple, list)):
             assert len(fuzz_mapping) > 0
             if all(isinstance(fm, dict) for fm in fuzz_mapping):
-                fuzz_mapping = [FuzzyMap(**fm) for fm in fuzz_mapping]
+                fuzz_mapping = [FuzzyMapping(**fm) for fm in fuzz_mapping]
 
-            if not isinstance(fuzz_mapping[0], FuzzyMap):
+            if not isinstance(fuzz_mapping[0], FuzzyMapping):
                 assert len(fuzz_mapping) <= 2
                 if len(fuzz_mapping) == 2:
                     assert isinstance(fuzz_mapping[0], str) and isinstance(fuzz_mapping[1], str)
-                    fuzz_mapping = [FuzzyMap(*fuzz_mapping)]
+                    fuzz_mapping = [FuzzyMapping(*fuzz_mapping)]
                 elif isinstance(fuzz_mapping[0], str):
-                    fuzz_mapping = [FuzzyMap(fuzz_mapping[0], fuzz_mapping[0])]
+                    fuzz_mapping = [FuzzyMapping(fuzz_mapping[0], fuzz_mapping[0])]
         elif isinstance(fuzz_mapping, str):
-            fuzz_mapping = [FuzzyMap(fuzz_mapping, fuzz_mapping)]
-        elif isinstance(fuzz_mapping, FuzzyMap):
+            fuzz_mapping = [FuzzyMapping(fuzz_mapping, fuzz_mapping)]
+        elif isinstance(fuzz_mapping, FuzzyMapping):
             fuzz_mapping = [fuzz_mapping]
         else:
             raise Exception('No valid join mapping as input')
         return fuzz_mapping
 
-    def __init__(self, join_mapping: List[FuzzyMap] | Tuple[str, str] | str, left_select: List[SelectInput] | List[str],
+    def __init__(self, join_mapping: List[FuzzyMapping] | Tuple[str, str] | str, left_select: List[SelectInput] | List[str],
                  right_select: List[SelectInput] | List[str], aggregate_output: bool = False, how: JoinStrategy = 'inner'):
         self.join_mapping = self.parse_fuzz_mapping(join_mapping)
         self.left_select = self.parse_select(left_select)
@@ -463,9 +453,9 @@ class FuzzyMatchInput(JoinInput):
         self.how = how
         for jm in self.join_mapping:
 
-            if jm.right_col not in self.right_select.old_cols:
+            if jm.right_col not in {v.old_name for v in self.right_select.renames}:
                 self.right_select.append(SelectInput(jm.right_col, keep=False, join_key=True))
-            if jm.left_col not in self.left_select.old_cols:
+            if jm.left_col not in {v.old_name for v in self.left_select.renames}:
                 self.left_select.append(SelectInput(jm.left_col, keep=False, join_key=True))
         [setattr(v, "join_key", v.old_name in self._left_join_keys) for v in self.left_select.renames]
         [setattr(v, "join_key", v.old_name in self._right_join_keys) for v in self.right_select.renames]
@@ -476,7 +466,7 @@ class FuzzyMatchInput(JoinInput):
         return self.left_select.new_cols & self.right_select.new_cols
 
     @property
-    def fuzzy_maps(self) -> List[FuzzyMap]:
+    def fuzzy_maps(self) -> List[FuzzyMapping]:
         """Returns the final fuzzy mappings after applying all column renames."""
         new_mappings = []
         left_rename_table, right_rename_table = self.left_select.rename_table, self.right_select.rename_table

flowfile_frame/__init__.py

@@ -1,12 +1,9 @@
 # flowframe/__init__.py
 """A Polars-like API for building ETL graphs."""
 
-from flowfile_core.configs.settings import OFFLOAD_TO_WORKER
-
-OFFLOAD_TO_WORKER.value = False
-
 # Core classes
 from flowfile_frame.flow_frame import FlowFrame   # noqa: F401
+from pl_fuzzy_frame_match.models import FuzzyMapping  # noqa: F401
 
 from flowfile_frame.utils import create_flow_graph  # noqa: F401
 

flowfile_frame/flow_frame.py

@@ -5,11 +5,13 @@ from typing import Any, Iterable, List, Literal, Optional, Tuple, Union, Dict, C
 import re
 
 import polars as pl
-from polars._typing import (CsvEncoding)
 from flowfile_frame.lazy_methods import add_lazyframe_methods
 
-from polars._typing import (FrameInitTypes, SchemaDefinition, SchemaDict, Orientation)
+from polars._typing import (CsvEncoding, FrameInitTypes, SchemaDefinition, SchemaDict, Orientation)
 from collections.abc import Iterator
+
+from pl_fuzzy_frame_match import FuzzyMapping, fuzzy_match_dfs
+
 from flowfile_core.flowfile.flow_graph import FlowGraph, add_connection
 from flowfile_core.flowfile.flow_graph_utils import combine_flow_graphs_with_mapping
 from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
@@ -20,8 +22,7 @@ from flowfile_frame.expr import Expr, Column, lit, col
 from flowfile_frame.selectors import Selector
 from flowfile_frame.group_frame import GroupByFrame
 from flowfile_frame.utils import (_parse_inputs_as_iterable, create_flow_graph, stringify_values,
-                                  ensure_inputs_as_iterable, generate_node_id,
-                                  set_node_id, data as node_id_data)
+                                  ensure_inputs_as_iterable, generate_node_id, data as node_id_data)
 from flowfile_frame.join import _normalize_columns_to_list, _create_join_mappings
 from flowfile_frame.utils import _check_if_convertible_to_code
 from flowfile_frame.config import logger
@@ -2109,6 +2110,34 @@ class FlowFrame:
 
         return self._create_child_frame(new_node_id)
 
+    def fuzzy_match(self,
+                    other: "FlowFrame",
+                    fuzzy_mappings: List[FuzzyMapping],
+                    description: str = None,
+                    ) -> "FlowFrame":
+        self._ensure_same_graph(other)
+
+        # Step 3: Generate new node ID
+        new_node_id = generate_node_id()
+        node_fuzzy_match = input_schema.NodeFuzzyMatch(flow_id=self.flow_graph.flow_id,
+                                                       node_id=new_node_id,
+                                                       join_input=
+                                                       transform_schema.FuzzyMatchInput(join_mapping=fuzzy_mappings,
+                                                                                        left_select=self.columns,
+                                                                                        right_select=other.columns),
+                                                       description=description or "Fuzzy match between two FlowFrames",
+                                                       depending_on_ids=[self.node_id, other.node_id],
+                                                       )
+        self.flow_graph.add_fuzzy_match(node_fuzzy_match)
+        self._add_connection(self.node_id, new_node_id, "main")
+        other._add_connection(other.node_id, new_node_id, "right")
+        return FlowFrame(
+            data=self.flow_graph.get_node(new_node_id).get_resulting_data().data_frame,
+            flow_graph=self.flow_graph,
+            node_id=new_node_id,
+            parent_node_id=self.node_id,
+        )
+
     def text_to_rows(
         self,
         column: str | Column,

flowfile_frame/flow_frame.pyi

@@ -208,6 +208,8 @@ class FlowFrame:
     # Get the first row of the DataFrame.
     def first(self, description: Optional[str] = None) -> 'FlowFrame': ...
 
+    def fuzzy_match(self, other: FlowFrame, fuzzy_mappings: typing.List[flowfile_core.schemas.transform_schema.FuzzyMap], description: str = None) -> 'FlowFrame': ...
+
     # Take every nth row in the LazyFrame and return as a new LazyFrame.
     def gather_every(self, n: int, offset: int = 0, description: Optional[str] = None) -> 'FlowFrame': ...
 

flowfile_worker/__init__.py

@@ -1,15 +1,15 @@
 from typing import Dict
-import tempfile
 import threading
 import multiprocessing
-import os
-import shutil
-multiprocessing.set_start_method('spawn', force=True)
+from shared.storage_config import storage
 
+multiprocessing.set_start_method('spawn', force=True)
 
 from multiprocessing import get_context
 from flowfile_worker.models import Status
+
 mp_context = get_context("spawn")
+
 status_dict: Dict[str, Status] = dict()
 process_dict = dict()
 
@@ -17,39 +17,10 @@ status_dict_lock = threading.Lock()
 process_dict_lock = threading.Lock()
 
 
-class SharedTempDirectory:
-    """A class that mimics tempfile.TemporaryDirectory but uses a fixed directory"""
-    def __init__(self, dir_path):
-        self._path = dir_path
-        os.makedirs(self._path, exist_ok=True)
-
-    @property
-    def name(self):
-        return self._path
-
-    def cleanup(self):
-        """Remove all contents of the temp directory"""
-        try:
-            shutil.rmtree(self._path)
-            os.makedirs(self._path, exist_ok=True)
-            print(f"Cleaned up temporary directory: {self._path}")
-        except Exception as e:
-            print(f"Error during cleanup: {e}")
-
-    def __enter__(self):
-        return self.name
-
-    def __exit__(self, exc, value, tb):
-        self.cleanup()
-
-
 CACHE_EXPIRATION_TIME = 24 * 60 * 60
 
 
-TEMP_DIR = os.getenv('TEMP_DIR')
-if TEMP_DIR:
-    CACHE_DIR = SharedTempDirectory(TEMP_DIR)
-else:
-    CACHE_DIR = tempfile.TemporaryDirectory()
+CACHE_DIR = storage.cache_directory
+
 
 PROCESS_MEMORY_USAGE: Dict[str, float] = dict()

flowfile_worker/funcs.py

@@ -2,8 +2,9 @@ import polars as pl
 import io
 from typing import List, Dict, Callable
 from multiprocessing import Array, Value, Queue
-from flowfile_worker.polars_fuzzy_match.matcher import fuzzy_match_dfs
-from flowfile_worker.polars_fuzzy_match.models import FuzzyMapping
+
+from pl_fuzzy_frame_match import fuzzy_match_dfs, FuzzyMapping
+
 from flowfile_worker.flow_logger import get_worker_logger
 from flowfile_worker.external_sources.sql_source.models import DatabaseWriteSettings
 from flowfile_worker.external_sources.sql_source.main import write_df_to_database
@@ -33,7 +34,10 @@ def fuzzy_join_task(left_serializable_object: bytes, right_serializable_object:
         flowfile_logger.info("Starting fuzzy join operation")
         left_df = pl.LazyFrame.deserialize(io.BytesIO(left_serializable_object))
         right_df = pl.LazyFrame.deserialize(io.BytesIO(right_serializable_object))
-        fuzzy_match_result = fuzzy_match_dfs(left_df, right_df, fuzzy_maps, flowfile_logger)
+        fuzzy_match_result = fuzzy_match_dfs(left_df=left_df,
+                                             right_df=right_df,
+                                             fuzzy_maps=fuzzy_maps,
+                                             logger=flowfile_logger)
         flowfile_logger.info("Fuzzy join operation completed successfully")
         fuzzy_match_result.write_ipc(file_path)
         with progress.get_lock():

flowfile_worker/main.py

@@ -4,8 +4,11 @@ import signal
 
 from contextlib import asynccontextmanager
 from fastapi import FastAPI
+
+from shared.storage_config import storage
+
 from flowfile_worker.routes import router
-from flowfile_worker import mp_context, CACHE_DIR
+from flowfile_worker import mp_context
 from flowfile_worker.configs import logger, FLOWFILE_CORE_URI, SERVICE_HOST, SERVICE_PORT
 
 
@@ -30,7 +33,7 @@ async def shutdown_handler(app: FastAPI):
                 logger.error(f"Error cleaning up process: {e}")
 
         try:
-            CACHE_DIR.cleanup()
+            storage.cleanup_directories()
         except Exception as e:
             print(f"Error cleaning up cache directory: {e}")
 

flowfile_worker/models.py

@@ -1,7 +1,9 @@
 from pydantic import BaseModel
 from typing import Optional, Literal, Any
 from base64 import decodebytes
-from flowfile_worker.polars_fuzzy_match.models import FuzzyMapping
+
+from pl_fuzzy_frame_match import FuzzyMapping
+
 from flowfile_worker.external_sources.sql_source.models import DatabaseWriteSettings
 from flowfile_worker.external_sources.s3_source.models import CloudStorageWriteSettings