Flowfile 0.2.2__py3-none-any.whl

flowfile_worker/external_sources/sql_source/models.py

@@ -0,0 +1,72 @@
+from typing import Optional, Literal
+from pydantic import BaseModel, SecretStr
+from flowfile_worker.secrets import decrypt_secret
+
+
+class DataBaseConnection(BaseModel):
+    """Database connection configuration with secure password handling."""
+    username: Optional[str] = None
+    password: Optional[SecretStr] = None  # Encrypted password
+    host: Optional[str] = None
+    port: Optional[int] = None
+    database: Optional[str] = None  # The database name
+    database_type: str = "postgresql"  # Database type (postgresql, mysql, etc.)
+    url: Optional[str] = None
+
+    def get_decrypted_secret(self) -> SecretStr:
+        return decrypt_secret(self.password.get_secret_value())
+
+    def create_uri(self) -> str:
+        """
+        Creates a database URI based on the connection details.
+        If url is provided, it returns that directly.
+        Otherwise, it constructs a URI from the individual components.
+
+        Returns:
+            str: The database URI
+        """
+        # If URL is already provided, use it
+        if self.url:
+            return self.url
+
+        # Validate that required fields are present
+        if not all([self.host, self.database_type]):
+            raise ValueError("Host and database type are required to create a URI")
+
+        # Create credential part if username is provided
+        credentials = ""
+        if self.username:
+            credentials = self.username
+            if self.password:
+                # Get the raw password string from SecretStr
+                password_value = decrypt_secret(self.password.get_secret_value()).get_secret_value()
+                credentials += f":{password_value}"
+            credentials += "@"
+
+        # Create port part if port is provided
+        port_section = ""
+        if self.port:
+            port_section = f":{self.port}"
+        if self.database:
+
+            base_uri = f"{self.database_type}://{credentials}{self.host}{port_section}/{self.database}"
+        else:
+            base_uri = f"{self.database_type}://{credentials}{self.host}{port_section}"
+        return base_uri
+
+
+class DatabaseReadSettings(BaseModel):
+    """Settings for SQL source."""
+    connection: DataBaseConnection
+    query: str
+    flowfile_flow_id: int = 1
+    flowfile_node_id: int | str = -1
+
+
+class DatabaseWriteSettings(BaseModel):
+    """Settings for SQL sink."""
+    connection: DataBaseConnection
+    table_name: str
+    if_exists: Literal['append', 'replace', 'fail'] = 'append'
+    flowfile_flow_id: int = 1
+    flowfile_node_id: int | str = -1

flowfile_worker/flow_logger.py

@@ -0,0 +1,58 @@
+import logging
+import requests
+from flowfile_worker.models import RawLogInput
+from flowfile_worker.configs import FLOWFILE_CORE_URI
+
+LOGGING_URL = FLOWFILE_CORE_URI + "/raw_logs"
+
+
+class FlowfileLogHandler(logging.Handler):
+    def __init__(self, flowfile_flow_id: int = 1, flowfile_node_id: int | str = -1):
+        super().__init__()
+        self.flowfile_flow_id = flowfile_flow_id
+        self.flowfile_node_id = flowfile_node_id
+
+    def emit(self, record):
+        try:
+            log_message = self.format(record)
+
+            extra = {"Node Id": self.flowfile_node_id}
+            for k, v in extra.items():
+                log_message = f"{k}: {v} - {log_message}"
+            raw_log_input = RawLogInput(
+                flowfile_flow_id=self.flowfile_flow_id,
+                log_message=log_message,
+                log_type=record.levelname.upper(),
+                extra={
+                }
+            )
+            if self.flowfile_flow_id != -1 and self.flowfile_node_id != -1:
+                response = requests.post(LOGGING_URL, json=raw_log_input.__dict__,
+                                         headers={"Content-Type": "application/json"})
+                if response.status_code != 200:
+                    raise Exception(f"Failed to send log: {response.text}")
+        except Exception as e:
+            print(f"Error sending log to {LOGGING_URL}: {e}")
+
+
+def get_worker_logger(flowfile_flow_id: int, flowfile_node_id: int | str) -> logging.Logger:
+    logger_name = f"NodeLog: {flowfile_node_id}"
+    logger = logging.getLogger(logger_name)
+    logger.propagate = False  # Prevent propagation to parent loggers
+    logger.setLevel(logging.DEBUG)
+
+    # Only add handlers if they don't already exist to avoid duplicates
+    if not logger.handlers:
+        stream_handler = logging.StreamHandler()
+        stream_handler.setLevel(logging.DEBUG)
+        stream_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        stream_handler.setFormatter(stream_formatter)
+        logger.addHandler(stream_handler)
+
+        http_handler = FlowfileLogHandler(flowfile_flow_id=flowfile_flow_id, flowfile_node_id = flowfile_node_id)
+        http_handler.setLevel(logging.INFO)
+        http_formatter = logging.Formatter('%(message)s')
+        http_handler.setFormatter(http_formatter)
+        logger.addHandler(http_handler)
+
+    return logger

flowfile_worker/funcs.py

@@ -0,0 +1,327 @@
+import polars as pl
+import io
+from typing import List, Dict, Callable
+from multiprocessing import Array, Value, Queue
+from flowfile_worker.polars_fuzzy_match.matcher import fuzzy_match_dfs
+from flowfile_worker.polars_fuzzy_match.models import FuzzyMapping
+from flowfile_worker.flow_logger import get_worker_logger
+from flowfile_worker.external_sources.sql_source.models import DatabaseWriteSettings
+from flowfile_worker.external_sources.sql_source.main import write_serialized_df_to_database, write_df_to_database
+from base64 import encodebytes
+from logging import Logger
+import logging
+import os
+from flowfile_worker.utils import collect_lazy_frame, collect_lazy_frame_and_get_streaming_info
+
+
+# 'store', 'calculate_schema', 'calculate_number_of_records', 'write_output', 'fuzzy', 'store_sample']
+
+logging.basicConfig(format='%(asctime)s: %(message)s')
+logger = logging.getLogger('Spawner')
+logger.setLevel(logging.INFO)
+
+
+def fuzzy_join_task(left_serializable_object: bytes, right_serializable_object: bytes,
+                    fuzzy_maps: List[FuzzyMapping], error_message: Array, file_path: str,
+                    progress: Value,
+                    queue: Queue, flowfile_flow_id: int, flowfile_node_id: int | str,
+                    ):
+    flowfile_logger = get_worker_logger(flowfile_flow_id, flowfile_node_id)
+    try:
+        flowfile_logger.info("Starting fuzzy join operation")
+        left_df = pl.LazyFrame.deserialize(io.BytesIO(left_serializable_object))
+        right_df = pl.LazyFrame.deserialize(io.BytesIO(right_serializable_object))
+        fuzzy_match_result = fuzzy_match_dfs(left_df, right_df, fuzzy_maps, flowfile_logger)
+        flowfile_logger.info("Fuzzy join operation completed successfully")
+        fuzzy_match_result.write_ipc(file_path)
+        with progress.get_lock():
+            progress.value = 100
+    except Exception as e:
+        error_msg = str(e).encode()[:256]
+        with error_message.get_lock():
+            error_message[:len(error_msg)] = error_msg
+        with progress.get_lock():
+            progress.value = -1
+        flowfile_logger.error(f'Error during fuzzy join operation: {str(e)}')
+    lf = pl.scan_ipc(file_path)
+    number_of_records = collect_lazy_frame(lf.select(pl.len()))[0, 0]
+    flowfile_logger.info(f'Number of records after fuzzy match: {number_of_records}')
+    queue.put(encodebytes(lf.serialize()))
+
+
+def process_and_cache(polars_serializable_object: io.BytesIO, progress: Value, error_message: Array,
+                      file_path: str, flowfile_logger: Logger) -> bytes:
+    try:
+        lf = pl.LazyFrame.deserialize(polars_serializable_object)
+        collect_lazy_frame(lf).write_ipc(file_path)
+        flowfile_logger.info("Process operation completed successfully")
+        with progress.get_lock():
+            progress.value = 100
+    except Exception as e:
+        error_msg = str(e).encode()[:1024]  # Limit error message length
+        flowfile_logger.error(f'Error during process and cache operation: {str(e)}')
+        with error_message.get_lock():
+            error_message[:len(error_msg)] = error_msg
+        with progress.get_lock():
+            progress.value = -1  # Indicate error
+        return error_msg
+
+
+def store_sample(polars_serializable_object: bytes,
+                 progress: Value,
+                 error_message: Array,
+                 queue: Queue,
+                 file_path: str,
+                 sample_size: int,
+                 flowfile_flow_id: int,
+                 flowfile_node_id: int | str
+                 ):
+    flowfile_logger = get_worker_logger(flowfile_flow_id, flowfile_node_id)
+    flowfile_logger.info("Starting store sample operation")
+    try:
+        lf = pl.LazyFrame.deserialize(io.BytesIO(polars_serializable_object))
+        collect_lazy_frame(lf.limit(sample_size)).write_ipc(file_path)
+        flowfile_logger.info("Store sample operation completed successfully")
+        with progress.get_lock():
+            progress.value = 100
+    except Exception as e:
+        flowfile_logger.error(f'Error during store sample operation: {str(e)}')
+        error_msg = str(e).encode()[:1024]  # Limit error message length
+        with error_message.get_lock():
+            error_message[:len(error_msg)] = error_msg
+        with progress.get_lock():
+            progress.value = -1  # Indicate error
+        return error_msg
+
+
+def store(polars_serializable_object: bytes, progress: Value, error_message: Array, queue: Queue,  file_path: str,
+          flowfile_flow_id: int, flowfile_node_id: int | str):
+    flowfile_logger = get_worker_logger(flowfile_flow_id, flowfile_node_id)
+    flowfile_logger.info("Starting store operation")
+    polars_serializable_object_io = io.BytesIO(polars_serializable_object)
+    process_and_cache(polars_serializable_object_io, progress, error_message, file_path, flowfile_logger)
+    lf = pl.scan_ipc(file_path)
+    number_of_records = collect_lazy_frame(lf.select(pl.len()))[0, 0]
+    flowfile_logger.info(f'Number of records processed: {number_of_records}')
+    queue.put(encodebytes(lf.serialize()))
+
+
+def calculate_schema_logic(df: pl.LazyFrame, optimize_memory: bool = True, flowfile_logger: Logger = None) -> List[Dict]:
+    if flowfile_logger is None:
+        raise ValueError('flowfile_logger is required')
+    schema = df.collect_schema()
+    schema_stats = [dict(column_name=k, pl_datatype=str(v), col_index=i) for i, (k, v) in
+                    enumerate(schema.items())]
+    flowfile_logger.info('Starting to calculate the number of records')
+    collected_streaming_info = collect_lazy_frame_and_get_streaming_info(df.select(pl.len()))
+    n_records = collected_streaming_info.df[0, 0]
+    if n_records < 10_000:
+        flowfile_logger.info('Collecting the whole dataset')
+        df = collect_lazy_frame(df).lazy()
+    if optimize_memory and n_records > 1_000_000:
+        df = df.head(1_000_000)
+    null_cols = [col for col, data_type in schema.items() if data_type is pl.Null]
+    if not (n_records == 0 and df.width == 0):
+        if len(null_cols) == 0:
+            pl_stats = df.describe()
+        else:
+            df = df.drop(null_cols)
+            pl_stats = df.describe()
+        n_unique_per_cols = list(df.select(pl.all().approx_n_unique()).collect(
+            engine="streaming" if collected_streaming_info.streaming_collect_available else "auto").to_dicts()[0].values()
+                                 )
+        stats_headers = pl_stats.drop_in_place('statistic').to_list()
+        stats = {v['column_name']: v for v in pl_stats.transpose(include_header=True, header_name='column_name',
+                                                                 column_names=stats_headers).to_dicts()}
+        for i, (col_stat, n_unique_values) in enumerate(zip(stats.values(), n_unique_per_cols)):
+            col_stat['n_unique'] = n_unique_values
+            col_stat['examples'] = ', '.join({str(col_stat['min']), str(col_stat['max'])})
+            col_stat['null_count'] = int(float(col_stat['null_count']))
+            col_stat['count'] = int(float(col_stat['count']))
+
+        for schema_stat in schema_stats:
+            deep_stat = stats.get(schema_stat['column_name'])
+            if deep_stat:
+                schema_stat.update(deep_stat)
+        del df
+    else:
+        schema_stats = []
+    return schema_stats
+
+
+def calculate_schema(polars_serializable_object: bytes, progress: Value, error_message: Array, queue: Queue,
+                     flowfile_flow_id: int, flowfile_node_id: int | str, *args, **kwargs):
+    polars_serializable_object_io = io.BytesIO(polars_serializable_object)
+    flowfile_logger = get_worker_logger(flowfile_flow_id, flowfile_node_id)
+    flowfile_logger.info("Starting schema calculation")
+    try:
+        lf = pl.LazyFrame.deserialize(polars_serializable_object_io)
+        schema_stats = calculate_schema_logic(lf, flowfile_logger=flowfile_logger)
+        flowfile_logger.info('schema_stats', schema_stats)
+        queue.put(schema_stats)
+        flowfile_logger.info("Schema calculation completed successfully")
+        with progress.get_lock():
+            progress.value = 100
+    except Exception as e:
+        error_msg = str(e).encode()[:256]  # Limit error message length
+        flowfile_logger.error('error', e)
+        with error_message.get_lock():
+            error_message[:len(error_msg)] = error_msg
+        with progress.get_lock():
+            progress.value = -1  # Indicate error
+
+
+def calculate_number_of_records(polars_serializable_object: bytes, progress: Value, error_message: Array,
+                                queue: Queue, flowfile_flow_id: int, *args, **kwargs):
+    flowfile_logger = get_worker_logger(flowfile_flow_id, -1)
+    flowfile_logger.info("Starting number of records calculation")
+    polars_serializable_object_io = io.BytesIO(polars_serializable_object)
+    try:
+        lf = pl.LazyFrame.deserialize(polars_serializable_object_io)
+        n_records = collect_lazy_frame(lf.select(pl.len()))[0, 0]
+        queue.put(n_records)
+        flowfile_logger.debug("Number of records calculation completed successfully")
+        flowfile_logger.debug(f'n_records {n_records}')
+        with progress.get_lock():
+            progress.value = 100
+    except Exception as e:
+        flowfile_logger.error('error', e)
+        error_msg = str(e).encode()[:256]  # Limit error message length
+        with error_message.get_lock():
+            error_message[:len(error_msg)] = error_msg
+        with progress.get_lock():
+            progress.value = -1  # Indicate error
+        return b'error'
+
+
+def execute_write_method(write_method: Callable, path: str, data_type: str = None, sheet_name: str = None,
+                         delimiter: str = None,
+                         write_mode: str = 'create', flowfile_logger: Logger = None):
+    flowfile_logger.info('executing write method')
+    if data_type == 'excel':
+        logger.info('Writing as excel file')
+        write_method(path, worksheet=sheet_name)
+    elif data_type == 'csv':
+        logger.info('Writing as csv file')
+        if write_mode == 'append':
+            with open(path, 'ab') as f:
+                write_method(file=f, separator=delimiter, quote_style='always')
+        else:
+            write_method(file=path, separator=delimiter, quote_style='always')
+    elif data_type == 'parquet':
+        logger.info('Writing as parquet file')
+        write_method(path)
+
+
+def write_to_database(polars_serializable_object: bytes,
+                      progress: Value,
+                      error_message: Array,
+                      queue: Queue,
+                      file_path: str,
+                      database_write_settings: DatabaseWriteSettings,
+                      flowfile_flow_id: int = -1,
+                      flowfile_node_id: int | str = -1
+                      ):
+    """
+    Writes a Polars DataFrame to a SQL database.
+    """
+    flowfile_logger = get_worker_logger(flowfile_flow_id, flowfile_node_id)
+    flowfile_logger.info(f"Starting write operation to: {database_write_settings.table_name}")
+    df = collect_lazy_frame(pl.LazyFrame.deserialize(io.BytesIO(polars_serializable_object)))
+    flowfile_logger.info(f"Starting to write {len(df)} records")
+    try:
+        write_df_to_database(df, database_write_settings)
+        flowfile_logger.info("Write operation completed successfully")
+        with progress.get_lock():
+            progress.value = 100
+    except Exception as e:
+        error_msg = str(e).encode()[:1024]
+        flowfile_logger.error(f'Error during write operation: {str(e)}')
+        with error_message.get_lock():
+            error_message[:len(error_msg)] = error_msg
+        with progress.get_lock():
+            progress.value = -1
+
+
+def write_output(polars_serializable_object: bytes,
+                 progress: Value,
+                 error_message: Array,
+                 queue: Queue,
+                 file_path: str,
+                 data_type: str,
+                 path: str,
+                 write_mode: str,
+                 sheet_name: str = None,
+                 delimiter: str = None,
+                 flowfile_flow_id: int = -1,
+                 flowfile_node_id: int | str = -1
+                 ):
+    flowfile_logger = get_worker_logger(flowfile_flow_id, flowfile_node_id)
+    flowfile_logger.info(f"Starting write operation to: {path}")
+    try:
+        df = pl.LazyFrame.deserialize(io.BytesIO(polars_serializable_object))
+        if isinstance(df, pl.LazyFrame):
+            flowfile_logger.info(f'Execution plan explanation:\n{df.explain(format="plain")}')
+        flowfile_logger.info("Successfully deserialized dataframe")
+        is_lazy = False
+        sink_method_str = 'sink_'+data_type
+        write_method_str = 'write_'+data_type
+        has_sink_method = hasattr(df, sink_method_str)
+        write_method = None
+        if os.path.exists(path) and write_mode == 'create':
+            raise Exception('File already exists')
+        if has_sink_method and is_lazy:
+            write_method = getattr(df, 'sink_' + data_type)
+        elif not is_lazy or not has_sink_method:
+            if isinstance(df, pl.LazyFrame):
+                df = collect_lazy_frame(df)
+            write_method = getattr(df, write_method_str)
+        if write_method is not None:
+            execute_write_method(write_method, path=path, data_type=data_type, sheet_name=sheet_name,
+                                 delimiter=delimiter, write_mode=write_mode, flowfile_logger=flowfile_logger)
+            number_of_records_written = (collect_lazy_frame(df.select(pl.len()))[0, 0]
+                                         if isinstance(df, pl.LazyFrame) else df.height)
+            flowfile_logger.info(f'Number of records written: {number_of_records_written}')
+        else:
+            raise Exception('Write method not found')
+        with progress.get_lock():
+            progress.value = 100
+    except Exception as e:
+        logger.info(f'Error during write operation: {str(e)}')
+        error_message[:len(str(e))] = str(e).encode()
+
+
+def generic_task(func: Callable,
+                 progress: Value,
+                 error_message: Array,
+                 queue: Queue,
+                 file_path: str,
+                 flowfile_flow_id: int,
+                 flowfile_node_id: int | str,
+                 *args, **kwargs):
+    print(kwargs)
+    flowfile_logger = get_worker_logger(flowfile_flow_id, flowfile_node_id)
+    flowfile_logger.info("Starting generic task")
+    try:
+        df = func(*args, **kwargs)
+        if isinstance(df, pl.LazyFrame):
+            collect_lazy_frame(df).write_ipc(file_path)
+        elif isinstance(df, pl.DataFrame):
+            df.write_ipc(file_path)
+        else:
+            raise Exception('Returned object is not a DataFrame or LazyFrame')
+        with progress.get_lock():
+            progress.value = 100
+        flowfile_logger.info("Task completed successfully")
+    except Exception as e:
+        flowfile_logger.error(f'Error during task execution: {str(e)}')
+        error_msg = str(e).encode()[:1024]
+        with error_message.get_lock():
+            error_message[:len(error_msg)] = error_msg
+        with progress.get_lock():
+            progress.value = -1
+
+    lf = pl.scan_ipc(file_path)
+    number_of_records = collect_lazy_frame(lf.select(pl.len()))[0, 0]
+    flowfile_logger.info(f'Number of records processed: {number_of_records}')
+    queue.put(encodebytes(lf.serialize()))

flowfile_worker/main.py

@@ -0,0 +1,108 @@
+import asyncio
+import uvicorn
+import signal
+
+from contextlib import asynccontextmanager
+from fastapi import FastAPI
+from flowfile_worker.routes import router
+from flowfile_worker import mp_context, CACHE_DIR
+from flowfile_worker.configs import logger, FLOWFILE_CORE_URI, SERVICE_HOST, SERVICE_PORT
+
+
+should_exit = False
+server_instance = None
+
+
+@asynccontextmanager
+async def shutdown_handler(app: FastAPI):
+    """Handle application startup and shutdown"""
+    logger.info('Starting application...')
+    try:
+        yield
+    finally:
+        logger.info('Shutting down application...')
+        logger.info("Cleaning up worker resources...")
+        for p in mp_context.active_children():
+            try:
+                p.terminate()
+                p.join()
+            except Exception as e:
+                logger.error(f"Error cleaning up process: {e}")
+
+        try:
+            CACHE_DIR.cleanup()
+        except Exception as e:
+            print(f"Error cleaning up cache directory: {e}")
+
+        await asyncio.sleep(0.1)
+
+
+app = FastAPI(lifespan=shutdown_handler)
+app.include_router(router)
+
+
+@app.post("/shutdown")
+async def shutdown():
+    """Endpoint to handle graceful shutdown"""
+    if server_instance:
+        # Schedule the shutdown
+        await asyncio.create_task(trigger_shutdown())
+    return {"message": "Shutting down"}
+
+
+async def trigger_shutdown():
+    """Trigger the actual shutdown after responding to the client"""
+    await asyncio.sleep(1)  # Give time for the response to be sent
+    if server_instance:
+        server_instance.should_exit = True
+
+
+def signal_handler(signum, frame):
+    """Handle shutdown signals"""
+    logger.info(f"Received signal {signum}")
+    if server_instance:
+        server_instance.should_exit = True
+
+
+def run(host: str = None, port: int = None):
+    """Run the FastAPI app with graceful shutdown"""
+    global server_instance
+
+    # Use values from settings if not explicitly provided
+    if host is None:
+        host = SERVICE_HOST
+    if port is None:
+        port = SERVICE_PORT
+
+    # Log service configuration
+    logger.info(f"Starting worker service on {host}:{port}")
+    logger.info(f"Core service configured at {FLOWFILE_CORE_URI}")
+
+    signal.signal(signal.SIGTERM, signal_handler)
+    signal.signal(signal.SIGINT, signal_handler)
+
+    config = uvicorn.Config(
+        app,
+        host=host,
+        port=port,
+        loop="asyncio"
+    )
+    server = uvicorn.Server(config)
+    server_instance = server  # Store server instance globally
+
+    logger.info('Starting server...')
+    logger.info('Server started')
+
+    try:
+        server.run()
+    except KeyboardInterrupt:
+        logger.info("Received interrupt signal, shutting down...")
+    finally:
+        server_instance = None
+        logger.info("Server shutdown complete")
+
+
+if __name__ == "__main__":
+    import multiprocessing
+    multiprocessing.freeze_support()
+    run()

flowfile_worker/models.py

@@ -0,0 +1,95 @@
+from pydantic import BaseModel
+from typing import Optional, Literal, Any
+from base64 import decodebytes
+from flowfile_worker.polars_fuzzy_match.models import FuzzyMapping
+from flowfile_worker.external_sources.sql_source.models import DatabaseWriteSettings
+
+
+OperationType = Literal[
+    'store', 'calculate_schema', 'calculate_number_of_records', 'write_output', 'fuzzy', 'store_sample',
+    'write_to_database']
+ResultType = Literal['polars', 'other']
+
+
+class PolarsOperation(BaseModel):
+    operation: bytes
+    flowfile_flow_id: Optional[int] = 1
+    flowfile_node_id: Optional[int | str] = -1
+    def polars_serializable_object(self):
+        return decodebytes(self.operation)
+
+
+class PolarsScript(PolarsOperation):
+    task_id: Optional[str] = None
+    cache_dir: Optional[str] = None
+    operation_type: OperationType
+
+
+class PolarsScriptSample(PolarsScript):
+    sample_size: Optional[int] = 100
+
+
+class PolarsScriptWrite(BaseModel):
+    operation: bytes
+    data_type: str
+    path: str
+    write_mode: str
+    sheet_name: Optional[str] = None
+    delimiter: Optional[str] = None
+    flowfile_flow_id: Optional[int] = -1
+    flowfile_node_id: Optional[int | str] = -1
+
+    def polars_serializable_object(self):
+        return decodebytes(self.operation)
+
+
+class DatabaseScriptWrite(DatabaseWriteSettings):
+    operation: bytes
+
+    def polars_serializable_object(self):
+        return decodebytes(self.operation)
+
+    def get_database_write_settings(self) -> DatabaseWriteSettings:
+        """
+        Converts the current instance to a DatabaseWriteSettings object.
+        Returns:
+            DatabaseWriteSettings: The corresponding DatabaseWriteSettings object.
+        """
+
+        return DatabaseWriteSettings(
+            connection=self.connection,
+            table_name=self.table_name,
+            if_exists=self.if_exists,
+            flowfile_flow_id=self.flowfile_flow_id,
+            flowfile_node_id=self.flowfile_node_id
+        )
+
+
+class FuzzyJoinInput(BaseModel):
+    task_id: Optional[str] = None
+    cache_dir: Optional[str] = None
+    left_df_operation: PolarsOperation
+    right_df_operation: PolarsOperation
+    fuzzy_maps: list[FuzzyMapping]
+    flowfile_flow_id: Optional[int] = 1
+    flowfile_node_id: Optional[int | str] = -1
+
+
+class Status(BaseModel):
+    background_task_id: str
+    status: Literal['Processing', 'Completed', 'Error', 'Unknown Error', 'Starting']  # Type alias for status
+    file_ref: str
+    progress: Optional[int] = 0
+    error_message: Optional[str] = None  # Add error_message field
+    results: Optional[Any] = None
+    result_type: Optional[ResultType] = 'polars'
+
+    def __hash__(self):
+        return hash(self.file_ref)
+
+
+class RawLogInput(BaseModel):
+    flowfile_flow_id: int
+    log_message: str
+    log_type: Literal["INFO", "ERROR"]
+    extra: Optional[dict] = None