Flowfile 0.3.1.2__py3-none-any.whl → 0.3.3__py3-none-any.whl

flowfile_core/flowfile/flow_data_engine/sample_data.py

@@ -1,14 +1,27 @@
 from faker import Faker
 from functools import partial
+from math import ceil
 from random import randint
 import polars as pl
 from typing import List, Dict, Any, Generator
 
 
-def create_fake_data(n_records: int = 1000) -> pl.DataFrame:
+def create_fake_data(n_records: int = 1000, optimized: bool = True) -> pl.DataFrame:
+    """
+
+    Args:
+        n_records (): Number of records to return
+        optimized (): Indicator if creation should be optimized, will result in more identical rows when True
+
+    Returns:
+        pl.DataFrame
+    """
     fake = Faker()
-    selector = partial(randint,0)
-    min_range = partial(min, n_records)
+    selector = partial(randint, 0)
+
+    max_n_records = min(10_000, n_records) if optimized else n_records
+
+    min_range = partial(min, max_n_records)
     # Pre-generation of static data
     cities = [fake.city() for _ in range(min_range(7000))]
     companies = [fake.company() for _ in range(min_range(100_000))]
@@ -19,7 +32,7 @@ def create_fake_data(n_records: int = 1000) -> pl.DataFrame:
     first_names = [fake.first_name() for _ in range(min_range(100_000))]
     last_names = [fake.last_name() for _ in range(min_range(50_000))]
     domain_names = [fake.domain_name() for _ in range(10)]
-    sales_data = [fake.random_int(0, 1000) for _ in range(n_records)]
+    sales_data = [fake.random_int(0, 1000) for _ in range(max_n_records)]
 
     def generate_name():
         return f"{first_names[selector(min_range(100_000))-1]} {last_names[selector(min_range(50_000))-1]}"
@@ -32,9 +45,8 @@ def create_fake_data(n_records: int = 1000) -> pl.DataFrame:
 
     def generate_phone_number():
         return fake.phone_number()
-
     data = []
-    for i in range(n_records):
+    for i in range(max_n_records):
         name = generate_name()
         data.append(dict(
             ID=randint(1, 1000000),
@@ -47,8 +59,14 @@ def create_fake_data(n_records: int = 1000) -> pl.DataFrame:
             Work=companies[selector(min_range(100_000))-1],
             Zipcode=zipcodes[selector(min_range(200_000))-1],
             Country=countries[selector(min_range(50))-1],
-            sales_data=sales_data[selector(n_records)-1]
+            sales_data=sales_data[selector(max_n_records)-1]
         ))
+    if max_n_records < n_records:
+        n_duplicates: int = ceil(n_records / max_n_records)
+        output = []
+        for _ in range(n_duplicates):
+            output.extend(data)
+        data = output[:n_records]
 
     return pl.DataFrame(data)
 

flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py

@@ -190,7 +190,7 @@ class BaseFetcher:
                 logger.info('Already running the fetching')
                 return
 
-            sleep_time = 1
+            sleep_time = .5
             self.running = True
             while not self.stop_event.is_set():
                 try:
@@ -205,7 +205,8 @@ class BaseFetcher:
                             break
                         elif status.status == 'Unknown Error':
                             self._handle_error(-1,
-                                               'There was an unknown error with the process, and the process got killed by the server')
+                                               'There was an unknown error with the process, '
+                                               'and the process got killed by the server')
                             break
                     else:
                         self._handle_error(2, r.text)
@@ -284,7 +285,7 @@ class ExternalDfFetcher(BaseFetcher):
 
     def __init__(self, flow_id: int, node_id: int | str, lf: pl.LazyFrame | pl.DataFrame, file_ref: str = None,
                  wait_on_completion: bool = True,
-                 operation_type: OperationType = 'store'):
+                 operation_type: OperationType = 'store', offload_to_worker: bool = True):
         super().__init__(file_ref=file_ref)
         lf = lf.lazy() if isinstance(lf, pl.DataFrame) else lf
         r = trigger_df_operation(lf=lf, file_ref=self.file_ref, operation_type=operation_type,

flowfile_core/flowfile/flow_data_engine/utils.py

@@ -210,3 +210,4 @@ def match_order(l: List[str], ref: List[str]) -> List[str]:
         org_order = ref_order.get(v, float('inf'))
         order.append(org_order)
     return [v for _, v in sorted(zip(order, l))]
+

flowfile_core/flowfile/flow_graph_utils.py

@@ -0,0 +1,320 @@
+from typing import Dict, Tuple, Optional, List, Set, Callable
+from copy import deepcopy
+from flowfile_core.schemas import input_schema, schemas
+from flowfile_core.flowfile.FlowfileFlow import FlowGraph, add_connection
+
+
+def combine_flow_graphs_with_mapping(
+        *flow_graphs: FlowGraph,
+        target_flow_id: Optional[int] = None) -> Tuple[FlowGraph, Dict[Tuple[int, int], int]]:
+    # Validate input parameters
+    _validate_input(flow_graphs)
+
+    # Generate a unique flow ID if not provided
+    if target_flow_id is None:
+        target_flow_id = _generate_unique_flow_id(flow_graphs)
+
+    flow_settings = _create_flow_settings(flow_graphs[0], target_flow_id)
+    combined_graph = FlowGraph(flow_id=target_flow_id, flow_settings=flow_settings)
+    node_id_mapping = _create_node_id_mapping(flow_graphs)
+    _add_nodes_to_combined_graph(flow_graphs, combined_graph, node_id_mapping, target_flow_id)
+    _add_connections_to_combined_graph(flow_graphs, combined_graph, node_id_mapping)
+    return combined_graph, node_id_mapping
+
+
+def combine_flow_graphs(*flow_graphs: FlowGraph, target_flow_id: Optional[int] = None) -> FlowGraph:
+    """
+    Combine multiple flow graphs into a single graph, ensuring node IDs don't overlap.
+
+    Args:
+        *flow_graphs: Multiple FlowGraph instances to combine
+        target_flow_id: Optional ID for the new combined graph. If None, a new ID will be generated.
+
+    Returns:
+        A new FlowGraph containing all nodes and edges from the input graphs with remapped IDs
+
+    Raises:
+        ValueError: If no flow graphs are provided
+    """
+    # Validate input parameters
+    _validate_input(flow_graphs)
+
+    # Generate a unique flow ID if not provided
+    if target_flow_id is None:
+        target_flow_id = _generate_unique_flow_id(flow_graphs)
+
+    flow_settings = _create_flow_settings(flow_graphs[0], target_flow_id)
+    combined_graph = FlowGraph(flow_id=target_flow_id, flow_settings=flow_settings)
+    node_id_mapping = _create_node_id_mapping(flow_graphs)
+    _add_nodes_to_combined_graph(flow_graphs, combined_graph, node_id_mapping, target_flow_id)
+    _add_connections_to_combined_graph(flow_graphs, combined_graph, node_id_mapping)
+
+    return combined_graph
+
+
+def _validate_input(flow_graphs: Tuple[FlowGraph, ...]) -> None:
+    """
+    Validate input parameters.
+
+    Args:
+        flow_graphs: Flow graphs to validate
+
+    Raises:
+        ValueError: If validation fails
+    """
+    if not flow_graphs:
+        raise ValueError("At least one FlowGraph must be provided")
+
+    # Check for duplicate flow IDs
+    flow_ids = [fg.flow_id for fg in flow_graphs]
+    if len(flow_ids) != len(set(flow_ids)):
+        raise ValueError("Cannot combine flows with duplicate flow IDs")
+
+
+def _generate_unique_flow_id(flow_graphs: Tuple[FlowGraph, ...]) -> int:
+    """
+    Generate a unique flow ID based on the input flow graphs.
+
+    Args:
+        flow_graphs: Flow graphs to generate ID from
+
+    Returns:
+        int: A new unique flow ID
+    """
+    return abs(hash(tuple(fg.flow_id for fg in flow_graphs))) % 1000000
+
+
+def _create_flow_settings(base_flow_graph: FlowGraph, target_flow_id: int) -> schemas.FlowSettings:
+    """
+    Create flow settings for the combined graph based on an existing graph.
+
+    Args:
+        base_flow_graph: Flow graph to base settings on
+        target_flow_id: The new flow ID
+
+    Returns:
+        schemas.FlowSettings: Flow settings for the combined graph
+    """
+    flow_settings = deepcopy(base_flow_graph.flow_settings)
+    flow_settings.flow_id = target_flow_id
+    flow_settings.name = f"Combined Flow {target_flow_id}"
+    return flow_settings
+
+
+def _create_node_id_mapping(flow_graphs: Tuple[FlowGraph, ...]) -> Dict[Tuple[int, int], int]:
+    """
+    Create a mapping from (flow_id, original_node_id) to new unique node IDs.
+
+    Args:
+        flow_graphs: Flow graphs to process
+
+    Returns:
+        Dict: Mapping from (flow_id, node_id) to new node ID
+    """
+    node_id_mapping = {}
+    next_node_id = _get_next_available_node_id(flow_graphs)
+
+    for fg in flow_graphs:
+        for node in fg.nodes:
+            node_id_mapping[(fg.flow_id, node.node_id)] = next_node_id
+            next_node_id += 1
+
+    return node_id_mapping
+
+
+def _get_next_available_node_id(flow_graphs: Tuple[FlowGraph, ...]) -> int:
+    """
+    Find the next available node ID.
+
+    Args:
+        flow_graphs: Flow graphs to examine
+
+    Returns:
+        int: Next available node ID
+    """
+    max_id = 0
+    for fg in flow_graphs:
+        for node in fg.nodes:
+            max_id = max(max_id, node.node_id)
+    return max_id + 1
+
+
+def _add_nodes_to_combined_graph(
+        flow_graphs: Tuple[FlowGraph, ...],
+        combined_graph: FlowGraph,
+        node_id_mapping: Dict[Tuple[int, int], int],
+        target_flow_id: int
+) -> None:
+    """
+    Add all nodes from source graphs to the combined graph.
+
+    Args:
+        flow_graphs: Source flow graphs
+        combined_graph: Target combined graph
+        node_id_mapping: Mapping of node IDs
+        target_flow_id: Target flow ID
+    """
+    processed_nodes = set()
+
+    for fg in flow_graphs:
+        for node in fg.nodes:
+            # Skip if already processed
+            if (fg.flow_id, node.node_id) in processed_nodes:
+                continue
+
+            # Generate new node ID
+            new_node_id = node_id_mapping[(fg.flow_id, node.node_id)]
+
+            # Create and update setting input
+            setting_input = _create_updated_setting_input(
+                node.setting_input,
+                new_node_id,
+                target_flow_id,
+                fg.flow_id,
+                node_id_mapping
+            )
+
+            # Add node to combined graph
+            _add_node_to_graph(combined_graph, new_node_id, target_flow_id, node.node_type, setting_input)
+
+            processed_nodes.add((fg.flow_id, node.node_id))
+
+
+def _create_updated_setting_input(
+        original_setting_input: any,
+        new_node_id: int,
+        target_flow_id: int,
+        source_flow_id: int,
+        node_id_mapping: Dict[Tuple[int, int], int]
+) -> any:
+    """
+    Create an updated setting input with new node and flow IDs.
+
+    Args:
+        original_setting_input: Original setting input
+        new_node_id: New node ID
+        target_flow_id: Target flow ID
+        source_flow_id: Source flow ID
+        node_id_mapping: Mapping of node IDs
+
+    Returns:
+        Updated setting input
+    """
+    setting_input = deepcopy(original_setting_input)
+
+    # Update node ID
+    if hasattr(setting_input, 'node_id'):
+        setting_input.node_id = new_node_id
+
+    # Update flow ID
+    if hasattr(setting_input, 'flow_id'):
+        setting_input.flow_id = target_flow_id
+
+    # Update depending_on_id if present
+    if hasattr(setting_input, 'depending_on_id') and setting_input.depending_on_id != -1:
+        orig_depending_id = setting_input.depending_on_id
+        setting_input.depending_on_id = node_id_mapping.get((source_flow_id, orig_depending_id), -1)
+
+    # Update depending_on_ids list if present
+    if hasattr(setting_input, 'depending_on_ids'):
+        setting_input.depending_on_ids = [
+            node_id_mapping.get((source_flow_id, dep_id), -1)
+            for dep_id in setting_input.depending_on_ids
+            if dep_id != -1
+        ]
+
+    return setting_input
+
+
+def _add_node_to_graph(
+        graph: FlowGraph,
+        node_id: int,
+        flow_id: int,
+        node_type: str,
+        setting_input: any
+) -> None:
+    """
+    Add a node to the graph.
+
+    Args:
+        graph: Target graph
+        node_id: Node ID
+        flow_id: Flow ID
+        node_type: Node type
+        setting_input: Setting input
+    """
+    # Add node promise to graph
+    node_promise = input_schema.NodePromise(
+        node_id=node_id,
+        flow_id=flow_id,
+        node_type=node_type,
+        is_setup=True,
+        pos_x=getattr(setting_input, 'pos_x', 0),
+        pos_y=getattr(setting_input, 'pos_y', 0),
+        description=getattr(setting_input, 'description', '')
+    )
+    graph.add_node_promise(node_promise)
+
+    # Get node type-specific add method
+    add_method_name = f"add_{node_type}"
+    if hasattr(graph, add_method_name):
+        add_method = getattr(graph, add_method_name)
+        add_method(setting_input)
+
+
+def _add_connections_to_combined_graph(
+        flow_graphs: Tuple[FlowGraph, ...],
+        combined_graph: FlowGraph,
+        node_id_mapping: Dict[Tuple[int, int], int]
+) -> None:
+    """
+    Add all connections from source graphs to the combined graph.
+
+    Args:
+        flow_graphs: Source flow graphs
+        combined_graph: Target combined graph
+        node_id_mapping: Mapping of node IDs
+    """
+    for fg in flow_graphs:
+        for connection in fg.node_connections:
+            source_id, target_id = connection
+            new_source_id = node_id_mapping.get((fg.flow_id, source_id))
+            new_target_id = node_id_mapping.get((fg.flow_id, target_id))
+
+            if new_source_id is not None and new_target_id is not None:
+                input_type = _determine_connection_input_type(fg, source_id, target_id)
+
+                # Create connection in combined graph
+                node_connection = input_schema.NodeConnection.create_from_simple_input(
+                    from_id=new_source_id,
+                    to_id=new_target_id,
+                    input_type=input_type
+                )
+                add_connection(combined_graph, node_connection)
+
+
+def _determine_connection_input_type(
+        flow_graph: FlowGraph,
+        source_id: int,
+        target_id: int
+) -> str:
+    """
+    Determine the input type for a connection.
+
+    Args:
+        flow_graph: Source flow graph
+        source_id: Source node ID
+        target_id: Target node ID
+
+    Returns:
+        str: Input type (main, left, right)
+    """
+    from_node = flow_graph.get_node(source_id)
+    to_node = flow_graph.get_node(target_id)
+
+    if from_node and to_node:
+        input_types = to_node.get_input_type(from_node.node_id)
+        if input_types:
+            return input_types[0]
+
+    return "main"

flowfile_core/flowfile/flow_node/flow_node.py

@@ -146,6 +146,7 @@ class FlowNode:
         self.node_settings.renew_schema = True
         if hasattr(setting_input, 'cache_results'):
             self.node_settings.cache_results = setting_input.cache_results
+
         self.setting_input = setting_input
         self.results.errors = None
         self.add_lead_to_in_depend_source()
@@ -174,7 +175,7 @@ class FlowNode:
         self.set_node_information()
         if self.node_type == 'manual_input' and isinstance(self._setting_input, input_schema.NodeManualInput):
             if self.hash != self.calculate_hash(setting_input) or not self.node_stats.has_run:
-                self.function = self.function.__class__(setting_input.raw_data)
+                self.function = self.function.__class__(setting_input.raw_data_format)
                 self.reset()
                 self.get_predicted_schema()
         elif self._setting_input is not None:

flowfile_core/flowfile/sources/external_sources/airbyte_sources/models.py

@@ -1,7 +1,7 @@
 from typing import Any, Dict, List, Optional, TYPE_CHECKING, Union
 from pydantic import BaseModel, field_validator, ConfigDict
 import polars as pl
-from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import type_to_polars_str
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import cast_str_to_polars_type
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
 from flowfile_core.schemas.input_schema import MinimalFieldInfo
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.polars_type import PlType
@@ -56,7 +56,7 @@ class JsonSchema(BaseModel):
                 dtype = 'string'
             else:
                 dtype = type_mapping.get(self.type[0] if isinstance(self.type, list) else self.type, 'string')
-        return type_to_polars_str(dtype)
+        return cast_str_to_polars_type(dtype)
 
 
 class AirbyteProperty(BaseModel):

flowfile_core/flowfile/sources/external_sources/custom_external_sources/__init__.py

@@ -1,2 +1 @@
-from flowfile_core.flowfile.sources.external_sources.custom_external_sources import google_sheet
 from flowfile_core.flowfile.sources.external_sources.custom_external_sources import sample_users

flowfile_core/flowfile/sources/external_sources/sql_source/sql_source.py

@@ -4,7 +4,7 @@ from flowfile_core.configs import logger
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
 from flowfile_core.schemas.input_schema import MinimalFieldInfo, DatabaseSettings
 from sqlalchemy import Engine, inspect, create_engine, text
-from flowfile_core.secrets.secrets import get_encrypted_secret, decrypt_secret
+from flowfile_core.secret_manager.secret_manager import get_encrypted_secret, decrypt_secret
 
 from flowfile_core.flowfile.sources.external_sources.base_class import ExternalDataSource
 from flowfile_core.flowfile.sources.external_sources.sql_source.utils import get_polars_type, construct_sql_uri

flowfile_core/flowfile/utils.py

@@ -1,9 +1,15 @@
-from typing import List
 import os
-import shutil
 import hashlib
-from datetime import datetime
 import json
+import polars as pl
+import shutil
+
+from datetime import datetime, date, time
+from typing import List
+from decimal import Decimal
+
+from flowfile_core.flowfile.flow_data_engine.utils import standardize_col_dtype
+from flowfile_core.schemas import input_schema
 
 
 def generate_sha256_hash(data: bytes):
@@ -26,8 +32,16 @@ def snake_case_to_camel_case(text: str) -> str:
 def json_default(val):
     if isinstance(val, datetime):
         return val.isoformat(timespec='microseconds')
+    elif isinstance(val, date):
+        return val.isoformat()
+    elif isinstance(val, time):
+        return val.isoformat()
     elif hasattr(val, '__dict__'):
         return val.__dict__
+    elif isinstance(val, Decimal):
+        if val.as_integer_ratio()[1] == 1:
+            return int(val)
+        return float(val)
     else:
         raise Exception('Value is not serializable')
 
@@ -104,3 +118,20 @@ def batch_generator(input_list: List, batch_size: int = 10000):
             input_list = []
             run = False
 
+
+def _handle_raw_data(node_manual_input: input_schema.NodeManualInput):
+    """Ensure compatibility with the new typed raw data and the old dict form data type"""
+    if (not (hasattr(node_manual_input, "raw_data_format") and node_manual_input.raw_data_format)
+            and (hasattr(node_manual_input, 'raw_data') and node_manual_input.raw_data)):
+        values = [standardize_col_dtype([vv for vv in c]) for c in zip(*(r.values()
+                                                                         for r in node_manual_input.raw_data))]
+        data_types = (pl.DataType.from_python(type(next((v for v in column_values), None))) for column_values in values)
+        _columns = [input_schema.MinimalFieldInfo(name=c, data_type=str(next(data_types))) for c in
+                    node_manual_input.raw_data[0].keys()]
+
+        node_manual_input.raw_data_format = input_schema.RawData(columns=_columns, data=values)
+    elif ((hasattr(node_manual_input, "raw_data_format") and node_manual_input.raw_data_format)
+          and not (hasattr(node_manual_input, 'raw_data') and node_manual_input.raw_data)):
+        node_manual_input.raw_data = [{c.name: node_manual_input.raw_data_format.data[ci][ri] for ci, c in
+                                       enumerate(node_manual_input.raw_data_format.columns)}
+                                      for ri in range(len(node_manual_input.raw_data_format.data[0]))]

flowfile_core/main.py

@@ -8,7 +8,8 @@ from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
 from flowfile_core import ServerRun
-from flowfile_core.configs.settings import SERVER_HOST, SERVER_PORT, WORKER_HOST, WORKER_PORT, WORKER_URL
+from flowfile_core.configs.settings import (SERVER_HOST, SERVER_PORT, WORKER_HOST, WORKER_PORT, WORKER_URL,
+                                            OFFLOAD_TO_WORKER)
 
 from flowfile_core.routes.auth import router as auth_router
 from flowfile_core.routes.secrets import router as secrets_router
@@ -107,7 +108,6 @@ def run(host: str = None, port: int = None):
         host = SERVER_HOST
     if port is None:
         port = SERVER_PORT
-
     print(f"Starting server on {host}:{port}")
     print(f"Worker configured at {WORKER_URL} (host: {WORKER_HOST}, port: {WORKER_PORT})")
 
@@ -120,7 +120,6 @@ def run(host: str = None, port: int = None):
         host=host,
         port=port,
         loop="asyncio",
-        log_level="warning",
     )
     server = uvicorn.Server(config)
     server_instance = server  # Store server instance globally

flowfile_core/routes/secrets.py

@@ -10,7 +10,7 @@ from flowfile_core.auth.jwt import get_current_active_user
 from flowfile_core.auth.models import Secret, SecretInput
 from flowfile_core.database import models as db_models
 from flowfile_core.database.connection import get_db
-from flowfile_core.secrets.secrets import encrypt_secret, store_secret, delete_secret as delete_secret_action
+from flowfile_core.secret_manager.secret_manager import encrypt_secret, store_secret, delete_secret as delete_secret_action
 
 router = APIRouter(dependencies=[Depends(get_current_active_user)])
 

flowfile_core/schemas/input_schema.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Literal
+from typing import List, Optional, Literal, Iterator
 from flowfile_core.schemas import transform_schema
 from pathlib import Path
 import os
@@ -15,6 +15,7 @@ InputConnectionClass = Literal['input-0', 'input-1', 'input-2', 'input-3', 'inpu
 
 InputType = Literal["main", "left", "right"]
 
+
 class NewDirectory(BaseModel):
     source_path: str
     dir_name: str
@@ -60,7 +61,7 @@ class ReceivedTableBase(BaseModel):
             return self.path
 
     def set_absolute_filepath(self):
-        base_path = Path(self.path)
+        base_path = Path(self.path).expanduser()
         # Check if the path is relative, resolve it with the current working directory
         if not base_path.is_absolute():
             base_path = Path.cwd() / base_path
@@ -96,7 +97,7 @@ class ReceivedJsonTable(ReceivedCsvTable):
     pass
 
 
-class ReceivedParquetTable(BaseModel):
+class ReceivedParquetTable(ReceivedTableBase):
     file_type: str = 'parquet'
 
 
@@ -246,8 +247,14 @@ class NodeDatasource(NodeBase):
     file_ref: str = None
 
 
+class RawData(BaseModel):
+    columns: List[MinimalFieldInfo] = None
+    data: List[List]  # List of list where each inner list is a column of data. This ensures more efficient storage
+
+
 class NodeManualInput(NodeBase):
-    raw_data: List = None
+    raw_data: Optional[List] = None
+    raw_data_format: Optional[RawData] = None
 
 
 class NodeRead(NodeBase):
@@ -341,15 +348,6 @@ class SampleUsers(ExternalSource):
     size: int = 100
 
 
-class GoogleSheet(ExternalSource):
-    GOOGLE_SHEET: bool
-    class_name: str = "google_sheet"
-    access_token: SecretStr = None
-    sheet_id: str
-    worksheet_name: str
-    sheet_name: str
-
-
 class AirbyteReader(AirbyteConfig):
     class_name: Optional[str] = "airbyte_reader"
     fields: Optional[List[MinimalFieldInfo]] = None
@@ -362,7 +360,7 @@ class AccessToken(BaseModel):
 
 class NodeExternalSource(NodeBase):
     identifier: str
-    source_settings: SampleUsers | GoogleSheet
+    source_settings: SampleUsers
 
 
 class NodeAirbyteReader(NodeExternalSource):