Flowfile 0.3.4.1__py3-none-any.whl → 0.3.6__py3-none-any.whl

flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py

@@ -1,9 +1,10 @@
+
 from dataclasses import dataclass
-from typing import Optional, Any, List, Dict, Literal
+from typing import Optional, Any, List, Dict, Literal, Iterable
+
 from flowfile_core.schemas import input_schema
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import cast_str_to_polars_type
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.polars_type import PlType
-from polars import datatypes
 import polars as pl
 # TODO: rename flow_file_column to flowfile_column
 DataTypeGroup = Literal['numeric', 'str', 'date']
@@ -175,3 +176,12 @@ def convert_stats_to_column_info(stats: List[Dict]) -> List[FlowfileColumn]:
 def convert_pl_schema_to_raw_data_format(pl_schema: pl.Schema) -> List[input_schema.MinimalFieldInfo]:
     return [FlowfileColumn.create_from_polars_type(PlType(column_name=k, pl_datatype=v)).get_minimal_field_info()
             for k, v in pl_schema.items()]
+
+
+def assert_if_flowfile_schema(obj: Iterable) -> bool:
+    """
+    Assert that the object is a valid iterable of FlowfileColumn objects.
+    """
+    if isinstance(obj, (list, set, tuple)):
+        return all(isinstance(item, FlowfileColumn) for item in obj)
+    return False

flowfile_core/flowfile/flow_data_engine/fuzzy_matching/settings_validator.py

@@ -32,7 +32,7 @@ def calculate_fuzzy_match_schema(fm_input: transform_schema.FuzzyMatchInput,
             output_schema.append(FlowfileColumn.from_input(column.new_name, column_schema.data_type,
                                                            example_values=column_schema.example_values))
 
-    for i, fm in enumerate(fm_input.join_mappings):
+    for i, fm in enumerate(fm_input.join_mapping):
         output_schema.append(FlowfileColumn.from_input(f'fuzzy_score_{i}', 'Float64'))
     return output_schema
 

flowfile_core/flowfile/flow_data_engine/join/__init__.py

@@ -1 +1,2 @@
-from flowfile_core.flowfile.flow_data_engine.join.verify_integrity import *
+from flowfile_core.flowfile.flow_data_engine.join.verify_integrity import *
+from flowfile_core.flowfile.flow_data_engine.join.utils import *

flowfile_core/flowfile/flow_data_engine/join/utils.py

@@ -0,0 +1,25 @@
+# Standard library imports
+from typing import Dict, Tuple, TypeVar
+
+# Third-party imports
+import polars as pl
+
+from flowfile_core.schemas import (
+    transform_schema as transform_schemas
+)
+
+T = TypeVar('T', pl.DataFrame, pl.LazyFrame)
+
+
+def rename_df_table_for_join(left_df: T, right_df: T,  join_key_rename: transform_schemas.FullJoinKeyResponse) -> Tuple[T, T]:
+    return (left_df.rename({r[0]: r[1] for r in join_key_rename.left.join_key_renames}),
+            right_df.rename({r[0]: r[1] for r in join_key_rename.right.join_key_renames}))
+
+
+def get_undo_rename_mapping_join(join_input: transform_schemas.JoinInput) -> Dict[str, str]:
+    join_key_rename = join_input.get_join_key_renames(True)
+    return {r[1]: r[0] for r in join_key_rename.right.join_key_renames + join_key_rename.left.join_key_renames}
+
+
+def get_col_name_to_delete(col: transform_schemas.SelectInput, side: transform_schemas.SideLit):
+    return col.new_name if not col.join_key else transform_schemas.construct_join_key_name(side, col.new_name)

flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py

@@ -18,9 +18,9 @@ from flowfile_core.flowfile.flow_data_engine.subprocess_operations.models import
     PolarsOperation,
     Status
 )
-from flowfile_core.flowfile.sources.external_sources.airbyte_sources.models import AirbyteSettings
 from flowfile_core.flowfile.sources.external_sources.sql_source.models import (DatabaseExternalReadSettings,
                                                                                DatabaseExternalWriteSettings)
+from flowfile_core.schemas.cloud_storage_schemas import CloudStorageWriteSettingsWorkerInterface
 from flowfile_core.schemas.input_schema import (
     ReceivedCsvTable,
     ReceivedExcelTable,
@@ -81,13 +81,6 @@ def trigger_create_operation(flow_id: int, node_id: int | str, received_table: R
     return Status(**f.json())
 
 
-def trigger_airbyte_collector(airbyte_settings: AirbyteSettings):
-    f = requests.post(url=f'{WORKER_URL}/store_airbyte_result', data=airbyte_settings.model_dump_json())
-    if not f.ok:
-        raise Exception(f'Could not cache the data, {f.text}')
-    return Status(**f.json())
-
-
 def trigger_database_read_collector(database_external_read_settings: DatabaseExternalReadSettings):
     f = requests.post(url=f'{WORKER_URL}/store_database_read_result',
                       data=database_external_read_settings.model_dump_json())
@@ -104,6 +97,14 @@ def trigger_database_write(database_external_write_settings: DatabaseExternalWri
     return Status(**f.json())
 
 
+def trigger_cloud_storage_write(database_external_write_settings: CloudStorageWriteSettingsWorkerInterface):
+    f = requests.post(url=f'{WORKER_URL}/write_data_to_cloud',
+                      data=database_external_write_settings.model_dump_json())
+    if not f.ok:
+        raise Exception(f'Could not cache the data, {f.text}')
+    return Status(**f.json())
+
+
 def get_results(file_ref: str) -> Status | None:
     f = requests.get(f'{WORKER_URL}/status/{file_ref}')
     if f.status_code == 200:
@@ -113,11 +114,15 @@ def get_results(file_ref: str) -> Status | None:
 
 
 def results_exists(file_ref: str):
-    f = requests.get(f'{WORKER_URL}/status/{file_ref}')
-    if f.status_code == 200:
-        if f.json()['status'] == 'Completed':
-            return True
-    return False
+    try:
+        f = requests.get(f'{WORKER_URL}/status/{file_ref}')
+        if f.status_code == 200:
+            if f.json()['status'] == 'Completed':
+                return True
+        return False
+    except requests.RequestException as e:
+        logger.error(f"Failed to check results existence: {str(e)}")
+        return False
 
 
 def get_df_result(encoded_df: str) -> pl.LazyFrame:
@@ -336,15 +341,6 @@ class ExternalCreateFetcher(BaseFetcher):
             _ = self.get_result()
 
 
-class ExternalAirbyteFetcher(BaseFetcher):
-    def __init__(self, airbyte_settings: AirbyteSettings, wait_on_completion: bool = True):
-        r = trigger_airbyte_collector(airbyte_settings)
-        super().__init__(file_ref=r.background_task_id)
-        self.running = r.status == 'Processing'
-        if wait_on_completion:
-            _ = self.get_result()
-
-
 class ExternalDatabaseFetcher(BaseFetcher):
     def __init__(self, database_external_read_settings: DatabaseExternalReadSettings,
                  wait_on_completion: bool = True):
@@ -365,6 +361,17 @@ class ExternalDatabaseWriter(BaseFetcher):
             _ = self.get_result()
 
 
+class ExternalCloudWriter(BaseFetcher):
+
+    def __init__(self, cloud_storage_write_settings: CloudStorageWriteSettingsWorkerInterface,
+                 wait_on_completion: bool = True):
+        r = trigger_cloud_storage_write(database_external_write_settings=cloud_storage_write_settings)
+        super().__init__(file_ref=r.background_task_id)
+        self.running = r.status == 'Processing'
+        if wait_on_completion:
+            _ = self.get_result()
+
+
 class ExternalExecutorTracker:
     result: Optional[pl.LazyFrame]
     started: bool = False

flowfile_core/flowfile/flow_data_engine/utils.py

@@ -3,30 +3,13 @@ from flowfile_core.configs.settings import AVAILABLE_RAM, WORKER_URL
 from flowfile_core.configs import logger
 from flowfile_core.flowfile.flow_data_engine.subprocess_operations import ExternalDfFetcher
 from flowfile_core.flowfile.flow_data_engine.subprocess_operations import Status
+from flowfile_core.utils.utils import standardize_col_dtype
 import os
 from typing import List, Dict, Iterable, Callable, Any
-from itertools import chain
 import requests
 from base64 import encodebytes
 
 
-def convert_to_string(v):
-    try:
-        return str(v)
-    except:
-        return None
-
-
-def standardize_col_dtype(vals):
-    types = set(type(val) for val in vals)
-    if len(types) == 1:
-        return vals
-    elif int in types and float in types:
-        return vals
-    else:
-        return [convert_to_string(v) for v in vals]
-
-
 def get_data_type(vals: Iterable[Any]):
     types = set(type(val) for val in vals)
     if len(types) == 1:
@@ -37,28 +20,6 @@ def get_data_type(vals: Iterable[Any]):
         return 'str'
 
 
-def ensure_similarity_dicts(datas: List[Dict], respect_order: bool = True):
-    all_cols = (data.keys() for data in datas)
-    if not respect_order:
-        unique_cols = set(chain(*all_cols))
-    else:
-        col_store = set()
-        unique_cols = list()
-        for row in all_cols:
-            for col in row:
-                if col not in col_store:
-                    unique_cols.append(col)
-                    col_store.update((col,))
-    output = []
-    for data in datas:
-        new_record = dict()
-        for col in unique_cols:
-            val = data.get(col)
-            new_record[col] = val
-        output.append(new_record)
-    return output
-
-
 def calculate_schema(lf: pl.LazyFrame) -> List[Dict]:
     r = ExternalDfFetcher(lf=lf, operation_type='calculate_schema', wait_on_completion=False, flow_id=-1, node_id=-1)
     schema_stats: List[Dict] = r.get_result()

flowfile_core/flowfile/flow_graph.py

@@ -13,10 +13,10 @@ from pyarrow.parquet import ParquetFile
 from flowfile_core.configs import logger
 from flowfile_core.configs.flow_logger import FlowLogger
 from flowfile_core.flowfile.sources.external_sources.factory import data_source_factory
-from flowfile_core.flowfile.sources.external_sources.airbyte_sources.settings import airbyte_settings_from_config
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import cast_str_to_polars_type, FlowfileColumn
 from flowfile_core.flowfile.flow_data_engine.fuzzy_matching.settings_validator import (calculate_fuzzy_match_schema,
                                                                                        pre_calculate_pivot_schema)
+from flowfile_core.flowfile.flow_data_engine.cloud_storage_reader import CloudStorageReader
 from flowfile_core.utils.arrow_reader import get_read_top_n
 from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine, execute_polars_code
 from flowfile_core.flowfile.flow_data_engine.read_excel_tables import get_open_xlsx_datatypes, \
@@ -24,19 +24,22 @@ from flowfile_core.flowfile.flow_data_engine.read_excel_tables import get_open_x
 from flowfile_core.flowfile.sources import external_sources
 from flowfile_core.schemas import input_schema, schemas, transform_schema
 from flowfile_core.schemas.output_model import TableExample, NodeData, NodeResult, RunInformation
-from flowfile_core.flowfile.utils import snake_case_to_camel_case, _handle_raw_data
+from flowfile_core.schemas.cloud_storage_schemas import (CloudStorageReadSettingsInternal, FullCloudStorageConnection,
+                                                         get_cloud_storage_write_settings_worker_interface, AuthMethod)
+from flowfile_core.flowfile.utils import snake_case_to_camel_case
 from flowfile_core.flowfile.analytics.utils import create_graphic_walker_node_from_node_promise
 from flowfile_core.flowfile.flow_node.flow_node import FlowNode
 from flowfile_core.flowfile.util.execution_orderer import determine_execution_order
 from flowfile_core.flowfile.flow_data_engine.polars_code_parser import polars_code_parser
-from flowfile_core.flowfile.flow_data_engine.subprocess_operations.subprocess_operations import (ExternalAirbyteFetcher,
-                                                                                                 ExternalDatabaseFetcher,
+from flowfile_core.flowfile.flow_data_engine.subprocess_operations.subprocess_operations import (ExternalDatabaseFetcher,
                                                                                                  ExternalDatabaseWriter,
-                                                                                                 ExternalDfFetcher)
+                                                                                                 ExternalDfFetcher,
+                                                                                                 ExternalCloudWriter)
 from flowfile_core.secret_manager.secret_manager import get_encrypted_secret, decrypt_secret
 from flowfile_core.flowfile.sources.external_sources.sql_source import utils as sql_utils, models as sql_models
 from flowfile_core.flowfile.sources.external_sources.sql_source.sql_source import SqlSource, BaseSqlSource
-from flowfile_core.flowfile.database_connection_manager.db_connections import get_local_database_connection
+from flowfile_core.flowfile.database_connection_manager.db_connections import (get_local_database_connection,
+                                                                               get_local_cloud_connection)
 from flowfile_core.flowfile.util.calculate_layout import calculate_layered_layout
 
 
@@ -80,6 +83,16 @@ def get_xlsx_schema_callback(engine: str, file_path: str, sheet_name: str, start
                    start_column=start_column, end_row=end_row, end_column=end_column, has_headers=has_headers)
 
 
+def get_cloud_connection_settings(connection_name: str, user_id: int, auth_mode: AuthMethod) -> FullCloudStorageConnection:
+    cloud_connection_settings = get_local_cloud_connection(connection_name, user_id)
+    if cloud_connection_settings is None and auth_mode == "aws-cli":
+        # If the auth mode is aws-cli, we do not need connection settings
+        cloud_connection_settings = FullCloudStorageConnection(storage_type="s3", auth_method="aws-cli")
+    if cloud_connection_settings is None:
+        raise HTTPException(status_code=400, detail="Cloud connection settings not found")
+    return cloud_connection_settings
+
+
 class FlowGraph:
     """
        FlowGraph is a class that enables Extract, Transform and Load (ETL) operations
@@ -656,7 +669,7 @@ class FlowGraph:
                       setting_input: Any = None,
                       cache_results: bool = None,
                       schema_callback: Callable = None,
-                      input_node_ids: List[int] = None):
+                      input_node_ids: List[int] = None) -> FlowNode:
         existing_node = self.get_node(node_id)
         if existing_node is not None:
             if existing_node.node_type != node_type:
@@ -668,14 +681,13 @@ class FlowGraph:
             input_nodes = [self.get_node(node_id) for node_id in input_node_ids]
         else:
             input_nodes = None
-        if cache_results is None:
-            if hasattr(setting_input, 'cache_results'):
-                cache_results = getattr(setting_input, 'cache_results')
-                cache_results = False if cache_results is None else cache_results
         if isinstance(input_columns, str):
             input_columns = [input_columns]
-
-        if input_nodes is not None or function.__name__ in ('placeholder', 'analysis_preparation'):
+        if (
+                input_nodes is not None or
+                function.__name__ in ('placeholder', 'analysis_preparation') or
+                node_type == "cloud_storage_reader"
+        ):
 
             if not existing_node:
                 node = FlowNode(node_id=node_id,
@@ -703,6 +715,7 @@ class FlowGraph:
             raise Exception("No data initialized")
         self._node_db[node_id] = node
         self._node_ids.append(node_id)
+        return node
 
     def add_include_cols(self, include_columns: List[str]):
         for column in include_columns:
@@ -854,80 +867,107 @@ class FlowGraph:
             self._flow_starts.append(node)
             self._node_ids.append(node_database_reader.node_id)
 
-    def add_airbyte_reader(self, external_source_input: input_schema.NodeAirbyteReader):
-        logger.info('Adding airbyte reader')
-        node_type = 'airbyte_reader'
-        source_settings: input_schema.AirbyteReader = external_source_input.source_settings
-        airbyte_settings = airbyte_settings_from_config(source_settings, flow_id=self.flow_id,
-                                                        node_id=external_source_input.node_id)
+    def add_sql_source(self, external_source_input: input_schema.NodeExternalSource):
+        logger.info('Adding sql source')
+        self.add_external_source(external_source_input)
 
-        logger.info("Airbyte settings created")
-        airbyte_settings.fields = source_settings.fields
-        external_source = data_source_factory(source_type='airbyte', airbyte_settings=airbyte_settings)
+    def add_cloud_storage_writer(self, node_cloud_storage_writer: input_schema.NodeCloudStorageWriter) -> None:
 
-        def _func():
-            logger.info('Calling external source')
-            external_fetcher = ExternalAirbyteFetcher(airbyte_settings, wait_on_completion=False)
-            node._fetch_cached_df = external_fetcher
-            fl = FlowDataEngine(external_fetcher.get_result())
-            external_source_input.source_settings.fields = [c.get_minimal_field_info() for c in fl.schema]
-            return fl
+        node_type = "cloud_storage_writer"
+
+        def _func(df: FlowDataEngine):
+            df.lazy = True
+            cloud_connection_settings = get_cloud_connection_settings(
+                connection_name=node_cloud_storage_writer.cloud_storage_settings.connection_name,
+                user_id=node_cloud_storage_writer.user_id,
+                auth_mode=node_cloud_storage_writer.cloud_storage_settings.auth_mode
+            )
+            full_cloud_storage_connection = FullCloudStorageConnection(
+                storage_type=cloud_connection_settings.storage_type,
+                auth_method=cloud_connection_settings.auth_method,
+                aws_allow_unsafe_html=cloud_connection_settings.aws_allow_unsafe_html,
+                **CloudStorageReader.get_storage_options(cloud_connection_settings)
+            )
+            settings = get_cloud_storage_write_settings_worker_interface(
+                write_settings=node_cloud_storage_writer.cloud_storage_settings,
+                connection=full_cloud_storage_connection,
+                lf=df.data_frame,
+                flowfile_node_id=node_cloud_storage_writer.node_id,
+                flowfile_flow_id=self.flow_id)
+            external_database_writer = ExternalCloudWriter(settings, wait_on_completion=False)
+            node._fetch_cached_df = external_database_writer
+            external_database_writer.get_result()
+            return df
 
         def schema_callback():
-            return [FlowfileColumn.from_input(f.name, f.data_type) for f in external_source.schema]
+            logger.info("Starting to run the schema callback for cloud storage writer")
+            if self.get_node(node_cloud_storage_writer.node_id).is_correct:
+                return self.get_node(node_cloud_storage_writer.node_id).node_inputs.main_inputs[0].schema
+            else:
+                return [FlowfileColumn.from_input(column_name="__error__", data_type="String")]
 
-        node = self.get_node(external_source_input.node_id)
-        if node:
-            node.node_type = node_type
-            node.name = node_type
-            node.function = _func
-            node.setting_input = external_source_input
-            node.node_settings.cache_results = external_source_input.cache_results
-            if external_source_input.node_id not in set(start_node.node_id for start_node in self._flow_starts):
-                self._flow_starts.append(node)
-            node.schema_callback = schema_callback
-        else:
-            node = FlowNode(external_source_input.node_id, function=_func,
-                            setting_input=external_source_input,
-                            name=node_type, node_type=node_type, parent_uuid=self.uuid,
-                            schema_callback=schema_callback)
-            self._node_db[external_source_input.node_id] = node
-            self._flow_starts.append(node)
-            self._node_ids.append(external_source_input.node_id)
-        if external_source_input.source_settings.fields and len(external_source_input.source_settings.fields) > 0:
-            logger.info('Using provided schema in the node')
+        self.add_node_step(
+            node_id=node_cloud_storage_writer.node_id,
+            function=_func,
+            input_columns=[],
+            node_type=node_type,
+            setting_input=node_cloud_storage_writer,
+            schema_callback=schema_callback,
+            input_node_ids=[node_cloud_storage_writer.depending_on_id]
+        )
 
+        node = self.get_node(node_cloud_storage_writer.node_id)
 
-    def add_sql_source(self, external_source_input: input_schema.NodeExternalSource):
-        logger.info('Adding sql source')
-        self.add_external_source(external_source_input)
+    def add_cloud_storage_reader(self, node_cloud_storage_reader: input_schema.NodeCloudStorageReader) -> None:
+        """
+        Adds a cloud storage read node to the flow graph.
+        Args:
+            node_cloud_storage_reader (input_schema.NodeCloudStorageReader):
+            The settings for the cloud storage read node.
+        Returns:
+        """
+        node_type = "cloud_storage_reader"
+        logger.info("Adding cloud storage reader")
+        cloud_storage_read_settings = node_cloud_storage_reader.cloud_storage_settings
+
+        def _func():
+            logger.info("Starting to run the schema callback for cloud storage reader")
+            self.flow_logger.info("Starting to run the schema callback for cloud storage reader")
+            settings = CloudStorageReadSettingsInternal(read_settings=cloud_storage_read_settings,
+                                                        connection=get_cloud_connection_settings(
+                                                            connection_name=cloud_storage_read_settings.connection_name,
+                                                            user_id=node_cloud_storage_reader.user_id,
+                                                            auth_mode=cloud_storage_read_settings.auth_mode
+                                                        ))
+            fl = FlowDataEngine.from_cloud_storage_obj(settings)
+            return fl
+
+        node = self.add_node_step(node_id=node_cloud_storage_reader.node_id,
+                                  function=_func,
+                                  cache_results=node_cloud_storage_reader.cache_results,
+                                  setting_input=node_cloud_storage_reader,
+                                  node_type=node_type,
+                                  )
+        if node_cloud_storage_reader.node_id not in set(start_node.node_id for start_node in self._flow_starts):
+            self._flow_starts.append(node)
 
     def add_external_source(self,
-                            external_source_input: input_schema.NodeExternalSource | input_schema.NodeAirbyteReader):
-
-        custom_source_type = external_source_input.identifier != 'airbyte'
-        if custom_source_type:
-            node_type = 'external_source'
-            external_source_script = getattr(external_sources.custom_external_sources, external_source_input.identifier)
-            source_settings = (getattr(input_schema, snake_case_to_camel_case(external_source_input.identifier)).
-                               model_validate(external_source_input.source_settings))
-            if hasattr(external_source_script, 'initial_getter'):
-                initial_getter = getattr(external_source_script, 'initial_getter')(source_settings)
-            else:
-                initial_getter = None
-            data_getter = external_source_script.getter(source_settings)
-            external_source = data_source_factory(source_type='custom',
-                                                  data_getter=data_getter,
-                                                  initial_data_getter=initial_getter,
-                                                  orientation=external_source_input.source_settings.orientation,
-                                                  schema=None)
+                            external_source_input: input_schema.NodeExternalSource):
+
+        node_type = 'external_source'
+        external_source_script = getattr(external_sources.custom_external_sources, external_source_input.identifier)
+        source_settings = (getattr(input_schema, snake_case_to_camel_case(external_source_input.identifier)).
+                           model_validate(external_source_input.source_settings))
+        if hasattr(external_source_script, 'initial_getter'):
+            initial_getter = getattr(external_source_script, 'initial_getter')(source_settings)
         else:
-            node_type = 'airbyte_reader'
-            source_settings: input_schema.AirbyteReader = external_source_input.source_settings
-            airbyte_settings = airbyte_settings_from_config(source_settings, flow_id=self.flow_id,
-                                                            node_id=external_source_input.node_id)
-            airbyte_settings.fields = source_settings.fields
-            external_source = data_source_factory(source_type='airbyte', airbyte_settings=airbyte_settings)
+            initial_getter = None
+        data_getter = external_source_script.getter(source_settings)
+        external_source = data_source_factory(source_type='custom',
+                                              data_getter=data_getter,
+                                              initial_data_getter=initial_getter,
+                                              orientation=external_source_input.source_settings.orientation,
+                                              schema=None)
 
         def _func():
             logger.info('Calling external source')
@@ -984,8 +1024,8 @@ class FlowGraph:
                 input_data = FlowDataEngine.create_from_path(input_file.received_file)
             else:
                 input_data = FlowDataEngine.create_from_path_worker(input_file.received_file,
-                                                                   node_id=input_file.node_id,
-                                                                   flow_id=self.flow_id)
+                                                                    node_id=input_file.node_id,
+                                                                    flow_id=self.flow_id)
             input_data.name = input_file.received_file.name
             return input_data
 
@@ -1039,7 +1079,6 @@ class FlowGraph:
 
     def add_datasource(self, input_file: input_schema.NodeDatasource | input_schema.NodeManualInput):
         if isinstance(input_file, input_schema.NodeManualInput):
-            _handle_raw_data(input_file)
             input_data = FlowDataEngine(input_file.raw_data_format)
             ref = 'manual_input'
         else:
@@ -1051,10 +1090,8 @@ class FlowGraph:
             node.name = ref
             node.function = input_data
             node.setting_input = input_file
-
             if not input_file.node_id in set(start_node.node_id for start_node in self._flow_starts):
                 self._flow_starts.append(node)
-
         else:
             input_data.collect()
             node = FlowNode(input_file.node_id, function=input_data,