Flowfile 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

flowfile_core/schemas/input_schema.py

@@ -3,8 +3,11 @@ from flowfile_core.schemas import transform_schema
 from pathlib import Path
 import os
 from flowfile_core.schemas.analysis_schemas import graphic_walker_schemas as gs_schemas
-from flowfile_core.schemas.external_sources.airbyte_schemas import AirbyteConfig
+from flowfile_core.schemas.cloud_storage_schemas import CloudStorageReadSettings, CloudStorageWriteSettings
+from flowfile_core.schemas.schemas import SecretRef
+from flowfile_core.utils.utils import ensure_similarity_dicts, standardize_col_dtype
 from pydantic import BaseModel, Field, model_validator, SecretStr, ConfigDict
+import polars as pl
 
 
 OutputConnectionClass = Literal['output-0', 'output-1', 'output-2', 'output-3', 'output-4',
@@ -17,26 +20,31 @@ InputType = Literal["main", "left", "right"]
 
 
 class NewDirectory(BaseModel):
+    """Defines the information required to create a new directory."""
     source_path: str
     dir_name: str
 
 
 class RemoveItem(BaseModel):
+    """Represents a single item to be removed from a directory or list."""
     path: str
     id: int = -1
 
 
 class RemoveItemsInput(BaseModel):
+    """Defines a list of items to be removed."""
     paths: List[RemoveItem]
     source_path: str
 
 
 class MinimalFieldInfo(BaseModel):
+    """Represents the most basic information about a data field (column)."""
     name: str
-    data_type: str
+    data_type: str = "String"
 
 
 class ReceivedTableBase(BaseModel):
+    """Base model for defining a table received from an external source."""
     id: Optional[int] = None
     name: Optional[str]
     path: str  # This can be an absolute or relative path
@@ -49,36 +57,37 @@ class ReceivedTableBase(BaseModel):
 
     @classmethod
     def create_from_path(cls, path: str):
+        """Creates an instance from a file path string."""
         filename = Path(path).name
-        # Create an instance of ReceivedTableBase with the extracted filename and path
         return cls(name=filename, path=path)
 
     @property
     def file_path(self) -> str:
+        """Constructs the full file path from the directory and name."""
         if not self.name in self.path:
             return os.path.join(self.path, self.name)
         else:
             return self.path
 
     def set_absolute_filepath(self):
+        """Resolves the path to an absolute file path."""
         base_path = Path(self.path).expanduser()
-        # Check if the path is relative, resolve it with the current working directory
         if not base_path.is_absolute():
             base_path = Path.cwd() / base_path
-
         if self.name and self.name not in base_path.name:
             base_path = base_path / self.name
-
         self.abs_file_path = str(base_path.resolve())
 
     @model_validator(mode='after')
     def populate_abs_file_path(self):
+        """Ensures the absolute file path is populated after validation."""
         if not self.abs_file_path:
             self.set_absolute_filepath()
         return self
 
 
 class ReceivedCsvTable(ReceivedTableBase):
+    """Defines settings for reading a CSV file."""
     file_type: str = 'csv'
     reference: str = ''
     starting_from_line: int = 0
@@ -94,82 +103,88 @@ class ReceivedCsvTable(ReceivedTableBase):
 
 
 class ReceivedJsonTable(ReceivedCsvTable):
+    """Defines settings for reading a JSON file (inherits from CSV settings)."""
     pass
 
 
 class ReceivedParquetTable(ReceivedTableBase):
+    """Defines settings for reading a Parquet file."""
     file_type: str = 'parquet'
 
 
 class ReceivedExcelTable(ReceivedTableBase):
+    """Defines settings for reading an Excel file."""
     sheet_name: Optional[str] = None
-    start_row: int = 0  # optional
-    start_column: int = 0  # optional
-    end_row: int = 0  # optional
-    end_column: int = 0  # optional
-    has_headers: bool = True  # optional
-    type_inference: bool = False  # optional
+    start_row: int = 0
+    start_column: int = 0
+    end_row: int = 0
+    end_column: int = 0
+    has_headers: bool = True
+    type_inference: bool = False
 
     def validate_range_values(self):
-        # Validate that start and end rows/columns are non-negative integers
+        """Validates that the Excel cell range is logical."""
         for attribute in [self.start_row, self.start_column, self.end_row, self.end_column]:
             if not isinstance(attribute, int) or attribute < 0:
                 raise ValueError("Row and column indices must be non-negative integers")
-
-        # Validate that start is before end if end is specified (non-zero)
         if (self.end_row > 0 and self.start_row > self.end_row) or \
-                (self.end_column > 0 and self.start_column > self.end_column):
-            raise ValueError("Start row/column must not be greater than end row/column if specified")
+           (self.end_column > 0 and self.start_column > self.end_column):
+            raise ValueError("Start row/column must not be greater than end row/column")
 
 
 class ReceivedTable(ReceivedExcelTable, ReceivedCsvTable, ReceivedParquetTable):
+    """A comprehensive model that can represent any type of received table."""
     ...
 
 
 class OutputCsvTable(BaseModel):
+    """Defines settings for writing a CSV file."""
     file_type: str = 'csv'
     delimiter: str = ','
     encoding: str = 'utf-8'
 
 
 class OutputParquetTable(BaseModel):
+    """Defines settings for writing a Parquet file."""
     file_type: str = 'parquet'
 
 
 class OutputExcelTable(BaseModel):
+    """Defines settings for writing an Excel file."""
     file_type: str = 'excel'
     sheet_name: str = 'Sheet1'
 
 
 class OutputSettings(BaseModel):
+    """Defines the complete settings for an output node."""
     name: str
     directory: str
     file_type: str
     fields: Optional[List[str]] = Field(default_factory=list)
     write_mode: str = 'overwrite'
-    output_csv_table: OutputCsvTable
-    output_parquet_table: OutputParquetTable
-    output_excel_table: OutputExcelTable
+    output_csv_table: Optional[OutputCsvTable] = Field(default_factory=OutputCsvTable)
+    output_parquet_table: OutputParquetTable = Field(default_factory=OutputParquetTable)
+    output_excel_table: OutputExcelTable = Field(default_factory=OutputExcelTable)
     abs_file_path: Optional[str] = None
 
     def set_absolute_filepath(self):
+        """Resolves the output directory and name into an absolute path."""
         base_path = Path(self.directory)
-
         if not base_path.is_absolute():
             base_path = Path.cwd() / base_path
-
         if self.name and self.name not in base_path.name:
             base_path = base_path / self.name
-
         self.abs_file_path = str(base_path.resolve())
 
     @model_validator(mode='after')
     def populate_abs_file_path(self):
+        """Ensures the absolute file path is populated after validation."""
         self.set_absolute_filepath()
         return self
 
 
 class NodeBase(BaseModel):
+    """Base model for all nodes in a FlowGraph. Contains common metadata."""
     model_config = ConfigDict(arbitrary_types_allowed=True)
     flow_id: int
     node_id: int
@@ -181,48 +196,51 @@ class NodeBase(BaseModel):
     user_id: Optional[int] = None
     is_flow_output: Optional[bool] = False
 
-    @classmethod
-    def overridden_hash(cls):
-        if getattr(cls, '__hash__'):
-            return BaseModel.__hash__ is not getattr(cls, '__hash__')
-        return False
-
 
 class NodeSingleInput(NodeBase):
+    """A base model for any node that takes a single data input."""
     depending_on_id: Optional[int] = -1
 
 
 class NodeMultiInput(NodeBase):
+    """A base model for any node that takes multiple data inputs."""
     depending_on_ids: Optional[List[int]] = [-1]
 
 
 class NodeSelect(NodeSingleInput):
+    """Settings for a node that selects, renames, and reorders columns."""
     keep_missing: bool = True
     select_input: List[transform_schema.SelectInput] = Field(default_factory=list)
     sorted_by: Optional[Literal['none', 'asc', 'desc']] = 'none'
 
 
 class NodeFilter(NodeSingleInput):
+    """Settings for a node that filters rows based on a condition."""
     filter_input: transform_schema.FilterInput
 
 
 class NodeSort(NodeSingleInput):
+    """Settings for a node that sorts the data by one or more columns."""
     sort_input: List[transform_schema.SortByInput] = Field(default_factory=list)
 
 
 class NodeTextToRows(NodeSingleInput):
+    """Settings for a node that splits a text column into multiple rows."""
     text_to_rows_input: transform_schema.TextToRowsInput
 
 
 class NodeSample(NodeSingleInput):
+    """Settings for a node that samples a subset of the data."""
     sample_size: int = 1000
 
 
 class NodeRecordId(NodeSingleInput):
+    """Settings for a node that adds a unique record ID column."""
     record_id_input: transform_schema.RecordIdInput
 
 
 class NodeJoin(NodeMultiInput):
+    """Settings for a node that performs a standard SQL-style join."""
     auto_generate_selection: bool = True
     verify_integrity: bool = True
     join_input: transform_schema.JoinInput
@@ -232,6 +250,7 @@ class NodeJoin(NodeMultiInput):
 
 
 class NodeCrossJoin(NodeMultiInput):
+    """Settings for a node that performs a cross join."""
     auto_generate_selection: bool = True
     verify_integrity: bool = True
     cross_join_input: transform_schema.CrossJoinInput
@@ -241,31 +260,52 @@ class NodeCrossJoin(NodeMultiInput):
 
 
 class NodeFuzzyMatch(NodeJoin):
+    """Settings for a node that performs a fuzzy join based on string similarity."""
     join_input: transform_schema.FuzzyMatchInput
 
 
 class NodeDatasource(NodeBase):
+    """Base settings for a node that acts as a data source."""
     file_ref: str = None
 
 
 class RawData(BaseModel):
+    """Represents data in a raw, columnar format for manual input."""
     columns: List[MinimalFieldInfo] = None
-    data: List[List]  # List of list where each inner list is a column of data. This ensures more efficient storage
+    data: List[List]
+
+    @classmethod
+    def from_pylist(cls, pylist: List[dict]):
+        """Creates a RawData object from a list of Python dictionaries."""
+        if len(pylist) == 0:
+            return cls(columns=[], data=[])
+        pylist = ensure_similarity_dicts(pylist)
+        values = [standardize_col_dtype([vv for vv in c]) for c in
+                  zip(*(r.values() for r in pylist))]
+        data_types = (pl.DataType.from_python(type(next((v for v in column_values), None))) for column_values in values)
+        columns = [MinimalFieldInfo(name=c, data_type=str(next(data_types))) for c in pylist[0].keys()]
+        return cls(columns=columns, data=values)
+
+    def to_pylist(self) -> List[dict]:
+        """Converts the RawData object back into a list of Python dictionaries."""
+        return [{c.name: self.data[ci][ri] for ci, c in enumerate(self.columns)} for ri in range(len(self.data[0]))]
 
 
 class NodeManualInput(NodeBase):
-    raw_data: Optional[List] = None
+    """Settings for a node that allows direct data entry in the UI."""
     raw_data_format: Optional[RawData] = None
 
 
 class NodeRead(NodeBase):
+    """Settings for a node that reads data from a file."""
     received_file: ReceivedTable
 
 
 class DatabaseConnection(BaseModel):
-    database_type: str = "postgresql"  # Database type (postgresql, mysql, etc.)
+    """Defines the connection parameters for a database."""
+    database_type: str = "postgresql"
     username: Optional[str] = None
-    password_ref: Optional[str] = None
+    password_ref: Optional[SecretRef] = None
     host: Optional[str] = None
     port: Optional[int] = None
     database: Optional[str] = None
@@ -273,8 +313,9 @@ class DatabaseConnection(BaseModel):
 
 
 class FullDatabaseConnection(BaseModel):
+    """A complete database connection model including the secret password."""
     connection_name: str
-    database_type: str = "postgresql"  # Database type (postgresql, mysql, etc.)
+    database_type: str = "postgresql"
     username: str
     password: SecretStr
     host: Optional[str] = None
@@ -285,8 +326,9 @@ class FullDatabaseConnection(BaseModel):
 
 
 class FullDatabaseConnectionInterface(BaseModel):
+    """A database connection model intended for UI display, omitting the password."""
     connection_name: str
-    database_type: str = "postgresql"  # Database type (postgresql, mysql, etc.)
+    database_type: str = "postgresql"
     username: str
     host: Optional[str] = None
     port: Optional[int] = None
@@ -296,6 +338,7 @@ class FullDatabaseConnectionInterface(BaseModel):
 
 
 class DatabaseSettings(BaseModel):
+    """Defines settings for reading from a database, either via table or query."""
     connection_mode: Optional[Literal['inline', 'reference']] = 'inline'
     database_connection: Optional[DatabaseConnection] = None
     database_connection_name: Optional[str] = None
@@ -321,6 +364,7 @@ class DatabaseSettings(BaseModel):
 
 
 class DatabaseWriteSettings(BaseModel):
+    """Defines settings for writing data to a database table."""
     connection_mode: Optional[Literal['inline', 'reference']] = 'inline'
     database_connection: Optional[DatabaseConnection] = None
     database_connection_name: Optional[str] = None
@@ -330,140 +374,146 @@ class DatabaseWriteSettings(BaseModel):
 
 
 class NodeDatabaseReader(NodeBase):
+    """Settings for a node that reads from a database."""
     database_settings: DatabaseSettings
     fields: Optional[List[MinimalFieldInfo]] = None
 
 
 class NodeDatabaseWriter(NodeSingleInput):
+    """Settings for a node that writes data to a database."""
     database_write_settings: DatabaseWriteSettings
 
 
+class NodeCloudStorageReader(NodeBase):
+    """Settings for a node that reads from a cloud storage service (S3, GCS, etc.)."""
+    cloud_storage_settings: CloudStorageReadSettings
+    fields: Optional[List[MinimalFieldInfo]] = None
+
+
+class NodeCloudStorageWriter(NodeSingleInput):
+    """Settings for a node that writes to a cloud storage service."""
+    cloud_storage_settings: CloudStorageWriteSettings
+
+
 class ExternalSource(BaseModel):
+    """Base model for data coming from a predefined external source."""
     orientation: str = 'row'
     fields: Optional[List[MinimalFieldInfo]] = None
 
 
 class SampleUsers(ExternalSource):
+    """Settings for generating a sample dataset of users."""
     SAMPLE_USERS: bool
     class_name: str = "sample_users"
     size: int = 100
 
 
-class AirbyteReader(AirbyteConfig):
-    class_name: Optional[str] = "airbyte_reader"
-    fields: Optional[List[MinimalFieldInfo]] = None
-
-
-class AccessToken(BaseModel):
-    user_id: str
-    access_token: SecretStr = None
-
-
 class NodeExternalSource(NodeBase):
+    """Settings for a node that connects to a registered external data source."""
     identifier: str
     source_settings: SampleUsers
 
 
-class NodeAirbyteReader(NodeExternalSource):
-    identifier: str = 'airbyte'
-    source_settings: AirbyteReader
-
-
 class NodeFormula(NodeSingleInput):
+    """Settings for a node that applies a formula to create/modify a column."""
     function: transform_schema.FunctionInput = None
 
 
 class NodeGroupBy(NodeSingleInput):
+    """Settings for a node that performs a group-by and aggregation operation."""
     groupby_input: transform_schema.GroupByInput = None
 
 
 class NodePromise(NodeBase):
+    """A placeholder node for an operation that has not yet been configured."""
     is_setup: bool = False
     node_type: str
 
 
 class NodeInputConnection(BaseModel):
+    """Represents the input side of a connection between two nodes."""
     node_id: int
     connection_class: InputConnectionClass
 
     def get_node_input_connection_type(self) -> Literal['main', 'right', 'left']:
+        """Determines the semantic type of the input (e.g., for a join)."""
         match self.connection_class:
-            case 'input-0':
-                return 'main'
-            case 'input-1':
-                return 'right'
-            case 'input-2':
-                return 'left'
-            case _:
-                raise ValueError(f"Unexpected connection_class: {self.connection_class}")
+            case 'input-0': return 'main'
+            case 'input-1': return 'right'
+            case 'input-2': return 'left'
+            case _: raise ValueError(f"Unexpected connection_class: {self.connection_class}")
 
 
 class NodePivot(NodeSingleInput):
+    """Settings for a node that pivots data from a long to a wide format."""
     pivot_input: transform_schema.PivotInput = None
     output_fields: Optional[List[MinimalFieldInfo]] = None
 
 
 class NodeUnpivot(NodeSingleInput):
+    """Settings for a node that unpivots data from a wide to a long format."""
     unpivot_input: transform_schema.UnpivotInput = None
 
 
 class NodeUnion(NodeMultiInput):
+    """Settings for a node that concatenates multiple data inputs."""
     union_input: transform_schema.UnionInput = Field(default_factory=transform_schema.UnionInput)
 
 
 class NodeOutput(NodeSingleInput):
+    """Settings for a node that writes its input to a file."""
     output_settings: OutputSettings
 
 
 class NodeOutputConnection(BaseModel):
+    """Represents the output side of a connection between two nodes."""
     node_id: int
     connection_class: OutputConnectionClass
 
 
 class NodeConnection(BaseModel):
+    """Represents a connection (edge) between two nodes in the graph."""
     input_connection: NodeInputConnection
     output_connection: NodeOutputConnection
 
     @classmethod
     def create_from_simple_input(cls, from_id: int, to_id: int, input_type: InputType = "input-0"):
-
+        """Creates a standard connection between two nodes."""
         match input_type:
-            case "main":
-                connection_class: InputConnectionClass = "input-0"
-            case "right":
-                connection_class: InputConnectionClass = "input-1"
-            case "left":
-                connection_class: InputConnectionClass = "input-2"
-            case _:
-                connection_class: InputConnectionClass = "input-0"
+            case "main": connection_class: InputConnectionClass = "input-0"
+            case "right": connection_class: InputConnectionClass = "input-1"
+            case "left": connection_class: InputConnectionClass = "input-2"
+            case _: connection_class: InputConnectionClass = "input-0"
         node_input = NodeInputConnection(node_id=to_id, connection_class=connection_class)
         node_output = NodeOutputConnection(node_id=from_id, connection_class='output-0')
         return cls(input_connection=node_input, output_connection=node_output)
 
 
 class NodeDescription(BaseModel):
+    """A simple model for updating a node's description text."""
     description: str = ''
 
 
 class NodeExploreData(NodeBase):
+    """Settings for a node that provides an interactive data exploration interface."""
     graphic_walker_input: Optional[gs_schemas.GraphicWalkerInput] = None
-    _hash_overrule: int = 0
-
-    def __hash__(self):
-        return 0
 
 
 class NodeGraphSolver(NodeSingleInput):
+    """Settings for a node that solves graph-based problems (e.g., connected components)."""
     graph_solver_input: transform_schema.GraphSolverInput
 
 
 class NodeUnique(NodeSingleInput):
+    """Settings for a node that returns the unique rows from the data."""
     unique_input: transform_schema.UniqueInput
 
 
 class NodeRecordCount(NodeSingleInput):
+    """Settings for a node that counts the number of records."""
     pass
 
 
 class NodePolarsCode(NodeMultiInput):
+    """Settings for a node that executes arbitrary user-provided Polars code."""
     polars_code_input: transform_schema.PolarsCodeInput

flowfile_core/schemas/output_model.py

@@ -5,8 +5,9 @@ import time
 
 
 class NodeResult(BaseModel):
+    """Represents the execution result of a single node in a FlowGraph run."""
     node_id: int
-    node_name: str = None
+    node_name: Optional[str] = None
     start_timestamp: float = Field(default_factory=time.time)
     end_timestamp: float = 0
     success: Optional[bool] = None
@@ -16,6 +17,7 @@ class NodeResult(BaseModel):
 
 
 class RunInformation(BaseModel):
+    """Contains summary information about a complete FlowGraph execution."""
     flow_id: int
     start_time: Optional[datetime] = Field(default_factory=datetime.now)
     end_time: Optional[datetime] = None
@@ -26,6 +28,7 @@ class RunInformation(BaseModel):
 
 
 class BaseItem(BaseModel):
+    """A base model for any item in a file system, like a file or directory."""
     name: str
     path: str
     size: Optional[int] = None
@@ -37,6 +40,7 @@ class BaseItem(BaseModel):
 
 
 class FileColumn(BaseModel):
+    """Represents detailed schema and statistics for a single column (field)."""
     name: str
     data_type: str
     is_unique: bool
@@ -49,6 +53,7 @@ class FileColumn(BaseModel):
 
 
 class TableExample(BaseModel):
+    """Represents a preview of a table, including schema and sample data."""
     node_id: int
     number_of_records: int
     number_of_columns: int
@@ -59,6 +64,10 @@ class TableExample(BaseModel):
 
 
 class NodeData(BaseModel):
+    """A comprehensive model holding the complete state and data for a single node.
+
+    This includes its input/output data previews, settings, and run status.
+    """
     flow_id: int
     node_id: int
     flow_type: str
@@ -74,19 +83,23 @@ class NodeData(BaseModel):
 
 
 class OutputFile(BaseItem):
+    """Represents a single file in an output directory, extending BaseItem."""
     ext: Optional[str] = None
     mimetype: Optional[str] = None
 
 
 class OutputFiles(BaseItem):
+    """Represents a collection of files, typically within a directory."""
     files: List[OutputFile] = Field(default_factory=list)
 
 
 class OutputTree(OutputFiles):
+    """Represents a directory tree, including subdirectories."""
     directories: List[OutputFiles] = Field(default_factory=list)
 
 
 class ItemInfo(OutputFile):
+    """Provides detailed information about a single item in an output directory."""
     id: int = -1
     type: str
     analysis_file_available: bool = False
@@ -95,21 +108,24 @@ class ItemInfo(OutputFile):
 
 
 class OutputDir(BaseItem):
+    """Represents the contents of a single output directory."""
     all_items: List[str]
     items: List[ItemInfo]
 
 
 class ExpressionRef(BaseModel):
+    """A reference to a single Polars expression, including its name and docstring."""
     name: str
     doc: Optional[str]
 
 
 class ExpressionsOverview(BaseModel):
+    """Represents a categorized list of available Polars expressions."""
     expression_type: str
     expressions: List[ExpressionRef]
 
 
 class InstantFuncResult(BaseModel):
+    """Represents the result of a function that is expected to execute instantly."""
     success: Optional[bool] = None
-    result: str
-
+    result: str