Flowfile 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

flowfile_frame/flow_frame_methods.py

@@ -1,22 +1,21 @@
-import logging
+import io
 import os
-from typing import Any, Iterable, List, Literal, Optional, Tuple, Union, Dict, Callable
 from pathlib import Path
+from typing import Any, List, Optional, Union, Dict, Callable, Literal
 
-import io
 import polars as pl
-from polars._typing import (SchemaDict, IO,PolarsDataType,
+from polars._typing import (SchemaDict, IO, PolarsDataType,
                             Sequence, CsvEncoding)
 
-from flowfile_core.flowfile.flow_graph import FlowGraph
 from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
-from flowfile_core.schemas import input_schema, transform_schema
-
+from flowfile_core.flowfile.flow_graph import FlowGraph
+from flowfile_core.schemas import input_schema, transform_schema, cloud_storage_schemas
+from flowfile_frame.config import logger
 from flowfile_frame.expr import col
-
+from flowfile_frame.flow_frame import FlowFrame
 from flowfile_frame.utils import create_flow_graph
-from flowfile_frame.flow_frame import generate_node_id, FlowFrame
-from flowfile_frame.config import logger
+from flowfile_frame.cloud_storage.secret_manager import get_current_user_id
+from flowfile_frame.utils import generate_node_id
 
 def sum(expr):
     """Sum aggregation function."""
@@ -140,11 +139,10 @@ def read_csv(
     Returns:
         A FlowFrame with the CSV data.
     """
-    node_id = generate_node_id() # Assuming generate_node_id is defined
+    node_id = generate_node_id()
     if flow_graph is None:
-        flow_graph = create_flow_graph() # Assuming create_flow_graph is defined
+        flow_graph = create_flow_graph()
     flow_id = flow_graph.flow_id
-
     current_source_path_for_native = None
     if isinstance(source, (str, os.PathLike)):
         current_source_path_for_native = str(source)
@@ -216,11 +214,14 @@ def read_csv(
             description=read_node_description
         )
         flow_graph.add_read(read_node)
+        flow_graph.get_node(1)
+
         result_frame = FlowFrame(
             data=flow_graph.get_node(node_id).get_resulting_data().data_frame,
             flow_graph=flow_graph,
             node_id=node_id
         )
+        flow_graph.get_node(1)
         return result_frame
     else:
         polars_source_arg = source
@@ -278,6 +279,7 @@ def read_csv(
             node_id=node_id,
         )
 
+
 def _build_polars_code_args(
     source: Union[str, Path, IO[bytes], bytes, List[Union[str, Path, IO[bytes], bytes]]],
     separator: str,
@@ -377,13 +379,13 @@ def _build_polars_code_args(
     return polars_code
 
 
-def read_parquet(file_path, *, flow_graph: FlowGraph = None, description: str = None,
+def read_parquet(source, *, flow_graph: FlowGraph = None, description: str = None,
                  convert_to_absolute_path: bool = True, **options) -> FlowFrame:
     """
     Read a Parquet file into a FlowFrame.
 
     Args:
-        file_path: Path to Parquet file
+        source: Path to Parquet file
         flow_graph: if you want to add it to an existing graph
         description: if you want to add a readable name in the frontend (advised)
         convert_to_absolute_path: If the path needs to be set to a fixed location
@@ -392,8 +394,8 @@ def read_parquet(file_path, *, flow_graph: FlowGraph = None, description: str =
     Returns:
         A FlowFrame with the Parquet data
     """
-    if '~' in file_path:
-        file_path = os.path.expanduser(file_path)
+    if '~' in source:
+        file_path = os.path.expanduser(source)
     node_id = generate_node_id()
 
     if flow_graph is None:
@@ -403,8 +405,8 @@ def read_parquet(file_path, *, flow_graph: FlowGraph = None, description: str =
 
     received_table = input_schema.ReceivedTable(
         file_type='parquet',
-        path=file_path,
-        name=Path(file_path).name,
+        path=source,
+        name=Path(source).name,
     )
     if convert_to_absolute_path:
         received_table.path = received_table.abs_file_path
@@ -449,7 +451,7 @@ def from_dict(data, *, flow_graph: FlowGraph = None, description: str = None) ->
     input_node = input_schema.NodeManualInput(
         flow_id=flow_id,
         node_id=node_id,
-        raw_data=FlowDataEngine(data).to_pylist(),
+        raw_data_format=FlowDataEngine(data).to_raw_data(),
         pos_x=100,
         pos_y=100,
         is_setup=True,
@@ -592,7 +594,7 @@ def scan_csv(
 
 
 def scan_parquet(
-        file_path,
+        source,
         *,
         flow_graph: FlowGraph = None,
         description: str = None,
@@ -608,10 +610,146 @@ def scan_parquet(
     See read_parquet for full documentation.
     """
     return read_parquet(
-        file_path=file_path,
+        source=source,
         flow_graph=flow_graph,
         description=description,
         convert_to_absolute_path=convert_to_absolute_path,
         **options
     )
 
+
+def scan_parquet_from_cloud_storage(
+        source: str,
+        *,
+        flow_graph: Optional[FlowGraph] = None,
+        connection_name: Optional[str] = None,
+        scan_mode: Literal["single_file", "directory", None] = None,
+        description: Optional[str] = None
+) -> FlowFrame:
+    node_id = generate_node_id()
+
+    if scan_mode is None:
+        if source[-1] in ("*", "/"):
+            scan_mode: Literal["single_file", "directory"] = "directory"
+        else:
+            scan_mode: Literal["single_file", "directory"] = "single_file"
+
+    if flow_graph is None:
+        flow_graph = create_flow_graph()
+
+    flow_id = flow_graph.flow_id
+    settings = input_schema.NodeCloudStorageReader(
+        flow_id=flow_id,
+        node_id=node_id,
+        cloud_storage_settings=cloud_storage_schemas.CloudStorageReadSettings(resource_path=source,
+                                                                              scan_mode=scan_mode,
+                                                                              connection_name=connection_name,
+                                                                              file_format="parquet"),
+        user_id=get_current_user_id(),
+        description=description)
+    flow_graph.add_cloud_storage_reader(settings)
+    return FlowFrame(
+        data=flow_graph.get_node(node_id).get_resulting_data().data_frame,
+        flow_graph=flow_graph,
+        node_id=node_id
+    )
+
+
+def scan_csv_from_cloud_storage(
+        source: str,
+        *,
+        flow_graph: Optional[FlowGraph] = None,
+        connection_name: Optional[str] = None,
+        scan_mode: Literal["single_file", "directory", None] = None,
+        delimiter: str = ";",
+        has_header: Optional[bool] = True,
+        encoding: Optional[CsvEncoding] = "utf8") -> FlowFrame:
+    node_id = generate_node_id()
+
+    if scan_mode is None:
+        if source[-1] in ("*", "/"):
+            scan_mode: Literal["single_file", "directory"] = "directory"
+        else:
+            scan_mode: Literal["single_file", "directory"] = "single_file"
+
+    if flow_graph is None:
+        flow_graph = create_flow_graph()
+    flow_id = flow_graph.flow_id
+    settings = input_schema.NodeCloudStorageReader(
+        flow_id=flow_id,
+        node_id=node_id,
+        cloud_storage_settings=cloud_storage_schemas.CloudStorageReadSettings(resource_path=source,
+                                                                              scan_mode=scan_mode,
+                                                                              connection_name=connection_name,
+                                                                              csv_delimiter=delimiter,
+                                                                              csv_encoding=encoding,
+                                                                              csv_has_header=has_header,
+                                                                              file_format="csv"),
+        user_id=get_current_user_id())
+    flow_graph.add_cloud_storage_reader(settings)
+    return FlowFrame(
+        data=flow_graph.get_node(node_id).get_resulting_data().data_frame,
+        flow_graph=flow_graph,
+        node_id=node_id
+    )
+
+
+def scan_delta(
+        source: str,
+        *,
+        flow_graph: Optional[FlowGraph] = None,
+        connection_name: Optional[str] = None,
+        version: int = None) -> FlowFrame:
+    node_id = generate_node_id()
+    if flow_graph is None:
+        flow_graph = create_flow_graph()
+    flow_id = flow_graph.flow_id
+    settings = input_schema.NodeCloudStorageReader(
+        flow_id=flow_id,
+        node_id=node_id,
+        cloud_storage_settings=cloud_storage_schemas.CloudStorageReadSettings(resource_path=source,
+                                                                              connection_name=connection_name,
+                                                                              file_format="delta",
+                                                                              delta_version=version),
+        user_id=get_current_user_id())
+    flow_graph.add_cloud_storage_reader(settings)
+    return FlowFrame(
+        data=flow_graph.get_node(node_id).get_resulting_data().data_frame,
+        flow_graph=flow_graph,
+        node_id=node_id
+    )
+
+
+def scan_json_from_cloud_storage(
+        source: str,
+        *,
+        flow_graph: Optional[FlowGraph] = None,
+        connection_name: Optional[str] = None,
+        scan_mode: Literal["single_file", "directory", None] = None,
+) -> FlowFrame:
+    node_id = generate_node_id()
+
+    if scan_mode is None:
+        if source[-1] in ("*", "/"):
+            scan_mode: Literal["single_file", "directory"] = "directory"
+        else:
+            scan_mode: Literal["single_file", "directory"] = "single_file"
+
+    if flow_graph is None:
+        flow_graph = create_flow_graph()
+    flow_id = flow_graph.flow_id
+    settings = input_schema.NodeCloudStorageReader(
+        flow_id=flow_id,
+        node_id=node_id,
+        cloud_storage_settings=cloud_storage_schemas.CloudStorageReadSettings(resource_path=source,
+                                                                              scan_mode=scan_mode,
+                                                                              connection_name=connection_name,
+                                                                              file_format="json"),
+        user_id=get_current_user_id())
+    flow_graph.add_cloud_storage_reader(settings)
+    return FlowFrame(
+        data=flow_graph.get_node(node_id).get_resulting_data().data_frame,
+        flow_graph=flow_graph,
+        node_id=node_id
+    )
+

flowfile_frame/group_frame.py

@@ -91,6 +91,8 @@ class GroupByFrame:
             if isinstance(col_expr, str):
                 agg_cols.append(transform_schema.AggColl(old_name=col_expr, agg="groupby"))
             elif isinstance(col_expr, Expr):
+                if col_expr.is_complex:
+                    return False
                 agg_cols.append(transform_schema.AggColl(old_name=col_expr.column_name, agg="groupby"))
             elif isinstance(col_expr, Selector):
                 return False
@@ -151,6 +153,7 @@ class GroupByFrame:
     def _create_agg_node(self, node_id_to_use: int, can_be_converted: bool, agg_cols: list, agg_expressions,
                          named_agg_exprs, convertable_to_code: bool, description: str):
         """Create node for explicit aggregations via self.agg()."""
+
         if can_be_converted:
             group_by_settings = input_schema.NodeGroupBy(
                 flow_id=self.parent.flow_graph.flow_id,

flowfile_frame/utils.py

@@ -88,14 +88,23 @@ def _generate_id() -> int:
     return int(uuid.uuid4().int % 100000)
 
 
-def create_flow_graph() -> FlowGraph:
-    flow_id = _generate_id()
+def create_flow_graph(flow_id: int = None) -> FlowGraph:
+    """
+    Create a new FlowGraph instance with a unique flow ID.
+    Parameters
+       - flow_id (int): Optional flow ID. If not provided, a new unique ID will be generated.
+    Returns
+       - FlowGraph: A new instance of FlowGraph with the specified or generated flow ID.
+
+    """
+    if flow_id is None:
+        flow_id = _generate_id()
     flow_settings = schemas.FlowSettings(
         flow_id=flow_id,
         name=f"Flow_{flow_id}",
         path=f"flow_{flow_id}"
     )
-    flow_graph = FlowGraph(flow_id=flow_id, flow_settings=flow_settings)
+    flow_graph = FlowGraph(flow_settings=flow_settings)
     flow_graph.flow_settings.execution_location = 'local'  # always create a local frame so that the run time does not attempt to use the flowfile_worker process
     return flow_graph
 
@@ -119,3 +128,16 @@ def stringify_values(v: Any) -> str:
     else:
         # Handle any other types
         return str(v)
+
+
+data = {"c": 0}
+
+
+def generate_node_id() -> int:
+    data["c"] += 1
+    return data["c"]
+
+
+def set_node_id(node_id):
+    """Set the node ID to a specific value."""
+    data["c"] = node_id

flowfile_worker/external_sources/s3_source/main.py

@@ -0,0 +1,216 @@
+"""Cloud storage writer module for FlowFile Worker.
+
+This module provides functionality to write Polars LazyFrames to various cloud storage
+services (S3, Azure ADLS, Google Cloud Storage) in different file formats.
+"""
+
+import polars as pl
+from typing import Dict, Any
+from logging import Logger
+
+from flowfile_worker.external_sources.s3_source.models import (
+    CloudStorageWriteSettings,
+    WriteSettings
+)
+from flowfile_worker.utils import collect_lazy_frame
+
+
+def _write_parquet_to_cloud(
+    df: pl.LazyFrame,
+    resource_path: str,
+    storage_options: Dict[str, Any],
+    write_settings: WriteSettings,
+    logger: Logger
+) -> None:
+    """Write LazyFrame to a Parquet file in cloud storage.
+
+    Args:
+        df: Polars LazyFrame to write.
+        resource_path: Cloud storage path where the file will be written.
+        storage_options: Storage-specific options for authentication and configuration.
+        write_settings: Write configuration including compression settings.
+        logger: Logger instance for logging operations.
+
+    Raises:
+        Exception: If writing fails, wrapped with a descriptive error message.
+    """
+    try:
+        sink_kwargs = {
+            "path": resource_path,
+            "compression": write_settings.parquet_compression,
+        }
+        if storage_options:
+            sink_kwargs["storage_options"] = storage_options
+
+        try:
+            # Try to use sink_parquet for lazy execution
+            df.sink_parquet(**sink_kwargs)
+        except Exception as e:
+            # Fall back to collecting and writing if sink fails
+            logger.warning(f"Failed to use sink_parquet, falling back to collect and write: {str(e)}")
+            pl_df = collect_lazy_frame(df)
+            sink_kwargs['file'] = sink_kwargs.pop("path")
+            pl_df.write_parquet(**sink_kwargs)
+
+    except Exception as e:
+        logger.error(f"Failed to write Parquet to {resource_path}: {str(e)}")
+        raise Exception(f"Failed to write Parquet to cloud storage: {str(e)}")
+
+
+def _write_delta_to_cloud(
+    df: pl.LazyFrame,
+    resource_path: str,
+    storage_options: Dict[str, Any],
+    write_settings: WriteSettings,
+    logger: Logger
+) -> None:
+    """Write LazyFrame to Delta Lake format in cloud storage.
+
+    Args:
+        df: Polars LazyFrame to write.
+        resource_path: Cloud storage path where the Delta table will be written.
+        storage_options: Storage-specific options for authentication and configuration.
+        write_settings: Write configuration including write mode.
+        logger: Logger instance for logging operations.
+    """
+    sink_kwargs = {
+        "target": resource_path,
+        "mode": write_settings.write_mode,
+    }
+    if storage_options:
+        sink_kwargs["storage_options"] = storage_options
+
+    # Delta format requires collecting the LazyFrame first
+    collect_lazy_frame(df).write_delta(**sink_kwargs)
+
+
+def _write_csv_to_cloud(
+    df: pl.LazyFrame,
+    resource_path: str,
+    storage_options: Dict[str, Any],
+    write_settings: WriteSettings,
+    logger: Logger
+) -> None:
+    """Write LazyFrame to a CSV file in cloud storage.
+
+    Args:
+        df: Polars LazyFrame to write.
+        resource_path: Cloud storage path where the CSV file will be written.
+        storage_options: Storage-specific options for authentication and configuration.
+        write_settings: Write configuration including delimiter settings.
+        logger: Logger instance for logging operations.
+
+    Raises:
+        Exception: If writing fails, wrapped with a descriptive error message.
+    """
+    try:
+        sink_kwargs = {
+            "path": resource_path,
+            "separator": write_settings.csv_delimiter,
+        }
+        if storage_options:
+            sink_kwargs["storage_options"] = storage_options
+
+        # sink_csv executes the lazy query and writes the result
+        df.sink_csv(**sink_kwargs)
+
+    except Exception as e:
+        logger.error(f"Failed to write CSV to {resource_path}: {str(e)}")
+        raise Exception(f"Failed to write CSV to cloud storage: {str(e)}")
+
+
+def _write_json_to_cloud(
+    df: pl.LazyFrame,
+    resource_path: str,
+    storage_options: Dict[str, Any],
+    write_settings: WriteSettings,
+    logger: Logger
+) -> None:
+    """Write LazyFrame to a line-delimited JSON (NDJSON) file in cloud storage.
+
+    Args:
+        df: Polars LazyFrame to write.
+        resource_path: Cloud storage path where the NDJSON file will be written.
+        storage_options: Storage-specific options for authentication and configuration.
+        write_settings: Write configuration settings.
+        logger: Logger instance for logging operations.
+
+    Raises:
+        Exception: If writing fails, wrapped with a descriptive error message.
+    """
+    try:
+        sink_kwargs = {"path": resource_path}
+        if storage_options:
+            sink_kwargs["storage_options"] = storage_options
+
+        try:
+            # Try to use sink_ndjson for lazy execution
+            df.sink_ndjson(**sink_kwargs)
+        except Exception as e:
+            # Fall back to collecting and writing if sink fails
+            pl_df = collect_lazy_frame(df)
+            sink_kwargs['file'] = sink_kwargs.pop("path")
+            pl_df.write_ndjson(**sink_kwargs)
+            logger.error(f"Failed to use sink_ndjson, falling back to collect and write: {str(e)}")
+
+    except Exception as e:
+        logger.error(f"Failed to write JSON to {resource_path}: {str(e)}")
+        raise Exception(f"Failed to write JSON to cloud storage: {str(e)}")
+
+writers = {
+    "parquet": _write_parquet_to_cloud,
+    "delta": _write_delta_to_cloud,
+    "csv": _write_csv_to_cloud,
+    "json": _write_json_to_cloud,
+}
+
+
+def write_df_to_cloud(
+    df: pl.LazyFrame,
+    settings: CloudStorageWriteSettings,
+    logger: Logger
+) -> None:
+    """Write a Polars LazyFrame to an object in cloud storage.
+
+    Supports writing to S3, Azure ADLS, and Google Cloud Storage. Currently supports
+    'overwrite' write mode. The 'append' mode is not yet implemented for most formats.
+
+    Args:
+        df: Polars LazyFrame to write to cloud storage.
+        settings: Cloud storage write settings containing connection details and write options.
+        logger: Logger instance for logging operations.
+
+    Raises:
+        ValueError: If the specified file format is not supported.
+        NotImplementedError: If 'append' write mode is used for non-delta formats.
+        Exception: If writing to cloud storage fails.
+    """
+    connection = settings.connection
+    write_settings = settings.write_settings
+    logger.info(
+        f"Writing to {connection.storage_type} storage: {write_settings.resource_path}"
+    )
+    # Validate write mode
+    if write_settings.write_mode == 'append' and write_settings.file_format != "delta":
+        raise NotImplementedError(
+            "The 'append' write mode is not yet supported for this destination."
+        )
+
+    storage_options = connection.get_storage_options()
+
+    # Dispatch to the appropriate writer
+    writer_func = writers.get(write_settings.file_format)
+    if not writer_func:
+        raise ValueError(
+            f"Unsupported file format for writing: {write_settings.file_format}"
+        )
+
+    writer_func(
+        df,
+        write_settings.resource_path,
+        storage_options,
+        write_settings,
+        logger
+    )
+
+    logger.info(f"Successfully wrote data to {write_settings.resource_path}")

flowfile_worker/external_sources/s3_source/models.py

@@ -0,0 +1,142 @@
+"""Cloud storage connection schemas for S3, ADLS, and other cloud providers."""
+
+from typing import Optional, Literal, Dict, Any
+import boto3
+from pydantic import BaseModel, SecretStr
+from flowfile_worker.secrets import decrypt_secret
+
+CloudStorageType = Literal["s3", "adls", "gcs"]
+AuthMethod = Literal["access_key", "iam_role", "service_principal", "managed_identity", "sas_token", "aws-cli", "env_vars"]
+
+
+def create_storage_options_from_boto_credentials(profile_name: Optional[str],
+                                                 region_name: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Create a storage options dictionary from AWS credentials using a boto3 profile.
+    This is the most robust way to handle profile-based authentication as it
+    bypasses Polars' internal credential provider chain, avoiding conflicts.
+
+    Parameters
+    ----------
+    profile_name
+        The name of the AWS profile in ~/.aws/credentials.
+    region_name
+        The AWS region to use.
+
+    Returns
+    -------
+    Dict[str, Any]
+        A storage options dictionary for Polars with explicit credentials.
+    """
+    session = boto3.Session(profile_name=profile_name, region_name=region_name)
+    credentials = session.get_credentials()
+    frozen_creds = credentials.get_frozen_credentials()
+
+    storage_options = {
+        "aws_access_key_id": frozen_creds.access_key,
+        "aws_secret_access_key": frozen_creds.secret_key,
+        "aws_session_token": frozen_creds.token,
+    }
+    # Use the session's region if one was resolved, otherwise use the provided one
+    if session.region_name:
+        storage_options["aws_region"] = session.region_name
+
+    print("Boto3: Successfully created storage options with explicit credentials.")
+    return storage_options
+
+
+class FullCloudStorageConnection(BaseModel):
+    """Internal model with decrypted secrets"""
+    storage_type: CloudStorageType
+    auth_method: AuthMethod
+    connection_name: Optional[str] = "None"  # This is the reference to the item we will fetch that contains the data
+
+    # AWS S3
+    aws_region: Optional[str] = None
+    aws_access_key_id: Optional[str] = None
+    aws_secret_access_key: Optional[SecretStr] = None
+    aws_role_arn: Optional[str] = None
+    aws_allow_unsafe_html: Optional[bool] = None
+
+    # Azure ADLS
+    azure_account_name: Optional[str] = None
+    azure_account_key: Optional[SecretStr] = None
+    azure_tenant_id: Optional[str] = None
+    azure_client_id: Optional[str] = None
+    azure_client_secret: Optional[SecretStr] = None
+
+    # Common
+    endpoint_url: Optional[str] = None
+    verify_ssl: bool = True
+
+    def get_storage_options(self) -> Dict[str, Any]:
+        """
+        Build storage options dict based on the connection type and auth method.
+
+        Returns:
+            Dict containing appropriate storage options for the provider
+        """
+        if self.storage_type == "s3":
+            return self._get_s3_storage_options()
+
+    def _get_s3_storage_options(self) -> Dict[str, Any]:
+        """Build S3-specific storage options."""
+        auth_method = self.auth_method
+        print(f"Building S3 storage options for auth_method: '{auth_method}'")
+
+        if auth_method == "aws-cli":
+            return create_storage_options_from_boto_credentials(
+                profile_name=self.connection_name,
+                region_name=self.aws_region
+            )
+
+        storage_options = {}
+        if self.aws_region:
+            storage_options["aws_region"] = self.aws_region
+        if self.endpoint_url:
+            storage_options["endpoint_url"] = self.endpoint_url
+        if not self.verify_ssl:
+            storage_options["verify"] = "False"
+        if self.aws_allow_unsafe_html:  # Note: Polars uses aws_allow_http
+            storage_options["aws_allow_http"] = "true"
+
+        if auth_method == "access_key":
+            storage_options["aws_access_key_id"] = self.aws_access_key_id
+            storage_options["aws_secret_access_key"] = decrypt_secret(
+                self.aws_secret_access_key.get_secret_value()).get_secret_value()
+            # Explicitly clear any session token from the environment
+            storage_options["aws_session_token"] = ""
+
+        elif auth_method == "iam_role":
+            # Correctly implement IAM role assumption using boto3 STS client.
+            sts_client = boto3.client('sts', region_name=self.aws_region)
+            assumed_role_object = sts_client.assume_role(
+                RoleArn=self.aws_role_arn,
+                RoleSessionName="PolarsCloudStorageReaderSession"  # A descriptive session name
+            )
+            credentials = assumed_role_object['Credentials']
+            storage_options["aws_access_key_id"] = credentials['AccessKeyId']
+            storage_options["aws_secret_access_key"] = decrypt_secret(credentials['SecretAccessKey']).get_secret_value()
+            storage_options["aws_session_token"] = decrypt_secret(credentials['SessionToken']).get_secret_value()
+
+        return storage_options
+
+
+class WriteSettings(BaseModel):
+    """Settings for writing to cloud storage"""
+    resource_path: str  # s3://bucket/path/to/file.csv
+
+    write_mode: Literal["overwrite", "append"] = "overwrite"
+    file_format: Literal["csv", "parquet", "json", "delta"] = "parquet"
+
+    parquet_compression: Literal["snappy", "gzip", "brotli", "lz4", "zstd"] = "snappy"
+
+    csv_delimiter: str = ","
+    csv_encoding: str = "utf8"
+
+
+class CloudStorageWriteSettings(BaseModel):
+    write_settings: WriteSettings
+    connection: FullCloudStorageConnection
+    flowfile_flow_id: int = 1
+    flowfile_node_id: int | str = -1