Flowfile 0.3.5__py3-none-any.whl → 0.3.6__py3-none-any.whl

flowfile_worker/external_sources/s3_source/main.py

@@ -0,0 +1,216 @@
+"""Cloud storage writer module for FlowFile Worker.
+
+This module provides functionality to write Polars LazyFrames to various cloud storage
+services (S3, Azure ADLS, Google Cloud Storage) in different file formats.
+"""
+
+import polars as pl
+from typing import Dict, Any
+from logging import Logger
+
+from flowfile_worker.external_sources.s3_source.models import (
+    CloudStorageWriteSettings,
+    WriteSettings
+)
+from flowfile_worker.utils import collect_lazy_frame
+
+
+def _write_parquet_to_cloud(
+    df: pl.LazyFrame,
+    resource_path: str,
+    storage_options: Dict[str, Any],
+    write_settings: WriteSettings,
+    logger: Logger
+) -> None:
+    """Write LazyFrame to a Parquet file in cloud storage.
+
+    Args:
+        df: Polars LazyFrame to write.
+        resource_path: Cloud storage path where the file will be written.
+        storage_options: Storage-specific options for authentication and configuration.
+        write_settings: Write configuration including compression settings.
+        logger: Logger instance for logging operations.
+
+    Raises:
+        Exception: If writing fails, wrapped with a descriptive error message.
+    """
+    try:
+        sink_kwargs = {
+            "path": resource_path,
+            "compression": write_settings.parquet_compression,
+        }
+        if storage_options:
+            sink_kwargs["storage_options"] = storage_options
+
+        try:
+            # Try to use sink_parquet for lazy execution
+            df.sink_parquet(**sink_kwargs)
+        except Exception as e:
+            # Fall back to collecting and writing if sink fails
+            logger.warning(f"Failed to use sink_parquet, falling back to collect and write: {str(e)}")
+            pl_df = collect_lazy_frame(df)
+            sink_kwargs['file'] = sink_kwargs.pop("path")
+            pl_df.write_parquet(**sink_kwargs)
+
+    except Exception as e:
+        logger.error(f"Failed to write Parquet to {resource_path}: {str(e)}")
+        raise Exception(f"Failed to write Parquet to cloud storage: {str(e)}")
+
+
+def _write_delta_to_cloud(
+    df: pl.LazyFrame,
+    resource_path: str,
+    storage_options: Dict[str, Any],
+    write_settings: WriteSettings,
+    logger: Logger
+) -> None:
+    """Write LazyFrame to Delta Lake format in cloud storage.
+
+    Args:
+        df: Polars LazyFrame to write.
+        resource_path: Cloud storage path where the Delta table will be written.
+        storage_options: Storage-specific options for authentication and configuration.
+        write_settings: Write configuration including write mode.
+        logger: Logger instance for logging operations.
+    """
+    sink_kwargs = {
+        "target": resource_path,
+        "mode": write_settings.write_mode,
+    }
+    if storage_options:
+        sink_kwargs["storage_options"] = storage_options
+
+    # Delta format requires collecting the LazyFrame first
+    collect_lazy_frame(df).write_delta(**sink_kwargs)
+
+
+def _write_csv_to_cloud(
+    df: pl.LazyFrame,
+    resource_path: str,
+    storage_options: Dict[str, Any],
+    write_settings: WriteSettings,
+    logger: Logger
+) -> None:
+    """Write LazyFrame to a CSV file in cloud storage.
+
+    Args:
+        df: Polars LazyFrame to write.
+        resource_path: Cloud storage path where the CSV file will be written.
+        storage_options: Storage-specific options for authentication and configuration.
+        write_settings: Write configuration including delimiter settings.
+        logger: Logger instance for logging operations.
+
+    Raises:
+        Exception: If writing fails, wrapped with a descriptive error message.
+    """
+    try:
+        sink_kwargs = {
+            "path": resource_path,
+            "separator": write_settings.csv_delimiter,
+        }
+        if storage_options:
+            sink_kwargs["storage_options"] = storage_options
+
+        # sink_csv executes the lazy query and writes the result
+        df.sink_csv(**sink_kwargs)
+
+    except Exception as e:
+        logger.error(f"Failed to write CSV to {resource_path}: {str(e)}")
+        raise Exception(f"Failed to write CSV to cloud storage: {str(e)}")
+
+
+def _write_json_to_cloud(
+    df: pl.LazyFrame,
+    resource_path: str,
+    storage_options: Dict[str, Any],
+    write_settings: WriteSettings,
+    logger: Logger
+) -> None:
+    """Write LazyFrame to a line-delimited JSON (NDJSON) file in cloud storage.
+
+    Args:
+        df: Polars LazyFrame to write.
+        resource_path: Cloud storage path where the NDJSON file will be written.
+        storage_options: Storage-specific options for authentication and configuration.
+        write_settings: Write configuration settings.
+        logger: Logger instance for logging operations.
+
+    Raises:
+        Exception: If writing fails, wrapped with a descriptive error message.
+    """
+    try:
+        sink_kwargs = {"path": resource_path}
+        if storage_options:
+            sink_kwargs["storage_options"] = storage_options
+
+        try:
+            # Try to use sink_ndjson for lazy execution
+            df.sink_ndjson(**sink_kwargs)
+        except Exception as e:
+            # Fall back to collecting and writing if sink fails
+            pl_df = collect_lazy_frame(df)
+            sink_kwargs['file'] = sink_kwargs.pop("path")
+            pl_df.write_ndjson(**sink_kwargs)
+            logger.error(f"Failed to use sink_ndjson, falling back to collect and write: {str(e)}")
+
+    except Exception as e:
+        logger.error(f"Failed to write JSON to {resource_path}: {str(e)}")
+        raise Exception(f"Failed to write JSON to cloud storage: {str(e)}")
+
+writers = {
+    "parquet": _write_parquet_to_cloud,
+    "delta": _write_delta_to_cloud,
+    "csv": _write_csv_to_cloud,
+    "json": _write_json_to_cloud,
+}
+
+
+def write_df_to_cloud(
+    df: pl.LazyFrame,
+    settings: CloudStorageWriteSettings,
+    logger: Logger
+) -> None:
+    """Write a Polars LazyFrame to an object in cloud storage.
+
+    Supports writing to S3, Azure ADLS, and Google Cloud Storage. Currently supports
+    'overwrite' write mode. The 'append' mode is not yet implemented for most formats.
+
+    Args:
+        df: Polars LazyFrame to write to cloud storage.
+        settings: Cloud storage write settings containing connection details and write options.
+        logger: Logger instance for logging operations.
+
+    Raises:
+        ValueError: If the specified file format is not supported.
+        NotImplementedError: If 'append' write mode is used for non-delta formats.
+        Exception: If writing to cloud storage fails.
+    """
+    connection = settings.connection
+    write_settings = settings.write_settings
+    logger.info(
+        f"Writing to {connection.storage_type} storage: {write_settings.resource_path}"
+    )
+    # Validate write mode
+    if write_settings.write_mode == 'append' and write_settings.file_format != "delta":
+        raise NotImplementedError(
+            "The 'append' write mode is not yet supported for this destination."
+        )
+
+    storage_options = connection.get_storage_options()
+
+    # Dispatch to the appropriate writer
+    writer_func = writers.get(write_settings.file_format)
+    if not writer_func:
+        raise ValueError(
+            f"Unsupported file format for writing: {write_settings.file_format}"
+        )
+
+    writer_func(
+        df,
+        write_settings.resource_path,
+        storage_options,
+        write_settings,
+        logger
+    )
+
+    logger.info(f"Successfully wrote data to {write_settings.resource_path}")

flowfile_worker/external_sources/s3_source/models.py

@@ -0,0 +1,142 @@
+"""Cloud storage connection schemas for S3, ADLS, and other cloud providers."""
+
+from typing import Optional, Literal, Dict, Any
+import boto3
+from pydantic import BaseModel, SecretStr
+from flowfile_worker.secrets import decrypt_secret
+
+CloudStorageType = Literal["s3", "adls", "gcs"]
+AuthMethod = Literal["access_key", "iam_role", "service_principal", "managed_identity", "sas_token", "aws-cli", "env_vars"]
+
+
+def create_storage_options_from_boto_credentials(profile_name: Optional[str],
+                                                 region_name: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Create a storage options dictionary from AWS credentials using a boto3 profile.
+    This is the most robust way to handle profile-based authentication as it
+    bypasses Polars' internal credential provider chain, avoiding conflicts.
+
+    Parameters
+    ----------
+    profile_name
+        The name of the AWS profile in ~/.aws/credentials.
+    region_name
+        The AWS region to use.
+
+    Returns
+    -------
+    Dict[str, Any]
+        A storage options dictionary for Polars with explicit credentials.
+    """
+    session = boto3.Session(profile_name=profile_name, region_name=region_name)
+    credentials = session.get_credentials()
+    frozen_creds = credentials.get_frozen_credentials()
+
+    storage_options = {
+        "aws_access_key_id": frozen_creds.access_key,
+        "aws_secret_access_key": frozen_creds.secret_key,
+        "aws_session_token": frozen_creds.token,
+    }
+    # Use the session's region if one was resolved, otherwise use the provided one
+    if session.region_name:
+        storage_options["aws_region"] = session.region_name
+
+    print("Boto3: Successfully created storage options with explicit credentials.")
+    return storage_options
+
+
+class FullCloudStorageConnection(BaseModel):
+    """Internal model with decrypted secrets"""
+    storage_type: CloudStorageType
+    auth_method: AuthMethod
+    connection_name: Optional[str] = "None"  # This is the reference to the item we will fetch that contains the data
+
+    # AWS S3
+    aws_region: Optional[str] = None
+    aws_access_key_id: Optional[str] = None
+    aws_secret_access_key: Optional[SecretStr] = None
+    aws_role_arn: Optional[str] = None
+    aws_allow_unsafe_html: Optional[bool] = None
+
+    # Azure ADLS
+    azure_account_name: Optional[str] = None
+    azure_account_key: Optional[SecretStr] = None
+    azure_tenant_id: Optional[str] = None
+    azure_client_id: Optional[str] = None
+    azure_client_secret: Optional[SecretStr] = None
+
+    # Common
+    endpoint_url: Optional[str] = None
+    verify_ssl: bool = True
+
+    def get_storage_options(self) -> Dict[str, Any]:
+        """
+        Build storage options dict based on the connection type and auth method.
+
+        Returns:
+            Dict containing appropriate storage options for the provider
+        """
+        if self.storage_type == "s3":
+            return self._get_s3_storage_options()
+
+    def _get_s3_storage_options(self) -> Dict[str, Any]:
+        """Build S3-specific storage options."""
+        auth_method = self.auth_method
+        print(f"Building S3 storage options for auth_method: '{auth_method}'")
+
+        if auth_method == "aws-cli":
+            return create_storage_options_from_boto_credentials(
+                profile_name=self.connection_name,
+                region_name=self.aws_region
+            )
+
+        storage_options = {}
+        if self.aws_region:
+            storage_options["aws_region"] = self.aws_region
+        if self.endpoint_url:
+            storage_options["endpoint_url"] = self.endpoint_url
+        if not self.verify_ssl:
+            storage_options["verify"] = "False"
+        if self.aws_allow_unsafe_html:  # Note: Polars uses aws_allow_http
+            storage_options["aws_allow_http"] = "true"
+
+        if auth_method == "access_key":
+            storage_options["aws_access_key_id"] = self.aws_access_key_id
+            storage_options["aws_secret_access_key"] = decrypt_secret(
+                self.aws_secret_access_key.get_secret_value()).get_secret_value()
+            # Explicitly clear any session token from the environment
+            storage_options["aws_session_token"] = ""
+
+        elif auth_method == "iam_role":
+            # Correctly implement IAM role assumption using boto3 STS client.
+            sts_client = boto3.client('sts', region_name=self.aws_region)
+            assumed_role_object = sts_client.assume_role(
+                RoleArn=self.aws_role_arn,
+                RoleSessionName="PolarsCloudStorageReaderSession"  # A descriptive session name
+            )
+            credentials = assumed_role_object['Credentials']
+            storage_options["aws_access_key_id"] = credentials['AccessKeyId']
+            storage_options["aws_secret_access_key"] = decrypt_secret(credentials['SecretAccessKey']).get_secret_value()
+            storage_options["aws_session_token"] = decrypt_secret(credentials['SessionToken']).get_secret_value()
+
+        return storage_options
+
+
+class WriteSettings(BaseModel):
+    """Settings for writing to cloud storage"""
+    resource_path: str  # s3://bucket/path/to/file.csv
+
+    write_mode: Literal["overwrite", "append"] = "overwrite"
+    file_format: Literal["csv", "parquet", "json", "delta"] = "parquet"
+
+    parquet_compression: Literal["snappy", "gzip", "brotli", "lz4", "zstd"] = "snappy"
+
+    csv_delimiter: str = ","
+    csv_encoding: str = "utf8"
+
+
+class CloudStorageWriteSettings(BaseModel):
+    write_settings: WriteSettings
+    connection: FullCloudStorageConnection
+    flowfile_flow_id: int = 1
+    flowfile_node_id: int | str = -1

flowfile_worker/funcs.py

@@ -6,7 +6,9 @@ from flowfile_worker.polars_fuzzy_match.matcher import fuzzy_match_dfs
 from flowfile_worker.polars_fuzzy_match.models import FuzzyMapping
 from flowfile_worker.flow_logger import get_worker_logger
 from flowfile_worker.external_sources.sql_source.models import DatabaseWriteSettings
-from flowfile_worker.external_sources.sql_source.main import write_serialized_df_to_database, write_df_to_database
+from flowfile_worker.external_sources.sql_source.main import write_df_to_database
+from flowfile_worker.external_sources.s3_source.main import write_df_to_cloud
+from flowfile_worker.external_sources.s3_source.models import CloudStorageWriteSettings
 from base64 import encodebytes
 from logging import Logger
 import logging
@@ -205,9 +207,9 @@ def execute_write_method(write_method: Callable, path: str, data_type: str = Non
         logger.info('Writing as csv file')
         if write_mode == 'append':
             with open(path, 'ab') as f:
-                write_method(file=f, separator=delimiter, quote_style='always')
+                write_method(f, separator=delimiter, quote_style='always')
         else:
-            write_method(file=path, separator=delimiter, quote_style='always')
+            write_method(path, separator=delimiter, quote_style='always')
     elif data_type == 'parquet':
         logger.info('Writing as parquet file')
         write_method(path)
@@ -243,6 +245,49 @@ def write_to_database(polars_serializable_object: bytes,
             progress.value = -1
 
 
+def write_to_cloud_storage(polars_serializable_object: bytes,
+                           progress: Value,
+                           error_message: Array,
+                           queue: Queue,
+                           file_path: str,
+                           cloud_write_settings: CloudStorageWriteSettings,
+                           flowfile_flow_id: int = -1,
+                           flowfile_node_id: int | str = -1
+                           ) -> None:
+    """
+    Writes a Polars DataFrame to cloud storage using the provided settings.
+    Args:
+        polars_serializable_object ():  # Serialized Polars DataFrame object
+        progress (): Multiprocessing Value to track progress
+        error_message (): Array to store error messages
+        queue (): Queue to send results back
+        file_path (): Path to the file where the DataFrame will be written
+        cloud_write_settings (): CloudStorageWriteSettings object containing write settings and connection details
+        flowfile_flow_id (): Flowfile flow ID for logging
+        flowfile_node_id (): Flowfile node ID for logging
+
+    Returns:
+        None
+    """
+    flowfile_logger = get_worker_logger(flowfile_flow_id, flowfile_node_id)
+    flowfile_logger.info(f"Starting write operation to: {cloud_write_settings.write_settings.resource_path}")
+    df = pl.LazyFrame.deserialize(io.BytesIO(polars_serializable_object))
+    flowfile_logger.info(f"Starting to sync the data to cloud, execution plan: \n"
+                         f"{df.explain(format='plain')}")
+    try:
+        write_df_to_cloud(df, cloud_write_settings, flowfile_logger)
+        flowfile_logger.info("Write operation completed successfully")
+        with progress.get_lock():
+            progress.value = 100
+    except Exception as e:
+        error_msg = str(e).encode()[:1024]
+        flowfile_logger.error(f'Error during write operation: {str(e)}')
+        with error_message.get_lock():
+            error_message[:len(error_msg)] = error_msg
+        with progress.get_lock():
+            progress.value = -1
+
+
 def write_output(polars_serializable_object: bytes,
                  progress: Value,
                  error_message: Array,
@@ -263,16 +308,16 @@ def write_output(polars_serializable_object: bytes,
         if isinstance(df, pl.LazyFrame):
             flowfile_logger.info(f'Execution plan explanation:\n{df.explain(format="plain")}')
         flowfile_logger.info("Successfully deserialized dataframe")
-        is_lazy = False
         sink_method_str = 'sink_'+data_type
         write_method_str = 'write_'+data_type
         has_sink_method = hasattr(df, sink_method_str)
         write_method = None
         if os.path.exists(path) and write_mode == 'create':
             raise Exception('File already exists')
-        if has_sink_method and is_lazy:
+        if has_sink_method and write_method != 'append':
+            flowfile_logger.info(f'Using sink method: {sink_method_str}')
             write_method = getattr(df, 'sink_' + data_type)
-        elif not is_lazy or not has_sink_method:
+        elif not has_sink_method:
             if isinstance(df, pl.LazyFrame):
                 df = collect_lazy_frame(df)
             write_method = getattr(df, write_method_str)

flowfile_worker/models.py

@@ -3,11 +3,12 @@ from typing import Optional, Literal, Any
 from base64 import decodebytes
 from flowfile_worker.polars_fuzzy_match.models import FuzzyMapping
 from flowfile_worker.external_sources.sql_source.models import DatabaseWriteSettings
+from flowfile_worker.external_sources.s3_source.models import CloudStorageWriteSettings
 
 
 OperationType = Literal[
     'store', 'calculate_schema', 'calculate_number_of_records', 'write_output', 'fuzzy', 'store_sample',
-    'write_to_database']
+    'write_to_database', "write_to_cloud_storage",]
 ResultType = Literal['polars', 'other']
 
 
@@ -55,7 +56,6 @@ class DatabaseScriptWrite(DatabaseWriteSettings):
         Returns:
             DatabaseWriteSettings: The corresponding DatabaseWriteSettings object.
         """
-
         return DatabaseWriteSettings(
             connection=self.connection,
             table_name=self.table_name,
@@ -65,6 +65,26 @@ class DatabaseScriptWrite(DatabaseWriteSettings):
         )
 
 
+class CloudStorageScriptWrite(CloudStorageWriteSettings):
+    operation: bytes
+
+    def polars_serializable_object(self):
+        return decodebytes(self.operation)
+
+    def get_cloud_storage_write_settings(self) -> CloudStorageWriteSettings:
+        """
+        Converts the current instance to a DatabaseWriteSettings object.
+        Returns:
+            DatabaseWriteSettings: The corresponding DatabaseWriteSettings object.
+        """
+        return CloudStorageWriteSettings(
+            write_settings=self.write_settings,
+            connection=self.connection,
+            flowfile_flow_id=self.flowfile_flow_id,
+            flowfile_node_id=self.flowfile_node_id
+        )
+
+
 class FuzzyJoinInput(BaseModel):
     task_id: Optional[str] = None
     cache_dir: Optional[str] = None

flowfile_worker/routes.py

@@ -10,10 +10,8 @@ from flowfile_worker import models
 from flowfile_worker.spawner import start_process, start_fuzzy_process, start_generic_process, process_manager
 from flowfile_worker.create import table_creator_factory_method, received_table_parser, FileType
 from flowfile_worker.configs import logger
-from flowfile_worker.external_sources.airbyte_sources.models import AirbyteSettings
 from flowfile_worker.external_sources.sql_source.models import DatabaseReadSettings
 from flowfile_worker.external_sources.sql_source.main import read_sql_source, write_serialized_df_to_database
-from flowfile_worker.external_sources.airbyte_sources.main import read_airbyte_source
 
 
 router = APIRouter()
@@ -74,6 +72,44 @@ def store_sample(polars_script: models.PolarsScriptSample, background_tasks: Bac
         raise HTTPException(status_code=500, detail=str(e))
 
 
+@router.post("/write_data_to_cloud/")
+def write_data_to_cloud(cloud_storage_script_write: models.CloudStorageScriptWrite,
+                        background_tasks: BackgroundTasks) -> models.Status:
+    """
+    Write polars dataframe to a file in cloud storage.
+    Args:
+        cloud_storage_script_write (): Contains dataframe and write options for cloud storage
+        background_tasks (): FastAPI background tasks handler
+
+    Returns:
+        models.Status: Status object tracking the write operation
+    """
+    try:
+        logger.info("Starting write operation to: cloud storage")
+        task_id = str(uuid.uuid4())
+        polars_serializable_object = cloud_storage_script_write.polars_serializable_object()
+        status = models.Status(background_task_id=task_id, status="Starting", file_ref='',
+                               result_type="other")
+        status_dict[task_id] = status
+        background_tasks.add_task(
+            start_process,
+            polars_serializable_object=polars_serializable_object,
+            task_id=task_id,
+            operation="write_to_cloud_storage",
+            file_ref='',
+            flowfile_flow_id=cloud_storage_script_write.flowfile_flow_id,
+            flowfile_node_id=cloud_storage_script_write.flowfile_node_id,
+            kwargs=dict(cloud_write_settings=cloud_storage_script_write.get_cloud_storage_write_settings()),
+        )
+        logger.info(
+            f"Started write task: {task_id} to database"
+        )
+        return status
+    except Exception as e:
+        logger.error(f"Error in write operation: {str(e)}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
 @router.post('/store_database_write_result/')
 def store_in_database(database_script_write: models.DatabaseScriptWrite, background_tasks: BackgroundTasks) -> models.Status:
     """
@@ -158,44 +194,10 @@ def write_results(polars_script_write: models.PolarsScriptWrite, background_task
         raise HTTPException(status_code=500, detail=str(e))
 
 
-@router.post('/store_airbyte_result')
-def store_airbyte_result(airbyte_settings: AirbyteSettings, background_tasks: BackgroundTasks) -> models.Status:
-    """
-    Store the result of an Airbyte source operation.
-
-    Args:
-        airbyte_settings (AirbyteSettings): Settings for the Airbyte source operation
-        background_tasks (BackgroundTasks): FastAPI background tasks handler
-
-    Returns:
-        models.Status: Status object tracking the Airbyte source operation
-    """
-    logger.info("Processing Airbyte source operation")
-
-    try:
-        task_id = str(uuid.uuid4())
-        file_path = os.path.join(CACHE_DIR.name, f"{task_id}.arrow")
-        status = models.Status(background_task_id=task_id, status="Starting", file_ref=file_path,
-                               result_type="polars")
-        status_dict[task_id] = status
-        logger.info(f"Starting Airbyte source task: {task_id}")
-        background_tasks.add_task(start_generic_process, func_ref=read_airbyte_source, file_ref=file_path,
-                                  flowfile_flow_id=airbyte_settings.flowfile_flow_id,
-                                  flowfile_node_id=airbyte_settings.flowfile_node_id,
-                                  task_id=task_id, kwargs=dict(airbyte_settings=airbyte_settings))
-        logger.info(f"Started Airbyte source task: {task_id}")
-
-        return status
-
-    except Exception as e:
-        logger.error(f"Error processing Airbyte source: {str(e)}", exc_info=True)
-        raise HTTPException(status_code=500, detail=str(e))
-
-
 @router.post('/store_database_read_result')
 def store_sql_db_result(database_read_settings: DatabaseReadSettings, background_tasks: BackgroundTasks) -> models.Status:
     """
-    Store the result of an Airbyte source operation.
+    Store the result of an sql source operation.
 
     Args:
         database_read_settings (SQLSourceSettings): Settings for the SQL source operation
@@ -204,7 +206,7 @@ def store_sql_db_result(database_read_settings: DatabaseReadSettings, background
     Returns:
         models.Status: Status object tracking the Sql operation
     """
-    logger.info("Processing Airbyte source operation")
+    logger.info("Processing Sql source operation")
 
     try:
         task_id = str(uuid.uuid4())

flowfile_worker/utils.py

@@ -7,7 +7,7 @@ def collect_lazy_frame(lf: pl.LazyFrame) -> pl.DataFrame:
     try:
         return lf.collect(engine="streaming")
     except PanicException:
-        return lf.collect(engine="auto")
+        return lf.collect(engine="in-memory")
 
 
 @dataclass

test_utils/s3/commands.py

@@ -0,0 +1,46 @@
+import logging
+
+# Set up logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S'
+)
+logger = logging.getLogger("postgres_commands")
+
+
+def start_minio():
+    """Start MinIO container for S3 testing"""
+    from . import fixtures
+    if not fixtures.is_docker_available():
+        logger.warning("Docker is not available. Cannot start PostgreSQL container.")
+        print("\n" + "=" * 50)
+        print("SKIPPING: Docker is not available on this system")
+        print("Tests requiring Docker will need to be skipped")
+        print("=" * 50 + "\n")
+        return 0  # Return success to allow pipeline to continue
+
+
+    if fixtures.start_minio_container():
+        print(f"MinIO started at http://localhost:{fixtures.MINIO_PORT}")
+        print(f"Access Key: {fixtures.MINIO_ACCESS_KEY}")
+        return 0
+    return 1
+
+
+def stop_minio():
+    """Stop MinIO container"""
+    from . import fixtures
+
+    if not fixtures.is_docker_available():
+        logger.warning("Docker is not available. Cannot stop MinIO container.")
+        print("\n" + "=" * 50)
+        print("SKIPPING: Docker is not available on this system")
+        print("Tests requiring Docker will need to be skipped")
+        print("=" * 50 + "\n")
+        return 0
+
+    if fixtures.stop_minio_container():
+        print("MinIO stopped successfully")
+        return 0
+    return 1