flowtask 5.8.4__cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/DownloadFromSmartSheet.py

@@ -0,0 +1,114 @@
+"""
+  DownloadFromSmartSheet
+  Download an Excel file from SmartSheet.
+
+
+        Example:
+
+        ```yaml
+        DownloadFromSmartSheet:
+          comments: Download an SmartSheet Tab into an Excel file.
+          file_id: '373896609326980'
+          file_format: application/vnd.ms-excel
+          destination:
+            filename: WORP MPL 2022.xlsx
+            directory: /home/ubuntu/symbits/worp/files/stores/
+        ```
+
+    """
+import asyncio
+from collections.abc import Callable
+from pathlib import Path
+from ..exceptions import ComponentError, FileNotFound
+from .DownloadFrom import DownloadFromBase
+from ..interfaces.smartsheet import SmartSheetClient
+
+class DownloadFromSmartSheet(SmartSheetClient, DownloadFromBase):
+    """
+    DownloadFromSmartSheet
+
+    Overview
+
+        Download an Excel file or CSV file from SmartSheet.
+
+    Properties (inherited from DownloadFromBase)
+
+    .. table:: Properties
+        :widths: auto
+
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+        | Name               | Required | Summary                                                                          |
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+        | credentials        |   Yes    | Credentials to establish connection with SharePoint site (username and password) |
+        |                    |          | get credentials from environment if null.                                        |
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+        | create_destination |   No     | Boolean flag indicating whether to create the destination directory if it        |
+        |                    |          | doesn't exist (default: True).                                                   |
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+        | api_key            |   No     | The SmartSheet API key (can be provided as an environment variable or directly   |
+        |                    |          | set as a property). If not provided, tries to use the `SMARTSHEET_API_KEY`       |
+        |                    |          | environment variable.                                                            |
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+        | url                |   No     | Base URL for the SmartSheet Sheets API (default:                                 |
+        |                    |          | https://api.smartsheet.com/2.0/sheets/).                                         |
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+        | file_id            |   Yes    | The ID of the SmartSheet file to download.                                       |
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+        | file_format        |   No     | The desired file format for the downloaded data (default:                        |
+        |                    |          | "application/vnd.ms-excel"). Supported formats are:                              |
+        |                    |          | * "application/vnd.ms-excel" (Excel)                                             |
+        |                    |          | * "text/csv" (CSV)                                                               |
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+        | filename           |   Yes    | The filename to use for the downloaded file.                                     |
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+
+        Save the downloaded files on the new destination.
+
+    """  # noqa
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        if self.file_format not in ["application/vnd.ms-excel", "text/csv"]:
+            # only supported
+            raise ComponentError(
+                f"SmartSheet: Format {self.file_format} is not suported"
+            )
+        try:
+            self.accept = (
+                "text/csv" if self.file_format == "dataframe" else self.file_format
+            )
+        except Exception as err:
+            print(err)
+        await super(DownloadFromSmartSheet, self).start(**kwargs)
+        return True
+
+    async def close(self):
+        pass
+
+    async def run(self):
+        self._result = None
+        try:
+            self._logger.info(f"Downloading SmartSheet file: {self.file_id}")
+            # check if self.filename is a relative path:
+            if isinstance(self.filename, str):
+                self.filename = Path(self.filename)
+            if not self.filename.is_absolute():
+                self.filename = self.directory.joinpath(self.filename)
+            if await self.download_file(file_id=self.file_id, destination=self.filename):
+                self._filenames = [str(self.filename)]
+                self._result = self._filenames
+                self.add_metric("SMARTSHEET_FILE", self.filename)
+            return self._result
+        except ComponentError as err:
+            raise FileNotFound(f"DownloadFromSmartSheet Error: {err}") from err
+        except Exception as err:
+            raise ComponentError(
+                f"DownloadFromSmartSheet: Unknown Error: {err}"
+            ) from err

flowtask/components/DownloadS3File.py

@@ -0,0 +1,229 @@
+from typing import Optional, Callable
+import os
+import mimetypes
+
+import asyncio
+from io import BytesIO
+import aiofiles
+from botocore.exceptions import ClientError
+from urllib.parse import urlparse
+from PIL import Image
+from PIL.ExifTags import TAGS, GPSTAGS
+from ..interfaces.Boto3Client import Boto3Client
+from ..interfaces.dataframes.pandas import PandasDataframe
+from .flow import FlowComponent
+from ..exceptions import FileNotFound, ComponentError
+
+_EXTENSION_MAP = {
+    # fall‑backs when `mimetypes` lacks something you need
+    'image/heic': '.heic',
+    'image/webp': '.webp',
+}
+
+class DownloadS3File(Boto3Client, FlowComponent, PandasDataframe):
+    """
+    Download a file from an S3 bucket.
+    """
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """
+        Initialize the DownloadS3File component.
+
+        Args:
+            loop (asyncio.AbstractEventLoop): The event loop to use.
+            job (Callable): The job to run.
+            stat (Callable): The statistics to collect.
+            **kwargs: Additional arguments.
+        """
+        self.path_column = kwargs.pop('path_column', 'file_path')
+        self.content_type = kwargs.pop('content_type', 'application/octet-stream')
+        self.filename_column: Optional[str] = kwargs.pop('filename_column', None)
+        self.download_dir: str = kwargs.pop('directory', '/tmp')
+        self.as_bytes: bool = kwargs.pop('as_bytes', False)
+        self.chunk_size: int = kwargs.get('chunk_size', 100)
+        self.task_parts: int = kwargs.get('task_parts', 10)
+        self.extract_exif: bool = kwargs.pop('extract_exif', False)
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+        self.files_dict = {}  # Dictionary to store downloaded files
+
+    async def start(self, **kwargs):
+        """
+        start.
+
+            Initialize Task.
+        """
+        self.processing_credentials()
+        await super().start(**kwargs)
+        if self.previous:
+            self.data = self.input
+        # check if self.filename_column and self.path_column are in self.data dataframe:
+        if self.filename_column not in self.data.columns:
+            raise ValueError(f'Filename column {self.filename_column} not found in data.')
+        if self.path_column not in self.data.columns:
+            raise ValueError(f'Path column {self.path_column} not found in data.')
+        # check if self.data is empty:
+        if self.data.empty:
+            raise ValueError('Dataframe is empty.')
+        # evaluate the download_dir:
+        if self.download_dir:
+            self.download_dir = self.download_dir.strip()
+            self.download_dir = self.mask_replacement(self.download_dir)
+        return True
+
+    def parse_s3_url(self, url: str) -> tuple:
+        """
+        Parse an S3 URL into bucket and key.
+
+        Args:
+            url (str): The S3 URL to parse.
+
+        Returns:
+            tuple: A tuple containing the bucket name and the key.
+        """
+        parsed_url = urlparse(url)
+        if parsed_url.scheme == 's3':
+            # s3://bucket-name/key
+            bucket_name = parsed_url.netloc
+            object_key = parsed_url.path.lstrip('/')
+        elif parsed_url.netloc.endswith('.s3.amazonaws.com'):
+            # https://bucket-name.s3.amazonaws.com/key
+            bucket_name = parsed_url.netloc.split('.s3.amazonaws.com')[0]
+            object_key = parsed_url.path.lstrip('/')
+        elif '.s3.' in parsed_url.netloc:
+            # https://s3.region.amazonaws.com/bucket-name/key or
+            # https://bucket-name.s3.region.amazonaws.com/key
+            if parsed_url.netloc.startswith('s3.'):
+                # https://s3.region.amazonaws.com/bucket-name/key
+                path_parts = parsed_url.path.lstrip('/').split('/', 1)
+                bucket_name = path_parts[0]
+                object_key = path_parts[1] if len(path_parts) > 1 else ''
+            else:
+                # https://bucket-name.s3.region.amazonaws.com/key
+                bucket_name = parsed_url.netloc.split('.s3.')[0]
+                object_key = parsed_url.path.lstrip('/')
+        elif self.prefix:
+            # Custom domain pointing to S3
+            # Use provided bucket or extract from path
+            bucket_name = self.bucket
+            object_key = self.prefix + parsed_url.path.lstrip('/')
+        else:
+            # Custom domain pointing to S3
+            # Use provided bucket or extract from path
+            bucket_name = self.bucket
+            object_key = parsed_url.path.lstrip('/')
+        return bucket_name, object_key
+
+    def _filename_from_key(
+        self,
+        object_key: str,
+        content_type: str | None = None
+    ) -> str:
+        """
+        Return a filename with an extension derived from either the key
+        or the S3 object's Content‑Type.
+        """
+        # 1) basename from the key
+        fname = os.path.basename(object_key)
+        root, ext = os.path.splitext(fname)
+
+        if ext:                         # key already has an extension
+            return fname
+
+        # 2) derive from Content‑Type
+        if content_type:
+            ext = mimetypes.guess_extension(content_type.split(';')[0].strip()) \
+                or _EXTENSION_MAP.get(content_type)
+
+        # 3) final fall‑back
+        ext = ext or '.jpg'
+        return f"{root}{ext}"
+
+    async def download_file(self, row, idx, client, as_bytes: bool = False):
+        path = row[self.path_column]
+        filename = row[self.filename_column]
+        if not path or not filename:
+            raise ValueError(f'Path or filename is empty for row {idx}.')
+        bucket_name, object_key = self.parse_s3_url(path)
+        if bucket_name != self.bucket:
+            raise ValueError(f'Bucket name {bucket_name} does not match {self.bucket}.')
+        try:
+            obj = await client.get_s3_object(
+                bucket=bucket_name,
+                filename=object_key,
+            )
+            content_type = obj.get("ContentType", self.content_type)
+            filename = self._filename_from_key(object_key, content_type)
+
+            self.data.at[idx, 'file_obj'] = obj
+            file_data = None
+            with obj["Body"] as stream:
+                file_data = stream.read()
+            if as_bytes:
+                # Read the file into BytesIO
+                output = BytesIO()
+                output.write(file_data)
+                output.seek(0)
+                self.data.at[idx, 'file_data'] = output
+            if self.download_dir:
+                # save file in download_dir:
+                output_path = f"{self.download_dir}/{filename}"
+                async with aiofiles.open(output_path, 'wb') as f:
+                    await f.write(file_data)
+                self.data.at[idx, 'file_path'] = output_path
+            return row
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'NoSuchKey':
+                raise FileNotFound(
+                    f"File '{object_key}' not found in bucket '{bucket_name}'."
+                ) from e
+            elif e.response['Error']['Code'] == 'NoSuchBucket':
+                raise FileNotFound(
+                    f"Bucket '{bucket_name}' does not exist."
+                ) from e
+            else:
+                raise ComponentError(
+                    f"Error downloading file from S3: {str(e)}"
+                ) from e
+        except Exception as e:
+            self._logger.error(f'Error downloading file {filename}: {e}')
+            raise e
+
+    async def run(self):
+        """
+        run.
+
+            Download the file from S3.
+        """
+        async with await self.open() as client:
+            if self.bucket is None:
+                raise ValueError('Bucket name is required.')
+            if self.filename_column is None:
+                raise ValueError('Filename column is required.')
+            if self.path_column is None:
+                raise ValueError('Path column is required.')
+
+            # Create the new columns:
+            self.column_exists('file_obj')
+            self.column_exists('file_path')
+            self.column_exists('file_data')
+            # generate tasks from dataframe:
+            tasks = self._create_tasks(
+                self.data,
+                self.download_file,
+                client=client,
+                as_bytes=self.as_bytes
+            )
+            # process tasks:
+            results = await self._processing_tasks(tasks)
+            # process results:
+            for result in results:
+                if isinstance(result, Exception):
+                    raise result
+        self._print_data_(self.data, ' DownloadS3File ')
+        self._result = self.data
+        return self._result

flowtask/components/Dummy.py

@@ -0,0 +1,59 @@
+from ..utils import cPrint, SafeDict
+from .flow import FlowComponent
+
+
+class Dummy(FlowComponent):
+    async def start(self, **kwargs):
+        """
+        start.
+
+            Initialize (if needed) a task
+        
+
+        Example:
+
+        ```yaml
+        Dummy:
+          message: 'Dummy Date: {firstdate} and {lastdate}'
+          masks:
+            firstdate:
+            - date_diff_dow
+            - day_of_week: monday
+              diff: 8
+              mask: '%Y-%m-%d'
+            lastdate:
+            - date_diff_dow
+            - day_of_week: monday
+              diff: 2
+              mask: '%Y-%m-%d'
+        ```
+
+    """
+        return True
+
+    async def run(self):
+        """
+        run.
+
+        Close (if needed) a task
+        """
+        try:
+            self.message = self.message.format_map(SafeDict(**self._mask))
+            cPrint(f"Message IS: {self.message}")
+            self.add_metric("message", self.message)
+        except Exception:
+            self.save_traceback()
+            raise
+        return True
+
+    async def close(self):
+        """
+        close.
+
+        Close (if needed) a task
+        """
+
+    def save_stats(self):
+        """
+        Extension to save stats for this component
+        """