flowtask 5.13.6__tar.gz → 5.13.7__tar.gz

{flowtask-5.13.6 → flowtask-5.13.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowtask
-Version: 5.13.6
+Version: 5.13.7
 Summary: Framework for Task orchestration
 Author-email: Jesus Lara <jesuslarag@gmail.com>
 License: Apache-2.0
@@ -87,7 +87,7 @@ Requires-Dist: gmqtt==0.7.0
 Requires-Dist: selectorlib==0.16.0
 Requires-Dist: playwright==1.52.0
 Requires-Dist: fuzzywuzzy==0.18.0
-Requires-Dist: praw==7.8.1
+Requires-Dist: praw==8.0.3
 Requires-Dist: prawcore==2.4.0
 Requires-Dist: osmnx==2.0.1
 Requires-Dist: osmium==4.0.2

{flowtask-5.13.6 → flowtask-5.13.7}/flowtask/components/AddDataset.py

@@ -388,7 +388,7 @@ class AddDataset(QSSupport, DBSupport, TemplateSupport, FlowComponent):
             df[u.columns] = u.fillna("")
 
         for col in df.columns:
-            if df[col].dtype == 'object':
+            if pd.api.types.is_object_dtype(df[col]) or pd.api.types.is_string_dtype(df[col]):
                 first_val = df[col].dropna().astype(str).head(1)
                 if not first_val.empty and first_val.iloc[0].strip().startswith(('{', '[')):
                     def _try_parse(v):

{flowtask-5.13.6 → flowtask-5.13.7}/flowtask/components/DownloadFromFTP.py

@@ -1,6 +1,6 @@
 """
-  DownloadFromSmartSheet
-  Download an Excel file from SmartSheet.
+  DownloadFromFTP
+  Downloads files from a remote FTP server.
 
 
         Example:

flowtask-5.13.7/flowtask/components/DownloadFromSmartSheet.py

@@ -0,0 +1,315 @@
+"""
+  DownloadFromSmartSheet
+  Export a SmartSheet sheet to a file, or download row attachments from it.
+
+
+        Example:
+
+        ```yaml
+        DownloadFromSmartSheet:
+          comments: Download an SmartSheet Tab into an Excel file.
+          api_key: TRACFONE_SMARTSHEET_API_KEY
+          file_id: '373896609326980'
+          file_format: application/vnd.ms-excel
+          destination:
+            filename: WORP MPL 2022.xlsx
+            directory: /home/ubuntu/symbits/worp/files/stores/
+        ```
+
+    """
+import asyncio
+from collections.abc import Callable
+from pathlib import Path
+from typing import Dict, List, Optional
+from ..exceptions import ComponentError, ConfigError, FileNotFound
+from ..interfaces.download_from import DownloadFromBase
+from ..interfaces.smartsheet import SmartSheetClient
+
+
+class DownloadFromSmartSheet(SmartSheetClient, DownloadFromBase):
+    r"""
+    DownloadFromSmartSheet
+
+    Overview
+
+        Export a Smartsheet sheet to Excel/CSV (``file_id``), or download row attachments
+        (``sheet_id``) with filename/column/extension filters. Exactly one of the two ids
+        selects the mode. Export mode returns a list with the saved file path; attachments
+        mode returns a dict (filenames, skipped, failed, total, filtered_by_column,
+        filtered_by_filename, directory).
+
+        Attachments filename format: {row_id}_{column1_value}_{column2_value}_{original_filename}
+
+    Properties
+
+        :widths: auto
+
+        | api_key            |   No     | Name of the environment variable holding the key (or a raw key).                 |
+        |                    |          | Default: the `SMARTSHEET_API_KEY` environment variable.                          |
+        | credentials        |   No     | Alternative form: {token: ENV_NAME, scheme: Bearer}. Wins over api_key.          |
+        | create_destination |   No     | Create the destination directory if missing (default: False).                    |
+        | url                |   No     | Base URL for the SmartSheet Sheets API                                           |
+        |                    |          | (default: https://api.smartsheet.com/2.0/sheets/).                               |
+        | file_id            |   No*    | Export mode: id of the sheet to export. *Required unless sheet_id is given.      |
+        | file_format        |   No     | Export mode: "application/vnd.ms-excel" (default) or "text/csv".                 |
+        | destination        |   No     | Export mode: {filename, directory} of the saved file.                            |
+        | sheet_id           |   No*    | Attachments mode: id of the sheet. *Required unless file_id is given.            |
+        | directory          |   No     | Attachments mode: directory where attachments are saved (required in that mode). |
+        | row_id             |   No     | Attachments mode: download only from this row (default: all rows).               |
+        | overwrite          |   No     | Attachments mode: overwrite existing files (default: False).                     |
+        | attachment_filter  |   No     | Attachments mode: list of file extensions to keep (e.g., ['.pdf', '.xlsx']).     |
+        | filename_columns   |   No     | Attachments mode: columns used in the filename. Default: lead_id, Company Name.  |
+        | filename_filter    |   No     | Attachments mode: regex the attachment filename must match.                      |
+        | column_filter      |   No     | Attachments mode: dict of column_name: value; only matching rows are used.       |
+        | use_row_number     |   No     | Attachments mode: use the row number instead of the row id in filenames.         |
+
+    :consumes: none
+    :produces: file
+
+        Example (export):
+
+        ```yaml
+        DownloadFromSmartSheet:
+          comments: Download an SmartSheet Tab into an Excel file.
+          api_key: TRACFONE_SMARTSHEET_API_KEY
+          file_id: '373896609326980'
+          file_format: application/vnd.ms-excel
+          destination:
+            filename: WORP MPL 2022.xlsx
+            directory: /home/ubuntu/symbits/worp/files/stores/
+        ```
+
+        Example (attachments):
+
+        ```yaml
+        DownloadFromSmartSheet:
+          comments: Download only "Full Packet.pdf" files from approved rows
+          api_key: TRACFONE_SMARTSHEET_API_KEY
+          sheet_id: '8504624500658052'
+          directory: /home/ubuntu/symbits/smartsheet/attachments/
+          overwrite: false
+          filename_filter: 'Full Packet\.pdf$'
+          column_filter:
+            Document Status: 'Send to TROC Processing'
+          filename_columns:
+            - lead_id
+            - Company Name
+        ```
+    """
+    _version = "1.1.0"
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        # Attachments-mode parameters
+        self.sheet_id: str = kwargs.pop('sheet_id', None)
+        self.row_id: Optional[str] = kwargs.pop('row_id', None)
+        self.directory: Path = kwargs.pop('directory', None)
+        _overwrite: bool = kwargs.pop('overwrite', False)
+        self.attachment_filter: Optional[List[str]] = kwargs.pop('attachment_filter', None)
+        self.filename_columns: Optional[List[str]] = kwargs.pop('filename_columns', None)
+        self.filename_filter: Optional[str] = kwargs.pop('filename_filter', None)
+        self.column_filter: Optional[Dict[str, str]] = kwargs.pop('column_filter', None)
+        self.use_row_number: bool = kwargs.pop('use_row_number', False)
+        self._mode: str = None
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+        # DownloadFromBase.__init__ resets overwrite to True; honour the documented default.
+        self.overwrite: bool = _overwrite
+
+    async def start(self, **kwargs) -> bool:
+        """Select the mode, validate its parameters, then run the base lifecycle."""
+        if bool(self.file_id) == bool(self.sheet_id):
+            raise ConfigError(
+                "DownloadFromSmartSheet: set exactly one of 'file_id' (export) or 'sheet_id' (attachments)"
+            )
+        self._mode = "export" if self.file_id else "attachments"
+        if self._mode == "export":
+            if self.file_format not in ["application/vnd.ms-excel", "text/csv"]:
+                # only supported
+                raise ComponentError(
+                    f"SmartSheet: Format {self.file_format} is not suported"
+                )
+            try:
+                self.accept = (
+                    "text/csv" if self.file_format == "dataframe" else self.file_format
+                )
+            except Exception as err:
+                self._logger.warning(f"SmartSheet: could not set Accept header: {err}")
+        else:
+            if not self.directory:
+                raise ConfigError(
+                    "DownloadFromSmartSheet: directory is required"
+                )
+            if isinstance(self.directory, str):
+                self.directory = Path(self.directory)
+            if self.attachment_filter:
+                if not isinstance(self.attachment_filter, list):
+                    raise ConfigError(
+                        "DownloadFromSmartSheet: attachment_filter must be a list of file extensions"
+                    )
+                # Ensure extensions start with a dot
+                self.attachment_filter = [
+                    ext if ext.startswith('.') else f'.{ext}'
+                    for ext in self.attachment_filter
+                ]
+            if self.filename_columns is not None:
+                if not isinstance(self.filename_columns, list):
+                    raise ConfigError(
+                        "DownloadFromSmartSheet: filename_columns must be a list of column names"
+                    )
+            if self.filename_filter is not None:
+                if not isinstance(self.filename_filter, str):
+                    raise ConfigError(
+                        "DownloadFromSmartSheet: filename_filter must be a string regex pattern"
+                    )
+            if self.column_filter is not None:
+                if not isinstance(self.column_filter, dict):
+                    raise ConfigError(
+                        "DownloadFromSmartSheet: column_filter must be a dict of column_name: value pairs"
+                    )
+        await super().start(**kwargs)
+        return True
+
+    async def close(self) -> bool:
+        """No resources to release."""
+        return True
+
+    async def run(self):
+        """Dispatch on the mode; each mode keeps its historical return shape."""
+        self._result = None
+        if self._mode == "export":
+            return await self._run_export()
+        return await self._run_attachments()
+
+    async def _run_export(self):
+        """Export a sheet by ``file_id``; returns ``[path]``."""
+        try:
+            self._logger.info(f"Downloading SmartSheet file: {self.file_id}")
+            # check if self.filename is a relative path:
+            if isinstance(self.filename, str):
+                self.filename = Path(self.filename)
+            if not self.filename.is_absolute():
+                self.filename = self.directory.joinpath(self.filename)
+            if await self.download_file(file_id=self.file_id, destination=self.filename):
+                self._filenames = [str(self.filename)]
+                self._result = self._filenames
+                self.add_metric("SMARTSHEET_FILE", self.filename)
+            return self._result
+        except ComponentError as err:
+            raise FileNotFound(f"DownloadFromSmartSheet Error: {err}") from err
+        except Exception as err:
+            raise ComponentError(
+                f"DownloadFromSmartSheet: Unknown Error: {err}"
+            ) from err
+
+    async def _run_attachments(self):
+        """Download row attachments of ``sheet_id``; returns the results dict."""
+        self._logger.info(
+            f"Starting SmartSheet attachment download for sheet {self.sheet_id}"
+        )
+
+        if self.row_id:
+            self._logger.info(f"Downloading attachments from row {self.row_id}")
+        else:
+            self._logger.info("Downloading all sheet attachments")
+
+        # Log filter configuration
+        if self.column_filter:
+            filter_str = ", ".join([f"{k}='{v}'" for k, v in self.column_filter.items()])
+            self._logger.info(f"Applying column filter: {filter_str}")
+
+        if self.filename_filter:
+            self._logger.info(f"Applying filename pattern: '{self.filename_filter}'")
+
+        if self.attachment_filter:
+            self._logger.info(f"Filtering extensions: {', '.join(self.attachment_filter)}")
+
+        # Log filename column configuration
+        if self.filename_columns:
+            self._logger.info(
+                f"Using columns for unique filenames: {', '.join(self.filename_columns)}"
+            )
+        else:
+            self._logger.info("Using default columns: lead_id, Company Name")
+
+        try:
+            # Call the download_attachments method from SmartSheetClient
+            results = await self.download_attachments(
+                sheet_id=self.sheet_id,
+                row_id=self.row_id,
+                directory=self.directory,
+                overwrite=self.overwrite,
+                attachment_filter=self.attachment_filter,
+                filename_columns=self.filename_columns,
+                filename_filter=self.filename_filter,
+                column_filter=self.column_filter,
+                use_row_number=self.use_row_number
+            )
+
+            # Add directory to results
+            results['directory'] = str(self.directory)
+
+            # Log summary
+            self._logger.info(
+                f"Download complete: {len(results['filenames'])} downloaded, "
+                f"{len(results['skipped'])} skipped, "
+                f"{len(results['failed'])} failed out of {results['total']} total attachments"
+            )
+
+            self.add_metric(
+                "smartsheet_attachments_downloaded", len(results['filenames'])
+            )
+
+            # Log filter statistics
+            if results.get('filtered_by_column', 0) > 0:
+                self._logger.info(
+                    f"Filtered by column criteria: {results['filtered_by_column']} attachments"
+                )
+
+            if results.get('filtered_by_filename', 0) > 0:
+                self._logger.info(
+                    f"Filtered by filename pattern: {results['filtered_by_filename']} attachments"
+                )
+
+            # Log downloaded files (show first 10)
+            file_list = results.get('filenames', [])
+            self.add_metric("smartsheet_downloaded_files", file_list)
+
+            if results['filenames']:
+                self._logger.notice("Successfully downloaded files:")
+                display_count = min(10, len(results['filenames']))
+                for file_path in results['filenames'][:display_count]:
+                    filename = Path(file_path).name
+                    self._logger.notice(f"  - {filename}")
+                if len(results['filenames']) > 10:
+                    self._logger.notice(f"  ... and {len(results['filenames']) - 10} more files")
+
+            # Log skipped files (show first 5)
+            if results['skipped']:
+                self._logger.info(f"Skipped {len(results['skipped'])} files:")
+                display_count = min(5, len(results['skipped']))
+                for item in results['skipped'][:display_count]:
+                    self._logger.info(f"  - {item['file']}: {item['reason']}")
+                if len(results['skipped']) > 5:
+                    self._logger.info(f"  ... and {len(results['skipped']) - 5} more skipped")
+
+            # Log failed files
+            if results['failed']:
+                self._logger.warning(f"Failed {len(results['failed'])} downloads:")
+                for item in results['failed']:
+                    self._logger.warning(f"  - {item['file']}: {item['error']}")
+
+            # Store result for later access
+            self._result = results
+            return results
+
+        except ComponentError:
+            raise
+        except Exception as err:
+            raise ComponentError(
+                f"DownloadFromSmartSheet: Unexpected error during execution: {err}"
+            ) from err

{flowtask-5.13.6 → flowtask-5.13.7}/flowtask/components/OdooInjector.py

@@ -1,12 +1,11 @@
 import asyncio
-import math
 from collections.abc import Callable
 from urllib.parse import urljoin
-import numpy as np
 import pandas as pd
 from ..interfaces.flow import FlowComponent
 from ..interfaces.http import HTTPService
 from ..exceptions import ComponentError, TaskError
+from ..utils.dataframes import split_dataframe
 
 
 class OdooInjector(HTTPService, FlowComponent):
@@ -209,10 +208,4 @@ class OdooInjector(HTTPService, FlowComponent):
         list: A list of DataFrame chunks.
             If the DataFrame is empty, returns an empty list.
         """
-        if df.empty:
-            return []
-
-        split_n = math.ceil(len(df) / chunk_size)
-
-        # Split into chunks of n rows
-        return np.array_split(df, split_n)  # Returns list of DataFrames
+        return split_dataframe(df, chunk_size)

{flowtask-5.13.6 → flowtask-5.13.7}/flowtask/components/SentimentAnalysis.py

@@ -2,7 +2,6 @@ import asyncio
 from collections.abc import Callable
 from typing import List
 import contextlib
-import numpy as np
 import pandas as pd
 from concurrent.futures import ThreadPoolExecutor
 from transformers import (
@@ -20,6 +19,7 @@ from nltk.tokenize import sent_tokenize
 import torch
 from ..exceptions import ComponentError
 from ..interfaces.flow import FlowComponent
+from ..utils.dataframes import split_dataframe
 
 
 class ModelPrediction:
@@ -661,8 +661,7 @@ class SentimentAnalysis(FlowComponent):
         """
         print("[DEBUG] Starting sentiment analysis with single predictor instance")
         # Split the dataframe into larger chunks
-        num_chunks = np.ceil(len(self.data) / self.chunk_size).astype(int)
-        chunks = np.array_split(self.data, num_chunks)
+        chunks = split_dataframe(self.data, self.chunk_size)
 
         # Process chunks sequentially using the shared predictor
         processed_chunks = []

{flowtask-5.13.6 → flowtask-5.13.7}/flowtask/components/tDiff.py

@@ -1,5 +1,4 @@
 import asyncio
-import warnings
 from collections.abc import Callable
 from typing import Any, Optional, Union
 import numpy as np
@@ -389,10 +388,8 @@ class tDiff(tPandas):
         elif len(parts) == 1:
             out = parts[0].reset_index(drop=True)
         else:
-            with warnings.catch_warnings():
-                # A-only columns of deleted rows are all-NA: pandas upcasting is accepted (spec §8)
-                warnings.simplefilter("ignore", FutureWarning)
-                out = pd.concat(parts, ignore_index=True)
+            # A-only columns of deleted rows are all-NA; pandas 3 concatenates them without warning
+            out = pd.concat(parts, ignore_index=True)
         self._variables[f"{self.StepName}_NUMROWS"] = len(out)
         self._variables[f"{self.StepName}_NEW"] = n_new
         self._variables[f"{self.StepName}_MODIFIED"] = n_mod

{flowtask-5.13.6 → flowtask-5.13.7}/flowtask/components/tMap/functions.py

@@ -50,7 +50,9 @@ def concat(df: pd.DataFrame, columns: List[str], sep: str = " ") -> pd.Series:
     :param sep: The separator to use between the concatenated values (default is a space)
     :return: A Series with the concatenated values
     """
-    combined = df[columns[0]].astype(str)
+    # ``map(str)`` keeps the legacy ``'nan'`` text for missing values: since pandas 3
+    # ``astype(str)`` preserves NaN, which would turn the whole concatenation into NaN.
+    combined = df[columns[0]].map(str)
     for col in columns[1:]:
-        combined += sep + df[col].astype(str)
+        combined += sep + df[col].map(str)
     return combined

{flowtask-5.13.6 → flowtask-5.13.7}/flowtask/documentation/categories.json

@@ -63,7 +63,6 @@
     "DownloadFromS3",
     "DownloadFromSFTP",
     "DownloadFromSharepoint",
-    "DownloadFromSmartsheet",
     "DownloadFromSmartSheet"
   ],
   "Upload": [

flowtask-5.13.7/flowtask/documentation/generated/components/DownloadFromSmartSheet.doc.json

@@ -0,0 +1,78 @@
+{
+  "name": "DownloadFromSmartSheet",
+  "version": "1.1.0",
+  "category": "Download",
+  "description": "Overview Export a Smartsheet sheet to Excel/CSV (``file_id``), or download row attachments (``sheet_id``) with filename/column/extension filters. Exactly one of the two ids selects the mode. Export mode returns a list with the saved file path; attachments mode returns a dict (filenames, skipped, failed, total, filtered_by_column, filtered_by_filename, directory). Attachments filename format: {row_id}_{column1_value}_{column2_value}_{original_filename} Properties",
+  "examples": [
+    "DownloadFromSmartSheet:\n          comments: Download an SmartSheet Tab into an Excel file.\n          api_key: TRACFONE_SMARTSHEET_API_KEY\n          file_id: '373896609326980'\n          file_format: application/vnd.ms-excel\n          destination:\n            filename: WORP MPL 2022.xlsx\n            directory: /home/ubuntu/symbits/worp/files/stores/",
+    "DownloadFromSmartSheet:\n          comments: Download only \"Full Packet.pdf\" files from approved rows\n          api_key: TRACFONE_SMARTSHEET_API_KEY\n          sheet_id: '8504624500658052'\n          directory: /home/ubuntu/symbits/smartsheet/attachments/\n          overwrite: false\n          filename_filter: 'Full Packet\\.pdf$'\n          column_filter:\n            Document Status: 'Send to TROC Processing'\n          filename_columns:\n            - lead_id\n            - Company Name"
+  ],
+  "schema": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "type": "object",
+    "title": "DownloadFromSmartSheet",
+    "description": "Overview Export a Smartsheet sheet to Excel/CSV (``file_id``), or download row attachments (``sheet_id``) with filename/column/extension filters. Exactly one of the two ids selects the mode. Export mode returns a list with the saved file path; attachments mode returns a dict (filenames, skipped, failed, total, filtered_by_column, filtered_by_filename, directory). Attachments filename format: {row_id}_{column1_value}_{column2_value}_{original_filename} Properties",
+    "properties": {
+      "api_key": {
+        "type": "string",
+        "description": "Name of the environment variable holding the key (or a raw key). Default: the `SMARTSHEET_API_KEY` environment variable."
+      },
+      "credentials": {
+        "type": "string",
+        "description": "Alternative form: {token: ENV_NAME, scheme: Bearer}. Wins over api_key."
+      },
+      "create_destination": {
+        "type": "string",
+        "description": "Create the destination directory if missing (default: False)."
+      },
+      "url": {
+        "type": "string",
+        "description": "Base URL for the SmartSheet Sheets API (default: https://api.smartsheet.com/2.0/sheets/)."
+      },
+      "file_format": {
+        "type": "string",
+        "description": "Export mode: \"application/vnd.ms-excel\" (default) or \"text/csv\"."
+      },
+      "destination": {
+        "type": "string",
+        "description": "Export mode: {filename, directory} of the saved file."
+      },
+      "directory": {
+        "type": "string",
+        "description": "Attachments mode: directory where attachments are saved (required in that mode)."
+      },
+      "row_id": {
+        "type": "string",
+        "description": "Attachments mode: download only from this row (default: all rows)."
+      },
+      "overwrite": {
+        "type": "string",
+        "description": "Attachments mode: overwrite existing files (default: False)."
+      },
+      "attachment_filter": {
+        "type": "string",
+        "description": "Attachments mode: list of file extensions to keep (e.g., ['.pdf', '.xlsx'])."
+      },
+      "filename_columns": {
+        "type": "string",
+        "description": "Attachments mode: columns used in the filename. Default: lead_id, Company Name."
+      },
+      "filename_filter": {
+        "type": "string",
+        "description": "Attachments mode: regex the attachment filename must match."
+      },
+      "column_filter": {
+        "type": "string",
+        "description": "Attachments mode: dict of column_name: value; only matching rows are used."
+      },
+      "use_row_number": {
+        "type": "string",
+        "description": "Attachments mode: use the row number instead of the row id in filenames."
+      }
+    },
+    "x-flowtask-io": {
+      "consumes": "none",
+      "produces": "file"
+    }
+  }
+}