flowtask 5.13.5__tar.gz → 5.13.7__tar.gz

{flowtask-5.13.5 → flowtask-5.13.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowtask
-Version: 5.13.5
+Version: 5.13.7
 Summary: Framework for Task orchestration
 Author-email: Jesus Lara <jesuslarag@gmail.com>
 License: Apache-2.0
@@ -87,7 +87,7 @@ Requires-Dist: gmqtt==0.7.0
 Requires-Dist: selectorlib==0.16.0
 Requires-Dist: playwright==1.52.0
 Requires-Dist: fuzzywuzzy==0.18.0
-Requires-Dist: praw==7.8.1
+Requires-Dist: praw==8.0.3
 Requires-Dist: prawcore==2.4.0
 Requires-Dist: osmnx==2.0.1
 Requires-Dist: osmium==4.0.2
@@ -105,7 +105,7 @@ Requires-Dist: pillow-heif<2,>=1.3.0
 Requires-Dist: pgvector==0.4.1
 Requires-Dist: proxylists>=0.14.0
 Requires-Dist: async-notify[all]>=1.4.0
-Requires-Dist: querysource>=3.17.0
+Requires-Dist: querysource>=5.2.5
 Requires-Dist: asyncdb[all]>=2.11.7
 Requires-Dist: navconfig[default,uvloop]>=1.7.13
 Requires-Dist: navigator-auth>=0.15.8

{flowtask-5.13.5 → flowtask-5.13.7}/flowtask/components/AddDataset.py

@@ -388,7 +388,7 @@ class AddDataset(QSSupport, DBSupport, TemplateSupport, FlowComponent):
             df[u.columns] = u.fillna("")
 
         for col in df.columns:
-            if df[col].dtype == 'object':
+            if pd.api.types.is_object_dtype(df[col]) or pd.api.types.is_string_dtype(df[col]):
                 first_val = df[col].dropna().astype(str).head(1)
                 if not first_val.empty and first_val.iloc[0].strip().startswith(('{', '[')):
                     def _try_parse(v):

{flowtask-5.13.5 → flowtask-5.13.7}/flowtask/components/CopyToBigQuery.py

@@ -4,6 +4,7 @@ import datetime as dt
 import math
 import time
 import pandas as pd
+from querysource.types.dt.transforms import parse_datetime_series, to_wall_clock
 from asyncdb import AsyncDB
 from asyncdb.exceptions import (
     StatementError,
@@ -640,6 +641,17 @@ class CopyToBigQuery(CopyTo, PandasDataframe):
             self._logger.warning(f'CopyTo: Could not retrieve table schema: {err}')
         return column_types
 
+    @staticmethod
+    def _is_date_objects(series: pd.Series) -> bool:
+        """Return True for an object column whose non-null values are all plain ``datetime.date``.
+
+        Such columns are already DATE-ready and are left unchanged.
+        """
+        if not pd.api.types.is_object_dtype(series):
+            return False
+        values = series.dropna()
+        return not values.empty and all(type(v) is dt.date for v in values)
+
     def _convert_dataframe_to_bigquery_types(self, df, column_types):
         """Convert DataFrame columns to match BigQuery schema types."""
         converted_df = df.copy()
@@ -668,8 +680,9 @@ class CopyToBigQuery(CopyTo, PandasDataframe):
                     self._logger.debug(
                         f"CopyTo: Converted {col} to DATE (from numeric values)"
                     )
-                elif not pd.api.types.is_object_dtype(converted_df[col]):
-                    dates = pd.to_datetime(converted_df[col], errors="coerce").dt.date
+                elif not self._is_date_objects(converted_df[col]):
+                    # str / string / object of strings or timestamps: local calendar date
+                    dates = to_wall_clock(converted_df[col], field=col).dt.date
                     converted_df[col] = dates.astype(object).where(
                         dates.notna(), None
                     )
@@ -679,13 +692,20 @@ class CopyToBigQuery(CopyTo, PandasDataframe):
 
             # Handle DATETIME type
             elif bq_type == "DATETIME":
-                if pd.api.types.is_datetime64_any_dtype(converted_df[col]):
-                    converted_df[col] = converted_df[col].dt.tz_localize(None)
+                if not pd.api.types.is_numeric_dtype(converted_df[col]):
+                    # civil value: every entry keeps its own local wall-clock
+                    converted_df[col] = to_wall_clock(converted_df[col], field=col)
 
             # Handle TIMESTAMP type
             elif bq_type == "TIMESTAMP":
-                if pd.api.types.is_datetime64_any_dtype(converted_df[col]):
-                    converted_df[col] = converted_df[col].dt.tz_localize(None)
+                if not pd.api.types.is_numeric_dtype(converted_df[col]):
+                    # instant: normalise to UTC before dropping the tz (BigQuery reads naive as UTC)
+                    values = parse_datetime_series(
+                        converted_df[col], infer_tz=False, field=col
+                    )
+                    if isinstance(values.dtype, pd.DatetimeTZDtype):
+                        values = values.dt.tz_convert("UTC").dt.tz_localize(None)
+                    converted_df[col] = values
 
             # Handle INTEGER type (GoogleSQL: INT64; legacy: INTEGER)
             elif bq_type in ("INT64", "INTEGER"):

{flowtask-5.13.5 → flowtask-5.13.7}/flowtask/components/CopyToPg.py

@@ -16,6 +16,7 @@ from pandas.api.types import (
 )
 import numpy as np
 from typing import Optional
+from querysource.types.dt.transforms import to_wall_clock
 import orjson
 import asyncpg
 from datamodel import Field
@@ -393,6 +394,23 @@ class CopyToPg(CopyTo):
         if converted:
             self._logger.debug(f"CopyToPg: coerced to text for {self.schema}.{self.tablename}: {converted}")
 
+    def _apply_naive_tz(self) -> None:
+        """Convert every ``naive_tz`` column to its local wall-clock (naive datetime64).
+
+        Missing columns are skipped with a warning; naive datetime columns are left as
+        they are; tz-aware, string and object columns go through QuerySource's
+        ``to_wall_clock`` (each value keeps its own local time). Must run before the
+        NaT -> None cleanup so converted NaT values are cleaned too.
+        """
+        for col in self._naive_columns:
+            if col not in self.data.columns:
+                self._logger.warning(f"CopyToPg: naive_tz column {col!r} not found; skipped")
+                continue
+            series = self.data[col]
+            if is_datetime64_any_dtype(series) and not isinstance(series.dtype, pd.DatetimeTZDtype):
+                continue
+            self.data[col] = to_wall_clock(series, field=col)
+
     async def _copy_dataframe(self):
         # insert data directly into table
         columns = list(self.data.columns)
@@ -547,6 +565,9 @@ class CopyToPg(CopyTo):
                         # Get PostgreSQL column types
                         pg_column_types = await self._get_column_types(conn)
 
+                    if self._naive_columns:
+                        self._apply_naive_tz()
+
                     # Handle datetime columns - replace NaT with None
                     datetime_cols = self.data.select_dtypes(include=['datetime64[ns]', 'datetime64[ns, UTC]'])
                     if not datetime_cols.empty:
@@ -554,24 +575,6 @@ class CopyToPg(CopyTo):
                             pd.notnull(datetime_cols), None
                         )
 
-                    if self._naive_columns:
-                        # Remove the timezone on tz-naive columns:
-                        for col in self._naive_columns:
-                            if col not in self.data.columns:
-                                continue
-
-                        # If the column is not datetime-like, coerce it first (optional)
-                        if not is_datetime64_any_dtype(self.data[col]):
-                            self.data[col] = pd.to_datetime(
-                                self.data[col],
-                                errors="coerce",  # bad values → NaT
-                                utc=True  # parse as UTC if a tz string is present
-                            )
-                        # After the coercion, act only on tz-aware columns
-                        if isinstance(self.data[col].dtype, pd.DatetimeTZDtype):
-                            # tz_localize(None) drops the tz info but keeps the *wall time*
-                            # e.g. 2025-03-20 19:11:09+00:00 → 2025-03-20 19:11:09
-                            self.data[col] = self.data[col].dt.tz_localize(None)
                     if self._json_columns:
                         for col in self._json_columns:
                             if col in self.data.columns:

{flowtask-5.13.5 → flowtask-5.13.7}/flowtask/components/DownloadFromFTP.py

@@ -1,6 +1,6 @@
 """
-  DownloadFromSmartSheet
-  Download an Excel file from SmartSheet.
+  DownloadFromFTP
+  Downloads files from a remote FTP server.
 
 
         Example:

flowtask-5.13.7/flowtask/components/DownloadFromSmartSheet.py

@@ -0,0 +1,315 @@
+"""
+  DownloadFromSmartSheet
+  Export a SmartSheet sheet to a file, or download row attachments from it.
+
+
+        Example:
+
+        ```yaml
+        DownloadFromSmartSheet:
+          comments: Download an SmartSheet Tab into an Excel file.
+          api_key: TRACFONE_SMARTSHEET_API_KEY
+          file_id: '373896609326980'
+          file_format: application/vnd.ms-excel
+          destination:
+            filename: WORP MPL 2022.xlsx
+            directory: /home/ubuntu/symbits/worp/files/stores/
+        ```
+
+    """
+import asyncio
+from collections.abc import Callable
+from pathlib import Path
+from typing import Dict, List, Optional
+from ..exceptions import ComponentError, ConfigError, FileNotFound
+from ..interfaces.download_from import DownloadFromBase
+from ..interfaces.smartsheet import SmartSheetClient
+
+
+class DownloadFromSmartSheet(SmartSheetClient, DownloadFromBase):
+    r"""
+    DownloadFromSmartSheet
+
+    Overview
+
+        Export a Smartsheet sheet to Excel/CSV (``file_id``), or download row attachments
+        (``sheet_id``) with filename/column/extension filters. Exactly one of the two ids
+        selects the mode. Export mode returns a list with the saved file path; attachments
+        mode returns a dict (filenames, skipped, failed, total, filtered_by_column,
+        filtered_by_filename, directory).
+
+        Attachments filename format: {row_id}_{column1_value}_{column2_value}_{original_filename}
+
+    Properties
+
+        :widths: auto
+
+        | api_key            |   No     | Name of the environment variable holding the key (or a raw key).                 |
+        |                    |          | Default: the `SMARTSHEET_API_KEY` environment variable.                          |
+        | credentials        |   No     | Alternative form: {token: ENV_NAME, scheme: Bearer}. Wins over api_key.          |
+        | create_destination |   No     | Create the destination directory if missing (default: False).                    |
+        | url                |   No     | Base URL for the SmartSheet Sheets API                                           |
+        |                    |          | (default: https://api.smartsheet.com/2.0/sheets/).                               |
+        | file_id            |   No*    | Export mode: id of the sheet to export. *Required unless sheet_id is given.      |
+        | file_format        |   No     | Export mode: "application/vnd.ms-excel" (default) or "text/csv".                 |
+        | destination        |   No     | Export mode: {filename, directory} of the saved file.                            |
+        | sheet_id           |   No*    | Attachments mode: id of the sheet. *Required unless file_id is given.            |
+        | directory          |   No     | Attachments mode: directory where attachments are saved (required in that mode). |
+        | row_id             |   No     | Attachments mode: download only from this row (default: all rows).               |
+        | overwrite          |   No     | Attachments mode: overwrite existing files (default: False).                     |
+        | attachment_filter  |   No     | Attachments mode: list of file extensions to keep (e.g., ['.pdf', '.xlsx']).     |
+        | filename_columns   |   No     | Attachments mode: columns used in the filename. Default: lead_id, Company Name.  |
+        | filename_filter    |   No     | Attachments mode: regex the attachment filename must match.                      |
+        | column_filter      |   No     | Attachments mode: dict of column_name: value; only matching rows are used.       |
+        | use_row_number     |   No     | Attachments mode: use the row number instead of the row id in filenames.         |
+
+    :consumes: none
+    :produces: file
+
+        Example (export):
+
+        ```yaml
+        DownloadFromSmartSheet:
+          comments: Download an SmartSheet Tab into an Excel file.
+          api_key: TRACFONE_SMARTSHEET_API_KEY
+          file_id: '373896609326980'
+          file_format: application/vnd.ms-excel
+          destination:
+            filename: WORP MPL 2022.xlsx
+            directory: /home/ubuntu/symbits/worp/files/stores/
+        ```
+
+        Example (attachments):
+
+        ```yaml
+        DownloadFromSmartSheet:
+          comments: Download only "Full Packet.pdf" files from approved rows
+          api_key: TRACFONE_SMARTSHEET_API_KEY
+          sheet_id: '8504624500658052'
+          directory: /home/ubuntu/symbits/smartsheet/attachments/
+          overwrite: false
+          filename_filter: 'Full Packet\.pdf$'
+          column_filter:
+            Document Status: 'Send to TROC Processing'
+          filename_columns:
+            - lead_id
+            - Company Name
+        ```
+    """
+    _version = "1.1.0"
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        # Attachments-mode parameters
+        self.sheet_id: str = kwargs.pop('sheet_id', None)
+        self.row_id: Optional[str] = kwargs.pop('row_id', None)
+        self.directory: Path = kwargs.pop('directory', None)
+        _overwrite: bool = kwargs.pop('overwrite', False)
+        self.attachment_filter: Optional[List[str]] = kwargs.pop('attachment_filter', None)
+        self.filename_columns: Optional[List[str]] = kwargs.pop('filename_columns', None)
+        self.filename_filter: Optional[str] = kwargs.pop('filename_filter', None)
+        self.column_filter: Optional[Dict[str, str]] = kwargs.pop('column_filter', None)
+        self.use_row_number: bool = kwargs.pop('use_row_number', False)
+        self._mode: str = None
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+        # DownloadFromBase.__init__ resets overwrite to True; honour the documented default.
+        self.overwrite: bool = _overwrite
+
+    async def start(self, **kwargs) -> bool:
+        """Select the mode, validate its parameters, then run the base lifecycle."""
+        if bool(self.file_id) == bool(self.sheet_id):
+            raise ConfigError(
+                "DownloadFromSmartSheet: set exactly one of 'file_id' (export) or 'sheet_id' (attachments)"
+            )
+        self._mode = "export" if self.file_id else "attachments"
+        if self._mode == "export":
+            if self.file_format not in ["application/vnd.ms-excel", "text/csv"]:
+                # only supported
+                raise ComponentError(
+                    f"SmartSheet: Format {self.file_format} is not suported"
+                )
+            try:
+                self.accept = (
+                    "text/csv" if self.file_format == "dataframe" else self.file_format
+                )
+            except Exception as err:
+                self._logger.warning(f"SmartSheet: could not set Accept header: {err}")
+        else:
+            if not self.directory:
+                raise ConfigError(
+                    "DownloadFromSmartSheet: directory is required"
+                )
+            if isinstance(self.directory, str):
+                self.directory = Path(self.directory)
+            if self.attachment_filter:
+                if not isinstance(self.attachment_filter, list):
+                    raise ConfigError(
+                        "DownloadFromSmartSheet: attachment_filter must be a list of file extensions"
+                    )
+                # Ensure extensions start with a dot
+                self.attachment_filter = [
+                    ext if ext.startswith('.') else f'.{ext}'
+                    for ext in self.attachment_filter
+                ]
+            if self.filename_columns is not None:
+                if not isinstance(self.filename_columns, list):
+                    raise ConfigError(
+                        "DownloadFromSmartSheet: filename_columns must be a list of column names"
+                    )
+            if self.filename_filter is not None:
+                if not isinstance(self.filename_filter, str):
+                    raise ConfigError(
+                        "DownloadFromSmartSheet: filename_filter must be a string regex pattern"
+                    )
+            if self.column_filter is not None:
+                if not isinstance(self.column_filter, dict):
+                    raise ConfigError(
+                        "DownloadFromSmartSheet: column_filter must be a dict of column_name: value pairs"
+                    )
+        await super().start(**kwargs)
+        return True
+
+    async def close(self) -> bool:
+        """No resources to release."""
+        return True
+
+    async def run(self):
+        """Dispatch on the mode; each mode keeps its historical return shape."""
+        self._result = None
+        if self._mode == "export":
+            return await self._run_export()
+        return await self._run_attachments()
+
+    async def _run_export(self):
+        """Export a sheet by ``file_id``; returns ``[path]``."""
+        try:
+            self._logger.info(f"Downloading SmartSheet file: {self.file_id}")
+            # check if self.filename is a relative path:
+            if isinstance(self.filename, str):
+                self.filename = Path(self.filename)
+            if not self.filename.is_absolute():
+                self.filename = self.directory.joinpath(self.filename)
+            if await self.download_file(file_id=self.file_id, destination=self.filename):
+                self._filenames = [str(self.filename)]
+                self._result = self._filenames
+                self.add_metric("SMARTSHEET_FILE", self.filename)
+            return self._result
+        except ComponentError as err:
+            raise FileNotFound(f"DownloadFromSmartSheet Error: {err}") from err
+        except Exception as err:
+            raise ComponentError(
+                f"DownloadFromSmartSheet: Unknown Error: {err}"
+            ) from err
+
+    async def _run_attachments(self):
+        """Download row attachments of ``sheet_id``; returns the results dict."""
+        self._logger.info(
+            f"Starting SmartSheet attachment download for sheet {self.sheet_id}"
+        )
+
+        if self.row_id:
+            self._logger.info(f"Downloading attachments from row {self.row_id}")
+        else:
+            self._logger.info("Downloading all sheet attachments")
+
+        # Log filter configuration
+        if self.column_filter:
+            filter_str = ", ".join([f"{k}='{v}'" for k, v in self.column_filter.items()])
+            self._logger.info(f"Applying column filter: {filter_str}")
+
+        if self.filename_filter:
+            self._logger.info(f"Applying filename pattern: '{self.filename_filter}'")
+
+        if self.attachment_filter:
+            self._logger.info(f"Filtering extensions: {', '.join(self.attachment_filter)}")
+
+        # Log filename column configuration
+        if self.filename_columns:
+            self._logger.info(
+                f"Using columns for unique filenames: {', '.join(self.filename_columns)}"
+            )
+        else:
+            self._logger.info("Using default columns: lead_id, Company Name")
+
+        try:
+            # Call the download_attachments method from SmartSheetClient
+            results = await self.download_attachments(
+                sheet_id=self.sheet_id,
+                row_id=self.row_id,
+                directory=self.directory,
+                overwrite=self.overwrite,
+                attachment_filter=self.attachment_filter,
+                filename_columns=self.filename_columns,
+                filename_filter=self.filename_filter,
+                column_filter=self.column_filter,
+                use_row_number=self.use_row_number
+            )
+
+            # Add directory to results
+            results['directory'] = str(self.directory)
+
+            # Log summary
+            self._logger.info(
+                f"Download complete: {len(results['filenames'])} downloaded, "
+                f"{len(results['skipped'])} skipped, "
+                f"{len(results['failed'])} failed out of {results['total']} total attachments"
+            )
+
+            self.add_metric(
+                "smartsheet_attachments_downloaded", len(results['filenames'])
+            )
+
+            # Log filter statistics
+            if results.get('filtered_by_column', 0) > 0:
+                self._logger.info(
+                    f"Filtered by column criteria: {results['filtered_by_column']} attachments"
+                )
+
+            if results.get('filtered_by_filename', 0) > 0:
+                self._logger.info(
+                    f"Filtered by filename pattern: {results['filtered_by_filename']} attachments"
+                )
+
+            # Log downloaded files (show first 10)
+            file_list = results.get('filenames', [])
+            self.add_metric("smartsheet_downloaded_files", file_list)
+
+            if results['filenames']:
+                self._logger.notice("Successfully downloaded files:")
+                display_count = min(10, len(results['filenames']))
+                for file_path in results['filenames'][:display_count]:
+                    filename = Path(file_path).name
+                    self._logger.notice(f"  - {filename}")
+                if len(results['filenames']) > 10:
+                    self._logger.notice(f"  ... and {len(results['filenames']) - 10} more files")
+
+            # Log skipped files (show first 5)
+            if results['skipped']:
+                self._logger.info(f"Skipped {len(results['skipped'])} files:")
+                display_count = min(5, len(results['skipped']))
+                for item in results['skipped'][:display_count]:
+                    self._logger.info(f"  - {item['file']}: {item['reason']}")
+                if len(results['skipped']) > 5:
+                    self._logger.info(f"  ... and {len(results['skipped']) - 5} more skipped")
+
+            # Log failed files
+            if results['failed']:
+                self._logger.warning(f"Failed {len(results['failed'])} downloads:")
+                for item in results['failed']:
+                    self._logger.warning(f"  - {item['file']}: {item['error']}")
+
+            # Store result for later access
+            self._result = results
+            return results
+
+        except ComponentError:
+            raise
+        except Exception as err:
+            raise ComponentError(
+                f"DownloadFromSmartSheet: Unexpected error during execution: {err}"
+            ) from err

{flowtask-5.13.5 → flowtask-5.13.7}/flowtask/components/DuplicatePhoto.py

@@ -13,7 +13,7 @@ from pgvector.utils import Vector
 from navigator.libs.json import JSONContent
 from ..interfaces.flow import FlowComponent
 from ..exceptions import ConfigError, ComponentError
-from ..conf import default_dsn
+from ..conf import default_dsn, pg_application_name
 
 IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
 
@@ -252,6 +252,9 @@ class DuplicatePhoto(FlowComponent):
             max_queries=100,
             init=self.pgvector_init,
             timeout=10,
+            server_settings={
+                "application_name": pg_application_name("DuplicatePhoto")
+            },
         )
         # Check if the table exists
         if not self.pool:

{flowtask-5.13.5 → flowtask-5.13.7}/flowtask/components/ExtractTranscript.py

@@ -17,7 +17,7 @@ from sqlalchemy import create_engine, text, bindparam
 from ..interfaces.flow import FlowComponent
 from ..interfaces.Boto3Client import Boto3Client
 from ..exceptions import ConfigError, ComponentError
-from ..conf import default_dsn
+from ..conf import default_dsn, pg_application_name
 
 # Import torch conditionally for CUDA cleanup
 try:
@@ -1179,7 +1179,12 @@ class ExtractTranscript(Boto3Client, FlowComponent):
                     dsn = str(dsn)
                 if isinstance(dsn, str) and dsn.startswith('postgres:'):
                     dsn = dsn.replace('postgres:', 'postgresql:')
-                self._lookup_engine = create_engine(dsn, echo=False)
+                engine_kwargs: dict = {}
+                if isinstance(dsn, str) and dsn.startswith('postgresql'):
+                    engine_kwargs['connect_args'] = {
+                        'application_name': pg_application_name('ExtractTranscript')
+                    }
+                self._lookup_engine = create_engine(dsn, echo=False, **engine_kwargs)
             except Exception as err:
                 self._logger.error(f"Error creating lookup engine: {err}")
                 return None

{flowtask-5.13.5 → flowtask-5.13.7}/flowtask/components/IcimsFolderCopy.py

@@ -4,6 +4,7 @@ import asyncio
 from navconfig.logging import logging
 from asyncdb.exceptions import ProviderError
 from asyncdb.drivers.pg import pg
+from ..conf import pg_application_name
 from ..interfaces.flow import FlowComponent
 from ..exceptions import ComponentError, DataNotFound
 from querysource.conf import default_dsn, DB_TIMEOUT
@@ -68,7 +69,13 @@ class IcimsFolderCopy(FlowComponent):
     async def run(self):
         """Execute the folder copying process based on database results using Pandas."""
         try:
-            db = pg(dsn=self.dsn, timeout=self.timeout)
+            db = pg(
+                dsn=self.dsn,
+                timeout=self.timeout,
+                server_settings={
+                    "application_name": pg_application_name("IcimsFolderCopy")
+                },
+            )
         except ProviderError as e:
             raise ComponentError(f"Error connecting to database: {e}")
 

{flowtask-5.13.5 → flowtask-5.13.7}/flowtask/components/MergeFiles.py

@@ -11,6 +11,7 @@ from ..exceptions import ComponentError
 from ..parsers.maps import open_model
 from ..interfaces.flow import FlowComponent
 from .OpenWithBase import detect_encoding, excel_based
+from .pandas_io import _post_parse_dates, _preserve_date_inputs
 
 
 class MergeFiles(FlowComponent):
@@ -176,13 +177,13 @@ class MergeFiles(FlowComponent):
                     file_engine = "pyxlsb"
                 else:
                     file_engine = "openpyxl"
+                dates = []
                 # get the Model (if any):
                 if hasattr(self, "model"):
                     columns = await open_model(
                         self.model, self._program, path=self._taskstore.get_path()
                     )
                     fields = []
-                    dates = []
                     for field, dtype in columns["fields"].items():
                         fields.append(field)
                         try:
@@ -192,8 +193,12 @@ class MergeFiles(FlowComponent):
                         if t in {"date", "datetime", "time"}:
                             dates.append(field)
                     args["names"] = fields
-                    if dates:
-                        args["parse_dates"] = dates
+                reader_options = _preserve_date_inputs(args, dates)
+                date_options = {
+                    key: reader_options.pop(key)
+                    for key in ("dayfirst", "date_format")
+                    if dates and key in reader_options
+                }
                 files = []
                 file_stats = {}
                 for file in self.data:
@@ -205,8 +210,9 @@ class MergeFiles(FlowComponent):
                             na_values=["TBD", "NULL", "null", "", "#NA"],
                             engine=file_engine,
                             keep_default_na=True,
-                            **args,
+                            **reader_options,
                         )
+                        df = _post_parse_dates(df, dates, **date_options)
                         file_args = {}
                         if hasattr(self, 'file_stats'):
                             first_row = df.iloc[0][self.file_stats['columns']].to_dict()

{flowtask-5.13.5 → flowtask-5.13.7}/flowtask/components/OdooInjector.py

@@ -1,12 +1,11 @@
 import asyncio
-import math
 from collections.abc import Callable
 from urllib.parse import urljoin
-import numpy as np
 import pandas as pd
 from ..interfaces.flow import FlowComponent
 from ..interfaces.http import HTTPService
 from ..exceptions import ComponentError, TaskError
+from ..utils.dataframes import split_dataframe
 
 
 class OdooInjector(HTTPService, FlowComponent):
@@ -209,10 +208,4 @@ class OdooInjector(HTTPService, FlowComponent):
         list: A list of DataFrame chunks.
             If the DataFrame is empty, returns an empty list.
         """
-        if df.empty:
-            return []
-
-        split_n = math.ceil(len(df) / chunk_size)
-
-        # Split into chunks of n rows
-        return np.array_split(df, split_n)  # Returns list of DataFrames
+        return split_dataframe(df, chunk_size)