flowtask 5.13.5__tar.gz → 5.13.6__tar.gz

{flowtask-5.13.5 → flowtask-5.13.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowtask
-Version: 5.13.5
+Version: 5.13.6
 Summary: Framework for Task orchestration
 Author-email: Jesus Lara <jesuslarag@gmail.com>
 License: Apache-2.0
@@ -105,7 +105,7 @@ Requires-Dist: pillow-heif<2,>=1.3.0
 Requires-Dist: pgvector==0.4.1
 Requires-Dist: proxylists>=0.14.0
 Requires-Dist: async-notify[all]>=1.4.0
-Requires-Dist: querysource>=3.17.0
+Requires-Dist: querysource>=5.2.5
 Requires-Dist: asyncdb[all]>=2.11.7
 Requires-Dist: navconfig[default,uvloop]>=1.7.13
 Requires-Dist: navigator-auth>=0.15.8

{flowtask-5.13.5 → flowtask-5.13.6}/flowtask/components/CopyToBigQuery.py

@@ -4,6 +4,7 @@ import datetime as dt
 import math
 import time
 import pandas as pd
+from querysource.types.dt.transforms import parse_datetime_series, to_wall_clock
 from asyncdb import AsyncDB
 from asyncdb.exceptions import (
     StatementError,
@@ -640,6 +641,17 @@ class CopyToBigQuery(CopyTo, PandasDataframe):
             self._logger.warning(f'CopyTo: Could not retrieve table schema: {err}')
         return column_types
 
+    @staticmethod
+    def _is_date_objects(series: pd.Series) -> bool:
+        """Return True for an object column whose non-null values are all plain ``datetime.date``.
+
+        Such columns are already DATE-ready and are left unchanged.
+        """
+        if not pd.api.types.is_object_dtype(series):
+            return False
+        values = series.dropna()
+        return not values.empty and all(type(v) is dt.date for v in values)
+
     def _convert_dataframe_to_bigquery_types(self, df, column_types):
         """Convert DataFrame columns to match BigQuery schema types."""
         converted_df = df.copy()
@@ -668,8 +680,9 @@ class CopyToBigQuery(CopyTo, PandasDataframe):
                     self._logger.debug(
                         f"CopyTo: Converted {col} to DATE (from numeric values)"
                     )
-                elif not pd.api.types.is_object_dtype(converted_df[col]):
-                    dates = pd.to_datetime(converted_df[col], errors="coerce").dt.date
+                elif not self._is_date_objects(converted_df[col]):
+                    # str / string / object of strings or timestamps: local calendar date
+                    dates = to_wall_clock(converted_df[col], field=col).dt.date
                     converted_df[col] = dates.astype(object).where(
                         dates.notna(), None
                     )
@@ -679,13 +692,20 @@ class CopyToBigQuery(CopyTo, PandasDataframe):
 
             # Handle DATETIME type
             elif bq_type == "DATETIME":
-                if pd.api.types.is_datetime64_any_dtype(converted_df[col]):
-                    converted_df[col] = converted_df[col].dt.tz_localize(None)
+                if not pd.api.types.is_numeric_dtype(converted_df[col]):
+                    # civil value: every entry keeps its own local wall-clock
+                    converted_df[col] = to_wall_clock(converted_df[col], field=col)
 
             # Handle TIMESTAMP type
             elif bq_type == "TIMESTAMP":
-                if pd.api.types.is_datetime64_any_dtype(converted_df[col]):
-                    converted_df[col] = converted_df[col].dt.tz_localize(None)
+                if not pd.api.types.is_numeric_dtype(converted_df[col]):
+                    # instant: normalise to UTC before dropping the tz (BigQuery reads naive as UTC)
+                    values = parse_datetime_series(
+                        converted_df[col], infer_tz=False, field=col
+                    )
+                    if isinstance(values.dtype, pd.DatetimeTZDtype):
+                        values = values.dt.tz_convert("UTC").dt.tz_localize(None)
+                    converted_df[col] = values
 
             # Handle INTEGER type (GoogleSQL: INT64; legacy: INTEGER)
             elif bq_type in ("INT64", "INTEGER"):

{flowtask-5.13.5 → flowtask-5.13.6}/flowtask/components/CopyToPg.py

@@ -16,6 +16,7 @@ from pandas.api.types import (
 )
 import numpy as np
 from typing import Optional
+from querysource.types.dt.transforms import to_wall_clock
 import orjson
 import asyncpg
 from datamodel import Field
@@ -393,6 +394,23 @@ class CopyToPg(CopyTo):
         if converted:
             self._logger.debug(f"CopyToPg: coerced to text for {self.schema}.{self.tablename}: {converted}")
 
+    def _apply_naive_tz(self) -> None:
+        """Convert every ``naive_tz`` column to its local wall-clock (naive datetime64).
+
+        Missing columns are skipped with a warning; naive datetime columns are left as
+        they are; tz-aware, string and object columns go through QuerySource's
+        ``to_wall_clock`` (each value keeps its own local time). Must run before the
+        NaT -> None cleanup so converted NaT values are cleaned too.
+        """
+        for col in self._naive_columns:
+            if col not in self.data.columns:
+                self._logger.warning(f"CopyToPg: naive_tz column {col!r} not found; skipped")
+                continue
+            series = self.data[col]
+            if is_datetime64_any_dtype(series) and not isinstance(series.dtype, pd.DatetimeTZDtype):
+                continue
+            self.data[col] = to_wall_clock(series, field=col)
+
     async def _copy_dataframe(self):
         # insert data directly into table
         columns = list(self.data.columns)
@@ -547,6 +565,9 @@ class CopyToPg(CopyTo):
                         # Get PostgreSQL column types
                         pg_column_types = await self._get_column_types(conn)
 
+                    if self._naive_columns:
+                        self._apply_naive_tz()
+
                     # Handle datetime columns - replace NaT with None
                     datetime_cols = self.data.select_dtypes(include=['datetime64[ns]', 'datetime64[ns, UTC]'])
                     if not datetime_cols.empty:
@@ -554,24 +575,6 @@ class CopyToPg(CopyTo):
                             pd.notnull(datetime_cols), None
                         )
 
-                    if self._naive_columns:
-                        # Remove the timezone on tz-naive columns:
-                        for col in self._naive_columns:
-                            if col not in self.data.columns:
-                                continue
-
-                        # If the column is not datetime-like, coerce it first (optional)
-                        if not is_datetime64_any_dtype(self.data[col]):
-                            self.data[col] = pd.to_datetime(
-                                self.data[col],
-                                errors="coerce",  # bad values → NaT
-                                utc=True  # parse as UTC if a tz string is present
-                            )
-                        # After the coercion, act only on tz-aware columns
-                        if isinstance(self.data[col].dtype, pd.DatetimeTZDtype):
-                            # tz_localize(None) drops the tz info but keeps the *wall time*
-                            # e.g. 2025-03-20 19:11:09+00:00 → 2025-03-20 19:11:09
-                            self.data[col] = self.data[col].dt.tz_localize(None)
                     if self._json_columns:
                         for col in self._json_columns:
                             if col in self.data.columns:

{flowtask-5.13.5 → flowtask-5.13.6}/flowtask/components/DuplicatePhoto.py

@@ -13,7 +13,7 @@ from pgvector.utils import Vector
 from navigator.libs.json import JSONContent
 from ..interfaces.flow import FlowComponent
 from ..exceptions import ConfigError, ComponentError
-from ..conf import default_dsn
+from ..conf import default_dsn, pg_application_name
 
 IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
 
@@ -252,6 +252,9 @@ class DuplicatePhoto(FlowComponent):
             max_queries=100,
             init=self.pgvector_init,
             timeout=10,
+            server_settings={
+                "application_name": pg_application_name("DuplicatePhoto")
+            },
         )
         # Check if the table exists
         if not self.pool:

{flowtask-5.13.5 → flowtask-5.13.6}/flowtask/components/ExtractTranscript.py

@@ -17,7 +17,7 @@ from sqlalchemy import create_engine, text, bindparam
 from ..interfaces.flow import FlowComponent
 from ..interfaces.Boto3Client import Boto3Client
 from ..exceptions import ConfigError, ComponentError
-from ..conf import default_dsn
+from ..conf import default_dsn, pg_application_name
 
 # Import torch conditionally for CUDA cleanup
 try:
@@ -1179,7 +1179,12 @@ class ExtractTranscript(Boto3Client, FlowComponent):
                     dsn = str(dsn)
                 if isinstance(dsn, str) and dsn.startswith('postgres:'):
                     dsn = dsn.replace('postgres:', 'postgresql:')
-                self._lookup_engine = create_engine(dsn, echo=False)
+                engine_kwargs: dict = {}
+                if isinstance(dsn, str) and dsn.startswith('postgresql'):
+                    engine_kwargs['connect_args'] = {
+                        'application_name': pg_application_name('ExtractTranscript')
+                    }
+                self._lookup_engine = create_engine(dsn, echo=False, **engine_kwargs)
             except Exception as err:
                 self._logger.error(f"Error creating lookup engine: {err}")
                 return None

{flowtask-5.13.5 → flowtask-5.13.6}/flowtask/components/IcimsFolderCopy.py

@@ -4,6 +4,7 @@ import asyncio
 from navconfig.logging import logging
 from asyncdb.exceptions import ProviderError
 from asyncdb.drivers.pg import pg
+from ..conf import pg_application_name
 from ..interfaces.flow import FlowComponent
 from ..exceptions import ComponentError, DataNotFound
 from querysource.conf import default_dsn, DB_TIMEOUT
@@ -68,7 +69,13 @@ class IcimsFolderCopy(FlowComponent):
     async def run(self):
         """Execute the folder copying process based on database results using Pandas."""
         try:
-            db = pg(dsn=self.dsn, timeout=self.timeout)
+            db = pg(
+                dsn=self.dsn,
+                timeout=self.timeout,
+                server_settings={
+                    "application_name": pg_application_name("IcimsFolderCopy")
+                },
+            )
         except ProviderError as e:
             raise ComponentError(f"Error connecting to database: {e}")
 

{flowtask-5.13.5 → flowtask-5.13.6}/flowtask/components/MergeFiles.py

@@ -11,6 +11,7 @@ from ..exceptions import ComponentError
 from ..parsers.maps import open_model
 from ..interfaces.flow import FlowComponent
 from .OpenWithBase import detect_encoding, excel_based
+from .pandas_io import _post_parse_dates, _preserve_date_inputs
 
 
 class MergeFiles(FlowComponent):
@@ -176,13 +177,13 @@ class MergeFiles(FlowComponent):
                     file_engine = "pyxlsb"
                 else:
                     file_engine = "openpyxl"
+                dates = []
                 # get the Model (if any):
                 if hasattr(self, "model"):
                     columns = await open_model(
                         self.model, self._program, path=self._taskstore.get_path()
                     )
                     fields = []
-                    dates = []
                     for field, dtype in columns["fields"].items():
                         fields.append(field)
                         try:
@@ -192,8 +193,12 @@ class MergeFiles(FlowComponent):
                         if t in {"date", "datetime", "time"}:
                             dates.append(field)
                     args["names"] = fields
-                    if dates:
-                        args["parse_dates"] = dates
+                reader_options = _preserve_date_inputs(args, dates)
+                date_options = {
+                    key: reader_options.pop(key)
+                    for key in ("dayfirst", "date_format")
+                    if dates and key in reader_options
+                }
                 files = []
                 file_stats = {}
                 for file in self.data:
@@ -205,8 +210,9 @@ class MergeFiles(FlowComponent):
                             na_values=["TBD", "NULL", "null", "", "#NA"],
                             engine=file_engine,
                             keep_default_na=True,
-                            **args,
+                            **reader_options,
                         )
+                        df = _post_parse_dates(df, dates, **date_options)
                         file_args = {}
                         if hasattr(self, 'file_stats'):
                             first_row = df.iloc[0][self.file_stats['columns']].to_dict()

{flowtask-5.13.5 → flowtask-5.13.6}/flowtask/components/OpenWithPandas.py

@@ -10,6 +10,8 @@ import numpy as np
 from ..utils import check_empty
 from ..exceptions import ComponentError, DataNotFound, EmptyFile
 from .OpenWithBase import OpenWithBase, excel_based
+from querysource.types.dt.transforms import parse_datetime_series, to_wall_clock
+from .pandas_io import _post_parse_dates, _preserve_date_inputs, _split_parse_dates
 
 
 # Suppress specific warning
@@ -387,6 +389,13 @@ class OpenWithPandas(OpenWithBase):
         # uses async aiofiles and a chunked iterator — kept inline).
         if hasattr(self, "bigfile"):
             try:
+                date_kwargs, date_columns = _split_parse_dates(self.parse_dates)
+                reader_options = _preserve_date_inputs({**add_columns, **date_kwargs, **self.args}, date_columns)
+                date_options = {
+                    key: reader_options.pop(key)
+                    for key in ("dayfirst", "date_format")
+                    if date_columns and key in reader_options
+                }
                 tp = pandas.read_csv(
                     filename,
                     sep=self.separator,
@@ -399,11 +408,16 @@ class OpenWithPandas(OpenWithBase):
                     skipinitialspace=True,
                     iterator=True,
                     chunksize=int(self.chunksize),
-                    **add_columns,
-                    **self.parse_dates,
-                    **self.args,
+                    **reader_options,
+                )
+                return _post_parse_dates(
+                    pandas.concat(
+                        tp,
+                        ignore_index=reader_options.get("index_col") is None or reader_options.get("index_col") is False,
+                    ),
+                    date_columns,
+                    **date_options,
                 )
-                return pandas.concat(tp, ignore_index=True)
             except pandas.errors.EmptyDataError as err:
                 raise ComponentError(
                     f"Empty Data File on: {filename}, error: {err}"
@@ -459,7 +473,10 @@ class OpenWithPandas(OpenWithBase):
                 for file in self._data:
                     try:
                         df = pandas.DataFrame(
-                            data=file, **add_columns, **self.parse_dates, **self.args
+                            data=file, **add_columns, **self.args
+                        )
+                        df = _post_parse_dates(
+                            df, _split_parse_dates(self.parse_dates)[1]
                         )
                         result.append(df)
                     except pandas.errors.EmptyDataError as err:
@@ -605,9 +622,17 @@ class OpenWithPandas(OpenWithBase):
                         or dtype == "timestamp with time zone"
                         or dtype == "date"
                     ):
-                        if df[column].dtype != "datetime64[ns]":
-                            df[column] = pandas.to_datetime(df[column], errors="coerce")
-                            df[column] = df[column].astype("datetime64[ns]")
+                        if dtype == "timestamp with time zone":
+                            values = parse_datetime_series(
+                                df[column], infer_tz=False, field=column
+                            )
+                            if isinstance(values.dtype, pandas.DatetimeTZDtype):
+                                values = values.dt.tz_convert("UTC")
+                            else:
+                                values = values.dt.tz_localize("UTC")
+                            df[column] = values
+                        else:
+                            df[column] = to_wall_clock(df[column], field=column)
                     elif (
                         dtype == "character varying"
                         or dtype == "character"

{flowtask-5.13.5 → flowtask-5.13.6}/flowtask/components/QueryToInsert.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from typing import Any
+from typing import Any, Optional
 from collections.abc import Callable
 import asyncio
 from datetime import datetime
@@ -7,11 +7,11 @@ from pathlib import PosixPath
 from tqdm import tqdm
 import orjson
 from datamodel.parsers.json import json_encoder
-from asyncdb.drivers.postgres import postgres
+from asyncdb.drivers.pg import pg
 from asyncdb.exceptions import ProviderError
 from ..exceptions import ComponentError
 from ..utils import SafeDict
-from ..conf import default_dsn
+from ..conf import default_dsn, pg_application_name
 from ..interfaces.flow import FlowComponent
 
 
@@ -197,9 +197,20 @@ class QueryToInsert(FlowComponent):
         )
         return True
 
-    async def get_connection(self):
+    async def get_connection(self) -> pg:
+        """Open a ``pg`` connection named ``<APP_NAME>.QueryToInsert``.
+
+        Raises:
+            ProviderError: when the connection cannot be opened.
+        """
         try:
-            self._connection = postgres(dsn=default_dsn, loop=self._loop)
+            self._connection = pg(
+                dsn=default_dsn,
+                loop=self._loop,
+                server_settings={
+                    "application_name": pg_application_name("QueryToInsert")
+                },
+            )
             await self._connection.connection()
         except Exception as err:
             raise ProviderError(f"Error configuring pg Connection: {err!s}") from err
@@ -214,7 +225,7 @@ class QueryToInsert(FlowComponent):
             raise ComponentError(
                 'Query Error "{}": {}'.format(self.query, err)
             )
-        colinfo = self.column_info("{}.{}".format(self.schema, self.tablename))
+        colinfo = await self.column_info("{}.{}".format(self.schema, self.tablename))
         if res:
             query = []
             if self.action == "insert":
@@ -277,21 +288,30 @@ class QueryToInsert(FlowComponent):
         """Method."""
         pass
 
-    def column_info(self, table):
+    async def column_info(self, table: str) -> Optional[dict[str, Any]]:
+        """Return ``{column_name: data_type}`` for ``table`` or ``None`` when it does not exist."""
         result = None
         sql = f"""
         SELECT attname AS column_name, atttypid::regtype AS data_type, attnotnull::boolean as notnull
         FROM pg_attribute WHERE attrelid = '{table}'::regclass AND attnum > 0 AND NOT attisdropped
         ORDER  BY attnum"""
-        db = postgres(dsn=default_dsn)
-        conn = db.connect()
-        result, error = conn.fetchall(sql)
+        db = pg(
+            dsn=default_dsn,
+            loop=self._loop,
+            server_settings={
+                "application_name": pg_application_name("QueryToInsert")
+            },
+        )
+        async with await db.connection() as conn:
+            result, error = await conn.query(sql)
         if error:
             raise ComponentError(f"Error executing query: {error}")
         if result:
             return {item["column_name"]: item["data_type"] for item in result}
         else:
-            print(f"The table {self.schema}.{self.tablename} does not exist")
+            self._logger.warning(
+                f"The table {self.schema}.{self.tablename} does not exist"
+            )
             return None
 
     def get_values(self, column, type, value):

{flowtask-5.13.5 → flowtask-5.13.6}/flowtask/components/SaveImageBank/__init__.py

@@ -9,7 +9,7 @@ from querysource.types.validators import Entity
 from navigator.libs.json import JSONContent
 from ...interfaces.flow import FlowComponent
 from ...exceptions import ConfigError, ComponentError
-from ...conf import default_dsn
+from ...conf import default_dsn, pg_application_name
 
 IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
 
@@ -168,6 +168,9 @@ class SaveImageBank(FlowComponent):
             max_queries=100,
             init=self.pgvector_init,
             timeout=10,
+            server_settings={
+                "application_name": pg_application_name("SaveImageBank")
+            },
         )
         # Check if the table exists
         if not self.pool:

{flowtask-5.13.5 → flowtask-5.13.6}/flowtask/components/TableDelete.py

@@ -23,6 +23,7 @@ from querysource.conf import (
 from sqlalchemy.orm import Session
 from sqlalchemy import create_engine, text
 from sqlalchemy.pool import NullPool
+from ..conf import pg_application_name
 from ..exceptions import ComponentError, DataNotFound
 from ..utils import AttrDict
 from ..interfaces.flow import FlowComponent
@@ -134,7 +135,14 @@ class TableDelete(QSSupport, FlowComponent):
         if not dsn:
             dsn = sqlalchemy_url
         try:
-            self._engine = create_engine(dsn, echo=False, poolclass=NullPool)
+            self._engine = create_engine(
+                dsn,
+                echo=False,
+                poolclass=NullPool,
+                connect_args={
+                    "application_name": pg_application_name("TableDelete")
+                },
+            )
             self._session = Session(bind=self._engine)
         except Exception as err:
             logging.exception(err)
@@ -159,23 +167,8 @@ class TableDelete(QSSupport, FlowComponent):
             columns = list(self.data.columns)
             for column in columns:
                 t = self.data[column].dtype
-                if isinstance(t, pd.core.dtypes.dtypes.DatetimeTZDtype):
-                    self.data[column] = pd.to_datetime(
-                        self.data[column],
-                        format="%Y-%m-%dT%H:%M:%S.%f%z",
-                        cache=True,
-                        utc=True,
-                    )
-                    self.data[column].dt.tz_convert("UTC")
-                elif str(t) == "datetime64[ns]":
-                    tmp_data = self.data.copy()
-                    tmp_data[column] = pd.to_datetime(
-                        self.data[column],
-                        format="%Y-%m-%dT%H:%M:%S.%f%z",
-                        cache=True,
-                        utc=True,
-                    )
-                    self.data = tmp_data.copy()
+                if isinstance(t, pd.DatetimeTZDtype):
+                    self.data[column] = self.data[column].dt.tz_convert("UTC")
         elif self.multi is True:
             # iteration over every Pandas DT:
             try:

{flowtask-5.13.5 → flowtask-5.13.6}/flowtask/components/TableOutput/TableOutput.py

@@ -5,6 +5,7 @@ import logging
 import json
 from sqlalchemy.exc import NoSuchTableError
 import pandas as pd
+from querysource.types.dt.transforms import parse_datetime_series
 from datamodel.typedefs.types import AttrDict
 from querysource.outputs.tables import (
     PgOutput,
@@ -193,41 +194,20 @@ class TableOutput(FlowComponent, CredentialsInterface):
             columns = list(self.data.columns)
             for column in columns:
                 t = self.data[column].dtype
-                if isinstance(t, pd.core.dtypes.dtypes.DatetimeTZDtype):
-                    self.data[column] = pd.to_datetime(
-                        self.data[column],
-                        format="%Y-%m-%dT%H:%M:%S.%f%z",
-                        cache=True,
-                        utc=True,
-                    )
-                    self.data[column].dt.tz_convert("UTC")
-                elif str(t) == "datetime64[ns]":
-                    tmp_data = self.data.copy()
-                    tmp_data[column] = pd.to_datetime(
-                        self.data[column],
-                        format="%Y-%m-%dT%H:%M:%S.%f%z",
-                        cache=True,
-                        utc=True,
-                    )
-                    self.data = tmp_data.copy()
+                if isinstance(t, pd.DatetimeTZDtype):
+                    self.data[column] = self.data[column].dt.tz_convert("UTC")
                 else:
                     # this is an special column from RethinkDB
                     # rethinkdb.ast.RqlTzinfo
                     if column == "inserted_at":
-                        try:
-                            self.data[column] = pd.to_datetime(
-                                self.data[column],
-                                format="%Y-%m-%dT%H:%M:%S.%f%z",
-                                cache=True,
-                                utc=True,
-                            )
-                        except ValueError:
-                            self.data[column] = pd.to_datetime(
-                                self.data[column],
-                                # format='%Y-%m-%d %H:%M:%S.%f+%z',
-                                cache=True,
-                                utc=True,
-                            )
+                        values = parse_datetime_series(
+                            self.data[column], infer_tz=False, field=column
+                        )
+                        if isinstance(values.dtype, pd.DatetimeTZDtype):
+                            values = values.dt.tz_convert("UTC")
+                        else:
+                            values = values.dt.tz_localize("UTC")
+                        self.data[column] = values
         elif self.multi is True:
             # iteration over every Pandas DT:
             try: