flowtask 5.12.14__tar.gz → 5.12.16__tar.gz

{flowtask-5.12.14 → flowtask-5.12.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowtask
-Version: 5.12.14
+Version: 5.12.16
 Summary: Framework for Task orchestration
 Author-email: Jesus Lara <jesuslarag@gmail.com>
 License: Apache-2.0
@@ -73,9 +73,8 @@ Requires-Dist: aioftp==0.23.1
 Requires-Dist: py7zr==0.22.0
 Requires-Dist: rarfile==4.2
 Requires-Dist: python-pptx==1.0.2
-Requires-Dist: gspread==6.2.0
-Requires-Dist: oauth2client>=4.1.3
 Requires-Dist: google-auth>=2.35.0
+Requires-Dist: google-api-python-client>=2.198.0
 Requires-Dist: google-cloud-language>=2.15.0
 Requires-Dist: googlemaps==4.10.0
 Requires-Dist: fastavro==1.9.7
@@ -119,6 +118,8 @@ Requires-Dist: microsoft-kiota-authentication-azure~=1.9
 Requires-Dist: msgraph-sdk~=1.22
 Requires-Dist: zendriver>=0.15.2
 Requires-Dist: pyarrow>=14.0
+Requires-Dist: redis>=5.2
+Requires-Dist: cloudpickle>=3.1
 Requires-Dist: scipy>=1.13
 Requires-Dist: scikit-learn>=1.5
 Requires-Dist: polyline>=2.0.4
@@ -145,6 +146,7 @@ Requires-Dist: pytest-xdist==3.3.1; extra == "dev"
 Requires-Dist: pytest-assume==2.4.3; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: aioresponses>=0.7.8; extra == "dev"
+Requires-Dist: fakeredis[aioredis]>=2.26; extra == "dev"
 Dynamic: license-file
 
 # FlowTask #

{flowtask-5.12.14 → flowtask-5.12.16}/flowtask/components/CopyToBigQuery.py

@@ -14,7 +14,6 @@ from ..interfaces.copy_to import CopyTo
 from ..interfaces.dataframes import PandasDataframe
 from ..exceptions import (
     ComponentError,
-    DataNotFound
 )
 from querysource.conf import (
     BIGQUERY_CREDENTIALS,
@@ -305,7 +304,7 @@ class CopyToBigQuery(CopyTo, PandasDataframe):
 
     async def _truncate_table(self):
         """Truncate the BigQuery table using the driver's built-in method."""
-        async with await self._connection.connection() as conn:
+        async with await self._connection.connection():
             await self._connection.truncate_table(
                 table_id=self.tablename,
                 dataset_id=self.schema
@@ -421,7 +420,7 @@ class CopyToBigQuery(CopyTo, PandasDataframe):
                 table_ref,
                 job_config=job_config
             )
-            result = load_job.result()  # Wait for completion
+            load_job.result()  # Wait for completion
             self._logger.info(f'CopyTo: Loaded {len(records)} rows into {table_ref}')
 
     async def _copy_with_json_loader(self, json_columns, column_types):
@@ -656,9 +655,27 @@ class CopyToBigQuery(CopyTo, PandasDataframe):
                 if pd.api.types.is_datetime64_any_dtype(converted_df[col]):
                     converted_df[col] = pd.to_datetime(converted_df[col]).dt.date
                     self._logger.debug(f'CopyTo: Converted {col} to DATE (BigQuery schema type)')
-                elif pd.api.types.is_integer_dtype(converted_df[col]):
-                    # Unix timestamp to date
-                    converted_df[col] = pd.to_datetime(converted_df[col], unit='s').dt.date
+                elif pd.api.types.is_numeric_dtype(converted_df[col]):
+                    # Numeric DATE values represent Unix timestamps. This also
+                    # converts an all-null float column to Python None values
+                    # instead of passing float64/NaN to PyArrow as DATE.
+                    dates = pd.to_datetime(
+                        converted_df[col], unit="s", errors="coerce"
+                    ).dt.date
+                    converted_df[col] = dates.astype(object).where(
+                        dates.notna(), None
+                    )
+                    self._logger.debug(
+                        f"CopyTo: Converted {col} to DATE (from numeric values)"
+                    )
+                elif not pd.api.types.is_object_dtype(converted_df[col]):
+                    dates = pd.to_datetime(converted_df[col], errors="coerce").dt.date
+                    converted_df[col] = dates.astype(object).where(
+                        dates.notna(), None
+                    )
+                    self._logger.debug(
+                        f"CopyTo: Converted {col} to DATE (BigQuery schema type)"
+                    )
 
             # Handle DATETIME type
             elif bq_type == "DATETIME":

{flowtask-5.12.14 → flowtask-5.12.16}/flowtask/components/DateList.py

@@ -1,15 +1,17 @@
 import asyncio
-import logging
 import datetime
 import builtins
 from dateutil.relativedelta import relativedelta
 from numbers import Number
 from collections.abc import Callable
 from asyncdb.exceptions import NoDataFound, ProviderError
-from querysource.exceptions import DataNotFound
 from querysource.utils.functions import format_date
 import querysource.utils.functions as qsfunctions
-from ..exceptions import ComponentError, NotSupported
+from ..exceptions import ComponentError, NotSupported, DataNotFound
+from ..interfaces.skip_policy import (
+    ConsecutiveFailureTracker,
+    SKIPPED_ITERATION,
+)
 from ..utils import fnExecutor
 from .IteratorBase import IteratorBase
 
@@ -228,47 +230,61 @@ class DateList(IteratorBase):
                 job.close()
 
     async def run(self):
-        """Async Run Method."""
-        status = False
+        """Itera el rango de fechas ejecutando el paso siguiente por fecha.
+
+        Contrato tras FEAT-554 (spec §3 M3):
+          - `DataNotFound`/`NoDataFound`: `continue` INCONDICIONAL (D3) y **no**
+            cuenta para el umbral -- verified: DateList.py:247-252
+          - Familias no-de-datos: `async_job` ya aplico `skipError`; si llega
+            una excepcion aqui, la politica era ENFORCE y se propaga
+          - Aborta cuando los fallos CONSECUTIVOS alcanzan
+            `self.max_consecutive_failures` (0/None desactiva), con
+            `tracker.last_error` como causa
+
+        Returns:
+            `True` si hubo al menos una iteracion util y no se abortó; `False`
+            si no se completo ninguna. El veredicto NO depende del `status` de
+            la ultima vuelta (S8, §7 #10).
+        """
         if not self._iterator:
             return False
-        if self.iterate:
-            # iterate over next task
-            step, target, params = self.get_step()
-            step_name = step.name
-            for d in self._iterator:
-                dt = format_date(d, self._format)
-
-                self.setAttributes(dt)
-                job = self.createJob(target, params, dt, d)
-                if job:
-                    try:
-                        status = await self.async_job(job, step_name)
-                    except (NoDataFound, DataNotFound) as err:
-                        # its a data component a no data was found
-                        logging.debug(
-                            f"Data not Found for Task {step_name}, Error: {err}"
-                        )
-                        continue
-                    except (ProviderError, ComponentError) as err:
-                        raise ComponentError(
-                            f"Error running Component {step_name}, error: {err}"
-                        ) from err
-                    except NotSupported as err:
-                        raise NotSupported(
-                            f"Not Supported: Error on Component {step_name}, error: {err}"
-                        ) from err
-                    except Exception as err:
-                        raise ComponentError(
-                            f"DateList: Component Error on {step_name}, error: {err}"
-                        ) from err
-                    finally:
-                        await self.close(job)
-            if status is False:
-                return False
-            else:
-                return True
-        else:
-            # return the list of date
+        if not self.iterate:
             self._result = list(self._iterator)
             return True
+        step, target, params = self.get_step()
+        step_name = step.name
+        tracker = ConsecutiveFailureTracker(self.max_consecutive_failures)
+        for d in self._iterator:
+            dt = format_date(d, self._format)
+            self.setAttributes(dt)
+            job = self.createJob(target, params, dt, d)
+            if not job:
+                continue
+            try:
+                status = await self.async_job(job, step_name)
+            except (NoDataFound, DataNotFound) as err:
+                # D3: continue INCONDICIONAL, y NO cuenta para el umbral.
+                self._logger.debug(
+                    f"Data not Found for Task {step_name} at {dt}, Error: {err}"
+                )
+                continue
+            except (ProviderError, ComponentError, NotSupported):
+                # async_job ya consulto skipError: si llega aqui era ENFORCE.
+                raise
+            except Exception as err:
+                raise ComponentError(
+                    f"DateList: Component Error on {step_name} at {dt}, error: {err}"
+                ) from err
+
+            if status is SKIPPED_ITERATION:
+                should_abort = tracker.record_skip(ComponentError(f"Skipped iteration at {dt}"))
+                if should_abort:
+                    tracker.publish(self)
+                    raise ComponentError(
+                        f"DateList: Aborted due to {tracker.consecutive} consecutive failures at {dt}. Last error: {tracker.last_error}"
+                    ) from tracker.last_error
+            else:
+                tracker.record_success()
+
+        tracker.publish(self)
+        return tracker.successes > 0

flowtask-5.12.16/flowtask/components/DownloadFromGoogleDrive.py

@@ -0,0 +1,154 @@
+import asyncio
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+from ..exceptions import FileError, FileNotFound
+from ..interfaces.GoogleDrive import GoogleDriveClient
+from ..interfaces.download_from import DownloadFromBase
+from ..utils import SafeDict
+
+
+class DownloadFromGoogleDrive(GoogleDriveClient, DownloadFromBase):
+    """
+    DownloadFromGoogleDrive.
+
+    **Overview**
+
+        Downloads (or, for native Google Docs/Sheets/Slides/Drawings,
+        exports) files from Google Drive using a service account. Supports
+        lookup by Drive ``file_id``, exact ``filename``, or
+        ``pattern``/``extension`` search scoped to a ``directory`` (folder
+        path, resolved by segment — Drive has no literal paths). When no
+        ``filename``/``pattern`` is given, matches every file directly under
+        ``directory``; with ``recursive: true``, descends into subfolders
+        too.
+
+    **Properties** (inherited from ``DownloadFromBase`` and ``GoogleClient``)
+
+        | Name          | Required | Summary                                                            |
+        |---------------|----------|---------------------------------------------------------------------|
+        | credentials   | No       | Service account credentials override (path, JSON string, or dict). |
+        |               |          | Defaults to ``GOOGLE_CREDENTIALS_FILE`` when omitted.               |
+        | file          | Yes*     | Search spec dict (or list of dicts): ``file_id`` \\| ``filename`` \\| |
+        |               |          | ``pattern``/``extension`` + ``directory`` + ``recursive``.          |
+        | source        | Yes*     | Alternate spelling of ``file`` (destination-style tasks).          |
+        | destination   | Yes      | ``directory`` (required), optional ``filename``/``export_format``. |
+
+        \\* Exactly one of ``file`` or ``source`` must be provided.
+
+        Example:
+
+        ```yaml
+        DownloadFromGoogleDrive:
+          file:
+            file_id: "1t8W6lNVTH1Z0DDG0IHpCgAu7ALtHMJNQz8YpaVz2J1o"
+          destination:
+            directory: "/home/ubuntu/symbits/pharmabox/files/"
+            filename: "MasterProductList.xlsx"
+            export_format: "xlsx"
+        ```
+
+        ```yaml
+        DownloadFromGoogleDrive:
+          file:
+            pattern: "*MPL*"
+            extension: "xlsx"
+            directory: "Pharmabox/Reports"
+          destination:
+            directory: "/home/ubuntu/symbits/pharmabox/files/"
+        ```
+    """
+    _version = "1.0.0"
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.export_format: Optional[str] = None
+        self._srcfiles: List[Dict[str, Any]] = []
+        self._filenames: List[str] = []
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    def processing_credentials(self):
+        """No-op override.
+
+        ``GoogleClient.__init__`` already fully resolves credentials into
+        ``credentials_file``/``credentials_dict``/``credentials_str`` from
+        the ``credentials`` constructor kwarg. There is no per-field
+        env-var substitution step to run the way
+        ``CredentialsInterface.processing_credentials()`` does for
+        username/password-style dict credentials, so this is intentionally
+        a no-op — it exists only so ``DownloadFromBase.start()`` (which
+        calls ``self.processing_credentials()`` unconditionally) has
+        something to call, since ``GoogleClient`` doesn't provide one.
+        """
+        return None
+
+    async def start(self, **kwargs):
+        await super(DownloadFromGoogleDrive, self).start(**kwargs)
+        self._started = True
+
+        self._srcfiles = []
+        if hasattr(self, "file"):
+            spec_source = self.file
+        elif hasattr(self, "source"):
+            spec_source = self.source
+        else:
+            raise FileError(
+                "DownloadFromGoogleDrive: either 'file' or 'source' must be provided"
+            )
+
+        if isinstance(spec_source, list):
+            for entry in spec_source:
+                if isinstance(entry, dict):
+                    self._srcfiles.append(dict(entry))
+                else:
+                    self._srcfiles.append({"filename": entry})
+        elif isinstance(spec_source, dict):
+            self._srcfiles.append(dict(spec_source))
+        else:
+            raise FileError(
+                "DownloadFromGoogleDrive: 'file'/'source' must be a dict or a list of dicts"
+            )
+
+        self.export_format = None
+        self._filenames = []
+        if hasattr(self, "destination"):
+            _dir = self.destination.get("directory", ".")
+            _direc = _dir.format_map(SafeDict(**self._variables))
+            _dir = self.mask_replacement(_direc)
+            self.directory = Path(_dir)
+            self.export_format = self.destination.get("export_format")
+
+            filename = self.destination.get("filename")
+            if isinstance(filename, list):
+                self._filenames = [self.mask_replacement(f) for f in filename]
+            elif filename:
+                self._filenames = [self.mask_replacement(filename)]
+        else:
+            self.directory = Path(".")
+
+        return True
+
+    async def close(self):
+        pass
+
+    async def run(self):
+        # GoogleClient never initializes self.credentials in __init__, so
+        # get_service()'s lazy `if not self.credentials: self.connection()`
+        # check would raise AttributeError on first use. Connect eagerly
+        # here instead of relying on that lazy path.
+        self.connection()
+
+        found = await self.search_files(self._srcfiles)
+        if not found:
+            raise FileNotFound("No files found to download")
+
+        self._result = await self.download_found_files(
+            found, self.directory, self._filenames, self.export_format
+        )
+        self.add_metric("GOOGLEDRIVE_FILES", self._result)
+        return self._result

{flowtask-5.12.14 → flowtask-5.12.16}/flowtask/components/DownloadFromSFTP.py

@@ -7,7 +7,6 @@ from ..exceptions import FileError, ComponentError, FileNotFound
 from ..interfaces.download_from import DownloadFromBase
 from ..interfaces.SSHClient import SSHClient
 
-
 class DownloadFromSFTP(SSHClient, DownloadFromBase):
     """
     DownloadFromSFTP.
@@ -15,6 +14,9 @@ class DownloadFromSFTP(SSHClient, DownloadFromBase):
     **Overview**
 
         Download a file or directory from an SFTP server using the functionality from DownloadFrom.
+        When no `file`, `source` or `filename` is defined, the remote path(s) to download are read from
+        the previous component (`self.input`): a single path (e.g. each item of an iterating `FileList`,
+        also inside a `Group`) or a list of paths (e.g. `FileList` with `iterate: false`).
 
     **Properties** (inherited from DownloadFromBase and SSHClient)
 
@@ -79,6 +81,26 @@ class DownloadFromSFTP(SSHClient, DownloadFromBase):
           - yesterday
           - mask: '%Y-%m-%d'
         ```
+
+        ```yaml
+          - FileList:
+              sftp_config:
+                host: 10.0.22.233
+                username: jlara
+                client_keys: ~/.ssh/jesuslara.pem
+                directory: /nfs/symbits/epson/files/sales
+              pattern: 'SELLTHRU_TROC_*.TXT'
+              iterate: true
+              to_group: download_sales
+          - DownloadFromSFTP:
+              Group: download_sales
+              host: 10.0.22.233
+              credentials:
+                username: jlara
+                client_keys: ~/.ssh/jesuslara.pem
+                known_hosts: null
+              directory: /home/ubuntu/symbits/epson/files/sales/
+        ```
     """
     _version = "1.0.0"
 
@@ -107,8 +129,38 @@ class DownloadFromSFTP(SSHClient, DownloadFromBase):
         self.whole_dir: bool = False
         super().__init__(loop=loop, job=job, stat=stat, **kwargs)
 
+    def _files_from_input(self) -> list[str]:
+        """Remote paths received from the previous component (`self.input`).
+
+        Accepts a single path (e.g. the current item of a `FileList`
+        iteration) or a list of paths (e.g. `FileList` with `iterate: false`).
+
+        Returns:
+            The remote paths as strings, empty when the input carries no paths.
+        """
+        if not self.previous:
+            return []
+        data = self.input
+        if isinstance(data, (str, PurePath)):
+            data = [data]
+        if not isinstance(data, (list, tuple)):
+            return []
+        if not all(isinstance(item, (str, PurePath)) for item in data):
+            return []
+        return [str(item) for item in data]
+
     async def start(self, **kwargs):
         await super(DownloadFromSFTP, self).start(**kwargs)
+        if not self._srcfiles and not self.filename:
+            # no source of its own: download what the previous component sent
+            self._srcfiles = self._files_from_input()
+            if not self._srcfiles:
+                raise ComponentError(
+                    "DownloadFromSFTP: no file to download, define file/source/filename "
+                    "or chain it after a component that outputs remote paths (e.g. FileList)"
+                )
+            if len(self._srcfiles) == 1:
+                self.filename = self._srcfiles[0]
         if hasattr(self, "source"):
             self.whole_dir = (
                 self.source["whole_dir"] if "whole_dir" in self.source else False
@@ -129,10 +181,13 @@ class DownloadFromSFTP(SSHClient, DownloadFromBase):
             self.rename = Path("{}/{}".format(self.directory, self.rename))
         if self.rename is not None:
             self.local_name = self.rename
-        else:
+        elif self.filename:
             self.local_name = Path(
                 "{}/{}".format(self.directory, PurePath(self.filename).name)
             )
+        else:
+            # several files from the input: there is no single local name
+            self.local_name = Path(self.directory)
         return True
 
     def download_progress(self, srcpath, dstpath, bytes_copied, total_bytes):

{flowtask-5.12.14 → flowtask-5.12.16}/flowtask/components/DownloadFromSmartsheet.py

@@ -1,4 +1,4 @@
-"""
+r"""
 DownloadFromSmartsheet
 Download attachments from SmartSheet sheets or rows with advanced filtering.
 
@@ -26,7 +26,7 @@ from ..interfaces.smartsheet import SmartSheetClient
 
 
 class DownloadFromSmartsheet(SmartSheetClient, FlowComponent):
-    """
+    r"""
     DownloadFromSmartsheet
 
     Overview

{flowtask-5.12.14 → flowtask-5.12.16}/flowtask/components/ExecuteSQL.py

@@ -19,7 +19,6 @@ from ..utils import SafeDict
 # TODO: migrate to FileStore component
 from ..interfaces.flow import FlowComponent
 from ..interfaces import TemplateSupport
-from ..conf import TASK_PATH
 from ..interfaces.qs import QSSupport
 
 
@@ -148,7 +147,8 @@ class ExecuteSQL(QSSupport, TemplateSupport, FlowComponent):
                 )
             for fs in qs:
                 self._logger.debug(f"Execute SQL File: {fs!s}")
-                file_path = TASK_PATH.joinpath(self._program, "sql", fs)
+                # sql files live in the task storage the task was loaded from
+                file_path = self._taskstore.get_path().joinpath(self._program, "sql", fs)
                 try:
                     sql = await self.open_sqlfile(file_path)
                     self._queries.append(sql)
@@ -243,7 +243,9 @@ class ExecuteSQL(QSSupport, TemplateSupport, FlowComponent):
                     )
                 else:
                     if self._driver == 'bigquery':
-                        return next(iter(result))
+                        # first row of a SELECT; None for DML/scripts, which
+                        # return no rows (next() without default raised StopIteration)
+                        return next(iter(result), None)
                     return result
         except StatementError as err:
             raise StatementError(f"Statement error: {err}") from err