flowtask 5.2.12__tar.gz → 5.3.3__tar.gz

{flowtask-5.2.12 → flowtask-5.3.3}/Makefile

@@ -1,5 +1,5 @@
 venv:
-	python3.10 -m venv .venv
+	python3.11 -m venv .venv
 	echo 'run `source .venv/bin/activate` to start develop FlowTask'
 
 install:
@@ -11,14 +11,15 @@ install:
 	pip install --upgrade navigator-auth
 	pip install --upgrade async-notify[all]
 	pip install --upgrade querysource
-	# dev
-	python -m pip install -Ur docs/requirements-dev.txt
 	# Install extra resources
 	pip install --upgrade qworker
 	# install FlowTask
 	pip install -e .
 	echo 'start develop FlowTask'
 
+develop:
+	pip install -Ur docs/requirements-dev.txt
+
 compile:
 	python setup.py build_ext --inplace
 

{flowtask-5.2.12 → flowtask-5.3.3}/PKG-INFO

@@ -1,27 +1,30 @@
 Metadata-Version: 2.1
 Name: flowtask
-Version: 5.2.12
+Version: 5.3.3
 Summary: Framework for running Tasks and from CLI and API for orchestation. Component-based Task builder/Runner for non-programmers.
 Home-page: https://github.com/phenobarbital/flowtask
 Author: Jesus Lara
-Author-email: jlara@trocglobal.com
-License: Apache License 2.0
+Author-email: "Jesus Lara G." <jesuslarag@gmail.com>
+License: Apache-2.0
 Project-URL: Source, https://github.com/phenobarbital/flowtask
 Project-URL: Funding, https://paypal.me/phenobarbital
 Project-URL: Say Thanks!, https://saythanks.io/to/phenobarbital
-Keywords: DataIntegration DataIntegration Task Orchestation Framework Task-Runner,Pipelines
+Keywords: DataIntegration,Task,Orchestation,Task-Runner,Pipelines,Data-Pipelines
 Platform: *nix
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: System Administrators
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python
+Classifier: Typing :: Typed
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Web Environment
+Classifier: Framework :: AsyncIO
 Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Software Development :: Build Tools
-Classifier: Topic :: Utilities
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
 Classifier: License :: OSI Approved :: BSD License
-Classifier: Framework :: AsyncIO
 Requires-Python: >=3.9.16
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -112,5 +115,3 @@ Please have a look at the Contribution Guide
 ### License ###
 
 Navigator is licensed under Apache 2.0 License. See the LICENSE file for more details.
-
-

{flowtask-5.2.12 → flowtask-5.3.3}/flowtask/__init__.py

@@ -5,9 +5,14 @@ Navigator Data-Integration.
 Tool for execution of Tasks.
 """
 from .version import (
-    __title__, __description__, __version__, __author__, __author_email__
+    __title__,
+    __description__,
+    __version__,
+    __author__,
+    __author_email__,
 )
 
+
 def version():
     """version.
     Returns:

{flowtask-5.2.12 → flowtask-5.3.3}/flowtask/__main__.py

@@ -24,16 +24,15 @@ def main():
     loop = asyncio.get_event_loop()
     loop.slow_callback_duration = 0.5  # Set threshold to 0.5 seconds
     # try:
-    result = loop.run_until_complete(
-        task(loop)
-    )
+    result = loop.run_until_complete(task(loop))
     if result:
-        cPrint(' === RESULT === ', level="DEBUG")
+        cPrint(" === RESULT === ", level="DEBUG")
         print(result.result)
-        cPrint('== Task stats === ', level="INFO")
+        cPrint("== Task stats === ", level="INFO")
         print(result.stats)
     # finally:
     #     loop.close()
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     main()

{flowtask-5.2.12 → flowtask-5.3.3}/flowtask/components/AddDataset.py

@@ -2,15 +2,9 @@ import asyncio
 from collections.abc import Callable
 import numpy as np
 import pandas as pd
-from asyncdb.exceptions import (
-    NoDataFound
-)
-from flowtask.exceptions import (
-    ComponentError,
-    DataNotFound,
-    TaskError
-)
-from flowtask.utils import cPrint
+from asyncdb.exceptions import NoDataFound
+from ..exceptions import ComponentError, DataNotFound, TaskError
+from ..utils import cPrint
 from .abstract import DtComponent
 
 
@@ -19,25 +13,24 @@ class AddDataset(DtComponent):
 
     Join two Datasets based on some criteria.
     """
+
     df1 = None
     df2 = None
 
     def __init__(
-            self,
-            loop: asyncio.AbstractEventLoop = None,
-            job: Callable = None,
-            stat: Callable = None,
-            **kwargs
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
     ) -> None:
         """Init Method."""
-        self.type: str = 'left'
+        self.type: str = "left"
         self._dtypes: dict = {}
         self.infer_types: bool = False
         self.to_string: bool = True
         self.as_objects: bool = False
-        super(AddDataset, self).__init__(
-            loop=loop, job=job, stat=stat, **kwargs
-        )
+        super(AddDataset, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
 
     async def start(self, **kwargs):
         """Obtain Pandas Dataframe."""
@@ -49,22 +42,18 @@ class AddDataset(DtComponent):
             else:
                 self.df1 = self.previous.output()
         except IndexError as ex:
-            raise ComponentError(
-                "Missing LEFT Dataframe"
-            ) from ex
+            raise ComponentError("Missing LEFT Dataframe") from ex
         ### check info for creating the second dataset:
         self.df2 = None
-        if not hasattr(self, 'fields'):
-            raise TaskError(
-                "Wrong Task configuration: need *fields* declaration."
-            )
-        if not hasattr(self, 'dataset'):
+        if not hasattr(self, "fields"):
+            raise TaskError("Wrong Task configuration: need *fields* declaration.")
+        if not hasattr(self, "dataset"):
             raise TaskError(
                 "Wrong Task configuration: need *dataset* name declaration."
             )
-        if not hasattr(self, 'datasource'):
-            self.datasource = 'datasets'
-        if self.datasource in ('datasets', 'vision'):
+        if not hasattr(self, "datasource"):
+            self.datasource = "datasets"
+        if self.datasource in ("datasets", "vision"):
             # using current datasets on pg database
             self.connection = self.pg_connection()
         else:
@@ -75,25 +64,21 @@ class AddDataset(DtComponent):
     async def run(self):
         args = {}
         if self.df1.empty:
-            raise DataNotFound(
-                "Data Was Not Found on Dataframe 1"
-            )
+            raise DataNotFound("Data Was Not Found on Dataframe 1")
         ## getting second dataset:
         try:
             ### TODO: instrumentation for getting dataset from different sources
             async with await self.connection.connection() as conn:
-                fields = ', '.join(self.fields)
-                if hasattr(self, 'distinct'):
-                    join = ', '.join(self.join)
+                fields = ", ".join(self.fields)
+                if hasattr(self, "distinct"):
+                    join = ", ".join(self.join)
                     query = f"SELECT DISTINCT ON ({join}) {fields} FROM {self.datasource}.{self.dataset}"
                 else:
                     query = f"SELECT {fields} FROM {self.datasource}.{self.dataset}"
-                self._logger.info(f'DATASET QUERY: {query}')
+                self._logger.info(f"DATASET QUERY: {query}")
                 result, error = await conn.query(query)
                 if error or not result:
-                    raise DataNotFound(
-                        "Empty Dataset"
-                    )
+                    raise DataNotFound("Empty Dataset")
                 ## converting on Dataframe:
                 self.df2 = await self.get_dataframe(result, infer_types=True)
         except (DataNotFound, NoDataFound) as exc:
@@ -101,44 +86,42 @@ class AddDataset(DtComponent):
             raise DataNotFound(str(exc)) from exc
         finally:
             self.connection = None
-        if self.type == 'left' and (self.df2 is None or self.df2.empty):
+        if self.type == "left" and (self.df2 is None or self.df2.empty):
             self._result = self.df1
             return True
         elif self.df2 is None or self.df2.empty:
-            raise DataNotFound(
-                "Data Was Not Found on Dataframe 2"
-            )
-        if hasattr(self, 'no_copy'):
-            args['copy'] = self.no_copy
+            raise DataNotFound("Data Was Not Found on Dataframe 2")
+        if hasattr(self, "no_copy"):
+            args["copy"] = self.no_copy
         if not self.type:
-            self.type = 'left'
-        elif self.type == 'inner':
-            args['left_index'] = True
-        if hasattr(self, 'args') and isinstance(self.args, dict):
+            self.type = "left"
+        elif self.type == "inner":
+            args["left_index"] = True
+        if hasattr(self, "args") and isinstance(self.args, dict):
             args = {**args, **self.args}
-        if hasattr(self, 'operator'):
+        if hasattr(self, "operator"):
             operator = self.operator
         else:
-            operator = 'and'
-            if hasattr(self, 'join'):
-                args['on'] = self.join
+            operator = "and"
+            if hasattr(self, "join"):
+                args["on"] = self.join
             else:
-                args['left_index'] = True
+                args["left_index"] = True
         # making a Join between 2 dataframes
         try:
-            if operator == 'and':
+            if operator == "and":
                 df = pd.merge(
                     self.df1,
                     self.df2,
                     how=self.type,
-                    suffixes=('_left', '_right'),
-                    **args
+                    suffixes=("_left", "_right"),
+                    **args,
                 )
             else:
-                if hasattr(self, 'join'):
-                    args['left_on'] = self.join
+                if hasattr(self, "join"):
+                    args["left_on"] = self.join
                 else:
-                    args['left_index'] = True
+                    args["left_index"] = True
                 ndf = self.df1
                 sdf = self.df2.copy()
                 merge = []
@@ -148,15 +131,21 @@ class AddDataset(DtComponent):
                         sdf,
                         right_on=key,
                         how=self.type,
-                        suffixes=('_left', None),
-                        **args
+                        suffixes=("_left", None),
+                        **args,
                     )
                     ndf = d[d[key].isnull()]
-                    ndf.drop(ndf.columns[ndf.columns.str.contains(
-                        '_left')], axis=1, inplace=True)
+                    ndf.drop(
+                        ndf.columns[ndf.columns.str.contains("_left")],
+                        axis=1,
+                        inplace=True,
+                    )
                     ddf = d[d[key].notnull()]
-                    ddf.drop(ddf.columns[ddf.columns.str.contains(
-                        '_left')], axis=1, inplace=True)
+                    ddf.drop(
+                        ddf.columns[ddf.columns.str.contains("_left")],
+                        axis=1,
+                        inplace=True,
+                    )
                     merge.append(ddf)
                 # merge the last (not matched) rows
                 merge.append(ndf)
@@ -164,25 +153,19 @@ class AddDataset(DtComponent):
                 df.reset_index(drop=True)
                 df.is_copy = None
         except (ValueError, KeyError) as err:
-            raise ComponentError(
-                f'Cannot Join with missing Column: {err!s}'
-            ) from err
+            raise ComponentError(f"Cannot Join with missing Column: {err!s}") from err
         except Exception as err:
-            raise ComponentError(
-                f"Unknown JOIN error {err!s}"
-            ) from err
+            raise ComponentError(f"Unknown JOIN error {err!s}") from err
         numrows = len(df.index)
         if numrows == 0:
-            raise DataNotFound(
-                "Cannot make any JOIN, returns zero coincidences"
-            )
-        self._variables[f'{self.TaskName}_NUMROWS'] = numrows
-        print('ON END> ', numrows)
-        self.add_metric('JOINED_ROWS', numrows)
+            raise DataNotFound("Cannot make any JOIN, returns zero coincidences")
+        self._variables[f"{self.TaskName}_NUMROWS"] = numrows
+        print("ON END> ", numrows)
+        self.add_metric("JOINED_ROWS", numrows)
         if self._debug is True:
-            print('::: Printing Column Information === ')
+            print("::: Printing Column Information === ")
             for column, t in df.dtypes.items():
-                print(column, '->', t, '->', df[column].iloc[0])
+                print(column, "->", t, "->", df[column].iloc[0])
         # helping some transformations
         df.is_copy = None
         self._result = df
@@ -198,63 +181,58 @@ class AddDataset(DtComponent):
         result = [dict(row) for row in result]
         try:
             if self.as_objects is True:
-                df = pd.DataFrame(
-                    result,
-                    dtype=object
-                )
+                df = pd.DataFrame(result, dtype=object)
             else:
-                df = pd.DataFrame(
-                    result,
-                    **self._dtypes
-                )
+                df = pd.DataFrame(result, **self._dtypes)
         except Exception as err:  # pylint: disable=W0718
             self._logger.exception(err, stack_info=True)
         # Attempt to infer better dtypes for object columns.
         if hasattr(self, "infer_types") or infer_types is True:
             df.infer_objects()
-            df = df.convert_dtypes(
-                convert_string=self.to_string
-            )
+            df = df.convert_dtypes(convert_string=self.to_string)
         if self._debug is True:
-            cPrint('Data Types:')
+            cPrint("Data Types:")
             print(df.dtypes)
         if hasattr(self, "drop_empty"):
-            df.dropna(axis=1, how='all', inplace=True)
-            df.dropna(axis=0, how='all', inplace=True)
-        if hasattr(self, 'dropna'):
-            df.dropna(subset=self.dropna, how='all', inplace=True)
-        if hasattr(self, 'clean_strings') and getattr(self, 'clean_strings', False) is True:
-            u = df.select_dtypes(include=['object', 'string'])
-            df[u.columns] = u.fillna('')
+            df.dropna(axis=1, how="all", inplace=True)
+            df.dropna(axis=0, how="all", inplace=True)
+        if hasattr(self, "dropna"):
+            df.dropna(subset=self.dropna, how="all", inplace=True)
+        if (
+            hasattr(self, "clean_strings")
+            and getattr(self, "clean_strings", False) is True
+        ):
+            u = df.select_dtypes(include=["object", "string"])
+            df[u.columns] = u.fillna("")
         return df
 
     def set_datatypes(self):
-        if hasattr(self, 'datatypes'):
+        if hasattr(self, "datatypes"):
             dtypes = {}
             for field, dtype in self.datatypes.items():
-                if dtype == 'uint8':
+                if dtype == "uint8":
                     dtypes[field] = np.uint8
-                elif dtype == 'uint16':
+                elif dtype == "uint16":
                     dtypes[field] = np.uint16
-                elif dtype == 'uint32':
+                elif dtype == "uint32":
                     dtypes[field] = np.uint32
-                elif dtype == 'int8':
+                elif dtype == "int8":
                     dtypes[field] = np.int8
-                elif dtype == 'int16':
+                elif dtype == "int16":
                     dtypes[field] = np.int16
-                elif dtype == 'int32':
+                elif dtype == "int32":
                     dtypes[field] = np.int32
-                elif dtype == 'float':
+                elif dtype == "float":
                     dtypes[field] = float
-                elif dtype == 'float32':
+                elif dtype == "float32":
                     dtypes[field] = float
-                elif dtype in ('string', 'varchar', 'str'):
+                elif dtype in ("string", "varchar", "str"):
                     dtypes[field] = str
                 else:
                     # invalid datatype
                     self._logger.warning(
-                        f'Invalid DataType value: {field} for field {dtype}'
+                        f"Invalid DataType value: {field} for field {dtype}"
                     )
                     continue
             if dtypes:
-                self._dtypes['dtype'] = dtypes
+                self._dtypes["dtype"] = dtypes

flowtask-5.3.3/flowtask/components/AutoTask.py

@@ -0,0 +1,226 @@
+import math
+import re
+from urllib.parse import urljoin
+import numpy as np
+import orjson
+import pandas as pd
+from flowtask.components.abstract import DtComponent
+from flowtask.components.interfaces.http import HTTPService
+from flowtask.exceptions import ComponentError
+
+
+def to_snake_case(s):
+    """Converts a string to snake_case format.
+
+    Args:
+        name: The string to convert.
+
+    Returns:
+        The converted string in snake_case format.
+    """
+    # Remove unwanted characters
+    s = re.sub(r"[^a-zA-Z0-9_\s]", "", s)
+
+    # Insert underscores before uppercase letters
+    s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", s)
+
+    # Replace remaining uppercase letters with underscores
+    return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower()
+
+
+class AutoTask(DtComponent, HTTPService):
+    _credentials: dict = {"API_INTEGRATION_CODE": str, "USERNAME": str, "SECRET": str}
+
+    async def start(self, **kwargs):
+        self.headers = None
+        self._proxies = None
+        self.auth = ""
+        self.auth_type = ""
+        self.download = None
+        self.timeout = 180
+        self.accept = "application/json"
+        self.query_json = self.mask_replacement_recursively(self.query_json)
+        self.processing_credentials()
+
+        self._base_url = (
+            f"https://{self.zone}.autotask.net/atservicesrest/v1.0/{self.entity}/"
+        )
+
+        self.headers = {
+            "Content-Type": "application/json",
+            "ApiIntegrationCode": self.credentials["API_INTEGRATION_CODE"],
+            "UserName": self.credentials["USERNAME"],
+            "Secret": self.credentials["SECRET"],
+        }
+
+        self.ids_chunks = []
+        if self.previous:
+            self.data = self.input
+
+            self.ids_chunks = self.filter_ids(
+                id_field=self.id_column_name,
+                items=self.data,
+                chunk_size=500,
+            )
+        elif getattr(self, "ids", None):
+            self._logger.info("Dropping specified Filters. Using ids instead.")
+            self.ids_chunks = [self.ids]
+
+        return True
+
+    async def run(self):
+        if not self.ids_chunks:
+            # Use the Filter specified in the task
+            df_items = await self.get_dataframe_from_entity(
+                payload=orjson.dumps(self.query_json),
+                id_column_name=self.id_column_name,
+            )
+        else:
+            # Use the ids from the previous component or from the ids argument
+            df_items = pd.DataFrame()
+            for ids_chunk in self.ids_chunks:
+                self.query_json.update(
+                    {
+                        "Filter": [
+                            {"op": "in", "field": "id", "value": ids_chunk.tolist()}
+                        ]
+                    }
+                )
+
+                items = await self.get_dataframe_from_entity(
+                    payload=orjson.dumps(self.query_json),
+                    id_column_name=self.id_column_name,
+                )
+
+                df_items = pd.concat([df_items, items], ignore_index=True)
+
+        if not df_items.empty:
+            for field in self.picklist_fields:
+                df_picklist_values = await self.get_picklist_values(field)
+
+                snake_case_field_name = to_snake_case(field)
+
+                df_items = df_items.astype({snake_case_field_name: "str"})
+
+                df_items = df_items.merge(
+                    df_picklist_values,
+                    how="left",
+                    on=snake_case_field_name,
+                )
+
+        self._result = df_items
+        return self._result
+
+    async def close(self):
+        pass
+
+    def processing_credentials(self):
+        if self.credentials:
+            for value, dtype in self._credentials.items():
+                try:
+                    if type(self.credentials[value]) == dtype:
+                        default = getattr(self, value, self.credentials[value])
+                        val = self.get_env_value(
+                            self.credentials[value], default=default
+                        )
+                        self.credentials[value] = val
+                except (TypeError, KeyError) as ex:
+                    self._logger.error(f"{__name__}: Wrong or missing Credentials")
+                    raise ComponentError(
+                        f"{__name__}: Wrong or missing Credentials"
+                    ) from ex
+
+    def filter_ids(self, id_field: str, items: pd.DataFrame, chunk_size):
+        data = items[id_field].dropna().unique().astype(int)
+
+        if data.size > 0:
+            split_n = math.ceil(data.size / chunk_size)
+
+            # Split into chunks of n items
+            return np.array_split(data, split_n)  # Convert to NumPy array and split
+
+        return [data]
+
+    def get_autotask_url(self, resource):
+        return urljoin(self._base_url, resource)
+
+    async def get_dataframe_from_entity(self, payload, id_column_name):
+        args = {
+            "url": self.get_autotask_url("query"),
+            "method": "post",
+            "data": payload,
+        }
+
+        results = []
+        while True:
+            result, error = await self.session(**args)
+
+            if error:
+                self._logger.error(f"{__name__}: Error getting {self.entity}")
+                raise ComponentError(f"{__name__}: Error getting {self.entity}") from error
+
+            results.extend(result.get("items", []))
+
+            args.update({"url": result["pageDetails"].get("nextPageUrl", None)})
+
+            if not args["url"]:
+                break
+
+        df_results = await self.create_dataframe(results)
+
+        if not df_results.empty:
+            for field in self.user_defined_fields:
+                df_results[field] = df_results["userDefinedFields"].apply(
+                    self.get_udf_value, args=[field]
+                )
+
+        df_results = (
+            df_results.drop("userDefinedFields", axis=1, errors="ignore")
+            .fillna(value=self.fillna_values)
+            .astype(self.map_field_type)
+            .rename(columns=lambda x: to_snake_case(x))
+            .rename(columns={"id": id_column_name})
+        )
+
+        return df_results
+
+    @staticmethod
+    def get_udf_value(udfs, field_name):
+        """
+        Extracts the value of a specific user-defined field from a item dictionary.
+
+        Args:
+            item (dict): A dictionary representing a item.
+            field_name (str): The name of the user-defined field to extract.
+
+        Returns:
+            str: The value of the specified user-defined field, or None if not found.
+        """
+        for udf in udfs:
+            if udf["name"] == field_name:
+                return udf["value"]
+
+        return None
+
+    async def get_picklist_values(self, field_name: str) -> pd.DataFrame:
+        result, error = await self.session(
+            url=self.get_autotask_url("entityInformation/fields"),
+            method="get",
+        )
+
+        if error:
+            self._logger.error(f"{__name__}: Error getting {self.entity}")
+            raise ComponentError(f"{__name__}: Error getting {self.entity}") from error
+
+        for field in result["fields"]:
+            if field["name"] == field_name:
+                df = await self.create_dataframe(field["picklistValues"])
+                df = df[["label", "value"]]
+                snake_case_field_name = to_snake_case(field_name)
+
+                return df.rename(
+                    columns={
+                        "label": f"{snake_case_field_name}_label",
+                        "value": snake_case_field_name,
+                    }
+                )