flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/SubTask.py

@@ -0,0 +1,182 @@
+import asyncio
+import logging
+from collections.abc import Callable
+import traceback
+import pprint
+from asyncdb.exceptions import NoDataFound
+from ..tasks.task import Task
+from ..utils.functions import check_empty
+from ..exceptions import (
+    DataNotFound,
+    NotSupported,
+    ComponentError,
+    TaskNotFound,
+    TaskDefinition,
+    TaskFailed,
+    FileNotFound,
+)
+from .flow import FlowComponent
+
+
+class SubTask(FlowComponent):
+    """
+    SubTask
+
+        Overview
+
+            The SubTask class is a component for executing a specified task as a sub-task within a workflow.
+            It allows passing configurations and parameters, including conditional steps, to dynamically manage 
+            and execute the named task in the specified program.
+
+        .. table:: Properties
+        :widths: auto
+
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | Name           | Required | Summary                                                                   |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | task           |   Yes    | The name of the task to execute as a sub-task.                            |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | program        |   Yes    | The name of the program under which the task is defined.                  |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | ignore_steps   |   No     | List of steps to ignore during the sub-task execution.                    |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | run_only       |   No     | List of specific steps to run, ignoring others.                           |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | conditions     |   No     | Dictionary of conditions to apply to the sub-task execution.              |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+
+        Returns
+
+            This component executes the specified task and returns the task’s output or state upon completion.
+            If the task encounters an error or lacks data, an appropriate exception is raised. The component tracks 
+            task and program details in the metrics, and logs the state of the task if debugging is enabled.
+    
+
+        Example:
+
+        ```yaml
+        SubTask:
+          task: forms
+          program: banco_chile
+        ```
+
+    """
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self._params = {}
+        self.task: str = ""
+        self._task = None
+        self.program: str = ""
+        self.conditions = {}
+        self._arguments = {}
+        self.ignore_steps: list = []
+        self.run_only: list = []
+        super(SubTask, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+        self.program = self._program
+
+    async def start(self, **kwargs):
+        program = self.program
+        self._program = program
+        try:
+            del self._params["program"]
+        except KeyError:
+            pass
+        try:
+            # task = self.task
+            del self._params["task"]
+        except KeyError:
+            pass
+        # remove the SkipError (avoid passed to SubTask.)
+        try:
+            del self._params["skipError"]
+        except KeyError:
+            pass
+        # passing variables to SubTask
+        params = {
+            "vars": self._vars,
+            "variables": self._variables,
+            "ignore_steps": self.ignore_steps,
+            "run_only": self.run_only,
+            "memory": self._memory,
+            "attributes": self._attributes,
+            "arguments": self._arguments,
+            "is_subtask": True,
+            "conditions": self.conditions,
+            "params": self._params,
+        }
+        if self._debug is True:
+            pp = pprint.PrettyPrinter(indent=4)
+            print(" Subtask Properties ==================")
+            pp.pprint(params)
+            print(" ==================")
+        if "debug" in self._params:
+            del self._params["debug"]
+        # definition of task
+        try:
+            self._task = Task(
+                task=self.task,
+                program=self.program,
+                loop=self._loop,
+                debug=self._debug,
+                stat=self.stat.parent(),
+                **params,
+            )
+            self.add_metric(
+                "TASK", f"{self.program}.{self.task}"
+            )
+            try:
+                if self._task:
+                    prepare = await self._task.start()
+                    if prepare:
+                        return True
+                    else:
+                        return False
+            except (TaskDefinition, TaskFailed) as err:
+                logging.exception(err, stack_info=True)
+        except TaskNotFound as err:
+            logging.exception(err, stack_info=True)
+        except Exception as err:
+            traceback.print_exc()
+            print(err, err.__class__)
+            logging.exception(err, stack_info=True)
+
+    async def close(self):
+        if self._task:
+            try:
+                await self._task.close()
+            except Exception as err:
+                raise TaskFailed(
+                    f"Error closing Task {self.task!s}, error: {err}"
+                ) from err
+
+    async def run(self):
+        result = False
+        if not self._task:
+            return False
+        try:
+            result = await self._task.run()
+        except (DataNotFound, NoDataFound, FileNotFound):
+            raise
+        except NotSupported as err:
+            raise NotSupported(
+                f"Error running Task {self.task}, error: {err}"
+            ) from err
+        except ComponentError:
+            raise
+        except Exception as err:
+            print("TYPE ", err, type(err))
+            raise ComponentError(
+                f"Error on Task {self.task}: {err}"
+            ) from err
+        if check_empty(result):
+            return False
+        else:
+            self._result = self
+            return result

flowtask/components/SuiteCRM.py

@@ -0,0 +1,48 @@
+from typing import Dict
+from navconfig.logging import logging
+from querysource.exceptions import DataNotFound as QSNotFound
+from ..exceptions import ComponentError, DataNotFound
+from .QSBase import QSBase
+
+
+class SuiteCRM(QSBase):
+    """SuiteCRM
+
+    Overview
+
+           This component captures the data from the SuiteCRM API to be
+           processed and stored in Navigator.
+
+    .. table:: Properties
+       :widths: auto
+
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | Name         | Required | Summary                                                           |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | type         |   Yes    |  Type of query                                                    |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | username     |   Yes    |  Credential Username                                              |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | password     |   Yes    |  Credential Password                                              |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | main_url     |   Yes    |  URL Base of the API                                              |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | module_name  |   Yes    |  Module to list                                                   |
+    +--------------+----------+-----------+-------------------------------------------------------+
+
+    Return the entry list of module
+
+    """
+
+    type = "entry_list"
+    _driver = "suitecrm"
+
+    async def entry_list(self):
+        try:
+            resultset = await self._qs.entry_list()
+            return resultset
+        except QSNotFound as err:
+            raise DataNotFound(f"CourseMerchant Not Found: {err}") from err
+        except Exception as err:
+            logging.exception(err)
+            raise ComponentError(f"CourseMerchant ERROR: {err!s}") from err

flowtask/components/Switch.py

@@ -0,0 +1,175 @@
+from collections.abc import Callable
+from jsonpath_ng import parse
+import asyncio
+import pandas as pd
+from ..exceptions import (
+    ComponentError,
+)
+from .BaseLoop import BaseLoop
+
+
+class Switch(BaseLoop):
+    """
+    Switch Component
+
+    Routes execution to a specific component based on user-defined conditions.
+
+    cases: Defines a list of conditions and their corresponding components.
+        Each condition is a Python-style logical expression evaluated against self.data.
+    default: Specifies the fallback component if none of the conditions match.
+
+    """
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self._cases = kwargs.get("cases", [])
+        super(Switch, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """
+        start.
+
+            Initialize (if needed) a task
+        """
+        await super(Switch, self).start(**kwargs)
+        # Add all case components to the tracker
+        self._define_tracking_components(*self._cases)
+        return True
+
+    def _evaluate_condition(self, condition: str) -> bool:
+        """
+        Evaluate a condition based on the type of self.data.
+
+        Args:
+            condition (str): A condition in the format "<jsonpath> <operator> <value>".
+
+        Returns:
+            bool: True if the condition is satisfied, else False.
+        """
+        if isinstance(self.data, dict):
+            # Parse the condition: split JSONPath and comparison
+            try:
+                # Split the condition into JSONPath and comparison
+                jsonpath_expr, operator, expected_value = self._parse_condition(condition)
+
+                # Evaluate the JSONPath
+                jsonpath = parse(jsonpath_expr)
+                matches = jsonpath.find(self.data)
+
+                # Get the first match (if any)
+                actual_value = matches[0].value if matches else None
+
+                # Evaluate the operator
+                return self._evaluate_operator(actual_value, operator, expected_value)
+
+            except Exception as err:
+                raise ComponentError(f"Error evaluating condition '{condition}': {err}")
+
+        elif isinstance(self.data, pd.DataFrame):
+            # For DataFrame, condition should be a valid query string
+            try:
+                filtered = self.data.query(condition)
+                return not filtered.empty
+            except Exception as err:
+                raise ComponentError(
+                    f"Error evaluating DataFrame condition '{condition}': {err}"
+                ) from err
+
+        elif isinstance(self.data, str):
+            # For string data, check if condition is a substring
+            return condition in self.data
+
+        else:
+            raise ComponentError(
+                f"Unsupported data type for condition evaluation: {type(self.data)}"
+            )
+
+    def _parse_condition(self, condition: str):
+        """
+        Parse the condition into JSONPath, operator, and expected value.
+
+        Args:
+            condition (str): A condition string, e.g., "$.metadata.type == 'recapDefinition'".
+
+        Returns:
+            tuple: (jsonpath_expr, operator, expected_value)
+        """
+        operators = ["==", "!=", ">", "<", ">=", "<="]
+        for op in operators:
+            if op in condition:
+                parts = condition.split(op)
+                if len(parts) != 2:
+                    raise ValueError(f"Invalid condition format: {condition}")
+                jsonpath_expr, expected_value = parts
+                expected_value = expected_value.strip().strip("'\"")  # Remove quotes
+                return jsonpath_expr.strip(), op, expected_value
+        raise ValueError(f"No valid operator found in condition: {condition}")
+
+    def _evaluate_operator(self, actual_value, operator, expected_value):
+        """
+        Evaluate the comparison operator.
+
+        Args:
+            actual_value: The value extracted from JSONPath.
+            operator: The comparison operator as a string.
+            expected_value: The value to compare against.
+
+        Returns:
+            bool: True if the comparison is satisfied, else False.
+        """
+        if operator == "==":
+            return actual_value == expected_value
+        elif operator == "!=":
+            return actual_value != expected_value
+        elif operator == ">":
+            return actual_value > expected_value
+        elif operator == "<":
+            return actual_value < expected_value
+        elif operator == ">=":
+            return actual_value >= expected_value
+        elif operator == "<=":
+            return actual_value <= expected_value
+        else:
+            raise ValueError(f"Unsupported operator: {operator}")
+
+    async def run(self):
+        """
+        Executes the appropriate component based on the conditions.
+        """
+        # Determine the component to execute
+        selected_component = None
+        for case in self._cases:
+            if self._evaluate_condition(case["condition"]):
+                selected_component = case["component"]
+                break
+
+        if not selected_component:
+            # Use the default component if no case matches
+            if not self._default:
+                raise ComponentError(
+                    "No matching case and no default component provided."
+                )
+            selected_component = self._default
+
+        # Execute the selected component
+        step, idx = self._TaskPile.getStepByName(selected_component)
+        if step is None:
+            raise ComponentError(
+                f"Component '{selected_component}' not found in Task Definition."
+            )
+        target, params, stat = self.get_component(step)
+        component_instance = self.create_component(
+            target,
+            value=self.data,
+            stat=stat,
+            **params
+        )
+        self._result = await self.exec_component(component_instance, step.name)
+        # at the end, remove all tracked components from TaskPile to prevent future execution
+        for component_name in self._tracked_components:
+            self._TaskPile.delStepByName(component_name)
+        return self._result

flowtask/components/TableBase.py

@@ -0,0 +1,148 @@
+import asyncio
+from collections.abc import Callable
+from pathlib import PurePath
+import aiofiles
+from sqlalchemy.orm import Session
+from sqlalchemy import create_engine, text
+from sqlalchemy.pool import NullPool
+from querysource.conf import sqlalchemy_url
+from ..conf import TASK_PATH
+from ..exceptions import FileNotFound, FileError
+from ..parsers.maps import open_map, open_model
+from .flow import FlowComponent
+from ..interfaces import TemplateSupport
+
+
+class TableBase(FlowComponent, TemplateSupport):
+    """
+    TableBase.
+
+    Abstract class for Using Pandas SQL features to manipulate data from databases.
+    """
+
+    flavor: str = "postgres"
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self._engine: Callable = None
+        self.params: dict = {}
+        self.use_template: bool = bool(kwargs.get('use_template', False))
+        super(TableBase, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def get_connection(self, dsn: str = None):
+        if not dsn:
+            dsn = sqlalchemy_url
+        self._engine = create_engine(dsn, echo=False, poolclass=NullPool)
+        self._session = Session(self._engine)
+
+    async def close(self):
+        """Closing Operations."""
+        if self._engine:
+            try:
+                self._engine.dispose()
+            except Exception as err:  # pylint: disable=W0703
+                print(err)
+
+    async def open_sqlfile(self, file: PurePath, **kwargs) -> str:
+        if file.exists() and file.is_file():
+            content = None
+            # open SQL File:
+            async with aiofiles.open(file, "r+") as afp:
+                content = await afp.read()
+                # check if we need to replace masks
+            content = self.mask_replacement(content)
+            if self.use_template is True:
+                content = self._templateparser.from_string(content, kwargs)
+            return content
+        else:
+            raise FileError(f"Table: Missing SQL File: {file}")
+
+    def column_info(self, tablename):
+        if self.flavor == "postgres":
+            discover = f"""SELECT attname AS column_name, atttypid::regtype AS data_type, attnotnull::boolean as notnull
+                    FROM pg_attribute WHERE attrelid = '{tablename}'::regclass AND attnum > 0
+                    AND NOT attisdropped ORDER  BY attnum;
+                    """
+        else:
+            raise ValueError(
+                f"Column Info: DB Flavor not supported yet: {self.flavor}"
+            )
+        discover = text(discover)
+        result = self._session.execute(discover)
+        if result:
+            # rows = result.fetchall()
+            rows = result.mappings()
+            # return {item['column_name']: item['data_type'] for item in rows}
+            return {row["column_name"]: row["data_type"] for row in rows}
+        else:
+            model = open_model(self.tablename, self.program)
+            if model:
+                fields = model["fields"]
+                return {field: fields[field]["data_type"] for field in fields}
+            else:
+                print(f"Table: table or Model {tablename} does not exists")
+                return None
+
+    async def start(self, **kwargs):
+        if self.previous:
+            self.data = self.input
+        await self.get_connection()
+        if hasattr(self, "tablename"):
+            # Using a Table:
+            if not hasattr(self, "schema"):
+                self.schema = self._program
+            # check for Table structure
+            try:
+                tablename = self.tablename
+                schema = self.schema
+                if self.flavor == "postgres":
+                    tablename = f"{schema}.{tablename}"
+                colinfo = self.column_info(tablename)
+            except KeyError:
+                if hasattr(self, "map"):
+                    mapping = self.map["map"]
+                    colinfo = open_map(mapping, self.program)
+                if not colinfo:
+                    colinfo = open_map(tablename, self.program)
+            if colinfo:
+                try:
+                    ignore = self.map["ignore"]
+                    colinfo = {k: v for k, v in colinfo.items() if k not in ignore}
+                except (KeyError, AttributeError):
+                    pass
+                self.params["columns"] = colinfo.keys()
+                parse_dates = []
+                for column, dtype in colinfo.items():
+                    if (
+                        dtype == "timestamp without time zone"
+                        or dtype == "timestamp with time zone"
+                        or dtype == "date"
+                    ):
+                        parse_dates.append(column)
+                if parse_dates:
+                    self.params["parse_dates"] = parse_dates
+        elif hasattr(self, "file_sql"):
+            file_path = TASK_PATH.joinpath(self.program, "sql", self.file_sql)
+            if not file_path.exists():
+                raise FileNotFound(f"Table: SQL File {file_path} was not found.")
+            self.query = await self.open_sqlfile(file_path)
+        elif hasattr(self, "query"):
+            ## passing query for mask conversion:
+            if hasattr(self, "masks"):
+                self.query = self.mask_replacement(self.query)
+            for val in self._variables:
+                if isinstance(self._variables[val], list):
+                    if isinstance(self._variables[val], int):
+                        self._variables[val] = ", ".join(self._variables[val])
+                    else:
+                        self._variables[val] = ", ".join(
+                            "'{}'".format(v) for v in self._variables[val]
+                        )
+                self.query = self.query.replace(
+                    "{{{}}}".format(str(val)), str(self._variables[val])
+                )