flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/AutoTask.py

@@ -0,0 +1,314 @@
+import math
+from urllib.parse import urljoin
+import numpy as np
+import orjson
+import pandas as pd
+from .flow import FlowComponent
+from ..interfaces.http import HTTPService
+from ..exceptions import ComponentError, DataNotFound
+from ..utils.transformations import to_snake_case
+
+
+class AutoTask(HTTPService, FlowComponent):
+    """
+        AutoTask Component
+
+            Overview
+
+            This component retrieves data from AutoTask using the Autotask REST API.
+            It supports filtering data based on a query or specific IDs, handles pagination, and converts picklist values to human-readable labels.
+
+            .. table:: Properties
+            :widths: auto
+
+        +---------------------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------------------------------+
+        | Name                            | Required | Summary                                                                                                                                           |
+        +---------------------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------------------------------+
+        |  credentials                    |   Yes    | Dictionary containing API credentials: "API_INTEGRATION_CODE", "USERNAME", and "SECRET". Credentials can be retrieved from environment variables. |
+        +---------------------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------------------------------+
+        |  zone                           |   Yes    | AutoTask zone (e.g., "na").                                                                                                                       |
+        +---------------------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------------------------------+
+        |  entity                         |   Yes    | AutoTask entity to query (e.g., "tickets").                                                                                                       |
+        +---------------------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------------------------------+
+        |  query_json                     |   No     | JSON object representing the AutoTask query filter (defaults to retrieving all items). Refer to AutoTask documentation for query syntax.          |
+        +---------------------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------------------------------+
+        |  id_column_name                 |   No     | Name of the column in the output DataFrame that should represent the AutoTask record ID (defaults to "id").                                       |
+        +---------------------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------------------------------+
+        |  ids                            |   No     | List of AutoTask record IDs to retrieve (overrides query_json filter if provided).                                                                |
+        +---------------------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------------------------------+
+        |  picklist_fields                |   No     | List of picklist fields in the entity to be converted to human-readable labels.                                                                   |
+        +---------------------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------------------------------+
+        |  user_defined_fields            |   No     | List of user-defined fields in the entity to extract (requires "userDefinedFields" field in the response).                                        |
+        +---------------------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------------------------------+
+        |  fillna_values                  |   No     | Default value to replace missing data (defaults to None).                                                                                         |
+        +---------------------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------------------------------+
+        |  map_field_type                 |   No     | Dictionary mapping field names to desired data types in the output DataFrame.                                                                     |
+        +---------------------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------------------------------+
+        | force_create_columns            |   No     | If True, ensures "IncludeFields" columns always exist, even if empty. Must be used with "IncludeFields" in "query_json".                          |
+        +---------------------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------------------------------+
+
+            Returns a pandas DataFrame containing the retrieved AutoTask data and additional columns for picklist labels (if applicable).
+
+    
+
+        Example:
+
+        ```yaml
+        AutoTask:
+          skipError: skip
+          credentials:
+            API_INTEGRATION_CODE: AUTOTASK_API_INTEGRATION_CODE
+            USERNAME: AUTOTASK_USERNAME
+            SECRET: AUTOTASK_SECRET
+          entity: TicketNotes
+          zone: webservices14
+          id_column_name: ticket_note_id
+          picklist_fields:
+          - noteType
+          - publish
+          ids: []
+          query_json:
+            IncludeFields:
+            - id
+            - createDateTime
+            - createdByContactID
+            - creatorResourceID
+            - description
+            - impersonatorCreatorResourceID
+            - impersonatorUpdaterResourceID
+            - lastActivityDate
+            - noteType
+            - publish
+            - ticketID
+            - title
+            Filter:
+            - op: gte
+              field: lastActivityDate
+              value: '{two_days_ago}'
+          masks:
+            '{two_days_ago}':
+            - date_diff
+            - value: current_date
+              diff: 48
+              mode: hours
+              mask: '%Y-%m-%d %H:%M:%S'
+        ```
+
+    """  # noqa
+    _credentials: dict = {"API_INTEGRATION_CODE": str, "USERNAME": str, "SECRET": str}
+    force_create_columns: bool = False
+
+    async def start(self, **kwargs):
+        self.headers = None
+        self._proxies = None
+        self.auth = ""
+        self.auth_type = ""
+        self.timeout = 180
+        self.accept = "application/json"
+        self.query_json = self.mask_replacement_recursively(self.query_json)
+        self.processing_credentials()
+
+        self._base_url = (
+            f"https://{self.zone}.autotask.net/atservicesrest/v1.0/{self.entity}/"
+        )
+
+        self.headers = {
+            "Content-Type": "application/json",
+            "ApiIntegrationCode": self.credentials["API_INTEGRATION_CODE"],
+            "UserName": self.credentials["USERNAME"],
+            "Secret": self.credentials["SECRET"],
+        }
+
+        self.ids_chunks = []
+        if self.previous:
+            self.data = self.input
+
+            self.ids_chunks = self.filter_ids(
+                id_field=self.id_column_name,
+                items=self.data,
+                chunk_size=500,
+            )
+        elif getattr(self, "ids", None):
+            self._logger.info("Dropping specified Filters. Using ids instead.")
+            self.ids_chunks = [self.ids]
+
+        return True
+
+    async def run(self):
+        if not self.ids_chunks:
+            # Use the Filter specified in the task
+            df_items = await self.get_dataframe_from_entity(
+                payload=orjson.dumps(self.query_json),
+                id_column_name=self.id_column_name,
+            )
+        else:
+            # Use the ids from the previous component or from the ids argument
+            df_items = pd.DataFrame()
+            for ids_chunk in self.ids_chunks:
+                self.query_json.update(
+                    {
+                        "Filter": [
+                            {"op": "in", "field": "id", "value": ids_chunk.tolist()}
+                        ]
+                    }
+                )
+
+                items = await self.get_dataframe_from_entity(
+                    payload=orjson.dumps(self.query_json),
+                    id_column_name=self.id_column_name,
+                )
+
+                df_items = pd.concat([df_items, items], ignore_index=True)
+
+        if not df_items.empty and self.picklist_fields:
+            df_picklist_values = await self.get_picklist_values(self.picklist_fields)
+
+            if not getattr(df_picklist_values, "empty", True):
+
+                for column in [to_snake_case(field) for field in self.picklist_fields]:
+                    df_filtered = df_picklist_values[df_picklist_values['field'] == column]
+
+                    # Merge the label into df_items
+                    df_items = df_items.merge(
+                        df_filtered[['label', 'value']],
+                        how='left',
+                        left_on=column,
+                        right_on='value'
+                    )
+
+                    # Rename the label column to reflect the original field
+                    df_items.rename(columns={'label': f'{column}_label'}, inplace=True)
+
+                    # Drop the 'value' column from the merge
+                    df_items.drop(columns=['value'], inplace=True)
+
+        if "IncludeFields" in self.query_json  and self.force_create_columns:
+            required_columns = self._get_force_create_columns()
+
+            if df_items.empty:
+                # Create a DataFrame with one row filled with NaNs based on IncludeFields
+                empty_dataset = {field: [pd.NA] for field in required_columns}
+                df_items = pd.DataFrame(empty_dataset)
+            else:
+                df_items = df_items.reindex(columns=required_columns, fill_value=pd.NA)
+
+        self._result = df_items
+            # Add Debugging Block
+        if self._debug is True:
+            print("::: Printing Result Data === ")
+            print("Data: ", self._result)
+            for column, t in df_items.dtypes.items():
+                print(column, "->", t, "->", df_items[column].iloc[0])
+        return self._result
+
+    async def close(self):
+        pass
+
+    def _get_force_create_columns(self):
+            required_columns = list(map(to_snake_case, self.query_json["IncludeFields"]))
+
+            try:
+                idx = required_columns.index("id")
+                required_columns[idx] = self.id_column_name
+                
+                return required_columns
+            
+            except ValueError:
+                self.logger.error("The item 'id' was not found in the list.")
+
+    def filter_ids(self, id_field: str, items: pd.DataFrame, chunk_size):
+        data = items[id_field].dropna().unique().astype(int)
+
+        if data.size > 0:
+            split_n = math.ceil(data.size / chunk_size)
+
+            # Split into chunks of n items
+            return np.array_split(data, split_n)  # Convert to NumPy array and split
+
+        return [data]
+
+    def get_autotask_url(self, resource):
+        return urljoin(self._base_url, resource)
+
+    async def get_dataframe_from_entity(self, payload, id_column_name):
+        args = {
+            "url": self.get_autotask_url("query"),
+            "method": "post",
+            "data": payload,
+        }
+
+        results = []
+        while True:
+            result, error = await self.session(**args)
+
+            if error:
+                self._logger.error(f"{__name__}: Error getting {self.entity}")
+                raise ComponentError(f"{__name__}: Error getting {self.entity}") from error
+
+            if result is None:
+                self._logger.error(f"API returned None or empty result for {args['url']}")
+                raise ComponentError(f"API returned None or empty result for {args['url']}")
+
+            if "items" not in result:
+                self._logger.error(f"'items' not found in API response: {result}")
+                raise ComponentError("'items' not found in API response")
+
+            results.extend(result.get("items", []))
+
+            args.update({"url": result["pageDetails"].get("nextPageUrl", None)})
+
+            if not args["url"]:
+                break
+        try:
+            df_results = await self.create_dataframe(results)
+        except DataNotFound as e:
+            if self.force_create_columns:
+                return pd.DataFrame()
+            
+            raise e
+
+        if not df_results.empty and "userDefinedFields" in df_results.columns:
+            df_results_udf = df_results["userDefinedFields"].apply(self.extract_udf)
+            df_results = df_results.drop("userDefinedFields", axis=1, errors="ignore").join(df_results_udf)
+
+        df_results = (
+            df_results.rename(columns=lambda x: to_snake_case(x))
+            .rename(columns={"id": id_column_name})
+        )
+
+        return df_results
+
+    @staticmethod
+    def extract_udf(row):
+        """ Extracts dictionary values into columns"""
+        return pd.Series({d['name']: d['value'] for d in row})
+
+    async def get_picklist_values(self, field_names: list[str]) -> pd.DataFrame:
+        result, error = await self.session(
+            url=self.get_autotask_url("entityInformation/fields"),
+            method="get",
+        )
+
+        if error:
+            self._logger.error(f"{__name__}: Error getting {self.entity}")
+            raise ComponentError(f"{__name__}: Error getting {self.entity}") from error
+
+        picklist_data = []
+
+        for field_name in field_names:
+            for field in result["fields"]:
+                if field["name"] == field_name:
+                    self._logger.info(f"Extracting picking list values for {field_name}")
+
+                    if not field["picklistValues"]:
+                        df = pd.DataFrame(columns=["label", "value"])
+                    else:
+                        df = await self.create_dataframe(field["picklistValues"])
+                        df = df[["label", "value"]]
+
+                    df["field"] = to_snake_case(field_name)
+                    picklist_data.append(df)
+
+        # Concatenate all DataFrames and reset the index
+        combined_df = pd.concat(picklist_data).reset_index(drop=True).astype({"value": "int"})
+        return combined_df[["field", "label", "value"]]

flowtask/components/Azure.py

@@ -0,0 +1,80 @@
+import asyncio
+from collections.abc import Callable
+from ..interfaces.AzureClient import AzureClient
+from ..interfaces.http import HTTPService
+from .flow import FlowComponent
+
+
+class Azure(AzureClient, HTTPService, FlowComponent):
+    """
+        Azure Component.
+
+            Overview
+
+            This component interacts with Azure services using the Azure SDK for Python.
+            It requires valid Azure credentials to establish a connection.
+
+            .. table:: Properties
+            :widths: auto
+
+        +--------------------------+----------+-----------+----------------------------------------------------------------+
+        | Name                     | Required | Summary                                                                    |
+        +--------------------------+----------+-----------+----------------------------------------------------------------+
+        |  credentials (optional)  |   Yes    | Dictionary containing Azure credentials: "client_id", "tenant_id",         |
+        |                          |          | and "client_secret". Credentials can be retrieved from environment         |
+        |                          |          | variables.                                                                 |
+        +--------------------------+----------+-----------+----------------------------------------------------------------+
+        |  as_dataframe (optional) |    No    | Specifies if the response should be converted to a pandas DataFrame        |
+        |                          |          | (default: False).                                                          |
+        +--------------------------+----------+-----------+----------------------------------------------------------------+
+
+            This component does not return any data directly. It interacts with
+            Azure services based on the configuration and potentially triggers
+            downstream components in a task.
+    """  # noqa: E501
+    accept: str = "application/json"
+    no_host: bool = True
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.as_dataframe: bool = kwargs.get("as_dataframe", False)
+        # Initialize parent classes explicitly
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+        if 'secret_id' in self.credentials:
+            self.credentials['client_secret'] = self.credentials.pop('secret_id')
+
+    async def close(self, timeout: int = 5):
+        """close.
+        Closing the connection.
+        """
+        pass
+
+    async def open(self, host: str, port: int, credentials: dict, **kwargs):
+        """open.
+        Starts (open) a connection to external resource.
+        """
+        self.app = self.get_msal_app()
+        return self
+
+    async def start(self, **kwargs):
+        """Start.
+
+        Processing variables and credentials.
+        """
+        await super(Azure, self).start(**kwargs)
+        self.processing_credentials()
+        try:
+            self.client_id, self.tenant_id, self.client_secret = (
+                self.credentials.get(key)
+                for key in ["client_id", "tenant_id", "client_secret"]
+            )
+        except Exception as err:
+            self._logger.error(err)
+            raise
+
+        return True

flowtask/components/AzureUsers.py

@@ -0,0 +1,106 @@
+import iso8601
+from datetime import datetime, time, timezone
+from .Azure import Azure
+from ..exceptions import ComponentError, DataNotFound
+
+class AzureUsers(Azure):
+    """
+
+    AzureUsers Component
+
+        Overview
+
+        This component retrieves a list of users from Azure Active Directory (AAD) using the Microsoft Graph API.
+
+        .. table:: Properties
+            :widths: auto
+
+    +------------------------+----------+-----------+---------------------------------------------------------------------------------+
+    | Name                   | Required | Summary                                                                                     |
+    +------------------------+----------+-----------+---------------------------------------------------------------------------------+
+    | credentials            |   Yes    | Dictionary placeholder for Azure AD application credentials.                                |
+    +------------------------+----------+-----------+---------------------------------------------------------------------------------+
+    | most_recent (optional) |    No    | ISO 8601-formatted date and time string to filter users created on or after that date/time. |                                                                                                                                |
+    +------------------------+----------+-----------+---------------------------------------------------------------------------------+
+
+    **Returns**
+
+        A pandas DataFrame containing the retrieved user information. Each row represents an Azure user, and columns might include properties like:
+
+        - `objectId`: Unique identifier for the user in AAD.
+        - `displayName`: User's display name.
+        - `userPrincipalName`: User's email address (usually the primary login).
+        - ... (other user properties depending on the AAD response)
+
+    **Error Handling**
+
+        - The component raises a `ComponentError` with a detailed message in case of errors during authentication, API calls, or data processing. This error will be surfaced within your Flowtask workflow.
+    
+
+        Example:
+
+        ```yaml
+        AzureUsers:
+          recent: '2023-10-01'
+        ```
+
+    """  # noqa: E501
+    accept: str = 'application/json'
+
+    async def get_users(self, most_recent: str = None):
+        async def fetch_users(url: str, all_users: list):
+            result, error = await self.async_request(url, self.method)
+            if error:
+                raise ComponentError(
+                    f"{__name__}: Error in AzureUsers: {error}"
+                )
+
+            all_users.extend(result.get("value", []))
+
+            next_link = result.get("@odata.nextLink")
+            if next_link:
+                await fetch_users(next_link, all_users)
+
+        all_users = []
+        if most_recent:
+            utc_datetime = (
+                iso8601.parse_date(most_recent).isoformat().replace("+00:00", "Z")
+            )
+            url = f"{self.users_info}?$filter=createdDateTime ge {utc_datetime}"
+        else:
+            url = f"{self.users_info}"
+        await fetch_users(url, all_users)
+        return all_users
+
+    async def run(self):
+        """Run Azure Connection for getting Users Info."""
+        self._logger.info(
+            f"<{__name__}>: Starting Azure Connection for getting Users Info."
+        )
+        self.app = self.get_msal_app()
+        token, self.token_type = self.get_token()
+        self.auth["apikey"] = token
+        self.headers["Content-Type"] = "application/json"
+        self.method = "GET"
+        try:
+            more_recent = None
+            if hasattr(self, "recent"):
+                more_recent = self.mask_replacement(self.recent)
+            result = await self.get_users(more_recent)
+            self._logger.info(f"<{__name__}>: Successfully got Users Info.")
+            self._result = await self.create_dataframe(result)
+            if self._result is None:
+                raise DataNotFound(
+                    "No data found for Azure Users."
+                )
+        except Exception as err:
+            self._logger.error(err)
+            raise ComponentError(f"{__name__}: Error in AzureUsers: {err}") from err
+        numrows = len(self._result.index)
+        self.add_metric("NUM_USERS", numrows)
+        if self._debug is True:
+            print(self._result)
+            print("::: Printing Column Information === ")
+            for column, t in self._result.dtypes.items():
+                print(column, "->", t, "->", self._result[column].iloc[0])
+        return self._result

flowtask/components/BaseAction.py

@@ -0,0 +1,91 @@
+import asyncio
+from abc import ABC
+from typing import List, Dict, Union
+from collections.abc import Callable
+from ..exceptions import DataNotFound, ComponentError
+from .flow import FlowComponent
+
+
+class BaseAction(FlowComponent, ABC):
+    """
+    BaseAction Component
+
+        Overview
+            Basic component for making RESTful queries to URLs.
+            This component serves as a foundation for building more specific action components.
+            It allows you to define methods (functions) that can be executed asynchronously.
+
+            .. table:: Properties
+                :widths: auto
+
+        +-----------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------+
+        | Name                  | Required | Summary                                                                                                                   |
+        +-----------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------+
+        | loop (optional)       | No       | Event loop to use for asynchronous operations (defaults to the current event loop).                                       |
+        +-----------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------+
+        | job (optional)        | No       | Reference to a job object for logging and tracking purposes.                                                              |
+        +-----------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------+
+        | stat (optional)       | No       | Reference to a stat object for custom metrics collection.                                                                 |
+        +-----------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------+
+        | method (from kwargs)  | Yes      | Name of the method (function) within the component to be executed. Specified as a keyword argument during initialization. |
+        +-----------------------+----------+-----------+---------------------------------------------------------------------------------------------------------------+
+
+        **Returns**
+
+        The output data can vary depending on the implemented method. It can be a list, dictionary, or any data structure returned by the executed method.
+
+        **Error Handling**
+
+        - `DataNotFound`: This exception is raised if the executed method doesn't return any data.
+        - `ComponentError`: This exception is raised for any other errors encountered during execution.
+
+
+    """
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ) -> None:
+        """Init Method."""
+        self._result: Union[List, Dict] = None
+        self._method: str = kwargs.pop("method", None)
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+        args = self._attrs.get("args", {})
+        keys_to_remove = ["loop", "stat", "debug", "memory", "comment", "Group"]
+        self._kwargs = {k: v for k, v in args.items() if k not in keys_to_remove}
+
+    async def start(self, **kwargs):
+        if not hasattr(self, self._method):
+            raise ComponentError(f"{self.__name__} Error: has no Method {self._method}")
+        # Getting the method to be called
+        self._fn = getattr(self, self._method)
+        await super(BaseAction, self).start(**kwargs)
+        # Processing Variables:
+        self._kwargs = self.var_replacement(self._kwargs)
+        return True
+
+    async def close(self):
+        pass
+
+    async def run(self):
+        try:
+            result, error = await self._fn(**self._kwargs)
+            if error:
+                self._logger.warning(
+                    f"Error {self.__name__}.{self._fn.__name__}: {error}"
+                )
+                return False
+            if result is None:
+                raise DataNotFound(
+                    f"No data found for {self.__name__}.{self._fn.__name__}"
+                )
+            self._result = result
+            self.add_metric(f"{self.__name__}.{self._fn.__name__}", result)
+        except DataNotFound:
+            raise
+        except Exception as e:
+            raise ComponentError(f"Error running {self.__name__}: {e}")
+        return self._result