flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/interfaces/d2l.py

@@ -0,0 +1,239 @@
+"""
+D2L.
+
+Making operations over D2L.
+
+"""
+import re
+from datetime import datetime, timezone
+from ..exceptions import (
+    ComponentError
+)
+import random
+from .http import HTTPService, ua
+from .cache import CacheSupport
+
+class D2LClient(CacheSupport, HTTPService):
+    '''
+        Manage Connection to D2L
+    '''
+    def __init__(self, *args, **kwargs):
+        self.token_type: str = "Bearer"
+        self.auth_type: str = "apikey"
+        self.domain: str = kwargs.pop('domain', None)
+        self.domain = self.get_env_value(self.domain, self.domain)
+        self.url_login: str = f'https://{self.domain}/d2l/lp/auth/login/login.d2l'
+        self.url_token: str = f'https://{self.domain}/d2l/lp/auth/oauth2/token'
+        self.file_format: str = 'application/zip'
+        self.create_destination: bool = True  # by default
+        self.dataset: str = kwargs.pop('dataset', None)
+        self.plugin: str = kwargs.pop('plugin', None)
+        self.d2l_session_val: str = None
+        self.d2l_secure_session_val: str = None
+        self.csrf_token: str = None
+        self.auth: dict = {'apikey': None}
+        super().__init__(*args, **kwargs)
+        self.processing_credentials()
+        self.username: str = self.credentials.get('username', None)
+        self.password: str = self.credentials.get('password', None)
+        self.headers = {
+            "Accept-Encoding": "gzip, deflate",
+            "DNT": "1",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": random.choice(ua)
+        }
+
+    async def get_bearer_token(self):
+        # Try to get API Key from REDIS
+        with await self.open() as redis:
+            api_key = redis.get('D2L_API_KEY')
+            d2l_session_val = redis.get('D2L_SESSION_VAL')
+            d2l_secure_session_val = redis.get('D2L_SECURE_SESSION_VAL')
+            if api_key:
+                self.auth['apikey'] = api_key
+                self.cookies = {
+                    "d2lSessionVal": d2l_session_val,
+                    "d2lSecureSessionVal": d2l_secure_session_val
+                }
+                self.headers["Authorization"] = f"Bearer {self.auth.get('apikey')}"
+                return api_key
+        # Get cookies values
+        login_headers = {
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Origin": f"https://{self.domain}",
+            "Referer": f"https://{self.domain}/d2l/login",
+            "User-Agent": random.choice(ua)
+        }
+        login_data = {
+            "d2l_referrer": f"https://{self.domain}/d2l/login",
+            "loginPath": "/d2l/login",
+            "userName": self.username,
+            "password": self.password,
+        }
+        try:
+            response = await self._post(
+                url=self.url_login,
+                cookies=None,
+                headers=login_headers,
+                data=login_data,
+                follow_redirects=False,
+                raise_for_status=False,
+                use_proxy= False
+            )
+            d2l_session_val = response.cookies.get("d2lSessionVal", "")
+            d2l_secure_session_val = response.cookies.get("d2lSecureSessionVal", "")
+            self.cookies = {
+                "d2lSessionVal": d2l_session_val,
+                "d2lSecureSessionVal": d2l_secure_session_val
+            }
+            response = await self._post(
+                url=self.url_login,
+                cookies=self.cookies,
+                headers=login_headers,
+                data=login_data,
+                follow_redirects=True,
+                raise_for_status=True,
+                use_proxy= False
+            )
+            csrf_token_match = re.search(r"localStorage\.setItem\('XSRF\.Token','(.*?)'\)", response.text)
+            csrf_token = csrf_token_match.group(1) if csrf_token_match else ""
+        except Exception as err:
+            raise ComponentError(
+                f"D2L: Error getting data from URL {err}"
+            )
+        # Get Bearer Token
+        token_headers = {
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Origin": f"https://{self.domain}",
+            "Referer": f"https://{self.domain}/d2l/home",
+            "x-csrf-token": csrf_token,
+            "User-Agent": random.choice(ua)
+        }
+        token_data = {"scope": "*:*:*"}
+        try:
+            response = await self._post(
+                url=self.url_token,
+                cookies=self.cookies,
+                headers=token_headers,
+                data=token_data,
+                use_proxy= False
+            )
+            json = response.json()
+            with await self.open() as redis:
+                current_timestamp = int(datetime.now(timezone.utc).timestamp())
+                expires_at = json.get('expires_at', (current_timestamp - 600))
+                timeout = expires_at - current_timestamp
+                self.setexp('D2L_API_KEY', json.get("access_token", None), f'{timeout}s')
+                self.setexp('D2L_SESSION_VAL', d2l_session_val, f'{timeout}s')
+                self.setexp('D2L_SECURE_SESSION_VAL', d2l_secure_session_val, f'{timeout}s')
+            self.auth['apikey'] = json.get("access_token", None)
+            self.headers["Authorization"] = f"Bearer {self.auth.get('apikey')}"
+            return self.auth['apikey']
+        except Exception as err:
+            raise ComponentError(
+                f"D2L: Error getting data from URL {err}"
+            )
+
+    async def download_file(self):
+        # Get Download URL
+        url = f'https://{self.domain}/d2l/api/lp/1.45/datasets/bds/{self.schema}/extracts'
+        response = await self.api_get(
+            url=url,
+            headers=self.headers,
+            use_proxy=False
+        )
+        download_link = ''
+        if hasattr(self, "masks") and hasattr(self, 'date'):
+            self.date = self.mask_replacement(self.date)
+        for obj in response.get('Objects', []):
+            if hasattr(self, "date") and self.date:
+                target_date = datetime.strptime(self.date, "%Y-%m-%d").date()
+                if obj.get("BdsType") == "Differential":
+                    created_date = datetime.strptime(obj.get("CreatedDate", ""), "%Y-%m-%dT%H:%M:%S.%fZ").date()
+                    if created_date == target_date:
+                        download_link = obj.get("DownloadLink")
+                        break
+            else:
+                if obj.get('BdsType', '') == 'Full':
+                    download_link = obj.get('DownloadLink', '')
+                    break
+        # Download ZIP File
+        self.download = True
+        await self.async_request(
+            url=download_link
+        )
+        return True
+
+    async def request_iterate(self, endpoint, api=True, **kwargs):
+        has_more_items = True
+        resultset = []
+        bookmark = 0
+        if api == True:
+            while has_more_items == True:
+                url = f'https://{self.domain}/d2l/api{endpoint}/?bookmark={bookmark}'
+                response = await self.api_get(
+                    url=url,
+                    headers=self.headers,
+                    use_proxy= False
+                )
+                if not response['Items']:
+                    break
+                else:
+                    bookmark = response['PagingInfo']['Bookmark']
+                    has_more_items = response['PagingInfo']['HasMoreItems']
+                    resultset = [*resultset, *response['Items']]
+                #break
+            return resultset
+        else:
+            item = kwargs.get('item', '')
+            while has_more_items == True:
+                url = f'https://{self.domain}/d2l{endpoint}/?limit=100&offset={bookmark}'
+                try:
+                    response = await self.api_get(
+                        url=url,
+                        headers=self.headers,
+                        cookies=self.cookies,
+                        use_proxy= False
+                    )
+                    if not response[item]:
+                        break
+                    else:
+                        bookmark += 100
+                        has_more_items = response['Metadata']['HasMore']
+                        resultset = [*resultset, *response[item]]
+                    #break
+                except Exception:
+                    break
+            return resultset
+
+    async def awards(self, org_units=None):
+        result = []
+        
+        if org_units is None:
+            org_units = await self.request_iterate('/lp/1.49/orgstructure', True)
+            org_units = [item["Identifier"] for item in org_units["Items"]]
+        
+        for org_unit_id in org_units:
+            users = await self.request_iterate(f'/awards/v1/{org_unit_id}/classlist', False, item='Users')
+            for user in users:
+                user_id = user['UserId']
+                org_unit_id = user['OrgUnitId']
+                
+                if user['Paging'].get('HasMore', False) == True:
+                    awards = await self.request_iterate(f'/awards/v1/{org_unit_id}/myAwards/{user_id}', False, item='Awards')
+                else:
+                    awards = user['Awards']
+                
+                for award in awards:
+                    result.append({
+                        'award_id': award['AwardId'],
+                        'user_id': user_id,
+                        'org_unit_id': org_unit_id,
+                        'certificate_id': award['CertificateId'],
+                        'achievement_id': award['AchievementId'],
+                        'achievement': award['Achievement'].get('Title	', None),
+                        'awarded_by': award['AwardedBy'],
+                        'issue_date': award['IssueDate'][14:-1]
+                    })
+        return result

flowtask/interfaces/databases/__init__.py

@@ -0,0 +1,5 @@
+from .db import DBSupport
+
+__all__ = (
+    "DBSupport",
+)

flowtask/interfaces/databases/db.py

@@ -0,0 +1,223 @@
+from typing import Optional
+import asyncio
+import pandas
+from asyncdb import AsyncDB
+from navconfig.logging import logging
+from querysource.conf import asyncpg_url, default_dsn
+from ..credentials import CredentialsInterface
+from ...utils.functions import is_empty, as_boolean
+from ...exceptions import DataNotFound
+
+
+class DBSupport(CredentialsInterface):
+    """DBSupport.
+
+        Interface for adding AsyncbDB-based Database Support to Components.
+    """
+    _service_name: str = 'Flowtask'
+    _credentials = {
+        "user": str,
+        "password": str,
+        "host": str,
+        "port": int,
+        "database": str,
+    }
+
+    def __init__(
+        self,
+        *args,
+        **kwargs
+    ):
+        self.as_dataframe: bool = as_boolean(kwargs.get("as_dataframe", False))
+        # using "string" instead objects in pandas
+        self.as_string: bool = as_boolean(kwargs.get("as_string", False))
+        # Infer types:
+        self.infer_types: bool = as_boolean(kwargs.get("infer_types", False))
+        if not hasattr(self, '_logger'):
+            self._logger = logging.getLogger(
+                'Flowtask.DBSupport'
+            )
+        super().__init__(*args, **kwargs)
+
+    def event_loop(
+        self, evt: Optional[asyncio.AbstractEventLoop] = None
+    ) -> asyncio.AbstractEventLoop:
+        if evt is not None:
+            asyncio.set_event_loop(evt)
+            return evt
+        else:
+            try:
+                return asyncio.get_event_loop()
+            except RuntimeError as exc:
+                try:
+                    evt = asyncio.new_event_loop()
+                    asyncio.set_event_loop(evt)
+                    return evt
+                except RuntimeError as exc:
+                    raise RuntimeError(
+                        f"There is no Event Loop: {exc}"
+                    ) from exc
+
+    def get_connection(
+        self,
+        driver: str = "pg",
+        dsn: Optional[str] = None,
+        params: Optional[dict] = None,
+        event_loop: Optional[asyncio.AbstractEventLoop] = None,
+        **kwargs,
+    ):
+        # TODO: datasources and credentials
+        if not kwargs and driver == "pg":
+            kwargs = {
+                "server_settings": {
+                    "application_name": f"{self._service_name}.DB",
+                    "client_min_messages": "notice",
+                    "max_parallel_workers": "512",
+                    "jit": "on",
+                }
+            }
+        if not event_loop:
+            event_loop = self.event_loop()
+        args = {
+            "loop": event_loop,
+            **kwargs
+        }
+        if dsn is not None:
+            args["dsn"] = dsn
+        if params:
+            args["params"] = params
+        return AsyncDB(
+            driver, **args
+        )
+
+    def db_connection(
+        self,
+        driver: str = "pg",
+        credentials: Optional[dict] = None,
+        event_loop: Optional[asyncio.AbstractEventLoop] = None,
+    ):
+        if not credentials:
+            credentials = {"dsn": default_dsn}
+        else:
+            credentials = {"params": credentials}
+        kwargs = {}
+        if driver == "pg":
+            kwargs = {
+                "server_settings": {
+                    "application_name": f"{self._service_name}.DB",
+                    "client_min_messages": "notice",
+                    "max_parallel_workers": "512",
+                    "jit": "on",
+                }
+            }
+        if not event_loop:
+            event_loop = self.event_loop()
+        return AsyncDB(
+            driver,
+            loop=event_loop,
+            **credentials,
+            **kwargs
+        )
+
+    def pg_connection(
+        self,
+        dsn: Optional[str] = None,
+        credentials: Optional[dict] = None,
+        event_loop: Optional[asyncio.AbstractEventLoop] = None,
+    ):
+        if not credentials:
+            if dsn is not None:
+                credentials = {"dsn": dsn}
+            else:
+                credentials = {"dsn": asyncpg_url}
+        else:
+            credentials = {"params": credentials}
+        kwargs: dict = {
+            "min_size": 2,
+            "server_settings": {
+                "application_name": f"{self._service_name}.DB",
+                "client_min_messages": "notice",
+                "max_parallel_workers": "512",
+                "jit": "on",
+            },
+        }
+        if not event_loop:
+            event_loop = self.event_loop()
+        return AsyncDB(
+            "pg",
+            loop=event_loop, **credentials, **kwargs
+        )
+
+    def get_default_driver(self, driver: str):
+        """get_default_driver.
+
+        Getting a default connection based on driver's name.
+        """
+        driver_path = f"querysource.datasources.drivers.{driver}"
+        drv = f"{driver}_default"
+        try:
+            driver_module = __import__(driver_path, fromlist=[driver])
+            drv_obj = getattr(driver_module, drv)
+            return drv_obj
+        except ImportError as err:
+            raise ImportError(
+                f"Error importing driver: {err!s}"
+            ) from err
+        except AttributeError as err:
+            raise AttributeError(
+                f"Error getting driver: {err!s}"
+            ) from err
+        except Exception as err:
+            raise Exception(
+                f"Error getting default connection: {err!s}"
+            ) from err
+
+    def default_connection(self, driver: str):
+        """default_connection.
+
+        Default Connection to Database.
+        """
+        credentials = {}
+        try:
+            driver = self.get_default_driver(driver)
+            credentials = driver.params()
+            if driver.driver == 'pg' and credentials.get('username', None) is not None:
+                credentials['user'] = credentials.pop('username')
+        except ImportError as err:
+            raise ImportError(
+                f"Error importing Default driver: {err!s}"
+            ) from err
+        try:
+            return self.get_connection(
+                driver=driver.driver,
+                params=credentials
+            )
+        except Exception as err:
+            raise Exception(
+                f"Error getting Default Connection: {err!s}"
+            ) from err
+
+    async def get_dataframe(self, result):
+        try:
+            df = pandas.DataFrame(result)
+        except Exception as err:  # pylint: disable=W0703
+            logging.exception(err, stack_info=True)
+        # Attempt to infer better dtypes for object columns.
+        if is_empty(df):
+            raise DataNotFound("DbClient: Data not Found")
+        df.infer_objects()
+        if self.infer_types is True:
+            df = df.convert_dtypes(convert_string=self.as_string)
+        if self._debug is True:
+            print(df.dtypes)
+        if hasattr(self, "drop_empty"):
+            df.dropna(axis=1, how="all", inplace=True)
+            df.dropna(axis=0, how="all", inplace=True)
+        if hasattr(self, "dropna"):
+            df.dropna(subset=self.dropna, how="all", inplace=True)
+        if (
+            hasattr(self, "clean_strings") and getattr(self, "clean_strings", False) is True
+        ):
+            u = df.select_dtypes(include=["object", "string"])
+            df[u.columns] = u.fillna("")
+        return df

flowtask/interfaces/databases/documentdb.py

@@ -0,0 +1,55 @@
+from typing import Optional
+from querysource.conf import (
+    DOCUMENTDB_HOSTNAME,
+    DOCUMENTDB_PORT,
+    DOCUMENTDB_DATABASE,
+    DOCUMENTDB_USERNAME,
+    DOCUMENTDB_PASSWORD,
+    DOCUMENTDB_TLSFILE,
+)
+from .db import DBSupport
+
+
+class DocumentDBSupport(DBSupport):
+    """DocumentDBSupport.
+
+    Interface for adding AWS DocumentDB Database Support to Components.
+    """
+    _service_name: str = 'Flowtask'
+    driver: str = 'mongo'
+    _credentials = {
+        "host": str,
+        "port": int,
+        "username": str,
+        "password": str,
+        "database": str,
+        "dbtype": "documentdb",
+        "ssl": True,
+        "tlsCAFile": str,
+    }
+
+    def default_connection(self, driver: str = 'mongo', credentials: Optional[dict] = None):
+        """default_connection.
+
+        Default Connection to RethinkDB.
+        """
+        if not credentials:
+            credentials = {
+                "host": DOCUMENTDB_HOSTNAME,
+                "port": DOCUMENTDB_PORT,
+                "username": DOCUMENTDB_USERNAME,
+                "password": DOCUMENTDB_PASSWORD,
+                "database": DOCUMENTDB_DATABASE,
+                "dbtype": "documentdb",
+                "ssl": True,
+                "tlsCAFile": DOCUMENTDB_TLSFILE,
+            }
+        try:
+            return self.get_connection(
+                driver=driver,
+                params=credentials
+            )
+        except Exception as err:
+            raise Exception(
+                f"Error getting Default DocumentDB Connection: {err!s}"
+            ) from err

flowtask/interfaces/databases/rethink.py

@@ -0,0 +1,39 @@
+from .db import DBSupport
+
+
+class RethinkDBSupport(DBSupport):
+    """RethinkDBSupport.
+
+    Interface for adding RethinkDB-based Database Support to Components.
+    """
+    _service_name: str = 'Flowtask'
+    _credentials = {
+        "user": str,
+        "password": str,
+        "host": str,
+        "port": int,
+        "database": str,
+    }
+
+    def default_connection(self, driver: str = 'rethink'):
+        """default_connection.
+
+        Default Connection to RethinkDB.
+        """
+        credentials = {}
+        try:
+            driver = self.get_default_driver(driver)
+            credentials = driver.params()
+        except ImportError as err:
+            raise ImportError(
+                f"Error importing RethinkDB driver: {err!s}"
+            ) from err
+        try:
+            return self.get_connection(
+                driver=driver.driver,
+                params=credentials
+            )
+        except Exception as err:
+            raise Exception(
+                f"Error getting Default Rethink Connection: {err!s}"
+            ) from err

flowtask/interfaces/dataframes/__init__.py

@@ -0,0 +1,11 @@
+from .pandas import PandasDataframe
+from .polars import PolarsDataframe
+from .arrow import ArrowDataframe
+from .dt import DtDataframe
+
+__all__ = (
+    "PandasDataframe",
+    "PolarsDataframe",
+    "ArrowDataframe",
+    "DtDataframe",
+)

flowtask/interfaces/dataframes/abstract.py

@@ -0,0 +1,21 @@
+from typing import Any, Union, ParamSpec
+from abc import ABC, abstractmethod
+from navconfig.logging import logging
+
+
+P = ParamSpec("P")
+
+
+class BaseDataframe(ABC):
+
+    @abstractmethod
+    async def create_dataframe(
+        self, result: Union[dict, bytes, Any], *args: P.args, **kwargs: P.kwargs
+    ) -> Any:
+        """
+        Converts any result into a DataFrame.
+
+        :param result: The result data to be converted into a DataFrame.
+        :return: A DataFrame containing the result data.
+        """
+        pass

flowtask/interfaces/dataframes/arrow.py

@@ -0,0 +1,71 @@
+from typing import Union, Any, ParamSpec
+import pandas as pd
+import orjson
+import pyarrow as pa
+import pyarrow.csv as pc
+import pyarrow.parquet as pq
+from io import BytesIO
+from .abstract import BaseDataframe
+from ...exceptions import ComponentError, DataNotFound
+
+
+P = ParamSpec("P")
+
+
+def is_empty(obj):
+    """check_empty.
+    Check if a basic object, a Apache Arrow Table is empty or not.
+    """
+    if isinstance(obj, pa.Table):
+        return obj.num_rows == 0
+    else:
+        return bool(not obj)
+
+
+class ArrowDataframe(BaseDataframe):
+    """ArrowDataframe.
+
+    Converts any result into a Arrow DataFrame.
+    """
+
+    async def create_dataframe(
+        self, result: Union[dict, bytes, Any], *args: P.args, **kwargs: P.kwargs
+    ) -> Any:
+        """
+        Converts any result into a Arrow DataFrame.
+
+        :param result: The result data to be converted into a Arrow DataFrame.
+        :return: A DataFrame containing the result data.
+        """
+        if is_empty(result):
+            raise DataNotFound("DataFrame: No Data was Found.")
+        try:
+            if isinstance(result, str):
+                try:
+                    result = orjson.loads(result)
+                except Exception:
+                    pass
+            if isinstance(result, list):
+                names = list(result[0].keys())
+                df = pa.Table.from_arrays(result, names=names, **kwargs)
+            elif isinstance(result, bytes) or isinstance(result, BytesIO):
+                try:
+                    # Reset the pointer to the start of the stream
+                    result.seek(0)
+                    df = pc.read_csv(result, **kwargs)
+                except pa.lib.ArrowInvalid as e:
+                    # Use pyarrow.parquet.read_table to read Parquet data
+                    df = pq.read_table(result)
+            else:
+                df = pa.Table.from_pandas(pd.DataFrame(result, **kwargs))
+            numrows = df.num_rows
+            numcols = df.num_columns
+            try:
+                self._variables["_numRows_"] = numrows
+                self.add_metric("NUM_ROWS", numrows)
+                self.add_metric("NUM_COLS", numcols)
+            except Exception:
+                pass
+            return df
+        except Exception as err:
+            raise ComponentError(f"Error Creating Dataframe: {err!s}")