dscin-ppy 0.0.1__tar.gz

dscin_ppy-0.0.1/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.1
+Name: dscin_ppy
+Version: 0.0.1
+Summary: Collection of utility functions
+Home-page: UNKNOWN
+Author: Andrea Chiappo
+Author-email: chiappo.andrea@gmail.com
+License: UNKNOWN
+Platform: UNKNOWN
+
+UNKNOWN
+

dscin_ppy-0.0.1/dscin_ppy/__init__.py

@@ -0,0 +1,12 @@
+from functions import *
+from db_utils import *
+from etl_job import *
+from aws_utils import *
+
+import os
+bucket = os.getenv("S3_TRANSDICT_BUCKET")
+key = os.getenv("S3_TRANSDICT_KEY")
+
+download_object_from_s3(
+    bucket, key, "src/translation_dictionaries.py"
+)

dscin_ppy-0.0.1/dscin_ppy/aws_utils.py

@@ -0,0 +1,86 @@
+import boto3
+
+
+def build_s3_url(bucket, key):
+    return f's3a://{bucket}/{key}'
+
+
+def read_object_from_s3(bucket, key):
+    s3_client = boto3.client("s3")
+    try:
+        # Get the file inside the S3 Bucket
+        s3_response = s3_client.get_object(Bucket=bucket, Key=key)
+        return s3_response["Body"].read()
+    except s3_client.exceptions.NoSuchBucket as e:
+        print("The S3 bucket does not exist.")
+        print(e)
+    except s3_client.exceptions.NoSuchKey as e:
+        print("The S3 objects does not exist in the S3 bucket.")
+        print(e)
+
+
+def open_object_from_s3(bucket, key):
+    s3_client = boto3.client("s3")
+    try:
+        # Get the file inside the S3 Bucket
+        s3_response = s3_client.get_object(Bucket=bucket, Key=key)
+        return s3_response["Body"]
+    except s3_client.exceptions.NoSuchBucket as e:
+        print("The S3 bucket does not exist.")
+        print(e)
+    except s3_client.exceptions.NoSuchKey as e:
+        print("The S3 objects does not exist in the S3 bucket.")
+        print(e)
+
+
+def write_dataframe_to_s3(dataframe, s3_url, file_type):
+    if file_type == "parquet":
+        dataframe.to_parquet(s3_url, engine='pyarrow', index=False)
+    elif file_type == "csv":
+        dataframe.to_csv(s3_url, index=False)
+    elif file_type == "xlsx":
+        dataframe.to_excel(s3_url, index=False)
+    else:
+        raise Exception("Format not supported")
+
+
+def get_value_from_parameter_store(key, decrypt=False):
+    ssm_client = boto3.client("ssm")
+    value = ssm_client.get_parameter(Name=key, WithDecryption=decrypt)
+    return value["Parameter"]["Value"]
+
+
+def save_to_s3(df, s3_bucket, s3_key, file_format):
+    s3_url = build_s3_url(s3_bucket, s3_key)
+    write_dataframe_to_s3(df, s3_url, file_format)
+
+
+def copy_s3_object(copy_source, target_bucket, target_key):
+    s3_resource = boto3.resource("s3")
+    s3_bucket = s3_resource.Bucket(target_bucket)
+    s3_bucket.copy(copy_source, target_key)
+
+
+def list_s3_objects(s3_bucket, s3_key_prefix):
+    s3_client = boto3.client("s3")
+    objects = s3_client.list_objects_v2(Bucket=s3_bucket, Prefix=s3_key_prefix)
+    # Extract the keys of the objects
+    objects_list = [
+        obj['Key'] for obj in objects.get('Contents', []) if obj
+    ]
+    return objects_list
+
+
+def upload_object_to_s3(obj, s3_bucket, s3_key):
+    s3_client = boto3.client("s3")
+    s3_client.upload_fileobj(obj, s3_bucket, s3_key)
+
+
+def remove_object_from_s3(bucket, key):
+    s3_client = boto3.client('s3')
+    s3_client.delete_object(Bucket=bucket, Key=key)
+
+
+def download_object_from_s3(bucket, key, filename):
+    s3_client = boto3.client('s3')
+    s3_client.download_file(Bucket=bucket, Key=key, Filename=filename)

dscin_ppy-0.0.1/dscin_ppy/constants.py

@@ -0,0 +1,10 @@
+# INPUT PARAMS
+ENV_PARAM = "env"
+EXPORT_DATA_PARAM = "export_data"
+LAST_X_DAYS_PARAM = "last_x_days"
+SEASON_YEAR = "season_year"
+HANA_SEASON = "hana_season"
+FILE_NAME_PARAM = "file_name"
+PREV_SEASONS_PARAM = "prev_seasons"
+SEND_EMAIL ="send_email"
+MODEL_NAME_EXT = "model_name_ext"

dscin_ppy-0.0.1/dscin_ppy/data_extracts.py

@@ -0,0 +1,234 @@
+# This module contains functions to save and load data
+import os
+import csv
+import time
+import pickle
+import datetime
+import pandas as pd
+
+import logging
+log = logging.getLogger(__name__)
+
+
+def remove_special_charachers(text,  special=" ", replace=None):
+    if replace is None:
+        text = text.replace(special, "")
+    else:
+        text = text.replace(special, replace)
+    return text
+
+def construct_path(*path):
+    path = os.path.join(*path)
+    path = (
+        os.path.join(os.path.dirname(__file__), path)
+        if not os.path.isabs(path)
+        else path
+    )
+    return path
+
+def write_pickle(obj, *path):
+    """
+    Filepath in .pkl
+    """
+    path = construct_path(*path)
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, "wb") as f:
+        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)
+
+def read_pickle(*path):
+    path = construct_path(*path)
+    with open(path, "rb") as f:
+        return pickle.load(f)
+
+def append_csv(dataframe, path):
+    os.path.isfile(path)
+    path = construct_path(path)
+    if not os.path.exists(os.path.dirname(path)):
+        os.makedirs(os.path.dirname(path))
+    if os.path.isfile(path):
+        dataframe.to_csv(path, index=False, mode="a", header=False, sep=";")
+    else:
+        dataframe.to_csv(path, index=False, mode="a", header=True, sep=";")
+
+def write_csv(dataframe: pd.DataFrame, *path, **kwargs):
+    """
+    Writes a DataFrame to disk as csv file, maintaining project standards.
+
+    Args:
+        DataFrame (pandas.DataFrame): the DataFrame to save
+        path: file location
+    """
+    path = construct_path(*path)
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    dataframe.to_csv(path, index=False, **kwargs)
+
+def read_from_table(connection, table_name, cols_list=None, sum_list=None,
+                    filter_condition=None, date_cols=None, extract_csv=False,
+                    path="table_extraction.csv"):
+    """
+    Function that extracts defined table
+    If {extract_csv} is True, Extracted file saved as csv to the defined {path}
+
+    Args:
+    - connection: pyhdb.connect(host, port, user, password)
+    - table_name: table name in the database
+    - col_list: list of columns to query
+    - sum_list: list of columns to sum from the aggregation
+    - filter_condition: string containing the filtering instruction
+    - date_cols: list of column date in the data
+    - extract_csv: Boolean to control whether to save extraction to csv
+    - path: path string where to save the extracted data to csv
+
+    Returns:
+    - df (pd.DataFrame) extracted data
+
+    """
+    try:
+        if cols_list is not None:
+            if sum_list is not None:
+                group_list = set(cols_list) - set(sum_list)
+                group_str = '","'.join(group_list)
+                group_str = '"' + group_str + '"'
+                cols_list = set(cols_list) - set(sum_list)
+            cols_string = '","'.join(cols_list)
+            cols_string = '"' + cols_string + '"'
+
+            if sum_list is not None:
+                sum_list = ['sum("' + s + '")' for s in sum_list]
+                sum_list = ",".join(sum_list)
+                cols_string = cols_string + "," + sum_list
+        else:
+            cols_string = "*"
+
+        if filter_condition is not None and sum_list is None:
+            extract_query = "SELECT {} FROM {} WHERE {}".format(
+                cols_string, table_name, filter_condition
+            )
+        elif filter_condition is not None and sum_list is not None:
+            extract_query = "SELECT {} FROM {} WHERE {} GROUP BY {}".format(
+                cols_string, table_name, filter_condition, group_str
+            )
+        elif filter_condition is None and sum_list is not None:
+            extract_query = "SELECT {} FROM {} GROUP BY {}".format(
+                cols_string, table_name, group_str
+            )
+        else:
+            extract_query = "SELECT {} FROM {}".format(
+                cols_string, table_name
+            )
+
+        log.info(extract_query)
+        no_of_zerorecord_check:int = 1
+        while no_of_zerorecord_check != 0:
+            log.info(
+                str("no_of_zerorecord_check :" + str(no_of_zerorecord_check))
+            )
+            log.info(
+                str("Extraction start at :" + str(datetime.datetime.now()))
+            )
+            _data = pd.DataFrame()
+            data_list = []
+
+            if date_cols is None:
+                for chunk in pd.read_sql_query(extract_query, connection, chunksize=25000):
+                    log.info(
+                        str(
+                            "Extracting Data from the table " +
+                            table_name +
+                            "as chunksize of 25000"
+                        )
+                    )
+                    if extract_csv:
+                        append_csv(chunk, path)
+                    else:
+                        data_list.append(chunk)
+                    del chunk
+            else:
+                for chunk in pd.read_sql_query(extract_query, connection, chunksize=25000, parse_dates=date_cols):
+                    log.info(
+                        str(
+                            "Extracting Data from the table " +
+                            table_name +
+                            "as chunksize of 25000"
+                        )
+                    )
+                    if extract_csv:
+                        append_csv(chunk, path)
+                    else:
+                        data_list.append(chunk)
+                    del chunk
+
+            if extract_csv is False:
+                _data = pd.concat(data_list, ignore_index=True)
+
+            numOfRows = len(_data.index)
+
+            if numOfRows == 0:
+                # job sleeps for 1 minutes
+                log.info("Connection has been broken, wait for 10 secs")
+                time.sleep(10)
+            else:
+                break
+
+            no_of_zerorecord_check = no_of_zerorecord_check - 1
+
+        log.info(
+            str("Extraction end at :" + str(datetime.datetime.now()))
+        )
+
+        return _data
+
+    except Exception as e:
+        raise e
+
+def read_csv(file_path, sep=",", decimal=".", header=0, index_col=""):
+    try:
+        if os.path.isfile(file_path):
+            if index_col != "":
+                _data = pd.read_csv(
+                    file_path,
+                    sep=sep,
+                    decimal=decimal,
+                    header=header,
+                    quoting=csv.QUOTE_NONE,
+                    index_col=index_col,
+                    dtype="str"
+                )
+            else:
+                _data = pd.read_csv(
+                    file_path,
+                    sep=sep,
+                    decimal=decimal,
+                    header=header,
+                    quoting=csv.QUOTE_NONE,
+                    dtype="str"
+                )
+            return _data
+        else:
+            raise Exception
+    except Exception:
+        raise
+
+def read_excel(file_path, sheet_name=0, skiprows=None):
+    try:
+        if os.path.isfile(file_path):
+            _data = pd.read_excel(
+                file_path, sheet_name=sheet_name, skiprows=skiprows
+            )
+            return _data
+        else:
+            raise Exception
+    except Exception:
+        raise
+
+def write_excel(dataframe: pd.DataFrame, *path, **kwargs):
+    """
+    Writes a DataFrame to disk as excel file, maintaining project standards.
+
+    Args:
+        DataFrame (pandas.DataFrame): the DataFrame to save
+        path: file location
+    """
+    path = construct_path(*path)
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    dataframe.to_excel(path, index=False, **kwargs)

dscin_ppy-0.0.1/dscin_ppy/db_utils.py

@@ -0,0 +1,14 @@
+from hdbcli import dbapi
+
+from config import db_config
+
+def get_connection():
+    conn = dbapi.connect(
+        address=db_config.sap_hana_url,
+        port=db_config.sap_hana_port,
+        user=db_config.sap_hana_username,
+        password=db_config.sap_hana_password,
+    )
+    _ = conn.cursor()
+
+    return conn

dscin_ppy-0.0.1/dscin_ppy/errors.py

@@ -0,0 +1,2 @@
+class ValidationError(Exception):
+    pass

dscin_ppy-0.0.1/dscin_ppy/etl_job.py

@@ -0,0 +1,14 @@
+import job_parameters
+
+class EtlJob:
+
+    def __init__(self):
+        self.params = job_parameters.gather_parameters()
+
+    def get_parameter(self, parameter, required: bool, default=None):
+        parameter_input_value = self.params.get(parameter.parameter_name)
+        parameter.validate(parameter_input_value, default, required)
+        return parameter.get_value_or_else(parameter_input_value, default)
+
+    def run(self):
+        pass

dscin_ppy-0.0.1/dscin_ppy/exposing_reports.py

@@ -0,0 +1,55 @@
+from config.env_config import env_config
+from aws_utils import copy_s3_object
+
+import logging
+log = logging.getLogger(__name__)
+
+
+def copy_report_to_email_bucket(env_param, file_name, s3_key, datatype='cid'):
+    s3_bucket = env_config[env_param]["feature-store-bucket"]
+    s3_exp_bucket = env_config[env_param]["exposed-bucket"]
+    s3_email_root_key = env_config[env_param]["exposed-send-to-email-path"]
+    if datatype == 'cid':
+        report_prefix = env_config[env_param]["exposed-cid-report-prefix"]
+    elif datatype == 'cod':
+        report_prefix = env_config[env_param]["exposed-cod-report-prefix"]
+    elif datatype == 'd2s':
+        report_prefix = env_config[env_param]["exposed-demand-to-supply-prefix"]
+    else:
+        raise ValueError(f"unkwon data type {datatype}")
+
+    copy_source = {"Bucket": s3_bucket, "Key": s3_key}
+    exp_file_name = f"{report_prefix}_{file_name}"
+    s3_exp_key = f"{s3_email_root_key}/{exp_file_name}"
+
+    log.info(f"Started file upload {exp_file_name} to email bucket.")
+    copy_s3_object(copy_source, s3_exp_bucket, s3_exp_key)
+    log.info(f"Finished file upload {exp_file_name} to email bucket.")
+
+
+def copy_report_to_tableau_bucket(env_param, file_name, s3_key):
+    s3_bucket = env_config[env_param]["feature-store-bucket"]
+    s3_exp_bucket = env_config[env_param]["exposed-bucket"]
+    s3_tableau_root_key = env_config[env_param]["exposed-send-to-tableau-path"]
+
+    copy_source = {"Bucket": s3_bucket, "Key": s3_key}
+    s3_exp_key = f"{s3_tableau_root_key}/{file_name}"
+
+    log.info(f"Started file upload {file_name} to tableau bucket.")
+    copy_s3_object(copy_source, s3_exp_bucket, s3_exp_key)
+    log.info(f"Finished file upload {file_name} to tableau bucket.")
+
+
+def copy_report_to_email_bucket_marketplace(env_param, file_name, s3_key):
+    s3_bucket = env_config[env_param]["feature-store-bucket"]
+    s3_exp_bucket = env_config[env_param]["exposed-bucket"]
+    s3_email_root_key = env_config[env_param]["exposed-send-to-email-path"]
+    report_prefix = env_config[env_param]["exposed-cod-report-prefix_compact"]
+
+    copy_source = {"Bucket": s3_bucket, "Key": s3_key}
+    exp_file_name = f"{report_prefix}_{file_name}"
+    s3_exp_key = f"{s3_email_root_key}/{exp_file_name}"
+
+    log.info(f"Started file upload {exp_file_name} to email bucket.")
+    copy_s3_object(copy_source, s3_exp_bucket, s3_exp_key)
+    log.info(f"Finished file upload {exp_file_name} to email bucket.")

dscin_ppy-0.0.1/dscin_ppy/functions.py

@@ -0,0 +1,206 @@
+
+import numpy as np
+import pandas as pd
+
+from time import time
+from functools import wraps
+from datetime import datetime
+
+
+def timing(f):
+    @wraps(f)
+    def wrap(*args, **kw):
+        ts = time()
+        result = f(*args, **kw)
+        te = time()
+        print('func:%r args:[%r, %r] took: %2.4f sec' % \
+          (f.__name__, args, kw, te-ts))
+        return result
+    return wrap
+
+
+def fillna_using_dtype(df):
+    column_types = df.dtypes.copy()
+
+    for k,v in df.dtypes.items():
+        if v == object:
+            column_types[k] = "#"
+        elif v == float or v == int:
+            column_types[k] = 0
+        elif v == datetime:
+            column_types[k] = pd.NaT
+        else:
+            column_types[k] = "#"
+
+    df.fillna(column_types, inplace=True)
+
+    return df
+
+
+def replace_comma_with_semicolon(x):
+    if isinstance(x, str):
+        return x.replace(',', ';')
+    else:
+        return x
+
+
+def extract_datetime_columns_from_dictionary(translation_dictionary):
+    """
+    Function to extract from {translation_dictionary} the columns with
+    datetime type and return a list of the corresponding technical names
+
+    Args:
+    - translation_dictionary: (dict)
+        translation dictionary, consisting of key-value pairs
+        {technical column name: [human column name (str), column data type]}
+
+    Returns:
+    - date_cols: (list[str])
+        list of all technical column names with data type datetime
+
+    Notes:
+    - An error is returned if {translation_dictionary} is empty
+    """
+
+    assert len(translation_dictionary) > 0, "translation_dictionary is empty"
+
+    date_cols = [
+        key for key, val in translation_dictionary.items() if val[1] == datetime
+    ]
+
+    return date_cols
+
+
+def extract_dtype_columns_from_dictionary(translation_dictionary, data_type):
+    """
+    Function to extract from {translation_dictionary} the columns with
+    {data_type} type and return a list of the corresponding technical names
+
+    Args:
+    - translation_dictionary: (dict)
+        translation dictionary, consisting of key-value pairs
+        {technical column name: [human column name (str), column data type]}
+    - data_type: (str)
+        data type of the desired columns (for example: str, float, int,...)
+
+    Returns:
+    - data_type_cols: (list[str])
+        list of all technical column names with data type {data_type}
+
+    Notes:
+    - An error is returned if {translation_dictionary} is empty
+    """
+
+    assert len(translation_dictionary) > 0, "translation_dictionary is empty"
+
+    ### Create a list of the d_non_technical_column_names that match data_type
+    data_type_cols = [
+        key for key, val in translation_dictionary.items() if val[1] == data_type
+    ]
+
+    return data_type_cols
+
+
+def extract_column_translations_from_dictionary(translation_dictionary):
+    """
+    Function to extract from {translation_dictionary} the technical names of
+    columns and their corresponding human-readable name and return a dictionary
+
+    Args:
+    - translation_dictionary: (dict)
+        translation dictionary, consisting of key-value pairs
+        {technical column name: [human column name (str), column data type]}
+    
+    Returns:
+    - column_translation_dictionary: (dict{str: str}])
+       dictionary mapping columns' technical and human-readable names
+
+    Notes:
+    - An error is returned if {translation_dictionary} is empty
+    """
+
+    assert len(translation_dictionary) > 0, "translation_dictionary is empty"
+
+    # Create a dictionary of technical_column_name: non_technical_column_name
+    column_translation_dictionary = {
+        key: val[0] for key, val in translation_dictionary.items()
+    }
+
+    return column_translation_dictionary
+
+
+def convert_column_dtype_to_str(df, translation_dictionary):
+    """
+    Function to convert to string the columns of {df} using the
+    {translation_dictionary} as a lookup reference for data types
+
+    Args:
+    - df: (pd.DataFrame)
+        Pandas.DataFrame whose columns to convert to str
+    - translation_dictionary: (dict)
+        reference dictionary for looking up the columns with str data type
+
+    Returns:
+    - df (pd.DataFrame):
+        staring Pandas.DataFrame with columns converted to str data type
+
+    Notes:
+    - An error is returned if {translation_dictionary} is empty
+    """
+
+    assert len(translation_dictionary) != 0, "translation_dictionary is empty"
+
+    ### Extract the str columns from translation_dictionary and store in list
+    str_cols = extract_dtype_columns_from_dictionary(translation_dictionary, str)
+    str_cols = [col for col in str_cols if col in df.columns.tolist()]
+    
+    if len(str_cols) != len(set(str_cols)):
+        raise ValueError("Duplicate column names found in str_cols.")
+
+    ### Convert type
+    df[str_cols] = df[str_cols].astype(str)
+
+    return df
+
+
+def convert_column_dtype_to_numeric(df, translation_dictionary, data_type):
+    """
+    Function to convert to numeric {data_type} the columns of {df} using 
+    the {translation_dictionary} as a lookup reference for data types
+
+    Args:
+    - df: (pd.DataFrame)
+        Pandas.DataFrame whose columns to convert to {data_type}
+    - translation_dictionary: (dict)
+        reference dictionary for looking up the columns with {data_type} data type
+    - data_type: (str)
+        numeric data type object to which convert the columns of {df}
+
+    Returns:
+    - df (pd.DataFrame):
+        staring Pandas.DataFrame with columns converted to {data_type} data type
+
+    Notes:
+    - Empty values are replaced with 0's
+    - {data_type} needs to be of type float or int
+    - An error is returned if {translation_dictionary} is empty
+    """
+
+    assert data_type == float or data_type == int, "data_type must be either int or float!"
+    assert len(translation_dictionary) > 0, "translation_dictionary is empty"
+
+    ### Extract the int columns from translation_dictionary and store in list
+    num_cols = extract_dtype_columns_from_dictionary(translation_dictionary, data_type)
+    num_cols = [col for col in num_cols if col in df.columns.tolist()]
+
+    ### Replace empty values in num_cols with 0
+    df[num_cols] = df[num_cols].replace(r"^\s*$", np.nan, regex=True)
+    df[num_cols] = df[num_cols].fillna(0)
+
+    ### Convert type
+    if data_type == float:
+        df[num_cols] = df[num_cols].astype(float)
+    elif data_type == int:
+        df[num_cols] = df[num_cols].astype(float).astype(int)
+
+    return df

dscin_ppy-0.0.1/dscin_ppy/job_parameters.py

@@ -0,0 +1,278 @@
+
+import argparse
+
+import constants
+from errors import ValidationError
+
+
+class Parameter:
+
+    def get_value_or_else(self, value, default):
+        return value or default
+
+    def validate(self, value, default, required):
+        pass
+
+    def cast_value(self, value):
+        pass
+
+
+class EnvParameter(Parameter):
+
+    def __init__(self) -> None:
+        self.parameter_name = constants.ENV_PARAM
+
+    def validate(self, value, default, required):
+        if not value:
+            value = default
+        if not value and required:
+            raise ValidationError(
+                f"Parameter is required: {self.parameter_name}"
+            )
+        if value not in ['dev', 'stage', 'prod']:
+            raise ValidationError(
+                f"{self.parameter_name} available values: 'dev', 'stage', 'prod'"
+            )
+
+    def get_value_or_else(self, value, default):
+        value = super().get_value_or_else(value, default)
+        return self.cast_value(value)
+
+    def cast_value(self, value):
+        return str(value)
+
+
+class ExportDataParameter(Parameter):
+
+    def __init__(self) -> None:
+        self.parameter_name = constants.EXPORT_DATA_PARAM
+
+    def validate(self, value, default, required):
+        if not value:
+            value = default
+        if not value and required:
+            raise ValidationError(
+                f"Parameter is required: {self.parameter_name}"
+            )
+        if value not in ['True', 'False']:
+            raise ValidationError(
+                f"{self.parameter_name} available values: True/False"
+            )
+
+    def get_value_or_else(self, value, default):
+        value = super().get_value_or_else(value, default)
+        return self.cast_value(value)
+
+    # TODO: cast u boolean?
+    def cast_value(self, value):
+        return str(value)
+
+
+class LastXDaysParameter(Parameter):
+    def __init__(self) -> None:
+        self.parameter_name = constants.LAST_X_DAYS_PARAM
+
+    def validate(self, value, default, required):
+        if not value:
+            value = default
+        if not value and required:
+            raise ValidationError(
+                f"Parameter is required: {self.parameter_name}"
+            )
+        if not value.isnumeric():
+            raise ValidationError(
+                f"Value of {self.parameter_name} must be a number."
+            )
+
+    def get_value_or_else(self, value, default):
+        value = super().get_value_or_else(value, default)
+        return self.cast_value(value)
+
+    def cast_value(self, value):
+        return str(value)
+
+
+class SeasonYearParameter(Parameter):
+    def __init__(self) -> None:
+        self.parameter_name = constants.SEASON_YEAR
+
+    def validate(self, value, default, required):
+        if not value:
+            value = default
+        if not value and required:
+            raise ValidationError(
+                f"Parameter is required: {self.parameter_name}"
+            )
+        if not value.isnumeric():
+            raise ValidationError(
+                f"Value of {self.parameter_name} must be a number"
+            )
+
+    def get_value_or_else(self, value, default):
+        value = super().get_value_or_else(value, default)
+        return self.cast_value(value)
+
+    def cast_value(self, value):
+        return int(value)
+
+
+class HanaSeasonParameter(Parameter):
+    def __init__(self) -> None:
+        self.parameter_name = constants.HANA_SEASON
+    def validate(self, value, default, required):
+        if not value:
+            value = default
+        if not value and required:
+            raise ValidationError(
+                f"Parameter is required: {self.parameter_name}"
+            )
+        if value not in ['WI', 'FA', 'SP', 'SU']:
+            raise ValidationError(
+                f"{self.parameter_name} available values: 'WI','SP','SU','FA'"
+            )
+
+    def get_value_or_else(self, value, default):
+        value = super().get_value_or_else(value, default)
+        return self.cast_value(value)
+
+    def cast_value(self, value):
+        return str(value)
+
+
+class FileNameParameter(Parameter):
+    def __init__(self) -> None:
+        self.parameter_name = constants.FILE_NAME_PARAM
+
+    def validate(self, value, default, required):
+        if not value:
+            value = default
+        if not value and required:
+            raise ValidationError(
+                f"Parameter is required: {self.parameter_name}"
+            )
+        if not isinstance(value, str):
+            raise ValidationError(
+                f"{self.parameter_name} should be the name of the file in string format"
+            )
+
+    def get_value_or_else(self, value, default):
+        value = super().get_value_or_else(value, default)
+        return self.cast_value(value)
+
+    def cast_value(self, value):
+        return str(value)
+
+
+class PreviousSeasonParameter(Parameter):
+    def __init__(self) -> None:
+        self.parameter_name = constants.PREV_SEASONS_PARAM
+
+    def validate(self, value, default, required):
+        if not value:
+            value = default
+        if not value and required:
+            raise ValidationError(
+                f"Parameter is required: {self.parameter_name}"
+            )
+
+    def get_value_or_else(self, value, default):
+        value = super().get_value_or_else(value, default)
+        return self.cast_value(value)
+
+    def cast_value(self, value):
+        return int(value)
+
+
+class SendEmailParameter(Parameter):
+    def __init__(self) -> None:
+        self.parameter_name = constants.SEND_EMAIL
+
+    def validate(self, value, default, required):
+        if value is None:
+            value = default
+        if not value and required:
+            raise ValidationError(
+                f"Parameter is required: {self.parameter_name}"
+            )
+
+    def get_value_or_else(self, value, default):
+        if value is None:
+            value = super().get_value_or_else(value, default)
+        return self.cast_value(value)
+
+    def cast_value(self, value):
+        return value
+
+
+class ModelNameExt(Parameter):
+    def __init__(self) -> None:
+        self.parameter_name = constants.MODEL_NAME_EXT
+
+    def validate(self, value, default, required):
+        if not value:
+            value = default
+        if not value and required:
+            raise ValidationError(
+                f"Parameter is required: {self.parameter_name}"
+            )
+
+    def get_value_or_else(self, value, default):
+        value = super().get_value_or_else(value, default)
+        return self.cast_value(value)
+
+    def cast_value(self, value):
+        return value
+
+
+def gather_parameters():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        f"--{constants.ENV_PARAM}",
+        help="environment in which job is running",
+    )
+
+    parser.add_argument(
+        f"--{constants.EXPORT_DATA_PARAM}",
+        help="save data parameter",
+    )
+
+    parser.add_argument(
+        f"--{constants.LAST_X_DAYS_PARAM}",
+        help="parameter to determine the number of days considered in the query",
+    )
+
+    parser.add_argument(
+        f"--{constants.SEASON_YEAR}",
+        help="parameter to determine the current season year",
+    )
+
+    parser.add_argument(
+        f"--{constants.HANA_SEASON}",
+        help="parameter to determine the current Hana season",
+    )
+
+    parser.add_argument(
+        f"--{constants.FILE_NAME_PARAM}",
+        help="file name parameter",
+    )
+
+    parser.add_argument(
+        f"--{constants.PREV_SEASONS_PARAM}",
+        help="number of previous seasons to consider",
+    )
+
+    parser.add_argument(
+        f"--{constants.SEND_EMAIL}",
+        action="store_false",
+        help="DO NOT send out email with link to download file",
+    )
+
+    parser.add_argument(
+        f"--{constants.MODEL_NAME_EXT}",
+        help="suffix to append to model name",
+    )
+
+    args = parser.parse_args()
+    params_dictionary = vars(args)
+
+    return params_dictionary

dscin_ppy-0.0.1/dscin_ppy.egg-info/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.1
+Name: dscin-ppy
+Version: 0.0.1
+Summary: Collection of utility functions
+Home-page: UNKNOWN
+Author: Andrea Chiappo
+Author-email: chiappo.andrea@gmail.com
+License: UNKNOWN
+Platform: UNKNOWN
+
+UNKNOWN
+

dscin_ppy-0.0.1/dscin_ppy.egg-info/SOURCES.txt

@@ -0,0 +1,17 @@
+README.md
+setup.py
+dscin_ppy/__init__.py
+dscin_ppy/aws_utils.py
+dscin_ppy/constants.py
+dscin_ppy/data_extracts.py
+dscin_ppy/db_utils.py
+dscin_ppy/errors.py
+dscin_ppy/etl_job.py
+dscin_ppy/exposing_reports.py
+dscin_ppy/functions.py
+dscin_ppy/job_parameters.py
+dscin_ppy.egg-info/PKG-INFO
+dscin_ppy.egg-info/SOURCES.txt
+dscin_ppy.egg-info/dependency_links.txt
+dscin_ppy.egg-info/requires.txt
+dscin_ppy.egg-info/top_level.txt

dscin_ppy-0.0.1/dscin_ppy.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

dscin_ppy-0.0.1/dscin_ppy.egg-info/requires.txt

@@ -0,0 +1,4 @@
+boto3==1.35.15
+pandas==2.0.1
+numpy==1.23.5
+hdbcli==2.17.22

dscin_ppy-0.0.1/dscin_ppy.egg-info/top_level.txt

@@ -0,0 +1 @@
+dscin_ppy

dscin_ppy-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

dscin_ppy-0.0.1/setup.py

@@ -0,0 +1,16 @@
+from setuptools import find_packages, setup
+
+setup(
+    name="dscin_ppy",
+    version="0.0.1",
+    author="Andrea Chiappo",
+    author_email="chiappo.andrea@gmail.com",
+    description="Collection of utility functions",
+    packages=find_packages(include=["dscin_ppy"]),
+    install_requires=[
+        "boto3==1.35.15",
+        "pandas==2.0.1",
+        "numpy==1.23.5",
+        "hdbcli==2.17.22"
+    ]
+)