customeranalytics 0.1__py3-none-any.whl

customeranalytics/data_storage_configurations/__init__.py

@@ -0,0 +1,245 @@
+import sys, os, inspect
+from os.path import join, dirname, abspath, exists
+from sqlalchemy import create_engine, MetaData
+from pandas import read_sql, DataFrame
+import subprocess
+import requests
+from dateutil.parser import parse
+
+currentdir = dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+parentdir = dirname(currentdir)
+sys.path.insert(0, parentdir)
+
+from customeranalytics.data_storage_configurations.query_es import QueryES
+from customeranalytics.data_storage_configurations.data_access import GetData
+from customeranalytics.data_storage_configurations.es_create_index import CreateIndex
+from customeranalytics.data_storage_configurations.schedule_data_integration import Scheduler
+from customeranalytics.configs import elasticsearch_connection_refused_comment, query_path, acception_column_count
+from customeranalytics.utils import read_yaml, sqlite_string_converter, abspath_for_sample_data
+
+
+engine = create_engine('sqlite://///' + join(abspath_for_sample_data(), "web", 'db.sqlite3'), connect_args={'check_same_thread': False})
+metadata = MetaData()
+con = engine.connect()
+
+
+sample_data_columns = read_yaml(query_path, "queries.yaml")['columns']
+
+
+def create_connection_columns(index='orders'):
+    return [index + i for i in ['_data_source_tag', '_data_source_type',
+                                '_data_query_path', '_password', '_user', '_port', '_host', '_db']]
+
+
+def create_data_access_parameters(connection, index='orders', date=None, test=False):
+    _ds = {'data_source': connection[index + '_data_source_type'],
+            'date': date,
+            'data_query_path': sqlite_string_converter(connection[index + '_data_query_path'], back_to_normal=True),
+            'test': test,
+            'config': {'host': connection[index + '_host'],
+                       'port': connection[index + '_port'],
+                       'password': connection[index + '_password'],
+                       'user': connection[index + '_user'], 'db': connection[index + '_db']}}
+    return _ds
+
+
+def get_data_connection_arguments():
+    conn = read_sql("SELECT * FROM data_connection  ", con).to_dict('resutls')[-1]
+    columns = read_sql("SELECT  *  FROM data_columns_integration", con).to_dict('records')[-1]
+
+    data_configs = {'orders': create_data_access_parameters(conn, index='orders', date=None, test=False),
+                    'downloads': create_data_access_parameters(conn, index='downloads', date=None, test=False),
+                    'products': create_data_access_parameters(conn, index='products', date=None, test=False),
+                    'deliveries': create_data_access_parameters(conn, index='deliveries', date=None, test=False)
+                    }
+    return conn, columns, data_configs
+
+
+def get_action_name():
+    actions = read_sql("SELECT * FROM actions ", con)
+    a_orders, a_downloads = actions.query("data_type == 'orders'"), actions.query("data_type == 'downloads'")
+    check_actions = lambda a: list(a['action_name']) if len(a) != 0 else []
+    return {'orders': check_actions(a_orders), 'downloads': check_actions(a_downloads)}
+
+
+def get_ea_and_ml_config(ea_configs, ml_configs, has_product_conn, has_promotion_conn, has_delivery_connection):
+    """
+        ea_configs = {"date": None,
+                      "funnel": {"actions": ["download", "signup"],
+                                 "purchase_actions": ["has_basket", "order_screen"],
+                                 "host": 'localhost',
+                                 "port": '9200',
+                                 'download_index': 'downloads',
+                                 'order_index': 'orders'},
+                      "cohort": {"has_download": True, "host": 'localhost', "port": '9200',
+                                 'download_index': 'downloads', 'order_index': 'orders'},
+                      "product": {"has_product_connection": True, "has_download": True,
+                                   "host": 'localhost', "port": '9200'},
+                      "promotions": {"has_promotion_connection": True,
+                             "host": 'localhost', "port": '9200',
+                             "download_index": 'downloads', "order_index": 'orders'},
+                      "rfm": {"host": 'localhost', "port": '9200',
+                              'download_index': 'downloads', 'order_index': 'orders'},
+                      "stats": {"host": 'localhost', "port": '9200',
+                               'download_index': 'downloads', 'order_index': 'orders'}
+             }
+
+        ml_configs = {"date": None,
+                      'time_period': 'weekly',
+                      "segmentation": {"host": 'localhost', "port": '9200',
+                                       'download_index': 'downloads', 'order_index': 'orders'},
+                      "clv_prediction": {"temporary_export_path": None,
+                                         "host": 'localhost', "port": '9200',
+                                         'download_index': 'downloads', 'order_index': 'orders', 'time_period': 'weekly'},
+                      "abtest": {"has_product_connection": True, "temporary_export_path": None,
+                                 "host": 'localhost', "port": '9200', 'download_index': 'downloads', 'order_index': 'orders'}
+                     }
+
+    """
+    es_tag_conn = read_sql(" SELECT  * FROM es_connection ", con)
+    port, host, directory = [list(es_tag_conn[i])[0] for i in ['port', 'host', 'directory']]
+    actions = get_action_name()
+
+    configs = []
+    for conf in [ea_configs, ml_configs]:
+        for ea in conf:
+            if ea not in ['date', 'time_period']:
+                conf[ea]['host'] = host
+                conf[ea]['port'] = port
+            if ea == 'funnel':
+                conf[ea]['actions'] = actions['downloads']
+                conf[ea]['purchase_actions'] = actions['orders']
+            if ea in ['abtest', 'clv_prediction', 'delivery_anomaly']:
+                conf[ea]['temporary_export_path'] = directory
+            if not has_product_conn:
+                if ea in ['products', 'abtest']:
+                    conf[ea]['has_product_connection'] = False
+            if not has_promotion_conn:
+                if ea in ['abtest', 'promotions']:
+                    conf[ea]['has_promotion_connection'] = False
+            if not has_delivery_connection:
+                if ea == 'delivery_anomaly':
+                    conf[ea]['has_delivery_connection'] = False
+
+        configs += [conf]
+    return configs + [actions]
+
+
+def decision_for_product_conn(data_configs):
+    return True if data_configs['products']['data_source'] not in [None, 'None'] else False
+
+
+def decision_for_delivery_conn(data_configs):
+    return True if data_configs['deliveries']['data_source'] not in [None, 'None'] else False
+
+
+def decision_for_promotion_conn(columns):
+    return True if columns['promotion_id'] not in [None, 'None'] else False
+
+
+def create_index(tag, ea_configs, ml_configs):
+    """
+
+    :return:
+    """
+    columns, data_configs = get_data_connection_arguments()[1:]
+    has_product_connection = decision_for_product_conn(data_configs)
+    has_delivery_connection = decision_for_delivery_conn(data_configs)
+    has_promotion_connection = decision_for_promotion_conn(columns)
+    _ea_configs, _ml_configs, _actions = get_ea_and_ml_config(ea_configs, ml_configs,
+                                                              has_product_connection,
+                                                              has_promotion_connection, has_delivery_connection)
+    s = Scheduler(es_tag=tag,
+                  data_connection_structure=data_configs,
+                  ea_connection_structure=_ea_configs,
+                  ml_connection_structure=_ml_configs, data_columns=columns, actions=_actions)
+    s.run_schedule_on_thread(function=s.execute_schedule)
+
+
+def get_columns_condition(request, _columns, index):
+    desire_column_count = 0
+    if index == 'orders':
+        a_col_count, p_col_count, d_col_count = 0, 0, 0
+        if request.get('actions', None) is not None:
+            a_col_count = len(request['actions'].split(","))
+        if request.get('dimension', None) is not None:
+            d_col_count = 1
+        if request.get('promotion', None) is not None:
+            p_col_count = 1
+        desire_column_count = p_col_count + a_col_count + d_col_count + acception_column_count['orders']
+
+    if index == 'downloads':
+        a_col_count = 0
+        if request.get('actions', None) is not None:
+            a_col_count = len(request['actions'].split(","))
+        desire_column_count = a_col_count + acception_column_count['downloads']
+
+    if index == 'products':
+        desire_column_count = acception_column_count['products']
+
+    if index == 'deliveries':
+        desire_column_count = acception_column_count['deliveries']
+
+    if len(_columns) >= desire_column_count:
+        return True
+    else:
+        return False
+
+
+def connection_check(request, index='orders', type=''):
+    """
+
+    :param tag: elasticsearch connected tag name
+    :return:
+    """
+    accept, message, data, raw_columns = False, "Connection Failed", None, []
+    try:
+        args = create_data_access_parameters(request, index=index, date=None, test=5)
+        gd = GetData(**args)
+        gd.query_data_source()
+        if gd.data is not None:
+            if len(gd.data) != 0:
+                _columns = list(gd.data.columns)
+                _df = gd.data
+                # required list; order_id, client, s_start_date, amount, has_purchased
+                if get_columns_condition(request, _columns, index):
+                    accept, message, data, raw_columns = True, 'Connected', _df.to_dict('records'), gd.data.columns.values
+    except Exception as e:
+        print(e)
+    return accept, message, data, raw_columns
+
+
+def check_data_integration(data, index):
+    columns = list(data.columns)
+    for col in sample_data_columns[index + '_sample_data'][1:]:
+        if col not in columns:
+            data[col] = '....'
+    data = data[sample_data_columns[index + '_sample_data'][1:]]
+
+    return data
+
+
+def check_elasticsearch(port, host, directory):
+    """
+
+    :return:
+    """
+    message = 'connected'
+    connection = True
+    if exists(directory):
+        es = QueryES(port=port, host=host)
+        if not es.es.ping():
+            message = 'pls check the ElasticSearch connection.'
+            connection = False
+
+    else:
+        message = 'pls check the directory.'
+        connection = False
+    return connection, message
+
+
+
+
+
+
+

customeranalytics/data_storage_configurations/data_access.py

@@ -0,0 +1,165 @@
+import pandas as pd
+from dateutil.parser import parse
+from os.path import join, dirname
+from os import listdir
+import sys, os, inspect
+from os.path import abspath
+import glob
+
+currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+parentdir = os.path.dirname(currentdir)
+sys.path.insert(0, parentdir)
+
+from customeranalytics.configs import default_dask_partitions
+
+
+class GetData:
+    """
+    Collecting data for storing Into Orders & Downloads Index
+    """
+    def __init__(self,
+                 config=None,
+                 data_source=None,
+                 date=None,
+                 data_query_path=None,
+                 test=None,
+                 dask=False):
+        self.data_source = data_source if data_source is not None else 'csv'
+        self.data_query_path = data_query_path
+        self.dask = dask
+        self.conn = None
+        self.data = pd.DataFrame()
+        self.nrows = test
+        self.date = date
+        self.n_partitions = default_dask_partitions
+        self.config = config
+
+    def get_connection(self):
+        if self.data_source in ['postgresql', 'awsredshift', 'mysql']:
+            server, db, user, pw, port = str(self.config['host']), str(self.config['db']), \
+                                         str(self.config['user']), str(self.config['password']),\
+                                         int(self.config['port'])
+        if self.data_source == 'mysql':
+            from mysql import connector
+            self.conn = connector.connect(host=server, database=db, user=user, password=pw)
+        if self.data_source in ['postgresql', 'awsredshift']:
+            import psycopg2
+            self.conn = psycopg2.connect(user=user, password=pw, host=server, port=port, database=db)
+        if self.data_source == 'googlebigquery':
+            from google.cloud.bigquery.client import Client
+            os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = self.config['db']
+            self.conn = Client()
+        print("db connection is done!")
+
+    def read_csv(self, f, partition=False):
+        _data = pd.DataFrame()
+        try:
+            for sep in [',', ';', ':', '|']:
+                print(sep)
+                _data = pd.read_csv(filepath_or_buffer=f,
+                                    error_bad_lines=False,
+                                    encoding="ISO-8859-1",
+                                    sep=sep,
+                                    nrows=self.nrows)
+                if partition:
+                    _data = pd.read_csv(filepath_or_buffer=f,
+                                        error_bad_lines=False,
+                                        encoding="ISO-8859-1",
+                                        sep=sep,
+                                        nrows=self.nrows, index_col=None, header=0)
+                print(_data.head())
+                print(len(_data.columns))
+                if len(_data.columns) >= 2:
+                    break
+        except Exception as e:
+            print(e)
+            print("yeeee")
+        return _data
+
+    def query_data_source(self):
+        # import data via pandas
+        if self.data_source in ['mysql', 'postgresql', 'awsredshift']:
+            self.get_connection()
+            counter = 0
+            try_count = 10 if self.nrows else 100
+            while counter < try_count:
+                try:
+                    self.data = pd.read_sql(self.data_query_path + " LIMIT " + str(self.nrows) if self.nrows else self.data_query_path, self.conn)
+                except Exception as e:
+                    print(e)
+                if len(self.data) > 0:
+                    counter = try_count
+                counter += 1
+        # import data via google
+        if self.data_source == 'googlebigquery':
+            self.get_connection()
+            self.data = self.conn.query(self.data_query_path + " LIMIT " + str(self.nrows) if self.nrows else self.data_query_path).to_dataframe()
+
+        # import via pandas
+        if self.data_source == 'csv':
+            li = []
+            if self.data_query_path.split(".")[-1] != 'csv':
+                _files = glob.glob(join(self.data_query_path, "*.csv"))
+                if self.nrows:
+                    self.data = self.read_csv(self.data_query_path)
+                else:
+                    for f in _files:
+                        li.append(self.read_csv(f, partition=True))
+                    self.data = pd.concat(li, axis=0, ignore_index=True)
+            else:
+                self.data = self.read_csv(self.data_query_path)
+
+        # import data via pyarrow
+        if self.data_source == 'parquet':
+            if self.data_query_path.split(".")[-1] == 'parquet':
+                if self.nrows:
+                    self.data = pd.read_parquet(self.data_query_path).iloc[:self.nrows]
+                else:
+                    self.data = pd.read_parquet(self.data_query_path)
+            else:
+                _files = glob.glob(join(self.data_query_path, "*.parquet"))
+                if self.nrows:
+                    self.data = pd.read_parquet(_files[0]).iloc[:self.nrows]
+                else:
+                    self.data = pd.read_parquet(_files)
+
+        if self.data_source == 'hdf5':
+            if self.data_query_path.split(".")[-1] == 'hdf5':
+                if self.nrows:
+                    self.data = pd.read_hdf(self.data_query_path).iloc[:self.nrows]
+                else:
+                    self.data = pd.read_hdf(self.data_query_path)
+            else:
+                _files = glob.glob(join(self.data_query_path, "*.h5"))
+                if self.nrows:
+                    self.data = pd.read_hdf(_files[0]).iloc[:self.nrows]
+                else:
+                    self.data = pd.read_hdf(_files)
+
+    def n_partitions_decision(self):
+        if self.data_source == 'dask':
+            total_usage_mb = sum(self.data.memory_usage()) / 100000000
+            if total_usage_mb < 2:
+                self.n_partitions = 3
+            else: self.n_partitions = int(total_usage_mb) + 1
+
+    def use_dask(self):
+        self.n_partitions_decision()
+        if self.dask:
+            if self.nrows is not None:
+                self.data = dd.from_pandas(self.data, npartitions=self.n_partitions)
+
+    def data_execute(self):
+        self.query_data_source()
+        self.use_dask()
+
+
+
+
+
+
+
+
+
+
+

customeranalytics/data_storage_configurations/data_works_pipeline.py

@@ -0,0 +1,207 @@
+import sys, os, inspect, logging
+currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+parentdir = os.path.dirname(currentdir)
+sys.path.insert(0, parentdir)
+import schedule
+import threading
+import time
+import pandas as pd
+from numpy import unique
+from sqlalchemy import create_engine, MetaData
+from os.path import abspath, join
+from flask_login import current_user
+
+try:
+    from exploratory_analysis.__init__ import create_exploratory_analysis, create_exploratory_analyse
+except Exception as e:
+    from customeranalytics.exploratory_analysis import create_exploratory_analysis, create_exploratory_analyse
+try:
+    from ml_process.__init__ import create_ml
+except Exception as e:
+    from customeranalytics.ml_process import create_ml
+
+from customeranalytics.utils import current_date_to_day, convert_to_day, abspath_for_sample_data, read_yaml
+from customeranalytics.configs import query_path, DATA_WORKS_READABLE_FORM
+from customeranalytics.data_storage_configurations.es_create_index import CreateIndex
+from customeranalytics.data_storage_configurations.query_es import QueryES
+
+
+engine = create_engine('sqlite://///' + join(abspath_for_sample_data(), "web", 'db.sqlite3'), connect_args={'check_same_thread': False})
+metadata = MetaData()
+con = engine.connect()
+
+
+class DataPipelines:
+    def __init__(self,
+                 es_tag,
+                 data_connection_structure,
+                 ea_connection_structure,
+                 ml_connection_structure,
+                 data_columns,
+                 actions):
+        """
+        :param es_tag: elasticsearch tag name that is created on web interface 'ElasticSearch Configuration' page
+        :param data_connection_structure: check :data_configs above
+        :param ea_connection_structure: check :ea_configs above
+        :param ml_connection_structure: check :ml_configs above
+        :param data_columns: check :data_columns above
+        :param actions: whole actions which are stored into the actions table in sqlite
+        """
+        self.es_tag = es_tag
+        self.data_connection_structure = data_connection_structure
+        self.ea_connection_structure = ea_connection_structure
+        self.ml_connection_structure = ml_connection_structure
+        self.actions = actions
+        self.data_columns = data_columns
+        self.es_con = pd.read_sql("select * from es_connection", con).to_dict('records')[-1]
+        self.create_index = CreateIndex(data_connection_structure=data_connection_structure,
+                                        data_columns=data_columns, actions=actions)
+        self.query_es = QueryES(host=self.es_con['host'], port=self.es_con['port'])
+        self.unique_dimensions = []
+        self.schedule = True
+        self.sqlite_queries = read_yaml(query_path, "queries.yaml")
+        self.tables = pd.read_sql(self.sqlite_queries['tables'], con)
+        self.separator = lambda dim: [print("*" * 20) for i in range(3)] + [
+            print("*" * 10, " ", " DIMENSION : ", dim, "*" * 10)]
+        self.info_logs_for_chat = lambda info: {'user': 'info',
+                                                'date': str(current_date_to_day())[0:19],
+                                                'user_logo': 'info.jpeg',
+                                                'chat_type': 'info', 'chart': '',
+                                                'general_message': info, 'message': ''}
+        self.suc_log_for_ea = """
+                               Exploratory Analysis are Created! Check Funnel, Cohort, 
+                               Descriptive Stats sections.
+                              """
+        self.suc_log_for_data_works = lambda x: " {0} is created! Check {0} sections. ".format(
+            DATA_WORKS_READABLE_FORM[x])
+        self.fail_log_for_data_works = lambda x, e: " {0} is failed! Check {0} sections. - {1}".format(
+            DATA_WORKS_READABLE_FORM[x], e)
+
+    def check_for_table_exits(self, table):
+        """
+        checking sqlite if table is created before. If it not, table is created.
+        :params table: checking table name in sqlite
+        """
+
+        try:
+            if table not in list(self.tables['name']):
+                con.execute(self.sqlite_queries[table])
+        except Exception as e: print(e)
+
+    def insert_query(self, table, columns, values):
+        """
+        insert sqlite tables with given table column and values
+
+        :param table: tables to be inserted the row
+        :param columns: tables of columns (all columns)
+        :param values: values for each column in the table
+        """
+        values = [values[col] for col in columns]
+        _query = "INSERT INTO " + table + " "
+        _query += " (" + ", ".join(columns) + ") "
+        _query += " VALUES (" + ", ".join([" '{}' ".format(v) for v in values]) + ") "
+        _query = _query.replace("\\", "")
+        return _query
+
+    def logs_update(self, logs):
+        """
+        logs table in sqlite table is updated.
+        chats table in sqlite table is updated.
+        """
+        try: self.check_for_table_exits(table='logs')
+        except Exception as e: print(e)
+
+        try: self.check_for_table_exits(table='chat')
+        except Exception as e: print(e)
+
+        try:
+            logs['login_user'] = current_user
+            logs['log_time'] = str(current_date_to_day())[0:19]
+            con.execute(self.insert_query(table='logs',
+                                          columns=self.sqlite_queries['columns']['logs'][1:],
+                                          values=logs
+                                          ))
+        except Exception as e: print(e)
+
+        try:
+            con.execute(self.insert_query(table='chat', columns=self.sqlite_queries['columns']['chat'][1:],
+                                          values=self.info_logs_for_chat(logs['info'])))
+        except Exception as e: print(e)
+
+    def info_log_create(self, type, dim=None):
+        """
+        Logging system on creating E.A. or ML Works. This print processes are also shown at profile.html activities.
+        This gives the information of the process are done.
+        """
+        _info = self.suc_log_for_data_works(type)
+        if dim is not None:
+            _info += " || dimension : " + dim
+        print("-- Process Info --")
+        print(_info)
+        self.logs_update(logs={"page": "data-execute", "info": _info, "color": "green"})
+
+    def fail_log_create(self, e, type, dim=None):
+        """
+        Logging system on creating E.A. or ML Works. This print processes are also shown at profile.html activities.
+        This gives the information of the process are failed.
+        """
+        e_str = ''
+        if e is not None:
+            e_str = str(e)
+            e_str = e_str[:min(len(e_str), 100)]
+        fail_message = self.fail_log_for_data_works(type, e_str)
+        if dim is not None:
+            fail_message += " || dimension : " + dim
+        print(" FAIL ---- !!!!!!!!")
+        print("-- message :", fail_message)
+        print(" ----- description ::::::", e_str)
+        self.logs_update(logs={"page": "data-execute",
+                               "info": fail_message.replace("'", " "),
+                               "color": "red"})
+
+    def execute_pipe(self, _conf, execution, type, dim=None):
+        try:
+            execution(_conf, type)
+            self.info_log_create(type=type)
+        except Exception as e:
+            print(e)
+            self.fail_log_create(e=e, type=type, dim=dim)
+
+    def pipe_1(self, ml_connection_structure, ea_connection_structure, dim=None):
+        self.execute_pipe(ml_connection_structure, create_ml, 'clv_prediction')
+
+    def pipe_2(self, ml_connection_structure, ea_connection_structure, dim=None):
+        self.execute_pipe(ea_connection_structure, create_exploratory_analyse, 'rfm', dim=dim)
+        self.execute_pipe(ml_connection_structure, create_ml, 'segmentation', dim=dim)
+
+    def pipe_3(self, ml_connection_structure, ea_connection_structure, dim=None):
+        self.execute_pipe(ea_connection_structure, create_exploratory_analyse, 'funnel', dim=dim)
+        self.execute_pipe(ea_connection_structure, create_exploratory_analyse, 'cohort', dim=dim)
+
+    def pipe_4(self, ml_connection_structure, ea_connection_structure, dim=None):
+        self.execute_pipe(ea_connection_structure, create_exploratory_analyse, 'stats', dim=dim)
+        self.execute_pipe(ea_connection_structure, create_exploratory_analyse, 'churn', dim=dim)
+
+    def pipe_5(self, ml_connection_structure, ea_connection_structure, dim=None):
+        self.execute_pipe(ml_connection_structure, create_ml, 'abtest', dim=dim)
+        self.execute_pipe(ea_connection_structure, create_exploratory_analyse, 'products', dim=dim)
+        self.execute_pipe(ea_connection_structure, create_exploratory_analyse, 'promotions', dim=dim)
+
+    def pipe_6(self, ml_connection_structure, ea_connection_structure, dim=None):
+        self.execute_pipe(ml_connection_structure, create_ml, 'anomaly', dim=dim)
+        self.execute_pipe(ml_connection_structure, create_ml, 'delivery_anomaly', dim=dim)
+
+    def data_work_pipelines_execution(self, ml_connection_structure, ea_connection_structure, dim=None):
+        _kwargs = {"ml_connection_structure": ml_connection_structure,
+                   'ea_connection_structure': ea_connection_structure, 'dim': dim}
+        if dim is None:
+            self.pipe_1(**_kwargs)
+        self.pipe_2(**_kwargs)
+        pipes = [self.pipe_3, self.pipe_4]
+        for pipe in pipes:
+            process = threading.Thread(target=pipe, kwargs=_kwargs)
+            process.daemon = True
+            process.start()
+        process.join()
+        self.pipe_5(**_kwargs)
+        self.pipe_6(**_kwargs)