MindsDB 1.2.9__py3-none-any.whl → 24.12.4.0__py3-none-any.whl

mindsdb/libs/controllers/transaction.py

@@ -1,254 +0,0 @@
-from mindsdb.libs.helpers.general_helpers import unpickle_obj
-from mindsdb.libs.constants.mindsdb import *
-from mindsdb.libs.helpers.general_helpers import *
-from mindsdb.libs.data_types.transaction_data import TransactionData
-from mindsdb.libs.data_types.transaction_output_data import PredictTransactionOutputData, TrainTransactionOutputData
-from mindsdb.libs.data_types.mindsdb_logger import log
-from mindsdb.libs.helpers.probabilistic_validator import ProbabilisticValidator
-from mindsdb.config import CONFIG
-
-import time
-import _thread
-import traceback
-import importlib
-import copy
-import pickle
-import datetime
-import sys
-
-
-class Transaction:
-
-    def __init__(self, session, light_transaction_metadata, heavy_transaction_metadata, logger =  log):
-        """
-        A transaction is the interface to start some MindsDB operation within a session
-
-        :param session:
-        :type session: utils.controllers.session_controller.SessionController
-        :param transaction_type:
-        :param transaction_metadata:
-        :type transaction_metadata: dict
-        :type heavy_transaction_metadata: dict
-        """
-
-        self.session = session
-        self.lmd = light_transaction_metadata
-        self.lmd['created_at'] = str(datetime.datetime.now())
-        self.hmd = heavy_transaction_metadata
-
-        # variables to de defined by setup
-        self.error = None
-        self.errorMsg = None
-
-        self.input_data = TransactionData()
-        self.output_data = TrainTransactionOutputData()
-
-        # variables that can be persisted
-
-
-        self.log = logger
-
-        self.run()
-
-    # @TODO Make it more generic, move to general helpers, use inside predictor instead of linline loading
-    def load_metadata(self):
-        try:
-            import resource
-            resource.setrlimit(resource.RLIMIT_STACK, [0x10000000, resource.RLIM_INFINITY])
-            sys.setrecursionlimit(0x100000)
-        except:
-            pass
-
-        with open(os.path.join(CONFIG.MINDSDB_STORAGE_PATH, self.lmd['name'] + '_light_model_metadata.pickle'), 'rb') as fp:
-            self.lmd = pickle.load(fp)
-
-        with open(os.path.join(CONFIG.MINDSDB_STORAGE_PATH, self.hmd['name'] + '_heavy_model_metadata.pickle'), 'rb') as fp:
-            self.hmd = pickle.load(fp)
-
-    # @TODO Make it more generic, move to general helpers
-    def save_metadata(self):
-        with open(os.path.join(CONFIG.MINDSDB_STORAGE_PATH, self.lmd['name'] + '_light_model_metadata.pickle'), 'wb') as fp:
-            self.lmd['updated_at'] = str(datetime.datetime.now())
-            pickle.dump(self.lmd, fp,protocol=pickle.HIGHEST_PROTOCOL)
-
-        with open(os.path.join(CONFIG.MINDSDB_STORAGE_PATH, self.hmd['name'] + '_heavy_model_metadata.pickle'), 'wb') as fp:
-            # Don't save data for now
-            save_hmd = {}
-            null_out_fields = ['test_from_data', 'from_data']
-            for k in null_out_fields:
-                save_hmd[k] = None
-
-            for k in self.hmd:
-                if k not in null_out_fields:
-                    save_hmd[k] = self.hmd[k]
-
-            # Don't save data for now
-            pickle.dump(save_hmd, fp,protocol=pickle.HIGHEST_PROTOCOL)
-
-    def _call_phase_module(self, clean_exit, module_name, **kwargs):
-        """
-        Loads the module and runs it
-
-        :param module_name:
-        :return:
-        """
-
-        self.lmd['is_active'] = True
-        module_path = convert_cammelcase_to_snake_string(module_name)
-        module_full_path = 'mindsdb.libs.phases.{module_path}.{module_path}'.format(module_path=module_path)
-        try:
-            main_module = importlib.import_module(module_full_path)
-            module = getattr(main_module, module_name)
-            return module(self.session, self)(**kwargs)
-        except:
-            error = 'Could not load module {module_name}'.format(module_name=module_name)
-            self.log.error('Could not load module {module_name}'.format(module_name=module_name))
-            self.log.error(traceback.format_exc())
-            if clean_exit:
-                exit(1)
-            else:
-                raise ValueError(error)
-                return None
-        finally:
-            self.lmd['is_active'] = False
-
-
-    def _execute_learn(self):
-        """
-        :return:
-        """
-        try:
-            self.lmd['current_phase'] = MODEL_STATUS_PREPARING
-            self.save_metadata()
-
-            self._call_phase_module(clean_exit=True, module_name='DataExtractor')
-            self.save_metadata()
-
-            self.lmd['current_phase'] = MODEL_STATUS_DATA_ANALYSIS
-            if 'skip_stats_generation' in self.lmd and self.lmd['skip_stats_generation'] == True:
-                self.load_metadata()
-            else:
-                self.save_metadata()
-                self._call_phase_module(clean_exit=True, module_name='StatsGenerator', input_data=self.input_data, modify_light_metadata=True, hmd=self.hmd)
-                self.save_metadata()
-
-            self._call_phase_module(clean_exit=True, module_name='DataTransformer', input_data=self.input_data, mode='train')
-
-            self.lmd['current_phase'] = MODEL_STATUS_TRAINING
-            self.save_metadata()
-            self._call_phase_module(clean_exit=True, module_name='ModelInterface', mode='train')
-
-            self.lmd['current_phase'] = MODEL_STATUS_ANALYZING
-            self.save_metadata()
-            self._call_phase_module(clean_exit=True, module_name='ModelAnalyzer')
-
-            self.lmd['current_phase'] = MODEL_STATUS_TRAINED
-            self.save_metadata()
-            return
-
-        except Exception as e:
-            self.lmd['is_active'] = False
-            self.lmd['current_phase'] = MODEL_STATUS_ERROR
-            self.lmd['error_msg'] = traceback.print_exc()
-            self.log.error(str(e))
-            raise e
-
-
-    def _execute_predict(self):
-        """
-        :return:
-        """
-        old_lmd = {}
-        for k in self.lmd: old_lmd[k] = self.lmd[k]
-
-        old_hmd = {}
-        for k in self.hmd: old_hmd[k] = self.hmd[k]
-        self.load_metadata()
-
-        for k in old_lmd:
-            if old_lmd[k] is not None:
-                self.lmd[k] = old_lmd[k]
-            else:
-                if k not in self.lmd:
-                    self.lmd[k] = None
-
-        for k in old_hmd:
-            if old_hmd[k] is not None:
-                self.hmd[k] = old_hmd[k]
-            else:
-                if k not in self.hmd:
-                    self.hmd[k] = None
-
-        if self.lmd is None:
-            self.log.error('No metadata found for this model')
-            return
-
-        self._call_phase_module(clean_exit=True, module_name='DataExtractor')
-
-        if self.input_data.data_frame.shape[0] <= 0:
-            self.log.error('No input data provided !')
-            return
-
-        self._call_phase_module(clean_exit=True, module_name='DataTransformer', input_data=self.input_data)
-
-        self._call_phase_module(clean_exit=True, module_name='ModelInterface', mode='predict')
-
-        output_data = {col: [] for col in self.lmd['columns']}
-        evaluations = {}
-
-        for column in self.input_data.data_frame.columns:
-            output_data[column] = list(self.input_data.data_frame[column])
-
-        for predicted_col in self.lmd['predict_columns']:
-            output_data[predicted_col] = list(self.hmd['predictions'][predicted_col])
-
-            probabilistic_validator = unpickle_obj(self.hmd['probabilistic_validators'][predicted_col])
-            confidence_column_name = f'{predicted_col}_confidence'
-            output_data[confidence_column_name] = [None] * len(output_data[predicted_col])
-            evaluations[predicted_col] = [None] * len(output_data[predicted_col])
-
-            for row_number, predicted_value in enumerate(output_data[predicted_col]):
-
-                # Compute the feature existance vector
-                input_columns = [col for col in self.input_data.columns if col not in self.lmd['predict_columns']]
-                features_existance_vector = [False if output_data[col][row_number] is None else True for col in input_columns if col not in self.lmd['malformed_columns']['names']]
-
-                # Create the probabilsitic evaluation
-                prediction_evaluation = probabilistic_validator.evaluate_prediction_accuracy(features_existence=features_existance_vector, predicted_value=predicted_value)
-                if type(prediction_evaluation) == float:
-                    output_data[confidence_column_name][row_number] = prediction_evaluation
-                    evaluations[predicted_col][row_number] = None
-                else:
-                    output_data[confidence_column_name][row_number] = prediction_evaluation.most_likely_probability
-                    evaluations[predicted_col][row_number] = prediction_evaluation
-
-        self.output_data = PredictTransactionOutputData(transaction=self, data=output_data, evaluations=evaluations)
-
-        return
-
-
-    def run(self):
-        """
-
-        :return:
-        """
-
-        if self.lmd['type'] == TRANSACTION_BAD_QUERY:
-            self.log.error(self.errorMsg)
-            self.error = True
-            return
-
-        if self.lmd['type'] == TRANSACTION_LEARN:
-            self.output_data.data_frame = [['Model ' + self.lmd['name'] + ' training.']]
-            self.output_data.columns = ['Status']
-
-            if CONFIG.EXEC_LEARN_IN_THREAD == False:
-                self._execute_learn()
-            else:
-                _thread.start_new_thread(self._execute_learn, ())
-            return
-
-        elif self.lmd['type'] == TRANSACTION_PREDICT:
-            self._execute_predict()
-        elif self.lmd['type'] == TRANSACTION_NORMAL_SELECT:
-            self._execute_normal_select()

mindsdb/libs/data_sources/file_ds.py

@@ -1,196 +0,0 @@
-import pandas
-import re
-from io import BytesIO, StringIO
-import csv
-import codecs
-import json
-import traceback
-import codecs
-import requests
-
-from mindsdb.libs.data_types.data_source import DataSource
-from pandas.io.json import json_normalize
-from mindsdb.libs.data_types.mindsdb_logger import log
-
-
-class FileDS(DataSource):
-    
-    def cleanRow(self, row):
-        n_row = []
-        for cell in row:
-            if str(cell) in ['', ' ', '  ', 'NaN', 'nan', 'NA']:
-                cell = None
-            n_row.append(cell)
-
-        return n_row
-
-    def _getDataIo(self, file):
-        """
-        This gets a file either url or local file and defiens what the format is as well as dialect
-        :param file: file path or url
-        :return: data_io, format, dialect
-        """
-
-        ############
-        # get file as io object
-        ############
-
-        data = BytesIO()
-
-        # get data from either url or file load in memory
-        if file[:5] == 'http:' or file[:6] == 'https:':
-            r = requests.get(file, stream=True)
-            if r.status_code == 200:
-                for chunk in r:
-                    data.write(chunk)
-            data.seek(0)
-
-        # else read file from local file system
-        else:
-            try:
-                data = open(file, 'rb')
-            except Exception as e:
-                error = 'Could not load file, possible exception : {exception}'.format(exception = e)
-                log.error(error)
-                raise ValueError(error)
-
-
-        dialect = None
-
-        ############
-        # check for file type
-        ############
-
-        # try to guess if its an excel file
-        xlsx_sig = b'\x50\x4B\x05\06'
-        xlsx_sig2 = b'\x50\x4B\x03\x04'
-        xls_sig = b'\x09\x08\x10\x00\x00\x06\x05\x00'
-
-        # different whence, offset, size for different types
-        excel_meta = [ ('xls', 0, 512, 8), ('xlsx', 2, -22, 4)]
-
-        for filename, whence, offset, size in excel_meta:
-
-            try:
-                data.seek(offset, whence)  # Seek to the offset.
-                bytes = data.read(size)  # Capture the specified number of bytes.
-                data.seek(0)
-                codecs.getencoder('hex')(bytes)
-
-                if bytes == xls_sig:
-                    return data, 'xls', dialect
-                elif bytes == xlsx_sig:
-                    return data, 'xlsx', dialect
-
-            except:
-                data.seek(0)
-
-        # if not excel it can be a json file or a CSV, convert from binary to stringio
-
-        byte_str = data.read()
-        # Move it to StringIO
-        try:
-            # Handle Microsoft's BOM "special" UTF-8 encoding
-            if byte_str.startswith(codecs.BOM_UTF8):
-                data = StringIO(byte_str.decode('utf-8-sig'))
-            else:
-                data = StringIO(byte_str.decode('utf-8'))
-
-        except:
-            log.error(traceback.format_exc())
-            log.error('Could not load into string')
-
-        # see if its JSON
-        buffer = data.read(100)
-        data.seek(0)
-        text = buffer.strip()
-        # analyze first n characters
-        if len(text) > 0:
-            text = text.strip()
-            # it it looks like a json, then try to parse it
-            if text != "" and ((text[0] == "{") or (text[0] == "[")):
-                try:
-                    json.loads(data.read())
-                    data.seek(0)
-                    return data, 'json', dialect
-                except:
-                    data.seek(0)
-                    return data, None, dialect
-
-        # lets try to figure out if its a csv
-        try:
-            data.seek(0)
-            first_few_lines = []
-            i = 0
-            for line in data:
-                i += 1
-                first_few_lines.append(line)
-                if i > 0:
-                    break
-
-            accepted_delimiters = [',','\t']
-            dialect = csv.Sniffer().sniff(''.join(first_few_lines[0]), delimiters=accepted_delimiters)
-            data.seek(0)
-            # if csv dialect identified then return csv
-            if dialect:
-                return data, 'csv', dialect
-            else:
-                return data, None, dialect
-        except:
-            data.seek(0)
-            log.error('Could not detect format for this file')
-            log.error(traceback.format_exc())
-            # No file type identified
-            return data, None, dialect
-
-
-
-
-    def _setup(self,file, clean_rows = True, custom_parser = None):
-        """
-        Setup from file
-        :param file: fielpath or url
-        :param clean_rows:  if you want to clean rows for strange null values
-        :param custom_parser: if you want to parse the file with some custom parser
-        """
-
-        # get file data io, format and dialect
-        data, format, dialect = self._getDataIo(file)
-        data.seek(0) # make sure we are at 0 in file pointer
-
-        if format is None:
-            log.error('Could not laod file into any format, supported formats are csv, json, xls, xslx')
-
-        if custom_parser:
-            header, file_data = custom_parser(data, format)
-
-        elif format == 'csv':
-            csv_reader = list(csv.reader(data, dialect))
-            header = csv_reader[0]
-            file_data =  csv_reader[1:]
-
-        elif format in ['xlsx', 'xls']:
-            data.seek(0)
-            df = pandas.read_excel(data)
-            header = df.columns.values.tolist()
-            file_data = df.values.tolist()
-
-        elif format == 'json':
-            data.seek(0)
-            json_doc = json.loads(data.read())
-            df = json_normalize(json_doc)
-            header = df.columns.values.tolist()
-            file_data = df.values.tolist()
-
-        for col in header:
-            self._col_map[col] = col
-
-        if clean_rows == True:
-            file_list_data = []
-            for row in file_data:
-                row = self.cleanRow(row)
-                file_list_data.append(row)
-        else:
-            file_list_data = file_data
-
-        self.setDF(pandas.DataFrame(file_list_data, columns=header))

mindsdb/libs/data_types/data_source.py

@@ -1,59 +0,0 @@
-from mindsdb.libs.data_types.mindsdb_logger import log
-
-class DataSource:
-
-    def __init__(self, *args, **kwargs):
-        self._col_map = {} # you can store here if there were some columns renamed
-        self._setup(*args, **kwargs)
-        self.log = log
-
-    def _setup(self, df):
-        self._df = df
-
-    @property
-    def df(self):
-        return self._df
-
-    def setDF(self, df):
-        self._df = df
-
-    def dropColumns(self, column_list):
-        """
-        Drop columns by original names
-
-        :param column_list: a list of columns that you want to drop
-        :return: None
-        """
-
-        cols = [col if col not in self._col_map else self._col_map[col] for col in column_list]
-        self._df = self._df.drop(columns=cols)
-        
-    def __getattr__(self, item):
-        """
-        Map all other functions to the DataFrame
-
-        :param item: the attribute to get
-        :return: the dataframe attribute
-        """
-
-        return getattr(self._df, item)
-
-
-    def __getitem__(self, key):
-        """
-        Map all other items to the DataFrame
-
-        :param key: the key to get
-        :return: the dataframe attribute
-        """
-        return self._df[key]
-
-
-    def __setitem__(self, key, value):
-        """
-        Support item assignment, mapped ot dataframe
-        :param key:
-        :param value:
-        :return:
-        """
-        self._df[key] = value

mindsdb/libs/data_types/mindsdb_logger.py

@@ -1,127 +0,0 @@
-import pprint
-import logging
-import colorlog
-import uuid
-
-from mindsdb import CONFIG
-from mindsdb.libs.helpers.text_helpers import gen_chars
-from inspect import getframeinfo, stack
-
-
-class MindsdbLogger():
-    internal_logger = None
-    id = None
-
-    def __init__(self, log_level, uuid):
-        '''
-        # Initialize the log module, should only be called once at the begging of the program
-
-        :param log_level: What logs to display
-        :param uuid: The unique id for this MindsDB instance or training/prediction session
-        '''
-
-        self.id = uuid
-        self.internal_logger = logging.getLogger('mindsdb-logger-{}'.format(self.id))
-
-        self.internal_logger.handlers = []
-        self.internal_logger.propagate = False
-
-        stream_handler = logging.StreamHandler()
-        stream_handler.setFormatter(colorlog.ColoredFormatter('%(log_color)s%(levelname)s:%(name)s:%(message)s'))
-        self.internal_logger.addHandler(stream_handler)
-
-        self.internal_logger.setLevel(log_level)
-
-
-    def log_message(self, message, func):
-        '''
-        # Internal function used for logging, adds the id and caller to the log and prettifies the message
-
-        :param message: message that the logger shoud log
-        :param chracter: logger function to use (example: 'info' or 'error')
-        '''
-        caller = getframeinfo(stack()[2][0])
-        message = pprint.pformat(str(message))
-
-        call = getattr(self.internal_logger, func)
-        call("%s:%d - %s" % (caller.filename.split('mindsdb/')[-1], caller.lineno, message))
-
-    def debug(self, message):
-        self.log_message(message, 'debug')
-
-    def info(self, message):
-        self.log_message(message, 'info')
-
-    def warning(self, message):
-        self.log_message(message, 'warning')
-
-    def error(self, message):
-        self.log_message(message, 'error')
-
-    def infoChart(self, message, type, uid=None):
-        """
-        It will add the specific markdown plus tags or format it for stdout logs
-
-        :param message: its the chart payload
-        :param type: the type of chart
-        :param uid: the unique id of the chart so markdown can update properly
-
-        :return: None
-        """
-
-        if uid is None:
-            uid = str(uuid.uuid1())
-        else:
-            self.info(gen_chars(10, '-'))
-            if type in ['pie']:
-                total = sum([i[0] for i in message['subsets']]) if 'total' not in message else message['total'][0]
-                max_len = max([len(i[1]) for i in message['subsets']])
-                len_format = " {: >" + str(max_len) + "}: "
-
-                if 'label' in message:
-                    label = message['label']
-                    self.info('{label}'.format(label=label))
-                for i in message['subsets']:
-                    p = 100.0*i[0]/total
-                    l = int(p/5)
-                    info_str =len_format.format(i[1]) + "[{:-<20}]".format(gen_chars(l,'#')) + ' {val} ({p}% of Total)'.format(label=str(i[1]), val=i[0], p=format(p, '.2f'))
-                    self.info(info_str)
-                if 'total' in message:
-                    label = '{label} ({count})'.format(label=message['total'][1], count=message['total'][0])
-                    self.info(' {label}'.format(label=label))
-
-            if type in ['histogram']:
-
-                total = sum(message['y'])
-                max_len = max([len(str(i)) for i in message['x']])
-                len_format = " {: >" + str(max_len) + "}: "
-                max_val = max(message['y'])
-
-                if 'label' in message:
-                    label = message['label']
-                    self.info('{label}'.format(label=label))
-
-                for i,v in enumerate(message['y']):
-                    p = 100.0 * v / max_val
-                    prob = 100.0 *v /total
-                    l = int(p / 5)
-                    info_str = len_format.format(message['x'][i]) + "[{: <20}".format(
-                        gen_chars(l, '#')) + '  ({p}% likely)'.format(label=message['x'][i], p=format(prob, '.2f'))
-                    self.info(info_str)
-
-
-
-            elif type in ['list']:
-                max_len = max([len(i) for i in message.keys()])
-                len_format = " {: >" + str(max_len) + "}: "
-                for key in message:
-                    self.info(len_format.format(key) + '{val}'.format(val=message[key]))
-            else:
-                self.info(message)
-                self.info('info type: {type}'.format(type=type))
-            self.info(gen_chars(10, '-'))
-
-
-
-main_logger_uuid = 'core-logger'
-log = MindsdbLogger(log_level=CONFIG.DEFAULT_LOG_LEVEL, uuid=main_logger_uuid)

mindsdb/libs/data_types/probability_evaluation.py

@@ -1,56 +0,0 @@
-
-
-class ProbabilityEvaluation:
-
-    def __init__(self, buckets, evaluation_distribution = None, predicted_value = None, logger = None):
-
-        self.distribution = evaluation_distribution
-        self.predicted_value = predicted_value
-        self.buckets = buckets
-        self.most_likely_value = None
-        self.most_likely_probability = None
-        self.logger = logger
-
-        if evaluation_distribution is not None:
-            self.update(evaluation_distribution, predicted_value)
-
-    def explain(self):
-
-        data = {
-            'x': [i if type(i) == type('') else "{0:.2f}".format(i) for i in self.buckets],
-            'y': [i*100 for i in self.distribution],
-            'label': 'The probability distribution of the prediction'
-        }
-        if self.logger is not None:
-            self.logger.infoChart(data, type='histogram')
-
-
-
-    def update(self, distribution, predicted_value):
-        """
-        For a given distribution, update the most_likely_values and probability
-        :param distribution: the distribution values
-        :return:
-        """
-        self.predicted_value = predicted_value
-        self.distribution = distribution
-
-
-        # now that we have a distribution obtain the most likely value and its probability
-        max_prob_value = max(distribution)  # the highest probability in the distribution
-        max_prob_index = distribution.index(max_prob_value)  # the index for the highest probability
-        max_prob_bucket_left = self.buckets[
-            max_prob_index]  # the predicted value will fall in between the two ends of the bucket (if not a text)
-
-        if type(self.buckets[0]) == type(""):  # if we our buckets are text, then return the most likely label
-            most_likely_value = max_prob_bucket_left
-        else:  # else calcualte the value in between the buckets (optional future implementation: we can also calcualte a random value in between the buckets)
-            if max_prob_index >= len(
-                    self.buckets) - 1:  # if most likely is the far most right value in the buckets then dont average
-                most_likely_value = max_prob_bucket_left
-            else:
-                max_prob_bucket_right = self.buckets[max_prob_index + 1]
-                most_likely_value = max_prob_bucket_left + (max_prob_bucket_right - max_prob_bucket_left) / 2
-
-        self.most_likely_value = most_likely_value
-        self.most_likely_probability = max_prob_value