MindsDB 1.3.1__py3-none-any.whl → 25.1.2.0__py3-none-any.whl

mindsdb/libs/data_types/transaction_data.py

@@ -1,15 +0,0 @@
-
-class TransactionData():
-
-    def __init__(self):
-        self.data_frame = None
-        self.train_df = None
-        self.test_df = None
-        self.validation_df = None
-
-        self.train_indexes = {}
-        self.test_indexes = {}
-        self.validation_indexes = {}
-        self.all_indexes = {}
-
-        self.columns = []

mindsdb/libs/data_types/transaction_output_data.py

@@ -1,29 +0,0 @@
-from mindsdb.libs.constants.mindsdb import *
-
-from mindsdb.libs.data_types.mindsdb_logger import log
-from mindsdb.libs.data_types.transaction_output_row import TransactionOutputRow
-
-class TrainTransactionOutputData():
-    def __init__(self):
-        self.data_frame = None
-        self.columns = None
-
-
-class PredictTransactionOutputData():
-    def __init__(self, transaction, data, evaluations):
-        self.data = data
-        self.evaluations = evaluations
-        self.transaction = transaction
-
-    def __iter__(self):
-        for i, value in enumerate(self.data[self.transaction.lmd['columns'][0]]):
-            yield TransactionOutputRow(self, i)
-
-    def __getitem__(self, item):
-        return TransactionOutputRow(self, item)
-
-    def __str__(self):
-        return str(self.data)
-
-    def __len__(self):
-        return len(self.data[self.transaction.lmd['columns'][0]])

mindsdb/libs/data_types/transaction_output_row.py

@@ -1,35 +0,0 @@
-from mindsdb.libs.helpers.explain_prediction import explain_prediction
-
-
-class TransactionOutputRow:
-    def __init__(self, transaction_output, row_index):
-        self.transaction_output = transaction_output
-        self.row_index = row_index
-
-    def __getitem__(self, item):
-        return self.transaction_output.data[item][self.row_index]
-
-    def __contains__(self, item):
-        return item in self.transaction_output.data.keys()
-
-    def explain(self):
-        prediction_row = {col: self.transaction_output.data[col][self.row_index] for col in list(self.transaction_output.data.keys())}
-        #self.transaction_output.data.iloc[self.row_index]
-        return explain_prediction(self.transaction_output.transaction.lmd, prediction_row)
-
-    def why(self): return self.explain()
-
-    def __str__(self):
-        return str(self.as_dict())
-
-
-    def as_dict(self):
-        return {key: self.transaction_output.data[key][self.row_index] for key in list(self.transaction_output.data.keys())}
-
-    def as_list(self):
-        #Note that here we will not output the confidence columns
-        return [self.transaction_output.data[col][self.row_index] for col in list(self.transaction_output.data.keys())]
-
-    @property
-    def _predicted_values(self):
-        return {pred_col:self.transaction_output.evaluations[pred_col][self.row_index].predicted_value for pred_col in self.transaction_output.evaluations}

mindsdb/libs/helpers/explain_prediction.py

@@ -1,114 +0,0 @@
-import numpy as np
-from mindsdb.libs.constants.mindsdb import *
-
-
-def explain_prediction(lmd, prediction_row):
-    '''
-        Given a row contianing a prediction, it tries to use the information in it plus the metadata generated by the ModelAnalyzer and StatsGenerator in order to explain "why" the prediction took said value.
-        :param prediction_row: The row that was predicted by the model backend and processed by mindsdb
-        :return: A, hopefully human readable, string containing the explaination
-    '''
-    top_20_val = np.percentile(list(lmd['column_importances'].values()),80)
-    bottom_20_val = np.percentile(list(lmd['column_importances'].values()),20)
-
-    important_cols = [col for col in lmd['column_importances'] if lmd['column_importances'][col] >= top_20_val]
-    useless_cols = [col for col in lmd['column_importances'] if lmd['column_importances'][col] <= bottom_20_val]
-
-    explain_predictions = {}
-
-    for pcol in lmd['predict_columns']:
-        predicted_val = prediction_row[pcol]
-        col_stats = lmd['column_stats'][pcol]
-        col_type = col_stats['data_type']
-
-        if 'histogram' in col_stats:
-            histogram_x = None
-            histogram_keys = col_stats['histogram']['x']
-            if col_type == DATA_TYPES.NUMERIC:
-                for i in range(len(histogram_keys)):
-                    if i == 0:
-                        if histogram_keys[i] >= predicted_val:
-                            histogram_x = histogram_keys[0]
-                    if i == len(histogram_keys) - 1:
-                            if histogram_keys[i] <= predicted_val:
-                                histogram_x = histogram_keys[0]
-                    else:
-                        if histogram_keys[i] <= predicted_val or predicted_val > histogram_keys[i+1]:
-                            histogram_x = histogram_keys[0]
-            else:
-                histogram_x = predicted_val
-
-            bucket_occurances = col_stats['histogram']['y'][col_stats['histogram']['x'].index(histogram_x)]
-            total_occuraces = sum(col_stats['histogram']['y'])
-
-            percentage_bucket_percentage = round(100*bucket_occurances/total_occuraces, 2)
-            column_confidence = prediction_row[f'{pcol}_confidence']
-
-            confidence_str = 'very confident'
-            if column_confidence < 80:
-                confidence_str = 'confident'
-            if column_confidence < 60:
-                confidence_str = 'somewhat confident'
-            if column_confidence < 40:
-                confidence_str = 'not very confident'
-            if column_confidence < 20:
-                confidence_str = 'not confident'
-
-            if percentage_bucket_percentage < 2:
-                column_explaination = f'A similar value for the predicted column {pcol} occurs rarely in your dataset'
-
-                if column_confidence >= 70:
-                    column_explaination += ', in spite of this, due to the quality of the input data and the model, we are very confident this prediction is correct.'
-
-                if column_confidence < 70 and column_confidence > 30:
-                    column_explaination += ', it is partially because of this reason that we are only somewhat confident this prediction is correct.'
-
-                if column_confidence <= 30:
-                    column_explaination += ', it is partially because of this reason that we aren\'t confident this prediction is correct.'
-
-            elif percentage_bucket_percentage < 12.5:
-                column_explaination = f'A similar value for the predicted column {pcol} occurs a moderate amount of times in your dataset'
-
-                if column_confidence >= 70:
-                    column_explaination += ', we are very confident this prediction is correct.'
-
-                if column_confidence < 70 and column_confidence > 30:
-                    column_explaination += ', we are only somewhat confident this prediction is correct.'
-
-                if column_confidence <= 30:
-                    column_explaination += ', you\'r input data might be of sub-par qualkity, since we aren\'t confident this prediction is correct.'
-
-            else:
-                column_explaination = f'A similar value for the predicted column {pcol} occurs very often in your dataset'
-
-                if column_confidence >= 70:
-                    column_explaination += ', it\'s partially because of this plaethora of examples that we can be very confident this prediction is correct.'
-
-                if column_confidence < 70 and column_confidence > 30:
-                    column_explaination += ', still we can only be somewhat confident this prediction is correct.'
-
-                if column_confidence <= 30:
-                    column_explaination += ', in spite of this, possibly due to poor input data quality, we aren\'t confident this prediction is correct.'
-
-            explain_predictions[pcol] = column_explaination
-
-        else:
-            explain_predictions[pcol] = f'Column {pcol} is of type {col_type} and thus it\'s impossible for us to make statistical inferences about it.'
-
-        explain_inputs = {}
-        for icol in important_cols:
-            if prediction_row[icol] == None:
-                explain_inputs[icol] = f'The column {icol} is very important for this model to predict correctly. Since it\'s missing it\'s quite likely this prediction is a bit of wild guess.'
-            else:
-                explain_inputs[icol] = f'The value of the column {icol} played a large role in generating this prediction.'
-
-        for icol in useless_cols:
-            if prediction_row[icol] == None:
-                explain_inputs[icol] = f'The fact that {icol} is missing is probably not very relevant for this prediction.'
-            else:
-                explain_inputs[icol] = f'The column {icol} is probably not very relevant for this prediction.'
-
-        join_str = '\n\n*'
-        explaination = join_str + join_str.join(explain_predictions.values())
-        explaination += join_str + join_str.join(explain_inputs.values())
-    return explaination

mindsdb/libs/helpers/file_helpers.py

@@ -1,66 +0,0 @@
-"""
-*******************************************************
- * Copyright (C) 2017 MindsDB Inc. <copyright@mindsdb.com>
- *
- * This file is part of MindsDB Server.
- *
- * MindsDB Server can not be copied and/or distributed without the express
- * permission of MindsDB Inc
- *******************************************************
-"""
-
-import csv
-import sys
-import traceback
-
-from mindsdb.libs.data_types.mindsdb_logger import log
-
-
-def fixFileIfPossible(filepath):
-    """
-    Tries to fix a file header if it finds header or encoding issues
-    :param filepath: the filepath to fix if possible
-    :return: fixed, error
-    """
-    fixed = False
-    error = False
-    rows = []
-    try:
-        with open(filepath, newline='') as f:
-            reader = csv.reader(f)
-            header = None
-            max_len = 0
-            for row in reader:
-                if header is None:
-                    header = row
-                    for i, col in enumerate(row):
-                        if col in [None, '']:
-                            fixed = True
-                            header[i] = 'col_{i}'.format(i=i+1)
-                rows += [row]
-                length = int(len(row))
-                if length > max_len:
-                    max_len = length
-                    log.info(max_len)
-    except:
-        exc_type, exc_value, exc_traceback = sys.exc_info()
-        error = traceback.format_exception(exc_type, exc_value,
-                                           exc_traceback)
-        return fixed, error
-    if len(header) < max_len or fixed == True:
-        rightCell = lambda h, i: 'col_{i}'.format(i=i+1) if i > len(header) else h
-        row = [rightCell(header_col, i) for i, header_col in enumerate(header)]
-        rows[0] = row
-
-        with open(filepath, 'w', newline='') as f:
-            writer = csv.writer(f)
-            writer.writerows(rows)
-
-    return fixed, error
-
-def test():
-    log.info(fixFileIfPossible('/Users/jorge/Downloads/tweets (1).csv'))
-
-# only run the test if this file is called from debugger
-if __name__ == "__main__":
-    test()

mindsdb/libs/helpers/general_helpers.py

@@ -1,276 +0,0 @@
-import platform
-import re
-import urllib
-import uuid
-from pathlib import Path
-import pickle
-import requests
-from contextlib import contextmanager
-import ctypes
-import io
-import os, sys
-import tempfile
-
-
-from mindsdb.__about__ import __version__
-from mindsdb.config import CONFIG
-from mindsdb.libs.data_types.mindsdb_logger import log
-from mindsdb.libs.constants.mindsdb import *
-import imagehash
-from PIL import Image
-
-
-def get_key_for_val(key, dict_map):
-    for orig_col in dict_map:
-        if dict_map[orig_col] == key:
-            return orig_col
-
-    return key
-
-def convert_cammelcase_to_snake_string(cammel_string):
-    """
-    Converts snake string to cammelcase
-
-    :param cammel_string: as described
-    :return: the snake string AsSaid -> as_said
-    """
-
-    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', cammel_string)
-    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
-
-
-def get_label_index_for_value(value, labels):
-    """
-    This will define what index interval does a value belong to given a list of labels
-
-    :param value: value to evaluate
-    :param labels: the labels to compare against
-    :return: an index value
-    """
-    if  value is None:
-        return 0 # Todo: Add another index for None values
-
-    if type(value) == type(''):
-        try:
-            index = labels.index(value)
-        except:
-            index = 0
-    else:
-        for index, compare_to in enumerate(labels):
-            if value < compare_to:
-                index = index
-                break
-
-    return index
-
-def convert_snake_to_cammelcase_string(snake_str, first_lower = False):
-    """
-    Converts snake to cammelcase
-
-    :param snake_str: as described
-    :param first_lower: if you want to start the string with lowercase as_said -> asSaid
-    :return: cammelcase string
-    """
-
-    components = snake_str.split('_')
-
-    if first_lower == True:
-        # We capitalize the first letter of each component except the first one
-        # with the 'title' method and join them together.
-        return components[0] + ''.join(x.title() for x in components[1:])
-    else:
-        return ''.join(x.title() for x in components)
-
-def check_for_updates():
-    """
-    Check for updates of mindsdb
-    it will ask the mindsdb server if there are new versions, if there are it will log a message
-
-    :return: None
-    """
-
-    # tmp files
-    uuid_file = CONFIG.MINDSDB_STORAGE_PATH + '/../uuid.mdb_base'
-    mdb_file = CONFIG.MINDSDB_STORAGE_PATH + '/start.mdb_base'
-
-    uuid_file_path = Path(uuid_file)
-    if uuid_file_path.is_file():
-        uuid_str = open(uuid_file).read()
-    else:
-        uuid_str = str(uuid.uuid4())
-        try:
-            open(uuid_file, 'w').write(uuid_str)
-        except:
-            log.warning('Cannot store token, Please add write permissions to file:' + uuid_file)
-            uuid_str = uuid_str + '.NO_WRITE'
-
-    file_path = Path(mdb_file)
-    if file_path.is_file():
-        token = open(mdb_file).read()
-    else:
-        token = '{system}|{version}|{uid}'.format(system=platform.system(), version=__version__, uid=uuid_str)
-        try:
-            open(mdb_file,'w').write(token)
-        except:
-            log.warning('Cannot store token, Please add write permissions to file:'+mdb_file)
-            token = token+'.NO_WRITE'
-    extra = urllib.parse.quote_plus(token)
-    try:
-        r = requests.get('http://mindsdb.com/updates/check/{extra}'.format(extra=extra), headers={'referer': 'http://check.mindsdb.com/?token={token}'.format(token=token)})
-    except:
-        log.warning('Could not check for updates')
-        return
-    try:
-        # TODO: Extract version, compare with version in version.py
-        ret = r.json()
-
-        if 'version' in ret and ret['version']!= __version__:
-            pass
-            #log.warning("There is a new version of MindsDB {version}, please do:\n    pip3 uninstall mindsdb\n    pip3 install mindsdb --user".format(version=ret['version']))
-        else:
-            log.debug('MindsDB is up to date!')
-
-    except:
-        log.warning('could not check for MindsDB updates')
-
-def pickle_obj(object_to_pickle):
-    """
-    Returns a version of self that can be serialized into mongodb or tinydb
-    :return: The data of an object serialized via pickle and decoded as a latin1 string
-    """
-
-    return pickle.dumps(object_to_pickle,protocol=pickle.HIGHEST_PROTOCOL).decode(encoding='latin1')
-
-
-def unpickle_obj(pickle_string):
-    """
-    :param pickle_string: A latin1 encoded python str containing the pickle data
-    :return: Returns an object generated from the pickle string
-    """
-    return pickle.loads(pickle_string.encode(encoding='latin1'))
-
-
-def closest(arr, value):
-    """
-    :return: The index of the member of `arr` which is closest to `value`
-    """
-
-    for i,ele in enumerate(arr):
-        if value == None:
-            return -1
-        value = float(str(value).replace(',','.'))
-        if ele > value:
-            return i - 1
-
-    return len(arr)-1
-
-
-def get_value_bucket(value, buckets, col_stats):
-    """
-    :return: The bucket in the `histogram` in which our `value` falls
-    """
-    if buckets is None:
-        return None
-
-    if col_stats['data_subtype'] in (DATA_SUBTYPES.SINGLE, DATA_SUBTYPES.MULTIPLE):
-        if value in buckets:
-            bucket = buckets.index(value)
-        else:
-            bucket = len(buckets) # for null values
-
-    elif col_stats['data_subtype'] in (DATA_SUBTYPES.BINARY, DATA_SUBTYPES.INT, DATA_SUBTYPES.FLOAT):
-        bucket = closest(buckets, value)
-    elif col_stats['data_subtype'] in (DATA_SUBTYPES.IMAGE):
-        bucket = self.hmd['bucketing_algorithms'][col_name].predict(np.array(imagehash.phash(Image.open(value)).reshape(1, -1)))[0]
-    else:
-        bucket = len(buckets) # for null values
-
-    return bucket
-
-
-def evaluate_accuracy(predictions, full_dataset, col_stats, output_columns):
-    score = 0
-    for output_column in output_columns:
-        cummulative_scores = 0
-        if 'percentage_buckets' in col_stats[output_column]:
-            buckets = col_stats[output_column]['percentage_buckets']
-        else:
-            buckets = None
-
-        i = 0
-        for real_value in full_dataset[output_column]:
-            pred_val_bucket = get_value_bucket(predictions[output_column][i], buckets, col_stats[output_column])
-            if pred_val_bucket is None:
-                if predictions[output_column][i] == real_value:
-                    cummulative_scores += 1
-            elif pred_val_bucket == get_value_bucket(real_value, buckets, col_stats[output_column]):
-                cummulative_scores += 1
-            i += 1
-
-        score += cummulative_scores/len(predictions[output_column])
-    score = score/len(output_columns)
-    if score == 0:
-        score = 0.00000001
-    return score
-
-
-class suppress_stdout_stderr(object):
-    def __init__(self):
-        try:
-            crash = 'fileno' in dir(sys.stdout)
-            # Open a pair of null files
-            self.null_fds =  [os.open(os.devnull,os.O_RDWR) for x in range(2)]
-            # Save the actual stdout (1) and stderr (2) file descriptors.
-            self.c_stdout = sys.stdout.fileno() #int(ctypes.c_void_p.in_dll(ctypes.CDLL(None), 'stdout'))
-            self.c_stderr = sys.stderr.fileno() #int(ctypes.c_void_p.in_dll(ctypes.CDLL(None), 'stderr'))
-
-            self.save_fds = [os.dup(self.c_stdout), os.dup(self.c_stderr)]
-        except:
-            print('Can\'t disable output on Jupyter notebook')
-
-    def __enter__(self):
-        try:
-            crash = 'dup2' in dir(os)
-            # Assign the null pointers to stdout and stderr.
-            os.dup2(self.null_fds[0],self.c_stdout)
-            os.dup2(self.null_fds[1],self.c_stderr)
-        except:
-            print('Can\'t disable output on Jupyter notebook')
-
-    def __exit__(self, *_):
-        try:
-            crash = 'dup2' in dir(os)
-            # Re-assign the real stdout/stderr back to (1) and (2)
-            os.dup2(self.save_fds[0],self.c_stdout)
-            os.dup2(self.save_fds[1],self.c_stderr)
-            # Close all file descriptors
-            for fd in self.null_fds + self.save_fds:
-                os.close(fd)
-        except:
-            print('Can\'t disable output on Jupyter notebook')
-
-def get_tensorflow_colname(col):
-    replace_chars = """ ,./;'[]!@#$%^&*()+{-=+~`}\\|:"<>?"""
-
-    for char in replace_chars:
-        col = col.replace(char,'_')
-    col = re.sub('_+','_',col)
-
-    return col
-
-@contextmanager
-# @TODO: Make it work with mindsdb logger/log levels... maybe
-def disable_console_output(activate=True):
-    try:
-        try:
-            old_tf_loglevel = os.environ['TF_CPP_MIN_LOG_LEVEL']
-        except:
-            old_tf_loglevel = '2'
-        # Maybe get rid of this to not supress all errors and stdout
-        if activate:
-            with suppress_stdout_stderr():
-                yield
-        else:
-            yield
-    finally:
-        os.environ['TF_CPP_MIN_LOG_LEVEL'] = old_tf_loglevel

mindsdb/libs/helpers/multi_data_source.py

@@ -1,28 +0,0 @@
-from mindsdb.libs.data_types.data_source import DataSource
-from mindsdb.libs.data_sources.file_ds import FileDS
-from pandas import DataFrame
-
-from mindsdb.libs.data_types.mindsdb_logger import log
-
-
-def getDS(from_data):
-    '''
-    Get a datasource give the input
-    :param input: a string or an object
-    :return: a datasource
-
-    '''
-
-    if isinstance(from_data, DataSource):
-        from_ds = from_data
-
-    elif isinstance(from_data, DataFrame):
-        from_ds = DataSource(from_data)
-
-
-    else:  # assume is a file
-        from_ds = FileDS(from_data)
-        if from_ds is None:
-            log.error('No data matched the input data')
-
-    return from_ds