MindsDB 1.2.9__py3-none-any.whl → 24.12.4.0__py3-none-any.whl

mindsdb/libs/helpers/parser.py

@@ -1,161 +0,0 @@
-
-
-test = """
-
-SELECT DISTINCT c as cosa, col2,
-col3 as "year", col5 as "select", * FROM (
-  select * FROM schemab.my_teable WHERE   c = 10 AND col3 < 5 and col2 in (20, 30) Or ( c in (SELECT count(distinct names) from name_list where area = 1) and col2 >= 5 )
-) table_2
-
-where (col4 > 10   AND col5  lIKE '%noname''myname )'  ) OR ( col5 = '(select * Fom ) ()' AND cold3 = 2 )
-
-"""
-
-
-#first step replace '' for __ESC_QUOTES__
-
-# replace all text with __VAR_varindex__ build a map
-
-# replace all double space  or greater with single space
-# replace all '( ' and ' )' with '(' ')'
-# upper case all reserved words that are not enclosed by ""
-
-# replace each sub statement with  __SUB:INDEX.child.grandchild..___ build a map
-
-
-
-def replaceTexts(str):
-    """
-    The whole point of this function is to
-    replace strings with __TEXTVAR_index__ and return a variable map dictionary
-
-    :param str: the string that we want to make replacements on
-    :return: replaced_str, text_var_map
-    """
-
-    # these are the variables to be returned
-    text_var_map = {}
-    ret = ''
-
-
-    inside = False # as we walk the string, if this is true is because we are inside a text variable
-    str_len = len(str)
-    collected_text = '' # as we find one text variable we collect it here
-    text_var_count = 0 # the variable count
-    skip_next = False
-
-    # here we walk oall the string in search for ' and store its contents replace the string and update the map
-    for i,c in enumerate(str):
-
-        if skip_next == True:
-            skip_next = False
-            continue
-
-        next = '' if i>=(str_len-1) else str[i+1]
-        if c == "'" and not inside:
-            inside = True
-            text_var_count +=1
-            continue
-        if c == "'" and inside and next!="'":
-            inside = False
-            map_key = '__TEXTVAR_{text_var_count}__'.format(text_var_count=text_var_count)
-            text_var_map[map_key] = collected_text
-            collected_text = ''
-            ret += map_key
-            continue
-        elif c == "'" and inside and next=="'":
-            skip_next = True
-            collected_text += "'"
-
-        if inside == True:
-            collected_text += c
-        else:
-            ret += c
-
-    return ret, text_var_map
-
-
-def cleanStr(str):
-    """
-    Do some cleaning, remove double spaces new lines, clean commas and others
-    :param str:
-    :return:
-    """
-    str = str.replace("\n", ' ')
-
-    clean_space_from = ['( ', ' )', ', ', '[ ', ' ]']
-    for str_to_replace in clean_space_from:
-        actual = str_to_replace.replace(' ', '')
-        str = str.replace(actual, str_to_replace)
-
-    str = ' (' + str + ') '
-    str = ' '.join(str.split())
-
-    return str
-
-def replaceSubStatements(str, count = 0, node='0'):
-
-    ret = ''
-
-    ret_map = {}
-
-    for i, c in enumerate(str):
-
-        if c == '(':
-            count += 1
-            subnode = '.'.join(node.split('.')[-1])
-            node = '{node}.{count}'.format(node=subnode,count=count)
-
-            node_str, count,  map = replaceSubStatements(str[i+1:], count, node)
-            ret_map[node] = map
-
-            ret += '__NODE:{node}__'.format(node=node)
-
-            break
-
-        if c == ')':
-            count -= 1
-
-            return ret, count, ret_map
-
-
-        ret += c
-
-    return ret, count, ret_map
-
-
-
-def parse(str):
-
-    str, text_var_map = replaceTexts(str)
-    str = cleanStr(str)
-
-
-    log.info(str)
-
-
-'''
- ( select a from ( select u from b ) where u > 10 )
-'''
-
-
-
-map = {
-
-    'expr': '__0__',
-    'parts': [
-        {
-            'expr': ' select a from __0__ where u > 10 ',
-            'parts': [
-                {
-                    'expr': ' select u from b ',
-                    'parts': []
-                }
-            ]
-        }
-    ]
-}
-
-
-
-parse(test)

mindsdb/libs/helpers/probabilistic_validator.py

@@ -1,198 +0,0 @@
-from mindsdb.libs.constants.mindsdb import *
-from mindsdb.libs.data_types.probability_evaluation import ProbabilityEvaluation
-from mindsdb.libs.helpers.general_helpers import get_value_bucket
-
-from sklearn.naive_bayes import GaussianNB, ComplementNB, MultinomialNB
-import numpy as np
-
-
-class ProbabilisticValidator():
-    """
-    # The probabilistic validator is a quick to train model used for validating the predictions of our main model
-    # It is fit to the results our model gets on the validation set
-    """
-    _smoothing_factor = 0.5 # TODO: Autodetermine smotthing factor depending on the info we know about the dataset
-    _value_bucket_probabilities = {}
-    _probabilistic_model = None
-    X_buff = None
-    Y_buff = None
-
-
-    def __init__(self, col_stats, data_type=None):
-        """
-        Chose the algorithm to use for the rest of the model
-        As of right now we go with ComplementNB
-        """
-        # <--- Pick one of the 3
-        self._probabilistic_model = ComplementNB(alpha=self._smoothing_factor)
-        #, class_prior=[0.5,0.5]
-        #self._probabilistic_model = GaussianNB()
-        #self._probabilistic_model = MultinomialNB(alpha=self._smoothing_factor)
-        self.X_buff = []
-        self.Y_buff = []
-
-        self.col_stats = col_stats
-
-        if 'percentage_buckets' in col_stats:
-            self.buckets = col_stats['percentage_buckets']
-            self.bucket_keys = [i for i in range(len(self.buckets))]
-        else:
-            self.buckets = None
-
-        self.data_type = col_stats['data_type']
-
-        self.bucket_accuracy = {
-
-        }
-
-    def register_observation(self, features_existence, real_value, predicted_value):
-        """
-        # Register an observation in the validator's internal buffers
-
-        :param features_existence: A vector of 0 and 1 representing the existence of all the features (0 == not exists, 1 == exists)
-        :param real_value: The real value/label for this prediction
-        :param predicted_value: The predicted value/label
-        :param histogram: The histogram for the predicted column, which allows us to bucketize the `predicted_value` and `real_value`
-        """
-
-        predicted_value = predicted_value if self.data_type != DATA_TYPES.NUMERIC else float(predicted_value)
-        try:
-            real_value = real_value if self.data_type != DATA_TYPES.NUMERIC else float(str(real_value).replace(',','.'))
-        except:
-            real_value = None
-
-        if self.buckets is not None:
-            predicted_value_b = get_value_bucket(predicted_value, self.buckets, self.col_stats)
-            real_value_b = get_value_bucket(real_value, self.buckets, self.col_stats)
-            X = [False] * (len(self.buckets) + 1)
-            X[predicted_value_b] = True
-            X = X + features_existence
-            self.X_buff.append(X)
-
-            self.Y_buff.append(real_value_b)
-
-            # If no column is ignored, compute the accuracy for this bucket
-            nr_missing_features = len([x for x in features_existence if x is False or x is 0])
-            if nr_missing_features == 0:
-                if predicted_value_b not in self.bucket_accuracy:
-                    self.bucket_accuracy[predicted_value_b] = []
-                self.bucket_accuracy[predicted_value_b].append(int(real_value_b == predicted_value_b))
-        else:
-            predicted_value_b = predicted_value
-            real_value_b = real_value
-            self.X_buff.append(features_existence)
-            self.Y_buff.append(real_value_b == predicted_value_b)
-
-    def get_accuracy_histogram(self):
-        x = []
-        y = []
-
-        total_correct = 0
-        total_vals = 0
-
-        for bucket in self.bucket_accuracy:
-            total_correct += sum(self.bucket_accuracy[bucket])
-            total_vals += len(self.bucket_accuracy[bucket])
-            x.append(bucket)
-            y.append(sum(self.bucket_accuracy[bucket])/len(self.bucket_accuracy[bucket]))
-
-        validation_set_accuracy = total_correct/total_vals
-        return {
-            'buckets': x
-            ,'accuracies': y
-        }, validation_set_accuracy
-
-
-    def partial_fit(self):
-        """
-        # Fit the probabilistic validator on all observations recorder that haven't been taken into account yet
-        """
-        log_types = np.seterr()
-        np.seterr(divide='ignore')
-
-        if self.buckets is not None:
-            self._probabilistic_model.partial_fit(self.X_buff, self.Y_buff, classes=self.bucket_keys)
-        else:
-            self._probabilistic_model.partial_fit(self.X_buff, self.Y_buff, classes=[True, False])
-
-        np.seterr(divide=log_types['divide'])
-
-        self.X_buff= []
-        self.Y_buff= []
-
-    def fit(self):
-        """
-        # Fit the probabilistic validator on all observations recorder that haven't been taken into account yet
-        """
-        log_types = np.seterr()
-        np.seterr(divide='ignore')
-        self._probabilistic_model.fit(self.X_buff, self.Y_buff)
-        np.seterr(divide=log_types['divide'])
-
-        self.X_buff= []
-        self.Y_buff= []
-
-    def evaluate_prediction_accuracy(self, features_existence, predicted_value):
-        """
-        # Fit the probabilistic validator on an observation    def evaluate_prediction_accuracy(self, features_existence, predicted_value):
-        :param features_existence: A vector of 0 and 1 representing the existence of all the features (0 == not exists, 1 == exists)
-        :param predicted_value: The predicted value/label
-        :return: The probability (from 0 to 1) of our prediction being accurate (within the same histogram bucket as the real value)
-        """
-        if self.buckets is not None:
-            predicted_value_b = get_value_bucket(predicted_value, self.buckets, self.col_stats)
-            X = [False] * (len(self.buckets) + 1)
-            X[predicted_value_b] = True
-            X = [X + features_existence]
-        else:
-            X = [features_existence]
-
-        log_types = np.seterr()
-        np.seterr(divide='ignore')
-        distribution = self._probabilistic_model.predict_proba(np.array(X))
-        np.seterr(divide=log_types['divide'])
-
-        if self.buckets is not None:
-            return ProbabilityEvaluation(self.buckets, distribution[0].tolist(), predicted_value)
-        else:
-            return distribution[0][1]
-
-
-
-if __name__ == "__main__":
-    import random
-
-    values = [2,2,2,3,5,2,2,2,3,5]
-    predictions = [2,2,2,3,2,2,2,2,3,2]
-
-    feature_rows = [
-        [bool(random.getrandbits(1)), bool(random.getrandbits(1)), bool(random.getrandbits(1))]
-        for i in values
-    ]
-
-    pbv = ProbabilisticValidator(buckets=[1,2,3,4,5])
-
-    for i in range(len(feature_rows)):
-        pbv.register_observation(feature_rows[i],values[i], predictions[i])
-
-    pbv.partial_fit()
-    print(pbv.evaluate_prediction_accuracy([True,True,True], 2))
-
-    # Now test text tokens
-    values = ['2', '2', '2', '3', '5', '2', '2', '2', '3', '5']
-    predictions = ['2', '2', '2', '3', '2', '2', '2', '2', '3', '2']
-
-    feature_rows = [
-        [bool(random.getrandbits(1)), bool(random.getrandbits(1)), bool(random.getrandbits(1))]
-        for i in values
-    ]
-
-    print(feature_rows)
-
-    pbv = ProbabilisticValidator(buckets=['1', '2', '3', '4', '5'], data_type=DATA_TYPES.CATEGORICAL)
-
-    for i in range(len(feature_rows)):
-        pbv.register_observation(feature_rows[i], values[i], predictions[i])
-
-    pbv.partial_fit()
-    print(pbv.evaluate_prediction_accuracy([True, True, True], '2'))

mindsdb/libs/helpers/sqlite_helpers.py

@@ -1,33 +0,0 @@
-
-import json
-
-
-
-
-class FirstValueAgg:
-    def __init__(self):
-        self.first = None
-
-    def step(self, value):
-        if self.first == None:
-            self.first=value
-
-    def finalize(self):
-        return self.first
-
-class ArrayAggJSON:
-    def __init__(self):
-        self.array = []
-        self.limit = 80
-
-    def step(self, value, limit):
-        self.array += [value]
-        self.limit = limit
-
-    def finalize(self):
-
-        arr =  self.array[-self.limit or None: ]
-        if len(arr) < self.limit:
-            diff_arra = [None]*(self.limit-len(arr))
-            arr = diff_arra + arr
-        return json.dumps(arr)

mindsdb/libs/helpers/text_helpers.py

@@ -1,79 +0,0 @@
-"""
-*******************************************************
- * Copyright (C) 2017 MindsDB Inc. <copyright@mindsdb.com>
- *
- * This file is part of MindsDB Server.
- *
- * MindsDB Server can not be copied and/or distributed without the express
- * permission of MindsDB Inc
- *******************************************************
-"""
-
-from mindsdb.libs.constants.mindsdb import *
-import json
-import hashlib
-from dateutil.parser import parse as parse_datetime
-import numpy
-
-def clean_float(val):
-    if type(val) in [type(int(1)), type(1.0)] :
-        return float(val)
-
-    if isinstance(val, numpy.float64) or isinstance(val, float) or isinstance(val, int):
-        return val
-
-    val = str(val)
-    val = val.replace(',','.')
-    val = val.rstrip('"').lstrip('"')
-
-    if val == '' or val == 'None':
-        return None
-
-    return float(val)
-
-
-def gen_chars(length, character):
-    """
-    # lambda to Generates a string consisting of `length` consiting of repeating `character`
-    :param length:
-    :param character:
-    :return:
-    """
-    return ''.join([character for i in range(length)])
-
-def cast_string_to_python_type(string):
-    """ Returns an integer, float or a string from a string"""
-    try:
-        if string is None:
-            return None
-        return int(string)
-    except:
-        try:
-            return clean_float(string)
-        except ValueError:
-            if string == '':
-                return None
-            else:
-                return string
-
-def splitRecursive(word, tokens):
-    words = [str(word)]
-    for token in tokens:
-        new_split = []
-        for word in words:
-            new_split += word.split(token)
-        words = new_split
-    words = [word for word in words if word not in ['', None] ]
-    return words
-
-def hashtext(cell):
-    text = json.dumps(cell)
-    hash = hashlib.md5(text.encode('utf8')).hexdigest()
-    return hash
-
-def test():
-    log.info(splitRecursive('ABC.C HELLO, one:123.45 67', WORD_SEPARATORS))
-
-# only run the test if this file is called from debugger
-if __name__ == "__main__":
-    test()

mindsdb/libs/helpers/train_helpers.py

@@ -1,4 +0,0 @@
-def getOneColPermutations(possible_columns):
-    permutations = {col: 1 for col in possible_columns}
-    ret = [perm.split(':') for perm in list(permutations.keys())]
-    return ret

mindsdb/libs/phases/base_module.py

@@ -1,62 +0,0 @@
-"""
-*******************************************************
- * Copyright (C) 2017 MindsDB Inc. <copyright@mindsdb.com>
- *
- * This file is part of MindsDB Server.
- *
- * MindsDB Server can not be copied and/or distributed without the express
- * permission of MindsDB Inc
- *******************************************************
-"""
-
-"""
-This class works as an interface to code any module within MindsDB
-We use this to standarize the way that code parts of what makes mindsDB in a way such that those can be replaced so long the interfaces remain
-
-The principle is very simple, all you need within the module is set in the method __call__
-
-whatever you pass to call is the input and whatever it returns is the ourput of the module
-
-The interesting part is that in a module you should not instantiate data sources or any global resources
-Those should be available via self.session
-
-"""
-
-from mindsdb.libs.constants.mindsdb import *
-import time
-
-
-class BaseModule():
-    def __init__(self, session, transaction, **kwargs):
-        '''
-        Initialize the base module and the basic global variables
-        :param session: the session under which this transaction is happening
-        :type transaction: libs.controllers.transaction_controller.TransactionController
-        :param transaction: the transaction under which this phases is being called
-        :param kwargs: extra arguments passed to run
-        '''
-        self.kwargs = kwargs
-        self.session = session
-        self.transaction = transaction # type: controllers.transaction_controller.TransactionController
-        self.output = {}
-        self.setup(**kwargs)
-        self.log = self.transaction.log
-
-    def run(self):
-        pass
-
-    def __call__(self, **kwargs):
-        start = time.time()
-        class_name = type(self).__name__
-
-        self.log.info('[START] {class_name}'.format(class_name=class_name))
-
-        ret = self.run(**kwargs)
-        execution_time = time.time() - start
-
-        self.log.info('[END] {class_name}, execution time: {execution_time:.3f} seconds'.format(class_name=class_name, execution_time=execution_time))
-        return ret
-
-    def setup(self, **kwargs):
-        # This is to be implemented by the child classes
-        pass