MindsDB 1.3.1__py3-none-any.whl → 25.1.2.0__py3-none-any.whl

mindsdb/utilities/fs.py

@@ -0,0 +1,153 @@
+import os
+import time
+import tempfile
+import threading
+from pathlib import Path
+from typing import Optional, List, Tuple
+
+import psutil
+
+from mindsdb.utilities import log
+
+logger = log.getLogger(__name__)
+
+
+def _get_process_mark_id(unified: bool = False) -> str:
+    """Creates a text that can be used to identify process+thread
+    Args:
+        unified: bool, if True then result will be same for same process+thread
+    Returns:
+        mark of process+thread
+    """
+    mark = f"{os.getpid()}-{threading.get_native_id()}"
+    if unified is True:
+        return mark
+    return f"{mark}-{str(time.time()).replace('.', '')}"
+
+
+def create_process_mark(folder="learn"):
+    p = Path(tempfile.gettempdir()).joinpath(f"mindsdb/processes/{folder}/")
+    p.mkdir(parents=True, exist_ok=True)
+    mark = _get_process_mark_id()
+    p.joinpath(mark).touch()
+    return mark
+
+
+def set_process_mark(folder: str, mark: str) -> None:
+    """touch new file which will be process mark
+
+    Args:
+        folder (str): where create the file
+        mark (str): file name
+
+    Returns:
+        str: process mark
+    """
+    p = Path(tempfile.gettempdir()).joinpath(f"mindsdb/processes/{folder}/")
+    p.mkdir(parents=True, exist_ok=True)
+    mark = f"{os.getpid()}-{threading.get_native_id()}-{mark}"
+    p.joinpath(mark).touch()
+    return mark
+
+
+def delete_process_mark(folder: str = "learn", mark: Optional[str] = None):
+    if mark is None:
+        mark = _get_process_mark_id()
+    p = (
+        Path(tempfile.gettempdir())
+        .joinpath(f"mindsdb/processes/{folder}/")
+        .joinpath(mark)
+    )
+    if p.exists():
+        p.unlink()
+
+
+def clean_process_marks():
+    """delete all existing processes marks"""
+    logger.debug("Deleting PIDs..")
+    p = Path(tempfile.gettempdir()).joinpath("mindsdb/processes/")
+    if p.exists() is False:
+        return
+    for path in p.iterdir():
+        if path.is_dir() is False:
+            return
+        for file in path.iterdir():
+            file.unlink()
+
+
+def get_processes_dir_files_generator() -> Tuple[Path, int, int]:
+    """Get files from processes dir
+
+    Yields:
+        Tuple[Path, int, int]: file object, process is and thread id
+    """
+    p = Path(tempfile.gettempdir()).joinpath("mindsdb/processes/")
+    if p.exists() is False:
+        return
+    for path in p.iterdir():
+        if path.is_dir() is False:
+            continue
+        for file in path.iterdir():
+            parts = file.name.split("-")
+            process_id = int(parts[0])
+            thread_id = int(parts[1])
+            yield file, process_id, thread_id
+
+
+def clean_unlinked_process_marks() -> List[int]:
+    """delete marks that does not have corresponded processes/threads
+
+    Returns:
+        List[int]: list with ids of unexisting processes
+    """
+    deleted_pids = []
+
+    for file, process_id, thread_id in get_processes_dir_files_generator():
+        try:
+            process = psutil.Process(process_id)
+            if process.status() in (psutil.STATUS_ZOMBIE, psutil.STATUS_DEAD):
+                raise psutil.NoSuchProcess(process_id)
+
+            threads = process.threads()
+            try:
+                next(t for t in threads if t.id == thread_id)
+            except StopIteration:
+                logger.warning(
+                    f"We have mark for process/thread {process_id}/{thread_id} but it does not exists"
+                )
+                deleted_pids.append(process_id)
+                file.unlink()
+
+        except psutil.AccessDenied:
+            logger.warning(f"access to {process_id} denied")
+
+            continue
+
+        except psutil.NoSuchProcess:
+            logger.warning(
+                f"We have mark for process/thread {process_id}/{thread_id} but it does not exists"
+            )
+            deleted_pids.append(process_id)
+            file.unlink()
+    return deleted_pids
+
+
+def __is_within_directory(directory, target):
+    abs_directory = os.path.abspath(directory)
+    abs_target = os.path.abspath(target)
+    prefix = os.path.commonprefix([abs_directory, abs_target])
+    return prefix == abs_directory
+
+
+def safe_extract(tarfile, path=".", members=None, *, numeric_owner=False):
+    # for py >= 3.12
+    if hasattr(tarfile, 'data_filter'):
+        tarfile.extractall(path, members=members, numeric_owner=numeric_owner, filter='data')
+        return
+
+    # for py < 3.12
+    for member in tarfile.getmembers():
+        member_path = os.path.join(path, member.name)
+        if not __is_within_directory(path, member_path):
+            raise Exception("Attempted Path Traversal in Tar File")
+    tarfile.extractall(path, members=members, numeric_owner=numeric_owner)

mindsdb/utilities/functions.py

@@ -0,0 +1,197 @@
+import os
+import base64
+import hashlib
+import datetime
+import textwrap
+from functools import wraps
+from collections.abc import Callable
+
+import requests
+from cryptography.fernet import Fernet
+from mindsdb_sql_parser.ast import Identifier
+
+from mindsdb.utilities.fs import create_process_mark, delete_process_mark, set_process_mark
+from mindsdb.utilities import log
+from mindsdb.utilities.config import Config
+
+
+logger = log.getLogger(__name__)
+
+
+def get_handler_install_message(handler_name):
+    if Config().use_docker_env:
+        container_id = os.environ.get("HOSTNAME", "<container_id>")
+        return textwrap.dedent(f"""\
+            To install the {handler_name} handler, run the following in your terminal outside the docker container
+            ({container_id} is the ID of this container):
+
+                docker exec {container_id} pip install 'mindsdb[{handler_name}]'""")
+    else:
+        return textwrap.dedent(f"""\
+            To install the {handler_name} handler, run the following in your terminal:
+
+                pip install 'mindsdb[{handler_name}]'  # If you installed mindsdb via pip
+                pip install '.[{handler_name}]'        # If you installed mindsdb from source""")
+
+
+def cast_row_types(row, field_types):
+    '''
+    '''
+    keys = [x for x in row.keys() if x in field_types]
+    for key in keys:
+        t = field_types[key]
+        if t == 'Timestamp' and isinstance(row[key], (int, float)):
+            timestamp = datetime.datetime.utcfromtimestamp(row[key])
+            row[key] = timestamp.strftime('%Y-%m-%d %H:%M:%S')
+        elif t == 'Date' and isinstance(row[key], (int, float)):
+            timestamp = datetime.datetime.utcfromtimestamp(row[key])
+            row[key] = timestamp.strftime('%Y-%m-%d')
+        elif t == 'Int' and isinstance(row[key], (int, float, str)):
+            try:
+                logger.debug(f'cast {row[key]} to {int(row[key])}')
+                row[key] = int(row[key])
+            except Exception:
+                pass
+
+
+def mark_process(name: str, custom_mark: str = None) -> Callable:
+    def mark_process_wrapper(func: Callable) -> Callable:
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            if custom_mark is None:
+                mark = create_process_mark(name)
+            else:
+                mark = set_process_mark(name, custom_mark)
+
+            try:
+                return func(*args, **kwargs)
+            finally:
+                delete_process_mark(name, mark)
+        return wrapper
+    return mark_process_wrapper
+
+
+def get_versions_where_predictors_become_obsolete():
+    """ Get list of MindsDB versions in which predictors should be retrained
+        Returns:
+            list of str or False
+    """
+    versions_for_updating_predictors = []
+    try:
+        try:
+            res = requests.get(
+                'https://mindsdb-cloud-public-service-files.s3.us-east-2.amazonaws.com/version_for_updating_predictors.txt',
+                timeout=0.5
+            )
+        except (ConnectionError, requests.exceptions.ConnectionError) as e:
+            logger.error(f'Is no connection. {e}')
+            raise
+        except Exception as e:
+            logger.error(f'Is something wrong with getting version_for_updating_predictors.txt: {e}')
+            raise
+
+        if res.status_code != 200:
+            logger.error(f'Cant get version_for_updating_predictors.txt: returned status code = {res.status_code}')
+            raise
+
+        try:
+            versions_for_updating_predictors = res.text.replace(' \t\r', '').split('\n')
+        except Exception as e:
+            logger.error(f'Cant decode version_for_updating_predictors.txt: {e}')
+            raise
+    except Exception:
+        return False, versions_for_updating_predictors
+
+    versions_for_updating_predictors = [x for x in versions_for_updating_predictors if len(x) > 0]
+    return True, versions_for_updating_predictors
+
+
+def init_lexer_parsers():
+    from mindsdb_sql_parser.lexer import MindsDBLexer
+    from mindsdb_sql_parser.parser import MindsDBParser
+    return MindsDBLexer(), MindsDBParser()
+
+
+def resolve_table_identifier(identifier: Identifier, default_database: str = None) -> tuple:
+    parts = identifier.parts
+
+    parts_count = len(parts)
+    if parts_count == 1:
+        return (None, parts[0])
+    elif parts_count == 2:
+        return (parts[0], parts[1])
+    else:
+        raise Exception(f'Table identifier must contain max 2 parts: {parts}')
+
+
+def resolve_model_identifier(identifier: Identifier) -> tuple:
+    """ split model name to parts
+
+        Identifier may be:
+
+        Examples:
+            >>> resolve_model_identifier(['a', 'b'])
+            ('a', 'b', None)
+
+            >>> resolve_model_identifier(['a', '1'])
+            (None, 'a', 1)
+
+            >>> resolve_model_identifier(['a'])
+            (None, 'a', None)
+
+            >>> resolve_model_identifier(['a', 'b', 'c'])
+            (None, None, None)  # not found
+
+        Args:
+            name (Identifier): Identifier parts
+
+        Returns:
+            tuple: (database_name, model_name, model_version)
+    """
+    parts = identifier.parts
+    database_name = None
+    model_name = None
+    model_version = None
+
+    parts_count = len(parts)
+    if parts_count == 1:
+        database_name = None
+        model_name = parts[0]
+        model_version = None
+    elif parts_count == 2:
+        if parts[-1].isdigit():
+            database_name = None
+            model_name = parts[0]
+            model_version = int(parts[-1])
+        else:
+            database_name = parts[0]
+            model_name = parts[1]
+            model_version = None
+    elif parts_count == 3:
+        database_name = parts[0]
+        model_name = parts[1]
+        if parts[2].isdigit():
+            model_version = int(parts[2])
+        else:
+            # not found
+            return None, None, None
+
+    return database_name, model_name, model_version
+
+
+def encrypt(string: bytes, key: str) -> bytes:
+    hashed_string = hashlib.sha256(key.encode()).digest()
+
+    fernet_key = base64.urlsafe_b64encode(hashed_string)
+
+    cipher = Fernet(fernet_key)
+    return cipher.encrypt(string)
+
+
+def decrypt(encripted: bytes, key: str) -> bytes:
+    hashed_string = hashlib.sha256(key.encode()).digest()
+
+    fernet_key = base64.urlsafe_b64encode(hashed_string)
+
+    cipher = Fernet(fernet_key)
+    return cipher.decrypt(encripted)

mindsdb/utilities/hooks/__init__.py

@@ -0,0 +1,34 @@
+def empty_fn(*args, **kwargs):
+    pass
+
+
+try:
+    from mindsdb.utilities.hooks.after_predict import after_predict
+except ImportError:
+    after_predict = empty_fn
+
+
+try:
+    from mindsdb.utilities.hooks.after_api_query import after_api_query
+except ImportError:
+    after_api_query = empty_fn
+    after_palm_query = empty_fn
+
+
+try:
+    from mindsdb.utilities.hooks.profiling import send_profiling_results
+except ImportError:
+    send_profiling_results = empty_fn
+
+
+try:
+    from mindsdb.utilities.hooks.openai_query import before_openai_query
+except ImportError:
+    before_openai_query = empty_fn
+    before_palm_query = empty_fn
+
+try:
+    from mindsdb.utilities.hooks.openai_query import after_openai_query
+except ImportError:
+    after_openai_query = empty_fn
+    after_palm_query = empty_fn

mindsdb/utilities/hooks/profiling.py

@@ -0,0 +1,74 @@
+import os
+import json
+
+import psycopg
+
+from mindsdb.utilities import log
+
+logger = log.getLogger(__name__)
+
+
+MINDSDB_PROFILING_ENABLED = os.environ.get("MINDSDB_PROFILING_ENABLED") in ("1", "true")
+MINDSDB_PROFILING_DB_HOST = os.environ.get("MINDSDB_PROFILING_DB_HOST")
+MINDSDB_PROFILING_DB_USER = os.environ.get("MINDSDB_PROFILING_DB_USER")
+MINDSDB_PROFILING_DB_PASSWORD = os.environ.get("MINDSDB_PROFILING_DB_PASSWORD")
+
+
+def set_level(node, level, internal_id):
+    internal_id["id"] += 1
+    node["level"] = level
+    node["value"] = node["stop_at"] - node["start_at"]
+    node["value_thread"] = node["stop_at_thread"] - node["start_at_thread"]
+    node["value_process"] = node["stop_at_process"] - node["start_at_process"]
+    node["internal_id"] = internal_id["id"]
+
+    accum = 0
+    for child_node in node["children"]:
+        set_level(child_node, level + 1, internal_id)
+        accum += child_node["value"]
+    node["self"] = node["value"] - accum
+
+
+def send_profiling_results(profiling_data: dict):
+    if MINDSDB_PROFILING_ENABLED is False:
+        return
+
+    profiling = profiling_data
+    set_level(profiling["tree"], 0, {"id": 0})
+
+    time_start_at = profiling["tree"]["time_start_at"]
+    del profiling["tree"]["time_start_at"]
+
+    try:
+        connection = psycopg.connect(
+            host=MINDSDB_PROFILING_DB_HOST,
+            port=5432,
+            user=MINDSDB_PROFILING_DB_USER,
+            password=MINDSDB_PROFILING_DB_PASSWORD,
+            dbname="postgres",
+            connect_timeout=5
+        )
+    except Exception:
+        logger.error('cant get acceess to profiling database')
+        return
+    cur = connection.cursor()
+    cur.execute("""
+        insert into profiling
+            (data, query, time, hostname, environment, api, total_time, company_id, instance_id)
+        values
+            (%s, %s, %s, %s, %s, %s, %s, %s, %s)
+    """, (
+        json.dumps(profiling["tree"]),
+        profiling.get("query", "?"),
+        time_start_at,
+        profiling["hostname"],
+        profiling.get("environment", "?"),
+        profiling.get("api", "?"),
+        profiling["tree"]["value"],
+        profiling["company_id"],
+        profiling["instance_id"]
+    ))
+
+    connection.commit()
+    cur.close()
+    connection.close()

mindsdb/utilities/json_encoder.py

@@ -0,0 +1,33 @@
+from datetime import datetime, date, timedelta
+from decimal import Decimal
+import numpy as np
+import pandas as pd
+import json
+
+from flask.json.provider import DefaultJSONProvider
+
+
+class CustomJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, timedelta):
+            return str(obj)
+        if isinstance(obj, datetime):
+            return obj.strftime("%Y-%m-%d %H:%M:%S.%f")
+        if isinstance(obj, date):
+            return obj.strftime("%Y-%m-%d")
+        if isinstance(obj, np.bool_):
+            return bool(obj)
+        if isinstance(obj, np.int8) or isinstance(obj, np.int16) or isinstance(obj, np.int32) or isinstance(obj, np.int64):
+            return int(obj)
+        if isinstance(obj, np.float16) or isinstance(obj, np.float32) or isinstance(obj, np.float64) or isinstance(obj, Decimal):
+            return float(obj)
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
+        if pd.isnull(obj):
+            return None
+
+        return str(obj)
+
+
+class CustomJSONProvider(CustomJSONEncoder, DefaultJSONProvider):
+    ...

mindsdb/utilities/log.py

@@ -0,0 +1,97 @@
+import logging
+from logging.config import dictConfig
+
+from mindsdb.utilities.config import config as app_config
+
+
+logging_initialized = False
+
+
+class ColorFormatter(logging.Formatter):
+    green = "\x1b[32;20m"
+    default = "\x1b[39;20m"
+    yellow = "\x1b[33;20m"
+    red = "\x1b[31;20m"
+    bold_red = "\x1b[31;1m"
+    reset = "\x1b[0m"
+    format = "%(asctime)s %(processName)15s %(levelname)-8s %(name)s: %(message)s"
+
+    FORMATS = {
+        logging.DEBUG: logging.Formatter(green + format + reset),
+        logging.INFO: logging.Formatter(default + format + reset),
+        logging.WARNING: logging.Formatter(yellow + format + reset),
+        logging.ERROR: logging.Formatter(red + format + reset),
+        logging.CRITICAL: logging.Formatter(bold_red + format + reset),
+    }
+
+    def format(self, record):
+        log_fmt = self.FORMATS.get(record.levelno)
+        return log_fmt.format(record)
+
+
+def configure_logging():
+    handlers_config = {}
+    console_handler_config = app_config['logging']['handlers']['console']
+    console_handler_config_level = getattr(logging, console_handler_config["level"])
+    if console_handler_config['enabled'] is True:
+        handlers_config['console'] = {
+            "class": "logging.StreamHandler",
+            "formatter": "f",
+            "level": console_handler_config_level
+        }
+    file_handler_config = app_config['logging']['handlers']['file']
+    file_handler_config_level = getattr(logging, file_handler_config["level"])
+    if file_handler_config['enabled'] is True:
+        handlers_config['file'] = {
+            "class": "logging.handlers.RotatingFileHandler",
+            "formatter": "file",
+            "level": file_handler_config_level,
+            "filename": app_config.paths["log"] / file_handler_config["filename"],
+            "maxBytes": file_handler_config["maxBytes"],  # 0.5 Mb
+            "backupCount": file_handler_config["backupCount"]
+        }
+
+    mindsdb_log_level = min(console_handler_config_level, file_handler_config_level)
+
+    logging_config = dict(
+        version=1,
+        formatters={
+            "f": {"()": ColorFormatter},
+            "file": {
+                "format": "%(asctime)s %(processName)15s %(levelname)-8s %(name)s: %(message)s"
+            }
+        },
+        handlers=handlers_config,
+        loggers={
+            "": {  # root logger
+                "handlers": list(handlers_config.keys()),
+                "level": logging.WARNING,
+            },
+            "__main__": {
+                "level": mindsdb_log_level,
+            },
+            "mindsdb": {
+                "level": mindsdb_log_level,
+            },
+            "alembic": {
+                "level": mindsdb_log_level,
+            },
+        },
+    )
+
+    dictConfig(logging_config)
+
+
+# I would prefer to leave code to use logging.getLogger(), but there are a lot of complicated situations
+# in MindsDB with processes being spawned that require logging to be configured again in a lot of cases.
+# Using a custom logger-getter like this lets us do that logic here, once.
+def getLogger(name=None):
+    """
+    Get a new logger, configuring logging first if it hasn't been done yet.
+    """
+    global logging_initialized
+    if not logging_initialized:
+        configure_logging()
+        logging_initialized = True
+
+    return logging.getLogger(name)

mindsdb/utilities/log_controller.py

@@ -0,0 +1,39 @@
+from mindsdb.interfaces.storage import db
+from mindsdb.utilities.context import context as ctx
+
+
+def fmt_log_record(log_record):
+    return {
+        'log_from': 'mindsdb',
+        'level': log_record.log_type,
+        'context': 'unknown',
+        'text': log_record.payload,
+        'created_at': str(log_record.created_at).split('.')[0]
+    }
+
+
+def get_logs(min_timestamp, max_timestamp, context, level, log_from, limit):
+    logs = db.session.query(db.Log).filter(
+        db.Log.company_id == ctx.company_id,
+        db.Log.created_at > min_timestamp
+    )
+
+    if max_timestamp is not None:
+        logs = logs.filter(db.Log.created_at < max_timestamp)
+
+    if context is not None:
+        # e.g. datasource/predictor and assoicated id
+        pass
+
+    if level is not None:
+        logs = logs.filter(db.Log.log_type == level)
+
+    if log_from is not None:
+        # mindsdb/native/lightwood/all
+        pass
+
+    if limit is not None:
+        logs = logs.limit(limit)
+
+    logs = [fmt_log_record(x) for x in logs]
+    return logs

mindsdb/utilities/ml_task_queue/__init__.py

@@ -0,0 +1,57 @@
+"""
+    Tasks queue allows to limit load by ordering tasks through a queue.
+    Current implementation use Redis as backend for the queue. To run MindsDB with tasks queue need:
+    1. config mindsdb to use tasks queue by one of:
+        - fill 'ml_task_queue' key in config.json:
+            {
+                "ml_task_queue": {
+                    "type": "redis",    // required
+                    "host": "...",
+                    "port": "...",
+                    "db": "...",
+                    "username": "...",
+                    "password": "..."
+                }
+            }
+        - or set env vars:
+            MINDSDB_ML_QUEUE_TYPE=redis  # required
+            MINDSDB_ML_QUEUE_HOST=...
+            MINDSDB_ML_QUEUE_PORT=...
+            MINDSDB_ML_QUEUE_DB=...
+            MINDSDB_ML_QUEUE_USERNAME=...
+            MINDSDB_ML_QUEUE_PASSWORD=...
+    2. run mindsdb with arg --ml_task_queue_consumer
+
+    In redis there is two types of entities used: streams (for distributing tasks) and regular
+    key-value storage with ttl (to transfer dataframes and some other data). Dataframes are not
+    transfer via streams to make stream messages lightweight.
+
+    Taks queue may work in single instnace to limit load on it, ot it may work in distributed
+    system. In that case mindsdb may be splitted into two modules: parser/planner/executioner (PPE)
+    and ML.
+
+  ┌─────────────┐ ┌─────────────┐
+  │             │ │             │
+  │ MindsDB PPE │ │ MindsDB PPE │
+  │             │ │             │
+  └───────────┬─┘ └─┬─────┬─▲───┘
+              │     │     │ │
+             ┌▼─────▼┐    │ │
+             │       │  ┌─▼─┴─────────┐
+             │ Queue │  │             │
+             │       │  │  Cache      │
+             ├───────┤  │             │
+             │ Task  │  ├─────────────┤
+             ├───────┤  │  Dataframe  │
+             │ Task  │  ├─────────────┤
+             ├───────┤  │  Status     │
+             │ Task  │  └─┬─▲─────────┘
+             └┬─────┬┘    │ │
+              │     │     │ │
+              │     │     │ │
+  ┌───────────▼─┐ ┌─▼─────▼─┴───┐
+  │             │ │             │
+  │ MindsDB ML  │ │ MindsDB ML  │
+  │             │ │             │
+  └─────────────┘ └─────────────┘
+"""