MindsDB 1.3.1__py3-none-any.whl → 24.12.4.0__py3-none-any.whl

mindsdb/interfaces/database/log.py

@@ -0,0 +1,297 @@
+from copy import deepcopy
+from abc import ABC, abstractmethod
+from typing import List, Union, Tuple
+from collections import OrderedDict
+
+import pandas as pd
+
+from mindsdb_sql_parser import parse_sql
+from mindsdb_sql_parser.ast import Select, Identifier, Star, BinaryOperation, Constant, Join, Function
+from mindsdb_sql_parser.utils import JoinType
+from mindsdb.utilities.render.sqlalchemy_render import SqlalchemyRender
+from mindsdb.integrations.utilities.query_traversal import query_traversal
+
+from mindsdb.utilities.functions import resolve_table_identifier
+from mindsdb.api.executor.utilities.sql import get_query_tables
+from mindsdb.utilities.exception import EntityNotExistsError
+import mindsdb.interfaces.storage.db as db
+from mindsdb.utilities.context import context as ctx
+from mindsdb.api.executor.datahub.classes.tables_row import (
+    TABLES_ROW_TYPE,
+    TablesRow,
+)
+
+
+class LogTable(ABC):
+    """Base class for 'table' entitie in internal 'log' database
+
+    Attributes:
+        name (str): name of the table
+        deletable (bool): is it possible to delete a table
+        visible (bool): should be table visible in GUI sidebar
+        kind (str): type of the table/view
+    """
+
+    name: str
+    deletable: bool = False
+    visible: bool = True
+    kind: str = 'table'
+
+    @staticmethod
+    @abstractmethod
+    def _get_base_subquery() -> Select:
+        """Get a query that returns the table from internal db
+
+        Returns:
+            Select: 'select' query that returns table
+        """
+        pass
+
+    @staticmethod
+    def company_id_comparison(table_a: str, table_b: str) -> BinaryOperation:
+        """Make statement for 'safe' comparison of company_id of two tables
+
+        Args:
+            table_a (str): name of first table
+            table_b (str): name of second table
+
+        Returns:
+            BinaryOperation: statement that can be used for 'safe' comparison
+        """
+        return BinaryOperation(
+            op='=',
+            args=(
+                Function(
+                    op='coalesce',
+                    args=(
+                        Identifier(f'{table_a}.company_id'),
+                        0
+                    )
+                ),
+                Function(
+                    op='coalesce',
+                    args=(
+                        Identifier(f'{table_b}.company_id'),
+                        0
+                    )
+                )
+            )
+        )
+
+
+class LLMLogTable(LogTable):
+    name = 'llm_log'
+
+    columns = [
+        'API_KEY', 'MODEL_NAME', 'INPUT', 'OUTPUT', 'START_TIME', 'END_TIME',
+        'PROMPT_TOKENS', 'COMPLETION_TOKENS', 'TOTAL_TOKENS', 'SUCCESS'
+    ]
+
+    types_map = {
+        'SUCCESS': 'boolean',
+        'START_TIME': 'datetime64[ns]',
+        'END_TIME': 'datetime64[ns]'
+    }
+
+    @staticmethod
+    def _get_base_subquery() -> Select:
+        query = Select(
+            targets=[
+                Identifier('llm_log.api_key', alias=Identifier('api_key')),
+                Identifier('predictor.name', alias=Identifier('model_name')),
+                Identifier('llm_log.input', alias=Identifier('input')),
+                Identifier('llm_log.output', alias=Identifier('output')),
+                Identifier('llm_log.start_time', alias=Identifier('start_time')),
+                Identifier('llm_log.end_time', alias=Identifier('end_time')),
+                Identifier('llm_log.prompt_tokens', alias=Identifier('prompt_tokens')),
+                Identifier('llm_log.completion_tokens', alias=Identifier('completion_tokens')),
+                Identifier('llm_log.total_tokens', alias=Identifier('total_tokens')),
+                Identifier('llm_log.success', alias=Identifier('success'))
+            ],
+            from_table=Join(
+                left=Identifier('llm_log'),
+                right=Identifier('predictor'),
+                join_type=JoinType.LEFT_JOIN,
+                condition=BinaryOperation(
+                    op='and',
+                    args=(
+                        LLMLogTable.company_id_comparison('llm_log', 'predictor'),
+                        BinaryOperation(
+                            op='=',
+                            args=(
+                                Identifier('llm_log.model_id'),
+                                Identifier('predictor.id')
+                            )
+                        )
+                    )
+                )
+            ),
+            where=BinaryOperation(
+                op='is' if ctx.company_id is None else '=',
+                args=(Identifier('llm_log.company_id'), Constant(ctx.company_id))
+            ),
+            alias=Identifier('llm_log')
+        )
+        return query
+
+
+class JobsHistoryTable(LogTable):
+    name = 'jobs_history'
+
+    columns = ['NAME', 'PROJECT', 'RUN_START', 'RUN_END', 'ERROR', 'QUERY']
+    types_map = {
+        'RUN_START': 'datetime64[ns]',
+        'RUN_END': 'datetime64[ns]'
+    }
+
+    @staticmethod
+    def _get_base_subquery() -> Select:
+        query = Select(
+            targets=[
+                Identifier('jobs.name', alias=Identifier('name')),
+                Identifier('project.name', alias=Identifier('project')),
+                Identifier('jobs_history.start_at', alias=Identifier('run_start')),
+                Identifier('jobs_history.end_at', alias=Identifier('run_end')),
+                Identifier('jobs_history.error', alias=Identifier('error')),
+                Identifier('jobs_history.query_str', alias=Identifier('query'))
+            ],
+            from_table=Join(
+                left=Join(
+                    left=Identifier('jobs_history'),
+                    right=Identifier('jobs'),
+                    join_type=JoinType.LEFT_JOIN,
+                    condition=BinaryOperation(
+                        op='and',
+                        args=(
+                            LLMLogTable.company_id_comparison('jobs_history', 'jobs'),
+                            BinaryOperation(
+                                op='=',
+                                args=(
+                                    Identifier('jobs_history.job_id'),
+                                    Identifier('jobs.id')
+                                )
+                            )
+                        )
+                    )
+                ),
+                right=Identifier('project'),
+                join_type=JoinType.LEFT_JOIN,
+                condition=BinaryOperation(
+                    op='and',
+                    args=(
+                        LLMLogTable.company_id_comparison('project', 'jobs'),
+                        BinaryOperation(
+                            op='=',
+                            args=(
+                                Identifier('project.id'),
+                                Identifier('jobs.project_id')
+                            )
+                        )
+                    )
+                )
+            ),
+            where=BinaryOperation(
+                op='is' if ctx.company_id is None else '=',
+                args=(Identifier('jobs_history.company_id'), Constant(ctx.company_id))
+            ),
+            alias=Identifier('jobs_history')
+        )
+        return query
+
+
+class LogDBController:
+    def __init__(self):
+        self._tables = OrderedDict()
+        self._tables['llm_log'] = LLMLogTable
+        self._tables['jobs_history'] = JobsHistoryTable
+
+    def get_list(self) -> List[LogTable]:
+        return list(self._tables.values())
+
+    def get(self, name: str = None) -> LogTable:
+        try:
+            return self._tables[name]
+        except KeyError:
+            raise EntityNotExistsError(f'Table log.{name} does not exists')
+
+    def get_tables(self) -> OrderedDict:
+        return self._tables
+
+    def get_tables_rows(self) -> List[TablesRow]:
+        return [
+            TablesRow(TABLE_TYPE=TABLES_ROW_TYPE.SYSTEM_VIEW, TABLE_NAME=table_name)
+            for table_name in self._tables.keys()
+        ]
+
+    def query(self, query: Select = None, native_query: str = None,
+              session=None, return_as: str = 'split') -> Union[pd.DataFrame, Tuple[pd.DataFrame, list]]:
+        if native_query is not None:
+            if query is not None:
+                raise Exception("'query' and 'native_query' arguments can not be used together")
+            query = parse_sql(native_query)
+        else:
+            query = deepcopy(query)
+
+        if type(query) is not Select:
+            raise Exception("Only 'SELECT' is allowed for tables in log database")
+        tables = get_query_tables(query)
+        if len(tables) != 1:
+            raise Exception("Only one table may be in query to log database")
+        table = tables[0]
+        if table[0] is not None and table[0].lower() != 'log':
+            raise Exception("This is not a query to the log database")
+        if table[1].lower() not in self._tables.keys():
+            raise Exception(f"There is no table '{table[1]}' in the log database")
+
+        log_table = self._tables[table[1].lower()]
+
+        # region check that only allowed identifiers are used in the query
+        available_columns_names = [column.lower() for column in log_table.columns]
+
+        def check_columns(node, is_table, **kwargs):
+            # region replace * to available columns
+            if type(node) is Select:
+                new_targets = []
+                for target in node.targets:
+                    if type(target) is Star:
+                        new_targets += [Identifier(name) for name in available_columns_names]
+                    else:
+                        new_targets.append(target)
+                node.targets = new_targets
+            # endregion
+
+            if type(node) is Identifier and is_table is False:
+                parts = resolve_table_identifier(node)
+                if parts[0] is not None and parts[0].lower() not in self._tables:
+                    raise Exception(f"Table '{parts[0]}' can not be used in query")
+                if parts[1].lower() not in available_columns_names:
+                    raise Exception(f"Column '{parts[1]}' can not be used in query")
+
+        query_traversal(query, check_columns)
+        # endregion
+
+        query.from_table = log_table._get_base_subquery()
+
+        render_engine = db.engine.name
+        if render_engine == "postgresql":
+            'postgres'
+        render = SqlalchemyRender(render_engine)
+        query_str = render.get_string(query, with_failback=False)
+        df = pd.read_sql_query(query_str, db.engine)
+
+        # region cast columns values to proper types
+        for column_name, column_type in log_table.types_map.items():
+            for df_column_name in df.columns:
+                if df_column_name.lower() == column_name.lower() and df[df_column_name].dtype != column_type:
+                    df[df_column_name] = df[df_column_name].astype(column_type)
+        # endregion
+
+        if return_as != 'split':
+            return df
+
+        columns_info = [{
+            'name': k,
+            'type': v
+        } for k, v in df.dtypes.items()]
+
+        return df, columns_info

mindsdb/interfaces/database/projects.py

@@ -0,0 +1,397 @@
+import datetime
+from copy import deepcopy
+from typing import List, Optional
+from collections import OrderedDict
+
+import sqlalchemy as sa
+import numpy as np
+
+from mindsdb_sql_parser.ast.base import ASTNode
+from mindsdb_sql_parser import parse_sql
+
+from mindsdb.interfaces.storage import db
+from mindsdb.utilities.config import Config
+from mindsdb.interfaces.model.model_controller import ModelController
+from mindsdb.interfaces.database.views import ViewController
+from mindsdb.utilities.context import context as ctx
+from mindsdb.utilities.exception import EntityExistsError, EntityNotExistsError
+import mindsdb.utilities.profiler as profiler
+
+
+class Project:
+    @staticmethod
+    def from_record(db_record: db.Project):
+        p = Project()
+        p.record = db_record
+        p.name = db_record.name
+        p.company_id = db_record.company_id
+        p.id = db_record.id
+        return p
+
+    def create(self, name: str):
+        name = name.lower()
+        existing_record = db.Project.query.filter(
+            (sa.func.lower(db.Project.name) == name)
+            & (db.Project.company_id == ctx.company_id)
+            & (db.Project.deleted_at == sa.null())
+        ).first()
+        if existing_record is not None:
+            raise EntityExistsError('Project already exists', name)
+
+        existing_record = db.Integration.query.filter(
+            sa.func.lower(db.Integration.name) == name,
+            db.Integration.company_id == ctx.company_id
+        ).first()
+        if existing_record is not None:
+            raise EntityExistsError('Database exists with this name ', name)
+
+        record = db.Project(
+            name=name,
+            company_id=ctx.company_id
+        )
+
+        self.record = record
+        self.name = name
+        self.company_id = ctx.company_id
+
+        db.session.add(record)
+        db.session.commit()
+
+        self.id = record.id
+
+    def save(self):
+        db.session.commit()
+
+    def delete(self):
+        tables = self.get_tables()
+        tables = [key for key, val in tables.items() if val['type'] != 'table']
+        if len(tables) > 0:
+            raise Exception(f"Project '{self.name}' can not be deleted, because it contains tables: {', '.join(tables)}")
+
+        is_cloud = Config().get('cloud', False)
+        if is_cloud is True:
+            self.record.deleted_at = datetime.datetime.now()
+        else:
+            db.session.delete(self.record)
+            self.record = None
+            self.name = None
+            self.company_id = None
+            self.id = None
+        db.session.commit()
+
+    def drop_model(self, name: str):
+        ModelController().delete_model(
+            name,
+            project_name=self.name
+        )
+
+    def drop_view(self, name: str):
+        ViewController().delete(
+            name,
+            project_name=self.name
+        )
+
+    def create_view(self, name: str, query: str):
+        ViewController().add(
+            name,
+            query=query,
+            project_name=self.name
+        )
+
+    def update_view(self, name: str, query: str):
+        ViewController().update(
+            name,
+            query=query,
+            project_name=self.name
+        )
+
+    def delete_view(self, name: str):
+        ViewController().delete(
+            name,
+            project_name=self.name
+        )
+
+    def query_view(self, query: ASTNode) -> ASTNode:
+        view_name = query.from_table.parts[-1]
+        view_meta = ViewController().get(
+            name=view_name,
+            project_name=self.name
+        )
+        view_meta['query_ast'] = parse_sql(view_meta['query'])
+        return view_meta
+
+    @staticmethod
+    def _get_model_data(predictor_record, integraion_record, with_secrets: bool = True):
+        from mindsdb.interfaces.database.integrations import integration_controller
+
+        predictor_data = predictor_record.data or {}
+        training_time = None
+        if (
+            predictor_record.training_start_at is not None
+            and predictor_record.training_stop_at is None
+            and predictor_record.status != 'error'
+        ):
+            training_time = round((datetime.datetime.now() - predictor_record.training_start_at).total_seconds(), 3)
+        elif (
+            predictor_record.training_start_at is not None
+            and predictor_record.training_stop_at is not None
+        ):
+            training_time = round((predictor_record.training_stop_at - predictor_record.training_start_at).total_seconds(), 3)
+
+        # regon Hide sensitive info
+        training_options = predictor_record.learn_args
+        handler_module = integration_controller.get_handler_module(integraion_record.engine)
+
+        if with_secrets is False and handler_module:
+
+            model_using_args = getattr(handler_module, 'model_using_args', None)
+            if (
+                isinstance(model_using_args, dict)
+                and isinstance(training_options, dict)
+                and isinstance(training_options.get('using'), dict)
+            ):
+                training_options['using'] = deepcopy(training_options['using'])
+                for key, value in model_using_args.items():
+                    if key in training_options['using'] and value.get('secret', False):
+                        training_options['using'][key] = '******'
+        # endregion
+
+        predictor_meta = {
+            'type': 'model',
+            'id': predictor_record.id,
+            'engine': integraion_record.engine,
+            'engine_name': integraion_record.name,
+            'active': predictor_record.active,
+            'version': predictor_record.version,
+            'status': predictor_record.status,
+            'accuracy': None,
+            'predict': predictor_record.to_predict[0],
+            'update_status': predictor_record.update_status,
+            'mindsdb_version': predictor_record.mindsdb_version,
+            'error': predictor_data.get('error'),
+            'select_data_query': predictor_record.fetch_data_query,
+            'training_options': training_options,
+            'deletable': True,
+            'label': predictor_record.label,
+            'current_training_phase': predictor_record.training_phase_current,
+            'total_training_phases': predictor_record.training_phase_total,
+            'training_phase_name': predictor_record.training_phase_name,
+            'training_time': training_time
+        }
+        if predictor_data.get('accuracies', None) is not None:
+            if len(predictor_data['accuracies']) > 0:
+                predictor_meta['accuracy'] = float(np.mean(list(predictor_data['accuracies'].values())))
+        return {
+            'name': predictor_record.name,
+            'metadata': predictor_meta,
+            'created_at': predictor_record.created_at
+        }
+
+    def get_model(self, name: str):
+        record = (
+            db.session.query(db.Predictor, db.Integration).filter_by(
+                project_id=self.id,
+                active=True,
+                name=name,
+                deleted_at=sa.null(),
+                company_id=ctx.company_id
+            )
+            .join(db.Integration, db.Integration.id == db.Predictor.integration_id)
+            .order_by(db.Predictor.name, db.Predictor.id)
+            .first()
+        )
+        if record is None:
+            return None
+        return self._get_model_data(record[0], record[1])
+
+    def get_model_by_id(self, model_id: int):
+        record = (
+            db.session.query(db.Predictor, db.Integration).filter_by(
+                project_id=self.id,
+                id=model_id,
+                deleted_at=sa.null(),
+                company_id=ctx.company_id
+            )
+            .join(db.Integration, db.Integration.id == db.Predictor.integration_id)
+            .order_by(db.Predictor.name, db.Predictor.id)
+            .first()
+        )
+        if record is None:
+            return None
+        return self._get_model_data(record[0], record[1])
+
+    def get_models(self, active: bool = True, with_secrets: bool = True):
+        query = db.session.query(db.Predictor, db.Integration).filter_by(
+            project_id=self.id,
+            deleted_at=sa.null(),
+            company_id=ctx.company_id
+        )
+        if isinstance(active, bool):
+            query = query.filter_by(active=active)
+
+        query = query.join(
+            db.Integration, db.Integration.id == db.Predictor.integration_id
+        ).order_by(db.Predictor.name, db.Predictor.id)
+
+        data = []
+
+        for predictor_record, integraion_record in query.all():
+            data.append(
+                self._get_model_data(predictor_record, integraion_record, with_secrets)
+            )
+
+        return data
+
+    def get_agents(self):
+        records = (
+            db.session.query(db.Agents).filter(
+                db.Agents.project_id == self.id,
+                db.Agents.company_id == ctx.company_id,
+                db.Agents.deleted_at == sa.null()
+            )
+            .order_by(db.Agents.name)
+            .all()
+        )
+        data = [
+            {
+                'name': record.name,
+                'query': record.query,
+                'metadata': {
+                    'type': 'agent',
+                    'id': record.id,
+                    'deletable': True
+                }
+            }
+            for record in records
+        ]
+        return data
+
+    def get_views(self):
+        records = (
+            db.session.query(db.View).filter_by(
+                project_id=self.id,
+                company_id=ctx.company_id
+            )
+            .order_by(db.View.name, db.View.id)
+            .all()
+        )
+        data = [{
+            'name': view_record.name,
+            'query': view_record.query,
+            'metadata': {
+                'type': 'view',
+                'id': view_record.id,
+                'deletable': True
+            }}
+            for view_record in records
+        ]
+        return data
+
+    def get_view(self, name):
+        view_record = db.session.query(db.View).filter(
+            db.View.project_id == self.id,
+            db.View.company_id == ctx.company_id,
+            sa.func.lower(db.View.name) == name.lower(),
+        ).one_or_none()
+        if view_record is None:
+            return view_record
+        return {
+            'name': view_record.name,
+            'query': view_record.query,
+            'metadata': {
+                'type': 'view',
+                'id': view_record.id,
+                'deletable': True
+            }
+        }
+
+    @profiler.profile()
+    def get_tables(self):
+        data = OrderedDict()
+        data['models'] = {'type': 'table', 'deletable': False}
+
+        models = self.get_models()
+        for model in models:
+            if model['metadata']['active'] is True:
+                data[model['name']] = model['metadata']
+
+        views = self.get_views()
+        for view in views:
+            data[view['name']] = view['metadata']
+
+        agents = self.get_agents()
+        for agent in agents:
+            data[agent['name']] = agent['metadata']
+
+        return data
+
+    def get_columns(self, table_name: str):
+        # at the moment it works only for models
+        predictor_record = db.Predictor.query.filter_by(
+            company_id=ctx.company_id,
+            project_id=self.id,
+            name=table_name
+        ).first()
+        columns = []
+        if predictor_record is not None:
+            if isinstance(predictor_record.dtype_dict, dict):
+                columns = list(predictor_record.dtype_dict.keys())
+            elif predictor_record.to_predict is not None:
+                # no dtype_dict, use target
+                columns = predictor_record.to_predict
+                if not isinstance(columns, list):
+                    columns = [columns]
+        else:
+            # is it agent?
+            agent = db.Agents.query.filter_by(
+                company_id=ctx.company_id,
+                project_id=self.id,
+                name=table_name
+            ).first()
+            if agent is not None:
+                from mindsdb.interfaces.agents.constants import ASSISTANT_COLUMN, USER_COLUMN
+                columns = [ASSISTANT_COLUMN, USER_COLUMN]
+
+        return columns
+
+
+class ProjectController:
+    def __init__(self):
+        pass
+
+    def get_list(self) -> List[Project]:
+        records = db.Project.query.filter(
+            (db.Project.company_id == ctx.company_id)
+            & (db.Project.deleted_at == sa.null())
+        ).order_by(db.Project.name)
+
+        return [Project.from_record(x) for x in records]
+
+    def get(self, id: Optional[int] = None, name: Optional[str] = None, deleted: bool = False) -> Project:
+        if id is not None and name is not None:
+            raise ValueError("Both 'id' and 'name' is None")
+
+        q = db.Project.query.filter_by(company_id=ctx.company_id)
+
+        if id is not None:
+            q = q.filter_by(id=id)
+        elif name is not None:
+            q = q.filter(
+                (sa.func.lower(db.Project.name) == sa.func.lower(name))
+            )
+
+        if deleted is True:
+            q = q.filter((db.Project.deleted_at != sa.null()))
+        else:
+            q = q.filter_by(deleted_at=sa.null())
+
+        record = q.first()
+
+        if record is None:
+            raise EntityNotExistsError(f'Project not found: {name}')
+        return Project.from_record(record)
+
+    def add(self, name: str) -> Project:
+        project = Project()
+        project.create(name=name)
+        return project