MindsDB 24.12.2.0__tar.gz → 24.12.2.1__tar.gz

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1/MindsDB.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: MindsDB
-Version: 24.12.2.0
+Version: 24.12.2.1
 Summary: MindsDB's AI SQL Server enables developers to build AI tools that need access to real-time data to perform their tasks
 Home-page: https://github.com/mindsdb/mindsdb
 Download-URL: https://pypi.org/project/mindsdb/

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/MindsDB.egg-info/SOURCES.txt

@@ -1691,7 +1691,6 @@ mindsdb/integrations/libs/api_handler.py
 mindsdb/integrations/libs/api_handler_exceptions.py
 mindsdb/integrations/libs/base.py
 mindsdb/integrations/libs/const.py
-mindsdb/integrations/libs/handler_helpers.py
 mindsdb/integrations/libs/ml_exec_base.py
 mindsdb/integrations/libs/process_cache.py
 mindsdb/integrations/libs/realtime_chat_handler.py

{MindsDB-24.12.2.0/MindsDB.egg-info → MindsDB-24.12.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: MindsDB
-Version: 24.12.2.0
+Version: 24.12.2.1
 Summary: MindsDB's AI SQL Server enables developers to build AI tools that need access to real-time data to perform their tasks
 Home-page: https://github.com/mindsdb/mindsdb
 Download-URL: https://pypi.org/project/mindsdb/

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/mindsdb/__about__.py

@@ -1,6 +1,6 @@
 __title__ = 'MindsDB'
 __package_name__ = 'mindsdb'
-__version__ = '24.12.2.0'
+__version__ = '24.12.2.1'
 __description__ = "MindsDB's AI SQL Server enables developers to build AI tools that need access to real-time data to perform their tasks"
 __email__ = "jorge@mindsdb.com"
 __author__ = 'MindsDB Inc'

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/mindsdb/__main__.py

@@ -22,6 +22,7 @@ from typing import Callable, Optional, Tuple, List
 from sqlalchemy.orm.attributes import flag_modified
 
 from mindsdb.__about__ import __version__ as mindsdb_version
+from mindsdb.utilities.config import Config
 from mindsdb.api.http.start import start as start_http
 from mindsdb.api.mysql.start import start as start_mysql
 from mindsdb.api.mongo.start import start as start_mongo
@@ -29,7 +30,6 @@ from mindsdb.api.postgres.start import start as start_postgres
 from mindsdb.interfaces.tasks.task_monitor import start as start_tasks
 from mindsdb.utilities.ml_task_queue.consumer import start as start_ml_task_queue
 from mindsdb.interfaces.jobs.scheduler import start as start_scheduler
-from mindsdb.utilities.config import Config
 from mindsdb.utilities.ps import is_pid_listen_port, get_child_pids
 from mindsdb.utilities.functions import args_parse, get_versions_where_predictors_become_obsolete
 from mindsdb.interfaces.database.integrations import integration_controller
@@ -357,23 +357,7 @@ if __name__ == '__main__':
             set_error_model_status_by_pids(unexisting_pids)
         set_error_model_status_for_unfinished()
 
-        # region creating permanent integrations
-        for (
-            integration_name,
-            handler,
-        ) in integration_controller.get_handlers_metadata().items():
-            if handler.get("permanent"):
-                integration_meta = integration_controller.get(name=integration_name)
-                if integration_meta is None:
-                    integration_record = db.Integration(
-                        name=integration_name,
-                        data={},
-                        engine=integration_name,
-                        company_id=None,
-                    )
-                    db.session.add(integration_record)
-                    db.session.commit()
-        # endregion
+        integration_controller.create_permanent_integrations()
 
         # region Mark old predictors as outdated
         is_modified = False

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/mindsdb/api/http/initialize.py

@@ -290,7 +290,7 @@ def initialize_app(config, no_studio):
 
     @app.before_request
     def before_request():
-        logger.debug(f"HTTP: {request.path}")
+        logger.debug(f"HTTP {request.method}: {request.path}")
         ctx.set_default()
         config = Config()
 

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/mindsdb/api/http/namespaces/sql.py

@@ -1,3 +1,4 @@
+from http import HTTPStatus
 import traceback
 
 from flask import request
@@ -5,6 +6,7 @@ from flask_restx import Resource
 
 import mindsdb.utilities.hooks as hooks
 import mindsdb.utilities.profiler as profiler
+from mindsdb.api.http.utils import http_error
 from mindsdb.api.http.namespaces.configs.sql import ns_conf
 from mindsdb.api.mysql.mysql_proxy.classes.fake_mysql_proxy import FakeMysqlProxy
 from mindsdb.api.executor.data_types.response_type import (
@@ -31,6 +33,14 @@ class Query(Resource):
         query = request.json["query"]
         context = request.json.get("context", {})
 
+        if isinstance(query, str) is False or isinstance(context, dict) is False:
+            return http_error(
+                HTTPStatus.BAD_REQUEST,
+                'Wrong arguments',
+                'Please provide "query" with the request.'
+            )
+        logger.debug(f'Incoming query: {query}')
+
         if context.get("profiling") is True:
             profiler.enable()
 
@@ -85,7 +95,7 @@ class Query(Resource):
                     "error_code": 0,
                     "error_message": str(e),
                 }
-                logger.error(f"Error profiling query: \n{traceback.format_exc()}")
+                logger.debug(f"Error query processing: \n{traceback.format_exc()}")
 
             if query_response.get("type") == SQL_RESPONSE_TYPE.ERROR:
                 error_type = "expected"

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/mindsdb/api/mysql/mysql_proxy/executor/mysql_executor.py

@@ -96,14 +96,11 @@ class Executor:
 
     @profiler.profile()
     def query_execute(self, sql):
-        logger.info("%s.query_execute: sql - %s", self.__class__.__name__, sql)
-
         self.parse(sql)
         self.do_execute()
 
     @profiler.profile()
     def parse(self, sql):
-        logger.info("%s.parse: sql - %s", self.__class__.__name__, sql)
         self.sql = sql
         sql_lower = sql.lower()
         self.sql_lower = sql_lower.replace("`", "")
@@ -112,10 +109,11 @@ class Executor:
             self.query = parse_sql(sql)
         except Exception as mdb_error:
             # not all statements are parsed by parse_sql
-            logger.warning(f"SQL statement is not parsed by mindsdb_sql_parser: {sql}")
+            logger.warning('Failed to parse SQL query')
+            logger.debug(f'Query that cannot be parsed: {sql}')
 
             raise ErSqlSyntaxError(
-                f"SQL statement cannot be parsed by mindsdb_sql_parser - {sql}: {mdb_error}"
+                f"The SQL statement cannot be parsed - {sql}: {mdb_error}"
             ) from mdb_error
 
             # == a place for workarounds ==
@@ -124,7 +122,6 @@ class Executor:
     @profiler.profile()
     def do_execute(self):
         # it can be already run at prepare state
-        logger.info("%s.do_execute", self.__class__.__name__)
         if self.is_executed:
             return
 

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py

@@ -343,7 +343,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
             data = []
         packets = []
         for i, column in enumerate(columns):
-            logger.info(
+            logger.debug(
                 "%s._get_column_defenition_packets: handling column - %s of %s type",
                 self.__class__.__name__,
                 column,
@@ -737,7 +737,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
                 if p.type.value == COMMANDS.COM_QUERY:
                     sql = self.decode_utf(p.sql.value)
                     sql = SqlStatementParser.clear_sql(sql)
-                    logger.debug(f"COM_QUERY: {sql}")
+                    logger.debug(f'Incoming query: {sql}')
                     profiler.set_meta(
                         query=sql, api="mysql", environment=Config().get("environment")
                     )

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/mindsdb/api/postgres/postgres_proxy/executor/executor.py

@@ -46,10 +46,11 @@ class Executor:
             self.query = parse_sql(sql)
         except Exception as mdb_error:
             # not all statements are parsed by parse_sql
-            self.logger.warning(f"SQL statement is not parsed by mindsdb_sql_parser: {sql}")
+            self.logger.warning('Failed to parse SQL query')
+            self.logger.debug(f'Query that cannot be parsed: {sql}')
 
             raise SqlApiException(
-                f"SQL statement cannot be parsed by mindsdb_sql_parser - {sql}: {mdb_error}"
+                f"The SQL statement cannot be parsed - {sql}: {mdb_error}"
             ) from mdb_error
 
     def stmt_execute(self, param_values):

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/mindsdb/integrations/handlers/pgvector_handler/pgvector_handler.py

@@ -36,6 +36,7 @@ class PgVectorHandler(VectorStoreHandler, PostgresHandler):
 
         super().__init__(name=name, **kwargs)
         self._is_shared_db = False
+        self._is_vector_registered = False
         self.connect()
 
     def _make_connection_args(self):
@@ -77,6 +78,8 @@ class PgVectorHandler(VectorStoreHandler, PostgresHandler):
         Handles the connection to a PostgreSQL database instance.
         """
         self.connection = super().connect()
+        if self._is_vector_registered:
+            return self.connection
 
         with self.connection.cursor() as cur:
             try:
@@ -93,6 +96,7 @@ class PgVectorHandler(VectorStoreHandler, PostgresHandler):
 
         # register vector type with psycopg2 connection
         register_vector(self.connection)
+        self._is_vector_registered = True
 
         return self.connection
 

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/mindsdb/integrations/handlers/slack_handler/__about__.py

@@ -6,4 +6,4 @@ __author__ = 'Tarun Chawla'
 __github__ = 'https://github.com/mindsdb/mindsdb'
 __pypi__ = 'https://pypi.org/project/mindsdb/'
 __license__ = 'MIT'
-__copyright__ = 'Copyright 2023 - mindsdb'
+__copyright__ = 'Copyright 2023 - mindsdb'

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/mindsdb/integrations/handlers/slack_handler/tests/test_slack.py

@@ -22,5 +22,6 @@ class SlackHandlerTest(unittest.TestCase):
             json={'channel': '#test-channel', 'text': 'Test message'}
         )
 
+
 if __name__ == '__main__':
     unittest.main()

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/mindsdb/integrations/utilities/rag/chains/map_reduce_summarizer_chain.py

@@ -23,7 +23,7 @@ logger = log.getLogger(__name__)
 Summary = namedtuple('Summary', ['source_id', 'content'])
 
 
-def create_map_reduce_documents_chain(summarization_config: SummarizationConfig) -> MapReduceDocumentsChain:
+def create_map_reduce_documents_chain(summarization_config: SummarizationConfig, input: str) -> MapReduceDocumentsChain:
     '''Creats a chain that map reduces documents into a single consolidated summary
 
     Args:
@@ -39,11 +39,17 @@ def create_map_reduce_documents_chain(summarization_config: SummarizationConfig)
     })
     map_prompt_template = summarization_config.map_prompt_template
     map_prompt = PromptTemplate.from_template(map_prompt_template)
+    # Langchain needs a template with only a variable for docs so we use a partial.
+    if 'input' in map_prompt.input_variables:
+        map_prompt = map_prompt.partial(input=input)
     # Handles summarization of individual chunks.
     map_chain = LLMChain(llm=summarization_llm, prompt=map_prompt)
 
     reduce_prompt_template = summarization_config.reduce_prompt_template
     reduce_prompt = PromptTemplate.from_template(reduce_prompt_template)
+    # Langchain needs a template with only a variable for docs so we use a partial.
+    if 'input' in reduce_prompt.input_variables:
+        reduce_prompt = reduce_prompt.partial(input=input)
     # Combines summarizations from multiple chunks into a consolidated summary.
     reduce_chain = LLMChain(llm=summarization_llm, prompt=reduce_prompt)
 
@@ -74,6 +80,7 @@ class MapReduceSummarizerChain(Chain):
     context_key: str = 'context'
     metadata_key: str = 'metadata'
     doc_id_key: str = 'original_row_id'
+    question_key: str = 'question'
 
     vector_store_handler: VectorStoreHandler
     table_name: str = 'embeddings'
@@ -81,15 +88,16 @@ class MapReduceSummarizerChain(Chain):
     content_column_name: str = 'content'
     metadata_column_name: str = 'metadata'
 
-    map_reduce_documents_chain: MapReduceDocumentsChain
+    summarization_config: SummarizationConfig
+    map_reduce_documents_chain: Optional[MapReduceDocumentsChain] = None
 
     @property
     def input_keys(self) -> List[str]:
-        return [self.context_key, 'question']
+        return [self.context_key, self.question_key]
 
     @property
     def output_keys(self) -> List[str]:
-        return [self.context_key, 'question']
+        return [self.context_key, self.question_key]
 
     def _get_document_ids_from_chunks(self, chunks: List[Document]) -> List[str]:
         unique_document_ids = set()
@@ -112,7 +120,7 @@ class MapReduceSummarizerChain(Chain):
     def _select_chunks_from_vector_store(self, conditions: List[FilterCondition]) -> DataFrame:
         return self.vector_store_handler.select(
             self.table_name,
-            columns=[self.content_column_name],
+            columns=[self.content_column_name, self.metadata_column_name],
             conditions=conditions
         )
 
@@ -126,11 +134,14 @@ class MapReduceSummarizerChain(Chain):
         all_source_chunks = await asyncio.get_event_loop().run_in_executor(None, self._select_chunks_from_vector_store, [id_filter_condition])
         document_chunks = []
         for _, row in all_source_chunks.iterrows():
-            document_chunks.append(Document(page_content=row[self.content_column_name]))
+            metadata = row.get(self.metadata_column_name, {})
+            document_chunks.append(Document(page_content=row[self.content_column_name], metadata=metadata))
+        # Sort by chunk index if present in metadata so the full document is in its original order.
+        document_chunks.sort(key=lambda doc: doc.metadata.get('chunk_index', 0) if doc.metadata else 0)
         logger.debug(f"Found {len(document_chunks)} chunks for document ID {id}")
         return document_chunks
 
-    async def _get_source_summary(self, source_id: str) -> Summary:
+    async def _get_source_summary(self, source_id: str, map_reduce_documents_chain: MapReduceDocumentsChain) -> Summary:
         if not source_id:
             logger.warning("Received empty source_id, returning empty summary")
             return Summary(source_id='', content='')
@@ -143,14 +154,14 @@ class MapReduceSummarizerChain(Chain):
             return Summary(source_id=source_id, content='')
 
         logger.debug(f"Summarizing {len(source_chunks)} chunks for source ID: {source_id}")
-        summary = await self.map_reduce_documents_chain.ainvoke(source_chunks)
+        summary = await map_reduce_documents_chain.ainvoke(source_chunks)
         content = summary.get('output_text', '')
         logger.debug(f"Generated summary for source ID {source_id}: {content[:100]}...")
         return Summary(source_id=source_id, content=content)
 
-    async def _get_source_summaries(self, source_ids: List[str]) -> List[Summary]:
+    async def _get_source_summaries(self, source_ids: List[str], map_reduce_documents_chain: MapReduceDocumentsChain) -> List[Summary]:
         summaries = await asyncio.gather(
-            *[self._get_source_summary(source_id) for source_id in source_ids]
+            *[self._get_source_summary(source_id, map_reduce_documents_chain) for source_id in source_ids]
         )
         return summaries
 
@@ -159,6 +170,8 @@ class MapReduceSummarizerChain(Chain):
         inputs: Dict[str, Any],
         run_manager: Optional[CallbackManagerForChainRun] = None
     ) -> Dict[str, Any]:
+        # Explicitly connect to make sure vectors are registered.
+        _ = self.vector_store_handler.connect()
         logger.debug(f"Processing inputs with keys: {list(inputs.keys())}")
         context_chunks = inputs.get(self.context_key, [])
         logger.debug(f"Found {len(context_chunks)} context chunks")
@@ -166,17 +179,21 @@ class MapReduceSummarizerChain(Chain):
         unique_document_ids = self._get_document_ids_from_chunks(context_chunks)
         logger.debug(f"Extracted {len(unique_document_ids)} unique document IDs")
 
+        question = inputs.get(self.question_key, '')
+        map_reduce_documents_chain = self.map_reduce_documents_chain
+        if map_reduce_documents_chain is None:
+            map_reduce_documents_chain = create_map_reduce_documents_chain(self.summarization_config, question)
         # For each document ID associated with one or more chunks, build the full document by
         # getting ALL chunks associated with that ID. Then, map reduce summarize the complete document.
         try:
             logger.debug("Starting async summary generation")
-            summaries = asyncio.get_event_loop().run_until_complete(self._get_source_summaries(unique_document_ids))
+            summaries = asyncio.get_event_loop().run_until_complete(self._get_source_summaries(unique_document_ids, map_reduce_documents_chain))
         except RuntimeError:
             logger.info("No event loop available, creating new one")
             # If no event loop is available, create a new one
             loop = asyncio.new_event_loop()
             asyncio.set_event_loop(loop)
-            summaries = loop.run_until_complete(self._get_source_summaries(unique_document_ids))
+            summaries = loop.run_until_complete(self._get_source_summaries(unique_document_ids, map_reduce_documents_chain))
 
         source_id_to_summary = {}
         for summary in summaries:

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/mindsdb/integrations/utilities/rag/pipelines/rag.py

@@ -7,7 +7,7 @@ from langchain.retrievers import ContextualCompressionRetriever
 from langchain_core.prompts import ChatPromptTemplate
 from langchain_core.runnables import RunnableParallel, RunnablePassthrough, RunnableSerializable
 
-from mindsdb.integrations.utilities.rag.chains.map_reduce_summarizer_chain import create_map_reduce_documents_chain, MapReduceSummarizerChain
+from mindsdb.integrations.utilities.rag.chains.map_reduce_summarizer_chain import MapReduceSummarizerChain
 from mindsdb.integrations.utilities.rag.retrievers.auto_retriever import AutoRetriever
 from mindsdb.integrations.utilities.rag.retrievers.multi_vector_retriever import MultiVectorRetriever
 from mindsdb.integrations.utilities.rag.rerankers.reranker_compressor import LLMReranker
@@ -55,11 +55,10 @@ class LangChainRAGPipeline:
         self.vector_store_config = vector_store_config
         knowledge_base_table = self.vector_store_config.kb_table if self.vector_store_config is not None else None
         if summarization_config is not None and knowledge_base_table is not None:
-            map_reduce_documents_chain = create_map_reduce_documents_chain(summarization_config)
             self.summarizer = MapReduceSummarizerChain(
                 vector_store_handler=knowledge_base_table.get_vector_db(),
                 table_name=knowledge_base_table.get_vector_db_table_name(),
-                map_reduce_documents_chain=map_reduce_documents_chain
+                summarization_config=summarization_config
             )
 
     def with_returned_sources(self) -> RunnableSerializable:

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/mindsdb/integrations/utilities/rag/settings.py

@@ -68,12 +68,18 @@ Answer:'''
 
 DEFAULT_MAP_PROMPT_TEMPLATE = '''The following is a set of documents
 {docs}
-Based on this list of docs, please identify the main themes
+Based on this list of docs, please summarize based on the user input.
+
+User input: {input}
+
 Helpful Answer:'''
 
 DEFAULT_REDUCE_PROMPT_TEMPLATE = '''The following is set of summaries:
 {docs}
-Take these and distill it into a final, consolidated summary of the main themes.
+Take these and distill it into a final, consolidated summary related to the user input.
+
+User input: {input}
+
 Helpful Answer:'''
 
 

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/mindsdb/interfaces/database/integrations.py

@@ -585,11 +585,6 @@ class IntegrationController:
             handler_storage=handler_storage
         )
 
-        logger.info(
-            "%s.get_handler: create a client to db service of %s type, args - %s",
-            self.__class__.__name__,
-            integration_engine, handler_ars
-        )
         HandlerClass = self.handler_modules[integration_engine].Handler
         handler = HandlerClass(**handler_ars)
         if connect:
@@ -731,6 +726,7 @@ class IntegrationController:
                     'dependencies': dependencies,
                 },
                 'name': handler_name,
+                'permanent': handler_info.get('permanent', False),
                 'connection_args': handler_info.get('connection_args', None),
                 'class_type': handler_info.get('class_type', None),
                 'type': handler_info.get('type')
@@ -914,5 +910,22 @@ class IntegrationController:
         if handler_meta["import"]["success"]:
             return self.handler_modules[handler_name]
 
+    def create_permanent_integrations(self):
+        for (
+            integration_name,
+            handler,
+        ) in self.get_handlers_metadata().items():
+            if handler.get("permanent"):
+                integration_meta = integration_controller.get(name=integration_name)
+                if integration_meta is None:
+                    integration_record = db.Integration(
+                        name=integration_name,
+                        data={},
+                        engine=integration_name,
+                        company_id=None,
+                    )
+                    db.session.add(integration_record)
+        db.session.commit()
+
 
 integration_controller = IntegrationController()

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/mindsdb/utilities/config.py

@@ -4,8 +4,9 @@ import datetime
 from copy import deepcopy
 from pathlib import Path
 
+from appdirs import user_data_dir
+
 from mindsdb.utilities import log
-from mindsdb.utilities.fs import create_directory, get_or_create_data_dir
 
 
 logger = log.getLogger(__name__)
@@ -28,6 +29,29 @@ def _merge_configs(original_config, override_config):
     return original_config
 
 
+def create_directory(path):
+    path = Path(path)
+    path.mkdir(mode=0o777, exist_ok=True, parents=True)
+
+
+def get_or_create_data_dir():
+    data_dir = user_data_dir("mindsdb", "mindsdb")
+    mindsdb_data_dir = os.path.join(data_dir, "var/")
+
+    if os.path.exists(mindsdb_data_dir) is False:
+        create_directory(mindsdb_data_dir)
+
+    try:
+        assert os.path.exists(mindsdb_data_dir)
+        assert os.access(mindsdb_data_dir, os.W_OK) is True
+    except Exception:
+        raise Exception(
+            "MindsDB storage directory does not exist and could not be created"
+        )
+
+    return mindsdb_data_dir
+
+
 config = None
 config_mtime = -1
 

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/mindsdb/utilities/fs.py

@@ -6,42 +6,18 @@ from pathlib import Path
 from typing import Optional, List, Tuple
 
 import psutil
-from appdirs import user_data_dir
 
 from mindsdb.utilities import log
 
 logger = log.getLogger(__name__)
 
 
-def create_directory(path):
-    path = Path(path)
-    path.mkdir(mode=0o777, exist_ok=True, parents=True)
-
-
-def get_or_create_data_dir():
-    data_dir = user_data_dir("mindsdb", "mindsdb")
-    mindsdb_data_dir = os.path.join(data_dir, "var/")
-
-    if os.path.exists(mindsdb_data_dir) is False:
-        create_directory(mindsdb_data_dir)
-
-    try:
-        assert os.path.exists(mindsdb_data_dir)
-        assert os.access(mindsdb_data_dir, os.W_OK) is True
-    except Exception:
-        raise Exception(
-            "MindsDB storage directory does not exist and could not be created"
-        )
-
-    return mindsdb_data_dir
-
-
 def create_dirs_recursive(path):
     if isinstance(path, dict):
         for p in path.values():
             create_dirs_recursive(p)
     elif isinstance(path, str):
-        create_directory(path)
+        Path(path).mkdir(mode=0o777, exist_ok=True, parents=True)
     else:
         raise ValueError(f"Wrong path: {path}")
 

{MindsDB-24.12.2.0 → MindsDB-24.12.2.1}/mindsdb/utilities/log.py

@@ -1,12 +1,11 @@
-import logging
 import os
+import logging
 from logging.config import dictConfig
 
 logging_initialized = False
 
 
 class ColorFormatter(logging.Formatter):
-
     green = "\x1b[32;20m"
     default = "\x1b[39;20m"
     yellow = "\x1b[33;20m"
@@ -29,7 +28,8 @@ class ColorFormatter(logging.Formatter):
 
 
 def configure_logging():
-    mindsdb_level = os.environ.get("MINDSDB_LOG_LEVEL", None)
+    mindsdb_level = os.environ.get("MINDSDB_LOG_LEVEL")
+
     if mindsdb_level is not None:
         mindsdb_level = getattr(logging, mindsdb_level)
     else:
@@ -37,11 +37,14 @@ def configure_logging():
 
     logging_config = dict(
         version=1,
-        formatters={"f": {"()": ColorFormatter}},
+        formatters={
+            "f": {"()": ColorFormatter}
+        },
         handlers={
             "console": {
                 "class": "logging.StreamHandler",
                 "formatter": "f",
+                "level": mindsdb_level
             }
         },
         loggers={

MindsDB-24.12.2.0/mindsdb/integrations/libs/handler_helpers.py

@@ -1,55 +0,0 @@
-import time
-import importlib
-from threading import Thread
-
-import requests
-
-from mindsdb.utilities import log
-logger = log.getLogger(__name__)
-
-
-def action_logger(logger, loglevel="info"):
-    def decorator(func):
-        def wrapper(*args, **kwargs):
-            log_func = getattr(logger, loglevel)
-            instance = args[0]
-            log_func("%s.%s: calling with args - %s, kwargs - %s", instance.__class__.__name__, func.__name__, args[1:], kwargs)
-            res = func(*args, **kwargs)
-            log_func("%s.%s: returning - %s", instance.__class__.__name__, func.__name__, res)
-            return res
-        return wrapper
-    return decorator
-
-
-def get_handler(_type):
-    _type = _type.lower()
-    # a crutch to fix bug in handler naming convention
-    if _type == "files":
-        _type = "file"
-    handler_folder_name = _type + "_handler"
-    logger.debug("get_handler: handler_folder - %s", handler_folder_name)
-
-    try:
-        handler_module = importlib.import_module(f'mindsdb.integrations.handlers.{handler_folder_name}')
-        logger.debug("get_handler: handler module - %s", handler_module)
-        handler = handler_module.Handler
-        if handler is None:
-            logger.error("get_handler: import error - %s", handler_module.import_error)
-        logger.debug("get_handler: found handler - %s", handler)
-        return handler
-    except Exception as e:
-        raise e
-
-
-def registry(url, data, interval):
-    def worker():
-        while True:
-            try:
-                requests.post(url, json=data, headers={"Content-Type": "application/json"})
-            except Exception as e:
-                logger.error("handler_register: unable to register the service - %s", e)
-            time.sleep(interval)
-
-    _th_worker = Thread(target=worker)
-    _th_worker.start()
-    logger.info("handler_register: registering service. url - %s, data - %s, interval - %s", url, data, interval)