MindsDB 25.9.2.0a1__py3-none-any.whl → 25.9.3rc1__py3-none-any.whl

mindsdb/integrations/libs/ml_handler_process/learn_process.py

@@ -1,6 +1,5 @@
 import os
 import importlib
-import traceback
 import datetime as dt
 
 from sqlalchemy.orm.attributes import flag_modified
@@ -25,56 +24,52 @@ from mindsdb.integrations.libs.ml_handler_process.handlers_cacher import handler
 logger = log.getLogger(__name__)
 
 
-@mark_process(name='learn')
-def learn_process(data_integration_ref: dict, problem_definition: dict, fetch_data_query: str,
-                  project_name: str, model_id: int, integration_id: int, base_model_id: int,
-                  set_active: bool, module_path: str):
-    ctx.profiling = {
-        'level': 0,
-        'enabled': True,
-        'pointer': None,
-        'tree': None
-    }
-    profiler.set_meta(query='learn_process', api='http', environment=Config().get('environment'))
-    with profiler.Context('learn_process'):
+@mark_process(name="learn")
+def learn_process(
+    data_integration_ref: dict,
+    problem_definition: dict,
+    fetch_data_query: str,
+    project_name: str,
+    model_id: int,
+    integration_id: int,
+    base_model_id: int,
+    set_active: bool,
+    module_path: str,
+):
+    ctx.profiling = {"level": 0, "enabled": True, "pointer": None, "tree": None}
+    profiler.set_meta(query="learn_process", api="http", environment=Config().get("environment"))
+    with profiler.Context("learn_process"):
         from mindsdb.interfaces.database.database import DatabaseController
 
         try:
             predictor_record = db.Predictor.query.with_for_update().get(model_id)
-            predictor_record.training_metadata['process_id'] = os.getpid()
-            flag_modified(predictor_record, 'training_metadata')
+            predictor_record.training_metadata["process_id"] = os.getpid()
+            flag_modified(predictor_record, "training_metadata")
             db.session.commit()
 
-            target = problem_definition.get('target', None)
+            target = problem_definition.get("target", None)
             training_data_df = None
             if data_integration_ref is not None:
                 database_controller = DatabaseController()
                 sql_session = make_sql_session()
-                if data_integration_ref['type'] == 'integration':
-                    integration_name = database_controller.get_integration(data_integration_ref['id'])['name']
+                if data_integration_ref["type"] == "integration":
+                    integration_name = database_controller.get_integration(data_integration_ref["id"])["name"]
                     query = Select(
                         targets=[Star()],
-                        from_table=NativeQuery(
-                            integration=Identifier(integration_name),
-                            query=fetch_data_query
-                        )
+                        from_table=NativeQuery(integration=Identifier(integration_name), query=fetch_data_query),
                     )
                     sqlquery = SQLQuery(query, session=sql_session)
-                if data_integration_ref['type'] == 'system':
+                if data_integration_ref["type"] == "system":
                     query = Select(
-                        targets=[Star()],
-                        from_table=NativeQuery(
-                            integration=Identifier('log'),
-                            query=fetch_data_query
-                        )
+                        targets=[Star()], from_table=NativeQuery(integration=Identifier("log"), query=fetch_data_query)
                     )
                     sqlquery = SQLQuery(query, session=sql_session)
-                elif data_integration_ref['type'] == 'view':
+                elif data_integration_ref["type"] == "view":
                     project = database_controller.get_project(project_name)
                     query_ast = parse_sql(fetch_data_query)
                     view_meta = project.get_view_meta(query_ast)
-                    sqlquery = SQLQuery(view_meta['query_ast'], session=sql_session)
-                elif data_integration_ref['type'] == 'project':
+                    sqlquery = SQLQuery(view_meta["query_ast"], session=sql_session)
+                elif data_integration_ref["type"] == "project":
                     query_ast = parse_sql(fetch_data_query)
                     sqlquery = SQLQuery(query_ast, session=sql_session)
 
@@ -97,17 +92,13 @@ def learn_process(data_integration_ref: dict, problem_definition: dict, fetch_da
 
             handlerStorage = HandlerStorage(integration_id)
             modelStorage = ModelStorage(model_id)
-            modelStorage.fileStorage.push()     # FIXME
+            modelStorage.fileStorage.push()  # FIXME
 
             kwargs = {}
             if base_model_id is not None:
-                kwargs['base_model_storage'] = ModelStorage(base_model_id)
-                kwargs['base_model_storage'].fileStorage.pull()
-            ml_handler = module.Handler(
-                engine_storage=handlerStorage,
-                model_storage=modelStorage,
-                **kwargs
-            )
+                kwargs["base_model_storage"] = ModelStorage(base_model_id)
+                kwargs["base_model_storage"].fileStorage.pull()
+            ml_handler = module.Handler(engine_storage=handlerStorage, model_storage=modelStorage, **kwargs)
             handlers_cacher[predictor_record.id] = ml_handler
 
             if not ml_handler.generative and target is not None:
@@ -119,18 +110,19 @@ def learn_process(data_integration_ref: dict, problem_definition: dict, fetch_da
                         training_data_df.rename(columns={target_found: target}, inplace=True)
                     else:
                         raise Exception(
-                            f'Prediction target "{target}" not found in training dataframe: {list(training_data_df.columns)}')
+                            f'Prediction target "{target}" not found in training dataframe: {list(training_data_df.columns)}'
+                        )
 
             # create new model
             if base_model_id is None:
-                with profiler.Context('create'):
+                with profiler.Context("create"):
                     ml_handler.create(target, df=training_data_df, args=problem_definition)
 
             # fine-tune (partially train) existing model
             else:
                 # load model from previous version, use it as starting point
-                with profiler.Context('finetune'):
-                    problem_definition['base_model_id'] = base_model_id
+                with profiler.Context("finetune"):
+                    problem_definition["base_model_id"] = base_model_id
                     ml_handler.finetune(df=training_data_df, args=problem_definition)
 
             predictor_record.status = PREDICTOR_STATUS.COMPLETE
@@ -139,9 +131,7 @@ def learn_process(data_integration_ref: dict, problem_definition: dict, fetch_da
             # if retrain and set_active after success creation
             if set_active is True:
                 models = get_model_records(
-                    name=predictor_record.name,
-                    project_id=predictor_record.project_id,
-                    active=None
+                    name=predictor_record.name, project_id=predictor_record.project_id, active=None
                 )
                 for model in models:
                     model.active = False
@@ -149,7 +139,7 @@ def learn_process(data_integration_ref: dict, problem_definition: dict, fetch_da
                 models.sort(key=lambda x: x.created_at)
                 models[-1].active = True
         except Exception as e:
-            logger.error(traceback.format_exc())
+            logger.exception("Error during 'learn' process:")
             error_message = format_exception_error(e)
 
             predictor_record = db.Predictor.query.with_for_update().get(model_id)

mindsdb/integrations/libs/ml_handler_process/update_engine_process.py

@@ -13,16 +13,13 @@ def update_engine_process(connection_args: dict, integration_id: int, module_pat
 
     result = None
 
-    if hasattr(module.Handler, 'update_engine'):
+    if hasattr(module.Handler, "update_engine"):
         engine_storage = HandlerStorage(integration_id)
         try:
-            result = module.Handler(
-                engine_storage=engine_storage,
-                model_storage=None
-            ).update_engine(connection_args=connection_args)
+            result = module.Handler(engine_storage=engine_storage, model_storage=None).update_engine(
+                connection_args=connection_args
+            )
         except NotImplementedError:
             return None
-        except Exception as e:
-            raise e
 
     return result

mindsdb/integrations/libs/ml_handler_process/update_process.py

@@ -11,17 +11,12 @@ def update_process(args: dict, integration_id: int, module_path: str, model_id:
 
     result = None
 
-    if hasattr(module.Handler, 'upgate'):
+    if hasattr(module.Handler, "upgate"):
         engine_storage = HandlerStorage(integration_id)
         model_storage = ModelStorage(model_id)
         try:
-            result = module.Handler(
-                engine_storage=engine_storage,
-                model_storage=model_storage
-            ).upgate(args=args)
+            result = module.Handler(engine_storage=engine_storage, model_storage=model_storage).upgate(args=args)
         except NotImplementedError:
             return None
-        except Exception as e:
-            raise e
 
     return result