MindsDB 1.2.9__py3-none-any.whl → 24.12.4.0__py3-none-any.whl

mindsdb/integrations/handlers/lightfm_handler/helpers.py

@@ -0,0 +1,175 @@
+import lightfm
+import numpy as np
+import pandas as pd
+
+
+def get_item_user_idx(args: dict, n_users, n_items, item_ids=None, user_ids=None):
+    """
+    gets item and user idxs from item_ids and user_ids
+    :param args:
+    :param n_users:
+    :param n_items:
+    :param item_ids:
+    :param user_ids:
+    :return:
+    """
+    if item_ids and user_ids:
+        item_idx = np.array(
+            [int(args["item_id_to_idx_map"][item_id]) for item_id in item_ids]
+        )
+        user_idx = np.array(
+            [int(args["user_id_to_idx_map"][user_id]) for user_id in user_ids]
+        )
+
+        # repeat each user id index n_items times
+        user_idxs = np.repeat(user_idx, n_items)
+
+        # repeat the full list of item indexes n_user times
+        item_idxs = np.tile(item_idx, n_users)
+
+    elif item_ids and not user_ids:
+        item_idx = np.array(
+            [int(args["item_id_to_idx_map"][item_id]) for item_id in item_ids]
+        )
+
+        user_idxs = np.repeat([i for i in range(0, n_users)], n_items)
+        item_idxs = np.tile(item_idx, n_users)
+
+    elif user_ids and not item_ids:
+        user_idx = np.array(
+            [int(args["user_id_to_idx_map"][user_id]) for user_id in user_ids]
+        )
+
+        user_idxs = np.repeat(user_idx, n_items)
+        item_idxs = np.tile([i for i in range(0, n_items)], n_users)
+
+    else:
+        user_idxs = np.repeat([i for i in range(0, n_users)], n_items)
+        item_idxs = np.tile([i for i in range(0, n_items)], n_users)
+
+    return item_idxs, user_idxs
+
+
+def get_user_item_recommendations(
+    n_users: int, n_items: int, args: dict, item_ids, user_ids, model: lightfm.LightFM
+):
+    """
+    gets N user-item recommendations for a given model
+    :param n_users:
+    :param n_items:
+    :param args:
+    :param item_ids:
+    :param user_ids:
+    :param model:
+
+    :return:
+    """
+    # get idxs for user-item pairs
+    item_idxs, user_idxs = get_item_user_idx(args, n_users, n_items, item_ids, user_ids)
+
+    scores = model.predict(user_ids=user_idxs, item_ids=item_idxs)
+
+    # map scores to user-item pairs, sort by score and return top N recommendations per user
+    user_item_recommendations_df = (
+        pd.DataFrame({"user_idx": user_idxs, "item_idx": item_idxs, "score": scores})
+        .groupby("user_idx")
+        .apply(
+            lambda x: x.sort_values("score", ascending=False).head(
+                args["n_recommendations"]
+            )
+        )
+    )
+
+    # map idxs to item ids and user ids
+    user_item_recommendations_df["item_id"] = (
+        user_item_recommendations_df["item_idx"]
+        .astype("str")
+        .map(args["item_idx_to_id_map"])
+    )
+    user_item_recommendations_df["user_id"] = (
+        user_item_recommendations_df["user_idx"]
+        .astype("str")
+        .map(args["user_idx_to_id_map"])
+    )
+
+    return user_item_recommendations_df[["user_id", "item_id", "score"]]
+
+
+def get_item_item_recommendations(
+    model: lightfm.LightFM,
+    args: dict,
+    item_ids: list = None,
+    item_features=None,
+) -> pd.DataFrame:
+    """
+    gets similar items to a given item index inside user-item interaction matrix
+    NB by default it won't use item features,however if item features are provided
+    it will use them to get similar items
+
+    :param args:
+    :param model:
+    :param item_ids:
+    :param item_features:
+
+    :return:
+    """
+    # todo make sure its not slow across larger data
+    # todo break into smaller functions
+
+    n_recommendations = args["n_recommendations"]
+
+    similar_items_dfs = []
+
+    item_idx_to_id_map = args["item_idx_to_id_map"]
+
+    if item_ids:
+        # filter out item_ids that are not in the request
+        item_idx_to_id_map = {
+            key: val
+            for key, val in args["item_idx_to_id_map"].items()
+            if val in item_ids
+        }
+
+    for item_idx, item_id in item_idx_to_id_map.items():
+        # ensure item_idx is int
+        item_idx = int(item_idx)
+
+        item_biases, item_representations = model.get_item_representations(
+            features=item_features
+        )
+
+        # Cosine similarity
+        scores = item_representations.dot(item_representations[item_idx, :])
+
+        # normalize
+        item_norms = np.sqrt((item_representations * item_representations).sum(axis=1))
+        scores /= item_norms
+
+        # ensure n_recommendations is not greater than number of recommendations
+        N = min(len(scores), n_recommendations)
+
+        # get the top N items
+        best = np.argpartition(scores, -N)
+
+        # sort the scores
+        rec = sorted(
+            zip(best, scores[best] / item_norms[item_idx]), key=lambda x: -x[1]
+        )
+
+        intermediate_df = (
+            pd.DataFrame(rec, columns=["item_idx", "score"])
+            .tail(-1)  # remove the item itself
+            .head(N)
+        )
+
+        intermediate_df["item_id_one"] = item_id
+        similar_items_dfs.append(intermediate_df)
+
+    similar_items_df = pd.concat(similar_items_dfs, ignore_index=True)
+    similar_items_df["item_idx"] = similar_items_df["item_idx"].astype("str")
+
+    similar_items_df["item_id_two"] = similar_items_df["item_idx"].map(
+        args["item_idx_to_id_map"]
+    )
+
+    return similar_items_df[["item_id_one", "item_id_two", "score"]]

mindsdb/integrations/handlers/lightfm_handler/icon.svg

@@ -0,0 +1,14 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="40" height="40" viewBox="0 0 40 40" fill="none">
+  <path d="M1.66644 9.99988V22.0599" stroke="black" stroke-width="2.28144" stroke-linecap="round" stroke-linejoin="round"/>
+  <path d="M5.41522 15.8661L5.41522 22.0591" stroke="black" stroke-width="2.28144" stroke-linecap="round" stroke-linejoin="round"/>
+  <path d="M5.57779 12.6079L5.57779 13.2598" stroke="black" stroke-width="2.28144" stroke-linecap="round" stroke-linejoin="round"/>
+  <path d="M9.16315 13.9109L9.16315 24.0152" stroke="black" stroke-width="2.28144" stroke-linecap="round" stroke-linejoin="round"/>
+  <path d="M9.16315 27.2737L9.16315 29.2294" stroke="black" stroke-width="2.28144" stroke-linecap="round" stroke-linejoin="round"/>
+  <path d="M12.9103 9.99976L12.9103 22.0598" stroke="black" stroke-width="2.28144" stroke-linecap="round" stroke-linejoin="round"/>
+  <path d="M16.6597 13.9109L16.6597 22.0596" stroke="black" stroke-width="2.28144" stroke-linecap="round" stroke-linejoin="round"/>
+  <path d="M20.4076 9.99976L20.4076 22.0598" stroke="black" stroke-width="2.28144" stroke-linecap="round" stroke-linejoin="round"/>
+  <path d="M26.4372 9.99976L26.4372 27.6009" stroke="black" stroke-width="2.28144" stroke-linecap="round" stroke-linejoin="round"/>
+  <path d="M30.3476 13.9109L30.3476 22.0596" stroke="black" stroke-width="2.28144" stroke-linecap="round" stroke-linejoin="round"/>
+  <path d="M34.2587 13.9109L34.2587 22.0596" stroke="black" stroke-width="2.28144" stroke-linecap="round" stroke-linejoin="round"/>
+  <path d="M38.1698 13.9109L38.1698 22.0596" stroke="black" stroke-width="2.28144" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

mindsdb/integrations/handlers/lightfm_handler/lightfm_handler.py

@@ -0,0 +1,198 @@
+from typing import Dict, Optional
+
+import dill
+import pandas as pd
+from dataprep_ml.recommenders import RecommenderPreprocessor
+from lightfm import LightFM
+from lightfm.cross_validation import random_train_test_split
+from lightfm.evaluation import auc_score, precision_at_k, recall_at_k
+
+from mindsdb.integrations.handlers.lightfm_handler.helpers import (
+    get_item_item_recommendations,
+    get_user_item_recommendations,
+)
+from mindsdb.integrations.handlers.lightfm_handler.settings import ModelParameters
+from mindsdb.integrations.libs.base import BaseMLEngine
+
+
+class LightFMHandler(BaseMLEngine):
+    """
+    Integration with the lightfm Recommender library.
+    """
+
+    name = "lightfm"
+
+    # todo add ability to partially update model based on new data for existing users, items
+    # todo add support for hybrid recommender
+    def create(self, target: str, df: pd.DataFrame = None, args: Optional[Dict] = None):
+
+        args = args["using"]
+
+        # get model parameters if defined by user - else use default values
+
+        user_defined_model_params = list(
+            filter(lambda x: x in args, ["learning_rate", "loss", "epochs"])
+        )
+        args["model_params"] = {
+            model_param: args[model_param] for model_param in user_defined_model_params
+        }
+        model_parameters = ModelParameters(**args["model_params"])
+
+        # store model parameters
+        args["model_params"] = model_parameters.model_dump()
+
+        rec_preprocessor = RecommenderPreprocessor(
+            interaction_data=df,
+            user_id_column_name=args["user_id"],
+            item_id_column_name=args["item_id"],
+            threshold=args["threshold"],
+        )
+
+        # preprocess data
+        preprocessed_data = rec_preprocessor.preprocess()
+
+        args["n_users_items"] = rec_preprocessor.n_users_items
+
+        # get item idx to id and user idx to id maps
+        args["item_idx_to_id_map"] = preprocessed_data.idx_item_map
+        args["user_idx_to_id_map"] = preprocessed_data.idx_user_map
+
+        random_state = 42
+
+        # run evaluation if specified
+        if args.get("evaluation"):
+
+            evaluation_metrics = self.evaluate(
+                preprocessed_data.interaction_matrix, random_state, model_parameters
+            )
+            # convert to float to str so it can be stored in json
+            args["evaluation_metrics"] = {
+                k: str(v) for k, v in evaluation_metrics.items()
+            }
+
+        # train model
+        model = LightFM(
+            learning_rate=model_parameters.learning_rate,
+            loss=model_parameters.loss,
+            random_state=random_state,
+        )
+        model.fit(preprocessed_data.interaction_matrix, epochs=model_parameters.epochs)
+
+        self.model_storage.file_set("model", dill.dumps(model))
+        self.model_storage.json_set("args", args)
+
+    def predict(self, df: Optional[pd.DataFrame] = None, args: Optional[dict] = None):
+
+        predict_params = args["predict_params"]
+
+        # if user doesn't specify recommender type, default to user_item
+        try:
+            recommender_type = df["recommender_type"].tolist()[0]
+        except KeyError:
+            recommender_type = predict_params.get("recommender_type", "user_item")
+
+        args = self.model_storage.json_get("args")
+
+        # get item_id to idx and user_id to idx maps
+        args["item_id_to_idx_map"] = dict(
+            zip(args["item_idx_to_id_map"].values(), args["item_idx_to_id_map"].keys())
+        )
+        args["user_id_to_idx_map"] = dict(
+            zip(args["user_idx_to_id_map"].values(), args["user_idx_to_id_map"].keys())
+        )
+
+        model = dill.loads(self.model_storage.file_get("model"))
+
+        n_users = args["n_users_items"][0]
+        n_items = args["n_users_items"][1]
+        item_ids, user_ids = None, None
+
+        if recommender_type == "user_item":
+            if df is not None:
+
+                if args["item_id"] in df.columns:
+                    n_items = df[args["item_id"]].nunique()
+                    item_ids = df[args["item_id"]].unique().tolist()
+
+                if args["user_id"] in df.columns:
+                    n_users = df[args["user_id"]].nunique()
+                    user_ids = df[args["user_id"]].unique().tolist()
+
+            return get_user_item_recommendations(
+                n_users=n_users,
+                n_items=n_items,
+                args=args,
+                model=model,
+                item_ids=item_ids,
+                user_ids=user_ids,
+            )
+
+        elif recommender_type == "item_item":
+            if df is not None and args["item_id"] in df.columns:
+                item_ids = df[args["item_id"]].unique().tolist()
+
+            return get_item_item_recommendations(
+                model=model,
+                args=args,
+                item_ids=item_ids,
+            )
+
+        elif recommender_type == "user_user":
+            raise NotImplementedError(
+                "user_user recommendation type is not implemented yet"
+            )
+
+        else:
+            raise ValueError(
+                "recommender_type must be either 'user_item', 'item_item' or 'user_user'"
+            )
+
+    def describe(self, attribute=None):
+        model_args = self.model_storage.json_get("args")
+
+        if attribute == "model":
+            return pd.DataFrame({k: [model_args[k]] for k in ["model_params"]})
+
+        elif attribute == "features":
+            return pd.DataFrame(
+                {
+                    "n_users_items": [model_args["n_users_items"]],
+                }
+            )
+
+        elif attribute == "info":
+
+            model_metrics = model_args["evaluation_metrics"]
+
+            info_dict = {k: [model_metrics[k]] for k in ["auc", "precision", "recall"]}
+            info_dict["user_id"] = [model_args["user_id"]]
+            info_dict["item_id"] = [model_args["item_id"]]
+
+            return pd.DataFrame(info_dict)
+
+        else:
+            tables = ["info", "features", "model"]
+            return pd.DataFrame(tables, columns=["tables"])
+
+    def evaluate(self, interaction_matrix, random_state, model_parameters):
+
+        train, test = random_train_test_split(
+            interaction_matrix,
+            test_percentage=0.2,
+            random_state=random_state,
+        )
+        model = LightFM(
+            learning_rate=model_parameters.learning_rate,
+            loss=model_parameters.loss,
+            random_state=random_state,
+        )
+
+        model.fit(train, epochs=model_parameters.epochs)
+
+        evaluation_metrics = dict(
+            auc=auc_score(model, test, train).mean(),
+            precision=precision_at_k(model, test, train).mean(),
+            recall=recall_at_k(model, test, train).mean(),
+        )
+
+        return evaluation_metrics

mindsdb/integrations/handlers/lightfm_handler/requirements.txt

@@ -0,0 +1 @@
+lightfm==1.17

mindsdb/integrations/handlers/lightfm_handler/settings.py

@@ -0,0 +1,7 @@
+from pydantic import BaseModel
+
+
+class ModelParameters(BaseModel):
+    learning_rate: float = 0.05
+    loss: str = "warp"
+    epochs: int = 10

mindsdb/integrations/handlers/lightwood_handler/__about__.py

@@ -0,0 +1,9 @@
+__title__ = 'MindsDB Lightwood handler'
+__package_name__ = 'mindsdb_lightwood_handler'
+__version__ = '1.0.0'
+__description__ = "MindsDB handler for Lightwood"
+__author__ = 'MindsDB Inc'
+__github__ = 'https://github.com/mindsdb/mindsdb'
+__pypi__ = 'https://pypi.org/project/mindsdb/'
+__license__ = 'MIT'
+__copyright__ = 'Copyright 2022- mindsdb'

mindsdb/integrations/handlers/lightwood_handler/__init__.py

@@ -0,0 +1,19 @@
+from mindsdb.integrations.libs.const import HANDLER_TYPE
+
+from .__about__ import __description__ as description
+from .__about__ import __version__ as version
+
+try:
+    from .lightwood_handler import LightwoodHandler as Handler
+
+    import_error = None
+except Exception as e:
+    Handler = None
+    import_error = e
+
+title = 'Lightwood'
+name = 'lightwood'
+type = HANDLER_TYPE.ML
+icon_path = "icon.svg"
+
+__all__ = ['Handler', 'version', 'name', 'type', 'title', 'description', 'import_error', 'icon_path']

mindsdb/integrations/handlers/lightwood_handler/functions.py

@@ -0,0 +1,252 @@
+import dataclasses
+import json
+import os
+import tempfile
+import traceback
+from datetime import datetime
+from pathlib import Path
+
+import lightwood
+import pandas as pd
+import requests
+from lightwood.api.types import JsonAI, ProblemDefinition
+from pandas.core.frame import DataFrame
+
+import mindsdb.utilities.profiler as profiler
+from mindsdb.integrations.libs.const import PREDICTOR_STATUS
+from mindsdb.integrations.utilities.utils import format_exception_error
+from mindsdb.interfaces.model.functions import get_model_records
+from mindsdb.interfaces.storage import db
+from mindsdb.interfaces.storage.fs import RESOURCE_GROUP, FileStorage
+from mindsdb.interfaces.storage.json import get_json_storage
+from mindsdb.utilities import log
+from mindsdb.utilities.functions import mark_process
+
+from .utils import brack_to_mod, rep_recur, unpack_jsonai_old_args
+
+logger = log.getLogger(__name__)
+
+def create_learn_mark():
+    if os.name == 'posix':
+        p = Path(tempfile.gettempdir()).joinpath('mindsdb/learn_processes/')
+        p.mkdir(parents=True, exist_ok=True)
+        p.joinpath(f'{os.getpid()}').touch()
+
+
+def delete_learn_mark():
+    if os.name == 'posix':
+        p = (
+            Path(tempfile.gettempdir())
+            .joinpath('mindsdb/learn_processes/')
+            .joinpath(f'{os.getpid()}')
+        )
+        if p.exists():
+            p.unlink()
+
+
+@mark_process(name='learn')
+@profiler.profile()
+def run_generate(df: DataFrame, predictor_id: int, model_storage, args: dict = None):
+
+    model_storage.training_state_set(
+        current_state_num=1, total_states=5, state_name='Generating problem definition'
+    )
+    json_ai_override = args.pop('using', {})
+
+    if 'dtype_dict' in json_ai_override:
+        args['dtype_dict'] = json_ai_override.pop('dtype_dict')
+
+    if 'problem_definition' in json_ai_override:
+        args = {**args, **json_ai_override['problem_definition']}
+
+    if 'timeseries_settings' in args:
+        for tss_key in [
+            f.name for f in dataclasses.fields(lightwood.api.TimeseriesSettings)
+        ]:
+            k = f'timeseries_settings.{tss_key}'
+            if k in json_ai_override:
+                args['timeseries_settings'][tss_key] = json_ai_override.pop(k)
+
+    problem_definition = lightwood.ProblemDefinition.from_dict(args)
+
+    model_storage.training_state_set(
+        current_state_num=2, total_states=5, state_name='Generating JsonAI'
+    )
+    json_ai = lightwood.json_ai_from_problem(df, problem_definition)
+    json_ai = json_ai.to_dict()
+    unpack_jsonai_old_args(json_ai_override)
+    json_ai_override = brack_to_mod(json_ai_override)
+    rep_recur(json_ai, json_ai_override)
+    json_ai = JsonAI.from_dict(json_ai)
+
+    model_storage.training_state_set(
+        current_state_num=3, total_states=5, state_name='Generating code'
+    )
+    code = lightwood.code_from_json_ai(json_ai)
+
+    predictor_record = db.Predictor.query.with_for_update().get(predictor_id)
+    predictor_record.code = code
+    db.session.commit()
+
+    json_storage = get_json_storage(resource_id=predictor_id)
+    json_storage.set('json_ai', json_ai.to_dict())
+
+
+@mark_process(name='learn')
+@profiler.profile()
+def run_fit(predictor_id: int, df: pd.DataFrame, model_storage) -> None:
+    try:
+        predictor_record = db.Predictor.query.with_for_update().get(predictor_id)
+        assert predictor_record is not None
+
+        predictor_record.data = {'training_log': 'training'}
+        predictor_record.status = PREDICTOR_STATUS.TRAINING
+        db.session.commit()
+
+        model_storage.training_state_set(
+            current_state_num=4, total_states=5, state_name='Training model'
+        )
+        predictor: lightwood.PredictorInterface = lightwood.predictor_from_code(
+            predictor_record.code
+        )
+        predictor.learn(df)
+
+        db.session.refresh(predictor_record)
+
+        fs = FileStorage(
+            resource_group=RESOURCE_GROUP.PREDICTOR, resource_id=predictor_id, sync=True
+        )
+        predictor.save(fs.folder_path / fs.folder_name)
+        fs.push(compression_level=0)
+
+        predictor_record.data = predictor.model_analysis.to_dict()
+
+        # getting training time for each tried model. it is possible to do
+        # after training only
+        fit_mixers = list(
+            predictor.runtime_log[x]
+            for x in predictor.runtime_log
+            if isinstance(x, tuple) and x[0] == "fit_mixer"
+        )
+        submodel_data = predictor_record.data.get("submodel_data", [])
+        # add training time to other mixers info
+        if submodel_data and fit_mixers and len(submodel_data) == len(fit_mixers):
+            for i, tr_time in enumerate(fit_mixers):
+                submodel_data[i]["training_time"] = tr_time
+        predictor_record.data["submodel_data"] = submodel_data
+
+        model_storage.training_state_set(
+            current_state_num=5, total_states=5, state_name='Complete'
+        )
+        predictor_record.dtype_dict = predictor.dtype_dict
+        db.session.commit()
+    except Exception as e:
+        db.session.refresh(predictor_record)
+        predictor_record.data = {'error': f'{traceback.format_exc()}\nMain error: {e}'}
+        db.session.commit()
+        raise e
+
+
+@mark_process(name='learn')
+def run_learn_remote(df: DataFrame, predictor_id: int) -> None:
+    try:
+        serialized_df = json.dumps(df.to_dict())
+        predictor_record = db.Predictor.query.with_for_update().get(predictor_id)
+        resp = requests.post(
+            predictor_record.data['train_url'],
+            json={'df': serialized_df, 'target': predictor_record.to_predict[0]},
+        )
+
+        assert resp.status_code == 200
+        predictor_record.data['status'] = 'complete'
+    except Exception:
+        predictor_record.data['status'] = 'error'
+        predictor_record.data['error'] = str(resp.text)
+
+    db.session.commit()
+
+
+@mark_process(name='learn')
+def run_learn(df: DataFrame, args: dict, model_storage) -> None:
+    if df is None or df.shape[0] == 0:
+        raise Exception(
+            'No input data. Ensure the data source is healthy and try again.'
+        )
+
+    predictor_id = model_storage.predictor_id
+
+    predictor_record = db.Predictor.query.with_for_update().get(predictor_id)
+    predictor_record.training_start_at = datetime.now()
+    db.session.commit()
+
+    run_generate(df, predictor_id, model_storage, args)
+    run_fit(predictor_id, df, model_storage)
+
+    predictor_record.status = PREDICTOR_STATUS.COMPLETE
+    predictor_record.training_stop_at = datetime.now()
+    db.session.commit()
+
+
+@mark_process(name='finetune')
+def run_finetune(df: DataFrame, args: dict, model_storage):
+    try:
+        if df is None or df.shape[0] == 0:
+            raise Exception(
+                'No input data. Ensure the data source is healthy and try again.'
+            )
+
+        base_predictor_id = args['base_model_id']
+        base_predictor_record = db.Predictor.query.get(base_predictor_id)
+        if base_predictor_record.status != PREDICTOR_STATUS.COMPLETE:
+            raise Exception("Base model must be in status 'complete'")
+
+        predictor_id = model_storage.predictor_id
+        predictor_record = db.Predictor.query.get(predictor_id)
+
+        # TODO move this to ModelStorage (don't work with database directly)
+        predictor_record.data = {'training_log': 'training'}
+        predictor_record.training_start_at = datetime.now()
+        predictor_record.status = (
+            PREDICTOR_STATUS.FINETUNING
+        )  # TODO: parallel execution block
+        db.session.commit()
+
+        base_fs = FileStorage(
+            resource_group=RESOURCE_GROUP.PREDICTOR,
+            resource_id=base_predictor_id,
+            sync=True,
+        )
+        predictor = lightwood.predictor_from_state(
+            base_fs.folder_path / base_fs.folder_name, base_predictor_record.code
+        )
+        predictor.adjust(df, adjust_args=args.get('using', {}))
+
+        fs = FileStorage(
+            resource_group=RESOURCE_GROUP.PREDICTOR, resource_id=predictor_id, sync=True
+        )
+        predictor.save(fs.folder_path / fs.folder_name)
+        fs.push(compression_level=0)
+
+        predictor_record.data = (
+            predictor.model_analysis.to_dict()
+        )  # todo: update accuracy in LW as post-finetune hook
+        predictor_record.code = base_predictor_record.code
+        predictor_record.update_status = 'up_to_date'
+        predictor_record.status = PREDICTOR_STATUS.COMPLETE
+        predictor_record.training_stop_at = datetime.now()
+        db.session.commit()
+
+    except Exception as e:
+        logger.error(e)
+        predictor_id = model_storage.predictor_id
+        predictor_record = db.Predictor.query.with_for_update().get(predictor_id)
+        logger.error(traceback.format_exc())
+        error_message = format_exception_error(e)
+        predictor_record.data = {"error": error_message}
+        predictor_record.status = PREDICTOR_STATUS.ERROR
+        db.session.commit()
+        raise
+    finally:
+        if predictor_record.training_stop_at is None:
+            predictor_record.training_stop_at = datetime.now()
+            db.session.commit()