lecrapaud 0.2.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

{lecrapaud-0.2.1.dist-info → lecrapaud-0.3.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 lecrapaud/__init__.py,sha256=oCxbtw_nk8rlOXbXbWo0RRMlsh6w-hTiZ6e5PRG_wp0,28
-lecrapaud/api.py,sha256=VpVj6zRanLzHo2bwbkar-HcOKsJPEDghWhxbrqDOxp4,9749
+lecrapaud/api.py,sha256=3yAoHTJt1XtjCrLemwZnoY7UyTdsZo9a7-jgbkOnirc,9912
 lecrapaud/config.py,sha256=Jf7kknumJ9NudVoLugOxqLqB7zTSl67IVV2VVWA0Pv4,1027
 lecrapaud/db/__init__.py,sha256=82o9fMfaqKXPh2_rt44EzNRVZV1R4LScEnQYvj_TjK0,34
 lecrapaud/db/alembic/README,sha256=MVlc9TYmr57RbhXET6QxgyCcwWP7w-vLkEsirENqiIQ,38
@@ -16,7 +16,7 @@ lecrapaud/db/alembic/versions/2025_05_27_2047-6b6f2d38e9bc_double_instead_of_flo
 lecrapaud/db/alembic/versions/2025_05_31_1111-c175e4a36d68_generalise_stock_to_group.py,sha256=oezuATpCoIsWEuMNB4M19LIb6y_a3UHskFjf7nF_-gk,1312
 lecrapaud/db/alembic/versions/2025_05_31_1256-5681095bfc27_create_investment_run_and_portfolio_.py,sha256=6kdAkFkVQi7hJQx-B1YhA_og2HkGep1gHITXivMNZR4,2850
 lecrapaud/db/alembic/versions/2025_05_31_1806-339927587383_add_investment_run_id.py,sha256=AdN1DzPOgtMDQQFbM62URsv1IxbPUO1_kNzy1Yl1oIc,3346
-lecrapaud/db/alembic/versions/2025_05_31_1834-52b809a34371_make_nullablee.py,sha256=40luKBAdfTJxhSgzKogm3nPbo4xvJKJmIEKcoD9V7tk,1331
+lecrapaud/db/alembic/versions/2025_05_31_1834-52b809a34371_make_nullablee.py,sha256=mnwDhWUFd-7FECHSn0_OGI93jxOeK4H2L4gp3ioqTDE,1234
 lecrapaud/db/alembic/versions/2025_05_31_1849-3b8550297e8e_change_date_to_datetime.py,sha256=d5swE95kdKovIBClS_tmTgNtN8epgMMz1X7DxPI17Lw,1356
 lecrapaud/db/alembic/versions/2025_05_31_1852-e6b8c95d8243_add_date_to_portfolio_history.py,sha256=veTx2DTbJDZ-wUhQTHJRA9lKam6To45jtRKT1UTX93I,829
 lecrapaud/db/alembic/versions/2025_06_10_1136-db8cdd83563a_addnewsandoptiontodata.py,sha256=V_Wjm-XMfMyG988R-3ii3-fzHP_tb2FtZmc4C6WkQOM,907
@@ -32,19 +32,17 @@ lecrapaud/db/models/model_selection.py,sha256=Lj2WZBWvaVpJeDjdiAs8wvkoI-XrWv2IIj
 lecrapaud/db/models/model_training.py,sha256=2xv9SOh8IUV1ROSsKlooAGanl9G8nBdmr7vaV4GAWtw,1618
 lecrapaud/db/models/score.py,sha256=I6kA55ZQhReo9dA_JEgeRSqDoFKmYyL3l-zCffwsofs,1719
 lecrapaud/db/models/target.py,sha256=scEtXVO3oysEvg9uwm_ncsS3-4o2A6z6XiDVGrg5WHE,1644
-lecrapaud/db/session.py,sha256=eD3zNodgC_ZEiNrSnhooMsaMiyccK9xgeReABWrbAuk,1629
-lecrapaud/directory_management.py,sha256=wrHmAJ-cBKb0GhJifxrb_RoLhZJX8xdkeirrWs7jQHk,791
-lecrapaud/experiment.py,sha256=8Y7HX5MGifrcquOML1lYXG-q40Wn4WBBRqLfPs1tLuk,1994
-lecrapaud/feature_engineering.py,sha256=zia2qxe_KLBUZV2KYVPqGd2K55AjHiyeeCm_GhobJHE,30327
-lecrapaud/feature_selection.py,sha256=2h3S_c0hYZM-EKenpVRYTFFqT4T7fg2lJ5pY2gUDalk,43023
+lecrapaud/db/session.py,sha256=LW97WdE4W48Fkx5fMrgDUKjwq54M2pfjqHjKwvUvAi0,2078
+lecrapaud/directories.py,sha256=wrHmAJ-cBKb0GhJifxrb_RoLhZJX8xdkeirrWs7jQHk,791
+lecrapaud/experiment.py,sha256=ApsGtGaISYsQCdAGa1llzBslfODK-w1_zf44p6Es3Xs,1985
+lecrapaud/feature_engineering.py,sha256=MEPW8-AgO3xnbwMHV3dC_gMtrBeKQhaCDrAicWgXX6w,30251
+lecrapaud/feature_selection.py,sha256=RD016F9_r249VEAe5bLCpB4tHQSbhdMoXRFohkk3c5g,42429
 lecrapaud/integrations/openai_integration.py,sha256=hHLF3fk5Bps8KNbNrEL3NUFa945jwClE6LrLpuMZOd4,7459
 lecrapaud/jobs/__init__.py,sha256=ZkrsyTOR21c_wN7RY8jPhm8jCrL1oCEtTsf3VFIlQiE,292
 lecrapaud/jobs/config.py,sha256=AmO0j3RFjx8H66dfKw_7vnshaOJb9Ox5BAZ9cwwLFMY,377
 lecrapaud/jobs/scheduler.py,sha256=SiYWPxokpKnR8V6btLOO6gbK0PEjSRoeG0kCbQvYPf4,990
 lecrapaud/jobs/tasks.py,sha256=evzhOHpgp6Gvoz__jUipi-_HSNny7bWgAauHv2Hpxyk,1640
-lecrapaud/model_selection.py,sha256=3BGp9mM9pZECcEcxLvm5YBJZ9Q7CJxZEC2zVWOm9g2o,61153
-lecrapaud/predictions.py,sha256=Uw6Ghm05KcoZIB1m9YLBe4v7sV42_Hv1oPgYfKsF2iY,10754
-lecrapaud/preprocessing.py,sha256=GfOTdVpubf9w13bbgKjcx12oP3mefaTgm1HK1V3tVeQ,34424
+lecrapaud/model_selection.py,sha256=y01yOK5oB9hdeEJkjjavDJvYAUCR3PS6FWRAN43uDzo,61322
 lecrapaud/search_space.py,sha256=6kVXDSmvpTuLKTlqQkKIzrJOM2P7HpqRiO3PR37VrsM,34123
 lecrapaud/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lecrapaud/services/embedding_categorical.py,sha256=jPKl3GjJ8STk5u0ux9VHbUYq-XRuNULhMJuXdeJAePU,2593
@@ -55,9 +53,8 @@ lecrapaud/speed_tests/test-gpu-resnet.ipynb,sha256=27Vu7nYwujYeh3fOxBNCnKJn3MXNP
 lecrapaud/speed_tests/test-gpu-transformers.ipynb,sha256=k6MBSs_Um1h4PykvE-LTBcdpbWLbIFST_xl_AFW2jgI,8444
 lecrapaud/speed_tests/tests.ipynb,sha256=RjI7LDHSsbadUkea_hT14sD7ivljtIQk4NB5McXJ1bE,3835
 lecrapaud/speed_tests/trash.py,sha256=E4zrrRyVqeNEumWg8rYKquR9VTIULN52eCRqjmv_s58,1647
-lecrapaud/training.py,sha256=a-p-06-h-oRj5A728-xwekHZ7mxTnrPnKjAjoZhNZME,8071
-lecrapaud/utils.py,sha256=J2lMRo8J1CoZ1AYIPcdbFWXg-PxtQTs8qmeINioXWfk,8107
-lecrapaud-0.2.1.dist-info/LICENSE,sha256=MImCryu0AnqhJE_uAZD-PIDKXDKb8sT7v0i1NOYeHTM,11350
-lecrapaud-0.2.1.dist-info/METADATA,sha256=g2qZx1kl3Qf1GT7LE-NNmTOwJluyZel0tgL1FCSYmt4,4460
-lecrapaud-0.2.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-lecrapaud-0.2.1.dist-info/RECORD,,
+lecrapaud/utils.py,sha256=TMlEyU6aRhgiYRtdbvKrlq2t1L9Al5BXfQ3cPbeHlX8,8098
+lecrapaud-0.3.0.dist-info/LICENSE,sha256=MImCryu0AnqhJE_uAZD-PIDKXDKb8sT7v0i1NOYeHTM,11350
+lecrapaud-0.3.0.dist-info/METADATA,sha256=dcXHKrW9p_QPxsshz7iVrG47_FikmRBpQ7gY6peB-2o,4525
+lecrapaud-0.3.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+lecrapaud-0.3.0.dist-info/RECORD,,

lecrapaud/predictions.py

@@ -1,292 +0,0 @@
-import keras
-import pickle
-import pandas as pd
-from pathlib import Path
-import joblib
-from datetime import timedelta, datetime
-import logging
-
-from lecrapaud.search_space import ml_models, dl_recurrent_models
-from lecrapaud.data_sourcing import get_filtered_data
-from lecrapaud.preprocessing import feature_engineering
-from lecrapaud.feature_selection import (
-    encode_categorical_features,
-    reshape_df,
-    TARGETS_CLF,
-    reshape_time_series,
-)
-from lecrapaud.model_selection import predict, evaluate
-from lecrapaud.utils import logger
-from lecrapaud.db import Dataset
-from lecrapaud.config import LOGGING_LEVEL
-
-MODELS_LIST = ml_models + dl_recurrent_models
-
-
-def run_prediction(
-    dataset_id: str,
-    targets_numbers: list[int],
-    test: bool = True,
-    date: datetime = None,
-    verbose: int = 0,
-):
-    """Function to run prediction on several TARGETS using best models"""
-    if verbose == 0:
-        logger.setLevel(logging.WARNING)
-
-    logger.warning("Running prediction...")
-
-    dataset = Dataset.get(dataset_id)
-    dataset_dir = dataset.path
-    preprocessing_dir = f"{dataset_dir}/preprocessing"
-    list_of_groups = dataset.list_of_groups
-
-    features_dict = {}
-    scaler_y_dict = {}
-    model_dict = {}
-    threshold_dict = {}
-    for target_number in targets_numbers:
-        (
-            model_dict[target_number],
-            threshold_dict[target_number],
-            features_dict[target_number],
-            scaler_y_dict[target_number],
-            all_features,
-            scaler_x,
-        ) = load_model(dataset, target_number)
-
-    # get data for backtesting
-    if test:
-        train_data_dir = f"{dataset_dir}/data"
-        data_for_pred = joblib.load(f"{train_data_dir}/test.pkl")
-        data_for_pred_scaled = joblib.load(f"{train_data_dir}/test_scaled.pkl")
-
-        if any(
-            config["recurrent"]
-            for config in MODELS_LIST
-            if config["model_name"] in model_dict.values()
-        ):
-            train_scaled = joblib.load(f"{train_data_dir}/train_scaled.pkl")
-            val_scaled = joblib.load(f"{train_data_dir}/val_scaled.pkl")
-            test_scaled = joblib.load(f"{train_data_dir}/test_scaled.pkl")
-            reshaped_data = reshape_time_series(
-                train_scaled, val_scaled, test_scaled, all_features, timesteps=120
-            )
-            data_for_pred_reshaped = reshaped_data["x_train_reshaped"]
-
-        most_recent_data = joblib.load(f"{train_data_dir}/full.pkl")
-        most_recent_data = most_recent_data.loc[data_for_pred.index]
-
-        scores_clf = []
-        scores_reg = []
-    # get data for predicting future
-    else:
-        # TODO: if date is a bit more older, need more than 0 years
-        most_recent_data = get_filtered_data(
-            years_of_data=0, list_of_groups=list_of_groups
-        )
-
-        most_recent_data = feature_engineering(
-            most_recent_data, for_training=False, save_as_csv=True
-        )
-
-        data_for_pred = encode_categorical_features(
-            most_recent_data, save_dir=preprocessing_dir, fit=False
-        )
-
-        data_for_pred_scaled = pd.DataFrame(
-            scaler_x.transform(data_for_pred[all_features]),
-            columns=list(data_for_pred[all_features].columns),
-            index=data_for_pred.index,
-        )
-
-        # TODO: don't we need to have 120 days of data for each stock?
-        if any(
-            config["recurrent"]
-            for config in MODELS_LIST
-            if config["model_name"] in model_dict.values()
-        ):
-            # Count number of rows per stock
-            counts = data_for_pred["STOCK"].value_counts()
-
-            # Find stocks with insufficient history
-            insufficient_stocks = counts[counts < 120]
-
-            if not insufficient_stocks.empty:
-                raise ValueError(
-                    f"Insufficient history for stocks: {', '.join(insufficient_stocks.index)}"
-                )
-
-            data_for_pred_reshaped = reshape_df(
-                data_for_pred_scaled[all_features], data_for_pred["STOCK"], 120
-            )
-
-    # make prediction
-    for target_number in targets_numbers:
-
-        # Prepare variables and data
-        target_type = "classification" if target_number in TARGETS_CLF else "regression"
-        features = features_dict[target_number]
-        model = model_dict[target_number]
-        threshold = threshold_dict[target_number]
-
-        config = [
-            config for config in MODELS_LIST if config["model_name"] == model.model_name
-        ]
-        if config is None or len(config) == 0:
-            Exception(f"Model {model.model_name} was not found in search space.")
-        else:
-            config = config[0]
-
-        need_scaling = config["need_scaling"] and target_type == "regression"
-        if config["recurrent"]:
-            features_idx = [i for i, e in enumerate(all_features) if e in set(features)]
-            x_pred = data_for_pred_reshaped[:, :, features_idx]
-        else:
-            x_pred = (
-                data_for_pred_scaled[features]
-                if need_scaling
-                else data_for_pred[features]
-            )
-
-        # Predict
-        y_pred = predict(model, x_pred, target_type, config, threshold)
-
-        # Fix for recurrent model because x_val has no index as it is a 3D np array
-        if config["recurrent"]:
-            y_pred.index = (
-                most_recent_data.index
-            )  # TODO: not sure this will work for old dataset not aligned with data_for_training for test use case (done, this is why we decode the test set)
-
-        # Unscale prediction
-        if need_scaling or config["recurrent"]:
-            scaler_y = scaler_y_dict[target_number]
-            y_pred = pd.Series(
-                scaler_y.inverse_transform(y_pred.values.reshape(-1, 1)).flatten(),
-                index=most_recent_data.index,
-            )
-            y_pred.name = "PRED"
-
-        # Evaluate if test
-        if test:
-            prediction = pd.concat(
-                [most_recent_data[f"TARGET_{target_number}"], y_pred], axis=1
-            )
-            prediction.rename(
-                columns={f"TARGET_{target_number}": "TARGET"}, inplace=True
-            )
-            score = evaluate(prediction, target_type)
-            score["TARGET"] = f"TARGET_{target_number}"
-            (
-                scores_clf.append(score)
-                if target_type == "classification"
-                else scores_reg.append(score)
-            )
-
-        if isinstance(y_pred, pd.DataFrame):
-            y_pred.rename(
-                columns={"PRED": f"TARGET_{target_number}_PRED"}, inplace=True
-            )
-            most_recent_data = pd.concat(
-                [most_recent_data, y_pred[f"TARGET_{target_number}_PRED"]], axis=1
-            )
-
-        else:
-            y_pred.name = f"TARGET_{target_number}_PRED"
-            most_recent_data = pd.concat([most_recent_data, y_pred], axis=1)
-
-    # return result either for test set or for tomorrow prediction
-    result = most_recent_data
-
-    if verbose == 0:
-        logger.setLevel(LOGGING_LEVEL)
-
-    if test:
-        logger.info("Test results on test set")
-        scores_reg = pd.DataFrame(scores_reg).set_index("TARGET")
-        scores_clf = pd.DataFrame(scores_clf).set_index("TARGET")
-        return result, scores_reg, scores_clf, prediction
-    elif date:
-        date = date.replace(hour=0, minute=0, second=0, microsecond=0)
-        tomorrow = date + timedelta(days=1)
-        logger.info(f"Prediction for : {tomorrow.date()}")
-        result = result[result["DATE"] == date]
-        return result, None, None, None
-    else:
-        date = datetime.today()
-        max_date = result["DATE"].max()
-        if max_date.date() != date.date():
-            logger.info(
-                f"The maximum date found in the dataset is {max_date} and not {date}"
-            )
-        tomorrow = max_date + timedelta(days=1)
-        logger.info(f"Prediction for tomorrow : {tomorrow.date()}")
-
-        # Filter the DataFrame for the last date
-        filtered_result = result[result["DATE"] == max_date]
-
-        return filtered_result, None, None, None
-
-
-# Helpers
-def load_model(dataset: Dataset, target_number: int):
-    dataset_dir = dataset.path
-    training_target_dir = f"{dataset_dir}/TARGET_{target_number}"
-    preprocessing_dir = f"{dataset_dir}/preprocessing"
-
-    # Search for files that contain '.best' or '.keras' in the name
-    scores_tracking = pd.read_csv(f"{training_target_dir}/scores_tracking.csv")
-    training_target_dir = Path(training_target_dir)
-    best_files = list(training_target_dir.glob("*.best*")) + list(
-        training_target_dir.glob("*.keras*")
-    )
-    threshold = (
-        scores_tracking["THRESHOLD"].values[0]
-        if "THRESHOLD" in scores_tracking.columns
-        else None
-    )
-
-    # If any files are found, try loading the first one (or process as needed)
-    if best_files:
-        file_path = best_files[0]  # Assuming you want to open the first matching file
-        try:
-            # Attempt to load the file as a scikit-learn, XGBoost, or LightGBM model (Pickle format)
-            model = joblib.load(file_path)
-            logger.info(f"Loaded model {model.model_name} and threshold {threshold}")
-        except (pickle.UnpicklingError, EOFError):
-            # If it's not a pickle file, try loading it as a Keras model
-            try:
-                # Attempt to load the file as a Keras model
-                model = keras.models.load_model(file_path)
-                logger.info(
-                    f"Loaded model {model.model_name} and threshold {threshold}"
-                )
-            except Exception as e:
-                raise FileNotFoundError(
-                    f"Model could not be loaded from path: {file_path}: {e}"
-                )
-    else:
-        raise FileNotFoundError(
-            f"No files with '.best' or '.keras' found in the specified folder: {training_target_dir}"
-        )
-
-    if dataset.name == "data_28_X_X":
-        features = joblib.load(
-            f"{preprocessing_dir}/features_{target_number}.pkl"
-        )  # we keep this for backward compatibility
-    else:
-        features = dataset.get_features(target_number)
-
-    scaler_y = None
-    if target_number not in TARGETS_CLF:
-        scaler_y = joblib.load(f"{preprocessing_dir}/scaler_y_{target_number}.pkl")
-
-    if dataset.name == "data_28_X_X":
-        all_features = joblib.load(
-            f"{preprocessing_dir}/all_features.pkl"
-        )  # we keep this for backward compatibility
-    else:
-        all_features = dataset.get_all_features()
-    scaler_x = joblib.load(f"{preprocessing_dir}/scaler_x.pkl")
-
-    return model, threshold, features, scaler_y, all_features, scaler_x