lecrapaud 0.4.0__py3-none-any.whl → 0.4.1__py3-none-any.whl

lecrapaud/predictions.py

@@ -1,292 +0,0 @@
-import keras
-import pickle
-import pandas as pd
-from pathlib import Path
-import joblib
-from datetime import timedelta, datetime
-import logging
-
-from src.search_space import ml_models, dl_recurrent_models
-from src.data_sourcing import get_filtered_data
-from src.feature_engineering import feature_engineering
-from src.feature_selection import (
-    encode_categorical_features,
-    reshape_df,
-    TARGETS_CLF,
-    reshape_time_series,
-)
-from src.model_selection import predict, evaluate
-from src.utils import logger
-from src.db.models import Dataset
-from src.config import LOGGING_LEVEL
-
-MODELS_LIST = ml_models + dl_recurrent_models
-
-
-def run_prediction(
-    dataset_id: str,
-    targets_numbers: list[int],
-    test: bool = True,
-    date: datetime = None,
-    verbose: int = 0,
-):
-    """Function to run prediction on several TARGETS using best models"""
-    if verbose == 0:
-        logger.setLevel(logging.WARNING)
-
-    logger.warning("Running prediction...")
-
-    dataset = Dataset.get(dataset_id)
-    dataset_dir = dataset.path
-    preprocessing_dir = f"{dataset_dir}/preprocessing"
-    list_of_groups = dataset.list_of_groups
-
-    features_dict = {}
-    scaler_y_dict = {}
-    model_dict = {}
-    threshold_dict = {}
-    for target_number in targets_numbers:
-        (
-            model_dict[target_number],
-            threshold_dict[target_number],
-            features_dict[target_number],
-            scaler_y_dict[target_number],
-            all_features,
-            scaler_x,
-        ) = load_model(dataset, target_number)
-
-    # get data for backtesting
-    if test:
-        train_data_dir = f"{dataset_dir}/data"
-        data_for_pred = joblib.load(f"{train_data_dir}/test.pkl")
-        data_for_pred_scaled = joblib.load(f"{train_data_dir}/test_scaled.pkl")
-
-        if any(
-            config["recurrent"]
-            for config in MODELS_LIST
-            if config["model_name"] in model_dict.values()
-        ):
-            train_scaled = joblib.load(f"{train_data_dir}/train_scaled.pkl")
-            val_scaled = joblib.load(f"{train_data_dir}/val_scaled.pkl")
-            test_scaled = joblib.load(f"{train_data_dir}/test_scaled.pkl")
-            reshaped_data = reshape_time_series(
-                train_scaled, val_scaled, test_scaled, all_features, timesteps=120
-            )
-            data_for_pred_reshaped = reshaped_data["x_train_reshaped"]
-
-        most_recent_data = joblib.load(f"{train_data_dir}/full.pkl")
-        most_recent_data = most_recent_data.loc[data_for_pred.index]
-
-        scores_clf = []
-        scores_reg = []
-    # get data for predicting future
-    else:
-        # TODO: if date is a bit more older, need more than 0 years
-        most_recent_data = get_filtered_data(
-            years_of_data=0, list_of_groups=list_of_groups
-        )
-
-        most_recent_data = feature_engineering(
-            most_recent_data, for_training=False, save_as_csv=True
-        )
-
-        data_for_pred = encode_categorical_features(
-            most_recent_data, save_dir=preprocessing_dir, fit=False
-        )
-
-        data_for_pred_scaled = pd.DataFrame(
-            scaler_x.transform(data_for_pred[all_features]),
-            columns=list(data_for_pred[all_features].columns),
-            index=data_for_pred.index,
-        )
-
-        # TODO: don't we need to have 120 days of data for each stock?
-        if any(
-            config["recurrent"]
-            for config in MODELS_LIST
-            if config["model_name"] in model_dict.values()
-        ):
-            # Count number of rows per stock
-            counts = data_for_pred["STOCK"].value_counts()
-
-            # Find stocks with insufficient history
-            insufficient_stocks = counts[counts < 120]
-
-            if not insufficient_stocks.empty:
-                raise ValueError(
-                    f"Insufficient history for stocks: {', '.join(insufficient_stocks.index)}"
-                )
-
-            data_for_pred_reshaped = reshape_df(
-                data_for_pred_scaled[all_features], data_for_pred["STOCK"], 120
-            )
-
-    # make prediction
-    for target_number in targets_numbers:
-
-        # Prepare variables and data
-        target_type = "classification" if target_number in TARGETS_CLF else "regression"
-        features = features_dict[target_number]
-        model = model_dict[target_number]
-        threshold = threshold_dict[target_number]
-
-        config = [
-            config for config in MODELS_LIST if config["model_name"] == model.model_name
-        ]
-        if config is None or len(config) == 0:
-            Exception(f"Model {model.model_name} was not found in search space.")
-        else:
-            config = config[0]
-
-        need_scaling = config["need_scaling"] and target_type == "regression"
-        if config["recurrent"]:
-            features_idx = [i for i, e in enumerate(all_features) if e in set(features)]
-            x_pred = data_for_pred_reshaped[:, :, features_idx]
-        else:
-            x_pred = (
-                data_for_pred_scaled[features]
-                if need_scaling
-                else data_for_pred[features]
-            )
-
-        # Predict
-        y_pred = predict(model, x_pred, target_type, config, threshold)
-
-        # Fix for recurrent model because x_val has no index as it is a 3D np array
-        if config["recurrent"]:
-            y_pred.index = (
-                most_recent_data.index
-            )  # TODO: not sure this will work for old dataset not aligned with data_for_training for test use case (done, this is why we decode the test set)
-
-        # Unscale prediction
-        if need_scaling or config["recurrent"]:
-            scaler_y = scaler_y_dict[target_number]
-            y_pred = pd.Series(
-                scaler_y.inverse_transform(y_pred.values.reshape(-1, 1)).flatten(),
-                index=most_recent_data.index,
-            )
-            y_pred.name = "PRED"
-
-        # Evaluate if test
-        if test:
-            prediction = pd.concat(
-                [most_recent_data[f"TARGET_{target_number}"], y_pred], axis=1
-            )
-            prediction.rename(
-                columns={f"TARGET_{target_number}": "TARGET"}, inplace=True
-            )
-            score = evaluate(prediction, target_type)
-            score["TARGET"] = f"TARGET_{target_number}"
-            (
-                scores_clf.append(score)
-                if target_type == "classification"
-                else scores_reg.append(score)
-            )
-
-        if isinstance(y_pred, pd.DataFrame):
-            y_pred.rename(
-                columns={"PRED": f"TARGET_{target_number}_PRED"}, inplace=True
-            )
-            most_recent_data = pd.concat(
-                [most_recent_data, y_pred[f"TARGET_{target_number}_PRED"]], axis=1
-            )
-
-        else:
-            y_pred.name = f"TARGET_{target_number}_PRED"
-            most_recent_data = pd.concat([most_recent_data, y_pred], axis=1)
-
-    # return result either for test set or for tomorrow prediction
-    result = most_recent_data
-
-    if verbose == 0:
-        logger.setLevel(LOGGING_LEVEL)
-
-    if test:
-        logger.info("Test results on test set")
-        scores_reg = pd.DataFrame(scores_reg).set_index("TARGET")
-        scores_clf = pd.DataFrame(scores_clf).set_index("TARGET")
-        return result, scores_reg, scores_clf, prediction
-    elif date:
-        date = date.replace(hour=0, minute=0, second=0, microsecond=0)
-        tomorrow = date + timedelta(days=1)
-        logger.info(f"Prediction for : {tomorrow.date()}")
-        result = result[result["DATE"] == date]
-        return result, None, None, None
-    else:
-        date = datetime.today()
-        max_date = result["DATE"].max()
-        if max_date.date() != date.date():
-            logger.info(
-                f"The maximum date found in the dataset is {max_date} and not {date}"
-            )
-        tomorrow = max_date + timedelta(days=1)
-        logger.info(f"Prediction for tomorrow : {tomorrow.date()}")
-
-        # Filter the DataFrame for the last date
-        filtered_result = result[result["DATE"] == max_date]
-
-        return filtered_result, None, None, None
-
-
-# Helpers
-def load_model(dataset: Dataset, target_number: int):
-    dataset_dir = dataset.path
-    training_target_dir = f"{dataset_dir}/TARGET_{target_number}"
-    preprocessing_dir = f"{dataset_dir}/preprocessing"
-
-    # Search for files that contain '.best' or '.keras' in the name
-    scores_tracking = pd.read_csv(f"{training_target_dir}/scores_tracking.csv")
-    training_target_dir = Path(training_target_dir)
-    best_files = list(training_target_dir.glob("*.best*")) + list(
-        training_target_dir.glob("*.keras*")
-    )
-    threshold = (
-        scores_tracking["THRESHOLD"].values[0]
-        if "THRESHOLD" in scores_tracking.columns
-        else None
-    )
-
-    # If any files are found, try loading the first one (or process as needed)
-    if best_files:
-        file_path = best_files[0]  # Assuming you want to open the first matching file
-        try:
-            # Attempt to load the file as a scikit-learn, XGBoost, or LightGBM model (Pickle format)
-            model = joblib.load(file_path)
-            logger.info(f"Loaded model {model.model_name} and threshold {threshold}")
-        except (pickle.UnpicklingError, EOFError):
-            # If it's not a pickle file, try loading it as a Keras model
-            try:
-                # Attempt to load the file as a Keras model
-                model = keras.models.load_model(file_path)
-                logger.info(
-                    f"Loaded model {model.model_name} and threshold {threshold}"
-                )
-            except Exception as e:
-                raise FileNotFoundError(
-                    f"Model could not be loaded from path: {file_path}: {e}"
-                )
-    else:
-        raise FileNotFoundError(
-            f"No files with '.best' or '.keras' found in the specified folder: {training_target_dir}"
-        )
-
-    if dataset.name == "data_28_X_X":
-        features = joblib.load(
-            f"{preprocessing_dir}/features_{target_number}.pkl"
-        )  # we keep this for backward compatibility
-    else:
-        features = dataset.get_features(target_number)
-
-    scaler_y = None
-    if target_number not in TARGETS_CLF:
-        scaler_y = joblib.load(f"{preprocessing_dir}/scaler_y_{target_number}.pkl")
-
-    if dataset.name == "data_28_X_X":
-        all_features = joblib.load(
-            f"{preprocessing_dir}/all_features.pkl"
-        )  # we keep this for backward compatibility
-    else:
-        all_features = dataset.get_all_features()
-    scaler_x = joblib.load(f"{preprocessing_dir}/scaler_x.pkl")
-
-    return model, threshold, features, scaler_y, all_features, scaler_x

lecrapaud/training.py

@@ -1,151 +0,0 @@
-import logging
-import joblib
-from pathlib import Path
-import os
-from src.utils import logger
-
-from src.feature_engineering import feature_engineering
-from src.feature_selection import (
-    create_sets_from_data,
-    feature_selection,
-    scale_data,
-    reshape_time_series,
-)
-from src.model_selection import model_selection, test_hardware
-from src.data_sourcing import get_filtered_data
-from src.constants import stock_list_3, stock_list_1
-from src.search_space import ml_models, dl_recurrent_models
-from src.directory_management import tmp_dir
-from src.db.models import Dataset
-from src.config import PYTHON_ENV
-
-
-def run_training(
-    dataset_id=None,
-    years_of_data=2,
-    list_of_groups=stock_list_1,
-    percentile=15,
-    corr_threshold=80,
-    max_features=20,
-    max_timesteps=120,
-    targets_numbers=range(1, 15),
-    models_idx=range(len(ml_models)),
-    number_of_trials=20,
-    perform_hyperoptimization=True,
-    perform_crossval=False,
-    clean_dir=False,
-    preserve_model=False,
-    session_name="test",
-):
-    logging.captureWarnings(True)
-
-    if dataset_id is None:
-        # Get the data
-        logger.info("Getting data...")
-        data = get_filtered_data(
-            years_of_data=years_of_data,
-            list_of_groups=list_of_groups,
-        )
-
-        # preprocess & feature engineering
-        logger.info("Preprocessing...")
-        data_for_training = feature_engineering(
-            data, for_training=True, save_as_csv=True
-        )
-
-        # train / val / test sets
-        train, val, test, dataset = create_sets_from_data(
-            data_for_training,
-            percentile=percentile,
-            corr_threshold=corr_threshold,
-            max_features=max_features,
-        )
-        dataset_dir = dataset.path
-        dataset_id = dataset.id
-        train_data_dir = f"{dataset_dir}/data"
-        os.makedirs(train_data_dir, exist_ok=True)
-        preprocessing_dir = f"{dataset_dir}/preprocessing"
-
-        # feature selection
-        logger.info("Feature Selection...")
-        for target_number in targets_numbers:
-            feature_selection(
-                dataset_id=dataset_id,
-                train=train,
-                target_number=target_number,
-                single_process=True,
-            )
-
-        dataset = Dataset.get(dataset_id)
-        all_features = dataset.get_all_features()
-        columns_to_keep = all_features + [f"TARGET_{i}" for i in range(1, 15)]
-        logger.info(columns_to_keep)
-        duplicates = [
-            col for col in set(columns_to_keep) if columns_to_keep.count(col) > 1
-        ]
-
-        if duplicates:
-            raise ValueError(f"Doublons détectés dans columns_to_keep: {duplicates}")
-
-        train = train[columns_to_keep]
-        val = val[columns_to_keep]
-        test = test[columns_to_keep]
-
-        if PYTHON_ENV != "Test":
-            joblib.dump(train[columns_to_keep], f"{train_data_dir}/train.pkl")
-            joblib.dump(val[columns_to_keep], f"{train_data_dir}/val.pkl")
-            joblib.dump(test[columns_to_keep], f"{train_data_dir}/test.pkl")
-
-        # scaling features
-        logger.info("Scaling features...")
-        train_scaled, scaler_x, scalers_y = scale_data(
-            train, save_dir=preprocessing_dir
-        )
-        val_scaled, _, _ = scale_data(
-            val, save_dir=preprocessing_dir, scaler_x=scaler_x, scalers_y=scalers_y
-        )
-        test_scaled, _, _ = scale_data(
-            test, save_dir=preprocessing_dir, scaler_x=scaler_x, scalers_y=scalers_y
-        )
-
-        if PYTHON_ENV != "Test":
-            joblib.dump(train_scaled, f"{train_data_dir}/train_scaled.pkl")
-            joblib.dump(val_scaled, f"{train_data_dir}/val_scaled.pkl")
-            joblib.dump(test_scaled, f"{train_data_dir}/test_scaled.pkl")
-
-        data = {
-            "train": train,
-            "val": val,
-            "test": test,
-            "train_scaled": train_scaled,
-            "val_scaled": val_scaled,
-            "test_scaled": test_scaled,
-            "scalers_y": scalers_y,
-        }
-
-    list_models = ml_models + dl_recurrent_models
-    reshaped_data = None
-    if any(list_models[i].get("recurrent") for i in models_idx):
-        # reshaping data for recurrent models
-        logger.info("Reshaping data for recurrent models...")
-        reshaped_data = reshape_time_series(
-            train_scaled, val_scaled, test_scaled, all_features, timesteps=max_timesteps
-        )
-
-    # model selection and hyperoptimization
-    logger.info("Model Selection and Hyperoptimization...")
-    for target_number in targets_numbers:
-        model_selection(
-            dataset_id=dataset_id,
-            models_idx=models_idx,
-            target_number=target_number,
-            session_name=session_name,
-            perform_hyperoptimization=perform_hyperoptimization,
-            perform_crossval=perform_crossval,
-            number_of_trials=number_of_trials,
-            plot=False,
-            clean_dir=clean_dir,
-            preserve_model=preserve_model,
-            reshaped_data=reshaped_data,
-            data=(data or None),
-        )

lecrapaud-0.4.0.dist-info/METADATA

@@ -1,103 +0,0 @@
-Metadata-Version: 2.3
-Name: lecrapaud
-Version: 0.4.0
-Summary: Framework for machine and deep learning, with regression, classification and time series analysis
-License: Apache License
-Author: Pierre H. Gallet
-Requires-Python: ==3.12.*
-Classifier: License :: Other/Proprietary License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: backoff (>=2.2.1)
-Requires-Dist: category-encoders (>=2.8.1)
-Requires-Dist: celery (>=5.5.1)
-Requires-Dist: curl-cffi (>=0.11.1)
-Requires-Dist: deep-translator (>=1.11.4)
-Requires-Dist: degiro-connector (>=3.0.26)
-Requires-Dist: fake-useragent (>=2.1.0)
-Requires-Dist: ftfy (>=6.3.1)
-Requires-Dist: honeybadger (>=0.21)
-Requires-Dist: joblib (>=1.4.2)
-Requires-Dist: keras (>=3.9.0)
-Requires-Dist: keras-tcn (>=3.1.2)
-Requires-Dist: lightgbm (>=4.6.0)
-Requires-Dist: matplotlib (>=3.10.1)
-Requires-Dist: mlxtend (>=0.23.4)
-Requires-Dist: numpy (>=2.1.3)
-Requires-Dist: pandas (>=2.2.3)
-Requires-Dist: pandas-market-calendars (>=4.6.1)
-Requires-Dist: playwright (>=1.52.0)
-Requires-Dist: pydantic (>=2.10.6)
-Requires-Dist: python-dotenv (>=1.0.1)
-Requires-Dist: pytz (>=2025.1)
-Requires-Dist: ratelimit (>=2.2.1)
-Requires-Dist: scikit-learn (>=1.6.1)
-Requires-Dist: scipy (>=1.15.2)
-Requires-Dist: seaborn (>=0.13.2)
-Requires-Dist: sentence-transformers (>=3.4.1)
-Requires-Dist: sqlalchemy (>=2.0.39)
-Requires-Dist: tensorboardx (>=2.6.2.2)
-Requires-Dist: tensorflow (>=2.19.0)
-Requires-Dist: tf-keras (>=2.19.0)
-Requires-Dist: tqdm (>=4.67.1)
-Requires-Dist: xgboost (>=3.0.0)
-Requires-Dist: yahoo-fin (>=0.8.9.1)
-Requires-Dist: yfinance (>=0.2.55)
-Description-Content-Type: text/markdown
-
-# Stock
-
-## Overview
-
-## Project description
-
-## Quick Start
-
-1. Create environment
-
-```sh
-$ pip install virtualenv
-$ python -m venv .venv
-$ source .venv/bin/activate
-```
-
-2. Install dependencies
-
-```sh
-$ pip install -r requirements.txt
-$ pip freeze > requirements.txt
-```
-
-3. Deactivate virtualenv (if needed)
-
-```sh
-$ deactivate
-```
-
-## Reminders for Github usage
-
-1. Creating Github repository
-
-```sh
-$ brew install gh
-$ gh auth login
-$ gh repo create
-```
-
-2. Initializing git and first commit to distant repository
-
-```sh
-$ git init
-$ git add .
-$ git commit -m 'first commit'
-$ git remote add origin <YOUR_REPO_URL>
-$ git push -u origin master
-```
-
-3. use conventional commits
-https://www.conventionalcommits.org/en/v1.0.0/#summary
-
-
-***
-
-Pierre Gallet © 2024