lecrapaud 0.1.0__py3-none-any.whl

lecrapaud/speed_tests/tests.ipynb

@@ -0,0 +1,145 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import autosklearn.classification\n",
+    "import sklearn.datasets\n",
+    "import sklearn.metrics\n",
+    "from pprint import pprint\n",
+    "from tabpfn import TabPFNClassifier\n",
+    "import numpy as np\n",
+    "from pathlib import Path\n",
+    "import pandas as pd\n",
+    "import time\n",
+    "from sklearn.metrics import accuracy_score\n",
+    "from sklearn.datasets import load_breast_cancer\n",
+    "from sklearn.model_selection import train_test_split"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X, y = sklearn.datasets.load_breast_cancer(return_X_y=True)\n",
+    "X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(\n",
+    "    X, y, random_state=1\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "automl = autosklearn.classification.AutoSklearnClassifier(\n",
+    "    time_left_for_this_task=120,\n",
+    "    per_run_time_limit=30,\n",
+    "    tmp_folder=\"/tmp/autosklearn_interpretable_models_example_tmp\",\n",
+    "    include={\n",
+    "        \"classifier\": [\"decision_tree\", \"lda\", \"sgd\"],\n",
+    "        \"feature_preprocessor\": [\n",
+    "            \"no_preprocessing\",\n",
+    "            \"polynomial\",\n",
+    "            \"select_percentile_classification\",\n",
+    "        ],\n",
+    "    },\n",
+    "    ensemble_kwargs={\"ensemble_size\": 1},\n",
+    ")\n",
+    "automl.fit(X_train, y_train, dataset_name=\"breast_cancer\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pprint(automl.show_models(), indent=4)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "predictions = automl.predict(X_test)\n",
+    "print(\"Accuracy score:\", sklearn.metrics.accuracy_score(y_test, predictions))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# N_ensemble_configurations defines how many estimators are averaged, it is bounded by #features * #classes\n",
+    "# more ensemble members are slower, but more accurate\n",
+    "classifier = TabPFNClassifier(device=\"cuda\", N_ensemble_configurations=4)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "start = time.time()\n",
+    "classifier.fit(X_train, y_train)\n",
+    "y_eval, p_eval = classifier.predict(X_test, return_winning_probability=True)\n",
+    "print(\n",
+    "    \"Prediction time: \", time.time() - start, \"Accuracy\", accuracy_score(y_test, y_eval)\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# We also offer the `predict_proba` interface\n",
+    "classifier.predict_proba(X_test).shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "out_table = pd.DataFrame(X_test.copy().astype(str))\n",
+    "out_table[\"prediction\"] = [f\"{y_e} (p={p_e:.2f})\" for y_e, p_e in zip(y_eval, p_eval)]\n",
+    "out_table"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

lecrapaud/speed_tests/trash.py

@@ -0,0 +1,37 @@
+# def _get_weekly_return(y_true, y_pred):
+#     df = pd.concat([y_true, y_pred, stock_data[['YEARWEEK', 'STOCK', 'TARGET_1']]], join='inner', axis=1)
+#     df['PRED'] += 1
+#     df['TARGET'] += 1
+#     return df[['YEARWEEK', 'STOCK', 'PRED', 'TARGET']].groupby(['YEARWEEK', 'STOCK']).prod().reset_index()
+
+# def _calc_spread_return_per_week(df, portfolio_size):
+#     return (df.sort_values('PRED', ascending=False)['TARGET_1'][:portfolio_size] - 1).mean()
+
+# def sharpe_ratio_weekly(y_true, y_pred, portfolio_size:int=10):
+#     df = _get_weekly_return(y_true, y_pred)
+#     buf = df.groupby('YEARWEEK').apply(_calc_spread_return_per_week, portfolio_size)
+#     sharpe_ratio = (buf.mean() * 52) / (buf.std() * np.sqrt(52))
+#     buf += 1
+#     cumulated_roi = buf.prod() - 1
+#     cagr = buf.prod() ** (1 / (buf.shape[0]/52) ) - 1
+#     return sharpe_ratio, cumulated_roi, cagr
+
+
+def sharpe_ratio_daily(y_true, y_pred, portfolio_size: int = 10):
+    df = pd.concat(
+        [y_true, y_pred, stock_data[["DATE", "TARGET_1"]]], join="inner", axis=1
+    )
+
+    def _calc_spread_return_per_day(df: pd.DataFrame, portfolio_size: int):
+        # print(df.sort_values('PRED', ascending=False)[['PRED', 'TARGET', 'TARGET_1']].head(10))
+        return (
+            df.sort_values("PRED", ascending=False)["TARGET_1"].iloc[:portfolio_size]
+        ).mean()
+
+    buf = df.groupby("DATE").apply(_calc_spread_return_per_day, portfolio_size)
+
+    sharpe_ratio = (buf.mean() * 252) / (buf.std() * np.sqrt(252))
+    buf += 1
+    cumulated_roi = buf.prod() - 1
+    cagr = buf.prod() ** (1 / (buf.shape[0] / 252)) - 1
+    return sharpe_ratio, cumulated_roi, cagr

lecrapaud/training.py

@@ -0,0 +1,239 @@
+import logging
+import joblib
+from pathlib import Path
+import os
+
+from lecrapaud.experiment import create_dataset
+from lecrapaud.feature_engineering import (
+    feature_engineering,
+    encode_categorical_features,
+    add_pca_features,
+    summarize_dataframe,
+)
+from lecrapaud.feature_selection import (
+    feature_selection,
+    train_val_test_split,
+    train_val_test_split_time_series,
+    scale_data,
+    reshape_time_series,
+)
+from lecrapaud.model_selection import model_selection
+from lecrapaud.search_space import all_models
+from lecrapaud.directory_management import tmp_dir
+from lecrapaud.db import Dataset
+from lecrapaud.utils import logger
+from lecrapaud.config import PYTHON_ENV
+
+
+# Parameters
+columns_date = ["DATE"]
+columns_te_groupby = [["SECTOR", "DATE"], ["SUBINDUSTRY", "DATE"]]
+columns_te_target = ["RET", "VOLUME", "RESIDUAL_RET", "RELATIVE_VOLUME"] + [
+    f"{ind}_{p}"
+    for p in [9, 14, 21, 50]
+    for ind in [
+        "CUMUL_RET",
+        "SMA",
+        "EMA",
+        "VOLATILITY",
+        "ATR",
+        "ADX",
+        "%K",
+        "RSI",
+        "MFI",
+    ]
+]
+target_clf = [2, 4, 6, 8, 9, 10, 11]
+column_ordinal = ["STOCK"]
+column_binary = ["SECTOR", "SUBINDUSTRY", "LOCATION"]
+columns_pca = []
+target_numbers = range(1, 15)
+date_column = "DATE"
+group_column = "STOCK"
+
+
+def run_training(
+    get_data_function: function,
+    get_data_params: dict = None,
+    time_series: bool = False,
+    dataset_id=None,
+    years_of_data=2,
+    list_of_groups=None,
+    percentile=15,
+    corr_threshold=80,
+    max_features=20,
+    max_timesteps=120,
+    targets_numbers=range(1, 15),
+    models_idx=range(len(all_models)),
+    number_of_trials=20,
+    perform_hyperoptimization=True,
+    perform_crossval=False,
+    clean_dir=False,
+    preserve_model=False,
+    session_name="test",
+):
+    logging.captureWarnings(True)
+
+    if any(all_models[i].get("recurrent") for i in models_idx) and not time_series:
+        ValueError(
+            "You need to set time_series to true to use recurrent model, or remove recurrent models from models_idx chosen"
+        )
+
+    if dataset_id is None:
+        # Get the data
+        logger.info("Getting data...")
+        data = get_data_function(**get_data_params)
+
+        # # preprocess & feature engineering => Should be in get_data_function
+        # logger.info("Preprocessing...")
+        # preprocessed_data = preprocessing(data, for_training=True, save_as_csv=True)
+
+        logger.info(f"Feature engineering for {session_name}...")
+        data_for_training = feature_engineering(
+            data,
+            columns_date=columns_date,
+            columns_te_groupby=columns_te_groupby,
+            columns_te_target=columns_te_target,
+        )
+
+        # Split
+        if time_series:
+            train, val, test = train_val_test_split_time_series(data_for_training)
+        else:
+            train, val, test = train_val_test_split(
+                data, stratify_col=f"target_{target_numbers[0]}"
+            )  # TODO: only stratifying first target for now
+
+        # Create Dataset / Experiment (TODO: should be defined sooner)
+        dataset = create_dataset(
+            train, val, test, corr_threshold, percentile, max_features
+        )
+        dataset_dir = dataset.path
+        dataset_id = dataset.id
+        data_dir = f"{dataset_dir}/data"
+        preprocessing_dir = f"{dataset_dir}/preprocessing"
+        os.makedirs(data_dir, exist_ok=True)
+        os.makedirs(preprocessing_dir, exist_ok=True)
+
+        # PCA
+        train, pcas = add_pca_features(train, columns_pca)
+        val, _ = add_pca_features(val, columns_pca, pcas=pcas)
+        test, _ = add_pca_features(test, columns_pca, pcas=pcas)
+
+        if PYTHON_ENV != "Test":
+            joblib.dump(pcas, f"{preprocessing_dir}/pca.pkl")
+
+        # Encoding
+        train, transformer = encode_categorical_features(
+            train, column_ordinal=column_ordinal, column_binary=column_binary
+        )
+        val, _ = encode_categorical_features(
+            val,
+            column_ordinal=column_ordinal,
+            column_binary=column_binary,
+            transformer=transformer,
+        )
+        test, _ = encode_categorical_features(
+            test,
+            column_ordinal=column_ordinal,
+            column_binary=column_binary,
+            transformer=transformer,
+        )
+
+        if PYTHON_ENV != "Test":
+            joblib.dump(data_for_training, f"{data_dir}/full.pkl")
+            joblib.dump(transformer, f"{preprocessing_dir}/column_transformer.pkl")
+            summary = summarize_dataframe(train)
+            summary.to_csv(f"{dataset_dir}/feature_summary.csv", index=False)
+
+        # feature selection
+        logger.info("Feature Selection...")
+        for target_number in targets_numbers:
+            feature_selection(
+                dataset_id=dataset_id,
+                train=train,
+                target_number=target_number,
+                single_process=True,
+            )
+
+        dataset = Dataset.get(dataset_id)
+        all_features = dataset.get_all_features()
+        columns_to_keep = all_features + [f"TARGET_{i}" for i in target_numbers]
+
+        duplicates = [
+            col for col in set(columns_to_keep) if columns_to_keep.count(col) > 1
+        ]
+        if duplicates:
+            raise ValueError(f"Doublons détectés dans columns_to_keep: {duplicates}")
+
+        train = train[columns_to_keep]
+        val = val[columns_to_keep]
+        test = test[columns_to_keep]
+
+        # save data
+        if PYTHON_ENV != "Test":
+            joblib.dump(train[columns_to_keep], f"{data_dir}/train.pkl")
+            joblib.dump(val[columns_to_keep], f"{data_dir}/val.pkl")
+            joblib.dump(test[columns_to_keep], f"{data_dir}/test.pkl")
+
+        # scaling features
+        if any(t not in target_clf for t in target_numbers) and any(
+            all_models[i].get("need_scaling") for i in models_idx
+        ):
+            logger.info("Scaling features...")
+            train_scaled, scaler_x, scalers_y = scale_data(
+                train, save_dir=preprocessing_dir
+            )
+            val_scaled, _, _ = scale_data(
+                val, save_dir=preprocessing_dir, scaler_x=scaler_x, scalers_y=scalers_y
+            )
+            test_scaled, _, _ = scale_data(
+                test, save_dir=preprocessing_dir, scaler_x=scaler_x, scalers_y=scalers_y
+            )
+        else:
+            train_scaled = None
+            val_scaled = None
+            test_scaled = None
+
+        # save data
+        if PYTHON_ENV != "Test":
+            joblib.dump(train_scaled, f"{data_dir}/train_scaled.pkl")
+            joblib.dump(val_scaled, f"{data_dir}/val_scaled.pkl")
+            joblib.dump(test_scaled, f"{data_dir}/test_scaled.pkl")
+
+        data = {
+            "train": train,
+            "val": val,
+            "test": test,
+            "train_scaled": train_scaled,
+            "val_scaled": val_scaled,
+            "test_scaled": test_scaled,
+            "scalers_y": scalers_y,
+        }
+
+    # reshape data for time series
+    reshaped_data = None
+    if any(all_models[i].get("recurrent") for i in models_idx) and time_series:
+        # reshaping data for recurrent models
+        logger.info("Reshaping data for recurrent models...")
+        reshaped_data = reshape_time_series(
+            train_scaled, val_scaled, test_scaled, all_features, timesteps=max_timesteps
+        )
+
+    # model selection and hyperoptimization
+    logger.info("Model Selection and Hyperoptimization...")
+    for target_number in target_numbers:
+        model_selection(
+            dataset_id=dataset_id,
+            models_idx=models_idx,
+            target_number=target_number,
+            session_name=session_name,
+            perform_hyperoptimization=perform_hyperoptimization,
+            perform_crossval=perform_crossval,
+            number_of_trials=number_of_trials,
+            plot=False,
+            clean_dir=clean_dir,
+            preserve_model=preserve_model,
+            reshaped_data=reshaped_data,
+            data=(data or None),
+        )

lecrapaud/utils.py

@@ -0,0 +1,246 @@
+import pandas as pd
+import logging
+from logging.handlers import RotatingFileHandler
+import shutil
+import os
+import subprocess
+from datetime import datetime, date
+from ftfy import fix_text
+import unicodedata
+import re
+import string
+
+from lecrapaud.directory_management import logger_dir
+from lecrapaud.config import LOGGING_LEVEL, PYTHON_ENV
+
+_LOGGER_ALREADY_CONFIGURED = False
+
+
+def setup_logger():
+
+    global _LOGGER_ALREADY_CONFIGURED
+    if _LOGGER_ALREADY_CONFIGURED:  # ← bail out if done before
+
+        return logging.getLogger("stock" if PYTHON_ENV != "Worker" else "")
+
+    print(
+        f"Setting up logger with PYTHON_ENV {PYTHON_ENV} and LOGGING_LEVEL {LOGGING_LEVEL}"
+    )
+    # ------------------------------------------------------------------ #
+    #  Real configuration happens only on the FIRST call                 #
+    # ------------------------------------------------------------------ #
+    fmt = "%(asctime)s - %(name)s - %(levelname)s - %(funcName)s - %(message)s"
+    datefmt = "%Y-%m-%d %H:%M:%S"
+    logging.basicConfig(format=fmt, datefmt=datefmt)  # root format
+    formatter = logging.Formatter(fmt, datefmt=datefmt)
+
+    logger = logging.getLogger("" if PYTHON_ENV == "Worker" else "stock")
+
+    log_level = getattr(logging, LOGGING_LEVEL.upper(), logging.INFO)
+    logger.setLevel(log_level)
+
+    # pick a file according to environment
+    env_file = {
+        "Development": "dev.log",
+        "Production": "prod.log",
+        "Test": "test.log",
+        "Worker": "worker.log",
+    }.get(PYTHON_ENV, "app.log")
+
+    file_handler = RotatingFileHandler(
+        f"{logger_dir}/{env_file}",
+        maxBytes=5 * 1024 * 1024,
+        backupCount=3,
+    )
+    file_handler.setFormatter(formatter)
+    file_handler.setLevel(log_level)
+    logger.addHandler(file_handler)
+
+    _LOGGER_ALREADY_CONFIGURED = True
+    return logger
+
+
+logger = setup_logger()
+
+
+def get_df_name(obj, namespace):
+    return [name for name in namespace if namespace[name] is obj][0]
+
+
+def pprint(item):
+    with pd.option_context("display.max_rows", None):
+        logger.info(item)
+
+
+def object_to_dict(obj):
+    if isinstance(obj, dict):
+        return {k: object_to_dict(v) for k, v in obj.items()}
+    elif hasattr(obj, "__dict__"):
+        return {k: object_to_dict(v) for k, v in obj.__dict__.items()}
+    elif isinstance(obj, list):
+        return [object_to_dict(i) for i in obj]
+    else:
+        return obj
+
+
+def copy_any(src, dst):
+    if os.path.isdir(src):
+        # Copy folder using copytree
+        shutil.copytree(src, dst)
+    else:
+        # Copy file using copy2 (which preserves metadata)
+        shutil.copy2(src, dst)
+
+
+def contains_best(folder_path):
+    # Iterate over all files and folders in the specified directory
+    for root, dirs, files in os.walk(folder_path):
+        # Check each file and folder name for '.best' or '.keras'
+        for name in files + dirs:
+            if ".best" in name or ".keras" in name:
+                return True
+    return False
+
+
+def get_folder_sizes(directory=os.path.expanduser("~")):
+    folder_sizes = {}
+
+    for folder in os.listdir(directory):
+        folder_path = os.path.join(directory, folder)
+        if os.path.isdir(folder_path):
+            try:
+                size = (
+                    subprocess.check_output(["du", "-sk", folder_path])
+                    .split()[0]
+                    .decode("utf-8")
+                )
+                folder_sizes[folder] = int(size)
+            except subprocess.CalledProcessError:
+                logger.info(f"Skipping {folder_path}: Permission Denied")
+
+    sorted_folders = sorted(folder_sizes.items(), key=lambda x: x[1], reverse=True)
+    logger.info(f"{'Folder':<50}{'Size (MB)':>10}")
+    logger.info("=" * 60)
+    for folder, size in sorted_folders:
+        logger.info(f"{folder:<50}{size / (1024*1024):>10.2f}")
+
+
+def create_cron_job(
+    script_path,
+    venv_path,
+    log_file,
+    pythonpath,
+    cwd,
+    job_frequency="* * * * *",
+    cron_name="My Custom Cron Job",
+):
+    """
+    Creates a cron job to run a Python script with a virtual environment, logging output, and setting PYTHONPATH and CWD.
+
+    Parameters:
+    - script_path (str): Path to the Python script to run.
+    - venv_path (str): Path to the virtual environment's Python interpreter.
+    - log_file (str): Path to the log file for output.
+    - pythonpath (str): Value for the PYTHONPATH environment variable.
+    - cwd (str): Working directory from which the script should run.
+    - job_frequency (str): Cron timing syntax (default is every minute).
+    - cron_name (str): Name to identify the cron job.
+    """
+    # Construct the cron command
+    cron_command = (
+        f"{job_frequency} /bin/zsh -c 'pgrep -fl python | grep -q {os.path.basename(script_path)} "
+        f'|| (echo -e "Cron job {cron_name} started at $(date)" >> {log_file} && cd {cwd} && '
+        f"PYTHONPATH={pythonpath} {venv_path}/bin/python {script_path} >> {log_file} 2>&1)'"
+    )
+
+    # Check existing cron jobs and remove any with the same comment
+    subprocess.run(f"(crontab -l | grep -v '{cron_name}') | crontab -", shell=True)
+
+    # Add the new cron job with the comment
+    full_cron_job = f"{cron_command} # {cron_name}\n"
+    subprocess.run(f'(crontab -l; echo "{full_cron_job}") | crontab -', shell=True)
+    logger.info(f"Cron job created: {full_cron_job}")
+
+
+def remove_all_cron_jobs():
+    """
+    Removes all cron jobs for the current user.
+    """
+    try:
+        # Clear the user's crontab
+        subprocess.run("crontab -r", shell=True, check=True)
+        logger.info("All cron jobs have been removed successfully.")
+    except subprocess.CalledProcessError:
+        logger.info(
+            "Failed to remove cron jobs. There may not be any cron jobs to remove, or there could be a permissions issue."
+        )
+
+
+def serialize_timestamp(dict: dict):
+    def convert(obj):
+        if isinstance(obj, (datetime, date, pd.Timestamp)):
+            return obj.isoformat()
+
+        return obj
+
+    return [{k: convert(v) for k, v in item.items()} for item in dict]
+
+
+def remove_accents(text: str) -> str:
+    """
+    Cleans the text of:
+    - Broken Unicode
+    - Accents
+    - Control characters (including \x00, \u0000, etc.)
+    - Escape sequences
+    - Non-printable characters
+    - Excessive punctuation (like ........ or !!!!)
+    """
+
+    # Step 1: Fix mojibake and broken Unicode
+    text = fix_text(text)
+
+    # Step 2 bis: Normalize accents
+    text = unicodedata.normalize("NFKD", text)
+    text = text.encode("ASCII", "ignore").decode("utf8")
+
+    # Step 3: Remove known weird tokens
+    text = text.replace("<|endoftext|>", "")
+    text = text.replace("\u0000", "").replace("\x00", "")
+
+    # Step 4: Remove raw control characters (e.g., \x1f)
+    text = "".join(c for c in text if unicodedata.category(c)[0] != "C" or c == "\n")
+
+    # Step 5: Remove literal escape sequences like \xNN
+    text = re.sub(r"\\x[0-9a-fA-F]{2}", "", text)
+
+    # Step 6: Remove non-printable characters
+    printable = set(string.printable)
+    text = "".join(c for c in text if c in printable)
+
+    # Step 7: Collapse repeated punctuation (e.g., ........ → .)
+    text = re.sub(r"([!?.])\1{2,}", r"\1", text)  # !!!!!! → !
+    text = re.sub(r"([-—])\1{1,}", r"\1", text)  # ------ → -
+    text = re.sub(r"([,.]){4,}", r"\1", text)  # ...... → .
+
+    return text.strip()
+
+
+def serialize_for_json(obj):
+    """
+    Recursively convert any object into a JSON-serializable structure.
+    Classes and class instances are converted to readable strings like 'ClassName()'.
+    """
+    if isinstance(obj, (str, int, float, bool, type(None))):
+        return obj
+    elif isinstance(obj, dict):
+        return {str(k): serialize_for_json(v) for k, v in obj.items()}
+    elif isinstance(obj, (list, tuple, set)):
+        return [serialize_for_json(v) for v in obj]
+    elif isinstance(obj, type):
+        # A class/type object like int, str, etc.
+        return obj.__name__
+    elif hasattr(obj, "__class__"):
+        return f"{obj.__class__.__name__}()"
+    else:
+        return str(obj)