lecrapaud 0.5.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

lecrapaud/db/models/feature_selection.py

@@ -60,9 +60,9 @@ class FeatureSelection(Base):
     )
     training_time = Column(Integer)
     best_features_path = Column(String(255))
-    dataset_id = Column(
+    experiment_id = Column(
         BigInteger,
-        ForeignKey("lecrapaud_datasets.id", ondelete="CASCADE"),
+        ForeignKey("lecrapaud_experiments.id", ondelete="CASCADE"),
         nullable=False,
     )
     target_id = Column(
@@ -71,8 +71,8 @@ class FeatureSelection(Base):
         nullable=False,
     )
 
-    dataset = relationship(
-        "Dataset", back_populates="feature_selections", lazy="selectin"
+    experiment = relationship(
+        "Experiment", back_populates="feature_selections", lazy="selectin"
     )
     target = relationship(
         "Target", back_populates="feature_selections", lazy="selectin"
@@ -92,7 +92,7 @@ class FeatureSelection(Base):
 
     __table_args__ = (
         UniqueConstraint(
-            "dataset_id", "target_id", name="uq_feature_selection_composite"
+            "experiment_id", "target_id", name="uq_feature_selection_composite"
         ),
     )
 

lecrapaud/db/models/model_selection.py

@@ -43,9 +43,9 @@ class ModelSelection(Base):
         ForeignKey("lecrapaud_targets.id", ondelete="CASCADE"),
         nullable=False,
     )
-    dataset_id = Column(
+    experiment_id = Column(
         BigInteger,
-        ForeignKey("lecrapaud_datasets.id", ondelete="CASCADE"),
+        ForeignKey("lecrapaud_experiments.id", ondelete="CASCADE"),
         nullable=False,
     )
 
@@ -56,13 +56,13 @@ class ModelSelection(Base):
         cascade="all, delete-orphan",
         lazy="selectin",
     )
-    dataset = relationship(
-        "Dataset", back_populates="model_selections", lazy="selectin"
+    experiment = relationship(
+        "Experiment", back_populates="model_selections", lazy="selectin"
     )
     target = relationship("Target", back_populates="model_selections", lazy="selectin")
 
     __table_args__ = (
         UniqueConstraint(
-            "target_id", "dataset_id", name="uq_model_selection_composite"
+            "target_id", "experiment_id", name="uq_model_selection_composite"
         ),
     )

lecrapaud/db/models/score.py

@@ -6,6 +6,7 @@ from sqlalchemy import (
     ForeignKey,
     BigInteger,
     TIMESTAMP,
+    JSON,
 )
 from sqlalchemy import func
 from sqlalchemy.orm import relationship
@@ -43,9 +44,10 @@ class Score(Base):
     recall = Column(Float)
     f1 = Column(Float)
     roc_auc = Column(Float)
-    threshold = Column(Float)
+    thresholds = Column(JSON)
     precision_at_threshold = Column(Float)
     recall_at_threshold = Column(Float)
+    f1_at_threshold = Column(Float)
     model_training_id = Column(
         BigInteger,
         ForeignKey("lecrapaud_model_trainings.id", ondelete="CASCADE"),

lecrapaud/db/models/target.py

@@ -19,7 +19,7 @@ from sqlalchemy.orm import relationship, Mapped, mapped_column, DeclarativeBase
 
 from lecrapaud.db.session import get_db
 from lecrapaud.db.models.base import Base
-from lecrapaud.db.models.dataset import lecrapaud_dataset_target_association
+from lecrapaud.db.models.experiment import lecrapaud_experiment_target_association
 
 
 class Target(Base):
@@ -38,9 +38,9 @@ class Target(Base):
     type = Column(String(50), nullable=False)
     description = Column(String(255))
 
-    datasets = relationship(
-        "Dataset",
-        secondary=lecrapaud_dataset_target_association,
+    experiments = relationship(
+        "Experiment",
+        secondary=lecrapaud_experiment_target_association,
         back_populates="targets",
         lazy="selectin",
     )

lecrapaud/db/session.py

@@ -14,7 +14,7 @@ _engine = None
 _SessionLocal = None
 if DB_URI:
     DATABASE_URL = DB_URI
-elif DB_USER and DB_PASSWORD and DB_HOST and DB_PORT and DB_NAME:
+elif DB_USER:
     DATABASE_URL = (
         f"mysql+pymysql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
     )
@@ -23,11 +23,10 @@ else:
 
 
 def init_db(uri: str = None):
-    print(f"Initializing database with URI: {uri}")
     global _engine, _SessionLocal, DATABASE_URL
     if uri:
         DATABASE_URL = uri
-    elif DB_USER and DB_PASSWORD and DB_HOST and DB_PORT and DB_NAME:
+    elif DB_USER:
         DATABASE_URL = (
             f"mysql+pymysql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
         )
@@ -39,6 +38,7 @@ def init_db(uri: str = None):
             "DB_USER, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME or DB_URI, "
             "or provide a `uri` argument to LeCrapaud"
         )
+    print(f"Initializing database with URI: {DATABASE_URL}")
 
     # Use urlparse for robust parsing
     parsed = urlparse(DATABASE_URL)
@@ -77,7 +77,7 @@ def init_db(uri: str = None):
 @contextmanager
 def get_db():
     if _SessionLocal is None:
-        raise RuntimeError("Database not initialized. Call `init_db()` first.")
+        init_db()
     db = _SessionLocal()
     try:
         yield db

lecrapaud/directories.py

@@ -3,11 +3,9 @@ from pathlib import Path
 
 cwd = os.getcwd()
 tmp_dir = cwd + "/tmp"
-data_dir = tmp_dir + "/data"
 logger_dir = cwd + "/log"
 
 os.makedirs(tmp_dir, exist_ok=True)
-os.makedirs(data_dir, exist_ok=True)
 os.makedirs(logger_dir, exist_ok=True)
 
 

lecrapaud/experiment.py

@@ -1,27 +1,25 @@
-import pandas as pd
 import os
 from pathlib import Path
 
+import pandas as pd
+
+# Set up coverage file path
 os.environ["COVERAGE_FILE"] = str(Path(".coverage").resolve())
 
-# Internal
+# Internal imports
 from lecrapaud.directories import tmp_dir
-from lecrapaud.utils import logger
-from lecrapaud.config import PYTHON_ENV
-from lecrapaud.db import (
-    Dataset,
-    Target,
-)
+from lecrapaud.db import Experiment, Target
 from lecrapaud.db.session import get_db
 
 
-def create_dataset(
+def create_experiment(
     data: pd.DataFrame,
     corr_threshold,
     percentile,
     max_features,
     date_column,
     group_column,
+    experiment_name,
     **kwargs,
 ):
     dates = {}
@@ -37,20 +35,20 @@ def create_dataset(
     with get_db() as db:
         all_targets = Target.get_all(db=db)
         targets = [target for target in all_targets if target.name in data.columns]
-        dataset_name = f"data_{groups["number_of_groups"] if group_column else 'ng'}_{corr_threshold}_{percentile}_{max_features}_{dates['start_date'].date() if date_column else 'nd'}_{dates['end_date'].date() if date_column else 'nd'}"
+        experiment_name = f"{experiment_name}_{groups["number_of_groups"] if group_column else 'ng'}_{corr_threshold}_{percentile}_{max_features}_{dates['start_date'].date() if date_column else 'nd'}_{dates['end_date'].date() if date_column else 'nd'}"
 
-        dataset_dir = f"{tmp_dir}/{dataset_name}"
-        preprocessing_dir = f"{dataset_dir}/preprocessing"
-        data_dir = f"{dataset_dir}/data"
-        os.makedirs(dataset_dir, exist_ok=True)
+        experiment_dir = f"{tmp_dir}/{experiment_name}"
+        preprocessing_dir = f"{experiment_dir}/preprocessing"
+        data_dir = f"{experiment_dir}/data"
+        os.makedirs(experiment_dir, exist_ok=True)
         os.makedirs(preprocessing_dir, exist_ok=True)
         os.makedirs(data_dir, exist_ok=True)
 
-        dataset = Dataset.upsert(
+        experiment = Experiment.upsert(
             match_fields=["name"],
             db=db,
-            name=dataset_name,
-            path=Path(dataset_dir).resolve(),
+            name=experiment_name,
+            path=Path(experiment_dir).resolve(),
             type="training",
             size=data.shape[0],
             corr_threshold=corr_threshold,
@@ -59,6 +57,15 @@ def create_dataset(
             **groups,
             **dates,
             targets=targets,
+            context={
+                "corr_threshold": corr_threshold,
+                "percentile": percentile,
+                "max_features": max_features,
+                "date_column": date_column,
+                "group_column": group_column,
+                "experiment_name": experiment_name,
+                **kwargs,
+            },
         )
 
-        return dataset
+        return experiment

lecrapaud/feature_engineering.py

@@ -61,7 +61,7 @@ from lecrapaud.integrations.openai_integration import (
 )
 from lecrapaud.feature_selection import get_features_by_types
 from lecrapaud.utils import logger
-from lecrapaud.db import Target, Feature, Dataset
+from lecrapaud.db import Target, Feature, Experiment
 from lecrapaud.config import PYTHON_ENV
 
 
@@ -308,7 +308,7 @@ class PreprocessFeature:
     def __init__(
         self,
         data: pd.DataFrame,
-        dataset,
+        experiment,
         time_series: bool = False,
         date_column: str | None = None,
         group_column: str | None = None,
@@ -326,7 +326,7 @@ class PreprocessFeature:
         self.data = data
         self.data.columns = self.data.columns.str.upper()
 
-        self.dataset = dataset
+        self.experiment = experiment
         self.columns_pca = columns_pca
         self.columns_onehot = columns_onehot
         self.columns_binary = columns_binary
@@ -341,10 +341,10 @@ class PreprocessFeature:
         self.val_size = val_size
         self.test_size = test_size
 
-        self.dataset_dir = self.dataset.path
-        self.dataset_id = self.dataset.id
-        self.data_dir = f"{self.dataset_dir}/data"
-        self.preprocessing_dir = f"{self.dataset_dir}/preprocessing"
+        self.experiment_dir = self.experiment.path
+        self.experiment_id = self.experiment.id
+        self.data_dir = f"{self.experiment_dir}/data"
+        self.preprocessing_dir = f"{self.experiment_dir}/preprocessing"
 
     def run(self):
         # Split
@@ -358,8 +358,8 @@ class PreprocessFeature:
 
         # PCA
         train, pcas = self.add_pca_features(train)
-        val, _ = self.add_pca_features(test, pcas=pcas)
-        test, _ = self.add_pca_features(val, pcas=pcas)
+        val, _ = self.add_pca_features(val, pcas=pcas)
+        test, _ = self.add_pca_features(test, pcas=pcas)
 
         joblib.dump(pcas, f"{self.preprocessing_dir}/pcas.pkl")
 
@@ -377,7 +377,7 @@ class PreprocessFeature:
         joblib.dump(self.data, f"{self.data_dir}/full.pkl")
         joblib.dump(transformer, f"{self.preprocessing_dir}/column_transformer.pkl")
         summary = summarize_dataframe(train)
-        summary.to_csv(f"{self.dataset_dir}/feature_summary.csv", index=False)
+        summary.to_csv(f"{self.experiment_dir}/feature_summary.csv", index=False)
 
         return train, val, test
 
@@ -431,9 +431,9 @@ class PreprocessFeature:
                 f"{data.shape} {name} data from {dates[f"{name}_start_date"].strftime('%d/%m/%Y')} to {dates[f"{name}_end_date"].strftime('%d/%m/%Y')}"
             )
 
-        Dataset.update(
+        Experiment.upsert(
             match_fields=["id"],
-            id=self.dataset_id,
+            id=self.experiment_id,
             train_size=len(train),
             val_size=len(val),
             test_size=len(test),
@@ -454,7 +454,7 @@ class PreprocessFeature:
         Splits a DataFrame into train, validation, and test sets.
 
         Parameters:
-            df (pd.DataFrame): The full dataset
+            df (pd.DataFrame): The full experiment
             val_size (float): Proportion of validation set (default 0.1)
             test_size (float): Proportion of test set (default 0.1)
             random_state (int): Random seed for reproducibility
@@ -646,23 +646,52 @@ class PreprocessFeature:
 
         # Insert features in db
         if save_in_db:
-            # TODO: in bulk
+            # Get feature types from transformed data
             categorical_features, numerical_features = get_features_by_types(
                 X_transformed
             )
-            for feature in categorical_features:
-                Feature.upsert(match_fields=["name"], name=feature, type="categorical")
-            for feature in numerical_features:
-                Feature.upsert(match_fields=["name"], name=feature, type="numerical")
-            for target in y.columns:
-                target_number = int(target.split("_")[1])
-                type = (
+
+            # Get column names from DataFrames
+            cat_feature_names = categorical_features.columns.tolist()
+            num_feature_names = numerical_features.columns.tolist()
+
+            # Combine all feature names and their types
+            all_feature_names = cat_feature_names + num_feature_names
+            all_feature_types = ["categorical"] * len(cat_feature_names) + [
+                "numerical"
+            ] * len(num_feature_names)
+
+            # Upsert features in bulk if we have any features
+            if all_feature_names:
+                Feature.upsert_bulk(
+                    match_fields=["name"],
+                    name=all_feature_names,
+                    type=all_feature_types,
+                )
+
+            # Upsert targets in bulk
+            target_names = y.columns.tolist()
+            target_types = [
+                (
                     "classification"
-                    if target_number in self.target_clf
+                    if int(target.split("_")[1]) in self.target_clf
                     else "regression"
                 )
-                # TODO: what about description here ?
-                Target.upsert(match_fields=["name", "type"], name=target, type=type)
+                for target in target_names
+            ]
+
+            Target.upsert_bulk(
+                match_fields=["name"], name=target_names, type=target_types
+            )
+
+            # Get all the upserted objects
+            targets = Target.filter(name__in=target_names)
+
+            # Update experiment with targets
+            experiment = Experiment.get(self.experiment_id)
+            if experiment:
+                experiment.targets = targets
+                experiment.save()
 
         return pd.concat([X_transformed, y], axis=1), transformer
 

lecrapaud/feature_selection.py

@@ -41,7 +41,7 @@ from lecrapaud.directories import tmp_dir, clean_directory
 from lecrapaud.utils import logger
 from lecrapaud.config import PYTHON_ENV
 from lecrapaud.db import (
-    Dataset,
+    Experiment,
     Target,
     Feature,
     FeatureSelection,
@@ -54,8 +54,8 @@ from lecrapaud.search_space import all_models
 warnings.filterwarnings("ignore", category=FutureWarning)
 
 
-def load_train_data(dataset_dir, target_number, target_type="regression"):
-    data_dir = f"{dataset_dir}/data"
+def load_train_data(experiment_dir, target_number, target_type="regression"):
+    data_dir = f"{experiment_dir}/data"
 
     logger.info("Loading data...")
     train = joblib.load(f"{data_dir}/train.pkl")
@@ -74,8 +74,8 @@ def load_train_data(dataset_dir, target_number, target_type="regression"):
 
 
 class FeatureSelectionEngine:
-    def __init__(self, train, dataset, target_number, target_clf, **kwargs):
-        self.dataset = dataset
+    def __init__(self, train, experiment, target_number, target_clf, **kwargs):
+        self.experiment = experiment
         self.train = train
         self.target_number = target_number
         self.target_clf = target_clf
@@ -83,16 +83,16 @@ class FeatureSelectionEngine:
         self.target_type = (
             "classification" if self.target_number in self.target_clf else "regression"
         )
-        self.percentile = self.dataset.percentile
-        self.corr_threshold = self.dataset.corr_threshold
-        self.max_features = self.dataset.max_features
-
-        self.dataset_dir = self.dataset.path
-        self.dataset_id = self.dataset.id
-        self.data_dir = f"{self.dataset_dir}/data"
-        self.preprocessing_dir = f"{self.dataset_dir}/preprocessing"
+        self.percentile = self.experiment.percentile
+        self.corr_threshold = self.experiment.corr_threshold
+        self.max_features = self.experiment.max_features
+
+        self.experiment_dir = self.experiment.path
+        self.experiment_id = self.experiment.id
+        self.data_dir = f"{self.experiment_dir}/data"
+        self.preprocessing_dir = f"{self.experiment_dir}/preprocessing"
         self.fs_dir_target = (
-            f"{self.dataset_dir}/{f"TARGET_{self.target_number}"}/feature_selection"
+            f"{self.experiment_dir}/{f"TARGET_{self.target_number}"}/feature_selection"
         )
         os.makedirs(self.fs_dir_target, exist_ok=True)
 
@@ -119,12 +119,14 @@ class FeatureSelectionEngine:
         max_features = self.max_features
 
         feature_selection = FeatureSelection.upsert(
-            match_fields=["target_id", "dataset_id"],
+            match_fields=["target_id", "experiment_id"],
             target_id=target.id,
-            dataset_id=self.dataset_id,
+            experiment_id=self.experiment_id,
         )
 
-        if feature_selection.best_features_path:
+        if feature_selection.best_features_path and os.path.exists(
+            feature_selection.best_features_path
+        ):
             return joblib.load(feature_selection.best_features_path)
 
         self.X = self.train.loc[:, ~self.train.columns.str.contains("^TARGET_")]
@@ -767,7 +769,7 @@ class PreprocessModel:
         train,
         val,
         test,
-        dataset,
+        experiment,
         target_numbers,
         target_clf,
         models_idx,
@@ -777,7 +779,10 @@ class PreprocessModel:
         date_column,
         **kwargs,
     ):
-        self.dataset = dataset
+        self.train = train
+        self.val = val
+        self.test = test
+        self.experiment = experiment
         self.target_numbers = target_numbers
         self.target_clf = target_clf
         self.models_idx = models_idx
@@ -786,13 +791,16 @@ class PreprocessModel:
         self.group_column = group_column
         self.date_column = date_column
 
-        self.dataset_dir = dataset.path
-        self.data_dir = f"{self.dataset_dir}/data"
-        self.preprocessing_dir = f"{self.dataset_dir}/preprocessing"
+        self.experiment_dir = experiment.path
+        self.data_dir = f"{self.experiment_dir}/data"
+        self.preprocessing_dir = f"{self.experiment_dir}/preprocessing"
 
-        self.all_features = dataset.get_all_features(
+        self.all_features = experiment.get_all_features(
             date_column=date_column, group_column=group_column
         )
+
+    def run(self):
+        # save data
         columns_to_keep = self.all_features + [
             f"TARGET_{i}" for i in self.target_numbers
         ]
@@ -801,15 +809,9 @@ class PreprocessModel:
         ]
         if duplicates:
             raise ValueError(f"Doublons détectés dans columns_to_keep: {duplicates}")
-
-        self.train = train[columns_to_keep]
-        if isinstance(val, pd.DataFrame):
-            self.val = val[columns_to_keep]
-        if isinstance(test, pd.DataFrame):
-            self.test = test[columns_to_keep]
-
-    def run(self):
-        # save data
+        self.train = self.train[columns_to_keep]
+        self.val = self.val[columns_to_keep]
+        self.test = self.test[columns_to_keep]
         joblib.dump(self.train, f"{self.data_dir}/train.pkl")
         joblib.dump(self.val, f"{self.data_dir}/val.pkl")
         joblib.dump(self.test, f"{self.data_dir}/test.pkl")
@@ -870,8 +872,11 @@ class PreprocessModel:
 
     def inference(self):
         # self.train is new data here
+        columns_to_keep = self.all_features
+        self.train = self.train[columns_to_keep]
+
         scaler_x = joblib.load(f"{self.preprocessing_dir}/scaler_x.pkl")
-        scaled_data = scaler_x.transform(self.train)
+        scaled_data = scaler_x.transform(self.train)  # TODO: utiliser scale_data
         scaled_data = pd.DataFrame(
             scaled_data, columns=self.train.columns, index=self.train.index
         )
@@ -1012,7 +1017,7 @@ class PreprocessModel:
             df_reshaped = df.apply(list, axis=1).apply(lambda x: [list(x)])
             df_reshaped = pd.concat([df_reshaped, group_series], axis=1)
 
-            logger.info("Grouping method stock and creating timesteps...")
+            logger.info("Grouping features and creating timesteps...")
             df_reshaped = (
                 df_reshaped.groupby(group_column)[0]
                 .apply(lambda x: shiftsum(x, timesteps))
@@ -1058,13 +1063,13 @@ class PreprocessModel:
 def feature_selection_analysis(feature_selection_id: int, n_components: int = 5):
 
     feature_selection = FeatureSelection.get(feature_selection_id)
-    dataset_dir = feature_selection.dataset.path
+    experiment_dir = feature_selection.experiment.path
     features = [f.name for f in feature_selection.features]
     target = feature_selection.target.name
     target_number = target.split("_")[1]
 
     train, val, train_scaled, val_scaled, _scaler_y = load_train_data(
-        dataset_dir, target_number, target_type=feature_selection.target.type
+        experiment_dir, target_number, target_type=feature_selection.target.type
     )
     train = train[features + [target]]
     train_scaled = train_scaled[features + [target]]

lecrapaud/jobs/tasks.py

@@ -2,7 +2,7 @@ from lecrapaud.jobs import app
 
 # from honeybadger import honeybadger
 from lecrapaud.send_daily_emails import send_daily_emails
-from lecrapaud.config import DATASET_ID, RECEIVER_EMAIL
+from lecrapaud.config import EXPERIMENT_ID, RECEIVER_EMAIL
 from lecrapaud.training import run_training
 from lecrapaud.constants import stock_list_3
 from lecrapaud.search_space import get_models_idx
@@ -18,9 +18,9 @@ from lecrapaud.search_space import get_models_idx
 def task_send_daily_emails(self):
     try:
         print(f"[Attempt #{self.request.retries}] task_send_daily_emails")
-        dataset_id = int(DATASET_ID)
+        experiment_id = int(EXPERIMENT_ID)
         email = RECEIVER_EMAIL
-        return send_daily_emails(email, dataset_id)
+        return send_daily_emails(email, experiment_id)
     except Exception as e:
         print(e)
         # honeybadger.notify(e)
@@ -49,7 +49,7 @@ def task_training_experiment(self):
             perform_hyperoptimization=True,
             perform_crossval=False,
             preserve_model=False,
-            session_name="20y_stock_list_3_linear_xgb",
+            experiment_name="20y_stock_list_3_linear_xgb",
         )
     except Exception as e:
         print(e)