lecrapaud 0.19.3__py3-none-any.whl → 0.20.1__py3-none-any.whl

lecrapaud/db/models/base.py

@@ -10,24 +10,27 @@ from sqlalchemy.orm.attributes import InstrumentedAttribute
 from lecrapaud.db.session import get_db
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.dialects.mysql import insert as mysql_insert
+from sqlalchemy import UniqueConstraint
+from sqlalchemy.inspection import inspect as sqlalchemy_inspect
 from lecrapaud.config import LECRAPAUD_TABLE_PREFIX
 
 
 def with_db(func):
     """Decorator to provide a database session to the wrapped function.
-    
+
     If a db parameter is already provided, it will be used. Otherwise,
     a new session will be created and automatically managed.
     """
+
     @wraps(func)
     def wrapper(*args, **kwargs):
         if "db" in kwargs and kwargs["db"] is not None:
             return func(*args, **kwargs)
-            
+
         with get_db() as db:
             kwargs["db"] = db
             return func(*args, **kwargs)
-            
+
     return wrapper
 
 
@@ -106,51 +109,6 @@ class Base(DeclarativeBase):
             ]
         return results
 
-    @classmethod
-    @with_db
-    def upsert_bulk(cls, db=None, match_fields: list[str] = None, **kwargs):
-        """
-        Performs a bulk upsert into the database using ON DUPLICATE KEY UPDATE.
-
-        Args:
-            db (Session): SQLAlchemy DB session
-            match_fields (list[str]): Fields to match on for deduplication
-            **kwargs: Column-wise keyword arguments (field_name=[...])
-        """
-        # Ensure all provided fields have values of equal length
-        value_lengths = [len(v) for v in kwargs.values()]
-        if not value_lengths or len(set(value_lengths)) != 1:
-            raise ValueError(
-                "All field values must be non-empty lists of the same length."
-            )
-
-        # Convert column-wise kwargs to row-wise list of dicts
-        items = [dict(zip(kwargs.keys(), row)) for row in zip(*kwargs.values())]
-        if not items:
-            return
-
-        stmt = mysql_insert(cls.__table__).values(items)
-
-        # Default to primary keys if match_fields not provided
-        if not match_fields:
-            match_fields = [col.name for col in cls.__table__.primary_key.columns]
-
-        # Ensure all columns to be updated are in the insert
-        update_dict = {
-            c.name: stmt.inserted[c.name]
-            for c in cls.__table__.columns
-            if c.name not in match_fields and c.name in items[0]
-        }
-
-        if not update_dict:
-            # Avoid triggering ON DUPLICATE KEY UPDATE with empty dict
-            db.execute(stmt.prefix_with("IGNORE"))
-        else:
-            upsert_stmt = stmt.on_duplicate_key_update(**update_dict)
-            db.execute(upsert_stmt)
-
-        db.commit()
-
     @classmethod
     @with_db
     def filter(cls, db=None, **kwargs):
@@ -198,33 +156,113 @@ class Base(DeclarativeBase):
 
     @classmethod
     @with_db
-    def upsert(cls, match_fields: list[str], db=None, **kwargs):
+    def upsert(cls, db=None, **kwargs):
         """
-        Upsert an instance of the model: update if found, else create.
+        Upsert an instance of the model using MySQL's ON DUPLICATE KEY UPDATE.
 
-        :param match_fields: list of field names to use for matching
         :param kwargs: all fields for creation or update
         """
-        filters = [
-            getattr(cls, field) == kwargs[field]
-            for field in match_fields
-            if field in kwargs
-        ]
+        # Use INSERT ... ON DUPLICATE KEY UPDATE
+        stmt = mysql_insert(cls.__table__).values(**kwargs)
+        stmt = stmt.on_duplicate_key_update(
+            **{k: v for k, v in kwargs.items() if k != "id"}
+        )
 
-        instance = db.query(cls).filter(*filters).first()
+        result = db.execute(stmt)
+        db.commit()
 
-        if instance:
-            for key, value in kwargs.items():
-                if key != "id":
-                    setattr(instance, key, value)
+        # Get the instance - either the newly inserted or updated one
+        # If updated, lastrowid is 0, so we need to query
+        if result.lastrowid and result.lastrowid > 0:
+            # New insert
+            instance = db.get(cls, result.lastrowid)
         else:
-            instance = cls(**kwargs)
-            db.add(instance)
+            # Updated - need to find it using unique constraint fields
+            mapper = sqlalchemy_inspect(cls)
+            instance = None
+
+            for constraint in mapper.mapped_table.constraints:
+                if isinstance(constraint, UniqueConstraint):
+                    col_names = [col.name for col in constraint.columns]
+                    if all(name in kwargs for name in col_names):
+                        filters = [
+                            getattr(cls, col_name) == kwargs[col_name]
+                            for col_name in col_names
+                        ]
+                        instance = db.query(cls).filter(*filters).first()
+                        if instance:
+                            break
+
+            # Check for single column unique constraints
+            if not instance:
+                for col in mapper.mapped_table.columns:
+                    if col.unique and col.name in kwargs:
+                        instance = (
+                            db.query(cls)
+                            .filter(getattr(cls, col.name) == kwargs[col.name])
+                            .first()
+                        )
+                        if instance:
+                            break
+
+            # If still not found, try to find by all kwargs (excluding None values)
+            if not instance:
+                instance = (
+                    db.query(cls)
+                    .filter_by(
+                        **{
+                            k: v
+                            for k, v in kwargs.items()
+                            if v is not None and k != "id"
+                        }
+                    )
+                    .first()
+                )
+
+        if instance:
+            db.refresh(instance)
 
-        db.commit()
-        db.refresh(instance)
         return instance
 
+    @classmethod
+    @with_db
+    def bulk_upsert(cls, rows: list[dict] = None, db=None, **kwargs):
+        """
+        Performs a bulk upsert into the database using ON DUPLICATE KEY UPDATE.
+
+        Args:
+            rows (list[dict]): List of dictionaries representing rows to upsert
+            db (Session): SQLAlchemy DB session
+            **kwargs: Column-wise keyword arguments (field_name=[...]) for backwards compatibility
+        """
+        # Handle both new format (rows) and legacy format (kwargs)
+        if rows is None and kwargs:
+            # Legacy format: convert column-wise kwargs to row-wise list of dicts
+            value_lengths = [len(v) for v in kwargs.values()]
+            if not value_lengths or len(set(value_lengths)) != 1:
+                raise ValueError(
+                    "All field values must be non-empty lists of the same length."
+                )
+            rows = [dict(zip(kwargs.keys(), row)) for row in zip(*kwargs.values())]
+
+        if not rows:
+            return 0
+
+        BATCH_SIZE = 200
+        total_affected = 0
+
+        for i in range(0, len(rows), BATCH_SIZE):
+            batch = rows[i : i + BATCH_SIZE]
+            stmt = mysql_insert(cls.__table__).values(batch)
+            stmt = stmt.on_duplicate_key_update(
+                **{key: stmt.inserted[key] for key in batch[0] if key != "id"}
+            )
+            result = db.execute(stmt)
+            total_affected += result.rowcount
+
+        db.commit()
+        return total_affected
+
     @classmethod
     @with_db
     def delete(cls, id: int, db=None):

lecrapaud/db/models/experiment.py

@@ -20,8 +20,7 @@ from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy import func
 from statistics import fmean as mean
 from lecrapaud.db.models.model_selection import ModelSelection
-from lecrapaud.db.models.model_training import ModelTraining
-from lecrapaud.db.models.score import Score
+from lecrapaud.db.models.model_selection_score import ModelSelectionScore
 
 from lecrapaud.db.models.base import Base, with_db
 from lecrapaud.db.models.utils import create_association_table
@@ -51,10 +50,43 @@ class Experiment(Base):
     )
     name = Column(String(255), nullable=False)
     path = Column(String(255))  # we do not have this at creation time
-    type = Column(String(50), nullable=False)
     size = Column(Integer, nullable=False)
     train_size = Column(Integer)
     val_size = Column(Integer)
+    test_size = Column(Integer)
+    number_of_groups = Column(Integer)
+    list_of_groups = Column(JSON)
+    number_of_targets = Column(Integer)
+    start_date = Column(DateTime)
+    end_date = Column(DateTime)
+    train_start_date = Column(DateTime)
+    train_end_date = Column(DateTime)
+    val_start_date = Column(DateTime)
+    val_end_date = Column(DateTime)
+    test_start_date = Column(DateTime)
+    test_end_date = Column(DateTime)
+    context = Column(JSON)
+
+    feature_selections = relationship(
+        "FeatureSelection",
+        back_populates="experiment",
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+
+    targets = relationship(
+        "Target",
+        secondary=lecrapaud_experiment_target_association,
+        back_populates="experiments",
+        lazy="selectin",
+    )
+
+    __table_args__ = (
+        UniqueConstraint(
+            "name",
+            name="uq_experiments_composite",
+        ),
+    )
 
     # Relationships
     model_selections = relationship(
@@ -69,18 +101,9 @@ class Experiment(Base):
         """Best RMSE score across all model selections and trainings."""
         # Get the minimum RMSE for each model selection
         min_scores = [
-            min(
-                score.rmse
-                for mt in ms.model_trainings
-                for score in mt.score
-                if score.rmse is not None
-            )
+            min(mss.rmse for mss in ms.model_selection_scores if mss.rmse is not None)
             for ms in self.model_selections
-            if any(
-                score.rmse is not None
-                for mt in ms.model_trainings
-                for score in mt.score
-            )
+            if any(mss.rmse is not None for mss in ms.model_selection_scores)
         ]
         return min(min_scores) if min_scores else None
 
@@ -90,17 +113,12 @@ class Experiment(Base):
         # Get the minimum LogLoss for each model selection
         min_scores = [
             min(
-                score.logloss
-                for mt in ms.model_trainings
-                for score in mt.score
-                if score.logloss is not None
+                mss.logloss
+                for mss in ms.model_selection_scores
+                if mss.logloss is not None
             )
             for ms in self.model_selections
-            if any(
-                score.logloss is not None
-                for mt in ms.model_trainings
-                for score in mt.score
-            )
+            if any(mss.logloss is not None for mss in ms.model_selection_scores)
         ]
         return min(min_scores) if min_scores else None
 
@@ -109,18 +127,9 @@ class Experiment(Base):
         """Average RMSE score across all model selections and trainings."""
         # Get the minimum RMSE for each model selection
         min_scores = [
-            min(
-                score.rmse
-                for mt in ms.model_trainings
-                for score in mt.score
-                if score.rmse is not None
-            )
+            min(mss.rmse for mss in ms.model_selection_scores if mss.rmse is not None)
             for ms in self.model_selections
-            if any(
-                score.rmse is not None
-                for mt in ms.model_trainings
-                for score in mt.score
-            )
+            if any(mss.rmse is not None for mss in ms.model_selection_scores)
         ]
         return mean(min_scores) if min_scores else None
 
@@ -130,57 +139,15 @@ class Experiment(Base):
         # Get the minimum LogLoss for each model selection
         min_scores = [
             min(
-                score.logloss
-                for mt in ms.model_trainings
-                for score in mt.score
-                if score.logloss is not None
+                mss.logloss
+                for mss in ms.model_selection_scores
+                if mss.logloss is not None
             )
             for ms in self.model_selections
-            if any(
-                score.logloss is not None
-                for mt in ms.model_trainings
-                for score in mt.score
-            )
+            if any(mss.logloss is not None for mss in ms.model_selection_scores)
         ]
         return mean(min_scores) if min_scores else None
 
-    test_size = Column(Integer)
-    corr_threshold = Column(Float, nullable=False)
-    max_features = Column(Integer, nullable=False)
-    percentile = Column(Float, nullable=False)
-    number_of_groups = Column(Integer)
-    list_of_groups = Column(JSON)
-    start_date = Column(DateTime)
-    end_date = Column(DateTime)
-    train_start_date = Column(DateTime)
-    train_end_date = Column(DateTime)
-    val_start_date = Column(DateTime)
-    val_end_date = Column(DateTime)
-    test_start_date = Column(DateTime)
-    test_end_date = Column(DateTime)
-    context = Column(JSON)
-
-    feature_selections = relationship(
-        "FeatureSelection",
-        back_populates="experiment",
-        cascade="all, delete-orphan",
-        lazy="selectin",
-    )
-
-    targets = relationship(
-        "Target",
-        secondary=lecrapaud_experiment_target_association,
-        back_populates="experiments",
-        lazy="selectin",
-    )
-
-    __table_args__ = (
-        UniqueConstraint(
-            "name",
-            name="uq_experiments_composite",
-        ),
-    )
-
     @classmethod
     @with_db
     def get_all_by_name(cls, name: str | None = None, limit: int = 1000, db=None):
@@ -353,7 +320,7 @@ class Experiment(Base):
             (ms for ms in self.model_selections if ms.target_id == target.id), None
         )
 
-        if not best_model_selection or not best_model_selection.model_trainings:
+        if not best_model_selection or not best_model_selection.model_selection_scores:
             return {
                 "experiment_name": self.name,
                 "target_number": target_number,
@@ -361,22 +328,31 @@ class Experiment(Base):
                 "scores": {},
             }
 
-        # Get the best model training (assuming the first one is the best)
-        best_training = best_model_selection.model_trainings[0]
-
-        # Get the validation score for this training
-        validation_scores = [s for s in best_training.score if s.type == "validation"]
+        # Get the best model score based on lowest logloss or rmse
+        model_scores = best_model_selection.model_selection_scores
 
-        if not validation_scores:
+        # Determine if we should use logloss or rmse based on what's available
+        if any(ms.logloss is not None for ms in model_scores):
+            # Classification: find lowest logloss
+            best_score = min(
+                (ms for ms in model_scores if ms.logloss is not None),
+                key=lambda x: x.logloss,
+            )
+        elif any(ms.rmse is not None for ms in model_scores):
+            # Regression: find lowest rmse
+            best_score = min(
+                (ms for ms in model_scores if ms.rmse is not None), key=lambda x: x.rmse
+            )
+        else:
             return {
                 "experiment_name": self.name,
                 "target_number": target_number,
-                "error": "No validation scores found for the best model",
+                "error": "No scores found for the best model",
                 "scores": {},
             }
 
-        # Get all available metrics from the first validation score
-        score = validation_scores[0]
+        # Use the best score found
+        score = best_score
         available_metrics = [
             "rmse",
             "mae",
@@ -397,13 +373,9 @@ class Experiment(Base):
 
         # Get the model info
         model_info = {
-            "model_type": (
-                best_training.model.model_type if best_training.model else "unknown"
-            ),
-            "model_name": (
-                best_training.model.name if best_training.model else "unknown"
-            ),
-            "training_time_seconds": best_training.training_time,
+            "model_type": (score.model.model_type if score.model else "unknown"),
+            "model_name": (score.model.name if score.model else "unknown"),
+            "training_time_seconds": score.training_time,
         }
 
         return {
@@ -413,7 +385,10 @@ class Experiment(Base):
             "scores": scores,
         }
 
-    def get_features(self, target_number: int):
+    @with_db
+    def get_features(self, target_number: int, db=None):
+        # Ensure we have a fresh instance attached to the session
+        self = db.merge(self)
         targets = [t for t in self.targets if t.name == f"TARGET_{target_number}"]
         if targets:
             target_id = targets[0].id
@@ -429,7 +404,12 @@ class Experiment(Base):
         features = joblib.load(f"{self.path}/TARGET_{target_number}/features.pkl")
         return features
 
-    def get_all_features(self, date_column: str = None, group_column: str = None):
+    @with_db
+    def get_all_features(
+        self, date_column: str = None, group_column: str = None, db=None
+    ):
+        # Ensure we have a fresh instance attached to the session
+        self = db.merge(self)
         target_idx = [target.id for target in self.targets]
         _all_features = chain.from_iterable(
             [f.name for f in fs.features]

lecrapaud/db/models/feature_selection.py

@@ -115,7 +115,4 @@ class FeatureSelection(Base):
             if feature not in self.features:
                 self.features.append(feature)
 
-        db.flush()
-        db.refresh(self)
-        print(self.features)
         return self

lecrapaud/db/models/feature_selection_rank.py

@@ -65,21 +65,3 @@ class FeatureSelectionRank(Base):
             name="uq_feature_selection_rank_composite",
         ),
     )
-
-    @classmethod
-    @with_db
-    def bulk_upsert(cls, rows, db=None):
-        stmt = insert(cls).values(rows)
-
-        update_fields = {
-            key: stmt.inserted[key]
-            for key in rows[0]
-            if key not in ("feature_selection_id", "feature_id", "method")
-        }
-
-        stmt = stmt.on_duplicate_key_update(**update_fields)
-
-        db.execute(stmt)
-        db.commit()
-
-        return len(rows)

lecrapaud/db/models/model_selection.py

@@ -54,8 +54,8 @@ class ModelSelection(Base):
     )
 
     best_model = relationship("Model", lazy="selectin")
-    model_trainings = relationship(
-        "ModelTraining",
+    model_selection_scores = relationship(
+        "ModelSelectionScore",
         back_populates="model_selection",
         cascade="all, delete-orphan",
         lazy="selectin",

lecrapaud/db/models/{score.py → model_selection_score.py}

@@ -3,10 +3,11 @@ from sqlalchemy import (
     Integer,
     String,
     Float,
+    JSON,
     ForeignKey,
     BigInteger,
     TIMESTAMP,
-    JSON,
+    UniqueConstraint,
 )
 from sqlalchemy import func
 from sqlalchemy.orm import relationship
@@ -14,7 +15,9 @@ from lecrapaud.db.models.base import Base
 from lecrapaud.config import LECRAPAUD_TABLE_PREFIX
 
 
-class Score(Base):
+class ModelSelectionScore(Base):
+    __tablename__ = f"{LECRAPAUD_TABLE_PREFIX}_model_selection_scores"
+    
     id = Column(BigInteger, primary_key=True, index=True, autoincrement=True)
     created_at = Column(
         TIMESTAMP(timezone=True), server_default=func.now(), nullable=False
@@ -25,10 +28,21 @@ class Score(Base):
         onupdate=func.now(),
         nullable=False,
     )
-    type = Column(
-        String(50), nullable=False
-    )  # either hyperopts or validation or crossval
+    
+    # From ModelTraining
+    best_params = Column(JSON)
+    model_path = Column(String(255))
     training_time = Column(Integer)
+    model_id = Column(
+        BigInteger, ForeignKey(f"{LECRAPAUD_TABLE_PREFIX}_models.id"), nullable=False
+    )
+    model_selection_id = Column(
+        BigInteger,
+        ForeignKey(f"{LECRAPAUD_TABLE_PREFIX}_model_selections.id", ondelete="CASCADE"),
+        nullable=False,
+    )
+    
+    # From Score (excluding type and training_time which is already in ModelTraining)
     eval_data_std = Column(Float)
     rmse = Column(Float)
     rmse_std_ratio = Column(Float)
@@ -50,12 +64,15 @@ class Score(Base):
     precision_at_threshold = Column(Float)
     recall_at_threshold = Column(Float)
     f1_at_threshold = Column(Float)
-    model_training_id = Column(
-        BigInteger,
-        ForeignKey(f"{LECRAPAUD_TABLE_PREFIX}_model_trainings.id", ondelete="CASCADE"),
-        nullable=False,
-    )
 
-    model_trainings = relationship(
-        "ModelTraining", back_populates="score", lazy="selectin"
+    # Relationships
+    model = relationship("Model", lazy="selectin")
+    model_selection = relationship(
+        "ModelSelection", back_populates="model_selection_scores", lazy="selectin"
     )
+
+    __table_args__ = (
+        UniqueConstraint(
+            "model_id", "model_selection_id", name="uq_model_selection_score_composite"
+        ),
+    )

lecrapaud/db/session.py

@@ -73,6 +73,7 @@ def init_db(uri: str = None):
         autocommit=False,
         autoflush=False,
         bind=_engine,
+        expire_on_commit=False,  # Prevent detached instance errors
     )
 
     # Step 5: Apply Alembic migrations programmatically

lecrapaud/experiment.py

@@ -3,6 +3,7 @@ from pathlib import Path
 
 import pandas as pd
 import joblib
+from datetime import datetime
 
 # Set up coverage file path
 os.environ["COVERAGE_FILE"] = str(Path(".coverage").resolve())
@@ -15,9 +16,6 @@ from lecrapaud.db.session import get_db
 
 def create_experiment(
     data: pd.DataFrame | str,
-    corr_threshold,
-    percentile,
-    max_features,
     date_column,
     group_column,
     experiment_name,
@@ -42,7 +40,10 @@ def create_experiment(
         targets = [
             target for target in all_targets if target.name in data.columns.str.upper()
         ]
-        experiment_name = f"{experiment_name}_{groups["number_of_groups"] if group_column else 'ng'}_{corr_threshold}_{percentile}_{max_features}_{dates['start_date'].date() if date_column else 'nd'}_{dates['end_date'].date() if date_column else 'nd'}"
+        experiment_name = (
+            f"{experiment_name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+        )
+        number_of_targets = len(targets)
 
         experiment_dir = f"{tmp_dir}/{experiment_name}"
         preprocessing_dir = f"{experiment_dir}/preprocessing"
@@ -50,23 +51,16 @@ def create_experiment(
         os.makedirs(preprocessing_dir, exist_ok=True)
         os.makedirs(data_dir, exist_ok=True)
 
+        # Create or update experiment (without targets relation)
         experiment = Experiment.upsert(
-            match_fields=["name"],
             db=db,
             name=experiment_name,
             path=Path(experiment_dir).resolve(),
-            type="training",
             size=data.shape[0],
-            corr_threshold=corr_threshold,
-            percentile=percentile,
-            max_features=max_features,
+            number_of_targets=number_of_targets,
             **groups,
             **dates,
-            targets=targets,
             context={
-                "corr_threshold": corr_threshold,
-                "percentile": percentile,
-                "max_features": max_features,
                 "date_column": date_column,
                 "group_column": group_column,
                 "experiment_name": experiment_name,
@@ -74,4 +68,8 @@ def create_experiment(
             },
         )
 
+        # Set targets relationship after creation/update
+        experiment.targets = targets
+        experiment.save(db=db)
+
         return experiment