lecrapaud 0.4.0__py3-none-any.whl

lecrapaud/db/models/dataset.py

@@ -0,0 +1,124 @@
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    DateTime,
+    Date,
+    Float,
+    JSON,
+    Table,
+    ForeignKey,
+    BigInteger,
+    Index,
+    TIMESTAMP,
+    UniqueConstraint,
+)
+from sqlalchemy import desc, asc, cast, text, func
+from sqlalchemy.orm import relationship, Mapped, mapped_column, DeclarativeBase
+from itertools import chain
+
+from src.db.setup import get_db
+from src.db.models.base import Base
+from src.db.crud import CRUDMixin, with_db
+
+# jointures
+dataset_target_association = Table(
+    "dataset_target_association",
+    Base.metadata,
+    Column(
+        "dataset_id",
+        BigInteger,
+        ForeignKey("datasets.id", ondelete="CASCADE"),
+        primary_key=True,
+    ),
+    Column(
+        "target_id",
+        BigInteger,
+        ForeignKey("targets.id", ondelete="CASCADE"),
+        primary_key=True,
+    ),
+)
+
+
+class Dataset(Base, CRUDMixin):
+    __tablename__ = "datasets"
+
+    id = Column(BigInteger, primary_key=True, index=True, autoincrement=True)
+    created_at = Column(
+        TIMESTAMP(timezone=True), server_default=func.now(), nullable=False
+    )
+    updated_at = Column(
+        TIMESTAMP(timezone=True),
+        server_default=func.now(),
+        onupdate=func.now(),
+        nullable=False,
+    )
+    name = Column(String(50), nullable=False)
+    path = Column(String(255))  # we do not have this at creation time
+    type = Column(String(50), nullable=False)
+    size = Column(Integer, nullable=False)
+    train_size = Column(Integer, nullable=False)
+    val_size = Column(Integer)
+    test_size = Column(Integer, nullable=False)
+    number_of_groups = Column(Integer, nullable=False)
+    list_of_groups = Column(JSON, nullable=False)
+    corr_threshold = Column(Float, nullable=False)
+    max_features = Column(Integer, nullable=False)
+    percentile = Column(Float, nullable=False)
+    start_date = Column(DateTime, nullable=False)
+    end_date = Column(DateTime, nullable=False)
+    train_start_date = Column(DateTime, nullable=False)
+    train_end_date = Column(DateTime, nullable=False)
+    val_start_date = Column(DateTime)
+    val_end_date = Column(DateTime)
+    test_start_date = Column(DateTime, nullable=False)
+    test_end_date = Column(DateTime, nullable=False)
+
+    feature_selections = relationship(
+        "FeatureSelection",
+        back_populates="dataset",
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+    model_selections = relationship(
+        "ModelSelection",
+        back_populates="dataset",
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+    targets = relationship(
+        "Target",
+        secondary=dataset_target_association,
+        back_populates="datasets",
+        lazy="selectin",
+    )
+
+    __table_args__ = (
+        UniqueConstraint(
+            "name",
+            name="uq_datasets_composite",
+        ),
+    )
+
+    def get_features(self, target_number: int):
+        feature_selections = self.feature_selections
+        target_id = [t for t in self.targets if t.name == f"TARGET_{target_number}"][
+            0
+        ].id
+        feature_selection = [
+            fs for fs in feature_selections if fs.target_id == target_id
+        ][0]
+        feature = [f.name for f in feature_selection.features]
+        return feature
+
+    def get_all_features(self):
+        target_idx = [target.id for target in self.targets]
+        all_features = chain.from_iterable(
+            [f.name for f in fs.features]
+            for fs in self.feature_selections
+            if fs.target_id in target_idx
+        )
+        all_features = ["DATE", "STOCK"] + list(all_features)
+        all_features = list(dict.fromkeys(all_features))
+
+        return all_features

lecrapaud/db/models/feature.py

@@ -0,0 +1,46 @@
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    DateTime,
+    Date,
+    Float,
+    JSON,
+    Table,
+    ForeignKey,
+    BigInteger,
+    Index,
+    TIMESTAMP,
+)
+from sqlalchemy import desc, asc, cast, text, func
+
+from sqlalchemy.orm import relationship, Mapped, mapped_column, DeclarativeBase
+
+from src.db.setup import get_db
+from src.db.models.base import Base
+from src.db.models.feature_selection import feature_selection_association
+from src.db.crud import CRUDMixin
+
+
+class Feature(Base, CRUDMixin):
+    __tablename__ = "features"
+
+    id = Column(BigInteger, primary_key=True, index=True, autoincrement=True)
+    created_at = Column(
+        TIMESTAMP(timezone=True), server_default=func.now(), nullable=False
+    )
+    updated_at = Column(
+        TIMESTAMP(timezone=True),
+        server_default=func.now(),
+        onupdate=func.now(),
+        nullable=False,
+    )
+    name = Column(String(50), nullable=False, unique=True)
+    type = Column(String(50))
+
+    feature_selections = relationship(
+        "FeatureSelection",
+        secondary=feature_selection_association,
+        back_populates="features",
+        lazy="selectin"
+    )

lecrapaud/db/models/feature_selection.py

@@ -0,0 +1,126 @@
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    DateTime,
+    Date,
+    Float,
+    JSON,
+    Table,
+    ForeignKey,
+    BigInteger,
+    Index,
+    TIMESTAMP,
+    UniqueConstraint,
+)
+from sqlalchemy import desc, asc, cast, text, func
+
+from sqlalchemy.orm import (
+    relationship,
+    Mapped,
+    mapped_column,
+    DeclarativeBase,
+    object_session,
+)
+from collections.abc import Iterable
+
+from src.db.setup import get_db
+from src.db.models.base import Base
+from src.db.crud import CRUDMixin, with_db
+
+# jointures
+feature_selection_association = Table(
+    "feature_selection_association",
+    Base.metadata,
+    Column(
+        "feature_selection_id",
+        BigInteger,
+        ForeignKey("feature_selections.id", ondelete="CASCADE"),
+        primary_key=True,
+    ),
+    Column(
+        "feature_id",
+        BigInteger,
+        ForeignKey("features.id", ondelete="CASCADE"),
+        primary_key=True,
+    ),
+)
+
+
+class FeatureSelection(Base, CRUDMixin):
+    __tablename__ = "feature_selections"
+
+    id = Column(BigInteger, primary_key=True, index=True, autoincrement=True)
+    created_at = Column(
+        TIMESTAMP(timezone=True), server_default=func.now(), nullable=False
+    )
+    updated_at = Column(
+        TIMESTAMP(timezone=True),
+        server_default=func.now(),
+        onupdate=func.now(),
+        nullable=False,
+    )
+    training_time = Column(Integer)
+    best_features_path = Column(String(255))
+    dataset_id = Column(
+        BigInteger, ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
+    )
+    target_id = Column(
+        BigInteger, ForeignKey("targets.id", ondelete="CASCADE"), nullable=False
+    )
+
+    dataset = relationship(
+        "Dataset", back_populates="feature_selections", lazy="selectin"
+    )
+    target = relationship(
+        "Target", back_populates="feature_selections", lazy="selectin"
+    )
+    feature_selection_ranks = relationship(
+        "FeatureSelectionRank",
+        back_populates="feature_selection",
+        cascade="all, delete-orphan",
+        lazy="selectin",
+    )
+    features = relationship(
+        "Feature",
+        secondary=feature_selection_association,
+        back_populates="feature_selections",
+        lazy="selectin",
+    )
+
+    __table_args__ = (
+        UniqueConstraint(
+            "dataset_id", "target_id", name="uq_feature_selection_composite"
+        ),
+    )
+
+    @with_db
+    def add_features(self, features: list, db=None):
+        self = db.merge(self)
+        for feature in features:
+            feature = db.merge(feature)
+            if feature not in self.features:
+                self.features.append(feature)
+        # db.flush()
+        # db.refresh(self)
+        return self
+
+    @with_db
+    def __lshift__(self, feature_or_list, db=None):
+        items = (
+            feature_or_list
+            if isinstance(feature_or_list, Iterable)
+            and not isinstance(feature_or_list, (str, bytes))
+            else [feature_or_list]
+        )
+
+        self = db.merge(self)
+        for feature in items:
+            feature = db.merge(feature)
+            if feature not in self.features:
+                self.features.append(feature)
+
+        db.flush()
+        db.refresh(self)
+        print(self.features)
+        return self

lecrapaud/db/models/feature_selection_rank.py

@@ -0,0 +1,80 @@
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    DateTime,
+    Date,
+    Float,
+    JSON,
+    Table,
+    ForeignKey,
+    BigInteger,
+    Index,
+    TIMESTAMP,
+    UniqueConstraint,
+)
+from sqlalchemy import desc, asc, cast, text, func
+
+from sqlalchemy.orm import relationship, Mapped, mapped_column, DeclarativeBase
+from sqlalchemy.dialects.mysql import insert
+
+from src.db.setup import get_db
+from src.db.models.base import Base
+from src.db.crud import CRUDMixin, with_db
+
+
+class FeatureSelectionRank(Base, CRUDMixin):
+    __tablename__ = "feature_selection_ranks"
+
+    id = Column(BigInteger, primary_key=True, index=True, autoincrement=True)
+    created_at = Column(
+        TIMESTAMP(timezone=True), server_default=func.now(), nullable=False
+    )
+    updated_at = Column(
+        TIMESTAMP(timezone=True),
+        server_default=func.now(),
+        onupdate=func.now(),
+        nullable=False,
+    )
+    score = Column(Float)
+    pvalue = Column(Float)
+    support = Column(Integer)
+    rank = Column(Integer)
+    method = Column(String(50))
+    training_time = Column(Integer)
+    feature_id = Column(BigInteger, ForeignKey("features.id", ondelete="CASCADE"))
+    feature_selection_id = Column(
+        BigInteger, ForeignKey("feature_selections.id", ondelete="CASCADE")
+    )
+
+    feature = relationship("Feature", lazy="selectin")
+    feature_selection = relationship(
+        "FeatureSelection", back_populates="feature_selection_ranks", lazy="selectin"
+    )
+
+    __table_args__ = (
+        UniqueConstraint(
+            "feature_id",
+            "feature_selection_id",
+            "method",
+            name="uq_feature_selection_rank_composite",
+        ),
+    )
+
+    @classmethod
+    @with_db
+    def bulk_upsert(cls, rows, db=None):
+        stmt = insert(cls).values(rows)
+
+        update_fields = {
+            key: stmt.inserted[key]
+            for key in rows[0]
+            if key not in ("feature_selection_id", "feature_id", "method")
+        }
+
+        stmt = stmt.on_duplicate_key_update(**update_fields)
+
+        db.execute(stmt)
+        db.commit()
+
+        return len(rows)

lecrapaud/db/models/model.py

@@ -0,0 +1,41 @@
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    DateTime,
+    Date,
+    Float,
+    JSON,
+    Table,
+    ForeignKey,
+    BigInteger,
+    Index,
+    TIMESTAMP,
+    UniqueConstraint,
+)
+from sqlalchemy import desc, asc, cast, text, func
+
+from sqlalchemy.orm import relationship, Mapped, mapped_column, DeclarativeBase
+
+from src.db.setup import get_db
+from src.db.models.base import Base
+from src.db.crud import CRUDMixin
+
+
+class Model(Base, CRUDMixin):
+    __tablename__ = "models"
+
+    id = Column(BigInteger, primary_key=True, index=True, autoincrement=True)
+    created_at = Column(
+        TIMESTAMP(timezone=True), server_default=func.now(), nullable=False
+    )
+    updated_at = Column(
+        TIMESTAMP(timezone=True),
+        server_default=func.now(),
+        onupdate=func.now(),
+        nullable=False,
+    )
+    name = Column(String(50), nullable=False)
+    type = Column(String(50), nullable=False)
+
+    __table_args__ = (UniqueConstraint("name", "type", name="uq_model_composite"),)

lecrapaud/db/models/model_selection.py

@@ -0,0 +1,56 @@
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    DateTime,
+    Date,
+    Float,
+    JSON,
+    Table,
+    ForeignKey,
+    BigInteger,
+    Index,
+    TIMESTAMP,
+    UniqueConstraint,
+)
+from sqlalchemy import desc, asc, cast, text, func
+
+from sqlalchemy.orm import relationship, Mapped, mapped_column, DeclarativeBase
+
+from src.db.setup import get_db
+from src.db.models.base import Base
+from src.db.crud import CRUDMixin
+
+
+class ModelSelection(Base, CRUDMixin):
+    __tablename__ = "model_selections"
+
+    id = Column(BigInteger, primary_key=True, index=True, autoincrement=True)
+    created_at = Column(
+        TIMESTAMP(timezone=True), server_default=func.now(), nullable=False
+    )
+    updated_at = Column(
+        TIMESTAMP(timezone=True),
+        server_default=func.now(),
+        onupdate=func.now(),
+        nullable=False,
+    )
+    best_model_params = Column(JSON)
+    best_model_path = Column(String(255))
+    best_model_id = Column(BigInteger, ForeignKey("models.id", ondelete="CASCADE"))
+    target_id = Column(BigInteger, ForeignKey("targets.id", ondelete="CASCADE"), nullable=False)
+    dataset_id = Column(BigInteger, ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False)
+
+    best_model = relationship("Model", lazy="selectin")
+    model_trainings = relationship("ModelTraining", back_populates="model_selection", cascade="all, delete-orphan", lazy="selectin")
+    dataset = relationship(
+        "Dataset", back_populates="model_selections", lazy="selectin"
+    )
+    target = relationship("Target", back_populates="model_selections", lazy="selectin")
+
+    __table_args__ = (
+        UniqueConstraint(
+            "target_id", "dataset_id", name="uq_model_selection_composite"
+        ),
+    )
+

lecrapaud/db/models/model_training.py

@@ -0,0 +1,54 @@
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    DateTime,
+    Date,
+    Float,
+    JSON,
+    Table,
+    ForeignKey,
+    BigInteger,
+    Index,
+    TIMESTAMP,
+    UniqueConstraint,
+)
+from sqlalchemy import desc, asc, cast, text, func
+
+from sqlalchemy.orm import relationship, Mapped, mapped_column, DeclarativeBase
+
+from src.db.setup import get_db
+from src.db.models.base import Base
+from src.db.crud import CRUDMixin
+
+
+class ModelTraining(Base, CRUDMixin):
+    __tablename__ = "model_trainings"
+
+    id = Column(BigInteger, primary_key=True, index=True, autoincrement=True)
+    created_at = Column(
+        TIMESTAMP(timezone=True), server_default=func.now(), nullable=False
+    )
+    updated_at = Column(
+        TIMESTAMP(timezone=True),
+        server_default=func.now(),
+        onupdate=func.now(),
+        nullable=False,
+    )
+    best_params = Column(JSON)
+    model_path = Column(String(255))
+    training_time = Column(Integer)
+    model_id = Column(BigInteger, ForeignKey("models.id"), nullable=False)
+    model_selection_id = Column(
+        BigInteger, ForeignKey("model_selections.id", ondelete="CASCADE"), nullable=False
+    )
+
+    model = relationship("Model", lazy="selectin")
+    model_selection = relationship("ModelSelection", back_populates="model_trainings", lazy="selectin")
+    score = relationship("Score", back_populates="model_trainings", cascade="all, delete-orphan", lazy="selectin")
+
+    __table_args__ = (
+        UniqueConstraint(
+            "model_id", "model_selection_id", name="uq_model_training_composite"
+        ),
+    )

lecrapaud/db/models/score.py

@@ -0,0 +1,62 @@
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    DateTime,
+    Date,
+    Float,
+    JSON,
+    Table,
+    ForeignKey,
+    BigInteger,
+    Index,
+    TIMESTAMP,
+)
+from sqlalchemy import desc, asc, cast, text, func
+
+from sqlalchemy.orm import relationship, Mapped, mapped_column, DeclarativeBase
+
+from src.db.setup import get_db
+from src.db.models.base import Base
+from src.db.crud import CRUDMixin
+
+
+class Score(Base, CRUDMixin):
+    __tablename__ = "scores"
+
+    id = Column(BigInteger, primary_key=True, index=True, autoincrement=True)
+    created_at = Column(
+        TIMESTAMP(timezone=True), server_default=func.now(), nullable=False
+    )
+    updated_at = Column(
+        TIMESTAMP(timezone=True),
+        server_default=func.now(),
+        onupdate=func.now(),
+        nullable=False,
+    )
+    type = Column(
+        String(50), nullable=False
+    )  # either hyperopts or validation or crossval
+    training_time = Column(Integer)
+    eval_data_std = Column(Float)
+    rmse = Column(Float)
+    rmse_std_ratio = Column(Float)
+    mae = Column(Float)
+    mape = Column(Float)
+    mam = Column(Float)
+    mad = Column(Float)
+    mae_mam_ratio = Column(Float)
+    mae_mad_ratio = Column(Float)
+    r2 = Column(Float)
+    logloss = Column(Float)
+    accuracy = Column(Float)
+    precision = Column(Float)
+    recall = Column(Float)
+    f1 = Column(Float)
+    roc_auc = Column(Float)
+    threshold = Column(Float)
+    precision_at_threshold = Column(Float)
+    recall_at_threshold = Column(Float)
+    model_training_id = Column(BigInteger, ForeignKey("model_trainings.id", ondelete="CASCADE"), nullable=False)
+
+    model_trainings = relationship("ModelTraining", back_populates="score", lazy="selectin")

lecrapaud/db/models/target.py

@@ -0,0 +1,59 @@
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    DateTime,
+    Date,
+    Float,
+    JSON,
+    Table,
+    ForeignKey,
+    BigInteger,
+    Index,
+    TIMESTAMP,
+    UniqueConstraint,
+)
+from sqlalchemy import desc, asc, cast, text, func
+
+from sqlalchemy.orm import relationship, Mapped, mapped_column, DeclarativeBase
+
+from src.db.setup import get_db
+from src.db.models.base import Base
+from src.db.crud import CRUDMixin
+from src.db.models.dataset import dataset_target_association
+
+
+class Target(Base, CRUDMixin):
+    __tablename__ = "targets"
+
+    id = Column(BigInteger, primary_key=True, index=True, autoincrement=True)
+    created_at = Column(
+        TIMESTAMP(timezone=True), server_default=func.now(), nullable=False
+    )
+    updated_at = Column(
+        TIMESTAMP(timezone=True),
+        server_default=func.now(),
+        onupdate=func.now(),
+        nullable=False,
+    )
+    name = Column(String(50), nullable=False)
+    type = Column(String(50), nullable=False)
+    description = Column(String(255))
+
+    datasets = relationship(
+        "Dataset", secondary=dataset_target_association, back_populates="targets", lazy="selectin"
+    )
+    feature_selections = relationship(
+        "FeatureSelection", back_populates="target", cascade="all, delete-orphan", lazy="selectin"
+    )
+    model_selections = relationship(
+        "ModelSelection", back_populates="target", cascade="all, delete-orphan", lazy="selectin"
+    )
+
+    __table_args__ = (
+        UniqueConstraint(
+            "name",
+            "type",
+            name="uq_target_composite",
+        ),
+    )