lecrapaud 0.4.0__py3-none-any.whl → 0.4.2__py3-none-any.whl

lecrapaud/__init__.py

@@ -0,0 +1 @@
+from lecrapaud.api import *

lecrapaud/api.py

@@ -0,0 +1,277 @@
+"""
+Main API class
+
+the way I want it to work :
+
+app = LeCrapaud()
+
+kwargs = {
+
+}
+
+experiment = app.create_experiment(**kwargs) # return a class Experiment()
+ou
+experiment = app.get_experiment(exp_id)
+
+best_features, artifacts, best_model = experiment.train(get_data, get_data_params)
+
+new_data + target_pred + target_proba (if classif) = experiment.predict(**new_data)
+
+On veut aussi pouvoir juste faire :
+
+experiment.feature_engineering(data) : feat eng, return data
+
+experiment.preprocess_feature(data) : split, encoding, pcas, return train, val, test df
+
+experiment.feature_selection(train) : return features
+
+experiment.preprocess_model(train, val, test) : return data = dict of df
+
+experiment.model_selection(data) : return best_model
+"""
+
+import joblib
+import pandas as pd
+import logging
+from lecrapaud.utils import logger
+from lecrapaud.db.session import init_db
+from lecrapaud.feature_selection import FeatureSelectionEngine, PreprocessModel
+from lecrapaud.model_selection import ModelSelectionEngine, ModelEngine
+from lecrapaud.feature_engineering import FeatureEngineeringEngine, PreprocessFeature
+from lecrapaud.experiment import create_dataset
+from lecrapaud.db import Dataset
+
+
+class LeCrapaud:
+    def __init__(self, uri: str = None):
+        init_db(uri=uri)
+
+    def create_experiment(self, **kwargs):
+        return Experiment(**kwargs)
+
+    def get_experiment(self, id: int):
+        return Experiment(id)
+
+
+class Experiment:
+    def __init__(self, id=None, **kwargs):
+        if id:
+            self.dataset = Dataset.get(id)
+        else:
+            self.dataset = create_dataset(**kwargs)
+
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
+        self.context = {
+            # generic
+            "dataset": self.dataset,
+            # for FeatureEngineering
+            "columns_drop": self.columns_drop,
+            "columns_boolean": self.columns_boolean,
+            "columns_date": self.columns_date,
+            "columns_te_groupby": self.columns_te_groupby,
+            "columns_te_target": self.columns_te_target,
+            # for PreprocessFeature
+            "time_series": self.time_series,
+            "date_column": self.date_column,
+            "group_column": self.group_column,
+            "val_size": self.val_size,
+            "test_size": self.test_size,
+            "columns_pca": self.columns_pca,
+            "columns_onehot": self.columns_onehot,
+            "columns_binary": self.columns_binary,
+            "columns_frequency": self.columns_frequency,
+            "columns_ordinal": self.columns_ordinal,
+            "target_numbers": self.target_numbers,
+            "target_clf": self.target_clf,
+            # for PreprocessModel
+            "models_idx": self.models_idx,
+            "max_timesteps": self.max_timesteps,
+            # for ModelSelection
+            "perform_hyperopt": self.perform_hyperopt,
+            "number_of_trials": self.number_of_trials,
+            "perform_crossval": self.perform_crossval,
+            "plot": self.plot,
+            "preserve_model": self.preserve_model,
+            # not yet
+            "target_mclf": self.target_mclf,
+        }
+
+    def train(self, data):
+        data_eng = self.feature_engineering(data)
+        train, val, test = self.preprocess_feature(data_eng)
+        all_features = self.feature_selection(train)
+        std_data, reshaped_data = self.preprocess_model(train, val, test)
+        self.model_selection(std_data, reshaped_data)
+
+    def predict(self, new_data, verbose: int = 0):
+        if verbose == 0:
+            logger.setLevel(logging.WARNING)
+
+        logger.warning("Running prediction...")
+
+        data = self.feature_engineering(
+            data=new_data,
+            for_training=False,
+        )
+        data = self.preprocess_feature(data, for_training=False)
+        data, scaled_data, reshaped_data = self.preprocess_model(
+            data, for_training=False
+        )
+
+        for target_number in self.target_numbers:
+
+            # loading model
+            training_target_dir = f"{self.dataset.path}/TARGET_{target_number}"
+            all_features = self.dataset.get_all_features(
+                date_column=self.date_column, group_column=self.group_column
+            )
+            if self.dataset.name == "data_28_X_X":
+                features = joblib.load(
+                    f"{self.dataset.path}/preprocessing/features_{target_number}.pkl"
+                )  # we keep this for backward compatibility
+            else:
+                features = self.dataset.get_features(target_number)
+            model = ModelEngine(path=training_target_dir)
+
+            # getting data
+            if model.recurrent:
+                features_idx = [
+                    i for i, e in enumerate(all_features) if e in set(features)
+                ]
+                x_pred = reshaped_data[:, :, features_idx]
+            else:
+                x_pred = scaled_data[features] if model.need_scaling else data[features]
+
+            # predicting
+            y_pred = model.predict(x_pred)
+
+            # fix for recurrent model because x_val has no index as it is a 3D np array
+            if model.recurrent:
+                y_pred.index = (
+                    new_data.index
+                )  # TODO: not sure this will work for old dataset not aligned with data_for_training for test use case (done, this is why we decode the test set)
+
+            # unscaling prediction
+            if (
+                model.need_scaling
+                and model.target_type == "regression"
+                and model.scaler_y is not None
+            ):
+                y_pred = pd.Series(
+                    model.scaler_y.inverse_transform(
+                        y_pred.values.reshape(-1, 1)
+                    ).flatten(),
+                    index=new_data.index,
+                )
+
+            # renaming pred column and concatenating with initial data
+            if isinstance(y_pred, pd.DataFrame):
+                y_pred.rename(
+                    columns={"PRED": f"TARGET_{target_number}_PRED"}, inplace=True
+                )
+                new_data = pd.concat(
+                    [new_data, y_pred[f"TARGET_{target_number}_PRED"]], axis=1
+                )
+
+            else:
+                y_pred.name = f"TARGET_{target_number}_PRED"
+                new_data = pd.concat([new_data, y_pred], axis=1)
+
+        return new_data
+
+    def feature_engineering(self, data, for_training=True):
+        app = FeatureEngineeringEngine(
+            data=data,
+            columns_drop=self.columns_drop,
+            columns_boolean=self.columns_boolean,
+            columns_date=self.columns_date,
+            columns_te_groupby=self.columns_te_groupby,
+            columns_te_target=self.columns_te_target,
+            for_training=for_training,
+        )
+        data = app.run()
+        return data
+
+    def preprocess_feature(self, data, for_training=True):
+        app = PreprocessFeature(
+            data=data,
+            dataset=self.dataset,
+            time_series=self.time_series,
+            date_column=self.date_column,
+            group_column=self.group_column,
+            val_size=self.val_size,
+            test_size=self.test_size,
+            columns_pca=self.columns_pca,
+            columns_onehot=self.columns_onehot,
+            columns_binary=self.columns_binary,
+            columns_frequency=self.columns_frequency,
+            columns_ordinal=self.columns_ordinal,
+            target_numbers=self.target_numbers,
+            target_clf=self.target_clf,
+        )
+        if for_training:
+            train, val, test = app.run()
+            return train, val, test
+        else:
+            data = app.inference()
+            return data
+
+    def feature_selection(self, train):
+        for target_number in self.target_numbers:
+            app = FeatureSelectionEngine(
+                train=train,
+                target_number=target_number,
+                dataset=self.dataset,
+                target_clf=self.target_clf,
+            )
+            app.run()
+        self.dataset = Dataset.get(self.dataset.id)
+        all_features = self.dataset.get_all_features(
+            date_column=self.date_column, group_column=self.group_column
+        )
+        return all_features
+
+    def preprocess_model(self, train, val=None, test=None, for_training=True):
+        app = PreprocessModel(
+            train=train,
+            val=val,
+            test=test,
+            dataset=self.dataset,
+            target_numbers=self.target_numbers,
+            target_clf=self.target_clf,
+            models_idx=self.models_idx,
+            time_series=self.time_series,
+            max_timesteps=self.max_timesteps,
+            date_column=self.date_column,
+            group_column=self.group_column,
+        )
+        if for_training:
+            data, reshaped_data = app.run()
+            return data, reshaped_data
+        else:
+            data, scaled_data, reshaped_data = app.inference()
+            return data, scaled_data, reshaped_data
+
+    def model_selection(self, data, reshaped_data):
+        for target_number in self.target_numbers:
+            app = ModelSelectionEngine(
+                data=data,
+                reshaped_data=reshaped_data,
+                target_number=target_number,
+                dataset=self.dataset,
+                target_clf=self.target_clf,
+                models_idx=self.models_idx,
+                time_series=self.time_series,
+                date_column=self.date_column,
+                group_column=self.group_column,
+            )
+            app.run(
+                self.session_name,
+                perform_hyperopt=self.perform_hyperopt,
+                number_of_trials=self.number_of_trials,
+                perform_crossval=self.perform_crossval,
+                plot=self.plot,
+                preserve_model=self.preserve_model,
+            )

lecrapaud/config.py

@@ -14,3 +14,13 @@ FA2 = os.getenv("2FA")
 INT = os.getenv("INT")
 LOGGING_LEVEL = os.getenv("LOGGING_LEVEL", "INFO")
 ALPHA_VENTAGE_API_KEY = os.getenv("ALPHA_VENTAGE_API_KEY")
+
+DB_USER = os.getenv("TEST_DB_USER") if PYTHON_ENV == "Test" else os.getenv("DB_USER")
+DB_PASSWORD = (
+    os.getenv("TEST_DB_PASSWORD") if PYTHON_ENV == "Test" else os.getenv("DB_PASSWORD")
+)
+DB_HOST = os.getenv("TEST_DB_HOST") if PYTHON_ENV == "Test" else os.getenv("DB_HOST")
+DB_PORT = os.getenv("TEST_DB_PORT") if PYTHON_ENV == "Test" else os.getenv("DB_PORT")
+DB_NAME = os.getenv("TEST_DB_NAME") if PYTHON_ENV == "Test" else os.getenv("DB_NAME")
+DB_URI = os.getenv("DB_URI", None)
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")

lecrapaud/db/__init__.py

@@ -0,0 +1 @@
+from lecrapaud.db.models import *

lecrapaud/db/alembic/env.py

@@ -4,7 +4,7 @@ from sqlalchemy import engine_from_config
 from sqlalchemy import pool
 
 from alembic import context
-from src.db.setup import DATABASE_URL
+from lecrapaud.db.session import DATABASE_URL
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -18,7 +18,7 @@ if config.config_file_name is not None:
 
 # add your model's MetaData object here
 # for 'autogenerate' support
-from src.db.models.base import Base
+from lecrapaud.db.models.base import Base
 
 target_metadata = Base.metadata
 

lecrapaud/db/alembic/versions/2025_05_31_1834-52b809a34371_make_nullablee.py

@@ -5,6 +5,7 @@ Revises: 339927587383
 Create Date: 2025-05-31 18:34:58.962966
 
 """
+
 from typing import Sequence, Union
 
 from alembic import op
@@ -12,27 +13,38 @@ import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
 # revision identifiers, used by Alembic.
-revision: str = '52b809a34371'
-down_revision: Union[str, None] = '339927587383'
+revision: str = "52b809a34371"
+down_revision: Union[str, None] = "339927587383"
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
-    op.alter_column('investment_runs', 'initial_portfolio',
-               existing_type=mysql.JSON(),
-               nullable=True)
-    op.create_index(op.f('ix_investment_runs_id'), 'investment_runs', ['id'], unique=False)
-    op.create_foreign_key(None, 'portfolios', 'investment_runs', ['investment_run_id'], ['id'], ondelete='CASCADE')
+    op.alter_column(
+        "investment_runs",
+        "initial_portfolio",
+        existing_type=mysql.JSON(),
+        nullable=True,
+    )
+    op.create_foreign_key(
+        None,
+        "portfolios",
+        "investment_runs",
+        ["investment_run_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )
     # ### end Alembic commands ###
 
 
 def downgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_constraint(None, 'portfolios', type_='foreignkey')
-    op.drop_index(op.f('ix_investment_runs_id'), table_name='investment_runs')
-    op.alter_column('investment_runs', 'initial_portfolio',
-               existing_type=mysql.JSON(),
-               nullable=False)
+    op.drop_constraint(None, "portfolios", type_="foreignkey")
+    op.alter_column(
+        "investment_runs",
+        "initial_portfolio",
+        existing_type=mysql.JSON(),
+        nullable=False,
+    )
     # ### end Alembic commands ###

lecrapaud/db/alembic/versions/2025_06_17_1652-c45f5e49fa2c_make_fields_nullable.py

@@ -0,0 +1,89 @@
+"""
+
+Revision ID: c45f5e49fa2c
+Revises: db8cdd83563a
+Create Date: 2025-06-17 16:52:45.042045
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision: str = "c45f5e49fa2c"
+down_revision: Union[str, None] = "db8cdd83563a"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "datasets", "train_size", existing_type=mysql.INTEGER(), nullable=True
+    )
+    op.alter_column(
+        "datasets", "test_size", existing_type=mysql.INTEGER(), nullable=True
+    )
+    op.alter_column(
+        "datasets", "number_of_groups", existing_type=mysql.INTEGER(), nullable=True
+    )
+    op.alter_column(
+        "datasets", "list_of_groups", existing_type=mysql.JSON(), nullable=True
+    )
+    op.alter_column(
+        "datasets", "start_date", existing_type=mysql.DATETIME(), nullable=True
+    )
+    op.alter_column(
+        "datasets", "end_date", existing_type=mysql.DATETIME(), nullable=True
+    )
+    op.alter_column(
+        "datasets", "train_start_date", existing_type=mysql.DATETIME(), nullable=True
+    )
+    op.alter_column(
+        "datasets", "train_end_date", existing_type=mysql.DATETIME(), nullable=True
+    )
+    op.alter_column(
+        "datasets", "test_start_date", existing_type=mysql.DATETIME(), nullable=True
+    )
+    op.alter_column(
+        "datasets", "test_end_date", existing_type=mysql.DATETIME(), nullable=True
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "datasets", "test_end_date", existing_type=mysql.DATETIME(), nullable=False
+    )
+    op.alter_column(
+        "datasets", "test_start_date", existing_type=mysql.DATETIME(), nullable=False
+    )
+    op.alter_column(
+        "datasets", "train_end_date", existing_type=mysql.DATETIME(), nullable=False
+    )
+    op.alter_column(
+        "datasets", "train_start_date", existing_type=mysql.DATETIME(), nullable=False
+    )
+    op.alter_column(
+        "datasets", "end_date", existing_type=mysql.DATETIME(), nullable=False
+    )
+    op.alter_column(
+        "datasets", "start_date", existing_type=mysql.DATETIME(), nullable=False
+    )
+    op.alter_column(
+        "datasets", "list_of_groups", existing_type=mysql.JSON(), nullable=False
+    )
+    op.alter_column(
+        "datasets", "number_of_groups", existing_type=mysql.INTEGER(), nullable=False
+    )
+    op.alter_column(
+        "datasets", "test_size", existing_type=mysql.INTEGER(), nullable=False
+    )
+    op.alter_column(
+        "datasets", "train_size", existing_type=mysql.INTEGER(), nullable=False
+    )
+    # ### end Alembic commands ###

lecrapaud/db/alembic.ini

@@ -0,0 +1,116 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = lecrapaud.db.alembic
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to src/db/alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:src/db/alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = %(DATABASE_URL)s
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
+# hooks = ruff
+# ruff.type = exec
+# ruff.executable = %(here)s/.venv/bin/ruff
+# ruff.options = --fix REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

lecrapaud/db/models/__init__.py

@@ -1,11 +1,11 @@
-from src.db.models.base import Base
+from lecrapaud.db.models.base import Base
 
-from src.db.models.dataset import Dataset
-from src.db.models.feature_selection_rank import FeatureSelectionRank
-from src.db.models.feature_selection import FeatureSelection
-from src.db.models.feature import Feature
-from src.db.models.model_selection import ModelSelection
-from src.db.models.model_training import ModelTraining
-from src.db.models.model import Model
-from src.db.models.score import Score
-from src.db.models.target import Target
+from lecrapaud.db.models.dataset import Dataset
+from lecrapaud.db.models.feature_selection_rank import FeatureSelectionRank
+from lecrapaud.db.models.feature_selection import FeatureSelection
+from lecrapaud.db.models.feature import Feature
+from lecrapaud.db.models.model_selection import ModelSelection
+from lecrapaud.db.models.model_training import ModelTraining
+from lecrapaud.db.models.model import Model
+from lecrapaud.db.models.score import Score
+from lecrapaud.db.models.target import Target