lecrapaud 0.1.0__py3-none-any.whl

lecrapaud/__init__.py

@@ -0,0 +1 @@
+from lecrapaud.api import *

lecrapaud/api.py

@@ -0,0 +1,271 @@
+"""
+Main API class
+
+the way I want it to work :
+
+app = LeCrapaud()
+
+kwargs = {
+
+}
+
+experiment = app.create_experiment(**kwargs) # return a class Experiment()
+ou
+experiment = app.get_experiment(exp_id)
+
+best_features, artifacts, best_model = experiment.train(get_data, get_data_params)
+
+new_data + target_pred + target_proba (if classif) = experiment.predict(**new_data)
+
+On veut aussi pouvoir juste faire :
+
+experiment.feature_engineering(data) : feat eng, return data
+
+experiment.preprocess_feature(data) : split, encoding, pcas, return train, val, test df
+
+experiment.feature_selection(train) : return features
+
+experiment.preprocess_model(train, val, test) : return data = dict of df
+
+experiment.model_selection(data) : return best_model
+"""
+
+import joblib
+import pandas as pd
+from lecrapaud.db.session import init_db
+from lecrapaud.feature_selection import FeatureSelectionEngine, PreprocessModel
+from lecrapaud.model_selection import ModelSelectionEngine, ModelEngine
+from lecrapaud.feature_engineering import FeatureEngineeringEngine, PreprocessFeature
+from lecrapaud.experiment import create_dataset
+from lecrapaud.db import Dataset
+
+
+class LeCrapaud:
+    def __init__(self, uri: str = None):
+        init_db(uri=uri)
+
+    def create_experiment(self, **kwargs):
+        return Experiment(**kwargs)
+
+    def get_experiment(self, id: int):
+        return Experiment(id)
+
+
+class Experiment:
+    def __init__(self, id=None, **kwargs):
+        if id:
+            self.dataset = Dataset.get(id)
+        else:
+            self.dataset = create_dataset(**kwargs)
+
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
+        self.context = {
+            # generic
+            "dataset": self.dataset,
+            # for FeatureEngineering
+            "columns_drop": self.columns_drop,
+            "columns_boolean": self.columns_boolean,
+            "columns_date": self.columns_date,
+            "columns_te_groupby": self.columns_te_groupby,
+            "columns_te_target": self.columns_te_target,
+            # for PreprocessFeature
+            "time_series": self.time_series,
+            "date_column": self.date_column,
+            "group_column": self.group_column,
+            "val_size": self.val_size,
+            "test_size": self.test_size,
+            "columns_pca": self.columns_pca,
+            "columns_onehot": self.columns_onehot,
+            "columns_binary": self.columns_binary,
+            "columns_frequency": self.columns_frequency,
+            "columns_ordinal": self.columns_ordinal,
+            "target_numbers": self.target_numbers,
+            "target_clf": self.target_clf,
+            # for PreprocessModel
+            "models_idx": self.models_idx,
+            "max_timesteps": self.max_timesteps,
+            # for ModelSelection
+            "perform_hyperopt": self.perform_hyperopt,
+            "number_of_trials": self.number_of_trials,
+            "perform_crossval": self.perform_crossval,
+            "plot": self.plot,
+            "preserve_model": self.preserve_model,
+            # not yet
+            "target_mclf": self.target_mclf,
+        }
+
+    def train(self, data):
+        data_eng = self.feature_engineering(data)
+        train, val, test = self.preprocess_feature(data_eng)
+        all_features = self.feature_selection(train)
+        std_data, reshaped_data = self.preprocess_model(train, val, test)
+        self.model_selection(std_data, reshaped_data)
+
+    def predict(self, new_data):
+        data = self.feature_engineering(
+            data=new_data,
+            for_training=False,
+        )
+        data = self.preprocess_feature(data, for_training=False)
+        data, scaled_data, reshaped_data = self.preprocess_model(
+            data, for_training=False
+        )
+
+        for target_number in self.target_numbers:
+
+            # loading model
+            training_target_dir = f"{self.dataset.path}/TARGET_{target_number}"
+            all_features = self.dataset.get_all_features(
+                date_column=self.date_column, group_column=self.group_column
+            )
+            if self.dataset.name == "data_28_X_X":
+                features = joblib.load(
+                    f"{self.dataset.path}/preprocessing/features_{target_number}.pkl"
+                )  # we keep this for backward compatibility
+            else:
+                features = self.dataset.get_features(target_number)
+            model = ModelEngine(path=training_target_dir)
+            model.load()
+
+            # getting data
+            if model.recurrent:
+                features_idx = [
+                    i for i, e in enumerate(all_features) if e in set(features)
+                ]
+                x_pred = reshaped_data[:, :, features_idx]
+            else:
+                x_pred = scaled_data[features] if model.need_scaling else data[features]
+
+            # predicting
+            y_pred = model.predict(x_pred)
+
+            # fix for recurrent model because x_val has no index as it is a 3D np array
+            if model.recurrent:
+                y_pred.index = (
+                    new_data.index
+                )  # TODO: not sure this will work for old dataset not aligned with data_for_training for test use case (done, this is why we decode the test set)
+
+            # unscaling prediction
+            if (
+                model.need_scaling
+                and model.target_type == "regression"
+                and model.scaler_y is not None
+            ):
+                y_pred = pd.Series(
+                    model.scaler_y.inverse_transform(
+                        y_pred.values.reshape(-1, 1)
+                    ).flatten(),
+                    index=new_data.index,
+                )
+
+            # renaming pred column and concatenating with initial data
+            if isinstance(y_pred, pd.DataFrame):
+                y_pred.rename(
+                    columns={"PRED": f"TARGET_{target_number}_PRED"}, inplace=True
+                )
+                new_data = pd.concat(
+                    [new_data, y_pred[f"TARGET_{target_number}_PRED"]], axis=1
+                )
+
+            else:
+                y_pred.name = f"TARGET_{target_number}_PRED"
+                new_data = pd.concat([new_data, y_pred], axis=1)
+
+        return new_data
+
+    def feature_engineering(self, data, for_training=True):
+        app = FeatureEngineeringEngine(
+            data=data,
+            columns_drop=self.columns_drop,
+            columns_boolean=self.columns_boolean,
+            columns_date=self.columns_date,
+            columns_te_groupby=self.columns_te_groupby,
+            columns_te_target=self.columns_te_target,
+            for_training=for_training,
+        )
+        data = app.run()
+        return data
+
+    def preprocess_feature(self, data, for_training=True):
+        app = PreprocessFeature(
+            data=data,
+            dataset=self.dataset,
+            time_series=self.time_series,
+            date_column=self.date_column,
+            group_column=self.group_column,
+            val_size=self.val_size,
+            test_size=self.test_size,
+            columns_pca=self.columns_pca,
+            columns_onehot=self.columns_onehot,
+            columns_binary=self.columns_binary,
+            columns_frequency=self.columns_frequency,
+            columns_ordinal=self.columns_ordinal,
+            target_numbers=self.target_numbers,
+            target_clf=self.target_clf,
+        )
+        if for_training:
+            train, val, test = app.run()
+            return train, val, test
+        else:
+            data = app.inference()
+            return data
+
+    def feature_selection(self, train):
+        for target_number in self.target_numbers:
+            app = FeatureSelectionEngine(
+                train=train,
+                target_number=target_number,
+                dataset=self.dataset,
+                target_clf=self.target_clf,
+            )
+            app.run()
+        self.dataset = Dataset.get(self.dataset.id)
+        all_features = self.dataset.get_all_features(
+            date_column=self.date_column, group_column=self.group_column
+        )
+        return all_features
+
+    def preprocess_model(self, train, val=None, test=None, for_training=True):
+        app = PreprocessModel(
+            train=train,
+            val=val,
+            test=test,
+            dataset=self.dataset,
+            target_numbers=self.target_numbers,
+            target_clf=self.target_clf,
+            models_idx=self.models_idx,
+            time_series=self.time_series,
+            max_timesteps=self.max_timesteps,
+            date_column=self.date_column,
+            group_column=self.group_column,
+        )
+        if for_training:
+            data, reshaped_data = app.run()
+            return data, reshaped_data
+        else:
+            data, scaled_data, reshaped_data = app.inference()
+            return data, scaled_data, reshaped_data
+
+    def model_selection(self, data, reshaped_data):
+        for target_number in self.target_numbers:
+            app = ModelSelectionEngine(
+                data=data,
+                reshaped_data=reshaped_data,
+                target_number=target_number,
+                dataset=self.dataset,
+                target_clf=self.target_clf,
+                models_idx=self.models_idx,
+                time_series=self.time_series,
+                date_column=self.date_column,
+                group_column=self.group_column,
+            )
+            app.run(
+                self.session_name,
+                perform_hyperopt=self.perform_hyperopt,
+                number_of_trials=self.number_of_trials,
+                perform_crossval=self.perform_crossval,
+                plot=self.plot,
+                preserve_model=self.preserve_model,
+            )

lecrapaud/config.py

@@ -0,0 +1,25 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv(override=False)
+
+PYTHON_ENV = os.getenv("PYTHON_ENV")
+REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379")
+EMAIL = os.getenv("EMAIL")
+DATASET_ID = os.getenv("DATASET_ID")
+RECEIVER_EMAIL = os.getenv("RECEIVER_EMAIL")
+USERNAME = os.getenv("USERNAME")
+FRAISE = os.getenv("FRAISE")
+FA2 = os.getenv("2FA")
+INT = os.getenv("INT")
+LOGGING_LEVEL = os.getenv("LOGGING_LEVEL", "INFO")
+ALPHA_VENTAGE_API_KEY = os.getenv("ALPHA_VENTAGE_API_KEY")
+
+DB_USER = os.getenv("TEST_DB_USER") if PYTHON_ENV == "Test" else os.getenv("DB_USER")
+DB_PASSWORD = (
+    os.getenv("TEST_DB_PASSWORD") if PYTHON_ENV == "Test" else os.getenv("DB_PASSWORD")
+)
+DB_HOST = os.getenv("TEST_DB_HOST") if PYTHON_ENV == "Test" else os.getenv("DB_HOST")
+DB_PORT = os.getenv("TEST_DB_PORT") if PYTHON_ENV == "Test" else os.getenv("DB_PORT")
+DB_NAME = os.getenv("TEST_DB_NAME") if PYTHON_ENV == "Test" else os.getenv("DB_NAME")
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")

lecrapaud/db/__init__.py

@@ -0,0 +1 @@
+from lecrapaud.db.models import *

lecrapaud/db/alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

lecrapaud/db/alembic/env.py

@@ -0,0 +1,78 @@
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from alembic import context
+from lecrapaud.db.session import DATABASE_URL
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+config.set_main_option("sqlalchemy.url", DATABASE_URL)
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+from lecrapaud.db.models.base import Base
+
+target_metadata = Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(connection=connection, target_metadata=target_metadata)
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

lecrapaud/db/alembic/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}