lecrapaud 0.4.0__tar.gz → 0.4.1__tar.gz

lecrapaud-0.4.1/PKG-INFO

@@ -0,0 +1,171 @@
+Metadata-Version: 2.3
+Name: lecrapaud
+Version: 0.4.1
+Summary: Framework for machine and deep learning, with regression, classification and time series analysis
+License: Apache License
+Author: Pierre H. Gallet
+Requires-Python: ==3.12.*
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: backoff (>=2.2.1)
+Requires-Dist: category-encoders (>=2.8.1)
+Requires-Dist: celery (>=5.5.1)
+Requires-Dist: curl-cffi (>=0.11.1)
+Requires-Dist: deep-translator (>=1.11.4)
+Requires-Dist: degiro-connector (>=3.0.26)
+Requires-Dist: fake-useragent (>=2.1.0)
+Requires-Dist: ftfy (>=6.3.1)
+Requires-Dist: honeybadger (>=0.21)
+Requires-Dist: joblib (>=1.4.2)
+Requires-Dist: keras (>=3.9.0)
+Requires-Dist: keras-tcn (>=3.1.2)
+Requires-Dist: lightgbm (>=4.6.0)
+Requires-Dist: matplotlib (>=3.10.1)
+Requires-Dist: mlxtend (>=0.23.4)
+Requires-Dist: numpy (>=2.1.3)
+Requires-Dist: openai (>=1.86.0)
+Requires-Dist: pandas (>=2.2.3)
+Requires-Dist: pandas-market-calendars (>=4.6.1)
+Requires-Dist: playwright (>=1.52.0)
+Requires-Dist: pydantic (>=2.10.6)
+Requires-Dist: python-dotenv (>=1.0.1)
+Requires-Dist: pytz (>=2025.1)
+Requires-Dist: ratelimit (>=2.2.1)
+Requires-Dist: scikit-learn (>=1.6.1)
+Requires-Dist: scipy (>=1.15.2)
+Requires-Dist: seaborn (>=0.13.2)
+Requires-Dist: sentence-transformers (>=3.4.1)
+Requires-Dist: sqlalchemy (>=2.0.39)
+Requires-Dist: tensorboardx (>=2.6.2.2)
+Requires-Dist: tensorflow (>=2.19.0)
+Requires-Dist: tf-keras (>=2.19.0)
+Requires-Dist: tiktoken (>=0.9.0)
+Requires-Dist: tqdm (>=4.67.1)
+Requires-Dist: xgboost (>=3.0.0)
+Requires-Dist: yahoo-fin (>=0.8.9.1)
+Requires-Dist: yfinance (>=0.2.55)
+Description-Content-Type: text/markdown
+
+<div align="center">
+
+<img src="https://s3.amazonaws.com/pix.iemoji.com/images/emoji/apple/ios-12/256/frog-face.png" width=120 alt="crapaud"/>
+
+## Welcome to LeCrapaud
+
+**An all-in-one machine learning framework**
+
+</div>
+
+## 🚀 Introduction
+
+LeCrapaud is a high-level Python library for end-to-end machine learning workflows on tabular data, with a focus on financial and stock datasets. It provides a simple API to handle feature engineering, model selection, training, and prediction, all in a reproducible and modular way.
+
+## ✨ Key Features
+
+- 🧩 Modular pipeline: Feature engineering, preprocessing, selection, and modeling as independent steps
+- 🤖 Automated model selection and hyperparameter optimization
+- 📊 Easy integration with pandas DataFrames
+- 🔬 Supports both regression and classification tasks
+- 🛠️ Simple API for both full pipeline and step-by-step usage
+- 📦 Ready for production and research workflows
+
+## ⚡ Quick Start
+
+
+### Install the package
+
+```sh
+pip install lecrapaud
+```
+
+### How it works
+
+This package provides a high-level API to manage experiments for feature engineering, model selection, and prediction on tabular data (e.g. stock data).
+
+### Typical workflow
+
+```python
+from lecrapaud import LeCrapaud
+
+# 1. Create the main app
+app = LeCrapaud()
+
+# 2. Define your experiment context (see your notebook or api.py for all options)
+context = {
+    "data": your_dataframe,
+    "columns_drop": [...],
+    "columns_date": [...],
+    # ... other config options
+}
+
+# 3. Create an experiment
+experiment = app.create_experiment(**context)
+
+# 4. Run the full training pipeline
+experiment.train(your_dataframe)
+
+# 5. Make predictions on new data
+predictions = experiment.predict(new_data)
+```
+
+### Modular usage
+
+You can also use each step independently:
+
+```python
+data_eng = experiment.feature_engineering(data)
+train, val, test = experiment.preprocess_feature(data_eng)
+features = experiment.feature_selection(train)
+std_data, reshaped_data = experiment.preprocess_model(train, val, test)
+experiment.model_selection(std_data, reshaped_data)
+```
+
+## 🤝 Contributing
+
+### Reminders for Github usage
+
+1. Creating Github repository
+
+```sh
+$ brew install gh
+$ gh auth login
+$ gh repo create
+```
+
+2. Initializing git and first commit to distant repository
+
+```sh
+$ git init
+$ git add .
+$ git commit -m 'first commit'
+$ git remote add origin <YOUR_REPO_URL>
+$ git push -u origin master
+```
+
+3. Use conventional commits  
+https://www.conventionalcommits.org/en/v1.0.0/#summary
+
+4. Create environment
+
+```sh
+$ pip install virtualenv
+$ python -m venv .venv
+$ source .venv/bin/activate
+```
+
+5. Install dependencies
+
+```sh
+$ make install
+```
+
+6. Deactivate virtualenv (if needed)
+
+```sh
+$ deactivate
+```
+
+---
+
+Pierre Gallet © 2025

lecrapaud-0.4.1/README.md

@@ -0,0 +1,122 @@
+<div align="center">
+
+<img src="https://s3.amazonaws.com/pix.iemoji.com/images/emoji/apple/ios-12/256/frog-face.png" width=120 alt="crapaud"/>
+
+## Welcome to LeCrapaud
+
+**An all-in-one machine learning framework**
+
+</div>
+
+## 🚀 Introduction
+
+LeCrapaud is a high-level Python library for end-to-end machine learning workflows on tabular data, with a focus on financial and stock datasets. It provides a simple API to handle feature engineering, model selection, training, and prediction, all in a reproducible and modular way.
+
+## ✨ Key Features
+
+- 🧩 Modular pipeline: Feature engineering, preprocessing, selection, and modeling as independent steps
+- 🤖 Automated model selection and hyperparameter optimization
+- 📊 Easy integration with pandas DataFrames
+- 🔬 Supports both regression and classification tasks
+- 🛠️ Simple API for both full pipeline and step-by-step usage
+- 📦 Ready for production and research workflows
+
+## ⚡ Quick Start
+
+
+### Install the package
+
+```sh
+pip install lecrapaud
+```
+
+### How it works
+
+This package provides a high-level API to manage experiments for feature engineering, model selection, and prediction on tabular data (e.g. stock data).
+
+### Typical workflow
+
+```python
+from lecrapaud import LeCrapaud
+
+# 1. Create the main app
+app = LeCrapaud()
+
+# 2. Define your experiment context (see your notebook or api.py for all options)
+context = {
+    "data": your_dataframe,
+    "columns_drop": [...],
+    "columns_date": [...],
+    # ... other config options
+}
+
+# 3. Create an experiment
+experiment = app.create_experiment(**context)
+
+# 4. Run the full training pipeline
+experiment.train(your_dataframe)
+
+# 5. Make predictions on new data
+predictions = experiment.predict(new_data)
+```
+
+### Modular usage
+
+You can also use each step independently:
+
+```python
+data_eng = experiment.feature_engineering(data)
+train, val, test = experiment.preprocess_feature(data_eng)
+features = experiment.feature_selection(train)
+std_data, reshaped_data = experiment.preprocess_model(train, val, test)
+experiment.model_selection(std_data, reshaped_data)
+```
+
+## 🤝 Contributing
+
+### Reminders for Github usage
+
+1. Creating Github repository
+
+```sh
+$ brew install gh
+$ gh auth login
+$ gh repo create
+```
+
+2. Initializing git and first commit to distant repository
+
+```sh
+$ git init
+$ git add .
+$ git commit -m 'first commit'
+$ git remote add origin <YOUR_REPO_URL>
+$ git push -u origin master
+```
+
+3. Use conventional commits  
+https://www.conventionalcommits.org/en/v1.0.0/#summary
+
+4. Create environment
+
+```sh
+$ pip install virtualenv
+$ python -m venv .venv
+$ source .venv/bin/activate
+```
+
+5. Install dependencies
+
+```sh
+$ make install
+```
+
+6. Deactivate virtualenv (if needed)
+
+```sh
+$ deactivate
+```
+
+---
+
+Pierre Gallet © 2025

lecrapaud-0.4.1/lecrapaud/__init__.py

@@ -0,0 +1 @@
+from lecrapaud.api import *

lecrapaud-0.4.1/lecrapaud/api.py

@@ -0,0 +1,277 @@
+"""
+Main API class
+
+the way I want it to work :
+
+app = LeCrapaud()
+
+kwargs = {
+
+}
+
+experiment = app.create_experiment(**kwargs) # return a class Experiment()
+ou
+experiment = app.get_experiment(exp_id)
+
+best_features, artifacts, best_model = experiment.train(get_data, get_data_params)
+
+new_data + target_pred + target_proba (if classif) = experiment.predict(**new_data)
+
+On veut aussi pouvoir juste faire :
+
+experiment.feature_engineering(data) : feat eng, return data
+
+experiment.preprocess_feature(data) : split, encoding, pcas, return train, val, test df
+
+experiment.feature_selection(train) : return features
+
+experiment.preprocess_model(train, val, test) : return data = dict of df
+
+experiment.model_selection(data) : return best_model
+"""
+
+import joblib
+import pandas as pd
+import logging
+from lecrapaud.utils import logger
+from lecrapaud.db.session import init_db
+from lecrapaud.feature_selection import FeatureSelectionEngine, PreprocessModel
+from lecrapaud.model_selection import ModelSelectionEngine, ModelEngine
+from lecrapaud.feature_engineering import FeatureEngineeringEngine, PreprocessFeature
+from lecrapaud.experiment import create_dataset
+from lecrapaud.db import Dataset
+
+
+class LeCrapaud:
+    def __init__(self, uri: str = None):
+        init_db(uri=uri)
+
+    def create_experiment(self, **kwargs):
+        return Experiment(**kwargs)
+
+    def get_experiment(self, id: int):
+        return Experiment(id)
+
+
+class Experiment:
+    def __init__(self, id=None, **kwargs):
+        if id:
+            self.dataset = Dataset.get(id)
+        else:
+            self.dataset = create_dataset(**kwargs)
+
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
+        self.context = {
+            # generic
+            "dataset": self.dataset,
+            # for FeatureEngineering
+            "columns_drop": self.columns_drop,
+            "columns_boolean": self.columns_boolean,
+            "columns_date": self.columns_date,
+            "columns_te_groupby": self.columns_te_groupby,
+            "columns_te_target": self.columns_te_target,
+            # for PreprocessFeature
+            "time_series": self.time_series,
+            "date_column": self.date_column,
+            "group_column": self.group_column,
+            "val_size": self.val_size,
+            "test_size": self.test_size,
+            "columns_pca": self.columns_pca,
+            "columns_onehot": self.columns_onehot,
+            "columns_binary": self.columns_binary,
+            "columns_frequency": self.columns_frequency,
+            "columns_ordinal": self.columns_ordinal,
+            "target_numbers": self.target_numbers,
+            "target_clf": self.target_clf,
+            # for PreprocessModel
+            "models_idx": self.models_idx,
+            "max_timesteps": self.max_timesteps,
+            # for ModelSelection
+            "perform_hyperopt": self.perform_hyperopt,
+            "number_of_trials": self.number_of_trials,
+            "perform_crossval": self.perform_crossval,
+            "plot": self.plot,
+            "preserve_model": self.preserve_model,
+            # not yet
+            "target_mclf": self.target_mclf,
+        }
+
+    def train(self, data):
+        data_eng = self.feature_engineering(data)
+        train, val, test = self.preprocess_feature(data_eng)
+        all_features = self.feature_selection(train)
+        std_data, reshaped_data = self.preprocess_model(train, val, test)
+        self.model_selection(std_data, reshaped_data)
+
+    def predict(self, new_data, verbose: int = 0):
+        if verbose == 0:
+            logger.setLevel(logging.WARNING)
+
+        logger.warning("Running prediction...")
+
+        data = self.feature_engineering(
+            data=new_data,
+            for_training=False,
+        )
+        data = self.preprocess_feature(data, for_training=False)
+        data, scaled_data, reshaped_data = self.preprocess_model(
+            data, for_training=False
+        )
+
+        for target_number in self.target_numbers:
+
+            # loading model
+            training_target_dir = f"{self.dataset.path}/TARGET_{target_number}"
+            all_features = self.dataset.get_all_features(
+                date_column=self.date_column, group_column=self.group_column
+            )
+            if self.dataset.name == "data_28_X_X":
+                features = joblib.load(
+                    f"{self.dataset.path}/preprocessing/features_{target_number}.pkl"
+                )  # we keep this for backward compatibility
+            else:
+                features = self.dataset.get_features(target_number)
+            model = ModelEngine(path=training_target_dir)
+
+            # getting data
+            if model.recurrent:
+                features_idx = [
+                    i for i, e in enumerate(all_features) if e in set(features)
+                ]
+                x_pred = reshaped_data[:, :, features_idx]
+            else:
+                x_pred = scaled_data[features] if model.need_scaling else data[features]
+
+            # predicting
+            y_pred = model.predict(x_pred)
+
+            # fix for recurrent model because x_val has no index as it is a 3D np array
+            if model.recurrent:
+                y_pred.index = (
+                    new_data.index
+                )  # TODO: not sure this will work for old dataset not aligned with data_for_training for test use case (done, this is why we decode the test set)
+
+            # unscaling prediction
+            if (
+                model.need_scaling
+                and model.target_type == "regression"
+                and model.scaler_y is not None
+            ):
+                y_pred = pd.Series(
+                    model.scaler_y.inverse_transform(
+                        y_pred.values.reshape(-1, 1)
+                    ).flatten(),
+                    index=new_data.index,
+                )
+
+            # renaming pred column and concatenating with initial data
+            if isinstance(y_pred, pd.DataFrame):
+                y_pred.rename(
+                    columns={"PRED": f"TARGET_{target_number}_PRED"}, inplace=True
+                )
+                new_data = pd.concat(
+                    [new_data, y_pred[f"TARGET_{target_number}_PRED"]], axis=1
+                )
+
+            else:
+                y_pred.name = f"TARGET_{target_number}_PRED"
+                new_data = pd.concat([new_data, y_pred], axis=1)
+
+        return new_data
+
+    def feature_engineering(self, data, for_training=True):
+        app = FeatureEngineeringEngine(
+            data=data,
+            columns_drop=self.columns_drop,
+            columns_boolean=self.columns_boolean,
+            columns_date=self.columns_date,
+            columns_te_groupby=self.columns_te_groupby,
+            columns_te_target=self.columns_te_target,
+            for_training=for_training,
+        )
+        data = app.run()
+        return data
+
+    def preprocess_feature(self, data, for_training=True):
+        app = PreprocessFeature(
+            data=data,
+            dataset=self.dataset,
+            time_series=self.time_series,
+            date_column=self.date_column,
+            group_column=self.group_column,
+            val_size=self.val_size,
+            test_size=self.test_size,
+            columns_pca=self.columns_pca,
+            columns_onehot=self.columns_onehot,
+            columns_binary=self.columns_binary,
+            columns_frequency=self.columns_frequency,
+            columns_ordinal=self.columns_ordinal,
+            target_numbers=self.target_numbers,
+            target_clf=self.target_clf,
+        )
+        if for_training:
+            train, val, test = app.run()
+            return train, val, test
+        else:
+            data = app.inference()
+            return data
+
+    def feature_selection(self, train):
+        for target_number in self.target_numbers:
+            app = FeatureSelectionEngine(
+                train=train,
+                target_number=target_number,
+                dataset=self.dataset,
+                target_clf=self.target_clf,
+            )
+            app.run()
+        self.dataset = Dataset.get(self.dataset.id)
+        all_features = self.dataset.get_all_features(
+            date_column=self.date_column, group_column=self.group_column
+        )
+        return all_features
+
+    def preprocess_model(self, train, val=None, test=None, for_training=True):
+        app = PreprocessModel(
+            train=train,
+            val=val,
+            test=test,
+            dataset=self.dataset,
+            target_numbers=self.target_numbers,
+            target_clf=self.target_clf,
+            models_idx=self.models_idx,
+            time_series=self.time_series,
+            max_timesteps=self.max_timesteps,
+            date_column=self.date_column,
+            group_column=self.group_column,
+        )
+        if for_training:
+            data, reshaped_data = app.run()
+            return data, reshaped_data
+        else:
+            data, scaled_data, reshaped_data = app.inference()
+            return data, scaled_data, reshaped_data
+
+    def model_selection(self, data, reshaped_data):
+        for target_number in self.target_numbers:
+            app = ModelSelectionEngine(
+                data=data,
+                reshaped_data=reshaped_data,
+                target_number=target_number,
+                dataset=self.dataset,
+                target_clf=self.target_clf,
+                models_idx=self.models_idx,
+                time_series=self.time_series,
+                date_column=self.date_column,
+                group_column=self.group_column,
+            )
+            app.run(
+                self.session_name,
+                perform_hyperopt=self.perform_hyperopt,
+                number_of_trials=self.number_of_trials,
+                perform_crossval=self.perform_crossval,
+                plot=self.plot,
+                preserve_model=self.preserve_model,
+            )

lecrapaud-0.4.1/lecrapaud/config.py

@@ -0,0 +1,26 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv(override=False)
+
+PYTHON_ENV = os.getenv("PYTHON_ENV")
+REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379")
+EMAIL = os.getenv("EMAIL")
+DATASET_ID = os.getenv("DATASET_ID")
+RECEIVER_EMAIL = os.getenv("RECEIVER_EMAIL")
+USERNAME = os.getenv("USERNAME")
+FRAISE = os.getenv("FRAISE")
+FA2 = os.getenv("2FA")
+INT = os.getenv("INT")
+LOGGING_LEVEL = os.getenv("LOGGING_LEVEL", "INFO")
+ALPHA_VENTAGE_API_KEY = os.getenv("ALPHA_VENTAGE_API_KEY")
+
+DB_USER = os.getenv("TEST_DB_USER") if PYTHON_ENV == "Test" else os.getenv("DB_USER")
+DB_PASSWORD = (
+    os.getenv("TEST_DB_PASSWORD") if PYTHON_ENV == "Test" else os.getenv("DB_PASSWORD")
+)
+DB_HOST = os.getenv("TEST_DB_HOST") if PYTHON_ENV == "Test" else os.getenv("DB_HOST")
+DB_PORT = os.getenv("TEST_DB_PORT") if PYTHON_ENV == "Test" else os.getenv("DB_PORT")
+DB_NAME = os.getenv("TEST_DB_NAME") if PYTHON_ENV == "Test" else os.getenv("DB_NAME")
+DB_URI = os.getenv("DB_URI", None)
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")

lecrapaud-0.4.1/lecrapaud/db/__init__.py

@@ -0,0 +1 @@
+from lecrapaud.db.models import *

{lecrapaud-0.4.0 → lecrapaud-0.4.1}/lecrapaud/db/alembic/env.py

@@ -4,7 +4,7 @@ from sqlalchemy import engine_from_config
 from sqlalchemy import pool
 
 from alembic import context
-from src.db.setup import DATABASE_URL
+from lecrapaud.db.session import DATABASE_URL
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -18,7 +18,7 @@ if config.config_file_name is not None:
 
 # add your model's MetaData object here
 # for 'autogenerate' support
-from src.db.models.base import Base
+from lecrapaud.db.models.base import Base
 
 target_metadata = Base.metadata
 

lecrapaud-0.4.1/lecrapaud/db/alembic/versions/2025_05_31_1834-52b809a34371_make_nullablee.py

@@ -0,0 +1,50 @@
+"""make_nullablee
+
+Revision ID: 52b809a34371
+Revises: 339927587383
+Create Date: 2025-05-31 18:34:58.962966
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision: str = "52b809a34371"
+down_revision: Union[str, None] = "339927587383"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "investment_runs",
+        "initial_portfolio",
+        existing_type=mysql.JSON(),
+        nullable=True,
+    )
+    op.create_foreign_key(
+        None,
+        "portfolios",
+        "investment_runs",
+        ["investment_run_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, "portfolios", type_="foreignkey")
+    op.alter_column(
+        "investment_runs",
+        "initial_portfolio",
+        existing_type=mysql.JSON(),
+        nullable=False,
+    )
+    # ### end Alembic commands ###