lecrapaud 0.4.2__py3-none-any.whl → 0.5.0__py3-none-any.whl

lecrapaud/db/session.py

@@ -3,41 +3,58 @@
 from contextlib import contextmanager
 from sqlalchemy import create_engine, text
 from sqlalchemy.orm import sessionmaker
-from urllib.parse import urlparse
 from alembic.config import Config
 from alembic import command
 import os
+from urllib.parse import urlparse
 
 from lecrapaud.config import DB_USER, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME, DB_URI
 
 _engine = None
 _SessionLocal = None
-DATABASE_URL = (
-    f"mysql+pymysql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}" or DB_URI
-)
+if DB_URI:
+    DATABASE_URL = DB_URI
+elif DB_USER and DB_PASSWORD and DB_HOST and DB_PORT and DB_NAME:
+    DATABASE_URL = (
+        f"mysql+pymysql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
+    )
+else:
+    DATABASE_URL = None
 
 
 def init_db(uri: str = None):
-    global _engine, _SessionLocal
-
-    uri = uri if uri else DATABASE_URL
-    # Extract DB name from URI to connect without it
-    parsed = urlparse(uri)
+    print(f"Initializing database with URI: {uri}")
+    global _engine, _SessionLocal, DATABASE_URL
+    if uri:
+        DATABASE_URL = uri
+    elif DB_USER and DB_PASSWORD and DB_HOST and DB_PORT and DB_NAME:
+        DATABASE_URL = (
+            f"mysql+pymysql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
+        )
+    elif DB_URI:
+        DATABASE_URL = DB_URI
+    else:
+        raise ValueError(
+            "No database configuration found, please set env variables "
+            "DB_USER, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME or DB_URI, "
+            "or provide a `uri` argument to LeCrapaud"
+        )
+
+    # Use urlparse for robust parsing
+    parsed = urlparse(DATABASE_URL)
     db_name = parsed.path.lstrip("/")  # remove leading slash
+    root_uri = f"{parsed.scheme}://{parsed.netloc}"
 
-    # Build root engine (no database in URI)
-    root_uri = uri.replace(f"/{db_name}", "/")
-
-    # Step 1: Connect to MySQL without a database
+    # Do not add trailing slash or database
     root_engine = create_engine(root_uri)
 
     # Step 2: Create database if it doesn't exist
     with root_engine.connect() as conn:
-        conn.execute(text(f"CREATE DATABASE IF NOT EXISTS {DB_NAME}"))
+        conn.execute(text(f"CREATE DATABASE IF NOT EXISTS `{db_name}`"))
         conn.commit()
 
     # Step 3: Connect to the newly created database
-    _engine = create_engine(uri, echo=False)
+    _engine = create_engine(DATABASE_URL, echo=False)
 
     # Step 4: Create session factory
     _SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=_engine)
@@ -45,10 +62,13 @@ def init_db(uri: str = None):
     # Step 5: Apply Alembic migrations programmatically
     current_dir = os.path.dirname(__file__)  # → lecrapaud/db
     alembic_ini_path = os.path.join(current_dir, "alembic.ini")
+    alembic_dir = os.path.join(
+        current_dir, "alembic"
+    )  # Use absolute path to alembic directory
 
     alembic_cfg = Config(alembic_ini_path)
-    alembic_cfg.set_main_option("script_location", "lecrapaud.db.alembic")
-    alembic_cfg.set_main_option("sqlalchemy.url", uri or os.getenv("DATABASE_URL"))
+    alembic_cfg.set_main_option("script_location", alembic_dir)  # Use absolute path
+    alembic_cfg.set_main_option("sqlalchemy.url", DATABASE_URL)
 
     command.upgrade(alembic_cfg, "head")
 

lecrapaud-0.5.0.dist-info/METADATA

@@ -0,0 +1,263 @@
+Metadata-Version: 2.3
+Name: lecrapaud
+Version: 0.5.0
+Summary: Framework for machine and deep learning, with regression, classification and time series analysis
+License: Apache License
+Author: Pierre H. Gallet
+Requires-Python: ==3.12.*
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: backoff (>=2.2.1)
+Requires-Dist: category-encoders (>=2.8.1)
+Requires-Dist: celery (>=5.5.1)
+Requires-Dist: curl-cffi (>=0.11.1)
+Requires-Dist: deep-translator (>=1.11.4)
+Requires-Dist: degiro-connector (>=3.0.26)
+Requires-Dist: fake-useragent (>=2.1.0)
+Requires-Dist: ftfy (>=6.3.1)
+Requires-Dist: honeybadger (>=0.21)
+Requires-Dist: joblib (>=1.4.2)
+Requires-Dist: keras (>=3.9.0)
+Requires-Dist: keras-tcn (>=3.1.2)
+Requires-Dist: lightgbm (>=4.6.0)
+Requires-Dist: matplotlib (>=3.10.1)
+Requires-Dist: mlxtend (>=0.23.4)
+Requires-Dist: numpy (>=2.1.3)
+Requires-Dist: openai (>=1.86.0)
+Requires-Dist: pandas (>=2.2.3)
+Requires-Dist: pandas-market-calendars (>=4.6.1)
+Requires-Dist: playwright (>=1.52.0)
+Requires-Dist: pydantic (>=2.10.6)
+Requires-Dist: python-dotenv (>=1.0.1)
+Requires-Dist: pytz (>=2025.1)
+Requires-Dist: ratelimit (>=2.2.1)
+Requires-Dist: scikit-learn (>=1.6.1)
+Requires-Dist: scipy (>=1.15.2)
+Requires-Dist: seaborn (>=0.13.2)
+Requires-Dist: sentence-transformers (>=3.4.1)
+Requires-Dist: sqlalchemy (>=2.0.39)
+Requires-Dist: tensorboardx (>=2.6.2.2)
+Requires-Dist: tensorflow (>=2.19.0)
+Requires-Dist: tf-keras (>=2.19.0)
+Requires-Dist: tiktoken (>=0.9.0)
+Requires-Dist: tqdm (>=4.67.1)
+Requires-Dist: xgboost (>=3.0.0)
+Requires-Dist: yahoo-fin (>=0.8.9.1)
+Requires-Dist: yfinance (>=0.2.55)
+Description-Content-Type: text/markdown
+
+<div align="center">
+
+<img src="https://s3.amazonaws.com/pix.iemoji.com/images/emoji/apple/ios-12/256/frog-face.png" width=120 alt="crapaud"/>
+
+## Welcome to LeCrapaud
+
+**An all-in-one machine learning framework**
+
+[![GitHub stars](https://img.shields.io/github/stars/pierregallet/lecrapaud.svg?style=flat&logo=github&colorB=blue&label=stars)](https://github.com/pierregallet/lecrapaud/stargazers)
+[![PyPI version](https://badge.fury.io/py/lecrapaud.svg)](https://badge.fury.io/py/lecrapaud)
+[![Python versions](https://img.shields.io/pypi/pyversions/lecrapaud.svg)](https://pypi.org/project/lecrapaud)
+[![License](https://img.shields.io/github/license/pierregallet/lecrapaud.svg)](https://github.com/pierregallet/lecrapaud/blob/main/LICENSE)
+[![codecov](https://codecov.io/gh/pierregallet/lecrapaud/branch/main/graph/badge.svg)](https://codecov.io/gh/pierregallet/lecrapaud)
+
+</div>
+
+## 🚀 Introduction
+
+LeCrapaud is a high-level Python library for end-to-end machine learning workflows on tabular data, with a focus on financial and stock datasets. It provides a simple API to handle feature engineering, model selection, training, and prediction, all in a reproducible and modular way.
+
+## ✨ Key Features
+
+- 🧩 Modular pipeline: Feature engineering, preprocessing, selection, and modeling as independent steps
+- 🤖 Automated model selection and hyperparameter optimization
+- 📊 Easy integration with pandas DataFrames
+- 🔬 Supports both regression and classification tasks
+- 🛠️ Simple API for both full pipeline and step-by-step usage
+- 📦 Ready for production and research workflows
+
+## ⚡ Quick Start
+
+
+### Install the package
+
+```sh
+pip install lecrapaud
+```
+
+### How it works
+
+This package provides a high-level API to manage experiments for feature engineering, model selection, and prediction on tabular data (e.g. stock data).
+
+### Typical workflow
+
+```python
+from lecrapaud import LeCrapaud
+
+# 1. Create the main app
+app = LeCrapaud(uri=uri)
+
+# 2. Define your experiment context (see your notebook or api.py for all options)
+context = {
+    "data": your_dataframe,
+    "columns_drop": [...],
+    "columns_date": [...],
+    # ... other config options
+}
+
+# 3. Create an experiment
+experiment = app.create_experiment(**context)
+
+# 4. Run the full training pipeline
+experiment.train(your_dataframe)
+
+# 5. Make predictions on new data
+predictions = experiment.predict(new_data)
+```
+
+### Database Configuration (Required)
+
+LeCrapaud requires access to a MySQL database to store experiments and results. You must either:
+
+- Pass a valid MySQL URI to the `LeCrapaud` constructor:
+  ```python
+  app = LeCrapaud(uri="mysql+pymysql://user:password@host:port/dbname")
+  ```
+- **OR** set the following environment variables before using the package:
+  - `DB_USER`, `DB_PASSWORD`, `DB_HOST`, `DB_PORT`, `DB_NAME`
+  - Or set `DB_URI` directly with your full connection string.
+
+If neither is provided, database operations will not work.
+
+### Using OpenAI Embeddings (Optional)
+
+If you want to use the `columns_pca` embedding feature (for advanced feature engineering), you must set the `OPENAI_API_KEY` environment variable with your OpenAI API key:
+
+```sh
+export OPENAI_API_KEY=sk-...
+```
+
+If this variable is not set, features relying on OpenAI embeddings will not be available.
+
+### Experiment Context Arguments
+
+Below are the main arguments you can pass to `create_experiment` (or the `Experiment` class):
+
+| Argument             | Type      | Description                                                                              | Example/Default    |
+| -------------------- | --------- | ---------------------------------------------------------------------------------------- | ------------------ |
+| `columns_binary`     | list      | Columns to treat as binary                                                               | `['flag']`         |
+| `columns_boolean`    | list      | Columns to treat as boolean                                                              | `['is_active']`    |
+| `columns_date`       | list      | Columns to treat as dates                                                                | `['date']`         |
+| `columns_drop`       | list      | Columns to drop during feature engineering                                               | `['col1', 'col2']` |
+| `columns_frequency`  | list      | Columns to frequency encode                                                              | `['category']`     |
+| `columns_onehot`     | list      | Columns to one-hot encode                                                                | `['sector']`       |
+| `columns_ordinal`    | list      | Columns to ordinal encode                                                                | `['grade']`        |
+| `columns_pca`        | list      | Columns to use for PCA/embeddings (requires `OPENAI_API_KEY` if using OpenAI embeddings) | `['text_col']`     |
+| `columns_te_groupby` | list      | Columns for target encoding groupby                                                      | `['sector']`       |
+| `columns_te_target`  | list      | Columns for target encoding target                                                       | `['target']`       |
+| `data`               | DataFrame | Your main dataset (required for new experiment)                                          | `your_dataframe`   |
+| `date_column`        | str       | Name of the date column                                                                  | `'date'`           |
+| `group_column`       | str       | Name of the group column                                                                 | `'stock_id'`       |
+| `max_timesteps`      | int       | Max timesteps for time series models                                                     | `30`               |
+| `models_idx`         | list      | Indices of models to use for model selection                                             | `[0, 1, 2]`        |
+| `number_of_trials`   | int       | Number of trials for hyperparameter optimization                                         | `20`               |
+| `perform_crossval`   | bool      | Whether to perform cross-validation                                                      | `True`/`False`     |
+| `perform_hyperopt`   | bool      | Whether to perform hyperparameter optimization                                           | `True`/`False`     |
+| `plot`               | bool      | Whether to plot results                                                                  | `True`/`False`     |
+| `preserve_model`     | bool      | Whether to preserve the best model                                                       | `True`/`False`     |
+| `session_name`       | str       | Name for the training session                                                            | `'my_session'`     |
+| `target_clf`         | list      | List of classification target column indices/names                                       | `[1, 2, 3]`        |
+| `target_mclf`        | list      | Multi-class classification targets (not yet implemented)                                 | `[11]`             |
+| `target_numbers`     | list      | List of regression target column indices/names                                           | `[1, 2, 3]`        |
+| `test_size`          | int/float | Test set size (count or fraction)                                                        | `0.2`              |
+| `time_series`        | bool      | Whether the data is time series                                                          | `True`/`False`     |
+| `val_size`           | int/float | Validation set size (count or fraction)                                                  | `0.2`              |
+
+**Note:**
+- Not all arguments are required; defaults may exist for some.
+- For `columns_pca` with OpenAI embeddings, you must set the `OPENAI_API_KEY` environment variable.
+
+
+
+### Modular usage
+
+You can also use each step independently:
+
+```python
+data_eng = experiment.feature_engineering(data)
+train, val, test = experiment.preprocess_feature(data_eng)
+features = experiment.feature_selection(train)
+std_data, reshaped_data = experiment.preprocess_model(train, val, test)
+experiment.model_selection(std_data, reshaped_data)
+```
+
+## ⚠️ Using Alembic in Your Project (Important for Integrators)
+
+If you use Alembic for migrations in your own project and you share the same database with LeCrapaud, you must ensure that Alembic does **not** attempt to drop or modify LeCrapaud tables (those prefixed with `lecrapaud_`).
+
+By default, Alembic's autogenerate feature will propose to drop any table that exists in the database but is not present in your project's models. To prevent this, add the following filter to your `env.py`:
+
+```python
+def include_object(object, name, type_, reflected, compare_to):
+    if type_ == "table" and name.startswith("lecrapaud_"):
+        return False  # Ignore LeCrapaud tables
+    return True
+
+context.configure(
+    # ... other options ...
+    include_object=include_object,
+)
+```
+
+This will ensure that Alembic ignores all tables created by LeCrapaud when generating migrations for your own project.
+
+---
+
+## 🤝 Contributing
+
+### Reminders for Github usage
+
+1. Creating Github repository
+
+```sh
+$ brew install gh
+$ gh auth login
+$ gh repo create
+```
+
+2. Initializing git and first commit to distant repository
+
+```sh
+$ git init
+$ git add .
+$ git commit -m 'first commit'
+$ git remote add origin <YOUR_REPO_URL>
+$ git push -u origin master
+```
+
+3. Use conventional commits  
+https://www.conventionalcommits.org/en/v1.0.0/#summary
+
+4. Create environment
+
+```sh
+$ pip install virtualenv
+$ python -m venv .venv
+$ source .venv/bin/activate
+```
+
+5. Install dependencies
+
+```sh
+$ make install
+```
+
+6. Deactivate virtualenv (if needed)
+
+```sh
+$ deactivate
+```
+
+---
+
+Pierre Gallet © 2025

lecrapaud-0.5.0.dist-info/RECORD

@@ -0,0 +1,46 @@
+lecrapaud/__init__.py,sha256=oCxbtw_nk8rlOXbXbWo0RRMlsh6w-hTiZ6e5PRG_wp0,28
+lecrapaud/api.py,sha256=3yAoHTJt1XtjCrLemwZnoY7UyTdsZo9a7-jgbkOnirc,9912
+lecrapaud/config.py,sha256=vtTxu0BTcAKfZWywl0jrdP2SAh6YYHDImncVIUXhf7E,858
+lecrapaud/db/__init__.py,sha256=82o9fMfaqKXPh2_rt44EzNRVZV1R4LScEnQYvj_TjK0,34
+lecrapaud/db/alembic/README,sha256=MVlc9TYmr57RbhXET6QxgyCcwWP7w-vLkEsirENqiIQ,38
+lecrapaud/db/alembic/env.py,sha256=rseEi8oR_eKXYYW3UwOKiCMuDEwT4lxsT7llySOUpgk,2305
+lecrapaud/db/alembic/script.py.mako,sha256=MEqL-2qATlST9TAOeYgscMn1uy6HUS9NFvDgl93dMj8,635
+lecrapaud/db/alembic/versions/2025_06_20_1924-1edada319fd7_initial_setup.py,sha256=zqFFB5VZEMNQstqQ0WAwQGUb2LGhw6-s60QiP60kr_k,13010
+lecrapaud/db/alembic.ini,sha256=zgvur-5jnLsT66_98FaTOTNgjwObGZCE0HqMwRAeJrs,3587
+lecrapaud/db/models/__init__.py,sha256=oiOQitb8zEA7TK262zWLN6xnbZWYfg4eOaKVfAVyAFA,540
+lecrapaud/db/models/base.py,sha256=5JT3W-vAcTmwi9I9JJbHjKjkOKQcJvAs-7d2ndKNecU,5725
+lecrapaud/db/models/dataset.py,sha256=rHzMmJzfJYx0cMZeLaV7X4hgzxfsnbm2iTg9jRXlTZs,3724
+lecrapaud/db/models/feature.py,sha256=5o77O2FyRObnLOCGNj8kaPSGM3pLv1Ov6mXXHYkmnYY,1136
+lecrapaud/db/models/feature_selection.py,sha256=VFsbdsVc4GV4_ivNuVqJuq-uWG6JADnCDD9hOhKoHtE,3300
+lecrapaud/db/models/feature_selection_rank.py,sha256=PvEpdv-JJt2wZMtX5TO0wyZ3IONlPkeDaC49i0VA-qU,2074
+lecrapaud/db/models/model.py,sha256=F0hyMjd4FFHCv6_arIWBEmBCGOfG3b6_uzU8ExtFE90,952
+lecrapaud/db/models/model_selection.py,sha256=DBwVIo6vZTNzeZ5BccXxrZ8CEv0FyLvema0N3XWt7Lc,1812
+lecrapaud/db/models/model_training.py,sha256=egggSfkW8C2nTadytc5DdjU7d2VEMT6LRRZxO1ZD5To,1600
+lecrapaud/db/models/score.py,sha256=VwAP06Ce_B2JF3WmXlIJV9Ej9sUptphyfmEFbYIfRYU,1555
+lecrapaud/db/models/target.py,sha256=LXzfIlX4A7elhi4bwpz_2911o9zc732RY5BkqQKPhn8,1634
+lecrapaud/db/session.py,sha256=bHzg_Dt4eRJvBeaDluR8KpMP0JVylg4njppDRawBxQU,2876
+lecrapaud/directories.py,sha256=wrHmAJ-cBKb0GhJifxrb_RoLhZJX8xdkeirrWs7jQHk,791
+lecrapaud/experiment.py,sha256=ApsGtGaISYsQCdAGa1llzBslfODK-w1_zf44p6Es3Xs,1985
+lecrapaud/feature_engineering.py,sha256=MEPW8-AgO3xnbwMHV3dC_gMtrBeKQhaCDrAicWgXX6w,30251
+lecrapaud/feature_selection.py,sha256=RD016F9_r249VEAe5bLCpB4tHQSbhdMoXRFohkk3c5g,42429
+lecrapaud/integrations/openai_integration.py,sha256=hHLF3fk5Bps8KNbNrEL3NUFa945jwClE6LrLpuMZOd4,7459
+lecrapaud/jobs/__init__.py,sha256=ZkrsyTOR21c_wN7RY8jPhm8jCrL1oCEtTsf3VFIlQiE,292
+lecrapaud/jobs/config.py,sha256=AmO0j3RFjx8H66dfKw_7vnshaOJb9Ox5BAZ9cwwLFMY,377
+lecrapaud/jobs/scheduler.py,sha256=SiYWPxokpKnR8V6btLOO6gbK0PEjSRoeG0kCbQvYPf4,990
+lecrapaud/jobs/tasks.py,sha256=evzhOHpgp6Gvoz__jUipi-_HSNny7bWgAauHv2Hpxyk,1640
+lecrapaud/model_selection.py,sha256=y01yOK5oB9hdeEJkjjavDJvYAUCR3PS6FWRAN43uDzo,61322
+lecrapaud/search_space.py,sha256=6kVXDSmvpTuLKTlqQkKIzrJOM2P7HpqRiO3PR37VrsM,34123
+lecrapaud/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lecrapaud/services/embedding_categorical.py,sha256=jPKl3GjJ8STk5u0ux9VHbUYq-XRuNULhMJuXdeJAePU,2593
+lecrapaud/services/indicators.py,sha256=1iDxCRzqAeBEwbazQOl_1LJnw_ceM7IiYYWkJbs14s4,9190
+lecrapaud/speed_tests/experiments.py,sha256=m8rz4a0ZwCMQKstK6GLU_FfTyBhFWvWQO807pS7Dz-s,4399
+lecrapaud/speed_tests/test-gpu-bilstm.ipynb,sha256=4nLuZRJVe2kn6kEmauhRiz5wkWT9AVrYhI9CEk_dYUY,9608
+lecrapaud/speed_tests/test-gpu-resnet.ipynb,sha256=27Vu7nYwujYeh3fOxBNCnKJn3MXNPKZU-U8oDDUbymg,4944
+lecrapaud/speed_tests/test-gpu-transformers.ipynb,sha256=k6MBSs_Um1h4PykvE-LTBcdpbWLbIFST_xl_AFW2jgI,8444
+lecrapaud/speed_tests/tests.ipynb,sha256=RjI7LDHSsbadUkea_hT14sD7ivljtIQk4NB5McXJ1bE,3835
+lecrapaud/speed_tests/trash.py,sha256=E4zrrRyVqeNEumWg8rYKquR9VTIULN52eCRqjmv_s58,1647
+lecrapaud/utils.py,sha256=TMlEyU6aRhgiYRtdbvKrlq2t1L9Al5BXfQ3cPbeHlX8,8098
+lecrapaud-0.5.0.dist-info/LICENSE,sha256=MImCryu0AnqhJE_uAZD-PIDKXDKb8sT7v0i1NOYeHTM,11350
+lecrapaud-0.5.0.dist-info/METADATA,sha256=EbLpYtW7ZsUMjynq_NAMULsExznxtPMwzRkJY4jRlXo,11623
+lecrapaud-0.5.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+lecrapaud-0.5.0.dist-info/RECORD,,