csv-to-db-py 0.1.0__py3-none-any.whl

csv_to_db_py/__init__.py

@@ -0,0 +1,4 @@
+from csv_to_db_py.core import get_connector, load_csv_to_db
+from csv_to_db_py.config import load_config
+
+__all__ = ["get_connector", "load_csv_to_db", "load_config"]

csv_to_db_py/config.py

@@ -0,0 +1,44 @@
+import json
+
+SUPPORTED_DB_TYPES = {"postgres", "mysql", "mssql", "oracle"}
+
+
+def validate_config(cfg: dict) -> None:
+    db_config = cfg.get("DATABASE_CONFIG")
+    if not isinstance(db_config, dict):
+        raise ValueError("config 'DATABASE_CONFIG' must be a dict")
+    db_type = db_config.get("type")
+    if db_type not in SUPPORTED_DB_TYPES:
+        raise ValueError(
+            f"config 'DATABASE_CONFIG.type' must be one of {', '.join(sorted(SUPPORTED_DB_TYPES))}, got {db_type!r}"
+        )
+    if "files_dir" not in cfg:
+        raise ValueError("config is missing required key 'files_dir'")
+
+
+def load_config(config_path: str) -> dict:
+    with open(config_path, "r") as f:
+        cfg = json.load(f)
+    validate_config(cfg)
+    return cfg
+
+
+PG_RESERVED = {
+    "insert",
+    "delete",
+    "update",
+    "select",
+    "drop",
+    "and",
+    "or",
+    "from",
+    "create",
+    "join",
+    "limit",
+    "table",
+    "order",
+    "user",
+    "not",
+    "where",
+    "group",
+}

csv_to_db_py/core.py

@@ -0,0 +1,90 @@
+import logging
+import os
+
+import polars as pl
+
+from csv_to_db_py.db_connectors import postgres, mysql, mssql, oracle
+from csv_to_db_py.db_connectors.base import BaseDBConnector
+from csv_to_db_py.utils.table_creator import create_table_from_csv
+from csv_to_db_py.utils.csv_loader import (
+    postgrestype_dict,
+    overwrite_table_with_csv_data,
+    convert_datetime_columns,
+)
+from csv_to_db_py.utils.cleaned_csv import apply_filter, get_dataframe_cleaned_new
+
+
+def get_connector(db_config: dict) -> BaseDBConnector:
+    db_type = db_config["type"]
+    if db_type == "postgres":
+        return postgres.PostgresConnector(db_config)
+    elif db_type == "mysql":
+        return mysql.MySQLConnector(db_config)
+    elif db_type == "mssql":
+        return mssql.MSSQLConnector(db_config)
+    elif db_type == "oracle":
+        return oracle.OracleConnector(db_config)
+    else:
+        raise ValueError("Base de données non supportée")
+
+
+def load_csv_to_db(config: dict, input_path: str | None = None) -> None:
+    if input_path is None:
+        input_path = config["files_dir"]
+
+    connector = get_connector(config["DATABASE_CONFIG"])
+    connector.connect()
+    engine = connector.get_engine()
+
+    # Détection du(s) fichier(s) CSV ou XLSX
+    if os.path.isfile(input_path) and (
+        input_path.endswith(".csv") or input_path.endswith(".xlsx")
+    ):
+        files = [input_path]
+
+    elif os.path.isdir(input_path):
+        files = [
+            os.path.join(input_path, f)
+            for f in os.listdir(input_path)
+            if f.endswith(".csv") or f.endswith(".xlsx")
+        ]
+
+    else:
+        raise ValueError(
+            "Le chemin fourni n'est ni un fichier CSV/XLSX existant ni un dossier valide"
+        )
+
+    print(f"Fichiers trouvés : {files}")
+    isUniqueFile = len(files) == 1
+    for file_path in files:
+        print(f"Traitement du fichier : {file_path}")
+        df = get_dataframe_cleaned_new(file_path, delimiter=config.get("delimiter"))
+
+        # Appliquer le filtre sur la colonne pour la valeur souhaitée
+        filter_config = config.get("FILTER", {})
+        df = apply_filter(df, filter_config)
+        if filter_config.get("column") and filter_config.get("value") is not None:
+            logging.info(
+                f"Filtre appliqué : {filter_config.get('column')} = {filter_config.get('value')} ({len(df)} lignes restantes)"
+            )
+
+        print(df.columns)
+        # ponytail: strptime runs on the whole file; polars is fast, re-sample only if it proves too slow
+        df = convert_datetime_columns(
+            df, threshold=config.get("datetime_detection_threshold", 1.0)
+        )
+        types_dict = postgrestype_dict(df)
+        print(types_dict)
+
+        if isUniqueFile:
+            table_name = config["table"]
+        else:
+            table_name = os.path.splitext(os.path.basename(file_path))[0]
+        create_table_from_csv(engine, df, table_name, types_dict)
+        print(f"Table {table_name} créée")
+        # Drop rows where all values are null (polars equivalent)
+        df = df.filter(~pl.all_horizontal(pl.all().is_null()))
+        overwrite_table_with_csv_data(
+            engine, df, table_name, batch_size=config.get("batch_size", 10000)
+        )
+        print(f"Données du fichier {file_path} chargées dans la table {table_name}")

csv_to_db_py/db_connectors/__init__.py

@@ -0,0 +1,5 @@
+# Expose les connecteurs pour un import simple
+from .postgres import PostgresConnector as PostgresConnector
+from .mysql import MySQLConnector as MySQLConnector
+from .mssql import MSSQLConnector as MSSQLConnector
+from .oracle import OracleConnector as OracleConnector

csv_to_db_py/db_connectors/base.py

@@ -0,0 +1,14 @@
+from abc import ABC, abstractmethod
+
+
+class BaseDBConnector(ABC):
+    def __init__(self, config):
+        self.config = config
+        self.engine = None
+
+    @abstractmethod
+    def connect(self):
+        pass
+
+    def get_engine(self):
+        return self.engine

csv_to_db_py/db_connectors/mssql.py

@@ -0,0 +1,17 @@
+from sqlalchemy import URL, create_engine
+from csv_to_db_py.db_connectors.base import BaseDBConnector
+
+
+class MSSQLConnector(BaseDBConnector):
+    def connect(self):
+        cfg = self.config
+        url = URL.create(
+            "mssql+pyodbc",
+            username=cfg["user"],
+            password=cfg["password"],
+            host=cfg["host"],
+            port=cfg["port"],
+            database=cfg["database"],
+            query={"driver": "ODBC Driver 17 for SQL Server"},
+        )
+        self.engine = create_engine(url)

csv_to_db_py/db_connectors/mysql.py

@@ -0,0 +1,16 @@
+from sqlalchemy import URL, create_engine
+from csv_to_db_py.db_connectors.base import BaseDBConnector
+
+
+class MySQLConnector(BaseDBConnector):
+    def connect(self):
+        cfg = self.config
+        url = URL.create(
+            "mysql+pymysql",
+            username=cfg["user"],
+            password=cfg["password"],
+            host=cfg["host"],
+            port=cfg["port"],
+            database=cfg["database"],
+        )
+        self.engine = create_engine(url)

csv_to_db_py/db_connectors/oracle.py

@@ -0,0 +1,17 @@
+from sqlalchemy import URL, create_engine
+from csv_to_db_py.db_connectors.base import BaseDBConnector
+
+
+class OracleConnector(BaseDBConnector):
+    def connect(self):
+        cfg = self.config
+        # DSN Oracle: host:port/service_name (ex: localhost:1521/XEPDB1)
+        url = URL.create(
+            "oracle+oracledb",
+            username=cfg["user"],
+            password=cfg["password"],
+            host=cfg["host"],
+            port=cfg["port"],
+            database=cfg["database"],
+        )
+        self.engine = create_engine(url)

csv_to_db_py/db_connectors/postgres.py

@@ -0,0 +1,17 @@
+from sqlalchemy import URL, create_engine
+from csv_to_db_py.db_connectors.base import BaseDBConnector
+
+
+class PostgresConnector(BaseDBConnector):
+    def connect(self):
+        cfg = self.config
+        # url = f"postgresql+psycopg2://{cfg['user']}:{cfg['password']}@{cfg['host']}:{cfg['port']}/{cfg['database']}"
+        url = URL.create(
+            "postgresql+psycopg2",
+            username=cfg["user"],
+            password=cfg["password"],
+            host=cfg["host"],
+            port=cfg["port"],
+            database=cfg["database"],
+        )
+        self.engine = create_engine(url)

csv_to_db_py/logger.py

@@ -0,0 +1,19 @@
+import logging
+from rich.logging import RichHandler
+
+
+def set_logger(level: str):
+    logging.basicConfig(
+        format="%(message)s",
+        level=level.upper(),
+        handlers=[RichHandler()],
+    )
+    logger = logging.getLogger()
+    return logger
+
+
+def get_logger():
+    return logging.getLogger(__name__)
+
+
+logger = get_logger()

csv_to_db_py/main.py

@@ -0,0 +1,32 @@
+import argparse
+import sys
+
+from csv_to_db_py.config import load_config
+from csv_to_db_py.core import load_csv_to_db
+from csv_to_db_py.logger import set_logger
+
+
+def parse_args(argv) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        prog="csv-to-db",
+        description="Load CSV/XLSX files into a database table",
+    )
+    parser.add_argument("input_path", nargs="?", default=None)
+    parser.add_argument("--config", default="mainConfig.json")
+    parser.add_argument(
+        "--log-level",
+        default="info",
+        choices=["debug", "info", "warning", "error"],
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv=None) -> None:
+    args = parse_args(sys.argv[1:] if argv is None else argv)
+    set_logger(args.log_level)
+    cfg = load_config(args.config)
+    load_csv_to_db(cfg, args.input_path)
+
+
+if __name__ == "__main__":
+    main()

csv_to_db_py/utils/__init__.py

@@ -0,0 +1,7 @@
+# Expose les fonctions utiles si tu veux les importer facilement
+from .table_creator import create_table_from_csv as create_table_from_csv
+from .csv_loader import overwrite_table_with_csv_data as overwrite_table_with_csv_data
+from .cleaned_csv import (
+    get_dataframe_cleaned_new as get_dataframe_cleaned_new,
+    normalize_column as normalize_column,
+)

csv_to_db_py/utils/cleaned_csv.py

@@ -0,0 +1,105 @@
+from collections import Counter
+import polars as pl
+import re
+from csv_to_db_py.config import PG_RESERVED
+from csv_to_db_py.logger import logger
+
+import csv
+
+
+def detect_delimiter(file_path):
+    with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+        sample = f.read(2048)
+        try:
+            delimiter = csv.Sniffer().sniff(sample).delimiter
+        except Exception:
+            delimiter = ";"  # fallback si détection impossible : tu peux mettre "," si tu préfères
+    return delimiter
+
+
+def quote_identifier(name: str) -> str:
+    return '"' + name.replace('"', '""') + '"'
+
+
+def normalize_column(name: str) -> str:
+    reserved_word = Counter()
+    name = name.strip().lower()  # lowercase + trim
+    name = re.sub(r"[^0-9a-zA-Z_]+", "_", name)  # replace non-alphanum with "_"
+    name = re.sub(r"_+", "_", name)  # collapse multiple "_"
+    name = name.strip("_")  # remove leading/trailing "_"
+
+    # si le nom est un mot réservé SQL → ajouter suffixe incrémental
+    if name in PG_RESERVED:
+        reserved_word[name] += 1
+        name = f"{name}_{reserved_word[name]}"
+
+    return name
+
+
+def get_dataframe_cleaned_new(file_path, delimiter=None):
+    # --- 1) Lecture du fichier selon l'extension ---
+    if file_path.endswith(".xlsx"):
+        print(f"Lecture Excel : {file_path}")
+        df = pl.read_excel(file_path)
+
+    elif file_path.endswith(".csv"):
+        print(f"Lecture CSV : {file_path}")
+
+        if delimiter is None:
+            delimiter = detect_delimiter(file_path)
+        print(f"Délimiteur utilisé pour {file_path}: '{delimiter}'")
+
+        # Essayer UTF-8, sinon fallback Latin-1
+        try:
+            df = pl.read_csv(file_path, separator=delimiter, encoding="utf-8")
+        except UnicodeDecodeError:
+            print("[ENCODING] UTF-8 échoué, tentative avec Latin-1...")
+            df = pl.read_csv(file_path, separator=delimiter, encoding="latin1")
+
+    else:
+        raise ValueError(f"Format non supporté : {file_path}")
+
+    # --- 2) Nettoyage des colonnes ---
+    new_columns = [
+        col.replace(" ", "").replace(".", "").replace("/", "") for col in df.columns
+    ]
+    new_columns = normalize_and_dedup_columns(new_columns)
+    df = df.rename(dict(zip(df.columns, new_columns)))
+
+    print(df)
+    return df
+
+
+def apply_filter(df: pl.DataFrame, filter_config: dict) -> pl.DataFrame:
+    filter_col = filter_config.get("column")
+    filter_val = filter_config.get("value")
+    if not filter_col or filter_val is None:
+        return df
+    normalized_col = normalize_column(filter_col)
+    if normalized_col not in df.columns:
+        logger.warning(
+            f"Filtre ignoré : colonne '{filter_col}' absente après normalisation."
+        )
+        return df
+    col = df[normalized_col]
+    try:
+        cast_val = pl.lit(filter_val).cast(col.dtype)
+        return df.filter(col == cast_val)
+    except Exception:
+        logger.warning(
+            f"Filtre ignoré : valeur '{filter_val}' non convertible pour la colonne '{filter_col}'."
+        )
+        return df
+
+
+def normalize_and_dedup_columns(columns):
+    seen = Counter()
+    new_cols = []
+    for col in columns:
+        normed = normalize_column(col)  # ta fonction de normalisation
+        seen[normed] += 1
+        if seen[normed] == 1:
+            new_cols.append(normed)
+        else:
+            new_cols.append(f"{normed}_{seen[normed]}")
+    return new_cols

csv_to_db_py/utils/csv_loader.py

@@ -0,0 +1,265 @@
+import polars as pl
+from sqlalchemy import inspect, text
+from csv_to_db_py.logger import logger
+from csv_to_db_py.utils.cleaned_csv import quote_identifier
+
+
+def _truncate_if_exists(engine, table_name, expected_columns):
+    """TRUNCATE if table exists → preserve schema & dependent views. Returns True if truncated."""
+    inspector = inspect(engine)
+    if not inspector.has_table(table_name):
+        return False
+    db_columns = [column["name"] for column in inspector.get_columns(table_name)]
+    if sorted(db_columns) != sorted(expected_columns):
+        # ponytail: column-name drift is guarded; TYPE drift is not (still fails loudly after truncate)
+        raise ValueError(
+            f"Schema drift on table {table_name!r}: existing columns {sorted(db_columns)} "
+            f"differ from CSV columns {sorted(expected_columns)}. "
+            "DROP TABLE ... CASCADE and recreate dependent views, or align the CSV columns."
+        )
+    with engine.begin() as conn:
+        conn.execute(text(f"TRUNCATE TABLE {quote_identifier(table_name)}"))
+    return True
+
+
+def map_polars_dtype_to_postgresql(series):
+    dtype = series.dtype
+    dtype_str = str(dtype)
+
+    if "Int" in dtype_str:
+        return "INTEGER"
+    elif "Float" in dtype_str:
+        # Check if all non-null values are integers
+        non_null = series.drop_nulls()
+        if len(non_null) > 0 and (non_null == non_null.cast(pl.Int64)).all():
+            return "INTEGER"
+        else:
+            return "FLOAT"
+    elif "Boolean" in dtype_str or "Bool" in dtype_str:
+        return "BOOLEAN"
+    elif "Datetime" in dtype_str or "Date" in dtype_str:
+        return "TIMESTAMP"
+    elif "Utf8" in dtype_str or "String" in dtype_str:
+        return "VARCHAR"
+    else:
+        return "VARCHAR"
+
+
+def detect_and_parse_datetime_column(dataframe, column_name, threshold=1.0):
+    """
+    Détecte automatiquement si une colonne string contient des dates et trouve le format approprié.
+    Utilise l'inférence automatique de Polars qui supporte de nombreux formats.
+
+    Logique de validation:
+    - Filtre les nulls et chaînes vides (valeurs non significatives pour l'analyse)
+    - Parmi les valeurs "utiles" restantes, au moins `datetime_detection_threshold` doivent être des dates valides
+    - Les valeurs invalides sont converties en null automatiquement (strict=False)
+    - Préserve tous les nulls et chaînes vides originaux
+    """
+    column = dataframe[column_name]
+    dtype_str = str(column.dtype)
+
+    # Only process string columns
+    if "Utf8" not in dtype_str and "String" not in dtype_str:
+        return dataframe
+
+    # Filter out nulls AND empty strings from original data
+    # Ces valeurs ne sont pas significatives pour déterminer si c'est une colonne de dates
+    useful_values = column.filter(
+        (column.is_not_null()) & (column.str.strip_chars() != "")
+    )
+
+    num_useful = len(useful_values)
+    if num_useful == 0:
+        return dataframe
+
+    # Seuil configurable pour la détection de dates (défaut: 1.0 = 100%, désactivé)
+    MIN_SUCCESS_RATE = threshold
+
+    # Try automatic datetime parsing with Polars (supports many formats automatically)
+    try:
+        # Polars can auto-detect many datetime formats
+        # strict=False means invalid values become null instead of raising errors
+        parsed = dataframe.with_columns(
+            pl.col(column_name).str.to_datetime(strict=False)
+        )
+
+        # Count how many useful values were successfully parsed as dates
+        # On compte les dates parsées DANS LE DATAFRAME ORIGINAL (pour exclure les nulls initiaux)
+        parsed_col = parsed[column_name]
+
+        # On doit compter combien de valeurs utiles ont été converties en dates
+        # Pour cela, on regarde la colonne parsée aux mêmes positions que les valeurs utiles
+        original_indices = (
+            column.to_frame()
+            .with_row_count("idx")
+            .filter((column.is_not_null()) & (column.str.strip_chars() != ""))["idx"]
+        )
+
+        parsed_useful = parsed_col.gather(original_indices)
+        parsed_useful = parsed_useful.filter(parsed_useful.is_not_null())
+        num_parsed = len(parsed_useful)
+
+        success_rate = num_parsed / num_useful
+
+        # Si au moins la part configurée des valeurs utiles est parsée, on convertit
+        if success_rate >= MIN_SUCCESS_RATE:
+            num_invalid = num_useful - num_parsed
+            logger.info(
+                f"Colonne '{column_name}' détectée comme datetime (format auto-détecté) - "
+                f"{num_parsed}/{num_useful} valeurs utiles parsées ({success_rate:.1%}), "
+                f"{num_invalid} valeur(s) invalide(s) convertie(s) en null"
+            )
+            return parsed
+    except Exception as e:
+        logger.debug(f"Auto-détection datetime échouée pour '{column_name}': {e}")
+
+    # Try common date formats manually if auto-detection fails
+    common_formats = [
+        # ISO formats (YYYY-MM-DD)
+        "%Y-%m-%d",
+        "%Y-%m-%d %H:%M:%S",
+        "%Y-%m-%dT%H:%M:%S",
+        "%Y-%m-%dT%H:%M:%S.%f",
+        # French/European formats (DD/MM/YYYY)
+        "%d/%m/%Y",
+        "%d/%m/%Y %H:%M:%S",
+        "%d/%m/%y",
+        # American formats (MM/DD/YYYY) - with AM/PM support
+        "%m/%d/%Y",
+        "%m/%d/%Y %H:%M:%S",
+        "%m/%d/%Y %I:%M:%S %p",  # 12-hour format with AM/PM
+        # European dot format (DD.MM.YYYY)
+        "%d.%m.%Y",
+        "%d.%m.%Y %H:%M:%S",
+        # Other formats
+        "%d-%m-%Y",
+        "%d-%m-%Y %H:%M:%S",
+        "%d-%m-%y",
+        "%Y/%m/%d",
+        "%Y/%m/%d %H:%M:%S",
+    ]
+
+    for date_format in common_formats:
+        try:
+            # strict=False: invalid values become null
+            parsed = dataframe.with_columns(
+                pl.col(column_name).str.strptime(
+                    pl.Datetime, format=date_format, strict=False
+                )
+            )
+
+            # Same logic as above
+            parsed_col = parsed[column_name]
+            original_indices = (
+                column.to_frame()
+                .with_row_count("idx")
+                .filter((column.is_not_null()) & (column.str.strip_chars() != ""))[
+                    "idx"
+                ]
+            )
+
+            parsed_useful = parsed_col.gather(original_indices)
+            parsed_useful = parsed_useful.filter(parsed_useful.is_not_null())
+            num_parsed = len(parsed_useful)
+            success_rate = num_parsed / num_useful
+
+            # Si au moins la part configurée des valeurs utiles est parsée, on accepte
+            if success_rate >= MIN_SUCCESS_RATE:
+                num_invalid = num_useful - num_parsed
+                logger.info(
+                    f"Colonne '{column_name}' détectée comme datetime (format: {date_format}) - "
+                    f"{num_parsed}/{num_useful} valeurs utiles parsées ({success_rate:.1%}), "
+                    f"{num_invalid} valeur(s) invalide(s) convertie(s) en null"
+                )
+                return parsed
+        except Exception:
+            continue
+
+    # If no format worked, return original dataframe
+    # Les nulls et valeurs vides sont préservés dans tous les cas
+    return dataframe
+
+
+def convert_datetime_columns(dataframe, threshold=1.0):
+    """Convert every string column that looks like a date, using the given detection threshold."""
+    for column_name in dataframe.columns:
+        dataframe = detect_and_parse_datetime_column(
+            dataframe, column_name, threshold=threshold
+        )
+    return dataframe
+
+
+def postgrestype_dict(dataframe, sample_size=10000):
+    """
+    Détermine les types PostgreSQL pour chaque colonne du DataFrame.
+
+    Args:
+        dataframe: DataFrame Polars
+        sample_size: Taille maximale de l'échantillon pour l'analyse des types.
+                     Si le DataFrame est plus grand, un échantillon aléatoire est utilisé.
+
+    Returns:
+        dict: Mapping {nom_colonne: type_postgresql}
+    """
+    column_types = {}
+
+    # Use sample if dataframe is larger than sample_size
+    num_rows = len(dataframe)
+    if num_rows > sample_size:
+        logger.info(
+            f"DataFrame contient {num_rows} lignes. Utilisation d'un échantillon aléatoire "
+            f"de {sample_size} lignes pour la détection des types."
+        )
+        df_sample = dataframe.sample(n=sample_size, seed=42)
+    else:
+        df_sample = dataframe
+
+    for column_name in df_sample.columns:
+        column_types[column_name] = map_polars_dtype_to_postgresql(
+            df_sample[column_name]
+        )
+
+    return column_types
+
+
+def overwrite_table_with_csv_data(engine, csv_data, table_name, batch_size=10000):
+    # Convert polars DataFrame to pandas for SQLAlchemy compatibility
+    pandas_data = csv_data.to_pandas()
+
+    total_rows = len(pandas_data)
+
+    # Typed table always pre-created (create_table_from_csv raises on failure); to_sql only appends.
+    truncated = _truncate_if_exists(engine, table_name, csv_data.columns)
+
+    if total_rows <= batch_size:
+        pandas_data.to_sql(name=table_name, con=engine, if_exists="append", index=False)
+        logger.info(
+            f"Table '{table_name}' {'vidée puis remplie' if truncated else 'créée'} "
+            f"avec {total_rows} lignes (insertion unique)"
+        )
+    else:
+        # Multi-batch insertion
+        num_batches = (total_rows + batch_size - 1) // batch_size  # Ceiling division
+        logger.info(
+            f"Insertion de {total_rows} lignes dans '{table_name}' par batch de {batch_size} "
+            f"({num_batches} batch(s))"
+        )
+
+        for i in range(0, total_rows, batch_size):
+            batch_num = (i // batch_size) + 1
+            end_idx = min(i + batch_size, total_rows)
+            batch_data = pandas_data.iloc[i:end_idx]
+
+            batch_data.to_sql(
+                name=table_name, con=engine, if_exists="append", index=False
+            )
+
+            logger.info(
+                f"  Batch {batch_num}/{num_batches}: {len(batch_data)} lignes insérées "
+                f"(lignes {i + 1} à {end_idx})"
+            )
+
+        logger.info(
+            f"Insertion terminée: {total_rows} lignes insérées dans '{table_name}'"
+        )

csv_to_db_py/utils/table_creator.py

@@ -0,0 +1,19 @@
+from csv_to_db_py.logger import logger
+from csv_to_db_py.utils.cleaned_csv import quote_identifier
+
+
+def create_table_from_csv(engine, dataframe, table, types_dict):
+    # Génération de la requête SQL pour créer la table
+    # query = f"CREATE TABLE {table} ({', '.join([column + ' '+types_dict[column] for column in columns])});"
+    columns = dataframe.columns
+    query = f"""CREATE TABLE IF NOT EXISTS {quote_identifier(table)} ({", ".join([quote_identifier(column) + " " + types_dict[column] for column in columns])});"""
+    with engine.begin() as conn:
+        conn.execute(query)
+
+    logger.info(f"Table '{table}' crée")
+
+    columns = dataframe.columns
+
+    logger.debug(columns)
+
+    logger.debug(query)

csv_to_db_py-0.1.0.dist-info/METADATA

@@ -0,0 +1,276 @@
+Metadata-Version: 2.5
+Name: csv-to-db-py
+Version: 0.1.0
+Summary: Convert CSV file or folder of CSV to DB with table filled with the CSV data
+Author: Karim Ounnoughi, Claude Fauconnet, Juste Bonzene
+License: MIT
+License-File: LICENSE.md
+Requires-Python: >=3.13
+Requires-Dist: fastexcel>=0.20.2
+Requires-Dist: openpyxl<4.0.0,>=3.1.5
+Requires-Dist: oracledb<3.0.0,>=2.0.0
+Requires-Dist: pandas<3.0.0,>=2.3.2
+Requires-Dist: polars>=1.20.0
+Requires-Dist: psycopg2<3.0.0,>=2.9.10
+Requires-Dist: pyarrow<19.0.0,>=18.0.0
+Requires-Dist: pymysql<2.0.0,>=1.1.0
+Requires-Dist: pyodbc<6.0.0,>=5.0.0
+Requires-Dist: requests<3.0.0,>=2.32.5
+Requires-Dist: rich<15.0.0,>=14.1.0
+Requires-Dist: sqlalchemy<3.0.0,>=2.0.43
+Description-Content-Type: text/markdown
+
+# CSV to DB
+
+Convert table files or folders to PostgreSQL database tables.The database need to already exists. It have a feature of autodection of types.
+ -dates (multiples formats cited on tests parts)
+ -int
+ -float
+ -string 
+ 
+
+## Prerequisites
+
+- Python 3.13+
+- UV (https://docs.astral.sh/uv/getting-started/installation/)
+- PostgreSQL database
+- files with CSV or XLSX format
+- MySQL/MSSQL/Oracle connections rely on bundled drivers: `pymysql`, `pyodbc`, `oracledb`
+- `pyodbc` additionally requires an ODBC driver; `oracledb` runs in thin mode (no Oracle client needed)
+
+
+
+## Installation
+
+```bash
+git clone https://github.com/ounnoughikarim/CSV_TO_DB_PY.git
+cd CSV_TO_DB_PY
+uv sync
+```
+
+## Install from PyPI
+
+The package is published to PyPI and can be installed with:
+
+```bash
+uv add csv-to-db-py
+```
+
+or with pip:
+
+```bash
+pip install csv-to-db-py
+```
+
+## Publishing
+
+To publish a new release to PyPI, push a tag matching the version in `pyproject.toml`:
+
+```bash
+git tag 0.1.0
+git push origin 0.1.0
+```
+
+The `publish` GitHub Actions workflow builds the package and publishes it to PyPI via trusted publishing.
+
+## Configuration
+
+Create `mainConfig.json` from the template:
+
+```json
+{
+    "DATABASE_CONFIG": {
+        "type": "postgres",
+        "host": "hostname",
+        "port": 5432,
+        "user": "postgres",
+        "password": "your_password",
+        "database": "your_database"
+    },
+    "files_dir": "C:\\path\\to\\your\\file",
+    "table": "table_name",
+    "delimiter": ";",
+    "datetime_detection_threshold": 1.0,
+    "batch_size": 100000
+}
+```
+
+### Configuration Parameters
+
+- `DATABASE_CONFIG`: Database connection settings
+  - `type`: Database type (postgres, mysql, mssql, oracle)
+  - `host`: Database hostname
+  - `port`: Database port
+  - `user`: Database username
+  - `password`: Database password
+  - `database`: Database name
+- `files_dir`: Path to a file or folder containing multiple files (CSV or XLSX)
+- `table`: Table name (optional - uses filename if not specified for single file)
+- `delimiter`: CSV delimiter character (e.g., ";", ",", "\t")
+- `datetime_detection_threshold`: *(optional)* Threshold for automatic datetime detection
+  - Range: `0.0` to `1.0` (0% to 100%)
+  - Default: `1.0` (100%, strict - only perfect matches)
+  - Recommended: `0.7` (70% threshold for flexible detection)
+  - A column is detected as datetime if at least X% of non-null, non-empty values can be parsed as valid dates
+  - Invalid values are automatically converted to NULL
+  - **Examples:**
+    - `1.0`: Only columns with 100% valid dates are detected (safest, default)
+    - `0.9`: Allows up to 10% invalid date values
+    - `0.7`: Allows up to 30% invalid date values (recommended for real-world data)
+- `batch_size`: *(optional)* Number of rows to insert per batch for large files
+  - Default: `10000` (10k rows per batch)
+  - Reduces memory usage for large datasets
+  - SQLAlchemy can struggle with very large files, batching prevents memory issues
+  - **Recommended values:**
+    - `100000`: Good for most cases (files up to several GB)
+    - `50000`: For memory-constrained environments
+    - `10000`: Default (10k rows per batch), good balance of speed and memory
+    - `500000`: For high-memory systems and faster insertions
+
+## Usage
+
+```bash
+uv run csv-to-db --config mainConfig.json
+```
+
+With custom log level:
+```bash
+uv run csv-to-db --config mainConfig.json --log-level info
+```
+
+## Testing
+
+Run the test suite with pytest:
+```bash
+uv run pytest
+```
+
+This suite tests automatic datetime detection with various date formats, using [tests/date_formats_test.csv](tests/date_formats_test.csv) containing 14 columns:
+
+**Date columns (should be detected as datetime):**
+- **FR_date**: French format (DD/MM/YYYY)
+- **FR_datetime**: French format with time (DD/MM/YYYY HH:MM:SS)
+- **ISO_date**: ISO 8601 date (YYYY-MM-DD)
+- **ISO_datetime**: ISO 8601 with time (YYYY-MM-DDTHH:MM:SS)
+- **USA_date**: American format (MM/DD/YYYY)
+- **USA_datetime**: American format with 12-hour time (MM/DD/YYYY hh:mm:ss AM/PM)
+- **EUR_date**: European format (DD.MM.YYYY)
+- **EUR_datetime**: European format with time (DD.MM.YYYY HH:MM:SS)
+- **ISO_time**: ISO format with space separator (YYYY-MM-DD HH:MM:SS)
+
+**Non-date columns (should NOT be detected as datetime):**
+- **int_column**: Integer values
+- **string_column**: Pure text strings
+- **mixed_column**: Mix of text with 1-2 dates (should not be detected as datetime)
+- **float_column**: Floating point numbers
+- **bool_column**: Boolean values
+
+The test demonstrates automatic format detection, handling of null values, conversion of invalid values to null, and proper rejection of non-date columns.
+
+### Batch Insertion for Large Files
+
+For large datasets, the tool automatically splits data insertion into batches to prevent memory issues:
+
+**How it works:**
+- If total rows ≤ `batch_size`: Single insertion (fastest)
+- If total rows > `batch_size`: Multiple insertions in batches
+
+**Example with 250,000 rows and `batch_size: 100000`:**
+```
+Pre-step: TRUNCATE table if it already exists (preserves schema & dependent views)
+Batch 1: Rows 1 to 100,000 (100,000 rows) - APPEND
+Batch 2: Rows 100,001 to 200,000 (100,000 rows) - APPEND
+Batch 3: Rows 200,001 to 250,000 (50,000 rows) - APPEND
+```
+
+**Benefits:**
+- ✅ Prevents memory overflow with large files
+- ✅ Better progress tracking through logs
+- ✅ Allows processing files larger than available RAM
+- ✅ Automatic recovery possible (first batch creates table structure)
+
+### Table Overwrite Strategy
+
+When a table already exists, the tool **does not DROP** it. Instead it runs `TRUNCATE` then appends the new data.
+
+**Why:**
+
+- Dropping a table fails if a view (or other object) depends on it.
+- TRUNCATE empties the rows while preserving the table schema and any dependent views/constraints.
+
+**Behavior:**
+
+- Table does not exist → pre-created by `create_table_from_csv` with detected types.
+- Table exists → `TRUNCATE TABLE "<name>"` then `INSERT` (append mode). If the existing table's columns differ from the CSV (schema drift), the run aborts with a clear error **before truncating**, so the old data is preserved.
+
+**Caveat — schema drift:**
+
+- TRUNCATE keeps the **existing** column definitions. Column-name drift (added/removed/renamed columns) is detected and blocks the overwrite before any data is lost. Type drift is not detected and still fails loudly after truncate.
+- To load a CSV whose schema differs, manually `DROP TABLE ... CASCADE` (and recreate dependent views) or alter the table to match the new schema.
+
+**Performance tips:**
+- Larger batch sizes = faster insertion but more memory usage
+- Smaller batch sizes = slower but safer for limited memory
+- Default 10k is optimal for most use cases
+
+### Type Detection Optimization (Sampling)
+
+For large datasets, type detection automatically uses random sampling to improve performance:
+
+**How it works:**
+- If DataFrame has ≤ 10,000 rows: Full dataset is analyzed
+- If DataFrame has > 10,000 rows: Random sample of 10,000 rows is used
+- Sample uses fixed seed (42) for reproducibility
+- Applies to both type detection and datetime format detection
+
+**Benefits:**
+- ✅ Significantly faster type detection for large files (millions of rows)
+- ✅ Sample size of 10,000 rows provides excellent type accuracy
+- ✅ Reduces processing time without compromising quality
+- ✅ Transparent logging when sampling is used
+
+**Example:**
+```
+DataFrame contient 5,000,000 lignes. Utilisation d'un échantillon aléatoire de 10000 lignes pour la détection des types.
+```
+
+**Note:** This optimization only affects type detection. The full dataset is still loaded and inserted into the database.
+
+### Datetime Detection Behavior
+
+The datetime detection threshold controls how strict the algorithm is when detecting date columns:
+
+**With `datetime_detection_threshold: 1.0` (default, 100%):**
+- Only columns where **ALL** non-null, non-empty values are valid dates will be detected
+- This is the safest option to avoid false positives
+- Use this when you want explicit control over which columns are dates
+
+**With `datetime_detection_threshold: 0.7` (recommended for flexible detection):**
+- Columns where **at least 70%** of non-null, non-empty values are valid dates will be detected
+- Invalid values are automatically converted to NULL
+- Useful when your data might contain occasional typos or invalid date entries
+- Example: A column with 9 valid dates and 1 "invalid" string → detected as datetime
+
+**Example scenarios:**
+```
+Column: ["2024-01-01", "2024-01-02", "invalid", null, ""]
+- 3 non-null, non-empty values total
+- 2 valid dates (66.7%)
+- With threshold 1.0: NOT detected (66.7% < 100%)
+- With threshold 0.7: NOT detected (66.7% < 70%)
+
+Column: ["2024-01-01", "2024-01-02", "2024-01-03", "invalid", null]
+- 4 non-null, non-empty values total
+- 3 valid dates (75%)
+- With threshold 1.0: NOT detected (75% < 100%)
+- With threshold 0.7: DETECTED (75% ≥ 70%), "invalid" becomes NULL
+```
+
+## Development
+
+Check code before committing:
+```bash
+uv run ruff check . --fix
+uv run ruff check .
+uv run ruff format
+```

csv_to_db_py-0.1.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+csv_to_db_py/__init__.py,sha256=wHSdd1uCl2Z9KWUYYOy1SjUIjTiZYkETXaEzf9OYdv0,166
+csv_to_db_py/config.py,sha256=ThLnOjgWFlKQz_AV8e0Ig_bgGjT-BR8BUNLccQ24Bds,986
+csv_to_db_py/core.py,sha256=gyeeENYIqVr2Uh-RKJFKO1BSYsgok_6-s0swIiwOc4o,3326
+csv_to_db_py/logger.py,sha256=WAZ4TyG4_7i9brjeUqkZSKWN9-fvsXXSwf18cL2Ayp0,340
+csv_to_db_py/main.py,sha256=M-uvY-sUVMmT00JmPmriJWJJNuwkE2eHhcDZ6tWXD6I,875
+csv_to_db_py/db_connectors/__init__.py,sha256=9tMXKzesBuar1Vt1ZfviYDxPehZLRc1YrU06YCDcyPA,267
+csv_to_db_py/db_connectors/base.py,sha256=09VWv1-VIVX0_-3rWWqhH4Cw-zIoFiqYq62UHmYwcFM,265
+csv_to_db_py/db_connectors/mssql.py,sha256=_--AcjHDYEB1tpP3H4atA9SAktCeOVZx4WvFU8veRxg,530
+csv_to_db_py/db_connectors/mysql.py,sha256=vdEw5wETA4qv6QmAUibRHNRjPS6eopJ_SiuuJ-EHmko,468
+csv_to_db_py/db_connectors/oracle.py,sha256=h7bKg9rf8xMWutYoeLAM2-9_gyyw7mEgMiD6k_t0rhk,544
+csv_to_db_py/db_connectors/postgres.py,sha256=w8gayoPoOAIvRr86qbvPiB506U8nD1cy5ZP41ASuUFY,596
+csv_to_db_py/utils/__init__.py,sha256=FRBa1lAxuuXDF8ZxaU5BQsnQ3e-jlaqVpid_33_n5Fg,357
+csv_to_db_py/utils/cleaned_csv.py,sha256=13qRbegpXdWMEZ0vncUO37HAy3SwvVbAdt9ojb8foD4,3426
+csv_to_db_py/utils/csv_loader.py,sha256=YMCZZKVW-PoL5wg1XxG8cF5VpxoFZMpctiB1iVUI2LU,10241
+csv_to_db_py/utils/table_creator.py,sha256=O2fszUoGd-6ONGdERUagRPnR0wDA-JsXvnH5pEMaT20,720
+csv_to_db_py-0.1.0.dist-info/METADATA,sha256=PfyE9KWW3fWxxSIMSCESCZteNlZoPbGBY6XMcHKTZVg,9864
+csv_to_db_py-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+csv_to_db_py-0.1.0.dist-info/entry_points.txt,sha256=5XTyokDAkfbkSQM0uIfW1IkKfHM6lKHcthbNTVMBGiQ,53
+csv_to_db_py-0.1.0.dist-info/licenses/LICENSE.md,sha256=xrhxfiSLPm17ApyK9kdDxAMHUYSN1zw-2JRkxvrp2qo,1073
+csv_to_db_py-0.1.0.dist-info/RECORD,,

csv_to_db_py-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

csv_to_db_py-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+csv-to-db = csv_to_db_py.main:main

csv_to_db_py-0.1.0.dist-info/licenses/LICENSE.md

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2025 OUNNOUGHI KARIM
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+