csv-detective 0.8.1.dev1460__py3-none-any.whl → 0.8.1.dev1482__py3-none-any.whl

csv_detective/detection/columns.py

@@ -0,0 +1,89 @@
+import logging
+from typing import TextIO
+from time import time
+
+from csv_detective.utils import display_logs_depending_process_time
+
+
+def detect_extra_columns(file: TextIO, sep: str):
+    """regarde s'il y a des colonnes en trop
+    Attention, file ne doit pas avoir de ligne vide"""
+    file.seek(0)
+    retour = False
+    nb_useless_col = 99999
+
+    for i in range(10):
+        line = file.readline()
+        # regarde si on a un retour
+        if retour:
+            assert line[-1] == "\n"
+        if line[-1] == "\n":
+            retour = True
+
+        # regarde le nombre de derniere colonne inutile
+        deb = 0 + retour
+        line = line[::-1][deb:]
+        k = 0
+        for sign in line:
+            if sign != sep:
+                break
+            k += 1
+        if k == 0:
+            return 0, retour
+        nb_useless_col = min(k, nb_useless_col)
+    return nb_useless_col, retour
+
+
+def detect_heading_columns(file: TextIO, sep: str, verbose: bool = False) -> int:
+    """Tests first 10 lines to see if there are empty heading columns"""
+    if verbose:
+        start = time()
+        logging.info("Detecting heading columns")
+    file.seek(0)
+    return_int = float("Inf")
+    for i in range(10):
+        line = file.readline()
+        return_int = min(return_int, len(line) - len(line.strip(sep)))
+        if return_int == 0:
+            if verbose:
+                display_logs_depending_process_time(
+                    f'No heading column detected in {round(time() - start, 3)}s',
+                    time() - start,
+                )
+            return 0
+    if verbose:
+        display_logs_depending_process_time(
+            f'{return_int} heading columns detected in {round(time() - start, 3)}s',
+            time() - start,
+        )
+    return return_int
+
+
+def detect_trailing_columns(file: TextIO, sep: str, heading_columns: int, verbose: bool = False) -> int:
+    """Tests first 10 lines to see if there are empty trailing columns"""
+    if verbose:
+        start = time()
+        logging.info("Detecting trailing columns")
+    file.seek(0)
+    return_int = float("Inf")
+    for i in range(10):
+        line = file.readline()
+        return_int = min(
+            return_int,
+            len(line.replace("\n", ""))
+            - len(line.replace("\n", "").strip(sep))
+            - heading_columns,
+        )
+        if return_int == 0:
+            if verbose:
+                display_logs_depending_process_time(
+                    f'No trailing column detected in {round(time() - start, 3)}s',
+                    time() - start,
+                )
+            return 0
+    if verbose:
+        display_logs_depending_process_time(
+            f'{return_int} trailing columns detected in {round(time() - start, 3)}s',
+            time() - start,
+        )
+    return return_int

csv_detective/detection/encoding.py

@@ -0,0 +1,27 @@
+import logging
+from time import time
+from io import BytesIO
+
+from cchardet import detect
+
+from csv_detective.utils import display_logs_depending_process_time
+
+
+def detect_encoding(binary_file: BytesIO, verbose: bool = False) -> str:
+    """
+    Detects file encoding using faust-cchardet (forked from the original cchardet)
+    """
+    if verbose:
+        start = time()
+        logging.info("Detecting encoding")
+    encoding_dict = detect(binary_file.read())
+    if not encoding_dict["encoding"]:
+        raise ValueError("Could not detect the file's encoding. Consider specifying it in the routine call.")
+    if verbose:
+        message = f'Detected encoding: "{encoding_dict["encoding"]}"'
+        message += f' in {round(time() - start, 3)}s (confidence: {round(encoding_dict["confidence"]*100)}%)'
+        display_logs_depending_process_time(
+            message,
+            time() - start,
+        )
+    return encoding_dict['encoding']

csv_detective/detection/engine.py

@@ -0,0 +1,46 @@
+from time import time
+from typing import Optional
+
+import magic
+import requests
+
+from csv_detective.utils import display_logs_depending_process_time, is_url
+
+COMPRESSION_ENGINES = ["gzip"]
+EXCEL_ENGINES = ["openpyxl", "xlrd", "odf"]
+engine_to_file = {
+    "openpyxl": "Excel",
+    "xlrd": "old Excel",
+    "odf": "OpenOffice",
+    "gzip": "csv.gz",
+}
+
+
+def detect_engine(file_path: str, verbose=False) -> Optional[str]:
+    if verbose:
+        start = time()
+    mapping = {
+        "application/gzip": "gzip",
+        "application/x-gzip": "gzip",
+        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'openpyxl',
+        'application/vnd.ms-excel': 'xlrd',
+        'application/vnd.oasis.opendocument.spreadsheet': 'odf',
+        # all these files could be recognized as zip, may need to check all cases then
+        'application/zip': 'openpyxl',
+    }
+    # if none of the above, we move forwards with the csv process
+    if is_url(file_path):
+        remote_content = requests.get(file_path).content
+        engine = mapping.get(magic.from_buffer(remote_content, mime=True))
+    else:
+        engine = mapping.get(magic.from_file(file_path, mime=True))
+    if verbose:
+        message = (
+            f"File is not csv, detected {engine_to_file.get(engine, 'csv')}"
+            if engine else "Processing the file as a csv"
+        )
+        display_logs_depending_process_time(
+            message,
+            time() - start,
+        )
+    return engine

csv_detective/detection/formats.py

@@ -0,0 +1,170 @@
+from collections import defaultdict
+import logging
+from typing import Union
+
+import numpy as np
+import pandas as pd
+from csv_detective.detection.variables import (
+    detect_categorical_variable,
+    # detect_continuous_variable,
+)
+from csv_detective.load_tests import return_all_tests
+from csv_detective.output.utils import prepare_output_dict
+from csv_detective.parsing.columns import test_col, test_label, MAX_ROWS_ANALYSIS
+from csv_detective.validate import validate
+
+
+def detect_formats(
+    table: pd.DataFrame,
+    analysis: dict,
+    file_path: str,
+    user_input_tests: Union[str, list[str]] = "ALL",
+    limited_output: bool = True,
+    skipna: bool = True,
+    verbose: bool = False,
+):
+    on_sample = len(table) > MAX_ROWS_ANALYSIS
+    if on_sample:
+        if verbose:
+            logging.warning(f"File is too long, analysing the {MAX_ROWS_ANALYSIS} first rows")
+        table = table.sample(n=MAX_ROWS_ANALYSIS, random_state=1)
+
+    if table.empty:
+        res_categorical = []
+        # res_continuous = []
+    else:
+        # Detects columns that are categorical
+        res_categorical, categorical_mask = detect_categorical_variable(table, verbose=verbose)
+        res_categorical = list(res_categorical)
+        # Detect columns that are continuous (we already know the categorical) :
+        # we don't need this for now, cuts processing time
+        # res_continuous = list(
+        #     detect_continuous_variable(table.iloc[:, ~categorical_mask.values], verbose=verbose)
+        # )
+
+    analysis.update({
+        "categorical": res_categorical,
+        # "continuous": res_continuous,
+    })
+
+    # list testing to be performed
+    all_tests_fields = return_all_tests(
+        user_input_tests, detect_type="detect_fields"
+    )  # list all tests for the fields
+    all_tests_labels = return_all_tests(
+        user_input_tests, detect_type="detect_labels"
+    )  # list all tests for the labels
+
+    # if no testing then return
+    if not all_tests_fields and not all_tests_labels:
+        return analysis
+
+    # Perform testing on fields
+    scores_table_fields = test_col(table, all_tests_fields, limited_output, skipna=skipna, verbose=verbose)
+    analysis["columns_fields"] = prepare_output_dict(scores_table_fields, limited_output)
+
+    # Perform testing on labels
+    scores_table_labels = test_label(table, all_tests_labels, limited_output, verbose=verbose)
+    analysis["columns_labels"] = prepare_output_dict(scores_table_labels, limited_output)
+
+    # Multiply the results of the fields by 1 + 0.5 * the results of the labels.
+    # This is because the fields are more important than the labels and yields a max
+    # of 1.5 for the final score.
+    scores_table = scores_table_fields * (
+        1
+        + scores_table_labels.reindex(
+            index=scores_table_fields.index, fill_value=0
+        ).values / 2
+    )
+
+    # To reduce false positives: ensure these formats are detected only if the label yields
+    # a detection (skipping the ones that have been excluded by the users).
+    formats_with_mandatory_label = [
+        f for f in [
+            "code_departement",
+            "code_commune_insee",
+            "code_postal",
+            "latitude_wgs",
+            "longitude_wgs",
+            "latitude_wgs_fr_metropole",
+            "longitude_wgs_fr_metropole",
+            "latitude_l93",
+            "longitude_l93",
+        ] if f in scores_table.index
+    ]
+    scores_table.loc[formats_with_mandatory_label, :] = np.where(
+        scores_table_labels.loc[formats_with_mandatory_label, :],
+        scores_table.loc[formats_with_mandatory_label, :],
+        0,
+    )
+    analysis["columns"] = prepare_output_dict(scores_table, limited_output)
+
+    metier_to_python_type = {
+        "booleen": "bool",
+        "int": "int",
+        "float": "float",
+        "string": "string",
+        "json": "json",
+        "json_geojson": "json",
+        "datetime": "datetime",
+        "datetime_iso": "datetime",
+        "datetime_rfc822": "datetime",
+        "date": "date",
+        "latitude": "float",
+        "latitude_l93": "float",
+        "latitude_wgs": "float",
+        "latitude_wgs_fr_metropole": "float",
+        "longitude": "float",
+        "longitude_l93": "float",
+        "longitude_wgs": "float",
+        "longitude_wgs_fr_metropole": "float",
+    }
+
+    if not limited_output:
+        for detection_method in ["columns_fields", "columns_labels", "columns"]:
+            analysis[detection_method] = {
+                col_name: [
+                    {
+                        "python_type": metier_to_python_type.get(
+                            detection["format"], "string"
+                        ),
+                        **detection,
+                    }
+                    for detection in detections
+                ]
+                for col_name, detections in analysis[detection_method].items()
+            }
+    else:
+        for detection_method in ["columns_fields", "columns_labels", "columns"]:
+            analysis[detection_method] = {
+                col_name: {
+                    "python_type": metier_to_python_type.get(
+                        detection["format"], "string"
+                    ),
+                    **detection,
+                }
+                for col_name, detection in analysis[detection_method].items()
+            }
+
+        # Add detection with formats as keys
+        analysis["formats"] = defaultdict(list)
+        for header, col_metadata in analysis["columns"].items():
+            analysis["formats"][col_metadata["format"]].append(header)
+
+    if on_sample:
+        if verbose:
+            logging.warning("Validating that analysis on the sample works on the whole file")
+        is_valid, _, _ = validate(
+            file_path=file_path,
+            previous_analysis=analysis,
+            num_rows=-1,
+            encoding=analysis.get("encoding"),
+            sep=analysis.get("separator"),
+            sheet_name=analysis.get("sheet_name"),
+            verbose=verbose,
+            skipna=skipna,
+        )
+        if not is_valid:
+            raise ValueError("Could not infer detected formats on the whole file")
+
+    return analysis

csv_detective/detection/headers.py

@@ -0,0 +1,32 @@
+import logging
+from time import time
+from typing import Optional, TextIO
+
+from csv_detective.utils import display_logs_depending_process_time
+
+
+def detect_headers(file: TextIO, sep: str, verbose: bool = False) -> tuple[int, Optional[list]]:
+    """Tests 10 first rows for possible header (in case header is not 1st row)"""
+    if verbose:
+        start = time()
+        logging.info("Detecting headers")
+    file.seek(0)
+    for i in range(10):
+        header = file.readline()
+        position = file.tell()
+        chaine = [c for c in header.replace("\n", "").split(sep) if c]
+        if chaine[-1] not in ["", "\n"] and all(
+            [mot not in ["", "\n"] for mot in chaine[1:-1]]
+        ):
+            next_row = file.readline()
+            file.seek(position)
+            if header != next_row:
+                if verbose:
+                    display_logs_depending_process_time(
+                        f'Detected headers in {round(time() - start, 3)}s',
+                        time() - start,
+                    )
+                return i, chaine
+    if verbose:
+        logging.info('No header detected')
+    return 0, None

csv_detective/detection/rows.py

@@ -0,0 +1,18 @@
+import pandas as pd
+
+
+def remove_empty_first_rows(table: pd.DataFrame) -> tuple[pd.DataFrame, int]:
+    """Analog process to detect_headers for csv files, determines how many rows to skip
+    to end up with the header at the right place"""
+    idx = 0
+    if all([str(c).startswith('Unnamed:') for c in table.columns]):
+        # there is on offset between the index in the file (idx here)
+        # and the index in the dataframe, because of the header
+        idx = 1
+        while table.iloc[idx - 1].isna().all():
+            idx += 1
+        cols = table.iloc[idx - 1]
+        table = table.iloc[idx:]
+        table.columns = cols.to_list()
+    # +1 here because the headers should count as a row
+    return table, idx

csv_detective/detection/separator.py

@@ -0,0 +1,44 @@
+import csv
+import logging
+from time import time
+from typing import TextIO
+
+from csv_detective.utils import display_logs_depending_process_time
+
+
+def detect_separator(file: TextIO, verbose: bool = False) -> str:
+    """Detects csv separator"""
+    # TODO: add a robust detection:
+    # si on a un point virgule comme texte et \t comme séparateur, on renvoie
+    # pour l'instant un point virgule
+    if verbose:
+        start = time()
+        logging.info("Detecting separator")
+    file.seek(0)
+    header = file.readline()
+    possible_separators = [";", ",", "|", "\t"]
+    sep_count = dict()
+    for sep in possible_separators:
+        sep_count[sep] = header.count(sep)
+    sep = max(sep_count, key=sep_count.get)
+    # testing that the first 10 (arbitrary) rows all have the same number of fields
+    # as the header. Prevents downstream unwanted behaviour where pandas can load
+    # the file (in a weird way) but the process is irrelevant.
+    file.seek(0)
+    reader = csv.reader(file, delimiter=sep)
+    rows_lengths = set()
+    for idx, row in enumerate(reader):
+        if idx > 10:
+            break
+        rows_lengths.add(len(row))
+    if len(rows_lengths) > 1:
+        raise ValueError(
+            f"Number of columns is not even across the first 10 rows (detected separator: {sep})."
+        )
+
+    if verbose:
+        display_logs_depending_process_time(
+            f'Detected separator: "{sep}" in {round(time() - start, 3)}s',
+            time() - start,
+        )
+    return sep

csv_detective/detection/variables.py

@@ -0,0 +1,98 @@
+from ast import literal_eval
+import logging
+from time import time
+
+import pandas as pd
+
+from csv_detective.utils import display_logs_depending_process_time
+
+
+def detect_continuous_variable(table: pd.DataFrame, continuous_th: float = 0.9, verbose: bool = False):
+    """
+    Detects whether a column contains continuous variables. We consider a continuous column
+    one that contains a considerable amount of float values.
+    We removed the integers as we then end up with postal codes, insee codes, and all sort
+    of codes and types.
+    This is not optimal but it will do for now.
+    """
+    # if we need this again in the future, could be first based on columns detected as int/float to cut time
+
+    def check_threshold(serie: pd.Series, continuous_th: float) -> bool:
+        count = serie.value_counts().to_dict()
+        total_nb = len(serie)
+        if float in count:
+            nb_floats = count[float]
+        else:
+            return False
+        if nb_floats / total_nb >= continuous_th:
+            return True
+        else:
+            return False
+
+    def parses_to_integer(value: str):
+        try:
+            value = value.replace(",", ".")
+            value = literal_eval(value)
+            return type(value)
+        # flake8: noqa
+        except:
+            return False
+
+    if verbose:
+        start = time()
+        logging.info("Detecting continuous columns")
+    res = table.apply(
+        lambda serie: check_threshold(serie.apply(parses_to_integer), continuous_th)
+    )
+    if verbose:
+        display_logs_depending_process_time(
+            f"Detected {sum(res)} continuous columns in {round(time() - start, 3)}s",
+            time() - start,
+        )
+    return res.index[res]
+
+
+def detect_categorical_variable(
+    table: pd.DataFrame,
+    threshold_pct_categorical: float = 0.05,
+    max_number_categorical_values: int = 25,
+    verbose: bool = False,
+):
+    """
+    Heuristically detects whether a table (df) contains categorical values according to
+    the number of unique values contained.
+    As the idea of detecting categorical values is to then try to learn models to predict
+    them, we limit categorical values to at most 25 different modes or at most 5% disparity.
+    Postal code, insee code, code region and so on, may be thus not considered categorical values.
+    :param table:
+    :param threshold_pct_categorical:
+    :param max_number_categorical_values:
+    :return:
+    """
+
+    def abs_number_different_values(column_values: pd.Series):
+        return column_values.nunique()
+
+    def rel_number_different_values(column_values: pd.Series):
+        return column_values.nunique() / len(column_values)
+
+    def detect_categorical(column_values: pd.Series):
+        abs_unique_values = abs_number_different_values(column_values)
+        rel_unique_values = rel_number_different_values(column_values)
+        if (
+            abs_unique_values <= max_number_categorical_values
+            or rel_unique_values <= threshold_pct_categorical
+        ):
+            return True
+        return False
+
+    if verbose:
+        start = time()
+        logging.info("Detecting categorical columns")
+    res = table.apply(lambda serie: detect_categorical(serie))
+    if verbose:
+        display_logs_depending_process_time(
+            f"Detected {sum(res)} categorical columns out of {len(table.columns)} in {round(time() - start, 3)}s",
+            time() - start,
+        )
+    return res.index[res], res

csv_detective/parsing/columns.py

@@ -0,0 +1,141 @@
+import logging
+from time import time
+from typing import Callable
+
+import pandas as pd
+
+from csv_detective.utils import display_logs_depending_process_time
+
+MAX_ROWS_ANALYSIS = 1e5
+
+
+def test_col_val(
+    serie: pd.Series,
+    test_func: Callable,
+    proportion: float = 0.9,
+    skipna: bool = True,
+    limited_output: bool = False,
+    verbose: bool = False,
+):
+    """Tests values of the serie using test_func.
+         - skipna : if True indicates that NaNs are not counted as False
+         - proportion :  indicates the proportion of values that have to pass the test
+    for the serie to be detected as a certain format
+    """
+    if verbose:
+        start = time()
+
+    # TODO : change for a cleaner method and only test columns in modules labels
+    def apply_test_func(serie: pd.Series, test_func: Callable, _range: int):
+        return serie.sample(n=_range).apply(test_func)
+    try:
+        if skipna:
+            serie = serie[serie.notnull()]
+        ser_len = len(serie)
+        if ser_len == 0:
+            return 0.0
+        if not limited_output:
+            result = apply_test_func(serie, test_func, ser_len).sum() / ser_len
+            return result if result >= proportion else 0.0
+        else:
+            if proportion == 1:  # Then try first 1 value, then 5, then all
+                for _range in [
+                    min(1, ser_len),
+                    min(5, ser_len),
+                    ser_len,
+                ]:  # Pour ne pas faire d'opérations inutiles, on commence par 1,
+                    # puis 5 valeurs puis la serie complète
+                    if all(apply_test_func(serie, test_func, _range)):
+                        # print(serie.name, ': check OK')
+                        pass
+                    else:
+                        return 0.0
+                return 1.0
+            else:
+                # if we have a proportion, statistically it's OK to analyse up to 10k rows
+                # (arbitrary number) and get a significant result
+                to_analyse = min(ser_len, MAX_ROWS_ANALYSIS)
+                result = apply_test_func(serie, test_func, to_analyse).sum() / to_analyse
+                return result if result >= proportion else 0.0
+    finally:
+        if verbose and time() - start > 3:
+            display_logs_depending_process_time(
+                f"\t/!\\ Column '{serie.name}' took too long ({round(time() - start, 3)}s)",
+                time() - start
+            )
+
+
+def test_col_label(label: str, test_func: Callable, proportion: float = 1, limited_output: bool = False):
+    """Tests label (from header) using test_func.
+    - proportion :  indicates the minimum score to pass the test for the serie
+    to be detected as a certain format
+    """
+    if not limited_output:
+        return test_func(label)
+    else:
+        result = test_func(label)
+        return result if result >= proportion else 0
+
+
+def test_col(table: pd.DataFrame, all_tests: list, limited_output: bool, skipna: bool = True, verbose: bool = False):
+    if verbose:
+        start = time()
+        logging.info("Testing columns to get types")
+    test_funcs = dict()
+    for test in all_tests:
+        name = test.__name__.split(".")[-1]
+        test_funcs[name] = {"func": test._is, "prop": test.PROPORTION}
+    return_table = pd.DataFrame(columns=table.columns)
+    for idx, (key, value) in enumerate(test_funcs.items()):
+        if verbose:
+            start_type = time()
+            logging.info(f"\t- Starting with type '{key}'")
+        # improvement lead : put the longest tests behind and make them only if previous tests not satisfactory
+        # => the following needs to change, "apply" means all columns are tested for one type at once
+        return_table.loc[key] = table.apply(
+            lambda serie: test_col_val(
+                serie,
+                value["func"],
+                value["prop"],
+                skipna=skipna,
+                limited_output=limited_output,
+                verbose=verbose,
+            )
+        )
+        if verbose:
+            display_logs_depending_process_time(
+                f'\t> Done with type "{key}" in {round(time() - start_type, 3)}s ({idx+1}/{len(test_funcs)})',
+                time() - start_type
+            )
+    if verbose:
+        display_logs_depending_process_time(f"Done testing columns in {round(time() - start, 3)}s", time() - start)
+    return return_table
+
+
+def test_label(table: pd.DataFrame, all_tests: list, limited_output: bool, verbose: bool = False):
+    if verbose:
+        start = time()
+        logging.info("Testing labels to get types")
+    test_funcs = dict()
+    for test in all_tests:
+        name = test.__name__.split(".")[-1]
+        test_funcs[name] = {"func": test._is, "prop": test.PROPORTION}
+
+    return_table = pd.DataFrame(columns=table.columns)
+    for idx, (key, value) in enumerate(test_funcs.items()):
+        if verbose:
+            start_type = time()
+        return_table.loc[key] = [
+            test_col_label(
+                col_name, value["func"], value["prop"], limited_output=limited_output
+            )
+            for col_name in table.columns
+        ]
+        if verbose:
+            display_logs_depending_process_time(
+                f'\t- Done with type "{key}" in {round(time() - start_type, 3)}s ({idx+1}/{len(test_funcs)})',
+                time() - start_type
+            )
+    if verbose:
+        display_logs_depending_process_time(f"Done testing labels in {round(time() - start, 3)}s", time() - start)
+    return return_table

csv_detective/parsing/compression.py

@@ -0,0 +1,11 @@
+import gzip
+from io import BytesIO
+
+
+def unzip(binary_file: BytesIO, engine: str) -> BytesIO:
+    if engine == "gzip":
+        with gzip.open(binary_file, mode="rb") as binary_file:
+            file_content = binary_file.read()
+    else:
+        raise NotImplementedError(f"{engine} is not yet supported")
+    return BytesIO(file_content)

csv_detective/parsing/csv.py

@@ -0,0 +1,55 @@
+import logging
+from time import time
+from typing import TextIO
+
+import pandas as pd
+
+from csv_detective.utils import display_logs_depending_process_time
+
+
+def parse_csv(
+    the_file: TextIO,
+    encoding: str,
+    sep: str,
+    num_rows: int,
+    skiprows: int,
+    random_state: int = 42,
+    verbose: bool = False,
+) -> tuple[pd.DataFrame, int, int]:
+    if verbose:
+        start = time()
+        logging.info("Parsing table")
+    table = None
+
+    if not isinstance(the_file, str):
+        the_file.seek(0)
+
+    total_lines = None
+    for encoding in [encoding, "ISO-8859-1", "utf-8"]:
+        if encoding is None:
+            continue
+
+        if "ISO-8859" in encoding:
+            encoding = "ISO-8859-1"
+        try:
+            table = pd.read_csv(
+                the_file, sep=sep, dtype="unicode", encoding=encoding, skiprows=skiprows
+            )
+            total_lines = len(table)
+            nb_duplicates = len(table.loc[table.duplicated()])
+            if num_rows > 0:
+                num_rows = min(num_rows - 1, total_lines)
+                table = table.sample(num_rows, random_state=random_state)
+            # else : table is unchanged
+            break
+        except TypeError:
+            print("Trying encoding : {encoding}".format(encoding=encoding))
+
+    if table is None:
+        raise ValueError("Could not load file")
+    if verbose:
+        display_logs_depending_process_time(
+            f'Table parsed successfully in {round(time() - start, 3)}s',
+            time() - start,
+        )
+    return table, total_lines, nb_duplicates

csv_detective/parsing/excel.py

@@ -0,0 +1,169 @@
+from io import BytesIO
+from time import time
+from typing import Optional
+
+import openpyxl
+import pandas as pd
+import requests
+import xlrd
+
+from csv_detective.detection.engine import engine_to_file
+from csv_detective.detection.rows import remove_empty_first_rows
+from csv_detective.utils import (
+    display_logs_depending_process_time,
+    is_url,
+)
+
+NEW_EXCEL_EXT = [".xlsx", ".xlsm", ".xltx", ".xltm"]
+OLD_EXCEL_EXT = [".xls"]
+OPEN_OFFICE_EXT = [".odf", ".ods", ".odt"]
+XLS_LIKE_EXT = NEW_EXCEL_EXT + OLD_EXCEL_EXT + OPEN_OFFICE_EXT
+
+
+def parse_excel(
+    file_path: str,
+    num_rows: int = -1,
+    engine: Optional[str] = None,
+    sheet_name: Optional[str] = None,
+    random_state: int = 42,
+    verbose: bool = False,
+) -> tuple[pd.DataFrame, int, int, str, str, int]:
+    """"Excel-like parsing is really slow, could be a good improvement for future development"""
+    if verbose:
+        start = time()
+    no_sheet_specified = sheet_name is None
+
+    if (
+        engine in ['openpyxl', 'xlrd'] or
+        any([file_path.endswith(k) for k in NEW_EXCEL_EXT + OLD_EXCEL_EXT])
+    ):
+        remote_content = None
+        if is_url(file_path):
+            r = requests.get(file_path)
+            r.raise_for_status()
+            remote_content = BytesIO(r.content)
+        if not engine:
+            if any([file_path.endswith(k) for k in NEW_EXCEL_EXT]):
+                engine = "openpyxl"
+            else:
+                engine = "xlrd"
+        if sheet_name is None:
+            if verbose:
+                display_logs_depending_process_time(
+                    f'Detected {engine_to_file[engine]} file, no sheet specified, reading the largest one',
+                    time() - start,
+                )
+            try:
+                if engine == "openpyxl":
+                    # openpyxl doesn't want to open files that don't have a valid extension
+                    # see: https://foss.heptapod.net/openpyxl/openpyxl/-/issues/2157
+                    # if the file is remote, we have a remote content anyway so it's fine
+                    if not remote_content and '.' not in file_path.split('/')[-1]:
+                        with open(file_path, 'rb') as f:
+                            remote_content = BytesIO(f.read())
+                    # faster than loading all sheets
+                    wb = openpyxl.load_workbook(remote_content or file_path, read_only=True)
+                    try:
+                        sizes = {s.title: s.max_row * s.max_column for s in wb.worksheets}
+                    except TypeError:
+                        # sometimes read_only can't get the info, so we have to open the file for real
+                        # this takes more time but it's for a limited number of files
+                        # and it's this or nothing
+                        wb = openpyxl.load_workbook(remote_content or file_path)
+                        sizes = {s.title: s.max_row * s.max_column for s in wb.worksheets}
+                else:
+                    if remote_content:
+                        wb = xlrd.open_workbook(file_contents=remote_content.read())
+                    else:
+                        wb = xlrd.open_workbook(file_path)
+                    sizes = {s.name: s.nrows * s.ncols for s in wb.sheets()}
+                sheet_name = max(sizes, key=sizes.get)
+            except xlrd.biffh.XLRDError:
+                # sometimes a xls file is recognized as ods
+                if verbose:
+                    display_logs_depending_process_time(
+                        'Could not read file with classic xls reader, trying with ODS',
+                        time() - start,
+                    )
+                engine = "odf"
+
+    if engine == "odf" or any([file_path.endswith(k) for k in OPEN_OFFICE_EXT]):
+        # for ODS files, no way to get sheets' sizes without
+        # loading the file one way or another (pandas or pure odfpy)
+        # so all in one
+        engine = "odf"
+        if sheet_name is None:
+            if verbose:
+                display_logs_depending_process_time(
+                    f'Detected {engine_to_file[engine]} file, no sheet specified, reading the largest one',
+                    time() - start,
+                )
+            tables = pd.read_excel(
+                file_path,
+                engine="odf",
+                sheet_name=None,
+                dtype="unicode",
+            )
+            sizes = {sheet_name: table.size for sheet_name, table in tables.items()}
+            sheet_name = max(sizes, key=sizes.get)
+            if verbose:
+                display_logs_depending_process_time(
+                    f'Going forwards with sheet "{sheet_name}"',
+                    time() - start,
+                )
+            table = tables[sheet_name]
+        else:
+            if verbose:
+                display_logs_depending_process_time(
+                    f'Detected {engine_to_file[engine]} file, reading sheet "{sheet_name}"',
+                    time() - start,
+                )
+            table = pd.read_excel(
+                file_path,
+                engine="odf",
+                sheet_name=sheet_name,
+                dtype="unicode",
+            )
+        table, header_row_idx = remove_empty_first_rows(table)
+        total_lines = len(table)
+        nb_duplicates = len(table.loc[table.duplicated()])
+        if num_rows > 0:
+            num_rows = min(num_rows - 1, total_lines)
+            table = table.sample(num_rows, random_state=random_state)
+        if verbose:
+            display_logs_depending_process_time(
+                f'Table parsed successfully in {round(time() - start, 3)}s',
+                time() - start,
+            )
+        return table, total_lines, nb_duplicates, sheet_name, engine, header_row_idx
+
+    # so here we end up with (old and new) excel files only
+    if verbose:
+        if no_sheet_specified:
+            display_logs_depending_process_time(
+                f'Going forwards with sheet "{sheet_name}"',
+                time() - start,
+            )
+        else:
+            display_logs_depending_process_time(
+                f'Detected {engine_to_file[engine]} file, reading sheet "{sheet_name}"',
+                time() - start,
+            )
+    table = pd.read_excel(
+        file_path,
+        engine=engine,
+        sheet_name=sheet_name,
+        dtype="unicode",
+    )
+    table, header_row_idx = remove_empty_first_rows(table)
+    total_lines = len(table)
+    nb_duplicates = len(table.loc[table.duplicated()])
+    if num_rows > 0:
+        num_rows = min(num_rows - 1, total_lines)
+        table = table.sample(num_rows, random_state=random_state)
+    if verbose:
+        display_logs_depending_process_time(
+            f'Table parsed successfully in {round(time() - start, 3)}s',
+            time() - start,
+        )
+    return table, total_lines, nb_duplicates, sheet_name, engine, header_row_idx

csv_detective/parsing/load.py

@@ -0,0 +1,97 @@
+from io import BytesIO, StringIO
+from typing import Optional, Union
+
+import pandas as pd
+import requests
+
+from csv_detective.detection.columns import detect_heading_columns, detect_trailing_columns
+from csv_detective.detection.encoding import detect_encoding
+from csv_detective.detection.engine import (
+    COMPRESSION_ENGINES,
+    EXCEL_ENGINES,
+    detect_engine,
+)
+from csv_detective.detection.headers import detect_headers
+from csv_detective.detection.separator import detect_separator
+from csv_detective.utils import is_url
+from .compression import unzip
+from .csv import parse_csv
+from .excel import (
+    XLS_LIKE_EXT,
+    parse_excel,
+)
+
+
+def load_file(
+    file_path: str,
+    num_rows: int = 500,
+    encoding: Optional[str] = None,
+    sep: Optional[str] = None,
+    verbose: bool = False,
+    sheet_name: Optional[Union[str, int]] = None,
+) -> tuple[pd.DataFrame, dict]:
+    file_name = file_path.split('/')[-1]
+    engine = None
+    if '.' not in file_name or not file_name.endswith("csv"):
+        # file has no extension, we'll investigate how to read it
+        engine = detect_engine(file_path, verbose=verbose)
+
+    if engine in EXCEL_ENGINES or any([file_path.endswith(k) for k in XLS_LIKE_EXT]):
+        table, total_lines, nb_duplicates, sheet_name, engine, header_row_idx = parse_excel(
+            file_path=file_path,
+            num_rows=num_rows,
+            engine=engine,
+            sheet_name=sheet_name,
+            verbose=verbose,
+        )
+        header = table.columns.to_list()
+        analysis = {
+            "engine": engine,
+            "sheet_name": sheet_name,
+        }
+    else:
+        # fetching or reading file as binary
+        if is_url(file_path):
+            r = requests.get(file_path, allow_redirects=True)
+            r.raise_for_status()
+            binary_file = BytesIO(r.content)
+        else:
+            binary_file = open(file_path, "rb")
+        # handling compression
+        if engine in COMPRESSION_ENGINES:
+            binary_file: BytesIO = unzip(binary_file=binary_file, engine=engine)
+        # detecting encoding if not specified
+        if encoding is None:
+            encoding: str = detect_encoding(binary_file, verbose=verbose)
+            binary_file.seek(0)
+        # decoding and reading file
+        if is_url(file_path) or engine in COMPRESSION_ENGINES:
+            str_file = StringIO(binary_file.read().decode(encoding=encoding))
+        else:
+            str_file = open(file_path, "r", encoding=encoding)
+        if sep is None:
+            sep = detect_separator(str_file, verbose=verbose)
+        header_row_idx, header = detect_headers(str_file, sep, verbose=verbose)
+        if header is None:
+            return {"error": True}
+        elif isinstance(header, list):
+            if any([x is None for x in header]):
+                return {"error": True}
+        heading_columns = detect_heading_columns(str_file, sep, verbose=verbose)
+        trailing_columns = detect_trailing_columns(str_file, sep, heading_columns, verbose=verbose)
+        table, total_lines, nb_duplicates = parse_csv(
+            str_file, encoding, sep, num_rows, header_row_idx, verbose=verbose
+        )
+        analysis = {
+            "encoding": encoding,
+            "separator": sep,
+            "heading_columns": heading_columns,
+            "trailing_columns": trailing_columns,
+        }
+    analysis.update({
+        "header_row_idx": header_row_idx,
+        "header": header,
+        "total_lines": total_lines,
+        "nb_duplicates": nb_duplicates,
+    })
+    return table, analysis

csv_detective/parsing/text.py

@@ -0,0 +1,61 @@
+from re import finditer
+
+
+def camel_case_split(identifier: str):
+    matches = finditer(
+        ".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)", identifier
+    )
+    return " ".join([m.group(0) for m in matches])
+
+
+translate_dict = {
+    " ": ["-", "_", "'", ",", "  "],
+    "a": ["à", "â"],
+    "c": ["ç"],
+    "e": ["é", "è", "ê", "é"],
+    "i": ["î", "ï"],
+    "o": ["ô", "ö"],
+    "u": ["ù", "û", "ü"],
+}
+
+
+# Process text
+def _process_text(val: str):
+    """Traitement des chaînes de caractères pour les standardiser.
+    Plusieurs alternatives ont été testées : .translate, unidecode.unidecode,
+    des méthodes hybrides, mais aucune ne s'est avérée plus performante."""
+    val = camel_case_split(val)
+    val = val.lower()
+    for target in translate_dict:
+        for source in translate_dict[target]:
+            val = val.replace(source, target)
+    val = val.strip()
+    return val
+
+
+def is_word_in_string(word: str, string: str):
+    # if the substring is too short, the test can become irrelevant
+    return len(word) > 2 and word in string
+
+
+def header_score(header: str, words_combinations_list: list[str]) -> float:
+    """Returns:
+    - 1 if the header is exactly in the specified list
+    - 0.5 if any of the words is within the header
+    - 0 otherwise"""
+    processed_header = _process_text(header)
+
+    header_matches_words_combination = float(
+        any(
+            words_combination == processed_header for words_combination in words_combinations_list
+        )
+    )
+    words_combination_in_header = 0.5 * (
+        any(
+            is_word_in_string(
+                words_combination, processed_header
+            ) for words_combination in words_combinations_list
+        )
+    )
+
+    return max(header_matches_words_combination, words_combination_in_header)

{csv_detective-0.8.1.dev1460.data → csv_detective-0.8.1.dev1482.data}/data/share/csv_detective/CHANGELOG.md

@@ -6,6 +6,7 @@
 - Refactor repo metadata and requirements [#120](https://github.com/datagouv/csv-detective/pull/120) [#122](https://github.com/datagouv/csv-detective/pull/122)
 - Better URL detection [#121](https://github.com/datagouv/csv-detective/pull/121)
 - For big files, analyse on sample then validate on whole file [#124](https://github.com/datagouv/csv-detective/pull/124)
+- Fix imports [#125](https://github.com/datagouv/csv-detective/pull/125) [#126](https://github.com/datagouv/csv-detective/pull/126)
 
 ## 0.8.0 (2025-05-20)
 

{csv_detective-0.8.1.dev1460.dist-info → csv_detective-0.8.1.dev1482.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csv_detective
-Version: 0.8.1.dev1460
+Version: 0.8.1.dev1482
 Summary: Detect tabular files column content
 Home-page: https://github.com/datagouv/csv_detective
 Author: Etalab

{csv_detective-0.8.1.dev1460.dist-info → csv_detective-0.8.1.dev1482.dist-info}/RECORD

@@ -121,16 +121,32 @@ csv_detective/detect_labels/temp/date/__init__.py,sha256=w0eeZIseAmPwL4OvCWzZXbx
 csv_detective/detect_labels/temp/datetime_iso/__init__.py,sha256=d0laZNzHx-kSARs9Re8TZ11GNs99aMz6gXc72CJ6ul4,440
 csv_detective/detect_labels/temp/datetime_rfc822/__init__.py,sha256=53ysj7QgsxXwG1le3zfSJd1oaTTf-Er3jBeYi_A4F9g,458
 csv_detective/detect_labels/temp/year/__init__.py,sha256=7uWaCZY7dOG7nolW46IgBWmcu8K-9jPED-pOlMlErfo,433
+csv_detective/detection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+csv_detective/detection/columns.py,sha256=vfE-DKESA6J9Rfsl-a8tjgZfE21VmzArO5TrbzL0KmE,2905
+csv_detective/detection/encoding.py,sha256=tpjJEMNM_2TcLXDzn1lNQPnSRnsWYjs83tQ8jNwTj4E,973
+csv_detective/detection/engine.py,sha256=HiIrU-l9EO5Fbc2Vh8W_Uy5-dpKcQQzlxCqMuWc09LY,1530
+csv_detective/detection/formats.py,sha256=5ZW7gmhyQt6BB7xLcVVhui17oGn1udAWI9w22EAOHy4,6337
+csv_detective/detection/headers.py,sha256=wrVII2RQpsVmHhrO1DHf3dmiu8kbtOjBlskf41cnQmc,1172
+csv_detective/detection/rows.py,sha256=3qvsbsBcMxiqqfSYYkOgsRpX777rk22tnRHDwUA97kU,742
+csv_detective/detection/separator.py,sha256=XjeDBqhiBxVfkCPJKem9BAgJqs_hOgQltc_pxrH_-Tg,1547
+csv_detective/detection/variables.py,sha256=3qEMtjZ_zyIFXvTnFgK7ZMDx8C12uQXKfFjEj2moyJc,3558
 csv_detective/output/__init__.py,sha256=5KTevPfp_4MRxByJyOntQjToNfeG7dPQn-_13wSq7EU,1910
 csv_detective/output/dataframe.py,sha256=89iQRE59cHQyQQEsujQVIKP2YAUYpPklWkdDOqZE-wE,2183
 csv_detective/output/example.py,sha256=EdPX1iqHhIG4DsiHuYdy-J7JxOkjgUh_o2D5nrfM5fA,8649
 csv_detective/output/profile.py,sha256=B8YU541T_YPDezJGh4dkHckOShiwHSrZd9GS8jbmz7A,2919
 csv_detective/output/schema.py,sha256=ZDBWDOD8IYp7rcB0_n8l9JXGIhOQ6bTZHFWfTmnNNEQ,13480
 csv_detective/output/utils.py,sha256=HbmvCCCmFo7NJxhD_UsJIveuw-rrfhrvYckv1CJn_10,2301
-csv_detective-0.8.1.dev1460.data/data/share/csv_detective/CHANGELOG.md,sha256=BsmO9YQAMi31co_c0I8aYRsm2m5Q5--vORWoJArdhOM,8725
-csv_detective-0.8.1.dev1460.data/data/share/csv_detective/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
-csv_detective-0.8.1.dev1460.data/data/share/csv_detective/README.md,sha256=gKLFmC8kuCCywS9eAhMak_JNriUWWNOsBKleAu5TIEY,8501
-csv_detective-0.8.1.dev1460.dist-info/licenses/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
+csv_detective/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+csv_detective/parsing/columns.py,sha256=e0xVmeXNvSC3su5HTFSNClgkz8PlFkoHmNwRYdS57mk,5670
+csv_detective/parsing/compression.py,sha256=Fnw5tj-PpBNI8NYsWj5gD-DUoWcVLnsVpiKm9MpxmIA,350
+csv_detective/parsing/csv.py,sha256=11mibDnJhIjykXLGZvA5ZEU5U7KgxIrbyO6BNv6jlro,1626
+csv_detective/parsing/excel.py,sha256=AslE2S1e67o8yTIAIhp-lAnJ6-XqeBBRz1-VMFqhZBM,7055
+csv_detective/parsing/load.py,sha256=u6fbGFZsL2GwPQRzhAXgt32JpUur7vbQdErREHxNJ-w,3661
+csv_detective/parsing/text.py,sha256=_TprGi0gHZlRsafizI3dqQhBehZW4BazqxmypMcAZ-o,1824
+csv_detective-0.8.1.dev1482.data/data/share/csv_detective/CHANGELOG.md,sha256=0kXmJeSwiHMY4mSOOtaZJ739S_Aj4ojWAzsJXYR_T98,8857
+csv_detective-0.8.1.dev1482.data/data/share/csv_detective/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
+csv_detective-0.8.1.dev1482.data/data/share/csv_detective/README.md,sha256=gKLFmC8kuCCywS9eAhMak_JNriUWWNOsBKleAu5TIEY,8501
+csv_detective-0.8.1.dev1482.dist-info/licenses/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_example.py,sha256=JeHxSK0IVDcSrOhSZlNGSQv4JAc_r6mzvJM8PfmLTMw,2018
 tests/test_fields.py,sha256=d2tNvjtal6ZbO646x1GDbp_CGgp-EIcdg2SgMG72J6E,10270
@@ -138,8 +154,8 @@ tests/test_file.py,sha256=FWVtYHlD5uU7tPeYsqlQg6O4lpU8Ct35vddkbzhvvjA,8508
 tests/test_labels.py,sha256=Nkr645bUewrj8hjNDKr67FQ6Sy_TID6f3E5Kfkl231M,464
 tests/test_structure.py,sha256=bv-tjgXohvQAxwmxzH0BynFpK2TyPjcxvtIAmIRlZmA,1393
 tests/test_validation.py,sha256=CTGonR6htxcWF9WH8MxumDD8cF45Y-G4hm94SM4lFjU,3246
-csv_detective-0.8.1.dev1460.dist-info/METADATA,sha256=Rhi872uRXV2PcYpcI64GJ9vw12TsYIEQJxf8H1srLic,10443
-csv_detective-0.8.1.dev1460.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-csv_detective-0.8.1.dev1460.dist-info/entry_points.txt,sha256=JjweTReFqKJmuvkegzlew2j3D5pZzfxvbEGOtGVGmaY,56
-csv_detective-0.8.1.dev1460.dist-info/top_level.txt,sha256=M0Nv646VHo-49zWjPkwo2C48UmtfddV8_9mEZeIxy8Q,20
-csv_detective-0.8.1.dev1460.dist-info/RECORD,,
+csv_detective-0.8.1.dev1482.dist-info/METADATA,sha256=xigKiWIsWZoH2rUZ0JGfvKW7eGm72SFQ1d7uFzWQcKs,10443
+csv_detective-0.8.1.dev1482.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+csv_detective-0.8.1.dev1482.dist-info/entry_points.txt,sha256=JjweTReFqKJmuvkegzlew2j3D5pZzfxvbEGOtGVGmaY,56
+csv_detective-0.8.1.dev1482.dist-info/top_level.txt,sha256=M0Nv646VHo-49zWjPkwo2C48UmtfddV8_9mEZeIxy8Q,20
+csv_detective-0.8.1.dev1482.dist-info/RECORD,,