csv-detective 0.8.1.dev1416__py3-none-any.whl → 0.8.1.dev1440__py3-none-any.whl

csv_detective/detection/variables.py

@@ -1,98 +0,0 @@
-from ast import literal_eval
-import logging
-from time import time
-
-import pandas as pd
-
-from csv_detective.utils import display_logs_depending_process_time
-
-
-def detect_continuous_variable(table: pd.DataFrame, continuous_th: float = 0.9, verbose: bool = False):
-    """
-    Detects whether a column contains continuous variables. We consider a continuous column
-    one that contains a considerable amount of float values.
-    We removed the integers as we then end up with postal codes, insee codes, and all sort
-    of codes and types.
-    This is not optimal but it will do for now.
-    """
-    # if we need this again in the future, could be first based on columns detected as int/float to cut time
-
-    def check_threshold(serie: pd.Series, continuous_th: float) -> bool:
-        count = serie.value_counts().to_dict()
-        total_nb = len(serie)
-        if float in count:
-            nb_floats = count[float]
-        else:
-            return False
-        if nb_floats / total_nb >= continuous_th:
-            return True
-        else:
-            return False
-
-    def parses_to_integer(value: str):
-        try:
-            value = value.replace(",", ".")
-            value = literal_eval(value)
-            return type(value)
-        # flake8: noqa
-        except:
-            return False
-
-    if verbose:
-        start = time()
-        logging.info("Detecting continuous columns")
-    res = table.apply(
-        lambda serie: check_threshold(serie.apply(parses_to_integer), continuous_th)
-    )
-    if verbose:
-        display_logs_depending_process_time(
-            f"Detected {sum(res)} continuous columns in {round(time() - start, 3)}s",
-            time() - start,
-        )
-    return res.index[res]
-
-
-def detect_categorical_variable(
-    table: pd.DataFrame,
-    threshold_pct_categorical: float = 0.05,
-    max_number_categorical_values: int = 25,
-    verbose: bool = False,
-):
-    """
-    Heuristically detects whether a table (df) contains categorical values according to
-    the number of unique values contained.
-    As the idea of detecting categorical values is to then try to learn models to predict
-    them, we limit categorical values to at most 25 different modes or at most 5% disparity.
-    Postal code, insee code, code region and so on, may be thus not considered categorical values.
-    :param table:
-    :param threshold_pct_categorical:
-    :param max_number_categorical_values:
-    :return:
-    """
-
-    def abs_number_different_values(column_values: pd.Series):
-        return column_values.nunique()
-
-    def rel_number_different_values(column_values: pd.Series):
-        return column_values.nunique() / len(column_values)
-
-    def detect_categorical(column_values: pd.Series):
-        abs_unique_values = abs_number_different_values(column_values)
-        rel_unique_values = rel_number_different_values(column_values)
-        if (
-            abs_unique_values <= max_number_categorical_values
-            or rel_unique_values <= threshold_pct_categorical
-        ):
-            return True
-        return False
-
-    if verbose:
-        start = time()
-        logging.info("Detecting categorical columns")
-    res = table.apply(lambda serie: detect_categorical(serie))
-    if verbose:
-        display_logs_depending_process_time(
-            f"Detected {sum(res)} categorical columns out of {len(table.columns)} in {round(time() - start, 3)}s",
-            time() - start,
-        )
-    return res.index[res], res

csv_detective/parsing/columns.py

@@ -1,139 +0,0 @@
-import logging
-from time import time
-from typing import Callable
-
-import pandas as pd
-
-from csv_detective.utils import display_logs_depending_process_time
-
-
-def test_col_val(
-    serie: pd.Series,
-    test_func: Callable,
-    proportion: float = 0.9,
-    skipna: bool = True,
-    limited_output: bool = False,
-    verbose: bool = False,
-):
-    """Tests values of the serie using test_func.
-         - skipna : if True indicates that NaNs are not counted as False
-         - proportion :  indicates the proportion of values that have to pass the test
-    for the serie to be detected as a certain format
-    """
-    if verbose:
-        start = time()
-
-    # TODO : change for a cleaner method and only test columns in modules labels
-    def apply_test_func(serie: pd.Series, test_func: Callable, _range: int):
-        return serie.sample(n=_range).apply(test_func)
-    try:
-        if skipna:
-            serie = serie[serie.notnull()]
-        ser_len = len(serie)
-        if ser_len == 0:
-            return 0.0
-        if not limited_output:
-            result = apply_test_func(serie, test_func, ser_len).sum() / ser_len
-            return result if result >= proportion else 0.0
-        else:
-            if proportion == 1:  # Then try first 1 value, then 5, then all
-                for _range in [
-                    min(1, ser_len),
-                    min(5, ser_len),
-                    ser_len,
-                ]:  # Pour ne pas faire d'opérations inutiles, on commence par 1,
-                    # puis 5 valeurs puis la serie complète
-                    if all(apply_test_func(serie, test_func, _range)):
-                        # print(serie.name, ': check OK')
-                        pass
-                    else:
-                        return 0.0
-                return 1.0
-            else:
-                # if we have a proportion, statistically it's OK to analyse up to 10k rows
-                # (arbitrary number) and get a significant result
-                to_analyse = min(ser_len, 10000)
-                result = apply_test_func(serie, test_func, to_analyse).sum() / to_analyse
-                return result if result >= proportion else 0.0
-    finally:
-        if verbose and time() - start > 3:
-            display_logs_depending_process_time(
-                f"\t/!\\ Column '{serie.name}' took too long ({round(time() - start, 3)}s)",
-                time() - start
-            )
-
-
-def test_col_label(label: str, test_func: Callable, proportion: float = 1, limited_output: bool = False):
-    """Tests label (from header) using test_func.
-    - proportion :  indicates the minimum score to pass the test for the serie
-    to be detected as a certain format
-    """
-    if not limited_output:
-        return test_func(label)
-    else:
-        result = test_func(label)
-        return result if result >= proportion else 0
-
-
-def test_col(table: pd.DataFrame, all_tests: list, limited_output: bool, skipna: bool = True, verbose: bool = False):
-    if verbose:
-        start = time()
-        logging.info("Testing columns to get types")
-    test_funcs = dict()
-    for test in all_tests:
-        name = test.__name__.split(".")[-1]
-        test_funcs[name] = {"func": test._is, "prop": test.PROPORTION}
-    return_table = pd.DataFrame(columns=table.columns)
-    for idx, (key, value) in enumerate(test_funcs.items()):
-        if verbose:
-            start_type = time()
-            logging.info(f"\t- Starting with type '{key}'")
-        # improvement lead : put the longest tests behind and make them only if previous tests not satisfactory
-        # => the following needs to change, "apply" means all columns are tested for one type at once
-        return_table.loc[key] = table.apply(
-            lambda serie: test_col_val(
-                serie,
-                value["func"],
-                value["prop"],
-                skipna=skipna,
-                limited_output=limited_output,
-                verbose=verbose,
-            )
-        )
-        if verbose:
-            display_logs_depending_process_time(
-                f'\t> Done with type "{key}" in {round(time() - start_type, 3)}s ({idx+1}/{len(test_funcs)})',
-                time() - start_type
-            )
-    if verbose:
-        display_logs_depending_process_time(f"Done testing columns in {round(time() - start, 3)}s", time() - start)
-    return return_table
-
-
-def test_label(table: pd.DataFrame, all_tests: list, limited_output: bool, verbose: bool = False):
-    if verbose:
-        start = time()
-        logging.info("Testing labels to get types")
-    test_funcs = dict()
-    for test in all_tests:
-        name = test.__name__.split(".")[-1]
-        test_funcs[name] = {"func": test._is, "prop": test.PROPORTION}
-
-    return_table = pd.DataFrame(columns=table.columns)
-    for idx, (key, value) in enumerate(test_funcs.items()):
-        if verbose:
-            start_type = time()
-        return_table.loc[key] = [
-            test_col_label(
-                col_name, value["func"], value["prop"], limited_output=limited_output
-            )
-            for col_name in table.columns
-        ]
-        if verbose:
-            display_logs_depending_process_time(
-                f'\t- Done with type "{key}" in {round(time() - start_type, 3)}s ({idx+1}/{len(test_funcs)})',
-                time() - start_type
-            )
-    if verbose:
-        display_logs_depending_process_time(f"Done testing labels in {round(time() - start, 3)}s", time() - start)
-    return return_table

csv_detective/parsing/compression.py

@@ -1,11 +0,0 @@
-import gzip
-from io import BytesIO
-
-
-def unzip(binary_file: BytesIO, engine: str) -> BytesIO:
-    if engine == "gzip":
-        with gzip.open(binary_file, mode="rb") as binary_file:
-            file_content = binary_file.read()
-    else:
-        raise NotImplementedError(f"{engine} is not yet supported")
-    return BytesIO(file_content)

csv_detective/parsing/csv.py

@@ -1,55 +0,0 @@
-import logging
-from time import time
-from typing import TextIO
-
-import pandas as pd
-
-from csv_detective.utils import display_logs_depending_process_time
-
-
-def parse_csv(
-    the_file: TextIO,
-    encoding: str,
-    sep: str,
-    num_rows: int,
-    skiprows: int,
-    random_state: int = 42,
-    verbose: bool = False,
-) -> tuple[pd.DataFrame, int, int]:
-    if verbose:
-        start = time()
-        logging.info("Parsing table")
-    table = None
-
-    if not isinstance(the_file, str):
-        the_file.seek(0)
-
-    total_lines = None
-    for encoding in [encoding, "ISO-8859-1", "utf-8"]:
-        if encoding is None:
-            continue
-
-        if "ISO-8859" in encoding:
-            encoding = "ISO-8859-1"
-        try:
-            table = pd.read_csv(
-                the_file, sep=sep, dtype="unicode", encoding=encoding, skiprows=skiprows
-            )
-            total_lines = len(table)
-            nb_duplicates = len(table.loc[table.duplicated()])
-            if num_rows > 0:
-                num_rows = min(num_rows - 1, total_lines)
-                table = table.sample(num_rows, random_state=random_state)
-            # else : table is unchanged
-            break
-        except TypeError:
-            print("Trying encoding : {encoding}".format(encoding=encoding))
-
-    if table is None:
-        raise ValueError("Could not load file")
-    if verbose:
-        display_logs_depending_process_time(
-            f'Table parsed successfully in {round(time() - start, 3)}s',
-            time() - start,
-        )
-    return table, total_lines, nb_duplicates

csv_detective/parsing/excel.py

@@ -1,169 +0,0 @@
-from io import BytesIO
-from time import time
-from typing import Optional
-
-import openpyxl
-import pandas as pd
-import requests
-import xlrd
-
-from csv_detective.detection.engine import engine_to_file
-from csv_detective.detection.rows import remove_empty_first_rows
-from csv_detective.utils import (
-    display_logs_depending_process_time,
-    is_url,
-)
-
-NEW_EXCEL_EXT = [".xlsx", ".xlsm", ".xltx", ".xltm"]
-OLD_EXCEL_EXT = [".xls"]
-OPEN_OFFICE_EXT = [".odf", ".ods", ".odt"]
-XLS_LIKE_EXT = NEW_EXCEL_EXT + OLD_EXCEL_EXT + OPEN_OFFICE_EXT
-
-
-def parse_excel(
-    file_path: str,
-    num_rows: int = -1,
-    engine: Optional[str] = None,
-    sheet_name: Optional[str] = None,
-    random_state: int = 42,
-    verbose: bool = False,
-) -> tuple[pd.DataFrame, int, int, str, str, int]:
-    """"Excel-like parsing is really slow, could be a good improvement for future development"""
-    if verbose:
-        start = time()
-    no_sheet_specified = sheet_name is None
-
-    if (
-        engine in ['openpyxl', 'xlrd'] or
-        any([file_path.endswith(k) for k in NEW_EXCEL_EXT + OLD_EXCEL_EXT])
-    ):
-        remote_content = None
-        if is_url(file_path):
-            r = requests.get(file_path)
-            r.raise_for_status()
-            remote_content = BytesIO(r.content)
-        if not engine:
-            if any([file_path.endswith(k) for k in NEW_EXCEL_EXT]):
-                engine = "openpyxl"
-            else:
-                engine = "xlrd"
-        if sheet_name is None:
-            if verbose:
-                display_logs_depending_process_time(
-                    f'Detected {engine_to_file[engine]} file, no sheet specified, reading the largest one',
-                    time() - start,
-                )
-            try:
-                if engine == "openpyxl":
-                    # openpyxl doesn't want to open files that don't have a valid extension
-                    # see: https://foss.heptapod.net/openpyxl/openpyxl/-/issues/2157
-                    # if the file is remote, we have a remote content anyway so it's fine
-                    if not remote_content and '.' not in file_path.split('/')[-1]:
-                        with open(file_path, 'rb') as f:
-                            remote_content = BytesIO(f.read())
-                    # faster than loading all sheets
-                    wb = openpyxl.load_workbook(remote_content or file_path, read_only=True)
-                    try:
-                        sizes = {s.title: s.max_row * s.max_column for s in wb.worksheets}
-                    except TypeError:
-                        # sometimes read_only can't get the info, so we have to open the file for real
-                        # this takes more time but it's for a limited number of files
-                        # and it's this or nothing
-                        wb = openpyxl.load_workbook(remote_content or file_path)
-                        sizes = {s.title: s.max_row * s.max_column for s in wb.worksheets}
-                else:
-                    if remote_content:
-                        wb = xlrd.open_workbook(file_contents=remote_content.read())
-                    else:
-                        wb = xlrd.open_workbook(file_path)
-                    sizes = {s.name: s.nrows * s.ncols for s in wb.sheets()}
-                sheet_name = max(sizes, key=sizes.get)
-            except xlrd.biffh.XLRDError:
-                # sometimes a xls file is recognized as ods
-                if verbose:
-                    display_logs_depending_process_time(
-                        'Could not read file with classic xls reader, trying with ODS',
-                        time() - start,
-                    )
-                engine = "odf"
-
-    if engine == "odf" or any([file_path.endswith(k) for k in OPEN_OFFICE_EXT]):
-        # for ODS files, no way to get sheets' sizes without
-        # loading the file one way or another (pandas or pure odfpy)
-        # so all in one
-        engine = "odf"
-        if sheet_name is None:
-            if verbose:
-                display_logs_depending_process_time(
-                    f'Detected {engine_to_file[engine]} file, no sheet specified, reading the largest one',
-                    time() - start,
-                )
-            tables = pd.read_excel(
-                file_path,
-                engine="odf",
-                sheet_name=None,
-                dtype="unicode",
-            )
-            sizes = {sheet_name: table.size for sheet_name, table in tables.items()}
-            sheet_name = max(sizes, key=sizes.get)
-            if verbose:
-                display_logs_depending_process_time(
-                    f'Going forwards with sheet "{sheet_name}"',
-                    time() - start,
-                )
-            table = tables[sheet_name]
-        else:
-            if verbose:
-                display_logs_depending_process_time(
-                    f'Detected {engine_to_file[engine]} file, reading sheet "{sheet_name}"',
-                    time() - start,
-                )
-            table = pd.read_excel(
-                file_path,
-                engine="odf",
-                sheet_name=sheet_name,
-                dtype="unicode",
-            )
-        table, header_row_idx = remove_empty_first_rows(table)
-        total_lines = len(table)
-        nb_duplicates = len(table.loc[table.duplicated()])
-        if num_rows > 0:
-            num_rows = min(num_rows - 1, total_lines)
-            table = table.sample(num_rows, random_state=random_state)
-        if verbose:
-            display_logs_depending_process_time(
-                f'Table parsed successfully in {round(time() - start, 3)}s',
-                time() - start,
-            )
-        return table, total_lines, nb_duplicates, sheet_name, engine, header_row_idx
-
-    # so here we end up with (old and new) excel files only
-    if verbose:
-        if no_sheet_specified:
-            display_logs_depending_process_time(
-                f'Going forwards with sheet "{sheet_name}"',
-                time() - start,
-            )
-        else:
-            display_logs_depending_process_time(
-                f'Detected {engine_to_file[engine]} file, reading sheet "{sheet_name}"',
-                time() - start,
-            )
-    table = pd.read_excel(
-        file_path,
-        engine=engine,
-        sheet_name=sheet_name,
-        dtype="unicode",
-    )
-    table, header_row_idx = remove_empty_first_rows(table)
-    total_lines = len(table)
-    nb_duplicates = len(table.loc[table.duplicated()])
-    if num_rows > 0:
-        num_rows = min(num_rows - 1, total_lines)
-        table = table.sample(num_rows, random_state=random_state)
-    if verbose:
-        display_logs_depending_process_time(
-            f'Table parsed successfully in {round(time() - start, 3)}s',
-            time() - start,
-        )
-    return table, total_lines, nb_duplicates, sheet_name, engine, header_row_idx

csv_detective/parsing/load.py

@@ -1,97 +0,0 @@
-from io import BytesIO, StringIO
-from typing import Optional, Union
-
-import pandas as pd
-import requests
-
-from csv_detective.detection.columns import detect_heading_columns, detect_trailing_columns
-from csv_detective.detection.encoding import detect_encoding
-from csv_detective.detection.engine import (
-    COMPRESSION_ENGINES,
-    EXCEL_ENGINES,
-    detect_engine,
-)
-from csv_detective.detection.headers import detect_headers
-from csv_detective.detection.separator import detect_separator
-from csv_detective.utils import is_url
-from .compression import unzip
-from .csv import parse_csv
-from .excel import (
-    XLS_LIKE_EXT,
-    parse_excel,
-)
-
-
-def load_file(
-    file_path: str,
-    num_rows: int = 500,
-    encoding: Optional[str] = None,
-    sep: Optional[str] = None,
-    verbose: bool = False,
-    sheet_name: Optional[Union[str, int]] = None,
-) -> tuple[pd.DataFrame, dict]:
-    file_name = file_path.split('/')[-1]
-    engine = None
-    if '.' not in file_name or not file_name.endswith("csv"):
-        # file has no extension, we'll investigate how to read it
-        engine = detect_engine(file_path, verbose=verbose)
-
-    if engine in EXCEL_ENGINES or any([file_path.endswith(k) for k in XLS_LIKE_EXT]):
-        table, total_lines, nb_duplicates, sheet_name, engine, header_row_idx = parse_excel(
-            file_path=file_path,
-            num_rows=num_rows,
-            engine=engine,
-            sheet_name=sheet_name,
-            verbose=verbose,
-        )
-        header = table.columns.to_list()
-        analysis = {
-            "engine": engine,
-            "sheet_name": sheet_name,
-        }
-    else:
-        # fetching or reading file as binary
-        if is_url(file_path):
-            r = requests.get(file_path, allow_redirects=True)
-            r.raise_for_status()
-            binary_file = BytesIO(r.content)
-        else:
-            binary_file = open(file_path, "rb")
-        # handling compression
-        if engine in COMPRESSION_ENGINES:
-            binary_file: BytesIO = unzip(binary_file=binary_file, engine=engine)
-        # detecting encoding if not specified
-        if encoding is None:
-            encoding: str = detect_encoding(binary_file, verbose=verbose)
-            binary_file.seek(0)
-        # decoding and reading file
-        if is_url(file_path) or engine in COMPRESSION_ENGINES:
-            str_file = StringIO(binary_file.read().decode(encoding=encoding))
-        else:
-            str_file = open(file_path, "r", encoding=encoding)
-        if sep is None:
-            sep = detect_separator(str_file, verbose=verbose)
-        header_row_idx, header = detect_headers(str_file, sep, verbose=verbose)
-        if header is None:
-            return {"error": True}
-        elif isinstance(header, list):
-            if any([x is None for x in header]):
-                return {"error": True}
-        heading_columns = detect_heading_columns(str_file, sep, verbose=verbose)
-        trailing_columns = detect_trailing_columns(str_file, sep, heading_columns, verbose=verbose)
-        table, total_lines, nb_duplicates = parse_csv(
-            str_file, encoding, sep, num_rows, header_row_idx, verbose=verbose
-        )
-        analysis = {
-            "encoding": encoding,
-            "separator": sep,
-            "heading_columns": heading_columns,
-            "trailing_columns": trailing_columns,
-        }
-    analysis.update({
-        "header_row_idx": header_row_idx,
-        "header": header,
-        "total_lines": total_lines,
-        "nb_duplicates": nb_duplicates,
-    })
-    return table, analysis

csv_detective/parsing/text.py

@@ -1,61 +0,0 @@
-from re import finditer
-
-
-def camel_case_split(identifier: str):
-    matches = finditer(
-        ".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)", identifier
-    )
-    return " ".join([m.group(0) for m in matches])
-
-
-translate_dict = {
-    " ": ["-", "_", "'", ",", "  "],
-    "a": ["à", "â"],
-    "c": ["ç"],
-    "e": ["é", "è", "ê", "é"],
-    "i": ["î", "ï"],
-    "o": ["ô", "ö"],
-    "u": ["ù", "û", "ü"],
-}
-
-
-# Process text
-def _process_text(val: str):
-    """Traitement des chaînes de caractères pour les standardiser.
-    Plusieurs alternatives ont été testées : .translate, unidecode.unidecode,
-    des méthodes hybrides, mais aucune ne s'est avérée plus performante."""
-    val = camel_case_split(val)
-    val = val.lower()
-    for target in translate_dict:
-        for source in translate_dict[target]:
-            val = val.replace(source, target)
-    val = val.strip()
-    return val
-
-
-def is_word_in_string(word: str, string: str):
-    # if the substring is too short, the test can become irrelevant
-    return len(word) > 2 and word in string
-
-
-def header_score(header: str, words_combinations_list: list[str]) -> float:
-    """Returns:
-    - 1 if the header is exactly in the specified list
-    - 0.5 if any of the words is within the header
-    - 0 otherwise"""
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            words_combination == processed_header for words_combination in words_combinations_list
-        )
-    )
-    words_combination_in_header = 0.5 * (
-        any(
-            is_word_in_string(
-                words_combination, processed_header
-            ) for words_combination in words_combinations_list
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)