csv-detective 0.7.5.dev1197__py3-none-any.whl → 0.7.5.dev1228__py3-none-any.whl

csv_detective/output/utils.py

@@ -0,0 +1,51 @@
+import pandas as pd
+
+
+def prepare_output_dict(return_table: pd.DataFrame, limited_output: bool):
+    return_dict_cols = return_table.to_dict("dict")
+    return_dict_cols_intermediary = {}
+    for column_name in return_dict_cols:
+        return_dict_cols_intermediary[column_name] = []
+        for detected_value_type in return_dict_cols[column_name]:
+            if return_dict_cols[column_name][detected_value_type] == 0:
+                continue
+            dict_tmp = {}
+            dict_tmp["format"] = detected_value_type
+            dict_tmp["score"] = return_dict_cols[column_name][detected_value_type]
+            return_dict_cols_intermediary[column_name].append(dict_tmp)
+
+        # Clean dict using priorities
+        formats_detected = {
+            x["format"] for x in return_dict_cols_intermediary[column_name]
+        }
+        formats_to_remove = set()
+        # Deprioritise float and int detection vs others
+        if len(formats_detected - {"float", "int"}) > 0:
+            formats_to_remove = formats_to_remove.union({"float", "int"})
+        if "int" in formats_detected:
+            formats_to_remove.add("float")
+        if "latitude_wgs_fr_metropole" in formats_detected:
+            formats_to_remove.add("latitude_l93")
+            formats_to_remove.add("latitude_wgs")
+        if "longitude_wgs_fr_metropole" in formats_detected:
+            formats_to_remove.add("longitude_l93")
+            formats_to_remove.add("longitude_wgs")
+        if "longitude_wgs" in formats_detected:
+            formats_to_remove.add("longitude_l93")
+        if "code_region" in formats_detected:
+            formats_to_remove.add("code_departement")
+
+        formats_to_keep = formats_detected - formats_to_remove
+
+        detections = return_dict_cols_intermediary[column_name]
+        detections = [x for x in detections if x["format"] in formats_to_keep]
+        if not limited_output:
+            return_dict_cols_intermediary[column_name] = detections
+        else:
+            return_dict_cols_intermediary[column_name] = (
+                max(detections, key=lambda x: x["score"])
+                if len(detections) > 0
+                else {"format": "string", "score": 1.0}
+            )
+
+    return return_dict_cols_intermediary

csv_detective/parsing/columns.py

@@ -0,0 +1,141 @@
+import logging
+from time import time
+from typing import Callable
+
+import pandas as pd
+
+from csv_detective.utils import display_logs_depending_process_time
+
+
+def test_col_val(
+    serie: pd.Series,
+    test_func: Callable,
+    proportion: float = 0.9,
+    skipna: bool = True,
+    limited_output: bool = False,
+    verbose: bool = False,
+):
+    """Tests values of the serie using test_func.
+         - skipna : if True indicates that NaNs are not counted as False
+         - proportion :  indicates the proportion of values that have to pass the test
+    for the serie to be detected as a certain format
+    """
+    if verbose:
+        start = time()
+
+    # TODO : change for a cleaner method and only test columns in modules labels
+    def apply_test_func(serie: pd.Series, test_func: Callable, _range: int):
+        return serie.sample(n=_range).apply(test_func)
+    try:
+        if skipna:
+            serie = serie[serie.notnull()]
+        ser_len = len(serie)
+        if ser_len == 0:
+            return 0.0
+        if not limited_output:
+            result = apply_test_func(serie, test_func, ser_len).sum() / ser_len
+            return result if result >= proportion else 0.0
+        else:
+            if proportion == 1:  # Then try first 1 value, then 5, then all
+                for _range in [
+                    min(1, ser_len),
+                    min(5, ser_len),
+                    ser_len,
+                ]:  # Pour ne pas faire d'opérations inutiles, on commence par 1,
+                    # puis 5 valeurs puis la serie complète
+                    if all(apply_test_func(serie, test_func, _range)):
+                        # print(serie.name, ': check OK')
+                        pass
+                    else:
+                        return 0.0
+                return 1.0
+            else:
+                # if we have a proportion, statistically it's OK to analyse up to 10k rows
+                # (arbitrary number) and get a significant result
+                to_analyse = min(ser_len, 10000)
+                result = apply_test_func(serie, test_func, to_analyse).sum() / to_analyse
+                return result if result >= proportion else 0.0
+    finally:
+        if verbose and time() - start > 3:
+            display_logs_depending_process_time(
+                f"\t/!\\ Column '{serie.name}' took too long ({round(time() - start, 3)}s)",
+                time() - start
+            )
+
+
+def test_col_label(label: str, test_func: Callable, proportion: float = 1, limited_output: bool = False):
+    """Tests label (from header) using test_func.
+    - proportion :  indicates the minimum score to pass the test for the serie
+    to be detected as a certain format
+    """
+    if not limited_output:
+        return test_func(label)
+    else:
+        result = test_func(label)
+        return result if result >= proportion else 0
+
+
+def test_col(table: pd.DataFrame, all_tests: list, limited_output: bool, skipna: bool = True, verbose: bool = False):
+    # Initialising dict for tests
+    if verbose:
+        start = time()
+        logging.info("Testing columns to get types")
+    test_funcs = dict()
+    for test in all_tests:
+        name = test.__name__.split(".")[-1]
+        test_funcs[name] = {"func": test._is, "prop": test.PROPORTION}
+    return_table = pd.DataFrame(columns=table.columns)
+    for idx, (key, value) in enumerate(test_funcs.items()):
+        if verbose:
+            start_type = time()
+            logging.info(f"\t- Starting with type '{key}'")
+        # improvement lead : put the longest tests behind and make them only if previous tests not satisfactory
+        # => the following needs to change, "apply" means all columns are tested for one type at once
+        return_table.loc[key] = table.apply(
+            lambda serie: test_col_val(
+                serie,
+                value["func"],
+                value["prop"],
+                skipna=skipna,
+                limited_output=limited_output,
+                verbose=verbose,
+            )
+        )
+        if verbose:
+            display_logs_depending_process_time(
+                f'\t> Done with type "{key}" in {round(time() - start_type, 3)}s ({idx+1}/{len(test_funcs)})',
+                time() - start_type
+            )
+    if verbose:
+        display_logs_depending_process_time(f"Done testing columns in {round(time() - start, 3)}s", time() - start)
+    return return_table
+
+
+def test_label(table: pd.DataFrame, all_tests: list, limited_output: bool, verbose: bool = False):
+    # Initialising dict for tests
+    if verbose:
+        start = time()
+        logging.info("Testing labels to get types")
+    test_funcs = dict()
+    for test in all_tests:
+        name = test.__name__.split(".")[-1]
+        test_funcs[name] = {"func": test._is, "prop": test.PROPORTION}
+
+    return_table = pd.DataFrame(columns=table.columns)
+    for idx, (key, value) in enumerate(test_funcs.items()):
+        if verbose:
+            start_type = time()
+        return_table.loc[key] = [
+            test_col_label(
+                col_name, value["func"], value["prop"], limited_output=limited_output
+            )
+            for col_name in table.columns
+        ]
+        if verbose:
+            display_logs_depending_process_time(
+                f'\t- Done with type "{key}" in {round(time() - start_type, 3)}s ({idx+1}/{len(test_funcs)})',
+                time() - start_type
+            )
+    if verbose:
+        display_logs_depending_process_time(f"Done testing labels in {round(time() - start, 3)}s", time() - start)
+    return return_table

csv_detective/parsing/compression.py

@@ -0,0 +1,11 @@
+import gzip
+from io import BytesIO
+
+
+def unzip(binary_file: BytesIO, engine: str) -> BytesIO:
+    if engine == "gzip":
+        with gzip.open(binary_file, mode="rb") as binary_file:
+            file_content = binary_file.read()
+    else:
+        raise NotImplementedError(f"{engine} is not yet supported")
+    return BytesIO(file_content)

csv_detective/parsing/csv.py

@@ -0,0 +1,55 @@
+import logging
+from time import time
+from typing import TextIO
+
+import pandas as pd
+
+from csv_detective.utils import display_logs_depending_process_time
+
+
+def parse_csv(
+    the_file: TextIO,
+    encoding: str,
+    sep: str,
+    num_rows: int,
+    skiprows: int,
+    random_state: int = 42,
+    verbose: bool = False,
+) -> tuple[pd.DataFrame, int, int]:
+    if verbose:
+        start = time()
+        logging.info("Parsing table")
+    table = None
+
+    if not isinstance(the_file, str):
+        the_file.seek(0)
+
+    total_lines = None
+    for encoding in [encoding, "ISO-8859-1", "utf-8"]:
+        if encoding is None:
+            continue
+
+        if "ISO-8859" in encoding:
+            encoding = "ISO-8859-1"
+        try:
+            table = pd.read_csv(
+                the_file, sep=sep, dtype="unicode", encoding=encoding, skiprows=skiprows
+            )
+            total_lines = len(table)
+            nb_duplicates = len(table.loc[table.duplicated()])
+            if num_rows > 0:
+                num_rows = min(num_rows - 1, total_lines)
+                table = table.sample(num_rows, random_state=random_state)
+            # else : table is unchanged
+            break
+        except TypeError:
+            print("Trying encoding : {encoding}".format(encoding=encoding))
+
+    if table is None:
+        raise ValueError("Could not load file")
+    if verbose:
+        display_logs_depending_process_time(
+            f'Table parsed successfully in {round(time() - start, 3)}s',
+            time() - start,
+        )
+    return table, total_lines, nb_duplicates

csv_detective/parsing/excel.py

@@ -0,0 +1,169 @@
+from io import BytesIO
+from time import time
+from typing import Optional
+
+import openpyxl
+import pandas as pd
+import requests
+import xlrd
+
+from csv_detective.detection.engine import engine_to_file
+from csv_detective.detection.rows import remove_empty_first_rows
+from csv_detective.utils import (
+    display_logs_depending_process_time,
+    is_url,
+)
+
+NEW_EXCEL_EXT = [".xlsx", ".xlsm", ".xltx", ".xltm"]
+OLD_EXCEL_EXT = [".xls"]
+OPEN_OFFICE_EXT = [".odf", ".ods", ".odt"]
+XLS_LIKE_EXT = NEW_EXCEL_EXT + OLD_EXCEL_EXT + OPEN_OFFICE_EXT
+
+
+def parse_excel(
+    file_path: str,
+    num_rows: int = -1,
+    engine: Optional[str] = None,
+    sheet_name: Optional[str] = None,
+    random_state: int = 42,
+    verbose: bool = False,
+) -> tuple[pd.DataFrame, int, int, str, str, int]:
+    """"Excel-like parsing is really slow, could be a good improvement for future development"""
+    if verbose:
+        start = time()
+    no_sheet_specified = sheet_name is None
+
+    if (
+        engine in ['openpyxl', 'xlrd'] or
+        any([file_path.endswith(k) for k in NEW_EXCEL_EXT + OLD_EXCEL_EXT])
+    ):
+        remote_content = None
+        if is_url(file_path):
+            r = requests.get(file_path)
+            r.raise_for_status()
+            remote_content = BytesIO(r.content)
+        if not engine:
+            if any([file_path.endswith(k) for k in NEW_EXCEL_EXT]):
+                engine = "openpyxl"
+            else:
+                engine = "xlrd"
+        if sheet_name is None:
+            if verbose:
+                display_logs_depending_process_time(
+                    f'Detected {engine_to_file[engine]} file, no sheet specified, reading the largest one',
+                    time() - start,
+                )
+            try:
+                if engine == "openpyxl":
+                    # openpyxl doesn't want to open files that don't have a valid extension
+                    # see: https://foss.heptapod.net/openpyxl/openpyxl/-/issues/2157
+                    # if the file is remote, we have a remote content anyway so it's fine
+                    if not remote_content and '.' not in file_path.split('/')[-1]:
+                        with open(file_path, 'rb') as f:
+                            remote_content = BytesIO(f.read())
+                    # faster than loading all sheets
+                    wb = openpyxl.load_workbook(remote_content or file_path, read_only=True)
+                    try:
+                        sizes = {s.title: s.max_row * s.max_column for s in wb.worksheets}
+                    except TypeError:
+                        # sometimes read_only can't get the info, so we have to open the file for real
+                        # this takes more time but it's for a limited number of files
+                        # and it's this or nothing
+                        wb = openpyxl.load_workbook(remote_content or file_path)
+                        sizes = {s.title: s.max_row * s.max_column for s in wb.worksheets}
+                else:
+                    if remote_content:
+                        wb = xlrd.open_workbook(file_contents=remote_content.read())
+                    else:
+                        wb = xlrd.open_workbook(file_path)
+                    sizes = {s.name: s.nrows * s.ncols for s in wb.sheets()}
+                sheet_name = max(sizes, key=sizes.get)
+            except xlrd.biffh.XLRDError:
+                # sometimes a xls file is recognized as ods
+                if verbose:
+                    display_logs_depending_process_time(
+                        'Could not read file with classic xls reader, trying with ODS',
+                        time() - start,
+                    )
+                engine = "odf"
+
+    if engine == "odf" or any([file_path.endswith(k) for k in OPEN_OFFICE_EXT]):
+        # for ODS files, no way to get sheets' sizes without
+        # loading the file one way or another (pandas or pure odfpy)
+        # so all in one
+        engine = "odf"
+        if sheet_name is None:
+            if verbose:
+                display_logs_depending_process_time(
+                    f'Detected {engine_to_file[engine]} file, no sheet specified, reading the largest one',
+                    time() - start,
+                )
+            tables = pd.read_excel(
+                file_path,
+                engine="odf",
+                sheet_name=None,
+                dtype="unicode",
+            )
+            sizes = {sheet_name: table.size for sheet_name, table in tables.items()}
+            sheet_name = max(sizes, key=sizes.get)
+            if verbose:
+                display_logs_depending_process_time(
+                    f'Going forwards with sheet "{sheet_name}"',
+                    time() - start,
+                )
+            table = tables[sheet_name]
+        else:
+            if verbose:
+                display_logs_depending_process_time(
+                    f'Detected {engine_to_file[engine]} file, reading sheet "{sheet_name}"',
+                    time() - start,
+                )
+            table = pd.read_excel(
+                file_path,
+                engine="odf",
+                sheet_name=sheet_name,
+                dtype="unicode",
+            )
+        table, header_row_idx = remove_empty_first_rows(table)
+        total_lines = len(table)
+        nb_duplicates = len(table.loc[table.duplicated()])
+        if num_rows > 0:
+            num_rows = min(num_rows - 1, total_lines)
+            table = table.sample(num_rows, random_state=random_state)
+        if verbose:
+            display_logs_depending_process_time(
+                f'Table parsed successfully in {round(time() - start, 3)}s',
+                time() - start,
+            )
+        return table, total_lines, nb_duplicates, sheet_name, engine, header_row_idx
+
+    # so here we end up with (old and new) excel files only
+    if verbose:
+        if no_sheet_specified:
+            display_logs_depending_process_time(
+                f'Going forwards with sheet "{sheet_name}"',
+                time() - start,
+            )
+        else:
+            display_logs_depending_process_time(
+                f'Detected {engine_to_file[engine]} file, reading sheet "{sheet_name}"',
+                time() - start,
+            )
+    table = pd.read_excel(
+        file_path,
+        engine=engine,
+        sheet_name=sheet_name,
+        dtype="unicode",
+    )
+    table, header_row_idx = remove_empty_first_rows(table)
+    total_lines = len(table)
+    nb_duplicates = len(table.loc[table.duplicated()])
+    if num_rows > 0:
+        num_rows = min(num_rows - 1, total_lines)
+        table = table.sample(num_rows, random_state=random_state)
+    if verbose:
+        display_logs_depending_process_time(
+            f'Table parsed successfully in {round(time() - start, 3)}s',
+            time() - start,
+        )
+    return table, total_lines, nb_duplicates, sheet_name, engine, header_row_idx

csv_detective/parsing/load.py

@@ -0,0 +1,97 @@
+from io import BytesIO, StringIO
+from typing import Union
+
+import pandas as pd
+import requests
+
+from csv_detective.detection.columns import detect_heading_columns, detect_trailing_columns
+from csv_detective.detection.encoding import detect_encoding
+from csv_detective.detection.engine import (
+    COMPRESSION_ENGINES,
+    EXCEL_ENGINES,
+    detect_engine,
+)
+from csv_detective.detection.headers import detect_headers
+from csv_detective.detection.separator import detect_separator
+from csv_detective.utils import is_url
+from .compression import unzip
+from .csv import parse_csv
+from .excel import (
+    XLS_LIKE_EXT,
+    parse_excel,
+)
+
+
+def load_file(
+    file_path: str,
+    num_rows: int = 500,
+    encoding: str = None,
+    sep: str = None,
+    verbose: bool = False,
+    sheet_name: Union[str, int] = None,
+) -> tuple[pd.DataFrame, dict]:
+    file_name = file_path.split('/')[-1]
+    engine = None
+    if '.' not in file_name or not file_name.endswith("csv"):
+        # file has no extension, we'll investigate how to read it
+        engine = detect_engine(file_path, verbose=verbose)
+
+    if engine in EXCEL_ENGINES or any([file_path.endswith(k) for k in XLS_LIKE_EXT]):
+        table, total_lines, nb_duplicates, sheet_name, engine, header_row_idx = parse_excel(
+            file_path=file_path,
+            num_rows=num_rows,
+            engine=engine,
+            sheet_name=sheet_name,
+            verbose=verbose,
+        )
+        header = table.columns.to_list()
+        analysis = {
+            "engine": engine,
+            "sheet_name": sheet_name,
+        }
+    else:
+        # fetching or reading file as binary
+        if is_url(file_path):
+            r = requests.get(file_path, allow_redirects=True)
+            r.raise_for_status()
+            binary_file = BytesIO(r.content)
+        else:
+            binary_file = open(file_path, "rb")
+        # handling compression
+        if engine in COMPRESSION_ENGINES:
+            binary_file: BytesIO = unzip(binary_file=binary_file, engine=engine)
+        # detecting encoding if not specified
+        if encoding is None:
+            encoding: str = detect_encoding(binary_file, verbose=verbose)
+            binary_file.seek(0)
+        # decoding and reading file
+        if is_url(file_path) or engine in COMPRESSION_ENGINES:
+            str_file = StringIO(binary_file.read().decode(encoding=encoding))
+        else:
+            str_file = open(file_path, "r", encoding=encoding)
+        if sep is None:
+            sep = detect_separator(str_file, verbose=verbose)
+        header_row_idx, header = detect_headers(str_file, sep, verbose=verbose)
+        if header is None:
+            return {"error": True}
+        elif isinstance(header, list):
+            if any([x is None for x in header]):
+                return {"error": True}
+        heading_columns = detect_heading_columns(str_file, sep, verbose=verbose)
+        trailing_columns = detect_trailing_columns(str_file, sep, heading_columns, verbose=verbose)
+        table, total_lines, nb_duplicates = parse_csv(
+            str_file, encoding, sep, num_rows, header_row_idx, verbose=verbose
+        )
+        analysis = {
+            "encoding": encoding,
+            "separator": sep,
+            "heading_columns": heading_columns,
+            "trailing_columns": trailing_columns,
+        }
+    analysis.update({
+        "header_row_idx": header_row_idx,
+        "header": header,
+        "total_lines": total_lines,
+        "nb_duplicates": nb_duplicates,
+    })
+    return table, analysis