csv-detective 0.6.7__py3-none-any.whl → 0.9.3.dev2438__py3-none-any.whl

csv_detective/output/utils.py

@@ -0,0 +1,74 @@
+import pandas as pd
+
+
+def prepare_output_dict(return_table: pd.DataFrame, limited_output: bool):
+    # -> dict[str, dict | list[dict]] (to be added when upgrading to python>=3.10)
+    return_dict_cols = return_table.to_dict("dict")
+    output_dict = {}
+    for column_name in return_dict_cols:
+        # keep only formats with a non-zero score
+        output_dict[column_name] = [
+            {
+                "format": detected_value_type,
+                "score": return_dict_cols[column_name][detected_value_type],
+            }
+            for detected_value_type in return_dict_cols[column_name]
+            if return_dict_cols[column_name][detected_value_type] > 0
+        ]
+        priorities = [
+            # no need to specify int and float everywhere, they are deprioritized anyway
+            ("int", ("float",)),
+            # bool over everything
+            (
+                "booleen",
+                (
+                    "latitude_l93",
+                    "latitude_wgs",
+                    "latitude_wgs_fr_metropole",
+                    "longitude_l93",
+                    "longitude_wgs",
+                    "longitude_wgs_fr_metropole",
+                ),
+            ),
+            ("geojson", ("json",)),
+            # latlon over lonlat if no longitude allows to discriminate
+            ("latlon_wgs", ("json", "lonlat_wgs")),
+            ("lonlat_wgs", ("json",)),
+            ("latitude_wgs_fr_metropole", ("latitude_l93", "latitude_wgs")),
+            ("longitude_wgs_fr_metropole", ("longitude_l93", "longitude_wgs")),
+            ("latitude_wgs", ("latitude_l93",)),
+            ("longitude_wgs", ("longitude_l93",)),
+            ("code_region", ("code_departement",)),
+            ("datetime_rfc822", ("datetime_aware",)),
+        ]
+        detected_formats = set(x["format"] for x in output_dict[column_name])
+        formats_to_remove = set()
+        # Deprioritise float and int detection vs others
+        if len(detected_formats - {"float", "int"}) > 0:
+            formats_to_remove = formats_to_remove.union({"float", "int"})
+        # Deprioritize less specific formats if:
+        # secondary score is even or worse
+        # or priority score is at least 1 (max of the field score)
+        for prio_format, secondary_formats in priorities:
+            if prio_format in detected_formats:
+                for secondary in secondary_formats:
+                    if secondary in detected_formats and (
+                        return_dict_cols[column_name][prio_format]
+                        >= return_dict_cols[column_name][secondary]
+                        or return_dict_cols[column_name][prio_format] >= 1
+                    ):
+                        formats_to_remove.add(secondary)
+
+        formats_to_keep = detected_formats - formats_to_remove
+
+        detections = [x for x in output_dict[column_name] if x["format"] in formats_to_keep]
+        if not limited_output:
+            output_dict[column_name] = detections
+        else:
+            output_dict[column_name] = (
+                max(detections, key=lambda x: x["score"])
+                if len(detections) > 0
+                else {"format": "string", "score": 1.0}
+            )
+
+    return output_dict

csv_detective/parsing/columns.py

@@ -0,0 +1,235 @@
+import logging
+from time import time
+from typing import Callable
+
+import pandas as pd
+from more_itertools import peekable
+
+from csv_detective.format import Format
+from csv_detective.parsing.csv import CHUNK_SIZE
+from csv_detective.utils import display_logs_depending_process_time
+
+# above this threshold, a column is not considered categorical
+MAX_NUMBER_CATEGORICAL_VALUES = 25
+
+
+def test_col_val(
+    serie: pd.Series,
+    format: Format,
+    skipna: bool = True,
+    limited_output: bool = False,
+    verbose: bool = False,
+) -> float:
+    """Tests values of the serie using test_func.
+         - skipna : if True indicates that NaNs are considered True
+    for the serie to be detected as a certain format
+    """
+    if verbose:
+        start = time()
+
+    # TODO : change for a cleaner method and only test columns in modules labels
+    def apply_test_func(serie: pd.Series, test_func: Callable, _range: int):
+        return serie.sample(n=_range).apply(test_func)
+
+    try:
+        if skipna:
+            serie = serie.loc[serie.notnull()]
+        ser_len = len(serie)
+        if ser_len == 0:
+            # being here means the whole column is NaN, so if skipna it's a pass
+            return 1.0 if skipna else 0.0
+        if not limited_output or format.proportion < 1:
+            # we want or have to go through the whole column to have the proportion
+            result: float = serie.apply(format.func).sum() / ser_len
+            return result if result >= format.proportion else 0.0
+        else:
+            # the whole column has to be valid so we have early stops (1 then 5 rows)
+            # to not waste time if directly unsuccessful
+            for _range in [
+                min(1, ser_len),
+                min(5, ser_len),
+                ser_len,
+            ]:
+                if not all(apply_test_func(serie, format.func, _range)):
+                    return 0.0
+            return 1.0
+    finally:
+        if verbose and time() - start > 3:
+            display_logs_depending_process_time(
+                f"\t/!\\ Column '{serie.name}' took too long ({round(time() - start, 3)}s)",
+                time() - start,
+            )
+
+
+def test_col(
+    table: pd.DataFrame,
+    formats: dict[str, Format],
+    limited_output: bool,
+    skipna: bool = True,
+    verbose: bool = False,
+):
+    if verbose:
+        start = time()
+        logging.info("Testing columns to get formats")
+    return_table = pd.DataFrame(columns=table.columns)
+    for idx, (label, format) in enumerate(formats.items()):
+        if verbose:
+            start_type = time()
+            logging.info(f"\t- Starting with format '{label}'")
+        # improvement lead : put the longest tests behind and make them only if previous tests not satisfactory
+        # => the following needs to change, "apply" means all columns are tested for one type at once
+        return_table.loc[label] = table.apply(
+            lambda serie: test_col_val(
+                serie,
+                format,
+                skipna=skipna,
+                limited_output=limited_output,
+                verbose=verbose,
+            )
+        )
+        if verbose:
+            display_logs_depending_process_time(
+                f'\t> Done with type "{label}" in {round(time() - start_type, 3)}s ({idx + 1}/{len(formats)})',
+                time() - start_type,
+            )
+    if verbose:
+        display_logs_depending_process_time(
+            f"Done testing columns in {round(time() - start, 3)}s", time() - start
+        )
+    return return_table
+
+
+def test_label(
+    columns: list[str], formats: dict[str, Format], limited_output: bool, verbose: bool = False
+):
+    if verbose:
+        start = time()
+        logging.info("Testing labels to get types")
+
+    return_table = pd.DataFrame(columns=columns)
+    for idx, (label, format) in enumerate(formats.items()):
+        if verbose:
+            start_type = time()
+        return_table.loc[label] = [format.is_valid_label(col_name) for col_name in columns]
+        if verbose:
+            display_logs_depending_process_time(
+                f'\t- Done with type "{label}" in {round(time() - start_type, 3)}s ({idx + 1}/{len(formats)})',
+                time() - start_type,
+            )
+    if verbose:
+        display_logs_depending_process_time(
+            f"Done testing labels in {round(time() - start, 3)}s", time() - start
+        )
+    return return_table
+
+
+def test_col_chunks(
+    table: pd.DataFrame,
+    file_path: str,
+    analysis: dict,
+    formats: dict[str, Format],
+    limited_output: bool,
+    skipna: bool = True,
+    verbose: bool = False,
+) -> tuple[pd.DataFrame, dict, dict[str, pd.Series]]:
+    def build_remaining_tests_per_col(return_table: pd.DataFrame) -> dict[str, list[str]]:
+        # returns a dict with the table's columns as keys and the list of remaining format labels to apply
+        return {
+            col: [
+                fmt_label
+                for fmt_label in return_table.index
+                if return_table.loc[fmt_label, col] > 0
+            ]
+            for col in return_table.columns
+        }
+
+    if verbose:
+        start = time()
+        logging.info("Testing columns to get formats on chunks")
+
+    # analysing the sample to get a first guess
+    return_table = test_col(table, formats, limited_output, skipna=skipna, verbose=verbose)
+    remaining_tests_per_col = build_remaining_tests_per_col(return_table)
+
+    # hashing rows to get nb_duplicates
+    row_hashes_count = table.apply(lambda row: hash(tuple(row)), axis=1).value_counts()
+    # getting values for profile to read the file only once
+    col_values = {col: table[col].value_counts(dropna=False) for col in table.columns}
+
+    # only csv files can end up here, can't chunk excel
+    chunks = pd.read_csv(
+        file_path,
+        dtype=str,
+        encoding=analysis["encoding"],
+        sep=analysis["separator"],
+        skiprows=analysis["header_row_idx"],
+        compression=analysis.get("compression"),
+        chunksize=CHUNK_SIZE,
+    )
+    analysis["total_lines"] = CHUNK_SIZE
+    batch, batch_number = [], 1
+    iterator = peekable(enumerate(chunks))
+    while iterator:
+        idx, chunk = next(iterator)
+        if idx == 0:
+            # we have read and analysed the first chunk already
+            continue
+        if len(batch) < 10:
+            # it's too slow to process chunks directly, but we want to keep the first analysis
+            # on a "small" chunk, so partial analyses are done on batches of chunks
+            batch.append(chunk)
+            # we don't know when the chunks end, and doing one additionnal step
+            # for the final batch is ugly
+            try:
+                iterator.peek()
+                continue
+            except StopIteration:
+                pass
+        if verbose:
+            logging.info(f"> Testing batch number {batch_number}")
+        batch = pd.concat(batch, ignore_index=True)
+        analysis["total_lines"] += len(batch)
+        row_hashes_count = row_hashes_count.add(
+            batch.apply(lambda row: hash(tuple(row)), axis=1).value_counts(),
+            fill_value=0,
+        )
+        for col in batch.columns:
+            col_values[col] = col_values[col].add(
+                batch[col].value_counts(dropna=False),
+                fill_value=0,
+            )
+        if not any(remaining_tests for remaining_tests in remaining_tests_per_col.values()):
+            # no more potential tests to do on any column, early stop
+            break
+        for col, fmt_labels in remaining_tests_per_col.items():
+            # testing each column with the tests that are still competing
+            # after previous batchs analyses
+            for label in fmt_labels:
+                batch_col_test = test_col_val(
+                    batch[col],
+                    formats[label],
+                    limited_output=limited_output,
+                    skipna=skipna,
+                )
+                return_table.loc[label, col] = (
+                    # if this batch's column tested 0 then test fails overall
+                    0
+                    if batch_col_test == 0
+                    # otherwise updating the score with weighted average
+                    else ((return_table.loc[label, col] * idx + batch_col_test) / (idx + 1))
+                )
+        remaining_tests_per_col = build_remaining_tests_per_col(return_table)
+        batch, batch_number = [], batch_number + 1
+    analysis["nb_duplicates"] = sum(row_hashes_count > 1)
+    analysis["categorical"] = [
+        col for col, values in col_values.items() if len(values) <= MAX_NUMBER_CATEGORICAL_VALUES
+    ]
+    # handling that empty columns score 1 everywhere
+    for col in return_table.columns:
+        if sum(return_table[col]) == len(return_table):
+            return_table[col] = 0
+    if verbose:
+        display_logs_depending_process_time(
+            f"Done testing chunks in {round(time() - start, 3)}s", time() - start
+        )
+    return return_table, analysis, col_values

csv_detective/parsing/compression.py

@@ -0,0 +1,11 @@
+import gzip
+from io import BytesIO
+
+
+def unzip(binary_file: BytesIO, engine: str) -> BytesIO:
+    if engine == "gzip":
+        with gzip.open(binary_file, mode="rb") as binary_file:
+            file_content = binary_file.read()
+    else:
+        raise NotImplementedError(f"{engine} is not yet supported")
+    return BytesIO(file_content)

csv_detective/parsing/csv.py

@@ -0,0 +1,56 @@
+import logging
+from time import time
+from typing import TextIO
+
+import pandas as pd
+
+from csv_detective.utils import display_logs_depending_process_time
+
+# the number of rows for the first analysis, and the number of rows per chunk of the df iterator
+CHUNK_SIZE = int(1e4)
+
+
+def parse_csv(
+    the_file: TextIO,
+    encoding: str,
+    sep: str,
+    num_rows: int,
+    skiprows: int,
+    random_state: int = 42,
+    verbose: bool = False,
+) -> tuple[pd.DataFrame, int | None, int | None]:
+    if verbose:
+        start = time()
+        logging.info("Parsing table")
+
+    if not isinstance(the_file, str):
+        the_file.seek(0)
+
+    try:
+        table = pd.read_csv(
+            the_file,
+            sep=sep,
+            dtype=str,
+            encoding=encoding,
+            skiprows=skiprows,
+            nrows=CHUNK_SIZE,
+        )
+        total_lines = len(table)
+        # branch between small and big files starts here
+        if total_lines == CHUNK_SIZE:
+            if verbose:
+                logging.warning(f"File is too long, analysing in chunks of {CHUNK_SIZE} rows")
+            total_lines, nb_duplicates = None, None
+        else:
+            nb_duplicates = len(table.loc[table.duplicated()])
+        if num_rows > 0:
+            num_rows = min(num_rows, total_lines or len(table))
+            table = table.sample(num_rows, random_state=random_state)
+    except Exception as e:
+        raise ValueError("Could not load file") from e
+    if verbose:
+        display_logs_depending_process_time(
+            f"Table parsed successfully in {round(time() - start, 3)}s",
+            time() - start,
+        )
+    return table, total_lines, nb_duplicates

csv_detective/parsing/excel.py

@@ -0,0 +1,167 @@
+from io import BytesIO
+from time import time
+
+import openpyxl
+import pandas as pd
+import requests
+import xlrd
+
+from csv_detective.detection.engine import engine_to_file
+from csv_detective.detection.rows import remove_empty_first_rows
+from csv_detective.utils import (
+    display_logs_depending_process_time,
+    is_url,
+)
+
+NEW_EXCEL_EXT = [".xlsx", ".xlsm", ".xltx", ".xltm"]
+OLD_EXCEL_EXT = [".xls"]
+OPEN_OFFICE_EXT = [".odf", ".ods", ".odt"]
+XLS_LIKE_EXT = NEW_EXCEL_EXT + OLD_EXCEL_EXT + OPEN_OFFICE_EXT
+
+
+def parse_excel(
+    file_path: str,
+    num_rows: int = -1,
+    engine: str | None = None,
+    sheet_name: str | None = None,
+    random_state: int = 42,
+    verbose: bool = False,
+) -> tuple[pd.DataFrame, int, int, str, str, int]:
+    """ "Excel-like parsing is really slow, could be a good improvement for future development"""
+    if verbose:
+        start = time()
+    no_sheet_specified = sheet_name is None
+
+    if engine in ["openpyxl", "xlrd"] or any(
+        [file_path.endswith(k) for k in NEW_EXCEL_EXT + OLD_EXCEL_EXT]
+    ):
+        remote_content = None
+        if is_url(file_path):
+            r = requests.get(file_path)
+            r.raise_for_status()
+            remote_content = BytesIO(r.content)
+        if not engine:
+            if any([file_path.endswith(k) for k in NEW_EXCEL_EXT]):
+                engine = "openpyxl"
+            else:
+                engine = "xlrd"
+        if sheet_name is None:
+            if verbose:
+                display_logs_depending_process_time(
+                    f"Detected {engine_to_file[engine]} file, no sheet specified, reading the largest one",
+                    time() - start,
+                )
+            try:
+                if engine == "openpyxl":
+                    # openpyxl doesn't want to open files that don't have a valid extension
+                    # see: https://foss.heptapod.net/openpyxl/openpyxl/-/issues/2157
+                    # if the file is remote, we have a remote content anyway so it's fine
+                    if not remote_content and "." not in file_path.split("/")[-1]:
+                        with open(file_path, "rb") as f:
+                            remote_content = BytesIO(f.read())
+                    # faster than loading all sheets
+                    wb = openpyxl.load_workbook(remote_content or file_path, read_only=True)
+                    try:
+                        sizes = {s.title: s.max_row * s.max_column for s in wb.worksheets}
+                    except TypeError:
+                        # sometimes read_only can't get the info, so we have to open the file for real
+                        # this takes more time but it's for a limited number of files
+                        # and it's this or nothing
+                        wb = openpyxl.load_workbook(remote_content or file_path)
+                        sizes = {s.title: s.max_row * s.max_column for s in wb.worksheets}
+                else:
+                    if remote_content:
+                        wb = xlrd.open_workbook(file_contents=remote_content.read())
+                    else:
+                        wb = xlrd.open_workbook(file_path)
+                    sizes = {s.name: s.nrows * s.ncols for s in wb.sheets()}
+                sheet_name = max(sizes, key=sizes.get)
+            except xlrd.biffh.XLRDError:
+                # sometimes a xls file is recognized as ods
+                if verbose:
+                    display_logs_depending_process_time(
+                        "Could not read file with classic xls reader, trying with ODS",
+                        time() - start,
+                    )
+                engine = "odf"
+
+    if engine == "odf" or any([file_path.endswith(k) for k in OPEN_OFFICE_EXT]):
+        # for ODS files, no way to get sheets' sizes without
+        # loading the file one way or another (pandas or pure odfpy)
+        # so all in one
+        engine = "odf"
+        if sheet_name is None:
+            if verbose:
+                display_logs_depending_process_time(
+                    f"Detected {engine_to_file[engine]} file, no sheet specified, reading the largest one",
+                    time() - start,
+                )
+            tables = pd.read_excel(
+                file_path,
+                engine="odf",
+                sheet_name=None,
+                dtype=str,
+            )
+            sizes = {sheet_name: table.size for sheet_name, table in tables.items()}
+            sheet_name = max(sizes, key=sizes.get)
+            if verbose:
+                display_logs_depending_process_time(
+                    f'Going forwards with sheet "{sheet_name}"',
+                    time() - start,
+                )
+            table = tables[sheet_name]
+        else:
+            if verbose:
+                display_logs_depending_process_time(
+                    f'Detected {engine_to_file[engine]} file, reading sheet "{sheet_name}"',
+                    time() - start,
+                )
+            table = pd.read_excel(
+                file_path,
+                engine="odf",
+                sheet_name=sheet_name,
+                dtype=str,
+            )
+        table, header_row_idx = remove_empty_first_rows(table)
+        total_lines = len(table)
+        nb_duplicates = len(table.loc[table.duplicated()])
+        if num_rows > 0:
+            num_rows = min(num_rows - 1, total_lines)
+            table = table.sample(num_rows, random_state=random_state)
+        if verbose:
+            display_logs_depending_process_time(
+                f"Table parsed successfully in {round(time() - start, 3)}s",
+                time() - start,
+            )
+        return table, total_lines, nb_duplicates, sheet_name, engine, header_row_idx
+
+    # so here we end up with (old and new) excel files only
+    if verbose:
+        if no_sheet_specified:
+            display_logs_depending_process_time(
+                f'Going forwards with sheet "{sheet_name}"',
+                time() - start,
+            )
+        else:
+            display_logs_depending_process_time(
+                f'Detected {engine_to_file[engine]} file, reading sheet "{sheet_name}"',
+                time() - start,
+            )
+    table = pd.read_excel(
+        file_path,
+        engine=engine,
+        sheet_name=sheet_name,
+        dtype=str,
+    )
+    table, header_row_idx = remove_empty_first_rows(table)
+    total_lines = len(table)
+    nb_duplicates = len(table.loc[table.duplicated()])
+    if num_rows > 0:
+        num_rows = min(num_rows - 1, total_lines)
+        table = table.sample(num_rows, random_state=random_state)
+    if verbose:
+        display_logs_depending_process_time(
+            f"Table parsed successfully in {round(time() - start, 3)}s",
+            time() - start,
+        )
+    return table, total_lines, nb_duplicates, sheet_name, engine, header_row_idx

csv_detective/parsing/load.py

@@ -0,0 +1,111 @@
+from io import BytesIO, StringIO
+
+import pandas as pd
+import requests
+
+from csv_detective.detection.columns import detect_heading_columns, detect_trailing_columns
+from csv_detective.detection.encoding import detect_encoding
+from csv_detective.detection.engine import (
+    COMPRESSION_ENGINES,
+    EXCEL_ENGINES,
+    detect_engine,
+)
+from csv_detective.detection.headers import detect_headers
+from csv_detective.detection.separator import detect_separator
+from csv_detective.parsing.compression import unzip
+from csv_detective.parsing.csv import parse_csv
+from csv_detective.parsing.excel import (
+    XLS_LIKE_EXT,
+    parse_excel,
+)
+from csv_detective.utils import is_url
+
+
+def load_file(
+    file_path: str,
+    num_rows: int = 500,
+    encoding: str | None = None,
+    sep: str | None = None,
+    verbose: bool = False,
+    sheet_name: str | int | None = None,
+) -> tuple[pd.DataFrame, dict]:
+    file_name = file_path.split("/")[-1]
+    engine = None
+    if "." not in file_name or not file_name.endswith("csv"):
+        # file has no extension, we'll investigate how to read it
+        engine = detect_engine(file_path, verbose=verbose)
+
+    if engine in EXCEL_ENGINES or any([file_path.endswith(k) for k in XLS_LIKE_EXT]):
+        table, total_lines, nb_duplicates, sheet_name, engine, header_row_idx = parse_excel(
+            file_path=file_path,
+            num_rows=num_rows,
+            engine=engine,
+            sheet_name=sheet_name,
+            verbose=verbose,
+        )
+        if table.empty:
+            raise ValueError("Table seems to be empty")
+        header = table.columns.to_list()
+        if any(col.startswith("Unnamed") for col in header):
+            raise ValueError("Could not retrieve headers")
+        analysis = {
+            "engine": engine,
+            "sheet_name": sheet_name,
+        }
+    else:
+        # fetching or reading file as binary
+        if is_url(file_path):
+            r = requests.get(file_path, allow_redirects=True)
+            r.raise_for_status()
+            binary_file = BytesIO(r.content)
+        else:
+            binary_file = open(file_path, "rb")
+        # handling compression
+        if engine in COMPRESSION_ENGINES:
+            binary_file: BytesIO = unzip(binary_file=binary_file, engine=engine)
+        # detecting encoding if not specified
+        if encoding is None:
+            encoding: str = detect_encoding(binary_file, verbose=verbose)
+            binary_file.seek(0)
+        # decoding and reading file
+        if is_url(file_path) or engine in COMPRESSION_ENGINES:
+            str_file = StringIO()
+            while True:
+                chunk = binary_file.read(1024**2)
+                if not chunk:
+                    break
+                str_file.write(chunk.decode(encoding=encoding))
+            del binary_file
+            str_file.seek(0)
+        else:
+            str_file = open(file_path, "r", encoding=encoding)
+        if sep is None:
+            sep = detect_separator(str_file, verbose=verbose)
+        header_row_idx, header = detect_headers(str_file, sep, verbose=verbose)
+        if header is None or (isinstance(header, list) and any([h is None for h in header])):
+            raise ValueError("Could not retrieve headers")
+        heading_columns = detect_heading_columns(str_file, sep, verbose=verbose)
+        trailing_columns = detect_trailing_columns(str_file, sep, heading_columns, verbose=verbose)
+        table, total_lines, nb_duplicates = parse_csv(
+            str_file, encoding, sep, num_rows, header_row_idx, verbose=verbose
+        )
+        del str_file
+        if table.empty:
+            raise ValueError("Table seems to be empty")
+        analysis = {
+            "encoding": encoding,
+            "separator": sep,
+            "heading_columns": heading_columns,
+            "trailing_columns": trailing_columns,
+        }
+        if engine is not None:
+            analysis["compression"] = engine
+    analysis |= {
+        "header_row_idx": header_row_idx,
+        "header": header,
+    }
+    if total_lines is not None:
+        analysis["total_lines"] = total_lines
+    if nb_duplicates is not None:
+        analysis["nb_duplicates"] = nb_duplicates
+    return table, analysis