csv-detective 0.6.7__py3-none-any.whl → 0.9.3.dev2438__py3-none-any.whl

csv_detective/__init__.py

@@ -1 +1,7 @@
-__version__ = '0.6.7'
+from csv_detective.explore_csv import routine, validate, validate_then_detect
+
+__all__ = [
+    "routine",
+    "validate",
+    "validate_then_detect",
+]

csv_detective/cli.py

@@ -4,41 +4,53 @@ Command line client for csv_detective
 
 import argparse
 import json
-from .explore_csv import routine
+
+from csv_detective.explore_csv import routine
 
 
 def run():
-    explorer = argparse.ArgumentParser(description='Get the arguments we want')
+    explorer = argparse.ArgumentParser(description="Analyse a tabular file")
+    explorer.add_argument("file_path", type=str, help="Enter path of tabular file to explore")
+    explorer.add_argument(
+        "-n",
+        "--num_rows",
+        dest="num_rows",
+        type=int,
+        nargs="?",
+        help="Number of rows to use for detection (default 500)",
+    )
     explorer.add_argument(
-        'file_path',
+        "-s",
+        "--sep",
+        dest="sep",
         type=str,
-        help='Enter path of csv file to explore'
+        nargs="?",
+        help="Columns separator (detected if not specified)",
     )
     explorer.add_argument(
-        '-n',
-        '--num_rows',
-        dest='num_rows',
+        "--save",
+        dest="save_results",
         type=int,
-        nargs='?',
-        help='Number of rows to use for detection'
+        nargs="?",
+        help="Whether to save the resulting analysis to json (1 = save, 0 = don't)",
     )
     explorer.add_argument(
-        '-t',
-        '--select_tests',
-        dest='city',
-        type=str,
-        nargs='*',
-        help='List of tests to be performed (use "" if you want to use the dash option to remove tests)'
+        "-v",
+        "--verbose",
+        dest="verbose",
+        type=int,
+        nargs="?",
+        help="Verbose (0 = quiet, 1 = details)",
     )
 
     opts = explorer.parse_args()
 
-    num_rows = opts.num_rows or 50
     inspection_results = routine(
-        opts.file_path,
-        num_rows=num_rows,
-        user_input_tests='ALL',
-        output_mode='ALL'
+        csv_file_path=opts.file_path,
+        num_rows=opts.num_rows,
+        sep=opts.sep,
+        save_results=bool(opts.save_results),
+        verbose=bool(opts.verbose),
     )
 
-    print(json.dumps(inspection_results, indent=4, sort_keys=True, ensure_ascii=False))
+    print(json.dumps(inspection_results, indent=4, ensure_ascii=False))

csv_detective/detection/columns.py

@@ -0,0 +1,89 @@
+import logging
+from time import time
+from typing import TextIO
+
+from csv_detective.utils import display_logs_depending_process_time
+
+
+def detect_extra_columns(file: TextIO, sep: str):
+    """regarde s'il y a des colonnes en trop
+    Attention, file ne doit pas avoir de ligne vide"""
+    file.seek(0)
+    retour = False
+    nb_useless_col = 99999
+
+    for i in range(10):
+        line = file.readline()
+        # regarde si on a un retour
+        if retour:
+            assert line[-1] == "\n"
+        if line[-1] == "\n":
+            retour = True
+
+        # regarde le nombre de derniere colonne inutile
+        deb = 0 + retour
+        line = line[::-1][deb:]
+        k = 0
+        for sign in line:
+            if sign != sep:
+                break
+            k += 1
+        if k == 0:
+            return 0, retour
+        nb_useless_col = min(k, nb_useless_col)
+    return nb_useless_col, retour
+
+
+def detect_heading_columns(file: TextIO, sep: str, verbose: bool = False) -> int:
+    """Tests first 10 lines to see if there are empty heading columns"""
+    if verbose:
+        start = time()
+        logging.info("Detecting heading columns")
+    file.seek(0)
+    return_int = float("Inf")
+    for i in range(10):
+        line = file.readline()
+        return_int = min(return_int, len(line) - len(line.strip(sep)))
+        if return_int == 0:
+            if verbose:
+                display_logs_depending_process_time(
+                    f"No heading column detected in {round(time() - start, 3)}s",
+                    time() - start,
+                )
+            return 0
+    if verbose:
+        display_logs_depending_process_time(
+            f"{return_int} heading columns detected in {round(time() - start, 3)}s",
+            time() - start,
+        )
+    return return_int
+
+
+def detect_trailing_columns(
+    file: TextIO, sep: str, heading_columns: int, verbose: bool = False
+) -> int:
+    """Tests first 10 lines to see if there are empty trailing columns"""
+    if verbose:
+        start = time()
+        logging.info("Detecting trailing columns")
+    file.seek(0)
+    return_int = float("Inf")
+    for i in range(10):
+        line = file.readline()
+        return_int = min(
+            return_int,
+            len(line.replace("\n", "")) - len(line.replace("\n", "").strip(sep)) - heading_columns,
+        )
+        if return_int == 0:
+            if verbose:
+                display_logs_depending_process_time(
+                    f"No trailing column detected in {round(time() - start, 3)}s",
+                    time() - start,
+                )
+            return 0
+    if verbose:
+        display_logs_depending_process_time(
+            f"{return_int} trailing columns detected in {round(time() - start, 3)}s",
+            time() - start,
+        )
+    return return_int

csv_detective/detection/encoding.py

@@ -0,0 +1,29 @@
+import logging
+from io import BytesIO
+from time import time
+
+from cchardet import detect
+
+from csv_detective.utils import display_logs_depending_process_time
+
+
+def detect_encoding(binary_file: BytesIO, verbose: bool = False) -> str:
+    """
+    Detects file encoding using faust-cchardet (forked from the original cchardet)
+    """
+    if verbose:
+        start = time()
+        logging.info("Detecting encoding")
+    encoding_dict = detect(binary_file.read())
+    if not encoding_dict["encoding"]:
+        raise ValueError(
+            "Could not detect the file's encoding. Consider specifying it in the routine call."
+        )
+    if verbose:
+        message = f'Detected encoding: "{encoding_dict["encoding"]}"'
+        message += f" in {round(time() - start, 3)}s (confidence: {round(encoding_dict['confidence'] * 100)}%)"
+        display_logs_depending_process_time(
+            message,
+            time() - start,
+        )
+    return encoding_dict["encoding"]

csv_detective/detection/engine.py

@@ -0,0 +1,46 @@
+from time import time
+
+import magic
+import requests
+
+from csv_detective.utils import display_logs_depending_process_time, is_url
+
+COMPRESSION_ENGINES = ["gzip"]
+EXCEL_ENGINES = ["openpyxl", "xlrd", "odf"]
+engine_to_file = {
+    "openpyxl": "Excel",
+    "xlrd": "old Excel",
+    "odf": "OpenOffice",
+    "gzip": "csv.gz",
+}
+
+
+def detect_engine(file_path: str, verbose=False) -> str | None:
+    if verbose:
+        start = time()
+    mapping = {
+        "application/gzip": "gzip",
+        "application/x-gzip": "gzip",
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "openpyxl",
+        "application/vnd.ms-excel": "xlrd",
+        "application/vnd.oasis.opendocument.spreadsheet": "odf",
+        # all these files could be recognized as zip, may need to check all cases then
+        "application/zip": "openpyxl",
+    }
+    # if none of the above, we move forwards with the csv process
+    if is_url(file_path):
+        remote_content = next(requests.get(file_path, stream=True).iter_content(chunk_size=1024))
+        engine = mapping.get(magic.from_buffer(remote_content, mime=True))
+    else:
+        engine = mapping.get(magic.from_file(file_path, mime=True))
+    if verbose:
+        message = (
+            f"File is not csv, detected {engine_to_file.get(engine, 'csv')}"
+            if engine
+            else "Processing the file as a csv"
+        )
+        display_logs_depending_process_time(
+            message,
+            time() - start,
+        )
+    return engine

csv_detective/detection/formats.py

@@ -0,0 +1,156 @@
+from collections import defaultdict
+
+import numpy as np
+import pandas as pd
+
+from csv_detective.detection.variables import (
+    detect_categorical_variable,
+    # detect_continuous_variable,
+)
+from csv_detective.format import Format, FormatsManager
+from csv_detective.output.utils import prepare_output_dict
+from csv_detective.parsing.columns import (
+    MAX_NUMBER_CATEGORICAL_VALUES,
+    test_col,
+    test_col_chunks,
+    test_label,
+)
+
+fmtm = FormatsManager()
+
+
+def detect_formats(
+    table: pd.DataFrame,
+    analysis: dict,
+    file_path: str,
+    tags: list[str] | None = None,
+    limited_output: bool = True,
+    skipna: bool = True,
+    verbose: bool = False,
+) -> tuple[dict, dict[str, pd.Series] | None]:
+    in_chunks = analysis.get("total_lines") is None
+
+    # list testing to be performed
+    formats: dict[str, Format] = (
+        fmtm.get_formats_from_tags(tags) if tags is not None else fmtm.formats
+    )
+
+    # if no testing then return
+    if len(formats) == 0:
+        return analysis, None
+
+    # Perform testing on fields
+    if not in_chunks:
+        # table is small enough to be tested in one go
+        scores_table_fields = test_col(
+            table=table,
+            formats=formats,
+            limited_output=limited_output,
+            skipna=skipna,
+            verbose=verbose,
+        )
+        res_categorical, _ = detect_categorical_variable(
+            table,
+            max_number_categorical_values=MAX_NUMBER_CATEGORICAL_VALUES,
+            verbose=verbose,
+        )
+        analysis["categorical"] = res_categorical
+        col_values = None
+    else:
+        scores_table_fields, analysis, col_values = test_col_chunks(
+            table=table,
+            file_path=file_path,
+            analysis=analysis,
+            formats=formats,
+            limited_output=limited_output,
+            skipna=skipna,
+            verbose=verbose,
+        )
+    analysis["columns_fields"] = prepare_output_dict(scores_table_fields, limited_output)
+
+    # Perform testing on labels
+    scores_table_labels = test_label(analysis["header"], formats, limited_output, verbose=verbose)
+    analysis["columns_labels"] = prepare_output_dict(scores_table_labels, limited_output)
+
+    # Multiply the results of the fields by 1 + 0.5 * the results of the labels.
+    # This is because the fields are more important than the labels and yields a max
+    # of 1.5 for the final score.
+    scores_table = scores_table_fields * (
+        1 + scores_table_labels.reindex(index=scores_table_fields.index, fill_value=0).values / 2
+    )
+
+    # To reduce false positives: ensure these formats are detected only if the label yields
+    # a detection (skipping the ones that have been excluded by the users).
+    formats_with_mandatory_label = [
+        f
+        for f in [
+            "code_departement",
+            "code_commune_insee",
+            "code_postal",
+            "code_fantoir",
+            "latitude_wgs",
+            "longitude_wgs",
+            "latitude_wgs_fr_metropole",
+            "longitude_wgs_fr_metropole",
+            "latitude_l93",
+            "longitude_l93",
+            "siren",
+            "siret",
+        ]
+        if f in scores_table.index
+    ]
+    scores_table.loc[formats_with_mandatory_label, :] = np.where(
+        scores_table_labels.loc[formats_with_mandatory_label, :],
+        scores_table.loc[formats_with_mandatory_label, :],
+        0,
+    )
+    analysis["columns"] = prepare_output_dict(scores_table, limited_output)
+
+    metier_to_python_type = {
+        "booleen": "bool",
+        "int": "int",
+        "float": "float",
+        "string": "string",
+        "json": "json",
+        "geojson": "json",
+        "datetime_aware": "datetime",
+        "datetime_naive": "datetime",
+        "datetime_rfc822": "datetime",
+        "date": "date",
+        "latitude_l93": "float",
+        "latitude_wgs": "float",
+        "latitude_wgs_fr_metropole": "float",
+        "longitude_l93": "float",
+        "longitude_wgs": "float",
+        "longitude_wgs_fr_metropole": "float",
+        "binary": "binary",
+    }
+
+    if not limited_output:
+        for detection_method in ["columns_fields", "columns_labels", "columns"]:
+            analysis[detection_method] = {
+                col_name: [
+                    {
+                        "python_type": metier_to_python_type.get(detection["format"], "string"),
+                        **detection,
+                    }
+                    for detection in detections
+                ]
+                for col_name, detections in analysis[detection_method].items()
+            }
+    else:
+        for detection_method in ["columns_fields", "columns_labels", "columns"]:
+            analysis[detection_method] = {
+                col_name: {
+                    "python_type": metier_to_python_type.get(detection["format"], "string"),
+                    **detection,
+                }
+                for col_name, detection in analysis[detection_method].items()
+            }
+
+        # Add detection with formats as keys
+        analysis["formats"] = defaultdict(list)
+        for header, col_metadata in analysis["columns"].items():
+            analysis["formats"][col_metadata["format"]].append(header)
+
+    return analysis, col_values

csv_detective/detection/headers.py

@@ -0,0 +1,28 @@
+import logging
+from time import time
+from typing import TextIO
+
+from csv_detective.utils import display_logs_depending_process_time
+
+
+def detect_headers(file: TextIO, sep: str, verbose: bool = False) -> tuple[int, list | None]:
+    """Tests 10 first rows for possible header (in case header is not 1st row)"""
+    if verbose:
+        start = time()
+        logging.info("Detecting headers")
+    file.seek(0)
+    for i in range(10):
+        row = file.readline()
+        position = file.tell()
+        headers = [c for c in row.replace("\n", "").split(sep) if c]
+        if not any(col == "" for col in headers):
+            next_row = file.readline()
+            file.seek(position)
+            if row != next_row:
+                if verbose:
+                    display_logs_depending_process_time(
+                        f"Detected headers in {round(time() - start, 3)}s",
+                        time() - start,
+                    )
+                return i, headers
+    raise ValueError("Could not retrieve headers")

csv_detective/detection/rows.py

@@ -0,0 +1,18 @@
+import pandas as pd
+
+
+def remove_empty_first_rows(table: pd.DataFrame) -> tuple[pd.DataFrame, int]:
+    """Analog process to detect_headers for csv files, determines how many rows to skip
+    to end up with the header at the right place"""
+    idx = 0
+    if all([str(c).startswith("Unnamed:") for c in table.columns]):
+        # there is on offset between the index in the file (idx here)
+        # and the index in the dataframe, because of the header
+        idx = 1
+        while table.iloc[idx - 1].isna().all():
+            idx += 1
+        cols = table.iloc[idx - 1]
+        table = table.iloc[idx:]
+        table.columns = cols.to_list()
+    # +1 here because the headers should count as a row
+    return table, idx

csv_detective/detection/separator.py

@@ -0,0 +1,44 @@
+import csv
+import logging
+from time import time
+from typing import TextIO
+
+from csv_detective.utils import display_logs_depending_process_time
+
+
+def detect_separator(file: TextIO, verbose: bool = False) -> str:
+    """Detects csv separator"""
+    # TODO: add a robust detection:
+    # si on a un point virgule comme texte et \t comme séparateur, on renvoie
+    # pour l'instant un point virgule
+    if verbose:
+        start = time()
+        logging.info("Detecting separator")
+    file.seek(0)
+    header = file.readline()
+    possible_separators = [";", ",", "|", "\t"]
+    sep_count = dict()
+    for sep in possible_separators:
+        sep_count[sep] = header.count(sep)
+    sep = max(sep_count, key=sep_count.get)
+    # testing that the first 10 (arbitrary) rows all have the same number of fields
+    # as the header. Prevents downstream unwanted behaviour where pandas can load
+    # the file (in a weird way) but the process is irrelevant.
+    file.seek(0)
+    reader = csv.reader(file, delimiter=sep)
+    rows_lengths = set()
+    for idx, row in enumerate(reader):
+        if idx > 10:
+            break
+        rows_lengths.add(len(row))
+    if len(rows_lengths) > 1:
+        raise ValueError(
+            f"Number of columns is not even across the first 10 rows (detected separator: {sep})."
+        )
+
+    if verbose:
+        display_logs_depending_process_time(
+            f'Detected separator: "{sep}" in {round(time() - start, 3)}s',
+            time() - start,
+        )
+    return sep

csv_detective/detection/variables.py

@@ -0,0 +1,97 @@
+import logging
+from ast import literal_eval
+from time import time
+
+import pandas as pd
+
+from csv_detective.utils import display_logs_depending_process_time
+
+
+def detect_continuous_variable(
+    table: pd.DataFrame, continuous_th: float = 0.9, verbose: bool = False
+):
+    """
+    Detects whether a column contains continuous variables. We consider a continuous column
+    one that contains a considerable amount of float values.
+    We removed the integers as we then end up with postal codes, insee codes, and all sort
+    of codes and types.
+    This is not optimal but it will do for now.
+    """
+    # if we need this again in the future, could be first based on columns detected as int/float to cut time
+
+    def check_threshold(serie: pd.Series, continuous_th: float) -> bool:
+        count = serie.value_counts().to_dict()
+        total_nb = len(serie)
+        if float in count:
+            nb_floats = count[float]
+        else:
+            return False
+        if nb_floats / total_nb >= continuous_th:
+            return True
+        else:
+            return False
+
+    def parses_to_integer(value: str):
+        try:
+            value = value.replace(",", ".")
+            value = literal_eval(value)
+            return type(value)
+        except Exception:
+            return False
+
+    if verbose:
+        start = time()
+        logging.info("Detecting continuous columns")
+    res = table.apply(lambda serie: check_threshold(serie.apply(parses_to_integer), continuous_th))
+    if verbose:
+        display_logs_depending_process_time(
+            f"Detected {sum(res)} continuous columns in {round(time() - start, 3)}s",
+            time() - start,
+        )
+    return res.index[res]
+
+
+def detect_categorical_variable(
+    table: pd.DataFrame,
+    threshold_pct_categorical: float = 0.05,
+    max_number_categorical_values: int = 25,
+    verbose: bool = False,
+) -> tuple[list[str], pd.DataFrame]:
+    """
+    Heuristically detects whether a table (df) contains categorical values according to
+    the number of unique values contained.
+    As the idea of detecting categorical values is to then try to learn models to predict
+    them, we limit categorical values to at most 25 different modes or at most 5% disparity.
+    Postal code, insee code, code region and so on, may be thus not considered categorical values.
+    :param table:
+    :param threshold_pct_categorical:
+    :param max_number_categorical_values:
+    :return:
+    """
+
+    def abs_number_different_values(column_values: pd.Series):
+        return column_values.nunique()
+
+    def rel_number_different_values(column_values: pd.Series):
+        return column_values.nunique() / len(column_values)
+
+    def detect_categorical(column_values: pd.Series):
+        abs_unique_values = abs_number_different_values(column_values)
+        rel_unique_values = rel_number_different_values(column_values)
+        if (
+            abs_unique_values <= max_number_categorical_values
+            or rel_unique_values <= threshold_pct_categorical
+        ):
+            return True
+        return False
+
+    if verbose:
+        start = time()
+        logging.info("Detecting categorical columns")
+    res = table.apply(lambda serie: detect_categorical(serie))
+    if verbose:
+        display_logs_depending_process_time(
+            f"Detected {sum(res)} categorical columns out of {len(table.columns)} in {round(time() - start, 3)}s",
+            time() - start,
+        )
+    return list(res.index[res]), res