csv-detective 0.8.1.dev1380__py3-none-any.whl → 0.8.1.dev1440__py3-none-any.whl

csv_detective/detect_fields/geo/iso_country_code_numeric/iso_country_code_numeric.txt

@@ -1,251 +0,0 @@
-004
-710
-248
-008
-012
-276
-020
-024
-660
-010
-028
-682
-032
-051
-533
-036
-040
-031
-044
-048
-050
-052
-112
-056
-084
-204
-060
-064
-068
-535
-070
-072
-074
-076
-096
-100
-854
-108
-136
-116
-120
-124
-132
-140
-152
-156
-162
-196
-166
-170
-174
-178
-180
-184
-410
-408
-188
-384
-191
-192
-531
-208
-262
-214
-212
-818
-Gum
-(Ré
-222
-784
-218
-232
-724
-233
-840
-231
-238
-234
-242
-246
-250
-266
-270
-268
-239
-288
-292
-300
-308
-304
-312
-316
-320
-831
-324
-624
-226
-328
-254
-332
-334
-340
-344
-348
-833
-581
-092
-850
-356
-360
-364
-368
-372
-352
-376
-380
-388
-392
-832
-400
-398
-404
-417
-296
-414
-418
-426
-428
-422
-430
-434
-438
-440
-442
-446
-807
-450
-458
-454
-462
-466
-470
-580
-504
-584
-474
-480
-478
-175
-484
-583
-498
-492
-496
-499
-500
-508
-104
-516
-520
-524
-558
-562
-566
-570
-574
-578
-540
-554
-086
-512
-800
-860
-586
-585
-275
-591
-598
-600
-528
-604
-608
-612
-616
-258
-630
-620
-634
-638
-642
-826
-643
-646
-732
-652
-659
-674
-663
-534
-666
-336
-670
-654
-662
-090
-882
-016
-678
-686
-688
-690
-694
-702
-703
-705
-706
-729
-728
-144
-752
-756
-740
-744
-748
-760
-762
-158
-834
-148
-203
-260
-764
-626
-768
-772
-776
-780
-788
-795
-796
-792
-798
-804
-858
-548
-862
-704
-876
-887
-894
-716

csv_detective/detection/columns.py

@@ -1,89 +0,0 @@
-import logging
-from typing import TextIO
-from time import time
-
-from csv_detective.utils import display_logs_depending_process_time
-
-
-def detect_extra_columns(file: TextIO, sep: str):
-    """regarde s'il y a des colonnes en trop
-    Attention, file ne doit pas avoir de ligne vide"""
-    file.seek(0)
-    retour = False
-    nb_useless_col = 99999
-
-    for i in range(10):
-        line = file.readline()
-        # regarde si on a un retour
-        if retour:
-            assert line[-1] == "\n"
-        if line[-1] == "\n":
-            retour = True
-
-        # regarde le nombre de derniere colonne inutile
-        deb = 0 + retour
-        line = line[::-1][deb:]
-        k = 0
-        for sign in line:
-            if sign != sep:
-                break
-            k += 1
-        if k == 0:
-            return 0, retour
-        nb_useless_col = min(k, nb_useless_col)
-    return nb_useless_col, retour
-
-
-def detect_heading_columns(file: TextIO, sep: str, verbose: bool = False) -> int:
-    """Tests first 10 lines to see if there are empty heading columns"""
-    if verbose:
-        start = time()
-        logging.info("Detecting heading columns")
-    file.seek(0)
-    return_int = float("Inf")
-    for i in range(10):
-        line = file.readline()
-        return_int = min(return_int, len(line) - len(line.strip(sep)))
-        if return_int == 0:
-            if verbose:
-                display_logs_depending_process_time(
-                    f'No heading column detected in {round(time() - start, 3)}s',
-                    time() - start,
-                )
-            return 0
-    if verbose:
-        display_logs_depending_process_time(
-            f'{return_int} heading columns detected in {round(time() - start, 3)}s',
-            time() - start,
-        )
-    return return_int
-
-
-def detect_trailing_columns(file: TextIO, sep: str, heading_columns: int, verbose: bool = False) -> int:
-    """Tests first 10 lines to see if there are empty trailing columns"""
-    if verbose:
-        start = time()
-        logging.info("Detecting trailing columns")
-    file.seek(0)
-    return_int = float("Inf")
-    for i in range(10):
-        line = file.readline()
-        return_int = min(
-            return_int,
-            len(line.replace("\n", ""))
-            - len(line.replace("\n", "").strip(sep))
-            - heading_columns,
-        )
-        if return_int == 0:
-            if verbose:
-                display_logs_depending_process_time(
-                    f'No trailing column detected in {round(time() - start, 3)}s',
-                    time() - start,
-                )
-            return 0
-    if verbose:
-        display_logs_depending_process_time(
-            f'{return_int} trailing columns detected in {round(time() - start, 3)}s',
-            time() - start,
-        )
-    return return_int

csv_detective/detection/encoding.py

@@ -1,27 +0,0 @@
-import logging
-from time import time
-from io import BytesIO
-
-from cchardet import detect
-
-from csv_detective.utils import display_logs_depending_process_time
-
-
-def detect_encoding(binary_file: BytesIO, verbose: bool = False) -> str:
-    """
-    Detects file encoding using faust-cchardet (forked from the original cchardet)
-    """
-    if verbose:
-        start = time()
-        logging.info("Detecting encoding")
-    encoding_dict = detect(binary_file.read())
-    if not encoding_dict["encoding"]:
-        raise ValueError("Could not detect the file's encoding. Consider specifying it in the routine call.")
-    if verbose:
-        message = f'Detected encoding: "{encoding_dict["encoding"]}"'
-        message += f' in {round(time() - start, 3)}s (confidence: {round(encoding_dict["confidence"]*100)}%)'
-        display_logs_depending_process_time(
-            message,
-            time() - start,
-        )
-    return encoding_dict['encoding']

csv_detective/detection/engine.py

@@ -1,46 +0,0 @@
-from time import time
-from typing import Optional
-
-import magic
-import requests
-
-from csv_detective.utils import display_logs_depending_process_time, is_url
-
-COMPRESSION_ENGINES = ["gzip"]
-EXCEL_ENGINES = ["openpyxl", "xlrd", "odf"]
-engine_to_file = {
-    "openpyxl": "Excel",
-    "xlrd": "old Excel",
-    "odf": "OpenOffice",
-    "gzip": "csv.gz",
-}
-
-
-def detect_engine(file_path: str, verbose=False) -> Optional[str]:
-    if verbose:
-        start = time()
-    mapping = {
-        "application/gzip": "gzip",
-        "application/x-gzip": "gzip",
-        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'openpyxl',
-        'application/vnd.ms-excel': 'xlrd',
-        'application/vnd.oasis.opendocument.spreadsheet': 'odf',
-        # all these files could be recognized as zip, may need to check all cases then
-        'application/zip': 'openpyxl',
-    }
-    # if none of the above, we move forwards with the csv process
-    if is_url(file_path):
-        remote_content = requests.get(file_path).content
-        engine = mapping.get(magic.from_buffer(remote_content, mime=True))
-    else:
-        engine = mapping.get(magic.from_file(file_path, mime=True))
-    if verbose:
-        message = (
-            f"File is not csv, detected {engine_to_file.get(engine, 'csv')}"
-            if engine else "Processing the file as a csv"
-        )
-        display_logs_depending_process_time(
-            message,
-            time() - start,
-        )
-    return engine

csv_detective/detection/formats.py

@@ -1,145 +0,0 @@
-from collections import defaultdict
-from typing import Union
-
-import numpy as np
-import pandas as pd
-from csv_detective.detection.variables import (
-    detect_categorical_variable,
-    # detect_continuous_variable,
-)
-from csv_detective.load_tests import return_all_tests
-from csv_detective.output.utils import prepare_output_dict
-from csv_detective.parsing.columns import test_col, test_label
-
-
-def detect_formats(
-    table: pd.DataFrame,
-    analysis: dict,
-    user_input_tests: Union[str, list[str]] = "ALL",
-    limited_output: bool = True,
-    skipna: bool = True,
-    verbose: bool = False,
-):
-
-    if table.empty:
-        res_categorical = []
-        # res_continuous = []
-    else:
-        # Detects columns that are categorical
-        res_categorical, categorical_mask = detect_categorical_variable(table, verbose=verbose)
-        res_categorical = list(res_categorical)
-        # Detect columns that are continuous (we already know the categorical) :
-        # we don't need this for now, cuts processing time
-        # res_continuous = list(
-        #     detect_continuous_variable(table.iloc[:, ~categorical_mask.values], verbose=verbose)
-        # )
-
-    analysis.update({
-        "categorical": res_categorical,
-        # "continuous": res_continuous,
-    })
-
-    # list testing to be performed
-    all_tests_fields = return_all_tests(
-        user_input_tests, detect_type="detect_fields"
-    )  # list all tests for the fields
-    all_tests_labels = return_all_tests(
-        user_input_tests, detect_type="detect_labels"
-    )  # list all tests for the labels
-
-    # if no testing then return
-    if not all_tests_fields and not all_tests_labels:
-        return analysis
-
-    # Perform testing on fields
-    scores_table_fields = test_col(table, all_tests_fields, limited_output, skipna=skipna, verbose=verbose)
-    analysis["columns_fields"] = prepare_output_dict(scores_table_fields, limited_output)
-
-    # Perform testing on labels
-    scores_table_labels = test_label(table, all_tests_labels, limited_output, verbose=verbose)
-    analysis["columns_labels"] = prepare_output_dict(scores_table_labels, limited_output)
-
-    # Multiply the results of the fields by 1 + 0.5 * the results of the labels.
-    # This is because the fields are more important than the labels and yields a max
-    # of 1.5 for the final score.
-    scores_table = scores_table_fields * (
-        1
-        + scores_table_labels.reindex(
-            index=scores_table_fields.index, fill_value=0
-        ).values / 2
-    )
-
-    # To reduce false positives: ensure these formats are detected only if the label yields
-    # a detection (skipping the ones that have been excluded by the users).
-    formats_with_mandatory_label = [
-        f for f in [
-            "code_departement",
-            "code_commune_insee",
-            "code_postal",
-            "latitude_wgs",
-            "longitude_wgs",
-            "latitude_wgs_fr_metropole",
-            "longitude_wgs_fr_metropole",
-            "latitude_l93",
-            "longitude_l93",
-        ] if f in scores_table.index
-    ]
-    scores_table.loc[formats_with_mandatory_label, :] = np.where(
-        scores_table_labels.loc[formats_with_mandatory_label, :],
-        scores_table.loc[formats_with_mandatory_label, :],
-        0,
-    )
-    analysis["columns"] = prepare_output_dict(scores_table, limited_output)
-
-    metier_to_python_type = {
-        "booleen": "bool",
-        "int": "int",
-        "float": "float",
-        "string": "string",
-        "json": "json",
-        "json_geojson": "json",
-        "datetime": "datetime",
-        "datetime_iso": "datetime",
-        "datetime_rfc822": "datetime",
-        "date": "date",
-        "latitude": "float",
-        "latitude_l93": "float",
-        "latitude_wgs": "float",
-        "latitude_wgs_fr_metropole": "float",
-        "longitude": "float",
-        "longitude_l93": "float",
-        "longitude_wgs": "float",
-        "longitude_wgs_fr_metropole": "float",
-    }
-
-    if not limited_output:
-        for detection_method in ["columns_fields", "columns_labels", "columns"]:
-            analysis[detection_method] = {
-                col_name: [
-                    {
-                        "python_type": metier_to_python_type.get(
-                            detection["format"], "string"
-                        ),
-                        **detection,
-                    }
-                    for detection in detections
-                ]
-                for col_name, detections in analysis[detection_method].items()
-            }
-    else:
-        for detection_method in ["columns_fields", "columns_labels", "columns"]:
-            analysis[detection_method] = {
-                col_name: {
-                    "python_type": metier_to_python_type.get(
-                        detection["format"], "string"
-                    ),
-                    **detection,
-                }
-                for col_name, detection in analysis[detection_method].items()
-            }
-
-        # Add detection with formats as keys
-        analysis["formats"] = defaultdict(list)
-        for header, col_metadata in analysis["columns"].items():
-            analysis["formats"][col_metadata["format"]].append(header)
-    return analysis

csv_detective/detection/headers.py

@@ -1,32 +0,0 @@
-import logging
-from time import time
-from typing import Optional, TextIO
-
-from csv_detective.utils import display_logs_depending_process_time
-
-
-def detect_headers(file: TextIO, sep: str, verbose: bool = False) -> tuple[int, Optional[list]]:
-    """Tests 10 first rows for possible header (in case header is not 1st row)"""
-    if verbose:
-        start = time()
-        logging.info("Detecting headers")
-    file.seek(0)
-    for i in range(10):
-        header = file.readline()
-        position = file.tell()
-        chaine = [c for c in header.replace("\n", "").split(sep) if c]
-        if chaine[-1] not in ["", "\n"] and all(
-            [mot not in ["", "\n"] for mot in chaine[1:-1]]
-        ):
-            next_row = file.readline()
-            file.seek(position)
-            if header != next_row:
-                if verbose:
-                    display_logs_depending_process_time(
-                        f'Detected headers in {round(time() - start, 3)}s',
-                        time() - start,
-                    )
-                return i, chaine
-    if verbose:
-        logging.info('No header detected')
-    return 0, None

csv_detective/detection/rows.py

@@ -1,18 +0,0 @@
-import pandas as pd
-
-
-def remove_empty_first_rows(table: pd.DataFrame) -> tuple[pd.DataFrame, int]:
-    """Analog process to detect_headers for csv files, determines how many rows to skip
-    to end up with the header at the right place"""
-    idx = 0
-    if all([str(c).startswith('Unnamed:') for c in table.columns]):
-        # there is on offset between the index in the file (idx here)
-        # and the index in the dataframe, because of the header
-        idx = 1
-        while table.iloc[idx - 1].isna().all():
-            idx += 1
-        cols = table.iloc[idx - 1]
-        table = table.iloc[idx:]
-        table.columns = cols.to_list()
-    # +1 here because the headers should count as a row
-    return table, idx

csv_detective/detection/separator.py

@@ -1,44 +0,0 @@
-import csv
-import logging
-from time import time
-from typing import TextIO
-
-from csv_detective.utils import display_logs_depending_process_time
-
-
-def detect_separator(file: TextIO, verbose: bool = False) -> str:
-    """Detects csv separator"""
-    # TODO: add a robust detection:
-    # si on a un point virgule comme texte et \t comme séparateur, on renvoie
-    # pour l'instant un point virgule
-    if verbose:
-        start = time()
-        logging.info("Detecting separator")
-    file.seek(0)
-    header = file.readline()
-    possible_separators = [";", ",", "|", "\t"]
-    sep_count = dict()
-    for sep in possible_separators:
-        sep_count[sep] = header.count(sep)
-    sep = max(sep_count, key=sep_count.get)
-    # testing that the first 10 (arbitrary) rows all have the same number of fields
-    # as the header. Prevents downstream unwanted behaviour where pandas can load
-    # the file (in a weird way) but the process is irrelevant.
-    file.seek(0)
-    reader = csv.reader(file, delimiter=sep)
-    rows_lengths = set()
-    for idx, row in enumerate(reader):
-        if idx > 10:
-            break
-        rows_lengths.add(len(row))
-    if len(rows_lengths) > 1:
-        raise ValueError(
-            f"Number of columns is not even across the first 10 rows (detected separator: {sep})."
-        )
-
-    if verbose:
-        display_logs_depending_process_time(
-            f'Detected separator: "{sep}" in {round(time() - start, 3)}s',
-            time() - start,
-        )
-    return sep