csv-detective 0.6.7__py3-none-any.whl → 0.9.3.dev2438__py3-none-any.whl

csv_detective/detection.py

@@ -1,361 +0,0 @@
-import pandas as pd
-import numpy as np
-from cchardet import detect
-from ast import literal_eval
-import logging
-from time import time
-from csv_detective.utils import display_logs_depending_process_time
-from csv_detective.detect_fields.other.float import float_casting
-
-logging.basicConfig(level=logging.INFO)
-
-def detect_continuous_variable(table, continuous_th=0.9, verbose: bool = False):
-    """
-    Detects whether a column contains continuous variables. We consider a continuous column
-    one that contains
-    a considerable amount of float values.
-    We removed the integers as we then end up with postal codes, insee codes, and all sort
-    of codes and types.
-    This is not optimal but it will do for now.
-    :param table:
-    :return:
-    """
-
-    def check_threshold(serie, continuous_th):
-        count = serie.value_counts().to_dict()
-        total_nb = len(serie)
-        if float in count:
-            nb_floats = count[float]
-        else:
-            return False
-        if nb_floats / total_nb >= continuous_th:
-            return True
-        else:
-            return False
-
-    def parses_to_integer(value):
-        try:
-            value = value.replace(",", ".")
-            value = literal_eval(value)
-            return type(value)
-        # flake8: noqa
-        except:
-            return False
-
-    if verbose:
-        start = time()
-        logging.info("Detecting continuous columns")
-    res = table.apply(
-        lambda serie: check_threshold(serie.apply(parses_to_integer), continuous_th)
-    )
-    if verbose:
-        display_logs_depending_process_time(
-            f"Detected {sum(res)} continuous columns in {round(time() - start, 3)}s",
-            time() - start
-        )
-    return res.index[res]
-
-
-def detetect_categorical_variable(
-    table, threshold_pct_categorical=0.05, max_number_categorical_values=25, verbose: bool = False
-):
-    """
-    Heuristically detects whether a table (df) contains categorical values according to
-    the number of unique values contained.
-    As the idea of detecting categorical values is to then try to learn models to predict
-    them, we limit categorical values to at most 25 different modes. Postal code, insee code,
-    code region and so on, may be thus not
-    considered categorical values.
-    :param table:
-    :param threshold_pct_categorical:
-    :param max_number_categorical_values:
-    :return:
-    """
-
-    def abs_number_different_values(column_values):
-        return column_values.nunique()
-
-    def rel_number_different_values(column_values):
-        return column_values.nunique() / len(column_values)
-
-    def detect_categorical(column_values):
-        abs_unique_values = abs_number_different_values(column_values)
-        rel_unique_values = rel_number_different_values(column_values)
-        if abs_unique_values < max_number_categorical_values:
-            if rel_unique_values < threshold_pct_categorical:
-                return True
-        return False
-
-    if verbose:
-        start = time()
-        logging.info("Detecting categorical columns")
-    res = table.apply(lambda serie: detect_categorical(serie))
-    if verbose:
-        display_logs_depending_process_time(
-            f"Detected {sum(res)} categorical columns out of {len(table.columns)} in {round(time() - start, 3)}s",
-            time() - start
-        )
-    return res.index[res], res
-
-
-def detect_separator(file, verbose: bool = False):
-    """Detects csv separator"""
-    # TODO: add a robust detection:
-    # si on a un point virgule comme texte et \t comme séparateur, on renvoit
-    # pour l'instant un point virgule
-    if verbose:
-        start = time()
-        logging.info("Detecting separator")
-    file.seek(0)
-    header = file.readline()
-    possible_separators = [";", ",", "|", "\t"]
-    sep_count = dict()
-    for sep in possible_separators:
-        sep_count[sep] = header.count(sep)
-    sep = max(sep_count, key=sep_count.get)
-    if verbose:
-        display_logs_depending_process_time(
-            f'Detected separator: "{sep}" in {round(time() - start, 3)}s',
-            time() - start
-        )
-    return sep
-
-
-def detect_encoding(the_file, verbose: bool = False):
-    """
-    Detects file encoding using faust-cchardet (forked from the original cchardet)
-    """
-    if verbose:
-        start = time()
-        logging.info("Detecting encoding")
-    encoding_dict = detect(the_file.read())
-    if verbose:
-        message = f'Detected encoding: "{encoding_dict["encoding"]}"'
-        message += f' in {round(time() - start, 3)}s (confidence: {round(encoding_dict["confidence"]*100)}%)'
-        display_logs_depending_process_time(
-            message,
-            time() - start
-        )
-    return encoding_dict['encoding']
-
-
-def parse_table(the_file, encoding, sep, num_rows, skiprows, random_state=42, verbose : bool = False):
-    # Takes care of some problems
-    if verbose:
-        start = time()
-        logging.info("Parsing table")
-    table = None
-
-    if not isinstance(the_file, str):
-        the_file.seek(0)
-
-    total_lines = None
-    for encoding in [encoding, "ISO-8859-1", "utf-8"]:
-        # TODO : modification systematique
-        if encoding is None:
-            continue
-
-        if "ISO-8859" in encoding:
-            encoding = "ISO-8859-1"
-        try:
-            table = pd.read_csv(
-                the_file, sep=sep, dtype="unicode", encoding=encoding, skiprows=skiprows
-            )
-            total_lines = len(table)
-            nb_duplicates = len(table.loc[table.duplicated()])
-            if num_rows > 0:
-                num_rows = min(num_rows - 1, total_lines)
-                table = table.sample(num_rows, random_state=random_state)
-            # else : table is unchanged
-            break
-        except TypeError:
-            print("Trying encoding : {encoding}".format(encoding=encoding))
-
-    if table is None:
-        logging.error("  >> encoding not found")
-        return table, "NA", "NA"
-    if verbose:
-        display_logs_depending_process_time(
-            f'Table parsed successfully in {round(time() - start, 3)}s',
-            time() - start
-        )
-    return table, total_lines, nb_duplicates
-
-
-def create_profile(table, dict_cols_fields, sep, encoding, num_rows, skiprows, verbose: bool = False):
-    if verbose:
-        start = time()
-        logging.info("Creating profile")
-    map_python_types = {
-        "string": str,
-        "int": float,
-        "float": float,
-    }
-
-    if num_rows > 0:
-        raise Exception("To create profiles num_rows has to be set to -1")
-    else:
-        safe_table = table.copy()
-        dtypes = {
-            k: map_python_types.get(v["python_type"], str)
-            for k, v in dict_cols_fields.items()
-        }
-        for c in safe_table.columns:
-            if dtypes[c] == float:
-                safe_table[c] = safe_table[c].apply(
-                    lambda s: float_casting(s) if isinstance(s, str) else s
-                )
-        profile = {}
-        for c in safe_table.columns:
-            profile[c] = {}
-            if map_python_types.get(dict_cols_fields[c]["python_type"], str) in [
-                float,
-                int,
-            ]:
-                profile[c].update(
-                    min=map_python_types.get(dict_cols_fields[c]["python_type"], str)(
-                        safe_table[c].min()
-                    ),
-                    max=map_python_types.get(dict_cols_fields[c]["python_type"], str)(
-                        safe_table[c].max()
-                    ),
-                    mean=map_python_types.get(dict_cols_fields[c]["python_type"], str)(
-                        safe_table[c].mean()
-                    ),
-                    std=map_python_types.get(dict_cols_fields[c]["python_type"], str)(
-                        safe_table[c].std()
-                    ),
-                )
-            tops_bruts = safe_table[safe_table[c].notna()][c] \
-                    .value_counts(dropna=True) \
-                    .reset_index() \
-                    .iloc[:10] \
-                    .to_dict(orient="records")
-            tops = []
-            for tb in tops_bruts:
-                top = {}
-                top["count"] = tb[c]
-                top["value"] = tb["index"]
-                tops.append(top)
-            profile[c].update(
-                tops=tops,
-                nb_distinct=safe_table[c].nunique(),
-                nb_missing_values=len(safe_table[c].loc[safe_table[c].isna()]),
-            )
-        if verbose:
-            display_logs_depending_process_time(
-                f"Created profile in {round(time() - start, 3)}s",
-                time() - start
-            )
-        return profile
-
-
-def detect_extra_columns(file, sep):
-    """regarde s'il y a des colonnes en trop
-    Attention, file ne doit pas avoir de ligne vide"""
-    file.seek(0)
-    retour = False
-    nb_useless_col = 99999
-
-    for i in range(10):
-        line = file.readline()
-        # regarde si on a un retour
-        if retour:
-            assert line[-1] == "\n"
-        if line[-1] == "\n":
-            retour = True
-
-        # regarde le nombre de derniere colonne inutile
-        deb = 0 + retour
-        line = line[::-1][deb:]
-        k = 0
-        for sign in line:
-            if sign != sep:
-                break
-            k += 1
-        if k == 0:
-            return 0, retour
-        nb_useless_col = min(k, nb_useless_col)
-    return nb_useless_col, retour
-
-
-def detect_headers(file, sep, verbose: bool = False):
-    """Tests 10 first rows for possible header (header not in 1st line)"""
-    if verbose:
-        start = time()
-        logging.info("Detecting headers")
-    file.seek(0)
-    for i in range(10):
-        header = file.readline()
-        position = file.tell()
-        chaine = [c for c in header.replace("\n", "").split(sep) if c]
-        if chaine[-1] not in ["", "\n"] and all(
-            [mot not in ["", "\n"] for mot in chaine[1:-1]]
-        ):
-            next_row = file.readline()
-            file.seek(position)
-            if header != next_row:
-                if verbose:
-                    display_logs_depending_process_time(
-                        f'Detected headers in {round(time() - start, 3)}s',
-                        time() - start
-                    )
-                return i, chaine
-    if verbose:
-        logging.info(f'No header detected')
-    return 0, None
-
-
-def detect_heading_columns(file, sep, verbose : bool = False):
-    """Tests first 10 lines to see if there are empty heading columns"""
-    if verbose:
-        start = time()
-        logging.info("Detecting heading columns")
-    file.seek(0)
-    return_int = float("Inf")
-    for i in range(10):
-        line = file.readline()
-        return_int = min(return_int, len(line) - len(line.strip(sep)))
-        if return_int == 0:
-            if verbose:
-                display_logs_depending_process_time(
-                    f'No heading column detected in {round(time() - start, 3)}s',
-                    time() - start
-                )
-            return 0
-    if verbose:
-        display_logs_depending_process_time(
-            f'{return_int} heading columns detected in {round(time() - start, 3)}s',
-            time() - start
-        )
-    return return_int
-
-
-def detect_trailing_columns(file, sep, heading_columns, verbose : bool = False):
-    """Tests first 10 lines to see if there are empty trailing columns"""
-    if verbose:
-        start = time()
-        logging.info("Detecting trailing columns")
-    file.seek(0)
-    return_int = float("Inf")
-    for i in range(10):
-        line = file.readline()
-        return_int = min(
-            return_int,
-            len(line.replace("\n", ""))
-            - len(line.replace("\n", "").strip(sep))
-            - heading_columns,
-        )
-        if return_int == 0:
-            if verbose:
-                display_logs_depending_process_time(
-                    f'No trailing column detected in {round(time() - start, 3)}s',
-                    time() - start
-                )
-            return 0
-    if verbose:
-        display_logs_depending_process_time(
-            f'{return_int} trailing columns detected in {round(time() - start, 3)}s',
-            time() - start
-        )
-    return return_int

csv_detective/process_text.py

@@ -1,39 +0,0 @@
-from re import finditer
-
-
-def camel_case_split(identifier):
-    matches = finditer(
-        ".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)", identifier
-    )
-    return " ".join([m.group(0) for m in matches])
-
-
-# Process text
-def _process_text(val):
-    """Traitement des chaînes de caractères pour les standardiser.
-    Plusieurs alternatives ont été testées : .translate, unidecode.unidecode,
-    des méthodes hybrides, mais aucune ne s'est avérée plus performante."""
-    val = camel_case_split(val)
-    val = val.lower()
-    val = val.replace("-", " ")
-    val = val.replace("_", " ")
-    val = val.replace("'", " ")
-    val = val.replace(",", " ")
-    val = val.replace("  ", " ")
-    val = val.replace("à", "a")
-    val = val.replace("â", "a")
-    val = val.replace("ç", "c")
-    val = val.replace("é", "e")
-    val = val.replace("é", "e")
-    val = val.replace("è", "e")
-    val = val.replace("ê", "e")
-    val = val.replace("î", "i")
-    val = val.replace("ï", "i")
-    val = val.replace("ô", "o")
-    val = val.replace("ö", "o")
-    val = val.replace("î", "i")
-    val = val.replace("û", "u")
-    val = val.replace("ù", "u")
-    val = val.replace("ü", "u")
-    val = val.strip()
-    return val

csv_detective/s3_utils.py

@@ -1,48 +0,0 @@
-import boto3
-import logging
-
-from botocore.client import Config
-from botocore.exceptions import ClientError
-
-
-def get_minio_url(netloc: str, bucket: str, key: str) -> str:
-    """Returns location of given resource in minio once it is saved"""
-    return netloc + "/" + bucket + "/" + key
-
-
-def get_s3_client(url: str, minio_user: str, minio_pwd: str) -> boto3.client:
-    return boto3.client(
-        "s3",
-        endpoint_url=url,
-        aws_access_key_id=minio_user,
-        aws_secret_access_key=minio_pwd,
-        config=Config(signature_version="s3v4"),
-    )
-
-
-def download_from_minio(
-    netloc: str, bucket: str, key: str, filepath: str, minio_user: str, minio_pwd: str
-) -> None:
-    logging.info("Downloading from minio")
-    s3 = get_s3_client(netloc, minio_user, minio_pwd)
-    try:
-        s3.download_file(bucket, key, filepath)
-        logging.info(
-            f"Resource downloaded from minio at {get_minio_url(netloc, bucket, key)}"
-        )
-    except ClientError as e:
-        logging.error(e)
-
-
-def upload_to_minio(
-    netloc: str, bucket: str, key: str, filepath: str, minio_user: str, minio_pwd: str
-) -> None:
-    logging.info("Saving to minio")
-    s3 = get_s3_client(netloc, minio_user, minio_pwd)
-    try:
-        s3.upload_file(filepath, bucket, key)
-        logging.info(
-            f"Resource saved into minio at {get_minio_url(netloc, bucket, key)}"
-        )
-    except ClientError as e:
-        logging.error(e)

csv_detective-0.6.7.data/data/share/csv_detective/CHANGELOG.md

@@ -1,118 +0,0 @@
-# Changelog
-
-## 0.6.7 (2024-01-15)
-
-- Add logs for columns that would take too much time within a specific test
-- Refactor some tests to improve performances and make detection more accurate
-- Try alternative ways to clean text
-
-## 0.6.6 (2023-11-24)
-
-- Change setup.py to better convey dependencies
-
-## 0.6.5 (2023-11-17)
-
-- Change encoding detection for faust-cchardet (forked from cchardet) [#66](https://github.com/etalab/csv-detective/pull/66)
-
-## 0.6.4 (2023-10-18)
-
-- Better handling of ints and floats (now not accepting blanks and "+" in string) [#62](https://github.com/etalab/csv-detective/pull/62)
-
-## 0.6.3 (2023-03-23)
-
-- Faster routine [#59](https://github.com/etalab/csv-detective/pull/59)
-
-## 0.6.2 (2023-02-10)
-
-- Catch OverflowError for latitude and longitude checks [#58](https://github.com/etalab/csv-detective/pull/58)
-
-## 0.6.0 (2023-02-10)
-
-- Add CI and upgrade dependencies [#49](https://github.com/etalab/csv-detective/pull/49)
-- Shuffle data before analysis [#56](https://github.com/etalab/csv-detective/pull/56)
-- Better discrimination between `code_departement` and `code_region` [#56](https://github.com/etalab/csv-detective/pull/56)
-- Add schema in output analysis [#57](https://github.com/etalab/csv-detective/pull/57)
-
-## 0.4.7 [#51](https://github.com/etalab/csv-detective/pull/51)
-
-- Allow possibility to analyze entire file instead of a limited number of rows [#48](https://github.com/etalab/csv-detective/pull/48)
-- Better boolean detection [#42](https://github.com/etalab/csv-detective/issues/42)
-- Differentiate python types and format for `date` and `datetime` [#43](https://github.com/etalab/csv-detective/issues/43)
-- Better `code_departement` and `code_commune_insee` detection [#44](https://github.com/etalab/csv-detective/issues/44)
-- Fix header line (`header_row_idx`) detection [#44](https://github.com/etalab/csv-detective/issues/44)
-- Allow possibility to get profile of csv [#46](https://github.com/etalab/csv-detective/issues/46)
-
-## 0.4.6 [#39](https://github.com/etalab/csv-detective/pull/39)
-
-- Fix tests
-- Prioritise lat / lon FR detection over more generic lat / lon.
-- To reduce false positives, prevent detection of the following if label detection is missing: `['code_departement', 'code_commune_insee', 'code_postal', 'latitude_wgs', 'longitude_wgs', 'latitude_wgs_fr_metropole', 'longitude_wgs_fr_metropole', 'latitude_l93', 'longitude_l93']`
-- Lower threshold of label detection so that if one relevant is detected in the label, it boosts the detection score.
-- Add ISO country alpha-3 and numeric detection
-- include camel case parsing in _process_text function
-- Support optional brackets in latlon format
-
-## 0.4.5 [#29](https://github.com/etalab/csv-detective/pull/29)
-
-- Use `netloc` instead of `url` in location dict
-
-## 0.4.4 [#24] (https://github.com/etalab/csv-detective/pull/28)
-
-- Prevent crash on empty CSVs
-- Add optional arguments encoding and sep to routine and routine_minio functions
-- Field detection improvements (code_csp_insee and datetime RFC 822)
-- Schema generation improvements with examples
-
-
-## 0.4.3 [#24] (https://github.com/etalab/csv-detective/pull/24)
-
-- Add uuid and MongoID detection
-- Add new function dedicated to interaction with minio data
-- Add table schema automatic generation (only on minio data)
-- Modification of calculated score (consider label detection as a boost for score)
-
-## 0.4.2 [#22] (https://github.com/etalab/csv-detective/pull/22)
-
-Add type detection by header name
-
-## 0.4.1 [#19] (https://github.com/etalab/csv-detective/pull/19)
-
-Fix bug
- * num_rows was causing problem when it was fix to other value than default - Fixed
-
-## 0.4.0 [#18] (https://github.com/etalab/csv_detective/pull/18)
-
-Add detailed output possibility
-
-Details :
- * two modes now for output report : "LIMITED" and "ALL"
- * "ALL" option give user information on found proportion for each column types and each columns
-
-## 0.3.0 [#15] (https://github.com/etalab/csv_detective/pull/15)
-
-Fix bugs
-
-Details :
- * Facilitate ML Integration
- * Add column types detection
- * Fix documentation
-
-## 0.2.1 - [#2](https://github.com/etalab/csv_detective/pull/2)
-
-Add continuous integration
-
-Details :
- * Add configuration for CircleCI
- * Add `CONTRIBUTING.md`
- * Push automatically new versions to PyPI
- * Use semantic versioning
-
-## 0.2 - [#1](https://github.com/etalab/csv_detective/pull/1)
-
-Port from python2 to python3
-
-Details :
- * Add license AGPLv3
- * Update requirements
-
-## 0.1