csv-detective 0.6.7__py3-none-any.whl → 0.9.3.dev2438__py3-none-any.whl

csv_detective/detect_fields/__init__.py

@@ -1,57 +0,0 @@
-# flake8: noqa
-from .FR.other import (
-    code_csp_insee,
-    csp_insee,
-    sexe,
-    siren,
-    tel_fr,
-    uai,
-    siret,
-    insee_ape700,
-    date_fr,
-    code_waldec,
-    code_rna
-)
-
-from .other import (
-    email,
-    url,
-    booleen,
-    mongo_object_id,
-    twitter,
-    float,
-    int,
-    uuid,
-    json
-)
-
-from .FR.geo import (
-    adresse,
-    code_commune_insee,
-    code_postal,
-    commune,
-    departement,
-    pays,
-    region,
-    code_departement,
-    code_fantoir,
-    longitude_wgs_fr_metropole,
-    latitude_wgs_fr_metropole,
-    code_region,
-    latitude_l93,
-    longitude_l93,
-    insee_canton
-)
-
-from .geo import (
-    iso_country_code_alpha2,
-    iso_country_code_alpha3,
-    iso_country_code_numeric,
-    latitude_wgs,
-    longitude_wgs,
-    latlon_wgs,
-    json_geojson
-)
-
-from .FR.temp import jour_de_la_semaine, mois_de_annee
-from .temp import year, date, datetime_iso, datetime_rfc822

csv_detective/detect_fields/geo/iso_country_code_alpha2/__init__.py

@@ -1,15 +0,0 @@
-from os.path import dirname, join
-import re
-
-PROPORTION = 1
-
-with open(join(dirname(__file__), 'iso_country_code_alpha2.txt'), 'r') as iofile:
-    liste_pays = iofile.read().split('\n')
-liste_pays = set(liste_pays)
-
-
-def _is(val):
-    '''Renvoie True si val peut etre un code iso pays alpha-2, False sinon'''
-    if not bool(re.match(r'[A-Z]{2}$', val)):
-        return False
-    return val in liste_pays

csv_detective/detect_fields/geo/iso_country_code_alpha3/__init__.py

@@ -1,14 +0,0 @@
-from os.path import dirname, join
-import re
-
-PROPORTION = 1
-
-with open(join(dirname(__file__), 'iso_country_code_alpha3.txt'), 'r') as iofile:
-    liste_pays = iofile.read().split('\n')
-
-
-def _is(val):
-    '''Renvoie True si val peut etre un code iso pays alpha-3, False sinon'''
-    if not bool(re.match(r'[A-Z]{3}$', val)):
-        return False
-    return val in set(liste_pays)

csv_detective/detect_fields/geo/iso_country_code_numeric/__init__.py

@@ -1,15 +0,0 @@
-from os.path import dirname, join
-import re
-
-PROPORTION = 1
-
-with open(join(dirname(__file__), 'iso_country_code_numeric.txt'), 'r') as iofile:
-    liste_pays = iofile.read().split('\n')
-liste_pays = set(liste_pays)
-
-
-def _is(val):
-    '''Renvoie True si val peut etre un code iso pays numerique, False sinon'''
-    if not bool(re.match(r'[0-9]{3}$', val)):
-        return False
-    return val in liste_pays

csv_detective/detect_fields/geo/json_geojson/__init__.py

@@ -1,22 +0,0 @@
-import json
-from json import JSONDecodeError
-
-PROPORTION = 0.9
-
-
-def _is(val):
-    '''Renvoie True si val peut etre geojson'''
-
-    try:
-        j = json.loads(val)
-        if 'type' in j and 'coordinates' in j:
-            return True
-        if 'geometry' in j:
-            if 'coordinates' in j['geometry']:
-                return True
-    except JSONDecodeError:
-        pass
-    except TypeError:
-        pass
-
-    return False

csv_detective/detect_fields/geo/latitude_wgs/__init__.py

@@ -1,13 +0,0 @@
-from csv_detective.detect_fields.other.float import _is as is_float
-
-PROPORTION = 0.9
-
-
-def _is(val):
-    '''Renvoie True si val peut etre une latitude'''
-    try:
-        return is_float(val) and float(val) >= -90 and float(val) <= 90
-    except ValueError:
-        return False
-    except OverflowError:
-        return False

csv_detective/detect_fields/geo/latlon_wgs/__init__.py

@@ -1,15 +0,0 @@
-import re
-
-PROPORTION = 0.9
-
-
-def _is(val):
-    '''Renvoie True si val peut etre une latitude,longitude'''
-
-    a = bool(
-        re.match(
-            r'^\[?[\+\-]?[0-8]?\d\.\d* ?, ?[\+\-]?(1[0-7]\d|\d{1,2})\.\d+\]?$', val
-        )
-    )
-
-    return a

csv_detective/detect_fields/geo/longitude_wgs/__init__.py

@@ -1,13 +0,0 @@
-from csv_detective.detect_fields.other.float import _is as is_float
-
-PROPORTION = 0.9
-
-
-def _is(val):
-    '''Renvoie True si val peut etre une longitude'''
-    try:
-        is_float(val) and float(val) >= -180 and float(val) <= 180
-    except ValueError:
-        return False
-    except OverflowError:
-        return False

csv_detective/detect_fields/other/booleen/__init__.py

@@ -1,21 +0,0 @@
-PROPORTION = 1
-liste_bool = {
-    '0',
-    '1',
-    'vrai',
-    'faux',
-    'true',
-    'false',
-    'oui',
-    'non',
-    'yes',
-    'no',
-    'y',
-    'n',
-    'o'
-}
-
-
-def _is(val):
-    '''Détection les booléens'''
-    return val.lower() in liste_bool

csv_detective/detect_fields/other/email/__init__.py

@@ -1,8 +0,0 @@
-import re
-
-PROPORTION = 1
-
-
-def _is(val):
-    '''Detects e-mails'''
-    return bool(re.match(r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}$', val))

csv_detective/detect_fields/other/float/__init__.py

@@ -1,17 +0,0 @@
-PROPORTION = 1
-
-
-def float_casting(str2cast):
-    return float(str2cast.replace(',', '.'))
-
-
-def _is(val):
-    '''Detects floats, assuming that tables will not have scientific
-    notations (3e6) or "+" in the string. "-" is still accepted.'''
-    try:
-        if any([k in val for k in ['_', '+', 'e', 'E']]):
-            return False
-        float_casting(val)
-        return True
-    except ValueError:
-        return False

csv_detective/detect_fields/other/int/__init__.py

@@ -1,12 +0,0 @@
-PROPORTION = 1
-
-
-def _is(val):
-    '''Detects integers'''
-    if any([v in val for v in ['.', '_', '+']]):
-        return False
-    try:
-        int(val)
-        return True
-    except ValueError:
-        return False

csv_detective/detect_fields/other/json/__init__.py

@@ -1,24 +0,0 @@
-import json
-from json import JSONDecodeError
-
-PROPORTION = 1
-
-
-def _is(val):
-    '''Detects json'''
-    try:
-        loaded = json.loads(val)
-        if isinstance(loaded, list) or (
-            isinstance(loaded, dict) and not (
-                any(
-                    [
-                        geo in loaded for geo in ['coordinates', 'geometry']
-                    ]
-                )
-            )
-        ):
-            return True
-        else:
-            return False
-    except JSONDecodeError:
-        return False

csv_detective/detect_fields/other/mongo_object_id/__init__.py

@@ -1,8 +0,0 @@
-import re
-
-PROPORTION = 0.8
-
-
-def _is(val):
-    '''Detects Mongo ObjectIds'''
-    return bool(re.match(r'^[0-9a-fA-F]{24}$', val))

csv_detective/detect_fields/other/twitter/__init__.py

@@ -1,8 +0,0 @@
-import re
-
-PROPORTION = 1
-
-
-def _is(val):
-    '''Detects twitter accounts'''
-    return bool(re.match(r'^@[A-Za-z0-9_]+$', val))

csv_detective/detect_fields/other/url/__init__.py

@@ -1,11 +0,0 @@
-
-PROPORTION = 1
-
-
-def _is(val):
-    '''Detects urls'''
-    a = 'http://' in val
-    b = 'www.' in val
-    c = any([x in val for x in ['.fr', '.com', '.org', '.gouv', '.net']])
-    d = not ('@' in val)
-    return (a or b or c) and d

csv_detective/detect_fields/other/uuid/__init__.py

@@ -1,11 +0,0 @@
-import re
-
-PROPORTION = 0.8
-
-
-def _is(val):
-    '''Detects UUIDs'''
-    return bool(re.match(
-        r'^[{]?[0-9a-fA-F]{8}' + '-?([0-9a-fA-F]{4}-?)' + '{3}[0-9a-fA-F]{12}[}]?$',
-        val
-    ))

csv_detective/detect_fields/temp/date/__init__.py

@@ -1,62 +0,0 @@
-import re
-from dateutil.parser import parse, ParserError
-from csv_detective.detect_fields.other.float import _is as is_float
-from unidecode import unidecode
-
-PROPORTION = 1
-# /!\ this is only for dates, not datetimes which are handled by other utils
-
-
-def is_dateutil_date(val: str) -> bool:
-    # we don't want to get datetimes here, so length restriction
-    # longest date string expected here is DD-septembre-YYYY, so 17 characters
-    if len(val) > 17:
-        return False
-    try:
-        res = parse(val, fuzzy=False)
-        if res.hour or res.minute or res.second:
-            return False
-        return True
-    except (ParserError, ValueError, TypeError, OverflowError):
-        return False
-
-
-def _is(val):
-    '''Renvoie True si val peut être une date, False sinon
-    On ne garde que les regex pour les cas où parse() ne convient pas'''
-
-    # matches 02/12 03 and 02_12 2003
-    a = bool(
-        re.match(
-            r'^(0[1-9]|[12][0-9]|3[01])[ -/_](0[1-9]|1[012])[ -/_]'
-            r'([0-9]{2}|(19|20)[0-9]{2}$)',
-            val
-        )
-    )
-
-    # matches 02052003
-    b = bool(
-        re.match(
-            r'^(0[1-9]|[12][0-9]|3[01])(0[1-9]|1[012])([0-9]{2}|'
-            r'(19|20){2}$)',
-            val
-        )
-    )
-
-    # matches JJ*MM*AAAA
-    c = bool(
-        re.match(
-            r'^(0[1-9]|[12][0-9]|3[01]).?(0[1-9]|1[012]).?(19|20)?\d\d$', val))
-
-    # matches JJ-mmm-AAAA and matches JJ-mmm...mm-AAAA
-    d = bool(
-        re.match(
-            r'^(0[1-9]|[12][0-9]|3[01])[ -/_;.:,](jan|fev|feb|mar|avr|apr'
-            r'|mai|may|jun|jui|jul|aou|aug|sep|oct|nov|dec|janvier|fevrier|mars|avril|'
-            r'mai|juin|jullet|aout|septembre|octobre|novembre|decembre)[ -/_;.:,]'
-            r'([0-9]{2}$|(19|20)[0-9]{2}$)',
-            unidecode(val)
-        )
-    )
-
-    return (is_dateutil_date(val) and not is_float(val)) or a or b or c or d

csv_detective/detect_fields/temp/datetime_iso/__init__.py

@@ -1,18 +0,0 @@
-import re
-
-PROPORTION = 1
-
-
-def _is(val):
-    '''Renvoie True si val peut être une date au format iso, False sinon
-    Exemple: 2023-01-15T12:30:45.123456Z'''
-    a = bool(
-        re.match(
-            r'^\d{4}-(0[1-9]|1[012])\-(0[1-9]|[12][0-9]|3[01])[Tt]'
-            r'([0-2])([0-9]):([0-5])([0-9]):([0-5])([0-9])'
-            r'(\.\d+)?([Zz]|[-+](0[0-9]|1[0-2]):[0-5][0-9])?$',
-            val
-        )
-    )
-
-    return a

csv_detective/detect_fields/temp/datetime_rfc822/__init__.py

@@ -1,21 +0,0 @@
-import re
-
-PROPORTION = 1
-
-
-def _is(val):
-    '''Renvoie True si val peut être une date au format rfc822, False sinon
-    Exemple: Tue, 19 Dec 2023 15:30:45 +0000'''
-
-    val = val.lower()
-    a = bool(
-        re.match(
-            r'^[A-Za-z]{3}, (0[1-9]|[1-2][0-9]|3[01]) [A-Za-z]{3} \d{4} '
-            r'([0-2])([0-9]):([0-5])([0-9]):([0-5])([0-9]) '
-            r'(ut|gmt|est|edt|cst|cdt|mst|mdt|pst|pdt|[+\-](0[0-9]|1[0-3])00)$',
-            val,
-            re.IGNORECASE
-        )
-    )
-
-    return a

csv_detective/detect_fields/temp/year/__init__.py

@@ -1,10 +0,0 @@
-PROPORTION = 1
-
-
-def _is(val):
-    '''Returns True if val can be a year'''
-    try:
-        val = int(val)
-    except ValueError:
-        return False
-    return (1800 <= val) and (val <= 2100)

csv_detective/detect_labels/FR/geo/adresse/__init__.py

@@ -1,40 +0,0 @@
-from csv_detective.utils import full_word_strictly_inside_string
-from csv_detective.process_text import _process_text
-
-PROPORTION = 0.5
-
-
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
-    words_combinations_list = [
-        'adresse',
-        'adresse postale',
-        'adresse geographique',
-        'adr',
-        'adresse complete',
-        'adresse station'
-    ]
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                full_word_strictly_inside_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)

csv_detective/detect_labels/FR/geo/code_commune_insee/__init__.py

@@ -1,42 +0,0 @@
-from csv_detective.utils import full_word_strictly_inside_string
-from csv_detective.process_text import _process_text
-
-PROPORTION = 0.5
-
-
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
-    words_combinations_list = [
-        'code commune insee',
-        'code insee',
-        'codes insee',
-        'code commune',
-        'code insee commune',
-        'insee',
-        'code com',
-        'com'
-    ]
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                full_word_strictly_inside_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)

csv_detective/detect_labels/FR/geo/code_departement/__init__.py

@@ -1,33 +0,0 @@
-from csv_detective.utils import full_word_strictly_inside_string
-from csv_detective.process_text import _process_text
-
-PROPORTION = 0.5
-
-
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-    # 'dep': Possible confusion with dep name?
-    words_combinations_list = ['code departement', 'code_departement', 'dep', 'departement', 'dept']
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                full_word_strictly_inside_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)

csv_detective/detect_labels/FR/geo/code_fantoir/__init__.py

@@ -1,33 +0,0 @@
-from csv_detective.utils import full_word_strictly_inside_string
-from csv_detective.process_text import _process_text
-
-PROPORTION = 0.5
-
-
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
-    words_combinations_list = ['cadastre1', 'code fantoir', 'fantoir']
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                full_word_strictly_inside_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)

csv_detective/detect_labels/FR/geo/code_postal/__init__.py

@@ -1,41 +0,0 @@
-from csv_detective.utils import full_word_strictly_inside_string
-from csv_detective.process_text import _process_text
-
-PROPORTION = 0.5
-
-
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-
-    words_combinations_list = [
-        'code postal',
-        'postal code',
-        'postcode',
-        'post code',
-        'cp',
-        'codes postaux',
-        'location postcode'
-    ]
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                full_word_strictly_inside_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)

csv_detective/detect_labels/FR/geo/code_region/__init__.py

@@ -1,33 +0,0 @@
-from csv_detective.utils import full_word_strictly_inside_string
-from csv_detective.process_text import _process_text
-
-PROPORTION = 0.5
-
-
-def _is(header):
-    '''
-    Returns 1 if the (processed) header matches one of the expected words combination,
-    else 0
-    '''
-    # 'reg' : possible confusion with region name?
-    words_combinations_list = ['code region', 'reg', 'code insee region', 'region']
-    processed_header = _process_text(header)
-
-    header_matches_words_combination = float(
-        any(
-            [
-                words_combination == processed_header for words_combination in words_combinations_list
-            ]
-        )
-    )
-    words_combination_in_header = 0.5 * float(
-        any(
-            [
-                full_word_strictly_inside_string(
-                    words_combination, processed_header
-                ) for words_combination in words_combinations_list
-            ]
-        )
-    )
-
-    return max(header_matches_words_combination, words_combination_in_header)