csv-detective 0.10.0__py3-none-any.whl → 0.10.1__py3-none-any.whl

csv_detective/formats/date.py

@@ -7,23 +7,24 @@ from dateutil.parser import parse as dateutil_parser
 
 proportion = 1
 tags = ["temp", "type"]
-SHARED_DATE_LABELS = [
-    "date",
-    "mise à jour",
-    "modifie",
-    "maj",
-    "datemaj",
-    "update",
-    "created",
-    "modified",
-]
-labels = SHARED_DATE_LABELS + [
-    "jour",
-    "periode",
-    "dpc",
-    "yyyymmdd",
-    "aaaammjj",
-]
+python_type = "date"
+SHARED_DATE_LABELS = {
+    "date": 1,
+    "mise à jour": 1,
+    "modifie": 1,
+    "maj": 0.75,
+    "datemaj": 1,
+    "update": 1,
+    "created": 1,
+    "modified": 1,
+}
+labels = SHARED_DATE_LABELS | {
+    "jour": 0.75,
+    "periode": 0.75,
+    "dpc": 0.5,
+    "yyyymmdd": 1,
+    "aaaammjj": 1,
+}
 
 
 def date_casting(val: str) -> datetime | None:

csv_detective/formats/date_fr.py

@@ -4,7 +4,7 @@ from csv_detective.parsing.text import _process_text
 
 proportion = 1
 tags = ["fr", "temp"]
-labels = ["date"]
+labels = {"date": 1}
 
 pattern = (
     r"^(0?[1-9]|[12][0-9]|3[01])[ \-/](janvier|fevrier|mars|avril|mai|juin|juillet|aout|septembre"

csv_detective/formats/datetime_aware.py

@@ -4,7 +4,8 @@ from csv_detective.formats.date import SHARED_DATE_LABELS, aaaammjj_pattern, dat
 
 proportion = 1
 tags = ["temp", "type"]
-labels = SHARED_DATE_LABELS + ["datetime", "timestamp"]
+python_type = "datetime"
+labels = SHARED_DATE_LABELS | {"datetime": 1, "timestamp": 1}
 
 threshold = 0.7
 pat = (
@@ -12,7 +13,9 @@ pat = (
     + r"(T|\s)(0\d|1[0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(.\d{1,6})"
     + r"?(([+-](0\d|1[0-9]|2[0-3]):([0-5][0-9]))|Z)$"
 )
-prefix = r"^\d{4}"
+# date_casting is very (too?) good at finding date(time)s where there sometimes is just a number
+# this prefix check asserts we only consider strings that have a somewhat fine structure trying to cast
+prefix = r"^\d{2}[-/:]?\d{2}"
 
 
 def _is(val):
@@ -41,6 +44,8 @@ _test_values = {
         "2000-12-21 10:20:10.1Z",
         "2024-12-19T10:53:36.428000+00:00",
         "1996/06/22 10:20:10 GMT",
+        "12/31/2022 12:00:00-04:00",
+        "12:00:00-04:00 12/31/2022",
     ],
     False: [
         "2021-06-22T30:20:10",

csv_detective/formats/datetime_naive.py

@@ -6,6 +6,7 @@ from csv_detective.formats.datetime_aware import labels, prefix  # noqa
 
 proportion = 1
 tags = ["temp", "type"]
+python_type = "datetime"
 threshold = 0.7
 
 # matches AAAA-MM-JJTHH:MM:SS(.dddddd)Z with any of the listed separators for the date OR NO SEPARATOR
@@ -36,6 +37,8 @@ _test_values = {
         "2021-06-22 10:20:10",
         "2030/06-22   00:00:00",
         "2030/06/22 00:00:00.0028",
+        "12/31/2022 12:00:00",
+        "12:00:00 12/31/2022",
     ],
     False: [
         "2021-06-22T30:20:10",

csv_detective/formats/datetime_rfc822.py

@@ -4,6 +4,7 @@ from csv_detective.formats.datetime_aware import labels  # noqa
 
 proportion = 1
 tags = ["temp", "type"]
+python_type = "datetime"
 
 
 def _is(val):

csv_detective/formats/departement.py

@@ -2,21 +2,21 @@ from frformat import Departement, Millesime, Options
 
 proportion = 0.9
 tags = ["fr", "geo"]
-labels = [
-    "departement",
-    "libelle du departement",
-    "deplib",
-    "nom dept",
-    "dept",
-    "libdepartement",
-    "nom departement",
-    "libelle dep",
-    "libelle departement",
-    "lb departements",
-    "dep libusage",
-    "lb departement",
-    "nom dep",
-]
+labels = {
+    "departement": 1,
+    "libelle du departement": 1,
+    "deplib": 1,
+    "nom dept": 1,
+    "dept": 0.75,
+    "libdepartement": 1,
+    "nom departement": 1,
+    "libelle dep": 1,
+    "libelle departement": 1,
+    "lb departements": 1,
+    "dep libusage": 1,
+    "lb departement": 1,
+    "nom dep": 1,
+}
 
 _options = Options(
     ignore_case=True,

csv_detective/formats/email.py

@@ -1,19 +1,19 @@
 import re
 
 proportion = 0.9
-labels = [
-    "email",
-    "mail",
-    "courriel",
-    "contact",
-    "mel",
-    "lieucourriel",
-    "coordinates.emailcontact",
-    "e mail",
-    "mo mail",
-    "adresse mail",
-    "adresse email",
-]
+labels = {
+    "email": 1,
+    "mail": 1,
+    "courriel": 1,
+    "contact": 1,
+    "mel": 1,
+    "lieucourriel": 1,
+    "coordinates.emailcontact": 1,
+    "e mail": 1,
+    "mo mail": 1,
+    "adresse mail": 1,
+    "adresse email": 1,
+}
 
 
 def _is(val):

csv_detective/formats/float.py

@@ -2,7 +2,8 @@ import re
 
 proportion = 1
 tags = ["type"]
-labels = ["part", "ratio", "taux"]
+python_type = "float"
+labels = {"part": 1, "ratio": 1, "taux": 1}
 
 scientific_notation_pattern = r"\d+\.\d+[e|E][+|-]?\d+"
 

csv_detective/formats/geojson.py

@@ -2,16 +2,16 @@ import json
 
 proportion = 1
 tags = ["geo"]
-labels = [
-    "json geojson",
-    "json",
-    "geojson",
-    "geo shape",
-    "geom",
-    "geometry",
-    "geo shape",
-    "geoshape",
-]
+python_type = "json"
+labels = {
+    "json geojson": 1,
+    "json": 1,
+    "geojson": 1,
+    "geo shape": 1,
+    "geom": 0.75,
+    "geometry": 1,
+    "geoshape": 1,
+}
 
 
 def _is(val) -> bool:

csv_detective/formats/insee_ape700.py

@@ -4,14 +4,16 @@ from csv_detective.parsing.text import _process_text
 
 proportion = 0.8
 tags = ["fr"]
-labels = [
-    "code ape",
-    "code activite (ape)",
-    "code naf",
-    "code naf organisme designe",
-    "code naf organisme designant",
-    "base sirene : code ape de l'etablissement siege",
-]
+labels = {
+    "code ape": 1,
+    "code activite (ape)": 1,
+    "code naf": 1,
+    "code naf organisme designe": 1,
+    "code naf organisme designant": 1,
+    "base sirene : code ape de l'etablissement siege": 1,
+    "naf": 0.75,
+    "ape": 0.5,
+}
 
 f = open(join(dirname(__file__), "data", "insee_ape700.txt"), "r")
 condes_insee_ape = f.read().split("\n")

csv_detective/formats/insee_canton.py

@@ -2,12 +2,12 @@ from frformat import Canton, Millesime, Options
 
 proportion = 0.9
 tags = ["fr", "geo"]
-labels = [
-    "insee canton",
-    "canton",
-    "cant",
-    "nom canton",
-]
+labels = {
+    "insee canton": 1,
+    "canton": 1,
+    "cant": 0.5,
+    "nom canton": 1,
+}
 
 _options = Options(
     ignore_case=True,

csv_detective/formats/int.py

@@ -1,5 +1,6 @@
-labels = ["nb", "nombre", "nbre"]
 tag = ["type"]
+python_type = "int"
+labels = {"nb": 0.75, "nombre": 1, "nbre": 0.75}
 
 
 def _is(val):

csv_detective/formats/iso_country_code_alpha2.py

@@ -3,28 +3,28 @@ from os.path import dirname, join
 
 proportion = 1
 tags = ["geo"]
-labels = [
-    "iso country code",
-    "code pays",
-    "pays",
-    "country",
-    "nation",
-    "pays code",
-    "code pays (iso)",
-]
+labels = {
+    "iso country code": 1,
+    "code pays": 1,
+    "pays": 1,
+    "country": 1,
+    "nation": 1,
+    "pays code": 1,
+    "code pays (iso)": 1,
+    "code": 0.5,
+}
 
 with open(join(dirname(__file__), "data", "iso_country_code_alpha2.txt"), "r") as iofile:
-    liste_pays = iofile.read().split("\n")
-liste_pays = set(liste_pays)
+    liste_pays = set(iofile.read().split("\n"))
 
 
 def _is(val):
-    if not isinstance(val, str) or not bool(re.match(r"[A-Z]{2}$", val)):
+    if not isinstance(val, str) or not bool(re.match(r"[a-zA-Z]{2}$", val)):
         return False
-    return val in liste_pays
+    return val.upper() in liste_pays
 
 
 _test_values = {
-    True: ["FR"],
+    True: ["FR", "sj"],
     False: ["XX", "A", "FRA"],
 }

csv_detective/formats/iso_country_code_alpha3.py

@@ -1,30 +1,23 @@
 import re
 from os.path import dirname, join
 
+from csv_detective.formats.iso_country_code_alpha2 import labels  # noqa
+
 proportion = 1
 tags = ["geo"]
-labels = [
-    "iso country code",
-    "code pays",
-    "pays",
-    "country",
-    "nation",
-    "pays code",
-    "code pays (iso)",
-]
 
 with open(join(dirname(__file__), "data", "iso_country_code_alpha3.txt"), "r") as iofile:
-    liste_pays = iofile.read().split("\n")
+    liste_pays = set(iofile.read().split("\n"))
 
 
 def _is(val):
     """Renvoie True si val peut etre un code iso pays alpha-3, False sinon"""
-    if not isinstance(val, str) or not bool(re.match(r"[A-Z]{3}$", val)):
+    if not isinstance(val, str) or not bool(re.match(r"[a-zA-Z]{3}$", val)):
         return False
-    return val in set(liste_pays)
+    return val.upper() in liste_pays
 
 
 _test_values = {
-    True: ["FRA"],
+    True: ["FRA", "brb"],
     False: ["XXX", "FR", "A"],
 }

csv_detective/formats/iso_country_code_numeric.py

@@ -1,17 +1,10 @@
 import re
 from os.path import dirname, join
 
+from csv_detective.formats.iso_country_code_alpha2 import labels  # noqa
+
 proportion = 1
 tags = ["geo"]
-labels = [
-    "iso country code",
-    "code pays",
-    "pays",
-    "country",
-    "nation",
-    "pays code",
-    "code pays (iso)",
-]
 
 with open(join(dirname(__file__), "data", "iso_country_code_numeric.txt"), "r") as iofile:
     liste_pays = iofile.read().split("\n")

csv_detective/formats/jour_de_la_semaine.py

@@ -1,14 +1,14 @@
 proportion = 0.8
 tags = ["fr", "temp"]
-labels = [
-    "jour semaine",
-    "type jour",
-    "jour de la semaine",
-    "saufjour",
-    "nomjour",
-    "jour",
-    "jour de fermeture",
-]
+labels = {
+    "jour semaine": 1,
+    "type jour": 1,
+    "jour de la semaine": 1,
+    "saufjour": 1,
+    "nomjour": 1,
+    "jour": 0.75,
+    "jour de fermeture": 1,
+}
 
 jours = {
     "lundi",
@@ -31,11 +31,10 @@ jours = {
 def _is(val):
     if not isinstance(val, str):
         return False
-    val = val.lower()
-    return val in jours
+    return val.lower() in jours
 
 
 _test_values = {
     True: ["lundi"],
-    False: ["jour de la biere"],
+    False: ["jour"],
 }

csv_detective/formats/json.py

@@ -2,7 +2,13 @@ import json
 from json import JSONDecodeError
 
 proportion = 1
+python_type = "json"
 tags = ["type"]
+labels = {
+    "list": 1,
+    "dict": 1,
+    "complex": 1,
+}
 
 
 def _is(val):

csv_detective/formats/latitude_l93.py

@@ -2,31 +2,17 @@ from frformat import LatitudeL93
 
 from csv_detective.formats.float import _is as is_float
 from csv_detective.formats.float import float_casting
+from csv_detective.formats.latitude_wgs import SHARED_LATITUDE_LABELS
 
 proportion = 1
 tags = ["fr", "geo"]
-labels = [
-    "latitude",
-    "lat",
-    "y",
-    "yf",
-    "yd",
-    "y l93",
-    "coordonnee y",
-    "latitude lb93",
-    "coord y",
-    "ycoord",
-    "geocodage y gps",
-    "location latitude",
-    "ylatitude",
-    "ylat",
-    "latitude (y)",
-    "latitudeorg",
-    "coordinates.latitude",
-    "googlemap latitude",
-    "latitudelieu",
-    "latitude googlemap",
-]
+mandatory_label = True
+python_type = "float"
+labels = SHARED_LATITUDE_LABELS | {
+    "y l93": 1,
+    "latitude lb93": 1,
+    "lamby": 1,
+}
 
 _latitudel93 = LatitudeL93()
 

csv_detective/formats/latitude_wgs.py

@@ -1,42 +1,44 @@
 from csv_detective.formats.float import _is as is_float
+from csv_detective.formats.int import _is as is_int
 
 proportion = 1
 tags = ["geo"]
-labels = [
-    "latitude",
-    "lat",
-    "y",
-    "yf",
-    "yd",
-    "coordonnee y",
-    "coord y",
-    "ycoord",
-    "geocodage y gps",
-    "location latitude",
-    "ylatitude",
-    "ylat",
-    "latitude (y)",
-    "latitudeorg",
-    "coordinates.latitude",
-    "googlemap latitude",
-    "latitudelieu",
-    "latitude googlemap",
-    "latitude wgs84",
-    "y wgs84",
-    "latitude (wgs84)",
-]
+mandatory_label = True
+python_type = "float"
+SHARED_LATITUDE_LABELS = {
+    "latitude": 1,
+    "lat": 0.75,
+    "y": 0.5,
+    "yf": 0.5,
+    "yd": 0.5,
+    "coordonnee y": 1,
+    "coord y": 1,
+    "ycoord": 1,
+    "ylat": 1,
+}
+labels = SHARED_LATITUDE_LABELS | {
+    "y gps": 1,
+    "latitude wgs84": 1,
+    "y wgs84": 1,
+    "wsg": 0.75,
+    "gps": 0.5,
+}
 
 
 def _is(val):
     try:
-        return is_float(val) and float(val) >= -90 and float(val) <= 90
-    except ValueError:
-        return False
-    except OverflowError:
+        return (
+            is_float(val)
+            and -90 <= float(val) <= 90
+            # we ideally would like a certain level of decimal precision
+            # but 1.200 is saved as 1.2 in csv so we just discriminate ints
+            and not is_int(val)
+        )
+    except Exception:
         return False
 
 
 _test_values = {
-    True: ["43.2", "-22"],
-    False: ["100"],
+    True: ["43.2872", "-22.61", "-3.0"],
+    False: ["100.1973", "40"],
 }

csv_detective/formats/latitude_wgs_fr_metropole.py

@@ -1,42 +1,19 @@
-from csv_detective.formats.float import _is as is_float
+from csv_detective.formats.latitude_wgs import _is as is_latitude, labels  # noqa
 
 proportion = 1
 tags = ["fr", "geo"]
-labels = [
-    "latitude",
-    "lat",
-    "y",
-    "yf",
-    "yd",
-    "coordonnee y",
-    "coord y",
-    "ycoord",
-    "geocodage y gps",
-    "location latitude",
-    "ylatitude",
-    "ylat",
-    "latitude (y)",
-    "latitudeorg",
-    "coordinates.latitude",
-    "googlemap latitude",
-    "latitudelieu",
-    "latitude googlemap",
-    "latitude wgs84",
-    "y wgs84",
-    "latitude (wgs84)",
-]
+mandatory_label = True
+python_type = "float"
 
 
 def _is(val):
     try:
-        return is_float(val) and float(val) >= 41.3 and float(val) <= 51.3
-    except ValueError:
-        return False
-    except OverflowError:
+        return is_latitude(val) and 41.3 <= float(val) <= 51.3
+    except Exception:
         return False
 
 
 _test_values = {
-    True: ["42.5"],
-    False: ["22.5", "62.5"],
+    True: ["42.576", "42.5"],
+    False: ["22.5"],
 }

csv_detective/formats/latlon_wgs.py

@@ -3,37 +3,39 @@ from csv_detective.formats.longitude_wgs import _is as is_lon
 
 proportion = 1
 tags = ["geo"]
+mandatory_label = True
+
+SHARED_COORDS_LABELS = {
+    "ban": 1,
+    "coordinates": 1,
+    "coordonnees": 1,
+    "coordonnees insee": 1,
+    "coord": 1,
+    "geo": 0.5,
+    "geopoint": 1,
+    "geoloc": 1,
+    "geolocalisation": 1,
+    "geom": 0.75,
+    "geometry": 1,
+    "gps": 1,
+    "localisation": 1,
+    "point": 1,
+    "position": 1,
+    "wgs84": 1,
+}
 
-SHARED_COORDS_LABELS = [
-    "ban",
-    "coordinates",
-    "coordonnees",
-    "coordonnees insee",
-    "geo",
-    "geopoint",
-    "geoloc",
-    "geolocalisation",
-    "geom",
-    "geometry",
-    "gps",
-    "localisation",
-    "point",
-    "position",
-    "wgs84",
-]
-
-specific = [
-    "latlon",
-    "lat lon",
-    "x y",
-    "xy",
-]
+specific = {
+    "latlon": 1,
+    "lat lon": 1,
+    "x y": 0.75,
+    "xy": 0.75,
+}
 
 # we aim wide to catch exact matches if possible for the highest possible score
 labels = (
     SHARED_COORDS_LABELS
-    + specific
-    + [w + sep + suf for suf in specific for w in SHARED_COORDS_LABELS for sep in ["", " "]]
+    | specific
+    | {w + sep + suf: 1 for suf in specific for w in SHARED_COORDS_LABELS for sep in ["", " "]}
 )
 
 
@@ -48,6 +50,6 @@ def _is(val):
 
 
 _test_values = {
-    True: ["43.2,-22.6", "-10.7,140", "-40.7, 10.8", "[12,-0.28]"],
-    False: ["0.1,192", "-102, 92", "[23.02,4.1", "23.02,4.1]", "160.1,-27"],
+    True: ["43.2,-22.6", "-10.71,140.0", "-40.791, 10.81", "[12.01,-0.28]"],
+    False: ["0.1,192", "-102, 92", "[23.02,4.1", "23.02,4.1]", "160.1,-27", "1,2", "43, -23"],
 }