csv-detective 0.6.7__py3-none-any.whl → 0.9.3.dev2438__py3-none-any.whl

csv_detective/formats/datetime_rfc822.py

@@ -0,0 +1,24 @@
+import re
+
+from csv_detective.formats.datetime_aware import labels  # noqa
+
+proportion = 1
+tags = ["temp", "type"]
+
+
+def _is(val):
+    return isinstance(val, str) and bool(
+        re.match(
+            r"^[A-Za-z]{3}, (0[1-9]|[1-2][0-9]|3[01]) [A-Za-z]{3} \d{4} "
+            r"([0-2])([0-9]):([0-5])([0-9]):([0-5])([0-9]) "
+            r"(ut|gmt|est|edt|cst|cdt|mst|mdt|pst|pdt|[+\-](0[0-9]|1[0-3])00)$",
+            val.lower(),
+            re.IGNORECASE,
+        )
+    )
+
+
+_test_values = {
+    True: ["Sun, 06 Nov 1994 08:49:37 GMT"],
+    False: ["2021-06-22T10:20:10"],
+}

csv_detective/formats/departement.py

@@ -0,0 +1,37 @@
+from frformat import Departement, Millesime, Options
+
+proportion = 0.9
+tags = ["fr", "geo"]
+labels = [
+    "departement",
+    "libelle du departement",
+    "deplib",
+    "nom dept",
+    "dept",
+    "libdepartement",
+    "nom departement",
+    "libelle dep",
+    "libelle departement",
+    "lb departements",
+    "dep libusage",
+    "lb departement",
+    "nom dep",
+]
+
+_options = Options(
+    ignore_case=True,
+    ignore_accents=True,
+    replace_non_alphanumeric_with_space=True,
+    ignore_extra_whitespace=True,
+)
+_departement = Departement(Millesime.LATEST, _options)
+
+
+def _is(val):
+    return isinstance(val, str) and _departement.is_valid(val)
+
+
+_test_values = {
+    True: ["essonne"],
+    False: ["alabama", "auvergne"],
+}

csv_detective/formats/email.py

@@ -0,0 +1,28 @@
+import re
+
+proportion = 0.9
+labels = [
+    "email",
+    "mail",
+    "courriel",
+    "contact",
+    "mel",
+    "lieucourriel",
+    "coordinates.emailcontact",
+    "e mail",
+    "mo mail",
+    "adresse mail",
+    "adresse email",
+]
+
+
+def _is(val):
+    return isinstance(val, str) and bool(
+        re.match(r"^[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}$", val, re.IGNORECASE)
+    )
+
+
+_test_values = {
+    True: ["cdo_intern@data.gouv.fr", "P.NOM@CIE.LONGDOMAIN"],
+    False: ["cdo@@gouv.sfd"],
+}

csv_detective/formats/float.py

@@ -0,0 +1,29 @@
+proportion = 1
+tags = ["type"]
+labels = ["part", "ratio", "taux"]
+
+
+def float_casting(val: str) -> float:
+    return float(val.replace(",", "."))
+
+
+def _is(val):
+    """Detects floats, assuming that tables will not have scientific
+    notations (3e6) or "+" in the string. "-" is still accepted."""
+    try:
+        if (
+            not isinstance(val, str)
+            or any([k in val for k in ["_", "+", "e", "E"]])
+            or (val.startswith("0") and len(val) > 1 and val[1] not in [".", ","])
+        ):
+            return False
+        float_casting(val)
+        return True
+    except ValueError:
+        return False
+
+
+_test_values = {
+    True: ["1", "0", "1764", "-24", "1.2", "1863.23", "-12.7", "0.1"],
+    False: ["01053", "01053.89", "1e3", "123_456", "123_456.78", "+35", "+35.9"],
+}

csv_detective/formats/geojson.py

@@ -0,0 +1,36 @@
+import json
+
+proportion = 1
+tags = ["geo"]
+labels = [
+    "json geojson",
+    "json",
+    "geojson",
+    "geo shape",
+    "geom",
+    "geometry",
+    "geo shape",
+    "geoshape",
+]
+
+
+def _is(val) -> bool:
+    try:
+        j = json.loads(val)
+        if isinstance(j, dict):
+            if "type" in j and "coordinates" in j:
+                return True
+            if "geometry" in j and "coordinates" in j["geometry"]:
+                return True
+    except Exception:
+        pass
+    return False
+
+
+_test_values = {
+    True: [
+        '{"coordinates": [45.783753, 3.049342], "type": "63870"}',
+        '{"geometry": {"coordinates": [45.783753, 3.049342]}}',
+    ],
+    False: ['{"pomme": "fruit", "reponse": 42}'],
+}

csv_detective/formats/insee_ape700.py

@@ -0,0 +1,31 @@
+from os.path import dirname, join
+
+from csv_detective.parsing.text import _process_text
+
+proportion = 0.8
+tags = ["fr"]
+labels = [
+    "code ape",
+    "code activite (ape)",
+    "code naf",
+    "code naf organisme designe",
+    "code naf organisme designant",
+    "base sirene : code ape de l'etablissement siege",
+]
+
+f = open(join(dirname(__file__), "data", "insee_ape700.txt"), "r")
+condes_insee_ape = f.read().split("\n")
+# removing empty str due to additionnal line in file
+del condes_insee_ape[-1]
+condes_insee_ape = set(condes_insee_ape)
+f.close()
+
+
+def _is(val):
+    if not isinstance(val, str):
+        return False
+    val = _process_text(val).upper()
+    return val in condes_insee_ape
+
+
+_test_values = {True: ["0116Z"], False: ["0116A"]}

csv_detective/formats/insee_canton.py

@@ -0,0 +1,28 @@
+from frformat import Canton, Millesime, Options
+
+proportion = 0.9
+tags = ["fr", "geo"]
+labels = [
+    "insee canton",
+    "canton",
+    "cant",
+    "nom canton",
+]
+
+_options = Options(
+    ignore_case=True,
+    ignore_accents=True,
+    replace_non_alphanumeric_with_space=True,
+    ignore_extra_whitespace=True,
+)
+_canton = Canton(Millesime.LATEST, _options)
+
+
+def _is(val):
+    return isinstance(val, str) and _canton.is_valid(val)
+
+
+_test_values = {
+    True: ["nantua"],
+    False: ["california"],
+}

csv_detective/formats/int.py

@@ -0,0 +1,23 @@
+labels = ["nb", "nombre", "nbre"]
+tag = ["type"]
+
+
+def _is(val):
+    """Detects integers"""
+    if (
+        not isinstance(val, str)
+        or any([v in val for v in [".", "_", "+"]])
+        or (val.startswith("0") and len(val) > 1)
+    ):
+        return False
+    try:
+        int(val)
+        return True
+    except ValueError:
+        return False
+
+
+_test_values = {
+    True: ["1", "0", "1764", "-24"],
+    False: ["01053", "1.2", "123_456", "+35"],
+}

csv_detective/formats/iso_country_code_alpha2.py

@@ -0,0 +1,30 @@
+import re
+from os.path import dirname, join
+
+proportion = 1
+tags = ["geo"]
+labels = [
+    "iso country code",
+    "code pays",
+    "pays",
+    "country",
+    "nation",
+    "pays code",
+    "code pays (iso)",
+]
+
+with open(join(dirname(__file__), "data", "iso_country_code_alpha2.txt"), "r") as iofile:
+    liste_pays = iofile.read().split("\n")
+liste_pays = set(liste_pays)
+
+
+def _is(val):
+    if not isinstance(val, str) or not bool(re.match(r"[A-Z]{2}$", val)):
+        return False
+    return val in liste_pays
+
+
+_test_values = {
+    True: ["FR"],
+    False: ["XX", "A", "FRA"],
+}

csv_detective/formats/iso_country_code_alpha3.py

@@ -0,0 +1,30 @@
+import re
+from os.path import dirname, join
+
+proportion = 1
+tags = ["geo"]
+labels = [
+    "iso country code",
+    "code pays",
+    "pays",
+    "country",
+    "nation",
+    "pays code",
+    "code pays (iso)",
+]
+
+with open(join(dirname(__file__), "data", "iso_country_code_alpha3.txt"), "r") as iofile:
+    liste_pays = iofile.read().split("\n")
+
+
+def _is(val):
+    """Renvoie True si val peut etre un code iso pays alpha-3, False sinon"""
+    if not isinstance(val, str) or not bool(re.match(r"[A-Z]{3}$", val)):
+        return False
+    return val in set(liste_pays)
+
+
+_test_values = {
+    True: ["FRA"],
+    False: ["XXX", "FR", "A"],
+}

csv_detective/formats/iso_country_code_numeric.py

@@ -0,0 +1,31 @@
+import re
+from os.path import dirname, join
+
+proportion = 1
+tags = ["geo"]
+labels = [
+    "iso country code",
+    "code pays",
+    "pays",
+    "country",
+    "nation",
+    "pays code",
+    "code pays (iso)",
+]
+
+with open(join(dirname(__file__), "data", "iso_country_code_numeric.txt"), "r") as iofile:
+    liste_pays = iofile.read().split("\n")
+liste_pays = set(liste_pays)
+
+
+def _is(val):
+    """Renvoie True si val peut etre un code iso pays numerique, False sinon"""
+    if not isinstance(val, str) or not bool(re.match(r"[0-9]{3}$", val)):
+        return False
+    return val in liste_pays
+
+
+_test_values = {
+    True: ["250"],
+    False: ["003"],
+}

csv_detective/formats/jour_de_la_semaine.py

@@ -0,0 +1,41 @@
+proportion = 0.8
+tags = ["fr", "temp"]
+labels = [
+    "jour semaine",
+    "type jour",
+    "jour de la semaine",
+    "saufjour",
+    "nomjour",
+    "jour",
+    "jour de fermeture",
+]
+
+jours = {
+    "lundi",
+    "mardi",
+    "mercredi",
+    "jeudi",
+    "vendredi",
+    "samedi",
+    "dimanche",
+    "lun",
+    "mar",
+    "mer",
+    "jeu",
+    "ven",
+    "sam",
+    "dim",
+}
+
+
+def _is(val):
+    if not isinstance(val, str):
+        return False
+    val = val.lower()
+    return val in jours
+
+
+_test_values = {
+    True: ["lundi"],
+    False: ["jour de la biere"],
+}

csv_detective/formats/json.py

@@ -0,0 +1,20 @@
+import json
+from json import JSONDecodeError
+
+proportion = 1
+tags = ["type"]
+
+
+def _is(val):
+    try:
+        loaded = json.loads(val)
+        # we don't want to consider integers for instance
+        return isinstance(loaded, (list, dict))
+    except (JSONDecodeError, TypeError):
+        return False
+
+
+_test_values = {
+    True: ['{"pomme": "fruit", "reponse": 42}', "[1,2,3,4]"],
+    False: ["5", '{"zefib":', '{"a"}'],
+}

csv_detective/formats/latitude_l93.py

@@ -0,0 +1,48 @@
+from frformat import LatitudeL93
+
+from csv_detective.formats.float import _is as is_float
+from csv_detective.formats.float import float_casting
+
+proportion = 1
+tags = ["fr", "geo"]
+labels = [
+    "latitude",
+    "lat",
+    "y",
+    "yf",
+    "yd",
+    "y l93",
+    "coordonnee y",
+    "latitude lb93",
+    "coord y",
+    "ycoord",
+    "geocodage y gps",
+    "location latitude",
+    "ylatitude",
+    "ylat",
+    "latitude (y)",
+    "latitudeorg",
+    "coordinates.latitude",
+    "googlemap latitude",
+    "latitudelieu",
+    "latitude googlemap",
+]
+
+_latitudel93 = LatitudeL93()
+
+
+def _is(val):
+    try:
+        if isinstance(val, str) and is_float(val):
+            return _latitudel93.is_valid(float_casting(val))
+
+        return False
+
+    except (ValueError, OverflowError):
+        return False
+
+
+_test_values = {
+    True: ["6037008", "7123528.5", "7124528,5"],
+    False: ["0", "-6734529.6", "7245669.8", "3422674,78", "32_34"],
+}

csv_detective/formats/latitude_wgs.py

@@ -0,0 +1,42 @@
+from csv_detective.formats.float import _is as is_float
+
+proportion = 1
+tags = ["geo"]
+labels = [
+    "latitude",
+    "lat",
+    "y",
+    "yf",
+    "yd",
+    "coordonnee y",
+    "coord y",
+    "ycoord",
+    "geocodage y gps",
+    "location latitude",
+    "ylatitude",
+    "ylat",
+    "latitude (y)",
+    "latitudeorg",
+    "coordinates.latitude",
+    "googlemap latitude",
+    "latitudelieu",
+    "latitude googlemap",
+    "latitude wgs84",
+    "y wgs84",
+    "latitude (wgs84)",
+]
+
+
+def _is(val):
+    try:
+        return is_float(val) and float(val) >= -90 and float(val) <= 90
+    except ValueError:
+        return False
+    except OverflowError:
+        return False
+
+
+_test_values = {
+    True: ["43.2", "-22"],
+    False: ["100"],
+}

csv_detective/formats/latitude_wgs_fr_metropole.py

@@ -0,0 +1,42 @@
+from csv_detective.formats.float import _is as is_float
+
+proportion = 1
+tags = ["fr", "geo"]
+labels = [
+    "latitude",
+    "lat",
+    "y",
+    "yf",
+    "yd",
+    "coordonnee y",
+    "coord y",
+    "ycoord",
+    "geocodage y gps",
+    "location latitude",
+    "ylatitude",
+    "ylat",
+    "latitude (y)",
+    "latitudeorg",
+    "coordinates.latitude",
+    "googlemap latitude",
+    "latitudelieu",
+    "latitude googlemap",
+    "latitude wgs84",
+    "y wgs84",
+    "latitude (wgs84)",
+]
+
+
+def _is(val):
+    try:
+        return is_float(val) and float(val) >= 41.3 and float(val) <= 51.3
+    except ValueError:
+        return False
+    except OverflowError:
+        return False
+
+
+_test_values = {
+    True: ["42.5"],
+    False: ["22.5", "62.5"],
+}

csv_detective/formats/latlon_wgs.py

@@ -0,0 +1,53 @@
+from csv_detective.formats.latitude_wgs import _is as is_lat
+from csv_detective.formats.longitude_wgs import _is as is_lon
+
+proportion = 1
+tags = ["geo"]
+
+SHARED_COORDS_LABELS = [
+    "ban",
+    "coordinates",
+    "coordonnees",
+    "coordonnees insee",
+    "geo",
+    "geopoint",
+    "geoloc",
+    "geolocalisation",
+    "geom",
+    "geometry",
+    "gps",
+    "localisation",
+    "point",
+    "position",
+    "wgs84",
+]
+
+specific = [
+    "latlon",
+    "lat lon",
+    "x y",
+    "xy",
+]
+
+# we aim wide to catch exact matches if possible for the highest possible score
+labels = (
+    SHARED_COORDS_LABELS
+    + specific
+    + [w + sep + suf for suf in specific for w in SHARED_COORDS_LABELS for sep in ["", " "]]
+)
+
+
+def _is(val):
+    if not isinstance(val, str) or val.count(",") != 1:
+        return False
+    lat, lon = val.split(",")
+    # handling [lat,lon]
+    if lat.startswith("[") and lon.endswith("]"):
+        lat, lon = lat[1:], lon[:-1]
+    return is_lat(lat) and is_lon(lon.replace(" ", ""))
+
+
+_test_values = {
+    True: ["43.2,-22.6", "-10.7,140", "-40.7, 10.8", "[12,-0.28]"],
+    False: ["0.1,192", "-102, 92", "[23.02,4.1", "23.02,4.1]", "160.1,-27"],
+}

csv_detective/formats/longitude_l93.py

@@ -0,0 +1,39 @@
+from frformat import LongitudeL93
+
+from csv_detective.formats.float import _is as is_float
+from csv_detective.formats.float import float_casting
+
+proportion = 1
+tags = ["fr", "geo"]
+labels = [
+    "longitude",
+    "lon",
+    "long",
+    "geocodage x gps",
+    "location longitude",
+    "xlongitude",
+    "lng",
+    "xlong",
+    "x",
+    "xf",
+    "xd",
+]
+
+_longitudel93 = LongitudeL93()
+
+
+def _is(val):
+    try:
+        if isinstance(val, str) and is_float(val):
+            return _longitudel93.is_valid(float_casting(val))
+
+        return False
+
+    except (ValueError, OverflowError):
+        return False
+
+
+_test_values = {
+    True: ["0", "-154", "1265783,45", "34723.4"],
+    False: ["1456669.8", "-776225", "346_3214"],
+}

csv_detective/formats/longitude_wgs.py

@@ -0,0 +1,32 @@
+from csv_detective.formats.float import _is as is_float
+
+proportion = 1
+tags = ["geo"]
+labels = [
+    "longitude",
+    "lon",
+    "long",
+    "geocodage x gps",
+    "location longitude",
+    "xlongitude",
+    "lng",
+    "xlong",
+    "x",
+    "xf",
+    "xd",
+]
+
+
+def _is(val):
+    try:
+        return is_float(val) and float(val) >= -180 and float(val) <= 180
+    except ValueError:
+        return False
+    except OverflowError:
+        return False
+
+
+_test_values = {
+    True: ["120", "-20.2"],
+    False: ["-200"],
+}

csv_detective/formats/longitude_wgs_fr_metropole.py

@@ -0,0 +1,32 @@
+from csv_detective.formats.float import _is as is_float
+
+proportion = 1
+tags = ["fr", "geo"]
+labels = [
+    "longitude",
+    "lon",
+    "long",
+    "geocodage x gps",
+    "location longitude",
+    "xlongitude",
+    "lng",
+    "xlong",
+    "x",
+    "xf",
+    "xd",
+]
+
+
+def _is(val):
+    try:
+        return is_float(val) and float(val) >= -5.5 and float(val) <= 9.8
+    except ValueError:
+        return False
+    except OverflowError:
+        return False
+
+
+_test_values = {
+    True: ["-2.5"],
+    False: ["12.8"],
+}