csv-detective 0.9.3.dev2258__py3-none-any.whl → 0.9.3.dev2348__py3-none-any.whl

csv_detective/formats/geojson.py

@@ -0,0 +1,36 @@
+import json
+
+proportion = 1
+tags = ["geo"]
+labels = [
+    "json geojson",
+    "json",
+    "geojson",
+    "geo shape",
+    "geom",
+    "geometry",
+    "geo shape",
+    "geoshape",
+]
+
+
+def _is(val) -> bool:
+    try:
+        j = json.loads(val)
+        if isinstance(j, dict):
+            if "type" in j and "coordinates" in j:
+                return True
+            if "geometry" in j and "coordinates" in j["geometry"]:
+                return True
+    except Exception:
+        pass
+    return False
+
+
+_test_values = {
+    True: [
+        '{"coordinates": [45.783753, 3.049342], "type": "63870"}',
+        '{"geometry": {"coordinates": [45.783753, 3.049342]}}',
+    ],
+    False: ['{"pomme": "fruit", "reponse": 42}'],
+}

csv_detective/{detect_fields/FR/other/insee_ape700/__init__.py → formats/insee_ape700.py}

@@ -1,19 +1,31 @@
-from os.path import dirname, join
-
-from csv_detective.parsing.text import _process_text
-
-PROPORTION = 1
-f = open(join(dirname(__file__), "insee_ape700.txt"), "r")
-condes_insee_ape = f.read().split("\n")
-# removing empty str due to additionnal line in file
-del condes_insee_ape[-1]
-condes_insee_ape = set(condes_insee_ape)
-f.close()
-
-
-def _is(val):
-    """Repère les codes APE700 de l'INSEE"""
-    if not isinstance(val, str):
-        return False
-    val = _process_text(val).upper()
-    return val in condes_insee_ape
+from os.path import dirname, join
+
+from csv_detective.parsing.text import _process_text
+
+proportion = 0.8
+tags = ["fr"]
+labels = [
+    "code ape",
+    "code activite (ape)",
+    "code naf",
+    "code naf organisme designe",
+    "code naf organisme designant",
+    "base sirene : code ape de l'etablissement siege",
+]
+
+f = open(join(dirname(__file__), "data", "insee_ape700.txt"), "r")
+condes_insee_ape = f.read().split("\n")
+# removing empty str due to additionnal line in file
+del condes_insee_ape[-1]
+condes_insee_ape = set(condes_insee_ape)
+f.close()
+
+
+def _is(val):
+    if not isinstance(val, str):
+        return False
+    val = _process_text(val).upper()
+    return val in condes_insee_ape
+
+
+_test_values = {True: ["0116Z"], False: ["0116A"]}

csv_detective/{detect_fields/FR/geo/insee_canton/__init__.py → formats/insee_canton.py}

@@ -1,15 +1,28 @@
-from frformat import Canton, Millesime, Options
-
-PROPORTION = 0.9
-_options = Options(
-    ignore_case=True,
-    ignore_accents=True,
-    replace_non_alphanumeric_with_space=True,
-    ignore_extra_whitespace=True,
-)
-_canton = Canton(Millesime.LATEST, _options)
-
-
-def _is(val):
-    """Match avec le nom des cantons"""
-    return isinstance(val, str) and _canton.is_valid(val)
+from frformat import Canton, Millesime, Options
+
+proportion = 0.9
+tags = ["fr", "geo"]
+labels = [
+    "insee canton",
+    "canton",
+    "cant",
+    "nom canton",
+]
+
+_options = Options(
+    ignore_case=True,
+    ignore_accents=True,
+    replace_non_alphanumeric_with_space=True,
+    ignore_extra_whitespace=True,
+)
+_canton = Canton(Millesime.LATEST, _options)
+
+
+def _is(val):
+    return isinstance(val, str) and _canton.is_valid(val)
+
+
+_test_values = {
+    True: ["nantua"],
+    False: ["california"],
+}

csv_detective/{detect_fields/other/int/__init__.py → formats/int.py}

@@ -1,16 +1,23 @@
-PROPORTION = 1
-
-
-def _is(val):
-    """Detects integers"""
-    if (
-        not isinstance(val, str)
-        or any([v in val for v in [".", "_", "+"]])
-        or (val.startswith("0") and len(val) > 1)
-    ):
-        return False
-    try:
-        int(val)
-        return True
-    except ValueError:
-        return False
+labels = ["nb", "nombre", "nbre"]
+tag = ["type"]
+
+
+def _is(val):
+    """Detects integers"""
+    if (
+        not isinstance(val, str)
+        or any([v in val for v in [".", "_", "+"]])
+        or (val.startswith("0") and len(val) > 1)
+    ):
+        return False
+    try:
+        int(val)
+        return True
+    except ValueError:
+        return False
+
+
+_test_values = {
+    True: ["1", "0", "1764", "-24"],
+    False: ["01053", "1.2", "123_456", "+35"],
+}

csv_detective/formats/iso_country_code_alpha2.py

@@ -0,0 +1,30 @@
+import re
+from os.path import dirname, join
+
+proportion = 1
+tags = ["geo"]
+labels = [
+    "iso country code",
+    "code pays",
+    "pays",
+    "country",
+    "nation",
+    "pays code",
+    "code pays (iso)",
+]
+
+with open(join(dirname(__file__), "data", "iso_country_code_alpha2.txt"), "r") as iofile:
+    liste_pays = iofile.read().split("\n")
+liste_pays = set(liste_pays)
+
+
+def _is(val):
+    if not isinstance(val, str) or not bool(re.match(r"[A-Z]{2}$", val)):
+        return False
+    return val in liste_pays
+
+
+_test_values = {
+    True: ["FR"],
+    False: ["XX", "A", "FRA"],
+}

csv_detective/formats/iso_country_code_alpha3.py

@@ -0,0 +1,30 @@
+import re
+from os.path import dirname, join
+
+proportion = 1
+tags = ["geo"]
+labels = [
+    "iso country code",
+    "code pays",
+    "pays",
+    "country",
+    "nation",
+    "pays code",
+    "code pays (iso)",
+]
+
+with open(join(dirname(__file__), "data", "iso_country_code_alpha3.txt"), "r") as iofile:
+    liste_pays = iofile.read().split("\n")
+
+
+def _is(val):
+    """Renvoie True si val peut etre un code iso pays alpha-3, False sinon"""
+    if not isinstance(val, str) or not bool(re.match(r"[A-Z]{3}$", val)):
+        return False
+    return val in set(liste_pays)
+
+
+_test_values = {
+    True: ["FRA"],
+    False: ["XXX", "FR", "A"],
+}

csv_detective/formats/iso_country_code_numeric.py

@@ -0,0 +1,31 @@
+import re
+from os.path import dirname, join
+
+proportion = 1
+tags = ["geo"]
+labels = [
+    "iso country code",
+    "code pays",
+    "pays",
+    "country",
+    "nation",
+    "pays code",
+    "code pays (iso)",
+]
+
+with open(join(dirname(__file__), "data", "iso_country_code_numeric.txt"), "r") as iofile:
+    liste_pays = iofile.read().split("\n")
+liste_pays = set(liste_pays)
+
+
+def _is(val):
+    """Renvoie True si val peut etre un code iso pays numerique, False sinon"""
+    if not isinstance(val, str) or not bool(re.match(r"[0-9]{3}$", val)):
+        return False
+    return val in liste_pays
+
+
+_test_values = {
+    True: ["250"],
+    False: ["003"],
+}

csv_detective/{detect_fields/FR/temp/jour_de_la_semaine/__init__.py → formats/jour_de_la_semaine.py}

@@ -1,25 +1,41 @@
-PROPORTION = 1
-jours = {
-    "lundi",
-    "mardi",
-    "mercredi",
-    "jeudi",
-    "vendredi",
-    "samedi",
-    "dimanche",
-    "lun",
-    "mar",
-    "mer",
-    "jeu",
-    "ven",
-    "sam",
-    "dim",
-}
-
-
-def _is(val):
-    """Renvoie True si les champs peuvent être des jours de la semaine"""
-    if not isinstance(val, str):
-        return False
-    val = val.lower()
-    return val in jours
+proportion = 0.8
+tags = ["fr", "temp"]
+labels = [
+    "jour semaine",
+    "type jour",
+    "jour de la semaine",
+    "saufjour",
+    "nomjour",
+    "jour",
+    "jour de fermeture",
+]
+
+jours = {
+    "lundi",
+    "mardi",
+    "mercredi",
+    "jeudi",
+    "vendredi",
+    "samedi",
+    "dimanche",
+    "lun",
+    "mar",
+    "mer",
+    "jeu",
+    "ven",
+    "sam",
+    "dim",
+}
+
+
+def _is(val):
+    if not isinstance(val, str):
+        return False
+    val = val.lower()
+    return val in jours
+
+
+_test_values = {
+    True: ["lundi"],
+    False: ["jour de la biere"],
+}

csv_detective/{detect_fields/other/json/__init__.py → formats/json.py}

@@ -1,14 +1,20 @@
-import json
-from json import JSONDecodeError
-
-PROPORTION = 1
-
-
-def _is(val):
-    """Detects json"""
-    try:
-        loaded = json.loads(val)
-        # we don't want to consider integers for instance
-        return isinstance(loaded, (list, dict))
-    except (JSONDecodeError, TypeError):
-        return False
+import json
+from json import JSONDecodeError
+
+proportion = 1
+tags = ["type"]
+
+
+def _is(val):
+    try:
+        loaded = json.loads(val)
+        # we don't want to consider integers for instance
+        return isinstance(loaded, (list, dict))
+    except (JSONDecodeError, TypeError):
+        return False
+
+
+_test_values = {
+    True: ['{"pomme": "fruit", "reponse": 42}', "[1,2,3,4]"],
+    False: ["5", '{"zefib":', '{"a"}'],
+}

csv_detective/formats/latitude_l93.py

@@ -0,0 +1,48 @@
+from frformat import LatitudeL93
+
+from csv_detective.formats.float import _is as is_float
+from csv_detective.formats.float import float_casting
+
+proportion = 1
+tags = ["fr", "geo"]
+labels = [
+    "latitude",
+    "lat",
+    "y",
+    "yf",
+    "yd",
+    "y l93",
+    "coordonnee y",
+    "latitude lb93",
+    "coord y",
+    "ycoord",
+    "geocodage y gps",
+    "location latitude",
+    "ylatitude",
+    "ylat",
+    "latitude (y)",
+    "latitudeorg",
+    "coordinates.latitude",
+    "googlemap latitude",
+    "latitudelieu",
+    "latitude googlemap",
+]
+
+_latitudel93 = LatitudeL93()
+
+
+def _is(val):
+    try:
+        if isinstance(val, str) and is_float(val):
+            return _latitudel93.is_valid(float_casting(val))
+
+        return False
+
+    except (ValueError, OverflowError):
+        return False
+
+
+_test_values = {
+    True: ["6037008", "7123528.5", "7124528,5"],
+    False: ["0", "-6734529.6", "7245669.8", "3422674,78", "32_34"],
+}

csv_detective/formats/latitude_wgs.py

@@ -0,0 +1,42 @@
+from csv_detective.formats.float import _is as is_float
+
+proportion = 1
+tags = ["geo"]
+labels = [
+    "latitude",
+    "lat",
+    "y",
+    "yf",
+    "yd",
+    "coordonnee y",
+    "coord y",
+    "ycoord",
+    "geocodage y gps",
+    "location latitude",
+    "ylatitude",
+    "ylat",
+    "latitude (y)",
+    "latitudeorg",
+    "coordinates.latitude",
+    "googlemap latitude",
+    "latitudelieu",
+    "latitude googlemap",
+    "latitude wgs84",
+    "y wgs84",
+    "latitude (wgs84)",
+]
+
+
+def _is(val):
+    try:
+        return is_float(val) and float(val) >= -90 and float(val) <= 90
+    except ValueError:
+        return False
+    except OverflowError:
+        return False
+
+
+_test_values = {
+    True: ["43.2", "-22"],
+    False: ["100"],
+}

csv_detective/formats/latitude_wgs_fr_metropole.py

@@ -0,0 +1,42 @@
+from csv_detective.formats.float import _is as is_float
+
+proportion = 1
+tags = ["fr", "geo"]
+labels = [
+    "latitude",
+    "lat",
+    "y",
+    "yf",
+    "yd",
+    "coordonnee y",
+    "coord y",
+    "ycoord",
+    "geocodage y gps",
+    "location latitude",
+    "ylatitude",
+    "ylat",
+    "latitude (y)",
+    "latitudeorg",
+    "coordinates.latitude",
+    "googlemap latitude",
+    "latitudelieu",
+    "latitude googlemap",
+    "latitude wgs84",
+    "y wgs84",
+    "latitude (wgs84)",
+]
+
+
+def _is(val):
+    try:
+        return is_float(val) and float(val) >= 41.3 and float(val) <= 51.3
+    except ValueError:
+        return False
+    except OverflowError:
+        return False
+
+
+_test_values = {
+    True: ["42.5"],
+    False: ["22.5", "62.5"],
+}

csv_detective/formats/latlon_wgs.py

@@ -0,0 +1,53 @@
+from csv_detective.formats.latitude_wgs import _is as is_lat
+from csv_detective.formats.longitude_wgs import _is as is_lon
+
+proportion = 1
+tags = ["geo"]
+
+SHARED_COORDS_LABELS = [
+    "ban",
+    "coordinates",
+    "coordonnees",
+    "coordonnees insee",
+    "geo",
+    "geopoint",
+    "geoloc",
+    "geolocalisation",
+    "geom",
+    "geometry",
+    "gps",
+    "localisation",
+    "point",
+    "position",
+    "wgs84",
+]
+
+specific = [
+    "latlon",
+    "lat lon",
+    "x y",
+    "xy",
+]
+
+# we aim wide to catch exact matches if possible for the highest possible score
+labels = (
+    SHARED_COORDS_LABELS
+    + specific
+    + [w + sep + suf for suf in specific for w in SHARED_COORDS_LABELS for sep in ["", " "]]
+)
+
+
+def _is(val):
+    if not isinstance(val, str) or val.count(",") != 1:
+        return False
+    lat, lon = val.split(",")
+    # handling [lat,lon]
+    if lat.startswith("[") and lon.endswith("]"):
+        lat, lon = lat[1:], lon[:-1]
+    return is_lat(lat) and is_lon(lon.replace(" ", ""))
+
+
+_test_values = {
+    True: ["43.2,-22.6", "-10.7,140", "-40.7, 10.8", "[12,-0.28]"],
+    False: ["0.1,192", "-102, 92", "[23.02,4.1", "23.02,4.1]", "160.1,-27"],
+}

csv_detective/formats/longitude_l93.py

@@ -0,0 +1,39 @@
+from frformat import LongitudeL93
+
+from csv_detective.formats.float import _is as is_float
+from csv_detective.formats.float import float_casting
+
+proportion = 1
+tags = ["fr", "geo"]
+labels = [
+    "longitude",
+    "lon",
+    "long",
+    "geocodage x gps",
+    "location longitude",
+    "xlongitude",
+    "lng",
+    "xlong",
+    "x",
+    "xf",
+    "xd",
+]
+
+_longitudel93 = LongitudeL93()
+
+
+def _is(val):
+    try:
+        if isinstance(val, str) and is_float(val):
+            return _longitudel93.is_valid(float_casting(val))
+
+        return False
+
+    except (ValueError, OverflowError):
+        return False
+
+
+_test_values = {
+    True: ["0", "-154", "1265783,45", "34723.4"],
+    False: ["1456669.8", "-776225", "346_3214"],
+}

csv_detective/formats/longitude_wgs.py

@@ -0,0 +1,32 @@
+from csv_detective.formats.float import _is as is_float
+
+proportion = 1
+tags = ["geo"]
+labels = [
+    "longitude",
+    "lon",
+    "long",
+    "geocodage x gps",
+    "location longitude",
+    "xlongitude",
+    "lng",
+    "xlong",
+    "x",
+    "xf",
+    "xd",
+]
+
+
+def _is(val):
+    try:
+        return is_float(val) and float(val) >= -180 and float(val) <= 180
+    except ValueError:
+        return False
+    except OverflowError:
+        return False
+
+
+_test_values = {
+    True: ["120", "-20.2"],
+    False: ["-200"],
+}

csv_detective/formats/longitude_wgs_fr_metropole.py

@@ -0,0 +1,32 @@
+from csv_detective.formats.float import _is as is_float
+
+proportion = 1
+tags = ["fr", "geo"]
+labels = [
+    "longitude",
+    "lon",
+    "long",
+    "geocodage x gps",
+    "location longitude",
+    "xlongitude",
+    "lng",
+    "xlong",
+    "x",
+    "xf",
+    "xd",
+]
+
+
+def _is(val):
+    try:
+        return is_float(val) and float(val) >= -5.5 and float(val) <= 9.8
+    except ValueError:
+        return False
+    except OverflowError:
+        return False
+
+
+_test_values = {
+    True: ["-2.5"],
+    False: ["12.8"],
+}