csv-detective 0.10.1__py3-none-any.whl → 0.10.1.dev2576__py3-none-any.whl

csv_detective/formats/longitude_l93.py

@@ -2,17 +2,22 @@ from frformat import LongitudeL93
 
 from csv_detective.formats.float import _is as is_float
 from csv_detective.formats.float import float_casting
-from csv_detective.formats.longitude_wgs import SHARED_LONGITUDE_LABELS
 
 proportion = 1
 tags = ["fr", "geo"]
-mandatory_label = True
-python_type = "float"
-labels = SHARED_LONGITUDE_LABELS | {
-    "x l93": 1,
-    "longitude lb93": 1,
-    "lambx": 1,
-}
+labels = [
+    "longitude",
+    "lon",
+    "long",
+    "geocodage x gps",
+    "location longitude",
+    "xlongitude",
+    "lng",
+    "xlong",
+    "x",
+    "xf",
+    "xd",
+]
 
 _longitudel93 = LongitudeL93()
 

csv_detective/formats/longitude_wgs.py

@@ -1,47 +1,32 @@
 from csv_detective.formats.float import _is as is_float
-from csv_detective.formats.int import _is as is_int
 
 proportion = 1
 tags = ["geo"]
-mandatory_label = True
-python_type = "float"
-SHARED_LONGITUDE_LABELS = {
-    "longitude": 1,
-    "long": 0.75,
-    "lon": 0.75,
-    "lng": 0.5,
-    "x": 0.5,
-    "xf": 0.5,
-    "xd": 0.5,
-    "coordonnee x": 1,
-    "coord x": 1,
-    "xcoord": 1,
-    "xlon": 1,
-    "xlong": 1,
-}
-labels = SHARED_LONGITUDE_LABELS | {
-    "x gps": 1,
-    "longitude wgs84": 1,
-    "x wgs84": 1,
-    "wsg": 0.75,
-    "gps": 0.5,
-}
+labels = [
+    "longitude",
+    "lon",
+    "long",
+    "geocodage x gps",
+    "location longitude",
+    "xlongitude",
+    "lng",
+    "xlong",
+    "x",
+    "xf",
+    "xd",
+]
 
 
 def _is(val):
     try:
-        return (
-            is_float(val)
-            and -180 <= float(val) <= 180
-            # we ideally would like a certain level of decimal precision
-            # but 1.200 is saved as 1.2 in csv so we just discriminate ints
-            and not is_int(val)
-        )
-    except Exception:
+        return is_float(val) and float(val) >= -180 and float(val) <= 180
+    except ValueError:
+        return False
+    except OverflowError:
         return False
 
 
 _test_values = {
-    True: ["120.8263", "-20.27", "31.0"],
-    False: ["-200", "20"],
+    True: ["120", "-20.2"],
+    False: ["-200"],
 }

csv_detective/formats/longitude_wgs_fr_metropole.py

@@ -1,19 +1,32 @@
-from csv_detective.formats.longitude_wgs import _is as is_longitude, labels  # noqa
+from csv_detective.formats.float import _is as is_float
 
 proportion = 1
 tags = ["fr", "geo"]
-mandatory_label = True
-python_type = "float"
+labels = [
+    "longitude",
+    "lon",
+    "long",
+    "geocodage x gps",
+    "location longitude",
+    "xlongitude",
+    "lng",
+    "xlong",
+    "x",
+    "xf",
+    "xd",
+]
 
 
 def _is(val):
     try:
-        return is_longitude(val) and -5.5 <= float(val) <= 9.8
-    except Exception:
+        return is_float(val) and float(val) >= -5.5 and float(val) <= 9.8
+    except ValueError:
+        return False
+    except OverflowError:
         return False
 
 
 _test_values = {
-    True: ["-2.01", "8.0"],
+    True: ["-2.5"],
     False: ["12.8"],
 }

csv_detective/formats/lonlat_wgs.py

@@ -4,20 +4,19 @@ from csv_detective.formats.longitude_wgs import _is as is_lon
 
 proportion = 1
 tags = ["geo"]
-mandatory_label = True
 
-specific = {
-    "lonlat": 1,
-    "lon lat": 1,
-    "y x": 0.75,
-    "yx": 0.75,
-}
+specific = [
+    "lonlat",
+    "lon lat",
+    "y x",
+    "yx",
+]
 
 # we aim wide to catch exact matches if possible for the highest possible score
-labels = (
+words = (
     SHARED_COORDS_LABELS
-    | specific
-    | {w + sep + suf: 1 for suf in specific for w in SHARED_COORDS_LABELS for sep in ["", " "]}
+    + specific
+    + [w + sep + suf for suf in specific for w in SHARED_COORDS_LABELS for sep in ["", " "]]
 )
 
 
@@ -32,6 +31,6 @@ def _is(val):
 
 
 _test_values = {
-    True: ["-22.6,43.012", "140.0,-10.70", "10.829, -40.71", "[-0.28,12.43]"],
-    False: ["192,0.1", "92, -102", "[4.1,23.02", "4.1,23.02]", "-27,160.1", "2,4", "-22, 43.0"],
+    True: ["-22.6,43.2", "140,-10.7", "10.8, -40.7", "[-0.28,12]"],
+    False: ["192,0.1", "92, -102", "[4.1,23.02", "4.1,23.02]", "-27,160.1"],
 }

csv_detective/formats/mois_de_lannee.py

@@ -2,7 +2,7 @@ from unidecode import unidecode
 
 proportion = 1
 tags = ["fr", "temp"]
-labels = {"mois": 1, "month": 1}
+labels = ["mois", "month"]
 
 mois = {
     "janvier",

csv_detective/formats/money.py

@@ -1,7 +1,7 @@
 from csv_detective.formats.float import _is as is_float
 
 proportion = 0.8
-labels = {"budget": 1, "salaire": 1, "euro": 1, "euros": 1, "prêt": 1, "montant": 1}
+labels = ["budget", "salaire", "euro", "euros", "prêt", "montant"]
 
 currencies = {"€", "$", "£", "¥"}
 

csv_detective/formats/mongo_object_id.py

@@ -1,7 +1,7 @@
 import re
 
 proportion = 0.8
-labels = {"id": 1, "objectid": 1}
+labels = ["id", "objectid"]
 
 
 def _is(val):

csv_detective/formats/pays.py

@@ -2,17 +2,19 @@ from frformat import Millesime, Options, Pays
 
 proportion = 0.6
 tags = ["fr", "geo"]
-labels = {
-    "pays": 1,
-    "payslieu": 1,
-    "paysorg": 1,
-    "country": 1,
-    "pays lib": 1,
-    "lieupays": 1,
-    "pays beneficiaire": 1,
-    "nom du pays": 1,
-    "libelle pays": 1,
-}
+labels = [
+    "pays",
+    "payslieu",
+    "paysorg",
+    "country",
+    "pays lib",
+    "lieupays",
+    "pays beneficiaire",
+    "nom du pays",
+    "journey start country",
+    "libelle pays",
+    "journey end country",
+]
 
 _options = Options(
     ignore_case=True,

csv_detective/formats/percent.py

@@ -1,7 +1,7 @@
 from csv_detective.formats.float import _is as is_float
 
 proportion = 0.8
-labels = {"pourcent": 1, "part": 0.75, "pct": 0.75}
+labels = []
 
 
 def _is(val):

csv_detective/formats/region.py

@@ -2,19 +2,19 @@ from frformat import Millesime, Options, Region
 
 proportion = 1
 tags = ["fr", "geo"]
-labels = {
-    "region": 1,
-    "libelle region": 1,
-    "nom region": 1,
-    "libelle reg": 1,
-    "nom reg": 1,
-    "reg libusage": 1,
-    "nom de la region": 1,
-    "regionorg": 1,
-    "regionlieu": 1,
-    "reg": 0.5,
-    "nom officiel region": 1,
-}
+labels = [
+    "region",
+    "libelle region",
+    "nom region",
+    "libelle reg",
+    "nom reg",
+    "reg libusage",
+    "nom de la region",
+    "regionorg",
+    "regionlieu",
+    "reg",
+    "nom officiel region",
+]
 
 _extra_valid_values_set = frozenset(
     {

csv_detective/formats/sexe.py

@@ -2,7 +2,7 @@ from csv_detective.parsing.text import _process_text
 
 proportion = 1
 tags = ["fr"]
-labels = {"sexe": 1, "sex": 1, "civilite": 1, "genre": 1}
+labels = ["sexe", "sex", "civilite", "genre", "id sexe"]
 
 
 def _is(val):

csv_detective/formats/siren.py

@@ -2,15 +2,16 @@ import re
 
 proportion = 0.9
 tags = ["fr"]
-mandatory_label = True
-labels = {
-    "siren": 1,
-    "n° siren": 1,
-    "siren organisme": 1,
-    "siren titulaire": 1,
-    "numero siren": 1,
-    "epci": 1,
-}
+labels = [
+    "siren",
+    "siren organisme designe",
+    "siren organisme designant",
+    "n° siren",
+    "siren organisme",
+    "siren titulaire",
+    "numero siren",
+    "epci",
+]
 
 
 def _is(val):

csv_detective/formats/siret.py

@@ -2,15 +2,15 @@ import re
 
 proportion = 0.8
 tags = ["fr"]
-mandatory_label = True
-labels = {
-    "siret": 1,
-    "num siret": 1,
-    "siretacheteur": 1,
-    "n° siret": 1,
-    "coll siret": 1,
-    "epci": 1,
-}
+labels = [
+    "siret",
+    "siret d",
+    "num siret",
+    "siretacheteur",
+    "n° siret",
+    "coll siret",
+    "epci",
+]
 
 
 def _is(val):

csv_detective/formats/tel_fr.py

@@ -2,13 +2,19 @@ import re
 
 proportion = 0.7
 tags = ["fr"]
-labels = {
-    "telephone": 1,
-    "tel": 1,
-    "phone": 1,
-    "num tel": 1,
-    "tel mob": 1,
-}
+labels = [
+    "telephone",
+    "tel",
+    "tel1",
+    "tel2",
+    "phone",
+    "num tel",
+    "tel mob",
+    "telephone sav",
+    "telephone1",
+    "coordinates.phone",
+    "telephone du lieu",
+]
 
 
 def _is(val):

csv_detective/formats/uai.py

@@ -2,23 +2,24 @@ import re
 
 proportion = 0.8
 tags = ["fr"]
-labels = {
-    "uai": 1,
-    "code etablissement": 1,
-    "code uai": 1,
-    "uai - identifiant": 1,
-    "numero uai": 1,
-    "rne": 0.75,
-    "numero de l'etablissement": 1,
-    "code rne": 1,
-    "codeetab": 1,
-    "code uai de l'etablissement": 1,
-    "ref uai": 1,
-    "cd rne": 1,
-    "numerouai": 1,
-    "numero d etablissement": 1,
-    "numero etablissement": 1,
-}
+labels = [
+    "uai",
+    "code etablissement",
+    "code uai",
+    "uai - identifiant",
+    "numero uai",
+    "rne",
+    "numero de l'etablissement",
+    "code rne",
+    "codeetab",
+    "code uai de l'etablissement",
+    "ref uai",
+    "cd rne",
+    "numerouai",
+    "numero d etablissement",
+    "code etablissement",
+    "numero etablissement",
+]
 
 
 def _is(val):

csv_detective/formats/url.py

@@ -1,22 +1,22 @@
 import re
 
 proportion = 1
-labels = {
-    "url": 1,
-    "url source": 1,
-    "site web": 1,
-    "source url": 1,
-    "site internet": 1,
-    "remote url": 1,
-    "web": 1,
-    "site": 1,
-    "lien": 1,
-    "site data": 1,
-    "lien url": 1,
-    "lien vers le fichier": 1,
-    "sitweb": 1,
-    "interneturl": 1,
-}
+labels = [
+    "url",
+    "url source",
+    "site web",
+    "source url",
+    "site internet",
+    "remote url",
+    "web",
+    "site",
+    "lien",
+    "site data",
+    "lien url",
+    "lien vers le fichier",
+    "sitweb",
+    "interneturl",
+]
 
 pattern = re.compile(
     r"^((https?|ftp)://|www\.)(([A-Za-z0-9-]+\.)+[A-Za-z]{2,6})"

csv_detective/formats/username.py

@@ -1,7 +1,7 @@
 import re
 
 proportion = 1
-labels = {"account": 1, "username": 1, "user": 0.75}
+labels = ["account", "username", "user"]
 
 
 def _is(val):

csv_detective/formats/uuid.py

@@ -1,7 +1,7 @@
 import re
 
 proportion = 0.8
-labels = {"id": 1, "identifiant": 1}
+labels = ["id", "identifiant"]
 
 
 def _is(val) -> bool:

csv_detective/formats/year.py

@@ -1,12 +1,17 @@
 proportion = 1
 tags = ["temp"]
-python_type = "int"
-labels = {
-    "year": 1,
-    "annee": 1,
-    "naissance": 1,
-    "exercice": 1,
-}
+labels = [
+    "year",
+    "annee",
+    "annee depot",
+    "an nais",
+    "exercice",
+    "data year",
+    "annee de publication",
+    "exercice comptable",
+    "annee de naissance",
+    "annee ouverture",
+]
 
 
 def _is(val):

csv_detective/output/dataframe.py

@@ -13,16 +13,11 @@ from csv_detective.parsing.csv import CHUNK_SIZE
 from csv_detective.utils import display_logs_depending_process_time
 
 
-def cast(value: str, _type: str) -> str | int | float | bool | date | datetime | bytes | None:
+def cast(value: str, _type: str) -> str | float | bool | date | datetime | bytes | None:
     if not isinstance(value, str) or not value:
         # None is the current default value in hydra, should we keep this?
         return None
     match _type:
-        case "string":
-            # not used here, convenience for external use (cc hydra)
-            return value
-        case "int":
-            return int(value)
         case "float":
             return float_casting(value)
         case "bool":

csv_detective/output/profile.py

@@ -81,11 +81,7 @@ def create_profile(
             del cast_col
         # for all formats we want most frequent values, nb unique values and nb missing values
         tops_bruts = (
-            (
-                table[c].value_counts()
-                if _col_values is None
-                else (s := _col_values[c]).loc[s.index.notna()].sort_values(ascending=False)
-            )
+            (table[c].value_counts() if _col_values is None else _col_values[c].sort_values())
             .reset_index(name=_count_col)
             .iloc[:10]
             .to_dict(orient="records")

csv_detective/parsing/text.py

@@ -36,22 +36,21 @@ def is_word_in_string(word: str, string: str):
     return len(word) > 2 and word in string
 
 
-def header_score(header: str, valid_headers: dict[str, float]) -> float:
+def header_score(header: str, words_combinations_list: list[str]) -> float:
     """Returns:
-    - the valid header's credibility if the header is exactly in the valid list
-    - 0.5*credibility if any of the words is within the valid list
+    - 1 if the header is exactly in the specified list
+    - 0.5 if any of the words is within the header
     - 0 otherwise"""
     processed_header = _process_text(header)
 
-    header_matches_valid = max(
-        (valid == processed_header) * credibility for valid, credibility in valid_headers.items()
+    header_matches_words_combination = float(
+        any(words_combination == processed_header for words_combination in words_combinations_list)
     )
-
-    return max(
-        header_matches_valid,
-        0.5
-        * max(
-            is_word_in_string(valid, processed_header) * credibility
-            for valid, credibility in valid_headers.items()
-        ),
+    words_combination_in_header = 0.5 * (
+        any(
+            is_word_in_string(words_combination, processed_header)
+            for words_combination in words_combinations_list
+        )
     )
+
+    return max(header_matches_words_combination, words_combination_in_header)

{csv_detective-0.10.1.dist-info → csv_detective-0.10.1.dev2576.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: csv-detective
-Version: 0.10.1
+Version: 0.10.1.dev2576
 Summary: Detect tabular files column content
 Keywords: CSV,data processing,encoding,guess,parser,tabular
 Author: data.gouv.fr
@@ -24,7 +24,7 @@ Requires-Dist: pytest>=8.3.0 ; extra == 'dev'
 Requires-Dist: responses>=0.25.0 ; extra == 'dev'
 Requires-Dist: ruff>=0.9.3 ; extra == 'dev'
 Requires-Python: >=3.10, <3.15
-Project-URL: Source, https://github.com/datagouv/csv-detective
+Project-URL: Source, https://github.com/datagouv/csv_detective
 Provides-Extra: dev
 Description-Content-Type: text/markdown