csv-detective 0.10.1.dev2581__py3-none-any.whl → 0.10.1.dev2599__py3-none-any.whl

csv_detective/detection/formats.py

@@ -82,22 +82,7 @@ def detect_formats(
     # To reduce false positives: ensure these formats are detected only if the label yields
     # a detection (skipping the ones that have been excluded by the users).
     formats_with_mandatory_label = [
-        f
-        for f in [
-            "code_departement",
-            "code_commune_insee",
-            "code_postal",
-            "code_fantoir",
-            "latitude_wgs",
-            "longitude_wgs",
-            "latitude_wgs_fr_metropole",
-            "longitude_wgs_fr_metropole",
-            "latitude_l93",
-            "longitude_l93",
-            "siren",
-            "siret",
-        ]
-        if f in scores_table.index
+        f for f in fmtm.get_formats_with_mandatory_label() if f in scores_table.index
     ]
     scores_table.loc[formats_with_mandatory_label, :] = np.where(
         scores_table_labels.loc[formats_with_mandatory_label, :],
@@ -106,32 +91,16 @@ def detect_formats(
     )
     analysis["columns"] = prepare_output_dict(scores_table, limited_output)
 
-    metier_to_python_type = {
-        "booleen": "bool",
-        "int": "int",
-        "float": "float",
-        "string": "string",
-        "json": "json",
-        "geojson": "json",
-        "datetime_aware": "datetime",
-        "datetime_naive": "datetime",
-        "datetime_rfc822": "datetime",
-        "date": "date",
-        "latitude_l93": "float",
-        "latitude_wgs": "float",
-        "latitude_wgs_fr_metropole": "float",
-        "longitude_l93": "float",
-        "longitude_wgs": "float",
-        "longitude_wgs_fr_metropole": "float",
-        "binary": "binary",
-    }
-
     if not limited_output:
         for detection_method in ["columns_fields", "columns_labels", "columns"]:
             analysis[detection_method] = {
                 col_name: [
                     {
-                        "python_type": metier_to_python_type.get(detection["format"], "string"),
+                        "python_type": (
+                            "string"
+                            if detection["format"] == "string"
+                            else fmtm.formats[detection["format"]].python_type
+                        ),
                         **detection,
                     }
                     for detection in detections
@@ -142,7 +111,11 @@ def detect_formats(
         for detection_method in ["columns_fields", "columns_labels", "columns"]:
             analysis[detection_method] = {
                 col_name: {
-                    "python_type": metier_to_python_type.get(detection["format"], "string"),
+                    "python_type": (
+                        "string"
+                        if detection["format"] == "string"
+                        else fmtm.formats[detection["format"]].python_type
+                    ),
                     **detection,
                 }
                 for col_name, detection in analysis[detection_method].items()

csv_detective/format.py

@@ -9,9 +9,11 @@ class Format:
         name: str,
         func: Callable[[Any], bool],
         _test_values: dict[bool, list[str]],
-        labels: list[str] = [],
+        labels: dict[str, float] = {},
         proportion: float = 1,
         tags: list[str] = [],
+        mandatory_label: bool = False,
+        python_type: str = "string",
     ) -> None:
         """
         Instanciates a Format object.
@@ -20,16 +22,18 @@ class Format:
             name: the name of the format.
             func: the value test for the format (returns whether a string is valid).
             _test_values: lists of valid and invalid values, used in the tests
-            labels: the list of hint headers for the header score
+            labels: the dict of hint headers and their credibilty for the header score (NB: credibility is relative witin a single format, should be used to rank the valid labels)
             proportion: the tolerance (between 0 and 1) to say a column is valid for a format. (1 => 100% of the column has to pass the func check for the column to be considered valid)
             tags: to allow users to submit a file to only a subset of formats
         """
         self.name: str = name
         self.func: Callable = func
         self._test_values: dict[bool, list[str]] = _test_values
-        self.labels: list[str] = labels
+        self.labels: dict[str, float] = labels
         self.proportion: float = proportion
         self.tags: list[str] = tags
+        self.mandatory_label: bool = mandatory_label
+        self.python_type: str = python_type
 
     def is_valid_label(self, val: str) -> float:
         return header_score(val, self.labels)
@@ -49,7 +53,7 @@ class FormatsManager:
                 _test_values=module._test_values,
                 **{
                     attr: val
-                    for attr in ["labels", "proportion", "tags"]
+                    for attr in ["labels", "proportion", "tags", "mandatory_label", "python_type"]
                     if (val := getattr(module, attr, None))
                 },
             )
@@ -63,5 +67,8 @@ class FormatsManager:
             if all(tag in fmt.tags for tag in tags)
         }
 
+    def get_formats_with_mandatory_label(self) -> dict[str, Format]:
+        return {label: fmt for label, fmt in self.formats.items() if fmt.mandatory_label}
+
     def available_tags(self) -> set[str]:
         return set(tag for format in self.formats.values() for tag in format.tags)

csv_detective/formats/adresse.py

@@ -2,15 +2,15 @@ from csv_detective.parsing.text import _process_text
 
 proportion = 0.55
 tags = ["fr", "geo"]
-labels = [
-    "adresse",
-    "localisation",
-    "adresse postale",
-    "adresse geographique",
-    "adr",
-    "adresse complete",
-    "adresse station",
-]
+labels = {
+    "adresse": 1,
+    "localisation": 1,
+    "adresse postale": 1,
+    "adresse geographique": 1,
+    "adr": 0.5,
+    "adresse complete": 1,
+    "adresse station": 1,
+}
 
 voies = {
     "aire ",

csv_detective/formats/binary.py

@@ -2,7 +2,8 @@ import codecs
 
 proportion = 1
 tags = ["type"]
-labels = ["bytes", "binary", "image", "encode", "content"]
+python_type = "binary"
+labels = {"bytes": 1, "binary": 1, "image": 1, "encode": 1, "content": 1}
 
 
 def binary_casting(val: str) -> bytes:

csv_detective/formats/booleen.py

@@ -1,6 +1,7 @@
 proportion = 1
 tags = ["type"]
-labels = ["is ", "has ", "est "]
+python_type = "bool"
+labels = {"is ": 1, "has ": 1, "est ": 1}
 
 bool_mapping = {
     "1": True,
@@ -21,7 +22,7 @@ bool_mapping = {
 liste_bool = set(bool_mapping.keys())
 
 
-def bool_casting(val: str) -> bool:
+def bool_casting(val: str) -> bool | None:
     return bool_mapping.get(val.lower())
 
 

csv_detective/formats/code_commune_insee.py

@@ -2,16 +2,18 @@ from frformat import CodeCommuneInsee, Millesime
 
 proportion = 0.75
 tags = ["fr", "geo"]
-labels = [
-    "code commune insee",
-    "code insee",
-    "codes insee",
-    "code commune",
-    "code insee commune",
-    "insee",
-    "code com",
-    "com",
-]
+mandatory_label = True
+labels = {
+    "code commune insee": 1,
+    "code insee": 1,
+    "codes insee": 1,
+    "code commune": 1,
+    "code insee commune": 1,
+    "insee": 0.75,
+    "code com": 1,
+    "com": 0.5,
+    "code": 0.5,
+}
 
 _code_commune_insee = CodeCommuneInsee(Millesime.LATEST)
 

csv_detective/formats/code_csp_insee.py

@@ -4,7 +4,7 @@ from csv_detective.parsing.text import _process_text
 
 proportion = 1
 tags = ["fr"]
-labels = ["code csp insee", "code csp"]
+labels = {"code csp insee": 1, "code csp": 1}
 
 
 def _is(val):

csv_detective/formats/code_departement.py

@@ -2,13 +2,14 @@ from frformat import Millesime, NumeroDepartement, Options
 
 proportion = 1
 tags = ["fr", "geo"]
-labels = [
-    "code departement",
-    "code_departement",
-    "dep",
-    "departement",
-    "dept",
-]
+mandatory_label = True
+labels = {
+    "code departement": 1,
+    "code_departement": 1,
+    "dep": 0.5,
+    "departement": 1,
+    "dept": 0.75,
+}
 
 _options = Options(
     ignore_case=True,

csv_detective/formats/code_fantoir.py

@@ -2,11 +2,12 @@ from frformat import CodeFantoir
 
 proportion = 1
 tags = ["fr", "geo"]
-labels = [
-    "cadastre1",
-    "code fantoir",
-    "fantoir",
-]
+mandatory_label = True
+labels = {
+    "cadastre1": 1,
+    "code fantoir": 1,
+    "fantoir": 1,
+}
 
 _code_fantoir = CodeFantoir()
 

csv_detective/formats/code_import.py

@@ -2,7 +2,7 @@ import re
 
 proportion = 0.9
 tags = ["fr"]
-labels = ["code"]
+labels = {"code": 0.5}
 
 regex = r"^(\d{3}[SP]\d{4,10}(.\w{1,3}\d{0,5})?|\d[A-Z0-9]\d[SP]\w(\w-?\w{0,2}\d{0,6})?)$"
 

csv_detective/formats/code_postal.py

@@ -2,15 +2,16 @@ from frformat import CodePostal
 
 proportion = 0.9
 tags = ["fr", "geo"]
-labels = [
-    "code postal",
-    "postal code",
-    "postcode",
-    "post code",
-    "cp",
-    "codes postaux",
-    "location postcode",
-]
+mandatory_label = True
+labels = {
+    "code postal": 1,
+    "postal code": 1,
+    "postcode": 1,
+    "post code": 1,
+    "cp": 0.5,
+    "codes postaux": 1,
+    "location postcode": 1,
+}
 
 _code_postal = CodePostal()
 

csv_detective/formats/code_region.py

@@ -2,12 +2,13 @@ from frformat import CodeRegion, Millesime
 
 proportion = 1
 tags = ["fr", "geo"]
-labels = [
-    "code region",
-    "reg",
-    "code insee region",
-    "region",
-]
+mandatory_label = True
+labels = {
+    "code region": 1,
+    "reg": 0.5,
+    "code insee region": 1,
+    "region": 1,
+}
 
 _code_region = CodeRegion(Millesime.LATEST)
 

csv_detective/formats/code_rna.py

@@ -2,12 +2,13 @@ from frformat import CodeRNA
 
 proportion = 0.9
 tags = ["fr"]
-labels = [
-    "code rna",
-    "rna",
-    "n° inscription association",
-    "identifiant association",
-]
+labels = {
+    "code rna": 1,
+    "rna": 1,
+    "n° inscription association": 1,
+    "identifiant association": 1,
+    "asso": 0.75,
+}
 
 _code_rna = CodeRNA()
 

csv_detective/formats/code_waldec.py

@@ -2,7 +2,7 @@ import re
 
 proportion = 0.9
 tags = ["fr"]
-labels = ["code waldec", "waldec"]
+labels = {"code waldec": 1, "waldec": 1}
 
 regex = r"^W\d[\dA-Z]\d{7}$"
 

csv_detective/formats/commune.py

@@ -2,11 +2,11 @@ from frformat import Commune, Millesime, Options
 
 proportion = 0.8
 tags = ["fr", "geo"]
-labels = [
-    "commune",
-    "ville",
-    "libelle commune",
-]
+labels = {
+    "commune": 1,
+    "ville": 1,
+    "libelle commune": 1,
+}
 
 _options = Options(
     ignore_case=True,

csv_detective/formats/csp_insee.py

@@ -4,11 +4,12 @@ from csv_detective.parsing.text import _process_text
 
 proportion = 1
 tags = ["fr"]
-labels = [
-    "csp insee",
-    "csp",
-    "categorie socioprofessionnelle",
-]
+labels = {
+    "csp insee": 1,
+    "csp": 0.75,
+    "categorie socioprofessionnelle": 1,
+    "sociopro": 1,
+}
 
 f = open(join(dirname(__file__), "data", "csp_insee.txt"), "r")
 codes_insee = f.read().split("\n")

csv_detective/formats/date.py

@@ -7,23 +7,24 @@ from dateutil.parser import parse as dateutil_parser
 
 proportion = 1
 tags = ["temp", "type"]
-SHARED_DATE_LABELS = [
-    "date",
-    "mise à jour",
-    "modifie",
-    "maj",
-    "datemaj",
-    "update",
-    "created",
-    "modified",
-]
-labels = SHARED_DATE_LABELS + [
-    "jour",
-    "periode",
-    "dpc",
-    "yyyymmdd",
-    "aaaammjj",
-]
+python_type = "date"
+SHARED_DATE_LABELS = {
+    "date": 1,
+    "mise à jour": 1,
+    "modifie": 1,
+    "maj": 0.75,
+    "datemaj": 1,
+    "update": 1,
+    "created": 1,
+    "modified": 1,
+}
+labels = SHARED_DATE_LABELS | {
+    "jour": 0.75,
+    "periode": 0.75,
+    "dpc": 0.5,
+    "yyyymmdd": 1,
+    "aaaammjj": 1,
+}
 
 
 def date_casting(val: str) -> datetime | None:

csv_detective/formats/date_fr.py

@@ -4,7 +4,7 @@ from csv_detective.parsing.text import _process_text
 
 proportion = 1
 tags = ["fr", "temp"]
-labels = ["date"]
+labels = {"date": 1}
 
 pattern = (
     r"^(0?[1-9]|[12][0-9]|3[01])[ \-/](janvier|fevrier|mars|avril|mai|juin|juillet|aout|septembre"

csv_detective/formats/datetime_aware.py

@@ -4,7 +4,8 @@ from csv_detective.formats.date import SHARED_DATE_LABELS, aaaammjj_pattern, dat
 
 proportion = 1
 tags = ["temp", "type"]
-labels = SHARED_DATE_LABELS + ["datetime", "timestamp"]
+python_type = "datetime"
+labels = SHARED_DATE_LABELS | {"datetime": 1, "timestamp": 1}
 
 threshold = 0.7
 pat = (

csv_detective/formats/datetime_naive.py

@@ -6,6 +6,7 @@ from csv_detective.formats.datetime_aware import labels, prefix  # noqa
 
 proportion = 1
 tags = ["temp", "type"]
+python_type = "datetime"
 threshold = 0.7
 
 # matches AAAA-MM-JJTHH:MM:SS(.dddddd)Z with any of the listed separators for the date OR NO SEPARATOR

csv_detective/formats/datetime_rfc822.py

@@ -4,6 +4,7 @@ from csv_detective.formats.datetime_aware import labels  # noqa
 
 proportion = 1
 tags = ["temp", "type"]
+python_type = "datetime"
 
 
 def _is(val):

csv_detective/formats/departement.py

@@ -2,21 +2,21 @@ from frformat import Departement, Millesime, Options
 
 proportion = 0.9
 tags = ["fr", "geo"]
-labels = [
-    "departement",
-    "libelle du departement",
-    "deplib",
-    "nom dept",
-    "dept",
-    "libdepartement",
-    "nom departement",
-    "libelle dep",
-    "libelle departement",
-    "lb departements",
-    "dep libusage",
-    "lb departement",
-    "nom dep",
-]
+labels = {
+    "departement": 1,
+    "libelle du departement": 1,
+    "deplib": 1,
+    "nom dept": 1,
+    "dept": 0.75,
+    "libdepartement": 1,
+    "nom departement": 1,
+    "libelle dep": 1,
+    "libelle departement": 1,
+    "lb departements": 1,
+    "dep libusage": 1,
+    "lb departement": 1,
+    "nom dep": 1,
+}
 
 _options = Options(
     ignore_case=True,

csv_detective/formats/email.py

@@ -1,19 +1,19 @@
 import re
 
 proportion = 0.9
-labels = [
-    "email",
-    "mail",
-    "courriel",
-    "contact",
-    "mel",
-    "lieucourriel",
-    "coordinates.emailcontact",
-    "e mail",
-    "mo mail",
-    "adresse mail",
-    "adresse email",
-]
+labels = {
+    "email": 1,
+    "mail": 1,
+    "courriel": 1,
+    "contact": 1,
+    "mel": 1,
+    "lieucourriel": 1,
+    "coordinates.emailcontact": 1,
+    "e mail": 1,
+    "mo mail": 1,
+    "adresse mail": 1,
+    "adresse email": 1,
+}
 
 
 def _is(val):

csv_detective/formats/float.py

@@ -2,7 +2,8 @@ import re
 
 proportion = 1
 tags = ["type"]
-labels = ["part", "ratio", "taux"]
+python_type = "float"
+labels = {"part": 1, "ratio": 1, "taux": 1}
 
 scientific_notation_pattern = r"\d+\.\d+[e|E][+|-]?\d+"
 

csv_detective/formats/geojson.py

@@ -2,16 +2,16 @@ import json
 
 proportion = 1
 tags = ["geo"]
-labels = [
-    "json geojson",
-    "json",
-    "geojson",
-    "geo shape",
-    "geom",
-    "geometry",
-    "geo shape",
-    "geoshape",
-]
+python_type = "json"
+labels = {
+    "json geojson": 1,
+    "json": 1,
+    "geojson": 1,
+    "geo shape": 1,
+    "geom": 0.75,
+    "geometry": 1,
+    "geoshape": 1,
+}
 
 
 def _is(val) -> bool:

csv_detective/formats/insee_ape700.py

@@ -4,14 +4,16 @@ from csv_detective.parsing.text import _process_text
 
 proportion = 0.8
 tags = ["fr"]
-labels = [
-    "code ape",
-    "code activite (ape)",
-    "code naf",
-    "code naf organisme designe",
-    "code naf organisme designant",
-    "base sirene : code ape de l'etablissement siege",
-]
+labels = {
+    "code ape": 1,
+    "code activite (ape)": 1,
+    "code naf": 1,
+    "code naf organisme designe": 1,
+    "code naf organisme designant": 1,
+    "base sirene : code ape de l'etablissement siege": 1,
+    "naf": 0.75,
+    "ape": 0.5,
+}
 
 f = open(join(dirname(__file__), "data", "insee_ape700.txt"), "r")
 condes_insee_ape = f.read().split("\n")

csv_detective/formats/insee_canton.py

@@ -2,12 +2,12 @@ from frformat import Canton, Millesime, Options
 
 proportion = 0.9
 tags = ["fr", "geo"]
-labels = [
-    "insee canton",
-    "canton",
-    "cant",
-    "nom canton",
-]
+labels = {
+    "insee canton": 1,
+    "canton": 1,
+    "cant": 0.5,
+    "nom canton": 1,
+}
 
 _options = Options(
     ignore_case=True,

csv_detective/formats/int.py

@@ -1,5 +1,6 @@
-labels = ["nb", "nombre", "nbre"]
 tag = ["type"]
+python_type = "int"
+labels = {"nb": 0.75, "nombre": 1, "nbre": 0.75}
 
 
 def _is(val):