csv-detective 0.10.2549__py3-none-any.whl → 0.10.12674__py3-none-any.whl

csv_detective/detection/formats.py

@@ -82,22 +82,7 @@ def detect_formats(
     # To reduce false positives: ensure these formats are detected only if the label yields
     # a detection (skipping the ones that have been excluded by the users).
     formats_with_mandatory_label = [
-        f
-        for f in [
-            "code_departement",
-            "code_commune_insee",
-            "code_postal",
-            "code_fantoir",
-            "latitude_wgs",
-            "longitude_wgs",
-            "latitude_wgs_fr_metropole",
-            "longitude_wgs_fr_metropole",
-            "latitude_l93",
-            "longitude_l93",
-            "siren",
-            "siret",
-        ]
-        if f in scores_table.index
+        f for f in fmtm.get_formats_with_mandatory_label() if f in scores_table.index
     ]
     scores_table.loc[formats_with_mandatory_label, :] = np.where(
         scores_table_labels.loc[formats_with_mandatory_label, :],
@@ -106,32 +91,16 @@ def detect_formats(
     )
     analysis["columns"] = prepare_output_dict(scores_table, limited_output)
 
-    metier_to_python_type = {
-        "booleen": "bool",
-        "int": "int",
-        "float": "float",
-        "string": "string",
-        "json": "json",
-        "geojson": "json",
-        "datetime_aware": "datetime",
-        "datetime_naive": "datetime",
-        "datetime_rfc822": "datetime",
-        "date": "date",
-        "latitude_l93": "float",
-        "latitude_wgs": "float",
-        "latitude_wgs_fr_metropole": "float",
-        "longitude_l93": "float",
-        "longitude_wgs": "float",
-        "longitude_wgs_fr_metropole": "float",
-        "binary": "binary",
-    }
-
     if not limited_output:
         for detection_method in ["columns_fields", "columns_labels", "columns"]:
             analysis[detection_method] = {
                 col_name: [
                     {
-                        "python_type": metier_to_python_type.get(detection["format"], "string"),
+                        "python_type": (
+                            "string"
+                            if detection["format"] == "string"
+                            else fmtm.formats[detection["format"]].python_type
+                        ),
                         **detection,
                     }
                     for detection in detections
@@ -142,7 +111,11 @@ def detect_formats(
         for detection_method in ["columns_fields", "columns_labels", "columns"]:
             analysis[detection_method] = {
                 col_name: {
-                    "python_type": metier_to_python_type.get(detection["format"], "string"),
+                    "python_type": (
+                        "string"
+                        if detection["format"] == "string"
+                        else fmtm.formats[detection["format"]].python_type
+                    ),
                     **detection,
                 }
                 for col_name, detection in analysis[detection_method].items()

csv_detective/explore_csv.py

@@ -1,5 +1,6 @@
 import logging
 from time import time
+from typing import Iterator
 
 import pandas as pd
 
@@ -27,7 +28,7 @@ def routine(
     cast_json: bool = True,
     verbose: bool = False,
     sheet_name: str | int | None = None,
-) -> dict | tuple[dict, pd.DataFrame]:
+) -> dict | tuple[dict, Iterator[pd.DataFrame]]:
     """
     Returns a dict with information about the table and possible column contents, and if requested the DataFrame with columns cast according to analysis.
 
@@ -115,7 +116,7 @@ def validate_then_detect(
     output_df: bool = False,
     cast_json: bool = True,
     verbose: bool = False,
-):
+) -> dict | tuple[dict, Iterator[pd.DataFrame]]:
     """
     Performs a validation of the given file against the given analysis.
     If the validation fails, performs a full analysis and return it.

csv_detective/format.py

@@ -9,9 +9,11 @@ class Format:
         name: str,
         func: Callable[[Any], bool],
         _test_values: dict[bool, list[str]],
-        labels: list[str] = [],
+        labels: dict[str, float] = {},
         proportion: float = 1,
         tags: list[str] = [],
+        mandatory_label: bool = False,
+        python_type: str = "string",
     ) -> None:
         """
         Instanciates a Format object.
@@ -20,16 +22,18 @@ class Format:
             name: the name of the format.
             func: the value test for the format (returns whether a string is valid).
             _test_values: lists of valid and invalid values, used in the tests
-            labels: the list of hint headers for the header score
+            labels: the dict of hint headers and their credibilty for the header score (NB: credibility is relative witin a single format, should be used to rank the valid labels)
             proportion: the tolerance (between 0 and 1) to say a column is valid for a format. (1 => 100% of the column has to pass the func check for the column to be considered valid)
             tags: to allow users to submit a file to only a subset of formats
         """
         self.name: str = name
         self.func: Callable = func
         self._test_values: dict[bool, list[str]] = _test_values
-        self.labels: list[str] = labels
+        self.labels: dict[str, float] = labels
         self.proportion: float = proportion
         self.tags: list[str] = tags
+        self.mandatory_label: bool = mandatory_label
+        self.python_type: str = python_type
 
     def is_valid_label(self, val: str) -> float:
         return header_score(val, self.labels)
@@ -49,7 +53,7 @@ class FormatsManager:
                 _test_values=module._test_values,
                 **{
                     attr: val
-                    for attr in ["labels", "proportion", "tags"]
+                    for attr in ["labels", "proportion", "tags", "mandatory_label", "python_type"]
                     if (val := getattr(module, attr, None))
                 },
             )
@@ -63,5 +67,8 @@ class FormatsManager:
             if all(tag in fmt.tags for tag in tags)
         }
 
+    def get_formats_with_mandatory_label(self) -> dict[str, Format]:
+        return {label: fmt for label, fmt in self.formats.items() if fmt.mandatory_label}
+
     def available_tags(self) -> set[str]:
         return set(tag for format in self.formats.values() for tag in format.tags)

csv_detective/formats/adresse.py

@@ -2,15 +2,15 @@ from csv_detective.parsing.text import _process_text
 
 proportion = 0.55
 tags = ["fr", "geo"]
-labels = [
-    "adresse",
-    "localisation",
-    "adresse postale",
-    "adresse geographique",
-    "adr",
-    "adresse complete",
-    "adresse station",
-]
+labels = {
+    "adresse": 1,
+    "localisation": 1,
+    "adresse postale": 1,
+    "adresse geographique": 1,
+    "adr": 0.5,
+    "adresse complete": 1,
+    "adresse station": 1,
+}
 
 voies = {
     "aire ",

csv_detective/formats/binary.py

@@ -2,7 +2,8 @@ import codecs
 
 proportion = 1
 tags = ["type"]
-labels = ["bytes", "binary", "image", "encode", "content"]
+python_type = "binary"
+labels = {"bytes": 1, "binary": 1, "image": 1, "encode": 1, "content": 1}
 
 
 def binary_casting(val: str) -> bytes:

csv_detective/formats/booleen.py

@@ -1,6 +1,7 @@
 proportion = 1
 tags = ["type"]
-labels = ["is ", "has ", "est "]
+python_type = "bool"
+labels = {"is ": 1, "has ": 1, "est ": 1}
 
 bool_mapping = {
     "1": True,
@@ -21,7 +22,7 @@ bool_mapping = {
 liste_bool = set(bool_mapping.keys())
 
 
-def bool_casting(val: str) -> bool:
+def bool_casting(val: str) -> bool | None:
     return bool_mapping.get(val.lower())
 
 

csv_detective/formats/code_commune_insee.py

@@ -2,16 +2,18 @@ from frformat import CodeCommuneInsee, Millesime
 
 proportion = 0.75
 tags = ["fr", "geo"]
-labels = [
-    "code commune insee",
-    "code insee",
-    "codes insee",
-    "code commune",
-    "code insee commune",
-    "insee",
-    "code com",
-    "com",
-]
+mandatory_label = True
+labels = {
+    "code commune insee": 1,
+    "code insee": 1,
+    "codes insee": 1,
+    "code commune": 1,
+    "code insee commune": 1,
+    "insee": 0.75,
+    "code com": 1,
+    "com": 0.5,
+    "code": 0.5,
+}
 
 _code_commune_insee = CodeCommuneInsee(Millesime.LATEST)
 

csv_detective/formats/code_csp_insee.py

@@ -4,7 +4,7 @@ from csv_detective.parsing.text import _process_text
 
 proportion = 1
 tags = ["fr"]
-labels = ["code csp insee", "code csp"]
+labels = {"code csp insee": 1, "code csp": 1}
 
 
 def _is(val):

csv_detective/formats/code_departement.py

@@ -2,13 +2,14 @@ from frformat import Millesime, NumeroDepartement, Options
 
 proportion = 1
 tags = ["fr", "geo"]
-labels = [
-    "code departement",
-    "code_departement",
-    "dep",
-    "departement",
-    "dept",
-]
+mandatory_label = True
+labels = {
+    "code departement": 1,
+    "code_departement": 1,
+    "dep": 0.5,
+    "departement": 1,
+    "dept": 0.75,
+}
 
 _options = Options(
     ignore_case=True,

csv_detective/formats/code_fantoir.py

@@ -2,11 +2,12 @@ from frformat import CodeFantoir
 
 proportion = 1
 tags = ["fr", "geo"]
-labels = [
-    "cadastre1",
-    "code fantoir",
-    "fantoir",
-]
+mandatory_label = True
+labels = {
+    "cadastre1": 1,
+    "code fantoir": 1,
+    "fantoir": 1,
+}
 
 _code_fantoir = CodeFantoir()
 

csv_detective/formats/code_import.py

@@ -2,7 +2,7 @@ import re
 
 proportion = 0.9
 tags = ["fr"]
-labels = ["code"]
+labels = {"code": 0.5}
 
 regex = r"^(\d{3}[SP]\d{4,10}(.\w{1,3}\d{0,5})?|\d[A-Z0-9]\d[SP]\w(\w-?\w{0,2}\d{0,6})?)$"
 

csv_detective/formats/code_postal.py

@@ -2,15 +2,16 @@ from frformat import CodePostal
 
 proportion = 0.9
 tags = ["fr", "geo"]
-labels = [
-    "code postal",
-    "postal code",
-    "postcode",
-    "post code",
-    "cp",
-    "codes postaux",
-    "location postcode",
-]
+mandatory_label = True
+labels = {
+    "code postal": 1,
+    "postal code": 1,
+    "postcode": 1,
+    "post code": 1,
+    "cp": 0.5,
+    "codes postaux": 1,
+    "location postcode": 1,
+}
 
 _code_postal = CodePostal()
 

csv_detective/formats/code_region.py

@@ -2,12 +2,13 @@ from frformat import CodeRegion, Millesime
 
 proportion = 1
 tags = ["fr", "geo"]
-labels = [
-    "code region",
-    "reg",
-    "code insee region",
-    "region",
-]
+mandatory_label = True
+labels = {
+    "code region": 1,
+    "reg": 0.5,
+    "code insee region": 1,
+    "region": 1,
+}
 
 _code_region = CodeRegion(Millesime.LATEST)
 

csv_detective/formats/code_rna.py

@@ -2,12 +2,13 @@ from frformat import CodeRNA
 
 proportion = 0.9
 tags = ["fr"]
-labels = [
-    "code rna",
-    "rna",
-    "n° inscription association",
-    "identifiant association",
-]
+labels = {
+    "code rna": 1,
+    "rna": 1,
+    "n° inscription association": 1,
+    "identifiant association": 1,
+    "asso": 0.75,
+}
 
 _code_rna = CodeRNA()
 

csv_detective/formats/code_waldec.py

@@ -2,7 +2,7 @@ import re
 
 proportion = 0.9
 tags = ["fr"]
-labels = ["code waldec", "waldec"]
+labels = {"code waldec": 1, "waldec": 1}
 
 regex = r"^W\d[\dA-Z]\d{7}$"
 

csv_detective/formats/commune.py

@@ -2,11 +2,11 @@ from frformat import Commune, Millesime, Options
 
 proportion = 0.8
 tags = ["fr", "geo"]
-labels = [
-    "commune",
-    "ville",
-    "libelle commune",
-]
+labels = {
+    "commune": 1,
+    "ville": 1,
+    "libelle commune": 1,
+}
 
 _options = Options(
     ignore_case=True,

csv_detective/formats/csp_insee.py

@@ -4,11 +4,12 @@ from csv_detective.parsing.text import _process_text
 
 proportion = 1
 tags = ["fr"]
-labels = [
-    "csp insee",
-    "csp",
-    "categorie socioprofessionnelle",
-]
+labels = {
+    "csp insee": 1,
+    "csp": 0.75,
+    "categorie socioprofessionnelle": 1,
+    "sociopro": 1,
+}
 
 f = open(join(dirname(__file__), "data", "csp_insee.txt"), "r")
 codes_insee = f.read().split("\n")

csv_detective/formats/data/insee_ape700.txt

@@ -1,6 +1,6 @@
 0000Z
 0000Z
-000Z 
+000Z
 0111Z
 0112Z
 0113Z