csv-detective 0.8.1.dev1578__py3-none-any.whl → 0.8.1.dev1599__py3-none-any.whl

csv_detective/detect_fields/geo/json_geojson/__init__.py

@@ -1,22 +1,18 @@
 import json
-from json import JSONDecodeError
 
 PROPORTION = 0.9
 
 
 def _is(val):
-    '''Renvoie True si val peut etre geojson'''
+    """Renvoie True si val peut etre un geojson"""
 
     try:
         j = json.loads(val)
-        if 'type' in j and 'coordinates' in j:
-            return True
-        if 'geometry' in j:
-            if 'coordinates' in j['geometry']:
+        if isinstance(j, dict):
+            if "type" in j and "coordinates" in j:
                 return True
-    except JSONDecodeError:
-        pass
-    except TypeError:
+            if "geometry" in j and "coordinates" in j["geometry"]:
+                return True
+    except Exception:
         pass
-
     return False

csv_detective/detect_fields/other/json/__init__.py

@@ -5,20 +5,10 @@ PROPORTION = 1
 
 
 def _is(val):
-    '''Detects json'''
+    """Detects json"""
     try:
         loaded = json.loads(val)
-        if isinstance(loaded, list) or (
-            isinstance(loaded, dict) and not (
-                any(
-                    [
-                        geo in loaded for geo in ['coordinates', 'geometry']
-                    ]
-                )
-            )
-        ):
-            return True
-        else:
-            return False
+        # we don't want to consider integers for instance
+        return isinstance(loaded, (list, dict))
     except (JSONDecodeError, TypeError):
         return False

csv_detective/detect_labels/other/booleen/__init__.py

@@ -4,5 +4,5 @@ PROPORTION = 0.5
 
 
 def _is(header: str) -> float:
-    words_combinations_list = ["is_", "has_", "est_"]
+    words_combinations_list = ["is ", "has ", "est "]
     return header_score(header, words_combinations_list)

csv_detective/detection/formats.py

@@ -110,11 +110,9 @@ def detect_formats(
         "datetime_naive": "datetime",
         "datetime_rfc822": "datetime",
         "date": "date",
-        "latitude": "float",
         "latitude_l93": "float",
         "latitude_wgs": "float",
         "latitude_wgs_fr_metropole": "float",
-        "longitude": "float",
         "longitude_l93": "float",
         "longitude_wgs": "float",
         "longitude_wgs_fr_metropole": "float",

csv_detective/output/utils.py

@@ -2,55 +2,73 @@ import pandas as pd
 
 
 def prepare_output_dict(return_table: pd.DataFrame, limited_output: bool):
+    # -> dict[str, dict | list[dict]] (to be added when upgrading to python>=3.10)
     return_dict_cols = return_table.to_dict("dict")
-    return_dict_cols_intermediary = {}
+    output_dict = {}
     for column_name in return_dict_cols:
-        return_dict_cols_intermediary[column_name] = []
-        for detected_value_type in return_dict_cols[column_name]:
-            if return_dict_cols[column_name][detected_value_type] == 0:
-                continue
-            dict_tmp = {}
-            dict_tmp["format"] = detected_value_type
-            dict_tmp["score"] = return_dict_cols[column_name][detected_value_type]
-            return_dict_cols_intermediary[column_name].append(dict_tmp)
-
-        # Clean dict using priorities
-        formats_detected = {
-            x["format"] for x in return_dict_cols_intermediary[column_name]
-        }
+        # keep only formats with a non-zero score
+        output_dict[column_name] = [
+            {
+                "format": detected_value_type,
+                "score": return_dict_cols[column_name][detected_value_type],
+            }
+            for detected_value_type in return_dict_cols[column_name]
+            if return_dict_cols[column_name][detected_value_type] > 0
+        ]
+        priorities = [
+            # no need to specify int and float everywhere, they are deprioritized anyway
+            ("int", ("float",)),
+            # bool over everything
+            ("booleen", (
+                "latitude_l93",
+                "latitude_wgs",
+                "latitude_wgs_fr_metropole",
+                "longitude_l93",
+                "longitude_wgs",
+                "longitude_wgs_fr_metropole",
+            )),
+            ("geojson", ("json",)),
+            # latlon over lonlat if no longitude allows to discriminate
+            ("latlon_wgs", ("json", "lonlat_wgs")),
+            ("lonlat_wgs", ("json",)),
+            ("latitude_wgs_fr_metropole", ("latitude_l93", "latitude_wgs")),
+            ("longitude_wgs_fr_metropole", ("longitude_l93", "longitude_wgs")),
+            ("latitude_wgs", ("latitude_l93",)),
+            ("longitude_wgs", ("longitude_l93",)),
+            ("code_region", ("code_departement",)),
+            ("datetime_rfc822", ("datetime_aware",)),
+        ]
+        detected_formats = set(x["format"] for x in output_dict[column_name])
         formats_to_remove = set()
         # Deprioritise float and int detection vs others
-        if len(formats_detected - {"float", "int"}) > 0:
+        if len(detected_formats - {"float", "int"}) > 0:
             formats_to_remove = formats_to_remove.union({"float", "int"})
-        if "int" in formats_detected:
-            formats_to_remove.add("float")
-        if "latitude_wgs_fr_metropole" in formats_detected:
-            formats_to_remove.add("latitude_l93")
-            formats_to_remove.add("latitude_wgs")
-        if "longitude_wgs_fr_metropole" in formats_detected:
-            formats_to_remove.add("longitude_l93")
-            formats_to_remove.add("longitude_wgs")
-        if "longitude_wgs" in formats_detected:
-            formats_to_remove.add("longitude_l93")
-        if "code_region" in formats_detected:
-            formats_to_remove.add("code_departement")
-        if "datetime_rfc822" in formats_detected:
-            formats_to_remove.add("datetime_aware")
-        # if there is no way to discriminate the case, default to latlon
-        if "latlon_wgs" in formats_detected:
-            formats_to_remove.add("lonlat_wgs")
+        # Deprioritize less specific formats if:
+        # secondary score is even or worse
+        # or priority score is at least 1 (max of the field score)
+        for prio_format, secondary_formats in priorities:
+            if prio_format in detected_formats:
+                for secondary in secondary_formats:
+                    if (
+                        secondary in detected_formats
+                        and (
+                            return_dict_cols[column_name][prio_format]
+                            >= return_dict_cols[column_name][secondary]
+                            or return_dict_cols[column_name][prio_format] >= 1
+                        )
+                    ):
+                        formats_to_remove.add(secondary)
 
-        formats_to_keep = formats_detected - formats_to_remove
+        formats_to_keep = detected_formats - formats_to_remove
 
-        detections = return_dict_cols_intermediary[column_name]
-        detections = [x for x in detections if x["format"] in formats_to_keep]
+        detections = [x for x in output_dict[column_name] if x["format"] in formats_to_keep]
         if not limited_output:
-            return_dict_cols_intermediary[column_name] = detections
+            output_dict[column_name] = detections
         else:
-            return_dict_cols_intermediary[column_name] = (
+            output_dict[column_name] = (
                 max(detections, key=lambda x: x["score"])
                 if len(detections) > 0
                 else {"format": "string", "score": 1.0}
             )
 
-    return return_dict_cols_intermediary
+    return output_dict

csv_detective/parsing/columns.py

@@ -46,7 +46,6 @@ def test_col_val(
                 ]:  # Pour ne pas faire d'opérations inutiles, on commence par 1,
                     # puis 5 valeurs puis la serie complète
                     if all(apply_test_func(serie, test_func, _range)):
-                        # print(serie.name, ': check OK')
                         pass
                     else:
                         return 0.0

{csv_detective-0.8.1.dev1578.data → csv_detective-0.8.1.dev1599.data}/data/share/csv_detective/CHANGELOG.md

@@ -11,6 +11,7 @@
 - Validate using the testing function, to consider PROPORTIONS [#131](https://github.com/datagouv/csv-detective/pull/131)
 - Remove `datetime_iso` format due to ambiguous cast in db (can be naive or aware) [#132](https://github.com/datagouv/csv-detective/pull/132)
 - Add `lonlat_wgs` format and handle optional brackets for `latlon_wgs` [#133](https://github.com/datagouv/csv-detective/pull/133)
+- Refactor format prioritizing [#134](https://github.com/datagouv/csv-detective/pull/134)
 
 ## 0.8.0 (2025-05-20)
 

{csv_detective-0.8.1.dev1578.dist-info → csv_detective-0.8.1.dev1599.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csv_detective
-Version: 0.8.1.dev1578
+Version: 0.8.1.dev1599
 Summary: Detect tabular files column content
 Home-page: https://github.com/datagouv/csv_detective
 Author: Etalab

{csv_detective-0.8.1.dev1578.dist-info → csv_detective-0.8.1.dev1599.dist-info}/RECORD

@@ -49,7 +49,7 @@ csv_detective/detect_fields/geo/iso_country_code_alpha3/__init__.py,sha256=u98rn
 csv_detective/detect_fields/geo/iso_country_code_alpha3/iso_country_code_alpha3.txt,sha256=aYqKSohgXuBtcIBfF52f8JWYDdxL_HV_Ol1srGnWBp4,1003
 csv_detective/detect_fields/geo/iso_country_code_numeric/__init__.py,sha256=wJAynAkGZN7jKeI3xOeLXQ_irxQBb_J56pRkLDYVClY,436
 csv_detective/detect_fields/geo/iso_country_code_numeric/iso_country_code_numeric.txt,sha256=2GtEhuporsHYV-pU4q9kfXU5iOtfW5C0GYBTTKQtnnA,1004
-csv_detective/detect_fields/geo/json_geojson/__init__.py,sha256=FPHOfTrfXJs62-NgeOcNGOvwPd7I1fEVp8lTdMNfj3w,433
+csv_detective/detect_fields/geo/json_geojson/__init__.py,sha256=6wlwlxQmsVIZ21g-THvH3nBj-I8FuoF2sBlZAoEMGiQ,393
 csv_detective/detect_fields/geo/latitude_wgs/__init__.py,sha256=ArS6PuYEd0atZwSqNDZhXZz1TwzdiwdV8ovRYTOacpg,327
 csv_detective/detect_fields/geo/latlon_wgs/__init__.py,sha256=IXDTqD4YFUJYI1FYZ5ZfkqXY6KvNY7sgBVFRAvgTHtI,454
 csv_detective/detect_fields/geo/longitude_wgs/__init__.py,sha256=G7afWOKiGh_Tv7gwDNGt1a4B_A8hkCBkIxn3THDCUFk,330
@@ -59,7 +59,7 @@ csv_detective/detect_fields/other/booleen/__init__.py,sha256=wn_yyTAmGxqo0l0b7JR
 csv_detective/detect_fields/other/email/__init__.py,sha256=O9tgJmq0O8Q-8iin63NqEEDhlsUJjxFZNaNFM4GZaws,178
 csv_detective/detect_fields/other/float/__init__.py,sha256=AT4Kpgwoz5PuAoLx00u0SL8DjjXZxsE8zSRbN18uAv4,578
 csv_detective/detect_fields/other/int/__init__.py,sha256=QN3kQJLYqLRBiubUK7g4Xq03PlA5wqVwx2pPPIO9FdI,320
-csv_detective/detect_fields/other/json/__init__.py,sha256=DhzyvT12kOqgum89silIu3uoSYXmC_s_AaxLtXAD4eU,540
+csv_detective/detect_fields/other/json/__init__.py,sha256=AkRWZAidEM1dWkVRFThEBI5M7kMUu5Yu12iCViGM8lU,310
 csv_detective/detect_fields/other/money/__init__.py,sha256=g_ZwBZXl9LhldwFYQotC5WqLiE8qQCZHtoI9eJvl_9M,232
 csv_detective/detect_fields/other/mongo_object_id/__init__.py,sha256=7fcrHsOZAqXp2_N0IjPskYJ_qi4xRlo9iyNNDQVLzsU,156
 csv_detective/detect_fields/other/percent/__init__.py,sha256=vgpekNOPBRuunoVBXMi81rwHv4uSOhe78pbVtQ5SBO8,177
@@ -115,7 +115,7 @@ csv_detective/detect_labels/geo/latlon_wgs/__init__.py,sha256=tDndlFyEM7qKS3ATxp
 csv_detective/detect_labels/geo/longitude_wgs/__init__.py,sha256=_8IV2FLtrOjzhQNsk-fsgc9-jbAgzKDVMr4tXu2P-s4,429
 csv_detective/detect_labels/geo/lonlat_wgs/__init__.py,sha256=NNKlFcMsKVqnUKEm_4flGxcNUGS2-iS3m6ihQf2AVTk,345
 csv_detective/detect_labels/other/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-csv_detective/detect_labels/other/booleen/__init__.py,sha256=BZwnfR-Zcv8dqscLrBKhttgwm4Dqq16M0PaGirxYWio,214
+csv_detective/detect_labels/other/booleen/__init__.py,sha256=zEkarex7L4T3vmYjR5hdhtnhugTVDsvkgG_it6nN0aA,214
 csv_detective/detect_labels/other/email/__init__.py,sha256=Poagn45-eC2a_Wdk5Qs6d2BgYdncCQKZp2yEB50IuNw,431
 csv_detective/detect_labels/other/float/__init__.py,sha256=X0axZN2GAfC_y01zRfIyvOfRsOy2KNQcQ-mlQAKxqT4,216
 csv_detective/detect_labels/other/int/__init__.py,sha256=_1AY7thEBCcgSBQQ2YbY4YaPaxGRQ71BtmaFaX088ig,215
@@ -132,7 +132,7 @@ csv_detective/detection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG
 csv_detective/detection/columns.py,sha256=vfE-DKESA6J9Rfsl-a8tjgZfE21VmzArO5TrbzL0KmE,2905
 csv_detective/detection/encoding.py,sha256=tpjJEMNM_2TcLXDzn1lNQPnSRnsWYjs83tQ8jNwTj4E,973
 csv_detective/detection/engine.py,sha256=HiIrU-l9EO5Fbc2Vh8W_Uy5-dpKcQQzlxCqMuWc09LY,1530
-csv_detective/detection/formats.py,sha256=3vf7VdjxTmdt5KaTqGBwT5GuZhHuw98R-sIemTcOIJg,6345
+csv_detective/detection/formats.py,sha256=c0LFTWbibWbEJSZaPy_86LIMOY3qRxj-I_agwpb4zbI,6284
 csv_detective/detection/headers.py,sha256=wrVII2RQpsVmHhrO1DHf3dmiu8kbtOjBlskf41cnQmc,1172
 csv_detective/detection/rows.py,sha256=3qvsbsBcMxiqqfSYYkOgsRpX777rk22tnRHDwUA97kU,742
 csv_detective/detection/separator.py,sha256=XjeDBqhiBxVfkCPJKem9BAgJqs_hOgQltc_pxrH_-Tg,1547
@@ -142,27 +142,27 @@ csv_detective/output/dataframe.py,sha256=89iQRE59cHQyQQEsujQVIKP2YAUYpPklWkdDOqZ
 csv_detective/output/example.py,sha256=EdPX1iqHhIG4DsiHuYdy-J7JxOkjgUh_o2D5nrfM5fA,8649
 csv_detective/output/profile.py,sha256=B8YU541T_YPDezJGh4dkHckOShiwHSrZd9GS8jbmz7A,2919
 csv_detective/output/schema.py,sha256=yC9K1vw6NUTULNv9a7CaMGns9iXmbzFLbtHI4wegqEc,13812
-csv_detective/output/utils.py,sha256=sAFc-oviDn3iR3kr8MdXQa14nCMWgAITsRYBW-f3WM0,2574
+csv_detective/output/utils.py,sha256=xPM2KYdqousmjU22-w7HnaF6AR74fj8lhQY77Y9xs7w,3310
 csv_detective/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-csv_detective/parsing/columns.py,sha256=rLzAU36cHMpVynEPhj8uMdr3IRO3_Yq58Yw7Z6oLPiQ,5693
+csv_detective/parsing/columns.py,sha256=aMdG6-G-2Tj_2JdHotAIveQwaG_r8chGcGieFiUaBRk,5634
 csv_detective/parsing/compression.py,sha256=Fnw5tj-PpBNI8NYsWj5gD-DUoWcVLnsVpiKm9MpxmIA,350
 csv_detective/parsing/csv.py,sha256=11mibDnJhIjykXLGZvA5ZEU5U7KgxIrbyO6BNv6jlro,1626
 csv_detective/parsing/excel.py,sha256=AslE2S1e67o8yTIAIhp-lAnJ6-XqeBBRz1-VMFqhZBM,7055
 csv_detective/parsing/load.py,sha256=u6fbGFZsL2GwPQRzhAXgt32JpUur7vbQdErREHxNJ-w,3661
 csv_detective/parsing/text.py,sha256=_TprGi0gHZlRsafizI3dqQhBehZW4BazqxmypMcAZ-o,1824
-csv_detective-0.8.1.dev1578.data/data/share/csv_detective/CHANGELOG.md,sha256=arQzNTnW3gZp5MyTCoDkb4fMBM7BaMhGnxVF_AhYPp4,9556
-csv_detective-0.8.1.dev1578.data/data/share/csv_detective/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
-csv_detective-0.8.1.dev1578.data/data/share/csv_detective/README.md,sha256=gKLFmC8kuCCywS9eAhMak_JNriUWWNOsBKleAu5TIEY,8501
-csv_detective-0.8.1.dev1578.dist-info/licenses/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
+csv_detective-0.8.1.dev1599.data/data/share/csv_detective/CHANGELOG.md,sha256=WQ8cTB2D5YkAJ9AsS2ziKtZL8m1sPclGPenTD1BxZ_g,9646
+csv_detective-0.8.1.dev1599.data/data/share/csv_detective/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
+csv_detective-0.8.1.dev1599.data/data/share/csv_detective/README.md,sha256=gKLFmC8kuCCywS9eAhMak_JNriUWWNOsBKleAu5TIEY,8501
+csv_detective-0.8.1.dev1599.dist-info/licenses/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_example.py,sha256=JeHxSK0IVDcSrOhSZlNGSQv4JAc_r6mzvJM8PfmLTMw,2018
-tests/test_fields.py,sha256=DzDtYXZDj2YJXNuKROHmoqA__PkE39sqETmnS3a01qQ,10887
+tests/test_fields.py,sha256=IwMpjOn8W5kDCvJYp3Cer4m571qomzjupOAvSRFMg_Q,11819
 tests/test_file.py,sha256=0bHV9wx9mSRoav_DVF19g694yohb1p0bw7rtcBeKG-8,8451
 tests/test_labels.py,sha256=Nkr645bUewrj8hjNDKr67FQ6Sy_TID6f3E5Kfkl231M,464
 tests/test_structure.py,sha256=bv-tjgXohvQAxwmxzH0BynFpK2TyPjcxvtIAmIRlZmA,1393
 tests/test_validation.py,sha256=CTGonR6htxcWF9WH8MxumDD8cF45Y-G4hm94SM4lFjU,3246
-csv_detective-0.8.1.dev1578.dist-info/METADATA,sha256=EcvSGV_PUHKAxG2HlebEXE05CuYh9ZND4erSbWCoIqo,10443
-csv_detective-0.8.1.dev1578.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-csv_detective-0.8.1.dev1578.dist-info/entry_points.txt,sha256=JjweTReFqKJmuvkegzlew2j3D5pZzfxvbEGOtGVGmaY,56
-csv_detective-0.8.1.dev1578.dist-info/top_level.txt,sha256=M0Nv646VHo-49zWjPkwo2C48UmtfddV8_9mEZeIxy8Q,20
-csv_detective-0.8.1.dev1578.dist-info/RECORD,,
+csv_detective-0.8.1.dev1599.dist-info/METADATA,sha256=NoE1tBjCZxO2uffbH9wSgkuNzOVOgLRA2qkjth7ynyk,10443
+csv_detective-0.8.1.dev1599.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+csv_detective-0.8.1.dev1599.dist-info/entry_points.txt,sha256=JjweTReFqKJmuvkegzlew2j3D5pZzfxvbEGOtGVGmaY,56
+csv_detective-0.8.1.dev1599.dist-info/top_level.txt,sha256=M0Nv646VHo-49zWjPkwo2C48UmtfddV8_9mEZeIxy8Q,20
+csv_detective-0.8.1.dev1599.dist-info/RECORD,,

tests/test_fields.py

@@ -72,6 +72,7 @@ from csv_detective.detection.variables import (
 )
 from csv_detective.load_tests import return_all_tests
 from csv_detective.output.dataframe import cast
+from csv_detective.output.utils import prepare_output_dict
 
 
 def test_all_tests_return_bool():
@@ -285,7 +286,7 @@ fields = {
     },
     json: {
         True: ['{"pomme": "fruit", "reponse": 42}', "[1,2,3,4]"],
-        False: ['{"coordinates": [45.783753, 3.049342], "citycode": "63870"}', "{zefib:"],
+        False: ["5", '{"zefib":', '{"a"}'],
     },
     money: {
         True: ["120€", "-20.2$"],
@@ -410,3 +411,27 @@ def test_fields_with_values(args):
 def test_cast(args):
     value, detected_type, cast_type = args
     assert isinstance(cast(value, detected_type), cast_type)
+
+
+@pytest.mark.parametrize(
+    "args",
+    (
+        # there is a specific numerical format => specific wins
+        ({"int": 1, "float": 1, "latitude_wgs": 1}, "latitude_wgs"),
+        # scores are equal for related formats => priority wins
+        ({"int": 1, "float": 1}, "int"),
+        # score is lower for priority format => secondary wins
+        ({"int": 0.5, "float": 1}, "float"),
+        # score is lower for priority format, but is 1 => priority wins
+        ({"int": 1, "float": 1.25}, "int"),
+        # two rounds of priority => highest priority wins
+        ({"latlon_wgs": 1, "lonlat_wgs": 1, "json": 1}, "latlon_wgs"),
+        # no detection => default to string
+        ({}, "string"),
+    ),
+)
+def test_priority(args):
+    detections, expected = args
+    col = "col1"
+    output = prepare_output_dict(pd.DataFrame({col: detections}), limited_output=True)
+    assert output[col]["format"] == expected