csv-detective 0.8.1.dev1535__py3-none-any.whl → 0.8.1.dev1549__py3-none-any.whl

csv_detective/detect_fields/__init__.py

@@ -57,4 +57,4 @@ from .geo import (
 )
 
 from .FR.temp import jour_de_la_semaine, mois_de_annee
-from .temp import year, date, datetime_aware, datetime_iso, datetime_naive, datetime_rfc822
+from .temp import year, date, datetime_aware, datetime_naive, datetime_rfc822

csv_detective/detect_fields/temp/datetime_aware/__init__.py

@@ -8,7 +8,9 @@ PROPORTION = 1
 def _is(val: Optional[Any]) -> bool:
     """Detects timezone-aware datetimes only"""
     # early stops, to cut processing time
-    if not isinstance(val, str) or len(val) > 30 or len(val) < 15:
+    # 21 is the minimal length of a datetime format YYMMDDTHH:MM:SS+HH:MM
+    # 32 is the maximal length of an ISO datetime format YYYY-MM-DDTHH:MM:SS.dddddd+HH:MM, keeping some slack
+    if not isinstance(val, str) or len(val) > 35 or len(val) < 21:
         return False
     threshold = 0.7
     if sum([char.isdigit() or char in {"-", "/", ":", " "} for char in val]) / len(val) < threshold:

csv_detective/detect_fields/temp/datetime_naive/__init__.py

@@ -8,6 +8,8 @@ PROPORTION = 1
 def _is(val: Optional[Any]) -> bool:
     """Detects naive datetimes only"""
     # early stops, to cut processing time
+    # 15 is the minimal length of a datetime format YYMMDDTHH:MM:SS
+    # 26 is the maximal length of an ISO datetime format YYYY-MM-DDTHH:MM:SS.dddddd, keeping some slack
     if not isinstance(val, str) or len(val) > 30 or len(val) < 15:
         return False
     threshold = 0.7

csv_detective/detect_labels/__init__.py

@@ -40,4 +40,4 @@ from .geo import (
     longitude_wgs
 )
 from .other import booleen, email, float, int, money, mongo_object_id, twitter, url, uuid
-from .temp import date, datetime_iso, datetime_rfc822, year
+from .temp import date, datetime_rfc822, year

csv_detective/detection/formats.py

@@ -107,7 +107,6 @@ def detect_formats(
         "json": "json",
         "json_geojson": "json",
         "datetime_aware": "datetime",
-        "datetime_iso": "datetime",
         "datetime_naive": "datetime",
         "datetime_rfc822": "datetime",
         "date": "date",

csv_detective/output/schema.py

@@ -61,7 +61,8 @@ def get_description(format: str) -> str:
         "url": "Adresse URL",
         "uuid": "Identifiant unique au format UUID",
         "date": "Date",
-        "datetime_iso": "Date au format datetime (ISO)",
+        "datetime_aware": "Date au format datetime avec fuseau horaire",
+        "datetime_naive": "Date au format datetime sans fuseau horaire",
         "datetime_rfc822": "Date au format datetime (RFC822)",
         "year": "Année",
     }
@@ -107,7 +108,6 @@ def get_validata_type(format: str) -> str:
         "string": "string",
         "date": "date",
         "datetime_aware": "datetime",
-        "datetime_iso": "datetime",
         "datetime_naive": "datetime",
         "datetime_rfc822": "datetime",
         "json_geojson": "geojson",
@@ -146,7 +146,8 @@ def get_example(format: str) -> str:
         "csp_insee": "anciens agriculteurs exploitants",
         "date": "2020-01-01",
         "date_fr": "12 janvier 2020",
-        "datetime_iso": "2020-01-01T00:00:00",
+        "datetime_aware": "2020-01-01T00:00:00+02:00",
+        "datetime_naive": "2020-01-01T00:00:00",
         "datetime_rfc822": "Tue, 1 Jan 2020 00:00:00 +0000",
         "departement": "Ain",
         "email": "example@example.com",

csv_detective/output/utils.py

@@ -34,8 +34,6 @@ def prepare_output_dict(return_table: pd.DataFrame, limited_output: bool):
             formats_to_remove.add("longitude_l93")
         if "code_region" in formats_detected:
             formats_to_remove.add("code_departement")
-        if "datetime_iso" in formats_detected:
-            formats_to_remove.add("datetime_naive")
         if "datetime_rfc822" in formats_detected:
             formats_to_remove.add("datetime_aware")
 

{csv_detective-0.8.1.dev1535.data → csv_detective-0.8.1.dev1549.data}/data/share/csv_detective/CHANGELOG.md

@@ -9,6 +9,7 @@
 - Fix imports [#125](https://github.com/datagouv/csv-detective/pull/125) [#126](https://github.com/datagouv/csv-detective/pull/126) [#127](https://github.com/datagouv/csv-detective/pull/127) [#128](https://github.com/datagouv/csv-detective/pull/128)
 - Split aware and naive datetimes for hydra to cast them separately [#130](https://github.com/datagouv/csv-detective/pull/130)
 - Validate using the testing function, to consider PROPORTIONS [#131](https://github.com/datagouv/csv-detective/pull/131)
+- Remove `datetime_iso` format due to ambiguous cast in db (can be naive or aware) [#132](https://github.com/datagouv/csv-detective/pull/132)
 
 ## 0.8.0 (2025-05-20)
 

{csv_detective-0.8.1.dev1535.dist-info → csv_detective-0.8.1.dev1549.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csv_detective
-Version: 0.8.1.dev1535
+Version: 0.8.1.dev1549
 Summary: Detect tabular files column content
 Home-page: https://github.com/datagouv/csv_detective
 Author: Etalab

{csv_detective-0.8.1.dev1535.dist-info → csv_detective-0.8.1.dev1549.dist-info}/RECORD

@@ -5,7 +5,7 @@ csv_detective/load_tests.py,sha256=GILvfkd4OVI-72mA4nzbPlZqgcXZ4wznOhGfZ1ucWkM,2
 csv_detective/s3_utils.py,sha256=1cIVdQUYY2ovErbMwp72Gqtqx2bkB8nfVhn-QaOFTT0,1451
 csv_detective/utils.py,sha256=-tIs9yV7RJPGj65lQ7LjRGch6Iws9UeuIPQsd2uUUJM,1025
 csv_detective/validate.py,sha256=5Li_vfvU9wdfoZjNjef-MBUoKcKoJ-c7381QoX9aDXY,2818
-csv_detective/detect_fields/__init__.py,sha256=HYSy0P_aH6R8Z8Hvd8aMaBAQaZ1QwcsWHT0YPm0iYs0,998
+csv_detective/detect_fields/__init__.py,sha256=jThGn0_HO8U0mMoSbf38x8l46ABRQcmHcNLvjZqQQdc,984
 csv_detective/detect_fields/FR/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 csv_detective/detect_fields/FR/geo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 csv_detective/detect_fields/FR/geo/adresse/__init__.py,sha256=NqV8ULf9gY9iFnA1deKR-1Yobr96WwCsn5JfbP_MjiY,1675
@@ -67,12 +67,11 @@ csv_detective/detect_fields/other/url/__init__.py,sha256=L7h9fZldh1w86XwCx0x3Q1T
 csv_detective/detect_fields/other/uuid/__init__.py,sha256=3-z0fDax29SJc57zPjNGR6DPICJu6gfuNGC5L3jh4d0,223
 csv_detective/detect_fields/temp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 csv_detective/detect_fields/temp/date/__init__.py,sha256=VC4_C5lQbjqTweC4T2p9GZAIO64zERhAuf53CPfXgw4,983
-csv_detective/detect_fields/temp/datetime_aware/__init__.py,sha256=Xi3fWiqm_S09AaMeHVrgx6bSieX1gEdjjM7GYsKqEx8,667
-csv_detective/detect_fields/temp/datetime_iso/__init__.py,sha256=DOfli-A7gPlZmiV2J6Ka5_yDUCaOgxis29LET_tfhA4,444
-csv_detective/detect_fields/temp/datetime_naive/__init__.py,sha256=q5Ow1yH9nCz8aY4uOHIKv8CCYIEPLUZlHzg8Nr59kBo,662
+csv_detective/detect_fields/temp/datetime_aware/__init__.py,sha256=bEfWvXx_GNCRUxMGJYqfOK4wRDr3WMaGVAmIa_C2pXE,853
+csv_detective/detect_fields/temp/datetime_naive/__init__.py,sha256=GtQo55SrrXfoT-L7ZXW63jrlAYvNT5m56wMfhuY3pyI,836
 csv_detective/detect_fields/temp/datetime_rfc822/__init__.py,sha256=JtUzg3BXYd-XJMLGxQ0P1OAJGOQ7DlYMD4fCU9yndg0,511
 csv_detective/detect_fields/temp/year/__init__.py,sha256=RjsiIHoplnI4Odi5587TzRhKTDT-FTqGOBpdartuShA,194
-csv_detective/detect_labels/__init__.py,sha256=BJjWlwTnnDe9nomABDUreu9EMu6IFG3T47d7YCJZbRc,878
+csv_detective/detect_labels/__init__.py,sha256=oVq2fiO6QkaWB0wZImL8YVW7oiwPky8ivmLZAFmK55Q,864
 csv_detective/detect_labels/FR/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 csv_detective/detect_labels/FR/geo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 csv_detective/detect_labels/FR/geo/adresse/__init__.py,sha256=fNWFW-Wo3n6azDBfmi0J0qnzP-p2StLxCc9eNiE9NNE,346
@@ -125,14 +124,13 @@ csv_detective/detect_labels/other/url/__init__.py,sha256=4Ajpdp8W0jS9aHZAAMyUlge
 csv_detective/detect_labels/other/uuid/__init__.py,sha256=kXVb4oMy-Zv-OYmAIEoNFrBA20l9hbUTdvTfjeMmhjk,213
 csv_detective/detect_labels/temp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 csv_detective/detect_labels/temp/date/__init__.py,sha256=w0eeZIseAmPwL4OvCWzZXbxGOIXYRKiZUhEtgHiBXd0,604
-csv_detective/detect_labels/temp/datetime_iso/__init__.py,sha256=d0laZNzHx-kSARs9Re8TZ11GNs99aMz6gXc72CJ6ul4,440
 csv_detective/detect_labels/temp/datetime_rfc822/__init__.py,sha256=53ysj7QgsxXwG1le3zfSJd1oaTTf-Er3jBeYi_A4F9g,458
 csv_detective/detect_labels/temp/year/__init__.py,sha256=7uWaCZY7dOG7nolW46IgBWmcu8K-9jPED-pOlMlErfo,433
 csv_detective/detection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 csv_detective/detection/columns.py,sha256=vfE-DKESA6J9Rfsl-a8tjgZfE21VmzArO5TrbzL0KmE,2905
 csv_detective/detection/encoding.py,sha256=tpjJEMNM_2TcLXDzn1lNQPnSRnsWYjs83tQ8jNwTj4E,973
 csv_detective/detection/engine.py,sha256=HiIrU-l9EO5Fbc2Vh8W_Uy5-dpKcQQzlxCqMuWc09LY,1530
-csv_detective/detection/formats.py,sha256=LDrstnAJccDeOEvGbWA5Ppx4gdlJrKbqd7qqWRG2tHI,6382
+csv_detective/detection/formats.py,sha256=3vf7VdjxTmdt5KaTqGBwT5GuZhHuw98R-sIemTcOIJg,6345
 csv_detective/detection/headers.py,sha256=wrVII2RQpsVmHhrO1DHf3dmiu8kbtOjBlskf41cnQmc,1172
 csv_detective/detection/rows.py,sha256=3qvsbsBcMxiqqfSYYkOgsRpX777rk22tnRHDwUA97kU,742
 csv_detective/detection/separator.py,sha256=XjeDBqhiBxVfkCPJKem9BAgJqs_hOgQltc_pxrH_-Tg,1547
@@ -141,8 +139,8 @@ csv_detective/output/__init__.py,sha256=5KTevPfp_4MRxByJyOntQjToNfeG7dPQn-_13wSq
 csv_detective/output/dataframe.py,sha256=89iQRE59cHQyQQEsujQVIKP2YAUYpPklWkdDOqZE-wE,2183
 csv_detective/output/example.py,sha256=EdPX1iqHhIG4DsiHuYdy-J7JxOkjgUh_o2D5nrfM5fA,8649
 csv_detective/output/profile.py,sha256=B8YU541T_YPDezJGh4dkHckOShiwHSrZd9GS8jbmz7A,2919
-csv_detective/output/schema.py,sha256=WxgajFuLfUTQQtmEdlO8ve2ULDzw2BYfz8QFwUsdDh0,13558
-csv_detective/output/utils.py,sha256=qFYhxJmkKrTUefdH7Owh-liZijswomCafic4cXYSyCg,2506
+csv_detective/output/schema.py,sha256=Hpav3RgIP7gOb93h154s1wNSlEZtHNJVzFDDwp54UcQ,13669
+csv_detective/output/utils.py,sha256=RcOkFQihwfmEIOD-gwrUKi2r5CwBbs17vkuAf8n7-Wo,2405
 csv_detective/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 csv_detective/parsing/columns.py,sha256=rLzAU36cHMpVynEPhj8uMdr3IRO3_Yq58Yw7Z6oLPiQ,5693
 csv_detective/parsing/compression.py,sha256=Fnw5tj-PpBNI8NYsWj5gD-DUoWcVLnsVpiKm9MpxmIA,350
@@ -150,19 +148,19 @@ csv_detective/parsing/csv.py,sha256=11mibDnJhIjykXLGZvA5ZEU5U7KgxIrbyO6BNv6jlro,
 csv_detective/parsing/excel.py,sha256=AslE2S1e67o8yTIAIhp-lAnJ6-XqeBBRz1-VMFqhZBM,7055
 csv_detective/parsing/load.py,sha256=u6fbGFZsL2GwPQRzhAXgt32JpUur7vbQdErREHxNJ-w,3661
 csv_detective/parsing/text.py,sha256=_TprGi0gHZlRsafizI3dqQhBehZW4BazqxmypMcAZ-o,1824
-csv_detective-0.8.1.dev1535.data/data/share/csv_detective/CHANGELOG.md,sha256=ECVXB1NkkzReQ4Bt_Bc3GrmGy3aMTSiP6VLxNlpAiN4,9283
-csv_detective-0.8.1.dev1535.data/data/share/csv_detective/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
-csv_detective-0.8.1.dev1535.data/data/share/csv_detective/README.md,sha256=gKLFmC8kuCCywS9eAhMak_JNriUWWNOsBKleAu5TIEY,8501
-csv_detective-0.8.1.dev1535.dist-info/licenses/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
+csv_detective-0.8.1.dev1549.data/data/share/csv_detective/CHANGELOG.md,sha256=1jO_wJx_-DK1TqmdmIu2bmbnvg2iJ2iX78MEb29MZYY,9425
+csv_detective-0.8.1.dev1549.data/data/share/csv_detective/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
+csv_detective-0.8.1.dev1549.data/data/share/csv_detective/README.md,sha256=gKLFmC8kuCCywS9eAhMak_JNriUWWNOsBKleAu5TIEY,8501
+csv_detective-0.8.1.dev1549.dist-info/licenses/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_example.py,sha256=JeHxSK0IVDcSrOhSZlNGSQv4JAc_r6mzvJM8PfmLTMw,2018
-tests/test_fields.py,sha256=zeEQbHs0ougLzydmZLZs1l2UdrhKBEtdCCK64B4dhSU,10700
+tests/test_fields.py,sha256=tTFOmlb9gewtCwcZV7B6Gc3aH6xXK5kMUFSEBi7iIy4,10638
 tests/test_file.py,sha256=0bHV9wx9mSRoav_DVF19g694yohb1p0bw7rtcBeKG-8,8451
 tests/test_labels.py,sha256=Nkr645bUewrj8hjNDKr67FQ6Sy_TID6f3E5Kfkl231M,464
 tests/test_structure.py,sha256=bv-tjgXohvQAxwmxzH0BynFpK2TyPjcxvtIAmIRlZmA,1393
 tests/test_validation.py,sha256=CTGonR6htxcWF9WH8MxumDD8cF45Y-G4hm94SM4lFjU,3246
-csv_detective-0.8.1.dev1535.dist-info/METADATA,sha256=MlEo_TETeNCT0LVS92Uu6f0W5BRMT70GqYXh2fhE12U,10443
-csv_detective-0.8.1.dev1535.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-csv_detective-0.8.1.dev1535.dist-info/entry_points.txt,sha256=JjweTReFqKJmuvkegzlew2j3D5pZzfxvbEGOtGVGmaY,56
-csv_detective-0.8.1.dev1535.dist-info/top_level.txt,sha256=M0Nv646VHo-49zWjPkwo2C48UmtfddV8_9mEZeIxy8Q,20
-csv_detective-0.8.1.dev1535.dist-info/RECORD,,
+csv_detective-0.8.1.dev1549.dist-info/METADATA,sha256=LQkj1jrN7dsdUjGOV3Z8BRKANccZTxdXdoRWtAvOa6w,10443
+csv_detective-0.8.1.dev1549.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+csv_detective-0.8.1.dev1549.dist-info/entry_points.txt,sha256=JjweTReFqKJmuvkegzlew2j3D5pZzfxvbEGOtGVGmaY,56
+csv_detective-0.8.1.dev1549.dist-info/top_level.txt,sha256=M0Nv646VHo-49zWjPkwo2C48UmtfddV8_9mEZeIxy8Q,20
+csv_detective-0.8.1.dev1549.dist-info/RECORD,,

tests/test_fields.py

@@ -61,7 +61,6 @@ from csv_detective.detect_fields.other import (
 from csv_detective.detect_fields.temp import (
     date,
     datetime_aware,
-    datetime_iso,
     datetime_naive,
     datetime_rfc822,
     year,
@@ -345,17 +344,18 @@ fields = {
         ],
     },
     datetime_aware: {
-        True: ["2021-06-22 10:20:10-04:00", "2030-06-22 00:00:00.0028+02:00", "1996/06/22 10:20:10 GMT"],
+        True: [
+            "2021-06-22 10:20:10-04:00",
+            "2030-06-22 00:00:00.0028+02:00",
+            "2024-12-19T10:53:36.428000+00:00",
+            "1996/06/22 10:20:10 GMT",
+        ],
         False: ["2021-06-22T30:20:10", "Sun, 06 Nov 1994 08:49:37 GMT", "2021-06-44 10:20:10"],
     },
     datetime_naive: {
         True: ["2021-06-22 10:20:10", "2030/06/22 00:00:00.0028"],
         False: ["2021-06-22T30:20:10", "Sun, 06 Nov 1994 08:49:37 GMT", "2021-06-44 10:20:10+02:00"],
     },
-    datetime_iso: {
-        True: ["2021-06-22T10:20:10"],
-        False: ["2021-06-22T30:20:10", "Sun, 06 Nov 1994 08:49:37 GMT"],
-    },
     datetime_rfc822: {
         True: ["Sun, 06 Nov 1994 08:49:37 GMT"],
         False: ["2021-06-22T10:20:10"],

csv_detective/detect_fields/temp/datetime_iso/__init__.py

@@ -1,16 +0,0 @@
-import re
-
-PROPORTION = 1
-
-
-def _is(val):
-    '''Renvoie True si val peut être une date au format iso, False sinon
-    Exemple: 2023-01-15T12:30:45.123456Z'''
-    return isinstance(val, str) and bool(
-        re.match(
-            r'^\d{4}-(0[1-9]|1[012])\-(0[1-9]|[12][0-9]|3[01])[Tt]'
-            r'([0-2])([0-9]):([0-5])([0-9]):([0-5])([0-9])'
-            r'(\.\d+)?([Zz]|[-+](0[0-9]|1[0-2]):[0-5][0-9])?$',
-            val
-        )
-    )

csv_detective/detect_labels/temp/datetime_iso/__init__.py

@@ -1,20 +0,0 @@
-from csv_detective.parsing.text import header_score
-
-PROPORTION = 0.5
-
-
-def _is(header: str) -> float:
-    words_combinations_list = [
-        "datetime iso",
-        "datetime",
-        "timestamp",
-        "osm_timestamp",
-        "date",
-        "created at",
-        "last update",
-        "date maj",
-        "createdat",
-        "date naissance",
-        "date donnees",
-    ]
-    return header_score(header, words_combinations_list)