csv-detective 0.10.3.dev1__py3-none-any.whl → 0.10.3.dev3__py3-none-any.whl

csv_detective/output/dataframe.py

@@ -14,8 +14,8 @@ from csv_detective.utils import display_logs_depending_process_time
 
 
 def cast(value: str, _type: str) -> str | int | float | bool | date | datetime | bytes | None:
-    if not isinstance(value, str) or not value:
-        # None is the current default value in hydra, should we keep this?
+    if not isinstance(value, str) or value in pd._libs.parsers.STR_NA_VALUES:
+        # STR_NA_VALUES are directly ingested as NaN by pandas, we avoid trying to cast them (into int for instance)
         return None
     match _type:
         case "string":

csv_detective/parsing/load.py

@@ -1,3 +1,4 @@
+import codecs
 from io import BytesIO, StringIO
 
 import pandas as pd
@@ -69,12 +70,13 @@ def load_file(
             binary_file.seek(0)
         # decoding and reading file
         if is_url(file_path) or engine in COMPRESSION_ENGINES:
+            decoder = codecs.getincrementaldecoder(encoding)()
             str_file = StringIO()
             while True:
                 chunk = binary_file.read(1024**2)
                 if not chunk:
                     break
-                str_file.write(chunk.decode(encoding=encoding))
+                str_file.write(decoder.decode(chunk))
             del binary_file
             str_file.seek(0)
         else:

{csv_detective-0.10.3.dev1.dist-info → csv_detective-0.10.3.dev3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csv-detective
-Version: 0.10.3.dev1
+Version: 0.10.3.dev3
 Summary: Detect tabular files column content
 Author-email: "data.gouv.fr" <opendatateam@data.gouv.fr>
 License: MIT

{csv_detective-0.10.3.dev1.dist-info → csv_detective-0.10.3.dev3.dist-info}/RECORD

@@ -74,7 +74,7 @@ csv_detective/formats/data/iso_country_code_alpha2.txt,sha256=mLt_qcQ6D8hfy9zdi7
 csv_detective/formats/data/iso_country_code_alpha3.txt,sha256=XFPdGBsyZCBg4D8IDn6VgwsycCwYVfuqPbyHfNeqGv0,1003
 csv_detective/formats/data/iso_country_code_numeric.txt,sha256=sdGpn0PqDMlc59-7prThkihHrf7mwB6j5uEHpxGvLFE,1003
 csv_detective/output/__init__.py,sha256=ALSq_tgX7rGyh--7rmbKz8wHkmResN0h7mNujndow3w,2103
-csv_detective/output/dataframe.py,sha256=QX5vplx0AOKgnwwJ6dKvDHWRX9IGPStax-svXEyweJ8,3584
+csv_detective/output/dataframe.py,sha256=juBMdj0eiL8c3OrJJ3kCf15Qs4-CFQfHqh91FnVbG9E,3656
 csv_detective/output/example.py,sha256=8LWheSBYCeDFfarbnmzBrdCbTd8Alh1U4pfXMKfabOw,8630
 csv_detective/output/profile.py,sha256=ADr5DwuvwcBYxugjN38fHm11l6ivfzGHXPd8a87Ht-s,4985
 csv_detective/output/schema.py,sha256=XoKljXPXP00DfqPCiz1ydwTHYGAFsvNxnaPCNBuuBIo,10443
@@ -84,12 +84,12 @@ csv_detective/parsing/columns.py,sha256=rb5JywbKnYCT3Jb0ZaG1BnyPVtB3gy5mSD-K7qcO
 csv_detective/parsing/compression.py,sha256=Fnw5tj-PpBNI8NYsWj5gD-DUoWcVLnsVpiKm9MpxmIA,350
 csv_detective/parsing/csv.py,sha256=5rw6gXZFQC1T4NT9CnW0AumidrYOkF8kjrfWGmk949I,1716
 csv_detective/parsing/excel.py,sha256=tb65I78tdYlZci_tzvvQt8U6bZSYKjeVdn2CEvsET1o,6972
-csv_detective/parsing/load.py,sha256=f-8aKiNpy_47qg4Lq-UZUR4NNrbJ_-KEGvcUQZ8cmb0,4317
+csv_detective/parsing/load.py,sha256=orW6PV5XUsHA093yVSxXkJl33LEUUArr3hP81U9Bzd4,4386
 csv_detective/parsing/text.py,sha256=yDAcop5xJQc25UtbZcV0guHXAZQfm-H8WuJORTy8Rr8,1734
-csv_detective-0.10.3.dev1.dist-info/licenses/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
+csv_detective-0.10.3.dev3.dist-info/licenses/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_example.py,sha256=uTWswvUzBWEADGXZmMAdZvKhKvIjvT5zWOVVABgCDN4,1987
-tests/test_fields.py,sha256=EuD2F1JUR8y88Hm-AYuJ5X7AKkGSyLIQfsGdxYgIWng,5680
+tests/test_fields.py,sha256=DSI-ZXDcRt69iZArYZZAr_3OEb-qvwgOVBZxmYAKIkI,5918
 tests/test_file.py,sha256=_ftEymft5-1keUVE5AUdF2XkVcChJo6oBjws3ye06FE,14543
 tests/test_labels.py,sha256=lgxRbLrGV1C-MkASf3KIQ120BG-UHzFQ4pqDWaeBvaw,539
 tests/test_structure.py,sha256=XDbviuuvk-0Mu9Y9PI6He2e5hry2dXVJ6yBVwEqF_2o,1043
@@ -104,8 +104,8 @@ tests/data/file.ods,sha256=4dR7zWptz5djALIBVeWHQ20GaZNfA63fevIJGFIk1_U,11832
 tests/data/file.xls,sha256=QYmNX3FF0QfcQSzYQMtaMJaepJf5EZpDa1miKc4wMdQ,21495
 tests/data/file.xlsx,sha256=naWzL02PK4pdIjMzfEyfSW9GQhkYYd_e7bpJvB8Pb2w,8314
 tests/data/xlsx_file,sha256=NyOyN_rIe7ryJuHQLqjxVdKCc8V4s5pxyHl6wWFykCM,8305
-csv_detective-0.10.3.dev1.dist-info/METADATA,sha256=gJcwR3wni376q4qqk8xOG6uy9W4fwA5enfmfT066990,11082
-csv_detective-0.10.3.dev1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-csv_detective-0.10.3.dev1.dist-info/entry_points.txt,sha256=JjweTReFqKJmuvkegzlew2j3D5pZzfxvbEGOtGVGmaY,56
-csv_detective-0.10.3.dev1.dist-info/top_level.txt,sha256=KDI4gyOpkmormGgUvSWrE3jen2e0unIsxR2b96DRvcw,25
-csv_detective-0.10.3.dev1.dist-info/RECORD,,
+csv_detective-0.10.3.dev3.dist-info/METADATA,sha256=aVDOx1LTRqvJIGf4wu4krkEjPl8HDyCICBbswfn_Wvg,11082
+csv_detective-0.10.3.dev3.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+csv_detective-0.10.3.dev3.dist-info/entry_points.txt,sha256=JjweTReFqKJmuvkegzlew2j3D5pZzfxvbEGOtGVGmaY,56
+csv_detective-0.10.3.dev3.dist-info/top_level.txt,sha256=KDI4gyOpkmormGgUvSWrE3jen2e0unIsxR2b96DRvcw,25
+csv_detective-0.10.3.dev3.dist-info/RECORD,,

{csv_detective-0.10.3.dev1.dist-info → csv_detective-0.10.3.dev3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

tests/test_fields.py

@@ -104,11 +104,17 @@ def test_fields_with_values(args):
         ("2022-08-01", "date", _date),
         ("2024-09-23 17:32:07", "datetime", _datetime),
         ("2024-09-23 17:32:07+02:00", "datetime", _datetime),
+        ("N/A", "int", None),
+        ("nan", "bool", None),
+        ("", "date", None),  # all NaN-like values should be cast as None for all type
     ),
 )
 def test_cast(args):
     value, detected_type, cast_type = args
-    assert isinstance(cast(value, detected_type), cast_type)
+    if cast_type is None:
+        assert cast(value, detected_type) is None
+    else:
+        assert isinstance(cast(value, detected_type), cast_type)
 
 
 @pytest.mark.parametrize(