csv-detective 0.9.3.dev2409__py3-none-any.whl → 0.9.3.dev2447__py3-none-any.whl

csv_detective/parsing/columns.py

@@ -38,23 +38,21 @@ def test_col_val(
         if ser_len == 0:
             # being here means the whole column is NaN, so if skipna it's a pass
             return 1.0 if skipna else 0.0
-        if not limited_output:
-            result = apply_test_func(serie, format.func, ser_len).sum() / ser_len
+        if not limited_output or format.proportion < 1:
+            # we want or have to go through the whole column to have the proportion
+            result: float = serie.apply(format.func).sum() / ser_len
             return result if result >= format.proportion else 0.0
         else:
-            if format.proportion == 1:
-                # early stops (1 then 5 rows) to not waste time if directly unsuccessful
-                for _range in [
-                    min(1, ser_len),
-                    min(5, ser_len),
-                    ser_len,
-                ]:
-                    if not all(apply_test_func(serie, format.func, _range)):
-                        return 0.0
-                return 1.0
-            else:
-                result = apply_test_func(serie, format.func, ser_len).sum() / ser_len
-                return result if result >= format.proportion else 0.0
+            # the whole column has to be valid so we have early stops (1 then 5 rows)
+            # to not waste time if directly unsuccessful
+            for _range in [
+                min(1, ser_len),
+                min(5, ser_len),
+                ser_len,
+            ]:
+                if not all(apply_test_func(serie, format.func, _range)):
+                    return 0.0
+            return 1.0
     finally:
         if verbose and time() - start > 3:
             display_logs_depending_process_time(
@@ -80,15 +78,14 @@ def test_col(
             logging.info(f"\t- Starting with format '{label}'")
         # improvement lead : put the longest tests behind and make them only if previous tests not satisfactory
         # => the following needs to change, "apply" means all columns are tested for one type at once
-        return_table.loc[label] = table.apply(
-            lambda serie: test_col_val(
-                serie,
+        for col in table.columns:
+            return_table.loc[label, col] = test_col_val(
+                table[col],
                 format,
                 skipna=skipna,
                 limited_output=limited_output,
                 verbose=verbose,
             )
-        )
         if verbose:
             display_logs_depending_process_time(
                 f'\t> Done with type "{label}" in {round(time() - start_type, 3)}s ({idx + 1}/{len(formats)})',

{csv_detective-0.9.3.dev2409.dist-info → csv_detective-0.9.3.dev2447.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: csv-detective
-Version: 0.9.3.dev2409
+Version: 0.9.3.dev2447
 Summary: Detect tabular files column content
 Keywords: CSV,data processing,encoding,guess,parser,tabular
 Author: data.gouv.fr

{csv_detective-0.9.3.dev2409.dist-info → csv_detective-0.9.3.dev2447.dist-info}/RECORD

@@ -78,7 +78,7 @@ csv_detective/output/profile.py,sha256=VUQp0VJ22dfY4R5TybTpuQW_TOX_rLEp98cOzu-Jf
 csv_detective/output/schema.py,sha256=XoKljXPXP00DfqPCiz1ydwTHYGAFsvNxnaPCNBuuBIo,10443
 csv_detective/output/utils.py,sha256=tbji3dEH7bDc6gLCeVSVquqU3xaHA1CQOMuaJT4Hub8,3297
 csv_detective/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-csv_detective/parsing/columns.py,sha256=CqtZRZYMYDNMopxnqs4eZLSABi-ms61wqv5M9vWJ7iU,9343
+csv_detective/parsing/columns.py,sha256=Eo8GUec5ykTuDTR15OC0S_hiMkcpuZnTPCPomW80aSQ,9244
 csv_detective/parsing/compression.py,sha256=Fnw5tj-PpBNI8NYsWj5gD-DUoWcVLnsVpiKm9MpxmIA,350
 csv_detective/parsing/csv.py,sha256=0T0gpaXzwJo-sq41IoLQD704GiMUYeDVVASVbat-zWg,1726
 csv_detective/parsing/excel.py,sha256=oAVTuoDccJc4-kVjHXiIPLQx3lq3aZRRZQxkG1c06JQ,6992
@@ -86,7 +86,7 @@ csv_detective/parsing/load.py,sha256=f-8aKiNpy_47qg4Lq-UZUR4NNrbJ_-KEGvcUQZ8cmb0
 csv_detective/parsing/text.py,sha256=uz8wfmNTQnOd_4fjrIZ_5rxmFmgrg343hJh2szB73Hc,1770
 csv_detective/utils.py,sha256=RJ_zFOJ1DRY8HtDrKPiCdNk5gU6-KwOrOKOyfSkBZZY,1118
 csv_detective/validate.py,sha256=XldlbGkUlPaIh0y4z9iaWlmmahwCrD1900s5Cxlq5wI,5430
-csv_detective-0.9.3.dev2409.dist-info/WHEEL,sha256=z-mOpxbJHqy3cq6SvUThBZdaLGFZzdZPtgWLcP2NKjQ,79
-csv_detective-0.9.3.dev2409.dist-info/entry_points.txt,sha256=1J86TQNCanjsLMboAufdEUla03qEQaC9QmVGYgt2FCQ,57
-csv_detective-0.9.3.dev2409.dist-info/METADATA,sha256=mcETENWniXaJkJwPuxDTnyUi3DNNq9yhnPKinyGQRpg,11063
-csv_detective-0.9.3.dev2409.dist-info/RECORD,,
+csv_detective-0.9.3.dev2447.dist-info/WHEEL,sha256=z-mOpxbJHqy3cq6SvUThBZdaLGFZzdZPtgWLcP2NKjQ,79
+csv_detective-0.9.3.dev2447.dist-info/entry_points.txt,sha256=1J86TQNCanjsLMboAufdEUla03qEQaC9QmVGYgt2FCQ,57
+csv_detective-0.9.3.dev2447.dist-info/METADATA,sha256=Pl4Yw1e2r6GcmmTp405LNmslBeHfsYAHNpc1kvGYz14,11063
+csv_detective-0.9.3.dev2447.dist-info/RECORD,,