csv-detective 0.8.1.dev1674__py3-none-any.whl → 0.8.1.dev1720__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. csv_detective/__init__.py +0 -2
  2. csv_detective/cli.py +6 -9
  3. csv_detective/detect_fields/FR/geo/adresse/__init__.py +78 -78
  4. csv_detective/detect_fields/FR/geo/code_departement/__init__.py +2 -2
  5. csv_detective/detect_fields/FR/geo/code_postal/__init__.py +0 -1
  6. csv_detective/detect_fields/FR/geo/code_region/__init__.py +1 -1
  7. csv_detective/detect_fields/FR/geo/commune/__init__.py +2 -2
  8. csv_detective/detect_fields/FR/geo/departement/__init__.py +2 -2
  9. csv_detective/detect_fields/FR/geo/insee_canton/__init__.py +2 -2
  10. csv_detective/detect_fields/FR/geo/latitude_l93/__init__.py +1 -2
  11. csv_detective/detect_fields/FR/geo/latitude_wgs_fr_metropole/__init__.py +1 -1
  12. csv_detective/detect_fields/FR/geo/longitude_l93/__init__.py +1 -2
  13. csv_detective/detect_fields/FR/geo/longitude_wgs_fr_metropole/__init__.py +1 -1
  14. csv_detective/detect_fields/FR/geo/pays/__init__.py +6 -6
  15. csv_detective/detect_fields/FR/geo/region/__init__.py +6 -4
  16. csv_detective/detect_fields/FR/other/code_csp_insee/__init__.py +15 -14
  17. csv_detective/detect_fields/FR/other/csp_insee/__init__.py +4 -3
  18. csv_detective/detect_fields/FR/other/date_fr/__init__.py +3 -3
  19. csv_detective/detect_fields/FR/other/insee_ape700/__init__.py +4 -3
  20. csv_detective/detect_fields/FR/other/sexe/__init__.py +2 -2
  21. csv_detective/detect_fields/FR/other/siren/__init__.py +3 -3
  22. csv_detective/detect_fields/FR/other/siret/__init__.py +3 -3
  23. csv_detective/detect_fields/FR/other/tel_fr/__init__.py +3 -3
  24. csv_detective/detect_fields/FR/other/uai/__init__.py +2 -2
  25. csv_detective/detect_fields/FR/temp/jour_de_la_semaine/__init__.py +15 -15
  26. csv_detective/detect_fields/FR/temp/mois_de_annee/__init__.py +27 -27
  27. csv_detective/detect_fields/geo/iso_country_code_alpha2/__init__.py +5 -5
  28. csv_detective/detect_fields/geo/iso_country_code_alpha3/__init__.py +5 -5
  29. csv_detective/detect_fields/geo/iso_country_code_numeric/__init__.py +5 -5
  30. csv_detective/detect_fields/geo/latitude_wgs/__init__.py +1 -1
  31. csv_detective/detect_fields/geo/longitude_wgs/__init__.py +1 -1
  32. csv_detective/detect_fields/other/booleen/__init__.py +1 -1
  33. csv_detective/detect_fields/other/email/__init__.py +4 -2
  34. csv_detective/detect_fields/other/int/__init__.py +3 -3
  35. csv_detective/detect_fields/other/mongo_object_id/__init__.py +2 -2
  36. csv_detective/detect_fields/other/twitter/__init__.py +2 -2
  37. csv_detective/detect_fields/other/uuid/__init__.py +4 -5
  38. csv_detective/detect_fields/temp/date/__init__.py +3 -2
  39. csv_detective/detect_fields/temp/datetime_rfc822/__init__.py +6 -6
  40. csv_detective/detect_fields/temp/year/__init__.py +1 -1
  41. csv_detective/detect_labels/FR/other/tel_fr/__init__.py +0 -1
  42. csv_detective/detect_labels/geo/lonlat_wgs/__init__.py +1 -0
  43. csv_detective/detect_labels/other/mongo_object_id/__init__.py +1 -1
  44. csv_detective/detection/columns.py +9 -9
  45. csv_detective/detection/encoding.py +6 -4
  46. csv_detective/detection/engine.py +6 -5
  47. csv_detective/detection/formats.py +19 -19
  48. csv_detective/detection/headers.py +3 -5
  49. csv_detective/detection/rows.py +1 -1
  50. csv_detective/detection/variables.py +4 -4
  51. csv_detective/explore_csv.py +7 -8
  52. csv_detective/load_tests.py +6 -14
  53. csv_detective/output/__init__.py +3 -7
  54. csv_detective/output/dataframe.py +9 -5
  55. csv_detective/output/example.py +13 -13
  56. csv_detective/output/profile.py +30 -23
  57. csv_detective/output/schema.py +20 -23
  58. csv_detective/output/utils.py +15 -15
  59. csv_detective/parsing/columns.py +23 -12
  60. csv_detective/parsing/csv.py +1 -1
  61. csv_detective/parsing/excel.py +10 -11
  62. csv_detective/parsing/load.py +11 -8
  63. csv_detective/parsing/text.py +4 -9
  64. csv_detective/s3_utils.py +3 -7
  65. csv_detective/utils.py +4 -2
  66. csv_detective/validate.py +18 -13
  67. csv_detective-0.8.1.dev1674.data/data/share/csv_detective/README.md → csv_detective-0.8.1.dev1720.dist-info/METADATA +32 -0
  68. {csv_detective-0.8.1.dev1674.dist-info → csv_detective-0.8.1.dev1720.dist-info}/RECORD +81 -81
  69. {csv_detective-0.8.1.dev1674.dist-info → csv_detective-0.8.1.dev1720.dist-info}/top_level.txt +2 -0
  70. tests/test_example.py +2 -6
  71. tests/test_fields.py +16 -10
  72. tests/test_file.py +10 -9
  73. tests/test_labels.py +3 -2
  74. tests/test_structure.py +3 -1
  75. tests/test_validation.py +9 -6
  76. venv/bin/activate_this.py +38 -0
  77. venv/bin/jp.py +54 -0
  78. venv/bin/runxlrd.py +410 -0
  79. csv_detective-0.8.1.dev1674.data/data/share/csv_detective/CHANGELOG.md +0 -186
  80. csv_detective-0.8.1.dev1674.dist-info/METADATA +0 -268
  81. csv_detective-0.8.1.dev1674.dist-info/licenses/LICENSE +0 -21
  82. {csv_detective-0.8.1.dev1674.dist-info → csv_detective-0.8.1.dev1720.dist-info}/WHEEL +0 -0
  83. {csv_detective-0.8.1.dev1674.dist-info → csv_detective-0.8.1.dev1720.dist-info}/entry_points.txt +0 -0
  84. {csv_detective-0.8.1.dev1674.data/data/share/csv_detective → csv_detective-0.8.1.dev1720.dist-info/licenses}/LICENSE +0 -0
venv/bin/runxlrd.py ADDED
@@ -0,0 +1,410 @@
1
+ #!/home/circleci/project/venv/bin/python
2
+ # Copyright (c) 2005-2012 Stephen John Machin, Lingfo Pty Ltd
3
+ # This script is part of the xlrd package, which is released under a
4
+ # BSD-style licence.
5
+
6
+ from __future__ import print_function
7
+
8
+ cmd_doc = """
9
+ Commands:
10
+
11
+ 2rows Print the contents of first and last row in each sheet
12
+ 3rows Print the contents of first, second and last row in each sheet
13
+ bench Same as "show", but doesn't print -- for profiling
14
+ biff_count[1] Print a count of each type of BIFF record in the file
15
+ biff_dump[1] Print a dump (char and hex) of the BIFF records in the file
16
+ fonts hdr + print a dump of all font objects
17
+ hdr Mini-overview of file (no per-sheet information)
18
+ hotshot Do a hotshot profile run e.g. ... -f1 hotshot bench bigfile*.xls
19
+ labels Dump of sheet.col_label_ranges and ...row... for each sheet
20
+ name_dump Dump of each object in book.name_obj_list
21
+ names Print brief information for each NAME record
22
+ ov Overview of file
23
+ profile Like "hotshot", but uses cProfile
24
+ show Print the contents of all rows in each sheet
25
+ version[0] Print versions of xlrd and Python and exit
26
+ xfc Print "XF counts" and cell-type counts -- see code for details
27
+
28
+ [0] means no file arg
29
+ [1] means only one file arg i.e. no glob.glob pattern
30
+ """
31
+
32
+ options = None
33
+ if __name__ == "__main__":
34
+ import xlrd
35
+ import sys
36
+ import time
37
+ import glob
38
+ import traceback
39
+ import gc
40
+
41
+ from xlrd.timemachine import xrange, REPR
42
+
43
+
44
+ class LogHandler(object):
45
+
46
+ def __init__(self, logfileobj):
47
+ self.logfileobj = logfileobj
48
+ self.fileheading = None
49
+ self.shown = 0
50
+
51
+ def setfileheading(self, fileheading):
52
+ self.fileheading = fileheading
53
+ self.shown = 0
54
+
55
+ def write(self, text):
56
+ if self.fileheading and not self.shown:
57
+ self.logfileobj.write(self.fileheading)
58
+ self.shown = 1
59
+ self.logfileobj.write(text)
60
+
61
+ null_cell = xlrd.empty_cell
62
+
63
+ def show_row(bk, sh, rowx, colrange, printit):
64
+ if bk.ragged_rows:
65
+ colrange = range(sh.row_len(rowx))
66
+ if not colrange: return
67
+ if printit: print()
68
+ if bk.formatting_info:
69
+ for colx, ty, val, cxfx in get_row_data(bk, sh, rowx, colrange):
70
+ if printit:
71
+ print("cell %s%d: type=%d, data: %r, xfx: %s"
72
+ % (xlrd.colname(colx), rowx+1, ty, val, cxfx))
73
+ else:
74
+ for colx, ty, val, _unused in get_row_data(bk, sh, rowx, colrange):
75
+ if printit:
76
+ print("cell %s%d: type=%d, data: %r" % (xlrd.colname(colx), rowx+1, ty, val))
77
+
78
+ def get_row_data(bk, sh, rowx, colrange):
79
+ result = []
80
+ dmode = bk.datemode
81
+ ctys = sh.row_types(rowx)
82
+ cvals = sh.row_values(rowx)
83
+ for colx in colrange:
84
+ cty = ctys[colx]
85
+ cval = cvals[colx]
86
+ if bk.formatting_info:
87
+ cxfx = str(sh.cell_xf_index(rowx, colx))
88
+ else:
89
+ cxfx = ''
90
+ if cty == xlrd.XL_CELL_DATE:
91
+ try:
92
+ showval = xlrd.xldate_as_tuple(cval, dmode)
93
+ except xlrd.XLDateError as e:
94
+ showval = "%s:%s" % (type(e).__name__, e)
95
+ cty = xlrd.XL_CELL_ERROR
96
+ elif cty == xlrd.XL_CELL_ERROR:
97
+ showval = xlrd.error_text_from_code.get(cval, '<Unknown error code 0x%02x>' % cval)
98
+ else:
99
+ showval = cval
100
+ result.append((colx, cty, showval, cxfx))
101
+ return result
102
+
103
+ def bk_header(bk):
104
+ print()
105
+ print("BIFF version: %s; datemode: %s"
106
+ % (xlrd.biff_text_from_num[bk.biff_version], bk.datemode))
107
+ print("codepage: %r (encoding: %s); countries: %r"
108
+ % (bk.codepage, bk.encoding, bk.countries))
109
+ print("Last saved by: %r" % bk.user_name)
110
+ print("Number of data sheets: %d" % bk.nsheets)
111
+ print("Use mmap: %d; Formatting: %d; On demand: %d"
112
+ % (bk.use_mmap, bk.formatting_info, bk.on_demand))
113
+ print("Ragged rows: %d" % bk.ragged_rows)
114
+ if bk.formatting_info:
115
+ print("FORMATs: %d, FONTs: %d, XFs: %d"
116
+ % (len(bk.format_list), len(bk.font_list), len(bk.xf_list)))
117
+ if not options.suppress_timing:
118
+ print("Load time: %.2f seconds (stage 1) %.2f seconds (stage 2)"
119
+ % (bk.load_time_stage_1, bk.load_time_stage_2))
120
+ print()
121
+
122
+ def show_fonts(bk):
123
+ print("Fonts:")
124
+ for x in xrange(len(bk.font_list)):
125
+ font = bk.font_list[x]
126
+ font.dump(header='== Index %d ==' % x, indent=4)
127
+
128
+ def show_names(bk, dump=0):
129
+ bk_header(bk)
130
+ if bk.biff_version < 50:
131
+ print("Names not extracted in this BIFF version")
132
+ return
133
+ nlist = bk.name_obj_list
134
+ print("Name list: %d entries" % len(nlist))
135
+ for nobj in nlist:
136
+ if dump:
137
+ nobj.dump(sys.stdout,
138
+ header="\n=== Dump of name_obj_list[%d] ===" % nobj.name_index)
139
+ else:
140
+ print("[%d]\tName:%r macro:%r scope:%d\n\tresult:%r\n"
141
+ % (nobj.name_index, nobj.name, nobj.macro, nobj.scope, nobj.result))
142
+
143
+ def print_labels(sh, labs, title):
144
+ if not labs:return
145
+ for rlo, rhi, clo, chi in labs:
146
+ print("%s label range %s:%s contains:"
147
+ % (title, xlrd.cellname(rlo, clo), xlrd.cellname(rhi-1, chi-1)))
148
+ for rx in xrange(rlo, rhi):
149
+ for cx in xrange(clo, chi):
150
+ print(" %s: %r" % (xlrd.cellname(rx, cx), sh.cell_value(rx, cx)))
151
+
152
+ def show_labels(bk):
153
+ # bk_header(bk)
154
+ hdr = 0
155
+ for shx in range(bk.nsheets):
156
+ sh = bk.sheet_by_index(shx)
157
+ clabs = sh.col_label_ranges
158
+ rlabs = sh.row_label_ranges
159
+ if clabs or rlabs:
160
+ if not hdr:
161
+ bk_header(bk)
162
+ hdr = 1
163
+ print("sheet %d: name = %r; nrows = %d; ncols = %d" %
164
+ (shx, sh.name, sh.nrows, sh.ncols))
165
+ print_labels(sh, clabs, 'Col')
166
+ print_labels(sh, rlabs, 'Row')
167
+ if bk.on_demand: bk.unload_sheet(shx)
168
+
169
+ def show(bk, nshow=65535, printit=1):
170
+ bk_header(bk)
171
+ if 0:
172
+ rclist = xlrd.sheet.rc_stats.items()
173
+ rclist = sorted(rclist)
174
+ print("rc stats")
175
+ for k, v in rclist:
176
+ print("0x%04x %7d" % (k, v))
177
+ if options.onesheet:
178
+ try:
179
+ shx = int(options.onesheet)
180
+ except ValueError:
181
+ shx = bk.sheet_by_name(options.onesheet).number
182
+ shxrange = [shx]
183
+ else:
184
+ shxrange = range(bk.nsheets)
185
+ # print("shxrange", list(shxrange))
186
+ for shx in shxrange:
187
+ sh = bk.sheet_by_index(shx)
188
+ nrows, ncols = sh.nrows, sh.ncols
189
+ colrange = range(ncols)
190
+ anshow = min(nshow, nrows)
191
+ print("sheet %d: name = %s; nrows = %d; ncols = %d" %
192
+ (shx, REPR(sh.name), sh.nrows, sh.ncols))
193
+ if nrows and ncols:
194
+ # Beat the bounds
195
+ for rowx in xrange(nrows):
196
+ nc = sh.row_len(rowx)
197
+ if nc:
198
+ sh.row_types(rowx)[nc-1]
199
+ sh.row_values(rowx)[nc-1]
200
+ sh.cell(rowx, nc-1)
201
+ for rowx in xrange(anshow-1):
202
+ if not printit and rowx % 10000 == 1 and rowx > 1:
203
+ print("done %d rows" % (rowx-1,))
204
+ show_row(bk, sh, rowx, colrange, printit)
205
+ if anshow and nrows:
206
+ show_row(bk, sh, nrows-1, colrange, printit)
207
+ print()
208
+ if bk.on_demand: bk.unload_sheet(shx)
209
+
210
+ def count_xfs(bk):
211
+ bk_header(bk)
212
+ for shx in range(bk.nsheets):
213
+ sh = bk.sheet_by_index(shx)
214
+ nrows = sh.nrows
215
+ print("sheet %d: name = %r; nrows = %d; ncols = %d" %
216
+ (shx, sh.name, sh.nrows, sh.ncols))
217
+ # Access all xfindexes to force gathering stats
218
+ type_stats = [0, 0, 0, 0, 0, 0, 0]
219
+ for rowx in xrange(nrows):
220
+ for colx in xrange(sh.row_len(rowx)):
221
+ xfx = sh.cell_xf_index(rowx, colx)
222
+ assert xfx >= 0
223
+ cty = sh.cell_type(rowx, colx)
224
+ type_stats[cty] += 1
225
+ print("XF stats", sh._xf_index_stats)
226
+ print("type stats", type_stats)
227
+ print()
228
+ if bk.on_demand: bk.unload_sheet(shx)
229
+
230
+ def main(cmd_args):
231
+ import optparse
232
+ global options
233
+ usage = "\n%prog [options] command [input-file-patterns]\n" + cmd_doc
234
+ oparser = optparse.OptionParser(usage)
235
+ oparser.add_option(
236
+ "-l", "--logfilename",
237
+ default="",
238
+ help="contains error messages")
239
+ oparser.add_option(
240
+ "-v", "--verbosity",
241
+ type="int", default=0,
242
+ help="level of information and diagnostics provided")
243
+ oparser.add_option(
244
+ "-m", "--mmap",
245
+ type="int", default=-1,
246
+ help="1: use mmap; 0: don't use mmap; -1: accept heuristic")
247
+ oparser.add_option(
248
+ "-e", "--encoding",
249
+ default="",
250
+ help="encoding override")
251
+ oparser.add_option(
252
+ "-f", "--formatting",
253
+ type="int", default=0,
254
+ help="0 (default): no fmt info\n"
255
+ "1: fmt info (all cells)\n",
256
+ )
257
+ oparser.add_option(
258
+ "-g", "--gc",
259
+ type="int", default=0,
260
+ help="0: auto gc enabled; 1: auto gc disabled, manual collect after each file; 2: no gc")
261
+ oparser.add_option(
262
+ "-s", "--onesheet",
263
+ default="",
264
+ help="restrict output to this sheet (name or index)")
265
+ oparser.add_option(
266
+ "-u", "--unnumbered",
267
+ action="store_true", default=0,
268
+ help="omit line numbers or offsets in biff_dump")
269
+ oparser.add_option(
270
+ "-d", "--on-demand",
271
+ action="store_true", default=0,
272
+ help="load sheets on demand instead of all at once")
273
+ oparser.add_option(
274
+ "-t", "--suppress-timing",
275
+ action="store_true", default=0,
276
+ help="don't print timings (diffs are less messy)")
277
+ oparser.add_option(
278
+ "-r", "--ragged-rows",
279
+ action="store_true", default=0,
280
+ help="open_workbook(..., ragged_rows=True)")
281
+ options, args = oparser.parse_args(cmd_args)
282
+ if len(args) == 1 and args[0] in ("version", ):
283
+ pass
284
+ elif len(args) < 2:
285
+ oparser.error("Expected at least 2 args, found %d" % len(args))
286
+ cmd = args[0]
287
+ xlrd_version = getattr(xlrd, "__VERSION__", "unknown; before 0.5")
288
+ if cmd == 'biff_dump':
289
+ xlrd.dump(args[1], unnumbered=options.unnumbered)
290
+ sys.exit(0)
291
+ if cmd == 'biff_count':
292
+ xlrd.count_records(args[1])
293
+ sys.exit(0)
294
+ if cmd == 'version':
295
+ print("xlrd: %s, from %s" % (xlrd_version, xlrd.__file__))
296
+ print("Python:", sys.version)
297
+ sys.exit(0)
298
+ if options.logfilename:
299
+ logfile = LogHandler(open(options.logfilename, 'w'))
300
+ else:
301
+ logfile = sys.stdout
302
+ mmap_opt = options.mmap
303
+ mmap_arg = xlrd.USE_MMAP
304
+ if mmap_opt in (1, 0):
305
+ mmap_arg = mmap_opt
306
+ elif mmap_opt != -1:
307
+ print('Unexpected value (%r) for mmap option -- assuming default' % mmap_opt)
308
+ fmt_opt = options.formatting | (cmd in ('xfc', ))
309
+ gc_mode = options.gc
310
+ if gc_mode:
311
+ gc.disable()
312
+ for pattern in args[1:]:
313
+ for fname in glob.glob(pattern):
314
+ print("\n=== File: %s ===" % fname)
315
+ if logfile != sys.stdout:
316
+ logfile.setfileheading("\n=== File: %s ===\n" % fname)
317
+ if gc_mode == 1:
318
+ n_unreachable = gc.collect()
319
+ if n_unreachable:
320
+ print("GC before open:", n_unreachable, "unreachable objects")
321
+ try:
322
+ t0 = time.time()
323
+ bk = xlrd.open_workbook(
324
+ fname,
325
+ verbosity=options.verbosity, logfile=logfile,
326
+ use_mmap=mmap_arg,
327
+ encoding_override=options.encoding,
328
+ formatting_info=fmt_opt,
329
+ on_demand=options.on_demand,
330
+ ragged_rows=options.ragged_rows,
331
+ )
332
+ t1 = time.time()
333
+ if not options.suppress_timing:
334
+ print("Open took %.2f seconds" % (t1-t0,))
335
+ except xlrd.XLRDError as e:
336
+ print("*** Open failed: %s: %s" % (type(e).__name__, e))
337
+ continue
338
+ except KeyboardInterrupt:
339
+ print("*** KeyboardInterrupt ***")
340
+ traceback.print_exc(file=sys.stdout)
341
+ sys.exit(1)
342
+ except BaseException as e:
343
+ print("*** Open failed: %s: %s" % (type(e).__name__, e))
344
+ traceback.print_exc(file=sys.stdout)
345
+ continue
346
+ t0 = time.time()
347
+ if cmd == 'hdr':
348
+ bk_header(bk)
349
+ elif cmd == 'ov': # OverView
350
+ show(bk, 0)
351
+ elif cmd == 'show': # all rows
352
+ show(bk)
353
+ elif cmd == '2rows': # first row and last row
354
+ show(bk, 2)
355
+ elif cmd == '3rows': # first row, 2nd row and last row
356
+ show(bk, 3)
357
+ elif cmd == 'bench':
358
+ show(bk, printit=0)
359
+ elif cmd == 'fonts':
360
+ bk_header(bk)
361
+ show_fonts(bk)
362
+ elif cmd == 'names': # named reference list
363
+ show_names(bk)
364
+ elif cmd == 'name_dump': # named reference list
365
+ show_names(bk, dump=1)
366
+ elif cmd == 'labels':
367
+ show_labels(bk)
368
+ elif cmd == 'xfc':
369
+ count_xfs(bk)
370
+ else:
371
+ print("*** Unknown command <%s>" % cmd)
372
+ sys.exit(1)
373
+ del bk
374
+ if gc_mode == 1:
375
+ n_unreachable = gc.collect()
376
+ if n_unreachable:
377
+ print("GC post cmd:", fname, "->", n_unreachable, "unreachable objects")
378
+ if not options.suppress_timing:
379
+ t1 = time.time()
380
+ print("\ncommand took %.2f seconds\n" % (t1-t0,))
381
+
382
+ return None
383
+
384
+ av = sys.argv[1:]
385
+ if not av:
386
+ main(av)
387
+ firstarg = av[0].lower()
388
+ if firstarg == "hotshot":
389
+ import hotshot
390
+ import hotshot.stats
391
+ av = av[1:]
392
+ prof_log_name = "XXXX.prof"
393
+ prof = hotshot.Profile(prof_log_name)
394
+ # benchtime, result = prof.runcall(main, *av)
395
+ result = prof.runcall(main, *(av, ))
396
+ print("result", repr(result))
397
+ prof.close()
398
+ stats = hotshot.stats.load(prof_log_name)
399
+ stats.strip_dirs()
400
+ stats.sort_stats('time', 'calls')
401
+ stats.print_stats(20)
402
+ elif firstarg == "profile":
403
+ import cProfile
404
+ av = av[1:]
405
+ cProfile.run('main(av)', 'YYYY.prof')
406
+ import pstats
407
+ p = pstats.Stats('YYYY.prof')
408
+ p.strip_dirs().sort_stats('cumulative').print_stats(30)
409
+ else:
410
+ main(av)
@@ -1,186 +0,0 @@
1
- # Changelog
2
-
3
- ## Current (in progress)
4
-
5
- - Refactor label testing [#119](https://github.com/datagouv/csv-detective/pull/119)
6
- - Refactor repo metadata and requirements [#120](https://github.com/datagouv/csv-detective/pull/120) [#122](https://github.com/datagouv/csv-detective/pull/122) [#135](https://github.com/datagouv/csv-detective/pull/135) [#136](https://github.com/datagouv/csv-detective/pull/136) [#138](https://github.com/datagouv/csv-detective/pull/138)
7
- - Better URL detection [#121](https://github.com/datagouv/csv-detective/pull/121)
8
- - For big files, analyse on sample then validate on whole file [#124](https://github.com/datagouv/csv-detective/pull/124) [#129](https://github.com/datagouv/csv-detective/pull/129)
9
- - Fix imports [#125](https://github.com/datagouv/csv-detective/pull/125) [#126](https://github.com/datagouv/csv-detective/pull/126) [#127](https://github.com/datagouv/csv-detective/pull/127) [#128](https://github.com/datagouv/csv-detective/pull/128)
10
- - Split aware and naive datetimes for hydra to cast them separately [#130](https://github.com/datagouv/csv-detective/pull/130)
11
- - Validate using the testing function, to consider PROPORTIONS [#131](https://github.com/datagouv/csv-detective/pull/131)
12
- - Remove `datetime_iso` format due to ambiguous cast in db (can be naive or aware) [#132](https://github.com/datagouv/csv-detective/pull/132)
13
- - Add `lonlat_wgs` format and handle optional brackets for `latlon_wgs` [#133](https://github.com/datagouv/csv-detective/pull/133)
14
- - Refactor format prioritizing [#134](https://github.com/datagouv/csv-detective/pull/134)
15
-
16
- ## 0.8.0 (2025-05-20)
17
-
18
- - New function that creates a csv from a list of fields and constraints, or from a TableSchema [#101](https://github.com/datagouv/csv-detective/pull/101)
19
- - Enable outputing loaded dataframe [#102](https://github.com/datagouv/csv-detective/pull/102)
20
- - Better naming, hint types and minor refactors [#103](https://github.com/datagouv/csv-detective/pull/103)
21
- - The returned dataframe has its columns properly cast to the detected types [#104](https://github.com/datagouv/csv-detective/pull/104)
22
- - Raise an error if the encoding could not be guessed [#106](https://github.com/datagouv/csv-detective/pull/106)
23
- - Fix CLI and minio routine [#107](https://github.com/datagouv/csv-detective/pull/107)
24
- - Allow to only specify tests to skip ("all but...") [#108](https://github.com/datagouv/csv-detective/pull/108)
25
- - Fix bool casting [#109](https://github.com/datagouv/csv-detective/pull/109)
26
- - Handle csv.gz files [#110](https://github.com/datagouv/csv-detective/pull/110)
27
- - Refactor file tests [#110](https://github.com/datagouv/csv-detective/pull/110)
28
- - Restructure repo (breaking changes) [#111](https://github.com/datagouv/csv-detective/pull/111)
29
- - Add validation function and associated flow [#112](https://github.com/datagouv/csv-detective/pull/112)
30
- - Better float detection [#113](https://github.com/datagouv/csv-detective/pull/113)
31
- - Refactor fields tests [#114](https://github.com/datagouv/csv-detective/pull/114)
32
- - Better code waldec and add code import [#116](https://github.com/datagouv/csv-detective/pull/116)
33
- - Better validation and refactors [#117](https://github.com/datagouv/csv-detective/pull/117)
34
- - Fix validation [#118](https://github.com/datagouv/csv-detective/pull/118)
35
-
36
- ## 0.7.4 (2024-11-15)
37
-
38
- - Enable calling main functions from base [#97](https://github.com/datagouv/csv-detective/pull/97)
39
- - Better detection of ints and floats [#94](https://github.com/datagouv/csv-detective/pull/94)
40
- - Better handle NaN values [#96](https://github.com/datagouv/csv-detective/pull/96)
41
- - Reshape exemple.py, clean up code and improve changelog [#98](https://github.com/datagouv/csv-detective/pull/98)
42
-
43
- ## 0.7.3 (2024-10-07)
44
-
45
- - Refactor tests import, now using folder arborescence instead of pre-made file [#93](https://github.com/datagouv/csv-detective/pull/93)
46
- - Fix inversion (count<=>value) in profile [#95](https://github.com/datagouv/csv-detective/pull/95)
47
-
48
- ## 0.7.2 (2024-08-27)
49
-
50
- - Outsource many formats to fr-format library [#87](https://github.com/datagouv/csv-detective/pull/87)
51
- - Better date detection [#89](https://github.com/datagouv/csv-detective/pull/89)
52
- - Update dependencies to make tests pass [#81](https://github.com/datagouv/csv-detective/pull/81)
53
- - Update readme [#81](https://github.com/datagouv/csv-detective/pull/81)
54
- - Hint type [#81](https://github.com/datagouv/csv-detective/pull/81)
55
- - Minor refactors [#81](https://github.com/datagouv/csv-detective/pull/81)
56
-
57
- ## 0.7.1 (2024-03-27)
58
-
59
- - Fixes after production release in hydra [#80](https://github.com/datagouv/csv-detective/pull/80)
60
-
61
- ## 0.7.0 (2024-03-21)
62
-
63
- - Handle other file formats: xls, xlsx, ods (and more) and analysis through URLs [#73](https://github.com/datagouv/csv-detective/pull/73)
64
- - Handle files with no extension (cc hydra) [#79](https://github.com/datagouv/csv-detective/pull/79)
65
-
66
- ## 0.6.8 (2024-01-18)
67
-
68
- - prevent exporting NaN values in profile [#72](https://github.com/datagouv/csv-detective/pull/72)
69
- - raise ValueError if analyzed file has various number of columns across first rows [#72](https://github.com/datagouv/csv-detective/pull/72)
70
-
71
- ## 0.6.7 (2024-01-15)
72
-
73
- - Add logs for columns that would take too much time within a specific test [#70](https://github.com/datagouv/csv-detective/pull/70)
74
- - Refactor some tests to improve performances and make detection more accurate [#69](https://github.com/datagouv/csv-detective/pull/69)
75
- - Try alternative ways to clean text [#71](https://github.com/datagouv/csv-detective/pull/71)
76
-
77
- ## 0.6.6 (2023-11-24)
78
-
79
- - Change setup.py to better convey dependencies [#67](https://github.com/datagouv/csv-detective/pull/67)
80
-
81
- ## 0.6.5 (2023-11-17)
82
-
83
- - Change encoding detection for faust-cchardet (forked from cchardet) [#66](https://github.com/etalab/csv-detective/pull/66)
84
-
85
- ## 0.6.4 (2023-10-18)
86
-
87
- - Better handling of ints and floats (now not accepting blanks and "+" in string) [#62](https://github.com/etalab/csv-detective/pull/62)
88
-
89
- ## 0.6.3 (2023-03-23)
90
-
91
- - Faster routine [#59](https://github.com/etalab/csv-detective/pull/59)
92
-
93
- ## 0.6.2 (2023-02-10)
94
-
95
- - Catch OverflowError for latitude and longitude checks [#58](https://github.com/etalab/csv-detective/pull/58)
96
-
97
- ## 0.6.0 (2023-02-10)
98
-
99
- - Add CI and upgrade dependencies [#49](https://github.com/etalab/csv-detective/pull/49)
100
- - Shuffle data before analysis [#56](https://github.com/etalab/csv-detective/pull/56)
101
- - Better discrimination between `code_departement` and `code_region` [#56](https://github.com/etalab/csv-detective/pull/56)
102
- - Add schema in output analysis [#57](https://github.com/etalab/csv-detective/pull/57)
103
-
104
- ## 0.4.7 [#51](https://github.com/etalab/csv-detective/pull/51)
105
-
106
- - Allow possibility to analyze entire file instead of a limited number of rows [#48](https://github.com/etalab/csv-detective/pull/48)
107
- - Better boolean detection [#42](https://github.com/etalab/csv-detective/issues/42)
108
- - Differentiate python types and format for `date` and `datetime` [#43](https://github.com/etalab/csv-detective/issues/43)
109
- - Better `code_departement` and `code_commune_insee` detection [#44](https://github.com/etalab/csv-detective/issues/44)
110
- - Fix header line (`header_row_idx`) detection [#44](https://github.com/etalab/csv-detective/issues/44)
111
- - Allow possibility to get profile of csv [#46](https://github.com/etalab/csv-detective/issues/46)
112
-
113
- ## 0.4.6 [#39](https://github.com/etalab/csv-detective/pull/39)
114
-
115
- - Fix tests
116
- - Prioritise lat / lon FR detection over more generic lat / lon.
117
- - To reduce false positives, prevent detection of the following if label detection is missing: `['code_departement', 'code_commune_insee', 'code_postal', 'latitude_wgs', 'longitude_wgs', 'latitude_wgs_fr_metropole', 'longitude_wgs_fr_metropole', 'latitude_l93', 'longitude_l93']`
118
- - Lower threshold of label detection so that if one relevant is detected in the label, it boosts the detection score.
119
- - Add ISO country alpha-3 and numeric detection
120
- - include camel case parsing in _process_text function
121
- - Support optional brackets in latlon format
122
-
123
- ## 0.4.5 [#29](https://github.com/etalab/csv-detective/pull/29)
124
-
125
- - Use `netloc` instead of `url` in location dict
126
-
127
- ## 0.4.4 [#24] (https://github.com/etalab/csv-detective/pull/28)
128
-
129
- - Prevent crash on empty CSVs
130
- - Add optional arguments encoding and sep to routine and routine_minio functions
131
- - Field detection improvements (code_csp_insee and datetime RFC 822)
132
- - Schema generation improvements with examples
133
-
134
-
135
- ## 0.4.3 [#24] (https://github.com/etalab/csv-detective/pull/24)
136
-
137
- - Add uuid and MongoID detection
138
- - Add new function dedicated to interaction with minio data
139
- - Add table schema automatic generation (only on minio data)
140
- - Modification of calculated score (consider label detection as a boost for score)
141
-
142
- ## 0.4.2 [#22] (https://github.com/etalab/csv-detective/pull/22)
143
-
144
- Add type detection by header name
145
-
146
- ## 0.4.1 [#19] (https://github.com/etalab/csv-detective/pull/19)
147
-
148
- Fix bug
149
- * num_rows was causing problem when it was fix to other value than default - Fixed
150
-
151
- ## 0.4.0 [#18] (https://github.com/etalab/csv_detective/pull/18)
152
-
153
- Add detailed output possibility
154
-
155
- Details :
156
- * two modes now for output report : "LIMITED" and "ALL"
157
- * "ALL" option give user information on found proportion for each column types and each columns
158
-
159
- ## 0.3.0 [#15] (https://github.com/etalab/csv_detective/pull/15)
160
-
161
- Fix bugs
162
-
163
- Details :
164
- * Facilitate ML Integration
165
- * Add column types detection
166
- * Fix documentation
167
-
168
- ## 0.2.1 - [#2](https://github.com/etalab/csv_detective/pull/2)
169
-
170
- Add continuous integration
171
-
172
- Details :
173
- * Add configuration for CircleCI
174
- * Add `CONTRIBUTING.md`
175
- * Push automatically new versions to PyPI
176
- * Use semantic versioning
177
-
178
- ## 0.2 - [#1](https://github.com/etalab/csv_detective/pull/1)
179
-
180
- Port from python2 to python3
181
-
182
- Details :
183
- * Add license AGPLv3
184
- * Update requirements
185
-
186
- ## 0.1