csv-detective 0.8.1.dev1674__py3-none-any.whl → 0.8.1.dev1720__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- csv_detective/__init__.py +0 -2
- csv_detective/cli.py +6 -9
- csv_detective/detect_fields/FR/geo/adresse/__init__.py +78 -78
- csv_detective/detect_fields/FR/geo/code_departement/__init__.py +2 -2
- csv_detective/detect_fields/FR/geo/code_postal/__init__.py +0 -1
- csv_detective/detect_fields/FR/geo/code_region/__init__.py +1 -1
- csv_detective/detect_fields/FR/geo/commune/__init__.py +2 -2
- csv_detective/detect_fields/FR/geo/departement/__init__.py +2 -2
- csv_detective/detect_fields/FR/geo/insee_canton/__init__.py +2 -2
- csv_detective/detect_fields/FR/geo/latitude_l93/__init__.py +1 -2
- csv_detective/detect_fields/FR/geo/latitude_wgs_fr_metropole/__init__.py +1 -1
- csv_detective/detect_fields/FR/geo/longitude_l93/__init__.py +1 -2
- csv_detective/detect_fields/FR/geo/longitude_wgs_fr_metropole/__init__.py +1 -1
- csv_detective/detect_fields/FR/geo/pays/__init__.py +6 -6
- csv_detective/detect_fields/FR/geo/region/__init__.py +6 -4
- csv_detective/detect_fields/FR/other/code_csp_insee/__init__.py +15 -14
- csv_detective/detect_fields/FR/other/csp_insee/__init__.py +4 -3
- csv_detective/detect_fields/FR/other/date_fr/__init__.py +3 -3
- csv_detective/detect_fields/FR/other/insee_ape700/__init__.py +4 -3
- csv_detective/detect_fields/FR/other/sexe/__init__.py +2 -2
- csv_detective/detect_fields/FR/other/siren/__init__.py +3 -3
- csv_detective/detect_fields/FR/other/siret/__init__.py +3 -3
- csv_detective/detect_fields/FR/other/tel_fr/__init__.py +3 -3
- csv_detective/detect_fields/FR/other/uai/__init__.py +2 -2
- csv_detective/detect_fields/FR/temp/jour_de_la_semaine/__init__.py +15 -15
- csv_detective/detect_fields/FR/temp/mois_de_annee/__init__.py +27 -27
- csv_detective/detect_fields/geo/iso_country_code_alpha2/__init__.py +5 -5
- csv_detective/detect_fields/geo/iso_country_code_alpha3/__init__.py +5 -5
- csv_detective/detect_fields/geo/iso_country_code_numeric/__init__.py +5 -5
- csv_detective/detect_fields/geo/latitude_wgs/__init__.py +1 -1
- csv_detective/detect_fields/geo/longitude_wgs/__init__.py +1 -1
- csv_detective/detect_fields/other/booleen/__init__.py +1 -1
- csv_detective/detect_fields/other/email/__init__.py +4 -2
- csv_detective/detect_fields/other/int/__init__.py +3 -3
- csv_detective/detect_fields/other/mongo_object_id/__init__.py +2 -2
- csv_detective/detect_fields/other/twitter/__init__.py +2 -2
- csv_detective/detect_fields/other/uuid/__init__.py +4 -5
- csv_detective/detect_fields/temp/date/__init__.py +3 -2
- csv_detective/detect_fields/temp/datetime_rfc822/__init__.py +6 -6
- csv_detective/detect_fields/temp/year/__init__.py +1 -1
- csv_detective/detect_labels/FR/other/tel_fr/__init__.py +0 -1
- csv_detective/detect_labels/geo/lonlat_wgs/__init__.py +1 -0
- csv_detective/detect_labels/other/mongo_object_id/__init__.py +1 -1
- csv_detective/detection/columns.py +9 -9
- csv_detective/detection/encoding.py +6 -4
- csv_detective/detection/engine.py +6 -5
- csv_detective/detection/formats.py +19 -19
- csv_detective/detection/headers.py +3 -5
- csv_detective/detection/rows.py +1 -1
- csv_detective/detection/variables.py +4 -4
- csv_detective/explore_csv.py +7 -8
- csv_detective/load_tests.py +6 -14
- csv_detective/output/__init__.py +3 -7
- csv_detective/output/dataframe.py +9 -5
- csv_detective/output/example.py +13 -13
- csv_detective/output/profile.py +30 -23
- csv_detective/output/schema.py +20 -23
- csv_detective/output/utils.py +15 -15
- csv_detective/parsing/columns.py +23 -12
- csv_detective/parsing/csv.py +1 -1
- csv_detective/parsing/excel.py +10 -11
- csv_detective/parsing/load.py +11 -8
- csv_detective/parsing/text.py +4 -9
- csv_detective/s3_utils.py +3 -7
- csv_detective/utils.py +4 -2
- csv_detective/validate.py +18 -13
- csv_detective-0.8.1.dev1674.data/data/share/csv_detective/README.md → csv_detective-0.8.1.dev1720.dist-info/METADATA +32 -0
- {csv_detective-0.8.1.dev1674.dist-info → csv_detective-0.8.1.dev1720.dist-info}/RECORD +81 -81
- {csv_detective-0.8.1.dev1674.dist-info → csv_detective-0.8.1.dev1720.dist-info}/top_level.txt +2 -0
- tests/test_example.py +2 -6
- tests/test_fields.py +16 -10
- tests/test_file.py +10 -9
- tests/test_labels.py +3 -2
- tests/test_structure.py +3 -1
- tests/test_validation.py +9 -6
- venv/bin/activate_this.py +38 -0
- venv/bin/jp.py +54 -0
- venv/bin/runxlrd.py +410 -0
- csv_detective-0.8.1.dev1674.data/data/share/csv_detective/CHANGELOG.md +0 -186
- csv_detective-0.8.1.dev1674.dist-info/METADATA +0 -268
- csv_detective-0.8.1.dev1674.dist-info/licenses/LICENSE +0 -21
- {csv_detective-0.8.1.dev1674.dist-info → csv_detective-0.8.1.dev1720.dist-info}/WHEEL +0 -0
- {csv_detective-0.8.1.dev1674.dist-info → csv_detective-0.8.1.dev1720.dist-info}/entry_points.txt +0 -0
- {csv_detective-0.8.1.dev1674.data/data/share/csv_detective → csv_detective-0.8.1.dev1720.dist-info/licenses}/LICENSE +0 -0
venv/bin/runxlrd.py
ADDED
|
@@ -0,0 +1,410 @@
|
|
|
1
|
+
#!/home/circleci/project/venv/bin/python
|
|
2
|
+
# Copyright (c) 2005-2012 Stephen John Machin, Lingfo Pty Ltd
|
|
3
|
+
# This script is part of the xlrd package, which is released under a
|
|
4
|
+
# BSD-style licence.
|
|
5
|
+
|
|
6
|
+
from __future__ import print_function
|
|
7
|
+
|
|
8
|
+
cmd_doc = """
|
|
9
|
+
Commands:
|
|
10
|
+
|
|
11
|
+
2rows Print the contents of first and last row in each sheet
|
|
12
|
+
3rows Print the contents of first, second and last row in each sheet
|
|
13
|
+
bench Same as "show", but doesn't print -- for profiling
|
|
14
|
+
biff_count[1] Print a count of each type of BIFF record in the file
|
|
15
|
+
biff_dump[1] Print a dump (char and hex) of the BIFF records in the file
|
|
16
|
+
fonts hdr + print a dump of all font objects
|
|
17
|
+
hdr Mini-overview of file (no per-sheet information)
|
|
18
|
+
hotshot Do a hotshot profile run e.g. ... -f1 hotshot bench bigfile*.xls
|
|
19
|
+
labels Dump of sheet.col_label_ranges and ...row... for each sheet
|
|
20
|
+
name_dump Dump of each object in book.name_obj_list
|
|
21
|
+
names Print brief information for each NAME record
|
|
22
|
+
ov Overview of file
|
|
23
|
+
profile Like "hotshot", but uses cProfile
|
|
24
|
+
show Print the contents of all rows in each sheet
|
|
25
|
+
version[0] Print versions of xlrd and Python and exit
|
|
26
|
+
xfc Print "XF counts" and cell-type counts -- see code for details
|
|
27
|
+
|
|
28
|
+
[0] means no file arg
|
|
29
|
+
[1] means only one file arg i.e. no glob.glob pattern
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
options = None
|
|
33
|
+
if __name__ == "__main__":
|
|
34
|
+
import xlrd
|
|
35
|
+
import sys
|
|
36
|
+
import time
|
|
37
|
+
import glob
|
|
38
|
+
import traceback
|
|
39
|
+
import gc
|
|
40
|
+
|
|
41
|
+
from xlrd.timemachine import xrange, REPR
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class LogHandler(object):
|
|
45
|
+
|
|
46
|
+
def __init__(self, logfileobj):
|
|
47
|
+
self.logfileobj = logfileobj
|
|
48
|
+
self.fileheading = None
|
|
49
|
+
self.shown = 0
|
|
50
|
+
|
|
51
|
+
def setfileheading(self, fileheading):
|
|
52
|
+
self.fileheading = fileheading
|
|
53
|
+
self.shown = 0
|
|
54
|
+
|
|
55
|
+
def write(self, text):
|
|
56
|
+
if self.fileheading and not self.shown:
|
|
57
|
+
self.logfileobj.write(self.fileheading)
|
|
58
|
+
self.shown = 1
|
|
59
|
+
self.logfileobj.write(text)
|
|
60
|
+
|
|
61
|
+
null_cell = xlrd.empty_cell
|
|
62
|
+
|
|
63
|
+
def show_row(bk, sh, rowx, colrange, printit):
|
|
64
|
+
if bk.ragged_rows:
|
|
65
|
+
colrange = range(sh.row_len(rowx))
|
|
66
|
+
if not colrange: return
|
|
67
|
+
if printit: print()
|
|
68
|
+
if bk.formatting_info:
|
|
69
|
+
for colx, ty, val, cxfx in get_row_data(bk, sh, rowx, colrange):
|
|
70
|
+
if printit:
|
|
71
|
+
print("cell %s%d: type=%d, data: %r, xfx: %s"
|
|
72
|
+
% (xlrd.colname(colx), rowx+1, ty, val, cxfx))
|
|
73
|
+
else:
|
|
74
|
+
for colx, ty, val, _unused in get_row_data(bk, sh, rowx, colrange):
|
|
75
|
+
if printit:
|
|
76
|
+
print("cell %s%d: type=%d, data: %r" % (xlrd.colname(colx), rowx+1, ty, val))
|
|
77
|
+
|
|
78
|
+
def get_row_data(bk, sh, rowx, colrange):
|
|
79
|
+
result = []
|
|
80
|
+
dmode = bk.datemode
|
|
81
|
+
ctys = sh.row_types(rowx)
|
|
82
|
+
cvals = sh.row_values(rowx)
|
|
83
|
+
for colx in colrange:
|
|
84
|
+
cty = ctys[colx]
|
|
85
|
+
cval = cvals[colx]
|
|
86
|
+
if bk.formatting_info:
|
|
87
|
+
cxfx = str(sh.cell_xf_index(rowx, colx))
|
|
88
|
+
else:
|
|
89
|
+
cxfx = ''
|
|
90
|
+
if cty == xlrd.XL_CELL_DATE:
|
|
91
|
+
try:
|
|
92
|
+
showval = xlrd.xldate_as_tuple(cval, dmode)
|
|
93
|
+
except xlrd.XLDateError as e:
|
|
94
|
+
showval = "%s:%s" % (type(e).__name__, e)
|
|
95
|
+
cty = xlrd.XL_CELL_ERROR
|
|
96
|
+
elif cty == xlrd.XL_CELL_ERROR:
|
|
97
|
+
showval = xlrd.error_text_from_code.get(cval, '<Unknown error code 0x%02x>' % cval)
|
|
98
|
+
else:
|
|
99
|
+
showval = cval
|
|
100
|
+
result.append((colx, cty, showval, cxfx))
|
|
101
|
+
return result
|
|
102
|
+
|
|
103
|
+
def bk_header(bk):
|
|
104
|
+
print()
|
|
105
|
+
print("BIFF version: %s; datemode: %s"
|
|
106
|
+
% (xlrd.biff_text_from_num[bk.biff_version], bk.datemode))
|
|
107
|
+
print("codepage: %r (encoding: %s); countries: %r"
|
|
108
|
+
% (bk.codepage, bk.encoding, bk.countries))
|
|
109
|
+
print("Last saved by: %r" % bk.user_name)
|
|
110
|
+
print("Number of data sheets: %d" % bk.nsheets)
|
|
111
|
+
print("Use mmap: %d; Formatting: %d; On demand: %d"
|
|
112
|
+
% (bk.use_mmap, bk.formatting_info, bk.on_demand))
|
|
113
|
+
print("Ragged rows: %d" % bk.ragged_rows)
|
|
114
|
+
if bk.formatting_info:
|
|
115
|
+
print("FORMATs: %d, FONTs: %d, XFs: %d"
|
|
116
|
+
% (len(bk.format_list), len(bk.font_list), len(bk.xf_list)))
|
|
117
|
+
if not options.suppress_timing:
|
|
118
|
+
print("Load time: %.2f seconds (stage 1) %.2f seconds (stage 2)"
|
|
119
|
+
% (bk.load_time_stage_1, bk.load_time_stage_2))
|
|
120
|
+
print()
|
|
121
|
+
|
|
122
|
+
def show_fonts(bk):
|
|
123
|
+
print("Fonts:")
|
|
124
|
+
for x in xrange(len(bk.font_list)):
|
|
125
|
+
font = bk.font_list[x]
|
|
126
|
+
font.dump(header='== Index %d ==' % x, indent=4)
|
|
127
|
+
|
|
128
|
+
def show_names(bk, dump=0):
|
|
129
|
+
bk_header(bk)
|
|
130
|
+
if bk.biff_version < 50:
|
|
131
|
+
print("Names not extracted in this BIFF version")
|
|
132
|
+
return
|
|
133
|
+
nlist = bk.name_obj_list
|
|
134
|
+
print("Name list: %d entries" % len(nlist))
|
|
135
|
+
for nobj in nlist:
|
|
136
|
+
if dump:
|
|
137
|
+
nobj.dump(sys.stdout,
|
|
138
|
+
header="\n=== Dump of name_obj_list[%d] ===" % nobj.name_index)
|
|
139
|
+
else:
|
|
140
|
+
print("[%d]\tName:%r macro:%r scope:%d\n\tresult:%r\n"
|
|
141
|
+
% (nobj.name_index, nobj.name, nobj.macro, nobj.scope, nobj.result))
|
|
142
|
+
|
|
143
|
+
def print_labels(sh, labs, title):
|
|
144
|
+
if not labs:return
|
|
145
|
+
for rlo, rhi, clo, chi in labs:
|
|
146
|
+
print("%s label range %s:%s contains:"
|
|
147
|
+
% (title, xlrd.cellname(rlo, clo), xlrd.cellname(rhi-1, chi-1)))
|
|
148
|
+
for rx in xrange(rlo, rhi):
|
|
149
|
+
for cx in xrange(clo, chi):
|
|
150
|
+
print(" %s: %r" % (xlrd.cellname(rx, cx), sh.cell_value(rx, cx)))
|
|
151
|
+
|
|
152
|
+
def show_labels(bk):
|
|
153
|
+
# bk_header(bk)
|
|
154
|
+
hdr = 0
|
|
155
|
+
for shx in range(bk.nsheets):
|
|
156
|
+
sh = bk.sheet_by_index(shx)
|
|
157
|
+
clabs = sh.col_label_ranges
|
|
158
|
+
rlabs = sh.row_label_ranges
|
|
159
|
+
if clabs or rlabs:
|
|
160
|
+
if not hdr:
|
|
161
|
+
bk_header(bk)
|
|
162
|
+
hdr = 1
|
|
163
|
+
print("sheet %d: name = %r; nrows = %d; ncols = %d" %
|
|
164
|
+
(shx, sh.name, sh.nrows, sh.ncols))
|
|
165
|
+
print_labels(sh, clabs, 'Col')
|
|
166
|
+
print_labels(sh, rlabs, 'Row')
|
|
167
|
+
if bk.on_demand: bk.unload_sheet(shx)
|
|
168
|
+
|
|
169
|
+
def show(bk, nshow=65535, printit=1):
|
|
170
|
+
bk_header(bk)
|
|
171
|
+
if 0:
|
|
172
|
+
rclist = xlrd.sheet.rc_stats.items()
|
|
173
|
+
rclist = sorted(rclist)
|
|
174
|
+
print("rc stats")
|
|
175
|
+
for k, v in rclist:
|
|
176
|
+
print("0x%04x %7d" % (k, v))
|
|
177
|
+
if options.onesheet:
|
|
178
|
+
try:
|
|
179
|
+
shx = int(options.onesheet)
|
|
180
|
+
except ValueError:
|
|
181
|
+
shx = bk.sheet_by_name(options.onesheet).number
|
|
182
|
+
shxrange = [shx]
|
|
183
|
+
else:
|
|
184
|
+
shxrange = range(bk.nsheets)
|
|
185
|
+
# print("shxrange", list(shxrange))
|
|
186
|
+
for shx in shxrange:
|
|
187
|
+
sh = bk.sheet_by_index(shx)
|
|
188
|
+
nrows, ncols = sh.nrows, sh.ncols
|
|
189
|
+
colrange = range(ncols)
|
|
190
|
+
anshow = min(nshow, nrows)
|
|
191
|
+
print("sheet %d: name = %s; nrows = %d; ncols = %d" %
|
|
192
|
+
(shx, REPR(sh.name), sh.nrows, sh.ncols))
|
|
193
|
+
if nrows and ncols:
|
|
194
|
+
# Beat the bounds
|
|
195
|
+
for rowx in xrange(nrows):
|
|
196
|
+
nc = sh.row_len(rowx)
|
|
197
|
+
if nc:
|
|
198
|
+
sh.row_types(rowx)[nc-1]
|
|
199
|
+
sh.row_values(rowx)[nc-1]
|
|
200
|
+
sh.cell(rowx, nc-1)
|
|
201
|
+
for rowx in xrange(anshow-1):
|
|
202
|
+
if not printit and rowx % 10000 == 1 and rowx > 1:
|
|
203
|
+
print("done %d rows" % (rowx-1,))
|
|
204
|
+
show_row(bk, sh, rowx, colrange, printit)
|
|
205
|
+
if anshow and nrows:
|
|
206
|
+
show_row(bk, sh, nrows-1, colrange, printit)
|
|
207
|
+
print()
|
|
208
|
+
if bk.on_demand: bk.unload_sheet(shx)
|
|
209
|
+
|
|
210
|
+
def count_xfs(bk):
|
|
211
|
+
bk_header(bk)
|
|
212
|
+
for shx in range(bk.nsheets):
|
|
213
|
+
sh = bk.sheet_by_index(shx)
|
|
214
|
+
nrows = sh.nrows
|
|
215
|
+
print("sheet %d: name = %r; nrows = %d; ncols = %d" %
|
|
216
|
+
(shx, sh.name, sh.nrows, sh.ncols))
|
|
217
|
+
# Access all xfindexes to force gathering stats
|
|
218
|
+
type_stats = [0, 0, 0, 0, 0, 0, 0]
|
|
219
|
+
for rowx in xrange(nrows):
|
|
220
|
+
for colx in xrange(sh.row_len(rowx)):
|
|
221
|
+
xfx = sh.cell_xf_index(rowx, colx)
|
|
222
|
+
assert xfx >= 0
|
|
223
|
+
cty = sh.cell_type(rowx, colx)
|
|
224
|
+
type_stats[cty] += 1
|
|
225
|
+
print("XF stats", sh._xf_index_stats)
|
|
226
|
+
print("type stats", type_stats)
|
|
227
|
+
print()
|
|
228
|
+
if bk.on_demand: bk.unload_sheet(shx)
|
|
229
|
+
|
|
230
|
+
def main(cmd_args):
|
|
231
|
+
import optparse
|
|
232
|
+
global options
|
|
233
|
+
usage = "\n%prog [options] command [input-file-patterns]\n" + cmd_doc
|
|
234
|
+
oparser = optparse.OptionParser(usage)
|
|
235
|
+
oparser.add_option(
|
|
236
|
+
"-l", "--logfilename",
|
|
237
|
+
default="",
|
|
238
|
+
help="contains error messages")
|
|
239
|
+
oparser.add_option(
|
|
240
|
+
"-v", "--verbosity",
|
|
241
|
+
type="int", default=0,
|
|
242
|
+
help="level of information and diagnostics provided")
|
|
243
|
+
oparser.add_option(
|
|
244
|
+
"-m", "--mmap",
|
|
245
|
+
type="int", default=-1,
|
|
246
|
+
help="1: use mmap; 0: don't use mmap; -1: accept heuristic")
|
|
247
|
+
oparser.add_option(
|
|
248
|
+
"-e", "--encoding",
|
|
249
|
+
default="",
|
|
250
|
+
help="encoding override")
|
|
251
|
+
oparser.add_option(
|
|
252
|
+
"-f", "--formatting",
|
|
253
|
+
type="int", default=0,
|
|
254
|
+
help="0 (default): no fmt info\n"
|
|
255
|
+
"1: fmt info (all cells)\n",
|
|
256
|
+
)
|
|
257
|
+
oparser.add_option(
|
|
258
|
+
"-g", "--gc",
|
|
259
|
+
type="int", default=0,
|
|
260
|
+
help="0: auto gc enabled; 1: auto gc disabled, manual collect after each file; 2: no gc")
|
|
261
|
+
oparser.add_option(
|
|
262
|
+
"-s", "--onesheet",
|
|
263
|
+
default="",
|
|
264
|
+
help="restrict output to this sheet (name or index)")
|
|
265
|
+
oparser.add_option(
|
|
266
|
+
"-u", "--unnumbered",
|
|
267
|
+
action="store_true", default=0,
|
|
268
|
+
help="omit line numbers or offsets in biff_dump")
|
|
269
|
+
oparser.add_option(
|
|
270
|
+
"-d", "--on-demand",
|
|
271
|
+
action="store_true", default=0,
|
|
272
|
+
help="load sheets on demand instead of all at once")
|
|
273
|
+
oparser.add_option(
|
|
274
|
+
"-t", "--suppress-timing",
|
|
275
|
+
action="store_true", default=0,
|
|
276
|
+
help="don't print timings (diffs are less messy)")
|
|
277
|
+
oparser.add_option(
|
|
278
|
+
"-r", "--ragged-rows",
|
|
279
|
+
action="store_true", default=0,
|
|
280
|
+
help="open_workbook(..., ragged_rows=True)")
|
|
281
|
+
options, args = oparser.parse_args(cmd_args)
|
|
282
|
+
if len(args) == 1 and args[0] in ("version", ):
|
|
283
|
+
pass
|
|
284
|
+
elif len(args) < 2:
|
|
285
|
+
oparser.error("Expected at least 2 args, found %d" % len(args))
|
|
286
|
+
cmd = args[0]
|
|
287
|
+
xlrd_version = getattr(xlrd, "__VERSION__", "unknown; before 0.5")
|
|
288
|
+
if cmd == 'biff_dump':
|
|
289
|
+
xlrd.dump(args[1], unnumbered=options.unnumbered)
|
|
290
|
+
sys.exit(0)
|
|
291
|
+
if cmd == 'biff_count':
|
|
292
|
+
xlrd.count_records(args[1])
|
|
293
|
+
sys.exit(0)
|
|
294
|
+
if cmd == 'version':
|
|
295
|
+
print("xlrd: %s, from %s" % (xlrd_version, xlrd.__file__))
|
|
296
|
+
print("Python:", sys.version)
|
|
297
|
+
sys.exit(0)
|
|
298
|
+
if options.logfilename:
|
|
299
|
+
logfile = LogHandler(open(options.logfilename, 'w'))
|
|
300
|
+
else:
|
|
301
|
+
logfile = sys.stdout
|
|
302
|
+
mmap_opt = options.mmap
|
|
303
|
+
mmap_arg = xlrd.USE_MMAP
|
|
304
|
+
if mmap_opt in (1, 0):
|
|
305
|
+
mmap_arg = mmap_opt
|
|
306
|
+
elif mmap_opt != -1:
|
|
307
|
+
print('Unexpected value (%r) for mmap option -- assuming default' % mmap_opt)
|
|
308
|
+
fmt_opt = options.formatting | (cmd in ('xfc', ))
|
|
309
|
+
gc_mode = options.gc
|
|
310
|
+
if gc_mode:
|
|
311
|
+
gc.disable()
|
|
312
|
+
for pattern in args[1:]:
|
|
313
|
+
for fname in glob.glob(pattern):
|
|
314
|
+
print("\n=== File: %s ===" % fname)
|
|
315
|
+
if logfile != sys.stdout:
|
|
316
|
+
logfile.setfileheading("\n=== File: %s ===\n" % fname)
|
|
317
|
+
if gc_mode == 1:
|
|
318
|
+
n_unreachable = gc.collect()
|
|
319
|
+
if n_unreachable:
|
|
320
|
+
print("GC before open:", n_unreachable, "unreachable objects")
|
|
321
|
+
try:
|
|
322
|
+
t0 = time.time()
|
|
323
|
+
bk = xlrd.open_workbook(
|
|
324
|
+
fname,
|
|
325
|
+
verbosity=options.verbosity, logfile=logfile,
|
|
326
|
+
use_mmap=mmap_arg,
|
|
327
|
+
encoding_override=options.encoding,
|
|
328
|
+
formatting_info=fmt_opt,
|
|
329
|
+
on_demand=options.on_demand,
|
|
330
|
+
ragged_rows=options.ragged_rows,
|
|
331
|
+
)
|
|
332
|
+
t1 = time.time()
|
|
333
|
+
if not options.suppress_timing:
|
|
334
|
+
print("Open took %.2f seconds" % (t1-t0,))
|
|
335
|
+
except xlrd.XLRDError as e:
|
|
336
|
+
print("*** Open failed: %s: %s" % (type(e).__name__, e))
|
|
337
|
+
continue
|
|
338
|
+
except KeyboardInterrupt:
|
|
339
|
+
print("*** KeyboardInterrupt ***")
|
|
340
|
+
traceback.print_exc(file=sys.stdout)
|
|
341
|
+
sys.exit(1)
|
|
342
|
+
except BaseException as e:
|
|
343
|
+
print("*** Open failed: %s: %s" % (type(e).__name__, e))
|
|
344
|
+
traceback.print_exc(file=sys.stdout)
|
|
345
|
+
continue
|
|
346
|
+
t0 = time.time()
|
|
347
|
+
if cmd == 'hdr':
|
|
348
|
+
bk_header(bk)
|
|
349
|
+
elif cmd == 'ov': # OverView
|
|
350
|
+
show(bk, 0)
|
|
351
|
+
elif cmd == 'show': # all rows
|
|
352
|
+
show(bk)
|
|
353
|
+
elif cmd == '2rows': # first row and last row
|
|
354
|
+
show(bk, 2)
|
|
355
|
+
elif cmd == '3rows': # first row, 2nd row and last row
|
|
356
|
+
show(bk, 3)
|
|
357
|
+
elif cmd == 'bench':
|
|
358
|
+
show(bk, printit=0)
|
|
359
|
+
elif cmd == 'fonts':
|
|
360
|
+
bk_header(bk)
|
|
361
|
+
show_fonts(bk)
|
|
362
|
+
elif cmd == 'names': # named reference list
|
|
363
|
+
show_names(bk)
|
|
364
|
+
elif cmd == 'name_dump': # named reference list
|
|
365
|
+
show_names(bk, dump=1)
|
|
366
|
+
elif cmd == 'labels':
|
|
367
|
+
show_labels(bk)
|
|
368
|
+
elif cmd == 'xfc':
|
|
369
|
+
count_xfs(bk)
|
|
370
|
+
else:
|
|
371
|
+
print("*** Unknown command <%s>" % cmd)
|
|
372
|
+
sys.exit(1)
|
|
373
|
+
del bk
|
|
374
|
+
if gc_mode == 1:
|
|
375
|
+
n_unreachable = gc.collect()
|
|
376
|
+
if n_unreachable:
|
|
377
|
+
print("GC post cmd:", fname, "->", n_unreachable, "unreachable objects")
|
|
378
|
+
if not options.suppress_timing:
|
|
379
|
+
t1 = time.time()
|
|
380
|
+
print("\ncommand took %.2f seconds\n" % (t1-t0,))
|
|
381
|
+
|
|
382
|
+
return None
|
|
383
|
+
|
|
384
|
+
av = sys.argv[1:]
|
|
385
|
+
if not av:
|
|
386
|
+
main(av)
|
|
387
|
+
firstarg = av[0].lower()
|
|
388
|
+
if firstarg == "hotshot":
|
|
389
|
+
import hotshot
|
|
390
|
+
import hotshot.stats
|
|
391
|
+
av = av[1:]
|
|
392
|
+
prof_log_name = "XXXX.prof"
|
|
393
|
+
prof = hotshot.Profile(prof_log_name)
|
|
394
|
+
# benchtime, result = prof.runcall(main, *av)
|
|
395
|
+
result = prof.runcall(main, *(av, ))
|
|
396
|
+
print("result", repr(result))
|
|
397
|
+
prof.close()
|
|
398
|
+
stats = hotshot.stats.load(prof_log_name)
|
|
399
|
+
stats.strip_dirs()
|
|
400
|
+
stats.sort_stats('time', 'calls')
|
|
401
|
+
stats.print_stats(20)
|
|
402
|
+
elif firstarg == "profile":
|
|
403
|
+
import cProfile
|
|
404
|
+
av = av[1:]
|
|
405
|
+
cProfile.run('main(av)', 'YYYY.prof')
|
|
406
|
+
import pstats
|
|
407
|
+
p = pstats.Stats('YYYY.prof')
|
|
408
|
+
p.strip_dirs().sort_stats('cumulative').print_stats(30)
|
|
409
|
+
else:
|
|
410
|
+
main(av)
|
|
@@ -1,186 +0,0 @@
|
|
|
1
|
-
# Changelog
|
|
2
|
-
|
|
3
|
-
## Current (in progress)
|
|
4
|
-
|
|
5
|
-
- Refactor label testing [#119](https://github.com/datagouv/csv-detective/pull/119)
|
|
6
|
-
- Refactor repo metadata and requirements [#120](https://github.com/datagouv/csv-detective/pull/120) [#122](https://github.com/datagouv/csv-detective/pull/122) [#135](https://github.com/datagouv/csv-detective/pull/135) [#136](https://github.com/datagouv/csv-detective/pull/136) [#138](https://github.com/datagouv/csv-detective/pull/138)
|
|
7
|
-
- Better URL detection [#121](https://github.com/datagouv/csv-detective/pull/121)
|
|
8
|
-
- For big files, analyse on sample then validate on whole file [#124](https://github.com/datagouv/csv-detective/pull/124) [#129](https://github.com/datagouv/csv-detective/pull/129)
|
|
9
|
-
- Fix imports [#125](https://github.com/datagouv/csv-detective/pull/125) [#126](https://github.com/datagouv/csv-detective/pull/126) [#127](https://github.com/datagouv/csv-detective/pull/127) [#128](https://github.com/datagouv/csv-detective/pull/128)
|
|
10
|
-
- Split aware and naive datetimes for hydra to cast them separately [#130](https://github.com/datagouv/csv-detective/pull/130)
|
|
11
|
-
- Validate using the testing function, to consider PROPORTIONS [#131](https://github.com/datagouv/csv-detective/pull/131)
|
|
12
|
-
- Remove `datetime_iso` format due to ambiguous cast in db (can be naive or aware) [#132](https://github.com/datagouv/csv-detective/pull/132)
|
|
13
|
-
- Add `lonlat_wgs` format and handle optional brackets for `latlon_wgs` [#133](https://github.com/datagouv/csv-detective/pull/133)
|
|
14
|
-
- Refactor format prioritizing [#134](https://github.com/datagouv/csv-detective/pull/134)
|
|
15
|
-
|
|
16
|
-
## 0.8.0 (2025-05-20)
|
|
17
|
-
|
|
18
|
-
- New function that creates a csv from a list of fields and constraints, or from a TableSchema [#101](https://github.com/datagouv/csv-detective/pull/101)
|
|
19
|
-
- Enable outputing loaded dataframe [#102](https://github.com/datagouv/csv-detective/pull/102)
|
|
20
|
-
- Better naming, hint types and minor refactors [#103](https://github.com/datagouv/csv-detective/pull/103)
|
|
21
|
-
- The returned dataframe has its columns properly cast to the detected types [#104](https://github.com/datagouv/csv-detective/pull/104)
|
|
22
|
-
- Raise an error if the encoding could not be guessed [#106](https://github.com/datagouv/csv-detective/pull/106)
|
|
23
|
-
- Fix CLI and minio routine [#107](https://github.com/datagouv/csv-detective/pull/107)
|
|
24
|
-
- Allow to only specify tests to skip ("all but...") [#108](https://github.com/datagouv/csv-detective/pull/108)
|
|
25
|
-
- Fix bool casting [#109](https://github.com/datagouv/csv-detective/pull/109)
|
|
26
|
-
- Handle csv.gz files [#110](https://github.com/datagouv/csv-detective/pull/110)
|
|
27
|
-
- Refactor file tests [#110](https://github.com/datagouv/csv-detective/pull/110)
|
|
28
|
-
- Restructure repo (breaking changes) [#111](https://github.com/datagouv/csv-detective/pull/111)
|
|
29
|
-
- Add validation function and associated flow [#112](https://github.com/datagouv/csv-detective/pull/112)
|
|
30
|
-
- Better float detection [#113](https://github.com/datagouv/csv-detective/pull/113)
|
|
31
|
-
- Refactor fields tests [#114](https://github.com/datagouv/csv-detective/pull/114)
|
|
32
|
-
- Better code waldec and add code import [#116](https://github.com/datagouv/csv-detective/pull/116)
|
|
33
|
-
- Better validation and refactors [#117](https://github.com/datagouv/csv-detective/pull/117)
|
|
34
|
-
- Fix validation [#118](https://github.com/datagouv/csv-detective/pull/118)
|
|
35
|
-
|
|
36
|
-
## 0.7.4 (2024-11-15)
|
|
37
|
-
|
|
38
|
-
- Enable calling main functions from base [#97](https://github.com/datagouv/csv-detective/pull/97)
|
|
39
|
-
- Better detection of ints and floats [#94](https://github.com/datagouv/csv-detective/pull/94)
|
|
40
|
-
- Better handle NaN values [#96](https://github.com/datagouv/csv-detective/pull/96)
|
|
41
|
-
- Reshape exemple.py, clean up code and improve changelog [#98](https://github.com/datagouv/csv-detective/pull/98)
|
|
42
|
-
|
|
43
|
-
## 0.7.3 (2024-10-07)
|
|
44
|
-
|
|
45
|
-
- Refactor tests import, now using folder arborescence instead of pre-made file [#93](https://github.com/datagouv/csv-detective/pull/93)
|
|
46
|
-
- Fix inversion (count<=>value) in profile [#95](https://github.com/datagouv/csv-detective/pull/95)
|
|
47
|
-
|
|
48
|
-
## 0.7.2 (2024-08-27)
|
|
49
|
-
|
|
50
|
-
- Outsource many formats to fr-format library [#87](https://github.com/datagouv/csv-detective/pull/87)
|
|
51
|
-
- Better date detection [#89](https://github.com/datagouv/csv-detective/pull/89)
|
|
52
|
-
- Update dependencies to make tests pass [#81](https://github.com/datagouv/csv-detective/pull/81)
|
|
53
|
-
- Update readme [#81](https://github.com/datagouv/csv-detective/pull/81)
|
|
54
|
-
- Hint type [#81](https://github.com/datagouv/csv-detective/pull/81)
|
|
55
|
-
- Minor refactors [#81](https://github.com/datagouv/csv-detective/pull/81)
|
|
56
|
-
|
|
57
|
-
## 0.7.1 (2024-03-27)
|
|
58
|
-
|
|
59
|
-
- Fixes after production release in hydra [#80](https://github.com/datagouv/csv-detective/pull/80)
|
|
60
|
-
|
|
61
|
-
## 0.7.0 (2024-03-21)
|
|
62
|
-
|
|
63
|
-
- Handle other file formats: xls, xlsx, ods (and more) and analysis through URLs [#73](https://github.com/datagouv/csv-detective/pull/73)
|
|
64
|
-
- Handle files with no extension (cc hydra) [#79](https://github.com/datagouv/csv-detective/pull/79)
|
|
65
|
-
|
|
66
|
-
## 0.6.8 (2024-01-18)
|
|
67
|
-
|
|
68
|
-
- prevent exporting NaN values in profile [#72](https://github.com/datagouv/csv-detective/pull/72)
|
|
69
|
-
- raise ValueError if analyzed file has various number of columns across first rows [#72](https://github.com/datagouv/csv-detective/pull/72)
|
|
70
|
-
|
|
71
|
-
## 0.6.7 (2024-01-15)
|
|
72
|
-
|
|
73
|
-
- Add logs for columns that would take too much time within a specific test [#70](https://github.com/datagouv/csv-detective/pull/70)
|
|
74
|
-
- Refactor some tests to improve performances and make detection more accurate [#69](https://github.com/datagouv/csv-detective/pull/69)
|
|
75
|
-
- Try alternative ways to clean text [#71](https://github.com/datagouv/csv-detective/pull/71)
|
|
76
|
-
|
|
77
|
-
## 0.6.6 (2023-11-24)
|
|
78
|
-
|
|
79
|
-
- Change setup.py to better convey dependencies [#67](https://github.com/datagouv/csv-detective/pull/67)
|
|
80
|
-
|
|
81
|
-
## 0.6.5 (2023-11-17)
|
|
82
|
-
|
|
83
|
-
- Change encoding detection for faust-cchardet (forked from cchardet) [#66](https://github.com/etalab/csv-detective/pull/66)
|
|
84
|
-
|
|
85
|
-
## 0.6.4 (2023-10-18)
|
|
86
|
-
|
|
87
|
-
- Better handling of ints and floats (now not accepting blanks and "+" in string) [#62](https://github.com/etalab/csv-detective/pull/62)
|
|
88
|
-
|
|
89
|
-
## 0.6.3 (2023-03-23)
|
|
90
|
-
|
|
91
|
-
- Faster routine [#59](https://github.com/etalab/csv-detective/pull/59)
|
|
92
|
-
|
|
93
|
-
## 0.6.2 (2023-02-10)
|
|
94
|
-
|
|
95
|
-
- Catch OverflowError for latitude and longitude checks [#58](https://github.com/etalab/csv-detective/pull/58)
|
|
96
|
-
|
|
97
|
-
## 0.6.0 (2023-02-10)
|
|
98
|
-
|
|
99
|
-
- Add CI and upgrade dependencies [#49](https://github.com/etalab/csv-detective/pull/49)
|
|
100
|
-
- Shuffle data before analysis [#56](https://github.com/etalab/csv-detective/pull/56)
|
|
101
|
-
- Better discrimination between `code_departement` and `code_region` [#56](https://github.com/etalab/csv-detective/pull/56)
|
|
102
|
-
- Add schema in output analysis [#57](https://github.com/etalab/csv-detective/pull/57)
|
|
103
|
-
|
|
104
|
-
## 0.4.7 [#51](https://github.com/etalab/csv-detective/pull/51)
|
|
105
|
-
|
|
106
|
-
- Allow possibility to analyze entire file instead of a limited number of rows [#48](https://github.com/etalab/csv-detective/pull/48)
|
|
107
|
-
- Better boolean detection [#42](https://github.com/etalab/csv-detective/issues/42)
|
|
108
|
-
- Differentiate python types and format for `date` and `datetime` [#43](https://github.com/etalab/csv-detective/issues/43)
|
|
109
|
-
- Better `code_departement` and `code_commune_insee` detection [#44](https://github.com/etalab/csv-detective/issues/44)
|
|
110
|
-
- Fix header line (`header_row_idx`) detection [#44](https://github.com/etalab/csv-detective/issues/44)
|
|
111
|
-
- Allow possibility to get profile of csv [#46](https://github.com/etalab/csv-detective/issues/46)
|
|
112
|
-
|
|
113
|
-
## 0.4.6 [#39](https://github.com/etalab/csv-detective/pull/39)
|
|
114
|
-
|
|
115
|
-
- Fix tests
|
|
116
|
-
- Prioritise lat / lon FR detection over more generic lat / lon.
|
|
117
|
-
- To reduce false positives, prevent detection of the following if label detection is missing: `['code_departement', 'code_commune_insee', 'code_postal', 'latitude_wgs', 'longitude_wgs', 'latitude_wgs_fr_metropole', 'longitude_wgs_fr_metropole', 'latitude_l93', 'longitude_l93']`
|
|
118
|
-
- Lower threshold of label detection so that if one relevant is detected in the label, it boosts the detection score.
|
|
119
|
-
- Add ISO country alpha-3 and numeric detection
|
|
120
|
-
- include camel case parsing in _process_text function
|
|
121
|
-
- Support optional brackets in latlon format
|
|
122
|
-
|
|
123
|
-
## 0.4.5 [#29](https://github.com/etalab/csv-detective/pull/29)
|
|
124
|
-
|
|
125
|
-
- Use `netloc` instead of `url` in location dict
|
|
126
|
-
|
|
127
|
-
## 0.4.4 [#24] (https://github.com/etalab/csv-detective/pull/28)
|
|
128
|
-
|
|
129
|
-
- Prevent crash on empty CSVs
|
|
130
|
-
- Add optional arguments encoding and sep to routine and routine_minio functions
|
|
131
|
-
- Field detection improvements (code_csp_insee and datetime RFC 822)
|
|
132
|
-
- Schema generation improvements with examples
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
## 0.4.3 [#24] (https://github.com/etalab/csv-detective/pull/24)
|
|
136
|
-
|
|
137
|
-
- Add uuid and MongoID detection
|
|
138
|
-
- Add new function dedicated to interaction with minio data
|
|
139
|
-
- Add table schema automatic generation (only on minio data)
|
|
140
|
-
- Modification of calculated score (consider label detection as a boost for score)
|
|
141
|
-
|
|
142
|
-
## 0.4.2 [#22] (https://github.com/etalab/csv-detective/pull/22)
|
|
143
|
-
|
|
144
|
-
Add type detection by header name
|
|
145
|
-
|
|
146
|
-
## 0.4.1 [#19] (https://github.com/etalab/csv-detective/pull/19)
|
|
147
|
-
|
|
148
|
-
Fix bug
|
|
149
|
-
* num_rows was causing problem when it was fix to other value than default - Fixed
|
|
150
|
-
|
|
151
|
-
## 0.4.0 [#18] (https://github.com/etalab/csv_detective/pull/18)
|
|
152
|
-
|
|
153
|
-
Add detailed output possibility
|
|
154
|
-
|
|
155
|
-
Details :
|
|
156
|
-
* two modes now for output report : "LIMITED" and "ALL"
|
|
157
|
-
* "ALL" option give user information on found proportion for each column types and each columns
|
|
158
|
-
|
|
159
|
-
## 0.3.0 [#15] (https://github.com/etalab/csv_detective/pull/15)
|
|
160
|
-
|
|
161
|
-
Fix bugs
|
|
162
|
-
|
|
163
|
-
Details :
|
|
164
|
-
* Facilitate ML Integration
|
|
165
|
-
* Add column types detection
|
|
166
|
-
* Fix documentation
|
|
167
|
-
|
|
168
|
-
## 0.2.1 - [#2](https://github.com/etalab/csv_detective/pull/2)
|
|
169
|
-
|
|
170
|
-
Add continuous integration
|
|
171
|
-
|
|
172
|
-
Details :
|
|
173
|
-
* Add configuration for CircleCI
|
|
174
|
-
* Add `CONTRIBUTING.md`
|
|
175
|
-
* Push automatically new versions to PyPI
|
|
176
|
-
* Use semantic versioning
|
|
177
|
-
|
|
178
|
-
## 0.2 - [#1](https://github.com/etalab/csv_detective/pull/1)
|
|
179
|
-
|
|
180
|
-
Port from python2 to python3
|
|
181
|
-
|
|
182
|
-
Details :
|
|
183
|
-
* Add license AGPLv3
|
|
184
|
-
* Update requirements
|
|
185
|
-
|
|
186
|
-
## 0.1
|