csv-detective 0.8.1.dev1617__py3-none-any.whl → 0.8.1.dev1703__py3-none-any.whl

csv_detective/__init__.py

@@ -1,4 +1,2 @@
 from csv_detective.explore_csv import routine, routine_minio, validate_then_detect  # noqa
 from csv_detective.output.example import create_example_csv_file  # noqa
-
-__version__ = '0.8.1.dev'

csv_detective-0.8.1.dev1617.data/data/share/csv_detective/README.md → csv_detective-0.8.1.dev1703.dist-info/METADATA

@@ -1,3 +1,34 @@
+Metadata-Version: 2.4
+Name: csv-detective
+Version: 0.8.1.dev1703
+Summary: Detect tabular files column content
+Author-email: Etalab <opendatateam@data.gouv.fr>
+License: MIT
+Project-URL: Source, https://github.com/datagouv/csv_detective
+Keywords: CSV,data processing,encoding,guess,parser,tabular
+Requires-Python: <3.14,>=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: boto3<2,>=1.34.0
+Requires-Dist: dateparser<2,>=1.2.0
+Requires-Dist: faust-cchardet==2.1.19
+Requires-Dist: pandas<3,>=2.2.0
+Requires-Dist: python-dateutil<3,>=2.8.2
+Requires-Dist: Unidecode<2,>=1.3.6
+Requires-Dist: openpyxl==3.1.5
+Requires-Dist: xlrd==2.0.1
+Requires-Dist: odfpy==1.4.1
+Requires-Dist: requests<3,>=2.32.3
+Requires-Dist: python-magic==0.4.27
+Requires-Dist: frformat==0.4.0
+Requires-Dist: Faker>=33.0.0
+Requires-Dist: rstr==3.2.2
+Provides-Extra: dev
+Requires-Dist: pytest>=8.3.0; extra == "dev"
+Requires-Dist: responses>=0.25.0; extra == "dev"
+Requires-Dist: bumpx>=0.3.10; extra == "dev"
+Dynamic: license-file
+
 # CSV Detective
 
 This is a package to **automatically detect column content in tabular files**. The script reads either the whole file or the first few rows and performs various checks to see for each column if it matches with various content types. This is currently done through regex and string comparison.

{csv_detective-0.8.1.dev1617.dist-info → csv_detective-0.8.1.dev1703.dist-info}/RECORD

@@ -1,4 +1,4 @@
-csv_detective/__init__.py,sha256=TwRP1gozmEmweSbK-lqihSsb-EqmCFSKUnJXz2x-dHE,191
+csv_detective/__init__.py,sha256=XY7pnoNHlocvyUiK8EQpJYPSQt5BRWWJD8KiPlvI9pU,164
 csv_detective/cli.py,sha256=VNztFz2nc90E3zkghF8PYtXTEZ6TrBSCQMi9v1ljkJs,1414
 csv_detective/explore_csv.py,sha256=VEeAJaz3FPOmGmQ-Yuf3FuSRRPULM03FrTf3qwZX52s,9222
 csv_detective/load_tests.py,sha256=GILvfkd4OVI-72mA4nzbPlZqgcXZ4wznOhGfZ1ucWkM,2385
@@ -150,10 +150,7 @@ csv_detective/parsing/csv.py,sha256=11mibDnJhIjykXLGZvA5ZEU5U7KgxIrbyO6BNv6jlro,
 csv_detective/parsing/excel.py,sha256=AslE2S1e67o8yTIAIhp-lAnJ6-XqeBBRz1-VMFqhZBM,7055
 csv_detective/parsing/load.py,sha256=u6fbGFZsL2GwPQRzhAXgt32JpUur7vbQdErREHxNJ-w,3661
 csv_detective/parsing/text.py,sha256=_TprGi0gHZlRsafizI3dqQhBehZW4BazqxmypMcAZ-o,1824
-csv_detective-0.8.1.dev1617.data/data/share/csv_detective/CHANGELOG.md,sha256=RT2mP1INh2SePA7bPsfu2U4ruoxAwQ5CVjJZbpN9uoU,9764
-csv_detective-0.8.1.dev1617.data/data/share/csv_detective/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
-csv_detective-0.8.1.dev1617.data/data/share/csv_detective/README.md,sha256=gKLFmC8kuCCywS9eAhMak_JNriUWWNOsBKleAu5TIEY,8501
-csv_detective-0.8.1.dev1617.dist-info/licenses/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
+csv_detective-0.8.1.dev1703.dist-info/licenses/LICENSE,sha256=A1dQrzxyxRHRih02KwibWj1khQyF7GeA6SqdOU87Gk4,1088
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_example.py,sha256=JeHxSK0IVDcSrOhSZlNGSQv4JAc_r6mzvJM8PfmLTMw,2018
 tests/test_fields.py,sha256=IwMpjOn8W5kDCvJYp3Cer4m571qomzjupOAvSRFMg_Q,11819
@@ -161,8 +158,11 @@ tests/test_file.py,sha256=0bHV9wx9mSRoav_DVF19g694yohb1p0bw7rtcBeKG-8,8451
 tests/test_labels.py,sha256=Nkr645bUewrj8hjNDKr67FQ6Sy_TID6f3E5Kfkl231M,464
 tests/test_structure.py,sha256=bv-tjgXohvQAxwmxzH0BynFpK2TyPjcxvtIAmIRlZmA,1393
 tests/test_validation.py,sha256=CTGonR6htxcWF9WH8MxumDD8cF45Y-G4hm94SM4lFjU,3246
-csv_detective-0.8.1.dev1617.dist-info/METADATA,sha256=4SAhn9DUS3Wz3RzNLJRxeICYW7prXrGWqMJNB92vu6I,10443
-csv_detective-0.8.1.dev1617.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-csv_detective-0.8.1.dev1617.dist-info/entry_points.txt,sha256=JjweTReFqKJmuvkegzlew2j3D5pZzfxvbEGOtGVGmaY,56
-csv_detective-0.8.1.dev1617.dist-info/top_level.txt,sha256=M0Nv646VHo-49zWjPkwo2C48UmtfddV8_9mEZeIxy8Q,20
-csv_detective-0.8.1.dev1617.dist-info/RECORD,,
+venv/bin/activate_this.py,sha256=NRy3waFmwW1pOaNUp33wNN0vD1Kzkd-zXX-Sgl4EiVI,1286
+venv/bin/jp.py,sha256=7z7dvRg0M7HzpZG4ssQID7nScjvQx7bcYTxJWDOrS6E,1717
+venv/bin/runxlrd.py,sha256=YlZMuycM_V_hzNt2yt3FyXPuwouMCmMhvj1oZaBeeuw,16092
+csv_detective-0.8.1.dev1703.dist-info/METADATA,sha256=HsL5tsoa92LIZSGCAth5zhUCRd-ovvKqHQSO2CaSrIo,9527
+csv_detective-0.8.1.dev1703.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+csv_detective-0.8.1.dev1703.dist-info/entry_points.txt,sha256=JjweTReFqKJmuvkegzlew2j3D5pZzfxvbEGOtGVGmaY,56
+csv_detective-0.8.1.dev1703.dist-info/top_level.txt,sha256=cYKb4Ok3XgYA7rMDOYtxysjSJp_iUA9lJjynhVzue8g,30
+csv_detective-0.8.1.dev1703.dist-info/RECORD,,

{csv_detective-0.8.1.dev1617.dist-info → csv_detective-0.8.1.dev1703.dist-info}/top_level.txt

@@ -1,2 +1,4 @@
 csv_detective
+dist
 tests
+venv

venv/bin/activate_this.py

@@ -0,0 +1,38 @@
+"""
+Activate virtualenv for current interpreter:
+
+import runpy
+runpy.run_path(this_file)
+
+This can be used when you must use an existing Python interpreter, not the virtualenv bin/python.
+"""  # noqa: D415
+
+from __future__ import annotations
+
+import os
+import site
+import sys
+
+try:
+    abs_file = os.path.abspath(__file__)
+except NameError as exc:
+    msg = "You must use import runpy; runpy.run_path(this_file)"
+    raise AssertionError(msg) from exc
+
+bin_dir = os.path.dirname(abs_file)
+base = bin_dir[: -len('bin') - 1]  # strip away the bin part from the __file__, plus the path separator
+
+# prepend bin to PATH (this file is inside the bin directory)
+os.environ["PATH"] = os.pathsep.join([bin_dir, *os.environ.get("PATH", "").split(os.pathsep)])
+os.environ["VIRTUAL_ENV"] = base  # virtual env is right above bin directory
+os.environ["VIRTUAL_ENV_PROMPT"] = '' or os.path.basename(base)
+
+# add the virtual environments libraries to the host python import mechanism
+prev_length = len(sys.path)
+for lib in '../lib/python3.9/site-packages'.split(os.pathsep):
+    path = os.path.realpath(os.path.join(bin_dir, lib))
+    site.addsitedir(path.decode("utf-8") if '' else path)
+sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length]
+
+sys.real_prefix = sys.prefix
+sys.prefix = base

venv/bin/jp.py

@@ -0,0 +1,54 @@
+#!/home/circleci/project/venv/bin/python
+
+import sys
+import json
+import argparse
+from pprint import pformat
+
+import jmespath
+from jmespath import exceptions
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('expression')
+    parser.add_argument('-f', '--filename',
+                        help=('The filename containing the input data.  '
+                              'If a filename is not given then data is '
+                              'read from stdin.'))
+    parser.add_argument('--ast', action='store_true',
+                        help=('Pretty print the AST, do not search the data.'))
+    args = parser.parse_args()
+    expression = args.expression
+    if args.ast:
+        # Only print the AST
+        expression = jmespath.compile(args.expression)
+        sys.stdout.write(pformat(expression.parsed))
+        sys.stdout.write('\n')
+        return 0
+    if args.filename:
+        with open(args.filename, 'r') as f:
+            data = json.load(f)
+    else:
+        data = sys.stdin.read()
+        data = json.loads(data)
+    try:
+        sys.stdout.write(json.dumps(
+            jmespath.search(expression, data), indent=4, ensure_ascii=False))
+        sys.stdout.write('\n')
+    except exceptions.ArityError as e:
+        sys.stderr.write("invalid-arity: %s\n" % e)
+        return 1
+    except exceptions.JMESPathTypeError as e:
+        sys.stderr.write("invalid-type: %s\n" % e)
+        return 1
+    except exceptions.UnknownFunctionError as e:
+        sys.stderr.write("unknown-function: %s\n" % e)
+        return 1
+    except exceptions.ParseError as e:
+        sys.stderr.write("syntax-error: %s\n" % e)
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())

venv/bin/runxlrd.py

@@ -0,0 +1,410 @@
+#!/home/circleci/project/venv/bin/python
+# Copyright (c) 2005-2012 Stephen John Machin, Lingfo Pty Ltd
+# This script is part of the xlrd package, which is released under a
+# BSD-style licence.
+
+from __future__ import print_function
+
+cmd_doc = """
+Commands:
+
+2rows           Print the contents of first and last row in each sheet
+3rows           Print the contents of first, second and last row in each sheet
+bench           Same as "show", but doesn't print -- for profiling
+biff_count[1]   Print a count of each type of BIFF record in the file
+biff_dump[1]    Print a dump (char and hex) of the BIFF records in the file
+fonts           hdr + print a dump of all font objects
+hdr             Mini-overview of file (no per-sheet information)
+hotshot         Do a hotshot profile run e.g. ... -f1 hotshot bench bigfile*.xls
+labels          Dump of sheet.col_label_ranges and ...row... for each sheet
+name_dump       Dump of each object in book.name_obj_list
+names           Print brief information for each NAME record
+ov              Overview of file
+profile         Like "hotshot", but uses cProfile
+show            Print the contents of all rows in each sheet
+version[0]      Print versions of xlrd and Python and exit
+xfc             Print "XF counts" and cell-type counts -- see code for details
+
+[0] means no file arg
+[1] means only one file arg i.e. no glob.glob pattern
+"""
+
+options = None
+if __name__ == "__main__":
+    import xlrd
+    import sys
+    import time
+    import glob
+    import traceback
+    import gc
+
+    from xlrd.timemachine import xrange, REPR
+
+
+    class LogHandler(object):
+
+        def __init__(self, logfileobj):
+            self.logfileobj = logfileobj
+            self.fileheading = None
+            self.shown = 0
+
+        def setfileheading(self, fileheading):
+            self.fileheading = fileheading
+            self.shown = 0
+
+        def write(self, text):
+            if self.fileheading and not self.shown:
+                self.logfileobj.write(self.fileheading)
+                self.shown = 1
+            self.logfileobj.write(text)
+
+    null_cell = xlrd.empty_cell
+
+    def show_row(bk, sh, rowx, colrange, printit):
+        if bk.ragged_rows:
+            colrange = range(sh.row_len(rowx))
+        if not colrange: return
+        if printit: print()
+        if bk.formatting_info:
+            for colx, ty, val, cxfx in get_row_data(bk, sh, rowx, colrange):
+                if printit:
+                    print("cell %s%d: type=%d, data: %r, xfx: %s"
+                        % (xlrd.colname(colx), rowx+1, ty, val, cxfx))
+        else:
+            for colx, ty, val, _unused in get_row_data(bk, sh, rowx, colrange):
+                if printit:
+                    print("cell %s%d: type=%d, data: %r" % (xlrd.colname(colx), rowx+1, ty, val))
+
+    def get_row_data(bk, sh, rowx, colrange):
+        result = []
+        dmode = bk.datemode
+        ctys = sh.row_types(rowx)
+        cvals = sh.row_values(rowx)
+        for colx in colrange:
+            cty = ctys[colx]
+            cval = cvals[colx]
+            if bk.formatting_info:
+                cxfx = str(sh.cell_xf_index(rowx, colx))
+            else:
+                cxfx = ''
+            if cty == xlrd.XL_CELL_DATE:
+                try:
+                    showval = xlrd.xldate_as_tuple(cval, dmode)
+                except xlrd.XLDateError as e:
+                    showval = "%s:%s" % (type(e).__name__, e)
+                    cty = xlrd.XL_CELL_ERROR
+            elif cty == xlrd.XL_CELL_ERROR:
+                showval = xlrd.error_text_from_code.get(cval, '<Unknown error code 0x%02x>' % cval)
+            else:
+                showval = cval
+            result.append((colx, cty, showval, cxfx))
+        return result
+
+    def bk_header(bk):
+        print()
+        print("BIFF version: %s; datemode: %s"
+            % (xlrd.biff_text_from_num[bk.biff_version], bk.datemode))
+        print("codepage: %r (encoding: %s); countries: %r"
+            % (bk.codepage, bk.encoding, bk.countries))
+        print("Last saved by: %r" % bk.user_name)
+        print("Number of data sheets: %d" % bk.nsheets)
+        print("Use mmap: %d; Formatting: %d; On demand: %d"
+            % (bk.use_mmap, bk.formatting_info, bk.on_demand))
+        print("Ragged rows: %d" % bk.ragged_rows)
+        if bk.formatting_info:
+            print("FORMATs: %d, FONTs: %d, XFs: %d"
+                % (len(bk.format_list), len(bk.font_list), len(bk.xf_list)))
+        if not options.suppress_timing:
+            print("Load time: %.2f seconds (stage 1) %.2f seconds (stage 2)"
+                % (bk.load_time_stage_1, bk.load_time_stage_2))
+        print()
+
+    def show_fonts(bk):
+        print("Fonts:")
+        for x in xrange(len(bk.font_list)):
+            font = bk.font_list[x]
+            font.dump(header='== Index %d ==' % x, indent=4)
+
+    def show_names(bk, dump=0):
+        bk_header(bk)
+        if bk.biff_version < 50:
+            print("Names not extracted in this BIFF version")
+            return
+        nlist = bk.name_obj_list
+        print("Name list: %d entries" % len(nlist))
+        for nobj in nlist:
+            if dump:
+                nobj.dump(sys.stdout,
+                    header="\n=== Dump of name_obj_list[%d] ===" % nobj.name_index)
+            else:
+                print("[%d]\tName:%r macro:%r scope:%d\n\tresult:%r\n"
+                    % (nobj.name_index, nobj.name, nobj.macro, nobj.scope, nobj.result))
+
+    def print_labels(sh, labs, title):
+        if not labs:return
+        for rlo, rhi, clo, chi in labs:
+            print("%s label range %s:%s contains:"
+                % (title, xlrd.cellname(rlo, clo), xlrd.cellname(rhi-1, chi-1)))
+            for rx in xrange(rlo, rhi):
+                for cx in xrange(clo, chi):
+                    print("    %s: %r" % (xlrd.cellname(rx, cx), sh.cell_value(rx, cx)))
+
+    def show_labels(bk):
+        # bk_header(bk)
+        hdr = 0
+        for shx in range(bk.nsheets):
+            sh = bk.sheet_by_index(shx)
+            clabs = sh.col_label_ranges
+            rlabs = sh.row_label_ranges
+            if clabs or rlabs:
+                if not hdr:
+                    bk_header(bk)
+                    hdr = 1
+                print("sheet %d: name = %r; nrows = %d; ncols = %d" %
+                    (shx, sh.name, sh.nrows, sh.ncols))
+                print_labels(sh, clabs, 'Col')
+                print_labels(sh, rlabs, 'Row')
+            if bk.on_demand: bk.unload_sheet(shx)
+
+    def show(bk, nshow=65535, printit=1):
+        bk_header(bk)
+        if 0:
+            rclist = xlrd.sheet.rc_stats.items()
+            rclist = sorted(rclist)
+            print("rc stats")
+            for k, v in rclist:
+                print("0x%04x %7d" % (k, v))
+        if options.onesheet:
+            try:
+                shx = int(options.onesheet)
+            except ValueError:
+                shx = bk.sheet_by_name(options.onesheet).number
+            shxrange = [shx]
+        else:
+            shxrange = range(bk.nsheets)
+        # print("shxrange", list(shxrange))
+        for shx in shxrange:
+            sh = bk.sheet_by_index(shx)
+            nrows, ncols = sh.nrows, sh.ncols
+            colrange = range(ncols)
+            anshow = min(nshow, nrows)
+            print("sheet %d: name = %s; nrows = %d; ncols = %d" %
+                (shx, REPR(sh.name), sh.nrows, sh.ncols))
+            if nrows and ncols:
+                # Beat the bounds
+                for rowx in xrange(nrows):
+                    nc = sh.row_len(rowx)
+                    if nc:
+                        sh.row_types(rowx)[nc-1]
+                        sh.row_values(rowx)[nc-1]
+                        sh.cell(rowx, nc-1)
+            for rowx in xrange(anshow-1):
+                if not printit and rowx % 10000 == 1 and rowx > 1:
+                    print("done %d rows" % (rowx-1,))
+                show_row(bk, sh, rowx, colrange, printit)
+            if anshow and nrows:
+                show_row(bk, sh, nrows-1, colrange, printit)
+            print()
+            if bk.on_demand: bk.unload_sheet(shx)
+
+    def count_xfs(bk):
+        bk_header(bk)
+        for shx in range(bk.nsheets):
+            sh = bk.sheet_by_index(shx)
+            nrows = sh.nrows
+            print("sheet %d: name = %r; nrows = %d; ncols = %d" %
+                (shx, sh.name, sh.nrows, sh.ncols))
+            # Access all xfindexes to force gathering stats
+            type_stats = [0, 0, 0, 0, 0, 0, 0]
+            for rowx in xrange(nrows):
+                for colx in xrange(sh.row_len(rowx)):
+                    xfx = sh.cell_xf_index(rowx, colx)
+                    assert xfx >= 0
+                    cty = sh.cell_type(rowx, colx)
+                    type_stats[cty] += 1
+            print("XF stats", sh._xf_index_stats)
+            print("type stats", type_stats)
+            print()
+            if bk.on_demand: bk.unload_sheet(shx)
+
+    def main(cmd_args):
+        import optparse
+        global options
+        usage = "\n%prog [options] command [input-file-patterns]\n" + cmd_doc
+        oparser = optparse.OptionParser(usage)
+        oparser.add_option(
+            "-l", "--logfilename",
+            default="",
+            help="contains error messages")
+        oparser.add_option(
+            "-v", "--verbosity",
+            type="int", default=0,
+            help="level of information and diagnostics provided")
+        oparser.add_option(
+            "-m", "--mmap",
+            type="int", default=-1,
+            help="1: use mmap; 0: don't use mmap; -1: accept heuristic")
+        oparser.add_option(
+            "-e", "--encoding",
+            default="",
+            help="encoding override")
+        oparser.add_option(
+            "-f", "--formatting",
+            type="int", default=0,
+            help="0 (default): no fmt info\n"
+                 "1: fmt info (all cells)\n",
+        )
+        oparser.add_option(
+            "-g", "--gc",
+            type="int", default=0,
+            help="0: auto gc enabled; 1: auto gc disabled, manual collect after each file; 2: no gc")
+        oparser.add_option(
+            "-s", "--onesheet",
+            default="",
+            help="restrict output to this sheet (name or index)")
+        oparser.add_option(
+            "-u", "--unnumbered",
+            action="store_true", default=0,
+            help="omit line numbers or offsets in biff_dump")
+        oparser.add_option(
+            "-d", "--on-demand",
+            action="store_true", default=0,
+            help="load sheets on demand instead of all at once")
+        oparser.add_option(
+            "-t", "--suppress-timing",
+            action="store_true", default=0,
+            help="don't print timings (diffs are less messy)")
+        oparser.add_option(
+            "-r", "--ragged-rows",
+            action="store_true", default=0,
+            help="open_workbook(..., ragged_rows=True)")
+        options, args = oparser.parse_args(cmd_args)
+        if len(args) == 1 and args[0] in ("version", ):
+            pass
+        elif len(args) < 2:
+            oparser.error("Expected at least 2 args, found %d" % len(args))
+        cmd = args[0]
+        xlrd_version = getattr(xlrd, "__VERSION__", "unknown; before 0.5")
+        if cmd == 'biff_dump':
+            xlrd.dump(args[1], unnumbered=options.unnumbered)
+            sys.exit(0)
+        if cmd == 'biff_count':
+            xlrd.count_records(args[1])
+            sys.exit(0)
+        if cmd == 'version':
+            print("xlrd: %s, from %s" % (xlrd_version, xlrd.__file__))
+            print("Python:", sys.version)
+            sys.exit(0)
+        if options.logfilename:
+            logfile = LogHandler(open(options.logfilename, 'w'))
+        else:
+            logfile = sys.stdout
+        mmap_opt = options.mmap
+        mmap_arg = xlrd.USE_MMAP
+        if mmap_opt in (1, 0):
+            mmap_arg = mmap_opt
+        elif mmap_opt != -1:
+            print('Unexpected value (%r) for mmap option -- assuming default' % mmap_opt)
+        fmt_opt = options.formatting | (cmd in ('xfc', ))
+        gc_mode = options.gc
+        if gc_mode:
+            gc.disable()
+        for pattern in args[1:]:
+            for fname in glob.glob(pattern):
+                print("\n=== File: %s ===" % fname)
+                if logfile != sys.stdout:
+                    logfile.setfileheading("\n=== File: %s ===\n" % fname)
+                if gc_mode == 1:
+                    n_unreachable = gc.collect()
+                    if n_unreachable:
+                        print("GC before open:", n_unreachable, "unreachable objects")
+                try:
+                    t0 = time.time()
+                    bk = xlrd.open_workbook(
+                        fname,
+                        verbosity=options.verbosity, logfile=logfile,
+                        use_mmap=mmap_arg,
+                        encoding_override=options.encoding,
+                        formatting_info=fmt_opt,
+                        on_demand=options.on_demand,
+                        ragged_rows=options.ragged_rows,
+                    )
+                    t1 = time.time()
+                    if not options.suppress_timing:
+                        print("Open took %.2f seconds" % (t1-t0,))
+                except xlrd.XLRDError as e:
+                    print("*** Open failed: %s: %s" % (type(e).__name__, e))
+                    continue
+                except KeyboardInterrupt:
+                    print("*** KeyboardInterrupt ***")
+                    traceback.print_exc(file=sys.stdout)
+                    sys.exit(1)
+                except BaseException as e:
+                    print("*** Open failed: %s: %s" % (type(e).__name__, e))
+                    traceback.print_exc(file=sys.stdout)
+                    continue
+                t0 = time.time()
+                if cmd == 'hdr':
+                    bk_header(bk)
+                elif cmd == 'ov': # OverView
+                    show(bk, 0)
+                elif cmd == 'show': # all rows
+                    show(bk)
+                elif cmd == '2rows': # first row and last row
+                    show(bk, 2)
+                elif cmd == '3rows': # first row, 2nd row and last row
+                    show(bk, 3)
+                elif cmd == 'bench':
+                    show(bk, printit=0)
+                elif cmd == 'fonts':
+                    bk_header(bk)
+                    show_fonts(bk)
+                elif cmd == 'names': # named reference list
+                    show_names(bk)
+                elif cmd == 'name_dump': # named reference list
+                    show_names(bk, dump=1)
+                elif cmd == 'labels':
+                    show_labels(bk)
+                elif cmd == 'xfc':
+                    count_xfs(bk)
+                else:
+                    print("*** Unknown command <%s>" % cmd)
+                    sys.exit(1)
+                del bk
+                if gc_mode == 1:
+                    n_unreachable = gc.collect()
+                    if n_unreachable:
+                        print("GC post cmd:", fname, "->", n_unreachable, "unreachable objects")
+                if not options.suppress_timing:
+                    t1 = time.time()
+                    print("\ncommand took %.2f seconds\n" % (t1-t0,))
+
+        return None
+
+    av = sys.argv[1:]
+    if not av:
+        main(av)
+    firstarg = av[0].lower()
+    if firstarg == "hotshot":
+        import hotshot
+        import hotshot.stats
+        av = av[1:]
+        prof_log_name = "XXXX.prof"
+        prof = hotshot.Profile(prof_log_name)
+        # benchtime, result = prof.runcall(main, *av)
+        result = prof.runcall(main, *(av, ))
+        print("result", repr(result))
+        prof.close()
+        stats = hotshot.stats.load(prof_log_name)
+        stats.strip_dirs()
+        stats.sort_stats('time', 'calls')
+        stats.print_stats(20)
+    elif firstarg == "profile":
+        import cProfile
+        av = av[1:]
+        cProfile.run('main(av)', 'YYYY.prof')
+        import pstats
+        p = pstats.Stats('YYYY.prof')
+        p.strip_dirs().sort_stats('cumulative').print_stats(30)
+    else:
+        main(av)

csv_detective-0.8.1.dev1617.data/data/share/csv_detective/CHANGELOG.md

@@ -1,186 +0,0 @@
-# Changelog
-
-## Current (in progress)
-
-- Refactor label testing [#119](https://github.com/datagouv/csv-detective/pull/119)
-- Refactor repo metadata and requirements [#120](https://github.com/datagouv/csv-detective/pull/120) [#122](https://github.com/datagouv/csv-detective/pull/122) [#135](https://github.com/datagouv/csv-detective/pull/135) [#136](https://github.com/datagouv/csv-detective/pull/136)
-- Better URL detection [#121](https://github.com/datagouv/csv-detective/pull/121)
-- For big files, analyse on sample then validate on whole file [#124](https://github.com/datagouv/csv-detective/pull/124) [#129](https://github.com/datagouv/csv-detective/pull/129)
-- Fix imports [#125](https://github.com/datagouv/csv-detective/pull/125) [#126](https://github.com/datagouv/csv-detective/pull/126) [#127](https://github.com/datagouv/csv-detective/pull/127) [#128](https://github.com/datagouv/csv-detective/pull/128)
-- Split aware and naive datetimes for hydra to cast them separately [#130](https://github.com/datagouv/csv-detective/pull/130)
-- Validate using the testing function, to consider PROPORTIONS [#131](https://github.com/datagouv/csv-detective/pull/131)
-- Remove `datetime_iso` format due to ambiguous cast in db (can be naive or aware) [#132](https://github.com/datagouv/csv-detective/pull/132)
-- Add `lonlat_wgs` format and handle optional brackets for `latlon_wgs` [#133](https://github.com/datagouv/csv-detective/pull/133)
-- Refactor format prioritizing [#134](https://github.com/datagouv/csv-detective/pull/134)
-
-## 0.8.0 (2025-05-20)
-
-- New function that creates a csv from a list of fields and constraints, or from a TableSchema [#101](https://github.com/datagouv/csv-detective/pull/101)
-- Enable outputing loaded dataframe [#102](https://github.com/datagouv/csv-detective/pull/102)
-- Better naming, hint types and minor refactors [#103](https://github.com/datagouv/csv-detective/pull/103)
-- The returned dataframe has its columns properly cast to the detected types [#104](https://github.com/datagouv/csv-detective/pull/104)
-- Raise an error if the encoding could not be guessed [#106](https://github.com/datagouv/csv-detective/pull/106)
-- Fix CLI and minio routine [#107](https://github.com/datagouv/csv-detective/pull/107)
-- Allow to only specify tests to skip ("all but...") [#108](https://github.com/datagouv/csv-detective/pull/108)
-- Fix bool casting [#109](https://github.com/datagouv/csv-detective/pull/109)
-- Handle csv.gz files [#110](https://github.com/datagouv/csv-detective/pull/110)
-- Refactor file tests [#110](https://github.com/datagouv/csv-detective/pull/110)
-- Restructure repo (breaking changes) [#111](https://github.com/datagouv/csv-detective/pull/111)
-- Add validation function and associated flow [#112](https://github.com/datagouv/csv-detective/pull/112)
-- Better float detection [#113](https://github.com/datagouv/csv-detective/pull/113)
-- Refactor fields tests [#114](https://github.com/datagouv/csv-detective/pull/114)
-- Better code waldec and add code import [#116](https://github.com/datagouv/csv-detective/pull/116)
-- Better validation and refactors [#117](https://github.com/datagouv/csv-detective/pull/117)
-- Fix validation [#118](https://github.com/datagouv/csv-detective/pull/118)
-
-## 0.7.4 (2024-11-15)
-
-- Enable calling main functions from base [#97](https://github.com/datagouv/csv-detective/pull/97)
-- Better detection of ints and floats [#94](https://github.com/datagouv/csv-detective/pull/94)
-- Better handle NaN values [#96](https://github.com/datagouv/csv-detective/pull/96)
-- Reshape exemple.py, clean up code and improve changelog [#98](https://github.com/datagouv/csv-detective/pull/98)
-
-## 0.7.3 (2024-10-07)
-
-- Refactor tests import, now using folder arborescence instead of pre-made file [#93](https://github.com/datagouv/csv-detective/pull/93)
-- Fix inversion (count<=>value) in profile [#95](https://github.com/datagouv/csv-detective/pull/95)
-
-## 0.7.2 (2024-08-27)
-
-- Outsource many formats to fr-format library [#87](https://github.com/datagouv/csv-detective/pull/87)
-- Better date detection [#89](https://github.com/datagouv/csv-detective/pull/89)
-- Update dependencies to make tests pass [#81](https://github.com/datagouv/csv-detective/pull/81)
-- Update readme [#81](https://github.com/datagouv/csv-detective/pull/81)
-- Hint type [#81](https://github.com/datagouv/csv-detective/pull/81)
-- Minor refactors [#81](https://github.com/datagouv/csv-detective/pull/81)
-
-## 0.7.1 (2024-03-27)
-
-- Fixes after production release in hydra [#80](https://github.com/datagouv/csv-detective/pull/80)
-
-## 0.7.0 (2024-03-21)
-
-- Handle other file formats: xls, xlsx, ods (and more) and analysis through URLs [#73](https://github.com/datagouv/csv-detective/pull/73)
-- Handle files with no extension (cc hydra) [#79](https://github.com/datagouv/csv-detective/pull/79)
-
-## 0.6.8 (2024-01-18)
-
-- prevent exporting NaN values in profile [#72](https://github.com/datagouv/csv-detective/pull/72)
-- raise ValueError if analyzed file has various number of columns across first rows [#72](https://github.com/datagouv/csv-detective/pull/72)
-
-## 0.6.7 (2024-01-15)
-
-- Add logs for columns that would take too much time within a specific test [#70](https://github.com/datagouv/csv-detective/pull/70)
-- Refactor some tests to improve performances and make detection more accurate [#69](https://github.com/datagouv/csv-detective/pull/69)
-- Try alternative ways to clean text [#71](https://github.com/datagouv/csv-detective/pull/71)
-
-## 0.6.6 (2023-11-24)
-
-- Change setup.py to better convey dependencies [#67](https://github.com/datagouv/csv-detective/pull/67)
-
-## 0.6.5 (2023-11-17)
-
-- Change encoding detection for faust-cchardet (forked from cchardet) [#66](https://github.com/etalab/csv-detective/pull/66)
-
-## 0.6.4 (2023-10-18)
-
-- Better handling of ints and floats (now not accepting blanks and "+" in string) [#62](https://github.com/etalab/csv-detective/pull/62)
-
-## 0.6.3 (2023-03-23)
-
-- Faster routine [#59](https://github.com/etalab/csv-detective/pull/59)
-
-## 0.6.2 (2023-02-10)
-
-- Catch OverflowError for latitude and longitude checks [#58](https://github.com/etalab/csv-detective/pull/58)
-
-## 0.6.0 (2023-02-10)
-
-- Add CI and upgrade dependencies [#49](https://github.com/etalab/csv-detective/pull/49)
-- Shuffle data before analysis [#56](https://github.com/etalab/csv-detective/pull/56)
-- Better discrimination between `code_departement` and `code_region` [#56](https://github.com/etalab/csv-detective/pull/56)
-- Add schema in output analysis [#57](https://github.com/etalab/csv-detective/pull/57)
-
-## 0.4.7 [#51](https://github.com/etalab/csv-detective/pull/51)
-
-- Allow possibility to analyze entire file instead of a limited number of rows [#48](https://github.com/etalab/csv-detective/pull/48)
-- Better boolean detection [#42](https://github.com/etalab/csv-detective/issues/42)
-- Differentiate python types and format for `date` and `datetime` [#43](https://github.com/etalab/csv-detective/issues/43)
-- Better `code_departement` and `code_commune_insee` detection [#44](https://github.com/etalab/csv-detective/issues/44)
-- Fix header line (`header_row_idx`) detection [#44](https://github.com/etalab/csv-detective/issues/44)
-- Allow possibility to get profile of csv [#46](https://github.com/etalab/csv-detective/issues/46)
-
-## 0.4.6 [#39](https://github.com/etalab/csv-detective/pull/39)
-
-- Fix tests
-- Prioritise lat / lon FR detection over more generic lat / lon.
-- To reduce false positives, prevent detection of the following if label detection is missing: `['code_departement', 'code_commune_insee', 'code_postal', 'latitude_wgs', 'longitude_wgs', 'latitude_wgs_fr_metropole', 'longitude_wgs_fr_metropole', 'latitude_l93', 'longitude_l93']`
-- Lower threshold of label detection so that if one relevant is detected in the label, it boosts the detection score.
-- Add ISO country alpha-3 and numeric detection
-- include camel case parsing in _process_text function
-- Support optional brackets in latlon format
-
-## 0.4.5 [#29](https://github.com/etalab/csv-detective/pull/29)
-
-- Use `netloc` instead of `url` in location dict
-
-## 0.4.4 [#24] (https://github.com/etalab/csv-detective/pull/28)
-
-- Prevent crash on empty CSVs
-- Add optional arguments encoding and sep to routine and routine_minio functions
-- Field detection improvements (code_csp_insee and datetime RFC 822)
-- Schema generation improvements with examples
-
-
-## 0.4.3 [#24] (https://github.com/etalab/csv-detective/pull/24)
-
-- Add uuid and MongoID detection
-- Add new function dedicated to interaction with minio data
-- Add table schema automatic generation (only on minio data)
-- Modification of calculated score (consider label detection as a boost for score)
-
-## 0.4.2 [#22] (https://github.com/etalab/csv-detective/pull/22)
-
-Add type detection by header name
-
-## 0.4.1 [#19] (https://github.com/etalab/csv-detective/pull/19)
-
-Fix bug
- * num_rows was causing problem when it was fix to other value than default - Fixed
-
-## 0.4.0 [#18] (https://github.com/etalab/csv_detective/pull/18)
-
-Add detailed output possibility
-
-Details :
- * two modes now for output report : "LIMITED" and "ALL"
- * "ALL" option give user information on found proportion for each column types and each columns
-
-## 0.3.0 [#15] (https://github.com/etalab/csv_detective/pull/15)
-
-Fix bugs
-
-Details :
- * Facilitate ML Integration
- * Add column types detection
- * Fix documentation
-
-## 0.2.1 - [#2](https://github.com/etalab/csv_detective/pull/2)
-
-Add continuous integration
-
-Details :
- * Add configuration for CircleCI
- * Add `CONTRIBUTING.md`
- * Push automatically new versions to PyPI
- * Use semantic versioning
-
-## 0.2 - [#1](https://github.com/etalab/csv_detective/pull/1)
-
-Port from python2 to python3
-
-Details :
- * Add license AGPLv3
- * Update requirements
-
-## 0.1

csv_detective-0.8.1.dev1617.dist-info/METADATA

@@ -1,267 +0,0 @@
-Metadata-Version: 2.4
-Name: csv_detective
-Version: 0.8.1.dev1617
-Summary: Detect tabular files column content
-Home-page: https://github.com/datagouv/csv_detective
-Author: Etalab
-Author-email: opendatateam@data.gouv.fr
-License: https://spdx.org/licenses/MIT.html#licenseText
-Project-URL: Source, https://github.com/datagouv/csv_detective
-Keywords: CSV data processing encoding guess parser tabular
-Classifier: Development Status :: 2 - Pre-Alpha
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Classifier: Programming Language :: Python :: Implementation :: CPython
-Classifier: Topic :: Scientific/Engineering :: Information Analysis
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: boto3<2,>=1.34.0
-Requires-Dist: dateparser<2,>=1.2.0
-Requires-Dist: faust-cchardet==2.1.19
-Requires-Dist: pandas<3,>=2.2.0
-Requires-Dist: python-dateutil<3,>=2.8.2
-Requires-Dist: Unidecode<2,>=1.3.6
-Requires-Dist: openpyxl==3.1.5
-Requires-Dist: xlrd==2.0.1
-Requires-Dist: odfpy==1.4.1
-Requires-Dist: requests<3,>=2.32.3
-Requires-Dist: python-magic==0.4.27
-Requires-Dist: frformat==0.4.0
-Requires-Dist: Faker>=33.0.0
-Requires-Dist: rstr==3.2.2
-Provides-Extra: dev
-Requires-Dist: pytest==8.3.0; extra == "dev"
-Requires-Dist: responses==0.25.0; extra == "dev"
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: keywords
-Dynamic: license
-Dynamic: license-file
-Dynamic: project-url
-Dynamic: provides-extra
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary
-
-# CSV Detective
-
-This is a package to **automatically detect column content in tabular files**. The script reads either the whole file or the first few rows and performs various checks to see for each column if it matches with various content types. This is currently done through regex and string comparison.
-
-Currently supported file types: csv, xls, xlsx, ods.
-
-You can also directly feed the URL of a remote file (from data.gouv.fr for instance).
-
-## How To ?
-
-### Install the package
-
-You need to have python >= 3.9 installed. We recommend using a virtual environement.
-
-```
-pip install csv-detective
-```
-
-### Detect some columns
-
-Say you have a tabular file located at `file_path`. This is how you could use `csv_detective`:
-
-```
-# Import the csv_detective package
-from csv_detective import routine
-import os # for this example only
-
-# Replace by your file path
-file_path = os.path.join('.', 'tests', 'code_postaux_v201410.csv')
-
-# Open your file and run csv_detective
-inspection_results = routine(
-  file_path, # or file URL
-  num_rows=-1, # Value -1 will analyze all lines of your file, you can change with the number of lines you wish to analyze
-  save_results=False, # Default False. If True, it will save result output into the same directory as the analyzed file, using the same name as your file and .json extension
-  output_profile=True, # Default False. If True, returned dict will contain a property "profile" indicating profile (min, max, mean, tops...) of every column of you csv
-  output_schema=True, # Default False. If True, returned dict will contain a property "schema" containing basic [tableschema](https://specs.frictionlessdata.io/table-schema/) of your file. This can be use to validate structure of other csv which should match same structure. 
-)
-```
-
-## So What Do You Get ?
-
-### Output
-
-The program creates a `Python` dictionnary with the following information :
-
-```
-{
-    "encoding": "windows-1252", 			        # Encoding detected
-    "separator": ";",						# Detected CSV separator
-    "header_row_idx": 0					# Index of the header (aka how many lines to skip to get it)
-    "headers": ['code commune INSEE', 'nom de la commune', 'code postal', "libellé d'acheminement"], # Header row
-    "total_lines": 42,					# Number of rows (excluding header)
-    "nb_duplicates": 0,					# Number of exact duplicates in rows
-    "heading_columns": 0,					# Number of heading columns
-    "trailing_columns": 0,					# Number of trailing columns
-    "categorical": ['Code commune']         # Columns that contain less than 25 different values (arbitrary threshold)
-    "columns": { # Property that conciliate detection from labels and content of a column
-        "Code commune": {
-            "python_type": "string",
-            "format": "code_commune_insee",
-            "score": 1.0
-        },
-    },
-    "columns_labels": { # Property that return detection from header columns
-        "Code commune": {
-            "python_type": "string",
-            "format": "code_commune_insee",
-            "score": 0.5
-        },
-    },
-    "columns_fields": { # Property that return detection from content columns
-        "Code commune": {
-            "python_type": "string",
-            "format": "code_commune_insee",
-            "score": 1.25
-        },
-    },
-    "profile": {
-      "column_name" : {
-        "min": 1, # only int and float
-        "max: 12, # only int and float
-        "mean": 5, # only int and float
-        "std": 5, # only int and float
-        "tops": [  # 10 most frequent values in the column
-          "xxx",
-          "yyy",
-          "..."
-        ],
-        "nb_distinct": 67, # number of distinct values
-        "nb_missing_values": 102 # number of empty cells in the column
-      }
-    },
-    "schema": { # TableSchema of the file if `output_schema` was set to `True`
-      "$schema": "https://frictionlessdata.io/schemas/table-schema.json",
-      "name": "",
-      "title": "",
-      "description": "",
-      "countryCode": "FR",
-      "homepage": "",
-      "path": "https://github.com/datagouv/csv-detective",
-      "resources": [],
-      "sources": [
-        {"title": "Spécification Tableschema", "path": "https://specs.frictionlessdata.io/table-schema"},
-        {"title": "schema.data.gouv.fr", "path": "https://schema.data.gouv.fr"}
-      ],
-      "created": "2023-02-10",
-      "lastModified": "2023-02-10",
-      "version": "0.0.1",
-      "contributors": [
-        {"title": "Table schema bot", "email": "schema@data.gouv.fr", "organisation": "data.gouv.fr", "role": "author"}
-      ],
-      "fields": [
-        {
-          "name": "Code commune",
-          "description": "Le code INSEE de la commune",
-          "example": "23150",
-          "type": "string",
-          "formatFR": "code_commune_insee",
-          "constraints": {
-            "required": False,
-            "pattern": "^([013-9]\\d|2[AB1-9])\\d{3}$",
-          }
-        }
-      ]
-    }
-}
-```
-
-The output slightly differs depending on the file format:
-- csv files have `encoding` and `separator`
-- xls, xls, ods files have `engine` and `sheet_name`
-
-### What Formats Can Be Detected
-
-Includes :
-
-- Communes, Départements, Régions, Pays
-- Codes Communes, Codes Postaux, Codes Departement, ISO Pays
-- Codes CSP, Description CSP, SIREN
-- E-Mails, URLs, Téléphones FR
-- Years, Dates, Jours de la Semaine FR
-- UUIDs, Mongo ObjectIds
-
-### Format detection and scoring
-For each column, 3 scores are computed for each format, the higher the score, the more likely the format:
-- the field score based on the values contained in the column (0.0 to 1.0).
-- the label score based on the header of the column (0.0 to 1.0).
-- the overall score, computed as `field_score * (1 + label_score/2)` (0.0 to 1.5).
-
-The overall score computation aims to give more weight to the column contents while
-still leveraging the column header.
-
-#### `limited_output` - Select the output mode you want for json report
-
-This option allows you to select the output mode you want to pass. To do so, you have to pass a `limited_output` argument to the `routine` function. This variable has two possible values:
-
-- `limited_output` defaults to `True` which means report will contain only detected column formats based on a pre-selected threshold proportion in data. Report result is the standard output (an example can be found above in 'Output' section).
-Only the format with highest score is present in the output.
-- `limited_output=False` means report will contain a full list of all column format possibilities for each input data columns with a value associated which match to the proportion of found column type in data. With this report, user can adjust its rules of detection based on a specific threshold and has a better vision of quality detection for each columns. Results could also be easily transformed into a dataframe (columns types in column / column names in rows) for analysis and test.
-
-## Improvement suggestions
-
-- Smarter refactors
-- Improve performances
-- Test other ways to load and process data (`pandas` alternatives)
-- Add more and more detection modules...
-
-Related ideas:
-
-- store column names to make a learning model based on column names for (possible pre-screen)
-- normalising data based on column prediction
-- entity resolution (good luck...)
-
-## Why Could This Be of Any Use ?
-
-Organisations such as [data.gouv.fr](http://data.gouv.fr) aggregate huge amounts of un-normalised data. Performing cross-examination across datasets can be difficult. This tool could help enrich the datasets metadata and facilitate linking them together.
-
-[`udata-hydra`](https://github.com/etalab/udata-hydra) is a crawler that checks, analyzes (using `csv-detective`) and APIfies all tabular files from [data.gouv.fr](http://data.gouv.fr).
-
-An early version of this analysis of all resources on data.gouv.fr can be found [here](https://github.com/Leobouloc/data.gouv-exploration).
-
-## Release
-
-The release process uses `bumpx`.
-
-```shell
-pip install -r requirements-build.txt
-```
-
-### Process
-
-1. `bumpx` will handle bumping the version according to your command (patch, minor, major)
-2. It will update the CHANGELOG according to the new version being published
-3. It will push a tag with the given version to github
-4. CircleCI will pickup this tag, build the package and publish it to pypi
-5. `bumpx` will have everything ready for the next version (version, changelog...)
-
-### Dry run
-
-```shell
-bumpx -d -v
-```
-
-### Release
-
-This will release a patch version:
-
-```shell
-bumpx -v
-```

csv_detective-0.8.1.dev1617.dist-info/licenses/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2025 data.gouv.fr
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.