undatum 1.0.17__py2.py3-none-any.whl

undatum/cmds/selector.py

@@ -0,0 +1,437 @@
+# -*- coding: utf8 -*-
+"""Data selection and filtering module."""
+import csv
+import logging
+import os
+import sys
+import zipfile
+
+import bson
+import dictquery as dq
+import duckdb
+import orjson
+from iterable.helpers.detect import detect_file_type, open_iterable
+
+from ..common.iterable import DataWriter, IterableData
+from ..constants import DUCKABLE_CODECS, DUCKABLE_FILE_TYPES
+from ..utils import (detect_encoding, dict_generator, get_dict_value,
+                     get_file_type, get_option, strip_dict_fields)
+
+LINEEND = '\n'.encode('utf8')
+
+
+ITERABLE_OPTIONS_KEYS = ['tagname', 'delimiter', 'encoding', 'start_line', 'page']
+
+
+def get_iterable_options(options):
+    """Extract iterable-specific options from options dictionary."""
+    out = {}
+    for k in ITERABLE_OPTIONS_KEYS:
+        if k in options.keys():
+            out[k] = options[k]
+    return out
+
+
+def _detect_engine(fromfile, engine, filetype):
+    """Detect the appropriate engine for processing."""
+    compression = 'raw'
+    if filetype is None:
+        ftype = detect_file_type(fromfile)
+        if ftype['success']:
+            filetype = ftype['datatype'].id()
+            if ftype['codec'] is not None:
+                compression = ftype['codec'].id()
+    logging.info(f'File filetype {filetype} and compression {compression}')
+    if engine == 'auto':
+        if filetype in DUCKABLE_FILE_TYPES and compression in DUCKABLE_CODECS:
+            return 'duckdb'
+        return 'iterable'
+    return engine
+
+
+def get_iterable_fields_uniq(iterable, fields, dolog=False, dq_instance=None):  # pylint: disable=unused-argument
+    """Returns all uniq values of the fields of iterable dictionary."""
+    # dq_instance kept for API compatibility
+    n = 0
+    uniqval = []
+    for row in iterable:
+        n += 1
+        if dolog and n % 1000 == 0:
+            logging.debug('uniq: processing %d records', n)
+        try:
+            allvals = []
+            for field in fields:
+                allvals.append(get_dict_value(row, field.split('.')))
+
+            for n1, _ in enumerate(allvals[0]):
+                k = []
+                for n2, _ in enumerate(allvals):
+                    k.append(str(allvals[n2][n1]))
+                if k not in uniqval:
+                    uniqval.append(k)
+        except KeyError:
+            pass
+    return uniqval
+
+
+def get_duckdb_fields_uniq(filename, fields, dolog=False, dq_instance=None):  # pylint: disable=unused-argument
+    """Returns all uniq values of the fields of the filename using DuckdDB."""
+    # dq_instance kept for API compatibility
+    uniqval = []
+    fieldstext = ','.join(fields)
+    query = (f"select unnest(grp) from (select distinct({fieldstext}) "
+             f"as grp from '{filename}')")
+    if dolog:
+        logging.info(query)
+    uniqval = duckdb.sql(query).fetchall()
+    return uniqval
+
+
+
+def get_iterable_fields_freq(iterable, fields, dolog=False, filter_expr=None, dq_instance=None):
+    """Iterates and returns most frequent values."""
+    n = 0
+    valuedict = {}
+    items = []
+    for r in iterable:
+        n += 1
+        if dolog and n % 10000 == 0:
+            logging.info('frequency: processing %d records', n)
+        if filter_expr is not None:
+            query_obj = dq_instance if dq_instance is not None else dq
+            if not query_obj.match(r, filter_expr):
+                continue
+        try:
+            allvals = []
+            for field in fields:
+                allvals.append(get_dict_value(r, field.split('.')))
+
+            for n1, _ in enumerate(allvals[0]):
+                k = []
+                for n2, _ in enumerate(allvals):
+                    k.append(str(allvals[n2][n1]))
+                kx = '\t'.join(k)
+                v = valuedict.get(kx, 0)
+                valuedict[kx] = v + 1
+        except KeyError:
+            pass
+    for k, v in valuedict.items():
+        row = k.split('\t')
+        row.append(v)
+        items.append(row)
+    items.sort(key=lambda x: x[-1], reverse=True)
+    return items
+
+def get_duckdb_fields_freq(filename, fields, dolog=False, dq_instance=None):  # pylint: disable=unused-argument
+    """Returns frequencies for the fields of the filename using DuckdDB."""
+    # dq_instance kept for API compatibility
+    uniqval = []
+    fieldstext = ','.join(fields)
+    query = (f"select {fieldstext}, count(*) as c from '{filename}' "
+             f"group by {fieldstext} order by c desc")
+    if dolog:
+        logging.info(query)
+    uniqval = duckdb.sql(query).fetchall()
+    return uniqval
+
+
+class Selector:
+    """Data selection and filtering handler."""
+    def __init__(self):
+        pass
+
+    def uniq(self, fromfile, options=None):
+        """Extracts unique values by field."""
+        if options is None:
+            options = {}
+        logging.debug('Processing %s', fromfile)
+        iterableargs = get_iterable_options(options)
+        filetype = get_option(options, 'filetype')
+        to_file = get_option(options, 'output')
+        engine = get_option(options, 'engine')
+        if to_file:
+            to_type = get_file_type(to_file)
+            if not to_file:
+                logging.debug('Output file type not supported')
+                return
+            out = open(to_file, 'w', encoding='utf8')
+        else:
+            to_type = 'csv'
+            out = sys.stdout
+        fields = options['fields'].split(',')
+        detected_engine = _detect_engine(fromfile, engine, filetype)
+        if detected_engine == 'duckdb':
+            output_type = 'duckdb'
+            uniqval = get_duckdb_fields_uniq(fromfile, fields, dolog=True)
+        elif detected_engine == 'iterable':
+            output_type = 'iterable'
+            iterable = open_iterable(fromfile, mode='r', iterableargs=iterableargs)
+            logging.info('uniq: looking for fields: %s' % (options['fields']))
+            uniqval = get_iterable_fields_uniq(iterable, fields, dolog=True)
+            iterable.close()
+        else:
+            logging.info('Engine not supported. Please choose duckdb or iterable')
+            return
+        logging.debug('%d unique values found' % (len(uniqval)))
+        writer = DataWriter(out, filetype=to_type, output_type=output_type, fieldnames=fields)
+        writer.write_items(uniqval)
+
+
+    def headers(self, fromfile, options=None):
+        """Extracts headers values."""
+        if options is None:
+            options = {}
+        limit = get_option(options, 'limit')
+        iterableargs = get_iterable_options(options)
+
+        iterable = open_iterable(fromfile, mode='r', iterableargs=iterableargs)
+        keys_set = set()  # Use set for O(1) lookup instead of O(n) list operations
+        n = 0
+        for item in iterable:
+            if limit and n > limit:
+                break
+            n += 1
+            dk = dict_generator(item)
+            for i in dk:
+                k = ".".join(i[:-1])
+                keys_set.add(k)
+        iterable.close()
+        keys = list(keys_set)  # Convert to list for backward compatibility
+        output = get_option(options, 'output')
+        if output:
+            with open(output, 'w', encoding=get_option(options, 'encoding')) as f:
+                f.write('\n'.join(keys))
+        else:
+            for x in keys:
+                print(x.encode('utf8').decode('utf8', 'ignore'))
+
+    def frequency(self, fromfile, options=None):
+        """Calculates frequency of the values in the file."""
+        if options is None:
+            options = {}
+        logging.debug('Processing %s', fromfile)
+        iterableargs = get_iterable_options(options)
+        filetype = get_option(options, 'filetype')
+        to_file = get_option(options, 'output')
+        engine = get_option(options, 'engine')
+        if to_file:
+            to_type = get_file_type(to_file)
+            if not to_file:
+                logging.debug('Output file type not supported')
+                return
+            out = open(to_file, 'w', encoding='utf8')
+        else:
+            to_type = 'csv'
+            out = sys.stdout
+        fields = options['fields'].split(',')
+        detected_engine = _detect_engine(fromfile, engine, filetype)
+        items = []
+        output_type = 'iterable'
+        if detected_engine == 'duckdb':
+            items = get_duckdb_fields_freq(fromfile, fields=fields, dolog=True)
+            output_type = 'duckdb'
+        elif detected_engine == 'iterable':
+            output_type = 'iterable'
+            iterable = open_iterable(fromfile, mode='r', iterableargs=iterableargs)
+            if iterable is not None:
+                items = get_iterable_fields_freq(iterable, fields, dolog=True)
+            else:
+                logging.info('File type not supported')
+                return
+        else:
+            logging.debug('Data processing engine is not set and not detected')
+            return
+        logging.debug('frequency: %d unique values found' % (len(items)))
+        fields.append('count')
+        writer = DataWriter(out, filetype=to_type, output_type=output_type, fieldnames=fields)
+        writer.write_items(items)
+
+    def select(self, fromfile, options=None):
+        """Select or re-order columns from file."""
+        if options is None:
+            options = {}
+        f_type = get_file_type(fromfile) if options['format_in'] is None else options['format_in']
+        iterable = IterableData(fromfile, options=options)
+        to_file = get_option(options, 'output')
+
+        if to_file:
+            to_type = get_file_type(to_file)
+            if not to_file:
+                print('Output file type not supported')
+                return
+            if to_type == 'bson':
+                out = open(to_file, 'wb')
+            if to_type == 'jsonl':
+                out = open(to_file, 'wb')
+            else:
+                out = open(to_file, 'w', encoding='utf8')
+        else:
+            to_type = f_type
+            out = sys.stdout
+        fields = options['fields'].split(',')
+        writer = DataWriter(out, filetype=to_type, fieldnames=fields)
+        if iterable:
+            n = 0
+            fields = [field.split('.') for field in fields]
+            chunk = []
+            for r in iterable.iter():
+                n += 1
+                if options['filter'] is not None:
+                    res = dq.match(r, options['filter'])
+                    #                    print(options['filter'], r)
+                    if not res:
+                        continue
+                r_selected = strip_dict_fields(r, fields, 0)
+                if n % 1000 == 0:
+                    logging.info('select: processing %d records of %s' % (n, fromfile))
+                    if len(chunk) > 0:
+                        writer.write_items(chunk)
+                        chunk = []
+                else:
+                    chunk.append(r_selected)
+            if len(chunk) > 0:
+                writer.write_items(chunk)
+        else:
+            logging.info('File type not supported')
+            return
+        logging.debug('select: %d records processed' % (n))
+        out.close()
+
+
+    def split_new(self, fromfile, options=None):
+        """Splits the given file with data into chunks based on chunk size or field value."""
+        if options is None:
+            options = {}
+        iterableargs = get_iterable_options(options)
+        open_iterable(fromfile, mode='r', iterableargs=iterableargs)
+        get_option(options, 'output')
+
+
+    def split(self, fromfile, options=None):
+        """Splits the given file with data into chunks based on chunk size or field value."""
+        if options is None:
+            options = {}
+        f_type = get_file_type(fromfile) if options['format_in'] is None else options['format_in']
+        if options['zipfile']:
+            z = zipfile.ZipFile(fromfile, mode='r')
+            fnames = z.namelist()
+            finfilename = fnames[0]
+            if f_type == 'bson':
+                infile = z.open(fnames[0], 'rb')
+            else:
+                infile = z.open(fnames[0], 'r')
+        elif options['gzipfile']:
+            import gzip
+            infile = gzip.open(fromfile, 'rb')
+            finfilename = fromfile.split('.', 1)[0] + '.' + f_type
+        else:
+            finfilename = fromfile
+            if f_type == 'bson':
+                infile = open(fromfile, 'rb')
+            else:
+                if 'encoding' in options.keys():
+                    infile = open(fromfile, 'r', encoding=get_option(options, 'encoding'))
+                else:
+                    detected_enc = detect_encoding(fromfile, limit=100000)
+                    if detected_enc:
+                        infile = open(fromfile, 'r', encoding=detected_enc['encoding'])
+                    else:
+                        infile = open(fromfile, 'r', encoding='utf8')
+        fields = options['fields'].split(',') if options['fields'] is not None else None
+        valuedict = {}
+        delimiter = get_option(options, 'delimiter')
+        if f_type == 'csv':
+            reader = csv.DictReader(infile, delimiter=delimiter)
+            n = 0
+            chunknum = 1
+            if options['fields'] is None:
+                splitname = finfilename.rsplit('.', 1)[0] + '_%d.csv' % (chunknum)
+                out = open(splitname, 'w', encoding=get_option(options, 'encoding'))
+                writer = csv.DictWriter(out, fieldnames=reader.fieldnames, delimiter=delimiter)
+                writer.writeheader()
+                for r in reader:
+                    n += 1
+                    if n % 10000 == 0:
+                        logging.info('split: processing %d records of %s' % (n, fromfile))
+                    if options['filter'] is not None:
+                        if not dq.match(r, options['filter']):
+                            continue
+                    writer.writerow(r)
+                    if n % options['chunksize'] == 0:
+                        out.close()
+                        chunknum += 1
+                        splitname = finfilename.rsplit('.', 1)[0] + '_%d.csv' % (
+                            chunknum)
+                        out = open(splitname, 'w',
+                                   encoding=get_option(options, 'encoding'))
+                        writer = csv.DictWriter(out, fieldnames=reader.fieldnames,
+                                                 delimiter=delimiter)
+                        writer.writeheader()
+        elif f_type == 'jsonl':
+            n = 0
+            chunknum = 1
+            if options['fields'] is None:
+                splitname = finfilename.rsplit('.', 1)[0] + '_%d.jsonl' % (chunknum)
+                out = open(splitname, 'wb')  # , encoding=get_option(options, 'encoding'))
+
+                for l in infile:
+                    n += 1
+                    if n % 10000 == 0:
+                        logging.info('split: processing %d records of %s' % (n, fromfile))
+                    r = orjson.loads(l)
+                    if options['filter'] is not None:
+                        if not dq.match(r, options['filter']):
+                            continue
+                    out.write(orjson.dumps(r, option=orjson.OPT_APPEND_NEWLINE))
+                    if n % options['chunksize'] == 0:
+                        out.close()
+                        chunknum += 1
+                        splitname = finfilename.rsplit('.', 1)[0] + '_%d.jsonl' % (chunknum)
+                        logging.info('split: new chunk %s' % splitname)
+                        out = open(splitname, 'wb') #, encoding=get_option(options, 'encoding'))
+            else:
+                for l in infile:
+                    n += 1
+                    if n % 10000 == 0:
+                        logging.info('split: processing %d records of %s' % (n, fromfile))
+                    r = orjson.loads(l)
+                    if options['filter'] is not None:
+                        if not dq.match(r, options['filter']):
+                            continue
+                    try:
+                        kx = get_dict_value(r, fields[0].split('.'))[0]
+                    except IndexError:
+                        continue
+                        kx = "None"
+                    if kx is None:
+                        continue
+                    kx = (kx.replace('\\', '-').replace('/', '-')
+                          .replace('?', '-').replace('<', '-')
+                          .replace('>', '-').replace('\n', ''))
+                    v = valuedict.get(kx, None)
+                    if v is None:
+                        # splitname = finfilename.rsplit('.', 1)[0] + '_%s.jsonl' % (kx)
+                        splitname = '%s.jsonl' % (kx)
+                        if options['dirname'] is not None:
+                            splitname = os.path.join(options['dirname'],
+                                                     splitname)
+                        valuedict[kx] = open(splitname, 'w', encoding='utf8')
+                    valuedict[kx].write(l)
+                #                    valuedict[kx].write(l.decode('utf8'))#.decode('utf8')#)
+                for opened in valuedict.values():
+                    opened.close()
+        elif f_type == 'bson':
+            bson_iter = bson.decode_file_iter(infile)
+            n = 0
+            for r in bson_iter:
+                n += 1
+                #                print(r)
+                strip_dict_fields(r, fields, 0)
+                #                out.write(json.dumps(r_selected)+'\n')
+                if n % 10000 == 0:
+                    logging.info('split: processing %d records of %s' % (n, fromfile))
+
+        else:
+            logging.info('File type not supported')
+            return
+        logging.debug('split: %d records processed' % (n))

undatum/cmds/statistics.py

@@ -0,0 +1,158 @@
+# -*- coding: utf8 -*-
+"""Statistical analysis module."""
+from ..utils import get_option, dict_generator, guess_datatype
+from ..constants import DEFAULT_DICT_SHARE
+import logging
+from qddate import DateParser
+#from ..common.iterable import IterableData
+from iterable.helpers.detect import open_iterable
+
+#STAT_READY_DATA_FORMATS = ['jsonl', 'bson', 'csv']
+
+ITERABLE_OPTIONS_KEYS = ['tagname', 'delimiter', 'encoding', 'start_line', 'page']
+
+
+def get_iterable_options(options):
+    """Extract iterable-specific options from options dictionary."""
+    out = {}
+    for k in ITERABLE_OPTIONS_KEYS:
+        if k in options.keys():
+            out[k] = options[k]
+    return out
+
+
+class StatProcessor:
+    """Statistical processing handler."""
+    def __init__(self, nodates=True):
+        if nodates:
+            self.qd = None
+        else:
+            self.qd = DateParser(generate=True)
+        pass
+
+    def stats(self, fromfile, options):
+        """Produces statistics and structure analysis of JSONlines, BSON or CSV file and produces stats"""
+        from rich import print
+        from rich.table import Table
+
+        iterableargs = get_iterable_options(options)
+        iterable = open_iterable(fromfile, mode='r', iterableargs=iterableargs)
+        dictshare = get_option(options, 'dictshare')
+
+        if dictshare and dictshare.isdigit():
+            dictshare = int(dictshare)
+        else:
+            dictshare = DEFAULT_DICT_SHARE
+
+        profile = {'version': 1.0}
+        fielddata = {}
+        fieldtypes = {}
+
+        #    data = json.load(open(profile['filename']))
+        count = 0
+        nfields = 0
+
+        # process data items one by one
+        logging.debug('Start processing %s' % (fromfile))
+        for item in iterable:
+            count += 1
+            dk = dict_generator(item)
+            if count % 1000 == 0: logging.debug('Processing %d records of %s' % (count, fromfile))
+            for i in dk:
+                #            print(i)
+                k = '.'.join(i[:-1])
+                if len(i) == 0: continue
+                if i[0].isdigit(): continue
+                if len(i[0]) == 1: continue
+                v = i[-1]
+                if k not in fielddata:  # Use direct dict membership check instead of list()
+                    fielddata[k] = {'key': k, 'uniq': {}, 'n_uniq': 0, 'total': 0, 'share_uniq': 0.0,
+                                    'minlen': None, 'maxlen': 0, 'avglen': 0, 'totallen': 0}
+                fd = fielddata[k]
+                uniqval = fd['uniq'].get(v, 0)
+                fd['uniq'][v] = uniqval + 1
+                fd['total'] += 1
+                if uniqval == 0:
+                    fd['n_uniq'] += 1
+                    fd['share_uniq'] = (fd['n_uniq'] * 100.0) / fd['total']
+                fl = len(str(v))
+                if fd['minlen'] is None:
+                    fd['minlen'] = fl
+                else:
+                    fd['minlen'] = fl if fl < fd['minlen'] else fd['minlen']
+                fd['maxlen'] = fl if fl > fd['maxlen'] else fd['maxlen']
+                fd['totallen'] += fl
+                fielddata[k] = fd
+                if k not in fieldtypes:  # Use direct dict membership check instead of list()
+                    fieldtypes[k] = {'key': k, 'types': {}}
+                fd = fieldtypes[k]
+                thetype = guess_datatype(v, self.qd)['base']
+                uniqval = fd['types'].get(thetype, 0)
+                fd['types'][thetype] = uniqval + 1
+                fieldtypes[k] = fd
+        #        print count
+        for k, v in fielddata.items():  # Use dict.items() directly, no list() conversion
+            fielddata[k]['share_uniq'] = (v['n_uniq'] * 100.0) / v['total']
+            fielddata[k]['avglen'] = v['totallen'] / v['total']
+        profile['count'] = count
+        profile['num_fields'] = nfields
+
+        # Determine field types first so we can use them when building dicts
+        finfields = {}
+        for fd in fieldtypes.values():  # Use dict.values() directly, no list() conversion
+            fdt = list(fd['types'].keys())  # Keep list() here as we need to check membership and modify
+            if 'empty' in fdt:
+                del fd['types']['empty']
+            types_keys = list(fd['types'].keys())  # Need list for len() and indexing
+            if len(types_keys) != 1:
+                ftype = 'str'
+            else:
+                ftype = types_keys[0]
+            finfields[fd['key']] = ftype
+
+        profile['fieldtypes'] = finfields
+
+        dictkeys = []
+        dicts = {}
+        #        print(profile)
+        profile['fields'] = []
+        for fd in fielddata.values():  # Use dict.values() directly, no list() conversion
+            #            print(fd['key'])  # , fd['n_uniq'], fd['share_uniq'], fieldtypes[fd['key']]
+            field = {'key': fd['key'], 'is_uniq': 0 if fd['share_uniq'] < 100 else 1}
+            profile['fields'].append(field)
+            if fd['share_uniq'] < dictshare:
+                dictkeys.append(fd['key'])
+                # Use determined field type instead of defaulting to 'str'
+                field_type = finfields.get(fd['key'], 'str')
+                dicts[fd['key']] = {'items': fd['uniq'], 'count': fd['n_uniq'],
+                                    'type': field_type}
+        #            for k, v in fd['uniq'].items():
+        #                print fd['key'], k, v
+        profile['dictkeys'] = dictkeys
+
+        for k, v in fielddata.items():  # Use dict.items() directly, no list() conversion
+            del v['uniq']
+            fielddata[k] = v
+        profile['debug'] = {'fieldtypes': fieldtypes.copy(), 'fielddata': fielddata}
+        table = []
+        for fd in fielddata.values():  # Use dict.values() directly, no list() conversion
+            field = [fd['key'], ]
+            field.append(finfields[fd['key']])
+            field.append(True if fd['key'] in dictkeys else False)
+            field.append(False if fd['share_uniq'] < 100 else True)
+            field.append(fd['n_uniq'])
+            field.append(fd['share_uniq'])
+            field.append(fd['minlen'])
+            field.append(fd['maxlen'])
+            field.append(fd['avglen'])
+            table.append(field)
+        headers = ('key', 'ftype', 'is_dictkey', 'is_uniq', 'n_uniq', 'share_uniq', 'minlen', 'maxlen', 'avglen')
+        reptable = Table(title="Statistics")
+        reptable.add_column(headers[0], justify="left", style="magenta")
+        for key in headers[1:-1]:
+            reptable.add_column(key, justify="left", style="cyan", no_wrap=True)
+        reptable.add_column(headers[-1], justify="right", style="cyan")
+        for row in table:
+            reptable.add_row(*map(str, row))
+        print(reptable)
+

undatum/cmds/textproc.py

@@ -0,0 +1,59 @@
+# -*- coding: utf8 -*-
+"""Text processing module."""
+from ..utils import get_file_type, get_option
+from iterable.helpers.detect import open_iterable
+
+#STAT_READY_DATA_FORMATS = ['jsonl', 'bson', 'csv']
+
+ITERABLE_OPTIONS_KEYS = ['tagname', 'delimiter', 'encoding', 'start_line', 'page']
+
+
+def get_iterable_options(options):
+    """Extract iterable-specific options from options dictionary."""
+    out = {}
+    for k in ITERABLE_OPTIONS_KEYS:
+        if k in options.keys():
+            out[k] = options[k]
+    return out
+
+
+def get_keys(adict, prefix=None):
+    """Extract all keys from nested dictionary."""
+    keys = {}
+    for k, v in adict.items():
+        fullk = '.'.join([prefix, k]) if prefix else k
+        keys[fullk] = 1
+        if isinstance(v, dict):
+            for ak in get_keys(v, fullk):
+                keys[ak] = 1
+        elif isinstance(v, list):
+            for item in v:
+                if isinstance(item, dict):
+                    for ak in get_keys(item, fullk):
+                        keys[ak] = 1
+        else:
+            print(('%s\t%s' % (fullk, str(v))))
+    return keys
+
+
+class TextProcessor:
+    """Text processing handler."""
+    def __init__(self):
+        pass
+
+    def flatten(self, filename, options):
+        """Flatten the data. One field - one line"""
+        get_file_type(filename) if options['format_in'] is None else options['format_in']
+        iterableargs = get_iterable_options(options)
+        iterable = open_iterable(fromfile, mode='r', iterableargs=iterableargs)
+        get_option(options, 'output')
+        i = 0
+        for rec in iterable:
+            allkeys = {}
+            i += 1
+            for k in get_keys(rec):
+                v = allkeys.get(k, 0)
+                allkeys[k] = v + 1
+            for k, v in allkeys.items():
+                print('\t'.join([k, str(v)]))
+