sacsv 1.5.1__tar.gz

sacsv-1.5.1/.gitignore

@@ -0,0 +1 @@
+*.pyc

sacsv-1.5.1/LICENSE.md

@@ -0,0 +1,5 @@
+
+Copyright (C) 2018 Gabor Nyeki
+
+This package is licensed under the Creative Commons Attribution 4.0 International License: http://creativecommons.org/licenses/by/4.0/.
+

sacsv-1.5.1/PKG-INFO

@@ -0,0 +1,62 @@
+Metadata-Version: 2.4
+Name: sacsv
+Version: 1.5.1
+Summary: Swiss Army csv: command-line tools to manipulate csv-formatted data
+Project-URL: Homepage, https://github.com/gn0/sacsv
+Author-email: Gabor Nyeki <gabor.nyeki@alumni.duke.edu>
+License-File: LICENSE.md
+Requires-Python: >=3.10
+Requires-Dist: argh>=0.31.3
+Description-Content-Type: text/markdown
+
+
+# sacsv: Swiss Army csv
+
+This Python package provides an assortment of command-line tools to manipulate csv-formatted data.
+The tools are:
+
+- `csv2jsonl`: converts csv input into jsonlines
+- `csvaddrandom`: adds a column with a random number
+- `csvadduniqueid`: adds a column with a unique record identifier
+- `csvaggregate`: applies an arbitrary Python function to every value of a column, possibly within groups
+- `csvappend`: appends two or more csv files
+- `csvdropdups`: drops duplicate records
+- `csvfindsortkey`: attempts to find the column that the input is sorted by
+- `csvkeepmax`: keeps the record that has the maximum value in a column
+- `csvleftjoin`: merges two csv files
+- `csvop`: applies an arbitrary Python function to every record and saves the return value in a new column
+- `csvparallel`: parallelizes arbitrary commands that read a csv input and write a csv output
+- `csvrename`: changes the name of a column
+- `csvreorder`: changes the order of columns
+- `csvsed`: applies a substitution rule, using regular expressions, to every value of a column
+- `csvsort`: sorts the input
+- `csvtranspose`: transposes the input
+- `fw2csv`: converts fixed-width input, potentially with multi-line records, into csv
+- `longcsv2wide`: converts the input from long to wide form
+- `widecsv2long`: converts the input from wide to long form
+
+![Illustration of `csvparallel` executing `f.py` in 8 jobs](examples/csvparallel.png)
+
+## Installation
+
+To install this package using pip, type
+
+```
+pip install git+https://github.com/gn0/sacsv
+```
+
+or, alternatively,
+
+```
+git clone https://github.com/gn0/sacsv
+pip install ./sacsv
+```
+
+## Author
+
+Gabor Nyeki.  Contact information is on http://www.gabornyeki.com/.
+
+## License
+
+This package is licensed under the Creative Commons Attribution 4.0 International License: http://creativecommons.org/licenses/by/4.0/.
+

sacsv-1.5.1/README.md

@@ -0,0 +1,51 @@
+
+# sacsv: Swiss Army csv
+
+This Python package provides an assortment of command-line tools to manipulate csv-formatted data.
+The tools are:
+
+- `csv2jsonl`: converts csv input into jsonlines
+- `csvaddrandom`: adds a column with a random number
+- `csvadduniqueid`: adds a column with a unique record identifier
+- `csvaggregate`: applies an arbitrary Python function to every value of a column, possibly within groups
+- `csvappend`: appends two or more csv files
+- `csvdropdups`: drops duplicate records
+- `csvfindsortkey`: attempts to find the column that the input is sorted by
+- `csvkeepmax`: keeps the record that has the maximum value in a column
+- `csvleftjoin`: merges two csv files
+- `csvop`: applies an arbitrary Python function to every record and saves the return value in a new column
+- `csvparallel`: parallelizes arbitrary commands that read a csv input and write a csv output
+- `csvrename`: changes the name of a column
+- `csvreorder`: changes the order of columns
+- `csvsed`: applies a substitution rule, using regular expressions, to every value of a column
+- `csvsort`: sorts the input
+- `csvtranspose`: transposes the input
+- `fw2csv`: converts fixed-width input, potentially with multi-line records, into csv
+- `longcsv2wide`: converts the input from long to wide form
+- `widecsv2long`: converts the input from wide to long form
+
+![Illustration of `csvparallel` executing `f.py` in 8 jobs](examples/csvparallel.png)
+
+## Installation
+
+To install this package using pip, type
+
+```
+pip install git+https://github.com/gn0/sacsv
+```
+
+or, alternatively,
+
+```
+git clone https://github.com/gn0/sacsv
+pip install ./sacsv
+```
+
+## Author
+
+Gabor Nyeki.  Contact information is on http://www.gabornyeki.com/.
+
+## License
+
+This package is licensed under the Creative Commons Attribution 4.0 International License: http://creativecommons.org/licenses/by/4.0/.
+

sacsv-1.5.1/pyproject.toml

@@ -0,0 +1,47 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "sacsv"
+version = "1.5.1"
+description = "Swiss Army csv: command-line tools to manipulate csv-formatted data"
+authors = [
+    { name = "Gabor Nyeki", email = "gabor.nyeki@alumni.duke.edu" }
+]
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "argh>=0.31.3",
+]
+
+[project.scripts]
+csv2jsonl = "sacsv.csv2jsonl:dispatch"
+csvaddrandom = "sacsv.csvaddrandom:dispatch"
+csvadduniqueid = "sacsv.csvadduniqueid:dispatch"
+csvaggregate = "sacsv.csvaggregate:dispatch"
+csvappend = "sacsv.csvappend:dispatch"
+csvcheck = "sacsv.csvcheck:dispatch"
+csvdropdups = "sacsv.csvdropdups:dispatch"
+csvfindsortkey = "sacsv.csvfindsortkey:dispatch"
+csvkeepmax = "sacsv.csvkeepmax:dispatch"
+csvleftjoin = "sacsv.csvleftjoin:dispatch"
+csvop = "sacsv.csvop:dispatch"
+csvparallel = "sacsv.csvparallel:dispatch"
+csvrename = "sacsv.csvrename:dispatch"
+csvreorder = "sacsv.csvreorder:dispatch"
+csvsed = "sacsv.csvsed:dispatch"
+csvsort = "sacsv.csvsort:dispatch"
+csvtranspose = "sacsv.csvtranspose:dispatch"
+fw2csv = "sacsv.fw2csv:dispatch"
+longcsv2wide = "sacsv.longcsv2wide:dispatch"
+widecsv2long = "sacsv.widecsv2long:dispatch"
+
+[project.urls]
+Homepage = "https://github.com/gn0/sacsv"
+
+[dependency-groups]
+dev = [
+    "pytest>=9.0.2",
+    "pytest-cov>=7.0.0",
+]

sacsv-1.5.1/src/sacsv/csv2jsonl.py

@@ -0,0 +1,33 @@
+import argh
+import csv
+import json
+import sys
+import collections
+
+
+def cast(obj):
+    try:
+        return float(obj)
+    except:
+        return obj
+
+
+@argh.arg("-a", "--auto-cast", default=False)
+def main(auto_cast=None):
+    reader = csv.reader(sys.stdin)
+    header = next(reader)
+
+    for record in reader:
+        obj = collections.OrderedDict((k, cast(v) if auto_cast else v)
+                                      for k, v in zip(header, record))
+        print(json.dumps(obj))
+
+    sys.stdout.flush()
+
+
+def dispatch():
+    argh.dispatch_command(main)
+
+
+if __name__ == "__main__":
+    dispatch()

sacsv-1.5.1/src/sacsv/csvaddrandom.py

@@ -0,0 +1,29 @@
+import argh
+import random
+import csv
+import sys
+
+
+@argh.arg("-s", "--seed", type=int, required=True)
+@argh.arg("-c", "--column-name", type=str, required=True)
+def main(column_name=None, seed=None):
+    random.seed(seed)
+
+    reader = csv.reader(sys.stdin)
+    header = next(reader)
+
+    writer = csv.writer(sys.stdout)
+    writer.writerow(
+        header + [column_name])
+
+    for record in reader:
+        writer.writerow(
+            record + [random.randint(1, 2**31)])
+
+
+def dispatch():
+    argh.dispatch_command(main)
+
+
+if __name__ == "__main__":
+    dispatch()

sacsv-1.5.1/src/sacsv/csvadduniqueid.py

@@ -0,0 +1,43 @@
+import argh
+import sys
+import csv
+import itertools as it
+
+
+@argh.arg("-g", "--group-by", type=str, nargs="+")
+@argh.arg("-s", "--sort-by", type=str, nargs="+")
+@argh.arg("-c", "--column-name", type=str, required=True)
+def main(group_by=None, sort_by=None, column_name=None):
+    reader = csv.reader(sys.stdin)
+    header = next(reader)
+
+    if group_by is None:
+        group_key = lambda r: 1
+    else:
+        group_indices = tuple(header.index(c) for c in group_by)
+        group_key = lambda r: tuple(r[i] for i in group_indices)
+
+    if sort_by is None:
+        sort_key = lambda r: 1
+    else:
+        sort_indices = tuple(header.index(c) for c in sort_by)
+        sort_key = lambda r: tuple(r[i] for i in sort_indices)
+
+    writer = csv.writer(sys.stdout)
+    writer.writerow(
+        [column_name] + header)
+
+    for group_id, group_iter in it.groupby(
+                                    sorted(reader, key=group_key),
+                                    group_key):
+        for k, record in enumerate(sorted(group_iter, key=sort_key), 1):
+            writer.writerow(
+                [k] + record)
+
+
+def dispatch():
+    argh.dispatch_command(main)
+
+
+if __name__ == "__main__":
+    dispatch()

sacsv-1.5.1/src/sacsv/csvaggregate.py

@@ -0,0 +1,59 @@
+import argh
+import importlib
+import csv
+import sys
+import operator as op
+import itertools as it
+
+
+@argh.arg("-m", "--import-mod", type=str, nargs="+", required=False)
+@argh.arg("-c", "--columns", type=str, nargs="+", required=True)
+@argh.arg("-g", "--group-by", type=str, nargs="+", required=False)
+@argh.arg("-f", "--func-def", type=str, required=True)
+def main(import_mod=None, columns=None, group_by=None, func_def=None):
+    for m in import_mod or tuple():
+        globals()[m.split(".")[0]] = importlib.import_module(m.split(".")[0])
+        importlib.import_module(m)
+
+    f = eval(func_def)
+
+    reader = csv.reader(sys.stdin)
+    header = next(reader)
+    pickers = tuple(op.itemgetter(header.index(c)) for c in columns)
+
+    writer = csv.writer(sys.stdout)
+
+    if group_by is None:
+        group_key = lambda r: 1
+        writer.writerow(columns)
+    else:
+        group_key = lambda r: tuple(r[header.index(c)] for c in group_by)
+        writer.writerow(group_by + columns)
+
+    for group_id, record_iter in it.groupby(
+                                     sorted(
+                                         reader,
+                                         key=group_key),
+                                     group_key):
+        values = list(list() for k in range(len(columns)))
+
+        for record in record_iter:
+            for k in range(len(columns)):
+                values[k].append(
+                    pickers[k](record))
+
+        if group_by is None:
+            writer.writerow(
+                tuple(f(values[k]) for k in range(len(columns))))
+        else:
+            writer.writerow(
+                group_id
+                + tuple(f(values[k]) for k in range(len(columns))))
+
+
+def dispatch():
+    argh.dispatch_command(main)
+
+
+if __name__ == "__main__":
+    dispatch()

sacsv-1.5.1/src/sacsv/csvappend.py

@@ -0,0 +1,52 @@
+import argh
+import csv
+import sys
+
+
+def get_fieldnames(fieldnames_iter):
+    return sorted(
+               set(f
+                   for fieldnames in fieldnames_iter
+                   for f in fieldnames))
+
+
+assert (get_fieldnames(
+            (("a", "b"), ("a", "b", "c")))
+        == ["a", "b", "c"])
+
+
+def make_get_fields(*fieldnames):
+    def get_fields(record):
+        return tuple(record.get(field) for field in fieldnames)
+
+    return get_fields
+
+
+assert make_get_fields("a", "b")(dict(a=1)) == (1, None)
+
+
+@argh.arg("csv_filename", nargs="+")
+def main(csv_filename):
+    dict_readers = tuple(csv.DictReader(open(filename, "r"))
+                         for filename in csv_filename)
+
+    fieldnames = get_fieldnames(r.fieldnames for r in dict_readers)
+    get_fields = make_get_fields(*fieldnames)
+
+    writer = csv.writer(sys.stdout)
+
+    writer.writerow(fieldnames)
+    writer.writerows(
+        get_fields(record)
+        for dict_reader in dict_readers
+        for record in dict_reader)
+
+    sys.stdout.flush()
+
+
+def dispatch():
+    argh.dispatch_command(main)
+
+
+if __name__ == "__main__":
+    dispatch()

sacsv-1.5.1/src/sacsv/csvcheck.py

@@ -0,0 +1,232 @@
+import argh
+import sys
+import csv
+
+from collections import namedtuple
+
+
+CheckResult = namedtuple(
+    "CheckResult",
+    ("success", "message", "problems"))
+
+
+def column_values(rows, index):
+    return tuple(row[index] for row in rows)
+
+
+def print_problems_markdown(header, records):
+    n_columns = len(header)
+
+    column_widths = tuple(
+        max(map(len, (header[i],) + column_values(records, i)))
+        for i in range(n_columns))
+
+    # Header.
+    #
+
+    print("|", end = "")
+    for i in range(n_columns):
+        print(
+            (" %%%ds |" % column_widths[i]) % header[i],
+            end="")
+    print()
+
+    print("|", end = "")
+    for i in range(n_columns):
+        print(" %s |" % ("-" * column_widths[i]), end="")
+    print()
+
+    # Body.
+    #
+
+    for record in records:
+        print("|", end = "")
+        for i in range(n_columns):
+            print(
+                (" %%%ds |" % column_widths[i]) % record[i],
+                end="")
+        print()
+
+    print()
+
+
+def share_samelength_among_nonmissing(values):
+    pass
+
+
+def is_nonmissing(value):
+    return value not in ("", "NA")
+
+
+def is_numeric(value):
+    try:
+        float(value)
+    except ValueError:
+        return False
+
+    return True
+
+
+def is_text(value):
+    return not is_numeric(value)
+
+
+def is_unique(value, values):
+    try:
+        values[values.index(value) + 1:].index(value)
+    except ValueError:
+        return True
+
+    return False
+
+
+def share_type_among_nonmissing(type_func, values):
+    indicators = tuple(
+        type_func(value)
+        for value in values
+        if is_nonmissing(value))
+
+    if len(indicators) == 0:
+        return None
+
+    return sum(indicators) / len(indicators)
+
+
+def share_numeric_among_nonmissing(values):
+    return share_type_among_nonmissing(
+        is_numeric, values)
+
+
+def share_text_among_nonmissing(values):
+    return share_type_among_nonmissing(
+        is_text, values)
+
+
+def share_unique_among_nonmissing(values):
+    return share_type_among_nonmissing(
+        lambda x: is_unique(x, values),
+        values)
+
+
+def mostly_samelength(values):
+    return False # TODO
+
+
+def mostly_numeric(values):
+    share = share_numeric_among_nonmissing(values)
+
+    return share is None or share >= .9
+
+
+def mostly_text(values):
+    share = share_text_among_nonmissing(values)
+
+    return share is None or share >= .9
+
+
+def mostly_unique(values):
+    share = share_unique_among_nonmissing(values)
+
+    return share is None or share >= .9
+
+
+def check_column_type_consistency(values):
+    """Checks that if most values in a column are a certain
+    type, then all values in that column are that type."""
+
+    success = True
+    message = None
+    problems = None
+
+    if mostly_samelength(values):
+        # E.g., phone numbers that each have 11 digits.
+        #
+        pass # TODO
+    elif mostly_numeric(values):
+        if share_text_among_nonmissing(values) > 0:
+            success = False
+            message = "Text value in mostly numeric column."
+            problems = tuple(
+                i
+                for i, value in enumerate(values)
+                if is_nonmissing(value) and is_text(value))
+    elif mostly_text(values):
+        if share_numeric_among_nonmissing(values) > 0:
+            success = False
+            message = "Numeric value in mostly text column."
+            problems = tuple(
+                i
+                for i, value in enumerate(values)
+                if is_nonmissing(value) and is_numeric(value))
+
+    return CheckResult(
+        success=success,
+        message=message,
+        problems=problems)
+
+
+def check_column_no_odd_value_repetitions(values):
+    """Checks that if most values in a column are unique, then
+    all values in that column are unique."""
+
+    success = True
+    message = None
+    problems = None
+
+    if mostly_unique(values):
+        if share_unique_among_nonmissing(values) < 1:
+            success = False
+            message = "Repeated values in column with mostly unique values."
+            problems = tuple(
+                i
+                for i, value in enumerate(values)
+                if is_nonmissing(value)
+                   and not is_unique(value, values))
+
+    return CheckResult(
+        success=success,
+        message=message,
+        problems=problems)
+
+
+def main():
+    reader = csv.reader(sys.stdin)
+
+    header = next(reader)
+    records = tuple(reader)
+
+    column_checks = (
+        check_column_type_consistency,
+        check_column_no_odd_value_repetitions)
+
+    # TODO
+    # row_checks = (
+    #     check_row_no_odd_value_repetitions,)
+
+    all_success = True
+
+    for check in column_checks:
+        for i, column in enumerate(header):
+            result = check(column_values(records, i))
+
+            if not result.success:
+                all_success = False
+
+                print("warning: %s: %s"
+                      % (column, result.message))
+                print_problems_markdown(
+                    header,
+                    tuple(
+                        records[index]
+                        for index in result.problems))
+
+    if not all_success:
+        sys.exit(1)
+
+
+def dispatch():
+    argh.dispatch_command(main)
+
+
+if __name__ == "__main__":
+    dispatch()

sacsv-1.5.1/src/sacsv/csvdropdups.py

@@ -0,0 +1,60 @@
+import argh
+import csv
+import sys
+import itertools as it
+
+
+def make_key(key_columns, columns):
+    if len(key_columns) == 0:
+        return lambda x: 1
+    else:
+        def key(item):
+            return tuple(item[columns.index(c)] for c in key_columns)
+
+        return key
+
+
+@argh.arg("-k", "--key", type=str, nargs="+", required=True)
+@argh.arg("-f", "--keep-first", type=str, nargs="*")
+@argh.arg("-l", "--keep-last", type=str, nargs="*")
+def main(key=None, keep_first=None, keep_last=None):
+    if keep_first is None and keep_last is None:
+        raise argh.CommandError(
+                  "Must specify either --keep-first or --keep-last.")
+    elif keep_first is not None and keep_last is not None:
+        raise argh.CommandError(
+                  "Must specify either --keep-first or --keep-last "
+                  + "but not both.")
+
+    reader = csv.reader(sys.stdin)
+    columns = next(reader)
+
+    primary_key = make_key(key, columns)
+
+    if keep_first is not None:
+        secondary_key = make_key(keep_first, columns)
+    else:
+        secondary_key = make_key(keep_last, columns)
+
+    writer = csv.writer(sys.stdout)
+    writer.writerow(columns)
+
+    for item_key, item_iter in it.groupby(
+                                   sorted(
+                                       reader,
+                                       key=primary_key),
+                                   primary_key):
+        items = sorted(item_iter, key=secondary_key)
+
+        if keep_first is not None:
+            writer.writerow(items[0])
+        else:
+            writer.writerow(items[-1])
+
+
+def dispatch():
+    argh.dispatch_command(main)
+
+
+if __name__ == "__main__":
+    dispatch()