skilleter-thingy 0.0.40__py3-none-any.whl → 0.0.42__py3-none-any.whl

skilleter_thingy/py_audit.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+
+""" Query api.osv.dev to determine whether a specified version of a particular
+    Python package is subject to known security vulnerabilities """
+
+################################################################################
+
+import os
+import sys
+import requests
+import subprocess
+import tempfile
+import re
+import glob
+import argparse
+
+################################################################################
+
+PIP_PACKAGES = ('pip', 'pkg_resources', 'setuptools')
+PIP_OPTIONS = '--no-cache-dir'
+
+QUERY_URL = "https://api.osv.dev/v1/query"
+QUERY_HEADERS = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'}
+
+ANSI_NORMAL = '\x1b[0m'
+ANSI_BOLD = '\x1b[1m'
+
+################################################################################
+
+def audit(package, version):
+    """ Main function """
+
+    # Query api.osv.dev for known vulnerabilties in this version of the package
+
+    payload = '{"version": "'+version+'", "package": {"name": "'+package+'", "ecosystem": "PyPI"}}'
+    result = requests.post(QUERY_URL, data=payload, headers=QUERY_HEADERS)
+
+    # Parse and report the results
+
+    details = result.json()
+
+    print('-' * 80)
+    if package in PIP_PACKAGES:
+        print(f'Package: {package} {version} (part of Pip)')
+    else:
+        print(f'Package: {package} {version}')
+
+    print()
+
+    if 'vulns' in details:
+        print(f'{len(details["vulns"])} known vulnerabilities')
+
+        for v in details['vulns']:
+            print()
+            print(f'{ANSI_BOLD}Vulnerability: {v["id"]}{ANSI_NORMAL}')
+
+            if 'summary' in v:
+                print(f'Summary:       {v["summary"]}')
+
+            if 'aliases' in v:
+                print('Aliases:       %s' % (', '.join(v['aliases'])))
+
+            if 'details' in v:
+                print()
+                print(v['details'])
+    else:
+        print(f'No known vulnerabilities')
+
+################################################################################
+
+def main():
+    """ Entry point """
+
+    parser = argparse.ArgumentParser(description='Query api.osv.dev to determine whether Python packagers in a requirments.txt file are subject to known security vulnerabilities')
+    parser.add_argument('requirements', nargs='*', type=str, action='store', help='The requirements file (if not specified, then the script searches for a requirements.txt file)')
+    args = parser.parse_args()
+
+    requirements = args.requirements or glob.glob('**/requirements.txt', recursive=True)
+
+    if not requirements:
+        print('No requirements.txt file(s) found')
+        sys.exit(0)
+
+    # Create a venv for each requirements file, install pip and the packages
+    # and prerequisites, get the list of installed package versions
+    # and check each one.
+
+    for requirement in requirements:
+        print('=' * 80)
+        print(f'{ANSI_BOLD}File: {requirement}{ANSI_NORMAL}')
+        print()
+
+        with tempfile.TemporaryDirectory() as env_dir:
+            with tempfile.NamedTemporaryFile() as package_list:
+                script = f'python3 -m venv {env_dir}' \
+                         f' && . {env_dir}/bin/activate' \
+                         f' && python3 -m pip {PIP_OPTIONS} install --upgrade pip setuptools' \
+                         f' && python3 -m pip {PIP_OPTIONS} install -r {requirement}' \
+                         f' && python3 -m pip {PIP_OPTIONS} list | tail -n+3 | tee {package_list.name}' \
+                          ' && deactivate'
+
+                result = subprocess.run(script, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
+
+                if result.returncode:
+                    print(f'ERROR #{result.returncode}: {result.stdout}')
+                    sys.exit(result.returncode)
+
+                with open(package_list.name) as infile:
+                    for package in infile.readlines():
+                        package_info = re.split(' +', package.strip())
+
+                        audit(package_info[0], package_info[1])
+
+################################################################################
+
+def py_audit():
+    """Entry point"""
+
+    try:
+        main()
+
+    except KeyboardInterrupt:
+        sys.exit(1)
+
+    except BrokenPipeError:
+        sys.exit(2)
+
+################################################################################
+
+if __name__ == '__main__':
+    py_audit()

skilleter_thingy/readable.py

@@ -0,0 +1,270 @@
+#! /usr/bin/env python3
+
+################################################################################
+""" Pipe for converting colour combinations to make them readable
+
+    Defaults to making things readable on a light background but has the
+    option to make things readable on a dark background or for removing
+    colours altogether
+"""
+
+# TODO: Not sure it works properly as a pipe
+# TODO: Error handling in file I/O
+
+################################################################################
+
+import sys
+import argparse
+import tempfile
+import os
+import re
+import filecmp
+import shutil
+
+import thingy.tidy as tidy
+import thingy.files as files
+
+################################################################################
+
+TF_OBJECTS_CHANGED_END = 'Terraform detected the following changes made outside of Terraform since the last "terraform apply":'
+TF_IGNORE_MSG = 'Unless you have made equivalent changes to your configuration, or ignored the relevant attributes using ignore_changes, the following plan may include actions to undo or respond to these changes.'
+
+TF_REFRESHING_AND_READING = re.compile(r'.*: (?:Refreshing state\.\.\.|Reading\.\.\.|Read complete after |Preparing import\.\.\.).*')
+TF_FINDING_AND_INSTALLING = re.compile(r'- (?:Finding .*[.]{3}|Installing .*[.]{3}|Installed .*)')
+
+TF_HAS_CHANGED = re.compile(r'  # .* has changed')
+TF_READ_DURING_APPLY = re.compile(r' +# [-a-z_.0-9\[\]]+ will be read during apply')
+TF_UNCHANGED_TAG = re.compile(r' +".*" += +".*"')
+
+TF_OBJECTS_CHANGED = 'Note: Objects have changed outside of Terraform'
+TF_UNCHANGED_HIDDEN = re.compile(r' +# \(\d+ unchanged (?:attribute|block|element)s? hidden\)')
+
+################################################################################
+
+def parse_command_line():
+    """ Parse, check and sanitise the command line arguments """
+
+    parser = argparse.ArgumentParser(description='Read from standard input and write to standard output modifying ANSI colour codes en-route.')
+
+    parser.add_argument('-l', '--light', action='store_true', help='Modify colours for a light background (the default)')
+    parser.add_argument('-d', '--dark', action='store_true', help='Modify colours for a dark background')
+    parser.add_argument('-n', '--none', action='store_true', help='Remove all colour codes')
+    parser.add_argument('-t', '--tidy', action='store_true',
+                        help='Remove colour codes and stuff that typically occurs in log files causing diffs, but is of no particular interest (e.g. SHA1 values, times, dates)')
+    parser.add_argument('-s', '--strip-blank', action='store_true', help='Strip all blank lines')
+    parser.add_argument('-D', '--debug', action='store_true', help='Replace colours with debug information')
+    parser.add_argument('-o', '--out', action='store_true', help='Output to standard output rather than overwriting input files')
+    parser.add_argument('-O', '--dir', action='store', default=None, help='Store output files in the specified directory (creating it if it doesn\'t exist)')
+    parser.add_argument('-a', '--aws', action='store_true', help='Remove AWS resource IDs')
+    parser.add_argument('-T', '--terraform', action='store_true', help='Clean Terraform plan/apply log files')
+    parser.add_argument('files', nargs='*', default=None, help='The files to convert (use stdin/stout if no input files are specified)')
+
+    args = parser.parse_args()
+
+    # Can't do more than one transformation
+
+    if args.light + args.dark + args.none + args.debug > 1:
+        print('ERROR: Only one colour conversion option can be specified')
+        sys.exit(1)
+
+    if args.tidy and (args.light or args.dark or args.debug):
+        print('ERROR: The tidy and colour conversion options cannot be specified together')
+        sys.exit(1)
+
+    # Default to doing cleanup - removing colour codes, times,
+
+    if not args.light and not args.dark and not args.tidy and not args.strip_blank and not args.terraform:
+        args.strip_blank = args.none = args.tidy = args.terraform = True
+
+    # Terraform option also removes ANSI, etc.
+
+    if args.terraform:
+        args.strip_blank = args.tidy = args.none = True
+
+    # Create the output directory, if required
+
+    if args.dir and not os.path.isdir(args.dir):
+        os.mkdir(args.dir)
+
+    return args
+
+################################################################################
+
+def cleanfile(args, infile, outfile):
+    """ Clean infile, writing to outfile.
+        Returns True if any changes were made. """
+
+    # Set if we are ignoring a block of adata
+
+    ignore_until = False
+
+    prev_line = None
+
+    ignore_until = None
+    collection = []
+
+    # Read, process and write stdin to stdout, converting appropriately
+
+    for data in infile:
+        # Remove the trailing newline
+
+        if data[-1] == '\n':
+            data = data[:-1]
+
+        # Strip trailing whitespace
+
+        data = data.rstrip()
+
+        # Do colour code conversion
+
+        if args.debug:
+            clean = tidy.debug_format(data)
+        elif args.light:
+            clean = tidy.convert_ansi(data, True)
+        elif args.dark:
+            clean = tidy.convert_ansi(data, False)
+        elif args.none:
+            clean = tidy.remove_ansi(data)
+        else:
+            clean = data
+
+        # Do tidying
+
+        if args.tidy:
+            clean = tidy.remove_sha256(clean)
+            clean = tidy.remove_sha1(clean)
+            clean = tidy.remove_times(clean)
+
+            if not args.light and not args.dark:
+                clean = tidy.remove_ansi(clean)
+
+        # Remove AWS ID values
+
+        if args.aws:
+            clean = tidy.remove_aws_ids(clean)
+
+        # Do things with Terraform log data
+
+        if args.terraform:
+            if TF_HAS_CHANGED.fullmatch(clean):
+                ignore_until = '    }'
+
+            elif TF_READ_DURING_APPLY.fullmatch(clean):
+                ignore_until = '    }'
+
+            elif clean == TF_OBJECTS_CHANGED:
+                ignore_until = TF_OBJECTS_CHANGED_END
+
+            elif TF_UNCHANGED_HIDDEN.fullmatch(clean):
+                clean = None
+
+            elif clean == TF_IGNORE_MSG:
+                clean = None
+
+            elif TF_UNCHANGED_TAG.fullmatch(clean):
+                clean = None
+
+            elif TF_REFRESHING_AND_READING.match(clean) or TF_FINDING_AND_INSTALLING.fullmatch(clean):
+                # Collect a block of non-deterministically-ordered data
+
+                collection.append(clean)
+                clean = None
+
+            elif collection:
+                # If we collected a block, write it out in sorted order
+
+                collection.sort()
+                for entry in collection:
+                    outfile.write(entry)
+                    outfile.write('\n')
+
+                collection = []
+
+        # Write normal output, skipping >1 blank lines
+
+        if clean is not None and not ignore_until:
+            if clean != '' or prev_line != '':
+                outfile.write(clean)
+                outfile.write('\n')
+
+            prev_line = clean
+
+        # Clear the ignore flag if we've hid the end marker of the block
+
+        if ignore_until and clean == ignore_until:
+            ignore_until = None
+
+################################################################################
+
+def main():
+    """ Main code """
+
+    # Process command line options
+
+    args = parse_command_line()
+
+    # We are either processing 1 or more files, or just piping stdin to stdout
+
+    if args.files:
+        try:
+            for filename in args.files:
+                with open(filename, 'rt') as infile:
+
+                    # Either write to stdout or to a temporary file
+
+                    if args.out:
+                        outfile = sys.stdout
+                        outfile_name = None
+                    else:
+                        outfile = tempfile.NamedTemporaryFile(mode='wt+', delete=False)
+                        outfile_name = outfile.name
+
+                    cleanfile(args, infile, outfile)
+
+                    if outfile_name:
+                        outfile.close()
+
+                # If we wrote to an output file then do something
+
+                if outfile_name:
+                    if args.dir:
+                        # Writing to a directory - just move the output file there unconditionally
+
+                        shutil.move(outfile_name, os.path.join(args.dir, os.path.basename(filename)))
+
+                    elif not filecmp.cmp(outfile_name, filename, shallow=False):
+                        # Only backup and write the source file if changes have been made (original is
+                        # left in place without a backup if nothing's changed)
+
+                        files.backup(filename)
+
+                        shutil.move(outfile_name, filename)
+
+                    else:
+                        # Output file hasn't been used, so just delete it
+
+                        os.unlink(outfile_name)
+
+        except FileNotFoundError as exc:
+            print(exc)
+            sys.exit(1)
+
+    else:
+        cleanfile(args, sys.stdin, sys.stdout)
+
+################################################################################
+
+def readable():
+    """Entry point"""
+
+    try:
+        main()
+    except KeyboardInterrupt:
+        sys.exit(1)
+    except BrokenPipeError:
+        sys.exit(2)
+
+################################################################################
+
+if __name__ == '__main__':
+    readable()

skilleter_thingy/remdir.py

@@ -0,0 +1,126 @@
+#! /usr/bin/env python3
+
+################################################################################
+""" remdir - remove empty directories
+
+    Given the name of a directory tree and, optionally, a list of files to ignore,
+    recursively deletes any directory in the tree that is either completely empty
+    or contains nothing but files matching the list.
+
+    For example:
+
+        remdir /backup --ignore '*.bak' --keep .stfolder
+
+    would remove any directory within the /backup tree that is empty
+    or contains only files that match the '*.bak' wildcard so long as
+    it isn't called '.stfolder'.
+
+    TODO: Using os.walk() means that, if a directory is deleted, it still shows
+          up in the list of directories in the parent directory (?)
+"""
+################################################################################
+
+import sys
+import os
+import argparse
+import fnmatch
+import shutil
+
+import thingy.colour as colour
+import thingy.logger as logger
+
+log = logger.init('remdir')
+
+################################################################################
+
+def main():
+    """ Entry point """
+
+    # Parse the command line
+
+    parser = argparse.ArgumentParser(description='Remove empty directories')
+    parser.add_argument('-D', '--dry-run', action='store_true', help='Dry-run - report what would be done without doing anything')
+    parser.add_argument('--debug', action='store_true', help='Output debug information')
+    parser.add_argument('--verbose', action='store_true', help='Output verbose information')
+    parser.add_argument('-I', '--ignore', action='append', help='Files to ignore when considering whether a directory is empty')
+    parser.add_argument('-K', '--keep', action='append', help='Directories that should be kept even if they are empty')
+    parser.add_argument('dirs', nargs='+', help='Directories to prune')
+    args = parser.parse_args()
+
+    if args.debug:
+        log.setLevel(logger.DEBUG)
+
+    # Go through each directory
+
+    for directory in args.dirs:
+        if not os.path.isdir(directory):
+            colour.write(f'"{directory}" is not a directory')
+            continue
+
+        # Walk through the directory tree in bottom-up order
+
+        for root, dirs, files in os.walk(directory, topdown=False):
+            log.debug('')
+            log.debug('Directory: %s', root)
+            log.debug('     Sub-directories : %s', dirs)
+            log.debug('     Files           : %s', files)
+
+            # Only consider directories with no subdirectories
+
+            if dirs:
+                log.debug('Ignoring directory "%s" as it has %d subdirectories', root, len(dirs))
+            else:
+                # Count of files (if any) to preserve in the directory
+
+                filecount = len(files)
+
+                # If any file matches an entry in the ignore list (if we have one) then decrement the file count
+
+                if args.ignore:
+                    for file in files:
+                        for ignore in args.ignore:
+                            if fnmatch.fnmatch(file, ignore):
+                                filecount -= 1
+                                break
+
+                # If no non-matching files then delete the directory unless it is in the keep list
+
+                if filecount == 0:
+                    keep_dir = False
+                    for keep in args.keep:
+                        if fnmatch.fnmatch(os.path.basename(root), keep):
+                            keep_dir = True
+                            break
+
+                    if keep_dir:
+                        colour.write(f'Keeping empty directory [BLUE:{root}]')
+                    else:
+                        log.debug('Deleting directory: %s', root)
+                        colour.write(f'Deleting "[BLUE:{root}]"')
+
+                        if not args.dry_run:
+                            # Delete the directory and contents
+
+                            try:
+                                shutil.rmtree(root)
+                            except OSError:
+                                colour.error('Unable to delete "[BLUE:{root}]"')
+                else:
+                    log.debug('Ignoring directory "%s" as it has %d non-ignorable files', root, filecount)
+
+################################################################################
+
+def remdir():
+    """Entry point"""
+
+    try:
+        main()
+    except KeyboardInterrupt:
+        sys.exit(1)
+    except BrokenPipeError:
+        sys.exit(2)
+
+################################################################################
+
+if __name__ == '__main__':
+    remdir()