skilleter-thingy 0.3.14__py3-none-any.whl

skilleter_thingy/py_audit.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+
+""" Query api.osv.dev to determine whether a specified version of a particular
+    Python package is subject to known security vulnerabilities """
+
+################################################################################
+
+import sys
+import requests
+import subprocess
+import tempfile
+import re
+import glob
+import argparse
+
+################################################################################
+
+PIP_PACKAGES = ('pip', 'pkg_resources', 'setuptools')
+PIP_OPTIONS = '--no-cache-dir'
+
+QUERY_URL = "https://api.osv.dev/v1/query"
+QUERY_HEADERS = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'}
+
+ANSI_NORMAL = '\x1b[0m'
+ANSI_BOLD = '\x1b[1m'
+
+################################################################################
+
+def audit(package, version):
+    """ Main function """
+
+    # Query api.osv.dev for known vulnerabilties in this version of the package
+
+    payload = '{"version": "'+version+'", "package": {"name": "'+package+'", "ecosystem": "PyPI"}}'
+    result = requests.post(QUERY_URL, data=payload, headers=QUERY_HEADERS)
+
+    # Parse and report the results
+
+    details = result.json()
+
+    print('-' * 80)
+    if package in PIP_PACKAGES:
+        print(f'Package: {package} {version} (part of Pip)')
+    else:
+        print(f'Package: {package} {version}')
+
+    print()
+
+    if 'vulns' in details:
+        print(f'{len(details["vulns"])} known vulnerabilities')
+
+        for v in details['vulns']:
+            print()
+            print(f'{ANSI_BOLD}Vulnerability: {v["id"]}{ANSI_NORMAL}')
+
+            if 'summary' in v:
+                print(f'Summary:       {v["summary"]}')
+
+            if 'aliases' in v:
+                print('Aliases:       %s' % (', '.join(v['aliases'])))
+
+            if 'details' in v:
+                print()
+                print(v['details'])
+    else:
+        print('No known vulnerabilities')
+
+################################################################################
+
+def main():
+    """ Entry point """
+
+    parser = argparse.ArgumentParser(
+        description='Query api.osv.dev to determine whether Python packagers in a requirments.txt file are subject to known security vulnerabilities')
+    parser.add_argument('requirements', nargs='*', type=str, action='store',
+                        help='The requirements file (if not specified, then the script searches for a requirements.txt file)')
+    args = parser.parse_args()
+
+    requirements = args.requirements or glob.glob('**/requirements.txt', recursive=True)
+
+    if not requirements:
+        print('No requirements.txt file(s) found')
+        sys.exit(0)
+
+    # Create a venv for each requirements file, install pip and the packages
+    # and prerequisites, get the list of installed package versions
+    # and check each one.
+
+    for requirement in requirements:
+        print('=' * 80)
+        print(f'{ANSI_BOLD}File: {requirement}{ANSI_NORMAL}')
+        print()
+
+        with tempfile.TemporaryDirectory() as env_dir:
+            with tempfile.NamedTemporaryFile() as package_list:
+                script = f'python3 -m venv {env_dir}' \
+                         f' && . {env_dir}/bin/activate' \
+                         f' && python3 -m pip {PIP_OPTIONS} install --upgrade pip setuptools' \
+                         f' && python3 -m pip {PIP_OPTIONS} install -r {requirement}' \
+                         f' && python3 -m pip {PIP_OPTIONS} list | tail -n+3 | tee {package_list.name}' \
+                          ' && deactivate'
+
+                try:
+                    subprocess.run(script, check=True, shell=True)
+
+                except subprocess.CalledProcessError as exc:
+                    print(f'ERROR #{exc.returncode}: {exc.stdout}')
+                    sys.exit(exc.returncode)
+
+                with open(package_list.name) as infile:
+                    for package in infile.readlines():
+                        package_info = re.split(' +', package.strip())
+
+                        audit(package_info[0], package_info[1])
+
+################################################################################
+
+def py_audit():
+    """Entry point"""
+
+    try:
+        main()
+
+    except KeyboardInterrupt:
+        sys.exit(1)
+
+    except BrokenPipeError:
+        sys.exit(2)
+
+################################################################################
+
+if __name__ == '__main__':
+    py_audit()

skilleter_thingy/remdir.py

@@ -0,0 +1,127 @@
+#! /usr/bin/env python3
+
+################################################################################
+""" remdir - remove empty directories
+
+    Given the name of a directory tree and, optionally, a list of files to ignore,
+    recursively deletes any directory in the tree that is either completely empty
+    or contains nothing but files matching the list.
+
+    For example:
+
+        remdir /backup --ignore '*.bak' --keep .stfolder
+
+    would remove any directory within the /backup tree that is empty
+    or contains only files that match the '*.bak' wildcard so long as
+    it isn't called '.stfolder'.
+
+    TODO: Using os.walk() means that, if a directory is deleted, it still shows
+          up in the list of directories in the parent directory (?)
+"""
+################################################################################
+
+import sys
+import os
+import argparse
+import fnmatch
+import shutil
+import logging
+
+from skilleter_modules import colour
+
+################################################################################
+
+def main():
+    """ Entry point """
+
+    # Parse the command line
+
+    parser = argparse.ArgumentParser(description='Remove empty directories')
+    parser.add_argument('--dry-run', '-D', action='store_true', help='Dry-run - report what would be done without doing anything')
+    parser.add_argument('--debug', action='store_true', help='Output debug information')
+    parser.add_argument('--verbose', action='store_true', help='Output verbose information')
+    parser.add_argument('--ignore', '-I', action='append', help='Files to ignore when considering whether a directory is empty')
+    parser.add_argument('--keep', '-K', action='append', help='Directories that should be kept even if they are empty')
+    parser.add_argument('dirs', nargs='+', help='Directories to prune')
+    args = parser.parse_args()
+
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+
+    if not args.keep:
+        args.keep = []
+
+    # Go through each directory
+
+    for directory in args.dirs:
+        if not os.path.isdir(directory):
+            colour.write(f'"{directory}" is not a directory')
+            continue
+
+        # Walk through the directory tree in bottom-up order
+
+        for root, dirs, files in os.walk(directory, topdown=False):
+            logging.debug('')
+            logging.debug('Directory: %s', root)
+            logging.debug('     Sub-directories : %s', dirs)
+            logging.debug('     Files           : %s', files)
+
+            # Only consider directories with no subdirectories
+
+            if dirs:
+                logging.debug('Ignoring directory "%s" as it has %d subdirectories', root, len(dirs))
+            else:
+                # Count of files (if any) to preserve in the directory
+
+                filecount = len(files)
+
+                # If any file matches an entry in the ignore list (if we have one) then decrement the file count
+
+                if args.ignore:
+                    for file in files:
+                        for ignore in args.ignore:
+                            if fnmatch.fnmatch(file, ignore):
+                                filecount -= 1
+                                break
+
+                # If no non-matching files then delete the directory unless it is in the keep list
+
+                if filecount == 0:
+                    keep_dir = False
+                    for keep in args.keep:
+                        if fnmatch.fnmatch(os.path.basename(root), keep):
+                            keep_dir = True
+                            break
+
+                    if keep_dir:
+                        colour.write(f'Keeping empty directory [BLUE:{root}]')
+                    else:
+                        logging.debug('Deleting directory: %s', root)
+                        colour.write(f'Deleting "[BLUE:{root}]"')
+
+                        if not args.dry_run:
+                            # Delete the directory and contents
+
+                            try:
+                                shutil.rmtree(root)
+                            except OSError:
+                                colour.error('Unable to delete "[BLUE:{root}]"')
+                else:
+                    logging.debug('Ignoring directory "%s" as it has %d non-ignorable files', root, filecount)
+
+################################################################################
+
+def remdir():
+    """Entry point"""
+
+    try:
+        main()
+    except KeyboardInterrupt:
+        sys.exit(1)
+    except BrokenPipeError:
+        sys.exit(2)
+
+################################################################################
+
+if __name__ == '__main__':
+    remdir()

skilleter_thingy/rpylint.py

@@ -0,0 +1,98 @@
+#! /usr/bin/env python3
+
+################################################################################
+""" Run pylint on all the Python source files in the current tree
+
+    Copyright (C) 2017-18 John Skilleter """
+################################################################################
+
+import os
+import sys
+import argparse
+import glob
+import subprocess
+
+################################################################################
+
+PYLINT = 'pylint'
+
+################################################################################
+
+def main():
+    """ Main code. Exits directly on failure to locate source files, or returns
+        the status code from Pylint otherwise. """
+
+    # Parse the command line
+
+    parser = argparse.ArgumentParser(description='Run pylint in the current (or specified) directory/ies')
+
+    parser.add_argument('paths', nargs='*', help='List of files or paths to lint')
+
+    args = parser.parse_args()
+
+    if not args.paths:
+        args.paths = ['.']
+
+    sourcefiles = []
+
+    # Use rgrep to find source files that have a Python 3 hashbang
+
+    for entry in args.paths:
+        if os.path.isdir(entry):
+            result = subprocess.run(['rgrep', '-E', '--exclude-dir=.git', '-l', '#![[:space:]]*/usr/bin/(env[[:space:]])?python3'] + args.paths,
+                                    capture_output=True, check=False, text=True)
+
+            if result.returncode == 1:
+                sys.stderr.write('No Python3 source files found\n')
+                sys.exit(2)
+
+            if result.returncode:
+                sys.stderr.write(f'ERROR #{result.returncode}: {result.stderr}')
+                sys.exit(1)
+
+            sourcefiles += result.stdout.split('\n')
+
+        elif os.path.isfile(entry):
+            sourcefiles.append(entry)
+        else:
+            files = glob.glob(entry)
+
+            if not files:
+                sys.stderr.write(f'No files found matching "{entry}"')
+                sys.exit(2)
+
+            sourcefiles += files
+
+    # Run pylint on all the files
+
+    try:
+        result = subprocess.run([PYLINT, '--output-format', 'parseable'] + sourcefiles, capture_output=False, check=False, text=True)
+
+    except FileNotFoundError:
+        sys.stderr.write(f'Unable to locate {PYLINT}\n')
+        sys.exit(1)
+
+    if result.returncode >= 64:
+        sys.stderr.write(f'Unexpected error: {result.returncode}\n')
+        sys.exit(3)
+
+    # Function return code is the status return from pylint
+
+    return result.returncode
+
+################################################################################
+
+def rpylint():
+    """Entry point"""
+
+    try:
+        sys.exit(main())
+    except KeyboardInterrupt:
+        sys.exit(1)
+    except BrokenPipeError:
+        sys.exit(2)
+
+################################################################################
+
+if __name__ == '__main__':
+    rpylint()

skilleter_thingy/strreplace.py

@@ -0,0 +1,82 @@
+#! /usr/bin/env python3
+
+################################################################################
+""" Textual search and replace
+
+   For those occasions when you want to search and replace strings with
+   regexppy characters that upset sed.
+
+   Copyright (C) 2018 John Skilleter """
+################################################################################
+
+import os
+import sys
+import argparse
+import tempfile
+
+################################################################################
+
+def main():
+    """ Main function """
+
+    parser = argparse.ArgumentParser(description='Textual search and replace')
+    parser.add_argument('--inplace', '-i', action='store_true', help='Do an in-place search and replace on the input file')
+    parser.add_argument('search', nargs=1, action='store', help='Search text')
+    parser.add_argument('replace', nargs=1, action='store', help='Replacment text')
+    parser.add_argument('infile', nargs='?', action='store', help='Input file')
+    parser.add_argument('outfile', nargs='?', action='store', help='Output file')
+
+    args = parser.parse_args()
+
+    # Sanity check
+
+    if args.inplace and not args.infile or args.outfile:
+        print('For in-place operations you must specify and input file and no output file')
+
+    # Open the input file
+
+    if args.infile:
+        infile = open(args.infile, 'r')
+    else:
+        infile = sys.stdin
+
+    # Open the output file, using a temporary file in the same directory as the input file
+    # if we are doing in-place operations
+
+    if args.outfile:
+        outfile = open(args.outfile, 'w')
+    elif args.inplace:
+        outfile = tempfile.NamedTemporaryFile(mode='w', delete=False, dir=os.path.dirname(args.infile))
+    else:
+        outfile = sys.stdout
+
+    # Perform the searchy-replacey-ness
+
+    for data in infile:
+        outfile.write(data.replace(args.search[0], args.replace[0]))
+
+    # If we doing in-place then juggle the temporary and input files
+
+    if args.inplace:
+        mode = os.stat(args.infile).st_mode
+        outfile.close()
+        infile.close()
+        os.rename(outfile.name, args.infile)
+        os.chmod(args.infile, mode)
+
+################################################################################
+
+def strreplace():
+    """Entry point"""
+
+    try:
+        main()
+    except KeyboardInterrupt:
+        sys.exit(1)
+    except BrokenPipeError:
+        sys.exit(2)
+
+################################################################################
+
+if __name__ == '__main__':
+    strreplace()

skilleter_thingy/test.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+print('Importing all Skilleter-thingy modules')
+import addpath
+import console_colours
+import ffind
+import ggit
+import ggrep
+import git_br
+import git_ca
+import git_cleanup
+import git_co
+import git_common
+import git_hold
+import git_parent
+import git_retag
+import git_review
+import git_update
+import git_wt
+import gitcmp_helper
+import gitprompt
+import linecount
+import multigit
+import py_audit
+import remdir
+import rpylint
+import strreplace
+import tfm
+import tfparse
+import trimpath
+import venv_create
+import xchmod
+import yamlcheck
+print('Import successful')