skilleter-thingy 0.0.22__py3-none-any.whl

skilleter_thingy/linecount.py

@@ -0,0 +1,155 @@
+#! /usr/bin/env python3
+
+################################################################################
+""" Count lines of code by filetype
+"""
+################################################################################
+
+import os
+import sys
+import argparse
+
+import filetype
+
+from skilleter_thingy import files
+
+################################################################################
+
+def guess_filetype(filepath):
+    """ Guess the type of a file """
+
+    binary = False
+
+    # Check for the filetype (usually detects binary files)
+
+    ftype = filetype.guess(filepath)
+
+    # If we have a type, store the extension and, as it is binary
+    # set the number of lines to 0
+    # Otherwise, work out the filetype and count the lines
+
+    if ftype:
+        ext = ftype.extension
+        binary = True
+    else:
+        filename = os.path.split(filepath)[1]
+
+        if '.' in filename:
+            ext = filename.split('.')[-1]
+        else:
+            if filename.startswith('Jenkins'):
+                ext = 'Jenkins'
+            elif filename.startswith('Docker'):
+                ext = 'Docker'
+            else:
+                ext = filename
+
+    return ext, binary
+
+################################################################################
+
+def determine_filetype(filepath):
+    """ Determine the type of a file """
+
+    file_type = files.file_type(filepath)
+
+    if file_type.startswith('a /usr/bin/env '):
+        file_type = file_type[len('a /usr/bin/env '):]
+    elif file_type.startswith('symbolic link to '):
+        file_type = 'Symbolic link'
+
+    if file_type[0].islower():
+        file_type = file_type.capitalize()
+
+    ext = file_type.split(',')[0]
+    binary = 'text' not in file_type
+
+    if file_type.startswith('ASCII text'):
+        return guess_filetype(filepath)
+
+    return ext, binary
+
+################################################################################
+
+def main():
+    """ Report Summary of files by name or extension """
+
+    parser = argparse.ArgumentParser(description='Summarise number of files, lines of text and total size of files in a directory tree')
+    parser.add_argument('-e-', '--ext', action='store_true', help='Identify file type using the file extension (faster but less accurrate)')
+
+    args = parser.parse_args()
+
+    filetypes = {}
+
+    # Wander down the tree
+
+    for dirpath, dirnames, filenames in os.walk('.'):
+        # Skip .git directories
+
+        if '.git' in dirnames:
+            dirnames.remove('.git')
+
+        for filename in filenames:
+            # Get the file path and size
+
+            filepath = os.path.join(dirpath, filename)
+            size = os.stat(filepath).st_size
+
+            if args.ext:
+                ext, binary = guess_filetype(filepath)
+            else:
+                ext, binary = determine_filetype(filepath)
+
+            if binary:
+                lines = 0
+            else:
+                with open(filepath, 'rb') as infile:
+                    lines = len(infile.readlines())
+
+            # Update the summary
+
+            if ext in filetypes:
+                filetypes[ext]['files'] += 1
+                filetypes[ext]['size'] += size
+                filetypes[ext]['lines'] += lines
+            else:
+                filetypes[ext] = {'files': 1, 'size': size, 'lines': lines}
+
+    # Work out the maximum size of each field of data
+
+    total_files = 0
+    total_lines = 0
+    total_size = 0
+
+    for ext in sorted(filetypes.keys()):
+        total_files += filetypes[ext]['files']
+        total_lines += filetypes[ext]['lines']
+        total_size += filetypes[ext]['size']
+
+        size = files.format_size(filetypes[ext]['size'])
+        print(f"{ext}: {filetypes[ext]['files']:,} files, {filetypes[ext]['lines']:,} lines, {size}")
+
+    size = files.format_size(total_size)
+
+    print()
+    print(f'Total files: {total_files:,}')
+    print(f'Total lines: {total_lines:,}')
+    print(f'Total size:  {size}')
+
+################################################################################
+
+def linecount():
+    """Entry point"""
+
+    try:
+        main()
+
+    except KeyboardInterrupt:
+        sys.exit(1)
+    except BrokenPipeError:
+        sys.exit(2)
+
+################################################################################
+
+if __name__ == '__main__':
+    linecount()

skilleter_thingy/logger.py

@@ -0,0 +1,112 @@
+#! /usr/bin/env python3
+
+################################################################################
+""" Thingy logging functionality - wraps the Pythong logging module
+
+    Copyright (c) 2017 John Skilleter
+
+    Licence: GPL v3 or later
+"""
+################################################################################
+
+import os
+
+import logging
+
+################################################################################
+
+CRITICAL = logging.CRITICAL
+ERROR = logging.ERROR
+WARNING = logging.WARNING
+INFO = logging.INFO
+DEBUG = logging.DEBUG
+NOTSET = logging.NOTSET
+
+LOG_LEVELS = {
+    'CRITICAL': CRITICAL, 'ERROR': ERROR, 'WARNING': WARNING, 'INFO': INFO,
+    'DEBUG': DEBUG, 'NOTSET': NOTSET
+}
+
+__config_done__ = False
+
+################################################################################
+
+def set_logging(log, name):
+    """ If an environment variable called NAME_DEBUG is set and defines a
+        log level that is more verbose than the current level then set
+        that level (you can only increase verbosity via the variable, not
+        decrease it). """
+
+    # Check whether there is an environment variable setting the debug level
+
+    env_name = '%s_DEBUG' % name.upper()
+
+    value = os.getenv(env_name, None)
+
+    if value is not None:
+        value = value.upper()
+
+        current = log.getEffectiveLevel()
+
+        # Check for a textual level in the value and if no match, try
+        # for an integer level ignoring invalid values.
+
+        if value in LOG_LEVELS:
+            if current > LOG_LEVELS[value]:
+                log.setLevel(LOG_LEVELS[value])
+        else:
+            try:
+                intlevel = int(value)
+
+                if current > intlevel:
+                    log.setLevel(intlevel)
+
+            except ValueError:
+                pass
+
+    return log
+
+################################################################################
+
+def init(name):
+    """ Initilise logging and create a logger.
+        If the environment variable NAME_DEBUG is set to a value in LOG_LEVELS
+        then the log level is set to that level. If NAME_DEBUG is an integer
+        then the same applies, otherwise, by default, the log level is CRITICAL """
+
+    # Create the new logger
+
+    log = logging.getLogger(name)
+
+    # Default log level is CRITICAL
+
+    log.setLevel(CRITICAL)
+
+    # Set logging according to the value of THINGY_DEBUG (if set) then
+    # override with the logger-specific variable (again, if set)
+
+    set_logging(log, 'THINGY')
+    set_logging(log, name)
+
+    return log
+
+################################################################################
+# Entry point
+
+# Ensure that the logging module is initialise
+
+if not __config_done__:
+    logging.basicConfig()
+    __config_done__ = True
+
+if __name__ == '__main__':
+    demo = init('wombat')
+
+    demo.critical('Critical error')
+
+    # These messages should only appear if the WOMBAT_DEBUG environment variable
+    # is set to an appropriate value (ERROR, WARNING or INFO)
+
+    demo.error('Error message')
+    demo.warning('Warning message')
+    demo.info('Info message')

skilleter_thingy/moviemover.py

@@ -0,0 +1,133 @@
+#! /usr/bin/env python3
+
+""" Search for files matching a wildcard in a directory tree and move them to an
+    equivalent location in a different tree """
+
+import argparse
+import os
+import sys
+import glob
+import pathlib
+import shutil
+import filecmp
+
+################################################################################
+
+def error(msg, status=1):
+    """ Exit with an error message """
+
+    print(msg)
+
+    sys.exit(status)
+
+################################################################################
+
+def parse_command_line():
+    """ Handle command line arguments """
+
+    parser = argparse.ArgumentParser(description='File relocation - move files by wildcard from one directory tree to another')
+
+    parser.add_argument('-s', '--source', type=str, required=True, help='Source directory')
+    parser.add_argument('-d', '--destination', type=str, required=True, help='Destination directory')
+    parser.add_argument('-D', '--dry-run', action='store_true', help='Report what files would be moved, without actually moving them')
+    parser.add_argument('files', nargs='*', help='List of wildcard matches')
+
+    args = parser.parse_args()
+
+    if not args.files:
+        print('You must specify at least one wildcard/regex parameter')
+
+    if not os.path.isdir(args.source):
+        error(f'{args.source} is not a directory')
+
+    if not os.path.isdir(args.destination):
+        error(f'{args.destination} is not a directory')
+
+    args.source_path = pathlib.Path(os.path.realpath(args.source))
+    args.destination_path = pathlib.Path(os.path.realpath(args.destination))
+
+    if args.source_path == args.destination_path:
+        error('Source and destination paths cannot be the same')
+
+    if args.source_path in args.destination_path.parents:
+        error('The destination directory cannot be within the source path')
+
+    if args.destination_path in args.source_path.parents:
+        error('The source directory cannot be within the destination path')
+
+    return args
+
+################################################################################
+
+def main():
+    """ Entry point """
+
+    args = parse_command_line()
+
+    # Process each wildcard
+
+    for wild in args.files:
+        # Find matching files in the source tree
+
+        for source_file in args.source_path.glob(f'**/{wild}'):
+            # Ignore anything that isn't a file
+
+            if source_file.is_file():
+                # Determine where to put it
+
+                dest_file = args.destination_path / source_file.relative_to(args.source_path)
+
+                if dest_file.exists():
+
+                    if filecmp.cmp(source_file, dest_file, shallow=False):
+                        print(f'Destination file {dest_file} already exists and is identical, so deleting source')
+                        if not args.dry_run:
+                            os.unlink(source_file)
+                    else:
+                        print(f'Destination file {dest_file} already exists and is DIFFERENT')
+                else:
+                    # If the destination directory doesn't exist, then create it
+
+                    if not dest_file.parent.is_dir():
+                        print(f'Creating directory {dest_file.parent}')
+
+                        if not args.dry_run:
+                            dest_file.parent.mkdir(parents=True, exist_ok=True)
+
+                    # Move the file
+
+                    print(f'Moving {source_file.name} to {dest_file.parent}')
+
+                    if not args.dry_run:
+                        try:
+                            shutil.move(source_file, dest_file)
+                        except PermissionError:
+                            print(f'WARNING: Permissions error moving {source_file}')
+
+                    # Delete the source directory if it is not empty
+
+                    source_dir = os.path.dirname(source_file)
+
+                    if not glob.glob(source_dir, recursive=True):
+                        print('Deleting directory "{source_dir}"')
+                        if not args.dry_run:
+                            os.path.unlink(source_dir)
+
+################################################################################
+
+def moviemover():
+    """Entry point"""
+
+    try:
+        main()
+
+    except KeyboardInterrupt:
+        sys.exit(1)
+
+    except BrokenPipeError:
+        sys.exit(2)
+
+################################################################################
+
+if __name__ == '__main__':
+    moviemover()

skilleter_thingy/path.py

@@ -0,0 +1,156 @@
+#! /usr/bin/env python3
+
+################################################################################
+""" Thingy file and directory functionality
+
+    Copyright (C) 2017-18 John Skilleter
+
+    Licence: GPL v3 or later
+"""
+################################################################################
+
+import os
+
+from skilleter_thingy import logger
+
+################################################################################
+
+class PathError(Exception):
+    """ Exception raised by the module """
+
+    def __init__(self, msg):
+        super(PathError, self).__init__(msg)
+        self.msg = msg
+
+################################################################################
+
+def is_subdirectory(root_path, sub_path):
+    """ Return True if sub_path is a sub-directory of root_path """
+
+    abs_sub_path = os.path.abspath(sub_path)
+    abs_root_path = os.path.abspath(root_path)
+
+    log.debug('root path: %s', abs_root_path)
+    log.debug('sub path : %s', abs_sub_path)
+
+    return abs_sub_path.startswith('%s/' % abs_root_path)
+
+################################################################################
+
+def trimpath(full_path, trim_width):
+    """ Trim a path to a specified maximum width, but always leaving the
+        lowest-level directory (even if it exceeds the trim width). """
+
+    log.debug('Path: "%s"', full_path)
+    log.debug('Required width: %d', trim_width)
+
+    full_path = os.path.abspath(full_path)
+
+    # Remove any trailing '/' from the path
+
+    if full_path != '/' and full_path[-1] == '/':
+        full_path = full_path[:-1]
+
+    # If the path starts with the user's home directory then convert the prefix
+    # into a '~'
+
+    home_dir = os.path.expanduser('~')
+
+    if full_path == home_dir:
+        full_path = '~'
+        log.debug('Converted path to "~"')
+
+    elif is_subdirectory(home_dir, full_path):
+        full_path = "~/%s" % full_path[len(home_dir) + 1:]
+
+        log.debug('Converted path to "%s"', full_path)
+
+    # If the path is too long then slice it into directories and cut sub-directories
+    # out of the middle until it is short enough. Always leave the last element
+    # in place, even if this means total length exceeds the requirement.
+
+    path_len = len(full_path)
+
+    log.debug('Path length: %d', path_len)
+
+    # Already within maximum width, so just return it
+
+    if path_len <= trim_width:
+        return full_path
+
+    # Split into an array of directories and trim out middle ones
+
+    directories = full_path.split('/')
+
+    log.debug('Path has %d elements: "%s"', len(directories), directories)
+
+    if len(directories) == 1:
+        # If there's only one element in the path, just give up
+
+        log.debug('Only 1 directory in the path, leaving it as-is')
+
+    elif len(directories) == 2:
+        # If there's only two elements in the path then replace the first
+        # element with '...' and give up
+
+        log.debug('Only 2 directories in the path, so setting the first to "..."')
+
+        directories[0] = '...'
+
+    else:
+        # Start in the middle and remove entries to the left and right until the total
+        # path length is shortened to a sufficient extent
+
+        right = len(directories) // 2
+        left = right - 1
+        first = True
+
+        while path_len > trim_width:
+
+            path_len -= len(directories[right]) + 1
+
+            if first:
+                path_len += 4
+                first = False
+
+            if right == len(directories) - 1:
+                break
+
+            right += 1
+
+            if path_len > trim_width:
+                path_len -= len(directories[left]) + 1
+
+                if left == 0:
+                    break
+
+                left -= 1
+
+        log.debug('Removing entries %d..%d from the path', left, right)
+
+        directories = directories[0:left + 1] + ['...'] + directories[right:]
+
+    full_path = '/'.join(directories)
+
+    log.debug('Calculated width is %d and actual width is %d', path_len, len(full_path))
+
+    return full_path
+
+################################################################################
+
+log = logger.init('tgy_path')
+
+if __name__ == '__main__':
+    PARENT = '/1/2/3/5'
+    CHILD = '/1/2/3/5/6'
+
+    print('Is %s a subdirectory of %s: %s (expecting True)' % (CHILD, PARENT, is_subdirectory(PARENT, CHILD)))
+    print('Is %s a subdirectory of %s: %s (expecting False)' % (PARENT, CHILD, is_subdirectory(CHILD, PARENT)))
+
+    LONG_PATH = '/home/jms/source/womble-biscuit-token-generation-service/subdirectory'
+
+    for pathname in (LONG_PATH, os.path.realpath('.')):
+        print('Full path: %s' % pathname)
+
+        for length in (80, 60, 40, 20, 16, 10):
+            print('Trimmed to %d characters: %s' % (length, trimpath(pathname, length)))

skilleter_thingy/photodupe.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""Hash photos to find closely-similar images and report them"""
+
+import sys
+import os
+import sys
+import pickle
+
+import PIL
+
+from collections import defaultdict
+
+from PIL import Image
+import imagehash
+
+################################################################################
+
+def read_image_hashes():
+    """Read all the specfied directories and hash every picture therein"""
+
+    hashes = defaultdict(list)
+
+    for directory in sys.argv[1:]:
+        for root, _, files in os.walk(directory):
+            for file in files:
+                filepath = os.path.join(root, file)
+
+                try:
+                    with Image.open(filepath) as image:
+                        hash_value = imagehash.average_hash(image, hash_size=12)
+
+                        size = os.stat(filepath).st_size
+                        hashes[hash_value].append({'path': filepath, 'width': image.width, 'height': image.height, 'size': size})
+
+                except PIL.UnidentifiedImageError:
+                    sys.stderr.write(f'ERROR: Unrecognized format {filepath}\n')
+
+                except OSError:
+                    sys.stderr.write(f'ERROR: Unable to read {filepath}\n')
+    return hashes
+
+################################################################################
+
+def main():
+    """Read the hashes and report duplicates in a vaguely civilised way"""
+
+    try:
+        print('Loading cached data')
+
+        with open('photodupe.pickle', 'rb') as pickles:
+            hashes = pickle.load(pickles)
+    except (FileNotFoundError, EOFError):
+        print('Scanning directories')
+
+        hashes = read_image_hashes()
+
+    print('Sorting hashes')
+
+    hash_values = sorted([str(hashval) for hashval in hashes])
+
+    for hash_value in hash_values:
+        if len(hashes[hash_value]) > 1:
+            print(hash_value)
+            max_len = 0
+            min_size = None
+
+            for entry in hashes[hash_value]:
+                max_len = max(max_len, len(entry['path']))
+
+                if min_size is None:
+                    min_size = entry['size']
+                else:
+                    min_size = min(min_size, entry['size'])
+
+                if min_size >= 1024 * 1024:
+                    size_suffix = 'MiB'
+                    size_div = 1024*1024
+
+                elif min_size > 1024:
+                    size_suffix = 'KiB'
+                    size_div = 1024
+                else:
+                    size_div = 1
+                    size_suffix = ''
+
+            for entry in hashes[hash_value]:
+                size = entry['size'] // size_div
+                print(f'    {entry["path"]:{max_len}} {size:>4} {size_suffix} ({entry["width"]}x{entry["height"]})')
+
+    with open('photodupe.pickle', 'wb') as pickles:
+        pickle.dump(hashes, pickles)
+
+################################################################################
+
+def photodupe():
+    """Entry point"""
+
+    try:
+        main()
+
+    except KeyboardInterrupt:
+        sys.exit(1)
+
+    except BrokenPipeError:
+        sys.exit(2)
+
+################################################################################
+
+if __name__ == '__main__':
+    photodupe()