skilleter-thingy 0.0.40__py3-none-any.whl → 0.0.41__py3-none-any.whl

skilleter_thingy/linecount.py

@@ -0,0 +1,155 @@
+#! /usr/bin/env python3
+
+################################################################################
+""" Count lines of code by filetype
+"""
+################################################################################
+
+import os
+import sys
+import argparse
+
+import filetype
+
+import thingy.files as files
+
+################################################################################
+
+def guess_filetype(filepath):
+    """ Guess the type of a file """
+
+    binary = False
+
+    # Check for the filetype (usually detects binary files)
+
+    ftype = filetype.guess(filepath)
+
+    # If we have a type, store the extension and, as it is binary
+    # set the number of lines to 0
+    # Otherwise, work out the filetype and count the lines
+
+    if ftype:
+        ext = ftype.extension
+        binary = True
+    else:
+        filename = os.path.split(filepath)[1]
+
+        if '.' in filename:
+            ext = filename.split('.')[-1]
+        else:
+            if filename.startswith('Jenkins'):
+                ext = 'Jenkins'
+            elif filename.startswith('Docker'):
+                ext = 'Docker'
+            else:
+                ext = filename
+
+    return ext, binary
+
+################################################################################
+
+def determine_filetype(filepath):
+    """ Determine the type of a file """
+
+    file_type = files.file_type(filepath)
+
+    if file_type.startswith('a /usr/bin/env '):
+        file_type = file_type[len('a /usr/bin/env '):]
+    elif file_type.startswith('symbolic link to '):
+        file_type = 'Symbolic link'
+
+    if file_type[0].islower():
+        file_type = file_type.capitalize()
+
+    ext = file_type.split(',')[0]
+    binary = 'text' not in file_type
+
+    if file_type.startswith('ASCII text'):
+        return guess_filetype(filepath)
+
+    return ext, binary
+
+################################################################################
+
+def main():
+    """ Report Summary of files by name or extension """
+
+    parser = argparse.ArgumentParser(description='Summarise number of files, lines of text and total size of files in a directory tree')
+    parser.add_argument('-e-', '--ext', action='store_true', help='Identify file type using the file extension (faster but less accurrate)')
+
+    args = parser.parse_args()
+
+    filetypes = {}
+
+    # Wander down the tree
+
+    for dirpath, dirnames, filenames in os.walk('.'):
+        # Skip .git directories
+
+        if '.git' in dirnames:
+            dirnames.remove('.git')
+
+        for filename in filenames:
+            # Get the file path and size
+
+            filepath = os.path.join(dirpath, filename)
+            size = os.stat(filepath).st_size
+
+            if args.ext:
+                ext, binary = guess_filetype(filepath)
+            else:
+                ext, binary = determine_filetype(filepath)
+
+            if binary:
+                lines = 0
+            else:
+                with open(filepath, 'rb') as infile:
+                    lines = len(infile.readlines())
+
+            # Update the summary
+
+            if ext in filetypes:
+                filetypes[ext]['files'] += 1
+                filetypes[ext]['size'] += size
+                filetypes[ext]['lines'] += lines
+            else:
+                filetypes[ext] = {'files': 1, 'size': size, 'lines': lines}
+
+    # Work out the maximum size of each field of data
+
+    total_files = 0
+    total_lines = 0
+    total_size = 0
+
+    for ext in sorted(filetypes.keys()):
+        total_files += filetypes[ext]['files']
+        total_lines += filetypes[ext]['lines']
+        total_size += filetypes[ext]['size']
+
+        size = files.format_size(filetypes[ext]['size'])
+        print(f"{ext}: {filetypes[ext]['files']:,} files, {filetypes[ext]['lines']:,} lines, {size}")
+
+    size = files.format_size(total_size)
+
+    print()
+    print(f'Total files: {total_files:,}')
+    print(f'Total lines: {total_lines:,}')
+    print(f'Total size:  {size}')
+
+################################################################################
+
+def linecount():
+    """Entry point"""
+
+    try:
+        main()
+
+    except KeyboardInterrupt:
+        sys.exit(1)
+    except BrokenPipeError:
+        sys.exit(2)
+
+################################################################################
+
+if __name__ == '__main__':
+    linecount()

skilleter_thingy/moviemover.py

@@ -0,0 +1,133 @@
+#! /usr/bin/env python3
+
+""" Search for files matching a wildcard in a directory tree and move them to an
+    equivalent location in a different tree """
+
+import argparse
+import os
+import sys
+import glob
+import pathlib
+import shutil
+import filecmp
+
+################################################################################
+
+def error(msg, status=1):
+    """ Exit with an error message """
+
+    print(msg)
+
+    sys.exit(status)
+
+################################################################################
+
+def parse_command_line():
+    """ Handle command line arguments """
+
+    parser = argparse.ArgumentParser(description='File relocation - move files by wildcard from one directory tree to another')
+
+    parser.add_argument('-s', '--source', type=str, required=True, help='Source directory')
+    parser.add_argument('-d', '--destination', type=str, required=True, help='Destination directory')
+    parser.add_argument('-D', '--dry-run', action='store_true', help='Report what files would be moved, without actually moving them')
+    parser.add_argument('files', nargs='*', help='List of wildcard matches')
+
+    args = parser.parse_args()
+
+    if not args.files:
+        print('You must specify at least one wildcard/regex parameter')
+
+    if not os.path.isdir(args.source):
+        error(f'{args.source} is not a directory')
+
+    if not os.path.isdir(args.destination):
+        error(f'{args.destination} is not a directory')
+
+    args.source_path = pathlib.Path(os.path.realpath(args.source))
+    args.destination_path = pathlib.Path(os.path.realpath(args.destination))
+
+    if args.source_path == args.destination_path:
+        error('Source and destination paths cannot be the same')
+
+    if args.source_path in args.destination_path.parents:
+        error('The destination directory cannot be within the source path')
+
+    if args.destination_path in args.source_path.parents:
+        error('The source directory cannot be within the destination path')
+
+    return args
+
+################################################################################
+
+def main():
+    """ Entry point """
+
+    args = parse_command_line()
+
+    # Process each wildcard
+
+    for wild in args.files:
+        # Find matching files in the source tree
+
+        for source_file in args.source_path.glob(f'**/{wild}'):
+            # Ignore anything that isn't a file
+
+            if source_file.is_file():
+                # Determine where to put it
+
+                dest_file = args.destination_path / source_file.relative_to(args.source_path)
+
+                if dest_file.exists():
+
+                    if filecmp.cmp(source_file, dest_file, shallow=False):
+                        print(f'Destination file {dest_file} already exists and is identical, so deleting source')
+                        if not args.dry_run:
+                            os.unlink(source_file)
+                    else:
+                        print(f'Destination file {dest_file} already exists and is DIFFERENT')
+                else:
+                    # If the destination directory doesn't exist, then create it
+
+                    if not dest_file.parent.is_dir():
+                        print(f'Creating directory {dest_file.parent}')
+
+                        if not args.dry_run:
+                            dest_file.parent.mkdir(parents=True, exist_ok=True)
+
+                    # Move the file
+
+                    print(f'Moving {source_file.name} to {dest_file.parent}')
+
+                    if not args.dry_run:
+                        try:
+                            shutil.move(source_file, dest_file)
+                        except PermissionError:
+                            print(f'WARNING: Permissions error moving {source_file}')
+
+                    # Delete the source directory if it is not empty
+
+                    source_dir = os.path.dirname(source_file)
+
+                    if not glob.glob(source_dir, recursive=True):
+                        print('Deleting directory "{source_dir}"')
+                        if not args.dry_run:
+                            os.path.unlink(source_dir)
+
+################################################################################
+
+def moviemover():
+    """Entry point"""
+
+    try:
+        main()
+
+    except KeyboardInterrupt:
+        sys.exit(1)
+
+    except BrokenPipeError:
+        sys.exit(2)
+
+################################################################################
+
+if __name__ == '__main__':
+    moviemover()

skilleter_thingy/photodupe.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+"""Hash photos to find closely-similar images and report them"""
+
+import sys
+import os
+import sys
+import pickle
+import argparse
+
+import PIL
+
+from collections import defaultdict
+
+from PIL import Image
+import imagehash
+
+################################################################################
+
+def read_image_hashes(directories):
+    """Read all the specfied directories and hash every picture therein"""
+
+    hashes = defaultdict(list)
+
+    # Walk each directory tree
+    
+    for directory in directories:
+        print(f'Scanning directory tree {directory}')
+        
+        for root, _, files in os.walk(directory):
+            print(f'Scanning directory {root}')
+            
+            for file in files:
+                filepath = os.path.join(root, file)
+                
+                fileext = os.path.splitext(file)[1]
+
+                if fileext.lower() not in ('.jbf', '.ini', '.xml', '.ffs_db'):
+                    # Calculate the hash and store path, dimensions and file size under the hash entry in the hashes table
+                    
+                    try:
+                        with Image.open(filepath) as image:
+                            hash_value = imagehash.average_hash(image, hash_size=12)
+
+                            size = os.stat(filepath).st_size
+                            hashes[hash_value].append({'path': filepath, 'width': image.width, 'height': image.height, 'size': size})
+
+                    except PIL.UnidentifiedImageError:
+                        sys.stderr.write(f'ERROR: Unrecognized format {filepath} (size={size})\n')
+
+                    except OSError:
+                        sys.stderr.write(f'ERROR: Unable to read {filepath} (size={size})\n')
+                    
+    # Return the hash table
+    
+    return hashes
+
+################################################################################
+
+def main():
+    """Read the hashes and report duplicates in a vaguely civilised way"""
+
+    parser = argparse.ArgumentParser(description='Search for similar images')
+    parser.add_argument('directories', nargs='*', action='store', help='Directories to search')
+
+    args = parser.parse_args()
+    
+    if not args.directories:
+        print('You must be specify at least one directory')
+        sys.exit(1)
+        
+    try:
+        print('Loading cached data')
+
+        with open('photodupe.pickle', 'rb') as pickles:
+            hashes = pickle.load(pickles)
+    except (FileNotFoundError, EOFError):
+        print('Scanning directories')
+
+        hashes = read_image_hashes(args.directories)
+
+    # Sort the list of hashes so that we can easily find close matches
+    
+    print('Sorting hashes')
+
+    hash_values = sorted([str(hashval) for hashval in hashes])
+
+    for hash_value in hash_values:
+        if len(hashes[hash_value]) > 1:
+            print(hash_value)
+            max_len = 0
+            min_size = None
+
+            for entry in hashes[hash_value]:
+                max_len = max(max_len, len(entry['path']))
+
+                if min_size is None:
+                    min_size = entry['size']
+                else:
+                    min_size = min(min_size, entry['size'])
+
+                if min_size >= 1024 * 1024:
+                    size_suffix = 'MiB'
+                    size_div = 1024*1024
+
+                elif min_size > 1024:
+                    size_suffix = 'KiB'
+                    size_div = 1024
+                else:
+                    size_div = 1
+                    size_suffix = ''
+
+            for entry in hashes[hash_value]:
+                size = entry['size'] // size_div
+                print(f'    {entry["path"]:{max_len}} {size:>4} {size_suffix} ({entry["width"]}x{entry["height"]})')
+
+    with open('photodupe.pickle', 'wb') as pickles:
+        pickle.dump(hashes, pickles)
+
+################################################################################
+
+def photodupe():
+    """Entry point"""
+
+    try:
+        main()
+
+    except KeyboardInterrupt:
+        sys.exit(1)
+
+    except BrokenPipeError:
+        sys.exit(2)
+
+################################################################################
+
+if __name__ == '__main__':
+    photodupe()

skilleter_thingy/phototidier.py

@@ -0,0 +1,248 @@
+#! /usr/bin/env python3
+
+""" Perform various tidying operations on a directory full of photos:
+    1. Remove leading '$' and '_' from filenames
+    2. Move files in hidden directories up 1 level
+    3. If the EXIF data in a photo indicates that it was taken on date that
+       doesn't match the name of the directory it is stored in (in YYYY-MM-DD format)
+       then it is moved to the correct directory, creating it if necessary.
+
+    All move/rename operations are carried out safely with the file being moved having
+    a numeric suffix added to the name if it conflicts with an existing file.
+
+    TODO: Ignore .stversions files
+
+"""
+
+################################################################################
+
+import argparse
+import os
+import sys
+import pathlib
+import re
+
+from PIL import UnidentifiedImageError
+from PIL import Image
+from PIL.ExifTags import TAGS
+
+import thingy.colour as colour
+
+################################################################################
+
+FILE_TYPES = ('.jpg', '.jpeg')
+
+DATE_RE = re.compile(r'[0-9]{4}-[0-9]{2}-[0-9]{2}')
+
+NUMBER_RE = re.compile(r'(.*) +\([0-9]+\).*')
+
+################################################################################
+
+def error(msg, status=1):
+    """ Exit with an error message """
+
+    print(msg)
+
+    sys.exit(status)
+
+################################################################################
+
+def parse_command_line():
+    """ Handle command line arguments """
+
+    parser = argparse.ArgumentParser(description='Re-organise photos into (hopefully) the correct folders.')
+
+    parser.add_argument('-D', '--dry-run', action='store_true', help='Report what files would be moved, without actually moving them')
+    parser.add_argument('path', nargs=1, help='Path to the picture storage directory')
+
+    args = parser.parse_args()
+
+    if not os.path.isdir(args.path[0]):
+        error(f'{args.path} is not a directory')
+
+    args.path = pathlib.Path(os.path.realpath(args.path[0]))
+
+    return args
+
+################################################################################
+
+def safe_rename(args, source_file, new_name):
+    """ Rename a file, adding a numeric suffix to avoid overwriting anything """
+
+    # If the destination file exists, add a numeric suffix to the new name
+    # until we find one that doesn't
+
+    index = 1
+    new_name_stem = new_name.stem
+
+    while new_name.exists():
+        new_name = new_name.with_name(f'{new_name_stem}-{index}{new_name.suffix}')
+        index += 1
+
+    colour.write(f'Rename [BLUE:{source_file}] to [BLUE:{new_name}]')
+
+    # Panic if the destination parent directory exists, but isn't actually a directory
+
+    if new_name.parent.exists and not new_name.parent.is_dir:
+        colour.write('[RED:WARNING]: Destination [BLUE:{new_name.parent}] exists, but is not a directory - [BLUE:{source_file}] will not be renamed')
+        return source_file
+
+    # Rename and return the new namem, creating the directory for it to go in, if necessary
+
+    if not args.dry_run:
+        new_name.parent.mkdir(parents=True, exist_ok=True)
+
+        source_file.rename(new_name)
+
+    return new_name
+
+################################################################################
+
+def get_exif_date(source_file):
+    """ Try an extract the daste when the photo was taken from the EXIF data
+        and return it in YYYY/YYYY-MM-DD format as the subdirectory where
+        the photo should be located """
+
+    # Get the EXIF data
+
+    try:
+        photo = Image.open(source_file)
+    except (OSError, UnidentifiedImageError):
+        colour.write(f'[RED:ERROR]: [BLUE:{source_file}] does not appear to be a valid image - ignoring EXIF data')
+        return None
+
+    exif = photo.getexif()
+
+    # Search for the original date/time tag
+
+    for tag_id in exif:
+        tag = TAGS.get(tag_id, tag_id)
+
+        if tag == 'DateTimeOriginal':
+            data = exif.get(tag_id)
+            if isinstance(data, bytes):
+                data = data.decode()
+
+            # Ignore dummy value
+
+            if data.startswith('0000:00:00'):
+                return None
+
+            # Convert to YYYY-MM-DD format, removing the time
+
+            date = f'{int(data[0:4]):04}-{int(data[5:7]):02}-{int(data[8:10]):02}'
+
+            return date
+
+    # No date tag found
+
+    return None
+
+################################################################################
+
+def fix_file(args, source_file):
+    """ Fix a file by moving or renaming it to fix naming or directory issues """
+
+    # Get the image date from the EXIF data
+
+    image_date = get_exif_date(source_file)
+
+    # If the file starts with $, ~, _ or ., rename it to remove it
+
+    while source_file.name[0] in ('$', '~', '_', '.'):
+        new_name = source_file.with_name(source_file.name[1:])
+
+        source_file = safe_rename(args, source_file, new_name)
+
+    # If filename contains '~' then truncate it
+
+    if '~' in source_file.name:
+        new_name = source_file.with_name(source_file.name.split('~')[0] + source_file.suffix)
+
+        source_file = safe_rename(args, source_file, new_name)
+
+    # If the directory name starts with . or $ move the file up 1 level
+
+    while source_file.parts[-2][0] in ('$', '.'):
+        new_name = source_file.parent.parent / source_file.name
+
+        source_file = safe_rename(args, source_file, new_name)
+
+    # If the filename has a number in parentheses, then remove it
+
+    num_match = NUMBER_RE.fullmatch(source_file.stem)
+    if num_match:
+        new_name = source_file.parent / (num_match.group(1) + source_file.suffix)
+
+        source_file = safe_rename(args, source_file, new_name)
+
+    # See if the date in the EXIF data matches the directory name prefix
+    # and move it to the correct location if it doesn't
+
+    if image_date:
+        image_year = image_date.split('-')[0]
+
+        image_path = args.path / image_year / image_date
+
+        # If the file isn't already in a directory with the correct year and date
+        # move it to one that it
+
+        if not str(source_file.parent).startswith(str(image_path)):
+            # If the source directory has a description after the date, append that
+            # to the destination directory
+            # Otherwise, if the source directory doesn't have a date, append the whole
+            # directory name.
+
+            source_parent_dir = source_file.parts[-2]
+
+            if DATE_RE.match(source_parent_dir):
+                if len(source_parent_dir) > 10:
+                    image_path = args.path / image_year / f'{image_date}{source_parent_dir[10:]}'
+            else:
+                image_path = args.path / image_year / f'{image_date} - {source_parent_dir}'
+
+            source_file = safe_rename(args, source_file, image_path / source_file.name)
+
+################################################################################
+
+def main():
+    """ Entry point """
+
+    args = parse_command_line()
+
+    # Disable the maximum image size in PIL
+
+    Image.MAX_IMAGE_PIXELS = None
+
+    # Find matching files in the source tree
+
+    print(f'Searching {args.path} with extension matching {", ".join(FILE_TYPES)}')
+
+    all_matches = args.path.glob('**/*')
+
+    matches = [file for file in all_matches if file.suffix.lower() in FILE_TYPES and file.is_file()]
+
+    print(f'Found {len(matches)} matching files')
+
+    for source_file in matches:
+        if '.stversions' not in source_file.parts:
+            fix_file(args, source_file)
+
+################################################################################
+
+def phototidier():
+    """Entry point"""
+
+    try:
+        main()
+
+    except KeyboardInterrupt:
+        sys.exit(1)
+
+    except BrokenPipeError:
+        sys.exit(2)
+
+################################################################################
+
+if __name__ == '__main__':
+    phototidier()