skilleter-thingy 0.0.22__py3-none-any.whl

skilleter_thingy/phototidier.py

@@ -0,0 +1,248 @@
+#! /usr/bin/env python3
+
+""" Perform various tidying operations on a directory full of photos:
+    1. Remove leading '$' and '_' from filenames
+    2. Move files in hidden directories up 1 level
+    3. If the EXIF data in a photo indicates that it was taken on date that
+       doesn't match the name of the directory it is stored in (in YYYY-MM-DD format)
+       then it is moved to the correct directory, creating it if necessary.
+
+    All move/rename operations are carried out safely with the file being moved having
+    a numeric suffix added to the name if it conflicts with an existing file.
+
+    TODO: Ignore .stversions files
+
+"""
+
+################################################################################
+
+import argparse
+import os
+import sys
+import pathlib
+import re
+
+from PIL import UnidentifiedImageError
+from PIL import Image
+from PIL.ExifTags import TAGS
+
+from skilleter_thingy import colour
+
+################################################################################
+
+FILE_TYPES = ('.jpg', '.jpeg')
+
+DATE_RE = re.compile(r'[0-9]{4}-[0-9]{2}-[0-9]{2}')
+
+NUMBER_RE = re.compile(r'(.*) +\([0-9]+\).*')
+
+################################################################################
+
+def error(msg, status=1):
+    """ Exit with an error message """
+
+    print(msg)
+
+    sys.exit(status)
+
+################################################################################
+
+def parse_command_line():
+    """ Handle command line arguments """
+
+    parser = argparse.ArgumentParser(description='Re-organise photos into (hopefully) the correct folders.')
+
+    parser.add_argument('-D', '--dry-run', action='store_true', help='Report what files would be moved, without actually moving them')
+    parser.add_argument('path', nargs=1, help='Path to the picture storage directory')
+
+    args = parser.parse_args()
+
+    if not os.path.isdir(args.path[0]):
+        error(f'{args.path} is not a directory')
+
+    args.path = pathlib.Path(os.path.realpath(args.path[0]))
+
+    return args
+
+################################################################################
+
+def safe_rename(args, source_file, new_name):
+    """ Rename a file, adding a numeric suffix to avoid overwriting anything """
+
+    # If the destination file exists, add a numeric suffix to the new name
+    # until we find one that doesn't
+
+    index = 1
+    new_name_stem = new_name.stem
+
+    while new_name.exists():
+        new_name = new_name.with_name(f'{new_name_stem}-{index}{new_name.suffix}')
+        index += 1
+
+    colour.write(f'Rename [BLUE:{source_file}] to [BLUE:{new_name}]')
+
+    # Panic if the destination parent directory exists, but isn't actually a directory
+
+    if new_name.parent.exists and not new_name.parent.is_dir:
+        colour.write('[RED:WARNING]: Destination [BLUE:{new_name.parent}] exists, but is not a directory - [BLUE:{source_file}] will not be renamed')
+        return source_file
+
+    # Rename and return the new namem, creating the directory for it to go in, if necessary
+
+    if not args.dry_run:
+        new_name.parent.mkdir(parents=True, exist_ok=True)
+
+        source_file.rename(new_name)
+
+    return new_name
+
+################################################################################
+
+def get_exif_date(source_file):
+    """ Try an extract the daste when the photo was taken from the EXIF data
+        and return it in YYYY/YYYY-MM-DD format as the subdirectory where
+        the photo should be located """
+
+    # Get the EXIF data
+
+    try:
+        photo = Image.open(source_file)
+    except (OSError, UnidentifiedImageError):
+        colour.write(f'[RED:ERROR]: [BLUE:{source_file}] does not appear to be a valid image - ignoring EXIF data')
+        return None
+
+    exif = photo.getexif()
+
+    # Search for the original date/time tag
+
+    for tag_id in exif:
+        tag = TAGS.get(tag_id, tag_id)
+
+        if tag == 'DateTimeOriginal':
+            data = exif.get(tag_id)
+            if isinstance(data, bytes):
+                data = data.decode()
+
+            # Ignore dummy value
+
+            if data.startswith('0000:00:00'):
+                return None
+
+            # Convert to YYYY-MM-DD format, removing the time
+
+            date = f'{int(data[0:4]):04}-{int(data[5:7]):02}-{int(data[8:10]):02}'
+
+            return date
+
+    # No date tag found
+
+    return None
+
+################################################################################
+
+def fix_file(args, source_file):
+    """ Fix a file by moving or renaming it to fix naming or directory issues """
+
+    # Get the image date from the EXIF data
+
+    image_date = get_exif_date(source_file)
+
+    # If the file starts with $, ~, _ or ., rename it to remove it
+
+    while source_file.name[0] in ('$', '~', '_', '.'):
+        new_name = source_file.with_name(source_file.name[1:])
+
+        source_file = safe_rename(args, source_file, new_name)
+
+    # If filename contains '~' then truncate it
+
+    if '~' in source_file.name:
+        new_name = source_file.with_name(source_file.name.split('~')[0] + source_file.suffix)
+
+        source_file = safe_rename(args, source_file, new_name)
+
+    # If the directory name starts with . or $ move the file up 1 level
+
+    while source_file.parts[-2][0] in ('$', '.'):
+        new_name = source_file.parent.parent / source_file.name
+
+        source_file = safe_rename(args, source_file, new_name)
+
+    # If the filename has a number in parentheses, then remove it
+
+    num_match = NUMBER_RE.fullmatch(source_file.stem)
+    if num_match:
+        new_name = source_file.parent / (num_match.group(1) + source_file.suffix)
+
+        source_file = safe_rename(args, source_file, new_name)
+
+    # See if the date in the EXIF data matches the directory name prefix
+    # and move it to the correct location if it doesn't
+
+    if image_date:
+        image_year = image_date.split('-')[0]
+
+        image_path = args.path / image_year / image_date
+
+        # If the file isn't already in a directory with the correct year and date
+        # move it to one that it
+
+        if not str(source_file.parent).startswith(str(image_path)):
+            # If the source directory has a description after the date, append that
+            # to the destination directory
+            # Otherwise, if the source directory doesn't have a date, append the whole
+            # directory name.
+
+            source_parent_dir = source_file.parts[-2]
+
+            if DATE_RE.match(source_parent_dir):
+                if len(source_parent_dir) > 10:
+                    image_path = args.path / image_year / f'{image_date}{source_parent_dir[10:]}'
+            else:
+                image_path = args.path / image_year / f'{image_date} - {source_parent_dir}'
+
+            source_file = safe_rename(args, source_file, image_path / source_file.name)
+
+################################################################################
+
+def main():
+    """ Entry point """
+
+    args = parse_command_line()
+
+    # Disable the maximum image size in PIL
+
+    Image.MAX_IMAGE_PIXELS = None
+
+    # Find matching files in the source tree
+
+    print(f'Searching {args.path} with extension matching {", ".join(FILE_TYPES)}')
+
+    all_matches = args.path.glob('**/*')
+
+    matches = [file for file in all_matches if file.suffix.lower() in FILE_TYPES and file.is_file()]
+
+    print(f'Found {len(matches)} matching files')
+
+    for source_file in matches:
+        if '.stversions' not in source_file.parts:
+            fix_file(args, source_file)
+
+################################################################################
+
+def phototidier():
+    """Entry point"""
+
+    try:
+        main()
+
+    except KeyboardInterrupt:
+        sys.exit(1)
+
+    except BrokenPipeError:
+        sys.exit(2)
+
+################################################################################
+
+if __name__ == '__main__':
+    phototidier()

skilleter_thingy/popup.py

@@ -0,0 +1,87 @@
+################################################################################
+""" Curses-based pop-up message
+
+Usage:
+    with PopUp(curses_screen, message, colour):
+        do_stuff
+
+Popup message is displayed for the duration of the with statement, and
+has optional parameters to wait for a keypress, and/or pause before removing
+the popup again.
+
+"""
+################################################################################
+
+import time
+import curses
+
+################################################################################
+
+class PopUp():
+    """ Class to enable popup windows to be used via with statements """
+
+    def __init__(self, screen, msg, colour, waitkey=False, sleep=True, centre=True, refresh=True):
+        """ Initialisation - just save the popup parameters """
+
+        self.panel = None
+        self.screen = screen
+        self.msg = msg
+        self.centre = centre
+        self.colour = curses.color_pair(colour)
+        self.refresh = refresh
+        self.sleep = sleep and not waitkey
+        self.waitkey = waitkey
+        self.start_time = 0
+
+    def __enter__(self):
+        """ Display the popup """
+
+        lines = self.msg.split('\n')
+        height = len(lines)
+
+        width = 0
+        for line in lines:
+            width = max(width, len(line))
+
+        width += 2
+        height += 2
+
+        size_y, size_x = self.screen.getmaxyx()
+
+        window = curses.newwin(height, width, (size_y - height) // 2, (size_x - width) // 2)
+        self.panel = curses.panel.new_panel(window)
+
+        window.bkgd(' ', self.colour)
+        for y_pos, line in enumerate(lines):
+            x_pos = (width - len(line)) // 2 if self.centre else 1
+            window.addstr(y_pos + 1, x_pos, line, self.colour)
+
+        self.panel.top()
+        curses.panel.update_panels()
+        self.screen.refresh()
+
+        self.start_time = time.monotonic()
+
+        if self.waitkey:
+            while True:
+                keypress = self.screen.getch()
+                if keypress != curses.KEY_RESIZE:
+                    break
+
+                curses.panel.update_panels()
+                self.screen.refresh()
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        """ Remove the popup """
+
+        if self.panel:
+            if self.sleep:
+                elapsed = time.monotonic() - self.start_time
+
+                if elapsed < 1:
+                    time.sleep(1 - elapsed)
+
+            del self.panel
+            
+        if self.refresh:
+            self.screen.refresh()

skilleter_thingy/process.py

@@ -0,0 +1,112 @@
+#! /usr/bin/env python3
+
+################################################################################
+""" Basic subprocess handling - simplified API to the Python subprocess module
+
+    Copyright (C) 2017-18 John Skilleter
+
+    Licence: GPL v3 or later
+"""
+################################################################################
+
+import subprocess
+import sys
+
+from skilleter_thingy import logger
+
+################################################################################
+
+class RunError(Exception):
+    """ Run exception """
+
+    def __init__(self, msg, status=1):
+        super(RunError, self).__init__(msg)
+        self.msg = msg
+        self.status = status
+
+################################################################################
+
+def run(command, foreground=False, shell=False):
+    """ Run the specified command and return the output.
+        command - the command to run as an array of command+arguments
+        foreground - set to True to run the command in the foreground, using stdin/err/out
+        shell - set to True to run the command inside a shell (allows the command to be specified
+                as a string, but needs spaces to bew quoted
+        Returns an empty array if foreground is True or an array of the output otherwise. """
+
+    # TODO: for background use subprocess.Popen but use devnull = open('/dev/null', 'w') for stdio and return proc instead of communicating with it?
+
+    log.info('Running "%s"', ' '.join(command))
+
+    # If running in the foreground, run the command and either return an empty value
+    # on success (output is to the console) or raise a RunError
+
+    if foreground:
+        try:
+            if shell:
+                # TODO: Handle command lines with parameters containing spaces
+                command = ' '.join(command)
+
+            status = subprocess.run(command, shell=shell).returncode
+            if status:
+                raise RunError('Error %d' % status, status)
+            else:
+                return []
+        except OSError as exc:
+            raise RunError(exc)
+    else:
+        # Run the command and capture stdout and stderr
+
+        try:
+            proc = subprocess.run(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True)
+        except OSError as exc:
+            raise RunError(exc)
+
+        log.info('Stdout: %s', proc.stdout)
+        log.info('Stderr: %s', proc.stderr)
+
+        # If it returned an error raise a RunError exception with the stdout text as the
+        # exception message
+
+        if proc.returncode:
+            raise RunError(proc.stderr)
+
+        # Otherwise return the stdout data or nothing
+
+        if proc.stdout:
+            output = proc.stdout.split('\n')
+        else:
+            output = []
+
+        # Remove trailing blank lines from the output
+
+        while output and output[-1] == '':
+            output = output[:-1]
+
+        log.info('Output: %s', output)
+        return output
+
+################################################################################
+
+log = logger.init('tgy_process')
+
+if __name__ == '__main__':
+    print('Run ls -l:')
+
+    cmd_output = run(['ls', '-l'])
+
+    for o in cmd_output:
+        print(o)
+
+    print('Run wombat (should fail):')
+    try:
+        run(['wombat'])
+    except RunError as exc:
+        print('Failed with error: %s' % exc.msg)
+
+    if sys.stdout.isatty():
+        print('Run vi in the foreground')
+
+        run(['vi'], foreground=True)
+    else:
+        print('Not testing call to run() with foreground=True as stdout is not a TTY')

skilleter_thingy/py_audit.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+
+""" Query api.osv.dev to determine whether a specified version of a particular
+    Python package is subject to known security vulnerabilities """
+
+################################################################################
+
+import os
+import sys
+import requests
+import subprocess
+import tempfile
+import re
+import glob
+import argparse
+
+################################################################################
+
+PIP_PACKAGES = ('pip', 'pkg_resources', 'setuptools')
+PIP_OPTIONS = '--no-cache-dir'
+
+QUERY_URL = "https://api.osv.dev/v1/query"
+QUERY_HEADERS = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'}
+
+ANSI_NORMAL = '\x1b[0m'
+ANSI_BOLD = '\x1b[1m'
+
+################################################################################
+
+def audit(package, version):
+    """ Main function """
+
+    # Query api.osv.dev for known vulnerabilties in this version of the package
+
+    payload = '{"version": "'+version+'", "package": {"name": "'+package+'", "ecosystem": "PyPI"}}'
+    result = requests.post(QUERY_URL, data=payload, headers=QUERY_HEADERS)
+
+    # Parse and report the results
+
+    details = result.json()
+
+    print('-' * 80)
+    if package in PIP_PACKAGES:
+        print(f'Package: {package} {version} (part of Pip)')
+    else:
+        print(f'Package: {package} {version}')
+
+    print()
+
+    if 'vulns' in details:
+        print(f'{len(details["vulns"])} known vulnerabilities')
+
+        for v in details['vulns']:
+            print()
+            print(f'{ANSI_BOLD}Vulnerability: {v["id"]}{ANSI_NORMAL}')
+
+            if 'summary' in v:
+                print(f'Summary:       {v["summary"]}')
+
+            if 'aliases' in v:
+                print('Aliases:       %s' % (', '.join(v['aliases'])))
+
+            if 'details' in v:
+                print()
+                print(v['details'])
+    else:
+        print(f'No known vulnerabilities')
+
+################################################################################
+
+def main():
+    """ Entry point """
+
+    parser = argparse.ArgumentParser(description='Query api.osv.dev to determine whether Python packagers in a requirments.txt file are subject to known security vulnerabilities')
+    parser.add_argument('requirements', nargs='*', type=str, action='store', help='The requirements file (if not specified, then the script searches for a requirements.txt file)')
+    args = parser.parse_args()
+
+    requirements = args.requirements or glob.glob('**/requirements.txt', recursive=True)
+
+    if not requirements:
+        print('No requirements.txt file(s) found')
+        sys.exit(0)
+
+    # Create a venv for each requirements file, install pip and the packages
+    # and prerequisites, get the list of installed package versions
+    # and check each one.
+
+    for requirement in requirements:
+        print('=' * 80)
+        print(f'{ANSI_BOLD}File: {requirement}{ANSI_NORMAL}')
+        print()
+
+        with tempfile.TemporaryDirectory() as env_dir:
+            with tempfile.NamedTemporaryFile() as package_list:
+                script = f'python3 -m venv {env_dir}' \
+                         f' && . {env_dir}/bin/activate' \
+                         f' && python3 -m pip {PIP_OPTIONS} install --upgrade pip setuptools' \
+                         f' && python3 -m pip {PIP_OPTIONS} install -r {requirement}' \
+                         f' && python3 -m pip {PIP_OPTIONS} list | tail -n+3 | tee {package_list.name}' \
+                          ' && deactivate'
+
+                result = subprocess.run(script, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
+
+                if result.returncode:
+                    print(f'ERROR #{result.returncode}: {result.stdout}')
+                    sys.exit(result.returncode)
+
+                with open(package_list.name) as infile:
+                    for package in infile.readlines():
+                        package_info = re.split(' +', package.strip())
+
+                        audit(package_info[0], package_info[1])
+
+################################################################################
+
+def py_audit():
+    """Entry point"""
+
+    try:
+        main()
+
+    except KeyboardInterrupt:
+        sys.exit(1)
+
+    except BrokenPipeError:
+        sys.exit(2)
+
+################################################################################
+
+if __name__ == '__main__':
+    py_audit()