skilleter-thingy 0.0.24__py3-none-any.whl → 0.0.26__py3-none-any.whl

skilleter_thingy/git_review.py

@@ -514,6 +514,12 @@ class GitReview():
     def show_file_list(self):
         """ Draw the current page of the file list """
 
+        def format_change(prefix, value):
+            """If value is 0 just return it as a string, otherwise apply the prefix and
+               return it (e.g. '+' or '-')"""
+
+            return f'{prefix}{value}' if value else '0'
+
         for ypos in range(0, self.file_list_h):
 
             normal_colour = curses.color_pair(COLOUR_NORMAL)
@@ -535,13 +541,13 @@ class GitReview():
                 # Diff stats, with or without non-whitespace changes
 
                 if self.show_none_whitespace_stats:
-                    added = f'+{current_file["non-ws added"]}'
-                    deleted = f'-{current_file["non-ws deleted"]}'
+                    added = format_change('+', current_file["non-ws added"])
+                    deleted = format_change('-', current_file["non-ws deleted"])
                 else:
-                    added = f'+{current_file["added"]}'
-                    deleted = f'-{current_file["deleted"]}'
+                    added = format_change('+', current_file["added"])
+                    deleted = format_change('-', current_file["deleted"])
 
-                status = f'{current_file["status"]} {added:>4}/{deleted:>4}'
+                status = f'{current_file["status"]} {deleted:>4}/{added:>4}'
 
                 abspath = os.path.join(self.working_tree_dir, filename)
 

skilleter_thingy/photodupe.py

@@ -5,6 +5,7 @@ import sys
 import os
 import sys
 import pickle
+import argparse
 
 import PIL
 
@@ -15,28 +16,42 @@ import imagehash
 
 ################################################################################
 
-def read_image_hashes():
+def read_image_hashes(directories):
     """Read all the specfied directories and hash every picture therein"""
 
     hashes = defaultdict(list)
 
-    for directory in sys.argv[1:]:
+    # Walk each directory tree
+    
+    for directory in directories:
+        print(f'Scanning directory tree {directory}')
+        
         for root, _, files in os.walk(directory):
+            print(f'Scanning directory {root}')
+            
             for file in files:
                 filepath = os.path.join(root, file)
-
-                try:
-                    with Image.open(filepath) as image:
-                        hash_value = imagehash.average_hash(image, hash_size=12)
-
-                        size = os.stat(filepath).st_size
-                        hashes[hash_value].append({'path': filepath, 'width': image.width, 'height': image.height, 'size': size})
-
-                except PIL.UnidentifiedImageError:
-                    sys.stderr.write(f'ERROR: Unrecognized format {filepath}\n')
-
-                except OSError:
-                    sys.stderr.write(f'ERROR: Unable to read {filepath}\n')
+                
+                fileext = os.path.splitext(file)[1]
+
+                if fileext.lower() not in ('.jbf', '.ini', '.xml', '.ffs_db'):
+                    # Calculate the hash and store path, dimensions and file size under the hash entry in the hashes table
+                    
+                    try:
+                        with Image.open(filepath) as image:
+                            hash_value = imagehash.average_hash(image, hash_size=12)
+
+                            size = os.stat(filepath).st_size
+                            hashes[hash_value].append({'path': filepath, 'width': image.width, 'height': image.height, 'size': size})
+
+                    except PIL.UnidentifiedImageError:
+                        sys.stderr.write(f'ERROR: Unrecognized format {filepath} (size={size})\n')
+
+                    except OSError:
+                        sys.stderr.write(f'ERROR: Unable to read {filepath} (size={size})\n')
+                    
+    # Return the hash table
+    
     return hashes
 
 ################################################################################
@@ -44,6 +59,15 @@ def read_image_hashes():
 def main():
     """Read the hashes and report duplicates in a vaguely civilised way"""
 
+    parser = argparse.ArgumentParser(description='Search for similar images')
+    parser.add_argument('directories', nargs='*', action='store', help='Directories to search')
+
+    args = parser.parse_args()
+    
+    if not args.directories:
+        print('You must be specify at least one directory')
+        sys.exit(1)
+        
     try:
         print('Loading cached data')
 
@@ -52,8 +76,10 @@ def main():
     except (FileNotFoundError, EOFError):
         print('Scanning directories')
 
-        hashes = read_image_hashes()
+        hashes = read_image_hashes(args.directories)
 
+    # Sort the list of hashes so that we can easily find close matches
+    
     print('Sorting hashes')
 
     hash_values = sorted([str(hashval) for hashval in hashes])

{skilleter_thingy-0.0.24.dist-info → skilleter_thingy-0.0.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: skilleter_thingy
-Version: 0.0.24
+Version: 0.0.26
 Summary: A collection of useful utilities, mainly aimed at making Git more friendly
 Author-email: John Skilleter <john@skilleter.org.uk>
 Project-URL: Home, https://skilleter.org.uk
@@ -10,7 +10,6 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: boto3
 Requires-Dist: imagehash
 Requires-Dist: inotify
 Requires-Dist: pillow

{skilleter_thingy-0.0.24.dist-info → skilleter_thingy-0.0.26.dist-info}/RECORD

@@ -1,6 +1,5 @@
 skilleter_thingy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 skilleter_thingy/addpath.py,sha256=4Yhhgjjz1XDI98j0dAiQpNA2ejLefeWUTeSg3nIXQq0,3842
-skilleter_thingy/aws.py,sha256=k08MT1866KuMjgG7ECr7LCuqcgWx78iPDcS9CmemIoA,18614
 skilleter_thingy/borger.py,sha256=AQX7OHeGXcUjkgyXEWE2h_oOey9eczZBbKjrreXvRAs,7832
 skilleter_thingy/colour.py,sha256=D-RTYsND6Xm6m3xl0mOe9QSrTNYsyY0K_a8x3id2gvg,7031
 skilleter_thingy/console_colours.py,sha256=dT5qc_B62VJaWs92yrFt1izoB7hs3a22t3lfrZFybG4,1786
@@ -25,7 +24,7 @@ skilleter_thingy/git_common.py,sha256=sjCRCNfyDhLPtxajjiq1l0SCEqB7d1L2uAgnuxLqc1
 skilleter_thingy/git_hold.py,sha256=i-mbY6wBZ-lRx6rS2kGyWlTCWqK6suhKape7Dnc4OAo,4630
 skilleter_thingy/git_mr.py,sha256=ik3uVecNos_73aYANgnaHutPdnW3PlGmWiAw9EJ9IGk,3100
 skilleter_thingy/git_parent.py,sha256=1VHbh_iH4tnw6vpafqHWEeZwHZVL_sZOIVZMojKTwlk,2696
-skilleter_thingy/git_review.py,sha256=LDCILatY8wNqySTSK2PsvSiR4Dd-PbtoH-GXzvHfSS0,51276
+skilleter_thingy/git_review.py,sha256=4pNf8DSVu3V2l9CDo97OIsEHaBIGxfINREbAYU9thg4,51566
 skilleter_thingy/git_update.py,sha256=comv8CR1j9W8yuCIeoksAKxH8fPTiROkIXVrHdpXWl4,13985
 skilleter_thingy/git_wt.py,sha256=VAcOgEAqVUUdmmkZ-zxaHi0Xp6LAt9NvZAZzUoJM3AU,3143
 skilleter_thingy/gitcmp_helper.py,sha256=rFCZiivZfYmd1_-0_3YwmYSfGDLWu6e6rK0I9hFXCUY,11279
@@ -37,7 +36,7 @@ skilleter_thingy/linecount.py,sha256=lw3vuXUUnMwrUihY6bHfZJsRKe6ZMCRz3952Z9N-ogI
 skilleter_thingy/logger.py,sha256=xKgPAq8KGXmtaXIFjFs1AmZJXtYrXJn2sqL3oxHZjfQ,3107
 skilleter_thingy/moviemover.py,sha256=j_Xb9_jFdgpFBAXcF4tEqbnKH_FonlnUU39LiCK980k,4470
 skilleter_thingy/path.py,sha256=3ba_e-QwYpAs-jFVWoV8sfjVjs_59uc5JZt-87Hqn6g,4737
-skilleter_thingy/photodupe.py,sha256=EiWTDLI4tFZp8ruwkbCayMTiwRfMI7O_kmbWgXwZGVQ,3178
+skilleter_thingy/photodupe.py,sha256=FssLgbLnqHPuSvMGtRyOa7bRdowufJOQaJJ56f9ybxk,4195
 skilleter_thingy/phototidier.py,sha256=1uj1XbUemJOalNC3PwTG2-_yUQp4WMAcmwRr7QXPn1w,7823
 skilleter_thingy/popup.py,sha256=jW-nbpdeswqEMTli7OmBv1J8XQsvFoMI0J33O6dOeu8,2529
 skilleter_thingy/process.py,sha256=RmXj2RrzwqP_mugeVejtjgF_T3PD7M10XsWuAnc99t0,3565
@@ -47,7 +46,6 @@ skilleter_thingy/remdir.py,sha256=zp5Nr0IMGXQ-b5iT48O5arqWoSjW65Xnr-SpKuav1Ac,46
 skilleter_thingy/rmdupe.py,sha256=tcX3w8XvliGwBMdSt9BUu07kuDtQEc0IiU8sCxmgzHA,17117
 skilleter_thingy/rpylint.py,sha256=na39x0yNXDwDkG9yP48BoM5FeTut-OS4AVsYixE0YZU,2639
 skilleter_thingy/run.py,sha256=EGYJSuMcOmUca6dpfVUFE41vG9C6ZNK8hzZlJCJE6Rs,12619
-skilleter_thingy/s3_sync.py,sha256=TITptjua_B-iwPlgTniuoxPvuEnQjyTKfs6l9CKHbXc,13849
 skilleter_thingy/splitpics.py,sha256=qRlJrqet7TEI6SodS4bkuKXQUpOdMaqmjE4c1CR7ouo,3266
 skilleter_thingy/strreplace.py,sha256=xsIWw0hc452rYEBtNEQFKIzmV03xjm_Taz-eDTmFFKI,2539
 skilleter_thingy/sysmon.py,sha256=XRZG6EVSzoVYan_N16qVB1l1RaU51uvLWlRA0CDjC54,11348
@@ -59,9 +57,9 @@ skilleter_thingy/trimpath.py,sha256=SAfOB75_dTldQHjam4kQy1J42209NYPYi8vVAaNn1e8,
 skilleter_thingy/window_rename.py,sha256=dCBgZqih_3YKHt35hsOAhARFp3QxOi8w8huC63sqJK8,3128
 skilleter_thingy/xchmod.py,sha256=F9_lxKuLqVlHHr3oBI3dkMoFOuwRzYDlpQMTmDcjpBI,4590
 skilleter_thingy/yamlcheck.py,sha256=FXylZ5NtHirDlPVhVEUZUZkTugVR-g51BbjaN06akAc,2868
-skilleter_thingy-0.0.24.dist-info/LICENSE,sha256=ljOS4DjXvqEo5VzGfdaRwgRZPbNScGBmfwyC8PChvmQ,32422
-skilleter_thingy-0.0.24.dist-info/METADATA,sha256=bdAQ2Q7iW4Y0-Otcl_StalppVtk2pCtP26dtAxzaNWA,5231
-skilleter_thingy-0.0.24.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-skilleter_thingy-0.0.24.dist-info/entry_points.txt,sha256=jf7hVfH0dojLxk0D4TjIazKoqdToditCGdCfyhIotME,1971
-skilleter_thingy-0.0.24.dist-info/top_level.txt,sha256=8-JhgToBBiWURunmvfpSxEvNkDHQQ7r25-aBXtZv61g,17
-skilleter_thingy-0.0.24.dist-info/RECORD,,
+skilleter_thingy-0.0.26.dist-info/LICENSE,sha256=ljOS4DjXvqEo5VzGfdaRwgRZPbNScGBmfwyC8PChvmQ,32422
+skilleter_thingy-0.0.26.dist-info/METADATA,sha256=tn3FjCTFSjHBom9bV7v__U8UFJ9QdVDCwtO9B7vhvy8,5210
+skilleter_thingy-0.0.26.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+skilleter_thingy-0.0.26.dist-info/entry_points.txt,sha256=jf7hVfH0dojLxk0D4TjIazKoqdToditCGdCfyhIotME,1971
+skilleter_thingy-0.0.26.dist-info/top_level.txt,sha256=8-JhgToBBiWURunmvfpSxEvNkDHQQ7r25-aBXtZv61g,17
+skilleter_thingy-0.0.26.dist-info/RECORD,,

skilleter_thingy/aws.py

@@ -1,548 +0,0 @@
-#! /usr/bin/env python3
-
-################################################################################
-""" Simple AWS module - provides an abstraction layer on top of the boto3 client
-    for simple operations such as reading/writing S3 buckets, sending emails
-    etc.
-
-    Currently this is a random collection of functions extracted from the S3
-    bucket ACL checking code but it is hoped it will evolve over time into
-    something more comprehensive.
-
-    Author: John Skilleter
-"""
-################################################################################
-
-# System modules
-
-import sys
-import logging
-import json
-
-# AWS modules
-
-import boto3
-import botocore
-
-################################################################################
-
-class GenericAWS:
-    """ Class providing generic S3 functionality from which the
-        other S3 classes are derived """
-
-    ################################################################################
-
-    def __init__(self, service, session=None, profile=None, createresource=True, createclient=True, region=None):
-        """ Initialisation - just create a client handle, optionally using
-            the specified profile """
-
-        if session:
-            self.session = session
-        elif profile:
-            self.session = boto3.session.Session(profile_name=profile)
-        else:
-            self.session = boto3
-
-        args = {}
-        if region:
-            args['region_name'] = region
-
-        if createresource:
-            self.resource = self.session.resource(service, **args)
-
-        if createclient:
-            self.client = self.session.client(service, **args)
-
-        self.log = logging.getLogger('%s:%s' % (__name__, service))
-
-    ################################################################################
-
-    def set_log_level(self, level):
-        """ Set logging for the module """
-
-        self.log.setLevel(level)
-
-################################################################################
-
-class SES(GenericAWS):
-    """ Class for AWS Simple Email Service """
-
-    ################################################################################
-
-    def __init__(self, session=None, profile=None):
-        """ Initialisation """
-
-        super().__init__('ses', session, profile, createresource=False)
-
-    ################################################################################
-
-    def send(self, sender, recipient, subject, body):
-        """ Send an email """
-
-        destination = \
-            {
-                'ToAddresses': [recipient]
-            }
-
-        message = \
-            {
-                'Subject':
-                {
-                    'Data': subject
-                },
-                'Body':
-                {
-                    'Text':
-                    {
-                        'Data': body
-                    }
-                }
-            }
-
-        # Attempt to send the email - just report the error and quit on failure
-
-        try:
-            self.log.info('Sending email from %s to %s', sender, recipient)
-
-            self.client.send_email(Source=sender,
-                                   Destination=destination,
-                                   Message=message)
-
-        except (botocore.exceptions.EndpointConnectionError,
-                self.client.exceptions.MessageRejected) as err:
-            print('')
-            print('Error sending email: %sn' % err)
-            print('    Sender:    %s' % sender)
-            print('    Recipient: %s' % recipient)
-            print('    Subject:   %s' % subject)
-            print('    Body:')
-
-            for txt in body.split('\n'):
-                print('        %s' % txt)
-
-            print('')
-
-            sys.exit(1)
-
-################################################################################
-
-class S3Bucket(GenericAWS):
-    """ Class providing access to S3 buckets """
-
-    ################################################################################
-
-    def __init__(self, session=None, profile=None):
-        """ Initialisation - just create a client handle, optionally using
-            the specified profile """
-
-        super().__init__('s3', session, profile)
-
-    ################################################################################
-
-    def read(self, bucket, key):
-        """ Read the specified data from the specified bucket.
-            Returns the data, or raises a boto3 exception on error. """
-
-        self.log.info('Get object from key %s in bucket %s', key, bucket)
-
-        response = self.client.get_object(Bucket=bucket, Key=key)
-
-        self.log.debug('Get object: %s', response)
-
-        return response['Body'].read()
-
-    ################################################################################
-
-    def write(self, bucket, key, data):
-        """ Write data into the specified key of the specified bucket
-            Raises a boto3 exception on error. """
-
-        self.log.info('Writing %d bytes of data to key %s in bucket %s', len(data), key, bucket)
-
-        self.client.put_object(Bucket=bucket, Key=key, Body=data)
-
-    ################################################################################
-
-    def get_tags(self, bucket):
-        """ Read the tags from a bucket. Returns a dictionary of tags, which will be
-            empty if the bucket has no tags. """
-
-        self.log.info('Reading tags from bucket %s', bucket)
-
-        try:
-            tags = self.client.get_bucket_tagging(Bucket=bucket)['TagSet']
-
-        except botocore.exceptions.ClientError as err:
-            # Any exception except NoSuchTagSet gets raised.
-            # NoSuchTagSet simply means that the bucket has no tags
-            # and isn't really an error, as such, so we just return
-            # an empty dictionary.
-
-            if err.response['Error']['Code'] != 'NoSuchTagSet':
-                self.log.error('Error reading tags for bucket %s: %s', bucket, err.response['Error']['Code'])
-                raise
-
-            tags = {}
-
-        self.log.info('Tags: %s', tags)
-
-        return tags
-
-    ################################################################################
-
-    def get_location(self, bucket):
-        """ Return the location of a bucket - note that the Boto3 returns the location
-            set to none, rather than us-east-1 if you query a bucket located in that
-            region. """
-
-        try:
-            location = self.client.get_bucket_location(Bucket=bucket)['LocationConstraint']
-        except botocore.exceptions.ClientError as err:
-            self.log.info('Error getting location of bucket %s: %s', bucket, err.response['Error']['Code'])
-            raise
-
-        if not location:
-            location = 'us-east-1'
-
-        return location
-
-    ################################################################################
-
-    def get_acl(self, bucket):
-        """ Return the bucket ACLs """
-
-        try:
-            return self.client.get_bucket_acl(Bucket=bucket)['Grants']
-        except botocore.exceptions.ClientError as err:
-            self.log.error('Error getting ACL for bucket %s: %s', bucket, err.response['Error']['Code'])
-            raise
-
-    ################################################################################
-
-    def get_policy(self, bucket):
-        """ Return bucket policy information as a list of policy dictionaries
-            Returns an empty list if the bucket has no policies (similar to
-            get_tags() above). """
-
-        try:
-            policy_data = self.client.get_bucket_policy(Bucket=bucket)
-
-        except botocore.exceptions.ClientError as err:
-            # Any exception which *isn't* a no-policy exception gets raised
-            # to the caller, the no-policy one causes the function to return
-            # an empty policy list.
-
-            if err.response['Error']['Code'] != 'NoSuchBucketPolicy':
-                self.log.error('Error reading policy for bucket %s: %s', bucket, err.response['Error']['Code'])
-                raise
-
-            return []
-
-        return json.loads(policy_data['Policy'])
-
-    ################################################################################
-
-    def get_buckets(self):
-        """ Return a list of all the available buckets """
-
-        return [bucket.name for bucket in self.resource.buckets.all()]
-
-    ################################################################################
-
-    def get_website(self, bucket):
-        """ Return the web site configuration for the bucket, or None if the bucket
-            is not configured for hosting """
-
-        try:
-            web = self.client.get_bucket_website(Bucket=bucket)
-
-        except botocore.exceptions.ClientError as err:
-            if err.response['Error']['Code'] != 'NoSuchWebsiteConfiguration':
-                print('>>>%s' % err.response)
-                raise
-
-            web = None
-
-        return web
-
-    ################################################################################
-
-    def get_objects(self, bucket, max_objects=None):
-        """ Yield a list of the details of the objects in a bucket, stopping after
-            returning max_objects (if specified). """
-
-        paginator = self.client.get_paginator('list_objects_v2')
-
-        pagesize = min(25, max_objects) if max_objects else 25
-
-        objects = paginator.paginate(Bucket=bucket, PaginationConfig={'PageSize': pagesize})
-
-        count = 0
-
-        for data in objects:
-            if 'Contents' in data:
-                for obj in data['Contents']:
-
-                    if max_objects:
-                        count += 1
-                        if count > max_objects:
-                            break
-
-                    yield obj
-
-    ################################################################################
-
-    def get_object_acl(self, bucket, obj):
-        """ Return the ACL data for an object in a bucket """
-
-        try:
-            return self.client.get_object_acl(Bucket=bucket, Key=obj)['Grants']
-        except botocore.exceptions.ClientError as err:
-            self.log.error('Error getting ACL for object %s in bucket %s: %s', obj, bucket, err.response['Error']['Code'])
-            raise
-
-    ################################################################################
-
-    def get_lifecycle(self, bucket):
-        """ Return the bucket lifecycle data """
-
-        try:
-            lifecycle = self.client.get_bucket_lifecycle_configuration(Bucket=bucket)
-        except botocore.exceptions.ClientError as err:
-            if err.response['Error']['Code'] == 'NoSuchLifecycleConfiguration':
-                lifecycle = None
-            else:
-                raise
-
-        return lifecycle
-
-################################################################################
-
-class STS(GenericAWS):
-    """ Class providing access to STS functionality """
-
-    def __init__(self, session=None, profile=None):
-        """ Initialise the STS client (there is no STS resource in boto3) """
-
-        super().__init__('sts', session, profile, createresource=False)
-
-    ################################################################################
-
-    def account(self):
-        """ Return the name of the current AWS account """
-
-        return self.client.get_caller_identity()['Account']
-
-################################################################################
-
-class IAM(GenericAWS):
-    """ Class providing access to IAM """
-
-    def __init__(self, session=None, profile=None):
-        """ Initialise the IAM client/resource """
-
-        super().__init__('iam', session, profile, createresource=False)
-
-    def role_exists(self, role):
-        """ Return True if the role exists """
-
-        try:
-            self.client.get_role(RoleName=role)
-        except botocore.exceptions.ClientError as err:
-            if err.response['Error']['Code'] != 'NoSuchEntity':
-                raise
-
-            return False
-        else:
-            return True
-
-    def create_role(self, role, description, policy):
-        """ Create a new role """
-
-        response = self.client.create_role(RoleName=role, Description=description, AssumeRolePolicyDocument=policy)
-
-        return response
-
-    def put_role_policy(self, role, policy_name, policy):
-        """ Update the policy in an existing role """
-
-        response = self.client.put_role_policy(RoleName=role, PolicyName=policy_name, PolicyDocument=policy)
-
-        return response
-
-    def get_role_policy(self, role, policy_name):
-        """ Read the policy in an existing role """
-
-        response = self.client.get_role_policy(RoleName=role, PolicyName=policy_name)
-
-        return response
-
-################################################################################
-
-class Events(GenericAWS):
-    """ Class providing access to CloudWatch Events """
-
-    def __init__(self, session=None, profile=None):
-        """ Initialise the CloudWatch Events client/resource """
-
-        super().__init__('events', session, profile, createresource=False)
-
-    def put_rule(self, name, schedule):
-        """ Create or update the specified rule """
-
-        self.client.put_rule(Name=name, ScheduleExpression=schedule)
-
-    def put_target(self, name, targets):
-        """ Add the specified target(s) to the specified rule, or update
-            existing targets """
-
-        self.client.put_targets(Rule=name, Targets=targets)
-
-################################################################################
-
-DEFAULT_LAMBDA_TIMEOUT = 3
-DEFAULT_LAMBDA_MEMORY = 128
-DEFAULT_LAMBDA_HANDLER = 'main.main'
-DEFAULT_LAMBDA_RUNTIME = 'python3.6'
-
-class Lambda(GenericAWS):
-    """ Class providing access to Lambda functions """
-
-    def __init__(self, session=None, profile=None, region=None):
-        """ Initialise the Lambda client/resource """
-
-        super().__init__('lambda', session, profile, createresource=False, region=region)
-
-    def exists(self, name):
-        """ Return True if the specified Lambda function exists """
-
-        try:
-            self.client.get_function(FunctionName=name)
-        except botocore.exceptions.ClientError as err:
-            if err.response['Error']['Code'] != 'ResourceNotFoundException':
-                raise
-            return False
-        else:
-            return True
-
-    def update_function(self, name, zipfile):
-        """ Update the specified Lambda function given a zip file """
-
-        with open(zipfile, 'rb') as zipper:
-            zipdata = zipper.read()
-
-        response = self.client.update_function_code(FunctionName=name, ZipFile=zipdata)
-
-        return response
-
-    def update_function_configuration(self, name,
-                                      handler=None,
-                                      environment=None):
-        """ Update the handler associated with a Lambda fucntion """
-
-        update_args = {}
-        update_args['FunctionName'] = name
-
-        if handler:
-            update_args['Handler'] = handler
-
-        if environment:
-            update_args['Environment'] = {'Variables': environment}
-
-        self.client.update_function_configuration(**update_args)
-
-    def create_function(self, name, role, description, zipfile,
-                        runtime=DEFAULT_LAMBDA_RUNTIME,
-                        handler=DEFAULT_LAMBDA_HANDLER,
-                        timeout=DEFAULT_LAMBDA_TIMEOUT,
-                        memory=DEFAULT_LAMBDA_MEMORY):
-        """ Create the specified Lambda function given a zip file"""
-
-        with open(zipfile, 'rb') as zipper:
-            zipdata = zipper.read()
-
-        response = self.client.create_function(FunctionName=name,
-                                               Runtime=runtime,
-                                               Role=role,
-                                               Handler=handler,
-                                               Code={'ZipFile': zipdata},
-                                               Description=description,
-                                               Timeout=timeout,
-                                               MemorySize=memory)
-
-        return response
-
-################################################################################
-
-def get_session(profile_name=None):
-    """ Wrapper for boto3.session.Session """
-
-    return boto3.session.Session(profile_name=profile_name)
-
-################################################################################
-
-def set_stream_logger(name='botocore', level=10, format_string=None):
-    """ Wrapper for boto3.set_stream_logger """
-
-    return boto3.set_stream_logger(name=name, level=level, format_string=format_string)
-
-################################################################################
-
-def set_default_region(region):
-    """ Set the default region for the module """
-
-    boto3.setup_default_session(region_name=region)
-
-################################################################################
-
-def get_regions(servicename):
-    """ Generate a list of regions where a service is supported """
-
-    return boto3.session.Session().get_available_regions(servicename)
-
-################################################################################
-
-def get_resources():
-    """ Devious method of getting the list of clients supported by boto3
-        Note that I'm somewhat embarrassed by this, but can't find a better
-        way of doing it! """
-
-    try:
-        boto3.resource('XXX-NOT-A-RESOURCE-XXX')
-    except boto3.exceptions.ResourceNotExistsError as err:
-        return str(err).replace('   - ', '').split('\n')[2:-1]
-
-################################################################################
-
-def get_clients():
-    """ Similarly devious way of getting the list of supported clients.
-        This is equally as yukky as the get_resources() function. """
-
-    try:
-        boto3.client('XXX-NOT-A-CLIENT-XXX')
-    except botocore.exceptions.UnknownServiceError as err:
-        return repr(err)[:-3].split(': ')[2].split(', ')
-
-################################################################################
-
-def test_function():
-    """ Module test function """
-
-    # Test STS
-
-    sts_client = STS()
-
-    print('Current account: %s' % sts_client.account())
-
-    # TODO: Test SES
-    # TODO: Test S3
-    # TODO: Test IAM
-    # TODO: Test Cloudwatch Events
-    # TODO: Test Lambda
-    # TODO: Test set_default_region(), get_session(), get_resources(), get_regions(), get_clients()
-
-################################################################################
-# Test code
-
-if __name__ == '__main__':
-    test_function()

skilleter_thingy/s3_sync.py

@@ -1,383 +0,0 @@
-#!/usr/bin/env python3
-
-"""Selectively synchronise an S3 bucket to a local destination.
-   Similar to the aws s3 sync CLI command, but faster, has better
-   options to filter files, only downloads from S3 to local and
-   doesn't support the huge range of command line options."""
-
-import os
-import argparse
-import sys
-import fnmatch
-import datetime
-import threading
-import queue
-
-from pathlib import Path
-
-import boto3
-
-from botocore.exceptions import ClientError
-
-################################################################################
-
-# Number of download threads to run - doing the downloads in threads is about
-# six times faster than doing so sequentially.
-
-NUM_THREADS = 12
-
-# Translate our environment names to AWS ARNs
-
-AWS_ACCOUNT_ARNS = {
-    'prod': 'arn:aws:iam::459580378985:role/ERSReadOnlyRole',
-    'test': 'arn:aws:iam::094438481629:role/ERSReadOnlyRole',
-    'dev': 'arn:aws:iam::402653103803:role/ERSReadOnlyRole',
-    'mgmt': 'arn:aws:iam::125943076446:role/ERSReadOnlyRole',
-    'audit': 'arn:aws:iam::229627323276:role/ERSReadOnlyRole',
-}
-
-################################################################################
-
-def error(msg, status=1):
-    """Report an error message and exit"""
-
-    print(f'ERROR: {msg}')
-    sys.exit(status)
-
-################################################################################
-
-def verbose(args, msg):
-    """Report a message in verbose mode"""
-
-    if not args or args.verbose:
-        print(msg)
-
-################################################################################
-
-def splitlist(lists, deliminator):
-    """Create a list from a list of deliminated strings"""
-
-    result = []
-
-    for item in lists or []:
-        result += item.split(deliminator)
-
-    return result
-
-################################################################################
-
-def configure():
-    """Parse the command line"""
-
-    parser = argparse.ArgumentParser(description='Selectively sync an S3 bucket to a local directory')
-
-    parser.add_argument('--verbose', '-v', action='store_true', help='Report verbose results (includes number of commits between branch and parent)')
-
-    parser.add_argument('--profile', '-p', action='store', help='Specify the AWS profile')
-
-    parser.add_argument('--include', '-i', action='append', help='Comma-separated list of wildcards to sync - if specified, only files matching one or more of these are synced')
-    parser.add_argument('--exclude', '-x', action='append', help='Comma-separated list of wildcards NOT to sync - if specified, only files NOT matching any of these are synced')
-
-    parser.add_argument('--include-type', '-I', action='append',
-                        help='Comma-separated list of file types to sync - if specified, only files matching one or more of these are synced')
-    parser.add_argument('--exclude-type', '-X', action='append',
-                        help='Comma-separated list of file types NOT to sync - if specified, only files NOT matching any of these are synced')
-
-    # TODO: parser.add_argument('--delete', '-d', action='store_true', help='Delete local files that don\'t exist in the bucket')
-    parser.add_argument('--force', '-f', action='store_true', help='Always overwrite locals files (by default files are only overwritten if they are older or a different size)')
-
-    parser.add_argument('--max-objects', '-m', action='store', type=int, help='Limit the number of objects to download')
-    parser.add_argument('--threads', '-t', action='store', type=int, default=NUM_THREADS, help='Number of parallel threads to run')
-    parser.add_argument('source', action='store', nargs=1, type=str, help='Name of the S3 bucket, optionally including path within the bucket')
-    parser.add_argument('destination', action='store', nargs=1, type=str, help='Name of the local directory to sync into')
-
-    args = parser.parse_args()
-
-    # Convert the arguments to single items, but 1-entry lists
-
-    args.source = args.source[0]
-    args.destination = args.destination[0]
-
-    # Convert the include/exclude parameters to lists
-
-    args.include = splitlist(args.include, ',')
-    args.exclude = splitlist(args.exclude, ',')
-
-    args.include_type = splitlist(args.include_type, ',')
-    args.exclude_type = splitlist(args.exclude_type, ',')
-
-    return args
-
-################################################################################
-
-def get_client(args):
-    """Create an S3 client for the specified profile"""
-
-    if args.profile:
-        profile = args.profile.split('-')[0]
-    else:
-        try:
-            profile = os.environ['AWS_PROFILE']
-        except KeyError:
-            error('The AWS profile must be specified via the AWS_PROFILE environment variable or the --profile command line option')
-
-    try:
-        arn = AWS_ACCOUNT_ARNS[profile]
-    except KeyError:
-        error(f'Invalid AWS profile "{profile}"')
-
-    sts_connection = boto3.client("sts")
-
-    try:
-        acct_b = sts_connection.assume_role(RoleArn=arn, RoleSessionName='s3-selective-sync')
-    except ClientError as exc:
-        error(f'{exc.response["Error"]["Message"]}')
-
-    access_key = acct_b["Credentials"]["AccessKeyId"]
-    secret_key = acct_b["Credentials"]["SecretAccessKey"]
-    session_token = acct_b["Credentials"]["SessionToken"]
-
-    session = boto3.Session(
-        aws_access_key_id=access_key,
-        aws_secret_access_key=secret_key,
-        aws_session_token=session_token)
-
-    return session.client('s3')
-
-################################################################################
-
-def download_filter(args, s3_client, s3_bucket, s3_object):
-    """Decide whether to download an object from S3
-       Returns True if the object should be downloaded, or False if it should be skipped."""
-
-    # Ignore directories
-
-    if s3_object['Key'][-1] == '/':
-        verbose(args, f'{s3_object["Key"]} is a prefix, so will be skipped')
-        return False
-
-    # Handle the object as a Path for simpicity
-
-    object_path = Path(s3_object['Key'])
-
-    # Filter according to wildcard
-
-    if args.include:
-        for wildcard in args.include:
-            if '/' in wildcard:
-                if fnmatch.fnmatch(s3_object['Key'], wildcard):
-                    break
-            elif fnmatch.fnmatch(object_path.name, wildcard):
-                break
-        else:
-            verbose(args, f'"{s3_object["Key"]}" does not match any include wildcards, so will be skipped')
-            return False
-
-    if args.exclude:
-        for wildcard in args.exclude:
-            if '/' in wildcard:
-                if fnmatch.fnmatch(s3_object['Key'], wildcard):
-                    verbose(args, f'"{s3_object["Key"]}" matches one or more exclude wildcards, so will be skipped')
-            elif fnmatch.fnmatch(object_path.name, wildcard):
-                verbose(args, f'"{s3_object["Key"]}" matches one or more exclude wildcards, so will be skipped')
-                return False
-
-    # Filter according to content type
-
-    if args.include_type or args.exclude_type:
-        object_type = s3_client.head_object(Bucket=s3_bucket, Key=s3_object["Key"])['ContentType']
-
-        if args.include_type:
-            for include_type in args.include_type:
-                if object_type == include_type:
-                    break
-            else:
-                verbose(args, f'"{s3_object["Key"]}" is of type "{object_type}" which does not match any entries in the the type include list, so will be skipped')
-                return False
-
-        if args.exclude_type:
-            for exclude_type in args.exclude_type:
-                if object_type == exclude_type:
-                    verbose(args, f'"{s3_object["Key"]}" is of type "{object_type}" which matches one of the entries in the type exclude list, so will be skipped')
-                    return False
-
-    # Unless we are in force-download mode, check if the destination file already exists and see if it needs to be overwritten
-
-    if not args.force:
-        dest_file = args.destination / object_path
-
-        if dest_file.exists():
-            # Overwrite if destination is older or a different size
-
-            dest_stat = dest_file.stat()
-            dest_timestamp = datetime.datetime.fromtimestamp(dest_stat.st_mtime, tz=datetime.timezone.utc)
-
-            if dest_timestamp >= s3_object['LastModified']:
-                verbose(args, f'Destination file already exists and is same age or newer, so "{s3_object["Key"]}" will be skipped')
-                return False
-
-    return True
-
-################################################################################
-
-def download(args, s3_client, mkdir_lock, bucket, s3_object):
-    """Attempt to download an object from S3 to an equivalent local location"""
-
-    local_path = Path(args.destination) / s3_object['Key']
-
-    with mkdir_lock:
-        if local_path.parent.exists():
-            if not local_path.parent.is_dir():
-                error(f'Unable to download "{s3_object["Key"]}" as the destination path is not a directory')
-        else:
-            local_path.parent.mkdir(parents=True)
-
-    # Download the object and the set the file timestamp to the same as the object
-
-    object_timestamp = s3_object['LastModified'].timestamp()
-    s3_client.download_file(bucket, s3_object['Key'], local_path)
-    os.utime(local_path, (object_timestamp, object_timestamp))
-
-################################################################################
-
-def downloader(args, s3_client, mkdir_lock, bucket, object_queue, error_queue, sem_counter, real_thread=True):
-    """Download thread"""
-
-    finished = False
-    while not finished:
-        # Get the next object to download (waiting for one to be added to the queue)
-
-        s3_object = object_queue.get()
-
-        # If it is a candidate for downloading (meetings the criteria specified on the command
-        # line and, unless force-downloading, hasn't already been downloaded) then attempt to
-        # download it.
-
-        # If the semaphore is being used to limit the number of downloads, attempt to acquire it
-        # If we couldn't, then we've reached the download limit so we'll finish.
-
-        if download_filter(args, s3_client, bucket, s3_object):
-
-            if not sem_counter or sem_counter.acquire(blocking=False):
-                print(f'Downloading "{s3_object["Key"]}"')
-                try:
-                    download(args, s3_client, mkdir_lock, bucket, s3_object)
-                except ClientError as exc:
-                    error_queue.put(f'Failed to download "{s3_object["Key"]}" - {exc.response["Error"]["Message"]}')
-
-                    if sem_counter:
-                        sem_counter.release()
-                else:
-                    print(f'       Done "{s3_object["Key"]}"')
-
-            else:
-                finished = True
-
-        # Indicate the queued item has been consumed
-
-        object_queue.task_done()
-
-    # If we were using a download semaphore then drain the queue (this will happen in all
-    # threads and will never terminate, but we're running as a daemon so it doesn't matter too much).
-
-    if sem_counter and real_thread:
-        while True:
-            object_queue.get()
-            object_queue.task_done()
-
-################################################################################
-
-def thread_exception_handler(args):
-    """Brute-force thread exception handler"""
-
-    _ = args
-    sys.exit(1)
-
-################################################################################
-
-def main():
-    """Entry point"""
-
-    args = configure()
-
-    s3_client = get_client(args)
-
-    bucket = args.source
-
-    # Remove the 's3://' prefix, if present so that we can split bucket and folder
-    # if specified
-
-    if bucket.startswith('s3://'):
-        bucket = bucket[5:]
-
-    if '/' in bucket:
-        bucket, prefix = bucket.split('/', 1)
-    else:
-        prefix = ''
-
-    # Semaphore to protect download counter
-
-    sem_counter = threading.Semaphore(value=args.max_objects) if args.max_objects else None
-
-    # Create the download queue and the worker threads
-
-    object_queue = queue.Queue()
-
-    # Create the queue for reporting errors back from the threads
-
-    error_queue = queue.Queue()
-
-    # Lock to prevent race conditions around directory creation
-
-    mkdir_lock = threading.Lock()
-
-    if args.threads > 1:
-        # Create threads
-
-        threading.excepthook = thread_exception_handler
-
-        for _ in range(NUM_THREADS):
-            thread = threading.Thread(target=downloader, daemon=True, args=(args, s3_client, mkdir_lock, bucket, object_queue, error_queue, sem_counter))
-            thread.start()
-
-    # Read all the objects in the bucket and queue them for consideration by the download workers
-
-    for page in s3_client.get_paginator('list_objects_v2').paginate(Bucket=bucket, Prefix=prefix):
-        for s3_object in page['Contents']:
-            object_queue.put(s3_object)
-
-    print('Finished queuing objects')
-
-    if args.threads > 1:
-        # Wait for the queues to drain
-
-        object_queue.join()
-    else:
-        downloader(args, s3_client, mkdir_lock, bucket, object_queue, error_queue, sem_counter, real_thread=False)
-
-    # Report any errors:
-
-    if not error_queue.empty():
-        sys.stderr.write('\nErrors were encountered downloading some of the objects:\n\n\n')
-
-        while not error_queue.empty():
-            error_msg = error_queue.get()
-            sys.stderr.write(f'{error_msg}\n')
-            error_queue.task_done()
-
-################################################################################
-
-def s3_sync():
-    """Entry point"""
-
-    try:
-        main()
-    except KeyboardInterrupt:
-        sys.exit(1)
-    except BrokenPipeError:
-        sys.exit(2)
-
-################################################################################
-
-if __name__ == '__main__':
-    s3_sync()