skilleter-thingy 0.0.23__py3-none-any.whl → 0.0.25__py3-none-any.whl

skilleter_thingy/imagedupe.py

@@ -0,0 +1,281 @@
+#!/usr/bin/env python3
+"""Hash photos to find closely-similar images and report them"""
+
+# TODO: Configurable hash type and stored in pickle data
+# TODO: Option to pickle duplicates
+# TODO: Check for files that are no longer present after scanning and remove them from the data
+# TODO: Display duplicates - borrow code from imagedup
+# TODO: Note that if a==b, we get reports for a==b and b==a so we need to do some filtering and decide on what's likely to be an original and what is a duplicate
+# TODO: Maybe store file date, size and image dimensions in 'updated' for better comparisons
+# TODO: Pickle data named for directories
+# TODO: Use database instead of pickling - very slow to load big pickles - would facilitate scanning/checking specific directories
+# TODO: If directories specified, the only scan and compare within those directories (but keep everything in pickle file (yes, it will get biggerer, and biggererer)
+
+import threading
+import queue
+import sys
+import os
+import pickle
+import argparse
+from pathlib import PurePath
+from typing import List, Union
+
+from imagededup.methods import PHash
+
+import matplotlib.gridspec as gridspec
+import matplotlib.pyplot as plt
+from matplotlib import figure
+
+import numpy as np
+from PIL import Image
+
+################################################################################
+
+PICKLE_FILE = 'imagedupe.pickle'
+
+IMAGE_EXT_LIST =  ('.jpeg', '.jpg', '.png', '.bmp', '.mpo', '.ppm', '.tif', '.tiff', '.gif', '.svg', '.pgm', '.pbm', '.webp')
+
+PICKLE_VERSION = 1
+
+NUM_THREADS = 24
+
+################################################################################
+
+def queue_files_to_hash(file_queue, directories):
+    """Read all the specfied directories queue every file therein"""
+
+    # Walk each directory tree
+
+    for directory in directories:
+        print(f'Scanning directory tree {directory}')
+
+        for root, _, files in os.walk(directory):
+            print(f'Scanning directory {root}')
+
+            for file in files:
+                file_queue.put(os.path.join(root, file))
+
+################################################################################
+
+def hasher_thread(method, file_queue, hash_queue, hashes, updated):
+    """Thread - reads a file from the queue and, if it is an unhashed or updated image
+       calculate the hash and post it and the modification time on the updated queue"""
+
+    while not file_queue.empty():
+        filepath = file_queue.get()
+
+        fileext = os.path.splitext(filepath)[1]
+        mod_time = os.path.getmtime(filepath)
+
+        # If the file type is an image and the file hasn't been hashed or the modification time has changed
+        # then save the hash and the modification time
+
+        if fileext.lower() in IMAGE_EXT_LIST and (filepath not in hashes or mod_time != updated[filepath]):
+            # Calculate the hash and store path, dimensions and file size under the hash entry in the hashes table
+
+            print(f'Calculating hash for {filepath}')
+            encoding = method.encode_image(image_file=filepath)
+
+            if encoding:
+                hash_queue.put({'filepath': filepath, 'encoding': encoding, 'updated': mod_time})
+            else:
+                print(f'Invalid image {filepath}')
+
+        file_queue.task_done()
+
+################################################################################
+
+def hash_directories(directories, method, hashes, updated):
+    """Scan for new files and calculate their hashes"""
+
+    # Create the I/O queues
+
+    file_queue = queue.Queue()
+    hash_queue = queue.Queue()
+
+    # Queue the list of files to hash
+
+    queue_files_to_hash(file_queue, directories)
+
+    # Start the threads hashing away
+
+    thread_list = []
+    for _ in range(NUM_THREADS):
+        thread = threading.Thread(target=hasher_thread, daemon=True, args=(method, file_queue, hash_queue, hashes, updated))
+        thread.start()
+        thread_list.append(thread)
+
+    # Wait for the threads to stop working
+
+    print('Waiting for threads to terminate')
+
+    for thread in thread_list:
+        thread.join()
+
+    # Process the results
+
+    while not hash_queue.empty():
+        entry = hash_queue.get()
+
+        filepath = entry['filepath']
+        hashes[filepath] = entry['encoding']
+        updated[filepath] = entry['updated']
+
+################################################################################
+
+def plot_images(
+    orig: str,
+    image_list: List,
+    scores: bool = False,
+    outfile: str = None,
+) -> None:
+    """
+    Plotting function for plot_duplicates() defined below.
+
+    Args:
+        orig: filename for which duplicates are to be plotted.
+        image_list: List of duplicate filenames, could also be with scores (filename, score).
+        scores: Whether only filenames are present in the image_list or scores as well.
+        outfile:  Name of the file to save the plot.
+    """
+
+    def formatter(val: Union[int, np.float32]):
+        """
+        For printing floats only upto 3rd precision. Ints are unchanged.
+        """
+        if isinstance(val, np.float32):
+            return f'{val:.3f}'
+
+        return val
+
+    n_ims = len(image_list)
+    ncols = 4  # fixed for a consistent layout
+    nrows = int(np.ceil(n_ims / ncols)) + 1
+    fig = figure.Figure(figsize=(10, 14))
+
+    gs = gridspec.GridSpec(nrows=nrows, ncols=ncols)
+    ax = plt.subplot(
+        gs[0, 1:3]
+    )  # Always plot the original image in the middle of top row
+    ax.imshow(Image.open(orig))
+    ax.set_title(f'Original Image: {format(orig)}')
+    ax.axis('off')
+
+    for i in range(0, n_ims):
+        row_num = (i // ncols) + 1
+        col_num = i % ncols
+
+        ax = plt.subplot(gs[row_num, col_num])
+        if scores:
+            ax.imshow(Image.open(image_list[i][0]))
+            val = formatter(image_list[i][1])
+            title = ' '.join([image_list[i][0], f'({val})'])
+        else:
+            ax.imshow(Image.open(image_list[i]))
+            title = image_list[i]
+
+        ax.set_title(title, fontsize=6)
+        ax.axis('off')
+    gs.tight_layout(fig)
+
+    if outfile:
+        plt.savefig(outfile)
+
+    plt.show()
+    plt.close()
+
+################################################################################
+
+def main():
+    """Read the hashes and report duplicates in a vaguely civilised way"""
+
+    # Hashing and comparison method
+
+    method = PHash()
+
+    # Handle command line arguments
+
+    parser = argparse.ArgumentParser(description='Search for similar images')
+    parser.add_argument('--no-scan', action='store_true', help='Use pickled scan data without updating it')
+    parser.add_argument('--no-compare', action='store_true', help='Use pickled comparison data without updating it')
+    parser.add_argument('--show', action='store_true', help='Show duplicate images')
+    parser.add_argument('directories', nargs='*', action='store', help='Directories to search')
+
+    args = parser.parse_args()
+
+    breakpoint()
+    
+    if not args.no_scan and not args.directories:
+        print('You must be specify at least one directory in order to perform a scan')
+        sys.exit(1)
+
+    # We pickle the current set of files, hashes and comparisons
+
+    try:
+        print('Loading cached data')
+
+        with open(PICKLE_FILE, 'rb') as pickles:
+            data = pickle.load(pickles)
+
+            if data['version'] != PICKLE_VERSION:
+                print(f'WARNING: Current version is {PICKLE_VERSION} but saved data is from version {data["version"]}. Interesting things could happen....')
+
+            hashes = data['hashes']
+            updated = data['updated']
+            duplicates = data.get('duplicates', None)
+
+    except (FileNotFoundError, EOFError):
+        if args.no_scan:
+            print('ERROR: Cannot use no-scan option as no cached scan data is available')
+            sys.exit(1)
+
+        hashes = {}
+        updated = {}
+        duplicates = {}
+
+    if args.no_compare and not duplicates:
+        print('ERROR: Cannot use no-compare option as no cached comparison data is available')
+        sys.exit(1)
+
+    if not args.no_scan:
+        # Scan for new values and calculate hashes
+
+        hash_directories(method, args.directories, hashes, updated)
+
+    if not args.no_compare:
+        # Look for duplicates
+
+        duplicates = method.find_duplicates(encoding_map=hashes)
+
+    # Pickle the updated results
+
+    with open(PICKLE_FILE, 'wb') as pickles:
+        dupe_data = {'hashes': hashes, 'updated': updated, 'version': PICKLE_VERSION, 'duplicates': duplicates}
+        pickle.dump(dupe_data, pickles)
+
+    # Report them
+
+    for entry in duplicates:
+        if duplicates[entry]:
+            print(f'{entry}: {duplicates[entry]}')
+            if args.show:
+                plot_duplicates(entry, duplicates[entry])
+
+################################################################################
+
+def photodupe():
+    """Entry point"""
+
+    try:
+        main()
+
+    except KeyboardInterrupt:
+        sys.exit(1)
+
+    except BrokenPipeError:
+        sys.exit(2)
+
+################################################################################
+
+if __name__ == '__main__':
+    photodupe()

skilleter_thingy/photodupe.py

@@ -5,6 +5,7 @@ import sys
 import os
 import sys
 import pickle
+import argparse
 
 import PIL
 
@@ -15,28 +16,42 @@ import imagehash
 
 ################################################################################
 
-def read_image_hashes():
+def read_image_hashes(directories):
     """Read all the specfied directories and hash every picture therein"""
 
     hashes = defaultdict(list)
 
-    for directory in sys.argv[1:]:
+    # Walk each directory tree
+    
+    for directory in directories:
+        print(f'Scanning directory tree {directory}')
+        
         for root, _, files in os.walk(directory):
+            print(f'Scanning directory {root}')
+            
             for file in files:
                 filepath = os.path.join(root, file)
-
-                try:
-                    with Image.open(filepath) as image:
-                        hash_value = imagehash.average_hash(image, hash_size=12)
-
-                        size = os.stat(filepath).st_size
-                        hashes[hash_value].append({'path': filepath, 'width': image.width, 'height': image.height, 'size': size})
-
-                except PIL.UnidentifiedImageError:
-                    sys.stderr.write(f'ERROR: Unrecognized format {filepath}\n')
-
-                except OSError:
-                    sys.stderr.write(f'ERROR: Unable to read {filepath}\n')
+                
+                fileext = os.path.splitext(file)[1]
+
+                if fileext.lower() not in ('.jbf', '.ini', '.xml', '.ffs_db'):
+                    # Calculate the hash and store path, dimensions and file size under the hash entry in the hashes table
+                    
+                    try:
+                        with Image.open(filepath) as image:
+                            hash_value = imagehash.average_hash(image, hash_size=12)
+
+                            size = os.stat(filepath).st_size
+                            hashes[hash_value].append({'path': filepath, 'width': image.width, 'height': image.height, 'size': size})
+
+                    except PIL.UnidentifiedImageError:
+                        sys.stderr.write(f'ERROR: Unrecognized format {filepath} (size={size})\n')
+
+                    except OSError:
+                        sys.stderr.write(f'ERROR: Unable to read {filepath} (size={size})\n')
+                    
+    # Return the hash table
+    
     return hashes
 
 ################################################################################
@@ -44,6 +59,15 @@ def read_image_hashes():
 def main():
     """Read the hashes and report duplicates in a vaguely civilised way"""
 
+    parser = argparse.ArgumentParser(description='Search for similar images')
+    parser.add_argument('directories', nargs='*', action='store', help='Directories to search')
+
+    args = parser.parse_args()
+    
+    if not args.directories:
+        print('You must be specify at least one directory')
+        sys.exit(1)
+        
     try:
         print('Loading cached data')
 
@@ -52,8 +76,10 @@ def main():
     except (FileNotFoundError, EOFError):
         print('Scanning directories')
 
-        hashes = read_image_hashes()
+        hashes = read_image_hashes(args.directories)
 
+    # Sort the list of hashes so that we can easily find close matches
+    
     print('Sorting hashes')
 
     hash_values = sorted([str(hashval) for hashval in hashes])

skilleter_thingy/sysmon.py

@@ -217,7 +217,7 @@ def show_cpu_load(scr, first, w, h, x, y):
         yo += 1
 
         if yo > h-2:
-            xo += w/3
+            xo += w//3
             yo = 0
 
     x += w//2

{skilleter_thingy-0.0.23.dist-info → skilleter_thingy-0.0.25.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: skilleter_thingy
-Version: 0.0.23
+Version: 0.0.25
 Summary: A collection of useful utilities, mainly aimed at making Git more friendly
 Author-email: John Skilleter <john@skilleter.org.uk>
 Project-URL: Home, https://skilleter.org.uk
@@ -10,7 +10,6 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: boto3
 Requires-Dist: imagehash
 Requires-Dist: inotify
 Requires-Dist: pillow
@@ -19,6 +18,7 @@ Requires-Dist: pyaml
 Requires-Dist: pygit2
 Requires-Dist: python-dateutil
 Requires-Dist: requests
+Requires-Dist: numpy
 
 # Thingy
 

{skilleter_thingy-0.0.23.dist-info → skilleter_thingy-0.0.25.dist-info}/RECORD

@@ -1,6 +1,5 @@
 skilleter_thingy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 skilleter_thingy/addpath.py,sha256=4Yhhgjjz1XDI98j0dAiQpNA2ejLefeWUTeSg3nIXQq0,3842
-skilleter_thingy/aws.py,sha256=k08MT1866KuMjgG7ECr7LCuqcgWx78iPDcS9CmemIoA,18614
 skilleter_thingy/borger.py,sha256=AQX7OHeGXcUjkgyXEWE2h_oOey9eczZBbKjrreXvRAs,7832
 skilleter_thingy/colour.py,sha256=D-RTYsND6Xm6m3xl0mOe9QSrTNYsyY0K_a8x3id2gvg,7031
 skilleter_thingy/console_colours.py,sha256=dT5qc_B62VJaWs92yrFt1izoB7hs3a22t3lfrZFybG4,1786
@@ -33,11 +32,12 @@ skilleter_thingy/gitlab.py,sha256=TKKkrQ3CfbWViEf_fnroQJSW8Nh-YwUeX6zo7GmzndI,60
 skilleter_thingy/gitprompt.py,sha256=m3WbeEd0WgNIyKbtx-wp-kHnOG83cWjq_KA8W6RmD4g,8925
 skilleter_thingy/gl.py,sha256=fb9OkxfPSuqyfnLYauN0F1jZYIJNLXhNwlU8PRjJ4j4,5964
 skilleter_thingy/gphotosync.py,sha256=-LbNXxi34sMSufmZ-NoStIe-6cKFKK2zvcDH7quI8W8,22040
+skilleter_thingy/imagedupe.py,sha256=60iSz4unS3REAWYHcWa4fnX0-kOrKVYcGuV2xBksj_E,9120
 skilleter_thingy/linecount.py,sha256=lw3vuXUUnMwrUihY6bHfZJsRKe6ZMCRz3952Z9N-ogI,4316
 skilleter_thingy/logger.py,sha256=xKgPAq8KGXmtaXIFjFs1AmZJXtYrXJn2sqL3oxHZjfQ,3107
 skilleter_thingy/moviemover.py,sha256=j_Xb9_jFdgpFBAXcF4tEqbnKH_FonlnUU39LiCK980k,4470
 skilleter_thingy/path.py,sha256=3ba_e-QwYpAs-jFVWoV8sfjVjs_59uc5JZt-87Hqn6g,4737
-skilleter_thingy/photodupe.py,sha256=EiWTDLI4tFZp8ruwkbCayMTiwRfMI7O_kmbWgXwZGVQ,3178
+skilleter_thingy/photodupe.py,sha256=FssLgbLnqHPuSvMGtRyOa7bRdowufJOQaJJ56f9ybxk,4195
 skilleter_thingy/phototidier.py,sha256=1uj1XbUemJOalNC3PwTG2-_yUQp4WMAcmwRr7QXPn1w,7823
 skilleter_thingy/popup.py,sha256=jW-nbpdeswqEMTli7OmBv1J8XQsvFoMI0J33O6dOeu8,2529
 skilleter_thingy/process.py,sha256=RmXj2RrzwqP_mugeVejtjgF_T3PD7M10XsWuAnc99t0,3565
@@ -47,10 +47,9 @@ skilleter_thingy/remdir.py,sha256=zp5Nr0IMGXQ-b5iT48O5arqWoSjW65Xnr-SpKuav1Ac,46
 skilleter_thingy/rmdupe.py,sha256=tcX3w8XvliGwBMdSt9BUu07kuDtQEc0IiU8sCxmgzHA,17117
 skilleter_thingy/rpylint.py,sha256=na39x0yNXDwDkG9yP48BoM5FeTut-OS4AVsYixE0YZU,2639
 skilleter_thingy/run.py,sha256=EGYJSuMcOmUca6dpfVUFE41vG9C6ZNK8hzZlJCJE6Rs,12619
-skilleter_thingy/s3_sync.py,sha256=TITptjua_B-iwPlgTniuoxPvuEnQjyTKfs6l9CKHbXc,13849
 skilleter_thingy/splitpics.py,sha256=qRlJrqet7TEI6SodS4bkuKXQUpOdMaqmjE4c1CR7ouo,3266
 skilleter_thingy/strreplace.py,sha256=xsIWw0hc452rYEBtNEQFKIzmV03xjm_Taz-eDTmFFKI,2539
-skilleter_thingy/sysmon.py,sha256=zSnR9oqGr1TwOhwn7Mcofq2fcnoVjlg5gRH56l2w1N0,11347
+skilleter_thingy/sysmon.py,sha256=XRZG6EVSzoVYan_N16qVB1l1RaU51uvLWlRA0CDjC54,11348
 skilleter_thingy/tfm.py,sha256=wG4oNhn1pBcLwPPzZc19x_HyPYsM1pl8E0skvj03IH8,33712
 skilleter_thingy/tfm_pane.py,sha256=BmyRDKZyr0mS89MbudbOUA4uBh4E6X3tGct6JX87vI8,19829
 skilleter_thingy/tfparse.py,sha256=y4MSipVPO-P12QXMSXAT92qy9YMqNDl_1Thum9j7S2g,2993
@@ -59,9 +58,9 @@ skilleter_thingy/trimpath.py,sha256=SAfOB75_dTldQHjam4kQy1J42209NYPYi8vVAaNn1e8,
 skilleter_thingy/window_rename.py,sha256=dCBgZqih_3YKHt35hsOAhARFp3QxOi8w8huC63sqJK8,3128
 skilleter_thingy/xchmod.py,sha256=F9_lxKuLqVlHHr3oBI3dkMoFOuwRzYDlpQMTmDcjpBI,4590
 skilleter_thingy/yamlcheck.py,sha256=FXylZ5NtHirDlPVhVEUZUZkTugVR-g51BbjaN06akAc,2868
-skilleter_thingy-0.0.23.dist-info/LICENSE,sha256=ljOS4DjXvqEo5VzGfdaRwgRZPbNScGBmfwyC8PChvmQ,32422
-skilleter_thingy-0.0.23.dist-info/METADATA,sha256=amrMSKYsBCKb8c3vm4e0e-pDl27Gamk9K6rn7oHy2oU,5231
-skilleter_thingy-0.0.23.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-skilleter_thingy-0.0.23.dist-info/entry_points.txt,sha256=jf7hVfH0dojLxk0D4TjIazKoqdToditCGdCfyhIotME,1971
-skilleter_thingy-0.0.23.dist-info/top_level.txt,sha256=8-JhgToBBiWURunmvfpSxEvNkDHQQ7r25-aBXtZv61g,17
-skilleter_thingy-0.0.23.dist-info/RECORD,,
+skilleter_thingy-0.0.25.dist-info/LICENSE,sha256=ljOS4DjXvqEo5VzGfdaRwgRZPbNScGBmfwyC8PChvmQ,32422
+skilleter_thingy-0.0.25.dist-info/METADATA,sha256=2n3rE7kBsRxBjonvItXcdpwaJf6stvUGB4g2hCmAWCM,5231
+skilleter_thingy-0.0.25.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+skilleter_thingy-0.0.25.dist-info/entry_points.txt,sha256=jf7hVfH0dojLxk0D4TjIazKoqdToditCGdCfyhIotME,1971
+skilleter_thingy-0.0.25.dist-info/top_level.txt,sha256=8-JhgToBBiWURunmvfpSxEvNkDHQQ7r25-aBXtZv61g,17
+skilleter_thingy-0.0.25.dist-info/RECORD,,