megadetector 5.0.12__py3-none-any.whl → 5.0.14__py3-none-any.whl

megadetector/detection/run_inference_with_yolov5_val.py

@@ -34,13 +34,6 @@ that permission and *doesn't* have admin privileges.  If you are running this sc
 Windows and you don't have admin privileges, use --no_use_symlinks, which will make copies of images,
 rather than using symlinks.
 
-TODO:
-
-* Multiple GPU support
-* Checkpointing
-* Support alternative class names at the command line (currently defaults to MD classes,
-  though other class names can be supplied programmatically)
-
 """
 
 #%% Imports
@@ -52,14 +45,19 @@ import glob
 import tempfile
 import shutil
 import json
+import copy
 
 from tqdm import tqdm
 
 from megadetector.utils import path_utils
 from megadetector.utils import process_utils
 from megadetector.utils import string_utils
+
+from megadetector.utils.ct_utils import is_iterable, split_list_into_fixed_size_chunks
+from megadetector.utils.path_utils import path_is_abs
 from megadetector.data_management import yolo_output_to_md_output
 from megadetector.detection.run_detector import try_download_known_detector
+from megadetector.postprocessing.combine_api_outputs import combine_api_output_files
 
 default_image_size_with_augmentation = int(1280 * 1.3)
 default_image_size_with_no_augmentation = 1280
@@ -73,93 +71,141 @@ class YoloInferenceOptions:
     the input/output filenames.
     """
     
-    ## Required ##
-    
-    #: Folder of images to process
-    input_folder = None
-    
-    #: Model filename (ending in .pt), or a well-known model name (e.g. "MDV5A")
-    model_filename = None
-    
-    #: .json output file, in MD results format
-    output_file = None
-    
-    
-    ## Optional ##
+    def __init__(self):
+        
+        ## Required-ish ##
+        
+        #: Folder of images to process (can be None if image_filename_list contains absolute paths)
+        self.input_folder = None
+        
+        #: If this is None, [input_folder] can't be None, we'll process all images in [input_folder].
+        #:            
+        #: If this is not None, and [input_folder] is not None, this should be a list of relative image 
+        #: paths within [input_folder] to process, or a .txt or .json file containing a list of 
+        #: relative image paths.
+        #:
+        #: If this is not None, and [input_folder] is None, this should be a list of absolute image
+        #: paths, or a .txt or .json file containing a list of absolute image paths.            
+        self.image_filename_list = None
+        
+        #: Model filename (ending in .pt), or a well-known model name (e.g. "MDV5A")
+        self.model_filename = None
+        
+        #: .json output file, in MD results format
+        self.output_file = None
+        
+        
+        ## Optional ##
+        
+        #: Required for older YOLOv5 inference, not for newer ulytralytics/YOLOv8 inference
+        self.yolo_working_folder = None
+        
+        #: Currently 'yolov5' and 'ultralytics' are supported, and really these are proxies for
+        #: "the yolov5 repo" and "the ultralytics repo".
+        self.model_type = 'yolov5' 
     
-    #: Required for older YOLOv5 inference, not for newer ulytralytics/YOLOv8 inference
-    yolo_working_folder = None
+        #: Image size to use; this is a single int, which in ultralytics's terminology means
+        #: "scale the long side of the image to this size, and preserve aspect ratio".
+        self.image_size = default_image_size_with_augmentation
+        
+        #: Detections below this threshold will not be included in the output file
+        self.conf_thres = '0.001'
+        
+        #: Batch size... has no impact on results, but may create memory issues if you set
+        #: this to large values
+        self.batch_size = 1
+        
+        #: Device string: typically '0' for GPU 0, '1' for GPU 1, etc., or 'cpu'
+        self.device_string = '0'
+        
+        #: Should we enable test-time augmentation?
+        self.augment = True
+        
+        #: Should we enable half-precision inference?
+        self.half_precision_enabled = None
+        
+        #: Where should we stash the temporary symlinks (or copies) used to give unique identifiers to image 
+        # files?
+        #:
+        #: If this is None, we'll create a folder in system temp space.
+        self.symlink_folder = None
+        
+        #: Should we use symlinks to give unique identifiers to image files (vs. copies)?
+        self.use_symlinks = True
+        
+        #: How should we guarantee that YOLO IDs (base filenames) are unique?  Choices are:
+        #:
+        #: * 'verify': assume image IDs are unique, but verify and error if they're not
+        #: * 'links': create symlinks (or copies, depending on use_symlinks) to enforce uniqueness
+        #: * 'auto': check whether IDs are unique, create links if necessary
+        self.unique_id_strategy = 'links'
+        
+        #: Temporary folder to stash intermediate YOLO results.
+        #:
+        #: If this is None, we'll create a folder in system temp space.    
+        self.yolo_results_folder = None
+        
+        #: Should we remove the symlink folder when we're done?
+        self.remove_symlink_folder = True
+        
+        #: Should we remove the intermediate results folder when we're done?
+        self.remove_yolo_results_folder = True
+        
+        #: These are deliberately offset from the standard MD categories; YOLOv5
+        #: needs categories IDs to start at 0.
+        #:
+        #: This can also be a string that points to a YOLO dataset.yaml file.
+        self.yolo_category_id_to_name = {0:'animal',1:'person',2:'vehicle'}
+        
+        #: What should we do if the output file already exists?
+        #:
+        #: Can be 'error', 'skip', or 'overwrite'.
+        self.overwrite_handling = 'skip'
+        
+        #: If True, we'll do a dry run that lets you preview the YOLO val command, without
+        #: actually running it.
+        self.preview_yolo_command_only = False
+        
+        #: By default, if any errors occur while we're copying images or creating symlinks, it's
+        #: game over.  If this is True, those errors become warnings, and we plow ahead.
+        self.treat_copy_failures_as_warnings = False
+        
+        #: Save YOLO console output
+        self.save_yolo_debug_output = False
+        
+        #: Whether to search for images recursively within [input_folder]
+        #:
+        #: Ignored if a list of files is provided.
+        self.recursive = True
+        
+        #: Maximum number of images to run in a single chunk
+        self.checkpoint_frequency = None
+        
+    # ...def __init__()
     
-    #: Currently 'yolov5' and 'ultralytics' are supported, and really these are proxies for
-    #: "the yolov5 repo" and "the ultralytics repo".
-    model_type = 'yolov5' 
+# ...YoloInferenceOptions()
 
-    #: Image size to use; this is a single int, which in ultralytics's terminology means
-    #: "scale the long side of the image to this size, and preserve aspect ratio".
-    image_size = default_image_size_with_augmentation
-    
-    #: Detections below this threshold will not be included in the output file
-    conf_thres = '0.001'
-    
-    #: Batch size... has no impact on results, but may create memory issues if you set
-    #: this to large values
-    batch_size = 1
-    
-    #: Device string: typically '0' for GPU 0, '1' for GPU 1, etc., or 'cpu'
-    device_string = '0'
-    
-    #: Should we enable test-time augmentation?
-    augment = True
-    
-    #: Should we enable half-precision inference?
-    half_precision_enabled = None
-    
-    #: Where should we stash the temporary symlinks used to give unique identifiers to image files?
-    #:
-    #: If this is None, we'll create a folder in system temp space.
-    symlink_folder = None
-    
-    #: Should we use symlinks to give unique identifiers to image files (vs. copies)?
-    use_symlinks = True
-    
-    #: Temporary folder to stash intermediate YOLO results.
-    #:
-    #: If this is None, we'll create a folder in system temp space.    
-    yolo_results_folder = None
-    
-    #: Should we remove the symlink folder when we're done?
-    remove_symlink_folder = True
-    
-    #: Should we remove the intermediate results folder when we're done?
-    remove_yolo_results_folder = True
-    
-    #: These are deliberately offset from the standard MD categories; YOLOv5
-    #: needs categories IDs to start at 0.
-    #:
-    #: This can also be a string that points to a YOLO dataset.yaml file.
-    yolo_category_id_to_name = {0:'animal',1:'person',2:'vehicle'}
-    
-    #: What should we do if the output file already exists?
-    #:
-    #: Can be 'error', 'skip', or 'overwrite'.
-    overwrite_handling = 'skip'
-    
-    #: If True, we'll do a dry run that lets you preview the YOLO val command, without
-    #: actually running it.
-    preview_yolo_command_only = False
-    
-    #: By default, if any errors occur while we're copying images or creating symlinks, it's
-    #: game over.  If this is True, those errors become warnings, and we plow ahead.
-    treat_copy_failures_as_warnings = False
-    
-    #: Save YOLO console output
-    save_yolo_debug_output = False
-    
-    #: Whether to search for images recursively within [input_folder]
-    recursive = True
-            
+
+#%% Support functions
+
+def _clean_up_temporary_folders(options,
+                                symlink_folder,yolo_results_folder,
+                                symlink_folder_is_temp_folder,yolo_folder_is_temp_folder):
+    """
+    Remove temporary symlink/results folders, unless the caller requested that we leave them in place.
+    """
     
-# ...YoloInferenceOptions()
+    if options.remove_symlink_folder:
+        shutil.rmtree(symlink_folder)
+    elif symlink_folder_is_temp_folder:
+        print('Warning: using temporary symlink folder {}, but not removing it'.format(
+            symlink_folder))
+        
+    if options.remove_yolo_results_folder:
+        shutil.rmtree(yolo_results_folder)
+    elif yolo_folder_is_temp_folder:
+        print('Warning: using temporary YOLO results folder {}, but not removing it'.format(
+            yolo_results_folder))
 
     
 #%% Main function
@@ -173,9 +219,15 @@ def run_inference_with_yolo_val(options):
         options (YoloInferenceOptions): all the parameters used to control this process,
             including filenames; see YoloInferenceOptions for details            
     """
-    
+        
     ##%% Input and path handling
     
+    default_options = YoloInferenceOptions()
+    
+    for k in options.__dict__.keys():
+        if k not in default_options.__dict__:
+            print('Warning: unexpected variable {} in options object'.format(k))
+            
     if options.model_type == 'yolov8':
         
         print('Warning: model type "yolov8" supplied, "ultralytics" is the preferred model type string for YOLOv8 models')
@@ -191,20 +243,24 @@ def run_inference_with_yolo_val(options):
         assert os.path.isdir(options.yolo_working_folder), \
             'Could not find working folder {}'.format(options.yolo_working_folder)
                 
-    assert os.path.isdir(options.input_folder) or os.path.isfile(options.input_folder), \
-        'Could not find input {}'.format(options.input_folder)
-    
     if options.half_precision_enabled is not None:
         assert options.half_precision_enabled in (0,1), \
             'Invalid value {} for --half_precision_enabled (should be 0 or 1)'.format(
                 options.half_precision_enabled)
-            
+    
     # If the model filename is a known model string (e.g. "MDv5A", download the model if necessary)
     model_filename = try_download_known_detector(options.model_filename)
     
     assert os.path.isfile(model_filename), \
         'Could not find model file {}'.format(model_filename)
     
+    assert (options.input_folder is not None) or (options.image_filename_list is not None), \
+        'You must specify a folder and/or a file list'
+        
+    if options.input_folder is not None:
+        assert os.path.isdir(options.input_folder), 'Could not find input folder {}'.format(
+            options.input_folder)
+    
     if os.path.exists(options.output_file):
         if options.overwrite_handling == 'skip':
             print('Warning: output file {} exists, skipping'.format(options.output_file))
@@ -215,13 +271,17 @@ def run_inference_with_yolo_val(options):
             raise ValueError('Output file {} exists'.format(options.output_file))
         else:
             raise ValueError('Unknown output handling method {}'.format(options.overwrite_handling))
-            
-    os.makedirs(os.path.dirname(options.output_file),exist_ok=True)
     
+    os.makedirs(os.path.dirname(options.output_file),exist_ok=True)
     
+    if options.input_folder is not None:
+        options.input_folder = options.input_folder.replace('\\','/')
+        
+
     ##%% Other input handling
     
     if isinstance(options.yolo_category_id_to_name,str):
+        
         assert os.path.isfile(options.yolo_category_id_to_name)
         yolo_dataset_file = options.yolo_category_id_to_name
         options.yolo_category_id_to_name = \
@@ -265,61 +325,273 @@ def run_inference_with_yolo_val(options):
 
     ##%% Enumerate images
     
-    if os.path.isdir(options.input_folder):
-        image_files_absolute = path_utils.find_images(options.input_folder,recursive=options.recursive)
+    image_files_relative = None
+    image_files_absolute = None
+    
+    if options.image_filename_list is None:
+        assert options.input_folder is not None and os.path.isdir(options.input_folder), \
+            'Could not find input folder {}'.format(options.input_folder)
+        image_files_relative = path_utils.find_images(options.input_folder,
+                                                      recursive=options.recursive,
+                                                      return_relative_paths=True,
+                                                      convert_slashes=True)
+        image_files_absolute = [os.path.join(options.input_folder,fn) for \
+                                fn in image_files_relative]
     else:
-        assert os.path.isfile(options.input_folder)
-        with open(options.input_folder,'r') as f:            
-            image_files_absolute = json.load(f)
-            assert isinstance(image_files_absolute,list)
-            for fn in image_files_absolute:
-                assert os.path.isfile(fn), 'Could not find image file {}'.format(fn)
+        
+        if is_iterable(options.image_filename_list):
+            
+            image_files_relative = options.image_filename_list
+            
+        else:
+            assert isinstance(options.image_filename_list,str), \
+                'Unrecognized image filename list object type: {}'.format(options.image_filename_list)
+            assert os.path.isfile(options.image_filename_list), \
+                'Could not find image filename list file: {}'.format(options.image_filename_list)
+            ext = os.path.splitext(options.image_filename_list).lower()
+            assert ext in ('.json','.txt'), \
+                'Unrecognized image filename list file extension: {}'.format(options.image_filename_list)
+            if ext == '.json':
+                with open(options.image_filename_list,'r') as f:
+                    image_files_relative = json.load(f)
+                    assert is_iterable(image_files_relative)
+            else:
+                assert ext == '.txt'
+                with open(options.image_filename_list,'r') as f:
+                    image_files_relative = f.readlines()
+                    image_files_relative = [s.strip() for s in image_files_relative]
+        
+        # ...whether the image filename list was supplied as list vs. a filename
+        
+        if options.input_folder is None:
+            image_files_absolute = image_files_relative
+        else:
+            # The list should be relative filenames
+            for fn in image_files_relative:
+                assert not path_is_abs(fn), \
+                    'When providing a folder and a list, paths in the list should be relative'
+                
+            image_files_absolute = \
+                [os.path.join(options.input_folder,fn) for fn in image_files_relative]
+        for fn in image_files_absolute:
+            assert os.path.isfile(fn), 'Could not find image file {}'.format(fn)
+    
+    # ...whether the caller supplied a list of filenames
+    
+    image_files_absolute = [fn.replace('\\','/') for fn in image_files_absolute]
+    del image_files_relative
+    
+    
+    ##%% Recurse if necessary to handle checkpoints
+    
+    if options.checkpoint_frequency is not None and options.checkpoint_frequency > 0:
+        
+        chunks = split_list_into_fixed_size_chunks(image_files_absolute,options.checkpoint_frequency)
+        
+        chunk_output_files = []
+        
+        # i_chunk = 0; chunk_files_abs = chunks[i_chunk]
+        for i_chunk,chunk_files_abs in enumerate(chunks):
+            
+            print('Processing {} images from chunk {} of {}'.format(
+                len(chunk_files_abs),i_chunk,len(chunks)))
+    
+            chunk_options = copy.deepcopy(options)
+            
+            # Run each chunk without checkpointing
+            chunk_options.checkpoint_frequency = None
+            
+            if options.input_folder is not None:
+                chunk_files_relative = \
+                    [os.path.relpath(fn,options.input_folder) for fn in chunk_files_abs]
+                chunk_options.image_filename_list = chunk_files_relative
+            else:
+                chunk_options.image_filename_list = chunk_files_abs
+                
+            chunk_options.image_filename_list = \
+                [fn.replace('\\','/') for fn in chunk_options.image_filename_list]
+                
+            chunk_string = 'chunk_{}'.format(str(i_chunk).zfill(5))
+            chunk_options.yolo_results_folder = yolo_results_folder + '_' + chunk_string
+            chunk_options.symlink_folder = symlink_folder + '_' + chunk_string
+            
+            # Put the output file in the parent job's scratch folder
+            chunk_output_file = os.path.join(yolo_results_folder,chunk_string + '_results_md_format.json')
+            chunk_output_files.append(chunk_output_file)
+            chunk_options.output_file = chunk_output_file
+            
+            if os.path.isfile(chunk_output_file):
+                
+                print('Chunk output file {} exists, checking completeness'.format(chunk_output_file))
+                
+                with open(chunk_output_file,'r') as f:
+                    chunk_results = json.load(f)
+                images_in_this_chunk_results_file = [im['file'] for im in chunk_results['images']]                    
+                assert len(images_in_this_chunk_results_file) == len(chunk_options.image_filename_list), \
+                    'Expected {} images in chunk results file {}, found {}, possibly this is left over from a previous job?'.format(
+                        len(chunk_options.image_filename_list),chunk_output_file,
+                        len(images_in_this_chunk_results_file))
+                for fn in images_in_this_chunk_results_file:
+                    assert fn in chunk_options.image_filename_list, \
+                        'Unexpected image {} in chunk results file {}, possibly this is left over from a previous job?'.format(
+                            fn,chunk_output_file)
+                
+                print('Chunk output file {} exists and is complete, skipping this chunk'.format(
+                    chunk_output_file))
+                
+            # ...if the outptut file exists
+            
+            else:
+                
+                run_inference_with_yolo_val(chunk_options)
+                
+            # ...if we do/don't have to run this chunk
+            
+            assert os.path.isfile(chunk_options.output_file)
+            
+        # ...for each chunk
     
+        # Merge
+        _ = combine_api_output_files(input_files=chunk_output_files,
+                                 output_file=options.output_file,
+                                 require_uniqueness=True,
+                                 verbose=True)
+        
+        # Validate
+        with open(options.output_file,'r') as f:
+            combined_results = json.load(f)
+        assert len(combined_results['images']) == len(image_files_absolute), \
+            'Expected {} images in merged output file, found {}'.format(
+                len(image_files_absolute),len(combined_results['images']))
+        
+        # Clean up
+        _clean_up_temporary_folders(options,
+                                    symlink_folder,yolo_results_folder,
+                                    symlink_folder_is_temp_folder,yolo_folder_is_temp_folder)
+        
+        return
     
-    ##%% Create symlinks to give a unique ID to each image
+    # ...if we need to make recursive calls for file chunks
     
+    
+    ##%% Create symlinks (or copy images) to give a unique ID to each image
+    
+    # Maps YOLO image IDs (base filename without extension as it will appear in YOLO .json output)
+    # to the *original full path* for each image (not the symlink path).
     image_id_to_file = {}  
+    
+    # Maps YOLO image IDs (base filename without extension as it will appear in YOLO .json output)
+    # to errors, including errors that happen before we run the model at all (e.g. file access errors).
     image_id_to_error = {}
     
-    if options.use_symlinks:
-        print('Creating {} symlinks in {}'.format(len(image_files_absolute),symlink_folder_inner))
+    create_links = True
+    
+    if options.unique_id_strategy == 'links':
+        
+        create_links = True
+        
     else:
-        print('Symlinks disabled, copying {} images to {}'.format(len(image_files_absolute),symlink_folder_inner))
         
-    # i_image = 0; image_fn = image_files_absolute[i_image]
-    for i_image,image_fn in tqdm(enumerate(image_files_absolute),total=len(image_files_absolute)):
+        assert options.unique_id_strategy in ('auto','verify'), \
+            'Unknown unique ID strategy {}'.format(options.unique_id_strategy)
+            
+        image_ids_are_unique = True
         
-        ext = os.path.splitext(image_fn)[1]
+        for i_image,image_fn in tqdm(enumerate(image_files_absolute),total=len(image_files_absolute)):        
+            
+            image_id = os.path.splitext(os.path.basename(image_fn))[0]
+            
+            # Is this image ID unique?
+            if image_id in image_id_to_file:
+                if options.unique_id_strategy == 'verify':
+                    raise ValueError('"verify" specified for image uniqueness, but ' +
+                                     'image ID {} occurs more than once:\n\n{}\n\n{}'.format(
+                                         image_id,image_fn,image_id_to_file[image_id]))
+                else:
+                    assert options.unique_id_strategy == 'auto'
+                    image_ids_are_unique = False
+                    image_id_to_file = {}
+                    break
+                
+            image_id_to_file[image_id] = image_fn
         
-        image_id = str(i_image).zfill(10)
-        image_id_to_file[image_id] = image_fn
-        symlink_name = image_id + ext
-        symlink_full_path = os.path.join(symlink_folder_inner,symlink_name)
+        # ...for each image
         
-        try:
+        if image_ids_are_unique:
+            
+            print('"{}" specified for image uniqueness and images are unique, skipping links'.format(
+                options.unique_id_strategy))
+            assert len(image_id_to_file) == len(image_files_absolute)
+            create_links = False
+            
+        else:
+            
+            assert options.unique_id_strategy == 'auto'
+            create_links = True
+            link_type = 'copies'
             if options.use_symlinks:
-                path_utils.safe_create_link(image_fn,symlink_full_path)
-            else:
-                shutil.copyfile(image_fn,symlink_full_path)
-        except Exception as e:
-            error_string = str(e)
-            image_id_to_error[image_id] = error_string
-            # Always break if the user is trying to create symlinks on Windows without
-            # permission, 100% of images will always fail in this case.
-            if ('a required privilege is not held by the client' in error_string.lower()) or \
-               (not options.treat_copy_failures_as_warnings):
-                   print('\nError copying/creating link for input file {}: {}'.format(
-                       image_fn,error_string))
-                   
-                   raise
-            else:
-                print('Warning: error copying/creating link for input file {}: {}'.format(
-                    image_fn,error_string))
-                continue
+                link_type = 'links'
+            print('"auto" specified for image uniqueness and images are not unique, defaulting to {}'.format(
+                link_type))
+            
+    # ...which unique ID strategy?
+    
+    if create_links:
         
-    # ...for each image
-
+        if options.use_symlinks:
+            print('Creating {} symlinks in {}'.format(len(image_files_absolute),symlink_folder_inner))
+        else:
+            print('Symlinks disabled, copying {} images to {}'.format(len(image_files_absolute),symlink_folder_inner))
+            
+        link_full_paths = []
+        
+        # i_image = 0; image_fn = image_files_absolute[i_image]
+        for i_image,image_fn in tqdm(enumerate(image_files_absolute),total=len(image_files_absolute)):
+            
+            ext = os.path.splitext(image_fn)[1]
+            
+            # YOLO .json output identifies images by the base filename without the extension
+            image_id = str(i_image).zfill(10)
+            image_id_to_file[image_id] = image_fn
+            symlink_name = image_id + ext
+            symlink_full_path = os.path.join(symlink_folder_inner,symlink_name)
+            link_full_paths.append(symlink_full_path)
+            
+            try:
+                
+                if options.use_symlinks:
+                    path_utils.safe_create_link(image_fn,symlink_full_path)
+                else:
+                    shutil.copyfile(image_fn,symlink_full_path)
+                    
+            except Exception as e:
+                
+                error_string = str(e)
+                image_id_to_error[image_id] = error_string
+                
+                # Always break if the user is trying to create symlinks on Windows without
+                # permission, 100% of images will always fail in this case.
+                if ('a required privilege is not held by the client' in error_string.lower()) or \
+                   (not options.treat_copy_failures_as_warnings):
+                       
+                       print('\nError copying/creating link for input file {}: {}'.format(
+                           image_fn,error_string))
+                       
+                       raise
+                       
+                else:
+                    
+                    print('Warning: error copying/creating link for input file {}: {}'.format(
+                        image_fn,error_string))
+                    continue
+            
+            # ...except
+            
+        # ...for each image
+                
+    # ...if we need to create links/copies
 
+    
     ##%% Create the dataset file if necessary
     
     # This may have been passed in as a string, but at this point, we should have
@@ -330,14 +602,40 @@ def run_inference_with_yolo_val(options):
     category_ids = sorted(list(options.yolo_category_id_to_name.keys()))
     assert category_ids[0] == 0
     assert len(category_ids) == 1 + category_ids[-1]
-    
+        
     yolo_dataset_file = os.path.join(yolo_results_folder,'dataset.yaml')
+    yolo_image_list_file = os.path.join(yolo_results_folder,'images.txt')
+    
     
+    with open(yolo_image_list_file,'w') as f:
+        
+        if create_links:
+            image_files_to_write = link_full_paths
+        else:
+            image_files_to_write = image_files_absolute
+            
+        for fn_abs in image_files_to_write:
+            # At least in YOLOv5 val (need to verify for YOLOv8 val), filenames in this 
+            # text file are treated as relative to the text file itself if they start with
+            # "./", otherwise they're treated as absolute paths.  Since we don't want to put this
+            # text file in the image folder, we'll use absolute paths.
+            # fn_relative = os.path.relpath(fn_abs,options.input_folder)
+            # f.write(fn_relative + '\n')
+            f.write(fn_abs + '\n')
+    
+    if create_links:
+        inference_folder = symlink_folder_inner
+    else:
+        # This doesn't matter, but it has to be a valid path
+        inference_folder = options.yolo_results_folder
+        
     with open(yolo_dataset_file,'w') as f:
-        f.write('path: {}\n'.format(symlink_folder_inner))
+        
+        f.write('path: {}\n'.format(inference_folder))        
+        # These need to be valid paths, even if you're not using them, and "." is always safe
         f.write('train: .\n')
         f.write('val: .\n')
-        f.write('test: .\n')
+        f.write('test: {}\n'.format(yolo_image_list_file))
         f.write('\n')
         f.write('nc: {}\n'.format(len(options.yolo_category_id_to_name)))
         f.write('\n')
@@ -425,13 +723,15 @@ def run_inference_with_yolo_val(options):
                 print('Warning: error removing YOLO results folder {}'.format(yolo_results_folder))
                 pass
         
-        sys.exit()
+        # sys.exit()
+        return
     
     execution_result = process_utils.execute_and_print(cmd,encoding='utf-8',verbose=True)
     assert execution_result['status'] == 0, 'Error running {}'.format(options.model_type)
     yolo_console_output = execution_result['output']
       
     if options.save_yolo_debug_output:
+        
         with open(os.path.join(yolo_results_folder,'yolo_console_output.txt'),'w') as f:
             for s in yolo_console_output:
                 f.write(s + '\n')
@@ -485,7 +785,7 @@ def run_inference_with_yolo_val(options):
     # image_file = yolo_read_failures[0]
     for image_file in yolo_read_failures:
         image_id = os.path.splitext(os.path.basename(image_file))[0]
-        assert image_id in image_id_to_file
+        assert image_id in image_id_to_file, 'Unexpected image ID {}'.format(image_id)
         if image_id not in image_id_to_error:
             image_id_to_error[image_id] = 'YOLO read failure'
     
@@ -499,23 +799,27 @@ def run_inference_with_yolo_val(options):
     assert len(json_files) == 1    
     yolo_json_file = json_files[0]
 
+    # Map YOLO image IDs to paths
     image_id_to_relative_path = {}
     for image_id in image_id_to_file:
-        fn = image_id_to_file[image_id]
-        if os.path.isdir(options.input_folder):
-            assert options.input_folder in fn
+        fn = image_id_to_file[image_id].replace('\\','/')
+        assert path_is_abs(fn)
+        if options.input_folder is not None:
+            assert os.path.isdir(options.input_folder)
+            assert options.input_folder in fn, 'Internal error: base folder {} not in file {}'.format(
+                options.input_folder,fn)
             relative_path = os.path.relpath(fn,options.input_folder)
         else:
-            assert os.path.isfile(options.input_folder)
             # We'll use the absolute path as a relative path, and pass '/'
             # as the base path in this case.
             relative_path = fn
         image_id_to_relative_path[image_id] = relative_path
         
-    if os.path.isdir(options.input_folder):
+    # Are we working with a base folder?
+    if options.input_folder is not None:
+        assert os.path.isdir(options.input_folder)
         image_base = options.input_folder
     else:
-        assert os.path.isfile(options.input_folder)
         image_base = '/'
         
     yolo_output_to_md_output.yolo_json_output_to_md_output(
@@ -530,18 +834,10 @@ def run_inference_with_yolo_val(options):
 
     ##%% Clean up
     
-    if options.remove_symlink_folder:
-        shutil.rmtree(symlink_folder)
-    elif symlink_folder_is_temp_folder:
-        print('Warning: using temporary symlink folder {}, but not removing it'.format(
-            symlink_folder))
+    _clean_up_temporary_folders(options,
+                                symlink_folder,yolo_results_folder,
+                                symlink_folder_is_temp_folder,yolo_folder_is_temp_folder)
         
-    if options.remove_yolo_results_folder:
-        shutil.rmtree(yolo_results_folder)
-    elif yolo_folder_is_temp_folder:
-        print('Warning: using temporary YOLO results folder {}, but not removing it'.format(
-            yolo_results_folder))
-            
 # ...def run_inference_with_yolo_val()
 
 
@@ -560,11 +856,14 @@ def main():
         help='model file name')
     parser.add_argument(
         'input_folder',type=str,
-        help='folder on which to recursively run the model, or a .json list of filenames')
+        help='folder on which to recursively run the model')
     parser.add_argument(
         'output_file',type=str,
         help='.json file where output will be written')
     
+    parser.add_argument(
+        '--image_filename_list',type=str,default=None,
+        help='.json or .txt file containing a list of relative image filenames within [input_folder]')
     parser.add_argument(
         '--yolo_working_folder',type=str,default=None,
         help='folder in which to execute val.py (not necessary for YOLOv8 inference)')
@@ -584,24 +883,30 @@ def main():
         help='use half-precision-inference (1 or 0) (default is the underlying model\'s default, probably full for YOLOv8 and half for YOLOv5')
     parser.add_argument(
         '--device_string', default=options.device_string, type=str,
-        help='CUDA device specifier, typically "0" or "1" for CUDA devices, "mps" for M1/M2 devices, or "cpu" (default {})'.format(options.device_string))
+        help='CUDA device specifier, typically "0" or "1" for CUDA devices, "mps" for M1/M2 devices, or "cpu" (default {})'.format(
+            options.device_string))
     parser.add_argument(
         '--overwrite_handling', default=options.overwrite_handling, type=str,
         help='action to take if the output file exists (skip, error, overwrite) (default {})'.format(
             options.overwrite_handling))
     parser.add_argument(
         '--yolo_dataset_file', default=None, type=str,
-        help='YOLOv5 dataset.yml file from which we should load category information ' + \
+        help='YOLOv5 dataset.yaml file from which we should load category information ' + \
             '(otherwise defaults to MD categories)')
     parser.add_argument(
         '--model_type', default=options.model_type, type=str,
-        help='Model type ("yolov5" or "ultralytics" ("yolov8" behaves the same as "ultralytics")) (default {})'.format(options.model_type))
+        help='model type ("yolov5" or "ultralytics" ("yolov8" behaves the same as "ultralytics")) (default {})'.format(
+            options.model_type))
 
+    parser.add_argument('--unique_id_strategy', default=options.unique_id_strategy, type=str,
+        help='how should we ensure that unique filenames are passed to the YOLO val script, ' + \
+             'can be "verify", "auto", or "links", see options class docs for details (default {})'.format(
+                 options.unique_id_strategy))
     parser.add_argument(
-        '--symlink_folder', type=str,
+        '--symlink_folder', default=None, type=str,
         help='temporary folder for symlinks (defaults to a folder in the system temp dir)')
     parser.add_argument(
-        '--yolo_results_folder', type=str,
+        '--yolo_results_folder', default=None, type=str,
         help='temporary folder for YOLO intermediate output (defaults to a folder in the system temp dir)')
     parser.add_argument(
         '--no_use_symlinks', action='store_true',
@@ -615,6 +920,10 @@ def main():
     parser.add_argument(
         '--save_yolo_debug_output', action='store_true',
         help='write yolo console output to a text file in the results folder, along with additional debug files')
+    parser.add_argument(
+        '--checkpoint_frequency', default=options.checkpoint_frequency, type=int,
+        help='break the job into chunks with no more than this many images (default {})'.format(
+            options.checkpoint_frequency))    
     
     parser.add_argument(
         '--nonrecursive', action='store_true',
@@ -658,13 +967,20 @@ def main():
     
     if args.yolo_dataset_file is not None:
         options.yolo_category_id_to_name = args.yolo_dataset_file
+        del options.yolo_dataset_file
         
     options.recursive = (not options.nonrecursive)
     options.remove_symlink_folder = (not options.no_remove_symlink_folder)
     options.remove_yolo_results_folder = (not options.no_remove_yolo_results_folder)
     options.use_symlinks = (not options.no_use_symlinks)
     options.augment = (options.augment_enabled > 0)        
-            
+    
+    del options.nonrecursive
+    del options.no_remove_symlink_folder
+    del options.no_remove_yolo_results_folder
+    del options.no_use_symlinks
+    del options.augment_enabled
+        
     print(options.__dict__)
     
     run_inference_with_yolo_val(options)    
@@ -673,58 +989,90 @@ if __name__ == '__main__':
     main()
 
 
-#%% Scrap
+#%% Interactive driver
 
 if False:
 
-    #%% Test driver (folder)
+    #%% Run inference on a folder
+    
+    input_folder = r'g:\temp\tegu-val-mini'.replace('\\','/')
+    model_filename = r'g:\temp\usgs-tegus-yolov5x-231003-b8-img1280-e3002-best.pt'
+    output_folder = r'g:\temp\tegu-scratch'
+    yolo_working_folder = r'c:\git\yolov5-tegus'
+    dataset_file = r'g:\temp\dataset.yaml'
+    
+    # This only impacts the output file name, it's not passed to the inference functio
+    job_name = 'yolo-inference-test'
     
-    project_name = 'KRU-test-corrupted'
-    input_folder = os.path.expanduser(f'~/data/{project_name}')
-    output_folder = os.path.expanduser(f'~/tmp/{project_name}')
-    model_filename = os.path.expanduser('~/models/camera_traps/megadetector/md_v5.0.0/md_v5a.0.0.pt')
-    yolo_working_folder = os.path.expanduser('~/git/yolov5')
     model_name = os.path.splitext(os.path.basename(model_filename))[0]
     
     symlink_folder = os.path.join(output_folder,'symlinks')
     yolo_results_folder = os.path.join(output_folder,'yolo_results')
     
     output_file = os.path.join(output_folder,'{}_{}-md_format.json'.format(
-        project_name,model_name))
+        job_name,model_name))
     
     options = YoloInferenceOptions()
     
     options.yolo_working_folder = yolo_working_folder
-    
+    options.input_folder = input_folder
     options.output_file = output_file
     
+    pass_image_filename_list = False    
+    pass_relative_paths = True
+    
+    if pass_image_filename_list:
+        if pass_relative_paths:
+            options.image_filename_list =  [
+                r"val#american_cardinal#american_cardinal#CaCa#31W.01_C83#2017-2019#C90 and C83_31W.01#(05) 18AUG17 - 05SEP17 FTC AEG#MFDC1949_000065.JPG",
+                r"val#american_cardinal#american_cardinal#CaCa#31W.01_C83#2017-2019#C90 and C83_31W.01#(04) 27JUL17 - 18AUG17 FTC AEG#MFDC1902_000064.JPG"
+            ]   
+        else:
+            options.image_filename_list =  [
+                r"g:/temp/tegu-val-mini/val#american_cardinal#american_cardinal#CaCa#31W.01_C83#2017-2019#C90 and C83_31W.01#(05) 18AUG17 - 05SEP17 FTC AEG#MFDC1949_000065.JPG",
+                r"g:/temp/tegu-val-mini/val#american_cardinal#american_cardinal#CaCa#31W.01_C83#2017-2019#C90 and C83_31W.01#(04) 27JUL17 - 18AUG17 FTC AEG#MFDC1902_000064.JPG"
+            ]
+    else:
+        options.image_filename_list = None    
+    
+    options.yolo_category_id_to_name = dataset_file
     options.augment = False
     options.conf_thres = '0.001'
     options.batch_size = 1
     options.device_string = '0'
+    options.unique_id_strategy = 'auto'
+    options.overwrite_handling = 'overwrite'
 
     if options.augment:
         options.image_size = round(1280 * 1.3)
     else:
         options.image_size = 1280
     
-    options.input_folder = input_folder
     options.model_filename = model_filename
     
     options.yolo_results_folder = yolo_results_folder # os.path.join(output_folder + 'yolo_results')        
     options.symlink_folder = symlink_folder # os.path.join(output_folder,'symlinks')
     options.use_symlinks = False
     
-    options.remove_temporary_symlink_folder = False
-    options.remove_yolo_results_file = False
+    options.remove_symlink_folder = True
+    options.remove_yolo_results_folder = True
+    
+    options.checkpoint_frequency = 5
     
     cmd = f'python run_inference_with_yolov5_val.py {model_filename} {input_folder} ' + \
           f'{output_file} --yolo_working_folder {yolo_working_folder} ' + \
           f' --image_size {options.image_size} --conf_thres {options.conf_thres} ' + \
           f' --batch_size {options.batch_size} ' + \
           f' --symlink_folder {options.symlink_folder} --yolo_results_folder {options.yolo_results_folder} ' + \
-          ' --no_remove_symlink_folder --no_remove_yolo_results_folder'
+          f' --yolo_dataset_file {options.yolo_category_id_to_name} ' + \
+          f' --unique_id_strategy {options.unique_id_strategy} --overwrite_handling {options.overwrite_handling}'
       
+    if not options.remove_symlink_folder:
+        cmd += ' --no_remove_symlink_folder'
+    if not options.remove_yolo_results_folder: 
+        cmd += ' --no_remove_yolo_results_folder'
+    if options.checkpoint_frequency is not None:
+        cmd += f' --checkpoint_frequency {options.checkpoint_frequency}'
     if not options.use_symlinks:
         cmd += ' --no_use_symlinks'
     if not options.augment:
@@ -736,182 +1084,4 @@ if False:
         run_inference_with_yolo_val(options)
     else:
         import clipboard; clipboard.copy(cmd)
-    
-    
-    #%% Test driver (folder) (YOLOv8 model)
-    
-    project_name = 'yolov8-inference-test'
-    input_folder = os.path.expanduser('~/data/usgs-kissel-training-resized/val')
-    dataset_file = os.path.expanduser('~/data/usgs-kissel-training-yolo/dataset.yaml')
-    output_folder = os.path.expanduser(f'~/tmp/{project_name}')
-    model_filename = os.path.expanduser(
-        '~/models/usgs-tegus/usgs-tegus-yolov8x-2023.10.25-b-1-img640-e200-best.pt')
-    model_name = os.path.splitext(os.path.basename(model_filename))[0]
-    
-    assert os.path.isdir(input_folder)
-    assert os.path.isfile(dataset_file)
-    assert os.path.isfile(model_filename)    
-    
-    symlink_folder = os.path.join(output_folder,'symlinks')
-    yolo_results_folder = os.path.join(output_folder,'yolo_results')
-    
-    output_file = os.path.join(output_folder,'{}_{}-md_format.json'.format(
-        project_name,model_name))
-    
-    options = YoloInferenceOptions()
-    
-    options.model_type = 'yolov8'    
-    options.yolo_category_id_to_name = dataset_file    
-    options.yolo_working_folder = None    
-    options.output_file = output_file
-    
-    options.augment = False
-    options.conf_thres = '0.001'
-    options.batch_size = 1
-    options.device_string = '0'
 
-    if options.augment:
-        options.image_size = round(640 * 1.3)
-    else:
-        options.image_size = 640
-    
-    options.input_folder = input_folder
-    options.model_filename = model_filename
-    
-    options.yolo_results_folder = yolo_results_folder 
-    options.symlink_folder = symlink_folder
-    options.use_symlinks = False
-    
-    options.remove_temporary_symlink_folder = False
-    options.remove_yolo_results_file = False
-    
-    cmd = f'python run_inference_with_yolov5_val.py {model_filename} ' + \
-          f'{input_folder} {output_file}' + \
-          f' --image_size {options.image_size} --conf_thres {options.conf_thres} ' + \
-          f' --batch_size {options.batch_size} --symlink_folder {options.symlink_folder} ' + \
-          f'--yolo_results_folder {options.yolo_results_folder} --model_type {options.model_type}' + \
-          f' --yolo_dataset_file {options.yolo_category_id_to_name}' + \
-          ' --no_remove_symlink_folder --no_remove_yolo_results_folder'
-      
-    if not options.use_symlinks:
-        cmd += ' --no_use_symlinks'
-    if not options.augment:
-        cmd += ' --augment_enabled 0'
-        
-    print(cmd)
-    execute_in_python = False
-    if execute_in_python:
-        run_inference_with_yolo_val(options)
-    else:
-        import clipboard; clipboard.copy(cmd)
-    
-    
-    #%% Preview results
-    
-    postprocessing_output_folder = os.path.join(output_folder,'yolo-val-preview')
-    md_json_file = options.output_file
-    
-    from megadetector.postprocessing.postprocess_batch_results import \
-        PostProcessingOptions, process_batch_results
-    
-    with open(md_json_file,'r') as f:
-        d = json.load(f)
-    
-    base_task_name = os.path.basename(md_json_file)
-    
-    pp_options = PostProcessingOptions()
-    pp_options.image_base_dir = input_folder
-    pp_options.include_almost_detections = True
-    pp_options.num_images_to_sample = None
-    pp_options.confidence_threshold = 0.1
-    pp_options.almost_detection_confidence_threshold = pp_options.confidence_threshold - 0.025
-    pp_options.ground_truth_json_file = None
-    pp_options.separate_detections_by_category = True
-    # pp_options.sample_seed = 0
-    
-    pp_options.parallelize_rendering = True
-    pp_options.parallelize_rendering_n_cores = 16
-    pp_options.parallelize_rendering_with_threads = False
-    
-    output_base = os.path.join(postprocessing_output_folder,
-        base_task_name + '_{:.3f}'.format(pp_options.confidence_threshold))
-    
-    os.makedirs(output_base, exist_ok=True)
-    print('Processing to {}'.format(output_base))
-    
-    pp_options.md_results_file = md_json_file
-    pp_options.output_dir = output_base
-    ppresults = process_batch_results(pp_options)
-    html_output_file = ppresults.output_html_file
-    
-    path_utils.open_file(html_output_file)
-    
-    # ...for each prediction file
-    
-    
-    #%% Compare results
-    
-    import itertools
-    
-    from megadetector.postprocessing.compare_batch_results import \
-        BatchComparisonOptions,PairwiseBatchComparisonOptions,compare_batch_results
-    
-    options = BatchComparisonOptions()
-    
-    organization_name = ''
-    project_name = ''
-    
-    options.job_name = f'{organization_name}-comparison'
-    options.output_folder = os.path.join(output_folder,'model_comparison')
-    options.image_folder = input_folder
-    
-    options.pairwise_options = []
-    
-    filenames = [
-        f'/home/user/tmp/{project_name}/{project_name}_md_v5a.0.0-md_format.json',
-        f'/home/user/postprocessing/{organization_name}/{organization_name}-2023-04-06-v5a.0.0/combined_api_outputs/{organization_name}-2023-04-06-v5a.0.0_detections.json',
-        f'/home/user/postprocessing/{organization_name}/{organization_name}-2023-04-06-v5b.0.0/combined_api_outputs/{organization_name}-2023-04-06-v5b.0.0_detections.json'
-        ]
-    
-    descriptions = ['YOLO w/augment','MDv5a','MDv5b']
-    
-    if False:
-        results = []
-        
-        for fn in filenames:
-            with open(fn,'r') as f:
-                d = json.load(f)
-            results.append(d)
-        
-    detection_thresholds = [0.1,0.1,0.1]
-    
-    assert len(detection_thresholds) == len(filenames)
-    
-    rendering_thresholds = [(x*0.6666) for x in detection_thresholds]
-    
-    # Choose all pairwise combinations of the files in [filenames]
-    for i, j in itertools.combinations(list(range(0,len(filenames))),2):
-            
-        pairwise_options = PairwiseBatchComparisonOptions()
-        
-        pairwise_options.results_filename_a = filenames[i]
-        pairwise_options.results_filename_b = filenames[j]
-        
-        pairwise_options.results_description_a = descriptions[i]
-        pairwise_options.results_description_b = descriptions[j]
-        
-        pairwise_options.rendering_confidence_threshold_a = rendering_thresholds[i]
-        pairwise_options.rendering_confidence_threshold_b = rendering_thresholds[j]
-        
-        pairwise_options.detection_thresholds_a = {'animal':detection_thresholds[i],
-                                                   'person':detection_thresholds[i],
-                                                   'vehicle':detection_thresholds[i]}
-        pairwise_options.detection_thresholds_b = {'animal':detection_thresholds[j],
-                                                   'person':detection_thresholds[j],
-                                                   'vehicle':detection_thresholds[j]}
-        options.pairwise_options.append(pairwise_options)
-    
-    results = compare_batch_results(options)
-    
-    from megadetector.utils.path_utils import open_file
-    open_file(results.html_output_file)