megadetector 5.0.8__py3-none-any.whl → 5.0.10__py3-none-any.whl

data_management/yolo_output_to_md_output.py

@@ -1,43 +1,38 @@
-########
-#
-# yolo_output_to_md_output.py
-#
-# Converts the output of YOLOv5's detect.py or val.py to the MD API output format.
-#
-# Command-line driver not done yet, this has only been run interactively.
-#
-########
-
-### Converting .txt files ###
-
-#
-# detect.py writes a .txt file per image, in YOLO training format.  Converting from this
-# format does not currently support recursive results, since detect.py doesn't save filenames
-# in a way that allows easy inference of folder names.  Requires access to the input
-# images, because the YOLO format uses the *absence* of a results file to indicate that
-# no detections are present.
-#
-# YOLOv5 output has one text file per image, like so:
-#
-# 0 0.0141693 0.469758 0.0283385 0.131552 0.761428 
-#
-# That's [class, x_center, y_center, width_of_box, height_of_box, confidence]
-#
-# val.py can write in this format as well, using the --save-txt argument.
-#
-# In both cases, a confidence value is only written to each line if you include the --save-conf
-# argument.  Confidence values are required by this conversion script.
-#
-
-### Converting .json files ###
-
-#
-# val.py can also write a .json file in COCO-ish format.  It's "COCO-ish" because it's
-# just the "images" portion of a COCO .json file.
-#
-# Converting from this format also requires access to the original images, since the format
-# written by YOLOv5 uses absolute coordinates, but MD results are in relative coordinates.
-#
+"""
+
+yolo_output_to_md_output.py
+
+Converts the output of YOLOv5's detect.py or val.py to the MD API output format.
+
+**Converting .txt files**
+
+detect.py writes a .txt file per image, in YOLO training format.  Converting from this
+format does not currently support recursive results, since detect.py doesn't save filenames
+in a way that allows easy inference of folder names.  Requires access to the input
+images, because the YOLO format uses the *absence* of a results file to indicate that
+no detections are present.
+
+YOLOv5 output has one text file per image, like so:
+
+0 0.0141693 0.469758 0.0283385 0.131552 0.761428 
+
+That's [class, x_center, y_center, width_of_box, height_of_box, confidence]
+
+val.py can write in this format as well, using the --save-txt argument.
+
+In both cases, a confidence value is only written to each line if you include the --save-conf
+argument.  Confidence values are required by this conversion script.
+
+
+**Converting .json files**
+
+val.py can also write a .json file in COCO-ish format.  It's "COCO-ish" because it's
+just the "images" portion of a COCO .json file.
+
+Converting from this format also requires access to the original images, since the format
+written by YOLOv5 uses absolute coordinates, but MD results are in relative coordinates.
+
+"""
 
 #%% Imports and constants
 
@@ -51,9 +46,7 @@ from tqdm import tqdm
 
 from md_utils import path_utils
 from md_utils import ct_utils
-
 from md_visualization import visualization_utils as vis_utils
-
 from detection.run_detector import CONF_DIGITS, COORD_DIGITS
 
 
@@ -61,9 +54,16 @@ from detection.run_detector import CONF_DIGITS, COORD_DIGITS
 
 def read_classes_from_yolo_dataset_file(fn):
     """
-    Read a dictionary mapping integer class IDs to class names from a YOLOv5/YOLOv8
+    Reads a dictionary mapping integer class IDs to class names from a YOLOv5/YOLOv8
     dataset.yaml file or a .json file.  A .json file should contain a dictionary mapping
     integer category IDs to string category names.
+    
+    Args:
+        fn (str): YOLOv5/YOLOv8 dataset file with a .yml or .yaml extension, or a .json file 
+            mapping integer category IDs to category names.
+            
+    Returns:
+        dict: a mapping from integer category IDs to category names
     """
         
     if fn.endswith('.yml') or fn.endswith('.yaml'):
@@ -92,45 +92,42 @@ def read_classes_from_yolo_dataset_file(fn):
         raise ValueError('Unrecognized category file type: {}'.format(fn))
         
     assert len(category_id_to_name) > 0, 'Failed to read class mappings from {}'.format(fn)
+    
     return category_id_to_name
     
 
-def yolo_json_output_to_md_output(yolo_json_file, image_folder,
-                                  output_file, yolo_category_id_to_name,                              
+def yolo_json_output_to_md_output(yolo_json_file, 
+                                  image_folder,
+                                  output_file, 
+                                  yolo_category_id_to_name,                              
                                   detector_name='unknown',
                                   image_id_to_relative_path=None,
                                   offset_yolo_class_ids=True,
                                   truncate_to_standard_md_precision=True,
                                   image_id_to_error=None):
     """
-    Convert a YOLOv5 .json file to MD .json format.
+    Converts a YOLOv5/YOLOv8 .json file to MD .json format.
     
     Args:
         
-    - yolo_json_file: the .json file to convert from YOLOv5 format to MD output format.
-    
-    - image_folder: the .json file contains relative path names, this is the path base.
-    
-    - yolo_category_id_to_name: the .json file contains only numeric identifiers for
-      categories, but we want names and numbers for the output format; this is a 
-      dict mapping numbers to names.  Can also be a YOLOv5 dataset.yaml file.
-    
-    - detector_name: a string that gets put in the output file, not otherwise used within
-      this function.
-    
-    - image_id_to_relative_path: YOLOv5 .json uses only basenames (e.g. abc1234.JPG);
-      by default these will be appended to the input path to create pathnames, so if you
-      have a flat folder, this is fine.  If you want to map base names to relative paths, use
-      this dict.
-    
-    - offset_yolo_class_ids: YOLOv5 class IDs always start at zero; if you want to make the 
-      output classes start at 1, set offset_yolo_class_ids to True.
-    
-    - truncate_to_standard_md_precision: YOLOv5 .json includes lots of (not-super-meaningful)
-      precision, set this to truncate to COORD_DIGITS and CONF_DIGITS.
-    
-    - image_id_to_error: if you want to include image IDs in the output file for which you couldn't
-      prepare the input file in the first place due to errors, include them here.
+        yolo_json_file (str): the .json file to convert from YOLOv5 format to MD output format
+        image_folder (str): the .json file contains relative path names, this is the path base
+        yolo_category_id_to_name (str or dict): the .json results file contains only numeric 
+            identifiers for categories, but we want names and numbers for the output format; 
+            yolo_category_id_to_name provides that mapping either as a dict or as a YOLOv5 
+            dataset.yaml file.
+        detector_name (str, optional): a string that gets put in the output file, not otherwise 
+            used within this function
+        image_id_to_relative_path (dict, optional): YOLOv5 .json uses only basenames (e.g. 
+            abc1234.JPG); by default these will be appended to the input path to create pathnames.
+            If you have a flat folder, this is fine.  If you want to map base names to relative paths in
+            a more complicated way, use this parameter.   
+        offset_yolo_class_ids (bool, optional): YOLOv5 class IDs always start at zero; if you want to 
+            make the output classes start at 1, set offset_yolo_class_ids to True.    
+        truncate_to_standard_md_precision (bool, optional): YOLOv5 .json includes lots of 
+            (not-super-meaningful) precision, set this to truncate to COORD_DIGITS and CONF_DIGITS.
+        image_id_to_error (dict, optional): if you want to include image IDs in the output file for which 
+            you couldn't prepare the input file in the first place due to errors, include them here.
     """    
         
     assert os.path.isfile(yolo_json_file), \
@@ -314,14 +311,25 @@ def yolo_json_output_to_md_output(yolo_json_file, image_folder,
 # ...def yolo_json_output_to_md_output(...)
     
 
-def yolo_txt_output_to_md_output(input_results_folder, image_folder,
-                                 output_file, detector_tag=None):
+def yolo_txt_output_to_md_output(input_results_folder, 
+                                 image_folder,
+                                 output_file, 
+                                 detector_tag=None):
     """
-    Converts a folder of YOLO-outptu .txt files to MD .json format.
+    Converts a folder of YOLO-output .txt files to MD .json format.
     
     Less finished than the .json conversion function; this .txt conversion assumes 
     a hard-coded mapping representing the standard MD categories (in MD indexing, 
     1/2/3=animal/person/vehicle; in YOLO indexing, 0/1/2=animal/person/vehicle).
+    
+    Args:
+        input_results_folder (str): the folder containing YOLO-output .txt files
+        image_folder (str): the folder where images live, may be the same as
+            [input_results_folder]
+        output_file (str): the MD-formatted .json file to which we should write
+            results
+        detector_tag (str, optional): string to put in the 'detector' field in the
+            output file            
     """
     
     assert os.path.isdir(input_results_folder)
@@ -426,3 +434,8 @@ if False:
     image_folder = os.path.expanduser('~/data/KRU-test')
     output_file = os.path.expanduser('~/data/mdv5a-yolo-pt-kru.json')    
     yolo_txt_output_to_md_output(input_results_folder,image_folder,output_file)
+
+
+#%% Command-line driver
+
+# TODO

data_management/yolo_to_coco.py

@@ -1,10 +1,10 @@
-########
-#
-# yolo_to_coco.py
-#
-# Converts a folder of YOLO-formatted annotation files to a COCO-formatted dataset. 
-#
-########
+"""
+
+yolo_to_coco.py
+
+Converts a folder of YOLO-formatted annotation files to a COCO-formatted dataset. 
+
+"""
 
 #%% Imports and constants
 
@@ -18,6 +18,8 @@ from functools import partial
 from tqdm import tqdm
 
 from md_utils.path_utils import find_images
+from md_utils.path_utils import recursive_file_list
+from md_utils.path_utils import find_image_strings
 from md_utils.ct_utils import invert_dictionary
 from md_visualization.visualization_utils import open_image
 from data_management.yolo_output_to_md_output import read_classes_from_yolo_dataset_file
@@ -25,9 +27,13 @@ from data_management.yolo_output_to_md_output import read_classes_from_yolo_data
 
 #%% Support functions
 
-def filename_to_image_id(fn):
+def _filename_to_image_id(fn):
+    """
+    Image IDs can't have spaces in them, replae spaces with underscores
+    """
     return fn.replace(' ','_')
 
+
 def _process_image(fn_abs,input_folder,category_id_to_name):
     """
     Internal support function for processing one image's labels.
@@ -35,7 +41,7 @@ def _process_image(fn_abs,input_folder,category_id_to_name):
     
     # Create the image object for this image
     fn_relative = os.path.relpath(fn_abs,input_folder)
-    image_id = filename_to_image_id(fn_relative)
+    image_id = _filename_to_image_id(fn_relative)
     
     # This is done in a separate loop now
     #
@@ -127,60 +133,28 @@ def _process_image(fn_abs,input_folder,category_id_to_name):
 # ...def _process_image(...)
 
 
-
-#%% Main conversion function
-
-def yolo_to_coco(input_folder,
-                 class_name_file,
-                 output_file=None,
-                 empty_image_handling='no_annotations',
-                 empty_image_category_name='empty',
-                 error_image_handling='no_annotations',
-                 allow_images_without_label_files=True,
-                 n_workers=1,
-                 pool_type='thread',
-                 recursive=True,
-                 exclude_string=None,
-                 include_string=None):
+def load_yolo_class_list(class_name_file):
     """
-    Convert the YOLO-formatted data in [input_folder] to a COCO-formatted dictionary,
-    reading class names from [class_name_file], which can be a flat list with a .txt
-    extension or a YOLO dataset.yml file.  Optionally writes the output dataset to [output_file].
-    
-    empty_image_handling can be:
-        
-    * 'no_annotations': include the image in the image list, with no annotations
-    
-    * 'empty_annotations': include the image in the image list, and add an annotation without
-      any bounding boxes, using a category called [empty_image_category_name].
-      
-    * 'skip': don't include the image in the image list
-    
-    * 'error': there shouldn't be any empty images
-    
-    error_image_handling can be:
-        
-    * 'skip': don't include the image at all
-    
-    * 'no_annotations': include with no annotations
-    
-    All images will be assigned an "error" value, usually None.
-        
-    Returns a COCO-formatted dictionary.
+    Loads a dictionary mapping zero-indexed IDs to class names from the text/yaml file
+    [class_name_file].    
+    
+    Args:
+        class_name_file (str or list): this can be:
+            - a .yaml or .yaml file in YOLO's dataset.yaml format
+            - a .txt or .data file containing a flat list of class names
+            - a list of class names
+            
+    Returns:
+        dict: A dict mapping zero-indexed integer IDs to class names
     """
     
-    ## Validate input
-    
-    assert os.path.isdir(input_folder)
-    assert os.path.isfile(class_name_file)
-    
-    assert empty_image_handling in \
-        ('no_annotations','empty_annotations','skip','error'), \
-            'Unrecognized empty image handling spec: {}'.format(empty_image_handling)
-     
+    # class_name_file can also be a list of class names
+    if isinstance(class_name_file,list):
+        category_id_to_name = {}
+        for i_name,name in enumerate(class_name_file):
+            category_id_to_name[i_name] = name
+        return category_id_to_name
             
-    ## Read class names
-    
     ext = os.path.splitext(class_name_file)[1][1:]
     assert ext in ('yml','txt','yaml','data'), 'Unrecognized class name file type {}'.format(
         class_name_file)
@@ -211,6 +185,267 @@ def yolo_to_coco(input_folder,
         
         assert ext in ('yml','yaml')
         category_id_to_name = read_classes_from_yolo_dataset_file(class_name_file)
+        
+    return category_id_to_name
+
+# ...load_yolo_class_list(...)
+
+
+def validate_label_file(label_file,category_id_to_name=None,verbose=False):
+    """"
+    Verifies that [label_file] is a valid YOLO label file.  Does not check the extension.
+    
+    Args:
+        label_file (str): the .txt file to validate
+        category_id_to_name (dict, optional): a dict mapping integer category IDs to names;
+            if this is not None, this function errors if the file uses a category that's not
+            in this dict
+        verbose (bool, optional): enable additional debug console output
+    
+    Returns:
+        dict: a dict with keys 'file' (the same as [label_file]) and 'errors' (a list of 
+        errors (if any) that we found in this file)
+    """
+    
+    label_result = {}
+    label_result['file'] = label_file
+    label_result['errors'] = []
+    
+    try:
+        with open(label_file,'r') as f:
+            lines = f.readlines()
+    except Exception as e:
+        label_result['errors'].append('Read error: {}'.format(str(e)))
+        return label_result
+    
+    # i_line 0; line = lines[i_line]
+    for i_line,line in enumerate(lines):
+        s = line.strip()
+        if len(s) == 0 or s[0] == '#':
+            continue
+        
+        try:
+        
+            tokens = s.split()
+            assert len(tokens) == 5, '{} tokens'.format(len(tokens))                
+            
+            if category_id_to_name is not None:
+                category_id = int(tokens[0])
+                assert category_id in category_id_to_name, \
+                    'Unrecognized category ID {}'.format(category_id)
+        
+            yolo_bbox = [float(x) for x in tokens[1:]]
+            
+        except Exception as e:
+            label_result['errors'].append('Token error at line {}: {}'.format(i_line,str(e)))
+            continue
+                            
+        normalized_x_center = yolo_bbox[0]
+        normalized_y_center = yolo_bbox[1]
+        normalized_width = yolo_bbox[2]
+        normalized_height = yolo_bbox[3]
+        
+        normalized_x_min = normalized_x_center - normalized_width / 2.0
+        normalized_x_max = normalized_x_center + normalized_width / 2.0
+        normalized_y_min = normalized_y_center - normalized_height / 2.0
+        normalized_y_max = normalized_y_center + normalized_height / 2.0
+        
+        if normalized_x_min < 0 or normalized_y_min < 0 or \
+            normalized_x_max > 1 or normalized_y_max > 1:
+            label_result['errors'].append('Invalid bounding box: {} {} {} {}'.format(
+                normalized_x_min,normalized_y_min,normalized_x_max,normalized_y_max))
+        
+    # ...for each line
+    
+    if verbose:
+        if len(label_result['errors']) > 0:
+            print('Errors for {}:'.format(label_file))
+            for error in label_result['errors']:
+                print(error)
+                
+    return label_result
+    
+# ...def validate_label_file(...)
+
+    
+def validate_yolo_dataset(input_folder, class_name_file, n_workers=1, pool_type='thread', verbose=False):
+    """
+    Verifies all the labels in a YOLO dataset folder.  
+    
+    Looks for:
+        
+    * Image files without label files
+    * Text files without image files
+    * Illegal classes in label files
+    * Invalid boxes in label files
+
+    Args:
+        input_folder (str): the YOLO dataset folder to validate
+        class_name_file (str or list): a list of classes, a flat text file, or a yolo 
+            dataset.yml/.yaml file.  If it's a dataset.yml file, that file should point to 
+            input_folder as the base folder, though this is not explicitly checked.
+        n_workers (int, optional): number of concurrent workers, set to <= 1 to disable
+            parallelization
+        pool_type (str, optional): 'thread' or 'process', worker type to use for parallelization;
+            not used if [n_workers] <= 1
+        verbose (bool, optional): enable additional debug console output
+    
+    Returns:
+        dict: validation results, as a dict with fields:        
+        
+        - image_files_without_label_files (list)
+        - label_files_without_image_files (list)
+        - label_results (list of dicts with field 'filename', 'errors') (list)
+    """
+    
+    # Validate arguments
+    assert os.path.isdir(input_folder), 'Could not find input folder {}'.format(input_folder)
+    if n_workers > 1:
+        assert pool_type in ('thread','process'), 'Illegal pool type {}'.format(pool_type)
+        
+    category_id_to_name = load_yolo_class_list(class_name_file)
+    
+    print('Enumerating files in {}'.format(input_folder))
+    
+    all_files = recursive_file_list(input_folder,recursive=True,return_relative_paths=False,
+                                    convert_slashes=True)
+    label_files = [fn for fn in all_files if fn.endswith('.txt')]
+    image_files = find_image_strings(all_files)
+    print('Found {} images files and {} label files in {}'.format(
+        len(image_files),len(label_files),input_folder))
+    
+    label_files_set = set(label_files)
+    
+    image_files_without_extension = set()
+    for fn in image_files:
+        image_file_without_extension = os.path.splitext(fn)[0]
+        assert image_file_without_extension not in image_files_without_extension, \
+            'Duplicate image file, likely with different extensions: {}'.format(fn)
+        image_files_without_extension.add(image_file_without_extension)
+        
+    print('Looking for missing image/label files')
+    
+    image_files_without_label_files = []
+    label_files_without_images = []
+    
+    for image_file in tqdm(image_files):
+        expected_label_file = os.path.splitext(image_file)[0] + '.txt'
+        if expected_label_file not in label_files_set:
+            image_files_without_label_files.append(image_file)
+            
+    for label_file in tqdm(label_files):
+        expected_image_file_without_extension = os.path.splitext(label_file)[0]
+        if expected_image_file_without_extension not in image_files_without_extension:
+            label_files_without_images.append(label_file)
+            
+    print('Found {} image files without labels, {} labels without images'.format(
+        len(image_files_without_label_files),len(label_files_without_images)))
+
+    print('Validating label files')
+    
+    if n_workers <= 1:
+        
+        label_results = []        
+        for fn_abs in tqdm(label_files):                
+            label_results.append(validate_label_file(fn_abs,
+                                                      category_id_to_name=category_id_to_name,
+                                                      verbose=verbose))
+            
+    else:
+        
+        assert pool_type in ('process','thread'), 'Illegal pool type {}'.format(pool_type)
+        
+        if pool_type == 'thread':
+            pool = ThreadPool(n_workers)
+        else:
+            pool = Pool(n_workers)
+        
+        print('Starting a {} pool of {} workers'.format(pool_type,n_workers))
+        
+        p = partial(validate_label_file,
+                    category_id_to_name=category_id_to_name,
+                    verbose=verbose)
+        label_results = list(tqdm(pool.imap(p, label_files),
+                                  total=len(label_files)))        
+    
+    assert len(label_results) == len(label_files)
+    
+    validation_results = {}
+    validation_results['image_files_without_label_files'] = image_files_without_label_files
+    validation_results['label_files_without_images'] = label_files_without_images
+    validation_results['label_results'] = label_results
+    
+    return validation_results
+    
+# ...validate_yolo_dataset(...)    
+
+
+#%% Main conversion function
+
+def yolo_to_coco(input_folder,
+                 class_name_file,
+                 output_file=None,
+                 empty_image_handling='no_annotations',
+                 empty_image_category_name='empty',
+                 error_image_handling='no_annotations',
+                 allow_images_without_label_files=True,
+                 n_workers=1,
+                 pool_type='thread',
+                 recursive=True,
+                 exclude_string=None,
+                 include_string=None):
+    """
+    Converts a YOLO-formatted dataset to a COCO-formatted dataset.
+    
+    All images will be assigned an "error" value, usually None.    
+    
+    Args:
+        input_folder (str): the YOLO dataset folder to validate
+        class_name_file (str or list): a list of classes, a flat text file, or a yolo 
+            dataset.yml/.yaml file.  If it's a dataset.yml file, that file should point to 
+            input_folder as the base folder, though this is not explicitly checked.
+        output_file (str, optional): .json file to which we should write COCO .json data
+        empty_image_handling (str, optional): how to handle images with no boxes; whether
+            this includes images with no .txt files depending on the value of 
+            [allow_images_without_label_files].  Can be:
+                
+            - 'no_annotations': include the image in the image list, with no annotations
+            - 'empty_annotations': include the image in the image list, and add an annotation without
+              any bounding boxes, using a category called [empty_image_category_name].
+            - 'skip': don't include the image in the image list
+            - 'error': there shouldn't be any empty images            
+        error_image_handling (str, optional): how to handle images that don't load properly; can
+            be:
+            
+            - 'skip': don't include the image at all
+            - 'no_annotations': include with no annotations
+            
+        n_workers (int, optional): number of concurrent workers, set to <= 1 to disable
+            parallelization
+        pool_type (str, optional): 'thread' or 'process', worker type to use for parallelization;
+            not used if [n_workers] <= 1
+        recursive (bool, optional): whether to recurse into [input_folder]
+        exclude_string (str, optional): exclude any images whose filename contains a string
+        include_string (str, optional): include only images whose filename contains a string
+    
+    Returns:
+        dict: COCO-formatted data, the same as what's written to [output_file]
+    """
+    
+    ## Validate input
+    
+    assert os.path.isdir(input_folder)
+    assert os.path.isfile(class_name_file)
+    
+    assert empty_image_handling in \
+        ('no_annotations','empty_annotations','skip','error'), \
+            'Unrecognized empty image handling spec: {}'.format(empty_image_handling)
+     
+            
+    ## Read class names
+    
+    category_id_to_name = load_yolo_class_list(class_name_file)
+    
     
     # Find or create the empty image category, if necessary
     empty_category_id = None
@@ -275,7 +510,7 @@ def yolo_to_coco(input_folder,
     for fn_abs in tqdm(image_files_abs):
         
         fn_relative = os.path.relpath(fn_abs,input_folder)
-        image_id = filename_to_image_id(fn_relative)
+        image_id = _filename_to_image_id(fn_relative)
         assert image_id not in image_ids, \
             'Oops, you have hit a very esoteric case where you have the same filename ' + \
             'with both spaces and underscores, this is not currently handled.'
@@ -434,3 +669,8 @@ if False:
     
     from md_utils.path_utils import open_file
     open_file(html_output_file)
+
+
+#%% Command-line driver
+
+# TODO