megadetector 5.0.29__py3-none-any.whl → 10.0.0__py3-none-any.whl

megadetector/classification/efficientnet/model.py

@@ -93,7 +93,7 @@ class MBConvBlock(nn.Module):
 
         Args:
             inputs (tensor): Input tensor.
-            drop_connect_rate (bool): Drop connect rate (float, between 0 and 1).
+            drop_connect_rate (bool, optional): Drop connect rate (float, between 0 and 1).
 
         Returns:
             Output of this block after processing.
@@ -135,7 +135,7 @@ class MBConvBlock(nn.Module):
         """Sets swish function as memory efficient (for training) or standard (for export).
 
         Args:
-            memory_efficient (bool): Whether to use memory-efficient version of swish.
+            memory_efficient (bool, optional): Whether to use memory-efficient version of swish.
         """
         self._swish = MemoryEfficientSwish() if memory_efficient else Swish()
 
@@ -221,7 +221,7 @@ class EfficientNet(nn.Module):
         """Sets swish function as memory efficient (for training) or standard (for export).
 
         Args:
-            memory_efficient (bool): Whether to use memory-efficient version of swish.
+            memory_efficient (bool, optional): Whether to use memory-efficient version of swish.
 
         """
         self._swish = MemoryEfficientSwish() if memory_efficient else Swish()
@@ -323,7 +323,7 @@ class EfficientNet(nn.Module):
 
         Args:
             model_name (str): Name for efficientnet.
-            in_channels (int): Input data's channel number.
+            in_channels (int, optional): Input data's channel number.
             override_params (other key word params):
                 Params to override model's global_params.
                 Optional key:
@@ -349,14 +349,14 @@ class EfficientNet(nn.Module):
 
         Args:
             model_name (str): Name for efficientnet.
-            weights_path (None or str):
+            weights_path (None or str, optional):
                 str: path to pretrained weights file on the local disk.
                 None: use pretrained weights downloaded from the Internet.
-            advprop (bool):
+            advprop (bool, optional):
                 Whether to load pretrained weights
                 trained with advprop (valid when weights_path is None).
-            in_channels (int): Input data's channel number.
-            num_classes (int):
+            in_channels (int, optional): Input data's channel number.
+            num_classes (int, optional):
                 Number of categories for classification.
                 It controls the output size for final linear layer.
             override_params (other key word params):

megadetector/classification/efficientnet/utils.py

@@ -194,7 +194,7 @@ def get_same_padding_conv2d(image_size=None):
        Static padding is necessary for ONNX exporting of models.
 
     Args:
-        image_size (int or tuple): Size of the image.
+        image_size (int or tuple, optional): Size of the image.
 
     Returns:
         Conv2dDynamicSamePadding or Conv2dStaticSamePadding.
@@ -274,7 +274,7 @@ def get_same_padding_maxPool2d(image_size=None):
        Static padding is necessary for ONNX exporting of models.
 
     Args:
-        image_size (int or tuple): Size of the image.
+        image_size (int or tuple, optional): Size of the image.
 
     Returns:
         MaxPool2dDynamicSamePadding or MaxPool2dStaticSamePadding.
@@ -579,11 +579,12 @@ def load_pretrained_weights(model, model_name, weights_path=None, load_fc=True,
     Args:
         model (Module): The whole model of efficientnet.
         model_name (str): Model name of efficientnet.
-        weights_path (None or str):
+        weights_path (None or str, optional):
             str: path to pretrained weights file on the local disk.
             None: use pretrained weights downloaded from the Internet.
-        load_fc (bool): Whether to load pretrained weights for fc layer at the end of the model.
-        advprop (bool): Whether to load pretrained weights
+        load_fc (bool, optional): Whether to load pretrained weights for fc layer at the end 
+            of the model.
+        advprop (bool, optional): Whether to load pretrained weights
                         trained with advprop (valid when weights_path is None).
     """
     if isinstance(weights_path, str):

megadetector/classification/prepare_classification_script_mc.py

@@ -99,7 +99,7 @@ for fn in input_files:
     crop_cmd += crop_comment
 
     crop_cmd += "python crop_detections.py \\\n" + \
-    	 input_file_path + ' \\\n' + \
+         input_file_path + ' \\\n' + \
          crop_path + ' \\\n' + \
          '--images-dir "' + image_base + '"' + ' \\\n' + \
          '--threshold "' + threshold_str + '"' + ' \\\n' + \
@@ -127,7 +127,7 @@ for fn in input_files:
     classify_cmd += classify_comment
 
     classify_cmd += "python run_classifier.py \\\n" + \
-    	 checkpoint_path + ' \\\n' + \
+         checkpoint_path + ' \\\n' + \
          crop_path + ' \\\n' + \
          classifier_output_path + ' \\\n' + \
          '--detections-json "' + input_file_path + '"' + ' \\\n' + \
@@ -205,7 +205,7 @@ for fn in input_files:
     merge_cmd += merge_comment
 
     merge_cmd += "python merge_classification_detection_output.py \\\n" + \
-    	 classifier_output_path_remapped + ' \\\n' + \
+         classifier_output_path_remapped + ' \\\n' + \
          output_label_index + ' \\\n' + \
          '--output-json "' + final_output_path + '"' + ' \\\n' + \
          '--detection-json "' + input_file_path + '"' + ' \\\n' + \

megadetector/data_management/annotations/annotation_constants.py

@@ -31,4 +31,3 @@ detector_bbox_category_name_to_id = {}
 for cat in detector_bbox_categories:
     detector_bbox_category_id_to_name[cat['id']] = cat['name']
     detector_bbox_category_name_to_id[cat['name']] = cat['id']
-

megadetector/data_management/camtrap_dp_to_coco.py

@@ -19,6 +19,8 @@ Currently supports only sequence-level labeling.
 
 import os
 import json
+import argparse
+
 import pandas as pd
 
 from dateutil import parser as dateparser
@@ -36,6 +38,10 @@ def camtrap_dp_to_coco(camtrap_dp_folder,output_file=None):
     the resulting COCO file.
 
     Optionally writes the results to [output_file]
+
+    Args:
+        camtrap_dp_folder (str): input folder, containing a CamtrapDP package
+        output_file (str, optional): COCO-formatted output file
     """
 
     required_files = ('datapackage.json','deployments.csv','events.csv','media.csv','observations.csv')
@@ -234,4 +240,31 @@ if False:
 
 #%% Command-line driver
 
-# TODO
+def main():
+    """
+    Command-line interface to convert Camtrap DP to COCO.
+    """
+
+    parser = argparse.ArgumentParser(description='Convert Camtrap DP to COCO format')
+    parser.add_argument('camtrap_dp_folder', type=str,
+                        help='Input folder, containing a CamtrapDP package')
+    parser.add_argument('--output_file', type=str, default=None,
+                        help='COCO-formatted output file (defaults to [camtrap_dp_folder]_coco.json)')
+
+    args = parser.parse_args()
+
+    if args.output_file is None:
+        # Default output file name: [camtrap_dp_folder]_coco.json
+        #
+        # Remove trailing slash if present
+        folder_name = args.camtrap_dp_folder.rstrip(os.sep)
+        output_file = folder_name + '_coco.json'
+    else:
+        output_file = args.output_file
+
+    camtrap_dp_to_coco(camtrap_dp_folder=args.camtrap_dp_folder, output_file=output_file)
+    print(f"Successfully converted Camtrap DP package at '{args.camtrap_dp_folder}' to " + \
+          f"COCO format at '{output_file}'")
+
+if __name__ == '__main__':
+    main()

megadetector/data_management/cct_json_utils.py

@@ -418,8 +418,8 @@ def create_sequences(image_info,options=None):
         image_info (str, dict, or list): a dict in CCT format, a CCT .json file, or just the
             'images' component of a CCT dataset (a list of dicts with  fields 'file_name' (str),
             'datetime' (datetime), and 'location' (str)).
-        options (SequenceOptions): options parameterizing the assembly of images into sequences;
-            see the SequenceOptions class for details.
+        options (SequenceOptions, optional): options parameterizing the assembly of images into
+            sequences; see the SequenceOptions class for details.
 
     Returns:
         image_info: if [image_info] is passed as a list, returns the list, otherwise returns

megadetector/data_management/coco_to_yolo.py

@@ -49,9 +49,15 @@ def write_yolo_dataset_file(yolo_dataset_file,
         class_list (list or str): an ordered list of class names (the first item will be class 0,
             etc.), or the name of a text file containing an ordered list of class names (one per
             line, starting from class zero).
-        train_folder_relative (str, optional): train folder name, used only to populate dataset.yaml
-        val_folder_relative (str, optional): val folder name, used only to populate dataset.yaml
-        test_folder_relative (str, optional): test folder name, used only to populate dataset.yaml
+        train_folder_relative (str, optional): train folder name, used only to
+            populate dataset.yaml.  Can also be a filename (e.g. a .txt file with image
+            files).
+        val_folder_relative (str, optional): val folder name, used only to
+            populate dataset.yaml.  Can also be a filename (e.g. a .txt file with image
+            files).
+        test_folder_relative (str, optional): test folder name, used only to
+            populate dataset.yaml.  Can also be a filename (e.g. a .txt file with image
+            files).
     """
 
     # Read class names
@@ -126,7 +132,7 @@ def coco_to_yolo(input_image_folder,
             either treat images as empty or error, depending on the value of [allow_empty_annotations].
             [allow_empty_annotations] has no effect if source_format is 'coco_camera_traps'.
         overwrite_images (bool, optional): over-write images in the output folder if they exist
-        create_image_and_label_folder (bool, optional): whether to create separate folders called 'images' and
+        create_image_and_label_folders (bool, optional): whether to create separate folders called 'images' and
             'labels' in the YOLO output folder.  If create_image_and_label_folders is False,
             a/b/c/image001.jpg will become a#b#c#image001.jpg, and the corresponding text file will
             be a#b#c#image001.txt.  If create_image_and_label_folders is True, a/b/c/image001.jpg will become
@@ -540,7 +546,9 @@ def coco_to_yolo(input_image_folder,
 # ...def coco_to_yolo(...)
 
 
-def create_yolo_symlinks(source_folder,images_folder,labels_folder,
+def create_yolo_symlinks(source_folder,
+                         images_folder,
+                         labels_folder,
                          class_list_file=None,
                          class_list_output_name='object.data',
                          force_lowercase_image_extension=False):
@@ -550,6 +558,15 @@ def create_yolo_symlinks(source_folder,images_folder,labels_folder,
     Used to support preview/editing tools that assume images and labels are in separate
     folders.
 
+    Args:
+        source_folder (str): input folder
+        images_folder (str): output folder with links to images
+        labels_folder (str): output folder with links to labels
+        class_list_file (str, optional): list to classes.txt file
+        class_list_output_name (str, optional): output file to write with class information
+        force_lowercase_image_extension (bool, False): create symlinks with, e.g., .jpg, even
+            if the input image is, e.g., .JPG
+
     :meta private:
     """
 

megadetector/data_management/databases/add_width_and_height_to_db.py

@@ -4,31 +4,104 @@ add_width_and_height_to_db.py
 
 Grabs width and height from actual image files for a .json database that is missing w/h.
 
-TODO: this is a one-off script waiting to be cleaned up for more general use.
-
 """
 
 #%% Imports and constants
 
+import os
+import sys
 import json
+import argparse
+
+from tqdm import tqdm
 from PIL import Image
+
 from megadetector.utils import ct_utils
 
-datafile = '/datadrive/snapshotserengeti/databases/snapshotserengeti.json'
-image_base = '/datadrive/snapshotserengeti/images/'
 
-def main(): # noqa
+#%% Main resizing function
+
+def add_width_and_height_to_db(input_file,output_file,image_base_folder):
+    """
+    Add width and height to images in the COCO db [input_file]
+    that don't have non-None w/h values.  Does not verify correctness
+    for images that already have non-None w/h values.  Ignores files that
+    fail to open.
+
+    Args:
+        input_file (str): the COCO .json file to process
+        output_file (str): the COCO .json file to write
+        image_base_folder (str): image filenames in [input_file] should be relative
+            to this folder
+
+    Returns:
+        list: the list of image dicts that were modified
+    """
+
+    with open(input_file,'r') as f:
+        d = json.load(f)
+
+    to_return = []
+
+    for im in tqdm(d['images']):
+
+        if ('height' not in im) or ('width' not in im) or \
+           (im['height'] is None) or (im['width'] is None) or \
+           (im['height'] <= 0) or (im['width'] <= 0):
+
+            fn_relative = im['file_name']
+            fn_abs = os.path.join(image_base_folder,fn_relative)
+
+            if not os.path.isfile(fn_abs):
+                print('Could not find image file {}'.format(fn_abs))
+                continue
 
-    with open(datafile,'r') as f:
-        data = json.load(f)
+            try:
+                im_w, im_h = Image.open(fn_abs).size
+            except Exception as e:
+                print('Error opening file {}: {}'.format(fn_abs,str(e)))
+                continue
+
+            assert isinstance(im_w,int) and isinstance(im_h,int) and \
+                im_w > 0 and im_h > 0, \
+                'Illegal size retrieved for {}'.format(fn_abs)
 
-    for im in data['images']:
-        if 'height' not in im:
-            im_w, im_h = Image.open(image_base+im['file_name']).size
             im['height'] = im_h
             im['width'] = im_w
+            to_return.append(im)
+
+        # ...if we need to add width and/or height to this image
+
+    # ...for each image
+
+    ct_utils.write_json(output_file, d)
+
+    print('Added size information to {} of {} images'.format(
+        len(to_return), len(d['images'])))
+
+    return to_return
 
-    ct_utils.write_json(datafile, data, indent=None)
+# ...def add_width_and_height_to_db(...)
+
+
+#%% Command-line driver
 
 if __name__ == '__main__':
-    main()
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('input_file', type=str,
+                        help='Input COCO-formatted .json file')
+    parser.add_argument('output_file', type=str,
+                        help='Output COCO-formatted .json file')
+    parser.add_argument('image_base_folder', type=str,
+                        help='Base directory for images')
+
+    if len(sys.argv[1:]) == 0:
+        parser.print_help()
+        parser.exit()
+
+    args = parser.parse_args()
+
+    add_width_and_height_to_db(args.input_file,
+                               args.output_file,
+                               args.image_base_folder)

megadetector/data_management/databases/combine_coco_camera_traps_files.py

@@ -72,8 +72,8 @@ def combine_cct_dictionaries(input_dicts, require_uniqueness=True):
     comment for details on merge rules.
 
     Args:
-        input_dicts: list of CCT dicts
-        require_uniqueness: bool, whether to require that the images in
+        input_dicts (list of dict): list of CCT dicts
+        require_uniqueness (bool, optional): whether to require that the images in
             each input_dict be unique
 
     Returns:

megadetector/data_management/databases/integrity_check_json_db.py

@@ -22,7 +22,8 @@ import json
 import os
 import sys
 
-from multiprocessing.pool import ThreadPool
+from functools import partial
+from multiprocessing.pool import Pool, ThreadPool
 from operator import itemgetter
 from tqdm import tqdm
 
@@ -61,6 +62,9 @@ class IntegrityCheckOptions:
         #: Number of threads to use for parallelization, set to <= 1 to disable parallelization
         self.nThreads = 10
 
+        #: Whether to use threads (rather than processes for parallelization)
+        self.parallelizeWithThreads = True
+
         #: Enable additional debug output
         self.verbose = True
 
@@ -70,9 +74,6 @@ class IntegrityCheckOptions:
         #: If True, error if the 'info' field is not present
         self.requireInfo = False
 
-# This is used in a medium-hacky way to share modified options across threads
-default_options = IntegrityCheckOptions()
-
 
 #%% Functions
 
@@ -93,7 +94,7 @@ def _check_image_existence_and_size(image,options=None):
     """
 
     if options is None:
-        options = default_options
+        options = IntegrityCheckOptions()
 
     assert options.bCheckImageExistence
 
@@ -108,7 +109,12 @@ def _check_image_existence_and_size(image,options=None):
             return s
 
         # width, height = Image.open(file_path).size
-        pil_im = open_image(file_path)
+        try:
+            pil_im = open_image(file_path)
+        except Exception as e:
+            s = 'Error opening {}: {}'.format(file_path,str(e))
+            return s
+
         width,height = pil_im.size
         if (not (width == image['width'] and height == image['height'])):
             s = 'Size mismatch for image {}: {} (reported {},{}, actual {},{})'.format(
@@ -125,6 +131,7 @@ def integrity_check_json_db(json_file, options=None):
 
     Args:
         json_file (str): filename to validate, or an already-loaded dict
+        options (IntegrityCheckOptions, optional): see IntegrityCheckOptions
 
     Returns:
         tuple: tuple containing:
@@ -172,7 +179,7 @@ def integrity_check_json_db(json_file, options=None):
     images = data['images']
     annotations = data['annotations']
     categories = data['categories']
-    
+
     if options.requireInfo:
         assert 'info' in data, 'No info struct in database'
 
@@ -219,7 +226,7 @@ def integrity_check_json_db(json_file, options=None):
     # ...for each category
 
     if options.verbose:
-        print('\nChecking images...')
+        print('\nChecking image records...')
 
     if options.iMaxNumImages > 0 and len(images) > options.iMaxNumImages:
 
@@ -321,15 +328,22 @@ def integrity_check_json_db(json_file, options=None):
             print('Checking image existence and/or image sizes...')
 
         if options.nThreads is not None and options.nThreads > 1:
+
+            if options.parallelizeWithThreads:
+                worker_string = 'threads'
+            else:
+                worker_string = 'processes'
+
             if options.verbose:
-                print('Starting a pool of {} workers'.format(options.nThreads))
-            pool = ThreadPool(options.nThreads)
-            # results = pool.imap_unordered(lambda x: fetch_url(x,nImages), indexedUrlList)
-            default_options.baseDir = options.baseDir
-            default_options.bCheckImageSizes = options.bCheckImageSizes
-            default_options.bCheckImageExistence = options.bCheckImageExistence
+                print('Starting a pool of {} {}'.format(options.nThreads,worker_string))
+            if options.parallelizeWithThreads:
+                pool = ThreadPool(options.nThreads)
+            else:
+                pool = Pool(options.nThreads)
             try:
-                results = tqdm(pool.imap(_check_image_existence_and_size, images), total=len(images))
+                results = list(tqdm(pool.imap(
+                    partial(_check_image_existence_and_size,options=options), images),
+                    total=len(images)))
             finally:
                 pool.close()
                 pool.join()

megadetector/data_management/generate_crops_from_cct.py

@@ -10,6 +10,7 @@ each bounding box.
 #%% Imports and constants
 
 import os
+import argparse
 import json
 
 from tqdm import tqdm
@@ -146,4 +147,52 @@ if False:
 
 #%% Command-line driver
 
-# TODO
+def main():
+    """
+    Command-line interface to generate crops from a COCO Camera Traps .json file.
+    """
+
+    parser = argparse.ArgumentParser(
+        description='Generate cropped images from a COCO Camera Traps .json file'
+    )
+    parser.add_argument(
+        'cct_file',
+        type=str,
+        help='COCO .json file to load data from'
+    )
+    parser.add_argument(
+        'image_dir',
+        type=str,
+        help='Folder where images are located'
+    )
+    parser.add_argument(
+        'output_dir',
+        type=str,
+        help='Folder to which we should write cropped images'
+    )
+    parser.add_argument(
+        '--padding',
+        type=int,
+        default=0,
+        help='Pixels to expand each box before cropping'
+    )
+    parser.add_argument(
+        '--flat_output',
+        action='store_true',
+        help='Flatten folder structure in output (preserves folder structure by default)'
+    )
+
+    args = parser.parse_args()
+
+    generate_crops_from_cct(
+        cct_file=args.cct_file,
+        image_dir=args.image_dir,
+        output_dir=args.output_dir,
+        padding=args.padding,
+        flat_output=args.flat_output
+    )
+
+    print(f'Generated crops in {args.output_dir}')
+
+if __name__ == '__main__':
+    main()

megadetector/data_management/labelme_to_coco.py

@@ -8,10 +8,10 @@ Converts a folder of labelme-formatted .json files to COCO.
 
 #%% Constants and imports
 
-import json
 import os
-import uuid
 import sys
+import json
+import uuid
 import argparse
 
 from multiprocessing.pool import Pool, ThreadPool
@@ -232,6 +232,8 @@ def labelme_to_coco(input_folder,
         category_id_to_category_name (dict, optional): dict mapping category IDs to category names;
             really used to map Labelme category names to COCO category IDs.  IDs will be auto-generated
             if this is None.
+        empty_category_name (str, optional): if images are present without boxes, the category name
+            we should use for whole-image (and not-very-COCO-like) empty categories.
         empty_category_id (int, optional): category ID to use for the not-very-COCO-like "empty" category;
             also see the no_json_handling parameter.
         info_struct (dict, optional): dict to stash in the "info" field of the resulting COCO dict