megadetector 5.0.21__tar.gz → 5.0.23__tar.gz

{megadetector-5.0.21/megadetector.egg-info → megadetector-5.0.23}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: megadetector
-Version: 5.0.21
+Version: 5.0.23
 Summary: MegaDetector is an AI model that helps conservation folks spend less time doing boring things with camera trap images.
 Author-email: Your friendly neighborhood MegaDetector team <cameratraps@lila.science>
 Maintainer-email: Your friendly neighborhood MegaDetector team <cameratraps@lila.science>
@@ -32,14 +32,14 @@ Keywords: camera traps,conservation,wildlife,ai,megadetector
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Requires-Python: <3.12,>=3.9
+Requires-Python: <=3.13,>=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: Pillow>=9.5
 Requires-Dist: tqdm>=4.64.0
 Requires-Dist: jsonpickle>=3.0.2
 Requires-Dist: humanfriendly>=10.0
-Requires-Dist: numpy<1.24,>=1.22
+Requires-Dist: numpy<2.0,>=1.26.4
 Requires-Dist: matplotlib>=3.8.0
 Requires-Dist: opencv-python>=4.8.0
 Requires-Dist: requests>=2.31.0
@@ -49,6 +49,7 @@ Requires-Dist: scikit-learn>=1.3.1
 Requires-Dist: pandas>=2.1.1
 Requires-Dist: PyYAML>=6.0.1
 Requires-Dist: ultralytics-yolov5==0.1.1
+Requires-Dist: python-dateutil
 
 # MegaDetector
 

{megadetector-5.0.21 → megadetector-5.0.23}/README.md

@@ -107,7 +107,7 @@ Here are a few of the organizations that have used MegaDetector... we're only li
 * [Wildlife Coexistence Lab](https://wildlife.forestry.ubc.ca/), University of British Columbia
 * [Wildlife Research](https://www.dfw.state.or.us/wildlife/research/index.asp), Oregon Department of Fish and Wildlife
 * [Wildlife Division](https://www.michigan.gov/dnr/about/contact/wildlife), Michigan Department of Natural Resources
-
+* [Kohl Wildlife Lab](https://kohlwildlifelab.com/), University of Georgia
 * Department of Ecology, TU Berlin
 * Ghost Cat Analytics
 * Protected Areas Unit, Canadian Wildlife Service

{megadetector-5.0.21 → megadetector-5.0.23}/megadetector/data_management/cct_json_utils.py

@@ -12,6 +12,8 @@ https://github.com/agentmorris/MegaDetector/blob/main/megadetector/data_manageme
 
 import json
 import os
+import datetime
+import dateutil
 
 from tqdm import tqdm
 from collections import defaultdict, OrderedDict
@@ -302,6 +304,106 @@ class SequenceOptions:
     
 #%% Functions
 
+def write_object_with_serialized_datetimes(d,json_fn):
+    """
+    Writes the object [d] to the .json file [json_fn] with a standard approach
+    to serializing Python datetime objects.
+    
+    Args:
+        d (obj): the object to write, typically a dict
+        json_fn (str): the output filename        
+    """
+    
+    # This writes datetimes as:
+    #
+    # 2022-12-31T09:52:50
+    def json_serialize_datetime(obj):
+        if isinstance(obj, (datetime.datetime, datetime.date)):
+            return obj.isoformat()
+        raise TypeError('Object {} (type {}) not serializable'.format(
+            str(obj),type(obj)))
+        
+    with open(json_fn,'w') as f:
+        json.dump(d,f,indent=1,default=json_serialize_datetime)
+
+
+def parse_datetimes_from_cct_image_list(images,conversion_failure_behavior='error'):
+    """
+    Given the "images" field from a COCO camera traps dictionary, converts all
+    string-formatted datetime fields to Python datetimes, making reasonable assumptions 
+    about datetime representations.  Modifies [images] in place.
+    
+    Args:
+        images (list): a list of dicts in CCT images format
+        conversion_failure_behavior (str, optional): determines what happens on a failed
+            conversion; can be "error" (raise an error), "str" (leave as a string), or
+            "none" (convert to None)
+    
+    Returns:
+        images: the input list, with datetimes converted (after modifying in place)
+    """
+    
+    assert isinstance(images,list)
+    
+    for im in images:
+        
+        if 'datetime' not in im:
+            continue
+        if isinstance(im['datetime'],datetime.datetime):
+            continue
+        try:
+            dt = dateutil.parser.parse(im['datetime'])
+            im['datetime'] = dt
+        except Exception as e:
+            s = 'could not parse datetime {}: {}'.format(str(im['datetime']),str(e))
+            if conversion_failure_behavior == 'error':
+                raise ValueError(s)
+            elif conversion_failure_behavior == 'str':
+                print('Warning: {}'.format(s))
+                pass
+            elif conversion_failure_behavior == 'none':
+                print('Warning: {}'.format(s))
+                im['datetime'] = None
+                
+    # ...for each image        
+    
+    return images
+    
+# ...def parse_datetimes_from_cct_image_list(...)
+
+
+def parse_datetimes_from_cct_dict(d,conversion_failure_behavior='error'):
+    """
+    Given a COCO camera traps dictionary that may just have been loaded from file,
+    converts all string-formatted datetime fields to Python datetimes, making
+    reasonable assumptions about datetime representations.  Modifies [d] in place
+    if [d] is supplied as a dict
+    
+    Args:
+        d (dict or str): a dict in CCT format or a filename pointing to a CCT .json file
+        conversion_failure_behavior (str, optional): determines what happens on a failed
+            conversion; can be "error" (raise an error), "str" (leave as a string), or
+            "none" (convert to None)
+    
+    Returns:
+        dict: the CCT dict with converted datetimes.
+    """
+    
+    if isinstance(d,str):        
+        assert os.path.isfile(d), 'Could not find .json file {}'.format(d)
+        with open(d,'r') as f:
+            d = json.load(f)
+    
+    images = d['images']
+    
+    # Modifies in place
+    _ = parse_datetimes_from_cct_image_list(images)
+    
+    return d
+            
+# ...def parse_datetimes_from_cct_dict(...)
+
+
 def create_sequences(image_info,options=None):
     """
     Synthesizes episodes/sequences/bursts for the images in [image_info].
@@ -310,21 +412,41 @@ def create_sequences(image_info,options=None):
     fields for each image.
     
     Args:
-        image_info (str, dict, or list): a dict in CCT format, a CCT .json file, or just the 'images' component
-            of a CCT dataset (a list of dicts with  fields 'file_name' (str), 'datetime' (datetime), and
-            'location' (str)).
+        image_info (str, dict, or list): a dict in CCT format, a CCT .json file, or just the 
+            'images' component of a CCT dataset (a list of dicts with  fields 'file_name' (str), 
+            'datetime' (datetime), and 'location' (str)).
+        options (SequenceOptions): options parameterizing the assembly of images into sequences;
+            see the SequenceOptions class for details.
+            
+    Returns:
+        image_info: if [image_info] is passed as a list, returns the list, otherwise returns
+            a CCT-formatted dict.
     """
     
     if options is None:
         options = SequenceOptions()
     
-    if isinstance(image_info,str):
+    to_return = None
+    
+    if isinstance(image_info,list):
+        to_return = image_info
+        
+    elif isinstance(image_info,str):
         with open(image_info,'r') as f:
-            image_info = json.load(f)
+            d = json.load(f)
+            to_return = d
+            image_info = d['images']
         
-    if isinstance(image_info,dict):
+    elif isinstance(image_info,dict):
+        to_return = image_info
         image_info = image_info['images']
+    
+    else:
+        raise ValueError('Unrecognized type for [image_info]')
         
+    # Modifies the images in place
+    _ = parse_datetimes_from_cct_image_list(image_info)
+    
     # Find all unique locations
     locations = set()
     for im in image_info:
@@ -401,4 +523,18 @@ def create_sequences(image_info,options=None):
     
     print('Created {} sequences from {} images'.format(len(all_sequences),len(image_info)))
     
-# ...create_sequences()
+    return to_return
+
+# ...def create_sequences(...)
+
+
+#%% Test drivers
+
+if False:
+    
+    pass
+
+    #%% 
+    
+    fn = r'g:\temp\test.json'
+    d = parse_datetimes_from_cct_dict(fn,conversion_failure_behavior='error')

{megadetector-5.0.21 → megadetector-5.0.23}/megadetector/data_management/cct_to_md.py

@@ -27,10 +27,13 @@ from tqdm import tqdm
 def cct_to_md(input_filename,output_filename=None):
     """
     "Converts" a COCO Camera Traps file to a MD results file.  Currently ignores
-    non-bounding-box annotations, and gives all annotations a confidence of 1.0.
+    non-bounding-box annotations.  If the semi-standard "score" field is present in
+    an annotation, or the totally non-standard "conf" field is present, it will be 
+    transferred to the output, otherwise a confidence value of 1.0 is assumed for
+    all annotations.
 
-    The only reason to do this is if you are going to add information to an existing
-    CCT-formatted dataset, and you want to do that in Timelapse.
+    The main reason to run this script the scenario where you are going to add information 
+    to an existing CCT-formatted dataset, and you want to do that in Timelapse.
 
     Currently assumes that width and height are present in the input data, does not
     read them from images.
@@ -113,8 +116,12 @@ def cct_to_md(input_filename,output_filename=None):
                    
                    det = {}
                    det['category'] = str(ann['category_id'])
-                   det['conf'] = 1.0
-                   # max_detection_conf = 1.0
+                   if 'score' in ann:
+                       det['conf'] = ann['score']
+                   elif 'conf' in ann:
+                       det['conf'] = ann['conf']
+                   else:
+                       det['conf'] = 1.0
                    
                    # MegaDetector: [x,y,width,height] (normalized, origin upper-left)
                    # CCT: [x,y,width,height] (absolute, origin upper-left)

{megadetector-5.0.21 → megadetector-5.0.23}/megadetector/data_management/databases/integrity_check_json_db.py

@@ -28,6 +28,7 @@ from tqdm import tqdm
 
 from megadetector.visualization.visualization_utils import open_image
 from megadetector.utils import ct_utils
+from megadetector.utils.path_utils import find_images
 
 
 #%% Classes and environment
@@ -67,7 +68,7 @@ class IntegrityCheckOptions:
         self.allowIntIDs = False
     
 # This is used in a medium-hacky way to share modified options across threads
-defaultOptions = IntegrityCheckOptions()
+default_options = IntegrityCheckOptions()
 
 
 #%% Functions
@@ -89,7 +90,7 @@ def _check_image_existence_and_size(image,options=None):
     """
 
     if options is None:        
-        options = defaultOptions
+        options = default_options
     
     assert options.bCheckImageExistence
     
@@ -124,9 +125,9 @@ def integrity_check_json_db(jsonFile, options=None):
     
     Returns:
         tuple: tuple containing:
-            - sortedCategories (dict): list of categories used in [jsonFile], sorted by frequency
+            - sorted_categories (dict): list of categories used in [jsonFile], sorted by frequency
             - data (dict): the data loaded from [jsonFile]
-            - errorInfo (dict): specific validation errors
+            - error_info (dict): specific validation errors
     """
     
     if options is None:       
@@ -141,7 +142,7 @@ def integrity_check_json_db(jsonFile, options=None):
     if options.baseDir is None:
         options.baseDir = ''
         
-    baseDir = options.baseDir
+    base_dir = options.baseDir
     
     
     ##%% Read .json file if necessary, integrity-check fields
@@ -156,7 +157,7 @@ def integrity_check_json_db(jsonFile, options=None):
     
         if options.verbose:
             print('Reading .json {} with base dir [{}]...'.format(
-                    jsonFile,baseDir))
+                    jsonFile,base_dir))
         
         with open(jsonFile,'r') as f:
             data = json.load(f) 
@@ -171,17 +172,17 @@ def integrity_check_json_db(jsonFile, options=None):
     # info = data['info']
     assert 'info' in data, 'No info struct in database'
     
-    if len(baseDir) > 0:        
-        assert os.path.isdir(baseDir), 'Base directory {} does not exist'.format(baseDir)
+    if len(base_dir) > 0:        
+        assert os.path.isdir(base_dir), 'Base directory {} does not exist'.format(base_dir)
         
         
     ##%% Build dictionaries, checking ID uniqueness and internal validity as we go
     
-    imageIdToImage = {}
-    annIdToAnn = {}
-    catIdToCat = {}
-    catNameToCat = {}
-    imageLocationSet = set()
+    image_id_to_image = {}
+    ann_id_to_ann = {}
+    category_id_to_category = {}
+    category_name_to_category = {}
+    image_location_set = set()
     
     if options.verbose:
         print('Checking categories...')
@@ -195,16 +196,16 @@ def integrity_check_json_db(jsonFile, options=None):
         assert isinstance(cat['id'],int), 'Illegal category ID type: [{}]'.format(str(cat['id']))
         assert isinstance(cat['name'],str), 'Illegal category name type [{}]'.format(str(cat['name']))
         
-        catId = cat['id']
-        catName = cat['name']
+        category_id = cat['id']
+        category_name = cat['name']
         
         # Confirm ID uniqueness
-        assert catId not in catIdToCat, 'Category ID {} is used more than once'.format(catId)
-        catIdToCat[catId] = cat
+        assert category_id not in category_id_to_category, 'Category ID {} is used more than once'.format(category_id)
+        category_id_to_category[category_id] = cat
         cat['_count'] = 0
         
-        assert catName not in catNameToCat, 'Category name {} is used more than once'.format(catName)
-        catNameToCat[catName] = cat        
+        assert category_name not in category_name_to_category, 'Category name {} is used more than once'.format(category_name)
+        category_name_to_category[category_name] = cat        
         
     # ...for each category
         
@@ -217,7 +218,7 @@ def integrity_check_json_db(jsonFile, options=None):
             print('Trimming image list to {}'.format(options.iMaxNumImages))
         images = images[0:options.iMaxNumImages]
         
-    imagePathsInJson = set()
+    image_paths_in_json = set()
     
     sequences = set()
     
@@ -230,9 +231,9 @@ def integrity_check_json_db(jsonFile, options=None):
         assert 'file_name' in image
         assert 'id' in image
 
-        image['file_name'] = os.path.normpath(image['file_name'])
+        image['file_name'] = image['file_name'].replace('\\','/')
                 
-        imagePathsInJson.add(image['file_name'])
+        image_paths_in_json.add(image['file_name'])
         
         assert isinstance(image['file_name'],str), 'Illegal image filename type'
         
@@ -242,12 +243,12 @@ def integrity_check_json_db(jsonFile, options=None):
         else:
             assert isinstance(image['id'],str), 'Illegal image ID type'
         
-        imageId = image['id']        
+        image_id = image['id']        
         
         # Confirm ID uniqueness
-        assert imageId not in imageIdToImage, 'Duplicate image ID {}'.format(imageId)
+        assert image_id not in image_id_to_image, 'Duplicate image ID {}'.format(image_id)
         
-        imageIdToImage[imageId] = image
+        image_id_to_image[image_id] = image
         
         if 'height' in image:
             assert 'width' in image, 'Image with height but no width: {}'.format(image['id'])
@@ -263,70 +264,74 @@ def integrity_check_json_db(jsonFile, options=None):
             # assert isinstance(image['location'], str) or isinstance(image['location'], int), \
             #  'Illegal image location type'
             assert isinstance(image['location'], str)
-            imageLocationSet.add(image['location'])
+            image_location_set.add(image['location'])
     
         if 'seq_id' in image:
             sequences.add(image['seq_id'])
             
         assert not ('sequence_id' in image or 'sequence' in image), 'Illegal sequence identifier'
         
-    unusedFiles = []
+    unused_files = []
                 
+    image_paths_relative = None
+    
     # Are we checking for unused images?
-    if (len(baseDir) > 0) and options.bFindUnusedImages:    
+    if (len(base_dir) > 0) and options.bFindUnusedImages:    
         
         if options.verbose:
             print('\nEnumerating images...')
         
-        # Recursively enumerate images
-        imagePaths = []
-        for root, dirs, files in os.walk(baseDir):
-            for file in files:
-                if file.lower().endswith(('.jpeg', '.jpg', '.png')):
-                    relDir = os.path.relpath(root, baseDir)
-                    relFile = os.path.join(relDir,file)
-                    relFile = os.path.normpath(relFile)
-                    if len(relFile) > 2 and \
-                        (relFile[0:2] == './' or relFile[0:2] == '.\\'):                     
-                            relFile = relFile[2:]
-                    imagePaths.append(relFile)
-          
-        for p in imagePaths:
-            if p not in imagePathsInJson:
-                # print('Image {} is unused'.format(p))
-                unusedFiles.append(p)
+        image_paths_relative = find_images(base_dir,return_relative_paths=True,recursive=True)
+        
+        for fn_relative in image_paths_relative:
+            if fn_relative not in image_paths_in_json:
+                unused_files.append(fn_relative)
                 
-    validationErrors = []
+    validation_errors = []
     
-    # Are we checking file existence and/or image size?
-    if options.bCheckImageSizes or options.bCheckImageExistence:
+    # If we're checking image existence but not image size, we don't need to read the images
+    if options.bCheckImageExistence and not options.bCheckImageSizes:
+        
+        if image_paths_relative is None:
+            image_paths_relative = find_images(base_dir,return_relative_paths=True,recursive=True)
+            
+        image_paths_relative_set = set(image_paths_relative)
+        
+        for im in images:
+            if im['file_name'] not in image_paths_relative_set:
+                validation_errors.append(im['file_name'])
+            
+    # If we're checking image size, we need to read the images
+    if options.bCheckImageSizes:
         
-        if len(baseDir) == 0:
+        if len(base_dir) == 0:
             print('Warning: checking image sizes without a base directory, assuming "."')
          
         if options.verbose:
             print('Checking image existence and/or image sizes...')
         
         if options.nThreads is not None and options.nThreads > 1:
+            if options.verbose:
+                print('Starting a pool of {} workers'.format(options.nThreads))
             pool = ThreadPool(options.nThreads)
             # results = pool.imap_unordered(lambda x: fetch_url(x,nImages), indexedUrlList)
-            defaultOptions.baseDir = options.baseDir
-            defaultOptions.bCheckImageSizes = options.bCheckImageSizes
-            defaultOptions.bCheckImageExistence = options.bCheckImageExistence
+            default_options.baseDir = options.baseDir
+            default_options.bCheckImageSizes = options.bCheckImageSizes
+            default_options.bCheckImageExistence = options.bCheckImageExistence
             results = tqdm(pool.imap(_check_image_existence_and_size, images), total=len(images))
         else:
             results = []
             for im in tqdm(images):
                 results.append(_check_image_existence_and_size(im,options))
                 
-        for iImage,r in enumerate(results):
-            if not r:            
-                validationErrors.append(os.path.join(options.baseDir,images[iImage]['file_name']))
+        for i_image,result in enumerate(results):
+            if result is not None:            
+                validation_errors.append(images[i_image]['file_name'])
                             
     # ...for each image
     
     if options.verbose:
-        print('{} validation errors (of {})'.format(len(validationErrors),len(images)))
+        print('{} validation errors (of {})'.format(len(validation_errors),len(images)))
         print('Checking annotations...')
     
     nBoxes = 0
@@ -355,22 +360,22 @@ def integrity_check_json_db(jsonFile, options=None):
         annId = ann['id']        
         
         # Confirm ID uniqueness
-        assert annId not in annIdToAnn
-        annIdToAnn[annId] = ann
+        assert annId not in ann_id_to_ann
+        ann_id_to_ann[annId] = ann
     
         # Confirm validity
-        assert ann['category_id'] in catIdToCat, \
+        assert ann['category_id'] in category_id_to_category, \
             'Category {} not found in category list'.format(ann['category_id'])
-        assert ann['image_id'] in imageIdToImage, \
+        assert ann['image_id'] in image_id_to_image, \
           'Image ID {} referred to by annotation {}, not available'.format(
             ann['image_id'],ann['id'])
     
-        imageIdToImage[ann['image_id']]['_count'] += 1
-        catIdToCat[ann['category_id']]['_count'] +=1 
+        image_id_to_image[ann['image_id']]['_count'] += 1
+        category_id_to_category[ann['category_id']]['_count'] +=1 
         
     # ...for each annotation
     
-    sortedCategories = sorted(categories, key=itemgetter('_count'), reverse=True)
+    sorted_categories = sorted(categories, key=itemgetter('_count'), reverse=True)
     
     
     ##%% Print statistics
@@ -390,18 +395,18 @@ def integrity_check_json_db(jsonFile, options=None):
         print('Found {} unannotated images, {} images with multiple annotations'.format(
                 nUnannotated,nMultiAnnotated))
         
-        if (len(baseDir) > 0) and options.bFindUnusedImages:
-            print('Found {} unused image files'.format(len(unusedFiles)))
+        if (len(base_dir) > 0) and options.bFindUnusedImages:
+            print('Found {} unused image files'.format(len(unused_files)))
             
-        nUnusedCategories = 0
+        n_unused_categories = 0
         
         # Find unused categories
         for cat in categories:
             if cat['_count'] == 0:
                 print('Unused category: {}'.format(cat['name']))
-                nUnusedCategories += 1
+                n_unused_categories += 1
         
-        print('Found {} unused categories'.format(nUnusedCategories))
+        print('Found {} unused categories'.format(n_unused_categories))
                 
         sequenceString = 'no sequence info'
         if len(sequences) > 0:
@@ -410,21 +415,21 @@ def integrity_check_json_db(jsonFile, options=None):
         print('\nDB contains {} images, {} annotations, {} bboxes, {} categories, {}\n'.format(
                 len(images),len(annotations),nBoxes,len(categories),sequenceString))
     
-        if len(imageLocationSet) > 0:
-            print('DB contains images from {} locations\n'.format(len(imageLocationSet)))
+        if len(image_location_set) > 0:
+            print('DB contains images from {} locations\n'.format(len(image_location_set)))
                 
         print('Categories and annotation (not image) counts:\n')
         
-        for cat in sortedCategories:
+        for cat in sorted_categories:
             print('{:6} {}'.format(cat['_count'],cat['name']))
         
         print('')
     
-    errorInfo = {}
-    errorInfo['unusedFiles'] = unusedFiles
-    errorInfo['validationErrors'] = validationErrors
+    error_info = {}
+    error_info['unused_files'] = unused_files
+    error_info['validation_errors'] = validation_errors
     
-    return sortedCategories, data, errorInfo
+    return sorted_categories, data, error_info
 
 # ...def integrity_check_json_db()
     
@@ -434,7 +439,8 @@ def integrity_check_json_db(jsonFile, options=None):
 def main():
         
     parser = argparse.ArgumentParser()
-    parser.add_argument('jsonFile')
+    parser.add_argument('jsonFile',type=str,
+                        help='COCO-formatted .json file to validate')
     parser.add_argument('--bCheckImageSizes', action='store_true', 
                         help='Validate image size, requires baseDir to be specified. ' + \
                              'Implies existence checking.')
@@ -490,4 +496,4 @@ if False:
     
     for json_file in json_files:
         
-        sortedCategories,data,_ = integrity_check_json_db(json_file, options)
+        sorted_categories,data,_ = integrity_check_json_db(json_file, options)