PyPI - megadetector - Versions diffs - 5.0.19__py3-none-any.whl → 5.0.21__py3-none-any.whl - Mend

megadetector 5.0.19py3-none-any.whl → 5.0.21py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of megadetector might be problematic. Click here for more details.

Files changed (34) hide show

megadetector/postprocessing/postprocess_batch_results.py CHANGED Viewed

@@ -92,16 +92,18 @@ class PostProcessingOptions:
         #: Optional .json file containing ground truth information
         self.ground_truth_json_file = ''
-        #: Classes we'll treat as negative
+        #: List of classes we'll treat as negative (defaults to "empty", typically includes
+        #: classes like "blank", "misfire", etc.).
         #:
         #: Include the token "#NO_LABELS#" to indicate that an image with no annotations
         #: should be considered empty.
         self.negative_classes = DEFAULT_NEGATIVE_CLASSES
-        #: Classes we'll treat as neither positive nor negative
+        #: List of classes we'll treat as neither positive nor negative (defaults to
+        #: "unknown", typically includes classes like "unidentifiable").
         self.unlabeled_classes = DEFAULT_UNKNOWN_CLASSES
-        #: A list of output sets that we should count, but not render images for.
+        #: List of output sets that we should count, but not render images for.
         #:
         #: Typically used to preview sets with lots of empties, where you don't want to
         #: subset but also don't want to render 100,000 empty images.
@@ -198,11 +200,16 @@ class PostProcessingOptions:
         #: When classification results are present, should be sort alphabetically by class name (False)
         #: or in descending order by frequency (True)?
-        self.sort_classification_results_by_count = False
+        self.sort_classification_results_by_count = False
         #: Should we split individual pages up into smaller pages if there are more than
         #: N images?
         self.max_figures_per_html_file = None
+        #: Footer text for the index page
+        # self.footer_text = '<br/><p style="font-size:80%;">Preview page created with the <a href="{}">MegaDetector Python package</a>.</p>'.\
+        #    format('https://megadetector.readthedocs.io')
+        self.footer_text = ''
     # ...__init__()
@@ -590,6 +597,7 @@ def _prepare_html_subpages(images_html, output_dir, options=None):
         html_image_list_options = {}
         html_image_list_options['maxFiguresPerHtmlFile'] = options.max_figures_per_html_file
         html_image_list_options['headerHtml'] = '<h1>{}</h1>'.format(res.upper())
+        html_image_list_options['pageTitle'] = '{}'.format(res.lower())
         # Don't write empty pages
         if len(array) == 0:
@@ -762,7 +770,7 @@ def _render_image_no_gt(file_info,detection_categories_to_results_name,
     if len(rendered_image_html_info) > 0:
         image_result = [[res, rendered_image_html_info]]
+        classes_rendered_this_image = set()
         max_conf = 0
         for det in detections:
@@ -782,11 +790,14 @@ def _render_image_no_gt(file_info,detection_categories_to_results_name,
                 # confidence threshold
                 if (options.classification_confidence_threshold < 0) or \
                     (top1_class_score >= options.classification_confidence_threshold):
-                    image_result.append(['class_{}'.format(top1_class_name),
-                                         rendered_image_html_info])
+                    class_string = 'class_{}'.format(top1_class_name)
                 else:
-                    image_result.append(['class_unreliable',
+                    class_string = 'class_unreliable'
+                if class_string not in classes_rendered_this_image:
+                    image_result.append([class_string,
                                          rendered_image_html_info])
+                    classes_rendered_this_image.add(class_string)
             # ...if this detection has classification info
@@ -1083,7 +1094,8 @@ def process_batch_results(options):
     output_html_file = ''
-    style_header = """<head>
+    style_header = """<head>
+        <title>Detection results preview</title>
         <style type="text/css">
         a { text-decoration: none; }
         body { font-family: segoe ui, calibri, "trebuchet ms", verdana, arial, sans-serif; }
@@ -1424,7 +1436,7 @@ def process_batch_results(options):
         else:
             confidence_threshold_string = str(options.confidence_threshold)
-        index_page = """<html>
+        index_page = """<html>
         {}
         <body>
         <h2>Evaluation</h2>
@@ -1509,7 +1521,7 @@ def process_batch_results(options):
             index_page += '</div>'
         # Close body and html tags
-        index_page += '</body></html>'
+        index_page += '{}</body></html>'.format(options.footer_text)
         output_html_file = os.path.join(output_dir, 'index.html')
         with open(output_html_file, 'w') as f:
             f.write(index_page)
@@ -1529,7 +1541,6 @@ def process_batch_results(options):
         # for each category
         images_html = collections.defaultdict(list)
         # Add default entries by accessing them for the first time
         # Maps sorted tuples of detection category IDs (string ints) - e.g. ("1"), ("1", "4", "7") - to
@@ -1637,14 +1648,15 @@ def process_batch_results(options):
                         files_to_render), total=len(files_to_render)))
         else:
             for file_info in tqdm(files_to_render):
-                rendering_results.append(_render_image_no_gt(file_info,
-                                                            detection_categories_to_results_name,
-                                                            detection_categories,
-                                                            classification_categories,
-                                                            options=options))
+                rendering_result = _render_image_no_gt(file_info,
+                                                       detection_categories_to_results_name,
+                                                       detection_categories,
+                                                       classification_categories,
+                                                       options=options)
+                rendering_results.append(rendering_result)
-        elapsed = time.time() - start_time
+        elapsed = time.time() - start_time
         # Do we have classification results in addition to detection results?
         has_classification_info = False
@@ -1793,7 +1805,7 @@ def process_batch_results(options):
                         cname, cname.lower(), ccount)
             index_page += '</div>\n'
-        index_page += '</body></html>'
+        index_page += '{}</body></html>'.format(options.footer_text)
         output_html_file = os.path.join(output_dir, 'index.html')
         with open(output_html_file, 'w') as f:
             f.write(index_page)

megadetector/postprocessing/validate_batch_results.py CHANGED Viewed

@@ -42,11 +42,13 @@ class ValidateBatchResultsOptions:
         #:
         #: If None, assumes absolute paths.
         self.relative_path_base = None
+        #: Should we return the loaded data, or just the validation results?
+        self.return_data = False
 # ...class ValidateBatchResultsOptions
 #%% Main function
 def validate_batch_results(json_filename,options=None):
@@ -55,11 +57,17 @@ def validate_batch_results(json_filename,options=None):
     Args:
         json_filename (str): the filename to validate
-        options (ValidateBatchResultsOptions, optionsl): all the parameters used to control this
+        options (ValidateBatchResultsOptions, optional): all the parameters used to control this
             process, see ValidateBatchResultsOptions for details
     Returns:
-        bool: reserved; currently always errors or returns True.
+        dict: a dict with a field called "validation_results", which is itself a dict.  The reason
+        it's a dict inside a dict is that if return_data is True, the outer dict also contains all
+        the loaded data.  The "validation_results" dict contains fields called "errors", "warnings",
+        and "filename".  "errors" and "warnings" are lists of strings, although "errors" will never
+        be longer than N=1, since validation fails at the first error.
     """
     if options is None:
@@ -68,75 +76,127 @@ def validate_batch_results(json_filename,options=None):
     with open(json_filename,'r') as f:
         d = json.load(f)
-    ## Info validation
+    validation_results = {}
+    validation_results['filename'] = json_filename
+    validation_results['warnings'] = []
+    validation_results['errors'] = []
-    assert 'info' in d
-    info = d['info']
+    if not isinstance(d,dict):
+        validation_results['errors'].append('Input data is not a dict')
+        to_return = {}
+        to_return['validation_results'] = validation_results
+        return to_return
-    assert isinstance(info,dict)
-    assert 'format_version' in info
-    format_version = float(info['format_version'])
-    assert format_version >= 1.3, 'This validator can only be used with format version 1.3 or later'
+    try:
+        ## Info validation
+        if not 'info' in d:
+            raise ValueError('Input does not contain info field')
-    print('Validating a .json results file with format version {}'.format(format_version))
-    ## Category validation
-    assert 'detection_categories' in d
-    for k in d['detection_categories'].keys():
-        # Categories should be string-formatted ints
-        assert isinstance(k,str)
-        _ = int(k)
-        assert isinstance(d['detection_categories'][k],str)
+        info = d['info']
+        if not isinstance(info,dict):
+            raise ValueError('Input contains invalid info field')
+        if 'format_version' not in info :
+            raise ValueError('Input does not specify format version')
+        format_version = float(info['format_version'])
+        if format_version < 1.3:
+            raise ValueError('This validator can only be used with format version 1.3 or later')
-    if 'classification_categories' in d:
-        for k in d['classification_categories'].keys():
-            # Categories should be string-formatted ints
-            assert isinstance(k,str)
+        ## Category validation
+        if 'detection_categories' not in d:
+            raise ValueError('Input does not contain detection_categories field')
+        for k in d['detection_categories'].keys():
+            # Category ID should be string-formatted ints
+            if not isinstance(k,str):
+                raise ValueError('Invalid detection category ID: {}'.format(k))
             _ = int(k)
-            assert isinstance(d['classification_categories'][k],str)
-    ## Image validation
-    assert 'images' in d
-    assert isinstance(d['images'],list)
-    # im = d['images'][0]
-    for im in d['images']:
+            if not isinstance(d['detection_categories'][k],str):
+                raise ValueError('Invalid detection category name: {}'.format(
+                    d['detection_categories'][k]))
+        if 'classification_categories' in d:
+            for k in d['classification_categories'].keys():
+                # Categories should be string-formatted ints
+                if not isinstance(k,str):
+                    raise ValueError('Invalid classification category ID: {}'.format(k))
+                _ = int(k)
+                if not isinstance(d['classification_categories'][k],str):
+                    raise ValueError('Invalid classification category name: {}'.format(
+                        d['classification_categories'][k]))
-        assert isinstance(im,dict)
-        assert 'file' in im
-        file = im['file']
+        ## Image validation
-        if options.check_image_existence:
-            if options.relative_path_base is None:
-                file_abs = file
-            else:
-                file_abs = os.path.join(options.relative_path_base,file)
-            assert os.path.isfile(file_abs), 'Cannot find file {}'.format(file_abs)
+        if 'images' not in d:
+            raise ValueError('images field not present')
+        if not isinstance(d['images'],list):
+            raise ValueError('Invalid images field')
+        # im = d['images'][0]
+        for i_im,im in enumerate(d['images']):
-        if 'detections' not in im or im['detections'] is None:
-            assert 'failure' in im and isinstance(im['failure'],str)
-        else:
-            assert isinstance(im['detections'],list)
+            if not isinstance(im,dict):
+                raise ValueError('Invalid image at index {}'.format(i_im))
+            if 'file' not in im:
+                raise ValueError('Image without filename at index {}'.format(i_im))
-        if is_video_file(im['file']) and (format_version >= 1.4):
-            assert 'frame_rate' in im
-            if 'detections' in im and im['detections'] is not None:
-                for det in im['detections']:
-                    assert 'frame_number' in det
+            file = im['file']
-    # ...for each image
+            if options.check_image_existence:
+                if options.relative_path_base is None:
+                    file_abs = file
+                else:
+                    file_abs = os.path.join(options.relative_path_base,file)
+                if not os.path.isfile(file_abs):
+                    raise ValueError('Cannot find file {}'.format(file_abs))
+            if ('detections' not in im) or (im['detections'] is None):
+                if not ('failure' in im and isinstance(im['failure'],str)):
+                    raise ValueError('Image {} has no detections and no failure'.format(im['file']))
+            else:
+                if not isinstance(im['detections'],list):
+                    raise ValueError('Invalid detections list for image {}'.format(im['file']))
+            if is_video_file(im['file']) and (format_version >= 1.4):
+                if 'frame_rate' not in im:
+                    raise ValueError('Video without frame rate: {}'.format(im['file']))
+                if 'detections' in im and im['detections'] is not None:
+                    for det in im['detections']:
+                        if 'frame_number' not in det:
+                            raise ValueError('Frame without frame number in video {}'.format(
+                                im['file']))
+        # ...for each image
+        ## Checking on other keys
+        for k in d.keys():
+            if (k not in typical_keys) and (k not in required_keys):
+                validation_results['warnings'].append(
+                    'Warning: non-standard key {} present at file level'.format(k))
+    except Exception as e:
+        validation_results['errors'].append(str(e))
+    if options.return_data:
+        to_return = d
+    else:
+        to_return = {}
-    ## Checking on other keys
+    to_return['validation_results'] = validation_results
-    for k in d.keys():
-        if k not in typical_keys and k not in required_keys:
-            print('Warning: non-standard key {} present at file level'.format(k))
+    return to_return
 # ...def validate_batch_results(...)

megadetector/taxonomy_mapping/map_new_lila_datasets.py CHANGED Viewed

@@ -15,10 +15,10 @@ import json
 # Created by get_lila_category_list.py
 input_lila_category_list_file = os.path.expanduser('~/lila/lila_categories_list/lila_dataset_to_categories.json')
-output_file = os.path.expanduser('~/lila/lila_additions_2024.07.16.csv')
+output_file = os.path.expanduser('~/lila/lila_additions_2024.10.05.csv')
 datasets_to_map = [
-    'Desert Lion Conservation Camera Traps'
+    'Ohio Small Animals'
     ]
@@ -127,13 +127,18 @@ output_df.to_csv(output_file, index=None, header=True)
 #%% Manual lookup
 if False:
+    #%%
+    from megadetector.utils.path_utils import open_file
+    open_file(output_file)
     #%%
     # q = 'white-throated monkey'
     # q = 'cingulata'
     # q = 'notamacropus'
-    q = 'aves'
+    q = 'thamnophis saurita saurita'
     taxonomy_preference = 'inat'
     m = get_preferred_taxonomic_match(q,taxonomy_preference)
     # print(m.scientific_name); import clipboard; clipboard.copy(m.scientific_name)

megadetector/taxonomy_mapping/prepare_lila_taxonomy_release.py CHANGED Viewed

@@ -68,7 +68,8 @@ if False:
     #%% Generate the final output file
-    assert not os.path.isfile(release_taxonomy_file)
+    assert not os.path.isfile(release_taxonomy_file), \
+        'File {} exists, delete it manually before proceeding'.format(release_taxonomy_file)
     known_levels = ['stateofmatter', #noqa
                         'kingdom',
@@ -88,7 +89,7 @@ if False:
                         'genus',
                         'species','subspecies','variety']
-    levels_to_exclude = ['stateofmatter','zoosection','parvorder']
+    levels_to_exclude = ['stateofmatter','zoosection','parvorder','complex']
     for s in levels_to_exclude:
         assert s not in levels_to_include

megadetector/taxonomy_mapping/preview_lila_taxonomy.py CHANGED Viewed

@@ -16,7 +16,7 @@ import os
 import pandas as pd
 # lila_taxonomy_file = r"c:\git\agentmorrisprivate\lila-taxonomy\lila-taxonomy-mapping.csv"
-lila_taxonomy_file = os.path.expanduser('~/lila/lila_additions_2024.07.16.csv')
+lila_taxonomy_file = os.path.expanduser('~/lila/lila_additions_2024.10.05.csv')
 preview_base = os.path.expanduser('~/lila/lila_taxonomy_preview')
 os.makedirs(preview_base,exist_ok=True)
@@ -383,6 +383,8 @@ for i_row,row in df.iterrows():
 #%% Download sample images for all scientific names
+# Takes ~1 minute per 10 rows
 remapped_queries = {'papio':'papio+baboon',
                     'damaliscus lunatus jimela':'damaliscus lunatus',
                     'mazama':'genus+mazama',

megadetector/utils/ct_utils.py CHANGED Viewed

@@ -105,6 +105,26 @@ def args_to_object(args, obj):
     return obj
+def dict_to_object(d, obj):
+    """
+    Copies all fields from a dict to an object. Skips fields starting with _.
+    Does not check existence in the target object.
+    Args:
+        d (dict): the dict to convert to an object
+        obj (object): object whose whose attributes will be updated
+    Returns:
+        object: the modified object (modified in place, but also returned)
+    """
+    for k in d.keys():
+        if not k.startswith('_'):
+            setattr(obj, k, d[k])
+    return obj
 def pretty_print_object(obj, b_print=True):
     """
     Converts an arbitrary object to .json, optionally printing the .json representation.

megadetector/utils/md_tests.py CHANGED Viewed

@@ -29,10 +29,6 @@ import subprocess
 import argparse
 import inspect
-#: IoU threshold used to determine whether boxes in two detection files likely correspond
-#: to the same box.
-iou_threshold_for_file_comparison = 0.9
 #%% Classes
@@ -106,6 +102,10 @@ class MDTestOptions:
         #: PYTHONPATH to set for CLI tests; if None, inherits from the parent process.  Only
         #: impacts the called functions, not the parent process.
         self.cli_test_pythonpath = None
+        #: IoU threshold used to determine whether boxes in two detection files likely correspond
+        #: to the same box.
+        self.iou_threshold_for_file_comparison = 0.85
 # ...class MDTestOptions()
@@ -177,7 +177,8 @@ def get_expected_results_filename(gpu_is_available,
 def download_test_data(options=None):
     """
-    Downloads the test zipfile if necessary, unzips if necessary.
+    Downloads the test zipfile if necessary, unzips if necessary.  Initializes
+    temporary fields in [options], particularly [options.scratch_dir].
     Args:
         options (MDTestOptions, optional): see MDTestOptions for details
@@ -409,7 +410,7 @@ def compare_detection_lists(detections_a,detections_b,options,bidirectional_comp
             iou = get_iou(det_a['bbox'],b_det['bbox'])
             # Is this likely the same detection as det_a?
-            if iou >= iou_threshold_for_file_comparison and iou > highest_iou:
+            if iou >= options.iou_threshold_for_file_comparison and iou > highest_iou:
                 matching_det_b = b_det
                 highest_iou = iou
@@ -528,12 +529,14 @@ def compare_results(inference_output_file,expected_results_file,options):
     if not options.warning_mode:
         assert max_conf_error <= options.max_conf_error, \
-            'Confidence error {} is greater than allowable ({}), on file:\n{}'.format(
-                max_conf_error,options.max_conf_error,max_conf_error_file)
+            'Confidence error {} is greater than allowable ({}), on file:\n{} ({},{})'.format(
+                max_conf_error,options.max_conf_error,max_conf_error_file,
+                inference_output_file,expected_results_file)
         assert max_coord_error <= options.max_coord_error, \
-            'Coord error {} is greater than allowable ({}), on file:\n{}'.format(
-                max_coord_error,options.max_coord_error,max_coord_error_file)
+            'Coord error {} is greater than allowable ({}), on file:\n{} ({},{})'.format(
+                max_coord_error,options.max_coord_error,max_coord_error_file,
+                inference_output_file,expected_results_file)
     print('Max conf error: {} (file {})'.format(
         max_conf_error,max_conf_error_file))
@@ -846,7 +849,7 @@ def run_python_tests(options):
         video_options.frame_rendering_folder = os.path.join(options.scratch_dir,'video_scratch/rendered_frame_folder')
         video_options.render_output_video = True
         # video_options.keep_rendered_frames = False
-        # video_options.keep_rendered_frames = False
+        # video_options.keep_extracted_frames = False
         video_options.force_extracted_frame_folder_deletion = True
         video_options.force_rendered_frame_folder_deletion = True
         # video_options.reuse_results_if_available = False
@@ -886,7 +889,7 @@ def run_python_tests(options):
         video_options.frame_rendering_folder = os.path.join(options.scratch_dir,'video_scratch/rendered_frame_folder')
         video_options.render_output_video = False
         video_options.keep_rendered_frames = False
-        video_options.keep_rendered_frames = False
+        video_options.keep_extracted_frames = False
         video_options.force_extracted_frame_folder_deletion = True
         video_options.force_rendered_frame_folder_deletion = True
         video_options.reuse_results_if_available = False
@@ -1347,11 +1350,13 @@ if False:
     options.force_data_download = False
     options.force_data_unzip = False
     options.warning_mode = False
-    options.max_coord_error = 0.001
-    options.max_conf_error = 0.005
-    options.cli_working_dir = r'c:\git\MegaDetector'
-    options.yolo_working_dir = r'c:\git\yolov5-md'
+    options.max_coord_error = 0.01 # 0.001
+    options.max_conf_error = 0.01 # 0.005
+    # options.cli_working_dir = r'c:\git\MegaDetector'
+    # options.yolo_working_dir = r'c:\git\yolov5-md'
+    options.cli_working_dir = os.path.expanduser('~')
+    # options.yolo_working_dir = '/mnt/c/git/yolov5-md'
+    options = download_test_data(options)
     #%%
@@ -1363,6 +1368,47 @@ if False:
     run_tests(options)
+    #%%
+    yolo_inference_options_dict = {'input_folder': '/tmp/md-tests/md-test-images',
+                                   'image_filename_list': None,
+                                   'model_filename': 'MDV5A',
+                                   'output_file': '/tmp/md-tests/folder_inference_output_yolo_val.json',
+                                   'yolo_working_folder': '/mnt/c/git/yolov5-md',
+                                   'model_type': 'yolov5',
+                                   'image_size': None,
+                                   'conf_thres': 0.005,
+                                   'batch_size': 1,
+                                   'device_string': '0',
+                                   'augment': False,
+                                   'half_precision_enabled': None,
+                                   'symlink_folder': None,
+                                   'use_symlinks': True,
+                                   'unique_id_strategy': 'links',
+                                   'yolo_results_folder': None,
+                                   'remove_symlink_folder': True,
+                                   'remove_yolo_results_folder': True,
+                                   'yolo_category_id_to_name': {0: 'animal', 1: 'person', 2: 'vehicle'},
+                                   'overwrite_handling': 'overwrite',
+                                   'preview_yolo_command_only': False,
+                                   'treat_copy_failures_as_warnings': False,
+                                   'save_yolo_debug_output': False,
+                                   'recursive': True,
+                                   'checkpoint_frequency': None}
+    from megadetector.utils.ct_utils import dict_to_object
+    from megadetector.detection.run_inference_with_yolov5_val import \
+        YoloInferenceOptions, run_inference_with_yolo_val
+    yolo_inference_options = YoloInferenceOptions()
+    yolo_inference_options = dict_to_object(yolo_inference_options_dict, yolo_inference_options)
+    os.makedirs(options.scratch_dir,exist_ok=True)
+    inference_output_file_yolo_val = os.path.join(options.scratch_dir,'folder_inference_output_yolo_val.json')
+    run_inference_with_yolo_val(yolo_inference_options)
 #%% Command-line driver

megadetector 5.0.19__py3-none-any.whl → 5.0.21__py3-none-any.whl

Potentially problematic release.

megadetector 5.0.19py3-none-any.whl → 5.0.21py3-none-any.whl