megadetector 5.0.7__py3-none-any.whl → 5.0.9__py3-none-any.whl

api/batch_processing/api_core/batch_service/score.py

@@ -435,6 +435,5 @@ def main():
     with open(task_output_path, 'w', encoding='utf-8') as f:
         json.dump(detections, f, ensure_ascii=False)
 
-
 if __name__ == '__main__':
     main()

api/batch_processing/api_core/server_job_status_table.py

@@ -148,6 +148,5 @@ class TestJobStatusTable(unittest.TestCase):
         item_read = table.read_job_status(job_id)
         self.assertIsNone(item_read)
 
-
 if __name__ == '__main__':
     unittest.main()

api/batch_processing/api_core_support/aggregate_results_manually.py

@@ -42,6 +42,5 @@ def main():
     output_file_urls_str = json.dumps(output_file_urls)
     print(output_file_urls_str)
 
-
 if __name__ == '__main__':
     main()

api/batch_processing/api_support/summarize_daily_activity.py

@@ -148,6 +148,5 @@ def main():
         send_message()
         time.sleep(24 * 60 * 60)
 
-
 if __name__ == '__main__':
     main()

api/batch_processing/data_preparation/manage_local_batch.py

@@ -1,64 +1,64 @@
-########
-#
-# manage_local_batch.py
-#    
-# Semi-automated process for managing a local MegaDetector job, including
-# standard postprocessing steps.
-#
-# This script is not intended to be run from top to bottom like a typical Python script,
-# it's a notebook disguised with a .py extension.  It's the Bestest Most Awesome way to
-# run MegaDetector, but it's also pretty subtle; if you want to play with this, you might
-# want to check in with cameratraps@lila.science for some tips.  Otherwise... YMMV.
-#
-# Some general notes on using this script, which I do in Spyder, though everything will be
-# the same if you are reading this in Jupyter Notebook (using the .ipynb version of the 
-# script):
-#
-# * Typically when I have a MegaDetector job to run, I make a copy of this script.  Let's 
-#   say I'm running a job for an organization called "bibblebop"; I have a big folder of
-#   job-specific copies of this script, and I might save a new one called "bibblebop-2023-07-26.py" 
-#   (the filename doesn't matter, it just helps me keep these organized).
-#
-# * There are three variables you need to set in this script before you start running code:
-#   "input_path", "organization_name_short", and "job_date".  You will get a sensible error if you forget 
-#   to set any of these.  In this case I might set those to "/data/bibblebobcamerastuff",
-#   "bibblebop", and "2023-07-26", respectively.
-#
-# * The defaults assume you want to split the job into two tasks (this is the default because I have 
-#   two GPUs).  Nothing bad will happen if you do this on a zero-GPU or single-GPU machine, but if you
-#   want everything to run in one logical task, change "n_gpus" and "n_jobs" to 1 (instead of 2).
-#
-# * After setting the required variables, I run the first few cells - up to and including the one 
-#   called "Generate commands" - which collectively take basically zero seconds.  After you run the
-#   "Generate commands" cell, you will have a folder that looks something like:
-#
-#   ~/postprocessing/bibblebop/bibblebop-2023-07-06-mdv5a/
-#  
-#   On Windows, this means:
-#
-#   ~/postprocessing/bibblebop/bibblebop-2023-07-06-mdv5a/    
-#
-#   Everything related to this job - scripts, outputs, intermediate stuff - will be in this folder.
-#   Specifically, after the "Generate commands" cell, you'll have scripts in that folder called something
-#   like:
-#
-#   run_chunk_000_gpu_00.sh (or .bat on Windows)
-#
-#   Personally, I like to run that script directly in a command prompt (I just leave Spyder open, though 
-#   it's OK if Spyder gets shut down while MD is running).  
-#
-#   At this point, once you get the hang of it, you've invested about zero seconds of human time,
-#   but possibly several days of unattended compute time, depending on the size of your job.
-#   
-# * Then when the jobs are done, back to the interactive environment!  I run the next few cells,
-#   which make sure the job finished OK, and the cell called "Post-processing (pre-RDE)", which 
-#   generates an HTML preview of the results.  You are very plausibly done at this point, and can ignore
-#   all the remaining cells.  If you want to do things like repeat detection elimination, or running 
-#   a classifier, or splitting your results file up in specialized ways, there are cells for all of those
-#   things, but now you're in power-user territory, so I'm going to leave this guide here.  Email
-#   cameratraps@lila.science with questions about the fancy stuff.
-#
-########
+"""
+
+manage_local_batch.py
+   
+Semi-automated process for managing a local MegaDetector job, including
+standard postprocessing steps.
+
+This script is not intended to be run from top to bottom like a typical Python script,
+it's a notebook disguised with a .py extension.  It's the Bestest Most Awesome way to
+run MegaDetector, but it's also pretty subtle; if you want to play with this, you might
+want to check in with cameratraps@lila.science for some tips.  Otherwise... YMMV.
+
+Some general notes on using this script, which I do in Spyder, though everything will be
+the same if you are reading this in Jupyter Notebook (using the .ipynb version of the 
+script):
+
+* Typically when I have a MegaDetector job to run, I make a copy of this script.  Let's 
+  say I'm running a job for an organization called "bibblebop"; I have a big folder of
+  job-specific copies of this script, and I might save a new one called "bibblebop-2023-07-26.py" 
+  (the filename doesn't matter, it just helps me keep these organized).
+
+* There are three variables you need to set in this script before you start running code:
+  "input_path", "organization_name_short", and "job_date".  You will get a sensible error if you forget 
+  to set any of these.  In this case I might set those to "/data/bibblebobcamerastuff",
+  "bibblebop", and "2023-07-26", respectively.
+
+* The defaults assume you want to split the job into two tasks (this is the default because I have 
+  two GPUs).  Nothing bad will happen if you do this on a zero-GPU or single-GPU machine, but if you
+  want everything to run in one logical task, change "n_gpus" and "n_jobs" to 1 (instead of 2).
+
+* After setting the required variables, I run the first few cells - up to and including the one 
+  called "Generate commands" - which collectively take basically zero seconds.  After you run the
+  "Generate commands" cell, you will have a folder that looks something like:
+
+  ~/postprocessing/bibblebop/bibblebop-2023-07-06-mdv5a/
+ 
+  On Windows, this means:
+
+  ~/postprocessing/bibblebop/bibblebop-2023-07-06-mdv5a/    
+
+  Everything related to this job - scripts, outputs, intermediate stuff - will be in this folder.
+  Specifically, after the "Generate commands" cell, you'll have scripts in that folder called something
+  like:
+
+  run_chunk_000_gpu_00.sh (or .bat on Windows)
+
+  Personally, I like to run that script directly in a command prompt (I just leave Spyder open, though 
+  it's OK if Spyder gets shut down while MD is running).  
+
+  At this point, once you get the hang of it, you've invested about zero seconds of human time,
+  but possibly several days of unattended compute time, depending on the size of your job.
+  
+* Then when the jobs are done, back to the interactive environment!  I run the next few cells,
+  which make sure the job finished OK, and the cell called "Post-processing (pre-RDE)", which 
+  generates an HTML preview of the results.  You are very plausibly done at this point, and can ignore
+  all the remaining cells.  If you want to do things like repeat detection elimination, or running 
+  a classifier, or splitting your results file up in specialized ways, there are cells for all of those
+  things, but now you're in power-user territory, so I'm going to leave this guide here.  Email
+  cameratraps@lila.science with questions about the fancy stuff.
+
+"""
 
 #%% Imports and constants
 
@@ -86,6 +86,8 @@ from api.batch_processing.postprocessing.postprocess_batch_results import (
 from detection.run_detector import get_detector_version_from_filename
 from md_utils.ct_utils import image_file_to_camera_folder
 
+## Inference options
+
 # To specify a non-default confidence threshold for including detections in the .json file
 json_threshold = None
 
@@ -109,6 +111,11 @@ quiet_mode = True
 # will use its default size, which is 1280 * 1.3, which is almost always what you want.
 image_size = None
 
+# Should we include image size, timestamp, and/or EXIF data in MD output?
+include_image_size = False
+include_image_timestamp = False
+include_exif_data = False
+
 # Only relevant when running on CPU
 ncores = 1
 
@@ -187,7 +194,7 @@ augment = False
 
 ## Constants related to tiled inference
 
-use_tiled_inference = True
+use_tiled_inference = False
 
 # Should we delete tiles after each job?  Only set this to False for debugging;
 # large jobs will take up a lot of space if you keep tiles around after each task.
@@ -234,7 +241,7 @@ checkpoint_frequency = 10000
 approx_images_per_second = estimate_md_images_per_second(model_file) 
     
 # Rough estimate for the inference time cost of augmentation    
-if augment:
+if augment and (approx_images_per_second is not None):
     approx_images_per_second = approx_images_per_second * 0.7
     
 base_task_name = organization_name_short + '-' + job_date + job_description_string + '-' + \
@@ -268,6 +275,10 @@ filename_base = os.path.join(base_output_folder_name, base_task_name)
 combined_api_output_folder = os.path.join(filename_base, 'combined_api_outputs')
 postprocessing_output_folder = os.path.join(filename_base, 'preview')
 
+combined_api_output_file = os.path.join(
+    combined_api_output_folder,
+    '{}_detections.json'.format(base_task_name))
+
 os.makedirs(filename_base, exist_ok=True)
 os.makedirs(combined_api_output_folder, exist_ok=True)
 os.makedirs(postprocessing_output_folder, exist_ok=True)
@@ -494,7 +505,14 @@ for i_task,task in enumerate(task_info):
         
         overwrite_handling_string = '--overwrite_handling {}'.format(overwrite_handling)        
         cmd = f'{cuda_string} python run_detector_batch.py "{model_file}" "{chunk_file}" "{output_fn}" {checkpoint_frequency_string} {checkpoint_path_string} {use_image_queue_string} {ncores_string} {quiet_string} {image_size_string} {confidence_threshold_string} {overwrite_handling_string}'
-                
+        
+        if include_image_size:
+            cmd += ' --include_image_size'
+        if include_image_timestamp:
+            cmd += ' --include_image_timestamp'
+        if include_exif_data:
+            cmd += ' --include_exif_data'
+        
     cmd_file = os.path.join(filename_base,'run_chunk_{}_gpu_{}{}'.format(str(i_task).zfill(3),
                             str(gpu_number).zfill(2),script_extension))
     
@@ -747,10 +765,6 @@ for im in combined_results['images']:
     else:
         im['file'] = im['file'].replace(input_path + '/','',1)
     
-combined_api_output_file = os.path.join(
-    combined_api_output_folder,
-    '{}_detections.json'.format(base_task_name))
-
 with open(combined_api_output_file,'w') as f:
     json.dump(combined_results,f,indent=1)
 
@@ -789,11 +803,11 @@ if render_animals_only:
 os.makedirs(output_base, exist_ok=True)
 print('Processing to {}'.format(output_base))
 
-options.api_output_file = combined_api_output_file
+options.md_results_file = combined_api_output_file
 options.output_dir = output_base
 ppresults = process_batch_results(options)
 html_output_file = ppresults.output_html_file
-path_utils.open_file(html_output_file,attempt_to_open_in_wsl_host=True)
+path_utils.open_file(html_output_file,attempt_to_open_in_wsl_host=True,browser_name='chrome')
 # import clipboard; clipboard.copy(html_output_file)
 
 
@@ -823,7 +837,7 @@ options.otherDetectionsThreshold = options.confidenceMin
 
 options.bRenderDetectionTiles = True
 options.maxOutputImageWidth = 2000
-options.detectionTilesMaxCrops = 300
+options.detectionTilesMaxCrops = 250
 
 # options.lineThickness = 5
 # options.boxExpansion = 8
@@ -925,12 +939,12 @@ os.makedirs(output_base, exist_ok=True)
 
 print('Processing post-RDE to {}'.format(output_base))
 
-options.api_output_file = filtered_output_filename
+options.md_results_file = filtered_output_filename
 options.output_dir = output_base
 ppresults = process_batch_results(options)
 html_output_file = ppresults.output_html_file
 
-path_utils.open_file(html_output_file,attempt_to_open_in_wsl_host=True)
+path_utils.open_file(html_output_file,attempt_to_open_in_wsl_host=True,browser_name='chrome')
 # import clipboard; clipboard.copy(html_output_file)
 
 
@@ -2000,10 +2014,10 @@ output_base = os.path.join(postprocessing_output_folder, folder_token + \
 os.makedirs(output_base, exist_ok=True)
 print('Processing {} to {}'.format(base_task_name, output_base))
 
-options.api_output_file = sequence_smoothed_classification_file
+options.md_results_file = sequence_smoothed_classification_file
 options.output_dir = output_base
 ppresults = process_batch_results(options)
-path_utils.open_file(ppresults.output_html_file,attempt_to_open_in_wsl_host=True)
+path_utils.open_file(ppresults.output_html_file,attempt_to_open_in_wsl_host=True,browser_name='chrome')
 # import clipboard; clipboard.copy(ppresults.output_html_file)
 
 #% Zip .json files
@@ -2071,7 +2085,7 @@ for i, j in itertools.combinations(list(range(0,len(filenames))),2):
 results = compare_batch_results(options)
 
 from md_utils.path_utils import open_file
-open_file(results.html_output_file,attempt_to_open_in_wsl_host=True)
+open_file(results.html_output_file,attempt_to_open_in_wsl_host=True,browser_name='chrome')
 
 
 #%% Merge in high-confidence detections from another results file
@@ -2120,12 +2134,12 @@ output_base_large_boxes = os.path.join(postprocessing_output_folder,
 os.makedirs(output_base_large_boxes, exist_ok=True)
 print('Processing post-RDE, post-size-separation to {}'.format(output_base_large_boxes))
 
-options.api_output_file = size_separated_file
+options.md_results_file = size_separated_file
 options.output_dir = output_base_large_boxes
 
 ppresults = process_batch_results(options)
 html_output_file = ppresults.output_html_file
-path_utils.open_file(html_output_file,attempt_to_open_in_wsl_host=True)
+path_utils.open_file(html_output_file,attempt_to_open_in_wsl_host=True,browser_name='chrome')
 
 
 #%% .json splitting
@@ -2280,7 +2294,7 @@ import nbformat as nbf
 if os.name == 'nt':
     git_base = r'c:\git'
 else:
-    git_base = os.path.expanduer('~/git')
+    git_base = os.path.expanduser('~/git')
     
 input_py_file = git_base + '/MegaDetector/api/batch_processing/data_preparation/manage_local_batch.py'
 assert os.path.isfile(input_py_file)

api/batch_processing/data_preparation/manage_video_batch.py

@@ -1,11 +1,11 @@
-########
-#
-# manage_video_batch.py
-#
-# Notebook-esque script to manage the process of running a local batch of videos
-# through MD.  Defers most of the heavy lifting to manage_local_batch.py .
-#
-########
+"""
+
+manage_video_batch.py
+
+Notebook-esque script to manage the process of running a local batch of videos
+through MD.  Defers most of the heavy lifting to manage_local_batch.py .
+
+"""
 
 #%% Imports and constants
 

api/batch_processing/integration/digiKam/xmp_integration.py

@@ -460,7 +460,6 @@ def main():
     else:
         process_input_data(options)
 
-
 if __name__ == '__main__':
     
     main()

api/batch_processing/integration/eMammal/test_scripts/push_annotations_to_emammal.py

@@ -121,7 +121,6 @@ def main():
             update_data(sql)
             mysql_connection.commit()
 
-
 if __name__ == '__main__':
     main()
 

api/batch_processing/postprocessing/add_max_conf.py

@@ -1,15 +1,15 @@
-########
-#
-# add_max_conf.py
-#
-# The MD output format included a "max_detection_conf" field with each image
-# up to and including version 1.2; it was removed as of version 1.3 (it's
-# redundant with the individual detection confidence values).
-#
-# Just in case someone took a dependency on that field, this script allows you
-# to add it back to an existing .json file.
-#
-########
+"""
+
+add_max_conf.py
+
+The MD output format included a "max_detection_conf" field with each image
+up to and including version 1.2; it was removed as of version 1.3 (it's
+redundant with the individual detection confidence values).
+
+Just in case someone took a dependency on that field, this script allows you
+to add it back to an existing .json file.
+
+"""
 
 #%% Imports and constants
 

api/batch_processing/postprocessing/categorize_detections_by_size.py

@@ -1,11 +1,11 @@
-########
-#
-# categorize_detections_by_size.py
-#
-# Given an API output .json file, creates a separate category for bounding boxes
-# above one or more size thresholds.
-# 
-########
+"""
+
+categorize_detections_by_size.py
+
+Given a MegaDetector .json file, creates a separate category for bounding boxes
+above one or more size thresholds.
+
+"""
 
 #%% Constants and imports
 
@@ -18,24 +18,42 @@ from tqdm import tqdm
 #%% Support classes
 
 class SizeCategorizationOptions:
-
-    # Should be sorted from smallest to largest
+    """
+    Options used to parameterize categorize_detections_by_size().
+    """
+    
+    #: Thresholds to use for separation, as a fraction of the image size.
+    #:
+    #: Should be sorted from smallest to largest.
     size_thresholds = [0.95]
     
-    # List of category numbers to use in separation; uses all categories if None
+    #: List of category numbers to use in separation; uses all categories if None
     categories_to_separate = None
     
-    # Can be "size", "width", or "height"
+    #: Dimension to use for thresholding; can be "size", "width", or "height"
     measurement = 'size'
     
-    # Should have the same length as "size_thresholds"
+    #: Categories to assign to thresholded ranges; should have the same length as
+    #: "size_thresholds".
     size_category_names = ['large_detection']
     
                                                  
 #%% Main functions
     
 def categorize_detections_by_size(input_file,output_file=None,options=None):
-    
+    """
+    Given a MegaDetector .json file, creates a separate category for bounding boxes
+    above one or more size thresholds, optionally writing results to [output_file].
+    
+    Args:
+        input_file (str): file to process
+        output_file (str, optional): optional output file
+        options (SizeCategorizationOptions): categorization parameters
+        
+    Returns:
+        dict: data loaded from [input_file], with the new size-based categories.        
+        Identical to what's written to [output_file], if [output_file] is not None.
+    """
     if options is None:
         options = SizeCategorizationOptions()
     

api/batch_processing/postprocessing/combine_api_outputs.py

@@ -1,54 +1,56 @@
-########
-#
-# combine_api_outputs.py
-#
-# Merges two or more .json files in batch API output format, optionally
-# writing the results to another .json file.
-#
-# * Concatenates image lists, erroring if images are not unique.
-# * Errors if class lists are conflicting; errors on unrecognized fields.
-# * Checks compatibility in info structs, within reason.
-# 
-# File format:
-# 
-# https://github.com/agentmorris/MegaDetector/tree/master/api/batch_processing#batch-processing-api-output-format
-# 
-# Command-line use:
-# 
-# combine_api_outputs input1.json input2.json ... inputN.json output.json
-# 
-# Also see combine_api_shard_files() (not exposed via the command line yet) to
-# combine the intermediate files created by the API.
-# 
-# This does no checking for redundancy; if you are looking to ensemble
-# the results of multiple model versions, see merge_detections.py.
-# 
-########
+"""
+
+combine_api_outputs.py
+
+Merges two or more .json files in batch API output format, optionally
+writing the results to another .json file.
+
+* Concatenates image lists, erroring if images are not unique.
+* Errors if class lists are conflicting; errors on unrecognized fields.
+* Checks compatibility in info structs, within reason.
+
+File format:
+
+https://github.com/agentmorris/MegaDetector/tree/master/api/batch_processing#batch-processing-api-output-format
+
+Command-line use:
+
+combine_api_outputs input1.json input2.json ... inputN.json output.json
+
+Also see combine_api_shard_files() (not exposed via the command line yet) to
+combine the intermediate files created by the API.
+
+This does no checking for redundancy; if you are looking to ensemble
+the results of multiple model versions, see merge_detections.py.
+
+"""
 
 #%% Constants and imports
 
 import argparse
 import sys
 import json
-from typing import Any, Dict, Iterable, Mapping, List, Optional
 
 
 #%% Merge functions
 
-def combine_api_output_files(input_files: List[str],
-                             output_file: Optional[str] = None,
-                             require_uniqueness: bool = True,
-                             verbose: bool = True
-                             ) -> Dict[str, Any]:
+def combine_api_output_files(input_files,
+                             output_file=None,
+                             require_uniqueness=True,
+                             verbose=True):
     """
-    Merges list of JSON API detection files *input_files* into a single
-    dictionary, optionally writing the result to *output_file*.
+    Merges the list of MD results files [input_files] into a single
+    dictionary, optionally writing the result to [output_file].
 
     Args:
-        input_files: list of str, paths to JSON detection files
-        output_file: optional str, path to write merged JSON
-        require_uniqueness: bool, whether to require that the images in
-            each input_dict be unique
+        input_files (list of str): paths to JSON detection files
+        output_file (str, optional): path to write merged JSON
+        require_uniqueness (bool): whether to require that the images in
+            each list of images be unique
+            
+    Returns:
+        dict: merged dictionaries loaded from [input_files], identical to what's 
+        written to [output_file] if [output_file] is not None
     """
     
     def print_if_verbose(s):
@@ -73,27 +75,27 @@ def combine_api_output_files(input_files: List[str],
     return merged_dict
 
 
-def combine_api_output_dictionaries(input_dicts: Iterable[Mapping[str, Any]],
-                                    require_uniqueness: bool = True
-                                    ) -> Dict[str, Any]:
+def combine_api_output_dictionaries(input_dicts, require_uniqueness=True):
     """
-    Merges the list of API detection dictionaries *input_dicts*.  See header
-    comment for details on merge rules.
+    Merges the list of MD results dictionaries [input_dicts] into a single dict.
+    See module header comment for details on merge rules.
 
     Args:
-        input_dicts: list of dicts, each dict is the JSON of the detections
-            output file from the Batch Processing API
-        require_uniqueness: bool, whether to require that the images in
-            each input_dict be unique
-
-    Returns: dict, represents the merged JSON
+        input_dicts (list of dicts): list of dicts in which each dict represents the 
+            contents of a MD output file
+        require_uniqueness (bool): whether to require that the images in
+            each input dict be unique; if this is True and image filenames are
+            not unique, an error is raised.
+
+    Returns
+        dict: merged MD results
     """
     
     # Map image filenames to detections, we'll convert to a list later
     images = {}
-    info: Dict[str, str] = {}
-    detection_categories: Dict[str, str] = {}
-    classification_categories: Dict[str, str] = {}
+    info = {}
+    detection_categories = {}
+    classification_categories = {}
     n_redundant_images = 0
     n_images = 0
 
@@ -182,8 +184,20 @@ def combine_api_output_dictionaries(input_dicts: Iterable[Mapping[str, Any]],
 
 def combine_api_shard_files(input_files, output_file=None):
     """
-    Merges the list of .json-formatted API shard files *input_files* into a single
-    list of dictionaries, optionally writing the result to *output_file*.
+    Merges the list of .json-formatted API shard files [input_files] into a single
+    list of dictionaries, optionally writing the result to [output_file].
+    
+    This operates on mostly-deprecated API shard files, not MegaDetector results files.
+    If you don't know what an API shard file is, you don't want this function.
+    
+    Args:
+        input_files (list of str): files to merge
+        output_file (str, optiona): file to which we should write merged results
+        
+    Returns:
+        dict: merged results
+        
+    :meta private:
     """
 
     input_lists = []
@@ -215,6 +229,7 @@ def combine_api_shard_files(input_files, output_file=None):
 #%% Command-line driver
 
 def main():
+    
     parser = argparse.ArgumentParser()
     parser.add_argument(
         'input_paths', nargs='+',
@@ -230,6 +245,5 @@ def main():
     args = parser.parse_args()
     combine_api_output_files(args.input_paths, args.output_path)
 
-
 if __name__ == '__main__':
     main()