megadetector 5.0.8__py3-none-any.whl → 5.0.9__py3-none-any.whl

api/batch_processing/api_core/batch_service/score.py

@@ -435,6 +435,5 @@ def main():
     with open(task_output_path, 'w', encoding='utf-8') as f:
         json.dump(detections, f, ensure_ascii=False)
 
-
 if __name__ == '__main__':
     main()

api/batch_processing/api_core/server_job_status_table.py

@@ -148,6 +148,5 @@ class TestJobStatusTable(unittest.TestCase):
         item_read = table.read_job_status(job_id)
         self.assertIsNone(item_read)
 
-
 if __name__ == '__main__':
     unittest.main()

api/batch_processing/api_core_support/aggregate_results_manually.py

@@ -42,6 +42,5 @@ def main():
     output_file_urls_str = json.dumps(output_file_urls)
     print(output_file_urls_str)
 
-
 if __name__ == '__main__':
     main()

api/batch_processing/api_support/summarize_daily_activity.py

@@ -148,6 +148,5 @@ def main():
         send_message()
         time.sleep(24 * 60 * 60)
 
-
 if __name__ == '__main__':
     main()

api/batch_processing/data_preparation/manage_local_batch.py

@@ -1,64 +1,64 @@
-########
-#
-# manage_local_batch.py
-#    
-# Semi-automated process for managing a local MegaDetector job, including
-# standard postprocessing steps.
-#
-# This script is not intended to be run from top to bottom like a typical Python script,
-# it's a notebook disguised with a .py extension.  It's the Bestest Most Awesome way to
-# run MegaDetector, but it's also pretty subtle; if you want to play with this, you might
-# want to check in with cameratraps@lila.science for some tips.  Otherwise... YMMV.
-#
-# Some general notes on using this script, which I do in Spyder, though everything will be
-# the same if you are reading this in Jupyter Notebook (using the .ipynb version of the 
-# script):
-#
-# * Typically when I have a MegaDetector job to run, I make a copy of this script.  Let's 
-#   say I'm running a job for an organization called "bibblebop"; I have a big folder of
-#   job-specific copies of this script, and I might save a new one called "bibblebop-2023-07-26.py" 
-#   (the filename doesn't matter, it just helps me keep these organized).
-#
-# * There are three variables you need to set in this script before you start running code:
-#   "input_path", "organization_name_short", and "job_date".  You will get a sensible error if you forget 
-#   to set any of these.  In this case I might set those to "/data/bibblebobcamerastuff",
-#   "bibblebop", and "2023-07-26", respectively.
-#
-# * The defaults assume you want to split the job into two tasks (this is the default because I have 
-#   two GPUs).  Nothing bad will happen if you do this on a zero-GPU or single-GPU machine, but if you
-#   want everything to run in one logical task, change "n_gpus" and "n_jobs" to 1 (instead of 2).
-#
-# * After setting the required variables, I run the first few cells - up to and including the one 
-#   called "Generate commands" - which collectively take basically zero seconds.  After you run the
-#   "Generate commands" cell, you will have a folder that looks something like:
-#
-#   ~/postprocessing/bibblebop/bibblebop-2023-07-06-mdv5a/
-#  
-#   On Windows, this means:
-#
-#   ~/postprocessing/bibblebop/bibblebop-2023-07-06-mdv5a/    
-#
-#   Everything related to this job - scripts, outputs, intermediate stuff - will be in this folder.
-#   Specifically, after the "Generate commands" cell, you'll have scripts in that folder called something
-#   like:
-#
-#   run_chunk_000_gpu_00.sh (or .bat on Windows)
-#
-#   Personally, I like to run that script directly in a command prompt (I just leave Spyder open, though 
-#   it's OK if Spyder gets shut down while MD is running).  
-#
-#   At this point, once you get the hang of it, you've invested about zero seconds of human time,
-#   but possibly several days of unattended compute time, depending on the size of your job.
-#   
-# * Then when the jobs are done, back to the interactive environment!  I run the next few cells,
-#   which make sure the job finished OK, and the cell called "Post-processing (pre-RDE)", which 
-#   generates an HTML preview of the results.  You are very plausibly done at this point, and can ignore
-#   all the remaining cells.  If you want to do things like repeat detection elimination, or running 
-#   a classifier, or splitting your results file up in specialized ways, there are cells for all of those
-#   things, but now you're in power-user territory, so I'm going to leave this guide here.  Email
-#   cameratraps@lila.science with questions about the fancy stuff.
-#
-########
+"""
+
+manage_local_batch.py
+   
+Semi-automated process for managing a local MegaDetector job, including
+standard postprocessing steps.
+
+This script is not intended to be run from top to bottom like a typical Python script,
+it's a notebook disguised with a .py extension.  It's the Bestest Most Awesome way to
+run MegaDetector, but it's also pretty subtle; if you want to play with this, you might
+want to check in with cameratraps@lila.science for some tips.  Otherwise... YMMV.
+
+Some general notes on using this script, which I do in Spyder, though everything will be
+the same if you are reading this in Jupyter Notebook (using the .ipynb version of the 
+script):
+
+* Typically when I have a MegaDetector job to run, I make a copy of this script.  Let's 
+  say I'm running a job for an organization called "bibblebop"; I have a big folder of
+  job-specific copies of this script, and I might save a new one called "bibblebop-2023-07-26.py" 
+  (the filename doesn't matter, it just helps me keep these organized).
+
+* There are three variables you need to set in this script before you start running code:
+  "input_path", "organization_name_short", and "job_date".  You will get a sensible error if you forget 
+  to set any of these.  In this case I might set those to "/data/bibblebobcamerastuff",
+  "bibblebop", and "2023-07-26", respectively.
+
+* The defaults assume you want to split the job into two tasks (this is the default because I have 
+  two GPUs).  Nothing bad will happen if you do this on a zero-GPU or single-GPU machine, but if you
+  want everything to run in one logical task, change "n_gpus" and "n_jobs" to 1 (instead of 2).
+
+* After setting the required variables, I run the first few cells - up to and including the one 
+  called "Generate commands" - which collectively take basically zero seconds.  After you run the
+  "Generate commands" cell, you will have a folder that looks something like:
+
+  ~/postprocessing/bibblebop/bibblebop-2023-07-06-mdv5a/
+ 
+  On Windows, this means:
+
+  ~/postprocessing/bibblebop/bibblebop-2023-07-06-mdv5a/    
+
+  Everything related to this job - scripts, outputs, intermediate stuff - will be in this folder.
+  Specifically, after the "Generate commands" cell, you'll have scripts in that folder called something
+  like:
+
+  run_chunk_000_gpu_00.sh (or .bat on Windows)
+
+  Personally, I like to run that script directly in a command prompt (I just leave Spyder open, though 
+  it's OK if Spyder gets shut down while MD is running).  
+
+  At this point, once you get the hang of it, you've invested about zero seconds of human time,
+  but possibly several days of unattended compute time, depending on the size of your job.
+  
+* Then when the jobs are done, back to the interactive environment!  I run the next few cells,
+  which make sure the job finished OK, and the cell called "Post-processing (pre-RDE)", which 
+  generates an HTML preview of the results.  You are very plausibly done at this point, and can ignore
+  all the remaining cells.  If you want to do things like repeat detection elimination, or running 
+  a classifier, or splitting your results file up in specialized ways, there are cells for all of those
+  things, but now you're in power-user territory, so I'm going to leave this guide here.  Email
+  cameratraps@lila.science with questions about the fancy stuff.
+
+"""
 
 #%% Imports and constants
 
@@ -803,7 +803,7 @@ if render_animals_only:
 os.makedirs(output_base, exist_ok=True)
 print('Processing to {}'.format(output_base))
 
-options.api_output_file = combined_api_output_file
+options.md_results_file = combined_api_output_file
 options.output_dir = output_base
 ppresults = process_batch_results(options)
 html_output_file = ppresults.output_html_file
@@ -939,7 +939,7 @@ os.makedirs(output_base, exist_ok=True)
 
 print('Processing post-RDE to {}'.format(output_base))
 
-options.api_output_file = filtered_output_filename
+options.md_results_file = filtered_output_filename
 options.output_dir = output_base
 ppresults = process_batch_results(options)
 html_output_file = ppresults.output_html_file
@@ -2014,7 +2014,7 @@ output_base = os.path.join(postprocessing_output_folder, folder_token + \
 os.makedirs(output_base, exist_ok=True)
 print('Processing {} to {}'.format(base_task_name, output_base))
 
-options.api_output_file = sequence_smoothed_classification_file
+options.md_results_file = sequence_smoothed_classification_file
 options.output_dir = output_base
 ppresults = process_batch_results(options)
 path_utils.open_file(ppresults.output_html_file,attempt_to_open_in_wsl_host=True,browser_name='chrome')
@@ -2134,7 +2134,7 @@ output_base_large_boxes = os.path.join(postprocessing_output_folder,
 os.makedirs(output_base_large_boxes, exist_ok=True)
 print('Processing post-RDE, post-size-separation to {}'.format(output_base_large_boxes))
 
-options.api_output_file = size_separated_file
+options.md_results_file = size_separated_file
 options.output_dir = output_base_large_boxes
 
 ppresults = process_batch_results(options)

api/batch_processing/data_preparation/manage_video_batch.py

@@ -1,11 +1,11 @@
-########
-#
-# manage_video_batch.py
-#
-# Notebook-esque script to manage the process of running a local batch of videos
-# through MD.  Defers most of the heavy lifting to manage_local_batch.py .
-#
-########
+"""
+
+manage_video_batch.py
+
+Notebook-esque script to manage the process of running a local batch of videos
+through MD.  Defers most of the heavy lifting to manage_local_batch.py .
+
+"""
 
 #%% Imports and constants
 

api/batch_processing/integration/digiKam/xmp_integration.py

@@ -460,7 +460,6 @@ def main():
     else:
         process_input_data(options)
 
-
 if __name__ == '__main__':
     
     main()

api/batch_processing/integration/eMammal/test_scripts/push_annotations_to_emammal.py

@@ -121,7 +121,6 @@ def main():
             update_data(sql)
             mysql_connection.commit()
 
-
 if __name__ == '__main__':
     main()
 

api/batch_processing/postprocessing/add_max_conf.py

@@ -1,15 +1,15 @@
-########
-#
-# add_max_conf.py
-#
-# The MD output format included a "max_detection_conf" field with each image
-# up to and including version 1.2; it was removed as of version 1.3 (it's
-# redundant with the individual detection confidence values).
-#
-# Just in case someone took a dependency on that field, this script allows you
-# to add it back to an existing .json file.
-#
-########
+"""
+
+add_max_conf.py
+
+The MD output format included a "max_detection_conf" field with each image
+up to and including version 1.2; it was removed as of version 1.3 (it's
+redundant with the individual detection confidence values).
+
+Just in case someone took a dependency on that field, this script allows you
+to add it back to an existing .json file.
+
+"""
 
 #%% Imports and constants
 

api/batch_processing/postprocessing/categorize_detections_by_size.py

@@ -1,11 +1,11 @@
-########
-#
-# categorize_detections_by_size.py
-#
-# Given an API output .json file, creates a separate category for bounding boxes
-# above one or more size thresholds.
-# 
-########
+"""
+
+categorize_detections_by_size.py
+
+Given a MegaDetector .json file, creates a separate category for bounding boxes
+above one or more size thresholds.
+
+"""
 
 #%% Constants and imports
 
@@ -18,24 +18,42 @@ from tqdm import tqdm
 #%% Support classes
 
 class SizeCategorizationOptions:
-
-    # Should be sorted from smallest to largest
+    """
+    Options used to parameterize categorize_detections_by_size().
+    """
+    
+    #: Thresholds to use for separation, as a fraction of the image size.
+    #:
+    #: Should be sorted from smallest to largest.
     size_thresholds = [0.95]
     
-    # List of category numbers to use in separation; uses all categories if None
+    #: List of category numbers to use in separation; uses all categories if None
     categories_to_separate = None
     
-    # Can be "size", "width", or "height"
+    #: Dimension to use for thresholding; can be "size", "width", or "height"
     measurement = 'size'
     
-    # Should have the same length as "size_thresholds"
+    #: Categories to assign to thresholded ranges; should have the same length as
+    #: "size_thresholds".
     size_category_names = ['large_detection']
     
                                                  
 #%% Main functions
     
 def categorize_detections_by_size(input_file,output_file=None,options=None):
-    
+    """
+    Given a MegaDetector .json file, creates a separate category for bounding boxes
+    above one or more size thresholds, optionally writing results to [output_file].
+    
+    Args:
+        input_file (str): file to process
+        output_file (str, optional): optional output file
+        options (SizeCategorizationOptions): categorization parameters
+        
+    Returns:
+        dict: data loaded from [input_file], with the new size-based categories.        
+        Identical to what's written to [output_file], if [output_file] is not None.
+    """
     if options is None:
         options = SizeCategorizationOptions()
     

api/batch_processing/postprocessing/combine_api_outputs.py

@@ -1,54 +1,56 @@
-########
-#
-# combine_api_outputs.py
-#
-# Merges two or more .json files in batch API output format, optionally
-# writing the results to another .json file.
-#
-# * Concatenates image lists, erroring if images are not unique.
-# * Errors if class lists are conflicting; errors on unrecognized fields.
-# * Checks compatibility in info structs, within reason.
-# 
-# File format:
-# 
-# https://github.com/agentmorris/MegaDetector/tree/master/api/batch_processing#batch-processing-api-output-format
-# 
-# Command-line use:
-# 
-# combine_api_outputs input1.json input2.json ... inputN.json output.json
-# 
-# Also see combine_api_shard_files() (not exposed via the command line yet) to
-# combine the intermediate files created by the API.
-# 
-# This does no checking for redundancy; if you are looking to ensemble
-# the results of multiple model versions, see merge_detections.py.
-# 
-########
+"""
+
+combine_api_outputs.py
+
+Merges two or more .json files in batch API output format, optionally
+writing the results to another .json file.
+
+* Concatenates image lists, erroring if images are not unique.
+* Errors if class lists are conflicting; errors on unrecognized fields.
+* Checks compatibility in info structs, within reason.
+
+File format:
+
+https://github.com/agentmorris/MegaDetector/tree/master/api/batch_processing#batch-processing-api-output-format
+
+Command-line use:
+
+combine_api_outputs input1.json input2.json ... inputN.json output.json
+
+Also see combine_api_shard_files() (not exposed via the command line yet) to
+combine the intermediate files created by the API.
+
+This does no checking for redundancy; if you are looking to ensemble
+the results of multiple model versions, see merge_detections.py.
+
+"""
 
 #%% Constants and imports
 
 import argparse
 import sys
 import json
-from typing import Any, Dict, Iterable, Mapping, List, Optional
 
 
 #%% Merge functions
 
-def combine_api_output_files(input_files: List[str],
-                             output_file: Optional[str] = None,
-                             require_uniqueness: bool = True,
-                             verbose: bool = True
-                             ) -> Dict[str, Any]:
+def combine_api_output_files(input_files,
+                             output_file=None,
+                             require_uniqueness=True,
+                             verbose=True):
     """
-    Merges list of JSON API detection files *input_files* into a single
-    dictionary, optionally writing the result to *output_file*.
+    Merges the list of MD results files [input_files] into a single
+    dictionary, optionally writing the result to [output_file].
 
     Args:
-        input_files: list of str, paths to JSON detection files
-        output_file: optional str, path to write merged JSON
-        require_uniqueness: bool, whether to require that the images in
+        input_files (list of str): paths to JSON detection files
+        output_file (str, optional): path to write merged JSON
+        require_uniqueness (bool): whether to require that the images in
             each list of images be unique
+            
+    Returns:
+        dict: merged dictionaries loaded from [input_files], identical to what's 
+        written to [output_file] if [output_file] is not None
     """
     
     def print_if_verbose(s):
@@ -73,27 +75,27 @@ def combine_api_output_files(input_files: List[str],
     return merged_dict
 
 
-def combine_api_output_dictionaries(input_dicts: Iterable[Mapping[str, Any]],
-                                    require_uniqueness: bool = True
-                                    ) -> Dict[str, Any]:
+def combine_api_output_dictionaries(input_dicts, require_uniqueness=True):
     """
-    Merges the list of API detection dictionaries *input_dicts*.  See header
-    comment for details on merge rules.
+    Merges the list of MD results dictionaries [input_dicts] into a single dict.
+    See module header comment for details on merge rules.
 
     Args:
-        input_dicts: list of dicts, each dict is the JSON of the detections
-            output file from the Batch Processing API
-        require_uniqueness: bool, whether to require that the images in
-            each input dict be unique
-
-    Returns: dict, represents the merged JSON
+        input_dicts (list of dicts): list of dicts in which each dict represents the 
+            contents of a MD output file
+        require_uniqueness (bool): whether to require that the images in
+            each input dict be unique; if this is True and image filenames are
+            not unique, an error is raised.
+
+    Returns
+        dict: merged MD results
     """
     
     # Map image filenames to detections, we'll convert to a list later
     images = {}
-    info: Dict[str, str] = {}
-    detection_categories: Dict[str, str] = {}
-    classification_categories: Dict[str, str] = {}
+    info = {}
+    detection_categories = {}
+    classification_categories = {}
     n_redundant_images = 0
     n_images = 0
 
@@ -182,8 +184,20 @@ def combine_api_output_dictionaries(input_dicts: Iterable[Mapping[str, Any]],
 
 def combine_api_shard_files(input_files, output_file=None):
     """
-    Merges the list of .json-formatted API shard files *input_files* into a single
-    list of dictionaries, optionally writing the result to *output_file*.
+    Merges the list of .json-formatted API shard files [input_files] into a single
+    list of dictionaries, optionally writing the result to [output_file].
+    
+    This operates on mostly-deprecated API shard files, not MegaDetector results files.
+    If you don't know what an API shard file is, you don't want this function.
+    
+    Args:
+        input_files (list of str): files to merge
+        output_file (str, optiona): file to which we should write merged results
+        
+    Returns:
+        dict: merged results
+        
+    :meta private:
     """
 
     input_lists = []
@@ -215,6 +229,7 @@ def combine_api_shard_files(input_files, output_file=None):
 #%% Command-line driver
 
 def main():
+    
     parser = argparse.ArgumentParser()
     parser.add_argument(
         'input_paths', nargs='+',
@@ -230,6 +245,5 @@ def main():
     args = parser.parse_args()
     combine_api_output_files(args.input_paths, args.output_path)
 
-
 if __name__ == '__main__':
     main()