megadetector 5.0.8__py3-none-any.whl → 5.0.10__py3-none-any.whl

api/batch_processing/postprocessing/subset_json_detector_output.py

@@ -1,61 +1,55 @@
-########
-#
-# subset_json_detector_output.py
-#
-# Creates one or more subsets of a detector API output file (.json), doing either
-# or both of the following (if both are requested, they happen in this order):
-#
-# 1) Retrieve all elements where filenames contain a specified query string, 
-#    optionally replacing that query with a replacement token. If the query is blank, 
-#    can also be used to prepend content to all filenames.
-#
-#    Does not support regex's, but supports a special case of ^string to indicate "must start with
-#    to match".
-#
-# 2) Create separate .jsons for each unique path, optionally making the filenames 
-#    in those .json's relative paths.  In this case, you specify an output directory, 
-#    rather than an output path.  All images in the folder blah\foo\bar will end up 
-#    in a .json file called blah_foo_bar.json.
-#
-# Can also apply a confidence threshold.
-#
-# Can also subset by categories above a threshold (programmatic invocation only, this is
-# not supported at the command line yet).
-#
-###
-#
-# Sample invocations (splitting into multiple json's):
-#
-# Read from "1800_idfg_statewide_wolf_detections_w_classifications.json", split up into 
-# individual .jsons in 'd:\temp\idfg\output', making filenames relative to their individual
-# folders:
-#
-# python subset_json_detector_output.py "d:\temp\idfg\1800_idfg_statewide_wolf_detections_w_classifications.json" "d:\temp\idfg\output" --split_folders --make_folder_relative
-#
-# Now do the same thing, but instead of writing .json's to d:\temp\idfg\output, write them to *subfolders*
-# corresponding to the subfolders for each .json file.
-#
-# python subset_json_detector_output.py "d:\temp\idfg\1800_detections_S2.json" "d:\temp\idfg\output_to_folders" --split_folders --make_folder_relative --copy_jsons_to_folders
-#
-###
-#
-# Sample invocations (creating a single subset matching a query):
-#
-# Read from "1800_detections.json", write to "1800_detections_2017.json"
-#
-# Include only images matching "2017", and change "2017" to "blah"
-#
-# python subset_json_detector_output.py "d:\temp\1800_detections.json" "d:\temp\1800_detections_2017_blah.json" --query 2017 --replacement blah
-#
-# Include all images, prepend with "prefix/"
-#
-# python subset_json_detector_output.py "d:\temp\1800_detections.json" "d:\temp\1800_detections_prefix.json" --replacement "prefix/"
-#
-###
-#
-# To subset a COCO Camera Traps .json database, see subset_json_db.py
-#
-########
+r"""
+
+subset_json_detector_output.py
+
+Creates one or more subsets of a detector results file (.json), doing either
+or both of the following (if both are requested, they happen in this order):
+
+1) Retrieve all elements where filenames contain a specified query string, 
+   optionally replacing that query with a replacement token. If the query is blank, 
+   can also be used to prepend content to all filenames.
+
+   Does not support regex's, but supports a special case of ^string to indicate "must start with
+   to match".
+
+2) Create separate .jsons for each unique path, optionally making the filenames 
+   in those .json's relative paths.  In this case, you specify an output directory, 
+   rather than an output path.  All images in the folder blah/foo/bar will end up 
+   in a .json file called blah_foo_bar.json.
+
+Can also apply a confidence threshold.
+
+Can also subset by categories above a threshold (programmatic invocation only, this is
+not supported at the command line yet).
+
+To subset a COCO Camera Traps .json database, see subset_json_db.py
+
+**Sample invocation (splitting into multiple json's)**
+
+Read from "1800_idfg_statewide_wolf_detections_w_classifications.json", split up into 
+individual .jsons in 'd:/temp/idfg/output', making filenames relative to their individual
+folders:
+
+python subset_json_detector_output.py "d:/temp/idfg/1800_idfg_statewide_wolf_detections_w_classifications.json" "d:/temp/idfg/output" --split_folders --make_folder_relative
+
+Now do the same thing, but instead of writing .json's to d:/temp/idfg/output, write them to *subfolders*
+corresponding to the subfolders for each .json file.
+
+python subset_json_detector_output.py "d:/temp/idfg/1800_detections_S2.json" "d:/temp/idfg/output_to_folders" --split_folders --make_folder_relative --copy_jsons_to_folders
+
+**Sample invocation (creating a single subset matching a query)**
+
+Read from "1800_detections.json", write to "1800_detections_2017.json"
+
+Include only images matching "2017", and change "2017" to "blah"
+
+python subset_json_detector_output.py "d:/temp/1800_detections.json" "d:/temp/1800_detections_2017_blah.json" --query 2017 --replacement blah
+
+Include all images, prepend with "prefix/"
+
+python subset_json_detector_output.py "d:/temp/1800_detections.json" "d:/temp/1800_detections_prefix.json" --replacement "prefix/"
+
+"""
 
 #%% Constants and imports
 
@@ -68,81 +62,85 @@ import re
 
 from tqdm import tqdm
 
-from md_utils.ct_utils import args_to_object
-from md_utils.ct_utils import get_max_conf
-from md_utils.ct_utils import invert_dictionary
+from md_utils.ct_utils import args_to_object, get_max_conf, invert_dictionary
+from md_utils.path_utils import top_level_folder
 
 
 #%% Helper classes
 
 class SubsetJsonDetectorOutputOptions:
+    """
+    Options used to parameterize subset_json_detector_output()
+    """
     
-    # Only process files containing the token 'query'
+    #: Only process files containing the token 'query'
     query = None
     
-    # Replace 'query' with 'replacement' if 'replacement' is not None.  If 'query' is None,
-    # prepend 'replacement'
+    #: Replace 'query' with 'replacement' if 'replacement' is not None.  If 'query' is None,
+    #: prepend 'replacement'
     replacement = None
     
-    # Should we split output into individual .json files for each folder?
+    #: Should we split output into individual .json files for each folder?
     split_folders = False
     
-    # Folder level to use for splitting ['bottom','top','n_from_bottom','n_from_top','dict']
-    #
-    # 'dict' requires 'split_folder_param' to be a dictionary mapping each filename
-    # to a token.
+    #: Folder level to use for splitting ['bottom','top','n_from_bottom','n_from_top','dict']
+    #:
+    #: 'dict' requires 'split_folder_param' to be a dictionary mapping each filename
+    #: to a token.
     split_folder_mode = 'bottom'  # 'top'
     
-    # When using the 'n_from_bottom' parameter to define folder splitting, this
-    # defines the number of directories from the bottom.  'n_from_bottom' with
-    # a parameter of zero is the same as 'bottom'.
-    #
-    # Same story with 'n_from_top'.
-    #
-    # When 'split_folder_mode' is 'dict', this should be a dictionary mapping each filename
-    # to a token.
+    #: When using the 'n_from_bottom' parameter to define folder splitting, this
+    #: defines the number of directories from the bottom.  'n_from_bottom' with
+    #: a parameter of zero is the same as 'bottom'.
+    #:
+    #: Same story with 'n_from_top'.
+    #:
+    #: When 'split_folder_mode' is 'dict', this should be a dictionary mapping each filename
+    #: to a token.
     split_folder_param = 0
     
-    # Only meaningful if split_folders is True: should we convert pathnames to be relative
-    # the folder for each .json file?
+    #: Only meaningful if split_folders is True: should we convert pathnames to be relative
+    #: the folder for each .json file?
     make_folder_relative = False
     
-    # Only meaningful if split_folders and make_folder_relative are True: if not None, 
-    # will copy .json files to their corresponding output directories, relative to 
-    # output_filename
+    #: Only meaningful if split_folders and make_folder_relative are True: if not None, 
+    #: will copy .json files to their corresponding output directories, relative to 
+    #: output_filename
     copy_jsons_to_folders = False
     
-    # Should we over-write .json files?
+    #: Should we over-write .json files?
     overwrite_json_files = False
     
-    # If copy_jsons_to_folders is true, do we require that directories already exist?
+    #: If copy_jsons_to_folders is true, do we require that directories already exist?
     copy_jsons_to_folders_directories_must_exist = True
     
-    # Threshold on confidence
+    #: Optional confidence threshold; if not None, detections below this confidence won't be
+    #: included in the output.
     confidence_threshold = None
     
-    # Should we remove failed images?
+    #: Should we remove failed images?
     remove_failed_images = False
     
-    # Either a list of category IDs (as string-ints) (not names), or a dictionary mapping category *IDs* 
-    # (as string-ints) (not names) to thresholds.  Removes  non-matching detections, does not 
-    # remove images.  Not technically mutually exclusize with category_names_to_keep, but it's an esoteric 
-    # scenario indeed where you would want to specify both.
+    #: Either a list of category IDs (as string-ints) (not names), or a dictionary mapping category *IDs* 
+    #: (as string-ints) (not names) to thresholds.  Removes  non-matching detections, does not 
+    #: remove images.  Not technically mutually exclusize with category_names_to_keep, but it's an esoteric 
+    #: scenario indeed where you would want to specify both.
     categories_to_keep = None
     
-    # Either a list of category names (not IDs), or a dictionary mapping category *names* (not IDs) to thresholds.  
-    # Removes non-matching detections, does not remove images.  Not technically mutually exclusize with 
-    # category_ids_to_keep, but it's an esoteric scenario indeed where you would want to specify both.
+    #: Either a list of category names (not IDs), or a dictionary mapping category *names* (not IDs) to thresholds.  
+    #: Removes non-matching detections, does not remove images.  Not technically mutually exclusize with 
+    #: category_ids_to_keep, but it's an esoteric scenario indeed where you would want to specify both.
     category_names_to_keep = None
     
+    #: Set to >0 during testing to limit the number of images that get processed.
     debug_max_images = -1
     
     
 #%% Main function
 
-def write_detection_results(data, output_filename, options):
+def _write_detection_results(data, output_filename, options):
     """
-    Write the detector-output-formatted dict *data* to *output_filename*.
+    Writes the detector-output-formatted dict *data* to *output_filename*.
     """
     
     if (not options.overwrite_json_files) and os.path.isfile(output_filename):
@@ -160,12 +158,19 @@ def write_detection_results(data, output_filename, options):
     with open(output_filename, 'w') as f:
         json.dump(data,f,indent=1)
 
-# ...write_detection_results()
+# ..._write_detection_results()
 
 
 def subset_json_detector_output_by_confidence(data, options):
     """
-    Remove all detections below options.confidence_threshold, update max confidences accordingly.
+    Removes all detections below options.confidence_threshold.
+    
+    Args:
+        data (dict): data loaded from a MD results file
+        options (SubsetJsonDetectorOutputOptions): parameters for subsetting
+    
+    Returns:
+        dict: Possibly-modified version of data (also modifies in place)
     """
     
     if options.confidence_threshold is None:
@@ -232,7 +237,14 @@ def subset_json_detector_output_by_confidence(data, options):
 
 def subset_json_detector_output_by_categories(data, options):
     """
-    Remove all detections without detections above a threshold for specific categories.
+    Removes all detections without detections above a threshold for specific categories.
+    
+    Args:
+        data (dict): data loaded from a MD results file
+        options (SubsetJsonDetectorOutputOptions): parameters for subsetting
+    
+    Returns:
+        dict: Possibly-modified version of data (also modifies in place)
     """
     
     # If categories_to_keep is supplied as a list, convert to a dict
@@ -334,6 +346,13 @@ def subset_json_detector_output_by_categories(data, options):
 def remove_failed_images(data,options):
     """
     Removed failed images from [data]
+    
+    Args:
+        data (dict): data loaded from a MD results file
+        options (SubsetJsonDetectorOutputOptions): parameters for subsetting
+    
+    Returns:
+        dict: Possibly-modified version of data (also modifies in place)
     """
     
     images_in = data['images']
@@ -365,8 +384,15 @@ def remove_failed_images(data,options):
 
 def subset_json_detector_output_by_query(data, options):
     """
-    Subset to images whose filename matches options.query; replace all instances of 
-    options.query with options.replacement.
+    Subsets to images whose filename matches options.query; replace all instances of 
+    options.query with options.replacement.  No-op if options.query_string is None or ''.
+    
+    Args:
+        data (dict): data loaded from a MD results file
+        options (SubsetJsonDetectorOutputOptions): parameters for subsetting
+    
+    Returns:
+        dict: Possibly-modified version of data (also modifies in place)
     """
     
     images_in = data['images']
@@ -415,74 +441,27 @@ def subset_json_detector_output_by_query(data, options):
 
 # ...subset_json_detector_output_by_query()
 
-
-def split_path(path, maxdepth=100):
-    """
-    Splits [path] into all its constituent tokens, e.g.:
-    
-    c:\blah\boo\goo.txt
-    
-    ...becomes:
-        
-    ['c:\\', 'blah', 'boo', 'goo.txt']
-    
-    http://nicks-liquid-soapbox.blogspot.com/2011/03/splitting-path-to-list-in-python.html
-    """
-    
-    (head, tail) = os.path.split(path)
-    return split_path(head, maxdepth - 1) + [tail] \
-        if maxdepth and head and head != path \
-        else [head or tail]
-
-# ...split_path()
-
-    
-def top_level_folder(p):
-    """
-    Gets the top-level folder from the path *p*; on Windows, will use the top-level folder
-    that isn't the drive.  E.g., top_level_folder(r"c:\blah\foo") returns "c:\blah".  Does not
-    include the leaf node, i.e. top_level_folder('/blah/foo') returns '/blah'.
-    """
-    
-    if p == '':
-        return ''
-    
-    # Path('/blah').parts is ('/','blah')
-    parts = split_path(p)
-    
-    if len(parts) == 1:
-        return parts[0]
-
-    # Handle paths like:
-    #
-    # /, \, /stuff, c:, c:\stuff
-    drive = os.path.splitdrive(p)[0]
-    if parts[0] == drive or parts[0] == drive + '/' or parts[0] == drive + '\\' or parts[0] in ['\\', '/']:
-        return os.path.join(parts[0], parts[1])
-    else:
-        return parts[0]
-
-# ...top_level_folder()
-
-    
-if False:  
-      
-    p = 'blah/foo/bar'; s = top_level_folder(p); print(s); assert s == 'blah'
-    p = '/blah/foo/bar'; s = top_level_folder(p); print(s); assert s == '/blah'
-    p = 'bar'; s = top_level_folder(p); print(s); assert s == 'bar'
-    p = ''; s = top_level_folder(p); print(s); assert s == ''
-    p = 'c:\\'; s = top_level_folder(p); print(s); assert s == 'c:\\'
-    p = r'c:\blah'; s = top_level_folder(p); print(s); assert s == 'c:\\blah'
-    p = r'c:\foo'; s = top_level_folder(p); print(s); assert s == 'c:\\foo'
-    p = r'c:/foo'; s = top_level_folder(p); print(s); assert s == 'c:/foo'
-    p = r'c:\foo/bar'; s = top_level_folder(p); print(s); assert s == 'c:\\foo'
-    
     
 def subset_json_detector_output(input_filename, output_filename, options, data=None):
     """
-    Main internal entry point
+    Main entry point; creates one or more subsets of a detector results file.  See the 
+    module header comment for more information about the available subsetting approaches.
         
     Makes a copy of [data] before modifying if a data dictionary is supplied.
+    
+    Args:
+        input_filename (str): filename to load and subset; can be None if [data] is supplied
+        output_filename (str): file or folder name (depending on [options]) to which we should
+            write subset results.
+        options (SubsetJsonDetectorOutputOptions): parameters for .json splitting/subsetting;
+            see SubsetJsonDetectorOutputOptions for details.
+        data (dict, optional): data loaded from a .json file; if this is not None, [input_filename]
+            will be ignored.  If supplied, this will be copied before it's modified.
+    
+    Returns:
+        dict: Results that are either loaded from [input_filename] and processed, or copied
+            from [data] and processed.
+    
     """
     
     if options is None:    
@@ -528,7 +507,7 @@ def subset_json_detector_output(input_filename, output_filename, options, data=N
         
     if not options.split_folders:
         
-        write_detection_results(data, output_filename, options)
+        _write_detection_results(data, output_filename, options)
         return data
     
     else:
@@ -558,7 +537,7 @@ def subset_json_detector_output(input_filename, output_filename, options, data=N
                 # Split string into folders, keeping delimiters
                 
                 # Don't use this, it removes delimiters
-                # tokens = split_path(fn)
+                # tokens = _split_path(fn)
                 tokens = re.split(r'([\\/])',fn)
                 
                 n_tokens_to_keep = ((options.split_folder_param + 1) * 2) - 1;
@@ -621,7 +600,7 @@ def subset_json_detector_output(input_filename, output_filename, options, data=N
             # forward-compatible in that I don't take dependencies on the other fields
             dir_data = data
             dir_data['images'] = folders_to_images[dirname]
-            write_detection_results(dir_data, json_fn, options)
+            _write_detection_results(dir_data, json_fn, options)
             print('Wrote {} images to {}'.format(len(dir_data['images']), json_fn))
             
         # ...for each directory
@@ -713,7 +692,5 @@ def main():
     
     subset_json_detector_output(args.input_file, args.output_file, options)
     
-
-if __name__ == '__main__':
-    
+if __name__ == '__main__':    
     main()

api/batch_processing/postprocessing/top_folders_to_bottom.py

@@ -1,26 +1,26 @@
-########
-#
-# top_folders_to_bottom.py
-#
-# Given a base folder with files like:
-#
-# A/1/2/a.jpg
-# B/3/4/b.jpg
-#
-# ...moves the top-level folders to the bottom in a new output folder, i.e., creates:
-#
-# 1/2/A/a.jpg
-# 3/4/B/b.jpg
-#
-# In practice, this is used to make this:
-# 
-# animal/camera01/image01.jpg
-#
-# ...look like:
-#
-# camera01/animal/image01.jpg
-#
-########
+"""
+
+top_folders_to_bottom.py
+
+Given a base folder with files like:
+
+* A/1/2/a.jpg
+* B/3/4/b.jpg
+
+...moves the top-level folders to the bottom in a new output folder, i.e., creates:
+
+* 1/2/A/a.jpg
+* 3/4/B/b.jpg
+
+In practice, this is used to make this:
+
+animal/camera01/image01.jpg
+
+...look like:
+
+camera01/animal/image01.jpg
+
+"""
 
 #%% Constants and imports
 
@@ -35,31 +35,46 @@ from tqdm import tqdm
 from functools import partial
 from multiprocessing.pool import ThreadPool
 
+from md_utils.path_utils import path_is_abs
+
+
+#%% Classes
+
 class TopFoldersToBottomOptions:
+    """
+    Options used to parameterize top_folders_to_bottom()
+    """
     
     def __init__(self,input_folder,output_folder,copy=True,n_threads=1):
+        
+        #: Whether to copy (True) vs. move (False) false when re-organizing
         self.copy = copy
+        
+        #: Number of worker threads to use, or <1 to disable parallelization
         self.n_threads = n_threads
+        
+        #: Input folder
         self.input_folder = input_folder
+        
+        #: Output folder
         self.output_folder = output_folder
-        self.overwrite = False
         
+        #: If this is False and an output file exists, throw an error
+        self.overwrite = False
         
-#%% Support functions
-
-def path_is_abs(p): return (len(p) > 1) and (p[0] == '/' or p[1] == ':')
-
 
 #%% Main functions
 
-def process_file(relative_filename,options,execute=True):
+def _process_file(relative_filename,options,execute=True):
     
-    assert ('/' in relative_filename) and ('\\' not in relative_filename) and (not path_is_abs(relative_filename))
+    assert ('/' in relative_filename) and \
+        ('\\' not in relative_filename) and \
+        (not path_is_abs(relative_filename))
     
     # Find top-level folder
     tokens = relative_filename.split('/')
-    top_level_folder = tokens.pop(0)
-    tokens.insert(len(tokens)-1,top_level_folder)
+    topmost_folder = tokens.pop(0)
+    tokens.insert(len(tokens)-1,topmost_folder)
     
     # Find file/folder names
     output_relative_path = '/'.join(tokens)
@@ -86,11 +101,35 @@ def process_file(relative_filename,options,execute=True):
 
     return output_absolute_path
     
-# ...def process_file()
+# ...def _process_file()
 
 
 def top_folders_to_bottom(options):
-    
+    """
+    top_folders_to_bottom.py
+
+    Given a base folder with files like:
+
+    * A/1/2/a.jpg
+    * B/3/4/b.jpg
+
+    ...moves the top-level folders to the bottom in a new output folder, i.e., creates:
+
+    * 1/2/A/a.jpg
+    * 3/4/B/b.jpg
+
+    In practice, this is used to make this:
+
+    animal/camera01/image01.jpg
+
+    ...look like:
+
+    camera01/animal/image01.jpg
+
+    Args:
+        options (TopFoldersToBottomOptions): See TopFoldersToBottomOptions for parameter details.
+
+    """
     os.makedirs(options.output_folder,exist_ok=True)
     
     # Enumerate input folder
@@ -112,7 +151,7 @@ def top_folders_to_bottom(options):
         relative_files = [s for s in relative_files if '/' in s]
     
     # Make sure each input file maps to a unique output file
-    absolute_output_files = [process_file(s, options, execute=False) for s in relative_files]
+    absolute_output_files = [_process_file(s, options, execute=False) for s in relative_files]
     assert len(absolute_output_files) == len(set(absolute_output_files)),\
         "Error: input filenames don't map to unique output filenames"
         
@@ -122,13 +161,13 @@ def top_folders_to_bottom(options):
     if options.n_threads <= 1:
         
         for relative_filename in tqdm(relative_files):
-            process_file(relative_filename,options)
+            _process_file(relative_filename,options)
     
     else:
         
         print('Starting a pool with {} threads'.format(options.n_threads))
         pool = ThreadPool(options.n_threads)
-        process_file_with_options = partial(process_file, options=options)
+        process_file_with_options = partial(_process_file, options=options)
         _ = list(tqdm(pool.imap(process_file_with_options, relative_files), total=len(relative_files)))
 
 # ...def top_folders_to_bottom()        
@@ -180,10 +219,5 @@ def main():
     
     top_folders_to_bottom(options)
     
-    
-if __name__ == '__main__':
-    
+if __name__ == '__main__':    
     main()
-    
-
-    

api/synchronous/api_core/animal_detection_api/api_backend.py

@@ -150,5 +150,3 @@ if __name__ == '__main__':
     print('\n')
 
     detect_process() 
-
-