megadetector 5.0.8__py3-none-any.whl → 5.0.10__py3-none-any.whl

taxonomy_mapping/retrieve_sample_image.py

@@ -1,15 +1,15 @@
-########
-#
-# retrieve_sample_image.py
-#
-# Downloader that retrieves images from Google images, used for verifying taxonomy
-# lookups and looking for egregious mismappings (e.g., "snake" being mapped to a fish called
-# "snake").
-#
-# Simple wrapper around simple_image_download, but I've had to swap in and out the underlying
-# downloader a few times.
-#
-########
+"""
+
+retrieve_sample_image.py
+
+Downloader that retrieves images from Google images, used for verifying taxonomy
+lookups and looking for egregious mismappings (e.g., "snake" being mapped to a fish called
+"snake").
+
+Simple wrapper around simple_image_download, but I've had to swap in and out the underlying
+downloader a few times.
+
+"""
 
 #%% Imports and environment
 

taxonomy_mapping/simple_image_download.py

@@ -1,14 +1,14 @@
-########
-#
-# simple_image_download.py
-#
-# Web image downloader, used in preview_lila_taxonomy.py
-#
-# Slightly modified from:
-#
-# https://github.com/RiddlerQ/simple_image_download
-#
-########
+"""
+
+simple_image_download.py
+
+Web image downloader, used in preview_lila_taxonomy.py
+
+Slightly modified from:
+
+https://github.com/RiddlerQ/simple_image_download
+
+"""
 
 #%% Imports
 

taxonomy_mapping/species_lookup.py

@@ -1,13 +1,13 @@
-########
-#
-# species_lookup.py
-#
-# Look up species names (common or scientific) in the GBIF and iNaturalist
-# taxonomies.
-#
-# Run initialize_taxonomy_lookup() before calling any other function.
-#
-########
+"""
+
+species_lookup.py
+
+Look up species names (common or scientific) in the GBIF and iNaturalist
+taxonomies.
+
+Run initialize_taxonomy_lookup() before calling any other function.
+
+"""
 
 #%% Constants and imports
 

taxonomy_mapping/taxonomy_csv_checker.py

@@ -1,21 +1,21 @@
-########
-#
-# taxonomy_csv_checker.py
-#
-# Checks the taxonomy CSV file to make sure that for each row:
-#
-# 1) The 'taxonomy_level' column matches the lowest-level taxon level in the
-#     'taxonomy_string' column.
-#
-# 2) The 'scientific_name' column matches the scientific name from the
-#     lowest-level taxon level in the 'taxonomy_string' column.
-# 
-# Prints out any mismatches.
-# 
-# Also prints out nodes that have 2 ambiguous parents. See "CASE 2" from the
-# module docstring of taxonomy_graph.py.
-#
-########
+"""
+
+taxonomy_csv_checker.py
+
+Checks the taxonomy CSV file to make sure that for each row:
+
+1) The 'taxonomy_level' column matches the lowest-level taxon level in the
+    'taxonomy_string' column.
+
+2) The 'scientific_name' column matches the scientific name from the
+    lowest-level taxon level in the 'taxonomy_string' column.
+
+Prints out any mismatches.
+
+Also prints out nodes that have 2 ambiguous parents. See "CASE 2" from the
+module docstring of taxonomy_graph.py.
+
+"""
 
 #%% Imports
 

taxonomy_mapping/taxonomy_graph.py

@@ -1,50 +1,50 @@
-########
-# 
-# taxonomy_graph.py
-#
-# Methods for transforming taxonomy CSV into a graph structure backed by
-# NetworkX.
-# 
-# We treat each taxon in the taxonomy as a node in a graph, represented by the
-# TaxonNode class. We use a NetworkX directed graph (nx.DiGraph) to keep track of
-# the edges (parent-child relationships) between the nodes.
-# 
-# In theory, the true biological taxonomy graph should be a tree, where every
-# taxon node has exactly 1 parent. However, because we use both GBIF and INAT
-# taxonomies, there are 2 situations where a taxon node ends up with two parents.
-# Thus, the graph is actually a "directed acyclic graph" (DAG) instead of a tree.
-# 
-# The two situations are explained in detail below. This module includes a
-# function dag_to_tree() which converts a DAG to a tree by heuristically removing
-# edges from the DAG so that each node only has 1 parent.
-# 
-# CASE 1: INAT and GBIF have different granularity in their taxonomy levels
-# ======
-# An example is shown below. In dag_to_tree(), the lower parent is kept, while
-# the higher-up parent is discarded. In this example, the "sciurini -> sciurus"
-# edge would be kept, while "sciuridae -> sciurus" would be removed.
-# 
-#         "eastern gray squirrel" (inat)     "squirrel" (gbif)
-#         ------------------------------     -----------------
-#     family:                        sciuridae
-#                                   /          \
-#     subfamily:          sciurinae             |  # skips subfamily
-#                                 |             |
-#     tribe:               sciurini             |  # skips tribe
-#                                   \          /
-#     genus:                          sciurus
-# 
-# 
-# CASE 2: INAT and GBIF have different taxonomies
-# ======
-# An example is shown below. In dag_to_tree(), the resolution to these
-# discrepancies are hard-coded.
-# 
-#     order:    cathartiformes (inat)     accipitriformes (gbif)
-#                            \           /
-#     family:                 cathartidae
-#
-########
+"""
+
+taxonomy_graph.py
+
+Methods for transforming taxonomy CSV into a graph structure backed by
+NetworkX.
+
+We treat each taxon in the taxonomy as a node in a graph, represented by the
+TaxonNode class. We use a NetworkX directed graph (nx.DiGraph) to keep track of
+the edges (parent-child relationships) between the nodes.
+
+In theory, the true biological taxonomy graph should be a tree, where every
+taxon node has exactly 1 parent. However, because we use both GBIF and INAT
+taxonomies, there are 2 situations where a taxon node ends up with two parents.
+Thus, the graph is actually a "directed acyclic graph" (DAG) instead of a tree.
+
+The two situations are explained in detail below. This module includes a
+function dag_to_tree() which converts a DAG to a tree by heuristically removing
+edges from the DAG so that each node only has 1 parent.
+
+CASE 1: INAT and GBIF have different granularity in their taxonomy levels
+======
+An example is shown below. In dag_to_tree(), the lower parent is kept, while
+the higher-up parent is discarded. In this example, the "sciurini -> sciurus"
+edge would be kept, while "sciuridae -> sciurus" would be removed.
+
+        "eastern gray squirrel" (inat)     "squirrel" (gbif)
+        ------------------------------     -----------------
+    family:                        sciuridae
+                                  /          \
+    subfamily:          sciurinae             |  # skips subfamily
+                                |             |
+    tribe:               sciurini             |  # skips tribe
+                                  \          /
+    genus:                          sciurus
+
+
+CASE 2: INAT and GBIF have different taxonomies
+======
+An example is shown below. In dag_to_tree(), the resolution to these
+discrepancies are hard-coded.
+
+    order:    cathartiformes (inat)     accipitriformes (gbif)
+                           \           /
+    family:                 cathartidae
+
+"""
 
 #%% Imports and constants
 

taxonomy_mapping/validate_lila_category_mappings.py

@@ -1,76 +1,83 @@
-########
-#
-# validate_lila_category_mappings.py
-#
-# Confirm that all category names on LILA have mappings in the taxonomy file.
-#
-########
-
-#%% Constants and imports
-
-import json
-import os
-
-from data_management.lila.lila_common import read_lila_taxonomy_mapping
-
-lila_local_base = os.path.expanduser('~/lila')
-
-metadata_dir = os.path.join(lila_local_base,'metadata')
-os.makedirs(metadata_dir,exist_ok=True)
-
-# Created by get_lila_category_list.py... contains counts for each category
-category_list_dir = os.path.join(lila_local_base,'lila_categories_list')
-lila_dataset_to_categories_file = os.path.join(category_list_dir,'lila_dataset_to_categories.json')
-
-assert os.path.isfile(lila_dataset_to_categories_file)
-    
-
-#%% Load category and taxonomy files
-
-with open(lila_dataset_to_categories_file,'r') as f:
-    lila_dataset_to_categories = json.load(f)
-
-taxonomy_df = read_lila_taxonomy_mapping(metadata_dir)
-
-
-#%% Map dataset names and category names to scientific names
-
-ds_query_to_scientific_name = {}
-
-unmapped_queries = set()
-
-# i_row = 1; row = taxonomy_df.iloc[i_row]; row
-for i_row,row in taxonomy_df.iterrows():
-    
-    ds_query = row['dataset_name'] + ':' + row['query']
-    ds_query = ds_query.lower()
-    
-    if not isinstance(row['scientific_name'],str):
-        unmapped_queries.add(ds_query)
-        ds_query_to_scientific_name[ds_query] = 'unmapped'
-        continue
-        
-    ds_query_to_scientific_name[ds_query] = row['scientific_name']
-    
-    
-#%% For each dataset, make sure we can map every category to the taxonomy
-
-# dataset_name = list(lila_dataset_to_categories.keys())[0]
-for _dataset_name in lila_dataset_to_categories.keys():
-    
-    if '_bbox' in _dataset_name:
-        dataset_name = _dataset_name.replace('_bbox','')
-    else:
-        dataset_name = _dataset_name
-    
-    categories = lila_dataset_to_categories[dataset_name]
-    
-    # c = categories[0]
-    for c in categories:
-        ds_query = dataset_name + ':' + c['name']
-        ds_query = ds_query.lower()
-        
-        if ds_query not in ds_query_to_scientific_name:
-            print('Could not find mapping for {}'.format(ds_query))            
-        else:
-            scientific_name = ds_query_to_scientific_name[ds_query]
+"""
+
+validate_lila_category_mappings.py
+
+Confirm that all category names on LILA have mappings in the taxonomy file.
+
+"""
+
+#%% Constants and imports
+
+import json
+import os
+
+from data_management.lila.lila_common import read_lila_taxonomy_mapping
+
+
+#%% Prevent execution during infrastructural imports
+
+if False:
+        
+    #%% Constants
+    
+    lila_local_base = os.path.expanduser('~/lila')
+
+    metadata_dir = os.path.join(lila_local_base,'metadata')
+    os.makedirs(metadata_dir,exist_ok=True)
+
+    # Created by get_lila_category_list.py... contains counts for each category
+    category_list_dir = os.path.join(lila_local_base,'lila_categories_list')
+    lila_dataset_to_categories_file = os.path.join(category_list_dir,'lila_dataset_to_categories.json')
+
+    assert os.path.isfile(lila_dataset_to_categories_file)
+        
+
+    #%% Load category and taxonomy files
+
+    with open(lila_dataset_to_categories_file,'r') as f:
+        lila_dataset_to_categories = json.load(f)
+
+    taxonomy_df = read_lila_taxonomy_mapping(metadata_dir)
+
+
+    #%% Map dataset names and category names to scientific names
+
+    ds_query_to_scientific_name = {}
+
+    unmapped_queries = set()
+
+    # i_row = 1; row = taxonomy_df.iloc[i_row]; row
+    for i_row,row in taxonomy_df.iterrows():
+        
+        ds_query = row['dataset_name'] + ':' + row['query']
+        ds_query = ds_query.lower()
+        
+        if not isinstance(row['scientific_name'],str):
+            unmapped_queries.add(ds_query)
+            ds_query_to_scientific_name[ds_query] = 'unmapped'
+            continue
+            
+        ds_query_to_scientific_name[ds_query] = row['scientific_name']
+        
+        
+    #%% For each dataset, make sure we can map every category to the taxonomy
+
+    # dataset_name = list(lila_dataset_to_categories.keys())[0]
+    for _dataset_name in lila_dataset_to_categories.keys():
+        
+        if '_bbox' in _dataset_name:
+            dataset_name = _dataset_name.replace('_bbox','')
+        else:
+            dataset_name = _dataset_name
+        
+        categories = lila_dataset_to_categories[dataset_name]
+        
+        # c = categories[0]
+        for c in categories:
+            ds_query = dataset_name + ':' + c['name']
+            ds_query = ds_query.lower()
+            
+            if ds_query not in ds_query_to_scientific_name:
+                print('Could not find mapping for {}'.format(ds_query))            
+            else:
+                scientific_name = ds_query_to_scientific_name[ds_query]

data_management/cct_json_to_filename_json.py

@@ -1,89 +0,0 @@
-########
-#
-# cct_json_to_filename_json.py
-#
-# Given a .json file in COCO Camera Traps format, outputs a .json-formatted list of
-# relative file names present in the CCT file.
-#
-########
-
-#%% Constants and environment
-
-import json
-import sys
-import os
-from itertools import compress
-
-
-#%% Main function
-
-def convertJsonToStringList(inputFilename,outputFilename=None,prepend='',bConfirmExists=False,
-                            bForceForwardSlash=True,imageBase=''):
-
-    assert os.path.isfile(inputFilename), '.json file {} does not exist'.format(inputFilename)
-    if outputFilename is None:
-        outputFilename = inputFilename + '_images.json'
-        
-    with open(inputFilename,'r') as f:
-        data = json.load(f)
-    
-    images = data['images']
-    
-    filenames = [im['file_name'] for im in images]
-    
-    if bConfirmExists:
-        bValid = [False] * len(filenames)
-        for iFile,f in enumerate(filenames):
-            fullPath = os.path.join(imageBase,f)
-            if os.path.isfile(fullPath):
-                bValid[iFile] = True
-        nFilesTotal = len(filenames)
-        filenames = list(compress(filenames, bValid))
-        nFilesValid = len(filenames)
-        print('Marking {} of {} as valid'.format(nFilesValid,nFilesTotal))
-
-    filenames = [prepend + s for s in filenames]
-    if bForceForwardSlash:
-        filenames = [s.replace('\\','/') for s in filenames]
-        
-    # json.dump(s,open(outputFilename,'w'))
-        
-    s =  json.dumps(filenames)    
-    with open(outputFilename, 'w') as f:
-        f.write(s)
-        
-    return s,outputFilename
-    
-    
-#%% Command-line driver
-
-import argparse
-
-def main():
-    
-    parser = argparse.ArgumentParser()
-    parser.add_argument('inputFilename')
-    parser.add_argument('outputFilename')
-
-    if len(sys.argv[1:]) == 0:
-        parser.print_help()
-        parser.exit()
-    
-    args = parser.parse_args()    
-    convertJsonToStringList(args.jsonFile,args)
-
-
-if __name__ == '__main__':
-    
-    main()
-
-#%% Interactive driver
-
-if False:
-
-    #%%    
-    prepend = '20190430cameratraps/'
-    inputFilename = r"D:\wildlife_data\awc\awc_imageinfo.json"
-    outputFilename = r"D:\wildlife_data\awc\awc_image_list.json"
-    convertJsonToStringList(inputFilename,outputFilename,prepend=prepend,bConfirmExists=True,imageBase=r'D:\wildlife_data\awc')
-    print('Finished converting {} to {}'.format(inputFilename,outputFilename))

data_management/cct_to_csv.py

@@ -1,140 +0,0 @@
-########
-#
-# cct_to_csv.py
-#
-# "Converts" a COCO Camera Traps .json file to .csv, in quotes because 
-# all kinds of assumptions are made here, and if you have a particular .csv
-# format in mind, YMMV.  Most notably, does not include any bounding box information
-# or any non-standard fields that may be present in the .json file.  Does not
-# propagate information about sequence-level vs. image-level annotations.
-#
-# Does not assume access to the images, therefore does not open .jpg files to find
-# datetime information if it's not in the metadata, just writes datetime as 'unknown'.
-#
-########
-
-#%% Imports
-
-import os
-import sys
-import json
-
-from tqdm import tqdm
-from collections import defaultdict 
-
-
-#%% Main function
-
-def cct_to_csv(input_file,output_file=None):
-
-    if output_file is None:
-        output_file = input_file + '.csv'
-
-    ##%% Read input
-    
-    print('Loading input data')
-    
-    with open(input_file,'r') as f:
-        input_data = json.load(f)
-    
-    
-    ##%% Build internal mappings
-    
-    print('Processing input data')    
-    
-    images = input_data['images']
-    
-    category_id_to_name = {cat['id']:cat['name'] for cat in input_data['categories']}
-    
-    image_id_to_class_names = defaultdict(set)
-    
-    annotations = input_data['annotations']
-                             
-    # annotation = annotations[0]
-    for annotation in tqdm(annotations):
-        image_id = annotation['image_id']
-        class_name = annotation['category_id']
-        image_id_to_class_names[image_id].add(
-            category_id_to_name[class_name])
-        
-    
-    ##%% Write output file
-    
-    print('Writing output file')
-    
-    with open(output_file,'w') as f:
-        
-        f.write('relative_path,datetime,location,sequence_id,class_name\n')
-        
-        # im = images[0]
-        for im in tqdm(images):
-            
-            file_name = im['file_name']
-            class_names_set = image_id_to_class_names[im['id']]
-            assert len(class_names_set) > 0
-            
-            if 'datetime' in im:
-                datetime = im['datetime']
-            else:
-                datetime = 'unknown'
-            
-            if 'location' in im:
-                location = im['location']
-            else:
-                location = 'unknown'
-            
-            if 'seq_id' in im:
-                sequence_id = im['seq_id']
-            else:
-                sequence_id = 'unknown'
-            
-            # Write out one line per class:
-            for class_name in class_names_set:
-                f.write('{},{},{},{},{}\n'.format(file_name,
-                   datetime,location,sequence_id,class_name))
-            
-            # ...for each class name
-            
-        # ...for each image
-        
-    # ...with open(output_file)    
-            
-# ...def cct_to_csv
-
-
-#%% Interactive driver
-
-if False:
-
-    #%%
-    
-    input_dir = r"G:\temp\cct-to-csv"
-    files = os.listdir(input_dir)
-    files = [s for s in files if s.endswith('.json')]
-    for fn in files:
-        input_file = os.path.join(input_dir,fn)
-        assert os.path.isfile(input_file)
-        cct_to_csv(input_file)
-
-
-#%% Command-line driver
-
-import argparse
-
-def main():
-
-    parser = argparse.ArgumentParser(description=(
-        '"Convert" a COCO Camera Traps .json file to .csv (read code to see why "convert" is in quotes)'))
-
-    parser.add_argument('input_file', type=str)                        
-    parser.add_argument('--output_file', type=str, default=None)
-
-    if len(sys.argv[1:]) == 0:
-        parser.print_help()
-        parser.exit()
-    
-    args = parser.parse_args()
-    cct_to_csv(args.input_file,args.output_file)
-    
-if __name__ == '__main__':
-    main()