megadetector 5.0.27__py3-none-any.whl → 5.0.29__py3-none-any.whl

megadetector/data_management/lila/get_lila_annotation_counts.py

@@ -3,7 +3,7 @@
 get_lila_annotation_counts.py
 
 Generates a .json-formatted dictionary mapping each LILA dataset to all categories
-that exist for that dataset, with counts for the number of occurrences of each category 
+that exist for that dataset, with counts for the number of occurrences of each category
 (the number of *annotations* for each category, not the number of *images*).
 
 Also loads the taxonomy mapping file, to include scientific names for each category.
@@ -17,8 +17,11 @@ get_lila_image_counts.py counts the number of *images* for each category in each
 import json
 import os
 
+from collections import defaultdict
+
 from megadetector.data_management.lila.lila_common import \
     read_lila_metadata, read_metadata_file_for_dataset, read_lila_taxonomy_mapping
+from megadetector.utils import ct_utils
 
 # cloud provider to use for downloading images; options are 'gcp', 'azure', or 'aws'
 preferred_cloud = 'gcp'
@@ -53,21 +56,21 @@ datasets_with_taxonomy_mapping = set()
 
 # i_row = 1; row = taxonomy_df.iloc[i_row]; row
 for i_row,row in taxonomy_df.iterrows():
-    
+
     datasets_with_taxonomy_mapping.add(row['dataset_name'])
-    
+
     ds_query = row['dataset_name'] + ':' + row['query']
     ds_query = ds_query.lower()
-    
+
     if not isinstance(row['scientific_name'],str):
         unmapped_queries.add(ds_query)
         ds_query_to_scientific_name[ds_query] = 'unmapped'
         continue
-        
+
     ds_query_to_scientific_name[ds_query] = row['scientific_name']
-    
+
 print('Loaded taxonomy mappings for {} datasets'.format(len(datasets_with_taxonomy_mapping)))
-    
+
 
 #%% Download and parse the metadata file
 
@@ -78,7 +81,7 @@ print('Loaded metadata URLs for {} datasets'.format(len(metadata_table)))
 
 #%% Download and extract metadata for each dataset
 
-for ds_name in metadata_table.keys():    
+for ds_name in metadata_table.keys():
     metadata_table[ds_name]['json_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
                                                                          metadata_dir=metadata_dir,
                                                                          metadata_table=metadata_table)
@@ -88,45 +91,43 @@ for ds_name in metadata_table.keys():
 
 # Takes ~5 minutes
 
-from collections import defaultdict
-
 dataset_to_categories = {}
 
 # ds_name = 'NACTI'
 for ds_name in metadata_table.keys():
-    
+
     taxonomy_mapping_available = (ds_name in datasets_with_taxonomy_mapping)
-    
+
     if not taxonomy_mapping_available:
         print('Warning: taxonomy mapping not available for {}'.format(ds_name))
-        
+
     print('Finding categories in {}'.format(ds_name))
 
     json_filename = metadata_table[ds_name]['json_filename']
     base_url = metadata_table[ds_name]['image_base_url_' + preferred_cloud]
     assert not base_url.endswith('/')
-    
-    # Open the metadata file    
+
+    # Open the metadata file
     with open(json_filename, 'r') as f:
         data = json.load(f)
-    
+
     # Collect list of categories and mappings to category name
     categories = data['categories']
-    
+
     category_id_to_count = defaultdict(int)
-    annotations = data['annotations']    
-    
+    annotations = data['annotations']
+
     # ann = annotations[0]
     for ann in annotations:
         category_id_to_count[ann['category_id']] = category_id_to_count[ann['category_id']] + 1
-    
+
     # c = categories[0]
     for c in categories:
-       count = category_id_to_count[c['id']] 
+       count = category_id_to_count[c['id']]
        if 'count' in c:
-           assert 'bbox' in ds_name or c['count'] == count       
+           assert 'bbox' in ds_name or c['count'] == count
        c['count'] = count
-       
+
        # Don't do taxonomy mapping for bbox data sets, which are sometimes just binary and are
        # always redundant with the class-level data sets.
        if 'bbox' in ds_name:
@@ -144,7 +145,7 @@ for ds_name in metadata_table.keys():
                sn = ds_query_to_scientific_name[taxonomy_query_string]
                assert sn is not None and len(sn) > 0
                c['scientific_name_from_taxonomy_mapping'] = sn
-    
+
     dataset_to_categories[ds_name] = categories
 
 # ...for each dataset
@@ -154,19 +155,18 @@ for ds_name in metadata_table.keys():
 
 # ds_name = list(dataset_to_categories.keys())[0]
 for ds_name in dataset_to_categories:
-    
+
     print('\n** Category counts for {} **\n'.format(ds_name))
-    
+
     categories = dataset_to_categories[ds_name]
     categories = sorted(categories, key=lambda x: x['count'], reverse=True)
-    
+
     for c in categories:
         print('{} ({}): {}'.format(c['name'],c['scientific_name_from_taxonomy_mapping'],c['count']))
-        
+
 # ...for each dataset
 
 
 #%% Save the results
 
-with open(output_file, 'w') as f:
-    json.dump(dataset_to_categories,f,indent=1)
+ct_utils.write_json(output_file, dataset_to_categories)

megadetector/data_management/lila/get_lila_image_counts.py

@@ -5,7 +5,7 @@ get_lila_image_counts.py
 Count the number of images and bounding boxes with each label in one or more LILA datasets.
 
 This script doesn't write these counts out anywhere other than the console, it's just intended
-as a template for doing operations like this on LILA data.  get_lila_annotation_counts.py writes 
+as a template for doing operations like this on LILA data.  get_lila_annotation_counts.py writes
 information out to a .json file, but it counts *annotations*, not *images*, for each category.
 
 """
@@ -40,53 +40,53 @@ metadata_table = read_lila_metadata(metadata_dir)
 if datasets_of_interest is None:
     datasets_of_interest = list(metadata_table.keys())
 
-for ds_name in datasets_of_interest:    
+for ds_name in datasets_of_interest:
     metadata_table[ds_name]['json_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
                                                                          metadata_dir=metadata_dir,
                                                                          metadata_table=metadata_table)
-    
-    
+
+
 #%% Count categories
 
 ds_name_to_category_counts = {}
 
 # ds_name = datasets_of_interest[0]
 for ds_name in datasets_of_interest:
-    
+
     category_to_image_count = {}
     category_to_bbox_count = {}
-    
+
     print('Counting categories in: ' + ds_name)
-    
+
     json_filename = metadata_table[ds_name]['json_filename']
     with open(json_filename, 'r') as f:
         data = json.load(f)
-    
+
     categories = data['categories']
     category_ids = [c['id'] for c in categories]
     for c in categories:
         category_id_to_name = {c['id']:c['name'] for c in categories}
     annotations = data['annotations']
     images = data['images']
-    
-    for category_id in category_ids:        
-        category_name = category_id_to_name[category_id]        
+
+    for category_id in category_ids:
+        category_name = category_id_to_name[category_id]
         category_to_image_count[category_name] = 0
         category_to_bbox_count[category_name] = 0
-        
+
     image_id_to_category_names = defaultdict(set)
-    
+
     # Go through annotations, marking each image with the categories that are present
     #
     # ann = annotations[0]
     for ann in annotations:
-        
+
         category_name = category_id_to_name[ann['category_id']]
         image_id_to_category_names[ann['image_id']].add(category_name)
 
     # Now go through images and count categories
     category_to_count = defaultdict(int)
-    
+
     # im = images[0]
     for im in images:
         categories_this_image = image_id_to_category_names[im['id']]
@@ -94,19 +94,19 @@ for ds_name in datasets_of_interest:
             category_to_count[category_name] += 1
 
     ds_name_to_category_counts[ds_name] = category_to_count
-    
+
 # ...for each dataset
-    
+
 
 #%% Print the results
 
 for ds_name in ds_name_to_category_counts:
-    
+
     print('\n** Category counts for {} **\n'.format(ds_name))
-    
+
     category_to_count = ds_name_to_category_counts[ds_name]
-    category_to_count = {k: v for k, v in sorted(category_to_count.items(), reverse=True, 
+    category_to_count = {k: v for k, v in sorted(category_to_count.items(), reverse=True,
                                                  key=lambda item: item[1])}
-    
-    for category_name in category_to_count.keys():        
+
+    for category_name in category_to_count.keys():
         print('{}: {}'.format(category_name,category_to_count[category_name]))

megadetector/data_management/lila/lila_common.py

@@ -53,30 +53,30 @@ for url in lila_base_urls.values():
 def read_wildlife_insights_taxonomy_mapping(metadata_dir, force_download=False):
     """
     Reads the WI taxonomy mapping file, downloading the .json data (and writing to .csv) if necessary.
-    
+
     Args:
         metadata_dir (str): folder to use for temporary LILA metadata files
-        force_download (bool, optional): download the taxonomy mapping file 
+        force_download (bool, optional): download the taxonomy mapping file
             even if the local file exists.
-        
+
     Returns:
         pd.dataframe: A DataFrame with taxonomy information
     """
-    
+
     wi_taxonomy_csv_path = os.path.join(metadata_dir,wildlife_insights_taxonomy_local_csv_filename)
-    
+
     if os.path.exists(wi_taxonomy_csv_path):
         df = pd.read_csv(wi_taxonomy_csv_path)
     else:
         wi_taxonomy_json_path = os.path.join(metadata_dir,wildlife_insights_taxonomy_local_json_filename)
-        download_url(wildlife_insights_taxonomy_url, wi_taxonomy_json_path, 
+        download_url(wildlife_insights_taxonomy_url, wi_taxonomy_json_path,
                      force_download=force_download)
         with open(wi_taxonomy_json_path,'r') as f:
             d = json.load(f)
-            
+
         # We haven't implemented paging, make sure that's not an issue
         assert d['meta']['totalItems'] < wildlife_insights_page_size
-            
+
         # d['data'] is a list of items that look like:
         """
          {'id': 2000003,
@@ -92,46 +92,46 @@ def read_wildlife_insights_taxonomy_mapping(metadata_dir, force_download=False):
         """
         df = pd.DataFrame(d['data'])
         df.to_csv(wi_taxonomy_csv_path,index=False)
-        
+
     return df
 
-    
+
 def read_lila_taxonomy_mapping(metadata_dir, force_download=False):
     """
     Reads the LILA taxonomy mapping file, downloading the .csv file if necessary.
-    
+
     Args:
         metadata_dir (str): folder to use for temporary LILA metadata files
-        force_download (bool, optional): download the taxonomy mapping file 
-            even if the local file exists.        
-        
+        force_download (bool, optional): download the taxonomy mapping file
+            even if the local file exists.
+
     Returns:
         pd.DataFrame: a DataFrame with one row per identification
     """
-    
+
     p = urlparse(lila_taxonomy_mapping_url)
     taxonomy_filename = os.path.join(metadata_dir,os.path.basename(p.path))
-    download_url(lila_taxonomy_mapping_url, taxonomy_filename, 
+    download_url(lila_taxonomy_mapping_url, taxonomy_filename,
                  force_download=force_download)
-    
+
     df = pd.read_csv(lila_taxonomy_mapping_url)
-    
+
     return df
 
-   
+
 def read_lila_metadata(metadata_dir, force_download=False):
     """
     Reads LILA metadata (URLs to each dataset), downloading the .csv file if necessary.
-    
+
     Args:
         metadata_dir (str): folder to use for temporary LILA metadata files
-        force_download (bool, optional): download the metadata file even if 
+        force_download (bool, optional): download the metadata file even if
             the local file exists.
-        
+
     Returns:
         dict: a dict mapping dataset names (e.g. "Caltech Camera Traps") to dicts
         with keys corresponding to the headers in the .csv file, currently:
-        
+
         - name
         - short_name
         - continent
@@ -153,65 +153,65 @@ def read_lila_metadata(metadata_dir, force_download=False):
         - md_results_with_rde
         - json_filename
     """
-    
+
     # Put the master metadata file in the same folder where we're putting images
     p = urlparse(lila_metadata_url)
     metadata_filename = os.path.join(metadata_dir,os.path.basename(p.path))
     download_url(lila_metadata_url, metadata_filename, force_download=force_download)
-    
+
     df = pd.read_csv(metadata_filename)
-    
+
     records = df.to_dict('records')
-    
+
     # Parse into a table keyed by dataset name
     metadata_table = {}
-    
+
     # r = records[0]
     for r in records:
         if is_empty(r['name']):
             continue
-        
+
         # Convert NaN's to None
         for k in r.keys():
             if is_empty(r[k]):
                 r[k] = None
-                
+
         metadata_table[r['name']] = r
-    
-    return metadata_table    
-    
+
+    return metadata_table
+
 
 def read_lila_all_images_file(metadata_dir, force_download=False):
     """
     Downloads if necessary - then unzips if necessary - the .csv file with label mappings for
     all LILA files, and opens the resulting .csv file as a Pandas DataFrame.
-    
+
     Args:
         metadata_dir (str): folder to use for temporary LILA metadata files
-        force_download (bool, optional): download the metadata file even if 
+        force_download (bool, optional): download the metadata file even if
             the local file exists.
-        
+
     Returns:
         pd.DataFrame: a DataFrame containing one row per identification in a LILA camera trap image
     """
-        
+
     p = urlparse(lila_all_images_url)
     lila_all_images_zip_filename = os.path.join(metadata_dir,os.path.basename(p.path))
     download_url(lila_all_images_url, lila_all_images_zip_filename,
                  force_download=force_download)
-    
+
     with zipfile.ZipFile(lila_all_images_zip_filename,'r') as z:
         files = z.namelist()
     assert len(files) == 1
-    
+
     unzipped_csv_filename = os.path.join(metadata_dir,files[0])
     if not os.path.isfile(unzipped_csv_filename):
         unzip_file(lila_all_images_zip_filename,metadata_dir)
     else:
-        print('{} already unzipped'.format(unzipped_csv_filename))    
-    
+        print('{} already unzipped'.format(unzipped_csv_filename))
+
     df = pd.read_csv(unzipped_csv_filename)
-    
+
     return df
 
 
@@ -223,94 +223,94 @@ def read_metadata_file_for_dataset(ds_name,
                                    force_download=False):
     """
     Downloads if necessary - then unzips if necessary - the .json file for a specific dataset.
-    
+
     Args:
         ds_name (str): the name of the dataset for which you want to retrieve metadata (e.g.
-            "Caltech Camera Traps")        
+            "Caltech Camera Traps")
         metadata_dir (str): folder to use for temporary LILA metadata files
         metadata_table (dict, optional): an optional dictionary already loaded via
             read_lila_metadata()
         json_url (str, optional): the URL of the metadata file, if None will be retrieved
             via read_lila_metadata()
         preferred_cloud (str, optional): 'gcp' (default), 'azure', or 'aws'
-        force_download (bool, optional): download the metadata file even if 
+        force_download (bool, optional): download the metadata file even if
             the local file exists.
-        
+
     Returns:
         str: the .json filename on the local disk
-    
+
     """
-    
+
     assert preferred_cloud in lila_base_urls.keys()
-    
+
     if json_url is None:
-        
+
         if metadata_table is None:
             metadata_table = read_lila_metadata(metadata_dir)
-            
+
         json_url = metadata_table[ds_name]['metadata_url_' + preferred_cloud]
-    
+
     p = urlparse(json_url)
     json_filename = os.path.join(metadata_dir,os.path.basename(p.path))
     download_url(json_url, json_filename, force_download=force_download)
-    
+
     # Unzip if necessary
     if json_filename.endswith('.zip'):
-        
+
         with zipfile.ZipFile(json_filename,'r') as z:
             files = z.namelist()
         assert len(files) == 1
         unzipped_json_filename = os.path.join(metadata_dir,files[0])
         if not os.path.isfile(unzipped_json_filename):
-            unzip_file(json_filename,metadata_dir)        
+            unzip_file(json_filename,metadata_dir)
         else:
             print('{} already unzipped'.format(unzipped_json_filename))
         json_filename = unzipped_json_filename
-    
+
     return json_filename
 
 
 #%% Interactive test driver
 
 if False:
-    
+
     pass
 
     #%% Verify that all base URLs exist
-    
+
     # LILA camera trap primary metadata file
     urls = (lila_metadata_url,
             lila_taxonomy_mapping_url,
             lila_all_images_url,
             wildlife_insights_taxonomy_url)
-    
+
     from megadetector.utils import url_utils
-    
+
     status_codes = url_utils.test_urls(urls,timeout=2.0)
     assert all([code == 200 for code in status_codes])
-    
-    
+
+
     #%% Verify that the metadata URLs exist for individual datasets
-    
+
     metadata_dir = os.path.expanduser('~/lila/metadata')
-    
+
     dataset_metadata = read_lila_metadata(metadata_dir)
-    
+
     urls_to_test = []
-    
+
     # ds_name = next(iter(dataset_metadata.keys()))
     for ds_name in dataset_metadata.keys():
-        
+
         ds_info = dataset_metadata[ds_name]
         for cloud_name in lila_base_urls.keys():
             urls_to_test.append(ds_info['metadata_url_' + cloud_name])
-            if ds_info['bbox_url_relative'] != None:
+            if ds_info['bbox_url_relative'] is not None:
                 urls_to_test.append(ds_info['bbox_url_' + cloud_name])
-            
+
     status_codes = url_utils.test_urls(urls_to_test,
                                        error_on_failure=True,
                                        n_workers=10,
                                        pool_type='process',
                                        timeout=2.0)
     assert all([code == 200 for code in status_codes])
-    
+

megadetector/data_management/lila/test_lila_metadata_urls.py

@@ -2,7 +2,7 @@
 
 test_lila_metadata_urls.py
 
-Test that all the metadata URLs for LILA camera trap datasets are valid, including MegaDetector 
+Test that all the metadata URLs for LILA camera trap datasets are valid, including MegaDetector
 results files.
 
 Also pick an arbitrary image from each dataset and make sure that URL is valid.
@@ -18,6 +18,7 @@ import os
 
 from megadetector.data_management.lila.lila_common import read_lila_metadata,\
     read_metadata_file_for_dataset, read_lila_taxonomy_mapping
+from megadetector.utils.url_utils import test_urls
 
 # We'll write images, metadata downloads, and temporary files here
 lila_local_base = os.path.expanduser('~/lila')
@@ -55,7 +56,7 @@ print('Loaded metadata URLs for {} datasets'.format(len(metadata_table)))
 
 # Takes ~60 seconds if everything needs to be downloaded and unzipped
 
-for ds_name in metadata_table.keys():    
+for ds_name in metadata_table.keys():
 
     # Download the main metadata file for this dataset
     metadata_table[ds_name]['json_filename'] = \
@@ -63,7 +64,7 @@ for ds_name in metadata_table.keys():
                                        metadata_dir=metadata_dir,
                                        metadata_table=metadata_table,
                                        force_download=force_download)
-        
+
     # Download MD results for this dataset
     for k in md_results_keys:
         md_results_url = metadata_table[ds_name][k]
@@ -93,13 +94,13 @@ image_index = 2000
 #
 # ds_name = list(metadata_table.keys())[0]
 for ds_name in metadata_table.keys():
-    
+
     if 'bbox' in ds_name:
         print('Skipping bbox dataset {}'.format(ds_name))
         continue
 
     print('Processing dataset {}'.format(ds_name))
-    
+
     json_filename = metadata_table[ds_name]['json_filename']
     with open(json_filename, 'r') as f:
         data = json.load(f)
@@ -108,20 +109,20 @@ for ds_name in metadata_table.keys():
         clouds = [preferred_cloud]
     else:
         clouds = ['gcp','aws','azure']
-        
+
     for cloud in clouds:
-        
+
         image_base_url = metadata_table[ds_name]['image_base_url_' + cloud]
         assert not image_base_url.endswith('/')
-        
+
         # Download a test image
         test_image_relative_path = data['images'][image_index]['file_name']
         test_image_url = image_base_url + '/' + test_image_relative_path
-        
+
         url_to_source[test_image_url] = ds_name + ' metadata ({})'.format(cloud)
-    
+
     # Grab an image from the MegaDetector results
-    
+
     # k = md_results_keys[2]
     for k in md_results_keys:
         k_fn = k + '_filename'
@@ -133,14 +134,12 @@ for ds_name in metadata_table.keys():
                 url_to_source[md_image_url] = ds_name + ' ' + k
             del md_results
     del data
-        
+
 # ...for each dataset
 
 
 #%% Test URLs
 
-from megadetector.utils.url_utils import test_urls
-
 urls_to_test = sorted(url_to_source.keys())
 urls_to_test = [fn.replace('\\','/') for fn in urls_to_test]