megadetector 5.0.7__py3-none-any.whl → 5.0.9__py3-none-any.whl

data_management/lila/create_lila_blank_set.py

@@ -1,12 +1,16 @@
-########
-#
-# create_lila_blank_set.py
-#
-# Create a folder of blank images sampled from LILA.  We'll aim for diversity, so less-common
-# locations will be oversampled relative to more common locations.  We'll also run MegaDetector
-# to minimize the chance that incorrectly-labeled non-empty images sneak into our blank set.
-#
-########
+"""
+
+create_lila_blank_set.py
+
+Create a folder of blank images sampled from LILA.  We'll aim for diversity, so less-common
+locations will be oversampled relative to more common locations.  We'll also run MegaDetector
+(with manual review) to remove some incorrectly-labeled, not-actually-empty images from our 
+blank set.
+
+We'll store location information for each image in a .json file, so we can split locations
+into train/val in downstream tasks.
+
+"""
 
 #%% Constants and imports
 
@@ -14,7 +18,6 @@ import os
 import random
 import math
 import json
-import shutil
 
 import numpy as np
 from tqdm import tqdm
@@ -22,8 +25,7 @@ from multiprocessing.pool import ThreadPool
 from urllib.parse import urlparse
 from collections import defaultdict
 
-from data_management.lila.lila_common import \
-    read_lila_all_images_file, azure_url_to_gcp_http_url
+from data_management.lila.lila_common import read_lila_all_images_file
 from md_utils.url_utils import download_url
 from md_visualization import visualization_utils as vis_utils
 from md_utils.path_utils import recursive_file_list
@@ -45,6 +47,14 @@ os.makedirs(confirmed_blanks_base,exist_ok=True)
 md_possible_non_blanks_folder = os.path.join(project_base,'candidate_non_blanks')
 os.makedirs(md_possible_non_blanks_folder,exist_ok=True)
 
+location_to_blank_image_urls_cache_file = os.path.join(project_base,
+                                                       'location_to_blank_image_urls.json')
+
+md_results_file = os.path.join(project_base,'lila_blanks_md_results.json')
+
+all_fn_relative_to_location_file = os.path.join(project_base,'all_fn_relative_to_location.json')
+confirmed_fn_relative_to_location_file = os.path.join(project_base,'confirmed_fn_relative_to_location.json')
+
 preferred_image_download_source = 'gcp'
 
 # Number of concurrent download threads
@@ -171,9 +181,6 @@ for s in original_labels_with_nan_common_names:
 
 #%% Map locations to blank images
 
-location_to_blank_image_urls_cache_file = os.path.join(project_base,
-                                                       'location_to_blank_image_urls.json')
-
 force_map_locations = False
 
 # Load from .json if available
@@ -275,7 +282,7 @@ print('Max samples per location: {}'.format(max_blanks_per_location))
 
 #%% Download those image files (prep)
 
-container_to_url_base = { 
+container_to_url_base = {
                          'lilablobssc.blob.core.windows.net':'/',
                          'storage.googleapis.com':'/public-datasets-lila/'
                          }
@@ -318,6 +325,21 @@ def download_relative_filename(url, output_base, verbose=False, url_base=None, o
     result['status'] = 'success'
     return result
 
+def azure_url_to_gcp_http_url(url,error_if_not_azure_url=True):
+    """
+    Most URLs point to Azure by default, but most files are available on both Azure and GCP.
+    This function converts an Azure URL to the corresponding GCP http:// url.
+    """
+    
+    lila_azure_storage_account = 'https://lilablobssc.blob.core.windows.net'
+    gcp_bucket_api_url = 'https://storage.googleapis.com/public-datasets-lila'
+    error_if_not_azure_url = False
+    
+    if error_if_not_azure_url:
+        assert url.startswith(lila_azure_storage_account)
+    gcp_url = url.replace(lila_azure_storage_account,gcp_bucket_api_url,1)
+    return gcp_url
+
 # Convert Azure URLs to GCP URLs if necessary
 if preferred_image_download_source != 'azure':
     assert preferred_image_download_source == 'gcp'
@@ -358,8 +380,6 @@ print('Errors on {} of {} downloads'.format(len(error_urls),len(results)))
 
 #%% Run MegaDetector on the folder
 
-md_results_file = os.path.join(project_base,'lila_blanks_md_results.json')
-
 cmd = 'python run_detector_batch.py MDV5A "{}" "{}"'.format(
     candidate_blanks_base,md_results_file)
 cmd += ' --recursive --output_relative_filenames'
@@ -419,6 +439,7 @@ for i_fn,source_file_relative in tqdm(enumerate(images_to_review_to_detections),
                                           confidence_threshold=min_threshold,
                                           target_size=(1280,-1))
 
+# This is a temporary file I just used during debugging
 with open(os.path.join(project_base,'output_file_to_source_file.json'),'w') as f:
     json.dump(output_file_to_source_file,f,indent=1)
     
@@ -442,33 +463,95 @@ for output_file in tqdm(output_file_to_source_file.keys()):
         source_file_relative = output_file_to_source_file[output_file]
         removed_blank_images_relative.append(source_file_relative)
         
+removed_blank_images_relative_set = set(removed_blank_images_relative)
 assert len(removed_blank_images_relative) + len(remaining_images) == len(output_file_to_source_file)
 
 
-#%% Copy all the confirmed blanks to the confirmed folder
+#%% Copy only the confirmed blanks to the confirmed folder
+
+from md_utils.path_utils import is_image_file
 
 all_candidate_blanks = recursive_file_list(candidate_blanks_base,return_relative_paths=True)
 print('Found {} candidate blanks'.format(len(all_candidate_blanks)))
 
+skipped_images_relative = []
+skipped_non_images = []
+
 for source_fn_relative in tqdm(all_candidate_blanks):
+    
+    # Skip anything we removed from the "candidate non-blanks" folder; these weren't really
+    # blank.
+    if source_fn_relative in removed_blank_images_relative_set:
+        skipped_images_relative.append(source_fn_relative)
+        continue
+    
+    if not is_image_file(source_fn_relative):
+        # Not a typo; "skipped images" really means "skipped files"
+        skipped_images_relative.append(source_fn_relative)
+        skipped_non_images.append(source_fn_relative)
+    
+    
     source_fn_abs = os.path.join(candidate_blanks_base,source_fn_relative)
     assert os.path.isfile(source_fn_abs)
     target_fn_abs = os.path.join(confirmed_blanks_base,source_fn_relative)
     os.makedirs(os.path.dirname(target_fn_abs),exist_ok=True)
-    shutil.copyfile(source_fn_abs,target_fn_abs)
+    # shutil.copyfile(source_fn_abs,target_fn_abs)
 
+print('Skipped {} files ({} non-image files)'.format(len(skipped_images_relative),
+                                                     len(skipped_non_images)))
 
-#%% Record location information for each file
 
-fn_relative_to_location = {}
-for location in location_to_blank_image_urls:
-    urls_this_location = location_to_blank_image_urls[location]
-    for url in urls_this_location:
-        fn_relative = url.split('//')[1]
-        fn_relative_to_location[fn_relative] = location
-        
-all_confirmed_blanks = recursive_file_list(confirmed_blanks_base,return_relative_paths=True)
+#%% Validate the folder of confirmed blanks
+
+from md_utils.path_utils import find_images
+# all_confirmed_blanks = recursive_file_list(confirmed_blanks_base,return_relative_paths=True)
+all_confirmed_blanks = find_images(confirmed_blanks_base,return_relative_paths=True,recursive=True)
+assert len(all_confirmed_blanks) < len(all_candidate_blanks)
 print('Found {} confirmed blanks'.format(len(all_confirmed_blanks)))
 
-for fn_relative in all_confirmed_blanks:
-    assert fn_relative in fn_relative_to_location
+
+#%% Manually review a few of the images we skipped
+
+# ...to make sure they're non-blank
+i_image = random.randint(0, len(skipped_images_relative))
+fn_relative = skipped_images_relative[i_image]
+fn_abs = os.path.join(candidate_blanks_base,fn_relative)
+assert os.path.isfile(fn_abs)
+import clipboard
+clipboard.copy('feh --scale-down "{}"'.format(fn_abs))
+
+
+#%% Record location information for each confirmed file
+
+# Map every URL's path to the corresponding location
+#
+# This is *all empty URLs*, not just the ones we downloaded
+all_fn_relative_to_location = {}
+
+# location = next(iter(location_to_blank_image_urls.keys()))
+for location in tqdm(location_to_blank_image_urls):
+    urls_this_location = location_to_blank_image_urls[location]
+    
+    # url = urls_this_location[0]
+    for url in urls_this_location:
+        # Turn:
+        # 
+        # https://lilablobssc.blob.core.windows.net/caltech-unzipped/cct_images/5968c0f9-23d2-11e8-a6a3-ec086b02610b.jpg'
+        #
+        # ...into:
+        #
+        # caltech-unzipped/cct_images/5968c0f9-23d2-11e8-a6a3-ec086b02610b.jpg'   
+        p = urlparse(url)
+        fn_relative = str(p.path)[1:]
+        all_fn_relative_to_location[fn_relative] = location
+
+# Build a much smaller mapping of just the confirmed blanks
+confirmed_fn_relative_to_location = {}        
+for i_fn,fn_relative in tqdm(enumerate(all_confirmed_blanks),total=len(all_confirmed_blanks)):
+    confirmed_fn_relative_to_location[fn_relative] = all_fn_relative_to_location[fn_relative]
+
+with open(all_fn_relative_to_location_file,'w') as f:
+    json.dump(all_fn_relative_to_location,f,indent=1)
+    
+with open(confirmed_fn_relative_to_location_file,'w') as f:
+    json.dump(confirmed_fn_relative_to_location,f,indent=1)    

data_management/lila/create_lila_test_set.py

@@ -1,11 +1,11 @@
-########
-#
-# create_lila_test_set.py
-#
-# Create a test set of camera trap images, containing N empty and N non-empty 
-# images from each LILA data set.
-#
-########
+"""
+
+create_lila_test_set.py
+
+Create a test set of camera trap images, containing N empty and N non-empty 
+images from each LILA data set.
+
+"""
 
 #%% Constants and imports
 

data_management/lila/create_links_to_md_results_files.py

@@ -1,106 +1,106 @@
-########
-#
-# create_links_to_md_results_files.py
-#
-# One-off script to populate the columns in the camera trap data .csv file that point to MD results.
-#
-########
-
-#%% Imports and constants
-
-import os
-
-import pandas as pd
-
-input_csv_file = r'g:\temp\lila_camera_trap_datasets_no_md_results.csv'
-output_csv_file = r'g:\temp\lila_camera_trap_datasets.csv'
-
-md_results_local_folder = r'g:\temp\lila-md-results'
-md_base_url = 'https://lila.science/public/lila-md-results/'
-assert md_base_url.endswith('/')
-
-# No RDE files for datasets with no location information        
-datasets_without_location_info = ('ena24','missouri-camera-traps')
-
-md_results_column_names = ['mdv4_results_raw','mdv5a_results_raw','mdv5b_results_raw','md_results_with_rde']
-
-validate_urls = False
-
-
-#%% Read input data
-
-df = pd.read_csv(input_csv_file)
-for s in md_results_column_names:
-    df[s] = ''
-    
-    
-#%% Find matching files locally, and create URLs
-
-local_files = os.listdir(md_results_local_folder)
-local_files = [fn for fn in local_files if fn.endswith('.zip')]
-
-# i_row = 0; row = df.iloc[i_row]
-for i_row,row in df.iterrows():
-    
-    if not isinstance(row['name'],str):
-        continue
-    
-    dataset_shortname = row['short_name']
-    matching_files = [fn for fn in local_files if dataset_shortname in fn]
-    
-    # No RDE files for datasets with no location information        
-    if dataset_shortname in datasets_without_location_info:
-        assert len(matching_files) == 2
-        mdv5a_files = [fn for fn in matching_files if 'mdv5a' in fn]
-        mdv5b_files = [fn for fn in matching_files if 'mdv5b' in fn]
-        assert len(mdv5a_files) == 1 and len(mdv5b_files) == 1
-        df.loc[i_row,'mdv5a_results_raw'] = md_base_url + mdv5a_files[0]
-        df.loc[i_row,'mdv5b_results_raw'] = md_base_url + mdv5b_files[0]
-    else:
-        # Exclude single-season files for snapshot-serengeti    
-        if dataset_shortname == 'snapshot-serengeti':
-            matching_files = [fn for fn in matching_files if '_S' not in fn]
-            assert len(matching_files) == 2        
-            assert all(['mdv4' in fn for fn in matching_files])
-            rde_files = [fn for fn in matching_files if 'rde' in fn]
-            raw_files = [fn for fn in matching_files if 'rde' not in fn]
-            assert len(rde_files) == 1 and len(raw_files) == 1
-            df.loc[i_row,'mdv4_results_raw'] = md_base_url + raw_files[0]
-            df.loc[i_row,'md_results_with_rde'] = md_base_url + rde_files[0]
-        else:
-            assert len(matching_files) == 3
-            mdv5a_files = [fn for fn in matching_files if 'mdv5a' in fn and 'rde' not in fn]
-            mdv5b_files = [fn for fn in matching_files if 'mdv5b' in fn and 'rde' not in fn]
-            rde_files = [fn for fn in matching_files if 'rde' in fn]
-            assert len(mdv5a_files) == 1 and len(mdv5b_files) == 1 and len(rde_files) == 1
-            df.loc[i_row,'mdv5a_results_raw'] = md_base_url + mdv5a_files[0]
-            df.loc[i_row,'mdv5b_results_raw'] = md_base_url + mdv5b_files[0]
-            df.loc[i_row,'md_results_with_rde'] = md_base_url + rde_files[0]
-            
-    print('Found {} matching files for {}'.format(len(matching_files),dataset_shortname))
-
-# ...for each row    
-
-
-#%% Validate URLs
-
-if validate_urls:
-    
-    from md_utils.url_utils import test_urls
-    
-    urls = set()
-    
-    for i_row,row in df.iterrows():
-        for column_name in md_results_column_names:
-            if len(row[column_name]) > 0:
-                assert row[column_name] not in urls        
-                urls.add(row[column_name])
-                
-    test_urls(urls,error_on_failure=True)            
-    
-    print('Validated {} URLs'.format(len(urls)))
-
-
-#%% Write new .csv file
-
-df.to_csv(output_csv_file,header=True,index=False)
+"""
+
+create_links_to_md_results_files.py
+
+One-off script to populate the columns in the camera trap data .csv file that point to MD results.
+
+"""
+
+#%% Imports and constants
+
+import os
+
+import pandas as pd
+
+input_csv_file = r'g:\temp\lila_camera_trap_datasets_no_md_results.csv'
+output_csv_file = r'g:\temp\lila_camera_trap_datasets.csv'
+
+md_results_local_folder = r'g:\temp\lila-md-results'
+md_base_url = 'https://lila.science/public/lila-md-results/'
+assert md_base_url.endswith('/')
+
+# No RDE files for datasets with no location information        
+datasets_without_location_info = ('ena24','missouri-camera-traps')
+
+md_results_column_names = ['mdv4_results_raw','mdv5a_results_raw','mdv5b_results_raw','md_results_with_rde']
+
+validate_urls = False
+
+
+#%% Read input data
+
+df = pd.read_csv(input_csv_file)
+for s in md_results_column_names:
+    df[s] = ''
+    
+    
+#%% Find matching files locally, and create URLs
+
+local_files = os.listdir(md_results_local_folder)
+local_files = [fn for fn in local_files if fn.endswith('.zip')]
+
+# i_row = 0; row = df.iloc[i_row]
+for i_row,row in df.iterrows():
+    
+    if not isinstance(row['name'],str):
+        continue
+    
+    dataset_shortname = row['short_name']
+    matching_files = [fn for fn in local_files if dataset_shortname in fn]
+    
+    # No RDE files for datasets with no location information        
+    if dataset_shortname in datasets_without_location_info:
+        assert len(matching_files) == 2
+        mdv5a_files = [fn for fn in matching_files if 'mdv5a' in fn]
+        mdv5b_files = [fn for fn in matching_files if 'mdv5b' in fn]
+        assert len(mdv5a_files) == 1 and len(mdv5b_files) == 1
+        df.loc[i_row,'mdv5a_results_raw'] = md_base_url + mdv5a_files[0]
+        df.loc[i_row,'mdv5b_results_raw'] = md_base_url + mdv5b_files[0]
+    else:
+        # Exclude single-season files for snapshot-serengeti    
+        if dataset_shortname == 'snapshot-serengeti':
+            matching_files = [fn for fn in matching_files if '_S' not in fn]
+            assert len(matching_files) == 2        
+            assert all(['mdv4' in fn for fn in matching_files])
+            rde_files = [fn for fn in matching_files if 'rde' in fn]
+            raw_files = [fn for fn in matching_files if 'rde' not in fn]
+            assert len(rde_files) == 1 and len(raw_files) == 1
+            df.loc[i_row,'mdv4_results_raw'] = md_base_url + raw_files[0]
+            df.loc[i_row,'md_results_with_rde'] = md_base_url + rde_files[0]
+        else:
+            assert len(matching_files) == 3
+            mdv5a_files = [fn for fn in matching_files if 'mdv5a' in fn and 'rde' not in fn]
+            mdv5b_files = [fn for fn in matching_files if 'mdv5b' in fn and 'rde' not in fn]
+            rde_files = [fn for fn in matching_files if 'rde' in fn]
+            assert len(mdv5a_files) == 1 and len(mdv5b_files) == 1 and len(rde_files) == 1
+            df.loc[i_row,'mdv5a_results_raw'] = md_base_url + mdv5a_files[0]
+            df.loc[i_row,'mdv5b_results_raw'] = md_base_url + mdv5b_files[0]
+            df.loc[i_row,'md_results_with_rde'] = md_base_url + rde_files[0]
+            
+    print('Found {} matching files for {}'.format(len(matching_files),dataset_shortname))
+
+# ...for each row    
+
+
+#%% Validate URLs
+
+if validate_urls:
+    
+    from md_utils.url_utils import test_urls
+    
+    urls = set()
+    
+    for i_row,row in df.iterrows():
+        for column_name in md_results_column_names:
+            if len(row[column_name]) > 0:
+                assert row[column_name] not in urls        
+                urls.add(row[column_name])
+                
+    test_urls(urls,error_on_failure=True)            
+    
+    print('Validated {} URLs'.format(len(urls)))
+
+
+#%% Write new .csv file
+
+df.to_csv(output_csv_file,header=True,index=False)