megadetector 5.0.28__py3-none-any.whl → 10.0.0__py3-none-any.whl

megadetector/data_management/lila/test_lila_metadata_urls.py

@@ -2,7 +2,7 @@
 
 test_lila_metadata_urls.py
 
-Test that all the metadata URLs for LILA camera trap datasets are valid, including MegaDetector 
+Test that all the metadata URLs for LILA camera trap datasets are valid, including MegaDetector
 results files.
 
 Also pick an arbitrary image from each dataset and make sure that URL is valid.
@@ -18,6 +18,7 @@ import os
 
 from megadetector.data_management.lila.lila_common import read_lila_metadata,\
     read_metadata_file_for_dataset, read_lila_taxonomy_mapping
+from megadetector.utils.url_utils import test_urls
 
 # We'll write images, metadata downloads, and temporary files here
 lila_local_base = os.path.expanduser('~/lila')
@@ -31,7 +32,7 @@ os.makedirs(metadata_dir,exist_ok=True)
 md_results_dir = os.path.join(lila_local_base,'md_results')
 os.makedirs(md_results_dir,exist_ok=True)
 
-md_results_keys = ['mdv4_results_raw','mdv5a_results_raw','mdv5b_results_raw',
+md_results_keys = ['mdv5a_results_raw','mdv5b_results_raw',
                    'md1000-redwood_results_raw','md_results_with_rde']
 
 preferred_cloud = None # 'gcp' # 'azure', 'aws'
@@ -53,19 +54,21 @@ print('Loaded metadata URLs for {} datasets'.format(len(metadata_table)))
 
 #%% Download and extract metadata and MD results for each dataset
 
-# Takes ~60 seconds if everything needs to be downloaded and unzipped
+# Takes ~10 minutes if everything needs to be downloaded and unzipped
 
-for ds_name in metadata_table.keys():    
+for ds_name in metadata_table.keys():
 
     # Download the main metadata file for this dataset
     metadata_table[ds_name]['json_filename'] = \
         read_metadata_file_for_dataset(ds_name=ds_name,
                                        metadata_dir=metadata_dir,
                                        metadata_table=metadata_table,
-                                       force_download=force_download)
-        
+                                       force_download=force_download,
+                                       preferred_cloud=preferred_cloud)
+
     # Download MD results for this dataset
     for k in md_results_keys:
+
         md_results_url = metadata_table[ds_name][k]
         if md_results_url is None:
             metadata_table[ds_name][k + '_filename'] = None
@@ -74,7 +77,10 @@ for ds_name in metadata_table.keys():
                 read_metadata_file_for_dataset(ds_name=ds_name,
                                                metadata_dir=md_results_dir,
                                                json_url=md_results_url,
-                                               force_download=force_download)
+                                               force_download=force_download,
+                                               preferred_cloud=preferred_cloud)
+
+    # ...for each MD results file
 
 # ...for each dataset
 
@@ -93,13 +99,13 @@ image_index = 2000
 #
 # ds_name = list(metadata_table.keys())[0]
 for ds_name in metadata_table.keys():
-    
+
     if 'bbox' in ds_name:
         print('Skipping bbox dataset {}'.format(ds_name))
         continue
 
     print('Processing dataset {}'.format(ds_name))
-    
+
     json_filename = metadata_table[ds_name]['json_filename']
     with open(json_filename, 'r') as f:
         data = json.load(f)
@@ -108,21 +114,23 @@ for ds_name in metadata_table.keys():
         clouds = [preferred_cloud]
     else:
         clouds = ['gcp','aws','azure']
-        
+
     for cloud in clouds:
-        
+
         image_base_url = metadata_table[ds_name]['image_base_url_' + cloud]
         assert not image_base_url.endswith('/')
-        
+
         # Download a test image
         test_image_relative_path = data['images'][image_index]['file_name']
         test_image_url = image_base_url + '/' + test_image_relative_path
-        
+
         url_to_source[test_image_url] = ds_name + ' metadata ({})'.format(cloud)
-    
+
+    # ...for each cloud
+
     # Grab an image from the MegaDetector results
-    
-    # k = md_results_keys[2]
+
+    # k = md_results_keys[0]
     for k in md_results_keys:
         k_fn = k + '_filename'
         if metadata_table[ds_name][k_fn] is not None:
@@ -133,14 +141,12 @@ for ds_name in metadata_table.keys():
                 url_to_source[md_image_url] = ds_name + ' ' + k
             del md_results
     del data
-        
+
 # ...for each dataset
 
 
 #%% Test URLs
 
-from megadetector.utils.url_utils import test_urls
-
 urls_to_test = sorted(url_to_source.keys())
 urls_to_test = [fn.replace('\\','/') for fn in urls_to_test]
 
@@ -154,3 +160,6 @@ for i_url,url in enumerate(urls_to_test):
     if status_codes[i_url] != 200:
         print('Status {} for {} ({})'.format(
             status_codes[i_url],url,url_to_source[url]))
+
+print('Tested {} URLs'.format(len(urls_to_test)))
+