megadetector 5.0.8__py3-none-any.whl → 5.0.10__py3-none-any.whl

data_management/lila/create_links_to_md_results_files.py

@@ -1,106 +1,106 @@
-########
-#
-# create_links_to_md_results_files.py
-#
-# One-off script to populate the columns in the camera trap data .csv file that point to MD results.
-#
-########
-
-#%% Imports and constants
-
-import os
-
-import pandas as pd
-
-input_csv_file = r'g:\temp\lila_camera_trap_datasets_no_md_results.csv'
-output_csv_file = r'g:\temp\lila_camera_trap_datasets.csv'
-
-md_results_local_folder = r'g:\temp\lila-md-results'
-md_base_url = 'https://lila.science/public/lila-md-results/'
-assert md_base_url.endswith('/')
-
-# No RDE files for datasets with no location information        
-datasets_without_location_info = ('ena24','missouri-camera-traps')
-
-md_results_column_names = ['mdv4_results_raw','mdv5a_results_raw','mdv5b_results_raw','md_results_with_rde']
-
-validate_urls = False
-
-
-#%% Read input data
-
-df = pd.read_csv(input_csv_file)
-for s in md_results_column_names:
-    df[s] = ''
-    
-    
-#%% Find matching files locally, and create URLs
-
-local_files = os.listdir(md_results_local_folder)
-local_files = [fn for fn in local_files if fn.endswith('.zip')]
-
-# i_row = 0; row = df.iloc[i_row]
-for i_row,row in df.iterrows():
-    
-    if not isinstance(row['name'],str):
-        continue
-    
-    dataset_shortname = row['short_name']
-    matching_files = [fn for fn in local_files if dataset_shortname in fn]
-    
-    # No RDE files for datasets with no location information        
-    if dataset_shortname in datasets_without_location_info:
-        assert len(matching_files) == 2
-        mdv5a_files = [fn for fn in matching_files if 'mdv5a' in fn]
-        mdv5b_files = [fn for fn in matching_files if 'mdv5b' in fn]
-        assert len(mdv5a_files) == 1 and len(mdv5b_files) == 1
-        df.loc[i_row,'mdv5a_results_raw'] = md_base_url + mdv5a_files[0]
-        df.loc[i_row,'mdv5b_results_raw'] = md_base_url + mdv5b_files[0]
-    else:
-        # Exclude single-season files for snapshot-serengeti    
-        if dataset_shortname == 'snapshot-serengeti':
-            matching_files = [fn for fn in matching_files if '_S' not in fn]
-            assert len(matching_files) == 2        
-            assert all(['mdv4' in fn for fn in matching_files])
-            rde_files = [fn for fn in matching_files if 'rde' in fn]
-            raw_files = [fn for fn in matching_files if 'rde' not in fn]
-            assert len(rde_files) == 1 and len(raw_files) == 1
-            df.loc[i_row,'mdv4_results_raw'] = md_base_url + raw_files[0]
-            df.loc[i_row,'md_results_with_rde'] = md_base_url + rde_files[0]
-        else:
-            assert len(matching_files) == 3
-            mdv5a_files = [fn for fn in matching_files if 'mdv5a' in fn and 'rde' not in fn]
-            mdv5b_files = [fn for fn in matching_files if 'mdv5b' in fn and 'rde' not in fn]
-            rde_files = [fn for fn in matching_files if 'rde' in fn]
-            assert len(mdv5a_files) == 1 and len(mdv5b_files) == 1 and len(rde_files) == 1
-            df.loc[i_row,'mdv5a_results_raw'] = md_base_url + mdv5a_files[0]
-            df.loc[i_row,'mdv5b_results_raw'] = md_base_url + mdv5b_files[0]
-            df.loc[i_row,'md_results_with_rde'] = md_base_url + rde_files[0]
-            
-    print('Found {} matching files for {}'.format(len(matching_files),dataset_shortname))
-
-# ...for each row    
-
-
-#%% Validate URLs
-
-if validate_urls:
-    
-    from md_utils.url_utils import test_urls
-    
-    urls = set()
-    
-    for i_row,row in df.iterrows():
-        for column_name in md_results_column_names:
-            if len(row[column_name]) > 0:
-                assert row[column_name] not in urls        
-                urls.add(row[column_name])
-                
-    test_urls(urls,error_on_failure=True)            
-    
-    print('Validated {} URLs'.format(len(urls)))
-
-
-#%% Write new .csv file
-
-df.to_csv(output_csv_file,header=True,index=False)
+"""
+
+create_links_to_md_results_files.py
+
+One-off script to populate the columns in the camera trap data .csv file that point to MD results.
+
+"""
+
+#%% Imports and constants
+
+import os
+
+import pandas as pd
+
+input_csv_file = r'g:\temp\lila_camera_trap_datasets_no_md_results.csv'
+output_csv_file = r'g:\temp\lila_camera_trap_datasets.csv'
+
+md_results_local_folder = r'g:\temp\lila-md-results'
+md_base_url = 'https://lila.science/public/lila-md-results/'
+assert md_base_url.endswith('/')
+
+# No RDE files for datasets with no location information        
+datasets_without_location_info = ('ena24','missouri-camera-traps')
+
+md_results_column_names = ['mdv4_results_raw','mdv5a_results_raw','mdv5b_results_raw','md_results_with_rde']
+
+validate_urls = False
+
+
+#%% Read input data
+
+df = pd.read_csv(input_csv_file)
+for s in md_results_column_names:
+    df[s] = ''
+    
+    
+#%% Find matching files locally, and create URLs
+
+local_files = os.listdir(md_results_local_folder)
+local_files = [fn for fn in local_files if fn.endswith('.zip')]
+
+# i_row = 0; row = df.iloc[i_row]
+for i_row,row in df.iterrows():
+    
+    if not isinstance(row['name'],str):
+        continue
+    
+    dataset_shortname = row['short_name']
+    matching_files = [fn for fn in local_files if dataset_shortname in fn]
+    
+    # No RDE files for datasets with no location information        
+    if dataset_shortname in datasets_without_location_info:
+        assert len(matching_files) == 2
+        mdv5a_files = [fn for fn in matching_files if 'mdv5a' in fn]
+        mdv5b_files = [fn for fn in matching_files if 'mdv5b' in fn]
+        assert len(mdv5a_files) == 1 and len(mdv5b_files) == 1
+        df.loc[i_row,'mdv5a_results_raw'] = md_base_url + mdv5a_files[0]
+        df.loc[i_row,'mdv5b_results_raw'] = md_base_url + mdv5b_files[0]
+    else:
+        # Exclude single-season files for snapshot-serengeti    
+        if dataset_shortname == 'snapshot-serengeti':
+            matching_files = [fn for fn in matching_files if '_S' not in fn]
+            assert len(matching_files) == 2        
+            assert all(['mdv4' in fn for fn in matching_files])
+            rde_files = [fn for fn in matching_files if 'rde' in fn]
+            raw_files = [fn for fn in matching_files if 'rde' not in fn]
+            assert len(rde_files) == 1 and len(raw_files) == 1
+            df.loc[i_row,'mdv4_results_raw'] = md_base_url + raw_files[0]
+            df.loc[i_row,'md_results_with_rde'] = md_base_url + rde_files[0]
+        else:
+            assert len(matching_files) == 3
+            mdv5a_files = [fn for fn in matching_files if 'mdv5a' in fn and 'rde' not in fn]
+            mdv5b_files = [fn for fn in matching_files if 'mdv5b' in fn and 'rde' not in fn]
+            rde_files = [fn for fn in matching_files if 'rde' in fn]
+            assert len(mdv5a_files) == 1 and len(mdv5b_files) == 1 and len(rde_files) == 1
+            df.loc[i_row,'mdv5a_results_raw'] = md_base_url + mdv5a_files[0]
+            df.loc[i_row,'mdv5b_results_raw'] = md_base_url + mdv5b_files[0]
+            df.loc[i_row,'md_results_with_rde'] = md_base_url + rde_files[0]
+            
+    print('Found {} matching files for {}'.format(len(matching_files),dataset_shortname))
+
+# ...for each row    
+
+
+#%% Validate URLs
+
+if validate_urls:
+    
+    from md_utils.url_utils import test_urls
+    
+    urls = set()
+    
+    for i_row,row in df.iterrows():
+        for column_name in md_results_column_names:
+            if len(row[column_name]) > 0:
+                assert row[column_name] not in urls        
+                urls.add(row[column_name])
+                
+    test_urls(urls,error_on_failure=True)            
+    
+    print('Validated {} URLs'.format(len(urls)))
+
+
+#%% Write new .csv file
+
+df.to_csv(output_csv_file,header=True,index=False)

data_management/lila/download_lila_subset.py

@@ -1,17 +1,11 @@
-########
-#
-# download_lila_subset.py
-#
-# Example of how to download a list of files from LILA, e.g. all the files
-# in a data set corresponding to a particular species.
-#
-# Organizes the downloaded images by dataset.  How you actually want to organize files,
-# what you want to query for, etc., is very application-specific; this is just meant as a 
-# demo.
-#
-# Can download from GCP (all datasets), AWS (all datasets), or Azure (most datasets).
-#
-########
+"""
+
+download_lila_subset.py
+
+Example of how to download a list of files from LILA, e.g. all the files
+in a data set corresponding to a particular species.
+
+"""
 
 #%% Constants and imports
 
@@ -19,11 +13,9 @@ import os
 import random
 
 from tqdm import tqdm
-from multiprocessing.pool import ThreadPool
 from collections import defaultdict
 
 from data_management.lila.lila_common import read_lila_all_images_file, is_empty, lila_base_urls
-from md_utils.url_utils import download_url
 
 for s in lila_base_urls.values():
     assert s.endswith('/')
@@ -43,70 +35,22 @@ os.makedirs(output_dir,exist_ok=True)
 # Number of concurrent download threads
 n_download_threads = 20
 
-verbose = False
-
 max_images_per_dataset = 10 # None
 
-# This impacts the data download, but not the metadata download
-#
-# Setting this to "Azure" really means "Azure if available"; some datasets are
-# not available on Azure.
 preferred_provider = 'gcp' # 'azure', 'gcp', 'aws'
 
 random.seed(0)
 
 
-#%% Support functions
-
-def download_relative_url(relative_url, output_base, provider='gcp', 
-                          verbose=False, overwrite=False):
-    """
-    Download a URL to output_base, preserving the path relative to the common LILA root.
-    """
-    
-    assert not relative_url.startswith('/')
-    
-    # Not all datasets are available on Azure, fall back in these cases.  The decision
-    # to fall back to GCP rather than AWS is arbitrary.
-    if provider == 'azure':
-        nominal_provider = relative_url_to_nominal_provider[relative_url]
-        if nominal_provider != 'azure':
-            if verbose:
-                print('URL {} not available on Azure, falling back to GCP'.format(
-                    relative_url))
-            provider = 'gcp'
-            
-    url = lila_base_urls[provider] + relative_url
-    
-    result = {'status':'unknown','url':url,'destination_filename':None}
-    
-    destination_filename = os.path.join(output_base,relative_url)
-    result['destination_filename'] = destination_filename
-    
-    if ((os.path.isfile(destination_filename)) and (not overwrite)):
-        result['status'] = 'skipped'
-        return result
-    try:
-        download_url(url, destination_filename, verbose=verbose, force_download=overwrite)
-    except Exception as e:
-        print('Warning: error downloading URL {}: {}'.format(
-            url,str(e)))     
-        result['status'] = 'error: {}'.format(str(e))
-        return result
-    
-    result['status'] = 'success'
-    return result
-
-
 #%% Download and open the giant table of image URLs and labels
 
-# ~60 seconds to download, unzip, and open
+# Takes ~60 seconds to download, unzip, and open
 df = read_lila_all_images_file(metadata_dir)
 
 
 #%% Find all the images we want to download
 
-# ~2 minutes
+# Takes ~2 minutes
 
 common_name_to_count = defaultdict(int)
 
@@ -119,6 +63,8 @@ def find_items(row):
     
     match = False
     
+    # This is the only bit of this file that's specific to a particular query.  In this case
+    # we're checking whether each row is on a list of species of interest, but you do you.
     for species_name in species_of_interest:
         if species_name in row['common_name']:
             match = True
@@ -126,7 +72,7 @@ def find_items(row):
             break
     
     if match:
-        ds_name_to_urls[row['dataset_name']].append(row['url'])
+        ds_name_to_urls[row['dataset_name']].append(row['url_' + preferred_provider])
 
 tqdm.pandas()
 _ = df.progress_apply(find_items,axis=1)
@@ -154,58 +100,47 @@ else:
             ds_name_to_urls[ds_name] = random.sample(ds_name_to_urls[ds_name],max_images_per_dataset)
 
 
-#%% Convert URLs to be relative to the common LILA base 
+#%% Choose target files for each URL
 
-all_urls = list(ds_name_to_urls.values())
-all_urls = [item for sublist in all_urls for item in sublist]
+from data_management.lila.lila_common import lila_base_urls
 
-all_urls_relative = []
+# We have a list of URLs per dataset, flatten that into a single list of URLs
+urls_to_download = set()
+for ds_name in ds_name_to_urls:
+    for url in ds_name_to_urls[ds_name]:
+        urls_to_download.add(url)
+urls_to_download = sorted(list(urls_to_download))        
 
-# Each file has a nominal URL in the .csv file.  For now, the only thing this tells is
-# is that if the nominal URL isn't an Azure URL, the file isn't on Azure.  All files are on
-# GCP and AWS.
+# A URL might look like this:
 #
-# Keep track of the nominal provider for each URL.
-relative_url_to_nominal_provider = {}
-
-for url in all_urls:
-    found_base = False
-    for provider in lila_base_urls.keys():
-        base = lila_base_urls[provider]
-        if url.startswith(base):
-            relative_url = url.replace(base,'')
-            all_urls_relative.append(relative_url)
-            relative_url_to_nominal_provider[relative_url] = provider
-            found_base = True
-            break
-    assert found_base
-    
-assert len(all_urls) == len(all_urls_relative)
+# https://storage.googleapis.com/public-datasets-lila/wcs-unzipped/animals/0667/0302.jpg
+# 
+# We'll write that to an output file that looks like this (relative to output_dir):
+#
+# wcs-unzipped/animals/0667/0302.jpg
+#
+# ...so we need to remove the base URL to get the target file.
+base_url = lila_base_urls[preferred_provider]
+assert base_url.endswith('/')
 
+url_to_target_file = {}
 
-#%% Download image files
+for url in urls_to_download:
+    assert url.startswith(base_url) 
+    target_fn_relative = url.replace(base_url,'')
+    target_fn_abs = os.path.join(output_dir,target_fn_relative)
+    url_to_target_file[url] = target_fn_abs
 
-print('Downloading {} images on {} workers, preferred provider is {}'.format(
-    len(all_urls),n_download_threads,preferred_provider))
 
-if n_download_threads <= 1:
+#%% Download image files
 
-    results = []
-    
-    # url_relative = all_urls_relative[0]
-    for url_relative in tqdm(all_urls_relative):        
-        result = download_relative_url(url_relative,
-                                       output_base=output_dir,
-                                       provider=preferred_provider,
-                                       verbose=verbose)
-        results.append(result)
-    
-else:
+from md_utils.url_utils import parallel_download_urls
 
-    pool = ThreadPool(n_download_threads)        
-    results = list(tqdm(pool.imap(lambda s: download_relative_url(
-        s,output_base=output_dir,provider=preferred_provider,verbose=verbose),
-        all_urls_relative), total=len(all_urls_relative)))
+download_results = parallel_download_urls(url_to_target_file=url_to_target_file,
+                                          verbose=False,
+                                          overwrite=False,
+                                          n_workers=n_download_threads,
+                                          pool_type='thread')
 
 
 #%% Scrap
@@ -240,4 +175,3 @@ if False:
     print('\nDatasets by count:\n')    
     for k in dataset_to_count:
         print('{} ({})'.format(k,dataset_to_count[k]))
-    

data_management/lila/generate_lila_per_image_labels.py

@@ -1,19 +1,19 @@
-########
-#
-# generate_lila_per_image_labels.py
-# 
-# Generate a .csv file with one row per annotation, containing full URLs to every
-# camera trap image on LILA, with taxonomically expanded labels.
-#
-# Typically there will be one row per image, though images with multiple annotations
-# will have multiple rows.
-#
-# Some images may not physically exist, particularly images that are labeled as "human".
-# This script does not validate image URLs.
-#
-# Does not include bounding box annotations.
-#
-########
+"""
+
+generate_lila_per_image_labels.py
+
+Generate a .csv file with one row per annotation, containing full URLs to every
+camera trap image on LILA, with taxonomically expanded labels.
+
+Typically there will be one row per image, though images with multiple annotations
+will have multiple rows.
+
+Some images may not physically exist, particularly images that are labeled as "human".
+This script does not validate image URLs.
+
+Does not include bounding box annotations.
+
+"""
 
 #%% Constants and imports
 
@@ -23,8 +23,6 @@ import pandas as pd
 import numpy as np
 import dateparser
 import csv
-import urllib
-import urllib.request
 
 from collections import defaultdict
 from tqdm import tqdm
@@ -36,7 +34,6 @@ from data_management.lila.lila_common import read_lila_metadata, \
 from md_utils import write_html_image_list
 from md_utils.path_utils import zip_file
 from md_utils.path_utils import open_file
-from md_utils.url_utils import download_url
 
 # We'll write images, metadata downloads, and temporary files here
 lila_local_base = os.path.expanduser('~/lila')
@@ -107,12 +104,15 @@ for i_row,row in taxonomy_df.iterrows():
 
 # Takes several hours
 
-header = ['dataset_name','url','image_id','sequence_id','location_id','frame_num','original_label',\
-          'scientific_name','common_name','datetime','annotation_level']
+# The order of these headers needs to match the order in which fields are added later in this cell;
+# don't mess with this order.
+header = ['dataset_name','url_gcp','url_aws','url_azure',
+          'image_id','sequence_id','location_id','frame_num',
+          'original_label','scientific_name','common_name','datetime','annotation_level']
 
 taxonomy_levels_to_include = \
     ['kingdom','phylum','subphylum','superclass','class','subclass','infraclass','superorder','order',
-     'suborder','infraorder','superfamily','family','subfamily','tribe','genus','species','subspecies',\
+     'suborder','infraorder','superfamily','family','subfamily','tribe','genus','species','subspecies',
      'variety']
     
 header.extend(taxonomy_levels_to_include)
@@ -179,10 +179,17 @@ with open(output_file,'w',encoding='utf-8',newline='') as f:
                 break
             
             file_name = im['file_name'].replace('\\','/')
-            base_url = metadata_table[ds_name]['image_base_url']
-            assert not base_url.endswith('/')
-            url = base_url + '/' + file_name
+            base_url_gcp = metadata_table[ds_name]['image_base_url_gcp']
+            base_url_aws = metadata_table[ds_name]['image_base_url_aws']
+            base_url_azure = metadata_table[ds_name]['image_base_url_azure']
+            assert not base_url_gcp.endswith('/')
+            assert not base_url_aws.endswith('/')
+            assert not base_url_azure.endswith('/')
             
+            url_gcp = base_url_gcp + '/' + file_name
+            url_aws = base_url_aws + '/' + file_name
+            url_azure = base_url_azure + '/' + file_name
+                        
             for k in im.keys():
                 if ('date' in k or 'time' in k) and (k not in ['datetime','date_captured']):
                     raise ValueError('Unrecognized datetime field')
@@ -297,7 +304,9 @@ with open(output_file,'w',encoding='utf-8',newline='') as f:
                 
                 row = []
                 row.append(ds_name)
-                row.append(url)
+                row.append(url_gcp)
+                row.append(url_aws)
+                row.append(url_azure)
                 row.append(image_id)
                 row.append(sequence_id)
                 row.append(location_id)
@@ -365,7 +374,8 @@ dataset_name_to_locations = defaultdict(set)
 def check_row(row):
     
     assert row['dataset_name'] in metadata_table.keys()
-    assert row['url'].startswith('https://')
+    for url_column in ['url_gcp','url_aws','url_azure']:
+        assert row[url_column].startswith('https://') or row[url_column].startswith('http://')
     assert ' : ' in row['image_id']
     assert 'seq' not in row['location_id'].lower()
     assert row['annotation_level'] in valid_annotation_levels
@@ -446,28 +456,31 @@ for ds_name in metadata_table.keys():
 print('Selected {} total images'.format(len(images_to_download)))
 
 
-#%% Download images
+#%% Download images (prep)
 
 # Expect a few errors for images with human or vehicle labels (or things like "ignore" that *could* be humans)
 
-# TODO: trivially parallelizable
-#
+preferred_cloud = 'aws'
+
+url_to_target_file = {}
+
 # i_image = 10; image = images_to_download[i_image]
 for i_image,image in tqdm(enumerate(images_to_download),total=len(images_to_download)):
     
-    url = image['url']
+    url = image['url_' + preferred_cloud]
     ext = os.path.splitext(url)[1]
-    image_file = os.path.join(preview_folder,'image_{}'.format(str(i_image).zfill(4)) + ext)
-    relative_file = os.path.relpath(image_file,preview_folder)
-    try:
-        download_url(url,image_file,verbose=False)
-        image['relative_file'] = relative_file
-    except urllib.error.HTTPError:
-        print('Image {} does not exist ({}:{})'.format(
-            i_image,image['dataset_name'],image['original_label']))
-        image['relative_file'] = None
+    fn_relative = 'image_{}'.format(str(i_image).zfill(4)) + ext
+    fn_abs = os.path.join(preview_folder,fn_relative)
+    image['relative_file'] = fn_relative
+    image['url'] = url
+    url_to_target_file[url] = fn_abs
+    
+
+#%% Download images (execution)
 
-# ...for each image we need to download
+from md_utils.url_utils import parallel_download_urls
+download_results = parallel_download_urls(url_to_target_file,verbose=False,overwrite=True,
+                                          n_workers=20,pool_type='thread')
 
 
 #%% Write preview HTML
@@ -499,4 +512,4 @@ open_file(html_filename)
 
 zipped_output_file = zip_file(output_file,verbose=True)
 
-print('Zipped {} to {}'.format(output_file,zipped_output_file))
+print('Zipped {} to {}'.format(output_file,zipped_output_file))

data_management/lila/get_lila_annotation_counts.py

@@ -1,16 +1,16 @@
-########
-#
-# get_lila_annotation_counts.py
-#
-# Generates a .json-formatted dictionary mapping each LILA dataset to all categories
-# that exist for that dataset, with counts for the number of occurrences of each category 
-# (the number of *annotations* for each category, not the number of *images*).
-#
-# Also loads the taxonomy mapping file, to include scientific names for each category.
-#
-# get_lila_image_counts.py counts the number of *images* for each category in each dataset.
-#
-########
+"""
+
+get_lila_annotation_counts.py
+
+Generates a .json-formatted dictionary mapping each LILA dataset to all categories
+that exist for that dataset, with counts for the number of occurrences of each category 
+(the number of *annotations* for each category, not the number of *images*).
+
+Also loads the taxonomy mapping file, to include scientific names for each category.
+
+get_lila_image_counts.py counts the number of *images* for each category in each dataset.
+
+"""
 
 #%% Constants and imports
 
@@ -20,6 +20,9 @@ import os
 from data_management.lila.lila_common import read_lila_metadata,\
     read_metadata_file_for_dataset, read_lila_taxonomy_mapping
 
+# cloud provider to use for downloading images; options are 'gcp', 'azure', or 'aws'
+preferred_cloud = 'gcp'
+
 # array to fill for output
 category_list = []
 
@@ -96,9 +99,9 @@ for ds_name in metadata_table.keys():
         print('Warning: taxonomy mapping not available for {}'.format(ds_name))
         
     print('Finding categories in {}'.format(ds_name))
-    
+
     json_filename = metadata_table[ds_name]['json_filename']
-    base_url = metadata_table[ds_name]['image_base_url']
+    base_url = metadata_table[ds_name]['image_base_url_' + preferred_cloud]
     assert not base_url.endswith('/')
     
     # Open the metadata file