megadetector 5.0.7__py3-none-any.whl → 5.0.9__py3-none-any.whl

data_management/lila/download_lila_subset.py

@@ -1,17 +1,11 @@
-########
-#
-# download_lila_subset.py
-#
-# Example of how to download a list of files from LILA, e.g. all the files
-# in a data set corresponding to a particular species.
-#
-# Organizes the downloaded images by dataset.  How you actually want to organize files,
-# what you want to query for, etc., is very application-specific; this is just meant as a 
-# demo.
-#
-# Can download from either Azure or GCP.
-#
-########
+"""
+
+download_lila_subset.py
+
+Example of how to download a list of files from LILA, e.g. all the files
+in a data set corresponding to a particular species.
+
+"""
 
 #%% Constants and imports
 
@@ -19,16 +13,15 @@ import os
 import random
 
 from tqdm import tqdm
-from multiprocessing.pool import ThreadPool
-from urllib.parse import urlparse
 from collections import defaultdict
 
-from data_management.lila.lila_common import \
-    read_lila_all_images_file, is_empty, azure_url_to_gcp_http_url
-from md_utils.url_utils import download_url
+from data_management.lila.lila_common import read_lila_all_images_file, is_empty, lila_base_urls
+
+for s in lila_base_urls.values():
+    assert s.endswith('/')
 
 # If any of these strings appear in the common name of a species, we'll download that image
-species_of_interest = ['grey fox','red fox','leopard cat','kiwi']
+species_of_interest = ['grey fox','gray fox','cape fox','red fox','kit fox']
 
 # We'll write images, metadata downloads, and temporary files here
 lila_local_base = os.path.expanduser('~/lila')
@@ -44,24 +37,22 @@ n_download_threads = 20
 
 max_images_per_dataset = 10 # None
 
-# This impacts the data download, but not the metadata download
-#
-# "Azure" really means "Azure if available"; recent datasets are only available
-# on GCP.
-image_download_source = 'azure' # 'azure' or 'gcp'
+preferred_provider = 'gcp' # 'azure', 'gcp', 'aws'
 
 random.seed(0)
 
 
 #%% Download and open the giant table of image URLs and labels
 
-# ~60 seconds to download, unzip, and open
+# Takes ~60 seconds to download, unzip, and open
 df = read_lila_all_images_file(metadata_dir)
 
 
 #%% Find all the images we want to download
 
-# ~2 minutes
+# Takes ~2 minutes
+
+common_name_to_count = defaultdict(int)
 
 ds_name_to_urls = defaultdict(list)
 
@@ -72,26 +63,33 @@ def find_items(row):
     
     match = False
     
+    # This is the only bit of this file that's specific to a particular query.  In this case
+    # we're checking whether each row is on a list of species of interest, but you do you.
     for species_name in species_of_interest:
         if species_name in row['common_name']:
             match = True
+            common_name_to_count[species_name] += 1
             break
     
     if match:
-        ds_name_to_urls[row['dataset_name']].append(row['url'])
+        ds_name_to_urls[row['dataset_name']].append(row['url_' + preferred_provider])
 
 tqdm.pandas()
 _ = df.progress_apply(find_items,axis=1)
 
+# We have a list of URLs for each dataset, flatten them all into a list of URLs
 all_urls = list(ds_name_to_urls.values())
 all_urls = [item for sublist in all_urls for item in sublist]
 print('Found {} matching URLs across {} datasets'.format(len(all_urls),len(ds_name_to_urls)))
 
+for common_name in common_name_to_count:
+    print('{}: {}'.format(common_name,common_name_to_count[common_name]))
+    
 from copy import deepcopy
 ds_name_to_urls_raw = deepcopy(ds_name_to_urls)
 
 
-#%% Trim to a fixed number of URLs per dataset
+#%% Optionally trim to a fixed number of URLs per dataset
 
 if max_images_per_dataset is None:
     pass
@@ -102,74 +100,78 @@ else:
             ds_name_to_urls[ds_name] = random.sample(ds_name_to_urls[ds_name],max_images_per_dataset)
 
 
-#%% Download those image files
+#%% Choose target files for each URL
 
-container_to_url_base = { 
-                         'lilablobssc.blob.core.windows.net':'/',
-                         'storage.googleapis.com':'/public-datasets-lila/'
-                         }
+from data_management.lila.lila_common import lila_base_urls
 
-def download_relative_filename(url, output_base, verbose=False, url_base=None, overwrite=False):
-    """
-    Download a URL to output_base, preserving relative path
-    """
-    
-    result = {'status':'unknown','url':url,'destination_filename':None}
-    
-    if url_base is None:
-        assert url.startswith('https://')
-        container = url.split('/')[2]
-        assert container in container_to_url_base
-        url_base = container_to_url_base[container]
-    
-    assert url_base.startswith('/') and url_base.endswith('/')
-    
-    p = urlparse(url)
-    relative_filename = str(p.path)
-    # remove the leading '/'
-    assert relative_filename.startswith(url_base)
-    relative_filename = relative_filename.replace(url_base,'',1)        
-    
-    destination_filename = os.path.join(output_base,relative_filename)
-    result['destination_filename'] = destination_filename
-    
-    if ((os.path.isfile(destination_filename)) and (not overwrite)):
-        result['status'] = 'skipped'
-        return result
-    try:
-        download_url(url, destination_filename, verbose=verbose)
-    except Exception as e:
-        print('Warning: error downloading URL {}: {}'.format(
-            url,str(e)))     
-        result['status'] = 'error: {}'.format(str(e))
-        return result
-    
-    result['status'] = 'success'
-    return result
+# We have a list of URLs per dataset, flatten that into a single list of URLs
+urls_to_download = set()
+for ds_name in ds_name_to_urls:
+    for url in ds_name_to_urls[ds_name]:
+        urls_to_download.add(url)
+urls_to_download = sorted(list(urls_to_download))        
 
+# A URL might look like this:
+#
+# https://storage.googleapis.com/public-datasets-lila/wcs-unzipped/animals/0667/0302.jpg
+# 
+# We'll write that to an output file that looks like this (relative to output_dir):
+#
+# wcs-unzipped/animals/0667/0302.jpg
+#
+# ...so we need to remove the base URL to get the target file.
+base_url = lila_base_urls[preferred_provider]
+assert base_url.endswith('/')
 
-# ds_name_to_urls maps dataset names to lists of URLs; flatten to a single list of URLs
-all_urls = list(ds_name_to_urls.values())
-all_urls = [item for sublist in all_urls for item in sublist]
+url_to_target_file = {}
 
-# Convert Azure URLs to GCP URLs if necessary
-if image_download_source != 'azure':
-    assert image_download_source == 'gcp'
-    all_urls = [azure_url_to_gcp_http_url(url) for url in all_urls]    
+for url in urls_to_download:
+    assert url.startswith(base_url) 
+    target_fn_relative = url.replace(base_url,'')
+    target_fn_abs = os.path.join(output_dir,target_fn_relative)
+    url_to_target_file[url] = target_fn_abs
 
-print('Downloading {} images on {} workers'.format(len(all_urls),n_download_threads))
 
-if n_download_threads <= 1:
+#%% Download image files
 
-    results = []
-    
-    # url = all_urls[0]
-    for url in tqdm(all_urls):        
-        results.append(download_relative_filename(url,output_dir,url_base=None))
+from md_utils.url_utils import parallel_download_urls
+
+download_results = parallel_download_urls(url_to_target_file=url_to_target_file,
+                                          verbose=False,
+                                          overwrite=False,
+                                          n_workers=n_download_threads,
+                                          pool_type='thread')
+
+
+#%% Scrap
+
+if False:
     
-else:
+    pass
+
+    #%% Find all the reptiles on LILA
 
-    pool = ThreadPool(n_download_threads)        
-    results = list(tqdm(pool.imap(lambda s: download_relative_filename(
-        s,output_dir,url_base=None), 
-        all_urls), total=len(all_urls)))
+    reptile_rows = df.loc[df['class'] == 'reptilia']
+    
+    # i_row = 0; row = reptile_rows.iloc[i_row]
+    
+    common_name_to_count = defaultdict(int)
+    dataset_to_count = defaultdict(int)
+    for i_row,row in reptile_rows.iterrows():
+        common_name_to_count[row['common_name']] += 1
+        dataset_to_count[row['dataset_name']] += 1
+        
+    from md_utils.ct_utils import sort_dictionary_by_value
+    
+    print('Found {} reptiles\n'.format(len(reptile_rows)))
+    
+    common_name_to_count = sort_dictionary_by_value(common_name_to_count,reverse=True)
+    dataset_to_count = sort_dictionary_by_value(dataset_to_count,reverse=True)
+    
+    print('Common names by count:\n')
+    for k in common_name_to_count:
+        print('{} ({})'.format(k,common_name_to_count[k]))
+    
+    print('\nDatasets by count:\n')    
+    for k in dataset_to_count:
+        print('{} ({})'.format(k,dataset_to_count[k]))

data_management/lila/generate_lila_per_image_labels.py

@@ -1,19 +1,19 @@
-########
-#
-# generate_lila_per_image_labels.py
-# 
-# Generate a .csv file with one row per annotation, containing full URLs to every
-# camera trap image on LILA, with taxonomically expanded labels.
-#
-# Typically there will be one row per image, though images with multiple annotations
-# will have multiple rows.
-#
-# Some images may not physically exist, particularly images that are labeled as "human".
-# This script does not validate image URLs.
-#
-# Does not include bounding box annotations.
-#
-########
+"""
+
+generate_lila_per_image_labels.py
+
+Generate a .csv file with one row per annotation, containing full URLs to every
+camera trap image on LILA, with taxonomically expanded labels.
+
+Typically there will be one row per image, though images with multiple annotations
+will have multiple rows.
+
+Some images may not physically exist, particularly images that are labeled as "human".
+This script does not validate image URLs.
+
+Does not include bounding box annotations.
+
+"""
 
 #%% Constants and imports
 
@@ -23,8 +23,6 @@ import pandas as pd
 import numpy as np
 import dateparser
 import csv
-import urllib
-import urllib.request
 
 from collections import defaultdict
 from tqdm import tqdm
@@ -36,7 +34,6 @@ from data_management.lila.lila_common import read_lila_metadata, \
 from md_utils import write_html_image_list
 from md_utils.path_utils import zip_file
 from md_utils.path_utils import open_file
-from md_utils.url_utils import download_url
 
 # We'll write images, metadata downloads, and temporary files here
 lila_local_base = os.path.expanduser('~/lila')
@@ -107,12 +104,15 @@ for i_row,row in taxonomy_df.iterrows():
 
 # Takes several hours
 
-header = ['dataset_name','url','image_id','sequence_id','location_id','frame_num','original_label',\
-          'scientific_name','common_name','datetime','annotation_level']
+# The order of these headers needs to match the order in which fields are added later in this cell;
+# don't mess with this order.
+header = ['dataset_name','url_gcp','url_aws','url_azure',
+          'image_id','sequence_id','location_id','frame_num',
+          'original_label','scientific_name','common_name','datetime','annotation_level']
 
 taxonomy_levels_to_include = \
     ['kingdom','phylum','subphylum','superclass','class','subclass','infraclass','superorder','order',
-     'suborder','infraorder','superfamily','family','subfamily','tribe','genus','species','subspecies',\
+     'suborder','infraorder','superfamily','family','subfamily','tribe','genus','species','subspecies',
      'variety']
     
 header.extend(taxonomy_levels_to_include)
@@ -179,10 +179,17 @@ with open(output_file,'w',encoding='utf-8',newline='') as f:
                 break
             
             file_name = im['file_name'].replace('\\','/')
-            base_url = metadata_table[ds_name]['image_base_url']
-            assert not base_url.endswith('/')
-            url = base_url + '/' + file_name
+            base_url_gcp = metadata_table[ds_name]['image_base_url_gcp']
+            base_url_aws = metadata_table[ds_name]['image_base_url_aws']
+            base_url_azure = metadata_table[ds_name]['image_base_url_azure']
+            assert not base_url_gcp.endswith('/')
+            assert not base_url_aws.endswith('/')
+            assert not base_url_azure.endswith('/')
             
+            url_gcp = base_url_gcp + '/' + file_name
+            url_aws = base_url_aws + '/' + file_name
+            url_azure = base_url_azure + '/' + file_name
+                        
             for k in im.keys():
                 if ('date' in k or 'time' in k) and (k not in ['datetime','date_captured']):
                     raise ValueError('Unrecognized datetime field')
@@ -297,7 +304,9 @@ with open(output_file,'w',encoding='utf-8',newline='') as f:
                 
                 row = []
                 row.append(ds_name)
-                row.append(url)
+                row.append(url_gcp)
+                row.append(url_aws)
+                row.append(url_azure)
                 row.append(image_id)
                 row.append(sequence_id)
                 row.append(location_id)
@@ -338,7 +347,7 @@ with open(output_file,'w',encoding='utf-8',newline='') as f:
 
 # ...with open()
 
-print('Processed {} datsets'.format(len(metadata_table)))
+print('Processed {} datasets'.format(len(metadata_table)))
 
 
 #%% Read the .csv back
@@ -365,7 +374,8 @@ dataset_name_to_locations = defaultdict(set)
 def check_row(row):
     
     assert row['dataset_name'] in metadata_table.keys()
-    assert row['url'].startswith('https://')
+    for url_column in ['url_gcp','url_aws','url_azure']:
+        assert row[url_column].startswith('https://') or row[url_column].startswith('http://')
     assert ' : ' in row['image_id']
     assert 'seq' not in row['location_id'].lower()
     assert row['annotation_level'] in valid_annotation_levels
@@ -446,28 +456,31 @@ for ds_name in metadata_table.keys():
 print('Selected {} total images'.format(len(images_to_download)))
 
 
-#%% Download images
+#%% Download images (prep)
 
 # Expect a few errors for images with human or vehicle labels (or things like "ignore" that *could* be humans)
 
-# TODO: trivially parallelizable
-#
+preferred_cloud = 'aws'
+
+url_to_target_file = {}
+
 # i_image = 10; image = images_to_download[i_image]
 for i_image,image in tqdm(enumerate(images_to_download),total=len(images_to_download)):
     
-    url = image['url']
+    url = image['url_' + preferred_cloud]
     ext = os.path.splitext(url)[1]
-    image_file = os.path.join(preview_folder,'image_{}'.format(str(i_image).zfill(4)) + ext)
-    relative_file = os.path.relpath(image_file,preview_folder)
-    try:
-        download_url(url,image_file,verbose=False)
-        image['relative_file'] = relative_file
-    except urllib.error.HTTPError:
-        print('Image {} does not exist ({}:{})'.format(
-            i_image,image['dataset_name'],image['original_label']))
-        image['relative_file'] = None
+    fn_relative = 'image_{}'.format(str(i_image).zfill(4)) + ext
+    fn_abs = os.path.join(preview_folder,fn_relative)
+    image['relative_file'] = fn_relative
+    image['url'] = url
+    url_to_target_file[url] = fn_abs
+    
+
+#%% Download images (execution)
 
-# ...for each image we need to download
+from md_utils.url_utils import parallel_download_urls
+download_results = parallel_download_urls(url_to_target_file,verbose=False,overwrite=True,
+                                          n_workers=20,pool_type='thread')
 
 
 #%% Write preview HTML
@@ -499,4 +512,4 @@ open_file(html_filename)
 
 zipped_output_file = zip_file(output_file,verbose=True)
 
-print('Zipped {} to {}'.format(output_file,zipped_output_file))
+print('Zipped {} to {}'.format(output_file,zipped_output_file))

data_management/lila/get_lila_annotation_counts.py

@@ -1,16 +1,16 @@
-########
-#
-# get_lila_annotation_counts.py
-#
-# Generates a .json-formatted dictionary mapping each LILA dataset to all categories
-# that exist for that dataset, with counts for the number of occurrences of each category 
-# (the number of *annotations* for each category, not the number of *images*).
-#
-# Also loads the taxonomy mapping file, to include scientific names for each category.
-#
-# get_lila_image_counts.py counts the number of *images* for each category in each dataset.
-#
-########
+"""
+
+get_lila_annotation_counts.py
+
+Generates a .json-formatted dictionary mapping each LILA dataset to all categories
+that exist for that dataset, with counts for the number of occurrences of each category 
+(the number of *annotations* for each category, not the number of *images*).
+
+Also loads the taxonomy mapping file, to include scientific names for each category.
+
+get_lila_image_counts.py counts the number of *images* for each category in each dataset.
+
+"""
 
 #%% Constants and imports
 
@@ -20,6 +20,9 @@ import os
 from data_management.lila.lila_common import read_lila_metadata,\
     read_metadata_file_for_dataset, read_lila_taxonomy_mapping
 
+# cloud provider to use for downloading images; options are 'gcp', 'azure', or 'aws'
+preferred_cloud = 'gcp'
+
 # array to fill for output
 category_list = []
 
@@ -96,9 +99,9 @@ for ds_name in metadata_table.keys():
         print('Warning: taxonomy mapping not available for {}'.format(ds_name))
         
     print('Finding categories in {}'.format(ds_name))
-    
+
     json_filename = metadata_table[ds_name]['json_filename']
-    base_url = metadata_table[ds_name]['image_base_url']
+    base_url = metadata_table[ds_name]['image_base_url_' + preferred_cloud]
     assert not base_url.endswith('/')
     
     # Open the metadata file    

data_management/lila/get_lila_image_counts.py

@@ -1,14 +1,14 @@
-########
-#
-# get_lila_image_counts.py
-#
-# Count the number of images and bounding boxes with each label in one or more LILA datasets.
-#
-# This script doesn't write these counts out anywhere other than the console, it's just intended
-# as a template for doing operations like this on LILA data.  get_lila_annotation_counts.py writes 
-# information out to a .json file, but it counts *annotations*, not *images*, for each category.
-#
-########
+"""
+
+get_lila_image_counts.py
+
+Count the number of images and bounding boxes with each label in one or more LILA datasets.
+
+This script doesn't write these counts out anywhere other than the console, it's just intended
+as a template for doing operations like this on LILA data.  get_lila_annotation_counts.py writes 
+information out to a .json file, but it counts *annotations*, not *images*, for each category.
+
+"""
 
 #%% Constants and imports