megadetector 5.0.5__py3-none-any.whl → 5.0.7__py3-none-any.whl

data_management/lila/download_lila_subset.py

@@ -24,11 +24,11 @@ from urllib.parse import urlparse
 from collections import defaultdict
 
 from data_management.lila.lila_common import \
-    read_lila_all_images_file, read_lila_metadata, is_empty, azure_url_to_gcp_http_url
+    read_lila_all_images_file, is_empty, azure_url_to_gcp_http_url
 from md_utils.url_utils import download_url
 
 # If any of these strings appear in the common name of a species, we'll download that image
-species_of_interest = ['grey fox','red fox','leopard cat']
+species_of_interest = ['grey fox','red fox','leopard cat','kiwi']
 
 # We'll write images, metadata downloads, and temporary files here
 lila_local_base = os.path.expanduser('~/lila')
@@ -40,30 +40,28 @@ output_dir = os.path.join(lila_local_base,'lila_downloads_by_dataset')
 os.makedirs(output_dir,exist_ok=True)
 
 # Number of concurrent download threads
-n_download_threads = 50
+n_download_threads = 20
 
 max_images_per_dataset = 10 # None
 
 # This impacts the data download, but not the metadata download
+#
+# "Azure" really means "Azure if available"; recent datasets are only available
+# on GCP.
 image_download_source = 'azure' # 'azure' or 'gcp'
 
 random.seed(0)
 
 
-#%% Download and open the giant table of image metadata
+#%% Download and open the giant table of image URLs and labels
 
-# Opening this huge .csv file make take ~30 seconds
+# ~60 seconds to download, unzip, and open
 df = read_lila_all_images_file(metadata_dir)
 
 
-#%% Download and parse the metadata file
-
-metadata_table = read_lila_metadata(metadata_dir)
-
-
 #%% Find all the images we want to download
 
-# Searching over the giant table can take a couple of minutes
+# ~2 minutes
 
 ds_name_to_urls = defaultdict(list)
 
@@ -106,13 +104,24 @@ else:
 
 #%% Download those image files
 
-def download_relative_filename(url, output_base, verbose=False, url_base=None):
+container_to_url_base = { 
+                         'lilablobssc.blob.core.windows.net':'/',
+                         'storage.googleapis.com':'/public-datasets-lila/'
+                         }
+
+def download_relative_filename(url, output_base, verbose=False, url_base=None, overwrite=False):
     """
     Download a URL to output_base, preserving relative path
     """
     
+    result = {'status':'unknown','url':url,'destination_filename':None}
+    
     if url_base is None:
-        url_base = '/'
+        assert url.startswith('https://')
+        container = url.split('/')[2]
+        assert container in container_to_url_base
+        url_base = container_to_url_base[container]
+    
     assert url_base.startswith('/') and url_base.endswith('/')
     
     p = urlparse(url)
@@ -122,29 +131,45 @@ def download_relative_filename(url, output_base, verbose=False, url_base=None):
     relative_filename = relative_filename.replace(url_base,'',1)        
     
     destination_filename = os.path.join(output_base,relative_filename)
-    download_url(url, destination_filename, verbose=verbose)
+    result['destination_filename'] = destination_filename
+    
+    if ((os.path.isfile(destination_filename)) and (not overwrite)):
+        result['status'] = 'skipped'
+        return result
+    try:
+        download_url(url, destination_filename, verbose=verbose)
+    except Exception as e:
+        print('Warning: error downloading URL {}: {}'.format(
+            url,str(e)))     
+        result['status'] = 'error: {}'.format(str(e))
+        return result
     
+    result['status'] = 'success'
+    return result
+
+
+# ds_name_to_urls maps dataset names to lists of URLs; flatten to a single list of URLs
 all_urls = list(ds_name_to_urls.values())
 all_urls = [item for sublist in all_urls for item in sublist]
 
-url_base = '/'
-
 # Convert Azure URLs to GCP URLs if necessary
 if image_download_source != 'azure':
     assert image_download_source == 'gcp'
-    url_base = '/public-datasets-lila/'
     all_urls = [azure_url_to_gcp_http_url(url) for url in all_urls]    
 
-print('Downloading {} images with Python requests'.format(len(all_urls)))
+print('Downloading {} images on {} workers'.format(len(all_urls),n_download_threads))
 
 if n_download_threads <= 1:
 
+    results = []
+    
     # url = all_urls[0]
     for url in tqdm(all_urls):        
-        download_relative_filename(url,output_dir,verbose=True,url_base=url_base)
+        results.append(download_relative_filename(url,output_dir,url_base=None))
     
 else:
 
     pool = ThreadPool(n_download_threads)        
-    tqdm(pool.imap(lambda s: download_relative_filename(s,output_dir,verbose=False,url_base=url_base), 
-                   all_urls), total=len(all_urls))
+    results = list(tqdm(pool.imap(lambda s: download_relative_filename(
+        s,output_dir,url_base=None), 
+        all_urls), total=len(all_urls)))

data_management/lila/generate_lila_per_image_labels.py

@@ -22,6 +22,9 @@ import json
 import pandas as pd
 import numpy as np
 import dateparser
+import csv
+import urllib
+import urllib.request
 
 from collections import defaultdict
 from tqdm import tqdm
@@ -30,6 +33,9 @@ from data_management.lila.lila_common import read_lila_metadata, \
     read_metadata_file_for_dataset, \
     read_lila_taxonomy_mapping
 
+from md_utils import write_html_image_list
+from md_utils.path_utils import zip_file
+from md_utils.path_utils import open_file
 from md_utils.url_utils import download_url
 
 # We'll write images, metadata downloads, and temporary files here
@@ -56,7 +62,7 @@ ds_name_to_annotation_level['NACTI'] = 'unknown'
 
 known_unmapped_labels = set(['WCS Camera Traps:#ref!'])
 
-debug_max_images_per_dataset = 0
+debug_max_images_per_dataset = -1
 if debug_max_images_per_dataset > 0:
     print('Running in debug mode')
     output_file = output_file.replace('.csv','_debug.csv')
@@ -72,7 +78,7 @@ if False:
     metadata_table = {k:metadata_table[k]}
 
 
-#%% Download and extract metadata for the datasets we're interested in
+#%% Download and extract metadata for each dataset
 
 for ds_name in metadata_table.keys():    
     metadata_table[ds_name]['metadata_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
@@ -101,8 +107,6 @@ for i_row,row in taxonomy_df.iterrows():
 
 # Takes several hours
 
-import csv
-
 header = ['dataset_name','url','image_id','sequence_id','location_id','frame_num','original_label',\
           'scientific_name','common_name','datetime','annotation_level']
 
@@ -122,7 +126,7 @@ def clearnan(v):
     assert isinstance(v,str)
     return v
 
-with open(output_file,'w') as f:
+with open(output_file,'w',encoding='utf-8',newline='') as f:
     
     csv_writer = csv.writer(f)
     csv_writer.writerow(header)
@@ -334,6 +338,8 @@ with open(output_file,'w') as f:
 
 # ...with open()
 
+print('Processed {} datsets'.format(len(metadata_table)))
+
 
 #%% Read the .csv back
 
@@ -352,6 +358,8 @@ def isint(v):
 
 valid_annotation_levels = set(['sequence','image','unknown'])
 
+# Collect a list of locations within each dataset; we'll use this
+# in the next cell to look for datasets that only have a single location
 dataset_name_to_locations = defaultdict(set)
 
 def check_row(row):
@@ -386,6 +394,8 @@ else:
 
 #%% Check for datasets that have only one location string
 
+# Expected: ENA24, Missouri Camera Traps
+
 for ds_name in dataset_name_to_locations.keys():
     if len(dataset_name_to_locations[ds_name]) == 1:
         print('No location information for {}'.format(ds_name))
@@ -440,8 +450,8 @@ print('Selected {} total images'.format(len(images_to_download)))
 
 # Expect a few errors for images with human or vehicle labels (or things like "ignore" that *could* be humans)
 
-import urllib.request
-
+# TODO: trivially parallelizable
+#
 # i_image = 10; image = images_to_download[i_image]
 for i_image,image in tqdm(enumerate(images_to_download),total=len(images_to_download)):
     
@@ -450,17 +460,17 @@ for i_image,image in tqdm(enumerate(images_to_download),total=len(images_to_down
     image_file = os.path.join(preview_folder,'image_{}'.format(str(i_image).zfill(4)) + ext)
     relative_file = os.path.relpath(image_file,preview_folder)
     try:
-        download_url(url,output_file,verbose=False)
+        download_url(url,image_file,verbose=False)
         image['relative_file'] = relative_file
     except urllib.error.HTTPError:
         print('Image {} does not exist ({}:{})'.format(
             i_image,image['dataset_name'],image['original_label']))
         image['relative_file'] = None
 
+# ...for each image we need to download
 
-#%% Write preview HTML
 
-from md_utils import write_html_image_list
+#%% Write preview HTML
 
 html_filename = os.path.join(preview_folder,'index.html')
 
@@ -475,19 +485,18 @@ for im in images_to_download:
     output_im = {}
     output_im['filename'] = im['relative_file']
     output_im['linkTarget'] = im['url']
-    output_im['title'] = str(im)
+    output_im['title'] = '<b>{}: {}</b><br/><br/>'.format(im['dataset_name'],im['original_label']) + str(im)
     output_im['imageStyle'] = 'width:600px;'
     output_im['textStyle'] = 'font-weight:normal;font-size:100%;'
     html_images.append(output_im)
     
 write_html_image_list.write_html_image_list(html_filename,html_images)
 
-from md_utils.path_utils import open_file
 open_file(html_filename)
 
 
 #%% Zip output file
 
-from md_utils.path_utils import zip_file
+zipped_output_file = zip_file(output_file,verbose=True)
 
-zip_file(output_file,verbose=True)
+print('Zipped {} to {}'.format(output_file,zipped_output_file))

data_management/lila/get_lila_annotation_counts.py

@@ -34,18 +34,9 @@ os.makedirs(metadata_dir,exist_ok=True)
 
 output_file = os.path.join(output_dir,'lila_dataset_to_categories.json')
 
-# Created by get_lila_category_list.py... contains counts for each category
-category_list_dir = os.path.join(lila_local_base,'lila_categories_list')
-lila_dataset_to_categories_file = os.path.join(category_list_dir,'lila_dataset_to_categories.json')
-
-assert os.path.isfile(lila_dataset_to_categories_file)
-
 
 #%% Load category and taxonomy files
 
-with open(lila_dataset_to_categories_file,'r') as f:
-    lila_dataset_to_categories = json.load(f)
-
 taxonomy_df = read_lila_taxonomy_mapping(metadata_dir)
 
 
@@ -55,9 +46,13 @@ ds_query_to_scientific_name = {}
 
 unmapped_queries = set()
 
+datasets_with_taxonomy_mapping = set()
+
 # i_row = 1; row = taxonomy_df.iloc[i_row]; row
 for i_row,row in taxonomy_df.iterrows():
     
+    datasets_with_taxonomy_mapping.add(row['dataset_name'])
+    
     ds_query = row['dataset_name'] + ':' + row['query']
     ds_query = ds_query.lower()
     
@@ -68,13 +63,17 @@ for i_row,row in taxonomy_df.iterrows():
         
     ds_query_to_scientific_name[ds_query] = row['scientific_name']
     
+print('Loaded taxonomy mappings for {} datasets'.format(len(datasets_with_taxonomy_mapping)))
     
+
 #%% Download and parse the metadata file
 
 metadata_table = read_lila_metadata(metadata_dir)
 
+print('Loaded metadata URLs for {} datasets'.format(len(metadata_table)))
+
 
-#%% Download and extract metadata for the datasets we're interested in
+#%% Download and extract metadata for each dataset
 
 for ds_name in metadata_table.keys():    
     metadata_table[ds_name]['json_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
@@ -91,6 +90,11 @@ dataset_to_categories = {}
 # ds_name = 'NACTI'
 for ds_name in metadata_table.keys():
     
+    taxonomy_mapping_available = (ds_name in datasets_with_taxonomy_mapping)
+    
+    if not taxonomy_mapping_available:
+        print('Warning: taxonomy mapping not available for {}'.format(ds_name))
+        
     print('Finding categories in {}'.format(ds_name))
     
     json_filename = metadata_table[ds_name]['json_filename']
@@ -122,6 +126,8 @@ for ds_name in metadata_table.keys():
        # always redundant with the class-level data sets.
        if 'bbox' in ds_name:
            c['scientific_name_from_taxonomy_mapping'] = None
+       elif not taxonomy_mapping_available:
+           c['scientific_name_from_taxonomy_mapping'] = None
        else:
            taxonomy_query_string = ds_name.lower().strip() + ':' + c['name'].lower()
            if taxonomy_query_string not in ds_query_to_scientific_name:
@@ -158,4 +164,4 @@ for ds_name in dataset_to_categories:
 #%% Save the results
 
 with open(output_file, 'w') as f:
-    json.dump(dataset_to_categories,f,indent=2)
+    json.dump(dataset_to_categories,f,indent=1)

data_management/lila/lila_common.py

@@ -21,7 +21,7 @@ from md_utils.path_utils import unzip_file
 
 # LILA camera trap primary metadata file
 lila_metadata_url = 'http://lila.science/wp-content/uploads/2023/06/lila_camera_trap_datasets.csv'
-lila_taxonomy_mapping_url = 'https://lila.science/wp-content/uploads/2022/07/lila-taxonomy-mapping_release.csv'
+lila_taxonomy_mapping_url = 'https://lila.science/public/lila-taxonomy-mapping_release.csv'
 lila_all_images_url = 'https://lila.science/public/lila_image_urls_and_labels.csv.zip'
 
 wildlife_insights_page_size = 30000
@@ -165,16 +165,18 @@ def read_lila_all_images_file(metadata_dir):
     return df
 
 
-def read_metadata_file_for_dataset(ds_name,metadata_dir,metadata_table=None):
+def read_metadata_file_for_dataset(ds_name,metadata_dir,metadata_table=None,json_url=None):
     """
     Downloads if necessary - then unzips if necessary - the .json file for a specific dataset.
     Returns the .json filename on the local disk.
     """
     
-    if metadata_table is None:
-        metadata_table = read_lila_metadata(metadata_dir)
+    if json_url is None:
         
-    json_url = metadata_table[ds_name]['metadata_url']
+        if metadata_table is None:
+            metadata_table = read_lila_metadata(metadata_dir)
+            
+        json_url = metadata_table[ds_name]['metadata_url']
     
     p = urlparse(json_url)
     json_filename = os.path.join(metadata_dir,os.path.basename(p.path))
@@ -196,25 +198,26 @@ def read_metadata_file_for_dataset(ds_name,metadata_dir,metadata_table=None):
     return json_filename
 
 
-def azure_url_to_gcp_http_url(url):
+def azure_url_to_gcp_http_url(url,error_if_not_azure_url=True):
     """
     Most URLs point to Azure by default, but most files are available on both Azure and GCP.
     This function converts an Azure URL to the corresponding GCP http:// url.
     """
     
-    assert url.startswith(lila_azure_storage_account)
+    if error_if_not_azure_url:
+        assert url.startswith(lila_azure_storage_account)
     gcp_url = url.replace(lila_azure_storage_account,gcp_bucket_api_url,1)
     return gcp_url
 
 
-def azure_url_to_gcp_gs_url(url):
+def azure_url_to_gcp_gs_url(url,error_if_not_azure_url=True):
     """
     Most URLs point to Azure by default, but most files are available on both Azure and GCP.
     This function converts an Azure URL to the corresponding GCP gs:// url.
     """
     
-    return azure_url_to_gcp_http_url(url).replace(gcp_bucket_api_url,
-                                                  gcp_bucket_gs_url,1)
+    return azure_url_to_gcp_http_url(url,error_if_not_azure_url).\
+        replace(gcp_bucket_api_url,gcp_bucket_gs_url,1)
 
 
 #%% Interactive test driver
@@ -261,4 +264,4 @@ if False:
         gcp_url = url.replace(lila_azure_storage_account,gcp_bucket_api_url,1)
         gcp_urls.append(gcp_url)
         
-    status_codes = url_utils.test_urls(gcp_urls)
+    status_codes = url_utils.test_urls(gcp_urls)

data_management/lila/test_lila_metadata_urls.py

@@ -0,0 +1,116 @@
+########
+#
+# test_lila_metadata_urls.py
+#
+# Test that all the metadata URLs for LILA camera trap datasets are valid, and 
+# test that at least one image within each URL is valid, including MegaDetector results
+# files.
+#
+########
+
+#%% Constants and imports
+
+import json
+import os
+
+from data_management.lila.lila_common import read_lila_metadata,\
+    read_metadata_file_for_dataset, read_lila_taxonomy_mapping
+
+# We'll write images, metadata downloads, and temporary files here
+lila_local_base = os.path.expanduser('~/lila')
+
+output_dir = os.path.join(lila_local_base,'lila_metadata_tests')
+os.makedirs(output_dir,exist_ok=True)
+
+metadata_dir = os.path.join(lila_local_base,'metadata')
+os.makedirs(metadata_dir,exist_ok=True)
+
+md_results_dir = os.path.join(lila_local_base,'md_results')
+os.makedirs(md_results_dir,exist_ok=True)
+
+md_results_keys = ['mdv4_results_raw','mdv5a_results_raw','mdv5b_results_raw','md_results_with_rde']
+
+
+#%% Load category and taxonomy files
+
+taxonomy_df = read_lila_taxonomy_mapping(metadata_dir)
+
+
+#%% Download and parse the metadata file
+
+metadata_table = read_lila_metadata(metadata_dir)
+
+print('Loaded metadata URLs for {} datasets'.format(len(metadata_table)))
+
+
+#%% Download and extract metadata and MD results for each dataset
+
+for ds_name in metadata_table.keys():    
+
+    metadata_table[ds_name]['json_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
+                                                                         metadata_dir=metadata_dir,
+                                                                         metadata_table=metadata_table)
+    for k in md_results_keys:
+        md_results_url = metadata_table[ds_name][k]
+        if md_results_url is None:
+            metadata_table[ds_name][k + '_filename'] = None
+        else:
+            metadata_table[ds_name][k + '_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
+                                                                        metadata_dir=md_results_dir,
+                                                                        json_url=md_results_url)
+
+
+#%% Build up a list of URLs to test
+
+url_to_source = {}
+
+# The first image in a dataset is disproportionately likely to be human (and thus 404)
+image_index = 1000
+
+# ds_name = list(metadata_table.keys())[0]
+for ds_name in metadata_table.keys():
+    
+    if 'bbox' in ds_name:
+        print('Skipping bbox dataset {}'.format(ds_name))
+        continue
+
+    print('Processing dataset {}'.format(ds_name))
+    
+    json_filename = metadata_table[ds_name]['json_filename']
+    with open(json_filename, 'r') as f:
+        data = json.load(f)
+
+    image_base_url = metadata_table[ds_name]['image_base_url']
+    assert not image_base_url.endswith('/')
+    # Download a test image
+    test_image_relative_path = data['images'][image_index]['file_name']
+    test_image_url = image_base_url + '/' + test_image_relative_path
+    
+    url_to_source[test_image_url] = ds_name + ' metadata'
+    
+    # k = md_results_keys[2]
+    for k in md_results_keys:
+        k_fn = k + '_filename'
+        if metadata_table[ds_name][k_fn] is not None:
+            with open(metadata_table[ds_name][k_fn],'r') as f:
+                md_results = json.load(f)
+                im = md_results['images'][image_index]
+                md_image_url = image_base_url + '/' + im['file']
+                url_to_source[md_image_url] = ds_name + ' ' + k
+    
+# ...for each dataset
+
+
+#%% Test URLs
+
+from md_utils.url_utils import test_urls
+
+urls_to_test = sorted(url_to_source.keys())
+urls_to_test = [fn.replace('\\','/') for fn in urls_to_test]
+
+status_codes = test_urls(urls_to_test,error_on_failure=False)
+
+for i_url,url in enumerate(urls_to_test):
+    if status_codes[i_url] != 200:
+        print('Status {} for {} ({})'.format(
+            status_codes[i_url],url,url_to_source[url]))