megadetector 5.0.7__py3-none-any.whl → 5.0.9__py3-none-any.whl

data_management/lila/lila_common.py

@@ -1,10 +1,10 @@
-########
-#
-# lila_common.py
-#
-# Common constants and functions related to LILA data management/retrieval.
-#
-########
+"""
+
+lila_common.py
+
+Common constants and functions related to LILA data management/retrieval.
+
+"""
 
 #%% Imports and constants
 
@@ -12,12 +12,12 @@ import os
 import json
 import zipfile
 import pandas as pd
-import numpy as np
 
 from urllib.parse import urlparse
 
 from md_utils.url_utils import download_url
 from md_utils.path_utils import unzip_file
+from md_utils.ct_utils import is_empty
 
 # LILA camera trap primary metadata file
 lila_metadata_url = 'http://lila.science/wp-content/uploads/2023/06/lila_camera_trap_datasets.csv'
@@ -31,9 +31,21 @@ wildlife_insights_taxonomy_local_json_filename = 'wi_taxonomy.json'
 wildlife_insights_taxonomy_local_csv_filename = \
     wildlife_insights_taxonomy_local_json_filename.replace('.json','.csv')
 
-lila_azure_storage_account = 'https://lilablobssc.blob.core.windows.net'
-gcp_bucket_api_url = 'https://storage.googleapis.com/public-datasets-lila'
-gcp_bucket_gs_url = 'gs://public-datasets-lila'
+# Filenames are consistent across clouds relative to these URLs
+lila_base_urls = {
+    'azure':'https://lilawildlife.blob.core.windows.net/lila-wildlife/',
+    'gcp':'https://storage.googleapis.com/public-datasets-lila/',
+    'aws':'http://us-west-2.opendata.source.coop.s3.amazonaws.com/agentmorris/lila-wildlife/'
+}
+
+lila_cloud_urls = {
+    'azure':'https://lilawildlife.blob.core.windows.net/lila-wildlife/',
+    'gcp':'gs://public-datasets-lila/',
+    'aws':'s3://us-west-2.opendata.source.coop/agentmorris/lila-wildlife/'
+}
+
+for url in lila_base_urls.values():
+    assert url.endswith('/')
 
 
 #%% Common functions
@@ -42,7 +54,11 @@ def read_wildlife_insights_taxonomy_mapping(metadata_dir):
     """
     Reads the WI taxonomy mapping file, downloading the .json data (and writing to .csv) if necessary.
     
-    Returns a Pandas dataframe.
+    Args:
+        metadata_dir (str): folder to use for temporary LILA metadata files
+        
+    Returns:
+        pd.dataframe: A DataFrame with taxonomy information
     """
     
     wi_taxonomy_csv_path = os.path.join(metadata_dir,wildlife_insights_taxonomy_local_csv_filename)
@@ -81,7 +97,11 @@ def read_lila_taxonomy_mapping(metadata_dir):
     """
     Reads the LILA taxonomy mapping file, downloading the .csv file if necessary.
     
-    Returns a Pandas dataframe, with one row per identification.
+    Args:
+        metadata_dir (str): folder to use for temporary LILA metadata files
+        
+    Returns:
+        pd.DataFrame: a DataFrame with one row per identification
     """
     
     p = urlparse(lila_taxonomy_mapping_url)
@@ -93,24 +113,38 @@ def read_lila_taxonomy_mapping(metadata_dir):
     return df
 
    
-def is_empty(v):
-    if v is None:
-        return True
-    if isinstance(v,str) and v == '':
-        return True
-    if isinstance(v,float) and np.isnan(v):
-        return True
-    return False
-
-
 def read_lila_metadata(metadata_dir):
     """
-    Reads LILA metadata (URLs to each dataset), downloading the txt file if necessary.
+    Reads LILA metadata (URLs to each dataset), downloading the .csv file if necessary.
     
-    Returns a dict mapping dataset names (e.g. "Caltech Camera Traps") to dicts
-    with keys corresponding to the headers in the .csv file, currently:
+    Args:
+        metadata_dir (str): folder to use for temporary LILA metadata files
+        
+    Returns:
+        dict: a dict mapping dataset names (e.g. "Caltech Camera Traps") to dicts
+        with keys corresponding to the headers in the .csv file, currently:
         
-    name,image_base_url,metadata_url,bbox_url,continent,country,region
+        - name
+        - short_name
+        - continent
+        - country
+        - region
+        - image_base_url_relative
+        - metadata_url_relative
+        - bbox_url_relative
+        - image_base_url_gcp
+        - metadata_url_gcp
+        - bbox_url_gcp
+        - image_base_url_aws
+        - metadata_url_aws
+        - bbox_url_aws
+        - image_base_url_azure
+        - metadata_url_azure
+        - box_url_azure
+        - mdv4_results_raw
+        - mdv5b_results_raw
+        - md_results_with_rde
+        - json_filename
     """
     
     # Put the master metadata file in the same folder where we're putting images
@@ -144,6 +178,12 @@ def read_lila_all_images_file(metadata_dir):
     """
     Downloads if necessary - then unzips if necessary - the .csv file with label mappings for
     all LILA files, and opens the resulting .csv file as a Pandas DataFrame.
+    
+    Args:
+        metadata_dir (str): folder to use for temporary LILA metadata files
+        
+    Returns:
+        pd.DataFrame: a DataFrame containing one row per identification in a LILA camera trap image
     """
         
     p = urlparse(lila_all_images_url)
@@ -165,18 +205,37 @@ def read_lila_all_images_file(metadata_dir):
     return df
 
 
-def read_metadata_file_for_dataset(ds_name,metadata_dir,metadata_table=None,json_url=None):
+def read_metadata_file_for_dataset(ds_name,
+                                   metadata_dir,
+                                   metadata_table=None,
+                                   json_url=None,
+                                   preferred_cloud='gcp'):
     """
     Downloads if necessary - then unzips if necessary - the .json file for a specific dataset.
-    Returns the .json filename on the local disk.
+    
+    Args:
+        ds_name (str): the name of the dataset for which you want to retrieve metadata (e.g.
+            "Caltech Camera Traps")        
+        metadata_dir (str): folder to use for temporary LILA metadata files
+        metadata_table (dict, optional): an optional dictionary already loaded via
+            read_lila_metadata()
+        json_url (str, optional): the URL of the metadata file, if None will be retrieved
+            via read_lila_metadata()
+        preferred_cloud (str, optional): 'gcp' (default), 'azure', or 'aws'
+        
+    Returns:
+        str: the .json filename on the local disk
+    
     """
     
+    assert preferred_cloud in lila_base_urls.keys()
+    
     if json_url is None:
         
         if metadata_table is None:
             metadata_table = read_lila_metadata(metadata_dir)
             
-        json_url = metadata_table[ds_name]['metadata_url']
+        json_url = metadata_table[ds_name]['metadata_url_' + preferred_cloud]
     
     p = urlparse(json_url)
     json_filename = os.path.join(metadata_dir,os.path.basename(p.path))
@@ -198,28 +257,6 @@ def read_metadata_file_for_dataset(ds_name,metadata_dir,metadata_table=None,json
     return json_filename
 
 
-def azure_url_to_gcp_http_url(url,error_if_not_azure_url=True):
-    """
-    Most URLs point to Azure by default, but most files are available on both Azure and GCP.
-    This function converts an Azure URL to the corresponding GCP http:// url.
-    """
-    
-    if error_if_not_azure_url:
-        assert url.startswith(lila_azure_storage_account)
-    gcp_url = url.replace(lila_azure_storage_account,gcp_bucket_api_url,1)
-    return gcp_url
-
-
-def azure_url_to_gcp_gs_url(url,error_if_not_azure_url=True):
-    """
-    Most URLs point to Azure by default, but most files are available on both Azure and GCP.
-    This function converts an Azure URL to the corresponding GCP gs:// url.
-    """
-    
-    return azure_url_to_gcp_http_url(url,error_if_not_azure_url).\
-        replace(gcp_bucket_api_url,gcp_bucket_gs_url,1)
-
-
 #%% Interactive test driver
 
 if False:
@@ -233,7 +270,8 @@ if False:
     
     from md_utils import url_utils
     
-    status_codes = url_utils.test_urls(urls)
+    status_codes = url_utils.test_urls(urls,timeout=2.0)
+    assert all([code == 200 for code in status_codes])
     
     
     #%% Verify that the metadata URLs exist for individual datasets
@@ -243,25 +281,20 @@ if False:
     dataset_metadata = read_lila_metadata(metadata_dir)
     
     urls_to_test = []
+    
     # ds_name = next(iter(dataset_metadata.keys()))
     for ds_name in dataset_metadata.keys():
         
         ds_info = dataset_metadata[ds_name]
-        urls_to_test.append(ds_info['metadata_url'])
-        if ds_info['bbox_url'] != None:
-            urls_to_test.append(ds_info['bbox_url'])
+        for cloud_name in lila_base_urls.keys():
+            urls_to_test.append(ds_info['metadata_url_' + cloud_name])
+            if ds_info['bbox_url_relative'] != None:
+                urls_to_test.append(ds_info['bbox_url_' + cloud_name])
             
-    status_codes = url_utils.test_urls(urls_to_test)    
-    
-    
-    #%% Verify that the GCP versions of all metadata files exist
-    
-    gcp_urls = []
-    
-    # url = urls_to_test[0]
-    for url in urls_to_test:
-        assert url.startswith(lila_azure_storage_account)
-        gcp_url = url.replace(lila_azure_storage_account,gcp_bucket_api_url,1)
-        gcp_urls.append(gcp_url)
-        
-    status_codes = url_utils.test_urls(gcp_urls)
+    status_codes = url_utils.test_urls(urls_to_test,
+                                       error_on_failure=True,
+                                       n_workers=10,
+                                       pool_type='process',
+                                       timeout=2.0)
+    assert all([code == 200 for code in status_codes])
+    

data_management/lila/test_lila_metadata_urls.py

@@ -1,116 +1,132 @@
-########
-#
-# test_lila_metadata_urls.py
-#
-# Test that all the metadata URLs for LILA camera trap datasets are valid, and 
-# test that at least one image within each URL is valid, including MegaDetector results
-# files.
-#
-########
-
-#%% Constants and imports
-
-import json
-import os
-
-from data_management.lila.lila_common import read_lila_metadata,\
-    read_metadata_file_for_dataset, read_lila_taxonomy_mapping
-
-# We'll write images, metadata downloads, and temporary files here
-lila_local_base = os.path.expanduser('~/lila')
-
-output_dir = os.path.join(lila_local_base,'lila_metadata_tests')
-os.makedirs(output_dir,exist_ok=True)
-
-metadata_dir = os.path.join(lila_local_base,'metadata')
-os.makedirs(metadata_dir,exist_ok=True)
-
-md_results_dir = os.path.join(lila_local_base,'md_results')
-os.makedirs(md_results_dir,exist_ok=True)
-
-md_results_keys = ['mdv4_results_raw','mdv5a_results_raw','mdv5b_results_raw','md_results_with_rde']
-
-
-#%% Load category and taxonomy files
-
-taxonomy_df = read_lila_taxonomy_mapping(metadata_dir)
-
-
-#%% Download and parse the metadata file
-
-metadata_table = read_lila_metadata(metadata_dir)
-
-print('Loaded metadata URLs for {} datasets'.format(len(metadata_table)))
-
-
-#%% Download and extract metadata and MD results for each dataset
-
-for ds_name in metadata_table.keys():    
-
-    metadata_table[ds_name]['json_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
-                                                                         metadata_dir=metadata_dir,
-                                                                         metadata_table=metadata_table)
-    for k in md_results_keys:
-        md_results_url = metadata_table[ds_name][k]
-        if md_results_url is None:
-            metadata_table[ds_name][k + '_filename'] = None
-        else:
-            metadata_table[ds_name][k + '_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
-                                                                        metadata_dir=md_results_dir,
-                                                                        json_url=md_results_url)
-
-
-#%% Build up a list of URLs to test
-
-url_to_source = {}
-
-# The first image in a dataset is disproportionately likely to be human (and thus 404)
-image_index = 1000
-
-# ds_name = list(metadata_table.keys())[0]
-for ds_name in metadata_table.keys():
-    
-    if 'bbox' in ds_name:
-        print('Skipping bbox dataset {}'.format(ds_name))
-        continue
-
-    print('Processing dataset {}'.format(ds_name))
-    
-    json_filename = metadata_table[ds_name]['json_filename']
-    with open(json_filename, 'r') as f:
-        data = json.load(f)
-
-    image_base_url = metadata_table[ds_name]['image_base_url']
-    assert not image_base_url.endswith('/')
-    # Download a test image
-    test_image_relative_path = data['images'][image_index]['file_name']
-    test_image_url = image_base_url + '/' + test_image_relative_path
-    
-    url_to_source[test_image_url] = ds_name + ' metadata'
-    
-    # k = md_results_keys[2]
-    for k in md_results_keys:
-        k_fn = k + '_filename'
-        if metadata_table[ds_name][k_fn] is not None:
-            with open(metadata_table[ds_name][k_fn],'r') as f:
-                md_results = json.load(f)
-                im = md_results['images'][image_index]
-                md_image_url = image_base_url + '/' + im['file']
-                url_to_source[md_image_url] = ds_name + ' ' + k
-    
-# ...for each dataset
-
-
-#%% Test URLs
-
-from md_utils.url_utils import test_urls
-
-urls_to_test = sorted(url_to_source.keys())
-urls_to_test = [fn.replace('\\','/') for fn in urls_to_test]
-
-status_codes = test_urls(urls_to_test,error_on_failure=False)
-
-for i_url,url in enumerate(urls_to_test):
-    if status_codes[i_url] != 200:
-        print('Status {} for {} ({})'.format(
-            status_codes[i_url],url,url_to_source[url]))
+"""
+
+test_lila_metadata_urls.py
+
+Test that all the metadata URLs for LILA camera trap datasets are valid, including MegaDetector 
+results files.
+
+Also pick an arbitrary image from each dataset and make sure that URL is valid.
+
+Also picks an arbitrary image from each dataset's MD results and make sure the corresponding URL is valid.
+
+"""
+
+#%% Constants and imports
+
+import json
+import os
+
+from data_management.lila.lila_common import read_lila_metadata,\
+    read_metadata_file_for_dataset, read_lila_taxonomy_mapping
+
+# We'll write images, metadata downloads, and temporary files here
+lila_local_base = os.path.expanduser('~/lila')
+
+output_dir = os.path.join(lila_local_base,'lila_metadata_tests')
+os.makedirs(output_dir,exist_ok=True)
+
+metadata_dir = os.path.join(lila_local_base,'metadata')
+os.makedirs(metadata_dir,exist_ok=True)
+
+md_results_dir = os.path.join(lila_local_base,'md_results')
+os.makedirs(md_results_dir,exist_ok=True)
+
+md_results_keys = ['mdv4_results_raw','mdv5a_results_raw','mdv5b_results_raw','md_results_with_rde']
+
+preferred_cloud = 'gcp' # 'azure', 'aws'
+
+
+#%% Load category and taxonomy files
+
+taxonomy_df = read_lila_taxonomy_mapping(metadata_dir)
+
+
+#%% Download and parse the metadata file
+
+metadata_table = read_lila_metadata(metadata_dir)
+
+print('Loaded metadata URLs for {} datasets'.format(len(metadata_table)))
+
+
+#%% Download and extract metadata and MD results for each dataset
+
+for ds_name in metadata_table.keys():    
+
+    metadata_table[ds_name]['json_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
+                                                                         metadata_dir=metadata_dir,
+                                                                         metadata_table=metadata_table)
+    for k in md_results_keys:
+        md_results_url = metadata_table[ds_name][k]
+        if md_results_url is None:
+            metadata_table[ds_name][k + '_filename'] = None
+        else:
+            metadata_table[ds_name][k + '_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
+                                                                        metadata_dir=md_results_dir,
+                                                                        json_url=md_results_url)
+
+
+#%% Build up a list of URLs to test
+
+# Takes ~15 mins, since it has to open all the giant .json files
+
+url_to_source = {}
+
+# The first image in a dataset is disproportionately likely to be human (and thus 404),
+# so we pick a semi-arbitrary image that isn't the first.  How about the 1000th?
+image_index = 1000
+
+# ds_name = list(metadata_table.keys())[0]
+for ds_name in metadata_table.keys():
+    
+    if 'bbox' in ds_name:
+        print('Skipping bbox dataset {}'.format(ds_name))
+        continue
+
+    print('Processing dataset {}'.format(ds_name))
+    
+    json_filename = metadata_table[ds_name]['json_filename']
+    with open(json_filename, 'r') as f:
+        data = json.load(f)
+
+    image_base_url = metadata_table[ds_name]['image_base_url_' + preferred_cloud]
+    assert not image_base_url.endswith('/')
+    # Download a test image
+    test_image_relative_path = data['images'][image_index]['file_name']
+    test_image_url = image_base_url + '/' + test_image_relative_path
+    
+    url_to_source[test_image_url] = ds_name + ' metadata'
+    
+    # Grab an image from the MegaDetector results
+    
+    # k = md_results_keys[2]
+    for k in md_results_keys:
+        k_fn = k + '_filename'
+        if metadata_table[ds_name][k_fn] is not None:
+            with open(metadata_table[ds_name][k_fn],'r') as f:
+                md_results = json.load(f)
+                im = md_results['images'][image_index]
+                md_image_url = image_base_url + '/' + im['file']
+                url_to_source[md_image_url] = ds_name + ' ' + k
+            del md_results
+    del data
+        
+# ...for each dataset
+
+
+#%% Test URLs
+
+from md_utils.url_utils import test_urls
+
+urls_to_test = sorted(url_to_source.keys())
+urls_to_test = [fn.replace('\\','/') for fn in urls_to_test]
+
+status_codes = test_urls(urls_to_test,
+                         error_on_failure=False,
+                         pool_type='thread',
+                         n_workers=10,
+                         timeout=2.0)
+
+for i_url,url in enumerate(urls_to_test):
+    if status_codes[i_url] != 200:
+        print('Status {} for {} ({})'.format(
+            status_codes[i_url],url,url_to_source[url]))