megadetector 5.0.11__py3-none-any.whl → 5.0.13__py3-none-any.whl

megadetector/data_management/lila/get_lila_image_counts.py

@@ -0,0 +1,112 @@
+"""
+
+get_lila_image_counts.py
+
+Count the number of images and bounding boxes with each label in one or more LILA datasets.
+
+This script doesn't write these counts out anywhere other than the console, it's just intended
+as a template for doing operations like this on LILA data.  get_lila_annotation_counts.py writes 
+information out to a .json file, but it counts *annotations*, not *images*, for each category.
+
+"""
+
+#%% Constants and imports
+
+import json
+import os
+
+from collections import defaultdict
+
+from megadetector.data_management.lila.lila_common import \
+    read_lila_metadata, read_metadata_file_for_dataset
+
+# If None, will use all datasets
+datasets_of_interest = None
+
+# We'll write images, metadata downloads, and temporary files here
+lila_local_base = os.path.expanduser('~/lila')
+
+metadata_dir = os.path.join(lila_local_base,'metadata')
+os.makedirs(metadata_dir,exist_ok=True)
+
+
+#%% Download and parse the metadata file
+
+metadata_table = read_lila_metadata(metadata_dir)
+
+
+#%% Download and extract metadata for the datasets we're interested in
+
+if datasets_of_interest is None:
+    datasets_of_interest = list(metadata_table.keys())
+
+for ds_name in datasets_of_interest:    
+    metadata_table[ds_name]['json_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
+                                                                         metadata_dir=metadata_dir,
+                                                                         metadata_table=metadata_table)
+    
+    
+#%% Count categories
+
+ds_name_to_category_counts = {}
+
+# ds_name = datasets_of_interest[0]
+for ds_name in datasets_of_interest:
+    
+    category_to_image_count = {}
+    category_to_bbox_count = {}
+    
+    print('Counting categories in: ' + ds_name)
+    
+    json_filename = metadata_table[ds_name]['json_filename']
+    with open(json_filename, 'r') as f:
+        data = json.load(f)
+    
+    categories = data['categories']
+    category_ids = [c['id'] for c in categories]
+    for c in categories:
+        category_id_to_name = {c['id']:c['name'] for c in categories}
+    annotations = data['annotations']
+    images = data['images']
+    
+    for category_id in category_ids:        
+        category_name = category_id_to_name[category_id]        
+        category_to_image_count[category_name] = 0
+        category_to_bbox_count[category_name] = 0
+        
+    image_id_to_category_names = defaultdict(set)
+    
+    # Go through annotations, marking each image with the categories that are present
+    #
+    # ann = annotations[0]
+    for ann in annotations:
+        
+        category_name = category_id_to_name[ann['category_id']]
+        image_id_to_category_names[ann['image_id']].add(category_name)
+
+    # Now go through images and count categories
+    category_to_count = defaultdict(int)
+    
+    # im = images[0]
+    for im in images:
+        categories_this_image = image_id_to_category_names[im['id']]
+        for category_name in categories_this_image:
+            category_to_count[category_name] += 1
+
+    ds_name_to_category_counts[ds_name] = category_to_count
+    
+# ...for each dataset
+    
+
+#%% Print the results
+
+for ds_name in ds_name_to_category_counts:
+    
+    print('\n** Category counts for {} **\n'.format(ds_name))
+    
+    category_to_count = ds_name_to_category_counts[ds_name]
+    category_to_count = {k: v for k, v in sorted(category_to_count.items(), reverse=True, 
+                                                 key=lambda item: item[1])}
+    
+    for category_name in category_to_count.keys():        
+        print('{}: {}'.format(category_name,category_to_count[category_name]))

megadetector/data_management/lila/lila_common.py

@@ -0,0 +1,300 @@
+"""
+
+lila_common.py
+
+Common constants and functions related to LILA data management/retrieval.
+
+"""
+
+#%% Imports and constants
+
+import os
+import json
+import zipfile
+import pandas as pd
+
+from urllib.parse import urlparse
+
+from megadetector.utils.url_utils import download_url
+from megadetector.utils.path_utils import unzip_file
+from megadetector.utils.ct_utils import is_empty
+
+# LILA camera trap primary metadata file
+lila_metadata_url = 'http://lila.science/wp-content/uploads/2023/06/lila_camera_trap_datasets.csv'
+lila_taxonomy_mapping_url = 'https://lila.science/public/lila-taxonomy-mapping_release.csv'
+lila_all_images_url = 'https://lila.science/public/lila_image_urls_and_labels.csv.zip'
+
+wildlife_insights_page_size = 30000
+wildlife_insights_taxonomy_url = 'https://api.wildlifeinsights.org/api/v1/taxonomy/taxonomies-all?fields=class,order,family,genus,species,authority,taxonomyType,uniqueIdentifier,commonNameEnglish&page[size]={}'.format(
+    wildlife_insights_page_size)
+wildlife_insights_taxonomy_local_json_filename = 'wi_taxonomy.json'
+wildlife_insights_taxonomy_local_csv_filename = \
+    wildlife_insights_taxonomy_local_json_filename.replace('.json','.csv')
+
+# Filenames are consistent across clouds relative to these URLs
+lila_base_urls = {
+    'azure':'https://lilawildlife.blob.core.windows.net/lila-wildlife/',
+    'gcp':'https://storage.googleapis.com/public-datasets-lila/',
+    'aws':'http://us-west-2.opendata.source.coop.s3.amazonaws.com/agentmorris/lila-wildlife/'
+}
+
+lila_cloud_urls = {
+    'azure':'https://lilawildlife.blob.core.windows.net/lila-wildlife/',
+    'gcp':'gs://public-datasets-lila/',
+    'aws':'s3://us-west-2.opendata.source.coop/agentmorris/lila-wildlife/'
+}
+
+for url in lila_base_urls.values():
+    assert url.endswith('/')
+
+
+#%% Common functions
+
+def read_wildlife_insights_taxonomy_mapping(metadata_dir):
+    """
+    Reads the WI taxonomy mapping file, downloading the .json data (and writing to .csv) if necessary.
+    
+    Args:
+        metadata_dir (str): folder to use for temporary LILA metadata files
+        
+    Returns:
+        pd.dataframe: A DataFrame with taxonomy information
+    """
+    
+    wi_taxonomy_csv_path = os.path.join(metadata_dir,wildlife_insights_taxonomy_local_csv_filename)
+    
+    if os.path.exists(wi_taxonomy_csv_path):
+        df = pd.read_csv(wi_taxonomy_csv_path)
+    else:
+        wi_taxonomy_json_path = os.path.join(metadata_dir,wildlife_insights_taxonomy_local_json_filename)
+        download_url(wildlife_insights_taxonomy_url, wi_taxonomy_json_path)
+        with open(wi_taxonomy_json_path,'r') as f:
+            d = json.load(f)
+            
+        # We haven't implemented paging, make sure that's not an issue
+        assert d['meta']['totalItems'] < wildlife_insights_page_size
+            
+        # d['data'] is a list of items that look like:
+        """
+         {'id': 2000003,
+         'class': 'Mammalia',
+         'order': 'Rodentia',
+         'family': 'Abrocomidae',
+         'genus': 'Abrocoma',
+         'species': 'bennettii',
+         'authority': 'Waterhouse, 1837',
+         'commonNameEnglish': "Bennett's Chinchilla Rat",
+         'taxonomyType': 'biological',
+         'uniqueIdentifier': '7a6c93a5-bdf7-4182-82f9-7a67d23f7fe1'}
+        """
+        df = pd.DataFrame(d['data'])
+        df.to_csv(wi_taxonomy_csv_path,index=False)
+        
+    return df
+
+    
+def read_lila_taxonomy_mapping(metadata_dir):
+    """
+    Reads the LILA taxonomy mapping file, downloading the .csv file if necessary.
+    
+    Args:
+        metadata_dir (str): folder to use for temporary LILA metadata files
+        
+    Returns:
+        pd.DataFrame: a DataFrame with one row per identification
+    """
+    
+    p = urlparse(lila_taxonomy_mapping_url)
+    taxonomy_filename = os.path.join(metadata_dir,os.path.basename(p.path))
+    download_url(lila_taxonomy_mapping_url, taxonomy_filename)
+    
+    df = pd.read_csv(lila_taxonomy_mapping_url)
+    
+    return df
+
+   
+def read_lila_metadata(metadata_dir):
+    """
+    Reads LILA metadata (URLs to each dataset), downloading the .csv file if necessary.
+    
+    Args:
+        metadata_dir (str): folder to use for temporary LILA metadata files
+        
+    Returns:
+        dict: a dict mapping dataset names (e.g. "Caltech Camera Traps") to dicts
+        with keys corresponding to the headers in the .csv file, currently:
+        
+        - name
+        - short_name
+        - continent
+        - country
+        - region
+        - image_base_url_relative
+        - metadata_url_relative
+        - bbox_url_relative
+        - image_base_url_gcp
+        - metadata_url_gcp
+        - bbox_url_gcp
+        - image_base_url_aws
+        - metadata_url_aws
+        - bbox_url_aws
+        - image_base_url_azure
+        - metadata_url_azure
+        - box_url_azure
+        - mdv4_results_raw
+        - mdv5b_results_raw
+        - md_results_with_rde
+        - json_filename
+    """
+    
+    # Put the master metadata file in the same folder where we're putting images
+    p = urlparse(lila_metadata_url)
+    metadata_filename = os.path.join(metadata_dir,os.path.basename(p.path))
+    download_url(lila_metadata_url, metadata_filename)
+    
+    df = pd.read_csv(metadata_filename)
+    
+    records = df.to_dict('records')
+    
+    # Parse into a table keyed by dataset name
+    metadata_table = {}
+    
+    # r = records[0]
+    for r in records:
+        if is_empty(r['name']):
+            continue
+        
+        # Convert NaN's to None
+        for k in r.keys():
+            if is_empty(r[k]):
+                r[k] = None
+                
+        metadata_table[r['name']] = r
+    
+    return metadata_table    
+    
+
+def read_lila_all_images_file(metadata_dir):
+    """
+    Downloads if necessary - then unzips if necessary - the .csv file with label mappings for
+    all LILA files, and opens the resulting .csv file as a Pandas DataFrame.
+    
+    Args:
+        metadata_dir (str): folder to use for temporary LILA metadata files
+        
+    Returns:
+        pd.DataFrame: a DataFrame containing one row per identification in a LILA camera trap image
+    """
+        
+    p = urlparse(lila_all_images_url)
+    lila_all_images_zip_filename = os.path.join(metadata_dir,os.path.basename(p.path))
+    download_url(lila_all_images_url, lila_all_images_zip_filename)
+    
+    with zipfile.ZipFile(lila_all_images_zip_filename,'r') as z:
+        files = z.namelist()
+    assert len(files) == 1
+    
+    unzipped_csv_filename = os.path.join(metadata_dir,files[0])
+    if not os.path.isfile(unzipped_csv_filename):
+        unzip_file(lila_all_images_zip_filename,metadata_dir)
+    else:
+        print('{} already unzipped'.format(unzipped_csv_filename))    
+    
+    df = pd.read_csv(unzipped_csv_filename)
+    
+    return df
+
+
+def read_metadata_file_for_dataset(ds_name,
+                                   metadata_dir,
+                                   metadata_table=None,
+                                   json_url=None,
+                                   preferred_cloud='gcp'):
+    """
+    Downloads if necessary - then unzips if necessary - the .json file for a specific dataset.
+    
+    Args:
+        ds_name (str): the name of the dataset for which you want to retrieve metadata (e.g.
+            "Caltech Camera Traps")        
+        metadata_dir (str): folder to use for temporary LILA metadata files
+        metadata_table (dict, optional): an optional dictionary already loaded via
+            read_lila_metadata()
+        json_url (str, optional): the URL of the metadata file, if None will be retrieved
+            via read_lila_metadata()
+        preferred_cloud (str, optional): 'gcp' (default), 'azure', or 'aws'
+        
+    Returns:
+        str: the .json filename on the local disk
+    
+    """
+    
+    assert preferred_cloud in lila_base_urls.keys()
+    
+    if json_url is None:
+        
+        if metadata_table is None:
+            metadata_table = read_lila_metadata(metadata_dir)
+            
+        json_url = metadata_table[ds_name]['metadata_url_' + preferred_cloud]
+    
+    p = urlparse(json_url)
+    json_filename = os.path.join(metadata_dir,os.path.basename(p.path))
+    download_url(json_url, json_filename)
+    
+    # Unzip if necessary
+    if json_filename.endswith('.zip'):
+        
+        with zipfile.ZipFile(json_filename,'r') as z:
+            files = z.namelist()
+        assert len(files) == 1
+        unzipped_json_filename = os.path.join(metadata_dir,files[0])
+        if not os.path.isfile(unzipped_json_filename):
+            unzip_file(json_filename,metadata_dir)        
+        else:
+            print('{} already unzipped'.format(unzipped_json_filename))
+        json_filename = unzipped_json_filename
+    
+    return json_filename
+
+
+#%% Interactive test driver
+
+if False:
+    
+    pass
+
+    #%% Verify that all base URLs exist
+    
+    # LILA camera trap primary metadata file
+    urls = (lila_metadata_url,lila_taxonomy_mapping_url,lila_all_images_url,wildlife_insights_taxonomy_url)
+    
+    from megadetector.utils import url_utils
+    
+    status_codes = url_utils.test_urls(urls,timeout=2.0)
+    assert all([code == 200 for code in status_codes])
+    
+    
+    #%% Verify that the metadata URLs exist for individual datasets
+    
+    metadata_dir = os.path.expanduser('~/lila/metadata')
+    
+    dataset_metadata = read_lila_metadata(metadata_dir)
+    
+    urls_to_test = []
+    
+    # ds_name = next(iter(dataset_metadata.keys()))
+    for ds_name in dataset_metadata.keys():
+        
+        ds_info = dataset_metadata[ds_name]
+        for cloud_name in lila_base_urls.keys():
+            urls_to_test.append(ds_info['metadata_url_' + cloud_name])
+            if ds_info['bbox_url_relative'] != None:
+                urls_to_test.append(ds_info['bbox_url_' + cloud_name])
+            
+    status_codes = url_utils.test_urls(urls_to_test,
+                                       error_on_failure=True,
+                                       n_workers=10,
+                                       pool_type='process',
+                                       timeout=2.0)
+    assert all([code == 200 for code in status_codes])
+    

megadetector/data_management/lila/test_lila_metadata_urls.py

@@ -0,0 +1,132 @@
+"""
+
+test_lila_metadata_urls.py
+
+Test that all the metadata URLs for LILA camera trap datasets are valid, including MegaDetector 
+results files.
+
+Also pick an arbitrary image from each dataset and make sure that URL is valid.
+
+Also picks an arbitrary image from each dataset's MD results and make sure the corresponding URL is valid.
+
+"""
+
+#%% Constants and imports
+
+import json
+import os
+
+from megadetector.data_management.lila.lila_common import read_lila_metadata,\
+    read_metadata_file_for_dataset, read_lila_taxonomy_mapping
+
+# We'll write images, metadata downloads, and temporary files here
+lila_local_base = os.path.expanduser('~/lila')
+
+output_dir = os.path.join(lila_local_base,'lila_metadata_tests')
+os.makedirs(output_dir,exist_ok=True)
+
+metadata_dir = os.path.join(lila_local_base,'metadata')
+os.makedirs(metadata_dir,exist_ok=True)
+
+md_results_dir = os.path.join(lila_local_base,'md_results')
+os.makedirs(md_results_dir,exist_ok=True)
+
+md_results_keys = ['mdv4_results_raw','mdv5a_results_raw','mdv5b_results_raw','md_results_with_rde']
+
+preferred_cloud = 'gcp' # 'azure', 'aws'
+
+
+#%% Load category and taxonomy files
+
+taxonomy_df = read_lila_taxonomy_mapping(metadata_dir)
+
+
+#%% Download and parse the metadata file
+
+metadata_table = read_lila_metadata(metadata_dir)
+
+print('Loaded metadata URLs for {} datasets'.format(len(metadata_table)))
+
+
+#%% Download and extract metadata and MD results for each dataset
+
+for ds_name in metadata_table.keys():    
+
+    metadata_table[ds_name]['json_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
+                                                                         metadata_dir=metadata_dir,
+                                                                         metadata_table=metadata_table)
+    for k in md_results_keys:
+        md_results_url = metadata_table[ds_name][k]
+        if md_results_url is None:
+            metadata_table[ds_name][k + '_filename'] = None
+        else:
+            metadata_table[ds_name][k + '_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
+                                                                        metadata_dir=md_results_dir,
+                                                                        json_url=md_results_url)
+
+
+#%% Build up a list of URLs to test
+
+# Takes ~15 mins, since it has to open all the giant .json files
+
+url_to_source = {}
+
+# The first image in a dataset is disproportionately likely to be human (and thus 404),
+# so we pick a semi-arbitrary image that isn't the first.  How about the 1000th?
+image_index = 1000
+
+# ds_name = list(metadata_table.keys())[0]
+for ds_name in metadata_table.keys():
+    
+    if 'bbox' in ds_name:
+        print('Skipping bbox dataset {}'.format(ds_name))
+        continue
+
+    print('Processing dataset {}'.format(ds_name))
+    
+    json_filename = metadata_table[ds_name]['json_filename']
+    with open(json_filename, 'r') as f:
+        data = json.load(f)
+
+    image_base_url = metadata_table[ds_name]['image_base_url_' + preferred_cloud]
+    assert not image_base_url.endswith('/')
+    # Download a test image
+    test_image_relative_path = data['images'][image_index]['file_name']
+    test_image_url = image_base_url + '/' + test_image_relative_path
+    
+    url_to_source[test_image_url] = ds_name + ' metadata'
+    
+    # Grab an image from the MegaDetector results
+    
+    # k = md_results_keys[2]
+    for k in md_results_keys:
+        k_fn = k + '_filename'
+        if metadata_table[ds_name][k_fn] is not None:
+            with open(metadata_table[ds_name][k_fn],'r') as f:
+                md_results = json.load(f)
+                im = md_results['images'][image_index]
+                md_image_url = image_base_url + '/' + im['file']
+                url_to_source[md_image_url] = ds_name + ' ' + k
+            del md_results
+    del data
+        
+# ...for each dataset
+
+
+#%% Test URLs
+
+from megadetector.utils.url_utils import test_urls
+
+urls_to_test = sorted(url_to_source.keys())
+urls_to_test = [fn.replace('\\','/') for fn in urls_to_test]
+
+status_codes = test_urls(urls_to_test,
+                         error_on_failure=False,
+                         pool_type='thread',
+                         n_workers=10,
+                         timeout=2.0)
+
+for i_url,url in enumerate(urls_to_test):
+    if status_codes[i_url] != 200:
+        print('Status {} for {} ({})'.format(
+            status_codes[i_url],url,url_to_source[url]))