megadetector 5.0.9__py3-none-any.whl → 5.0.11__py3-none-any.whl

data_management/lila/get_lila_image_counts.py

@@ -1,111 +0,0 @@
-"""
-
-get_lila_image_counts.py
-
-Count the number of images and bounding boxes with each label in one or more LILA datasets.
-
-This script doesn't write these counts out anywhere other than the console, it's just intended
-as a template for doing operations like this on LILA data.  get_lila_annotation_counts.py writes 
-information out to a .json file, but it counts *annotations*, not *images*, for each category.
-
-"""
-
-#%% Constants and imports
-
-import json
-import os
-
-from collections import defaultdict
-
-from data_management.lila.lila_common import read_lila_metadata, read_metadata_file_for_dataset
-
-# If None, will use all datasets
-datasets_of_interest = None
-
-# We'll write images, metadata downloads, and temporary files here
-lila_local_base = os.path.expanduser('~/lila')
-
-metadata_dir = os.path.join(lila_local_base,'metadata')
-os.makedirs(metadata_dir,exist_ok=True)
-
-
-#%% Download and parse the metadata file
-
-metadata_table = read_lila_metadata(metadata_dir)
-
-
-#%% Download and extract metadata for the datasets we're interested in
-
-if datasets_of_interest is None:
-    datasets_of_interest = list(metadata_table.keys())
-
-for ds_name in datasets_of_interest:    
-    metadata_table[ds_name]['json_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
-                                                                         metadata_dir=metadata_dir,
-                                                                         metadata_table=metadata_table)
-    
-    
-#%% Count categories
-
-ds_name_to_category_counts = {}
-
-# ds_name = datasets_of_interest[0]
-for ds_name in datasets_of_interest:
-    
-    category_to_image_count = {}
-    category_to_bbox_count = {}
-    
-    print('Counting categories in: ' + ds_name)
-    
-    json_filename = metadata_table[ds_name]['json_filename']
-    with open(json_filename, 'r') as f:
-        data = json.load(f)
-    
-    categories = data['categories']
-    category_ids = [c['id'] for c in categories]
-    for c in categories:
-        category_id_to_name = {c['id']:c['name'] for c in categories}
-    annotations = data['annotations']
-    images = data['images']
-    
-    for category_id in category_ids:        
-        category_name = category_id_to_name[category_id]        
-        category_to_image_count[category_name] = 0
-        category_to_bbox_count[category_name] = 0
-        
-    image_id_to_category_names = defaultdict(set)
-    
-    # Go through annotations, marking each image with the categories that are present
-    #
-    # ann = annotations[0]
-    for ann in annotations:
-        
-        category_name = category_id_to_name[ann['category_id']]
-        image_id_to_category_names[ann['image_id']].add(category_name)
-
-    # Now go through images and count categories
-    category_to_count = defaultdict(int)
-    
-    # im = images[0]
-    for im in images:
-        categories_this_image = image_id_to_category_names[im['id']]
-        for category_name in categories_this_image:
-            category_to_count[category_name] += 1
-
-    ds_name_to_category_counts[ds_name] = category_to_count
-    
-# ...for each dataset
-    
-
-#%% Print the results
-
-for ds_name in ds_name_to_category_counts:
-    
-    print('\n** Category counts for {} **\n'.format(ds_name))
-    
-    category_to_count = ds_name_to_category_counts[ds_name]
-    category_to_count = {k: v for k, v in sorted(category_to_count.items(), reverse=True, 
-                                                 key=lambda item: item[1])}
-    
-    for category_name in category_to_count.keys():        
-        print('{}: {}'.format(category_name,category_to_count[category_name]))

data_management/lila/lila_common.py

@@ -1,300 +0,0 @@
-"""
-
-lila_common.py
-
-Common constants and functions related to LILA data management/retrieval.
-
-"""
-
-#%% Imports and constants
-
-import os
-import json
-import zipfile
-import pandas as pd
-
-from urllib.parse import urlparse
-
-from md_utils.url_utils import download_url
-from md_utils.path_utils import unzip_file
-from md_utils.ct_utils import is_empty
-
-# LILA camera trap primary metadata file
-lila_metadata_url = 'http://lila.science/wp-content/uploads/2023/06/lila_camera_trap_datasets.csv'
-lila_taxonomy_mapping_url = 'https://lila.science/public/lila-taxonomy-mapping_release.csv'
-lila_all_images_url = 'https://lila.science/public/lila_image_urls_and_labels.csv.zip'
-
-wildlife_insights_page_size = 30000
-wildlife_insights_taxonomy_url = 'https://api.wildlifeinsights.org/api/v1/taxonomy/taxonomies-all?fields=class,order,family,genus,species,authority,taxonomyType,uniqueIdentifier,commonNameEnglish&page[size]={}'.format(
-    wildlife_insights_page_size)
-wildlife_insights_taxonomy_local_json_filename = 'wi_taxonomy.json'
-wildlife_insights_taxonomy_local_csv_filename = \
-    wildlife_insights_taxonomy_local_json_filename.replace('.json','.csv')
-
-# Filenames are consistent across clouds relative to these URLs
-lila_base_urls = {
-    'azure':'https://lilawildlife.blob.core.windows.net/lila-wildlife/',
-    'gcp':'https://storage.googleapis.com/public-datasets-lila/',
-    'aws':'http://us-west-2.opendata.source.coop.s3.amazonaws.com/agentmorris/lila-wildlife/'
-}
-
-lila_cloud_urls = {
-    'azure':'https://lilawildlife.blob.core.windows.net/lila-wildlife/',
-    'gcp':'gs://public-datasets-lila/',
-    'aws':'s3://us-west-2.opendata.source.coop/agentmorris/lila-wildlife/'
-}
-
-for url in lila_base_urls.values():
-    assert url.endswith('/')
-
-
-#%% Common functions
-
-def read_wildlife_insights_taxonomy_mapping(metadata_dir):
-    """
-    Reads the WI taxonomy mapping file, downloading the .json data (and writing to .csv) if necessary.
-    
-    Args:
-        metadata_dir (str): folder to use for temporary LILA metadata files
-        
-    Returns:
-        pd.dataframe: A DataFrame with taxonomy information
-    """
-    
-    wi_taxonomy_csv_path = os.path.join(metadata_dir,wildlife_insights_taxonomy_local_csv_filename)
-    
-    if os.path.exists(wi_taxonomy_csv_path):
-        df = pd.read_csv(wi_taxonomy_csv_path)
-    else:
-        wi_taxonomy_json_path = os.path.join(metadata_dir,wildlife_insights_taxonomy_local_json_filename)
-        download_url(wildlife_insights_taxonomy_url, wi_taxonomy_json_path)
-        with open(wi_taxonomy_json_path,'r') as f:
-            d = json.load(f)
-            
-        # We haven't implemented paging, make sure that's not an issue
-        assert d['meta']['totalItems'] < wildlife_insights_page_size
-            
-        # d['data'] is a list of items that look like:
-        """
-         {'id': 2000003,
-         'class': 'Mammalia',
-         'order': 'Rodentia',
-         'family': 'Abrocomidae',
-         'genus': 'Abrocoma',
-         'species': 'bennettii',
-         'authority': 'Waterhouse, 1837',
-         'commonNameEnglish': "Bennett's Chinchilla Rat",
-         'taxonomyType': 'biological',
-         'uniqueIdentifier': '7a6c93a5-bdf7-4182-82f9-7a67d23f7fe1'}
-        """
-        df = pd.DataFrame(d['data'])
-        df.to_csv(wi_taxonomy_csv_path,index=False)
-        
-    return df
-
-    
-def read_lila_taxonomy_mapping(metadata_dir):
-    """
-    Reads the LILA taxonomy mapping file, downloading the .csv file if necessary.
-    
-    Args:
-        metadata_dir (str): folder to use for temporary LILA metadata files
-        
-    Returns:
-        pd.DataFrame: a DataFrame with one row per identification
-    """
-    
-    p = urlparse(lila_taxonomy_mapping_url)
-    taxonomy_filename = os.path.join(metadata_dir,os.path.basename(p.path))
-    download_url(lila_taxonomy_mapping_url, taxonomy_filename)
-    
-    df = pd.read_csv(lila_taxonomy_mapping_url)
-    
-    return df
-
-   
-def read_lila_metadata(metadata_dir):
-    """
-    Reads LILA metadata (URLs to each dataset), downloading the .csv file if necessary.
-    
-    Args:
-        metadata_dir (str): folder to use for temporary LILA metadata files
-        
-    Returns:
-        dict: a dict mapping dataset names (e.g. "Caltech Camera Traps") to dicts
-        with keys corresponding to the headers in the .csv file, currently:
-        
-        - name
-        - short_name
-        - continent
-        - country
-        - region
-        - image_base_url_relative
-        - metadata_url_relative
-        - bbox_url_relative
-        - image_base_url_gcp
-        - metadata_url_gcp
-        - bbox_url_gcp
-        - image_base_url_aws
-        - metadata_url_aws
-        - bbox_url_aws
-        - image_base_url_azure
-        - metadata_url_azure
-        - box_url_azure
-        - mdv4_results_raw
-        - mdv5b_results_raw
-        - md_results_with_rde
-        - json_filename
-    """
-    
-    # Put the master metadata file in the same folder where we're putting images
-    p = urlparse(lila_metadata_url)
-    metadata_filename = os.path.join(metadata_dir,os.path.basename(p.path))
-    download_url(lila_metadata_url, metadata_filename)
-    
-    df = pd.read_csv(metadata_filename)
-    
-    records = df.to_dict('records')
-    
-    # Parse into a table keyed by dataset name
-    metadata_table = {}
-    
-    # r = records[0]
-    for r in records:
-        if is_empty(r['name']):
-            continue
-        
-        # Convert NaN's to None
-        for k in r.keys():
-            if is_empty(r[k]):
-                r[k] = None
-                
-        metadata_table[r['name']] = r
-    
-    return metadata_table    
-    
-
-def read_lila_all_images_file(metadata_dir):
-    """
-    Downloads if necessary - then unzips if necessary - the .csv file with label mappings for
-    all LILA files, and opens the resulting .csv file as a Pandas DataFrame.
-    
-    Args:
-        metadata_dir (str): folder to use for temporary LILA metadata files
-        
-    Returns:
-        pd.DataFrame: a DataFrame containing one row per identification in a LILA camera trap image
-    """
-        
-    p = urlparse(lila_all_images_url)
-    lila_all_images_zip_filename = os.path.join(metadata_dir,os.path.basename(p.path))
-    download_url(lila_all_images_url, lila_all_images_zip_filename)
-    
-    with zipfile.ZipFile(lila_all_images_zip_filename,'r') as z:
-        files = z.namelist()
-    assert len(files) == 1
-    
-    unzipped_csv_filename = os.path.join(metadata_dir,files[0])
-    if not os.path.isfile(unzipped_csv_filename):
-        unzip_file(lila_all_images_zip_filename,metadata_dir)
-    else:
-        print('{} already unzipped'.format(unzipped_csv_filename))    
-    
-    df = pd.read_csv(unzipped_csv_filename)
-    
-    return df
-
-
-def read_metadata_file_for_dataset(ds_name,
-                                   metadata_dir,
-                                   metadata_table=None,
-                                   json_url=None,
-                                   preferred_cloud='gcp'):
-    """
-    Downloads if necessary - then unzips if necessary - the .json file for a specific dataset.
-    
-    Args:
-        ds_name (str): the name of the dataset for which you want to retrieve metadata (e.g.
-            "Caltech Camera Traps")        
-        metadata_dir (str): folder to use for temporary LILA metadata files
-        metadata_table (dict, optional): an optional dictionary already loaded via
-            read_lila_metadata()
-        json_url (str, optional): the URL of the metadata file, if None will be retrieved
-            via read_lila_metadata()
-        preferred_cloud (str, optional): 'gcp' (default), 'azure', or 'aws'
-        
-    Returns:
-        str: the .json filename on the local disk
-    
-    """
-    
-    assert preferred_cloud in lila_base_urls.keys()
-    
-    if json_url is None:
-        
-        if metadata_table is None:
-            metadata_table = read_lila_metadata(metadata_dir)
-            
-        json_url = metadata_table[ds_name]['metadata_url_' + preferred_cloud]
-    
-    p = urlparse(json_url)
-    json_filename = os.path.join(metadata_dir,os.path.basename(p.path))
-    download_url(json_url, json_filename)
-    
-    # Unzip if necessary
-    if json_filename.endswith('.zip'):
-        
-        with zipfile.ZipFile(json_filename,'r') as z:
-            files = z.namelist()
-        assert len(files) == 1
-        unzipped_json_filename = os.path.join(metadata_dir,files[0])
-        if not os.path.isfile(unzipped_json_filename):
-            unzip_file(json_filename,metadata_dir)        
-        else:
-            print('{} already unzipped'.format(unzipped_json_filename))
-        json_filename = unzipped_json_filename
-    
-    return json_filename
-
-
-#%% Interactive test driver
-
-if False:
-    
-    pass
-
-    #%% Verify that all base URLs exist
-    
-    # LILA camera trap primary metadata file
-    urls = (lila_metadata_url,lila_taxonomy_mapping_url,lila_all_images_url,wildlife_insights_taxonomy_url)
-    
-    from md_utils import url_utils
-    
-    status_codes = url_utils.test_urls(urls,timeout=2.0)
-    assert all([code == 200 for code in status_codes])
-    
-    
-    #%% Verify that the metadata URLs exist for individual datasets
-    
-    metadata_dir = os.path.expanduser('~/lila/metadata')
-    
-    dataset_metadata = read_lila_metadata(metadata_dir)
-    
-    urls_to_test = []
-    
-    # ds_name = next(iter(dataset_metadata.keys()))
-    for ds_name in dataset_metadata.keys():
-        
-        ds_info = dataset_metadata[ds_name]
-        for cloud_name in lila_base_urls.keys():
-            urls_to_test.append(ds_info['metadata_url_' + cloud_name])
-            if ds_info['bbox_url_relative'] != None:
-                urls_to_test.append(ds_info['bbox_url_' + cloud_name])
-            
-    status_codes = url_utils.test_urls(urls_to_test,
-                                       error_on_failure=True,
-                                       n_workers=10,
-                                       pool_type='process',
-                                       timeout=2.0)
-    assert all([code == 200 for code in status_codes])
-    

data_management/lila/test_lila_metadata_urls.py

@@ -1,132 +0,0 @@
-"""
-
-test_lila_metadata_urls.py
-
-Test that all the metadata URLs for LILA camera trap datasets are valid, including MegaDetector 
-results files.
-
-Also pick an arbitrary image from each dataset and make sure that URL is valid.
-
-Also picks an arbitrary image from each dataset's MD results and make sure the corresponding URL is valid.
-
-"""
-
-#%% Constants and imports
-
-import json
-import os
-
-from data_management.lila.lila_common import read_lila_metadata,\
-    read_metadata_file_for_dataset, read_lila_taxonomy_mapping
-
-# We'll write images, metadata downloads, and temporary files here
-lila_local_base = os.path.expanduser('~/lila')
-
-output_dir = os.path.join(lila_local_base,'lila_metadata_tests')
-os.makedirs(output_dir,exist_ok=True)
-
-metadata_dir = os.path.join(lila_local_base,'metadata')
-os.makedirs(metadata_dir,exist_ok=True)
-
-md_results_dir = os.path.join(lila_local_base,'md_results')
-os.makedirs(md_results_dir,exist_ok=True)
-
-md_results_keys = ['mdv4_results_raw','mdv5a_results_raw','mdv5b_results_raw','md_results_with_rde']
-
-preferred_cloud = 'gcp' # 'azure', 'aws'
-
-
-#%% Load category and taxonomy files
-
-taxonomy_df = read_lila_taxonomy_mapping(metadata_dir)
-
-
-#%% Download and parse the metadata file
-
-metadata_table = read_lila_metadata(metadata_dir)
-
-print('Loaded metadata URLs for {} datasets'.format(len(metadata_table)))
-
-
-#%% Download and extract metadata and MD results for each dataset
-
-for ds_name in metadata_table.keys():    
-
-    metadata_table[ds_name]['json_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
-                                                                         metadata_dir=metadata_dir,
-                                                                         metadata_table=metadata_table)
-    for k in md_results_keys:
-        md_results_url = metadata_table[ds_name][k]
-        if md_results_url is None:
-            metadata_table[ds_name][k + '_filename'] = None
-        else:
-            metadata_table[ds_name][k + '_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
-                                                                        metadata_dir=md_results_dir,
-                                                                        json_url=md_results_url)
-
-
-#%% Build up a list of URLs to test
-
-# Takes ~15 mins, since it has to open all the giant .json files
-
-url_to_source = {}
-
-# The first image in a dataset is disproportionately likely to be human (and thus 404),
-# so we pick a semi-arbitrary image that isn't the first.  How about the 1000th?
-image_index = 1000
-
-# ds_name = list(metadata_table.keys())[0]
-for ds_name in metadata_table.keys():
-    
-    if 'bbox' in ds_name:
-        print('Skipping bbox dataset {}'.format(ds_name))
-        continue
-
-    print('Processing dataset {}'.format(ds_name))
-    
-    json_filename = metadata_table[ds_name]['json_filename']
-    with open(json_filename, 'r') as f:
-        data = json.load(f)
-
-    image_base_url = metadata_table[ds_name]['image_base_url_' + preferred_cloud]
-    assert not image_base_url.endswith('/')
-    # Download a test image
-    test_image_relative_path = data['images'][image_index]['file_name']
-    test_image_url = image_base_url + '/' + test_image_relative_path
-    
-    url_to_source[test_image_url] = ds_name + ' metadata'
-    
-    # Grab an image from the MegaDetector results
-    
-    # k = md_results_keys[2]
-    for k in md_results_keys:
-        k_fn = k + '_filename'
-        if metadata_table[ds_name][k_fn] is not None:
-            with open(metadata_table[ds_name][k_fn],'r') as f:
-                md_results = json.load(f)
-                im = md_results['images'][image_index]
-                md_image_url = image_base_url + '/' + im['file']
-                url_to_source[md_image_url] = ds_name + ' ' + k
-            del md_results
-    del data
-        
-# ...for each dataset
-
-
-#%% Test URLs
-
-from md_utils.url_utils import test_urls
-
-urls_to_test = sorted(url_to_source.keys())
-urls_to_test = [fn.replace('\\','/') for fn in urls_to_test]
-
-status_codes = test_urls(urls_to_test,
-                         error_on_failure=False,
-                         pool_type='thread',
-                         n_workers=10,
-                         timeout=2.0)
-
-for i_url,url in enumerate(urls_to_test):
-    if status_codes[i_url] != 200:
-        print('Status {} for {} ({})'.format(
-            status_codes[i_url],url,url_to_source[url]))