megadetector 10.0.13__py3-none-any.whl

megadetector/data_management/lila/create_lila_test_set.py

@@ -0,0 +1,187 @@
+"""
+
+create_lila_test_set.py
+
+Create a test set of camera trap images, containing N empty and N non-empty
+images from each LILA data set.
+
+"""
+
+#%% Constants and imports
+
+import json
+import os
+import random
+
+from megadetector.data_management.lila.lila_common import \
+    read_lila_metadata, read_metadata_file_for_dataset
+from megadetector.utils.url_utils import parallel_download_urls
+from megadetector.utils.path_utils import open_file
+
+n_empty_images_per_dataset = 1
+n_non_empty_images_per_dataset = 1
+
+# We'll write images, metadata downloads, and temporary files here
+lila_local_base = os.path.expanduser('~/lila')
+
+output_dir = os.path.join(lila_local_base,'lila_test_set')
+os.makedirs(output_dir,exist_ok=True)
+
+metadata_dir = os.path.join(lila_local_base,'metadata')
+os.makedirs(metadata_dir,exist_ok=True)
+
+random.seed(0)
+
+
+#%% Download and parse the metadata file
+
+metadata_table = read_lila_metadata(metadata_dir)
+
+
+#%% Download and extract metadata for every dataset
+
+for ds_name in metadata_table.keys():
+    metadata_table[ds_name]['metadata_filename'] = \
+        read_metadata_file_for_dataset(ds_name=ds_name,
+                                       metadata_dir=metadata_dir,
+                                       metadata_table=metadata_table)
+
+
+#%% Choose images from each dataset
+
+# Takes ~60 seconds
+
+empty_category_names = ['empty','blank']
+
+# ds_name = (list(metadata_table.keys()))[0]
+for ds_name in metadata_table.keys():
+
+    print('Choosing images for {}'.format(ds_name))
+
+    json_filename = metadata_table[ds_name]['metadata_filename']
+
+    with open(json_filename,'r') as f:
+        d = json.load(f)
+
+    category_id_to_name = {c['id']:c['name'] for c in d['categories']}
+    category_name_to_id = {c['name']:c['id'] for c in d['categories']}
+
+    ## Find empty images
+
+    empty_category_present = False
+    for category_name in category_name_to_id:
+        if category_name in empty_category_names:
+            empty_category_present = True
+            break
+    if not empty_category_present:
+        empty_annotations_to_download = []
+    else:
+        empty_category_id = None
+        for category_name in empty_category_names:
+            if category_name in category_name_to_id:
+                if empty_category_id is not None:
+                    print('Warning: multiple empty categories in dataset {}'.format(ds_name))
+                else:
+                    empty_category_id = category_name_to_id[category_name]
+        assert empty_category_id is not None
+        empty_annotations = [ann for ann in d['annotations'] if ann['category_id'] == empty_category_id]
+        try:
+            empty_annotations_to_download = random.sample(empty_annotations,n_empty_images_per_dataset)
+        except ValueError:
+            print('No empty images available for dataset {}'.format(ds_name))
+            empty_annotations_to_download = []
+
+    ## Find non-empty images
+
+    non_empty_annotations = [ann for ann in d['annotations'] if ann['category_id'] != empty_category_id]
+    try:
+        non_empty_annotations_to_download = random.sample(non_empty_annotations,n_non_empty_images_per_dataset)
+    except ValueError:
+        print('No non-empty images available for dataset {}'.format(ds_name))
+        non_empty_annotations_to_download = []
+
+
+    annotations_to_download = empty_annotations_to_download + non_empty_annotations_to_download
+
+    image_ids_to_download = set([ann['image_id'] for ann in annotations_to_download])
+    assert len(image_ids_to_download) == len(set(image_ids_to_download))
+
+    images_to_download = []
+    for im in d['images']:
+        if im['id'] in image_ids_to_download:
+            images_to_download.append(im)
+    assert len(images_to_download) == len(image_ids_to_download)
+
+    metadata_table[ds_name]['images_to_download'] = images_to_download
+
+# ...for each dataset
+
+
+#%% Convert to URLs
+
+preferred_cloud = 'gcp'
+
+# ds_name = (list(metadata_table.keys()))[0]
+for ds_name in metadata_table.keys():
+
+    base_url = metadata_table[ds_name]['image_base_url_' + preferred_cloud]
+    assert not base_url.endswith('/')
+
+    # Retrieve image file names
+    filenames = [im['file_name'] for im in metadata_table[ds_name]['images_to_download']]
+
+    urls_to_download = []
+
+    # Convert to URLs
+    for fn in filenames:
+        url = base_url + '/' + fn
+        urls_to_download.append(url)
+
+    metadata_table[ds_name]['urls_to_download'] = urls_to_download
+
+# ...for each dataset
+
+
+#%% Download image files (prep)
+
+url_to_target_file = {}
+
+# ds_name = (list(metadata_table.keys()))[0]
+for ds_name in metadata_table.keys():
+
+    base_url = metadata_table[ds_name]['image_base_url_' + preferred_cloud]
+    assert not base_url.endswith('/')
+    base_url += '/'
+
+    urls_to_download = metadata_table[ds_name]['urls_to_download']
+
+    # url = urls_to_download[0]
+    for url in urls_to_download:
+
+        assert base_url in url
+        output_file_relative = ds_name.lower().replace(' ','_') + \
+            '_' + url.replace(base_url,'').replace('/','_').replace('\\','_')
+        output_file_absolute = os.path.join(output_dir,output_file_relative)
+        url_to_target_file[url] = output_file_absolute
+
+    # ...for each url
+
+# ...for each dataset
+
+
+#%% Download image files (execution)
+
+download_results = parallel_download_urls(url_to_target_file,
+                                          verbose=False,
+                                          overwrite=False,
+                                          n_workers=20,
+                                          pool_type='thread')
+
+# r = download_results[0]
+for r in download_results:
+   assert r['status'] in ('skipped','success')
+
+
+#%% Open the test test
+
+open_file(output_dir)

megadetector/data_management/lila/create_links_to_md_results_files.py

@@ -0,0 +1,106 @@
+"""
+
+create_links_to_md_results_files.py
+
+One-off script to populate the columns in the camera trap data .csv file that point to MD results.
+
+"""
+
+#%% Imports and constants
+
+import os
+
+import pandas as pd
+
+input_csv_file = r'g:\temp\lila_camera_trap_datasets_no_md_results.csv'
+output_csv_file = r'g:\temp\lila_camera_trap_datasets.csv'
+
+md_results_local_folder = r'g:\temp\lila-md-results'
+md_base_url = 'https://lila.science/public/lila-md-results/'
+assert md_base_url.endswith('/')
+
+# No RDE files for datasets with no location information
+datasets_without_location_info = ('ena24','missouri-camera-traps')
+
+md_results_column_names = ['mdv4_results_raw','mdv5a_results_raw','mdv5b_results_raw','md_results_with_rde']
+
+validate_urls = False
+
+
+#%% Read input data
+
+df = pd.read_csv(input_csv_file)
+for s in md_results_column_names:
+    df[s] = ''
+
+
+#%% Find matching files locally, and create URLs
+
+local_files = os.listdir(md_results_local_folder)
+local_files = [fn for fn in local_files if fn.endswith('.zip')]
+
+# i_row = 0; row = df.iloc[i_row]
+for i_row,row in df.iterrows():
+
+    if not isinstance(row['name'],str):
+        continue
+
+    dataset_shortname = row['short_name']
+    matching_files = [fn for fn in local_files if dataset_shortname in fn]
+
+    # No RDE files for datasets with no location information
+    if dataset_shortname in datasets_without_location_info:
+        assert len(matching_files) == 2
+        mdv5a_files = [fn for fn in matching_files if 'mdv5a' in fn]
+        mdv5b_files = [fn for fn in matching_files if 'mdv5b' in fn]
+        assert len(mdv5a_files) == 1 and len(mdv5b_files) == 1
+        df.loc[i_row,'mdv5a_results_raw'] = md_base_url + mdv5a_files[0]
+        df.loc[i_row,'mdv5b_results_raw'] = md_base_url + mdv5b_files[0]
+    else:
+        # Exclude single-season files for snapshot-serengeti
+        if dataset_shortname == 'snapshot-serengeti':
+            matching_files = [fn for fn in matching_files if '_S' not in fn]
+            assert len(matching_files) == 2
+            assert all(['mdv4' in fn for fn in matching_files])
+            rde_files = [fn for fn in matching_files if 'rde' in fn]
+            raw_files = [fn for fn in matching_files if 'rde' not in fn]
+            assert len(rde_files) == 1 and len(raw_files) == 1
+            df.loc[i_row,'mdv4_results_raw'] = md_base_url + raw_files[0]
+            df.loc[i_row,'md_results_with_rde'] = md_base_url + rde_files[0]
+        else:
+            assert len(matching_files) == 3
+            mdv5a_files = [fn for fn in matching_files if 'mdv5a' in fn and 'rde' not in fn]
+            mdv5b_files = [fn for fn in matching_files if 'mdv5b' in fn and 'rde' not in fn]
+            rde_files = [fn for fn in matching_files if 'rde' in fn]
+            assert len(mdv5a_files) == 1 and len(mdv5b_files) == 1 and len(rde_files) == 1
+            df.loc[i_row,'mdv5a_results_raw'] = md_base_url + mdv5a_files[0]
+            df.loc[i_row,'mdv5b_results_raw'] = md_base_url + mdv5b_files[0]
+            df.loc[i_row,'md_results_with_rde'] = md_base_url + rde_files[0]
+
+    print('Found {} matching files for {}'.format(len(matching_files),dataset_shortname))
+
+# ...for each row
+
+
+#%% Validate URLs
+
+if validate_urls:
+
+    from megadetector.utils.url_utils import test_urls
+
+    urls = set()
+
+    for i_row,row in df.iterrows():
+        for column_name in md_results_column_names:
+            if len(row[column_name]) > 0:
+                assert row[column_name] not in urls
+                urls.add(row[column_name])
+
+    test_urls(urls,error_on_failure=True)
+
+    print('Validated {} URLs'.format(len(urls)))
+
+
+#%% Write new .csv file
+
+df.to_csv(output_csv_file,header=True,index=False)

megadetector/data_management/lila/download_lila_subset.py

@@ -0,0 +1,182 @@
+"""
+
+download_lila_subset.py
+
+Example of how to download a list of files from LILA, e.g. all the files
+in a data set corresponding to a particular species.
+
+"""
+
+#%% Constants and imports
+
+import os
+import random
+
+from tqdm import tqdm
+from collections import defaultdict
+from copy import deepcopy
+
+from megadetector.data_management.lila.lila_common import \
+    read_lila_all_images_file, is_empty, lila_base_urls
+from megadetector.utils.url_utils import parallel_download_urls
+from megadetector.utils.path_utils import open_file
+
+for s in lila_base_urls.values():
+    assert s.endswith('/')
+
+# If any of these strings appear in the common name of a species, we'll download that image
+# species_of_interest = ['grey fox','gray fox','cape fox','red fox','kit fox']
+species_of_interest = ['bear']
+
+# We'll write images, metadata downloads, and temporary files here
+lila_local_base = os.path.expanduser('~/lila')
+
+metadata_dir = os.path.join(lila_local_base,'metadata')
+os.makedirs(metadata_dir,exist_ok=True)
+
+output_dir = os.path.join(lila_local_base,'lila_downloads_by_dataset')
+os.makedirs(output_dir,exist_ok=True)
+
+# Number of concurrent download threads
+n_download_threads = 20
+
+max_images_per_dataset = 10 # None
+
+preferred_provider = 'gcp' # 'azure', 'gcp', 'aws'
+
+random.seed(0)
+
+
+#%% Download and open the giant table of image URLs and labels
+
+# Takes ~2 minutes to download, unzip, and open
+df = read_lila_all_images_file(metadata_dir)
+
+
+#%% Find all the images we want to download
+
+# Takes ~2 minutes
+
+common_name_to_count = defaultdict(int)
+
+ds_name_to_urls = defaultdict(list)
+
+def find_items(row): # noqa
+
+    if is_empty(row['common_name']):
+        return
+
+    match = False
+
+    # This is the only bit of this file that's specific to a particular query.  In this case
+    # we're checking whether each row is on a list of species of interest, but you do you.
+    for species_name in species_of_interest:
+        if species_name in row['common_name']:
+            match = True
+            common_name_to_count[species_name] += 1
+            break
+
+    if match:
+        ds_name_to_urls[row['dataset_name']].append(row['url_' + preferred_provider])
+
+tqdm.pandas()
+_ = df.progress_apply(find_items,axis=1)
+
+# We have a list of URLs for each dataset, flatten them all into a list of URLs
+all_urls = list(ds_name_to_urls.values())
+all_urls = [item for sublist in all_urls for item in sublist]
+print('Found {} matching URLs across {} datasets'.format(len(all_urls),len(ds_name_to_urls)))
+
+for common_name in common_name_to_count:
+    print('{}: {}'.format(common_name,common_name_to_count[common_name]))
+
+ds_name_to_urls_raw = deepcopy(ds_name_to_urls)
+
+
+#%% Optionally trim to a fixed number of URLs per dataset
+
+if max_images_per_dataset is None:
+    pass
+else:
+    # ds_name = next(iter(ds_name_to_urls.keys()))
+    for ds_name in ds_name_to_urls:
+        if len(ds_name_to_urls[ds_name]) > max_images_per_dataset:
+            ds_name_to_urls[ds_name] = random.sample(ds_name_to_urls[ds_name],max_images_per_dataset)
+
+
+#%% Choose target files for each URL
+
+# We have a list of URLs per dataset, flatten that into a single list of URLs
+urls_to_download = set()
+for ds_name in ds_name_to_urls:
+    for url in ds_name_to_urls[ds_name]:
+        urls_to_download.add(url)
+urls_to_download = sorted(list(urls_to_download))
+
+# A URL might look like this:
+#
+# https://storage.googleapis.com/public-datasets-lila/wcs-unzipped/animals/0667/0302.jpg
+#
+# We'll write that to an output file that looks like this (relative to output_dir):
+#
+# wcs-unzipped/animals/0667/0302.jpg
+#
+# ...so we need to remove the base URL to get the target file.
+base_url = lila_base_urls[preferred_provider]
+assert base_url.endswith('/')
+
+url_to_target_file = {}
+
+for url in urls_to_download:
+    assert url.startswith(base_url)
+    target_fn_relative = url.replace(base_url,'')
+    target_fn_abs = os.path.join(output_dir,target_fn_relative)
+    url_to_target_file[url] = target_fn_abs
+
+
+#%% Download image files
+
+download_results = parallel_download_urls(url_to_target_file=url_to_target_file,
+                                          verbose=False,
+                                          overwrite=False,
+                                          n_workers=n_download_threads,
+                                          pool_type='thread')
+
+
+#%% Open output folder
+
+open_file(output_dir)
+
+
+#%% Scrap
+
+if False:
+
+    pass
+
+    #%% Find all the reptiles on LILA
+
+    reptile_rows = df.loc[df['class'] == 'reptilia']
+
+    # i_row = 0; row = reptile_rows.iloc[i_row]
+
+    common_name_to_count = defaultdict(int)
+    dataset_to_count = defaultdict(int)
+    for i_row,row in reptile_rows.iterrows():
+        common_name_to_count[row['common_name']] += 1
+        dataset_to_count[row['dataset_name']] += 1
+
+    from megadetector.utils.ct_utils import sort_dictionary_by_value
+
+    print('Found {} reptiles\n'.format(len(reptile_rows)))
+
+    common_name_to_count = sort_dictionary_by_value(common_name_to_count,reverse=True)
+    dataset_to_count = sort_dictionary_by_value(dataset_to_count,reverse=True)
+
+    print('Common names by count:\n')
+    for k in common_name_to_count:
+        print('{} ({})'.format(k,common_name_to_count[k]))
+
+    print('\nDatasets by count:\n')
+    for k in dataset_to_count:
+        print('{} ({})'.format(k,dataset_to_count[k]))