megadetector 5.0.6__py3-none-any.whl → 5.0.7__py3-none-any.whl

data_management/lila/generate_lila_per_image_labels.py

@@ -22,6 +22,9 @@ import json
 import pandas as pd
 import numpy as np
 import dateparser
+import csv
+import urllib
+import urllib.request
 
 from collections import defaultdict
 from tqdm import tqdm
@@ -30,6 +33,9 @@ from data_management.lila.lila_common import read_lila_metadata, \
     read_metadata_file_for_dataset, \
     read_lila_taxonomy_mapping
 
+from md_utils import write_html_image_list
+from md_utils.path_utils import zip_file
+from md_utils.path_utils import open_file
 from md_utils.url_utils import download_url
 
 # We'll write images, metadata downloads, and temporary files here
@@ -56,7 +62,7 @@ ds_name_to_annotation_level['NACTI'] = 'unknown'
 
 known_unmapped_labels = set(['WCS Camera Traps:#ref!'])
 
-debug_max_images_per_dataset = 0
+debug_max_images_per_dataset = -1
 if debug_max_images_per_dataset > 0:
     print('Running in debug mode')
     output_file = output_file.replace('.csv','_debug.csv')
@@ -72,7 +78,7 @@ if False:
     metadata_table = {k:metadata_table[k]}
 
 
-#%% Download and extract metadata for the datasets we're interested in
+#%% Download and extract metadata for each dataset
 
 for ds_name in metadata_table.keys():    
     metadata_table[ds_name]['metadata_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
@@ -101,8 +107,6 @@ for i_row,row in taxonomy_df.iterrows():
 
 # Takes several hours
 
-import csv
-
 header = ['dataset_name','url','image_id','sequence_id','location_id','frame_num','original_label',\
           'scientific_name','common_name','datetime','annotation_level']
 
@@ -122,7 +126,7 @@ def clearnan(v):
     assert isinstance(v,str)
     return v
 
-with open(output_file,'w') as f:
+with open(output_file,'w',encoding='utf-8',newline='') as f:
     
     csv_writer = csv.writer(f)
     csv_writer.writerow(header)
@@ -334,6 +338,8 @@ with open(output_file,'w') as f:
 
 # ...with open()
 
+print('Processed {} datsets'.format(len(metadata_table)))
+
 
 #%% Read the .csv back
 
@@ -352,6 +358,8 @@ def isint(v):
 
 valid_annotation_levels = set(['sequence','image','unknown'])
 
+# Collect a list of locations within each dataset; we'll use this
+# in the next cell to look for datasets that only have a single location
 dataset_name_to_locations = defaultdict(set)
 
 def check_row(row):
@@ -386,6 +394,8 @@ else:
 
 #%% Check for datasets that have only one location string
 
+# Expected: ENA24, Missouri Camera Traps
+
 for ds_name in dataset_name_to_locations.keys():
     if len(dataset_name_to_locations[ds_name]) == 1:
         print('No location information for {}'.format(ds_name))
@@ -440,8 +450,8 @@ print('Selected {} total images'.format(len(images_to_download)))
 
 # Expect a few errors for images with human or vehicle labels (or things like "ignore" that *could* be humans)
 
-import urllib.request
-
+# TODO: trivially parallelizable
+#
 # i_image = 10; image = images_to_download[i_image]
 for i_image,image in tqdm(enumerate(images_to_download),total=len(images_to_download)):
     
@@ -450,17 +460,17 @@ for i_image,image in tqdm(enumerate(images_to_download),total=len(images_to_down
     image_file = os.path.join(preview_folder,'image_{}'.format(str(i_image).zfill(4)) + ext)
     relative_file = os.path.relpath(image_file,preview_folder)
     try:
-        download_url(url,output_file,verbose=False)
+        download_url(url,image_file,verbose=False)
         image['relative_file'] = relative_file
     except urllib.error.HTTPError:
         print('Image {} does not exist ({}:{})'.format(
             i_image,image['dataset_name'],image['original_label']))
         image['relative_file'] = None
 
+# ...for each image we need to download
 
-#%% Write preview HTML
 
-from md_utils import write_html_image_list
+#%% Write preview HTML
 
 html_filename = os.path.join(preview_folder,'index.html')
 
@@ -475,19 +485,18 @@ for im in images_to_download:
     output_im = {}
     output_im['filename'] = im['relative_file']
     output_im['linkTarget'] = im['url']
-    output_im['title'] = str(im)
+    output_im['title'] = '<b>{}: {}</b><br/><br/>'.format(im['dataset_name'],im['original_label']) + str(im)
     output_im['imageStyle'] = 'width:600px;'
     output_im['textStyle'] = 'font-weight:normal;font-size:100%;'
     html_images.append(output_im)
     
 write_html_image_list.write_html_image_list(html_filename,html_images)
 
-from md_utils.path_utils import open_file
 open_file(html_filename)
 
 
 #%% Zip output file
 
-from md_utils.path_utils import zip_file
+zipped_output_file = zip_file(output_file,verbose=True)
 
-zip_file(output_file,verbose=True)
+print('Zipped {} to {}'.format(output_file,zipped_output_file))

data_management/lila/get_lila_annotation_counts.py

@@ -34,18 +34,9 @@ os.makedirs(metadata_dir,exist_ok=True)
 
 output_file = os.path.join(output_dir,'lila_dataset_to_categories.json')
 
-# Created by get_lila_category_list.py... contains counts for each category
-category_list_dir = os.path.join(lila_local_base,'lila_categories_list')
-lila_dataset_to_categories_file = os.path.join(category_list_dir,'lila_dataset_to_categories.json')
-
-assert os.path.isfile(lila_dataset_to_categories_file)
-
 
 #%% Load category and taxonomy files
 
-with open(lila_dataset_to_categories_file,'r') as f:
-    lila_dataset_to_categories = json.load(f)
-
 taxonomy_df = read_lila_taxonomy_mapping(metadata_dir)
 
 
@@ -55,9 +46,13 @@ ds_query_to_scientific_name = {}
 
 unmapped_queries = set()
 
+datasets_with_taxonomy_mapping = set()
+
 # i_row = 1; row = taxonomy_df.iloc[i_row]; row
 for i_row,row in taxonomy_df.iterrows():
     
+    datasets_with_taxonomy_mapping.add(row['dataset_name'])
+    
     ds_query = row['dataset_name'] + ':' + row['query']
     ds_query = ds_query.lower()
     
@@ -68,13 +63,17 @@ for i_row,row in taxonomy_df.iterrows():
         
     ds_query_to_scientific_name[ds_query] = row['scientific_name']
     
+print('Loaded taxonomy mappings for {} datasets'.format(len(datasets_with_taxonomy_mapping)))
     
+
 #%% Download and parse the metadata file
 
 metadata_table = read_lila_metadata(metadata_dir)
 
+print('Loaded metadata URLs for {} datasets'.format(len(metadata_table)))
+
 
-#%% Download and extract metadata for the datasets we're interested in
+#%% Download and extract metadata for each dataset
 
 for ds_name in metadata_table.keys():    
     metadata_table[ds_name]['json_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
@@ -91,6 +90,11 @@ dataset_to_categories = {}
 # ds_name = 'NACTI'
 for ds_name in metadata_table.keys():
     
+    taxonomy_mapping_available = (ds_name in datasets_with_taxonomy_mapping)
+    
+    if not taxonomy_mapping_available:
+        print('Warning: taxonomy mapping not available for {}'.format(ds_name))
+        
     print('Finding categories in {}'.format(ds_name))
     
     json_filename = metadata_table[ds_name]['json_filename']
@@ -122,6 +126,8 @@ for ds_name in metadata_table.keys():
        # always redundant with the class-level data sets.
        if 'bbox' in ds_name:
            c['scientific_name_from_taxonomy_mapping'] = None
+       elif not taxonomy_mapping_available:
+           c['scientific_name_from_taxonomy_mapping'] = None
        else:
            taxonomy_query_string = ds_name.lower().strip() + ':' + c['name'].lower()
            if taxonomy_query_string not in ds_query_to_scientific_name:

data_management/lila/lila_common.py

@@ -21,7 +21,7 @@ from md_utils.path_utils import unzip_file
 
 # LILA camera trap primary metadata file
 lila_metadata_url = 'http://lila.science/wp-content/uploads/2023/06/lila_camera_trap_datasets.csv'
-lila_taxonomy_mapping_url = 'https://lila.science/wp-content/uploads/2022/07/lila-taxonomy-mapping_release.csv'
+lila_taxonomy_mapping_url = 'https://lila.science/public/lila-taxonomy-mapping_release.csv'
 lila_all_images_url = 'https://lila.science/public/lila_image_urls_and_labels.csv.zip'
 
 wildlife_insights_page_size = 30000
@@ -165,16 +165,18 @@ def read_lila_all_images_file(metadata_dir):
     return df
 
 
-def read_metadata_file_for_dataset(ds_name,metadata_dir,metadata_table=None):
+def read_metadata_file_for_dataset(ds_name,metadata_dir,metadata_table=None,json_url=None):
     """
     Downloads if necessary - then unzips if necessary - the .json file for a specific dataset.
     Returns the .json filename on the local disk.
     """
     
-    if metadata_table is None:
-        metadata_table = read_lila_metadata(metadata_dir)
+    if json_url is None:
         
-    json_url = metadata_table[ds_name]['metadata_url']
+        if metadata_table is None:
+            metadata_table = read_lila_metadata(metadata_dir)
+            
+        json_url = metadata_table[ds_name]['metadata_url']
     
     p = urlparse(json_url)
     json_filename = os.path.join(metadata_dir,os.path.basename(p.path))
@@ -196,25 +198,26 @@ def read_metadata_file_for_dataset(ds_name,metadata_dir,metadata_table=None):
     return json_filename
 
 
-def azure_url_to_gcp_http_url(url):
+def azure_url_to_gcp_http_url(url,error_if_not_azure_url=True):
     """
     Most URLs point to Azure by default, but most files are available on both Azure and GCP.
     This function converts an Azure URL to the corresponding GCP http:// url.
     """
     
-    assert url.startswith(lila_azure_storage_account)
+    if error_if_not_azure_url:
+        assert url.startswith(lila_azure_storage_account)
     gcp_url = url.replace(lila_azure_storage_account,gcp_bucket_api_url,1)
     return gcp_url
 
 
-def azure_url_to_gcp_gs_url(url):
+def azure_url_to_gcp_gs_url(url,error_if_not_azure_url=True):
     """
     Most URLs point to Azure by default, but most files are available on both Azure and GCP.
     This function converts an Azure URL to the corresponding GCP gs:// url.
     """
     
-    return azure_url_to_gcp_http_url(url).replace(gcp_bucket_api_url,
-                                                  gcp_bucket_gs_url,1)
+    return azure_url_to_gcp_http_url(url,error_if_not_azure_url).\
+        replace(gcp_bucket_api_url,gcp_bucket_gs_url,1)
 
 
 #%% Interactive test driver
@@ -261,4 +264,4 @@ if False:
         gcp_url = url.replace(lila_azure_storage_account,gcp_bucket_api_url,1)
         gcp_urls.append(gcp_url)
         
-    status_codes = url_utils.test_urls(gcp_urls)
+    status_codes = url_utils.test_urls(gcp_urls)

data_management/lila/test_lila_metadata_urls.py

@@ -0,0 +1,116 @@
+########
+#
+# test_lila_metadata_urls.py
+#
+# Test that all the metadata URLs for LILA camera trap datasets are valid, and 
+# test that at least one image within each URL is valid, including MegaDetector results
+# files.
+#
+########
+
+#%% Constants and imports
+
+import json
+import os
+
+from data_management.lila.lila_common import read_lila_metadata,\
+    read_metadata_file_for_dataset, read_lila_taxonomy_mapping
+
+# We'll write images, metadata downloads, and temporary files here
+lila_local_base = os.path.expanduser('~/lila')
+
+output_dir = os.path.join(lila_local_base,'lila_metadata_tests')
+os.makedirs(output_dir,exist_ok=True)
+
+metadata_dir = os.path.join(lila_local_base,'metadata')
+os.makedirs(metadata_dir,exist_ok=True)
+
+md_results_dir = os.path.join(lila_local_base,'md_results')
+os.makedirs(md_results_dir,exist_ok=True)
+
+md_results_keys = ['mdv4_results_raw','mdv5a_results_raw','mdv5b_results_raw','md_results_with_rde']
+
+
+#%% Load category and taxonomy files
+
+taxonomy_df = read_lila_taxonomy_mapping(metadata_dir)
+
+
+#%% Download and parse the metadata file
+
+metadata_table = read_lila_metadata(metadata_dir)
+
+print('Loaded metadata URLs for {} datasets'.format(len(metadata_table)))
+
+
+#%% Download and extract metadata and MD results for each dataset
+
+for ds_name in metadata_table.keys():    
+
+    metadata_table[ds_name]['json_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
+                                                                         metadata_dir=metadata_dir,
+                                                                         metadata_table=metadata_table)
+    for k in md_results_keys:
+        md_results_url = metadata_table[ds_name][k]
+        if md_results_url is None:
+            metadata_table[ds_name][k + '_filename'] = None
+        else:
+            metadata_table[ds_name][k + '_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
+                                                                        metadata_dir=md_results_dir,
+                                                                        json_url=md_results_url)
+
+
+#%% Build up a list of URLs to test
+
+url_to_source = {}
+
+# The first image in a dataset is disproportionately likely to be human (and thus 404)
+image_index = 1000
+
+# ds_name = list(metadata_table.keys())[0]
+for ds_name in metadata_table.keys():
+    
+    if 'bbox' in ds_name:
+        print('Skipping bbox dataset {}'.format(ds_name))
+        continue
+
+    print('Processing dataset {}'.format(ds_name))
+    
+    json_filename = metadata_table[ds_name]['json_filename']
+    with open(json_filename, 'r') as f:
+        data = json.load(f)
+
+    image_base_url = metadata_table[ds_name]['image_base_url']
+    assert not image_base_url.endswith('/')
+    # Download a test image
+    test_image_relative_path = data['images'][image_index]['file_name']
+    test_image_url = image_base_url + '/' + test_image_relative_path
+    
+    url_to_source[test_image_url] = ds_name + ' metadata'
+    
+    # k = md_results_keys[2]
+    for k in md_results_keys:
+        k_fn = k + '_filename'
+        if metadata_table[ds_name][k_fn] is not None:
+            with open(metadata_table[ds_name][k_fn],'r') as f:
+                md_results = json.load(f)
+                im = md_results['images'][image_index]
+                md_image_url = image_base_url + '/' + im['file']
+                url_to_source[md_image_url] = ds_name + ' ' + k
+    
+# ...for each dataset
+
+
+#%% Test URLs
+
+from md_utils.url_utils import test_urls
+
+urls_to_test = sorted(url_to_source.keys())
+urls_to_test = [fn.replace('\\','/') for fn in urls_to_test]
+
+status_codes = test_urls(urls_to_test,error_on_failure=False)
+
+for i_url,url in enumerate(urls_to_test):
+    if status_codes[i_url] != 200:
+        print('Status {} for {} ({})'.format(
+            status_codes[i_url],url,url_to_source[url]))

data_management/resize_coco_dataset.py

@@ -49,6 +49,8 @@ def resize_coco_dataset(input_folder,input_filename,
     of the way there, due to what appears to be a slight bias inherent to MD.  If a box extends
     within [right_edge_quantization_threshold] (a small number, from 0 to 1, but probably around 
     0.02) of the right edge of the image, it will be extended to the far right edge.
+    
+    Returns the COCO database with resized images.
     """
     
     # Read input data
@@ -62,7 +64,9 @@ def resize_coco_dataset(input_folder,input_filename,
                                           
     # For each image
     
-    # im = d['images'][1]
+    # TODO: this is trivially parallelizable
+    #
+    # im = d['images'][0]
     for im in tqdm(d['images']):
     
         input_fn_relative = im['file_name']
@@ -143,6 +147,8 @@ def resize_coco_dataset(input_folder,input_filename,
     with open(output_filename,'w') as f:
         json.dump(d,f,indent=1)
     
+    return d
+
 # ...def resize_coco_dataset(...)
     
 
@@ -153,17 +159,13 @@ if False:
     pass
 
     #%% Test resizing
-    
-    # input_filename = os.path.expanduser('~/tmp/labelme_to_coco_test.json')
-    # input_folder = os.path.expanduser('~/data/labelme-json-test')
-    # target_size = (600,-1)
-    
-    input_folder = os.path.expanduser('~/data/usgs-kissel-training')
-    input_filename = os.path.expanduser('~/data/usgs-tegus.json')
+        
+    input_folder = os.path.expanduser('~/data/usgs-tegus/usgs-kissel-training')
+    input_filename = os.path.expanduser('~/data/usgs-tegus/usgs-kissel-training.json')
     target_size = (1600,-1)
     
-    output_filename = insert_before_extension(input_filename,'resized')
-    output_folder = input_folder + '-resized'    
+    output_filename = insert_before_extension(input_filename,'resized-test')
+    output_folder = input_folder + '-resized-test'
     
     correct_size_image_handling = 'rewrite'
     

data_management/yolo_output_to_md_output.py

@@ -61,21 +61,37 @@ from detection.run_detector import CONF_DIGITS, COORD_DIGITS
 
 def read_classes_from_yolo_dataset_file(fn):
     """
-    Read a dictionary mapping integer class IDs to class names from a YOLOv5 dataset.yaml
-    file.
+    Read a dictionary mapping integer class IDs to class names from a YOLOv5/YOLOv8
+    dataset.yaml file or a .json file.  A .json file should contain a dictionary mapping
+    integer category IDs to string category names.
     """
         
-    with open(fn,'r') as f:
-        lines = f.readlines()
-            
-    category_id_to_name = {}
-    pat = '\d+:.+'
-    for s in lines:
-        if re.search(pat,s) is not None:
-            tokens = s.split(':')
-            assert len(tokens) == 2, 'Invalid token in category file {}'.format(fn)
-            category_id_to_name[int(tokens[0].strip())] = tokens[1].strip()
+    if fn.endswith('.yml') or fn.endswith('.yaml'):
+        
+        with open(fn,'r') as f:
+            lines = f.readlines()
+                
+        category_id_to_name = {}
+        pat = '\d+:.+'
+        for s in lines:
+            if re.search(pat,s) is not None:
+                tokens = s.split(':')
+                assert len(tokens) == 2, 'Invalid token in category file {}'.format(fn)
+                category_id_to_name[int(tokens[0].strip())] = tokens[1].strip()
+                
+    elif fn.endswith('.json'):
+        
+        with open(fn,'r') as f:
+            d_in = json.load(f)
+            category_id_to_name = {}
+            for k in d_in.keys():
+                category_id_to_name[int(k)] = d_in[k]
         
+    else:
+        
+        raise ValueError('Unrecognized category file type: {}'.format(fn))
+        
+    assert len(category_id_to_name) > 0, 'Failed to read class mappings from {}'.format(fn)
     return category_id_to_name
     
 
@@ -125,7 +141,8 @@ def yolo_json_output_to_md_output(yolo_json_file, image_folder,
     if image_id_to_error is None:
         image_id_to_error = {}
             
-    print('Converting {} to MD format'.format(yolo_json_file))
+    print('Converting {} to MD format and writing results to {}'.format(
+        yolo_json_file,output_file))
     
     if isinstance(yolo_category_id_to_name,str):
         assert os.path.isfile(yolo_category_id_to_name), \
@@ -194,6 +211,16 @@ def yolo_json_output_to_md_output(yolo_json_file, image_folder,
            
     # ...if image IDs are formatted as integers in YOLO output
     
+    # In a modified version of val.py, we use negative category IDs to indicate an error
+    # that happened during inference (typically truncated images with valid headers,
+    # so corruption was not detected during val.py's initial corruption check pass.
+    for det in detections:
+        if det['category_id'] < 0:
+            assert 'error' in det, 'Negative category ID present with no error string'
+            error_string = det['error']
+            print('Caught inference-time failure {} for image {}'.format(error_string,det['image_id']))
+            image_id_to_error[det['image_id']] = error_string
+            
     output_images = []
     
     # image_file_relative = image_files_relative[10]

data_management/yolo_to_coco.py

@@ -18,6 +18,7 @@ from PIL import Image
 from tqdm import tqdm
 
 from md_utils.path_utils import find_images
+from data_management.yolo_output_to_md_output import read_classes_from_yolo_dataset_file
 
 
 #%% Main conversion function
@@ -25,8 +26,10 @@ from md_utils.path_utils import find_images
 def yolo_to_coco(input_folder,class_name_file,output_file=None):
     """
     Convert the YOLO-formatted data in [input_folder] to a COCO-formatted dictionary,
-    reading class names from the flat list [class_name_file].  Optionally writes the output
-    dataset to [output_file].
+    reading class names from [class_name_file], which can be a flat list with a .txt
+    extension or a YOLO dataset.yml file.  Optionally writes the output dataset to [output_file].
+    
+    Returns a COCO-formatted dictionary.
     """
     
     # Validate input
@@ -35,29 +38,39 @@ def yolo_to_coco(input_folder,class_name_file,output_file=None):
     assert os.path.isfile(class_name_file)
     
     
-    # Class names
+    # Read class names
     
-    with open(class_name_file,'r') as f:
-        lines = f.readlines()
-    assert len(lines) > 0, 'Empty class name file {}'.format(class_name_file)
-    lines = [s.strip() for s in lines]
-    assert len(lines[0]) > 0, 'Empty class name file {} (empty first line)'.format(class_name_file)
+    ext = os.path.splitext(class_name_file)[1][1:]
+    assert ext in ('yml','txt','yaml'), 'Unrecognized class name file type {}'.format(
+        class_name_file)
     
-    # Blank lines should only appear at the end
-    b_found_blank = False
-    for s in lines:
-        if len(s) == 0:
-            b_found_blank = True
-        elif b_found_blank:
-            raise ValueError('Invalid class name file {}, non-blank line after the last blank line'.format(
-                class_name_file))
-            
-    category_id_to_name = {}
+    if ext == 'txt':
         
-    for i_category_id,category_name in enumerate(lines):
-        assert len(category_name) > 0
-        category_id_to_name[i_category_id] = category_name
+        with open(class_name_file,'r') as f:
+            lines = f.readlines()
+        assert len(lines) > 0, 'Empty class name file {}'.format(class_name_file)
+        class_names = [s.strip() for s in lines]
+        assert len(lines[0]) > 0, 'Empty class name file {} (empty first line)'.format(class_name_file)
         
+        # Blank lines should only appear at the end
+        b_found_blank = False
+        for s in lines:
+            if len(s) == 0:
+                b_found_blank = True
+            elif b_found_blank:
+                raise ValueError('Invalid class name file {}, non-blank line after the last blank line'.format(
+                    class_name_file))
+    
+        category_id_to_name = {}        
+        for i_category_id,category_name in enumerate(class_names):
+            assert len(category_name) > 0
+            category_id_to_name[i_category_id] = category_name
+            
+    else:
+        
+        assert ext in ('yml','yaml')
+        category_id_to_name = read_classes_from_yolo_dataset_file(class_name_file)
+            
         
     # Enumerate images