megadetector 5.0.28__py3-none-any.whl → 10.0.0__py3-none-any.whl

megadetector/data_management/importers/zamba_results_to_md_results.py

@@ -1,180 +0,0 @@
-"""
-
- zamba_results_to_md_results.py
-
- Convert a labels.csv file produced by Zamba Cloud to a MD results file suitable
- for import into Timelapse.
-
- Columns are expected to be:
-
- video_uuid (not used)
- original_filename (assumed to be a relative path name)
- top_k_label,top_k_probability, for k = 1..N
- [category name 1],[category name 2],...
- corrected_label
-
- Because the MD results file fundamentally stores detections, what we'll
- actually do is create bogus detections that fill the entire image.
-
- There is no special handling of empty/blank categories; because these results are
- based on a classifier, rather than a detector (where "blank" would be the absence of
- all other categories), "blank" can be queried in Timelapse just like any other class.
-
-"""
-
-#%% Imports and constants
-
-import pandas as pd
-import json
-
-
-#%% Main function
-
-def zamba_results_to_md_results(input_file,output_file=None):
-    """
-    Converts the .csv file [input_file] to the MD-formatted .json file [output_file].
-    
-    If [output_file] is None, '.json' will be appended to the input file.
-    """
-    
-    if output_file is None:
-        output_file = input_file + '.json'
-
-    df = pd.read_csv(input_file)
-    
-    expected_columns = ('video_uuid','corrected_label','original_filename')
-    for s in expected_columns:
-        assert s in df.columns,\
-            'Expected column {} not found, are you sure this is a Zamba results .csv file?'.format(
-                s)
-            
-    # How many results are included per file?
-    assert 'top_1_probability' in df.columns and 'top_1_label' in df.columns
-    top_k = 2
-    while(True):
-        p_string = 'top_' + str(top_k) + '_probability'
-        label_string = 'top_' + str(top_k) + '_label'
-        
-        if p_string in df.columns:
-            assert label_string in df.columns,\
-                'Oops, {} is a column but {} is not'.format(
-                    p_string,label_string)
-            top_k += 1
-            continue
-        else:
-            assert label_string not in df.columns,\
-                'Oops, {} is a column but {} is not'.format(
-                    label_string,p_string)
-            top_k -= 1
-            break
-            
-    print('Found {} probability column pairs'.format(top_k))
-    
-    # Category names start after the fixed columns and the probability columns
-    category_names = []
-    column_names = list(df.columns)
-    first_category_name_index = 0
-    while('top_' in column_names[first_category_name_index] or \
-          column_names[first_category_name_index] in expected_columns):
-        first_category_name_index += 1
-        
-    i_column = first_category_name_index
-    while( (i_column < len(column_names)) and (column_names[i_column] != 'corrected_label') ):
-        category_names.append(column_names[i_column])
-        i_column += 1
-        
-    print('Found {} categories:\n'.format(len(category_names)))
-    
-    for s in category_names:
-        print(s)
-    
-    info = {}
-    info['format_version'] = '1.3'
-    info['detector'] = 'Zamba Cloud'
-    info['classifier'] = 'Zamba Cloud'    
-    
-    detection_category_id_to_name = {}
-    for category_id,category_name in enumerate(category_names):
-        detection_category_id_to_name[str(category_id)] = category_name
-    detection_category_name_to_id = {v: k for k, v in detection_category_id_to_name.items()}
-    
-    images = []
-    
-    # i_row = 0; row = df.iloc[i_row]
-    for i_row,row in df.iterrows():
-        
-        im = {}
-        images.append(im)
-        im['file'] = row['original_filename']
-        
-        detections = []
-        
-        # k = 1
-        for k in range(1,top_k+1):
-            label = row['top_{}_label'.format(k)]
-            confidence = row['top_{}_probability'.format(k)]
-            det = {}
-            det['category'] = detection_category_name_to_id[label]
-            det['conf'] = confidence
-            det['bbox'] = [0,0,1.0,1.0]
-            detections.append(det)
-            
-        im['detections'] = detections
-        
-    # ...for each row
-    
-    results = {}
-    results['info'] = info
-    results['detection_categories'] = detection_category_id_to_name
-    results['images'] = images
-    
-    with open(output_file,'w') as f:
-        json.dump(results,f,indent=1)
-        
-# ...zamba_results_to_md_results(...)
-        
-
-#%% Interactive driver
-
-if False:
-
-    pass
-
-    #%%
-    
-    input_file = r"G:\temp\labels-job-b95a4b76-e332-4e17-ab40-03469392d36a-2023-11-04_16-28-50.060130.csv"
-    output_file = None
-    zamba_results_to_md_results(input_file,output_file)
-    
-
-#%% Command-line driver
-
-import sys,argparse
-
-def main():
-
-    parser = argparse.ArgumentParser(
-        description='Convert a Zamba-formatted .csv results file to a MD-formatted .json results file')
-    
-    parser.add_argument(
-        'input_file',
-        type=str,
-        help='input .csv file')
-    
-    parser.add_argument(
-        '--output_file',
-        type=str,
-        default=None,
-        help='output .json file (defaults to input file appended with ".json")')
-    
-    if len(sys.argv[1:]) == 0:
-        parser.print_help()
-        parser.exit()
-
-    args = parser.parse_args()
-
-    zamba_results_to_md_results(args.input_file,args.output_file)
-    
-if __name__ == '__main__':
-    main()
-    

megadetector/data_management/lila/add_locations_to_island_camera_traps.py

@@ -1,101 +0,0 @@
-"""
-
-add_locations_to_island_camera_traps.py
-
-The Island Conservation Camera Traps dataset had unique camera identifiers embedded
-in filenames, but not in the proper metadata fields.  This script copies that information
-to metadata.
-
-"""
-
-#%% Imports and constants
-
-import os
-import json
-from tqdm import tqdm
-
-input_fn = os.path.expanduser('~/lila/metadata/island_conservation.json')
-output_fn = os.path.expanduser('~/tmp/island_conservation.json')
-preview_folder = os.path.expanduser('~/tmp/island_conservation_preview')
-image_directory = os.path.expanduser('~/data/icct/public/')
-
-
-#%% Prevent imports during testing
-
-if False:
-
-    #%% Read input file
-    
-    with open(input_fn,'r') as f:
-        d = json.load(f)
-        
-    d['info']
-    d['info']['version'] = '1.01'
-    
-    
-    #%% Find locations
-    
-    images = d['images']
-    
-    locations = set()
-    
-    for i_image,im in tqdm(enumerate(images),total=len(images)):
-        tokens_fn = im['file_name'].split('/')
-        tokens_id = im['id'].split('_')
-        assert tokens_fn[0] == tokens_id[0]
-        assert tokens_fn[1] == tokens_id[1]
-        location = tokens_fn[0] + '_' + tokens_fn[1]
-        im['location'] = location
-        locations.add(location)
-    
-    locations = sorted(list(locations))
-        
-    for s in locations:
-        print(s)
-        
-        
-    #%% Write output file
-    
-    with open(output_fn,'w') as f:
-        json.dump(d,f,indent=1)
-        
-    
-    #%% Validate .json files
-    
-    from megadetector.data_management.databases import integrity_check_json_db
-    
-    options = integrity_check_json_db.IntegrityCheckOptions()
-    options.baseDir = image_directory
-    options.bCheckImageSizes = False
-    options.bCheckImageExistence = True
-    options.bFindUnusedImages = True
-    
-    sorted_categories, data, error_info = integrity_check_json_db.integrity_check_json_db(output_fn, options)
-    
-    
-    #%% Preview labels
-    
-    from megadetector.visualization import visualize_db
-    
-    viz_options = visualize_db.DbVizOptions()
-    viz_options.num_to_visualize = 2000
-    viz_options.trim_to_images_with_bboxes = False
-    viz_options.add_search_links = False
-    viz_options.sort_by_filename = False
-    viz_options.parallelize_rendering = True
-    viz_options.classes_to_exclude = ['test']
-    html_output_file, image_db = visualize_db.visualize_db(db_path=output_fn,
-                                                             output_dir=preview_folder,
-                                                             image_base_dir=image_directory,
-                                                             options=viz_options)
-    
-    from megadetector.utils import path_utils
-    path_utils.open_file(html_output_file)
-    
-    
-    #%% Zip output file
-    
-    from megadetector.utils.path_utils import zip_file
-    
-    zip_file(output_fn, verbose=True)
-    assert os.path.isfile(output_fn + '.zip')

megadetector/data_management/lila/add_locations_to_nacti.py

@@ -1,151 +0,0 @@
-"""
-
-add_locations_to_nacti.py
-
-As of 10.2023, NACTI metadata only has very coarse location information (e.g. "Florida"),
-but camera IDs are embedded in filenames.  This script pulls that information from filenames
-and adds it to metadata.
-
-"""
-
-#%% Imports and constants
-
-import os
-import json
-import shutil
-
-from tqdm import tqdm
-from collections import defaultdict
-
-input_file = r'd:\lila\nacti\nacti_metadata.json.1.13\nacti_metadata.json'
-output_file = r'g:\temp\nacti_metadata.1.14.json'
-
-
-#%% Prevent execution during testing
-
-if False:
-
-    #%% Read metadata
-    
-    with open(input_file,'r') as f:
-        d = json.load(f)
-        
-    assert d['info']['version'] == 1.13
-    
-    
-    #%% Map images to locations (according to the metadata)
-    
-    file_name_to_original_location = {}
-    
-    # im = dataset_labels['images'][0]
-    for im in tqdm(d['images']):
-        file_name_to_original_location[im['file_name']] = im['location']
-    
-    original_locations = set(file_name_to_original_location.values())
-    
-    print('Found {} locations in the original metadata:'.format(len(original_locations)))
-    for loc in original_locations:
-        print('[{}]'.format(loc))
-    
-    
-    #%% Map images to new locations
-    
-    def path_to_location(relative_path):
-        
-        relative_path = relative_path.replace('\\','/')
-        if relative_path in file_name_to_original_location:
-            location_name = file_name_to_original_location[relative_path]
-            if location_name == 'San Juan Mntns, Colorado':
-                # "part0/sub000/2010_Unit150_Ivan097_img0003.jpg"
-                tokens = relative_path.split('/')[-1].split('_')
-                assert tokens[1].startswith('Unit')
-                location_name = 'sanjuan_{}_{}_{}'.format(tokens[0],tokens[1],tokens[2])
-            elif location_name == 'Lebec, California':
-                # "part0/sub035/CA-03_08_13_2015_CA-03_0009738.jpg"
-                tokens = relative_path.split('/')[-1].split('_')
-                assert tokens[0].startswith('CA-') or tokens[0].startswith('TAG-')
-                location_name = 'lebec_{}'.format(tokens[0])   
-            elif location_name == 'Archbold, FL':
-                # "part1/sub110/FL-01_01_25_2016_FL-01_0040421.jpg"
-                tokens = relative_path.split('/')[-1].split('_')
-                assert tokens[0].startswith('FL-')
-                location_name = 'archbold_{}'.format(tokens[0])   
-            else:
-                assert location_name == ''
-                tokens = relative_path.split('/')[-1].split('_')
-                if tokens[0].startswith('CA-') or tokens[0].startswith('TAG-') or tokens[0].startswith('FL-'):
-                    location_name = '{}'.format(tokens[0])
-                
-        else:
-            
-            location_name = 'unknown'    
-        
-        # print('Returning location {} for file {}'.format(location_name,relative_path))          
-    
-        return location_name
-    
-    file_name_to_updated_location = {}
-    updated_location_to_count = defaultdict(int)
-    for im in tqdm(d['images']):
-        
-        updated_location = path_to_location(im['file_name'])
-        file_name_to_updated_location[im['file_name']] = updated_location
-        updated_location_to_count[updated_location] += 1
-    
-    updated_location_to_count = {k: v for k, v in sorted(updated_location_to_count.items(), 
-                             key=lambda item: item[1],
-                             reverse=True)}
-    
-    updated_locations = set(file_name_to_updated_location.values())
-    
-    print('Found {} updated locations in the original metadata:'.format(len(updated_locations)))
-    for loc in updated_location_to_count:
-        print('{}: {}'.format(loc,updated_location_to_count[loc]))
-    
-    
-    #%% Re-write metadata
-    
-    for im in d['images']:
-        im['location'] = file_name_to_updated_location[im['file_name']]
-    d['info']['version'] = 1.14
-    
-    with open(output_file,'w') as f:
-        json.dump(d,f,indent=1)
-        
-    
-    #%% For each location, sample some random images to make sure they look consistent
-    
-    input_base = r'd:\lila\nacti-unzipped'
-    assert os.path.isdir(input_base)
-    
-    location_to_images = defaultdict(list)
-    
-    for im in d['images']:
-        location_to_images[im['location']].append(im)
-        
-    n_to_sample = 10
-    import random
-    random.seed(0)
-    sampling_folder_base = r'g:\temp\nacti_samples'
-    
-    for location in tqdm(location_to_images):
-        
-        images_this_location = location_to_images[location]
-        if len(images_this_location) > n_to_sample:
-            images_this_location = random.sample(images_this_location,n_to_sample)
-            
-        for i_image,im in enumerate(images_this_location):
-            
-            fn_relative = im['file_name']
-            source_fn_abs = os.path.join(input_base,fn_relative)
-            assert os.path.isfile(source_fn_abs)
-            ext = os.path.splitext(fn_relative)[1]            
-            target_fn_abs = os.path.join(sampling_folder_base,'{}/{}'.format(
-                location,'image_{}{}'.format(str(i_image).zfill(2),ext)))
-            os.makedirs(os.path.dirname(target_fn_abs),exist_ok=True)
-            shutil.copyfile(source_fn_abs,target_fn_abs)
-            
-        # ...for each image
-        
-    # ...for each location
-                

megadetector/utils/azure_utils.py

@@ -1,178 +0,0 @@
-"""
-
-azure_utils.py
-
-Miscellaneous Azure Blob Storage utilities
-
-Requires azure-storage-blob>=12.4.0
-
-"""
-
-#%% Imports
-
-import json
-
-from typing import Any, Iterable, List, Optional, Tuple, Union
-from azure.storage.blob import BlobPrefix, ContainerClient
-
-from megadetector.utils import path_utils
-from megadetector.utils import sas_blob_utils
-
-
-#%% Functions
-
-def walk_container(container_client: ContainerClient,
-                   max_depth: int = -1,
-                   prefix: str = '',
-                   store_folders: bool = True,
-                   store_blobs: bool = True,
-                   debug_max_items: int = -1) -> Tuple[List[str], List[str]]:
-    """
-    Recursively walk folders a Azure Blob Storage container.
-
-    Based on:
-    https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/storage/azure-storage-blob/samples/blob_samples_walk_blob_hierarchy.py
-    """
-    
-    depth = 1
-
-    def walk_blob_hierarchy(prefix: str,
-                            folders: Optional[List[str]] = None,
-                            blobs: Optional[List[str]] = None
-                            ) -> Tuple[List[str], List[str]]:
-        if folders is None:
-            folders = []
-        if blobs is None:
-            blobs = []
-
-        nonlocal depth
-
-        if 0 < max_depth < depth:
-            return folders, blobs
-
-        for item in container_client.walk_blobs(name_starts_with=prefix):
-            short_name = item.name[len(prefix):]
-            if isinstance(item, BlobPrefix):
-                # print('F: ' + prefix + short_name)
-                if store_folders:
-                    folders.append(prefix + short_name)
-                depth += 1
-                walk_blob_hierarchy(item.name, folders=folders, blobs=blobs)
-                if (debug_max_items > 0
-                        and len(folders) + len(blobs) > debug_max_items):
-                    return folders, blobs
-                depth -= 1
-            else:
-                if store_blobs:
-                    blobs.append(prefix + short_name)
-
-        return folders, blobs
-
-    folders, blobs = walk_blob_hierarchy(prefix=prefix)
-
-    assert all(s.endswith('/') for s in folders)
-    folders = [s.strip('/') for s in folders]
-
-    return folders, blobs
-
-
-def list_top_level_blob_folders(container_client: ContainerClient) -> List[str]:
-    """
-    List all top-level folders in a container.
-    """
-    
-    top_level_folders, _ = walk_container(
-        container_client, max_depth=1, store_blobs=False)
-    return top_level_folders
-
-
-def concatenate_json_lists(input_files: Iterable[str],
-                           output_file: Optional[str] = None
-                           ) -> List[Any]:
-    """
-    Given a list of JSON files that contain lists (typically string
-    filenames), concatenates the lists into a single list and optionally
-    writes out this list to a new output JSON file.
-    """
-    
-    output_list = []
-    for fn in input_files:
-        with open(fn, 'r') as f:
-            file_list = json.load(f)
-        output_list.extend(file_list)
-    if output_file is not None:
-        with open(output_file, 'w') as f:
-            json.dump(output_list, f, indent=1)
-    return output_list
-
-
-def upload_file_to_blob(account_name: str,
-                        container_name: str,
-                        local_path: str,
-                        blob_name: str,
-                        sas_token: str,
-                        overwrite: bool=False) -> str:
-    """
-    Uploads a local file to Azure Blob Storage and returns the uploaded
-    blob URI with SAS token.
-    """
-    
-    container_uri = sas_blob_utils.build_azure_storage_uri(
-        account=account_name, container=container_name, sas_token=sas_token)
-    with open(local_path, 'rb') as data:
-        return sas_blob_utils.upload_blob(
-            container_uri=container_uri, blob_name=blob_name, data=data, 
-            overwrite=overwrite)
-
-
-def enumerate_blobs_to_file(
-        output_file: str,
-        account_name: str,
-        container_name: str,
-        sas_token: Optional[str] = None,
-        blob_prefix: Optional[str] = None,
-        blob_suffix: Optional[Union[str, Tuple[str]]] = None,
-        rsearch: Optional[str] = None,
-        limit: Optional[int] = None,
-        verbose: Optional[bool] = True
-        ) -> List[str]:
-    """
-    Enumerates blobs in a container, and writes the blob names to an output
-    file.
-
-    Args:
-        output_file: str, path to save list of files in container
-            If ends in '.json', writes a JSON string. Otherwise, writes a
-            newline-delimited list. Can be None, in which case this is just a 
-            convenient wrapper for blob enumeration.
-        account_name: str, Azure Storage account name
-        container_name: str, Azure Blob Storage container name
-        sas_token: optional str, container SAS token, leading ? will be removed if present.
-        blob_prefix: optional str, returned results will only contain blob names
-            to with this prefix
-        blob_suffix: optional str or tuple of str, returned results will only
-            contain blob names with this/these suffix(es). The blob names will
-            be lowercased first before comparing with the suffix(es).
-        rsearch: optional str, returned results will only contain blob names
-            that match this regex. Can also be a list of regexes, in which case
-            blobs matching *any* of the regex's will be returned.            
-        limit: int, maximum # of blob names to list
-            if None, then returns all blob names
-
-    Returns: list of str, sorted blob names, of length limit or shorter.
-    """
-    
-    if sas_token is not None and len(sas_token) > 9 and sas_token[0] == '?':
-        sas_token = sas_token[1:]
-        
-    container_uri = sas_blob_utils.build_azure_storage_uri(
-        account=account_name, container=container_name, sas_token=sas_token)
-    
-    matched_blobs = sas_blob_utils.list_blobs_in_container(
-        container_uri=container_uri, blob_prefix=blob_prefix,
-        blob_suffix=blob_suffix, rsearch=rsearch, limit=limit, verbose=verbose)
-    
-    if output_file is not None:
-        path_utils.write_list_to_file(output_file, matched_blobs)
-        
-    return matched_blobs