megadetector 5.0.28__py3-none-any.whl → 5.0.29__py3-none-any.whl

megadetector/data_management/importers/eMammal/copy_and_unzip_emammal.py

@@ -1,180 +0,0 @@
-'''
-copy_and_unzip_emammal.py
-
-Siyu Yang
-
-Script to copy all deployments in the emammal container (mounted on the VM or not) to data
-disk at /datadrive and unzip them, deleting the copied zip file.
-
-Need to add exception handling.
-'''
-
-#%% Imports and constants
-
-from datetime import datetime
-import itertools
-import json
-import multiprocessing
-from multiprocessing.dummy import Pool as ThreadPool  # this functions like threading
-import os
-from shutil import copy, copyfile
-from tqdm import tqdm
-from typing import Optional
-import zipfile
-
-from azure.storage.blob import BlobServiceClient
-
-
-# configurations and paths
-log_folder = '/home/lynx/logs'
-dest_folder = '/datadrive/emammal_robertlong'  # data disk attached where data is stored
-origin = 'cloud'  # 'cloud' or 'mounted'
-
-
-#%% Helper functions
-
-def _copy_unzip(source_path, dest_folder):
-    
-    try:
-        dest_subfolder = os.path.join(dest_folder, os.path.basename(source_path).split('.zip')[0])
-        if os.path.exists(dest_subfolder):
-            print('{} exists.'.format(dest_subfolder))
-            return('exists')
-
-        print('Copying...')
-        # dest_path = copy(source_path, dest_folder)
-        dest_path = os.path.join(dest_folder, os.path.basename(source_path))
-        copyfile(source_path, dest_path)
-
-        with zipfile.ZipFile(dest_path, 'r') as zip_ref:
-            zip_ref.extractall(dest_subfolder)
-
-        os.remove(dest_path)
-        print('{} copied and extracted'.format(dest_subfolder))
-        return None
-
-    except Exception:
-        try:
-            print('Retrying...')
-            dest_path = os.path.join(dest_folder, os.path.basename(source_path))
-            copyfile(source_path, dest_path)
-            with zipfile.ZipFile(dest_path, 'r') as zip_ref:
-                zip_ref.extractall(dest_subfolder)
-            os.remove(dest_path)
-            print('{} copied and extracted'.format(dest_subfolder))
-            return (None)
-        except Exception as e:
-            print('WARNING {} did not get copied. Exception: {}'.format(source_path, str(e)))
-            return source_path
-
-
-def copy_from_mounted_container(source_folder, dest_folder):
-    
-    sources = []
-
-    collections = sorted(os.listdir(source_folder))
-
-    for collection in collections:
-        collection_folder = os.path.join(source_folder, collection)
-        if not os.path.isdir(collection_folder):
-            continue
-
-        print('Processing collection {}'.format(collection))
-
-        for file in tqdm(sorted(os.listdir(collection_folder))):
-            source_path = os.path.join(collection_folder, file)
-            sources.append(source_path)
-
-    # num_workers = multiprocessing.cpu_count()
-    # pool = ThreadPool(num_workers)
-    # results = pool.starmap(_copy_unzip, zip(sources, itertools.repeat(dest_folder)))
-    #
-    # print('Waiting for processes to finish...')
-    # pool.close()
-    # pool.join()
-
-    # sequential
-    results = []
-    for source_path in sources:
-        result = _copy_unzip(source_path, dest_folder)
-        results.append(result)
-
-    cur_time = datetime.now().strftime('%Y%m%d-%H%M%S')
-    with open(os.path.join(log_folder, 'copy_unzip_results_{}.json'.format(cur_time)), 'w') as f:
-        json.dump(results, f)
-
-
-def _download_unzip(blob_service: BlobServiceClient,
-                    container: str,
-                    blob_path: str,
-                    dest_path: str) -> Optional[str]:
-    try:
-        with open(dest_path, 'wb') as f:
-            cc = blob_service.get_container_client(container)
-            print('Downloading...')
-            cc.download_blob(blob_path).readinto(f)
-
-        dest_subfolder = dest_path.split('.zip')[0]
-
-        with zipfile.ZipFile(dest_path, 'r') as zip_ref:
-            zip_ref.extractall(dest_subfolder)
-
-        os.remove(dest_path)
-        print('{} copied and extracted'.format(dest_subfolder))
-        return None
-
-    except Exception as e:
-        print('ERROR while downloading or unzipping {}. Exception: {}'.format(blob_path, str(e)))
-        return blob_path
-
-
-def download_from_container(dest_folder: str,
-                            blob_service: BlobServiceClient,
-                            container: str = 'emammal',
-                            desired_blob_prefix: str = '') -> None:
-    generator = blob_service.get_containre_client(container).list_blobs()
-    desired_blobs = [blob.name for blob in generator
-                     if blob.name.startswith(desired_blob_prefix)]
-
-    print('desired_blobs', desired_blobs)
-
-    results = []
-    for blob_path in tqdm(desired_blobs):
-        blob_name = blob_path.split('/')[2]
-        print('blob_name', blob_name)
-        dest_path = os.path.join(dest_folder, blob_name)
-        print('dest_path', dest_path)
-        result = _download_unzip(blob_service, container, blob_path, dest_path)
-        results.append(result)
-
-    cur_time = datetime.now().strftime('%Y%m%d-%H%M%S')
-    with open(os.path.join(log_folder, 'download_unzip_results_{}.json'.format(cur_time)), 'w') as f:
-        json.dump(results, f)
-
-
-#%% Command-line driver
-        
-if __name__ == '__main__':
-    
-    if origin == 'cloud':
-        container = 'wpz'
-        desired_blob_prefix = 'emammal/0Robert Long/'
-
-    print('Start timing...')
-    start_time = datetime.now()
-
-    if origin == 'mounted':
-        # if the blob container is already mounted on the VM
-        source_folder = '/home/yasiyu/mnt/wildlifeblobssc/emammal'  # blob container mounted
-        copy_from_mounted_container(source_folder, dest_folder)
-
-    elif origin == 'cloud':
-        # or you can download them using the storage Python SDK
-        # store storage account key in environment variable AZ_STORAGE_KEY
-        blob_service = BlobServiceClient(
-            account_url='wildlifeblobssc.blob.core.windows.net',
-            credential=os.environ["AZ_STORAGE_KEY"])
-        download_from_container(dest_folder, blob_service, container=container,
-                                desired_blob_prefix=desired_blob_prefix)
-
-    print('Process took {}.'.format(datetime.now() - start_time))

megadetector/data_management/importers/eMammal/eMammal_helpers.py

@@ -1,249 +0,0 @@
-#
-# eMammal_helpers.py
-#
-# Support functions for processing eMammal metadata
-#
-
-#%% Constants and imports
-
-from PIL import Image, ImageDraw
-import matplotlib.pyplot as plt
-import operator
-from datetime import datetime
-
-
-#%% Support functions
-
-def clean_species_name(common_name):
-    """
-    Converts various forms of "human" to the token "human", and various forms
-    of "empty" to the token "empty"
-    """
-    
-    _people_tags = {
-        'Bicycle',
-        'Calibration Photos',
-        'Camera Trapper',
-        'camera trappper',
-        'camera  trapper',
-        'Homo sapien',
-        'Homo sapiens',
-        'Human, non staff',
-        'Human, non-staff',
-        'camera trappe',
-        'Human non-staff',
-        'Setup Pickup',
-        'Vehicle'
-    }
-    PEOPLE_TAGS = {x.lower() for x in _people_tags}
-
-    _no_animal_tags = {'No Animal', 'no  animal', 'Time Lapse', 'Camera Misfire', 'False trigger', 'Blank'}
-    NO_ANIMAL_TAGS = {x.lower() for x in _no_animal_tags}
-
-    common_name = common_name.lower().strip()
-    if common_name in PEOPLE_TAGS:
-        return 'human'
-
-    if common_name in NO_ANIMAL_TAGS:
-        return 'empty'
-
-    return common_name
-
-
-def clean_frame_number(img_frame):
-    
-    # pad to a total of 3 digits if < 1000, or 4 digits otherwise
-    # img_frame is a string from the xml tree
-    length = len(img_frame)
-
-    assert length > 0
-    assert length < 5
-
-    # length 4 frame order is returned as is, others are left padded to be 3 digit long
-    # we need to make sure img_frame has length 3 when it's < 1000 so we can match it to the iMerit labels
-    if length == 1:
-        return '00' + img_frame
-    elif length == 2:
-        return '0' + img_frame
-    else:  # for '100' and '1000'
-        return img_frame
-
-
-def clean_frame_number_4_digit(img_frame):
-    
-    # pad to a total of 4 digits
-    # img_frame is a string from the xml tree
-    length = len(img_frame)
-
-    assert length > 0
-    assert length < 5
-
-    # length 4 frame order is returned as is, others are left padded to be 3 digit long
-    # we need to make sure img_frame has length 3 when it's < 1000 so we can match it to the iMerit labels
-    if length == 1:
-        return '000' + img_frame
-    elif length == 2:
-        return '00' + img_frame
-    elif length == 3:
-        return '0' + img_frame
-    else:  # for'1000'
-        return img_frame
-
-
-def get_img_size(img_path):
-    """
-    There are ways to size the image without loading it into memory by reading its headers
-    (https://github.com/scardine/image_size), but seems less reliable.
-
-    Returns (-1, -1) if PIL could not open the image
-    """
-
-    try:
-        im = Image.open(img_path)
-        width, height = im.size
-    except:
-        return (-1, -1)
-    return (width, height)
-
-
-def get_total_from_distribution(d):
-    
-    total = 0
-    for key, count in d.items():
-        total += int(key) * count
-    return total
-
-
-def sort_dict_val_desc(d, percent=False):    
-    """ Sort a dictionary by the values in descending order. Returns a list of tuples. """
-    
-    sorted_d = sorted(d.items(), key=operator.itemgetter(1), reverse=True)
-
-    if percent:
-        with_percent = []
-        total = sum([t[1] for t in sorted_d])
-        for k, v in sorted_d:
-            p = '{:.1f}%'.format(100 * float(v) / total)
-            with_percent.append((k, v, p))
-        return with_percent
-
-    return sorted_d
-
-
-def plot_distribution(d, title='', top=15):
-    
-    if top is None or top > len(d):
-        top = len(d)
-
-    sorted_d = sort_dict_val_desc(d)
-
-    top_d = sorted_d[:top]
-    x = [t[0] for t in top_d]
-    y = [t[1] for t in top_d]
-
-    # others column
-    others_d = sorted_d[top:]
-    others_sum = sum([t[1] for t in others_d])
-    x.append('others')
-    y.append(others_sum)
-
-    plt.bar(range(len(x)), y, align='center', facecolor='#57BC90', edgecolor=None)
-    plt.xticks(range(len(x)), x, rotation=90)
-    plt.title(title)
-    plt.show()
-
-
-def plot_histogram(l, title='', max_val=None, bins='auto'):
-    
-    if max_val:
-        l = [x for x in l if x < max_val]
-
-    plt.hist(l, bins=bins, facecolor='#57BC90', edgecolor=None)
-    plt.title(title)
-    plt.show()
-
-
-def draw_bboxes(image, bboxes, classes, thickness=4, show_label=False):    
-    """
-    Draw bounding boxes on top of an image
-    Args:
-        image : Path to image or a loaded PIL image
-        bboxes: A list of bboxes to draw on the image, each bbox is [top left x, top left y, width, height] in relative coordinates
-        classes: A list of classes corresponding to the bboxes
-        thickness: Thickness of the line to draw, minimum is 1
-    Outputs:
-        Image object with the bboxes and class labels annotated
-    """
-    if type(image) is str:
-        img = Image.open(image)
-    else:
-        img = image.copy()
-
-    draw = ImageDraw.Draw(img)
-    img_width, img_height = img.size
-
-    for i in range(len(bboxes)):
-        x_rel, y_rel, w_rel, h_rel = bboxes[i]
-        x = x_rel * img_width  # x and y are the top left
-        y = y_rel * img_height
-        w = w_rel * img_width
-        h = h_rel * img_height
-
-        if show_label:
-            c = classes[i]
-            draw.text((x + 15, y + 15), str(c), fill=(255, 0, 0, 255))
-
-        for j in range(thickness):
-            draw.rectangle(((x + j, y + j), (x + w + j, y + h + j)), outline='red')
-
-    return img
-
-
-def is_daytime(date_time):
-    """ 
-    Returns True if daytime as determined by the input timestamp, a rough 
-    decision based on two seasons
-    """
-    
-    # summer day hours: 6am - 7pm
-    # others day hours: 7am - 6pm
-
-    is_summer = True if date_time.month in [5, 6, 7, 8, 9] else False
-    if is_summer:
-        if date_time.hour >= 6 and date_time.hour <= 19:
-            return True
-        else:
-            return False
-    else:
-        if date_time.hour >= 7 and date_time.hour <= 18:
-            return True
-        else:
-            return False
-
-
-def parse_timestamp(time_str):    
-    """
-    There are three datetime string formats in eMammal, and some have an empty field.
-    Args:
-        time_str: text in the tag ImageDateTime
-
-    Returns:
-        datetime object, error (None if no error)
-    """
-
-    if time_str == '' or time_str is None:
-        return '', 'empty or None'
-    try:
-        res = datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S')
-        return res, None
-    except Exception:
-        try:
-            res = datetime.strptime(time_str, '%Y-%m-%d %H:%M:%S')
-            return res, None
-        except Exception:
-            try:
-                res = datetime.strptime(time_str, '%m/%d/%Y %H:%M')
-                return res, None
-            except:
-                print('WARNING, time_str cannot be parsed {}.'.format(time_str))
-                return time_str, 'cannot be parsed {}'.format(time_str)  # return original string

megadetector/data_management/importers/eMammal/make_eMammal_json.py

@@ -1,223 +0,0 @@
-#
-# make_eMammal_json.py
-#
-# Produces the COCO-formatted json database for an eMammal dataset, i.e. a 
-# collection of folders, each of which contains a deployment_manifest.xml file.
-#
-# In this process, each image needs to be loaded to size it.
-#
-# To add bounding box annotations to the resulting database, use 
-# add_annotations_to_eMammal_json.py.
-#
-
-#%% Constants and imports
-
-# Either add the eMammal directory to your path, or run from there
-# os.chdir(r'd:\git\MegaDetector\database_tools\eMammal')
-
-import json
-import multiprocessing
-import os
-# import warnings
-import eMammal_helpers as helpers
-
-from datetime import datetime, date
-from multiprocessing.dummy import Pool as ThreadPool
-from lxml import etree
-from tqdm import tqdm
-
-# ignoring all "PIL cannot read EXIF metainfo for the images" warnings
-# warnings.filterwarnings('ignore')
-
-# Should we run the image size retrieval in parallel?
-run_parallel = False
-
-output_dir_path = r'd:\path'
-deployments_path = r'd:\other_path'
-db_filename = 'apr.json'
-corrupt_images_db_filename = 'apr_corrupt.json'
-
-description = 'description'
-version = '1.0'
-contributor = 'contributor'
-curator = '.json created by curator'
-
-
-#%% Support functions
-
-def _add_image(entry, full_img_path):    
-    """ 
-    Open the image to get size information and add height and width to the image entry. 
-    """
-    
-    img_width, img_height = helpers.get_img_size(full_img_path)
-    if img_width == -1 or img_height == -1:
-        corrupt_images.append(full_img_path)
-        return None
-    entry['width'] = img_width
-    entry['height'] = img_height
-    pbar.update(1)
-    return entry
-
-
-#%% Main loop (metadata processing; image sizes are retrieved later)
-    
-print('Creating tasks to get all images...')
-start_time = datetime.now()
-tasks = []
-folders = os.listdir(deployments_path)
-
-all_species_strings = set()
-
-# deployment = folders[0]
-for deployment in tqdm(folders):
-    
-    deployment_path = os.path.join(deployments_path, deployment)
-    manifest_path = os.path.join(deployment_path, 'deployment_manifest.xml')
-
-    assert os.path.isfile(manifest_path)
-    
-    with open(manifest_path, 'r') as f:
-        tree = etree.parse(f)
-
-    root = tree.getroot()
-    project_id = root.findtext('ProjectId')
-    deployment_id = root.findtext('CameraDeploymentID')
-    deployment_location = root.findtext('CameraSiteName')
-
-    image_sequences = root.findall('ImageSequence')
-    
-    # sequence = image_sequences[0]
-    for sequence in image_sequences:
-        
-        seq_id = sequence.findtext('ImageSequenceId')
-
-        # get species info for this sequence
-        researcher_identifications = sequence.findall('ResearcherIdentifications')
-        species = set()
-
-        for researcher_id in researcher_identifications:
-            identifications = researcher_id.findall('Identification')
-            for id in identifications:
-                species_common_name = helpers.clean_species_name(id.findtext('SpeciesCommonName'))
-                species.add(species_common_name)
-
-        species_str = ';'.join(sorted(list(species)))
-        all_species_strings.add(species_str)
-        
-        # add each image's info to database
-        images = sequence.findall('Image')
-        seq_num_frames = len(images) # total number of frames in this sequence
-        assert isinstance(seq_num_frames,int) and seq_num_frames > 0 # historical quirk
-        
-        # img = images[0]
-        for img in images:
-            
-            img_id = img.findtext('ImageId')
-            img_file_name = img.findtext('ImageFileName')
-            assert img_file_name.lower().endswith('.jpg')
-
-            img_frame = img.findtext('ImageOrder')
-            if img_frame == '' or img_frame is None:
-                # some manifests don't have the ImageOrder info, but the info is in the file name
-                img_frame = img_file_name.split('i')[1].split('.')[0]
-
-            # full_img_id has no frame info
-            #
-            # frame number only used in requests to iMerit for ordering
-            full_img_id = 'datasetemammal.project{}.deployment{}.seq{}.img{}'.format(
-                project_id, deployment_id, seq_id, img_id)            
-            full_img_path = os.path.join(deployment_path, img_file_name)
-
-            img_datetime, datetime_err = helpers.parse_timestamp(img.findtext('ImageDateTime'))
-            if datetime_err:
-                print('WARNING datetime parsing error for image {}. Error: {}'.format(
-                    full_img_path, datetime_err))
-
-            entry = {
-                'id': full_img_id,
-                'width': 0,  # place holders
-                'height': 0,
-                'file_name': os.path.join(deployment, img_file_name),
-                'location': deployment_location,
-                'datetime': str(img_datetime),
-                'seq_id': seq_id,
-                'frame_num': int(img_frame),
-                'seq_num_frames': seq_num_frames,
-                'label': species_str  # extra field for eMammal
-            }
-
-            tasks.append((entry, full_img_path))
-        
-        # ...for each image
-        
-    # ...for each sequence
-    
-# ...for each deployment
-    
-print('Finished creating tasks to get images.')
-
-
-#%% Get image sizes
-
-# 'tasks' is currently a list of 2-tuples, with each entry as [image dictionary,path].
-# 
-# Go through that and copy just the image dictionaries to 'db_images', adding size
-# information to each entry.  Takes a couple hours.
-
-db_images = []
-corrupt_images = []
-pbar = tqdm(total=len(tasks))
-
-if run_parallel:
-    # opening each image seems too fast for this multi-threaded version to be faster than sequential code.
-    num_workers = multiprocessing.cpu_count()
-    pool = ThreadPool(num_workers)
-    db_images = pool.starmap(_add_image, tasks)
-    print('Waiting for image processes to finish...')
-    pool.close()
-    pool.join()
-else:
-    print('Finding image size sequentially')
-    for entry, full_img_path in tasks:
-        db_images.append(_add_image(entry, full_img_path))
-
-db_images = [i for i in db_images if i is not None]
-
-print('{} images could not be opened:'.format(len(corrupt_images)))
-print(corrupt_images)
-print('Done getting image sizes')
-
-
-#%% Assemble top-level dictionaries
-
-db_info = {
-    'year': 'unknown',
-    'version': version,
-    'description': description,
-    'contributor': contributor,
-    'curator': curator,
-    'date_created': str(date.today())
-}
-
-coco_formatted_json = {
-    'info': db_info,
-    'images': db_images
-}
-
-
-#%% Write out .json
-
-print('Saving the json database to disk...')
-with open(os.path.join(output_dir_path, db_filename), 'w') as f:
-    json.dump(coco_formatted_json, f, indent=4, sort_keys=True)
-print('...done')
-
-print('Saving list of corrupt images...')
-with open(os.path.join(output_dir_path, corrupt_images_db_filename), 'w') as f:
-    json.dump(corrupt_images, f, indent=4)    
-print('...done')
-
-print('Running the script took {}.'.format(datetime.now() - start_time))
-
-