megadetector 5.0.11__py3-none-any.whl → 5.0.13__py3-none-any.whl

megadetector/data_management/importers/eMammal/copy_and_unzip_emammal.py

@@ -0,0 +1,180 @@
+'''
+copy_and_unzip_emammal.py
+
+Siyu Yang
+
+Script to copy all deployments in the emammal container (mounted on the VM or not) to data
+disk at /datadrive and unzip them, deleting the copied zip file.
+
+Need to add exception handling.
+'''
+
+#%% Imports and constants
+
+from datetime import datetime
+import itertools
+import json
+import multiprocessing
+from multiprocessing.dummy import Pool as ThreadPool  # this functions like threading
+import os
+from shutil import copy, copyfile
+from tqdm import tqdm
+from typing import Optional
+import zipfile
+
+from azure.storage.blob import BlobServiceClient
+
+
+# configurations and paths
+log_folder = '/home/lynx/logs'
+dest_folder = '/datadrive/emammal_robertlong'  # data disk attached where data is stored
+origin = 'cloud'  # 'cloud' or 'mounted'
+
+
+#%% Helper functions
+
+def _copy_unzip(source_path, dest_folder):
+    
+    try:
+        dest_subfolder = os.path.join(dest_folder, os.path.basename(source_path).split('.zip')[0])
+        if os.path.exists(dest_subfolder):
+            print('{} exists.'.format(dest_subfolder))
+            return('exists')
+
+        print('Copying...')
+        # dest_path = copy(source_path, dest_folder)
+        dest_path = os.path.join(dest_folder, os.path.basename(source_path))
+        copyfile(source_path, dest_path)
+
+        with zipfile.ZipFile(dest_path, 'r') as zip_ref:
+            zip_ref.extractall(dest_subfolder)
+
+        os.remove(dest_path)
+        print('{} copied and extracted'.format(dest_subfolder))
+        return None
+
+    except Exception:
+        try:
+            print('Retrying...')
+            dest_path = os.path.join(dest_folder, os.path.basename(source_path))
+            copyfile(source_path, dest_path)
+            with zipfile.ZipFile(dest_path, 'r') as zip_ref:
+                zip_ref.extractall(dest_subfolder)
+            os.remove(dest_path)
+            print('{} copied and extracted'.format(dest_subfolder))
+            return (None)
+        except Exception as e:
+            print('WARNING {} did not get copied. Exception: {}'.format(source_path, str(e)))
+            return source_path
+
+
+def copy_from_mounted_container(source_folder, dest_folder):
+    
+    sources = []
+
+    collections = sorted(os.listdir(source_folder))
+
+    for collection in collections:
+        collection_folder = os.path.join(source_folder, collection)
+        if not os.path.isdir(collection_folder):
+            continue
+
+        print('Processing collection {}'.format(collection))
+
+        for file in tqdm(sorted(os.listdir(collection_folder))):
+            source_path = os.path.join(collection_folder, file)
+            sources.append(source_path)
+
+    # num_workers = multiprocessing.cpu_count()
+    # pool = ThreadPool(num_workers)
+    # results = pool.starmap(_copy_unzip, zip(sources, itertools.repeat(dest_folder)))
+    #
+    # print('Waiting for processes to finish...')
+    # pool.close()
+    # pool.join()
+
+    # sequential
+    results = []
+    for source_path in sources:
+        result = _copy_unzip(source_path, dest_folder)
+        results.append(result)
+
+    cur_time = datetime.now().strftime('%Y%m%d-%H%M%S')
+    with open(os.path.join(log_folder, 'copy_unzip_results_{}.json'.format(cur_time)), 'w') as f:
+        json.dump(results, f)
+
+
+def _download_unzip(blob_service: BlobServiceClient,
+                    container: str,
+                    blob_path: str,
+                    dest_path: str) -> Optional[str]:
+    try:
+        with open(dest_path, 'wb') as f:
+            cc = blob_service.get_container_client(container)
+            print('Downloading...')
+            cc.download_blob(blob_path).readinto(f)
+
+        dest_subfolder = dest_path.split('.zip')[0]
+
+        with zipfile.ZipFile(dest_path, 'r') as zip_ref:
+            zip_ref.extractall(dest_subfolder)
+
+        os.remove(dest_path)
+        print('{} copied and extracted'.format(dest_subfolder))
+        return None
+
+    except Exception as e:
+        print('ERROR while downloading or unzipping {}. Exception: {}'.format(blob_path, str(e)))
+        return blob_path
+
+
+def download_from_container(dest_folder: str,
+                            blob_service: BlobServiceClient,
+                            container: str = 'emammal',
+                            desired_blob_prefix: str = '') -> None:
+    generator = blob_service.get_containre_client(container).list_blobs()
+    desired_blobs = [blob.name for blob in generator
+                     if blob.name.startswith(desired_blob_prefix)]
+
+    print('desired_blobs', desired_blobs)
+
+    results = []
+    for blob_path in tqdm(desired_blobs):
+        blob_name = blob_path.split('/')[2]
+        print('blob_name', blob_name)
+        dest_path = os.path.join(dest_folder, blob_name)
+        print('dest_path', dest_path)
+        result = _download_unzip(blob_service, container, blob_path, dest_path)
+        results.append(result)
+
+    cur_time = datetime.now().strftime('%Y%m%d-%H%M%S')
+    with open(os.path.join(log_folder, 'download_unzip_results_{}.json'.format(cur_time)), 'w') as f:
+        json.dump(results, f)
+
+
+#%% Command-line driver
+        
+if __name__ == '__main__':
+    
+    if origin == 'cloud':
+        container = 'wpz'
+        desired_blob_prefix = 'emammal/0Robert Long/'
+
+    print('Start timing...')
+    start_time = datetime.now()
+
+    if origin == 'mounted':
+        # if the blob container is already mounted on the VM
+        source_folder = '/home/yasiyu/mnt/wildlifeblobssc/emammal'  # blob container mounted
+        copy_from_mounted_container(source_folder, dest_folder)
+
+    elif origin == 'cloud':
+        # or you can download them using the storage Python SDK
+        # store storage account key in environment variable AZ_STORAGE_KEY
+        blob_service = BlobServiceClient(
+            account_url='wildlifeblobssc.blob.core.windows.net',
+            credential=os.environ["AZ_STORAGE_KEY"])
+        download_from_container(dest_folder, blob_service, container=container,
+                                desired_blob_prefix=desired_blob_prefix)
+
+    print('Process took {}.'.format(datetime.now() - start_time))

megadetector/data_management/importers/eMammal/eMammal_helpers.py

@@ -0,0 +1,249 @@
+#
+# eMammal_helpers.py
+#
+# Support functions for processing eMammal metadata
+#
+
+#%% Constants and imports
+
+from PIL import Image, ImageDraw
+import matplotlib.pyplot as plt
+import operator
+from datetime import datetime
+
+
+#%% Support functions
+
+def clean_species_name(common_name):
+    """
+    Converts various forms of "human" to the token "human", and various forms
+    of "empty" to the token "empty"
+    """
+    
+    _people_tags = {
+        'Bicycle',
+        'Calibration Photos',
+        'Camera Trapper',
+        'camera trappper',
+        'camera  trapper',
+        'Homo sapien',
+        'Homo sapiens',
+        'Human, non staff',
+        'Human, non-staff',
+        'camera trappe',
+        'Human non-staff',
+        'Setup Pickup',
+        'Vehicle'
+    }
+    PEOPLE_TAGS = {x.lower() for x in _people_tags}
+
+    _no_animal_tags = {'No Animal', 'no  animal', 'Time Lapse', 'Camera Misfire', 'False trigger', 'Blank'}
+    NO_ANIMAL_TAGS = {x.lower() for x in _no_animal_tags}
+
+    common_name = common_name.lower().strip()
+    if common_name in PEOPLE_TAGS:
+        return 'human'
+
+    if common_name in NO_ANIMAL_TAGS:
+        return 'empty'
+
+    return common_name
+
+
+def clean_frame_number(img_frame):
+    
+    # pad to a total of 3 digits if < 1000, or 4 digits otherwise
+    # img_frame is a string from the xml tree
+    length = len(img_frame)
+
+    assert length > 0
+    assert length < 5
+
+    # length 4 frame order is returned as is, others are left padded to be 3 digit long
+    # we need to make sure img_frame has length 3 when it's < 1000 so we can match it to the iMerit labels
+    if length == 1:
+        return '00' + img_frame
+    elif length == 2:
+        return '0' + img_frame
+    else:  # for '100' and '1000'
+        return img_frame
+
+
+def clean_frame_number_4_digit(img_frame):
+    
+    # pad to a total of 4 digits
+    # img_frame is a string from the xml tree
+    length = len(img_frame)
+
+    assert length > 0
+    assert length < 5
+
+    # length 4 frame order is returned as is, others are left padded to be 3 digit long
+    # we need to make sure img_frame has length 3 when it's < 1000 so we can match it to the iMerit labels
+    if length == 1:
+        return '000' + img_frame
+    elif length == 2:
+        return '00' + img_frame
+    elif length == 3:
+        return '0' + img_frame
+    else:  # for'1000'
+        return img_frame
+
+
+def get_img_size(img_path):
+    """
+    There are ways to size the image without loading it into memory by reading its headers
+    (https://github.com/scardine/image_size), but seems less reliable.
+
+    Returns (-1, -1) if PIL could not open the image
+    """
+
+    try:
+        im = Image.open(img_path)
+        width, height = im.size
+    except:
+        return (-1, -1)
+    return (width, height)
+
+
+def get_total_from_distribution(d):
+    
+    total = 0
+    for key, count in d.items():
+        total += int(key) * count
+    return total
+
+
+def sort_dict_val_desc(d, percent=False):    
+    """ Sort a dictionary by the values in descending order. Returns a list of tuples. """
+    
+    sorted_d = sorted(d.items(), key=operator.itemgetter(1), reverse=True)
+
+    if percent:
+        with_percent = []
+        total = sum([t[1] for t in sorted_d])
+        for k, v in sorted_d:
+            p = '{:.1f}%'.format(100 * float(v) / total)
+            with_percent.append((k, v, p))
+        return with_percent
+
+    return sorted_d
+
+
+def plot_distribution(d, title='', top=15):
+    
+    if top is None or top > len(d):
+        top = len(d)
+
+    sorted_d = sort_dict_val_desc(d)
+
+    top_d = sorted_d[:top]
+    x = [t[0] for t in top_d]
+    y = [t[1] for t in top_d]
+
+    # others column
+    others_d = sorted_d[top:]
+    others_sum = sum([t[1] for t in others_d])
+    x.append('others')
+    y.append(others_sum)
+
+    plt.bar(range(len(x)), y, align='center', facecolor='#57BC90', edgecolor=None)
+    plt.xticks(range(len(x)), x, rotation=90)
+    plt.title(title)
+    plt.show()
+
+
+def plot_histogram(l, title='', max_val=None, bins='auto'):
+    
+    if max_val:
+        l = [x for x in l if x < max_val]
+
+    plt.hist(l, bins=bins, facecolor='#57BC90', edgecolor=None)
+    plt.title(title)
+    plt.show()
+
+
+def draw_bboxes(image, bboxes, classes, thickness=4, show_label=False):    
+    """
+    Draw bounding boxes on top of an image
+    Args:
+        image : Path to image or a loaded PIL image
+        bboxes: A list of bboxes to draw on the image, each bbox is [top left x, top left y, width, height] in relative coordinates
+        classes: A list of classes corresponding to the bboxes
+        thickness: Thickness of the line to draw, minimum is 1
+    Outputs:
+        Image object with the bboxes and class labels annotated
+    """
+    if type(image) is str:
+        img = Image.open(image)
+    else:
+        img = image.copy()
+
+    draw = ImageDraw.Draw(img)
+    img_width, img_height = img.size
+
+    for i in range(len(bboxes)):
+        x_rel, y_rel, w_rel, h_rel = bboxes[i]
+        x = x_rel * img_width  # x and y are the top left
+        y = y_rel * img_height
+        w = w_rel * img_width
+        h = h_rel * img_height
+
+        if show_label:
+            c = classes[i]
+            draw.text((x + 15, y + 15), str(c), fill=(255, 0, 0, 255))
+
+        for j in range(thickness):
+            draw.rectangle(((x + j, y + j), (x + w + j, y + h + j)), outline='red')
+
+    return img
+
+
+def is_daytime(date_time):
+    """ 
+    Returns True if daytime as determined by the input timestamp, a rough 
+    decision based on two seasons
+    """
+    
+    # summer day hours: 6am - 7pm
+    # others day hours: 7am - 6pm
+
+    is_summer = True if date_time.month in [5, 6, 7, 8, 9] else False
+    if is_summer:
+        if date_time.hour >= 6 and date_time.hour <= 19:
+            return True
+        else:
+            return False
+    else:
+        if date_time.hour >= 7 and date_time.hour <= 18:
+            return True
+        else:
+            return False
+
+
+def parse_timestamp(time_str):    
+    """
+    There are three datetime string formats in eMammal, and some have an empty field.
+    Args:
+        time_str: text in the tag ImageDateTime
+
+    Returns:
+        datetime object, error (None if no error)
+    """
+
+    if time_str == '' or time_str is None:
+        return '', 'empty or None'
+    try:
+        res = datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S')
+        return res, None
+    except Exception:
+        try:
+            res = datetime.strptime(time_str, '%Y-%m-%d %H:%M:%S')
+            return res, None
+        except Exception:
+            try:
+                res = datetime.strptime(time_str, '%m/%d/%Y %H:%M')
+                return res, None
+            except:
+                print('WARNING, time_str cannot be parsed {}.'.format(time_str))
+                return time_str, 'cannot be parsed {}'.format(time_str)  # return original string

megadetector/data_management/importers/eMammal/make_eMammal_json.py

@@ -0,0 +1,223 @@
+#
+# make_eMammal_json.py
+#
+# Produces the COCO-formatted json database for an eMammal dataset, i.e. a 
+# collection of folders, each of which contains a deployment_manifest.xml file.
+#
+# In this process, each image needs to be loaded to size it.
+#
+# To add bounding box annotations to the resulting database, use 
+# add_annotations_to_eMammal_json.py.
+#
+
+#%% Constants and imports
+
+# Either add the eMammal directory to your path, or run from there
+# os.chdir(r'd:\git\MegaDetector\database_tools\eMammal')
+
+import json
+import multiprocessing
+import os
+# import warnings
+import eMammal_helpers as helpers
+
+from datetime import datetime, date
+from multiprocessing.dummy import Pool as ThreadPool
+from lxml import etree
+from tqdm import tqdm
+
+# ignoring all "PIL cannot read EXIF metainfo for the images" warnings
+# warnings.filterwarnings('ignore')
+
+# Should we run the image size retrieval in parallel?
+run_parallel = False
+
+output_dir_path = r'd:\path'
+deployments_path = r'd:\other_path'
+db_filename = 'apr.json'
+corrupt_images_db_filename = 'apr_corrupt.json'
+
+description = 'description'
+version = '1.0'
+contributor = 'contributor'
+curator = '.json created by curator'
+
+
+#%% Support functions
+
+def _add_image(entry, full_img_path):    
+    """ 
+    Open the image to get size information and add height and width to the image entry. 
+    """
+    
+    img_width, img_height = helpers.get_img_size(full_img_path)
+    if img_width == -1 or img_height == -1:
+        corrupt_images.append(full_img_path)
+        return None
+    entry['width'] = img_width
+    entry['height'] = img_height
+    pbar.update(1)
+    return entry
+
+
+#%% Main loop (metadata processing; image sizes are retrieved later)
+    
+print('Creating tasks to get all images...')
+start_time = datetime.now()
+tasks = []
+folders = os.listdir(deployments_path)
+
+all_species_strings = set()
+
+# deployment = folders[0]
+for deployment in tqdm(folders):
+    
+    deployment_path = os.path.join(deployments_path, deployment)
+    manifest_path = os.path.join(deployment_path, 'deployment_manifest.xml')
+
+    assert os.path.isfile(manifest_path)
+    
+    with open(manifest_path, 'r') as f:
+        tree = etree.parse(f)
+
+    root = tree.getroot()
+    project_id = root.findtext('ProjectId')
+    deployment_id = root.findtext('CameraDeploymentID')
+    deployment_location = root.findtext('CameraSiteName')
+
+    image_sequences = root.findall('ImageSequence')
+    
+    # sequence = image_sequences[0]
+    for sequence in image_sequences:
+        
+        seq_id = sequence.findtext('ImageSequenceId')
+
+        # get species info for this sequence
+        researcher_identifications = sequence.findall('ResearcherIdentifications')
+        species = set()
+
+        for researcher_id in researcher_identifications:
+            identifications = researcher_id.findall('Identification')
+            for id in identifications:
+                species_common_name = helpers.clean_species_name(id.findtext('SpeciesCommonName'))
+                species.add(species_common_name)
+
+        species_str = ';'.join(sorted(list(species)))
+        all_species_strings.add(species_str)
+        
+        # add each image's info to database
+        images = sequence.findall('Image')
+        seq_num_frames = len(images) # total number of frames in this sequence
+        assert isinstance(seq_num_frames,int) and seq_num_frames > 0 # historical quirk
+        
+        # img = images[0]
+        for img in images:
+            
+            img_id = img.findtext('ImageId')
+            img_file_name = img.findtext('ImageFileName')
+            assert img_file_name.lower().endswith('.jpg')
+
+            img_frame = img.findtext('ImageOrder')
+            if img_frame == '' or img_frame is None:
+                # some manifests don't have the ImageOrder info, but the info is in the file name
+                img_frame = img_file_name.split('i')[1].split('.')[0]
+
+            # full_img_id has no frame info
+            #
+            # frame number only used in requests to iMerit for ordering
+            full_img_id = 'datasetemammal.project{}.deployment{}.seq{}.img{}'.format(
+                project_id, deployment_id, seq_id, img_id)            
+            full_img_path = os.path.join(deployment_path, img_file_name)
+
+            img_datetime, datetime_err = helpers.parse_timestamp(img.findtext('ImageDateTime'))
+            if datetime_err:
+                print('WARNING datetime parsing error for image {}. Error: {}'.format(
+                    full_img_path, datetime_err))
+
+            entry = {
+                'id': full_img_id,
+                'width': 0,  # place holders
+                'height': 0,
+                'file_name': os.path.join(deployment, img_file_name),
+                'location': deployment_location,
+                'datetime': str(img_datetime),
+                'seq_id': seq_id,
+                'frame_num': int(img_frame),
+                'seq_num_frames': seq_num_frames,
+                'label': species_str  # extra field for eMammal
+            }
+
+            tasks.append((entry, full_img_path))
+        
+        # ...for each image
+        
+    # ...for each sequence
+    
+# ...for each deployment
+    
+print('Finished creating tasks to get images.')
+
+
+#%% Get image sizes
+
+# 'tasks' is currently a list of 2-tuples, with each entry as [image dictionary,path].
+# 
+# Go through that and copy just the image dictionaries to 'db_images', adding size
+# information to each entry.  Takes a couple hours.
+
+db_images = []
+corrupt_images = []
+pbar = tqdm(total=len(tasks))
+
+if run_parallel:
+    # opening each image seems too fast for this multi-threaded version to be faster than sequential code.
+    num_workers = multiprocessing.cpu_count()
+    pool = ThreadPool(num_workers)
+    db_images = pool.starmap(_add_image, tasks)
+    print('Waiting for image processes to finish...')
+    pool.close()
+    pool.join()
+else:
+    print('Finding image size sequentially')
+    for entry, full_img_path in tasks:
+        db_images.append(_add_image(entry, full_img_path))
+
+db_images = [i for i in db_images if i is not None]
+
+print('{} images could not be opened:'.format(len(corrupt_images)))
+print(corrupt_images)
+print('Done getting image sizes')
+
+
+#%% Assemble top-level dictionaries
+
+db_info = {
+    'year': 'unknown',
+    'version': version,
+    'description': description,
+    'contributor': contributor,
+    'curator': curator,
+    'date_created': str(date.today())
+}
+
+coco_formatted_json = {
+    'info': db_info,
+    'images': db_images
+}
+
+
+#%% Write out .json
+
+print('Saving the json database to disk...')
+with open(os.path.join(output_dir_path, db_filename), 'w') as f:
+    json.dump(coco_formatted_json, f, indent=4, sort_keys=True)
+print('...done')
+
+print('Saving list of corrupt images...')
+with open(os.path.join(output_dir_path, corrupt_images_db_filename), 'w') as f:
+    json.dump(corrupt_images, f, indent=4)    
+print('...done')
+
+print('Running the script took {}.'.format(datetime.now() - start_time))
+
+