megadetector 5.0.10__py3-none-any.whl → 5.0.12__py3-none-any.whl

api/synchronous/api_core/animal_detection_api/md_utils/azure_utils.py

@@ -1,174 +0,0 @@
-########
-#
-# azure_utils.py
-#
-# Miscellaneous Azure Blob Storage utilities
-#
-# Requires azure-storage-blob>=12.4.0
-#
-########
-
-import json
-from md_utils import path_utils
-from typing import Any, Iterable, List, Optional, Tuple, Union
-
-from azure.storage.blob import BlobPrefix, ContainerClient
-
-from md_utils import sas_blob_utils
-
-
-def walk_container(container_client: ContainerClient,
-                   max_depth: int = -1,
-                   prefix: str = '',
-                   store_folders: bool = True,
-                   store_blobs: bool = True,
-                   debug_max_items: int = -1) -> Tuple[List[str], List[str]]:
-    """
-    Recursively walk folders a Azure Blob Storage container.
-
-    Based on:
-    https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/storage/azure-storage-blob/samples/blob_samples_walk_blob_hierarchy.py
-    """
-    
-    depth = 1
-
-    def walk_blob_hierarchy(prefix: str,
-                            folders: Optional[List[str]] = None,
-                            blobs: Optional[List[str]] = None
-                            ) -> Tuple[List[str], List[str]]:
-        if folders is None:
-            folders = []
-        if blobs is None:
-            blobs = []
-
-        nonlocal depth
-
-        if 0 < max_depth < depth:
-            return folders, blobs
-
-        for item in container_client.walk_blobs(name_starts_with=prefix):
-            short_name = item.name[len(prefix):]
-            if isinstance(item, BlobPrefix):
-                # print('F: ' + prefix + short_name)
-                if store_folders:
-                    folders.append(prefix + short_name)
-                depth += 1
-                walk_blob_hierarchy(item.name, folders=folders, blobs=blobs)
-                if (debug_max_items > 0
-                        and len(folders) + len(blobs) > debug_max_items):
-                    return folders, blobs
-                depth -= 1
-            else:
-                if store_blobs:
-                    blobs.append(prefix + short_name)
-
-        return folders, blobs
-
-    folders, blobs = walk_blob_hierarchy(prefix=prefix)
-
-    assert all(s.endswith('/') for s in folders)
-    folders = [s.strip('/') for s in folders]
-
-    return folders, blobs
-
-
-def list_top_level_blob_folders(container_client: ContainerClient) -> List[str]:
-    """
-    List all top-level folders in a container.
-    """
-    
-    top_level_folders, _ = walk_container(
-        container_client, max_depth=1, store_blobs=False)
-    return top_level_folders
-
-
-def concatenate_json_lists(input_files: Iterable[str],
-                           output_file: Optional[str] = None
-                           ) -> List[Any]:
-    """
-    Given a list of JSON files that contain lists (typically string
-    filenames), concatenates the lists into a single list and optionally
-    writes out this list to a new output JSON file.
-    """
-    
-    output_list = []
-    for fn in input_files:
-        with open(fn, 'r') as f:
-            file_list = json.load(f)
-        output_list.extend(file_list)
-    if output_file is not None:
-        with open(output_file, 'w') as f:
-            json.dump(output_list, f, indent=1)
-    return output_list
-
-
-def upload_file_to_blob(account_name: str,
-                        container_name: str,
-                        local_path: str,
-                        blob_name: str,
-                        sas_token: str,
-                        overwrite: bool=False) -> str:
-    """
-    Uploads a local file to Azure Blob Storage and returns the uploaded
-    blob URI with SAS token.
-    """
-    
-    container_uri = sas_blob_utils.build_azure_storage_uri(
-        account=account_name, container=container_name, sas_token=sas_token)
-    with open(local_path, 'rb') as data:
-        return sas_blob_utils.upload_blob(
-            container_uri=container_uri, blob_name=blob_name, data=data, 
-            overwrite=overwrite)
-
-
-def enumerate_blobs_to_file(
-        output_file: str,
-        account_name: str,
-        container_name: str,
-        sas_token: Optional[str] = None,
-        blob_prefix: Optional[str] = None,
-        blob_suffix: Optional[Union[str, Tuple[str]]] = None,
-        rsearch: Optional[str] = None,
-        limit: Optional[int] = None,
-        verbose: Optional[bool] = True
-        ) -> List[str]:
-    """
-    Enumerates blobs in a container, and writes the blob names to an output
-    file.
-
-    Args:
-        output_file: str, path to save list of files in container
-            If ends in '.json', writes a JSON string. Otherwise, writes a
-            newline-delimited list. Can be None, in which case this is just a 
-            convenient wrapper for blob enumeration.
-        account_name: str, Azure Storage account name
-        container_name: str, Azure Blob Storage container name
-        sas_token: optional str, container SAS token, leading ? will be removed if present.
-        blob_prefix: optional str, returned results will only contain blob names
-            to with this prefix
-        blob_suffix: optional str or tuple of str, returned results will only
-            contain blob names with this/these suffix(es). The blob names will
-            be lowercased first before comparing with the suffix(es).
-        rsearch: optional str, returned results will only contain blob names
-            that match this regex. Can also be a list of regexes, in which case
-            blobs matching *any* of the regex's will be returned.            
-        limit: int, maximum # of blob names to list
-            if None, then returns all blob names
-
-    Returns: list of str, sorted blob names, of length limit or shorter.
-    """
-    
-    if sas_token is not None and len(sas_token) > 9 and sas_token[0] == '?':
-        sas_token = sas_token[1:]
-        
-    container_uri = sas_blob_utils.build_azure_storage_uri(
-        account=account_name, container=container_name, sas_token=sas_token)
-    
-    matched_blobs = sas_blob_utils.list_blobs_in_container(
-        container_uri=container_uri, blob_prefix=blob_prefix,
-        blob_suffix=blob_suffix, rsearch=rsearch, limit=limit, verbose=verbose)
-    
-    if output_file is not None:
-        path_utils.write_list_to_file(output_file, matched_blobs)
-        
-    return matched_blobs

api/synchronous/api_core/animal_detection_api/md_utils/ct_utils.py

@@ -1,262 +0,0 @@
-########
-#
-# ct_utils.py
-#
-# Utility functions that don't depend on other things in this repo.
-#
-########
-
-#%% Imports and constants
-
-import argparse
-import inspect
-import json
-import math
-import os
-
-import jsonpickle
-import numpy as np
-
-# List of file extensions we'll consider images; comparisons will be case-insensitive
-# (i.e., no need to include both .jpg and .JPG on this list).
-image_extensions = ['.jpg', '.jpeg', '.gif', '.png']
-
-
-#%% Functions
-
-def truncate_float_array(xs, precision=3):
-    """
-    Vectorized version of truncate_float(...)
-
-    Args:
-    xs        (list of float) List of floats to truncate
-    precision (int)           The number of significant digits to preserve, should be
-                              greater or equal 1
-    """
-
-    return [truncate_float(x, precision=precision) for x in xs]
-
-
-def truncate_float(x, precision=3):
-    """
-    Truncates a floating-point value to a specific number of significant digits.
-    
-    For example: truncate_float(0.0003214884) --> 0.000321
-    
-    This function is primarily used to achieve a certain float representation
-    before exporting to JSON.
-
-    Args:
-    x         (float) Scalar to truncate
-    precision (int)   The number of significant digits to preserve, should be
-                      greater or equal 1
-    """
-
-    assert precision > 0
-
-    if np.isclose(x, 0):
-        
-        return 0
-    
-    else:
-        
-        # Determine the factor, which shifts the decimal point of x
-        # just behind the last significant digit.
-        factor = math.pow(10, precision - 1 - math.floor(math.log10(abs(x))))
-        
-        # Shift decimal point by multiplicatipon with factor, flooring, and
-        # division by factor.
-        return math.floor(x * factor)/factor
-
-
-def args_to_object(args: argparse.Namespace, obj: object) -> None:
-    """
-    Copies all fields from a Namespace (i.e., the output from parse_args) to an
-    object. Skips fields starting with _. Does not check existence in the target
-    object.
-
-    Args:
-        args: argparse.Namespace
-        obj: class or object whose whose attributes will be updated
-    """
-    
-    for n, v in inspect.getmembers(args):
-        if not n.startswith('_'):
-            setattr(obj, n, v)
-
-
-def pretty_print_object(obj, b_print=True):
-    """
-    Prints an arbitrary object as .json
-    """
-
-    # _ = pretty_print_object(obj)
-
-    # Sloppy that I'm making a module-wide change here...
-    jsonpickle.set_encoder_options('json', sort_keys=True, indent=2)
-    a = jsonpickle.encode(obj)
-    s = '{}'.format(a)
-    if b_print:
-        print(s)
-    return s
-
-
-def is_list_sorted(L,reverse=False):
-    """
-    Returns true if the list L appears to be sorted, otherwise False.
-    
-    Calling is_list_sorted(L,reverse=True) is the same as calling
-    is_list_sorted(L.reverse(),reverse=False).
-    """
-    
-    if reverse:
-        return all(L[i] >= L[i + 1] for i in range(len(L)-1))
-    else:
-        return all(L[i] <= L[i + 1] for i in range(len(L)-1))
-        
-
-def write_json(path, content, indent=1):
-    """
-    Standardized wrapper for json.dump
-    """
-    
-    with open(path, 'w') as f:
-        json.dump(content, f, indent=indent)
-
-
-def is_image_file(s):
-    """
-    Checks a file's extension against a hard-coded set of image file extensions; 
-    return True if it appears to be an image.
-    """
-
-    ext = os.path.splitext(s)[1]
-    return ext.lower() in image_extensions
-
-
-def convert_yolo_to_xywh(yolo_box):
-    """
-    Converts a YOLO format bounding box to [x_min, y_min, width_of_box, height_of_box].
-
-    Args:
-        yolo_box: bounding box of format [x_center, y_center, width_of_box, height_of_box].
-
-    Returns:
-        bbox with coordinates represented as [x_min, y_min, width_of_box, height_of_box].
-    """
-    
-    x_center, y_center, width_of_box, height_of_box = yolo_box
-    x_min = x_center - width_of_box / 2.0
-    y_min = y_center - height_of_box / 2.0
-    return [x_min, y_min, width_of_box, height_of_box]
-
-
-def convert_xywh_to_tf(api_box):
-    """
-    Converts an xywh bounding box to an [y_min, x_min, y_max, x_max] box that the TensorFlow
-    Object Detection API uses
-
-    Args:
-        api_box: bbox output by the batch processing API [x_min, y_min, width_of_box, height_of_box]
-
-    Returns:
-        bbox with coordinates represented as [y_min, x_min, y_max, x_max]
-    """
-    
-    x_min, y_min, width_of_box, height_of_box = api_box
-    x_max = x_min + width_of_box
-    y_max = y_min + height_of_box
-    return [y_min, x_min, y_max, x_max]
-
-
-def convert_xywh_to_xyxy(api_bbox):
-    """
-    Converts an xywh bounding box to an xyxy bounding box.
-
-    Note that this is also different from the TensorFlow Object Detection API coords format.
-    Args:
-        api_bbox: bbox output by the batch processing API [x_min, y_min, width_of_box, height_of_box]
-
-    Returns:
-        bbox with coordinates represented as [x_min, y_min, x_max, y_max]
-    """
-
-    x_min, y_min, width_of_box, height_of_box = api_bbox
-    x_max, y_max = x_min + width_of_box, y_min + height_of_box
-    return [x_min, y_min, x_max, y_max]
-
-
-def get_iou(bb1, bb2):
-    """
-    Calculates the Intersection over Union (IoU) of two bounding boxes.
-
-    Adapted from:
-        
-    https://stackoverflow.com/questions/25349178/calculating-percentage-of-bounding-box-overlap-for-image-detector-evaluation
-
-    Args:
-        bb1: [x_min, y_min, width_of_box, height_of_box]
-        bb2: [x_min, y_min, width_of_box, height_of_box]
-
-    Returns:
-        intersection_over_union, a float in [0, 1]
-    """
-
-    bb1 = convert_xywh_to_xyxy(bb1)
-    bb2 = convert_xywh_to_xyxy(bb2)
-
-    assert bb1[0] < bb1[2], 'Malformed bounding box (x2 >= x1)'
-    assert bb1[1] < bb1[3], 'Malformed bounding box (y2 >= y1)'
-
-    assert bb2[0] < bb2[2], 'Malformed bounding box (x2 >= x1)'
-    assert bb2[1] < bb2[3], 'Malformed bounding box (y2 >= y1)'
-
-    # Determine the coordinates of the intersection rectangle
-    x_left = max(bb1[0], bb2[0])
-    y_top = max(bb1[1], bb2[1])
-    x_right = min(bb1[2], bb2[2])
-    y_bottom = min(bb1[3], bb2[3])
-
-    if x_right < x_left or y_bottom < y_top:
-        return 0.0
-
-    # The intersection of two axis-aligned bounding boxes is always an
-    # axis-aligned bounding box
-    intersection_area = (x_right - x_left) * (y_bottom - y_top)
-
-    # Compute the area of both AABBs
-    bb1_area = (bb1[2] - bb1[0]) * (bb1[3] - bb1[1])
-    bb2_area = (bb2[2] - bb2[0]) * (bb2[3] - bb2[1])
-
-    # Compute the intersection over union by taking the intersection
-    # area and dividing it by the sum of prediction + ground-truth
-    # areas - the intersection area.
-    iou = intersection_area / float(bb1_area + bb2_area - intersection_area)
-    assert iou >= 0.0, 'Illegal IOU < 0'
-    assert iou <= 1.0, 'Illegal IOU > 1'
-    return iou
-
-
-def _get_max_conf_from_detections(detections):
-    """
-    Internal function used by get_max_conf(); don't call this directly.
-    """
-    
-    max_conf = 0.0
-    if detections is not None and len(detections) > 0:
-        confidences = [det['conf'] for det in detections]
-        max_conf = max(confidences)
-    return max_conf
-
-
-def get_max_conf(im):
-    """
-    Given an image dict in the format used by the batch API, compute the maximum detection
-    confidence for any class.  Returns 0.0 (not None) if there was a failure and 'detections'
-    isn't present.
-    """
-    
-    max_conf = 0.0
-    if 'detections' in im and im['detections'] is not None and len(im['detections']) > 0:
-        max_conf = _get_max_conf_from_detections(im['detections'])
-    return max_conf

api/synchronous/api_core/animal_detection_api/md_utils/directory_listing.py

@@ -1,251 +0,0 @@
-########
-#
-# directory_listing.py
-#
-# Script for creating HTML directory listings for a local directory and
-# all its subdirectories.  Primarily intended for use on mounted blob
-# containers, so it includes the ability to set the content-type property
-# on the generated html using the blob SDK, so it can be browser-viewable.
-#
-# Also includes a preview of a jpg file (the first in an alphabetical list),
-# if present.
-#
-# Sample invocation:
-#
-# python directory_listing.py /naipout/v002 --basepath /naipout/v002 \
-#   --enable_overwrite \
-#   --sas_url "https://naipblobs.blob.core.windows.net/naip/v002?sv=..."
-#
-########
-
-#%% Imports
-
-import os
-import sys
-import argparse
-import re
-
-import azure.common
-from azure.storage.blob import BlobServiceClient, ContentSettings
-
-
-#%% Directory enumeration functions
-
-def is_image_file(path):
-    """
-    Checks whether the provided file path points to an image by checking the
-    file extension. The following file extensions are considered images: jpg,
-    jpeg.  The check is case-insensitive.
-
-    Args:
-        path: string, path to image file or just the file name
-
-    Returns:
-        boolean, True if the file is an image
-    """
-
-    return os.path.splitext(path)[1].lower()[1:] in ['jpg', 'jpeg'] #, 'gif', 'tiff', 'tif', 'png']
-
-
-def create_plain_index(root, dirs, files, dirname=None):
-    """
-    Creates the fairly plain HTML folder index
-    including a preview of a single image file, if any is present.
-    Returns the HTML source as string.
-
-    Args:
-        root: string, path to the root directory, all paths in *dirs* and
-            *files* are relative to this one
-        dirs: list of strings, the directories in *root*
-        files: list of string, the files in *root*
-        dirname: name to print in the html, which may be different than *root*
-
-    Returns: HTML source of the directory listing
-    """
-
-    if dirname is None:
-        dirname = root or '/'
-
-    html = "<!DOCTYPE html>\n"
-    html += "<html lang='en'><head>"
-    html += "<title>Index of {}</title>\n".format(dirname)
-    html += "<meta charset='UTF-8'>\n"
-    html += "<style>\n"
-    html += "body { font-family: Segoe UI, Helvetica, Arial, sans-serif; }\n"
-    html += "a { text-decoration:none; }\n"
-    html += "</style>\n"
-    html += "</head><body>\n"
-
-    html += "<h1>Index of {}</h1>\n".format(dirname)
-
-    # Insert preview image
-    jpg_files = [f for f in files if is_image_file(f)]
-
-    if len(jpg_files) > 0:
-
-        # This is slow, so defaulting to just the first image:
-        #
-        # Use the largest image file as this is most likely to contain
-        # useful content.
-        #
-        # jpg_file_sizes = [os.path.getsize(f) for f in jpg_files]
-        # largest_file_index = max(range(len(jpg_files)), key=lambda x: jpg_file_sizes[x])
-
-        html += "<a href='{0}'><img style='height:200px; float:right;' src='{0}' alt='Preview image'></a>\n".format(jpg_files[0])
-    else:
-        html += "\n"
-        # html += "<p style='width:15em; float:right; margin:0;'>[No preview available]</p>\n"
-
-    if root:
-        html += "<p><a href='../index.html'>To parent directory</a></p>\n"
-    else:
-        html += "\n"
-        # html += "<p>This is the root directory.</p>\n"
-
-    html += "<h2>Folders</h2>\n"
-    if len(dirs) > 0:
-        html += "<ul style='list-style-type: none; padding-left:1em;'>\n"
-        for dir in sorted(dirs):
-            html += "<li>&#128193; <a href='{0}/index.html'>{0}</a></li>\n".format(dir)
-        html += "</ul>\n"
-    else:
-        html += "<p style='padding-left:1em;'>No folders</p>\n"
-
-    html += "<h2>Files</h2>\n"
-    if len(files) > 0:
-        html += "<ul style='list-style-type: none; padding-left:1em;'>\n"
-        for fname in sorted(files):
-            if is_image_file(fname):
-                html += "<li>&#128443; <a href='{0}'>{0}</a></li>\n".format(fname)
-            else:
-                html += "<li>&#128442; <a href='{0}'>{0}</a></li>\n".format(fname)
-        html += "</ul>\n"
-    else:
-        html += "<p style = 'padding-left:1em;'>No files</p>\n"
-
-    # Add some space at the bottom because the browser's status bar might hide stuff
-    html += "<p style='margin:2em;'>&nbsp;</p>\n"
-    html += "</body></html>\n"
-    return html
-
-
-def traverse_and_create_index(dir, sas_url=None, overwrite_files=False,
-                              template_fun=create_plain_index, basepath=None):
-    """
-    Recursively traverses the local directory *dir* and generates a index
-    file for each folder using *template_fun* to generate the HTML output.
-    Excludes hidden files.
-
-    Args:
-        dir: string, path to directory
-        template_fun: function taking three arguments (string, list of string, list of string)
-            representing the current root, the list of folders, and the list of files.
-            Should return the HTML source of the index file
-
-    Return:
-        None
-    """
-
-    print("Traversing {}".format(dir))
-
-    # Make sure we remove the trailing /
-    dir = os.path.normpath(dir)
-
-    # If we want to set the content type in blob storage using a SAS URL
-    if sas_url:
-
-        # Example: sas_url = 'https://accname.blob.core.windows.net/bname/path/to/folder?st=...&se=...&sp=...&...'
-        if '?' in sas_url:
-            # 'https://accname.blob.core.windows.net/bname/path/to/folder' and 'st=...&se=...&sp=...&...'
-            base_url, sas_token = sas_url.split('?', 1)
-        else:
-            # 'https://accname.blob.core.windows.net/bname/path/to/folder' and None
-            base_url, sas_token = sas_url, None
-        # Remove https:// from base url
-        # 'accname.blob.core.windows.net/bname/path/to/folder'
-        base_url = base_url.split("//", 1)[1]
-        # Everything up to the first dot is account name
-        # 'accname'
-        account_name = base_url.split(".", 1)[0]
-        # get everything after the first /
-        # 'bname/path/to/folder'
-        query_string = base_url.split("/", 1)[1]
-        # Get container name and subfolder
-        if '/' in query_string:
-            # 'bname', 'path/to/folder'
-            container_name, container_folder = query_string.split("/", 1)
-        else:
-            container_name, container_folder = query_string, ''
-
-        # Prepare the storage access
-        target_settings = ContentSettings(content_type='text/html')
-        blob_service = BlobServiceClient(
-            account_url=f'{account_name}.blob.core.windows.net',
-            credential=sas_token)
-
-    # Traverse directory and all sub directories, excluding hidden files
-    for root, dirs, files in os.walk(dir):
-
-        # Exclude files and folders that are hidden
-        files = [f for f in files if not f[0] == '.']
-        dirs[:] = [d for d in dirs if not d[0] == '.']
-
-        # Output is written to file *root*/index.html
-        output_file = os.path.join(root, "index.html")
-
-        if not overwrite_files and os.path.isfile(output_file):
-            print('Skipping {}, file exists'.format(output_file))
-            continue
-
-        print("Generating {}".format(output_file))
-
-        # Generate HTML with template function
-        dirname = None
-        if basepath is not None:
-            dirname = os.path.relpath(root,basepath)
-        html = template_fun(root[len(dir):], dirs, files, dirname)
-
-        # Write to file
-        with open(output_file, 'wt') as fi:
-            fi.write(html)
-
-        # Set content type in blob storage
-        if sas_url:
-            if container_folder:
-                output_blob_path = container_folder + '/' + output_file[len(dir) + 1:]
-            else:
-                output_blob_path = output_file[len(dir) + 1:]
-            try:
-                blob_client = blob_service.get_blob_client(container_name, output_blob_path)
-                blob_client.set_http_headers(content_settings=target_settings)
-            except azure.common.AzureMissingResourceHttpError:
-                print('ERROR: It seems the SAS URL is incorrect or does not allow setting properties.')
-                return
-
-
-#%% Command-line driver
-
-def parse_args(argv=sys.argv[1:]):
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument("directory", type=str, help='Path to directory which should be traversed.')
-    parser.add_argument("--basepath", type=str, help='Folder names will be printed relative to basepath, if specified', default=None)
-    parser.add_argument("--sas_url", type=str, help='Blobfuse does not set the content-type property ' + \
-        'properly and hence index.html won\'t be accessible in the browser. If you want to set the ' + \
-        'content-type in the corresponding blob storage, provide the SAS URL that corresponds to the ' + \
-        "directory, e.g. if *directory* is /mountpoint/path/to/folder, then *--sas_url* looks like " + \
-        "'https://accname.blob.core.windows.net/bname/path/to/folder?st=...&se=...&sp=...&...'")
-    parser.add_argument("--enable_overwrite", action='store_true', default=False,
-                        help='If set, the script will overwrite existing index.html files.')
-    return parser.parse_args(argv)
-
-
-if __name__ == '__main__':
-
-    args = parse_args()
-
-    assert os.path.isdir(args.directory), "{} is not a valid directory".format(args.directory)
-    assert re.match('https?://[^\.]+\.blob\.core\.windows\.net/.+', args.sas_url), "--sas_url does not " + \
-        "match the format https://accname.blob.core.windows.net/bname/path/to/folder?..."
-
-    traverse_and_create_index(args.directory, overwrite_files=args.enable_overwrite, sas_url=args.sas_url, basepath=args.basepath)