megadetector 5.0.9__py3-none-any.whl → 5.0.11__py3-none-any.whl

api/batch_processing/postprocessing/top_folders_to_bottom.py

@@ -1,223 +0,0 @@
-"""
-
-top_folders_to_bottom.py
-
-Given a base folder with files like:
-
-* A/1/2/a.jpg
-* B/3/4/b.jpg
-
-...moves the top-level folders to the bottom in a new output folder, i.e., creates:
-
-* 1/2/A/a.jpg
-* 3/4/B/b.jpg
-
-In practice, this is used to make this:
-
-animal/camera01/image01.jpg
-
-...look like:
-
-camera01/animal/image01.jpg
-
-"""
-
-#%% Constants and imports
-
-import os
-import sys
-import shutil
-import argparse
-
-from pathlib import Path
-from tqdm import tqdm
-
-from functools import partial
-from multiprocessing.pool import ThreadPool
-
-from md_utils.path_utils import path_is_abs
-
-
-#%% Classes
-
-class TopFoldersToBottomOptions:
-    """
-    Options used to parameterize top_folders_to_bottom()
-    """
-    
-    def __init__(self,input_folder,output_folder,copy=True,n_threads=1):
-        
-        #: Whether to copy (True) vs. move (False) false when re-organizing
-        self.copy = copy
-        
-        #: Number of worker threads to use, or <1 to disable parallelization
-        self.n_threads = n_threads
-        
-        #: Input folder
-        self.input_folder = input_folder
-        
-        #: Output folder
-        self.output_folder = output_folder
-        
-        #: If this is False and an output file exists, throw an error
-        self.overwrite = False
-        
-
-#%% Main functions
-
-def _process_file(relative_filename,options,execute=True):
-    
-    assert ('/' in relative_filename) and \
-        ('\\' not in relative_filename) and \
-        (not path_is_abs(relative_filename))
-    
-    # Find top-level folder
-    tokens = relative_filename.split('/')
-    topmost_folder = tokens.pop(0)
-    tokens.insert(len(tokens)-1,topmost_folder)
-    
-    # Find file/folder names
-    output_relative_path = '/'.join(tokens)
-    output_relative_folder = '/'.join(tokens[0:-1])
-    
-    output_absolute_folder = os.path.join(options.output_folder,output_relative_folder)
-    output_absolute_path = os.path.join(options.output_folder,output_relative_path)
-
-    if execute:
-        
-        os.makedirs(output_absolute_folder,exist_ok=True)
-        
-        input_absolute_path = os.path.join(options.input_folder,relative_filename)
-        
-        if not options.overwrite:
-            assert not os.path.isfile(output_absolute_path), \
-                'Error: output file {} exists'.format(output_absolute_path)
-            
-        # Move or copy
-        if options.copy:
-            shutil.copy(input_absolute_path, output_absolute_path)
-        else:
-            shutil.move(input_absolute_path, output_absolute_path)
-
-    return output_absolute_path
-    
-# ...def _process_file()
-
-
-def top_folders_to_bottom(options):
-    """
-    top_folders_to_bottom.py
-
-    Given a base folder with files like:
-
-    * A/1/2/a.jpg
-    * B/3/4/b.jpg
-
-    ...moves the top-level folders to the bottom in a new output folder, i.e., creates:
-
-    * 1/2/A/a.jpg
-    * 3/4/B/b.jpg
-
-    In practice, this is used to make this:
-
-    animal/camera01/image01.jpg
-
-    ...look like:
-
-    camera01/animal/image01.jpg
-
-    Args:
-        options (TopFoldersToBottomOptions): See TopFoldersToBottomOptions for parameter details.
-
-    """
-    os.makedirs(options.output_folder,exist_ok=True)
-    
-    # Enumerate input folder
-    print('Enumerating files...')
-    files = list(Path(options.input_folder).rglob('*'))
-    files = [p for p in files if not p.is_dir()]
-    files = [str(s) for s in files]
-    print('Enumerated {} files'.format(len(files)))
-    
-    # Convert absolute paths to relative paths
-    relative_files = [os.path.relpath(s,options.input_folder) for s in files]
-    
-    # Standardize delimiters
-    relative_files = [s.replace('\\','/') for s in relative_files]
-    
-    base_files = [s for s in relative_files if '/' not in s]
-    if len(base_files) > 0:
-        print('Warning: ignoring {} files in the base folder'.format(len(base_files)))
-        relative_files = [s for s in relative_files if '/' in s]
-    
-    # Make sure each input file maps to a unique output file
-    absolute_output_files = [_process_file(s, options, execute=False) for s in relative_files]
-    assert len(absolute_output_files) == len(set(absolute_output_files)),\
-        "Error: input filenames don't map to unique output filenames"
-        
-    # relative_filename = relative_files[0]
-    
-    # Loop
-    if options.n_threads <= 1:
-        
-        for relative_filename in tqdm(relative_files):
-            _process_file(relative_filename,options)
-    
-    else:
-        
-        print('Starting a pool with {} threads'.format(options.n_threads))
-        pool = ThreadPool(options.n_threads)
-        process_file_with_options = partial(_process_file, options=options)
-        _ = list(tqdm(pool.imap(process_file_with_options, relative_files), total=len(relative_files)))
-
-# ...def top_folders_to_bottom()        
-        
-
-#%% Interactive driver
-        
-if False:
-
-    pass
-
-    #%%
-    
-    input_folder = r"G:\temp\output"
-    output_folder = r"G:\temp\output-inverted"    
-    options = TopFoldersToBottomOptions(input_folder,output_folder,copy=True,n_threads=10)
-    
-    #%%
-    
-    top_folders_to_bottom(options)
-
-
-#%% Command-line driver   
-
-# python top_folders_to_bottom.py "g:\temp\separated_images" "g:\temp\separated_images_inverted" --n_threads 100
-
-def main():
-    
-    parser = argparse.ArgumentParser()
-    parser.add_argument('input_folder', type=str, help='Input image folder')
-    parser.add_argument('output_folder', type=str, help='Output image folder')
-
-    parser.add_argument('--copy', action='store_true', 
-                        help='Copy images, instead of moving (moving is the default)')
-    parser.add_argument('--overwrite', action='store_true', 
-                        help='Allow image overwrite (default=False)')
-    parser.add_argument('--n_threads', type=int, default=1,
-                        help='Number of threads to use for parallel operation (default=1)')
-    
-    if len(sys.argv[1:])==0:
-        parser.print_help()
-        parser.exit()
-        
-    args = parser.parse_args()    
-    
-    # Convert to an options object
-    options = TopFoldersToBottomOptions(
-        args.input_folder,args.output_folder,copy=args.copy,n_threads=args.n_threads)
-    
-    top_folders_to_bottom(options)
-    
-if __name__ == '__main__':    
-    main()

api/synchronous/api_core/animal_detection_api/api_backend.py

@@ -1,152 +0,0 @@
-#
-# api_backend.py
-#
-# Defines the model execution service, which pulls requests (one or more images)
-# from the shared Redis queue, and runs them through the TF model.
-#
-
-#%% Imports
-
-import os
-import json
-import time
-import redis
-import argparse
-import PIL
-
-from io import BytesIO
-
-from detection.run_detector import load_detector, convert_to_tf_coords
-import config
-import md_visualization.visualization_utils as vis_utils 
-
-#%% Initialization
-
-db = redis.StrictRedis(host=config.REDIS_HOST, port=config.REDIS_PORT)
-current_directory = os.path.dirname(os.path.realpath(__file__))
-
-
-#%% Main loop
-
-def detect_process():
-     
-     while True:
-        
-        # TODO: convert to a blocking read and eliminate the sleep() statement in this loop
-        serialized_entry = db.lpop(config.REDIS_QUEUE_NAME)
-        all_detection_results = []
-        inference_time_detector = []
-        
-        if serialized_entry:
-            
-            entry = json.loads(serialized_entry)
-            id = entry['id']
-            print('Processing images from request id:', id)
-            return_confidence_threshold = entry['return_confidence_threshold']
-
-            try:
-                
-                temp_direc = os.path.join(config.TEMP_FOLDER,id)
-                assert os.path.isdir(temp_direc), 'Could not find temporary folder {}'.format(temp_direc)
-                
-                for filename in os.listdir(temp_direc):
-                    
-                    image_path = f'{temp_direc}/{filename}'
-                    print('Reading image from {}'.format(image_path))
-                    image = open(image_path, 'rb')
-                    image = vis_utils.load_image(image)
-
-                    start_time = time.time()
-                    result = detector.generate_detections_one_image(image, filename, detection_threshold=config.DEFAULT_CONFIDENCE_THRESHOLD)
-                    all_detection_results.append(result)
-
-                    elapsed = time.time() - start_time
-                    inference_time_detector.append(elapsed)
-                    
-            except Exception as e:
-
-                print('Detection error: ' + str(e))
-                
-                db.set(entry['id'], json.dumps({ 
-                    'status': 500,
-                    'error': 'Detection error: ' + str(e)
-                }))
-
-                continue
-
-            # Filter the detections by the confidence threshold
-            #
-            # Each result is [ymin, xmin, ymax, xmax, confidence, category]
-            #
-            # Coordinates are relative, with the origin in the upper-left
-            detections = {}  
-            
-            try:
-                
-                for result in all_detection_results:
-                    
-                    image_name = result['file']
-                    _detections = result.get('detections', None)
-                    detections[image_name] = []
-
-                    if _detections is None:
-                        continue
-
-                    for d in _detections:
-                        if d['conf'] > return_confidence_threshold:
-                            res = convert_to_tf_coords(d['bbox'])
-                            res.append(d['conf'])
-                            res.append(int(d['category']))
-                            detections[image_name].append(res)
-    
-                db.set(entry['id'], json.dumps({ 
-                    'status': 200,
-                    'detections': detections,
-                    'inference_time_detector': inference_time_detector
-                }))
-              
-            except Exception as e:
-                print('Error consolidating the detection boxes: ' + str(e))
-                
-                db.set(entry['id'], json.dumps({ 
-                    'status': 500,
-                    'error': 'Error consolidating the detection boxes:' + str(e)
-                }))
-
-        # ...if serialized_entry
-        
-        else:
-            time.sleep(0.005)
-    
-    # ...while(True)
-    
-# ...def detect_process()    
-        
-
-#%% Command-line driver
-
-if __name__ == '__main__':
-    
-    parser = argparse.ArgumentParser(description='api backend')
-
-    # use --non-docker if you are testing without Docker
-    #
-    # python api_frontend.py --non-docker
-    parser.add_argument('--non-docker', action='store_true', default=False)
-    args = parser.parse_args()
-
-    if args.non_docker:
-        model_path = config.DETECTOR_MODEL_PATH_DEBUG
-    else:
-        model_path = config.DETECTOR_MODEL_PATH
-
-    detector = load_detector(model_path)
-
-    # run detections on a test image to load the model
-    print('Running initial detection to load model...')
-    test_image = PIL.Image.new(mode="RGB", size=(200, 200))
-    result = detector.generate_detections_one_image(test_image, "test_image", detection_threshold=config.DEFAULT_CONFIDENCE_THRESHOLD)
-    print(result)
-    print('\n')
-
-    detect_process() 

api/synchronous/api_core/animal_detection_api/api_frontend.py

@@ -1,266 +0,0 @@
-# 
-# api_frontend.py
-#
-# Defines the Flask app, which takes requests (one or more images) from
-# remote callers and pushes the images onto the shared Redis queue, to be processed
-# by the main service in api_backend.py .
-#
-
-#%% Imports
-
-import os
-import json
-import time
-import uuid
-import redis
-import shutil
-import argparse
-import traceback
-
-from io import BytesIO
-from flask import Flask, Response, jsonify, make_response, request
-from requests_toolbelt.multipart.encoder import MultipartEncoder
-
-import md_visualization.visualization_utils as vis_utils
-import config
-
-
-#%% Initialization
-
-app = Flask(__name__)
-db = redis.StrictRedis(host=config.REDIS_HOST, port=config.REDIS_PORT)
-
-
-#%% Support functions
-
-def _make_error_object(error_code, error_message):
-    
-    # Make a dict that the request_processing_function can return to the endpoint 
-    # function to notify it of an error
-    return {
-        'error_message': error_message,
-        'error_code': error_code
-    }
-
-
-def _make_error_response(error_code, error_message):
-
-    return make_response(jsonify({'error': error_message}), error_code)
-
-
-def has_access(request):
-    
-    if not os.path.exists(config.API_KEYS_FILE):
-        return True
-    else:
-        if not request.headers.get('key'):
-            print('Key header not available')
-            return False
-        else:
-            API_key = request.headers.get('key').strip().lower()
-            with open(config.API_KEYS_FILE, "r") as f:
-                for line in f:
-                    valid_key = line.strip().lower()
-                    if valid_key == API_key:
-                        return True
-
-    return False
-
-
-def check_posted_data(request):
- 
-    files = request.files
-    params = request.args
-
-    # Verify that the content uploaded is not too big
-    #
-    # request.content_length is the length of the total payload
-    content_length = request.content_length
-    if not content_length:
-        return _make_error_object(411, 'No image(s) were sent, or content length cannot be determined.')
-    if content_length > config.MAX_CONTENT_LENGTH_IN_MB * 1024 * 1024:
-        return _make_error_object(413, ('Payload size {:.2f} MB exceeds the maximum allowed of {} MB. '
-                    'Please upload fewer or more compressed images.').format(
-            content_length / (1024 * 1024), config.MAX_CONTENT_LENGTH_IN_MB))
-
-    render_boxes = True if params.get('render', '').lower() == 'true' else False
-
-    if 'min_confidence' in params:
-        return_confidence_threshold = float(params['min_confidence'])
-        print('runserver, post_detect_sync, user specified detection confidence: ', return_confidence_threshold)
-        if return_confidence_threshold < 0.0 or return_confidence_threshold > 1.0:
-            return _make_error_object(400, 'Detection confidence threshold {} is invalid, should be between 0.0 and 1.0.'.format(
-                return_confidence_threshold))
-    else:
-        return_confidence_threshold = config.DEFAULT_CONFIDENCE_THRESHOLD
-        
-    if 'min_rendering_confidence' in params:
-        rendering_confidence_threshold = float(params['min_rendering_confidence'])
-        print('runserver, post_detect_sync, user specified rendering confidence: ', rendering_confidence_threshold) 
-        if rendering_confidence_threshold < 0.0 or rendering_confidence_threshold > 1.0:
-            return _make_error_object(400, 'Rendering confidence threshold {} is invalid, should be between 0.0 and 1.0.'.format(
-                rendering_confidence_threshold))
-    else:
-        rendering_confidence_threshold =  config.DEFAULT_RENDERING_CONFIDENCE_THRESHOLD
-    
-    # Verify that the number of images is acceptable
-    num_images = sum([1 if file.content_type in config.IMAGE_CONTENT_TYPES else 0 for file in files.values()])
-    print('runserver, post_detect_sync, number of images received: ', num_images)
-
-    if num_images > config.MAX_IMAGES_ACCEPTED:
-        return _make_error_object(413, 'Too many images. Maximum number of images that can be processed in one call is {}.'.format(config.MAX_IMAGES_ACCEPTED))
-    elif num_images == 0:
-        return _make_error_object(400, 'No image(s) of accepted types (image/jpeg, image/png, application/octet-stream) received.')
-    
-    return {
-        'render_boxes': render_boxes,
-        'return_confidence_threshold': return_confidence_threshold,
-        'rendering_confidence_threshold': rendering_confidence_threshold
-    }
-
-# ...def check_posted_data(request)
-    
-
-#%% Main loop
-
-@app.route(config.API_PREFIX + '/detect', methods = ['POST'])
-def detect_sync():
-    
-    if not has_access(request):
-        print('Access denied, please provide a valid API key')
-        return _make_error_response(403, 'Access denied, please provide a valid API key')
-
-    # Check whether the request_processing_function had an error
-    post_data = check_posted_data(request)
-    if post_data.get('error_code', None) is not None:
-        return _make_error_response(post_data.get('error_code'), post_data.get('error_message'))
-
-    render_boxes = post_data.get('render_boxes')
-    return_confidence_threshold = post_data.get('return_confidence_threshold')
-    rendering_confidence_threshold = post_data.get('rendering_confidence_threshold')
-  
-    redis_id = str(uuid.uuid4())
-    d = {'id': redis_id, 'render_boxes': render_boxes, 'return_confidence_threshold': return_confidence_threshold}
-    temp_direc = os.path.join(config.TEMP_FOLDER, redis_id)
-    
-    try:
-        
-        try:
-            # Write images to temporary files
-            #
-            # TODO: read from memory rather than using intermediate files
-            os.makedirs(temp_direc,exist_ok=True)
-            for name, file in request.files.items():
-                if file.content_type in config.IMAGE_CONTENT_TYPES:
-                    filename = request.files[name].filename
-                    image_path = os.path.join(temp_direc, filename)
-                    print('Saving image {} to {}'.format(name,image_path))
-                    file.save(image_path)
-                    assert os.path.isfile(image_path),'Error creating file {}'.format(image_path)
-        
-        except Exception as e:
-            return _make_error_object(500, 'Error saving images: ' + str(e))
-        
-        # Submit the image(s) for processing by api_backend.py, who is waiting on this queue
-        db.rpush(config.REDIS_QUEUE_NAME, json.dumps(d))
-        
-        while True:
-            
-            # TODO: convert to a blocking read and eliminate the sleep() statement in this loop
-            result = db.get(redis_id)
-            
-            if result:
-                
-                result = json.loads(result.decode())
-                print('Processing result {}'.format(str(result)))
-                
-                if result['status'] == 200:
-                    detections = result['detections']
-                    db.delete(redis_id)
-                
-                else:
-                    db.delete(redis_id)
-                    print('Detection error: ' + str(result))
-                    return _make_error_response(500, 'Detection error: ' + str(result))
-
-                try:
-                    print('detect_sync: postprocessing and sending images back...')
-                    fields = {
-                        'detection_result': ('detection_result', json.dumps(detections), 'application/json'),
-                    }
-
-                    if render_boxes and result['status'] == 200:
-
-                        print('Rendering images')
-
-                        for image_name, detections in detections.items():
-                            
-                            #image = Image.open(os.path.join(temp_direc, image_name))
-                            image = open(f'{temp_direc}/{image_name}', "rb")
-                            image = vis_utils.load_image(image)
-                            width, height = image.size
-
-                            _detections = []
-                            for d in detections:
-                                y1,x1,y2,x2 = d[0:4]
-                                width = x2 - x1
-                                height = y2 - y1
-                                bbox = [x1,y1,width,height]
-                                _detections.append({'bbox': bbox, 'conf': d[4], 'category': d[5]}) 
-                            
-                            vis_utils.render_detection_bounding_boxes(_detections, image, 
-                            confidence_threshold=rendering_confidence_threshold)
-                            
-                            output_img_stream = BytesIO()
-                            image.save(output_img_stream, format='jpeg')
-                            output_img_stream.seek(0)
-                            fields[image_name] = (image_name, output_img_stream, 'image/jpeg')
-                        print('Done rendering images')
-                        
-                    m = MultipartEncoder(fields=fields)                    
-                    return Response(m.to_string(), mimetype=m.content_type)
-
-                except Exception as e:
-                    
-                    print(traceback.format_exc())
-                    print('Error returning result or rendering the detection boxes: ' + str(e))
-
-                finally:
-                    
-                    try:
-                        print('Removing temporary files')
-                        shutil.rmtree(temp_direc)
-                    except Exception as e:
-                        print('Error removing temporary folder {}: {}'.format(temp_direc,str(e)))
-                    
-            else:
-                time.sleep(0.005)
-                
-            # ...if we do/don't have a request available on the queue
-            
-        # ...while(True)
-
-    except Exception as e:
-        
-        print(traceback.format_exc())
-        return _make_error_object(500, 'Error processing images: ' + str(e))
-
-# ...def detect_sync()
-
-
-#%% Command-line driver
-    
-if __name__ == '__main__':
-    
-    parser = argparse.ArgumentParser(description='api frontend')
-
-    # use --non-docker if you are testing without Docker
-    #
-    # python api_frontend.py --non-docker
-    parser.add_argument('--non-docker', action="store_true", default=False)
-    args = parser.parse_args()
-
-    if args.non_docker:
-        app.run(host='0.0.0.0', port=5050)
-    else:
-        app.run()

api/synchronous/api_core/animal_detection_api/config.py

@@ -1,35 +0,0 @@
-## Camera trap real-time API configuration
-
-REDIS_HOST = 'localhost'
-
-REDIS_PORT = 6379
-
-# Full path to the temporary folder for image storage, only meaningful 
-# within the Docker container
-TEMP_FOLDER = '/app/temp'
-
-REDIS_QUEUE_NAME = 'camera-trap-queue'
-
-# Upper limit on total content length (all images and parameters)
-MAX_CONTENT_LENGTH_IN_MB = 5 * 8  # 5MB per image * number of images allowed
-
-MAX_IMAGES_ACCEPTED = 8
-
-IMAGE_CONTENT_TYPES = ['image/png', 'application/octet-stream', 'image/jpeg']
-
-DETECTOR_MODEL_PATH = '/app/animal_detection_api/model/md_v5a.0.0.pt'
-
-DETECTOR_MODEL_VERSION = 'v5a.0.0'
-
-# Minimum confidence threshold for detections
-DEFAULT_CONFIDENCE_THRESHOLD = 0.01
-
-# Minimum confidence threshold for showing a bounding box on the output image
-DEFAULT_RENDERING_CONFIDENCE_THRESHOLD = 0.2
-
-API_PREFIX = '/v1/camera-trap/sync'
-
-API_KEYS_FILE = 'allowed_keys.txt'
-
-# Use this when testing without Docker
-DETECTOR_MODEL_PATH_DEBUG = 'model/md_v5a.0.0.pt'