megadetector 5.0.29__py3-none-any.whl → 10.0.0__py3-none-any.whl

megadetector/api/batch_processing/api_support/summarize_daily_activity.py

@@ -1,152 +0,0 @@
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License.
-
-"""
-This script can be run in a separate process to monitor all instances of the batch API.
-It sends a digest of submissions within the past day to a Teams channel webhook.
-
-It requires the environment variables TEAMS_WEBHOOK, COSMOS_ENDPOINT and COSMOS_READ_KEY to be set.
-"""
-
-import time
-import os
-import json
-from datetime import datetime, timezone, timedelta
-from collections import defaultdict
-
-import requests
-from azure.cosmos.cosmos_client import CosmosClient
-
-
-# Cosmos DB `batch-api-jobs` table for job status
-COSMOS_ENDPOINT = os.environ['COSMOS_ENDPOINT']
-COSMOS_READ_KEY = os.environ['COSMOS_READ_KEY']
-
-TEAMS_WEBHOOK = os.environ['TEAMS_WEBHOOK']
-
-
-def send_message():
-    cosmos_client = CosmosClient(COSMOS_ENDPOINT, credential=COSMOS_READ_KEY)
-    db_client = cosmos_client.get_database_client('camera-trap')
-    db_jobs_client = db_client.get_container_client('batch_api_jobs')
-
-    yesterday = datetime.now(timezone.utc).date() - timedelta(days=1)
-
-    query = f'''
-    SELECT *
-    FROM job
-    WHERE job.job_submission_time >= "{yesterday.isoformat()}T00:00:00Z"
-    '''
-
-    result_iterable = db_jobs_client.query_items(query=query,
-                                                 enable_cross_partition_query=True)
-
-    # aggregate the number of images, country and organization names info from each job
-    # submitted during yesterday (UTC time)
-    instance_num_images = defaultdict(lambda: defaultdict(int))
-    instance_countries = defaultdict(set)
-    instance_orgs = defaultdict(set)
-
-    total_images_received = 0
-
-    for job in result_iterable:
-        api_instance = job['api_instance']
-        status = job['status']
-        call_params = job['call_params']
-
-        if status['request_status'] == 'completed':
-            instance_num_images[api_instance]['num_images_completed'] += status.get('num_images', 0)
-        instance_num_images[api_instance]['num_images_total'] += status.get('num_images', 0)
-        total_images_received += status.get('num_images', 0)
-
-        instance_countries[api_instance].add(call_params.get('country', 'unknown'))
-        instance_orgs[api_instance].add(call_params.get('organization_name', 'unknown'))
-
-    print(f'send_message, number of images received yesterday: {total_images_received}')
-
-    if total_images_received < 1:
-        print('send_message, no images submitted yesterday, not sending a summary')
-        print('')
-        return
-
-    # create the card
-    sections = []
-
-    for instance_name, num_images in instance_num_images.items():
-        entry = {
-            'activityTitle': f'API instance: {instance_name}',
-            'facts': [
-                {
-                    'name': 'Total images',
-                    'value': '{:,}'.format(num_images['num_images_total'])
-                },
-                {
-                    'name': 'Images completed',
-                    'value': '{:,}'.format(num_images['num_images_completed'])
-                },
-                {
-                    'name': 'Countries',
-                    'value': ', '.join(sorted(list(instance_countries[instance_name])))
-                },
-                {
-                    'name': 'Organizations',
-                    'value': ', '.join(sorted(list(instance_orgs[instance_name])))
-                }
-            ]
-        }
-        sections.append(entry)
-
-    card = {
-        '@type': 'MessageCard',
-        '@context': 'http://schema.org/extensions',
-        'themeColor': 'ffcdb2',
-        'summary': 'Digest of batch API activities over the past 24 hours',
-        'title': f'Camera traps batch API activities on {yesterday.strftime("%b %d, %Y")}',
-        'sections': sections,
-        'potentialAction': [
-            {
-                '@type': 'OpenUri',
-                'name': 'View Batch account in Azure Portal',
-                'targets': [
-                    {
-                        'os': 'default',
-                        'uri': 'https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/74d91980-e5b4-4fd9-adb6-263b8f90ec5b/resourcegroups/camera_trap_api_rg/providers/Microsoft.Batch/batchAccounts/cameratrapssc/accountOverview'
-                    }
-                ]
-            }
-        ]
-    }
-
-    response = requests.post(TEAMS_WEBHOOK, data=json.dumps(card))
-    print(f'send_message, card to send:')
-    print(json.dumps(card, indent=4))
-    print(f'send_message, sent summary to webhook, response code: {response.status_code}')
-    print('')
-
-
-def main(): # noqa
-    """
-    Wake up at 5 minutes past midnight UTC to send a summary of yesterday's activities if there were any.
-    Then goes in a loop to wake up and send a summary every 24 hours.
-    """
-    current = datetime.utcnow()
-    future = current.replace(day=current.day, hour=0, minute=5, second=0, microsecond=0) + timedelta(
-        days=1)  # current has been modified
-
-    current = datetime.utcnow()
-    duration = future - current
-
-    duration_hours = duration.seconds / (60 * 60)
-    print(f'Current time: {current}')
-    print(f'Will wake up at {future}, in {duration_hours} hours')
-    print('')
-
-    time.sleep(duration.seconds)
-
-    while True:
-        print(f'Woke up at {datetime.utcnow()}')
-        send_message()
-        time.sleep(24 * 60 * 60)
-
-if __name__ == '__main__':
-    main()

megadetector/api/synchronous/api_core/animal_detection_api/api_backend.py

@@ -1,151 +0,0 @@
-#
-# api_backend.py
-#
-# Defines the model execution service, which pulls requests (one or more images)
-# from the shared Redis queue, and runs them through the TF model.
-#
-
-#%% Imports
-
-import os
-import json
-import time
-import redis
-import argparse
-import PIL
-
-from detection.run_detector import load_detector
-from utils.ct_utils import convert_xywh_to_xyxy as convert_to_tf_coords
-import config
-import visualization.visualization_utils as vis_utils 
-
-#%% Initialization
-
-db = redis.StrictRedis(host=config.REDIS_HOST, port=config.REDIS_PORT)
-current_directory = os.path.dirname(os.path.realpath(__file__))
-
-
-#%% Main loop
-
-def detect_process():
-     
-     while True:
-        
-        # TODO: convert to a blocking read and eliminate the sleep() statement in this loop
-        serialized_entry = db.lpop(config.REDIS_QUEUE_NAME)
-        all_detection_results = []
-        inference_time_detector = []
-        
-        if serialized_entry:
-            
-            entry = json.loads(serialized_entry)
-            id = entry['id']
-            print('Processing images from request id:', id)
-            return_confidence_threshold = entry['return_confidence_threshold']
-
-            try:
-                
-                temp_direc = os.path.join(config.TEMP_FOLDER,id)
-                assert os.path.isdir(temp_direc), 'Could not find temporary folder {}'.format(temp_direc)
-                
-                for filename in os.listdir(temp_direc):
-                    
-                    image_path = f'{temp_direc}/{filename}'
-                    print('Reading image from {}'.format(image_path))
-                    image = open(image_path, 'rb')
-                    image = vis_utils.load_image(image)
-
-                    start_time = time.time()
-                    result = detector.generate_detections_one_image(image, filename, detection_threshold=config.DEFAULT_CONFIDENCE_THRESHOLD)
-                    all_detection_results.append(result)
-
-                    elapsed = time.time() - start_time
-                    inference_time_detector.append(elapsed)
-                    
-            except Exception as e:
-
-                print('Detection error: ' + str(e))
-                
-                db.set(entry['id'], json.dumps({ 
-                    'status': 500,
-                    'error': 'Detection error: ' + str(e)
-                }))
-
-                continue
-
-            # Filter the detections by the confidence threshold
-            #
-            # Each result is [ymin, xmin, ymax, xmax, confidence, category]
-            #
-            # Coordinates are relative, with the origin in the upper-left
-            detections = {}  
-            
-            try:
-                
-                for result in all_detection_results:
-                    
-                    image_name = result['file']
-                    _detections = result.get('detections', None)
-                    detections[image_name] = []
-
-                    if _detections is None:
-                        continue
-
-                    for d in _detections:
-                        if d['conf'] > return_confidence_threshold:
-                            res = convert_to_tf_coords(d['bbox'])
-                            res.append(d['conf'])
-                            res.append(int(d['category']))
-                            detections[image_name].append(res)
-    
-                db.set(entry['id'], json.dumps({ 
-                    'status': 200,
-                    'detections': detections,
-                    'inference_time_detector': inference_time_detector
-                }))
-              
-            except Exception as e:
-                print('Error consolidating the detection boxes: ' + str(e))
-                
-                db.set(entry['id'], json.dumps({ 
-                    'status': 500,
-                    'error': 'Error consolidating the detection boxes:' + str(e)
-                }))
-
-        # ...if serialized_entry
-        
-        else:
-            time.sleep(0.005)
-    
-    # ...while(True)
-    
-# ...def detect_process()    
-        
-
-#%% Command-line driver
-
-if __name__ == '__main__':
-    
-    parser = argparse.ArgumentParser(description='api backend')
-
-    # use --non-docker if you are testing without Docker
-    #
-    # python api_frontend.py --non-docker
-    parser.add_argument('--non-docker', action='store_true', default=False)
-    args = parser.parse_args()
-
-    if args.non_docker:
-        model_path = config.DETECTOR_MODEL_PATH_DEBUG
-    else:
-        model_path = config.DETECTOR_MODEL_PATH
-
-    detector = load_detector(model_path)
-
-    # run detections on a test image to load the model
-    print('Running initial detection to load model...')
-    test_image = PIL.Image.new(mode="RGB", size=(200, 200))
-    result = detector.generate_detections_one_image(test_image, "test_image", detection_threshold=config.DEFAULT_CONFIDENCE_THRESHOLD)
-    print(result)
-    print('\n')
-
-    detect_process() 

megadetector/api/synchronous/api_core/animal_detection_api/api_frontend.py

@@ -1,263 +0,0 @@
-# 
-# api_frontend.py
-#
-# Defines the Flask app, which takes requests (one or more images) from
-# remote callers and pushes the images onto the shared Redis queue, to be processed
-# by the main service in api_backend.py .
-#
-
-#%% Imports
-
-import os
-import json
-import time
-import uuid
-import redis
-import shutil
-import argparse
-import traceback
-
-from io import BytesIO
-from flask import Flask, Response, jsonify, make_response, request
-from requests_toolbelt.multipart.encoder import MultipartEncoder
-
-import visualization.visualization_utils as vis_utils
-import config
-
-
-#%% Initialization
-
-app = Flask(__name__)
-db = redis.StrictRedis(host=config.REDIS_HOST, port=config.REDIS_PORT)
-
-
-#%% Support functions
-
-def _make_error_object(error_code, error_message):
-    
-    # Make a dict that the request_processing_function can return to the endpoint 
-    # function to notify it of an error
-    return {
-        'error_message': error_message,
-        'error_code': error_code
-    }
-
-
-def _make_error_response(error_code, error_message):
-
-    return make_response(jsonify({'error': error_message}), error_code)
-
-
-def has_access(request):
-    
-    if not os.path.exists(config.API_KEYS_FILE):
-        return True
-    else:
-        if not request.headers.get('key'):
-            print('Key header not available')
-            return False
-        else:
-            API_key = request.headers.get('key').strip().lower()
-            with open(config.API_KEYS_FILE, "r") as f:
-                for line in f:
-                    valid_key = line.strip().lower()
-                    if valid_key == API_key:
-                        return True
-
-    return False
-
-
-def check_posted_data(request):
- 
-    files = request.files
-    params = request.args
-
-    # Verify that the content uploaded is not too big
-    #
-    # request.content_length is the length of the total payload
-    content_length = request.content_length
-    if not content_length:
-        return _make_error_object(411, 'No image(s) were sent, or content length cannot be determined.')
-    if content_length > config.MAX_CONTENT_LENGTH_IN_MB * 1024 * 1024:
-        return _make_error_object(413, ('Payload size {:.2f} MB exceeds the maximum allowed of {} MB. '
-                    'Please upload fewer or more compressed images.').format(
-            content_length / (1024 * 1024), config.MAX_CONTENT_LENGTH_IN_MB))
-
-    render_boxes = True if params.get('render', '').lower() == 'true' else False
-
-    if 'min_confidence' in params:
-        return_confidence_threshold = float(params['min_confidence'])
-        print('runserver, post_detect_sync, user specified detection confidence: ', return_confidence_threshold)
-        if return_confidence_threshold < 0.0 or return_confidence_threshold > 1.0:
-            return _make_error_object(400, 'Detection confidence threshold {} is invalid, should be between 0.0 and 1.0.'.format(
-                return_confidence_threshold))
-    else:
-        return_confidence_threshold = config.DEFAULT_CONFIDENCE_THRESHOLD
-        
-    if 'min_rendering_confidence' in params:
-        rendering_confidence_threshold = float(params['min_rendering_confidence'])
-        print('runserver, post_detect_sync, user specified rendering confidence: ', rendering_confidence_threshold) 
-        if rendering_confidence_threshold < 0.0 or rendering_confidence_threshold > 1.0:
-            return _make_error_object(400, 'Rendering confidence threshold {} is invalid, should be between 0.0 and 1.0.'.format(
-                rendering_confidence_threshold))
-    else:
-        rendering_confidence_threshold =  config.DEFAULT_RENDERING_CONFIDENCE_THRESHOLD
-    
-    # Verify that the number of images is acceptable
-    num_images = sum([1 if file.content_type in config.IMAGE_CONTENT_TYPES else 0 for file in files.values()])
-    print('runserver, post_detect_sync, number of images received: ', num_images)
-
-    if num_images > config.MAX_IMAGES_ACCEPTED:
-        return _make_error_object(413, 'Too many images. Maximum number of images that can be processed in one call is {}.'.format(config.MAX_IMAGES_ACCEPTED))
-    elif num_images == 0:
-        return _make_error_object(400, 'No image(s) of accepted types (image/jpeg, image/png, application/octet-stream) received.')
-    
-    return {
-        'render_boxes': render_boxes,
-        'return_confidence_threshold': return_confidence_threshold,
-        'rendering_confidence_threshold': rendering_confidence_threshold
-    }
-
-# ...def check_posted_data(request)
-    
-
-#%% Main loop
-
-@app.route(config.API_PREFIX + '/detect', methods = ['POST'])
-def detect_sync():
-    
-    if not has_access(request):
-        print('Access denied, please provide a valid API key')
-        return _make_error_response(403, 'Access denied, please provide a valid API key')
-
-    # Check whether the request_processing_function had an error
-    post_data = check_posted_data(request)
-    if post_data.get('error_code', None) is not None:
-        return _make_error_response(post_data.get('error_code'), post_data.get('error_message'))
-
-    render_boxes = post_data.get('render_boxes')
-    return_confidence_threshold = post_data.get('return_confidence_threshold')
-    rendering_confidence_threshold = post_data.get('rendering_confidence_threshold')
-  
-    redis_id = str(uuid.uuid4())
-    d = {'id': redis_id, 'render_boxes': render_boxes, 'return_confidence_threshold': return_confidence_threshold}
-    temp_direc = os.path.join(config.TEMP_FOLDER, redis_id)
-    
-    try:
-        
-        try:
-            # Write images to temporary files
-            os.makedirs(temp_direc,exist_ok=True)
-            for name, file in request.files.items():
-                if file.content_type in config.IMAGE_CONTENT_TYPES:
-                    filename = request.files[name].filename
-                    image_path = os.path.join(temp_direc, filename)
-                    print('Saving image {} to {}'.format(name,image_path))
-                    file.save(image_path)
-                    assert os.path.isfile(image_path),'Error creating file {}'.format(image_path)
-        
-        except Exception as e:
-            return _make_error_object(500, 'Error saving images: ' + str(e))
-        
-        # Submit the image(s) for processing by api_backend.py, who is waiting on this queue
-        db.rpush(config.REDIS_QUEUE_NAME, json.dumps(d))
-        
-        while True:
-            
-            result = db.get(redis_id)
-            
-            if result:
-                
-                result = json.loads(result.decode())
-                print('Processing result {}'.format(str(result)))
-                
-                if result['status'] == 200:
-                    detections = result['detections']
-                    db.delete(redis_id)
-                
-                else:
-                    db.delete(redis_id)
-                    print('Detection error: ' + str(result))
-                    return _make_error_response(500, 'Detection error: ' + str(result))
-
-                try:
-                    print('detect_sync: postprocessing and sending images back...')
-                    fields = {
-                        'detection_result': ('detection_result', json.dumps(detections), 'application/json'),
-                    }
-
-                    if render_boxes and result['status'] == 200:
-
-                        print('Rendering images')
-
-                        for image_name, detections in detections.items():
-                            
-                            #image = Image.open(os.path.join(temp_direc, image_name))
-                            image = open(f'{temp_direc}/{image_name}', "rb")
-                            image = vis_utils.load_image(image)
-                            width, height = image.size
-
-                            _detections = []
-                            for d in detections:
-                                y1,x1,y2,x2 = d[0:4]
-                                width = x2 - x1
-                                height = y2 - y1
-                                bbox = [x1,y1,width,height]
-                                _detections.append({'bbox': bbox, 'conf': d[4], 'category': d[5]}) 
-                            
-                            vis_utils.render_detection_bounding_boxes(_detections, image, 
-                            confidence_threshold=rendering_confidence_threshold)
-                            
-                            output_img_stream = BytesIO()
-                            image.save(output_img_stream, format='jpeg')
-                            output_img_stream.seek(0)
-                            fields[image_name] = (image_name, output_img_stream, 'image/jpeg')
-                        print('Done rendering images')
-                        
-                    m = MultipartEncoder(fields=fields)                    
-                    return Response(m.to_string(), mimetype=m.content_type)
-
-                except Exception as e:
-                    
-                    print(traceback.format_exc())
-                    print('Error returning result or rendering the detection boxes: ' + str(e))
-
-                finally:
-                    
-                    try:
-                        print('Removing temporary files')
-                        shutil.rmtree(temp_direc)
-                    except Exception as e:
-                        print('Error removing temporary folder {}: {}'.format(temp_direc,str(e)))
-                    
-            else:
-                time.sleep(0.005)
-                
-            # ...if we do/don't have a request available on the queue
-            
-        # ...while(True)
-
-    except Exception as e:
-        
-        print(traceback.format_exc())
-        return _make_error_object(500, 'Error processing images: ' + str(e))
-
-# ...def detect_sync()
-
-
-#%% Command-line driver
-    
-if __name__ == '__main__':
-    
-    parser = argparse.ArgumentParser(description='api frontend')
-
-    # use --non-docker if you are testing without Docker
-    #
-    # python api_frontend.py --non-docker
-    parser.add_argument('--non-docker', action="store_true", default=False)
-    args = parser.parse_args()
-
-    if args.non_docker:
-        app.run(host='0.0.0.0', port=5050)
-    else:
-        app.run()

megadetector/api/synchronous/api_core/animal_detection_api/config.py

@@ -1,35 +0,0 @@
-## Camera trap real-time API configuration
-
-REDIS_HOST = 'localhost'
-
-REDIS_PORT = 6379
-
-# Full path to the temporary folder for image storage, only meaningful 
-# within the Docker container
-TEMP_FOLDER = '/app/temp'
-
-REDIS_QUEUE_NAME = 'camera-trap-queue'
-
-# Upper limit on total content length (all images and parameters)
-MAX_CONTENT_LENGTH_IN_MB = 5 * 8  # 5MB per image * number of images allowed
-
-MAX_IMAGES_ACCEPTED = 8
-
-IMAGE_CONTENT_TYPES = ['image/png', 'application/octet-stream', 'image/jpeg']
-
-DETECTOR_MODEL_PATH = '/app/animal_detection_api/model/md_v5a.0.0.pt'
-
-DETECTOR_MODEL_VERSION = 'v5a.0.0'
-
-# Minimum confidence threshold for detections
-DEFAULT_CONFIDENCE_THRESHOLD = 0.01
-
-# Minimum confidence threshold for showing a bounding box on the output image
-DEFAULT_RENDERING_CONFIDENCE_THRESHOLD = 0.2
-
-API_PREFIX = '/v1/camera-trap/sync'
-
-API_KEYS_FILE = 'allowed_keys.txt'
-
-# Use this when testing without Docker
-DETECTOR_MODEL_PATH_DEBUG = 'model/md_v5a.0.0.pt'