megadetector 5.0.11__py3-none-any.whl → 5.0.12__py3-none-any.whl

megadetector/api/batch_processing/api_core/server_job_status_table.py

@@ -0,0 +1,152 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+"""
+A class to manage updating the status of an API request / Azure Batch Job using
+the Cosmos DB table "batch_api_jobs".
+"""
+
+import logging
+import os
+import unittest
+import uuid
+from typing import Union, Optional
+
+from azure.cosmos.cosmos_client import CosmosClient
+from azure.cosmos.exceptions import CosmosResourceNotFoundError
+
+from server_api_config import API_INSTANCE_NAME, COSMOS_ENDPOINT, COSMOS_WRITE_KEY
+from server_utils import get_utc_time
+
+
+log = logging.getLogger(os.environ['FLASK_APP'])
+
+
+class JobStatusTable:
+    """
+    A wrapper around the Cosmos DB client. Each item in the table "batch_api_jobs" represents
+    a request/Batch Job, and should have the following fields:
+        - id: this is the job_id
+        - api_instance
+        - status
+        - last_updated
+        - call_params: the dict representing the body of the POST request from the user
+    The 'status' field is a dict with the following fields:
+        - request_status
+        - message
+        - num_tasks  (present after Batch Job created)
+        - num_images  (present after Batch Job created)
+    """
+    # a job moves from created to running/problem after the Batch Job has been submitted
+    allowed_statuses = ['created', 'running', 'failed', 'problem', 'completed', 'canceled']
+
+    def __init__(self, api_instance=None):
+        self.api_instance = api_instance if api_instance is not None else API_INSTANCE_NAME
+        cosmos_client = CosmosClient(COSMOS_ENDPOINT, credential=COSMOS_WRITE_KEY)
+        db_client = cosmos_client.get_database_client('camera-trap')
+        self.db_jobs_client = db_client.get_container_client('batch_api_jobs')
+
+    def create_job_status(self, job_id: str, status: Union[dict, str], call_params: dict) -> dict:
+        assert 'request_status' in status and 'message' in status
+        assert status['request_status'] in JobStatusTable.allowed_statuses
+
+        # job_id should be unique across all instances, and is also the partition key
+        cur_time = get_utc_time()
+        item = {
+            'id': job_id,
+            'api_instance': self.api_instance,
+            'status': status,
+            'job_submission_time': cur_time,
+            'last_updated': cur_time,
+            'call_params': call_params
+        }
+        created_item = self.db_jobs_client.create_item(item)
+        return created_item
+
+    def update_job_status(self, job_id: str, status: Union[dict, str]) -> dict:
+        assert 'request_status' in status and 'message' in status
+        assert status['request_status'] in JobStatusTable.allowed_statuses
+
+        # TODO do not read the entry first to get the call_params when the Cosmos SDK add a
+        # patching functionality:
+        # https://feedback.azure.com/forums/263030-azure-cosmos-db/suggestions/6693091-be-able-to-do-partial-updates-on-document
+        item_old = self.read_job_status(job_id)
+        if item_old is None:
+            raise ValueError
+
+        # need to retain other fields in 'status' to be able to restart monitoring thread
+        if 'status' in item_old and isinstance(item_old['status'], dict):
+            # retain existing fields; update as needed
+            for k, v in item_old['status'].items():
+                if k not in status:
+                    status[k] = v
+        item = {
+            'id': job_id,
+            'api_instance': self.api_instance,
+            'status': status,
+            'job_submission_time': item_old['job_submission_time'],
+            'last_updated': get_utc_time(),
+            'call_params': item_old['call_params']
+        }
+        replaced_item = self.db_jobs_client.replace_item(job_id, item)
+        return replaced_item
+
+    def read_job_status(self, job_id) -> Optional[dict]:
+        """
+        Read the status of the job from the Cosmos DB table of job status.
+        Note that it does not check the actual status of the job on Batch, and just returns what
+        the monitoring thread wrote to the database.
+        job_id is also the partition key
+        """
+        try:
+            read_item = self.db_jobs_client.read_item(job_id, partition_key=job_id)
+            assert read_item['api_instance'] == self.api_instance, 'Job does not belong to this API instance'
+        except CosmosResourceNotFoundError:
+            return None  # job_id not a key
+        except Exception as e:
+            logging.error(f'server_job_status_table, read_job_status, exception: {e}')
+            raise
+        else:
+            item = {k: v for k, v in read_item.items() if not k.startswith('_')}
+            return item
+
+
+class TestJobStatusTable(unittest.TestCase):
+    api_instance = 'api_test'
+
+    def test_insert(self):
+        table = JobStatusTable(TestJobStatusTable.api_instance)
+        status = {
+            'request_status': 'running',
+            'message': 'this is a test'
+        }
+        job_id = uuid.uuid4().hex
+        item = table.create_job_status(job_id, status, {'container_sas': 'random_string'})
+        self.assertTrue(job_id == item['id'], 'Expect job_id to be the id of the item')
+        self.assertTrue(item['status']['request_status'] == 'running', 'Expect fields to be inserted correctly')
+
+    def test_update_and_read(self):
+        table = JobStatusTable(TestJobStatusTable.api_instance)
+        status = {
+            'request_status': 'running',
+            'message': 'this is a test'
+        }
+        job_id = uuid.uuid4().hex
+        res = table.create_job_status(job_id, status, {'container_sas': 'random_string'})
+
+        status = {
+            'request_status': 'completed',
+            'message': 'this is a test again'
+        }
+        res = table.update_job_status(job_id, status)
+        item_read = table.read_job_status(job_id)
+        self.assertTrue(item_read['status']['request_status'] == 'completed', 'Expect field to have updated')
+
+    def test_read_invalid_id(self):
+        table = JobStatusTable(TestJobStatusTable.api_instance)
+        job_id = uuid.uuid4().hex  # should not be in the database
+        item_read = table.read_job_status(job_id)
+        self.assertIsNone(item_read)
+
+if __name__ == '__main__':
+    unittest.main()

megadetector/api/batch_processing/api_core/server_orchestration.py

@@ -0,0 +1,360 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+"""
+Functions to submit images to the Azure Batch node pool for processing, monitor
+the Job and fetch results when completed.
+"""
+
+import io
+import json
+import threading
+import time
+import logging
+import os
+import urllib.parse
+from datetime import timedelta
+from random import shuffle
+
+import sas_blob_utils
+import requests
+from azure.storage.blob import ContainerClient, BlobSasPermissions, generate_blob_sas
+from tqdm import tqdm
+
+from server_utils import *
+import server_api_config as api_config
+from server_batch_job_manager import BatchJobManager
+from server_job_status_table import JobStatusTable
+
+
+# Gunicorn logger handler will get attached if needed in server.py
+log = logging.getLogger(os.environ['FLASK_APP'])
+
+
+def create_batch_job(job_id: str, body: dict):
+    """
+    This is the target to be run in a thread to submit a batch processing job and monitor progress
+    """
+    job_status_table = JobStatusTable()
+    try:
+        log.info(f'server_job, create_batch_job, job_id {job_id}, {body}')
+
+        input_container_sas = body.get('input_container_sas', None)
+
+        use_url = body.get('use_url', False)
+
+        images_requested_json_sas = body.get('images_requested_json_sas', None)
+
+        image_path_prefix = body.get('image_path_prefix', None)
+
+        first_n = body.get('first_n', None)
+        first_n = int(first_n) if first_n else None
+
+        sample_n = body.get('sample_n', None)
+        sample_n = int(sample_n) if sample_n else None
+
+        model_version = body.get('model_version', '')
+        if model_version == '':
+            model_version = api_config.DEFAULT_MD_VERSION
+
+        # request_name and request_submission_timestamp are for appending to
+        # output file names
+        job_name = body.get('request_name', '')  # in earlier versions we used "request" to mean a "job"
+        job_submission_timestamp = get_utc_time()
+
+        # image_paths can be a list of strings (Azure blob names or public URLs)
+        # or a list of length-2 lists where each is a [image_id, metadata] pair
+
+        # Case 1: listing all images in the container
+        # - not possible to have attached metadata if listing images in a blob
+        if images_requested_json_sas is None:
+            log.info('server_job, create_batch_job, listing all images to process.')
+
+            # list all images to process
+            image_paths = sas_blob_utils.list_blobs_in_container(
+                container_uri=input_container_sas,
+                blob_prefix=image_path_prefix,  # check will be case-sensitive
+                blob_suffix=api_config.IMAGE_SUFFIXES_ACCEPTED,  # check will be case-insensitive
+                limit=api_config.MAX_NUMBER_IMAGES_ACCEPTED_PER_JOB + 1
+                # + 1 so if the number of images listed > MAX_NUMBER_IMAGES_ACCEPTED_PER_JOB
+                # we will know and not proceed
+            )
+
+        # Case 2: user supplied a list of images to process; can include metadata
+        else:
+            log.info('server_job, create_batch_job, using provided list of images.')
+
+            response = requests.get(images_requested_json_sas) # could be a file hosted anywhere
+            image_paths = response.json()
+
+            log.info('server_job, create_batch_job, length of image_paths provided by the user: {}'.format(
+                len(image_paths)))
+            if len(image_paths) == 0:
+                job_status = get_job_status(
+                    'completed', '0 images found in provided list of images.')
+                job_status_table.update_job_status(job_id, job_status)
+                return
+
+            error, metadata_available = validate_provided_image_paths(image_paths)
+            if error is not None:
+                msg = 'image paths provided in the json are not valid: {}'.format(error)
+                raise ValueError(msg)
+
+            # filter down to those conforming to the provided prefix and accepted suffixes (image file types)
+            valid_image_paths = []
+            for p in image_paths:
+                locator = p[0] if metadata_available else p
+
+                # prefix is case-sensitive; suffix is not
+                if image_path_prefix is not None and not locator.startswith(image_path_prefix):
+                    continue
+
+                # Although urlparse(p).path preserves the extension on local paths, it will not work for
+                # blob file names that contains "#", which will be treated as indication of a query.
+                # If the URL is generated via Azure Blob Storage, the "#" char will be properly encoded
+                path = urllib.parse.urlparse(locator).path if use_url else locator
+
+                if path.lower().endswith(api_config.IMAGE_SUFFIXES_ACCEPTED):
+                    valid_image_paths.append(p)
+            image_paths = valid_image_paths
+            log.info(('server_job, create_batch_job, length of image_paths provided by user, '
+                      f'after filtering to jpg: {len(image_paths)}'))
+
+        # apply the first_n and sample_n filters
+        if first_n:
+            assert first_n > 0, 'parameter first_n is 0.'
+            # OK if first_n > total number of images
+            image_paths = image_paths[:first_n]
+
+        if sample_n:
+            assert sample_n > 0, 'parameter sample_n is 0.'
+            if sample_n > len(image_paths):
+                msg = ('parameter sample_n specifies more images than '
+                       'available (after filtering by other provided params).')
+                raise ValueError(msg)
+
+            # sample by shuffling image paths and take the first sample_n images
+            log.info('First path before shuffling:', image_paths[0])
+            shuffle(image_paths)
+            log.info('First path after shuffling:', image_paths[0])
+            image_paths = image_paths[:sample_n]
+
+        num_images = len(image_paths)
+        log.info(f'server_job, create_batch_job, num_images after applying all filters: {num_images}')
+
+        if num_images < 1:
+            job_status = get_job_status('completed', (
+                'Zero images found in container or in provided list of images '
+                'after filtering with the provided parameters.'))
+            job_status_table.update_job_status(job_id, job_status)
+            return
+        if num_images > api_config.MAX_NUMBER_IMAGES_ACCEPTED_PER_JOB:
+            job_status = get_job_status(
+                'failed',
+                (f'The number of images ({num_images}) requested for processing exceeds the maximum '
+                 f'accepted {api_config.MAX_NUMBER_IMAGES_ACCEPTED_PER_JOB} in one call'))
+            job_status_table.update_job_status(job_id, job_status)
+            return
+
+        # upload the image list to the container, which is also mounted on all nodes
+        # all sharding and scoring use the uploaded list
+        images_list_str_as_bytes = bytes(json.dumps(image_paths, ensure_ascii=False), encoding='utf-8')
+
+        container_url = sas_blob_utils.build_azure_storage_uri(account=api_config.STORAGE_ACCOUNT_NAME,
+                                                               container=api_config.STORAGE_CONTAINER_API)
+        with ContainerClient.from_container_url(container_url,
+                                                credential=api_config.STORAGE_ACCOUNT_KEY) as api_container_client:
+            _ = api_container_client.upload_blob(
+                name=f'api_{api_config.API_INSTANCE_NAME}/job_{job_id}/{job_id}_images.json',
+                data=images_list_str_as_bytes)
+
+        job_status = get_job_status('created', f'{num_images} images listed; submitting the job...')
+        job_status_table.update_job_status(job_id, job_status)
+
+    except Exception as e:
+        job_status = get_job_status('failed', f'Error occurred while preparing the Batch job: {e}')
+        job_status_table.update_job_status(job_id, job_status)
+        log.error(f'server_job, create_batch_job, Error occurred while preparing the Batch job: {e}')
+        return  # do not start monitoring
+
+    try:
+        batch_job_manager = BatchJobManager()
+
+        model_rel_path = api_config.MD_VERSIONS_TO_REL_PATH[model_version]
+        batch_job_manager.create_job(job_id,
+                                     model_rel_path,
+                                     input_container_sas,
+                                     use_url)
+
+        num_tasks, task_ids_failed_to_submit = batch_job_manager.submit_tasks(job_id, num_images)
+
+        # now request_status moves from created to running
+        job_status = get_job_status('running',
+                                    (f'Submitted {num_images} images to cluster in {num_tasks} shards. '
+                                     f'Number of shards failed to be submitted: {len(task_ids_failed_to_submit)}'))
+
+        # an extra field to allow the monitoring thread to restart after an API restart: total number of tasks
+        job_status['num_tasks'] = num_tasks
+        # also record the number of images to process for reporting
+        job_status['num_images'] = num_images
+
+        job_status_table.update_job_status(job_id, job_status)
+    except Exception as e:
+        job_status = get_job_status('problem', f'Please contact us. Error occurred while submitting the Batch job: {e}')
+        job_status_table.update_job_status(job_id, job_status)
+        log.error(f'server_job, create_batch_job, Error occurred while submitting the Batch job: {e}')
+        return
+
+    # start the monitor thread with the same name
+    try:
+        thread = threading.Thread(
+            target=monitor_batch_job,
+            name=f'job_{job_id}',
+            kwargs={
+                'job_id': job_id,
+                'num_tasks': num_tasks,
+                'model_version': model_version,
+                'job_name': job_name,
+                'job_submission_timestamp': job_submission_timestamp
+            }
+        )
+        thread.start()
+    except Exception as e:
+        job_status = get_job_status('problem', f'Error occurred while starting the monitoring thread: {e}')
+        job_status_table.update_job_status(job_id, job_status)
+        log.error(f'server_job, create_batch_job, Error occurred while starting the monitoring thread: {e}')
+        return
+
+
+def monitor_batch_job(job_id: str,
+                      num_tasks: int,
+                      model_version: str,
+                      job_name: str,
+                      job_submission_timestamp: str):
+
+    job_status_table = JobStatusTable()
+    batch_job_manager = BatchJobManager()
+
+    try:
+        num_checks = 0
+
+        while True:
+            time.sleep(api_config.MONITOR_PERIOD_MINUTES * 60)
+            num_checks += 1
+
+            # both succeeded and failed tasks are marked "completed" on Batch
+            num_tasks_succeeded, num_tasks_failed = batch_job_manager.get_num_completed_tasks(job_id)
+            job_status = get_job_status('running',
+                                        (f'Check number {num_checks}, '
+                                         f'{num_tasks_succeeded} out of {num_tasks} shards have completed '
+                                         f'successfully, {num_tasks_failed} shards have failed.'))
+            job_status_table.update_job_status(job_id, job_status)
+            log.info(f'job_id {job_id}. '
+                f'Check number {num_checks}, {num_tasks_succeeded} out of {num_tasks} shards completed, '
+                f'{num_tasks_failed} shards failed.')
+
+            if (num_tasks_succeeded + num_tasks_failed) >= num_tasks:
+                break
+
+            if num_checks > api_config.MAX_MONITOR_CYCLES:
+                job_status = get_job_status('problem',
+                    (
+                        f'Job unfinished after {num_checks} x {api_config.MONITOR_PERIOD_MINUTES} minutes, '
+                        f'please contact us to retrieve the results. Number of succeeded shards: {num_tasks_succeeded}')
+                    )
+                job_status_table.update_job_status(job_id, job_status)
+                log.warning(f'server_job, create_batch_job, MAX_MONITOR_CYCLES reached, ending thread')
+                break  # still aggregate the Tasks' outputs
+
+    except Exception as e:
+        job_status = get_job_status('problem', f'Error occurred while monitoring the Batch job: {e}')
+        job_status_table.update_job_status(job_id, job_status)
+        log.error(f'server_job, create_batch_job, Error occurred while monitoring the Batch job: {e}')
+        return
+
+    try:
+        output_sas_url = aggregate_results(job_id, model_version, job_name, job_submission_timestamp)
+        # preserving format from before, but SAS URL to 'failed_images' and 'images' are no longer provided
+        # failures should be contained in the output entries, indicated by an 'error' field
+        msg = {
+            'num_failed_shards': num_tasks_failed,
+            'output_file_urls': {
+                'detections': output_sas_url
+            }
+        }
+        job_status = get_job_status('completed', msg)
+        job_status_table.update_job_status(job_id, job_status)
+
+    except Exception as e:
+        job_status = get_job_status('problem',
+                        f'Please contact us to retrieve the results. Error occurred while aggregating results: {e}')
+        job_status_table.update_job_status(job_id, job_status)
+        log.error(f'server_job, create_batch_job, Error occurred while aggregating results: {e}')
+        return
+
+
+def aggregate_results(job_id: str,
+                      model_version: str,
+                      job_name: str,
+                      job_submission_timestamp: str) -> str:
+    log.info(f'server_job, aggregate_results starting, job_id: {job_id}')
+
+    container_url = sas_blob_utils.build_azure_storage_uri(account=api_config.STORAGE_ACCOUNT_NAME,
+                                                           container=api_config.STORAGE_CONTAINER_API)
+    # when people download this, the timestamp will have : replaced by _
+    output_file_path = f'api_{api_config.API_INSTANCE_NAME}/job_{job_id}/{job_id}_detections_{job_name}_{job_submission_timestamp}.json'
+
+    with ContainerClient.from_container_url(container_url,
+                                            credential=api_config.STORAGE_ACCOUNT_KEY) as container_client:
+        # check if the result blob has already been written (could be another instance of the API / worker thread)
+        # and if so, skip aggregating and uploading the results, and just generate the SAS URL, which
+        # could be needed still if the previous request_status was `problem`.
+        blob_client = container_client.get_blob_client(output_file_path)
+        if blob_client.exists():
+            log.warning(f'The output file already exists, likely because another monitoring thread already wrote it.')
+        else:
+            task_outputs_dir = f'api_{api_config.API_INSTANCE_NAME}/job_{job_id}/task_outputs/'
+            generator = container_client.list_blobs(name_starts_with=task_outputs_dir)
+
+            blobs = [i for i in generator if i.name.endswith('.json')]
+
+            all_results = []
+            for blob_props in tqdm(blobs):
+                with container_client.get_blob_client(blob_props) as blob_client:
+                    stream = io.BytesIO()
+                    blob_client.download_blob().readinto(stream)
+                    stream.seek(0)
+                    task_results = json.load(stream)
+                    all_results.extend(task_results)
+
+            api_output = {
+                'info': {
+                    'detector': f'megadetector_v{model_version}',
+                    'detection_completion_time': get_utc_time(),
+                    'format_version': api_config.OUTPUT_FORMAT_VERSION
+                },
+                'detection_categories': api_config.DETECTOR_LABEL_MAP,
+                'images': all_results
+            }
+
+            # upload the output JSON to the Job folder
+            api_output_as_bytes = bytes(json.dumps(api_output, ensure_ascii=False, indent=1), encoding='utf-8')
+            _ = container_client.upload_blob(name=output_file_path, data=api_output_as_bytes)
+
+    output_sas = generate_blob_sas(
+        account_name=api_config.STORAGE_ACCOUNT_NAME,
+        container_name=api_config.STORAGE_CONTAINER_API,
+        blob_name=output_file_path,
+        account_key=api_config.STORAGE_ACCOUNT_KEY,
+        permission=BlobSasPermissions(read=True, write=False),
+        expiry=datetime.utcnow() + timedelta(days=api_config.OUTPUT_SAS_EXPIRATION_DAYS)
+    )
+    output_sas_url = sas_blob_utils.build_azure_storage_uri(
+        account=api_config.STORAGE_ACCOUNT_NAME,
+        container=api_config.STORAGE_CONTAINER_API,
+        blob=output_file_path,
+        sas_token=output_sas
+    )
+    log.info(f'server_job, aggregate_results done, job_id: {job_id}')
+    log.info(f'output_sas_url: {output_sas_url}')
+    return output_sas_url

megadetector/api/batch_processing/api_core/server_utils.py

@@ -0,0 +1,92 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+"""
+Helper functions for the batch processing API.
+"""
+
+import logging
+import os
+from datetime import datetime
+from typing import Tuple, Any, Sequence, Optional
+
+import sas_blob_utils
+
+
+log = logging.getLogger(os.environ['FLASK_APP'])
+
+
+#%% helper classes and functions
+
+def make_error(error_code: int, error_message: str) -> Tuple[dict, int]:
+    # TODO log exception when we have more telemetry
+    log.error(f'Error {error_code} - {error_message}')
+    return {'error': error_message}, error_code
+
+
+def check_data_container_sas(input_container_sas: str) -> Optional[Tuple[int, str]]:
+    """
+    Returns a tuple (error_code, msg) if not a usable SAS URL, else returns None
+    """
+    # TODO check that the expiry date of input_container_sas is at least a month
+    # into the future
+    permissions = sas_blob_utils.get_permissions_from_uri(input_container_sas)
+    data = sas_blob_utils.get_all_query_parts(input_container_sas)
+
+    msg = ('input_container_sas provided does not have both read and list '
+           'permissions.')
+    if 'read' not in permissions or 'list' not in permissions:
+        if 'si' in data:
+            # if no permission specified explicitly but has an access policy, assumes okay
+            # TODO - check based on access policy as well
+            return None
+
+        return 400, msg
+
+    return None
+
+
+def get_utc_time() -> str:
+    # return current UTC time as a string in the ISO 8601 format (so we can query by
+    # timestamp in the Cosmos DB job status table.
+    # example: '2021-02-08T20:02:05.699689Z'
+    return datetime.utcnow().isoformat(timespec='microseconds') + 'Z'
+
+
+def get_job_status(request_status: str, message: Any) -> dict:
+    return {
+        'request_status': request_status,
+        'message': message
+    }
+
+
+def validate_provided_image_paths(image_paths: Sequence[Any]) -> Tuple[Optional[str], bool]:
+    """Given a list of image_paths (list length at least 1), validate them and
+    determine if metadata is available.
+    Args:
+        image_paths: a list of string (image_id) or a list of 2-item lists
+            ([image_id, image_metadata])
+    Returns:
+        error: None if checks passed, otherwise a string error message
+        metadata_available: bool, True if available
+    """
+    # image_paths will have length at least 1, otherwise would have ended before this step
+    first_item = image_paths[0]
+    metadata_available = False
+    if isinstance(first_item, str):
+        for i in image_paths:
+            if not isinstance(i, str):
+                error = 'Not all items in image_paths are of type string.'
+                return error, metadata_available
+        return None, metadata_available
+    elif isinstance(first_item, list):
+        metadata_available = True
+        for i in image_paths:
+            if len(i) != 2:  # i should be [image_id, metadata_string]
+                error = ('Items in image_paths are lists, but not all lists '
+                         'are of length 2 [image locator, metadata].')
+                return error, metadata_available
+        return None, metadata_available
+    else:
+        error = 'image_paths contain items that are not strings nor lists.'
+        return error, metadata_available

megadetector/api/batch_processing/api_core_support/aggregate_results_manually.py

@@ -0,0 +1,46 @@
+#
+# If a request has been sent to AML for batch scoring but the monitoring thread of the API was
+# interrupted (uncaught exception or having to re-start the API container), we could manually
+# aggregate results from each shard using this script, assuming all jobs submitted to AML have finished.
+#
+# Need to have set environment variables STORAGE_ACCOUNT_NAME and STORAGE_ACCOUNT_KEY to those of the
+# storage account backing the API. Also need to adjust the INTERNAL_CONTAINER, AML_CONTAINER and
+# AML_CONFIG fields in api_core/orchestrator_api/api_config.py to match the instance of the API that this
+# request was submitted to.
+#
+# May need to change the import statement in api_core/orchestrator_api/orchestrator.py
+# "from sas_blob_utils import SasBlob" to
+# "from .sas_blob_utils import SasBlob" to not confuse with the module in AI4Eutils;
+# and change "import api_config" to
+# "from api.batch_processing.api_core.orchestrator_api import api_config"
+
+# Execute this script from the root of the repository. You may need to add the repository to PYTHONPATH.
+
+import argparse
+import json
+
+from api.batch_processing.api_core.orchestrator_api.orchestrator import AMLMonitor
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('shortened_request_id', type=str,
+                        help='the request ID to restart monitoring')
+    parser.add_argument('model_version', type=str, help='version of megadetector used; this is used to fill in the meta info section of the output file')
+    parser.add_argument('request_name', type=str, help='easy to remember name for that job, optional', default='')
+    args = parser.parse_args()
+
+
+    # list_jobs_submitted cannot be serialized ("can't pickle _thread.RLock objects "), but
+    # do not need it for aggregating results
+    aml_monitor = AMLMonitor(request_id=args.request_id,
+                             list_jobs_submitted=None,
+                             request_name=args.request_name,
+                             request_submission_timestamp='',
+                             model_version=args.model_version)
+    output_file_urls = aml_monitor.aggregate_results()
+    output_file_urls_str = json.dumps(output_file_urls)
+    print(output_file_urls_str)
+
+if __name__ == '__main__':
+    main()