megadetector 5.0.11__py3-none-any.whl → 5.0.12__py3-none-any.whl

megadetector/api/batch_processing/api_core/server.py

@@ -0,0 +1,294 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import string
+import uuid
+import threading
+from datetime import timedelta
+
+import sas_blob_utils
+from flask import Flask, request, jsonify
+
+import server_api_config as api_config
+from server_app_config import AppConfig
+from server_batch_job_manager import BatchJobManager
+from server_orchestration import create_batch_job, monitor_batch_job
+from server_job_status_table import JobStatusTable
+from server_utils import *
+
+# %% Flask app
+app = Flask(__name__)
+
+# reference: https://trstringer.com/logging-flask-gunicorn-the-manageable-way/
+if __name__ != '__main__':
+    gunicorn_logger = logging.getLogger('gunicorn.error')
+    app.logger.handlers = gunicorn_logger.handlers
+    app.logger.setLevel(gunicorn_logger.level)
+
+
+API_PREFIX = api_config.API_PREFIX
+app.logger.info('server, created Flask application...')
+
+# %% Helper classes
+
+app_config = AppConfig()
+job_status_table = JobStatusTable()
+batch_job_manager = BatchJobManager()
+app.logger.info('server, finished instantiating helper classes')
+
+
+# %% Flask endpoints
+
+@app.route(f'{API_PREFIX}/')
+def hello():
+    return f'Camera traps batch processing API. Instance: {api_config.API_INSTANCE_NAME}'
+
+
+@app.route(f'{API_PREFIX}/request_detections', methods=['POST'])
+def request_detections():
+    """
+    Checks that the input parameters to this endpoint are valid, starts a thread
+    to launch the batch processing job, and return the job_id/request_id to the user.
+    """
+    if not request.is_json:
+        msg = 'Body needs to have a JSON mimetype (e.g., application/json).'
+        return make_error(415, msg)
+
+    try:
+        post_body = request.get_json()
+    except Exception as e:
+        return make_error(415, f'Error occurred reading POST request body: {e}.')
+
+    app.logger.info(f'server, request_detections, post_body: {post_body}')
+
+    # required params
+
+    caller_id = post_body.get('caller', None)
+    if caller_id is None or caller_id not in app_config.get_allowlist():
+        msg = ('Parameter caller is not supplied or is not on our allowlist. '
+               'Please email cameratraps@lila.science to request access.')
+        return make_error(401, msg)
+
+    use_url = post_body.get('use_url', False)
+    if use_url and isinstance(use_url, str):  # in case it is included but is intended to be False
+        if use_url.lower() in ['false', 'f', 'no', 'n']:
+            use_url = False
+
+    input_container_sas = post_body.get('input_container_sas', None)
+    if not input_container_sas and not use_url:
+        msg = ('input_container_sas with read and list access is a required '
+               'field when not using image URLs.')
+        return make_error(400, msg)
+
+    if input_container_sas is not None:
+        if not sas_blob_utils.is_container_uri(input_container_sas):
+            return make_error(400, 'input_container_sas provided is not for a container.')
+
+        result = check_data_container_sas(input_container_sas)
+        if result is not None:
+            return make_error(result[0], result[1])
+
+    # can be an URL to a file not hosted in an Azure blob storage container
+    images_requested_json_sas = post_body.get('images_requested_json_sas', None)
+
+    if images_requested_json_sas is not None:
+        if not images_requested_json_sas.startswith(('http://', 'https://')):
+            return make_error(400, 'images_requested_json_sas needs to be an URL.')
+
+    # if use_url, then images_requested_json_sas is required
+    if use_url and images_requested_json_sas is None:
+            return make_error(400, 'images_requested_json_sas is required since use_url is true.')
+
+    # optional params
+
+    # check model_version is among the available model versions
+    model_version = post_body.get('model_version', '')
+    if model_version != '':
+        model_version = str(model_version)  # in case user used an int
+        if model_version not in api_config.MD_VERSIONS_TO_REL_PATH:  # TODO use AppConfig to store model version info
+            return make_error(400, f'model_version {model_version} is not supported.')
+
+    # check request_name has only allowed characters
+    request_name = post_body.get('request_name', '')
+    if request_name != '':
+        if len(request_name) > 92:
+            return make_error(400, 'request_name is longer than 92 characters.')
+        allowed = set(string.ascii_letters + string.digits + '_' + '-')
+        if not set(request_name) <= allowed:
+            msg = ('request_name contains invalid characters (only letters, '
+                   'digits, - and _ are allowed).')
+            return make_error(400, msg)
+
+    # optional params for telemetry collection - logged to status table for now as part of call_params
+    country = post_body.get('country', None)
+    organization_name = post_body.get('organization_name', None)
+
+    # All API instances / node pools share a quota on total number of active Jobs;
+    # we cannot accept new Job submissions if we are at the quota
+    try:
+        num_active_jobs = batch_job_manager.get_num_active_jobs()
+        if num_active_jobs >= api_config.MAX_BATCH_ACCOUNT_ACTIVE_JOBS:
+            return make_error(503, f'Too many active jobs, please try again later')
+    except Exception as e:
+        return make_error(500, f'Error checking number of active jobs: {e}')
+
+    try:
+        job_id = uuid.uuid4().hex
+        job_status_table.create_job_status(
+            job_id=job_id,
+            status= get_job_status('created', 'Request received. Listing images next...'),
+            call_params=post_body
+        )
+    except Exception as e:
+        return make_error(500, f'Error creating a job status entry: {e}')
+
+    try:
+        thread = threading.Thread(
+            target=create_batch_job,
+            name=f'job_{job_id}',
+            kwargs={'job_id': job_id, 'body': post_body}
+        )
+        thread.start()
+    except Exception as e:
+        return make_error(500, f'Error creating or starting the batch processing thread: {e}')
+
+    return {'request_id': job_id}
+
+
+@app.route(f'{API_PREFIX}/cancel_request', methods=['POST'])
+def cancel_request():
+    """
+    Cancels a request/job given the job_id and caller_id
+    """
+    if not request.is_json:
+        msg = 'Body needs to have a JSON mimetype (e.g., application/json).'
+        return make_error(415, msg)
+    try:
+        post_body = request.get_json()
+    except Exception as e:
+        return make_error(415, f'Error occurred reading POST request body: {e}.')
+
+    app.logger.info(f'server, cancel_request received, body: {post_body}')
+
+    # required fields
+    job_id = post_body.get('request_id', None)
+    if job_id is None:
+        return make_error(400, 'request_id is a required field.')
+
+    caller_id = post_body.get('caller', None)
+    if caller_id is None or caller_id not in app_config.get_allowlist():
+        return make_error(401, 'Parameter caller is not supplied or is not on our allowlist.')
+
+    item_read = job_status_table.read_job_status(job_id)
+    if item_read is None:
+        return make_error(404, 'Task is not found.')
+    if 'status' not in item_read:
+        return make_error(404, 'Something went wrong. This task does not have a status field.')
+
+    request_status = item_read['status']['request_status']
+    if request_status not in ['running', 'problem']:
+        # request_status is either completed or failed
+        return make_error(400, f'Task has {request_status} and cannot be canceled')
+
+    try:
+        batch_job_manager.cancel_batch_job(job_id)
+        # the create_batch_job thread will stop when it wakes up the next time
+    except Exception as e:
+        return make_error(500, f'Error when canceling the request: {e}')
+    else:
+        job_status_table.update_job_status(job_id, {
+            'request_status': 'canceled',
+            'message': 'Request has been canceled by the user.'
+        })
+    return 'Canceling signal has been sent. You can verify the status at the /task endpoint'
+
+
+@app.route(f'{API_PREFIX}/task/<job_id>')
+def retrieve_job_status(job_id: str):
+    """
+    Does not require the "caller" field to avoid checking the allowlist in App Configurations.
+    Retains the /task endpoint name to be compatible with previous versions.
+    """
+    # Fix for Zooniverse - deleting any "-" characters in the job_id
+    job_id = job_id.replace('-', '')
+
+    item_read = job_status_table.read_job_status(job_id)  # just what the monitoring thread wrote to the DB
+    if item_read is None:
+        return make_error(404, 'Task is not found.')
+    if 'status' not in item_read or 'last_updated' not in item_read or 'call_params' not in item_read:
+        return make_error(404, 'Something went wrong. This task does not have a valid status.')
+
+    # If the status is running, it could be a Job submitted before the last restart of this
+    # API instance. If that is the case, we should start to monitor its progress again.
+    status = item_read['status']
+
+    last_updated = datetime.fromisoformat(item_read['last_updated'][:-1])  # get rid of "Z" (required by Cosmos DB)
+    time_passed = datetime.utcnow() - last_updated
+    job_is_unmonitored = True if time_passed > timedelta(minutes=(api_config.MONITOR_PERIOD_MINUTES + 1)) else False
+
+    if isinstance(status, dict) and \
+            'request_status' in status and \
+            status['request_status'] in ['running', 'problem'] and \
+            'num_tasks' in status and \
+            job_id not in get_thread_names() and \
+            job_is_unmonitored:
+        # WARNING model_version could be wrong (a newer version number gets written to the output file) around
+        # the time that  the model is updated, if this request was submitted before the model update
+        # and the API restart; this should be quite rare
+        model_version = item_read['call_params'].get('model_version', api_config.DEFAULT_MD_VERSION)
+
+        num_tasks = status['num_tasks']
+        job_name = item_read['call_params'].get('request_name', '')
+        job_submission_timestamp = item_read.get('job_submission_time', '')
+
+        thread = threading.Thread(
+            target=monitor_batch_job,
+            name=f'job_{job_id}',
+            kwargs={
+                'job_id': job_id,
+                'num_tasks': num_tasks,
+                'model_version': model_version,
+                'job_name': job_name,
+                'job_submission_timestamp': job_submission_timestamp
+            }
+        )
+        thread.start()
+        app.logger.info(f'server, started a new thread to monitor job {job_id}')
+
+    # conform to previous schemes
+    if 'num_tasks' in status:
+        del status['num_tasks']
+    item_to_return = {
+        'Status': status,
+        'Endpoint': f'{API_PREFIX}/request_detections',
+        'TaskId': job_id,
+        'Timestamp': item_read['last_updated']
+    }
+    return item_to_return
+
+
+@app.route(f'{API_PREFIX}/default_model_version')
+def get_default_model_version() -> str:
+    return api_config.DEFAULT_MD_VERSION
+
+
+@app.route(f'{API_PREFIX}/supported_model_versions')
+def get_supported_model_versions() -> str:
+    return jsonify(sorted(list(api_config.MD_VERSIONS_TO_REL_PATH.keys())))
+
+
+# %% undocumented endpoints
+
+def get_thread_names() -> list:
+    thread_names = []
+    for thread in threading.enumerate():
+        if thread.name.startswith('job_'):
+            thread_names.append(thread.name.split('_')[1])
+    return sorted(thread_names)
+
+
+@app.route(f'{API_PREFIX}/all_jobs')
+def get_all_jobs():
+    """List all Jobs being monitored since this API instance started"""
+    thread_names = get_thread_names()
+    return jsonify(thread_names)

megadetector/api/batch_processing/api_core/server_api_config.py

@@ -0,0 +1,98 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+"""
+A module to hold the configurations specific to an instance of the API.
+"""
+
+import os
+
+
+#%% instance-specific API settings
+# you likely need to modify these when deploying a new instance of the API
+
+API_INSTANCE_NAME = 'cm'  # 'internal', 'cm', 'camelot', 'zooniverse'
+POOL_ID = 'cm_1'  # name of the Batch pool created for this API instance
+
+MAX_NUMBER_IMAGES_ACCEPTED_PER_JOB = 4 * 1000 * 1000  # inclusive
+
+# Azure Batch for batch processing
+BATCH_ACCOUNT_NAME = 'cameratrapssc'
+BATCH_ACCOUNT_URL = 'https://cameratrapssc.southcentralus.batch.azure.com'
+
+
+#%% general API settings
+API_PREFIX = '/v4/camera-trap/detection-batch'  # URL to root is http://127.0.0.1:5000/v4/camera-trap/detection-batch/
+
+MONITOR_PERIOD_MINUTES = 10
+
+# if this number of times the thread wakes up to check is exceeded, stop the monitoring thread
+MAX_MONITOR_CYCLES = 4 * 7 * int((60 * 24) / MONITOR_PERIOD_MINUTES)  # 4 weeks
+
+IMAGE_SUFFIXES_ACCEPTED = ('.jpg', '.jpeg', '.png')  # case-insensitive
+assert isinstance(IMAGE_SUFFIXES_ACCEPTED, tuple)
+
+OUTPUT_FORMAT_VERSION = '1.1'
+
+NUM_IMAGES_PER_TASK = 2000
+
+OUTPUT_SAS_EXPIRATION_DAYS = 180
+
+# quota of active Jobs in our Batch account, which all node pools i.e. API instances share;
+# cannot accept job submissions if there are this many active Jobs already
+MAX_BATCH_ACCOUNT_ACTIVE_JOBS = 300
+
+
+#%% MegaDetector info
+DETECTION_CONF_THRESHOLD = 0.1
+
+# relative to the `megadetector_copies` folder in the container `models`
+# TODO add MD versions info to AppConfig
+MD_VERSIONS_TO_REL_PATH = {
+    '4.1': 'megadetector_v4_1/md_v4.1.0.pb',
+    '3': 'megadetector_v3/megadetector_v3_tf19.pb',
+    '2': 'megadetector_v2/frozen_inference_graph.pb'
+}
+DEFAULT_MD_VERSION = '4.1'
+assert DEFAULT_MD_VERSION in MD_VERSIONS_TO_REL_PATH
+
+# copied from TFDetector class in detection/run_detector.py
+DETECTOR_LABEL_MAP = {
+    '1': 'animal',
+    '2': 'person',
+    '3': 'vehicle'
+}
+
+
+#%% Azure Batch settings
+NUM_TASKS_PER_SUBMISSION = 20  # max for the Python SDK without extension is 100
+
+NUM_TASKS_PER_RESUBMISSION = 5
+
+
+#%% env variables for service credentials, and info related to these services
+
+# Cosmos DB `batch-api-jobs` table for job status
+COSMOS_ENDPOINT = os.environ['COSMOS_ENDPOINT']
+COSMOS_WRITE_KEY = os.environ['COSMOS_WRITE_KEY']
+
+# Service principal of this "application", authorized to use Azure Batch
+APP_TENANT_ID = os.environ['APP_TENANT_ID']
+APP_CLIENT_ID = os.environ['APP_CLIENT_ID']
+APP_CLIENT_SECRET = os.environ['APP_CLIENT_SECRET']
+
+# Blob storage account for storing Batch tasks' outputs and scoring script
+STORAGE_ACCOUNT_NAME = os.environ['STORAGE_ACCOUNT_NAME']
+STORAGE_ACCOUNT_KEY = os.environ['STORAGE_ACCOUNT_KEY']
+
+# STORAGE_CONTAINER_MODELS = 'models'  # names of the two containers supporting Batch
+STORAGE_CONTAINER_API = 'batch-api'
+
+# Azure Container Registry for Docker image used by our Batch node pools
+REGISTRY_SERVER = os.environ['REGISTRY_SERVER']
+REGISTRY_PASSWORD = os.environ['REGISTRY_PASSWORD']
+CONTAINER_IMAGE_NAME = 'cameratracrsppftkje.azurecr.io/tensorflow:1.14.0-gpu-py3'
+
+# Azure App Configuration instance to get configurations specific to
+# this instance of the API
+APP_CONFIG_CONNECTION_STR = os.environ['APP_CONFIG_CONNECTION_STR']

megadetector/api/batch_processing/api_core/server_app_config.py

@@ -0,0 +1,55 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+"""
+A class wrapping the Azure App Configuration client to get configurations
+for each instance of the API.
+"""
+import logging
+import os
+
+from server_api_config import APP_CONFIG_CONNECTION_STR, API_INSTANCE_NAME
+
+from azure.appconfiguration import AzureAppConfigurationClient
+
+
+log = logging.getLogger(os.environ['FLASK_APP'])
+
+
+class AppConfig:
+    """Wrapper around the Azure App Configuration client"""
+
+    def __init__(self):
+        self.client = AzureAppConfigurationClient.from_connection_string(APP_CONFIG_CONNECTION_STR)
+
+        self.api_instance = API_INSTANCE_NAME
+
+        # sentinel should change if new configurations are available
+        self.sentinel = self._get_sentinel()  # get initial sentinel and allowlist values
+        self.allowlist = self._get_allowlist()
+
+    def _get_sentinel(self):
+        return self.client.get_configuration_setting(key='batch_api:sentinel').value
+
+    def _get_allowlist(self):
+        filtered_listed = self.client.list_configuration_settings(key_filter='batch_api_allow:*')
+        allowlist = []
+        for item in filtered_listed:
+            if item.value == self.api_instance:
+                allowlist.append(item.key.split('batch_api_allow:')[1])
+        return allowlist
+
+    def get_allowlist(self):
+        try:
+            cur_sentinel = self._get_sentinel()
+            if cur_sentinel == self.sentinel:
+                # configs have not changed
+                return self.allowlist
+            else:
+                self.sentinel = cur_sentinel
+                self.allowlist = self._get_allowlist()
+                return self.allowlist
+
+        except Exception as e:
+            log.error(f'AppConfig, get_allowlist, exception so using old allowlist: {e}')
+            return self.allowlist

megadetector/api/batch_processing/api_core/server_batch_job_manager.py

@@ -0,0 +1,220 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+"""
+A class wrapping the Azure Batch client.
+"""
+
+import logging
+import os
+import math
+from typing import Tuple
+from datetime import datetime, timedelta
+
+import sas_blob_utils
+from azure.storage.blob import ContainerClient, ContainerSasPermissions, generate_container_sas
+from azure.batch import BatchServiceClient
+from azure.batch.models import *
+from azure.common.credentials import ServicePrincipalCredentials
+
+import server_api_config as api_config
+
+
+# Gunicorn logger handler will get attached if needed in server.py
+log = logging.getLogger(os.environ['FLASK_APP'])
+
+
+class BatchJobManager:
+    """Wrapper around the Azure App Configuration client"""
+
+    def __init__(self):
+        credentials = ServicePrincipalCredentials(
+            client_id=api_config.APP_CLIENT_ID,
+            secret=api_config.APP_CLIENT_SECRET,
+            tenant=api_config.APP_TENANT_ID,
+            resource='https://batch.core.windows.net/'
+        )
+        self.batch_client = BatchServiceClient(credentials=credentials,
+                                               batch_url=api_config.BATCH_ACCOUNT_URL)
+
+    def create_job(self, job_id: str, detector_model_rel_path: str,
+                   input_container_sas: str, use_url: bool):
+        log.info(f'BatchJobManager, create_job, job_id: {job_id}')
+        job = JobAddParameter(
+            id=job_id,
+            pool_info=PoolInformation(pool_id=api_config.POOL_ID),
+
+            # set for all tasks in the job
+            common_environment_settings=[
+                EnvironmentSetting(name='DETECTOR_REL_PATH', value=detector_model_rel_path),
+                EnvironmentSetting(name='API_INSTANCE_NAME', value=api_config.API_INSTANCE_NAME),
+                EnvironmentSetting(name='JOB_CONTAINER_SAS', value=input_container_sas),
+                EnvironmentSetting(name='JOB_USE_URL', value=str(use_url)),
+                EnvironmentSetting(name='DETECTION_CONF_THRESHOLD', value=api_config.DETECTION_CONF_THRESHOLD)
+            ]
+        )
+        self.batch_client.job.add(job)
+
+    def submit_tasks(self, job_id: str, num_images: int) -> Tuple[int, list]:
+        """
+        Shard the images and submit each shard as a Task under the Job pointed to by this job_id
+        Args:
+            job_id: ID of the Batch Job to submit the tasks to
+            num_images: total number of images to be processed in this Job
+
+        Returns:
+            num_task: total number of Tasks that should be in this Job
+            task_ids_failed_to_submit: which Tasks from the above failed to be submitted
+        """
+        log.info('BatchJobManager, submit_tasks')
+
+        # cannot execute the scoring script that is in the mounted directory; has to be copied to cwd
+        # not luck giving the commandline arguments via formatted string - set as env vars instead
+        score_command = '/bin/bash -c \"cp $AZ_BATCH_NODE_MOUNTS_DIR/batch-api/scripts/score.py . && python score.py\" '
+
+        num_images_per_task = api_config.NUM_IMAGES_PER_TASK
+
+        # form shards of images and assign each shard to a Task
+        num_tasks = math.ceil(num_images / num_images_per_task)
+
+        # for persisting stdout and stderr
+        permissions = ContainerSasPermissions(read=True, write=True, list=True)
+        access_duration_hrs = api_config.MONITOR_PERIOD_MINUTES * api_config.MAX_MONITOR_CYCLES / 60
+        container_sas_token = generate_container_sas(
+            account_name=api_config.STORAGE_ACCOUNT_NAME,
+            container_name=api_config.STORAGE_CONTAINER_API,
+            account_key=api_config.STORAGE_ACCOUNT_KEY,
+            permission=permissions,
+            expiry=datetime.utcnow() + timedelta(hours=access_duration_hrs))
+        container_sas_url = sas_blob_utils.build_azure_storage_uri(
+            account=api_config.STORAGE_ACCOUNT_NAME,
+            container=api_config.STORAGE_CONTAINER_API,
+            sas_token=container_sas_token)
+
+        tasks = []
+        for task_id in range(num_tasks):
+            begin_index = task_id * num_images_per_task
+            end_index = begin_index + num_images_per_task
+
+            # persist stdout and stderr (will be removed when node removed)
+            # paths are relative to the Task working directory
+            stderr_destination = OutputFileDestination(
+                container=OutputFileBlobContainerDestination(
+                    container_url=container_sas_url,
+                    path=f'api_{api_config.API_INSTANCE_NAME}/job_{job_id}/task_logs/job_{job_id}_task_{task_id}_stderr.txt'
+                )
+            )
+            stdout_destination = OutputFileDestination(
+                container=OutputFileBlobContainerDestination(
+                    container_url=container_sas_url,
+                    path=f'api_{api_config.API_INSTANCE_NAME}/job_{job_id}/task_logs/job_{job_id}_task_{task_id}_stdout.txt'
+                )
+            )
+            std_err_and_out = [
+                OutputFile(
+                    file_pattern='../stderr.txt',  # stderr.txt is at the same level as wd
+                    destination=stderr_destination,
+                    upload_options=OutputFileUploadOptions(upload_condition=OutputFileUploadCondition.task_completion)
+                    # can also just upload on failure
+                ),
+                OutputFile(
+                    file_pattern='../stdout.txt',
+                    destination=stdout_destination,
+                    upload_options=OutputFileUploadOptions(upload_condition=OutputFileUploadCondition.task_completion)
+                )
+            ]
+
+            task = TaskAddParameter(
+                id=str(task_id),
+                command_line=score_command,
+                container_settings=TaskContainerSettings(
+                    image_name=api_config.CONTAINER_IMAGE_NAME,
+                    working_directory='taskWorkingDirectory'
+                ),
+                environment_settings=[
+                    EnvironmentSetting(name='TASK_BEGIN_INDEX', value=begin_index),
+                    EnvironmentSetting(name='TASK_END_INDEX', value=end_index),
+                ],
+                output_files=std_err_and_out
+            )
+            tasks.append(task)
+
+        # first try submitting Tasks
+        task_ids_failed_to_submit = self._create_tasks(job_id, tasks, api_config.NUM_TASKS_PER_SUBMISSION, 1)
+
+        # retry submitting Tasks
+        if len(task_ids_failed_to_submit) > 0:
+            task_ids_failed_to_submit_set = set(task_ids_failed_to_submit)
+            tasks_to_retry = [t for t in tasks if t.id in task_ids_failed_to_submit_set]
+            task_ids_failed_to_submit = self._create_tasks(job_id,
+                                                           tasks_to_retry,
+                                                           api_config.NUM_TASKS_PER_RESUBMISSION,
+                                                           2)
+
+            if len(task_ids_failed_to_submit) > 0:
+                log.info('BatchJobManager, submit_tasks, after retry, '
+                      f'len of task_ids_failed_to_submit: {len(task_ids_failed_to_submit)}')
+            else:
+                log.info('BatchJobManager, submit_tasks, after retry, all Tasks submitted')
+        else:
+            log.info('BatchJobManager, submit_tasks, all Tasks submitted after first try')
+
+        # Change the Job's on_all_tasks_complete option to 'terminateJob' so the Job's status changes automatically
+        # after all submitted tasks are done
+        # This is so that we do not take up the quota for active Jobs in the Batch account.
+        job_patch_params = JobPatchParameter(
+            on_all_tasks_complete=OnAllTasksComplete.terminate_job
+        )
+        self.batch_client.job.patch(job_id, job_patch_params)
+
+        return num_tasks, task_ids_failed_to_submit
+
+    def _create_tasks(self, job_id, tasks, num_tasks_per_submission, n_th_try) -> list:
+        task_ids_failed_to_submit = []
+
+        for i in range(0, len(tasks), num_tasks_per_submission):
+            tasks_to_submit = tasks[i: i + num_tasks_per_submission]
+
+            # return type: TaskAddCollectionResult
+            collection_results = self.batch_client.task.add_collection(job_id, tasks_to_submit, threads=10)
+
+            for task_result in collection_results.value:
+                if task_result.status is not TaskAddStatus.success:
+                    # actually we should probably only re-submit if it's a server_error
+                    task_ids_failed_to_submit.append(task_result.task_id)
+                    log.info(f'task {task_result.task_id} failed to submitted after {n_th_try} try/tries, '
+                          f'status: {task_result.status}, error: {task_result.error}')
+
+        return task_ids_failed_to_submit
+
+    def get_num_completed_tasks(self, job_id: str) -> Tuple[int, int]:
+        """
+        Returns the number of completed tasks for the job of job_id, as a tuple:
+        (number of succeeded jobs, number of failed jobs) - both are considered "completed".=
+        """
+        # docs: # https://docs.microsoft.com/en-us/rest/api/batchservice/odata-filters-in-batch#list-tasks
+        tasks = self.batch_client.task.list(job_id,
+                                            task_list_options=TaskListOptions(
+                                                filter='state eq \'completed\'',
+                                                select='id, executionInfo'  # only the id field will be non-empty
+                                            ))
+        num_succeeded, num_failed = 0, 0
+        for task in tasks:
+            exit_code: int = task.execution_info.exit_code
+            if exit_code == 0:
+                num_succeeded += 1
+            else:
+                num_failed += 1
+        return num_succeeded, num_failed
+
+    def cancel_batch_job(self, job_id: str):
+        self.batch_client.job.terminate(job_id, terminate_reason='APIUserCanceled')
+
+    def get_num_active_jobs(self) -> int:
+        jobs_generator = self.batch_client.job.list(
+            job_list_options=JobListOptions(
+                filter='state eq \'active\'',
+                select='id'
+            ))
+        jobs_list = [j for j in jobs_generator]
+        return len(jobs_list)