megadetector 5.0.7__py3-none-any.whl → 5.0.9__py3-none-any.whl

api/synchronous/api_core/tests/load_test.py

@@ -1,110 +1,110 @@
-
-import os
-import json
-import io
-import random
-import requests
-
-from PIL import Image
-from multiprocessing import Pool
-from datetime import datetime
-from requests_toolbelt import MultipartEncoder
-from requests_toolbelt.multipart import decoder
-
-
-ip_address = '100.100.200.200'
-port = 5050
-
-base_url = 'http://{}:{}/v1/camera-trap/sync/'.format(ip_address, port)  
-
-
-def call_api(args):
-    start = datetime.now()
-    
-    index, url, params, data, headers = args['index'],args['url'], args['params'], args['data'], args['headers']
-    print('calling api: {} starttime: {}'.format(index, start))
-
-    response = requests.post(url, params=params, data=data, headers=headers)
-    elapsed_time = datetime.now() - start
-    print('\napi {} status code: {}, elapsed time in seconds {}'.format(index, response.status_code, elapsed_time.total_seconds()))
-    
-    get_detections(response)   
-    return response
-
-def get_detections(response):
-    results = decoder.MultipartDecoder.from_response(response)
-    text_results = {}
-    images = {}
-    for part in results.parts:
-        # part is a BodyPart object with b'Content-Type', and b'Content-Disposition', the later includes 'name' and 'filename' info
-        headers = {}
-        for k, v in part.headers.items():
-            headers[k.decode(part.encoding)] = v.decode(part.encoding)
-       
-        if headers.get('Content-Type', None) == 'application/json':
-            text_result = json.loads(part.content.decode())
-
-    print(text_result)
-
-
-def test_load(num_requests, params, max_images=1):
-    requests = []
-    
-    # read the images anew for each request
-    index = 0
-    for i in range(num_requests):
-        index += 1
-        files = {}
-        sample_input_dir = '../../../api/synchronous/sample_input/test_images'
-
-        image_files = os.listdir(sample_input_dir)
-        random.shuffle(image_files)
-
-        num_images = 0
-        for i, image_name in enumerate(image_files):
-            if not image_name.lower().endswith('.jpg'):
-                continue
-
-            if num_images >= max_images:
-                break
-            else:
-                num_images += 1
-
-            img_path = os.path.join(sample_input_dir, image_name)
-            with open(img_path, 'rb') as f:
-                content = f.read()
-            files[image_name] = (image_name, content, 'image/jpeg')
-
-        m = MultipartEncoder(fields=files)
-        args = {
-            'index': index,
-            'url': base_url + 'detect',
-            'params': params,
-            'data': m,
-            'headers': {'Content-Type': m.content_type}
-        }
-        requests.append(args)
-    
-    print('starting', num_requests, 'threads...')
-    # images are read and in each request by the time we call the API in map()
-    with Pool(num_requests) as pool:
-        results = pool.map(call_api, requests)
-
-    return results
-
-
-if __name__ == "__main__":
-    params = {
-    'min_confidence': 0.05,
-    'min_rendering_confidence': 0.2,
-    'render': True
-    }
-    
-    num_requests = 10
-    max_images = 1
-
-    start = datetime.now()
-    responses = test_load(num_requests, params, max_images=max_images)
-    end = datetime.now()
-    total_time = end - start
+
+import os
+import json
+import io
+import random
+import requests
+
+from PIL import Image
+from multiprocessing import Pool
+from datetime import datetime
+from requests_toolbelt import MultipartEncoder
+from requests_toolbelt.multipart import decoder
+
+
+ip_address = '100.100.200.200'
+port = 5050
+
+base_url = 'http://{}:{}/v1/camera-trap/sync/'.format(ip_address, port)  
+
+
+def call_api(args):
+    start = datetime.now()
+    
+    index, url, params, data, headers = args['index'],args['url'], args['params'], args['data'], args['headers']
+    print('calling api: {} starttime: {}'.format(index, start))
+
+    response = requests.post(url, params=params, data=data, headers=headers)
+    elapsed_time = datetime.now() - start
+    print('\napi {} status code: {}, elapsed time in seconds {}'.format(index, response.status_code, elapsed_time.total_seconds()))
+    
+    get_detections(response)   
+    return response
+
+def get_detections(response):
+    results = decoder.MultipartDecoder.from_response(response)
+    text_results = {}
+    images = {}
+    for part in results.parts:
+        # part is a BodyPart object with b'Content-Type', and b'Content-Disposition', the later includes 'name' and 'filename' info
+        headers = {}
+        for k, v in part.headers.items():
+            headers[k.decode(part.encoding)] = v.decode(part.encoding)
+       
+        if headers.get('Content-Type', None) == 'application/json':
+            text_result = json.loads(part.content.decode())
+
+    print(text_result)
+
+
+def test_load(num_requests, params, max_images=1):
+    requests = []
+    
+    # read the images anew for each request
+    index = 0
+    for i in range(num_requests):
+        index += 1
+        files = {}
+        sample_input_dir = '../../../api/synchronous/sample_input/test_images'
+
+        image_files = os.listdir(sample_input_dir)
+        random.shuffle(image_files)
+
+        num_images = 0
+        for i, image_name in enumerate(image_files):
+            if not image_name.lower().endswith('.jpg'):
+                continue
+
+            if num_images >= max_images:
+                break
+            else:
+                num_images += 1
+
+            img_path = os.path.join(sample_input_dir, image_name)
+            with open(img_path, 'rb') as f:
+                content = f.read()
+            files[image_name] = (image_name, content, 'image/jpeg')
+
+        m = MultipartEncoder(fields=files)
+        args = {
+            'index': index,
+            'url': base_url + 'detect',
+            'params': params,
+            'data': m,
+            'headers': {'Content-Type': m.content_type}
+        }
+        requests.append(args)
+    
+    print('starting', num_requests, 'threads...')
+    # images are read and in each request by the time we call the API in map()
+    with Pool(num_requests) as pool:
+        results = pool.map(call_api, requests)
+
+    return results
+
+
+if __name__ == "__main__":
+    params = {
+    'min_confidence': 0.05,
+    'min_rendering_confidence': 0.2,
+    'render': True
+    }
+    
+    num_requests = 10
+    max_images = 1
+
+    start = datetime.now()
+    responses = test_load(num_requests, params, max_images=max_images)
+    end = datetime.now()
+    total_time = end - start
     print('Total time for {} requests: {}'.format(num_requests, total_time))

classification/aggregate_classifier_probs.py

@@ -1,27 +1,16 @@
-########
-#
-# aggregate_classifier_probs.py
-#
-# Aggregate probabilities from a classifier's outputs according to a mapping
-# from the desired (target) categories to the classifier's categories.
-#
-# Using the mapping, create a new version of the classifier output CSV with
-# probabilities summed within each target category. Also output a new
-# "index-to-name" JSON file which identifies the sequential order of the target
-# categories.
-# 
-########
+"""
 
-#%%  Example usage
+aggregate_classifier_probs.py
 
-"""
-python aggregate_classifier_probs.py \
-    classifier_output.csv.gz \
-    --target-mapping target_to_classifier_labels.json \
-    --output-csv classifier_output_remapped.csv.gz \
-    --output-label-index label_index_remapped.json
-"""
+Aggregate probabilities from a classifier's outputs according to a mapping
+from the desired (target) categories to the classifier's categories.
 
+Using the mapping, create a new version of the classifier output CSV with
+probabilities summed within each target category. Also output a new
+"index-to-name" JSON file which identifies the sequential order of the target
+categories.
+
+"""
 
 #%% Imports
 
@@ -33,6 +22,15 @@ import json
 import pandas as pd
 from tqdm import tqdm
 
+#%%  Example usage
+
+"""
+python aggregate_classifier_probs.py \
+    classifier_output.csv.gz \
+    --target-mapping target_to_classifier_labels.json \
+    --output-csv classifier_output_remapped.csv.gz \
+    --output-label-index label_index_remapped.json
+"""
 
 #%% Main function
 
@@ -46,6 +44,7 @@ def main(classifier_results_csv_path: str,
     Because the output CSV is often very large, we process it in chunks of 1000
     rows at a time.
     """
+    
     chunked_df_iterator = pd.read_csv(
         classifier_results_csv_path, chunksize=1000, float_precision='high',
         index_col='path')
@@ -81,9 +80,7 @@ def main(classifier_results_csv_path: str,
 #%% Command-line driver
 
 def _parse_args() -> argparse.Namespace:
-    """
-    Parses arguments.
-    """
+    
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
         description='Aggregate classifier probabilities to target classes.')

classification/analyze_failed_images.py

@@ -1,20 +1,11 @@
-########
-#
-# analyze_failed_images.py
-#
-########
+"""
 
-#%% Example usage
+analyze_failed_images.py
 
-"""
-    python analyze_failed_images.py failed.json \
-        -a ACCOUNT -c CONTAINER -s SAS_TOKEN
 """
 
 #%% Imports and constants
 
-from __future__ import annotations
-
 import argparse
 from collections.abc import Mapping, Sequence
 from concurrent import futures
@@ -31,6 +22,14 @@ from data_management.megadb.megadb_utils import MegadbUtils
 from md_utils import path_utils
 from md_utils import sas_blob_utils
 
+
+#%% Example usage
+
+"""
+    python analyze_failed_images.py failed.json \
+        -a ACCOUNT -c CONTAINER -s SAS_TOKEN
+"""
+
 ImageFile.LOAD_TRUNCATED_IMAGES = False
 
 
@@ -191,8 +190,7 @@ def analyze_images(url_or_path: str, json_keys: Optional[Sequence[str]] = None,
 
 #%% Command-line driver
 
-def _parse_args() -> argparse.Namespace:
-    
+def _parse_args() -> argparse.Namespace:    
     
     parser = argparse.ArgumentParser(
         description='Analyze a list of images that failed to download or crop.')

classification/cache_batchapi_outputs.py

@@ -1,54 +1,54 @@
-########
-#
-# cache_batchapi_outputs.py
-#
-# Script to cache Batch Detection API outputs.
-# 
-# This script can handle either the Batch Detection API JSON Response or the
-# detections JSON.
-# 
-# Batch Detection API Response format:
-# 
-#     {
-#         "Status": {
-#             "request_status": "completed",
-#             "message": {
-#                 "num_failed_shards": 0,
-#                 "output_file_urls": {
-#                     "detections": "https://url/to/detections.json",
-#                     "failed_images": "https://url/to/failed_images.json",
-#                     "images": https://url/to/images.json",
-#                 }
-#             },
-#         },
-#         "Endpoint": "/v3/camera-trap/detection-batch/request_detections",
-#         "TaskId": "ea26326e-7e0d-4524-a9ea-f57a5799d4ba"
-#     }
-# 
-# Detections JSON format:
-#
-#     {
-#         "info": {...}
-#         "detection_categories": {...}
-#         "classification_categories": {...}
-#         "images": [
-#             {
-#                 "file": "path/from/base/dir/image1.jpg",
-#                 "max_detection_conf": 0.926,
-#                 "detections": [{
-#                         "category": "1",
-#                         "conf": 0.061,
-#                         "bbox": [0.0451, 0.1849, 0.3642, 0.4636]
-#                 }]
-#             }
-#         ]
-#     }
-# 
-# Batch Detection API Output Format:
-#
-# github.com/agentmorris/MegaDetector/tree/master/api/batch_processing#api-outputs
-#
-########
+"""
+
+cache_batchapi_outputs.py
+
+Script to cache Batch Detection API outputs.
+
+This script can handle either the Batch Detection API JSON Response or the
+detections JSON.
+
+Batch Detection API Response format:
+
+    {
+        "Status": {
+            "request_status": "completed",
+            "message": {
+                "num_failed_shards": 0,
+                "output_file_urls": {
+                    "detections": "https://url/to/detections.json",
+                    "failed_images": "https://url/to/failed_images.json",
+                    "images": https://url/to/images.json",
+                }
+            },
+        },
+        "Endpoint": "/v3/camera-trap/detection-batch/request_detections",
+        "TaskId": "ea26326e-7e0d-4524-a9ea-f57a5799d4ba"
+    }
+
+Detections JSON format:
+
+    {
+        "info": {...}
+        "detection_categories": {...}
+        "classification_categories": {...}
+        "images": [
+            {
+                "file": "path/from/base/dir/image1.jpg",
+                "max_detection_conf": 0.926,
+                "detections": [{
+                        "category": "1",
+                        "conf": 0.061,
+                        "bbox": [0.0451, 0.1849, 0.3642, 0.4636]
+                }]
+            }
+        ]
+    }
+
+Batch Detection API Output Format:
+
+github.com/agentmorris/MegaDetector/tree/master/api/batch_processing#api-outputs
+
+"""
 
 #%% Imports
 

classification/create_classification_dataset.py

@@ -1,73 +1,63 @@
-########
-#
-# create_classification_dataset.py
-# 
-# Creates a classification dataset CSV with a corresponding JSON file determining
-# the train/val/test split.
-# 
-# This script takes as input a "queried images" JSON file whose keys are paths to
-# images and values are dictionaries containing information relevant for training
-# a classifier, including labels and (optionally) ground-truth bounding boxes.
-# The image paths are in the format `<dataset-name>/<blob-name>` where we assume
-# that the dataset name does not contain '/'.
-# 
-# {
-#     "caltech/cct_images/59f79901-23d2-11e8-a6a3-ec086b02610b.jpg": {
-#         "dataset": "caltech",
-#         "location": 13,
-#         "class": "mountain_lion",  # class from dataset
-#         "bbox": [{"category": "animal",
-#                   "bbox": [0, 0.347, 0.237, 0.257]}],   # ground-truth bbox
-#         "label": ["monutain_lion"]  # labels to use in classifier
-#     },
-#     "caltech/cct_images/59f5fe2b-23d2-11e8-a6a3-ec086b02610b.jpg": {
-#         "dataset": "caltech",
-#         "location": 13,
-#         "class": "mountain_lion",  # class from dataset
-#         "label": ["monutain_lion"]  # labels to use in classifier
-#     },
-#     ...
-# }
-# 
-# We assume that the tuple (dataset, location) identifies a unique location. In
-# other words, we assume that no two datasets have overlapping locations. This
-# probably isn't 100% true, but it's pretty much the best we can do in terms of
-# avoiding overlapping locations between the train/val/test splits.
-# 
-# This script outputs 3 files to <output_dir>:
-# 
-# 1) classification_ds.csv, contains columns:
-#    
-#     - 'path': str, path to cropped images
-#     - 'dataset': str, name of dataset
-#     - 'location': str, location that image was taken, as saved in MegaDB
-#     - 'dataset_class': str, original class assigned to image, as saved in MegaDB
-#     - 'confidence': float, confidence that this crop is of an actual animal,
-#         1.0 if the crop is a "ground truth bounding box" (i.e., from MegaDB),
-#         <= 1.0 if the bounding box was detected by MegaDetector
-#     - 'label': str, comma-separated list of label(s) assigned to this crop for
-#         the sake of classification
-# 
-# 2) label_index.json: maps integer to label name
-#
-#     - keys are string representations of Python integers (JSON requires keys to
-#       be strings), numbered from 0 to num_labels-1
-#     - values are strings, label names
-# 
-# 3) splits.json: serialization of a Python dict that maps each split
-#    ['train', 'val', 'test'] to a list of length-2 lists, where each inner list
-#    is [<dataset>, <location>]
-#
-########
+"""
 
-#%% Example usage
+create_classification_dataset.py
+
+Creates a classification dataset CSV with a corresponding JSON file determining
+the train/val/test split.
+
+This script takes as input a "queried images" JSON file whose keys are paths to
+images and values are dictionaries containing information relevant for training
+a classifier, including labels and (optionally) ground-truth bounding boxes.
+The image paths are in the format `<dataset-name>/<blob-name>` where we assume
+that the dataset name does not contain '/'.
+
+{
+    "caltech/cct_images/59f79901-23d2-11e8-a6a3-ec086b02610b.jpg": {
+        "dataset": "caltech",
+        "location": 13,
+        "class": "mountain_lion",  # class from dataset
+        "bbox": [{"category": "animal",
+                  "bbox": [0, 0.347, 0.237, 0.257]}],   # ground-truth bbox
+        "label": ["monutain_lion"]  # labels to use in classifier
+    },
+    "caltech/cct_images/59f5fe2b-23d2-11e8-a6a3-ec086b02610b.jpg": {
+        "dataset": "caltech",
+        "location": 13,
+        "class": "mountain_lion",  # class from dataset
+        "label": ["monutain_lion"]  # labels to use in classifier
+    },
+    ...
+}
+
+We assume that the tuple (dataset, location) identifies a unique location. In
+other words, we assume that no two datasets have overlapping locations. This
+probably isn't 100% true, but it's pretty much the best we can do in terms of
+avoiding overlapping locations between the train/val/test splits.
+
+This script outputs 3 files to <output_dir>:
+
+1) classification_ds.csv, contains columns:
+   
+    - 'path': str, path to cropped images
+    - 'dataset': str, name of dataset
+    - 'location': str, location that image was taken, as saved in MegaDB
+    - 'dataset_class': str, original class assigned to image, as saved in MegaDB
+    - 'confidence': float, confidence that this crop is of an actual animal,
+        1.0 if the crop is a "ground truth bounding box" (i.e., from MegaDB),
+        <= 1.0 if the bounding box was detected by MegaDetector
+    - 'label': str, comma-separated list of label(s) assigned to this crop for
+        the sake of classification
+
+2) label_index.json: maps integer to label name
+
+    - keys are string representations of Python integers (JSON requires keys to
+      be strings), numbered from 0 to num_labels-1
+    - values are strings, label names
+
+3) splits.json: serialization of a Python dict that maps each split
+   ['train', 'val', 'test'] to a list of length-2 lists, where each inner list
+   is [<dataset>, <location>]
 
-"""
-    python create_classification_dataset.py \
-        run_idfg2 \
-        --queried-images-json run_idfg2/queried_images.json \
-        --cropped-images-dir /ssd/crops_sq \
-        -d $HOME/classifier-training/mdcache -v "4.1" -t 0.8
 """
 
 #%% Imports and constants
@@ -87,6 +77,17 @@ from tqdm import tqdm
 from classification import detect_and_crop
 
 
+#%% Example usage
+
+"""
+    python create_classification_dataset.py \
+        run_idfg2 \
+        --queried-images-json run_idfg2/queried_images.json \
+        --cropped-images-dir /ssd/crops_sq \
+        -d $HOME/classifier-training/mdcache -v "4.1" -t 0.8
+"""
+
+
 DATASET_FILENAME = 'classification_ds.csv'
 LABEL_INDEX_FILENAME = 'label_index.json'
 SPLITS_FILENAME = 'splits.json'