megadetector 5.0.28__py3-none-any.whl → 5.0.29__py3-none-any.whl

megadetector/api/batch_processing/api_core/batch_service/score.py

@@ -1,6 +1,7 @@
 import io
 import json
 import math
+from megadetector.utils import ct_utils
 import os
 import sys
 from datetime import datetime
@@ -353,7 +354,7 @@ class BatchScorer:
         return detections
 
 
-def main():
+def main(): # noqa
     print('score.py, main()')
 
     # information to determine input and output locations
@@ -387,8 +388,7 @@ def main():
     task_output_path = os.path.join(task_out_dir, f'job_{job_id}_task_{task_id}.json')
 
     # test that we can write to output path; also in case there is no image to process
-    with open(task_output_path, 'w') as f:
-        json.dump([], f)
+    ct_utils.write_json(task_output_path, [])
 
     # list images to process
     list_images_path = os.path.join(job_folder_mounted, f'{job_id}_images.json')
@@ -432,8 +432,7 @@ def main():
     except Exception as e:
         raise RuntimeError(f'score.py, main(), exception in score_images(): {e}')
 
-    with open(task_output_path, 'w', encoding='utf-8') as f:
-        json.dump(detections, f, ensure_ascii=False)
+    ct_utils.write_json(task_output_path, detections, ensure_ascii=False)
 
 if __name__ == '__main__':
     main()

megadetector/api/batch_processing/api_core_support/aggregate_results_manually.py

@@ -22,7 +22,7 @@ import json
 from api.batch_processing.api_core.orchestrator_api.orchestrator import AMLMonitor
 
 
-def main():
+def main(): # noqa
     parser = argparse.ArgumentParser()
     parser.add_argument('shortened_request_id', type=str,
                         help='the request ID to restart monitoring')

megadetector/api/batch_processing/api_support/summarize_daily_activity.py

@@ -124,7 +124,7 @@ def send_message():
     print('')
 
 
-def main():
+def main(): # noqa
     """
     Wake up at 5 minutes past midnight UTC to send a summary of yesterday's activities if there were any.
     Then goes in a loop to wake up and send a summary every 24 hours.

megadetector/api/batch_processing/integration/digiKam/xmp_integration.py

@@ -17,7 +17,7 @@ import inspect
 import os
 import sys
 import json
-import pyexiv2
+import pyexiv2 # type: ignore
 import ntpath
 import threading
 import traceback
@@ -432,7 +432,7 @@ def args_to_object(args,obj):
             setattr(obj, n, v)
 
 
-def main():
+def main(): # noqa
     
     parser = argparse.ArgumentParser()
     parser.add_argument('--input_file', help = 'Path to the MegaDetector .json file', default=None)

megadetector/api/batch_processing/integration/eMammal/test_scripts/push_annotations_to_emammal.py

@@ -51,7 +51,7 @@ def format_data_print_deployments(rows):
 
 #%% Command-line driver
 
-def main():
+def main(): # noqa
     
     parser = argparse.ArgumentParser()
     parser.add_argument('input_file', type=str, help='Input .json filename')

megadetector/api/batch_processing/integration/eMammal/test_scripts/select_images_for_testing.py

@@ -10,7 +10,7 @@ from tqdm import tqdm
 
 #%% Main function
 
-def main():
+def main(): # noqa
     
     parser = argparse.ArgumentParser()
     parser.add_argument('input_file', type=str, help='Input .json filename')

megadetector/api/synchronous/api_core/tests/load_test.py

@@ -1,4 +1,3 @@
-
 import os
 import json
 import io
@@ -47,7 +46,7 @@ def get_detections(response):
     print(text_result)
 
 
-def test_load(num_requests, params, max_images=1):
+def load_test(num_requests, params, max_images=1):
     requests = []
     
     # read the images anew for each request
@@ -104,7 +103,7 @@ if __name__ == "__main__":
     max_images = 1
 
     start = datetime.now()
-    responses = test_load(num_requests, params, max_images=max_images)
+    responses = load_test(num_requests, params, max_images=max_images)
     end = datetime.now()
     total_time = end - start
     print('Total time for {} requests: {}'.format(num_requests, total_time))

megadetector/classification/aggregate_classifier_probs.py

@@ -44,7 +44,7 @@ def main(classifier_results_csv_path: str,
     Because the output CSV is often very large, we process it in chunks of 1000
     rows at a time.
     """
-    
+
     chunked_df_iterator = pd.read_csv(
         classifier_results_csv_path, chunksize=1000, float_precision='high',
         index_col='path')
@@ -80,7 +80,7 @@ def main(classifier_results_csv_path: str,
 #%% Command-line driver
 
 def _parse_args() -> argparse.Namespace:
-    
+
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
         description='Aggregate classifier probabilities to target classes.')
@@ -100,7 +100,7 @@ def _parse_args() -> argparse.Namespace:
 
 
 if __name__ == '__main__':
-    
+
     args = _parse_args()
     main(classifier_results_csv_path=args.classifier_results_csv,
          target_mapping_json_path=args.target_mapping,

megadetector/classification/analyze_failed_images.py

@@ -62,7 +62,7 @@ def check_image_condition(img_path: str,
         'bad': image exists, but cannot be opened even when setting
             ImageFile.LOAD_TRUNCATED_IMAGES=True
     """
-    
+
     if (account is None) or (container is None) or (datasets_table is not None):
         assert account is None
         assert container is None
@@ -133,7 +133,7 @@ def analyze_images(url_or_path: str, json_keys: Optional[Sequence[str]] = None,
         sas_token: str, optional SAS token (without leading '?') if the
             container is not publicly accessible
     """
-    
+
     datasets_table = None
     if (account is None) or (container is None):
         assert account is None
@@ -190,8 +190,8 @@ def analyze_images(url_or_path: str, json_keys: Optional[Sequence[str]] = None,
 
 #%% Command-line driver
 
-def _parse_args() -> argparse.Namespace:    
-    
+def _parse_args() -> argparse.Namespace:
+
     parser = argparse.ArgumentParser(
         description='Analyze a list of images that failed to download or crop.')
     parser.add_argument(
@@ -220,7 +220,7 @@ def _parse_args() -> argparse.Namespace:
 
 
 if __name__ == '__main__':
-    
+
     args = _parse_args()
     analyze_images(url_or_path=args.failed_images, json_keys=args.json_keys,
                    account=args.account, container=args.container,

megadetector/classification/cache_batchapi_outputs.py

@@ -66,6 +66,7 @@ from api.batch_processing.data_preparation.prepare_api_submission import (
     TaskStatus, Task)
 from api.batch_processing.postprocessing.combine_api_outputs import (
     combine_api_output_dictionaries)
+from megadetector.utils import ct_utils
 
 
 #%% Support functions
@@ -84,7 +85,7 @@ def cache_json(json_path: str,
         detector_output_cache_base_dir: str
         detector_version: str
     """
-    
+
     with open(json_path, 'r') as f:
         js = json.load(f)
 
@@ -138,7 +139,7 @@ def cache_detections(detections: Mapping[str, Any], dataset: str,
 
     Returns: str, message
     """
-    
+
     # combine detections with cache
     dataset_cache_path = os.path.join(
         detector_output_cache_dir, f'{dataset}.json')
@@ -155,8 +156,7 @@ def cache_detections(detections: Mapping[str, Any], dataset: str,
                f'{dataset_cache_path}')
 
     # write combined detections back out to cache
-    with open(dataset_cache_path, 'w') as f:
-        json.dump(merged_dataset_cache, f, indent=1)
+    ct_utils.write_json(dataset_cache_path, merged_dataset_cache)
     return msg
 
 
@@ -188,7 +188,7 @@ def _parse_args() -> argparse.Namespace:
 
 
 if __name__ == '__main__':
-    
+
     args = _parse_args()
     cache_json(
         json_path=args.json_file,

megadetector/classification/create_classification_dataset.py

@@ -37,7 +37,7 @@ avoiding overlapping locations between the train/val/test splits.
 This script outputs 3 files to <output_dir>:
 
 1) classification_ds.csv, contains columns:
-   
+
     - 'path': str, path to cropped images
     - 'dataset': str, name of dataset
     - 'location': str, location that image was taken, as saved in MegaDB
@@ -75,6 +75,7 @@ import pandas as pd
 from tqdm import tqdm
 
 from megadetector.classification import detect_and_crop
+from megadetector.utils import ct_utils
 
 
 #%% Example usage
@@ -108,7 +109,7 @@ def main(output_dir: str,
          test_frac: Optional[float],
          splits_method: Optional[str],
          label_spec_json_path: Optional[str]) -> None:
-    
+
     # input validation
     assert set(mode) <= {'csv', 'splits'}
     if label_spec_json_path is not None:
@@ -160,9 +161,8 @@ def main(output_dir: str,
             labels = labels.map(lambda x: x.split(',')).explode()
             # look into sklearn.preprocessing.MultiLabelBinarizer
         label_names = sorted(labels.unique())
-        with open(os.path.join(output_dir, LABEL_INDEX_FILENAME), 'w') as f:
-            # Note: JSON always saves keys as strings!
-            json.dump(dict(enumerate(label_names)), f, indent=1)
+        # Note: JSON always saves keys as strings!
+        ct_utils.write_json(os.path.join(output_dir, LABEL_INDEX_FILENAME), dict(enumerate(label_names)))
 
     if 'splits' in mode:
         assert splits_method is not None
@@ -181,8 +181,7 @@ def main(output_dir: str,
             split_to_locs = create_splits_smallest_label_first(
                 df, val_frac, test_frac, test_split=test_set_locs,
                 label_spec_json_path=label_spec_json_path)
-        with open(os.path.join(output_dir, SPLITS_FILENAME), 'w') as f:
-            json.dump(split_to_locs, f, indent=1)
+        ct_utils.write_json(os.path.join(output_dir, SPLITS_FILENAME), split_to_locs)
 
 
 #%% Support functions
@@ -236,7 +235,7 @@ def create_classification_csv(
             'missing crops': list of tuple (img_path, i), where i is the
                 i-th crop index
     """
-    
+
     assert 0 <= confidence_threshold <= 1
 
     columns = [
@@ -359,7 +358,7 @@ def create_splits_random(df: pd.DataFrame, val_frac: float,
     Returns: dict, keys are ['train', 'val', 'test'], values are lists of locs,
         where each loc is a tuple (dataset, location)
     """
-    
+
     if test_split is not None:
         assert test_frac == 0
     train_frac = 1. - val_frac - test_frac
@@ -445,7 +444,7 @@ def create_splits_smallest_label_first(
     Returns: dict, keys are ['train', 'val', 'test'], values are lists of locs,
         where each loc is a tuple (dataset, location)
     """
-    
+
     # label => list of datasets to prioritize for test and validation sets
     prioritize = {}
     if label_spec_json_path is not None:
@@ -525,7 +524,7 @@ def sort_locs_by_size(loc_to_size: MutableMapping[tuple[str, str], int],
     Returns: list of (dataset, location) tuples, ordered from smallest size to
         largest. Locations from prioritized datasets come first.
     """
-    
+
     result = []
     if prioritize is not None:
         # modify loc_to_size in place, so copy its keys before iterating
@@ -610,7 +609,7 @@ def _parse_args() -> argparse.Namespace:
 
 
 if __name__ == '__main__':
-    
+
     args = _parse_args()
     main(output_dir=args.output_dir,
          mode=args.mode,

megadetector/classification/crop_detections.py

@@ -33,7 +33,7 @@ bounding box width or height. In the case that the square crop boundaries exceed
 the original image size, the crop is padded with 0s.
 
 This script outputs a log file to:
-   
+
    <output_dir>/crop_detections_log_{timestamp}.json
 
 ...which contains images that failed to download and crop properly.
@@ -107,7 +107,7 @@ def main(detections_json_path: str,
         threads: int, number of threads to use for downloading images
         logdir: str, path to directory to save log file
     """
-    
+
     # error checking
     assert 0 <= confidence_threshold <= 1, \
             'Invalid confidence threshold {}'.format(confidence_threshold)
@@ -149,7 +149,7 @@ def main(detections_json_path: str,
         for d in info_dict['detections']:
             if d['category'] not in detection_categories:
                 print('Warning: ignoring detection with category {} for image {}'.format(
-                    d['category'],img_path))                
+                    d['category'],img_path))
                 # This will be removed later when we filter for animals
                 d['category'] = 'unsupported'
             else:
@@ -235,7 +235,7 @@ def download_and_crop(
         total_downloads: int, number of images downloaded
         total_new_crops: int, number of new crops saved to cropped_images_dir
     """
-    
+
     # True for ground truth, False for MegaDetector
     # always save as .jpg for consistency
     crop_path_template = {
@@ -297,7 +297,7 @@ def load_local_image(img_path: str |  BinaryIO) -> Optional[Image.Image]:
     """
     Attempts to load an image from a local path.
     """
-    
+
     try:
         with Image.open(img_path) as img:
             img.load()
@@ -347,7 +347,7 @@ def load_and_crop(img_path: str,
         did_download: bool, whether image was downloaded from Azure Blob Storage
         num_new_crops: int, number of new crops successfully saved
     """
-    
+
     did_download = False
     num_new_crops = 0
 
@@ -393,7 +393,7 @@ def load_and_crop(img_path: str,
 
     assert img is not None, 'image "{}" failed to load or download properly'.format(
         debug_path)
-    
+
     if img.mode != 'RGB':
         img = img.convert(mode='RGB')  # always save as RGB for consistency
 
@@ -418,7 +418,7 @@ def save_crop(img: Image.Image, bbox_norm: Sequence[float], square_crop: bool,
 
     Returns: bool, True if a crop was saved, False otherwise
     """
-    
+
     img_w, img_h = img.size
     xmin = int(bbox_norm[0] * img_w)
     ymin = int(bbox_norm[1] * img_h)
@@ -456,7 +456,7 @@ def save_crop(img: Image.Image, bbox_norm: Sequence[float], square_crop: bool,
 #%% Command-line driver
 
 def _parse_args() -> argparse.Namespace:
-    
+
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
         description='Crop detections from MegaDetector.')
@@ -501,7 +501,7 @@ def _parse_args() -> argparse.Namespace:
 
 
 if __name__ == '__main__':
-    
+
     args = _parse_args()
     main(detections_json_path=args.detections_json,
          cropped_images_dir=args.cropped_images_dir,

megadetector/classification/csv_to_json.py

@@ -40,7 +40,7 @@ Example CSV input:
 
 Example JSON output:
 
-"    
+"
   {
       "cervid": {
           "dataset_labels": {
@@ -107,17 +107,17 @@ import json
 from typing import Any
 
 import pandas as pd
+from megadetector.utils import ct_utils
 
 
 #%% Main function
 
-def main():
+def main(): # noqa
     args = _parse_args()
     js = csv_to_jsondict(args.input_csv_file)
     for label in js:
         js[label] = order_spec_dict(js[label])
-    with open(args.output_json_path, 'w') as f:
-        json.dump(js, f, indent=args.json_indent)
+    ct_utils.write_json(args.output_json_path, js, indent=args.json_indent)
 
 
 #%% Support functions
@@ -126,7 +126,7 @@ def parse_csv_row(obj: dict[str, Any], rowtype: str, content: str) -> None:
     """
     Parses a row in the CSV.
     """
-    
+
     if rowtype == 'row':
         if 'dataset_labels' not in obj:
             obj['dataset_labels'] = defaultdict(list)
@@ -169,7 +169,7 @@ def csv_to_jsondict(csv_path: str) -> dict[str, dict[str, Any]]:
     """
     Converts CSV to json-style dictionary.
     """
-    
+
     df = pd.read_csv(csv_path, comment='#', skip_blank_lines=True)
     assert (df.columns == ['output_label', 'type', 'content']).all()
 
@@ -193,7 +193,7 @@ def order_spec_dict(spec_dict: dict[str, Any]) -> dict[str, Any]:
     """
     Returns spec_dict with keys in a specific order.
     """
-    
+
     if 'exclude' in spec_dict:
         spec_dict['exclude'] = order_spec_dict(spec_dict['exclude'])
     ordered_spec_dict: dict[str, Any] = {}
@@ -206,7 +206,7 @@ def order_spec_dict(spec_dict: dict[str, Any]) -> dict[str, Any]:
 #%% Command-line driver
 
 def _parse_args() -> argparse.Namespace:
-    
+
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
         description='Converts CSV to JSON format for label specification.')

megadetector/classification/detect_and_crop.py

@@ -118,6 +118,7 @@ from megadetector.classification.crop_detections import load_and_crop
 from megadetector.data_management.megadb import megadb_utils
 from megadetector.utils import path_utils
 from megadetector.utils import sas_blob_utils
+from megadetector.utils import ct_utils
 
 
 #%% Example usage
@@ -264,8 +265,7 @@ def main(queried_images_json_path: str,
     # save log of bad images
     date = datetime.now().strftime('%Y%m%d_%H%M%S')  # e.g., '20200722_110816'
     log_path = os.path.join(output_dir, f'detect_and_crop_log_{date}.json')
-    with open(log_path, 'w') as f:
-        json.dump(log, f, indent=1)
+    ct_utils.write_json(log_path, log)
 
 
 #%% Support functions
@@ -291,7 +291,7 @@ def load_detection_cache(detector_output_cache_dir: str,
             if no cached detections were found for the given dataset ds.
         detection_categories: dict, maps str category ID to str category name
     """
-    
+
     # cache of Detector outputs: dataset name => {img_path => detection_dict}
     detection_cache = {}
     detection_categories: dict[str, str] = {}
@@ -339,7 +339,7 @@ def filter_detected_images(
         detection_categories: dict, maps str category ID to str category name,
             empty dict if no cached detections are found
     """
-    
+
     datasets = set(img_path[:img_path.find('/')]
                    for img_path in potential_images_to_detect)
     detection_cache, detection_categories = load_detection_cache(
@@ -364,7 +364,7 @@ def split_images_list_by_dataset(images_to_detect: Iterable[str]
 
     Returns: dict, maps dataset name to a list of image paths
     """
-    
+
     images_by_dataset: dict[str, list[str]] = {}
     for img_path in images_to_detect:
         dataset = img_path[:img_path.find('/')]
@@ -402,7 +402,7 @@ def submit_batch_detection_api(images_to_detect: Iterable[str],
 
     Returns: dict, maps str dataset name to list of Task objects
     """
-    
+
     filtered_images_to_detect = [
         x for x in images_to_detect if path_utils.is_image_file(x)]
     not_images = set(images_to_detect) - set(filtered_images_to_detect)
@@ -450,8 +450,7 @@ def submit_batch_detection_api(images_to_detect: Iterable[str],
         for dataset in tasks_by_dataset
         for task in tasks_by_dataset[dataset]
     ]
-    with open(resume_file_path, 'w') as f:
-        json.dump(resume_json, f, indent=1)
+    ct_utils.write_json(resume_file_path, resume_json)
     return tasks_by_dataset
 
 
@@ -478,7 +477,7 @@ def submit_batch_detection_api_by_dataset(
 
     Returns: list of Task objects
     """
-    
+
     os.makedirs(task_lists_dir, exist_ok=True)
 
     date = datetime.now().strftime('%Y%m%d_%H%M%S')  # e.g., '20200722_110816'
@@ -522,7 +521,7 @@ def resume_tasks(resume_file_path: str, batch_detection_api_url: str
 
     Returns: dict, maps str dataset name to list of Task objects
     """
-    
+
     with open(resume_file_path, 'r') as f:
         resume_json = json.load(f)
 
@@ -559,7 +558,7 @@ def wait_for_tasks(tasks_by_dataset: Mapping[str, Iterable[Task]],
             saved to <output_dir>/batchapi_response/{task_id}.json
         poll_interval: int, # of seconds between pinging the task status API
     """
-    
+
     remaining_tasks: list[tuple[str, Task]] = [
         (dataset, task) for dataset, tasks in tasks_by_dataset.items()
         for task in tasks]
@@ -597,8 +596,7 @@ def wait_for_tasks(tasks_by_dataset: Mapping[str, Iterable[Task]],
                 if not os.path.exists(save_dir):
                     tqdm.write(f'Creating API output dir: {save_dir}')
                     os.makedirs(save_dir)
-                with open(os.path.join(save_dir, f'{task.id}.json'), 'w') as f:
-                    json.dump(task.response, f, indent=1)
+                ct_utils.write_json(os.path.join(save_dir, f'{task.id}.json'), task.response)
             message = task.response['Status']['message']
             num_failed_shards = message['num_failed_shards']
             if num_failed_shards != 0:
@@ -676,7 +674,7 @@ def download_and_crop(
     Returns: list of str, images with bounding boxes that failed to download or
         crop properly
     """
-    
+
     # error checking before we download and crop any images
     valid_img_paths = set(queried_images_json.keys())
     if images_missing_detections is not None:
@@ -838,7 +836,7 @@ def _parse_args() -> argparse.Namespace:
 
 
 if __name__ == '__main__':
-    
+
     args = _parse_args()
     main(queried_images_json_path=args.queried_images_json,
          output_dir=args.output_dir,

megadetector/classification/evaluate_model.py

@@ -52,6 +52,7 @@ import torchvision
 import tqdm
 
 from megadetector.classification import efficientnet, train_classifier
+from megadetector.utils import ct_utils
 
 
 #%% Example usage
@@ -75,7 +76,7 @@ def check_override(params: Mapping[str, Any], key: str,
     """
     Return desired value, with optional override.
     """
-    
+
     if override is None:
         return params[key]
     saved = params.get(key, None)
@@ -102,7 +103,7 @@ def trace_model(model_name: str, ckpt_path: str, num_classes: int,
         '/path/to/ckpt_16.pt', then the returned path is
         '/path/to/ckpt_16_compiled.pt'.
     """
-    
+
     root, ext = os.path.splitext(ckpt_path)
     compiled_path = root + '_compiled' + ext
     if os.path.exists(compiled_path):
@@ -135,7 +136,7 @@ def calc_per_label_stats(cm: np.ndarray, label_names: Sequence[str]
         recall values are in [0, 1], or np.nan if that label had 0 ground-truth
             observations
     """
-    
+
     tp = np.diag(cm)  # true positives
 
     predicted_positives = cm.sum(axis=0, dtype=np.float64)  # tp + fp
@@ -186,7 +187,7 @@ def test_epoch(model: torch.nn.Module,
         cm: np.ndarray, confusion matrix C such that C[i,j] is the # of
             observations known to be in group i and predicted to be in group j
     """
-    
+
     # set dropout and BN layers to eval mode
     model.eval()
 
@@ -395,8 +396,7 @@ def main(params_json_path: str, ckpt_path: str, output_dir: str,
             assert target_names == set(label_names) | {'other'}
             label_names.append('other')
 
-            with open(os.path.join(output_dir, 'label_index.json'), 'w') as f:
-                json.dump(dict(enumerate(label_names)), f)
+            ct_utils.write_json(os.path.join(output_dir, 'label_index.json'), dict(enumerate(label_names)), indent=None)
 
         with open(label_index_json_path, 'r') as f:
             idx_to_label = json.load(f)
@@ -510,7 +510,7 @@ def _parse_args() -> argparse.Namespace:
 
 
 if __name__ == '__main__':
-    
+
     args = _parse_args()
     main(params_json_path=args.params_json, ckpt_path=args.ckpt_path,
          output_dir=args.output_dir, splits=args.splits,