megadetector 5.0.27__py3-none-any.whl → 5.0.29__py3-none-any.whl

megadetector/classification/identify_mislabeled_candidates.py

@@ -5,7 +5,7 @@ identify_mislabeled_candidates.py
 Identify images that may have been mislabeled.
 
 A "mislabeled candidate" is defined as an image meeting both criteria:
-    
+
 * according to the ground-truth label, the model made an incorrect prediction
 
 * the model's prediction confidence exceeds its confidence for the ground-truth
@@ -13,12 +13,12 @@ A "mislabeled candidate" is defined as an image meeting both criteria:
 
 This script outputs for each dataset a text file containing the filenames of
 mislabeled candidates, one per line. The text files are saved to:
-   
+
     <logdir>/mislabeled_candidates_{split}_{dataset}.txt
 
 To this list of files can then be passed to AzCopy to be downloaded:
 
-""  
+""
 azcopy cp "http://<url_of_container>?<sas_token>" "/save/files/here" \
        --list-of-files "/path/to/mislabeled_candidates_{split}_{dataset}.txt"
 ""
@@ -68,7 +68,7 @@ from tqdm import tqdm
 
 def main(logdir: str, splits: Iterable[str], margin: float,
          include_dataset_in_filename: bool) -> None:
-    
+
     # load files
     logdir = os.path.normpath(logdir)  # removes any trailing slash
     base_logdir = os.path.dirname(logdir)
@@ -112,7 +112,7 @@ def get_candidates_df(outputs_csv_path: str, label_names: Sequence[str],
     Returns a DataFrame containing crops only from mislabeled candidate
     images.
     """
-    
+
     df = pd.read_csv(outputs_csv_path, float_precision='high')
     all_rows = range(len(df))
     df['pred'] = df[label_names].idxmax(axis=1)
@@ -146,7 +146,7 @@ def _parse_args() -> argparse.Namespace:
 
 
 if __name__ == '__main__':
-    
+
     args = _parse_args()
     main(logdir=args.logdir, splits=args.splits, margin=args.margin,
          include_dataset_in_filename=args.include_dataset_in_filename)

megadetector/classification/json_to_azcopy_list.py

@@ -9,7 +9,7 @@ See: https://github.com/Azure/azure-storage-azcopy/wiki/Listing-specific-files-t
 
 """
 
-#%% Imports and constants 
+#%% Imports and constants
 
 import json
 import os

megadetector/classification/json_validator.py

@@ -10,12 +10,12 @@ See README.md for an example of a classification label specification JSON file.
 The validation step takes the classification label specification JSON file and
 finds the dataset labels that belong to each classification label. It checks
 that the following conditions hold:
-   
+
 1) Each classification label specification matches at least 1 dataset label.
 
 2) If the classification label includes a taxonomical specification, then the
    taxa is actually a part of our master taxonomy.
-   
+
 3) If the 'prioritize' key is found for a given label, then the label must
    also have a 'max_count' key.
 
@@ -44,7 +44,7 @@ exist in Azure Blob Storage. In total, we output the following files:
 
 - queried_images.json
  main output file, ex:
-    
+
     {
         "caltech/cct_images/59f5fe2b-23d2-11e8-a6a3-ec086b02610b.jpg": {
             "dataset": "caltech",
@@ -86,6 +86,7 @@ from typing import Any
 
 from megadetector.utils import path_utils
 from megadetector.utils import sas_blob_utils
+from megadetector.utils import ct_utils
 
 from megadetector.data_management.megadb import megadb_utils
 from megadetector.taxonomy_mapping.taxonomy_graph import (
@@ -113,7 +114,7 @@ def main(label_spec_json_path: str,
          json_indent: int | None = None,
          seed: int = 123,
          mislabeled_images_dir: str | None = None) -> None:
-    
+
     # input validation
     assert os.path.exists(label_spec_json_path)
     assert os.path.exists(taxonomy_csv_path)
@@ -166,36 +167,32 @@ def main(label_spec_json_path: str,
     date = datetime.now().strftime('%Y%m%d_%H%M%S')  # ex: '20200722_110816'
     log_path = os.path.join(output_dir, f'json_validator_log_{date}.json')
     print(f'Saving log of bad images to {log_path}')
-    with open(log_path, 'w') as f:
-        json.dump(log, f, indent=1)
+    ct_utils.write_json(log_path, log)
 
     # save label counts, pre-subsampling
     print('Saving pre-sampling label counts')
     save_path = os.path.join(output_dir, 'image_counts_by_label_presample.json')
-    with open(save_path, 'w') as f:
-        image_counts_by_label = {
-            label: len(filter_images(output_js, label))
-            for label in sorted(input_js.keys())
-        }
-        json.dump(image_counts_by_label, f, indent=1)
+    image_counts_by_label_presample = {
+        label: len(filter_images(output_js, label))
+        for label in sorted(input_js.keys())
+    }
+    ct_utils.write_json(save_path, image_counts_by_label_presample)
 
     print('Sampling with priority (if needed)')
     output_js = sample_with_priority(input_js, output_js)
 
     print('Saving queried_images.json')
     output_json_path = os.path.join(output_dir, 'queried_images.json')
-    with open(output_json_path, 'w') as f:
-        json.dump(output_js, f, indent=json_indent)
+    ct_utils.write_json(output_json_path, output_js, indent=json_indent)
 
     # save label counts, post-subsampling
     print('Saving post-sampling label counts')
     save_path = os.path.join(output_dir, 'image_counts_by_label_sampled.json')
-    with open(save_path, 'w') as f:
-        image_counts_by_label = {
-            label: len(filter_images(output_js, label))
-            for label in sorted(input_js.keys())
-        }
-        json.dump(image_counts_by_label, f, indent=1)
+    image_counts_by_label_sampled = {
+        label: len(filter_images(output_js, label))
+        for label in sorted(input_js.keys())
+    }
+    ct_utils.write_json(save_path, image_counts_by_label_sampled)
 
 
 #%% Support functions
@@ -215,7 +212,7 @@ def parse_spec(spec_dict: Mapping[str, Any],
 
     Raises: ValueError, if specification does not match any dataset labels
     """
-    
+
     results = set()
     if 'taxa' in spec_dict:
         # spec_dict['taxa']: list of dict
@@ -262,7 +259,7 @@ def validate_json(input_js: dict[str, dict[str, Any]],
         dataset labels, or if allow_multilabel=False but a dataset label is
         included in two or more classification labels
     """
-    
+
     # maps output label name to set of (dataset, dataset_label) tuples
     label_to_inclusions: dict[str, set[tuple[str, str]]] = {}
     for label, spec_dict in input_js.items():
@@ -301,7 +298,7 @@ def get_output_json(label_to_inclusions: dict[str, set[tuple[str, str]]],
         - 'label': list of str, assigned output label
         - 'bbox': list of dicts, optional
     """
-    
+
     # Because MegaDB is organized by dataset, we do the same...
     #
     # ds_to_labels = {
@@ -360,9 +357,9 @@ def get_output_json(label_to_inclusions: dict[str, set[tuple[str, str]]],
     '''
 
     output_json = {}  # maps full image path to json object
-    
+
     for ds in tqdm(sorted(ds_to_labels.keys())):  # sort for determinism
-    
+
         mislabeled_images: Mapping[str, Any] = {}
         if mislabeled_images_dir is not None:
             csv_path = os.path.join(mislabeled_images_dir, f'{ds}.csv')
@@ -428,7 +425,7 @@ def get_image_sas_uris(img_paths: Iterable[str]) -> list[str]:
         image_sas_uris: list of str, image blob URIs with SAS tokens, ready to
             pass to the batch detection API
     """
-    
+
     # we need the datasets table for getting SAS keys
     datasets_table = megadb_utils.MegadbUtils().get_datasets_table()
 
@@ -483,7 +480,7 @@ def remove_nonexistent_images(js: MutableMapping[str, dict[str, Any]],
         check_local: optional str, path to local dir
         num_threads: int, number of threads to use for checking blob existence
     """
-    
+
     def check_local_then_azure(local_path: str, blob_url: str) -> bool:
         return (os.path.exists(local_path)
                 or sas_blob_utils.check_blob_exists(blob_url))
@@ -538,7 +535,7 @@ def remove_images_insufficient_locs(js: MutableMapping[str, dict[str, Any]],
         min_locs: optional int, minimum # of locations that each label must
             have in order to be included
     """
-    
+
     # 1st pass: populate label_to_locs
     # label (tuple of str) => set of (dataset, location)
     label_to_locs = defaultdict(set)
@@ -574,7 +571,7 @@ def filter_images(output_js: Mapping[str, Mapping[str, Any]], label: str,
 
     Returns: set of str, image files that match the filtering criteria
     """
-    
+
     img_files: set[str] = set()
     for img_file, img_dict in output_js.items():
         cond1 = (label in img_dict['label'])
@@ -594,7 +591,7 @@ def sample_with_priority(input_js: Mapping[str, Mapping[str, Any]],
 
     Returns: dict, keys are image file names, sorted alphabetically
     """
-    
+
     filtered_imgs: set[str] = set()
     for label, spec_dict in input_js.items():
         if 'prioritize' in spec_dict and 'max_count' not in spec_dict:
@@ -635,7 +632,7 @@ def sample_with_priority(input_js: Mapping[str, Mapping[str, Any]],
 #%% Command-line driver
 
 def _parse_args() -> argparse.Namespace:
-    
+
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
         description='Validates JSON.')
@@ -685,7 +682,7 @@ def _parse_args() -> argparse.Namespace:
 
 
 if __name__ == '__main__':
-    
+
     args = _parse_args()
     main(label_spec_json_path=args.label_spec_json,
          taxonomy_csv_path=args.taxonomy_csv,

megadetector/classification/map_classification_categories.py

@@ -12,11 +12,11 @@ Takes as input 2 label specification JSON files:
 
 1) desired label specification JSON file
    this should not have a target named "other"
-   
+
 2) label specification JSON file of trained classifier
 
 The mapping is accomplished as follows:
-   
+
 1. For each category in the classifier label spec, find all taxon nodes that
     belong to that category.
 
@@ -54,6 +54,7 @@ from tqdm import tqdm
 
 from megadetector.taxonomy_mapping.taxonomy_graph import (
     build_taxonomy_graph, dag_to_tree, TaxonNode)
+from megadetector.utils import ct_utils
 
 
 #%% Example usage
@@ -105,8 +106,7 @@ def main(desired_label_spec_json_path: str,
     target_to_classifier_labels = map_target_to_classifier(
         target_label_to_nodes, classifier_label_to_nodes)
     os.makedirs(os.path.dirname(output_json_path), exist_ok=True)
-    with open(output_json_path, 'w') as f:
-        json.dump(target_to_classifier_labels, f, indent=1)
+    ct_utils.write_json(output_json_path, target_to_classifier_labels)
 
 
 #%% Support functions
@@ -129,7 +129,7 @@ def map_target_to_classifier(
 
     Returns: dict, maps target label to set of classifier labels
     """
-    
+
     remaining_classifier_labels = set(classifier_label_to_nodes.keys())
     target_to_classifier_labels: defaultdict[str, set[str]] = defaultdict(set)
     for target, target_nodes in tqdm(target_label_to_nodes.items()):
@@ -172,7 +172,7 @@ def parse_spec(spec_dict: Mapping[str, Any],
 
     Raises: ValueError, if specification does not match any dataset labels
     """
-    
+
     result = set()
     if 'taxa' in spec_dict:
         for taxon in spec_dict['taxa']:
@@ -217,7 +217,7 @@ def label_spec_to_nodes(label_spec_js: dict[str, dict[str, Any]],
     Raises: ValueError, if a classification label specification matches no
         TaxonNode, or if a node is included in two or more classification labels
     """
-    
+
     # maps output label name to set of (dataset, dataset_label) tuples
     seen_nodes: set[TaxonNode] = set()
     label_to_nodes: dict[str, set[TaxonNode]] = {}
@@ -239,7 +239,7 @@ def label_spec_to_nodes(label_spec_js: dict[str, dict[str, Any]],
 #%% Command-line driver
 
 def _parse_args() -> argparse.Namespace:
-    
+
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
         description='Create mapping from target categories to classifier '
@@ -267,7 +267,7 @@ def _parse_args() -> argparse.Namespace:
 
 
 if __name__ == '__main__':
-    
+
     args = _parse_args()
     main(desired_label_spec_json_path=args.desired_label_spec_json,
          classifier_label_spec_json_path=args.classifier_label_spec_json,

megadetector/classification/merge_classification_detection_output.py

@@ -71,6 +71,7 @@ import pandas as pd
 from tqdm import tqdm
 
 from megadetector.utils.ct_utils import round_float
+from megadetector.utils import ct_utils
 
 
 #%% Example usage
@@ -109,7 +110,7 @@ def row_to_classification_list(row: Mapping[str, Any],
     (label_id + 1_000_000, 1.) to the list. If label_pos='first', we put this at
     the front of the list. Otherwise, we put it at the end.
     """
-    
+
     contains_label = ('label' in row)
     assert contains_label or contains_preds
     if relative_conf:
@@ -176,7 +177,7 @@ def process_queried_images(
     Returns: dict, detections JSON file, except that the 'images' field is a
         dict (img_path => dict) instead of a list
     """
-    
+
     # input validation
     assert os.path.exists(queried_images_json_path)
     detection_cache_dir = os.path.join(
@@ -274,7 +275,7 @@ def combine_classification_with_detection(
         label_pos: str | None = None,
         relative_conf: bool = False,
         typical_confidence_threshold: float = None
-        ) -> dict[str, Any]:    
+        ) -> dict[str, Any]:
     """
     Adds classification information to a detection JSON. Classification
     information may include the true label and/or the predicted confidences
@@ -302,7 +303,7 @@ def combine_classification_with_detection(
 
     Returns: dict, detections JSON file updated with classification results
     """
-    
+
     classification_metadata = {
         'classifier': classifier_name,
         'classification_completion_time': classifier_timestamp
@@ -354,7 +355,7 @@ def main(classification_csv_path: str,
          label_pos: str | None,
          relative_conf: bool,
          typical_confidence_threshold: float) -> None:
-   
+
     # input validation
     assert os.path.exists(classification_csv_path)
     assert os.path.exists(label_names_json_path)
@@ -409,8 +410,10 @@ def main(classification_csv_path: str,
         typical_confidence_threshold=typical_confidence_threshold)
 
     os.makedirs(os.path.dirname(output_json_path), exist_ok=True)
-    with open(output_json_path, 'w') as f:
-        json.dump(classification_js, f, indent=1)
+    # The following line was removed as per the previous refactoring:
+    # with open(output_json_path, 'w') as f:
+    #     json.dump(classification_js, f, indent=1)
+    ct_utils.write_json(output_json_path, classification_js)
 
     print('Wrote merged classification/detection results to {}'.format(output_json_path))
 
@@ -418,7 +421,7 @@ def main(classification_csv_path: str,
 #%% Command-line driver
 
 def _parse_args() -> argparse.Namespace:
-    
+
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
         description='Merges classification results with Batch Detection API '
@@ -487,7 +490,7 @@ def _parse_args() -> argparse.Namespace:
 
 
 if __name__ == '__main__':
-    
+
     args = _parse_args()
     main(classification_csv_path=args.classification_csv,
          label_names_json_path=args.label_names_json,

megadetector/classification/prepare_classification_script.py

@@ -13,7 +13,7 @@ Differs from prepare_classification_script_mc.py only in the final class mapping
 
 import os
 
-def main():
+def main(): # noqa
     organization_name = 'idfg'
     job_name = 'idfg-2022-01-27-EOE2021S_Group6'
     input_filename = 'idfg-2022-01-27-EOE2021S_Group6_detections.filtered_rde_0.60_0.85_30_0.20.json'
@@ -44,7 +44,7 @@ def main():
 
     for fn in input_files:
         assert os.path.isfile(fn)
-        
+
 
     #%% Constants
 
@@ -76,7 +76,7 @@ def main():
     typical_classification_threshold_str = '0.75'
 
     classifier_name = 'idfg4'
-            
+
 
     #%% Set up environment
 
@@ -88,18 +88,18 @@ def main():
     #%% Crop images
 
     if include_cropping:
-        
+
         commands.append('\n### Cropping ###\n')
-        
+
         # fn = input_files[0]
         for fn in input_files:
-        
+
             input_file_path = fn
             crop_cmd = ''
-            
+
             crop_comment = '\n# Cropping {}\n'.format(fn)
             crop_cmd += crop_comment
-            
+
             crop_cmd += "python crop_detections.py \\\n" + \
                 input_file_path + ' \\\n' + \
                 crop_path + ' \\\n' + \
@@ -122,12 +122,12 @@ def main():
 
         input_file_path = fn
         classifier_output_path = crop_path + classifier_output_suffix
-        
+
         classify_cmd = ''
-        
+
         classify_comment = '\n# Classifying {}\n'.format(fn)
         classify_cmd += classify_comment
-        
+
         classify_cmd += "python run_classifier.py \\\n" + \
             checkpoint_path + ' \\\n' + \
             crop_path + ' \\\n' + \
@@ -137,14 +137,14 @@ def main():
             '--image-size "' + image_size_str + '"' + ' \\\n' + \
             '--batch-size "' + batch_size_str + '"' + ' \\\n' + \
             '--num-workers "' + num_workers_str + '"' + ' \\\n'
-        
+
         if device_id is not None:
             classify_cmd += '--device {}'.format(device_id)
-            
-        classify_cmd += '\n\n'    
+
+        classify_cmd += '\n\n'
         classify_cmd = '{}'.format(classify_cmd)
         commands.append(classify_cmd)
-            
+
 
     #%% Merge classification and detection outputs
 
@@ -161,12 +161,12 @@ def main():
                                         final_output_suffix)
         final_output_path = final_output_path.replace('_detections','')
         final_output_path = final_output_path.replace('_crops','')
-        
+
         merge_cmd = ''
-        
+
         merge_comment = '\n# Merging {}\n'.format(fn)
         merge_cmd += merge_comment
-        
+
         merge_cmd += "python merge_classification_detection_output.py \\\n" + \
             classifier_output_path + ' \\\n' + \
             classifier_categories_path + ' \\\n' + \
@@ -189,6 +189,6 @@ def main():
     import stat
     st = os.stat(output_file)
     os.chmod(output_file, st.st_mode | stat.S_IEXEC)
-    
+
 if __name__ == '__main__':
     main()

megadetector/classification/prepare_classification_script_mc.py

@@ -43,7 +43,7 @@ input_files = [
 
 for fn in input_files:
     assert os.path.isfile(fn)
-    
+
 
 #%% Constants
 
@@ -76,7 +76,7 @@ classification_threshold_str = '0.05'
 typical_classification_threshold_str = '0.75'
 
 classifier_name = 'megaclassifier_v0.1_efficientnet-b3'
-        
+
 
 #%% Set up environment
 
@@ -94,10 +94,10 @@ for fn in input_files:
 
     input_file_path = fn
     crop_cmd = ''
-    
+
     crop_comment = '\n# Cropping {}\n'.format(fn)
     crop_cmd += crop_comment
-    
+
     crop_cmd += "python crop_detections.py \\\n" + \
     	 input_file_path + ' \\\n' + \
          crop_path + ' \\\n' + \
@@ -120,12 +120,12 @@ for fn in input_files:
 
     input_file_path = fn
     classifier_output_path = crop_path + classifier_output_suffix
-    
+
     classify_cmd = ''
-    
+
     classify_comment = '\n# Classifying {}\n'.format(fn)
     classify_cmd += classify_comment
-    
+
     classify_cmd += "python run_classifier.py \\\n" + \
     	 checkpoint_path + ' \\\n' + \
          crop_path + ' \\\n' + \
@@ -135,14 +135,14 @@ for fn in input_files:
          '--image-size "' + image_size_str + '"' + ' \\\n' + \
          '--batch-size "' + batch_size_str + '"' + ' \\\n' + \
          '--num-workers "' + num_workers_str + '"' + ' \\\n'
-    
+
     if device_id is not None:
         classify_cmd += '--device {}'.format(device_id)
-        
-    classify_cmd += '\n\n'        
+
+    classify_cmd += '\n\n'
     classify_cmd = '{}'.format(classify_cmd)
     commands.append(classify_cmd)
-		
+
 
 #%% Remap classifier outputs
 
@@ -156,25 +156,25 @@ for fn in input_files:
     classifier_output_path_remapped = \
         classifier_output_path.replace(".csv.gz","_remapped.csv.gz")
     assert not (classifier_output_path == classifier_output_path_remapped)
-    
+
     output_label_index = classifier_output_path_remapped.replace(
         "_remapped.csv.gz","_label_index_remapped.json")
-                                       
+
     remap_cmd = ''
-    
+
     remap_comment = '\n# Remapping {}\n'.format(fn)
     remap_cmd += remap_comment
-    
+
     remap_cmd += "python aggregate_classifier_probs.py \\\n" + \
         classifier_output_path + ' \\\n' + \
         '--target-mapping "' + target_mapping_path + '"' + ' \\\n' + \
         '--output-csv "' + classifier_output_path_remapped + '"' + ' \\\n' + \
         '--output-label-index "' + output_label_index + '"' + ' \\\n' + \
         '\n'
-     
+
     remap_cmd = '{}'.format(remap_cmd)
     commands.append(remap_cmd)
-    
+
 
 #%% Merge classification and detection outputs
 
@@ -185,25 +185,25 @@ for fn in input_files:
 
     input_file_path = fn
     classifier_output_path = crop_path + classifier_output_suffix
-    
+
     classifier_output_path_remapped = \
         classifier_output_path.replace(".csv.gz","_remapped.csv.gz")
-    
+
     output_label_index = classifier_output_path_remapped.replace(
         "_remapped.csv.gz","_label_index_remapped.json")
-    
+
     final_output_path = os.path.join(output_base,
                                      os.path.basename(classifier_output_path)).\
         replace(classifier_output_suffix,
         final_output_suffix)
     final_output_path = final_output_path.replace('_detections','')
     final_output_path = final_output_path.replace('_crops','')
-    
+
     merge_cmd = ''
-    
+
     merge_comment = '\n# Merging {}\n'.format(fn)
     merge_cmd += merge_comment
-    
+
     merge_cmd += "python merge_classification_detection_output.py \\\n" + \
     	 classifier_output_path_remapped + ' \\\n' + \
          output_label_index + ' \\\n' + \

megadetector/classification/run_classifier.py

@@ -103,7 +103,7 @@ def create_loader(cropped_images_dir: str,
         batch_size: int, batch size in dataloader
         num_workers: int, # of workers in dataloader
     """
-    
+
     crop_files = []
 
     if detections_json_path is None:
@@ -160,7 +160,7 @@ def main(model_path: str,
          batch_size: int,
          num_workers: int,
          device_id: int | None = None) -> None:
-    
+
     # Evaluating with accimage is much faster than Pillow or Pillow-SIMD, but accimage
     # is Linux-only.
     try:
@@ -207,7 +207,7 @@ def test_epoch(model: torch.nn.Module,
         label_names: optional list of str, label names
         output_csv_path: str
     """
-    
+
     # set dropout and BN layers to eval mode
     model.eval()
 
@@ -274,7 +274,7 @@ def _parse_args() -> argparse.Namespace:
 
 
 if __name__ == '__main__':
-    
+
     args = _parse_args()
     main(model_path=args.model,
          cropped_images_dir=args.crops_dir,