megadetector 5.0.28__py3-none-any.whl → 10.0.0__py3-none-any.whl

megadetector/classification/save_mislabeled.py

@@ -1,4 +1,4 @@
-"""
+r"""
 
 save_mislabeled.py
 
@@ -10,7 +10,7 @@ List of known mislabeled images is stored in Azure Blob Storage.
 * blob: megadb_mislabeled/{dataset}.csv, one file per dataset
 
 Each file megadb_mislabeled/{dataset}.csv has two columns:
-   
+
 * 'file': str, blob name
 
 * 'correct_class': optional str, correct dataset class
@@ -41,7 +41,7 @@ import pandas as pd
 #%% Main function
 
 def update_mislabeled_images(container_path: str, input_csv_path: str) -> None:
-    
+
     df = pd.read_csv(input_csv_path, index_col=False)
 
     # error checking
@@ -62,7 +62,7 @@ def update_mislabeled_images(container_path: str, input_csv_path: str) -> None:
     df['file'] = df['blob_dirname'] + '/' + df['File']
 
     for ds, ds_df in df.groupby('dataset'):
-        
+
         sr_path = os.path.join(container_path, 'megadb_mislabeled', f'{ds}.csv')
         if os.path.exists(sr_path):
             old_sr = pd.read_csv(sr_path, index_col='file', squeeze=True)
@@ -89,7 +89,7 @@ def update_mislabeled_images(container_path: str, input_csv_path: str) -> None:
 #%% Command-line driver
 
 def _parse_args() -> argparse.Namespace:
-    
+
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
         description='Merges classification results with Batch Detection API '
@@ -104,7 +104,7 @@ def _parse_args() -> argparse.Namespace:
 
 
 if __name__ == '__main__':
-    
+
     args = _parse_args()
     update_mislabeled_images(container_path=args.container_path,
                              input_csv_path=args.input_csv)

megadetector/classification/train_classifier.py

@@ -44,7 +44,7 @@ from megadetector.visualization import plot_utils
 
 #%% Example usage
 
-"""    
+"""
     python train_classifier.py run_idfg /ssd/crops_sq \
         -m "efficientnet-b0" --pretrained --finetune --label-weighted \
         --epochs 50 --batch-size 512 --lr 1e-4 \

megadetector/classification/train_classifier_tf.py

@@ -54,7 +54,7 @@ EFFICIENTNET_MODELS: Mapping[str, Mapping[str, Any]] = {
 
 #%% Example usage
 
-"""    
+"""
     python train_classifier_tf.py run_idfg /ssd/crops_sq \
         -m "efficientnet-b0" --pretrained --finetune --label-weighted \
         --epochs 50 --batch-size 512 --lr 1e-4 \
@@ -97,7 +97,7 @@ def create_dataset(
 
     Returns: tf.data.Dataset
     """
-    
+
     # images dataset
     img_ds = tf.data.Dataset.from_tensor_slices(img_files)
     img_ds = img_ds.map(lambda p: tf.io.read_file(img_base_dir + os.sep + p),
@@ -162,7 +162,7 @@ def create_dataloaders(
         datasets: dict, maps split to DataLoader
         label_names: list of str, label names in order of label id
     """
-    
+
     df, label_names, split_to_locs = load_dataset_csv(
         dataset_csv_path, label_index_json_path, splits_json_path,
         multilabel=multilabel, label_weighted=label_weighted,
@@ -238,7 +238,7 @@ def build_model(model_name: str, num_classes: int, img_size: int,
     """
     Creates a model with an EfficientNet base.
     """
-    
+
     class_name = EFFICIENTNET_MODELS[model_name]['cls']
     dropout = EFFICIENTNET_MODELS[model_name]['dropout']
 
@@ -279,7 +279,7 @@ def log_images_with_confidence(
         epoch: int
         tag: str
     """
-    
+
     for label_id, heap in heap_dict.items():
         label_name = label_names[label_id]
 
@@ -319,7 +319,7 @@ def track_extreme_examples(tp_heaps: dict[int, list[HeapItem]],
         img_files: tf.Tensor, shape [batch_size], type tf.string
         logits: tf.Tensor, shape [batch_size, num_classes]
     """
-    
+
     labels = labels.numpy().tolist()
     inputs = inputs.numpy().astype(np.uint8)
     img_files = img_files.numpy().astype(str).tolist()
@@ -480,7 +480,7 @@ def log_run(split: str, epoch: int, writer: tf.summary.SummaryWriter,
     Args:
         metrics: dict, keys already prefixed with {split}/
     """
-    
+
     per_class_recall = recall_from_confusion_matrix(cm, label_names)
     metrics.update(prefix_all_keys(per_class_recall, f'{split}/label_recall/'))
 
@@ -518,7 +518,7 @@ def main(dataset_dir: str,
          seed: Optional[int] = None,
          logdir: str = '',
          cache_splits: Sequence[str] = ()) -> None:
-    
+
     # input validation
     assert os.path.exists(dataset_dir)
     assert os.path.exists(cropped_images_dir)
@@ -597,7 +597,7 @@ def main(dataset_dir: str,
             model.base_model.trainable = True
 
         print('- train:')
-        
+
         train_metrics, train_heaps, train_cm = run_epoch(
             model, loader=loaders['train'], weighted=label_weighted,
             loss_fn=loss_fn, weight_decay=weight_decay, optimizer=optimizer,

megadetector/classification/train_utils.py

@@ -35,7 +35,7 @@ class HeapItem:
     """
     A wrapper over non-comparable data with a comparable priority value.
     """
-    
+
     priority: Any
     data: Any = dataclasses.field(compare=False, repr=False)
 
@@ -53,7 +53,7 @@ def add_to_heap(h: list[Any], item: HeapItem, k: Optional[int] = None) -> None:
         item: HeapItem
         k: int, desired capacity of the heap, or None for no limit
     """
-    
+
     if k is None or len(h) < k:
         heapq.heappush(h, item)
     else:
@@ -66,17 +66,17 @@ def prefix_all_keys(d: Mapping[str, Any], prefix: str) -> dict[str, Any]:
     """
     Returns a new dict where the keys are prefixed by <prefix>.
     """
-    
+
     return {f'{prefix}{k}': v for k, v in d.items()}
 
 
 def fig_to_img(fig: matplotlib.figure.Figure) -> np.ndarray:
     """
     Converts a matplotlib figure to an image represented by a numpy array.
-    
+
     Returns: np.ndarray, type uint8, shape [H, W, 3]
     """
-    
+
     with io.BytesIO() as b:
         fig.savefig(b, transparent=False, bbox_inches='tight', pad_inches=0,
                     format='png')
@@ -103,7 +103,7 @@ def imgs_with_confidences(imgs_list: list[tuple[Any, ...]],
         fig: matplotlib.figure.Figure
         img_files: list of str
     """
-    
+
     imgs, img_files, tags, titles = [], [], [], []
     for img, label_id, topk_conf, topk_preds, img_file in imgs_list:
         imgs.append(img)
@@ -140,7 +140,7 @@ def plot_img_grid(imgs: Sequence[Any], row_h: float, col_w: float,
 
     Returns: matplotlib.figure.Figure
     """
-    
+
     # input validation
     num_images = len(imgs)
     if tags is not None:
@@ -186,7 +186,7 @@ def load_splits(splits_json_path: str) -> dict[str, set[tuple[str, str]]]:
 
     Returns: dict, maps split to set of (dataset, location) tuples
     """
-    
+
     with open(splits_json_path, 'r') as f:
         split_to_locs_js = json.load(f)
     split_to_locs = {
@@ -235,7 +235,7 @@ def load_dataset_csv(dataset_csv_path: str,
         label_names: list of str, label names in order of label id
         split_to_locs: dict, maps split to set of (dataset, location) tuples
     """
-    
+
     # read in dataset CSV and create merged (dataset, location) col
     df = pd.read_csv(dataset_csv_path, index_col=False, float_precision='high')
     df['dataset_location'] = list(zip(df['dataset'], df['location']))
@@ -315,7 +315,7 @@ def recall_from_confusion_matrix(
 
     Returns: dict, label_name => recall
     """
-    
+
     result = {
         label_name: confusion_matrix[i, i] / (confusion_matrix[i].sum() + 1e-8)
         for i, label_name in enumerate(label_names)

megadetector/data_management/annotations/annotation_constants.py

@@ -4,7 +4,7 @@ annotation_constants.py
 
 Defines default categories for MegaDetector output boxes.
 
-Used throughout the repo; do not change unless you are Dan or Siyu.  In fact, do not change unless 
+Used throughout the repo; do not change unless you are Dan or Siyu.  In fact, do not change unless
 you are both Dan *and* Siyu.
 
 We use integer IDs here; this is different from the MD .json file format,
@@ -31,4 +31,3 @@ detector_bbox_category_name_to_id = {}
 for cat in detector_bbox_categories:
     detector_bbox_category_id_to_name[cat['id']] = cat['name']
     detector_bbox_category_name_to_id[cat['name']] = cat['id']
-

megadetector/data_management/camtrap_dp_to_coco.py

@@ -19,6 +19,8 @@ Currently supports only sequence-level labeling.
 
 import os
 import json
+import argparse
+
 import pandas as pd
 
 from dateutil import parser as dateparser
@@ -31,22 +33,26 @@ from collections import defaultdict
 def camtrap_dp_to_coco(camtrap_dp_folder,output_file=None):
     """
     Convert the Camtrap DP package in [camtrap_dp_folder] to COCO.
-    
+
     Does not validate images, just converts.  Use integrity_check_json_db to validate
-    the resulting COCO file.  
-    
+    the resulting COCO file.
+
     Optionally writes the results to [output_file]
+
+    Args:
+        camtrap_dp_folder (str): input folder, containing a CamtrapDP package
+        output_file (str, optional): COCO-formatted output file
     """
-    
+
     required_files = ('datapackage.json','deployments.csv','events.csv','media.csv','observations.csv')
-    
+
     for fn in required_files:
         fn_abs = os.path.join(camtrap_dp_folder,fn)
         assert os.path.isfile(fn_abs), 'Could not find required file {}'.format(fn_abs)
-        
+
     with open(os.path.join(camtrap_dp_folder,'datapackage.json'),'r') as f:
         datapackage = json.load(f)
-        
+
     assert datapackage['profile'] == 'https://raw.githubusercontent.com/tdwg/camtrap-dp/1.0/camtrap-dp-profile.json', \
         'I only know how to parse Camtrap DP 1.0 packages'
 
@@ -54,7 +60,7 @@ def camtrap_dp_to_coco(camtrap_dp_folder,output_file=None):
     events_file = None
     media_file = None
     observations_file = None
-    
+
     resources = datapackage['resources']
     for r in resources:
         if r['name'] == 'deployments':
@@ -70,19 +76,19 @@ def camtrap_dp_to_coco(camtrap_dp_folder,output_file=None):
     assert events_file is not None, 'No events file specified'
     assert media_file is not None, 'No media file specified'
     assert observations_file is not None, 'No observation file specified'
-    
+
     deployments_df = pd.read_csv(os.path.join(camtrap_dp_folder,deployments_file))
     events_df = pd.read_csv(os.path.join(camtrap_dp_folder,events_file))
     media_df = pd.read_csv(os.path.join(camtrap_dp_folder,media_file))
     observations_df = pd.read_csv(os.path.join(camtrap_dp_folder,observations_file))
-    
+
     print('Read {} deployment lines'.format(len(deployments_df)))
     print('Read {} events lines'.format(len(events_df)))
     print('Read {} media lines'.format(len(media_df)))
     print('Read {} observation lines'.format(len(observations_df)))
-    
+
     media_id_to_media_info = {}
-    
+
     # i_row = 0; row = media_df.iloc[i_row]
     for i_row,row in media_df.iterrows():
         media_info = {}
@@ -94,23 +100,23 @@ def camtrap_dp_to_coco(camtrap_dp_folder,output_file=None):
         media_info['frame_num'] = -1
         media_info['seq_num_frames'] = -1
         media_id_to_media_info[row['mediaID']] = media_info
-        
+
     event_id_to_media_ids = defaultdict(list)
-    
+
     # i_row = 0; row = events_df.iloc[i_row]
     for i_row,row in events_df.iterrows():
         media_id = row['mediaID']
         assert media_id in media_id_to_media_info
         event_id_to_media_ids[row['eventID']].append(media_id)
-    
+
     event_id_to_category_names = defaultdict(set)
-    
+
     # i_row = 0; row = observations_df.iloc[i_row]
     for i_row,row in observations_df.iterrows():
-        
+
         if row['observationLevel'] != 'event':
             raise ValueError("I don't know how to parse image-level events yet")
-            
+
         if row['observationType'] == 'blank':
             event_id_to_category_names[row['eventID']].add('empty')
         elif row['observationType'] == 'unknown':
@@ -122,7 +128,7 @@ def camtrap_dp_to_coco(camtrap_dp_folder,output_file=None):
             assert row['observationType'] == 'animal'
             assert isinstance(row['scientificName'],str)
             event_id_to_category_names[row['eventID']].add(row['scientificName'])
-    
+
     # Sort images within an event into frame numbers
     #
     # event_id = next(iter(event_id_to_media_ids))
@@ -134,7 +140,7 @@ def camtrap_dp_to_coco(camtrap_dp_folder,output_file=None):
             media_info['frame_num'] = i_media
             media_info['seq_num_frames'] = len(media_info_this_event)
             media_info['seq_id'] = event_id
-            
+
     # Create category names
     category_name_to_category_id = {'empty':0}
     for event_id in event_id_to_category_names:
@@ -142,18 +148,18 @@ def camtrap_dp_to_coco(camtrap_dp_folder,output_file=None):
         for name in category_names_this_event:
             if name not in category_name_to_category_id:
                 category_name_to_category_id[name] = len(category_name_to_category_id)
-    
+
     # Move everything into COCO format
     images = list(media_id_to_media_info.values())
-    
+
     categories = []
     for name in category_name_to_category_id:
         categories.append({'name':name,'id':category_name_to_category_id[name]})
     info = {'version':1.0,'description':datapackage['name']}
-    
+
     # Create annotations
     annotations = []
-    
+
     for event_id in event_id_to_media_ids.keys():
         i_ann = 0
         media_ids_this_event = event_id_to_media_ids[event_id]
@@ -168,23 +174,23 @@ def camtrap_dp_to_coco(camtrap_dp_folder,output_file=None):
                 ann['category_id'] = category_name_to_category_id[category_name]
                 ann['sequence_level_annotation'] = True
                 annotations.append(ann)
-    
+
     coco_data = {}
     coco_data['images'] = images
     coco_data['annotations'] = annotations
     coco_data['categories'] = categories
     coco_data['info'] = info
-    
+
     for im in coco_data['images']:
         im['datetime'] = str(im['datetime'] )
-        
+
     if output_file is not None:
         with open(output_file,'w') as f:
             json.dump(coco_data,f,indent=1)
-    
+
     return coco_data
-            
-    
+
+
 #%% Interactive driver
 
 if False:
@@ -192,19 +198,19 @@ if False:
     pass
 
     #%%
-    
+
     camtrap_dp_folder = r'C:\temp\pilot2\pilot2'
     coco_file = os.path.join(camtrap_dp_folder,'test-coco.json')
     coco_data = camtrap_dp_to_coco(camtrap_dp_folder,
                                    output_file=coco_file)
-    
+
     #%% Validate
-    
+
     from megadetector.data_management.databases.integrity_check_json_db import \
         integrity_check_json_db, IntegrityCheckOptions
-    
+
     options = IntegrityCheckOptions()
-    
+
     options.baseDir = camtrap_dp_folder
     options.bCheckImageSizes = False
     options.bCheckImageExistence = True
@@ -213,25 +219,52 @@ if False:
     options.iMaxNumImages = -1
     options.nThreads = 1
     options.verbose = True
-    
-    sortedCategories, data, errorInfo = integrity_check_json_db(coco_file,options)
+
+    sorted_categories, data, error_info = integrity_check_json_db(coco_file,options)
 
     #%% Preview
-    
+
     from megadetector.visualization.visualize_db import DbVizOptions, visualize_db
-    
+
     options = DbVizOptions()
     options.parallelize_rendering = True
     options.parallelize_rendering_with_threads = True
     options.parallelize_rendering_n_cores = 10
-    
+
     preview_dir = r'c:\temp\camtrapdp-preview'
-    htmlOutputFile,image_db = visualize_db(coco_file, preview_dir, camtrap_dp_folder, options=options)
-    
+    html_output_file, image_db = visualize_db(coco_file, preview_dir, camtrap_dp_folder, options=options)
+
     from megadetector.utils.path_utils import open_file
-    open_file(htmlOutputFile)
-    
-    
+    open_file(html_output_file)
+
+
 #%% Command-line driver
 
-# TODO
+def main():
+    """
+    Command-line interface to convert Camtrap DP to COCO.
+    """
+
+    parser = argparse.ArgumentParser(description='Convert Camtrap DP to COCO format')
+    parser.add_argument('camtrap_dp_folder', type=str,
+                        help='Input folder, containing a CamtrapDP package')
+    parser.add_argument('--output_file', type=str, default=None,
+                        help='COCO-formatted output file (defaults to [camtrap_dp_folder]_coco.json)')
+
+    args = parser.parse_args()
+
+    if args.output_file is None:
+        # Default output file name: [camtrap_dp_folder]_coco.json
+        #
+        # Remove trailing slash if present
+        folder_name = args.camtrap_dp_folder.rstrip(os.sep)
+        output_file = folder_name + '_coco.json'
+    else:
+        output_file = args.output_file
+
+    camtrap_dp_to_coco(camtrap_dp_folder=args.camtrap_dp_folder, output_file=output_file)
+    print(f"Successfully converted Camtrap DP package at '{args.camtrap_dp_folder}' to " + \
+          f"COCO format at '{output_file}'")
+
+if __name__ == '__main__':
+    main()