megadetector 5.0.10__py3-none-any.whl → 5.0.11__py3-none-any.whl

classification/prepare_classification_script_mc.py

@@ -1,228 +0,0 @@
-"""
-
-prepare_classification_script_mc.py
-
-Notebook-y script used to prepare a series of shell commands to run MegaClassifier
-on a MegaDetector result set.
-
-Differs from prepare_classification_script.py only in the final class mapping step.
-
-"""
-
-#%% Job options
-
-import os
-
-organization_name = 'idfg'
-job_name = 'idfg-2022-01-27-EOE2021S_Group6'
-input_filename = 'idfg-2022-01-27-EOE2021S_Group6_detections.filtered_rde_0.60_0.85_30_0.20.json'
-image_base = '/datadrive/idfg/EOE2021S_Group6'
-crop_path = os.path.join(os.path.expanduser('~/crops'),job_name + '_crops')
-device_id = 0
-
-working_dir_base = os.path.join(os.path.expanduser('~/postprocessing'),
-                                                   organization_name,
-                                                   job_name)
-
-output_base = os.path.join(working_dir_base,'combined_api_outputs')
-
-assert os.path.isdir(working_dir_base)
-assert os.path.isdir(output_base)
-
-output_file = os.path.join(working_dir_base,'run_megaclassifier_' + job_name +  '.sh')
-
-input_files = [
-    os.path.join(
-        os.path.expanduser('~/postprocessing'),
-                           organization_name,
-                           job_name,
-                           'combined_api_outputs',
-                           input_filename
-        )
-    ]
-
-for fn in input_files:
-    assert os.path.isfile(fn)
-    
-
-#%% Constants
-
-classifier_base = os.path.expanduser('~/models/camera_traps/megaclassifier/v0.1/')
-assert os.path.isdir(classifier_base)
-
-checkpoint_path = os.path.join(classifier_base,'v0.1_efficientnet-b3_compiled.pt')
-assert os.path.isfile(checkpoint_path)
-
-classifier_categories_path = os.path.join(classifier_base,'v0.1_index_to_name.json')
-assert os.path.isfile(classifier_categories_path)
-
-target_mapping_path = os.path.join(classifier_base,'idfg_to_megaclassifier_labels.json')
-assert os.path.isfile(target_mapping_path)
-
-classifier_output_suffix = '_megaclassifier_output.csv.gz'
-final_output_suffix = '_megaclassifier.json'
-
-threshold_str = '0.65'
-n_threads_str = '50'
-image_size_str = '300'
-batch_size_str = '64'
-num_workers_str = '8'
-logdir = working_dir_base
-
-classification_threshold_str = '0.05'
-
-# This is just passed along to the metadata in the output file, it has no impact
-# on how the classification scripts run.
-typical_classification_threshold_str = '0.75'
-
-classifier_name = 'megaclassifier_v0.1_efficientnet-b3'
-        
-
-#%% Set up environment
-
-commands = []
-# commands.append('cd MegaDetector/classification\n')
-# commands.append('conda activate cameratraps-classifier\n')
-
-
-#%% Crop images
-
-commands.append('\n### Cropping ###\n')
-
-# fn = input_files[0]
-for fn in input_files:
-
-    input_file_path = fn
-    crop_cmd = ''
-    
-    crop_comment = '\n# Cropping {}\n'.format(fn)
-    crop_cmd += crop_comment
-    
-    crop_cmd += "python crop_detections.py \\\n" + \
-    	 input_file_path + ' \\\n' + \
-         crop_path + ' \\\n' + \
-         '--images-dir "' + image_base + '"' + ' \\\n' + \
-         '--threshold "' + threshold_str + '"' + ' \\\n' + \
-         '--square-crops ' + ' \\\n' + \
-         '--threads "' + n_threads_str + '"' + ' \\\n' + \
-         '--logdir "' + logdir + '"' + ' \\\n' + \
-         '\n'
-    crop_cmd = '{}'.format(crop_cmd)
-    commands.append(crop_cmd)
-
-
-#%% Run classifier
-
-commands.append('\n### Classifying ###\n')
-
-# fn = input_files[0]
-for fn in input_files:
-
-    input_file_path = fn
-    classifier_output_path = crop_path + classifier_output_suffix
-    
-    classify_cmd = ''
-    
-    classify_comment = '\n# Classifying {}\n'.format(fn)
-    classify_cmd += classify_comment
-    
-    classify_cmd += "python run_classifier.py \\\n" + \
-    	 checkpoint_path + ' \\\n' + \
-         crop_path + ' \\\n' + \
-         classifier_output_path + ' \\\n' + \
-         '--detections-json "' + input_file_path + '"' + ' \\\n' + \
-         '--classifier-categories "' + classifier_categories_path + '"' + ' \\\n' + \
-         '--image-size "' + image_size_str + '"' + ' \\\n' + \
-         '--batch-size "' + batch_size_str + '"' + ' \\\n' + \
-         '--num-workers "' + num_workers_str + '"' + ' \\\n'
-    
-    if device_id is not None:
-        classify_cmd += '--device {}'.format(device_id)
-        
-    classify_cmd += '\n\n'        
-    classify_cmd = '{}'.format(classify_cmd)
-    commands.append(classify_cmd)
-		
-
-#%% Remap classifier outputs
-
-commands.append('\n### Remapping ###\n')
-
-# fn = input_files[0]
-for fn in input_files:
-
-    input_file_path = fn
-    classifier_output_path = crop_path + classifier_output_suffix
-    classifier_output_path_remapped = \
-        classifier_output_path.replace(".csv.gz","_remapped.csv.gz")
-    assert not (classifier_output_path == classifier_output_path_remapped)
-    
-    output_label_index = classifier_output_path_remapped.replace(
-        "_remapped.csv.gz","_label_index_remapped.json")
-                                       
-    remap_cmd = ''
-    
-    remap_comment = '\n# Remapping {}\n'.format(fn)
-    remap_cmd += remap_comment
-    
-    remap_cmd += "python aggregate_classifier_probs.py \\\n" + \
-        classifier_output_path + ' \\\n' + \
-        '--target-mapping "' + target_mapping_path + '"' + ' \\\n' + \
-        '--output-csv "' + classifier_output_path_remapped + '"' + ' \\\n' + \
-        '--output-label-index "' + output_label_index + '"' + ' \\\n' + \
-        '\n'
-     
-    remap_cmd = '{}'.format(remap_cmd)
-    commands.append(remap_cmd)
-    
-
-#%% Merge classification and detection outputs
-
-commands.append('\n### Merging ###\n')
-
-# fn = input_files[0]
-for fn in input_files:
-
-    input_file_path = fn
-    classifier_output_path = crop_path + classifier_output_suffix
-    
-    classifier_output_path_remapped = \
-        classifier_output_path.replace(".csv.gz","_remapped.csv.gz")
-    
-    output_label_index = classifier_output_path_remapped.replace(
-        "_remapped.csv.gz","_label_index_remapped.json")
-    
-    final_output_path = os.path.join(output_base,
-                                     os.path.basename(classifier_output_path)).\
-        replace(classifier_output_suffix,
-        final_output_suffix)
-    final_output_path = final_output_path.replace('_detections','')
-    final_output_path = final_output_path.replace('_crops','')
-    
-    merge_cmd = ''
-    
-    merge_comment = '\n# Merging {}\n'.format(fn)
-    merge_cmd += merge_comment
-    
-    merge_cmd += "python merge_classification_detection_output.py \\\n" + \
-    	 classifier_output_path_remapped + ' \\\n' + \
-         output_label_index + ' \\\n' + \
-         '--output-json "' + final_output_path + '"' + ' \\\n' + \
-         '--detection-json "' + input_file_path + '"' + ' \\\n' + \
-         '--classifier-name "' + classifier_name + '"' + ' \\\n' + \
-         '--threshold "' + classification_threshold_str + '"' + ' \\\n' + \
-         '--typical-confidence-threshold "' + typical_classification_threshold_str + '"' + ' \\\n' + \
-         '\n'
-    merge_cmd = '{}'.format(merge_cmd)
-    commands.append(merge_cmd)
-
-
-#%% Write everything out
-
-with open(output_file,'w') as f:
-    for s in commands:
-        f.write('{}'.format(s))
-
-import stat
-st = os.stat(output_file)
-os.chmod(output_file, st.st_mode | stat.S_IEXEC)

classification/run_classifier.py

@@ -1,286 +0,0 @@
-"""
-
-run_classifier.py
-
-Run a species classifier.
-
-This script is the classifier counterpart to detection/run_tf_detector_batch.py.
-This script takes as input:
-1) a detections JSON file, usually the output of run_tf_detector_batch.py or the
-    output of the Batch API in the "Batch processing API output format"
-2) a path to a directory containing crops of bounding boxes from the detections
-    JSON file
-3) a path to a PyTorch TorchScript compiled model file
-4) (if the model is EfficientNet) an image size
-
-By default, this script overwrites the detections JSON file, adding in
-classification results. To output a new JSON file, use the --output argument.
-
-"""
-
-#%% Imports
-
-from __future__ import annotations
-
-import argparse
-import json
-import os
-from tqdm import tqdm
-from typing import Any
-from collections.abc import Callable, Sequence
-
-import pandas as pd
-import PIL
-import torch
-import torch.utils
-import torchvision as tv
-from torchvision.datasets.folder import default_loader
-
-from classification import train_classifier
-
-
-#%% Example usage
-
-"""
-    python run_classifier.py \
-        detections.json \
-        /path/to/crops \
-        /path/to/model.pt \
-        --image-size 224
-"""
-
-
-#%% Classes
-
-class SimpleDataset(torch.utils.data.Dataset):
-    """
-    Very simple dataset.
-    """
-
-    def __init__(self, img_files: Sequence[str],
-                 images_dir: str | None = None,
-                 transform: Callable[[PIL.Image.Image], Any] | None = None):
-        """Creates a SimpleDataset."""
-        self.img_files = img_files
-        self.images_dir = images_dir
-        self.transform = transform
-
-    def __getitem__(self, index: int) -> tuple[Any, str]:
-        """
-        Returns: tuple, (img, img_file)
-        """
-        img_file = self.img_files[index]
-        if self.images_dir is not None:
-            img_path = os.path.join(self.images_dir, img_file)
-        else:
-            img_path = img_file
-        img = default_loader(img_path)
-        if self.transform is not None:
-            img = self.transform(img)
-        return img, img_file
-
-    def __len__(self) -> int:
-        return len(self.img_files)
-
-
-#%% Support functions
-
-def create_loader(cropped_images_dir: str,
-                  detections_json_path: str | None,
-                  img_size: int,
-                  batch_size: int,
-                  num_workers: int
-                  ) -> torch.utils.data.DataLoader:
-    """
-    Creates a DataLoader.
-
-    Args:
-        cropped_images_dir: str, path to image crops
-        detections_json_path: optional str, path to detections JSON
-        img_size: int, resizes smallest side of image to img_size,
-            then center-crops to (img_size, img_size)
-        batch_size: int, batch size in dataloader
-        num_workers: int, # of workers in dataloader
-    """
-    
-    crop_files = []
-
-    if detections_json_path is None:
-        # recursively find all files in cropped_images_dir
-        for subdir, _, files in os.walk(cropped_images_dir):
-            for file_name in files:
-                rel_dir = os.path.relpath(subdir, cropped_images_dir)
-                rel_file = os.path.join(rel_dir, file_name)
-                crop_files.append(rel_file)
-
-    else:
-        # only find crops of images from detections JSON
-        print('Loading detections JSON')
-        with open(detections_json_path, 'r') as f:
-            js = json.load(f)
-        detections = {img['file']: img for img in js['images']}
-        detector_version = js['info']['detector']
-
-        for img_file, info_dict in tqdm(detections.items()):
-            if 'detections' not in info_dict or info_dict['detections'] is None:
-                continue
-            for i in range(len(info_dict['detections'])):
-                crop_filename = img_file + f'___crop{i:02d}_{detector_version}.jpg'
-                crop_path = os.path.join(cropped_images_dir, crop_filename)
-                if os.path.exists(crop_path):
-                    crop_files.append(crop_filename)
-
-    transform = tv.transforms.Compose([
-        # resizes smaller edge to img_size
-        tv.transforms.Resize(img_size, interpolation=PIL.Image.BICUBIC),
-        tv.transforms.CenterCrop(img_size),
-        tv.transforms.ToTensor(),
-        tv.transforms.Normalize(mean=train_classifier.MEANS,
-                                std=train_classifier.STDS, inplace=True)
-    ])
-
-    dataset = SimpleDataset(img_files=crop_files, images_dir=cropped_images_dir,
-                            transform=transform)
-    assert len(dataset) > 0
-    loader = torch.utils.data.DataLoader(
-        dataset, batch_size=batch_size, num_workers=num_workers,
-        pin_memory=True)
-    return loader
-
-
-#%% Main function
-
-def main(model_path: str,
-         cropped_images_dir: str,
-         output_csv_path: str,
-         detections_json_path: str | None,
-         classifier_categories_json_path: str | None,
-         img_size: int,
-         batch_size: int,
-         num_workers: int,
-         device_id: int | None = None) -> None:
-    
-    # Evaluating with accimage is much faster than Pillow or Pillow-SIMD, but accimage
-    # is Linux-only.
-    try:
-        import accimage # noqa
-        tv.set_image_backend('accimage')
-    except:
-        print('Warning: could not start accimage backend (ignore this if you\'re not using Linux)')
-
-    # create dataset
-    print('Creating data loader')
-    loader = create_loader(
-        cropped_images_dir, detections_json_path=detections_json_path,
-        img_size=img_size, batch_size=batch_size, num_workers=num_workers)
-
-    label_names = None
-    if classifier_categories_json_path is not None:
-        with open(classifier_categories_json_path, 'r') as f:
-            categories = json.load(f)
-        label_names = [categories[str(i)] for i in range(len(categories))]
-
-    # create model
-    print('Loading saved model')
-    model = torch.jit.load(model_path)
-    model, device = train_classifier.prep_device(model, device_id=device_id)
-
-    test_epoch(model, loader, device=device, label_names=label_names,
-               output_csv_path=output_csv_path)
-
-
-def test_epoch(model: torch.nn.Module,
-               loader: torch.utils.data.DataLoader,
-               device: torch.device,
-               label_names: Sequence[str] | None,
-               output_csv_path: str) -> None:
-    """
-    Runs for 1 epoch.
-
-    Writes results to the output CSV in batches.
-
-    Args:
-        model: torch.nn.Module
-        loader: torch.utils.data.DataLoader
-        device: torch.device
-        label_names: optional list of str, label names
-        output_csv_path: str
-    """
-    
-    # set dropout and BN layers to eval mode
-    model.eval()
-
-    header = True
-    mode = 'w'  # new file on first write
-
-    with torch.no_grad():
-        for inputs, img_files in tqdm(loader):
-            inputs = inputs.to(device, non_blocking=True)
-            outputs = model(inputs)
-            probs = torch.nn.functional.softmax(outputs, dim=1).cpu().numpy()
-
-            if label_names is None:
-                label_names = [str(i) for i in range(probs.shape[1])]
-
-            df = pd.DataFrame(data=probs, columns=label_names,
-                              index=pd.Index(img_files, name='path'))
-            df.to_csv(output_csv_path, index=True, header=header, mode=mode)
-
-            if header:
-                header = False
-                mode = 'a'
-
-
-#%% Command-line driver
-
-def _parse_args() -> argparse.Namespace:
-
-    parser = argparse.ArgumentParser(
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-        description='Run classifier.')
-    parser.add_argument(
-        'model',
-        help='path to TorchScript compiled model')
-    parser.add_argument(
-        'crops_dir',
-        help='path to directory containing cropped images')
-    parser.add_argument(
-        'output',
-        help='path to save CSV file with classifier results (can use .csv.gz '
-             'extension for compression)')
-    parser.add_argument(
-        '-d', '--detections-json',
-        help='path to detections JSON file, used to filter paths within '
-             'crops_dir')
-    parser.add_argument(
-        '-c', '--classifier-categories',
-        help='path to JSON file for classifier categories. If not given, '
-             'classes are numbered "0", "1", "2", ...')
-    parser.add_argument(
-        '--image-size', type=int, default=224,
-        help='size of input image to model, usually 224px, but may be larger '
-             'especially for EfficientNet models')
-    parser.add_argument(
-        '--batch-size', type=int, default=1,
-        help='batch size for evaluating model')
-    parser.add_argument(
-        '--device', type=int, default=None,
-        help='preferred CUDA device')
-    parser.add_argument(
-        '--num-workers', type=int, default=8,
-        help='# of workers for data loading')
-    return parser.parse_args()
-
-
-if __name__ == '__main__':
-    
-    args = _parse_args()
-    main(model_path=args.model,
-         cropped_images_dir=args.crops_dir,
-         output_csv_path=args.output,
-         detections_json_path=args.detections_json,
-         classifier_categories_json_path=args.classifier_categories,
-         img_size=args.image_size,
-         batch_size=args.batch_size,
-         num_workers=args.num_workers,
-         device_id=args.device)

classification/save_mislabeled.py

@@ -1,110 +0,0 @@
-"""
-
-save_mislabeled.py
-
-Update the list of known mislabeled images in MegaDB.
-
-List of known mislabeled images is stored in Azure Blob Storage.
-* storage account: cameratrapsc
-* container: classifier-training
-* blob: megadb_mislabeled/{dataset}.csv, one file per dataset
-
-Each file megadb_mislabeled/{dataset}.csv has two columns:
-   
-* 'file': str, blob name
-
-* 'correct_class': optional str, correct dataset class
-
-  if empty, indicates that the existing class in MegaDB is inaccurate, but
-  the correct class is unknown.
-
-This script assumes that the classifier-training container is mounted locally.
-
-Takes as input a CSV file (output from Timelapse) with the following columns:
-
-* 'File': str, <blob_basename>
-* 'RelativePath': str, <dataset>\<blob_dirname>
-* 'mislabeled': str, values in ['true', 'false']
-* 'correct_class': either empty or str
-
-"""
-
-#%% Imports
-
-import argparse
-import os
-import pathlib
-
-import pandas as pd
-
-
-#%% Main function
-
-def update_mislabeled_images(container_path: str, input_csv_path: str) -> None:
-    
-    df = pd.read_csv(input_csv_path, index_col=False)
-
-    # error checking
-    assert df['mislabeled'].dtype == bool
-
-    # any row with 'correct_class' should be marked 'mislabeled'
-    tmp = (df['correct_class'].notna() & df['mislabeled']).sum()
-    assert df['correct_class'].notna().sum() == tmp
-
-    # filter to only the mislabeled rows
-    df = df[df['mislabeled']].copy()
-
-    # convert '\' to '/'
-    df['RelativePath'] = df['RelativePath'].map(
-        lambda p: pathlib.PureWindowsPath(p).as_posix())
-    df[['dataset', 'blob_dirname']] = df['RelativePath'].str.split(
-        '/', n=1, expand=True)
-    df['file'] = df['blob_dirname'] + '/' + df['File']
-
-    for ds, ds_df in df.groupby('dataset'):
-        
-        sr_path = os.path.join(container_path, 'megadb_mislabeled', f'{ds}.csv')
-        if os.path.exists(sr_path):
-            old_sr = pd.read_csv(sr_path, index_col='file', squeeze=True)
-        else:
-            old_sr = pd.Series(index=pd.Index([], name='file'),
-                               dtype='str', name='correct_class')
-
-        ds_sr = ds_df.set_index('file', verify_integrity=True)['correct_class']
-
-        # verify that overlapping indices are the same
-        overlap_index = ds_sr.index.intersection(old_sr.index)
-        assert ds_sr.loc[overlap_index].equals(old_sr.loc[overlap_index])
-
-        # "add" any new mislabelings
-        new_indices = ds_sr.index.difference(old_sr.index)
-        new_sr = pd.concat([old_sr, ds_sr.loc[new_indices]],
-                           verify_integrity=True)
-        new_sr.sort_index(inplace=True)
-
-        # write out results
-        new_sr.to_csv(sr_path, index=True)
-
-
-#%% Command-line driver
-
-def _parse_args() -> argparse.Namespace:
-    
-    parser = argparse.ArgumentParser(
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-        description='Merges classification results with Batch Detection API '
-                    'outputs.')
-    parser.add_argument(
-        'container_path',
-        help='path to locally-mounted classifier-training container')
-    parser.add_argument(
-        'input_csv',
-        help='path to CSV file output by Timelapse')
-    return parser.parse_args()
-
-
-if __name__ == '__main__':
-    
-    args = _parse_args()
-    update_mislabeled_images(container_path=args.container_path,
-                             input_csv_path=args.input_csv)