megadetector 5.0.9__py3-none-any.whl → 5.0.11__py3-none-any.whl

classification/evaluate_model.py

@@ -1,520 +0,0 @@
-"""
-
-evaluate_model.py
-
-Evaluate a species classifier.
-
-Currently the implementation of multi-label multi-class classification is
-non-functional.
-
-Outputs the following files:
-
-1) outputs_{split}.csv, one file per split, contains columns:
-    - 'path': str, path to cropped image
-    - 'label': str
-    - 'weight': float
-    - [label names]: float, confidence in each label
-
-2) overall_metrics.csv, contains columns:
-    - 'split': str
-    - 'loss': float, mean per-example loss over entire epoch
-    - 'acc_top{k}': float, accuracy@k over the entire epoch
-    - 'loss_weighted' and 'acc_weighted_top{k}': float, weighted versions
-
-3) confusion_matrices.npz
-    - keys ['train', 'val', 'test']
-    - values are np.ndarray, confusion matrices
-
-4) label_stats.csv, per-label statistics, columns
-    - 'split': str
-    - 'label': str
-    - 'precision': float
-    - 'recall': float
-
-"""
-
-#%% Imports and constants
-
-from __future__ import annotations
-
-import argparse
-from collections.abc import Mapping, Sequence
-import json
-import os
-from pprint import pprint
-from typing import Any, Optional
-
-import numpy as np
-import pandas as pd
-import sklearn.metrics
-import torch
-import torchvision
-import tqdm
-
-from classification import efficientnet, train_classifier
-
-
-#%% Example usage
-
-"""
-    python evaluate_model.py \
-        $BASE_LOGDIR/$LOGDIR/params.json \
-        $BASE_LOGDIR/$LOGDIR/ckpt_XX.pt \
-        --output-dir $BASE_LOGDIR/$LOGDIR \
-        --splits train val test \
-        --batch-size 256
-"""
-
-SPLITS = ['train', 'val', 'test']
-
-
-#%% Support functions
-
-def check_override(params: Mapping[str, Any], key: str,
-                   override: Optional[Any]) -> Any:
-    """
-    Return desired value, with optional override.
-    """
-    
-    if override is None:
-        return params[key]
-    saved = params.get(key, None)
-    print(f'Overriding saved {key}. Saved: {saved}. Override with: {override}.')
-    return override
-
-
-def trace_model(model_name: str, ckpt_path: str, num_classes: int,
-                img_size: int) -> str:
-    """
-    Use TorchScript tracing to compile trained model into standalone file.
-
-    For now, we have to use tracing instead of scripting. See
-        https://github.com/lukemelas/EfficientNet-PyTorch/issues/89
-        https://github.com/lukemelas/EfficientNet-PyTorch/issues/218
-
-    Args:
-        model_name: str
-        ckpt_path: str, path to checkpoint file
-        num_labels: int, number of classification classes
-        img_size: int, size of input image, used for tracing
-
-    Returns: str, name of file for compiled model. For example, if ckpt_path is
-        '/path/to/ckpt_16.pt', then the returned path is
-        '/path/to/ckpt_16_compiled.pt'.
-    """
-    
-    root, ext = os.path.splitext(ckpt_path)
-    compiled_path = root + '_compiled' + ext
-    if os.path.exists(compiled_path):
-        return compiled_path
-
-    model = train_classifier.build_model(model_name, num_classes=num_classes,
-                                         pretrained=ckpt_path, finetune=False)
-    if 'efficientnet' in model_name:
-        model.set_swish(memory_efficient=False)
-    model.eval()
-
-    ex_img = torch.rand(1, 3, img_size, img_size)
-    scripted_model = torch.jit.trace(model, (ex_img,))
-
-    scripted_model.save(compiled_path)
-    print('Saved TorchScript compiled model to', compiled_path)
-    return compiled_path
-
-
-def calc_per_label_stats(cm: np.ndarray, label_names: Sequence[str]
-                         ) -> pd.DataFrame:
-    """
-    Args:
-        cm: np.ndarray, type int, confusion matrix C such that C[i,j] is the #
-            of observations from group i that are predicted to be in group j
-        label_names: list of str, label names in order of label id
-
-    Returns: pd.DataFrame, index 'label', columns ['precision', 'recall']
-        precision values are in [0, 1]
-        recall values are in [0, 1], or np.nan if that label had 0 ground-truth
-            observations
-    """
-    
-    tp = np.diag(cm)  # true positives
-
-    predicted_positives = cm.sum(axis=0, dtype=np.float64)  # tp + fp
-    predicted_positives[predicted_positives == 0] += 1e-8
-
-    all_positives = cm.sum(axis=1, dtype=np.float64)  # tp + fn
-    all_positives[all_positives == 0] = np.nan
-
-    df = pd.DataFrame()
-    df['label'] = label_names
-    df['precision'] = tp / predicted_positives
-    df['recall'] = tp / all_positives
-    df.set_index('label', inplace=True)
-    return df
-
-
-def test_epoch(model: torch.nn.Module,
-               loader: torch.utils.data.DataLoader,
-               weighted: bool,
-               device: torch.device,
-               label_names: Sequence[str],
-               top: Sequence[int] = (1, 3),
-               loss_fn: Optional[torch.nn.Module] = None,
-               target_mapping: Mapping[int, Sequence[int]] = None
-               ) -> tuple[pd.DataFrame, pd.Series, np.ndarray]:
-    """
-    Runs for 1 epoch.
-
-    Args:
-        model: torch.nn.Module
-        loader: torch.utils.data.DataLoader
-        weighted: bool, whether to calculate weighted accuracy statistics
-        device: torch.device
-        label_names: list of str, label names in order of label id
-        top: tuple of int, list of values of k for calculating top-K accuracy
-        loss_fn: optional loss function, calculates per-example loss
-        target_mapping: optional dict, label_id => list of ids from classifier
-            that should map to the label_id
-
-    Returns:
-        df: pd.DataFrame, columns ['img_file', 'label', 'weight', label_names]
-        metrics: pd.Series, type float, index includes:
-            'loss': mean per-example loss over entire epoch,
-                only included if loss_fn is not None
-            'acc_top{k}': accuracy@k over the entire epoch
-            'loss_weighted' and 'acc_weighted_top{k}': weighted versions, only
-                included if weighted=True
-        cm: np.ndarray, confusion matrix C such that C[i,j] is the # of
-            observations known to be in group i and predicted to be in group j
-    """
-    
-    # set dropout and BN layers to eval mode
-    model.eval()
-
-    if loss_fn is not None:
-        losses = train_classifier.AverageMeter()
-    accuracies_topk = {k: train_classifier.AverageMeter() for k in top}  # acc@k
-    if weighted:
-        accs_weighted = {k: train_classifier.AverageMeter() for k in top}
-        losses_weighted = train_classifier.AverageMeter()
-
-    num_examples = len(loader.dataset)
-    num_labels = len(label_names)
-
-    all_img_files = []
-    all_probs = np.zeros([num_examples, len(label_names)], dtype=np.float32)
-    all_labels = np.zeros(num_examples, dtype=np.int32)
-    if weighted:
-        all_weights = np.zeros(num_examples, dtype=np.float32)
-
-    batch_slice = slice(0, 0)
-    tqdm_loader = tqdm.tqdm(loader)
-    with torch.no_grad():
-        for batch in tqdm_loader:
-            if weighted:
-                inputs, labels, img_files, weights = batch
-            else:
-                # even if batch contains sample weights, don't use them
-                inputs, labels, img_files = batch[0:3]
-                weights = None
-
-            all_img_files.append(img_files)
-
-            batch_size = labels.size(0)
-            batch_slice = slice(batch_slice.stop, batch_slice.stop + batch_size)
-            all_labels[batch_slice] = labels
-            if weighted:
-                all_weights[batch_slice] = weights
-                weights = weights.to(device, non_blocking=True)
-
-            inputs = inputs.to(device, non_blocking=True)
-            labels = labels.to(device, non_blocking=True)
-            outputs = model(inputs)
-
-            # Do target mapping on the outputs (unnormalized logits) instead of
-            # the normalized (softmax) probabilities, because the loss function
-            # uses unnormalized logits. Summing probabilities is equivalent to
-            # log-sum-exp of unnormalized logits.
-            if target_mapping is not None:
-                outputs_mapped = torch.zeros(
-                    [batch_size, num_labels], dtype=outputs.dtype,
-                    device=outputs.device)
-                for target, cols in target_mapping.items():
-                    outputs_mapped[:, target] = torch.logsumexp(
-                        outputs[:, cols], dim=1)
-                outputs = outputs_mapped
-
-            probs = torch.nn.functional.softmax(outputs, dim=1).cpu()
-            all_probs[batch_slice] = probs
-
-            desc = []
-            if loss_fn is not None:
-                loss = loss_fn(outputs, labels)
-                losses.update(loss.mean().item(), n=batch_size)
-                desc.append(f'Loss {losses.val:.3f} ({losses.avg:.3f})')
-                if weights is not None:
-                    loss_weighted = (loss * weights).mean()
-                    losses_weighted.update(loss_weighted.item(), n=batch_size)
-
-            top_correct = train_classifier.correct(
-                outputs, labels, weights=None, top=top)
-            for k, acc in accuracies_topk.items():
-                acc.update(top_correct[k] * (100. / batch_size), n=batch_size)
-                desc.append(f'Acc@{k} {acc.val:.2f} ({acc.avg:.2f})')
-
-            if weighted:
-                top_correct = train_classifier.correct(
-                    outputs, labels, weights=weights, top=top)
-                for k, acc in accs_weighted.items():
-                    acc.update(top_correct[k] * (100. / batch_size),
-                               n=batch_size)
-                    desc.append(f'Acc_w@{k} {acc.val:.2f} ({acc.avg:.2f})')
-
-            tqdm_loader.set_description(' '.join(desc))
-
-    # a confusion matrix C is such that C[i,j] is the # of observations known to
-    # be in group i and predicted to be in group j.
-    all_preds = all_probs.argmax(axis=1)
-    cm = sklearn.metrics.confusion_matrix(
-        y_true=all_labels, y_pred=all_preds, labels=np.arange(num_labels))
-
-    df = pd.DataFrame()
-    df['path'] = np.concatenate(all_img_files)
-    df['label'] = list(map(label_names.__getitem__, all_labels))
-    df['weight'] = all_weights
-    df[label_names] = all_probs
-
-    metrics = {}
-    if loss_fn is not None:
-        metrics['loss'] = losses.avg
-        if weighted:
-            metrics['loss_weighted'] = losses_weighted.avg
-    for k, acc in accuracies_topk.items():
-        metrics[f'acc_top{k}'] = acc.avg
-    if weighted:
-        for k, acc in accs_weighted.items():
-            metrics[f'acc_weighted_top{k}'] = acc.avg
-    return df, pd.Series(metrics), cm
-
-
-#%% Main function
-
-def main(params_json_path: str, ckpt_path: str, output_dir: str,
-         splits: Sequence[str], target_mapping_json_path: Optional[str] = None,
-         label_index_json_path: Optional[str] = None,
-         **kwargs: Any) -> None:
-
-    # input validation
-    assert os.path.exists(params_json_path)
-    assert os.path.exists(ckpt_path)
-    assert (target_mapping_json_path is None) == (label_index_json_path is None)
-    if target_mapping_json_path is not None:
-        assert label_index_json_path is not None
-        assert os.path.exists(target_mapping_json_path)
-        assert os.path.exists(label_index_json_path)
-
-    # Evaluating with accimage is much faster than Pillow or Pillow-SIMD, but accimage
-    # is Linux-only.
-    try:
-        import accimage # noqa
-        torchvision.set_image_backend('accimage')
-    except:
-        print('Warning: could not start accimage backend (ignore this if you\'re not using Linux)')
-
-    # create output directory
-    if not os.path.exists(output_dir):
-        print('Creating output directory:', output_dir)
-        os.makedirs(output_dir, exist_ok=True)
-
-    with open(params_json_path, 'r') as f:
-        params = json.load(f)
-    pprint(params)
-
-    # override saved params with kwargs
-    for key, new in kwargs.items():
-        if new is None:
-            continue
-        if key in params:
-            saved = params[key]
-            print(f'Overriding saved {key}. Saved: {saved}. '
-                  f'Override with: {new}.')
-        else:
-            print(f'Did not find {key} in saved params. Using value {new}.')
-        params[key] = new
-
-    model_name: str = params['model_name']
-    dataset_dir: str = params['dataset_dir']
-
-    if 'efficientnet' in model_name:
-        img_size = efficientnet.EfficientNet.get_image_size(model_name)
-    else:
-        img_size = 224
-
-    # For now, we don't weight crops by detection confidence during
-    # evaluation. But consider changing this.
-    print('Creating dataloaders')
-    loaders, label_names = train_classifier.create_dataloaders(
-        dataset_csv_path=os.path.join(dataset_dir, 'classification_ds.csv'),
-        label_index_json_path=os.path.join(dataset_dir, 'label_index.json'),
-        splits_json_path=os.path.join(dataset_dir, 'splits.json'),
-        cropped_images_dir=params['cropped_images_dir'],
-        img_size=img_size,
-        multilabel=params['multilabel'],
-        label_weighted=params['label_weighted'],
-        weight_by_detection_conf=False,
-        batch_size=params['batch_size'],
-        num_workers=params['num_workers'],
-        augment_train=False)
-    num_labels = len(label_names)
-
-    # create model, compile with TorchScript if given checkpoint is not compiled
-    print('Loading model from checkpoint')
-    try:
-        model = torch.jit.load(ckpt_path, map_location='cpu')
-    except RuntimeError:
-        compiled_path = trace_model(model_name, ckpt_path, num_labels, img_size)
-        model = torch.jit.load(compiled_path, map_location='cpu')
-    model, device = train_classifier.prep_device(model)
-
-    if len(splits) == 0:
-        print('No splits given! Exiting.')
-        return
-
-    target_cols_map = None
-    if target_mapping_json_path is not None:
-        assert label_index_json_path is not None
-
-        # verify that target names matches original "label names" from dataset
-        with open(target_mapping_json_path, 'r') as f:
-            target_names_map = json.load(f)
-        target_names = set(target_names_map.keys())
-
-        # if the dataset does not already have a 'other' category, then the
-        # 'other' category must come last in label_names to avoid conflicting
-        # with an existing label_id
-        if target_names != set(label_names):
-            assert target_names == set(label_names) | {'other'}
-            label_names.append('other')
-
-            with open(os.path.join(output_dir, 'label_index.json'), 'w') as f:
-                json.dump(dict(enumerate(label_names)), f)
-
-        with open(label_index_json_path, 'r') as f:
-            idx_to_label = json.load(f)
-        classifier_name_to_idx = {
-            idx_to_label[str(k)]: k for k in range(len(idx_to_label))
-        }
-
-        target_cols_map = {}
-        for i_target, label_name in enumerate(label_names):
-            classifier_names = target_names_map[label_name]
-            target_cols_map[i_target] = [
-                classifier_name_to_idx[classifier_name]
-                for classifier_name in classifier_names
-            ]
-
-    # define loss function (criterion)
-    loss_fn: torch.nn.Module
-    if params['multilabel']:
-        loss_fn = torch.nn.BCEWithLogitsLoss(reduction='none').to(device)
-    else:
-        loss_fn = torch.nn.CrossEntropyLoss(reduction='none').to(device)
-
-    split_metrics = {}
-    split_label_stats = {}
-    cms = {}
-    for split in splits:
-        print(f'Evaluating {split}...')
-        df, metrics, cm = test_epoch(
-            model, loaders[split], weighted=True, device=device,
-            label_names=label_names, loss_fn=loss_fn,
-            target_mapping=target_cols_map)
-
-        # this file ends up being huge, so we GZIP compress it
-        output_csv_path = os.path.join(output_dir, f'outputs_{split}.csv.gz')
-        df.to_csv(output_csv_path, index=False, compression='gzip')
-
-        split_metrics[split] = metrics
-        cms[split] = cm
-        split_label_stats[split] = calc_per_label_stats(cm, label_names)
-
-        # double check that the accuracy metrics are computed properly
-        preds = df[label_names].to_numpy().argmax(axis=1)
-        preds = np.asarray(label_names)[preds]
-        assert np.isclose(metrics['acc_top1'] / 100.,
-                          sum(preds == df['label']) / len(df))
-        assert np.isclose(metrics['acc_weighted_top1'] / 100.,
-                          sum((preds == df['label']) * df['weight']) / len(df))
-
-    metrics_df = pd.concat(split_metrics, names=['split']).unstack(level=1)
-    metrics_df.to_csv(os.path.join(output_dir, 'overall_metrics.csv'))
-
-    # save the confusion matrices to .npz
-    npz_path = os.path.join(output_dir, 'confusion_matrices.npz')
-    np.savez_compressed(npz_path, **cms)
-
-    # save per-label statistics
-    label_stats_df = pd.concat(
-        split_label_stats, names=['split', 'label']).reset_index()
-    label_stats_csv_path = os.path.join(output_dir, 'label_stats.csv')
-    label_stats_df.to_csv(label_stats_csv_path, index=False)
-
-
-#%% Command-line driver
-
-def _parse_args() -> argparse.Namespace:
-
-    parser = argparse.ArgumentParser(
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-        description='Evaluate trained model.')
-    parser.add_argument(
-        'params_json',
-        help='path to params.json')
-    parser.add_argument(
-        'ckpt_path',
-        help='path to checkpoint file (normal or TorchScript-compiled)')
-    parser.add_argument(
-        '-o', '--output-dir', required=True,
-        help='(required) path to output directory')
-    parser.add_argument(
-        '--splits', nargs='*', choices=SPLITS, default=[],
-        help='which splits to evaluate model on. If no splits are given, then '
-             'only compiles normal checkpoint file using TorchScript without '
-             'actually running the model.')
-
-    other_classifier = parser.add_argument_group(
-        'arguments for evaluating a model (e.g., MegaClassifier) on a '
-        'different set of labels than what the model was trained on')
-    other_classifier.add_argument(
-        '--target-mapping',
-        help='path to JSON file mapping target categories to classifier labels')
-    other_classifier.add_argument(
-        '--label-index',
-        help='path to label index JSON file for classifier')
-
-    override_group = parser.add_argument_group(
-        'optional arguments to override values in params_json')
-    override_group.add_argument(
-        '--model-name',
-        help='which EfficientNet or Resnet model')
-    override_group.add_argument(
-        '--batch-size', type=int,
-        help='batch size for evaluating model')
-    override_group.add_argument(
-        '--num-workers', type=int,
-        help='number of workers for data loading')
-    override_group.add_argument(
-        '--dataset-dir',
-        help='path to directory containing classification_ds.csv, '
-             'label_index.json, and splits.json')
-    return parser.parse_args()
-
-
-if __name__ == '__main__':
-    
-    args = _parse_args()
-    main(params_json_path=args.params_json, ckpt_path=args.ckpt_path,
-         output_dir=args.output_dir, splits=args.splits,
-         target_mapping_json_path=args.target_mapping,
-         label_index_json_path=args.label_index,
-         model_name=args.model_name, batch_size=args.batch_size,
-         num_workers=args.num_workers, dataset_dir=args.dataset_dir)

classification/identify_mislabeled_candidates.py

@@ -1,152 +0,0 @@
-"""
-
-identify_mislabeled_candidates.py
-
-Identify images that may have been mislabeled.
-
-A "mislabeled candidate" is defined as an image meeting both criteria:
-    
-* according to the ground-truth label, the model made an incorrect prediction
-
-* the model's prediction confidence exceeds its confidence for the ground-truth
-  label by at least <margin>
-
-This script outputs for each dataset a text file containing the filenames of
-mislabeled candidates, one per line. The text files are saved to:
-   
-    <logdir>/mislabeled_candidates_{split}_{dataset}.txt
-
-To this list of files can then be passed to AzCopy to be downloaded:
-
-""  
-azcopy cp "http://<url_of_container>?<sas_token>" "/save/files/here" \
-       --list-of-files "/path/to/mislabeled_candidates_{split}_{dataset}.txt"
-""
-
-To save the filename as <dataset_name>/<blob_name> (instead of just <blob_name>
-by default), pass the --include-dataset-in-filename flag. Then, the images can
-be downloaded with:
-
-""
-  python data_management/megadb/download_images.py txt \
-      "/path/to/mislabeled_candidates_{split}_{dataset}.txt" \
-      /save/files/here \
-      --threads 50
-""
-
-Assumes the following directory layout:
-    <base_logdir>/
-        label_index.json
-        <logdir>/
-            outputs_{split}.csv.gz
-
-"""
-
-#%% Imports
-
-from __future__ import annotations
-
-import argparse
-from collections import defaultdict
-from collections.abc import Iterable, Sequence
-import json
-import os
-
-import pandas as pd
-from tqdm import tqdm
-
-
-#%% Example usage
-
-"""
-    python identify_mislabeled_candidates.py <base_logdir>/<logdir> \
-        --margin 0.5 --splits val test
-"""
-
-
-#%% Main function
-
-def main(logdir: str, splits: Iterable[str], margin: float,
-         include_dataset_in_filename: bool) -> None:
-    
-    # load files
-    logdir = os.path.normpath(logdir)  # removes any trailing slash
-    base_logdir = os.path.dirname(logdir)
-    idx_to_label_json_path = os.path.join(base_logdir, 'label_index.json')
-    with open(idx_to_label_json_path, 'r') as f:
-        idx_to_label = json.load(f)
-    label_names = [idx_to_label[str(i)] for i in range(len(idx_to_label))]
-
-    for split in splits:
-        outputs_csv_path = os.path.join(logdir, f'outputs_{split}.csv.gz')
-        candidates_df = get_candidates_df(outputs_csv_path, label_names, margin)
-
-        # dataset => set of img_file
-        candidate_image_files: defaultdict[str, set[str]] = defaultdict(set)
-
-        for crop_path in tqdm(candidates_df['path']):
-            # crop_path: <dataset>/<img_file>___cropXX_mdvY.Y.jpg
-            #            [----<img_path>----]
-            img_path = crop_path.split('___crop')[0]
-            ds, img_file = img_path.split('/', maxsplit=1)
-            if include_dataset_in_filename:
-                candidate_image_files[ds].add(img_path)
-            else:
-                candidate_image_files[ds].add(img_file)
-
-        for ds in sorted(candidate_image_files.keys()):
-            img_files = candidate_image_files[ds]
-            print(f'{ds} contains {len(img_files)} mislabeled candidates.')
-            save_path = os.path.join(
-                logdir, f'mislabeled_candidates_{split}_{ds}.txt')
-            with open(save_path, 'w') as f:
-                for img_file in sorted(img_files):
-                    f.write(img_file + '\n')
-
-
-#%% Support functions
-
-def get_candidates_df(outputs_csv_path: str, label_names: Sequence[str],
-                      margin: float) -> pd.DataFrame:
-    """
-    Returns a DataFrame containing crops only from mislabeled candidate
-    images.
-    """
-    
-    df = pd.read_csv(outputs_csv_path, float_precision='high')
-    all_rows = range(len(df))
-    df['pred'] = df[label_names].idxmax(axis=1)
-    df['pred_conf'] = df.lookup(row_labels=all_rows, col_labels=df['pred'])
-    df['label_conf'] = df.lookup(row_labels=all_rows, col_labels=df['label'])
-    candidate_mask = df['pred_conf'] >= df['label_conf'] + margin
-    candidates_df = df[candidate_mask].copy()
-    return candidates_df
-
-
-#%% Command-line driver
-
-def _parse_args() -> argparse.Namespace:
-
-    parser = argparse.ArgumentParser(
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-        description='Identify mislabeled candidate images.')
-    parser.add_argument(
-        'logdir',
-        help='folder inside <base_logdir> containing `outputs_<split>.csv.gz`')
-    parser.add_argument(
-        '--margin', type=float, default=0.5,
-        help='confidence margin to count as a mislabeled candidate')
-    parser.add_argument(
-        '--splits', nargs='+', choices=['train', 'val', 'test'],
-        help='which splits to identify mislabeled candidates on')
-    parser.add_argument(
-        '-d', '--include-dataset-in-filename', action='store_true',
-        help='save filename as <dataset_name>/<blob_name>')
-    return parser.parse_args()
-
-
-if __name__ == '__main__':
-    
-    args = _parse_args()
-    main(logdir=args.logdir, splits=args.splits, margin=args.margin,
-         include_dataset_in_filename=args.include_dataset_in_filename)