vismatch 1.1.1__py3-none-any.whl

vismatch/third_party/keypt2subpx/submodules/glue_factory/gluefactory/eval/eth3d.py

@@ -0,0 +1,202 @@
+from collections import defaultdict
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+from omegaconf import OmegaConf
+from tqdm import tqdm
+
+from ..datasets import get_dataset
+from ..models.cache_loader import CacheLoader
+from ..settings import EVAL_PATH
+from ..utils.export_predictions import export_predictions
+from .eval_pipeline import EvalPipeline, load_eval
+from .io import get_eval_parser, load_model, parse_eval_args
+from .utils import aggregate_pr_results, get_tp_fp_pts
+
+
+def eval_dataset(loader, pred_file, suffix=""):
+    results = defaultdict(list)
+    results["num_pos" + suffix] = 0
+    cache_loader = CacheLoader({"path": str(pred_file), "collate": None}).eval()
+    for data in tqdm(loader):
+        pred = cache_loader(data)
+
+        if suffix == "":
+            scores = pred["matching_scores0"].numpy()
+            sort_indices = np.argsort(scores)[::-1]
+            gt_matches = pred["gt_matches0"].numpy()[sort_indices]
+            pred_matches = pred["matches0"].numpy()[sort_indices]
+        else:
+            scores = pred["line_matching_scores0"].numpy()
+            sort_indices = np.argsort(scores)[::-1]
+            gt_matches = pred["gt_line_matches0"].numpy()[sort_indices]
+            pred_matches = pred["line_matches0"].numpy()[sort_indices]
+        scores = scores[sort_indices]
+
+        tp, fp, scores, num_pos = get_tp_fp_pts(pred_matches, gt_matches, scores)
+        results["tp" + suffix].append(tp)
+        results["fp" + suffix].append(fp)
+        results["scores" + suffix].append(scores)
+        results["num_pos" + suffix] += num_pos
+
+    # Aggregate the results
+    return aggregate_pr_results(results, suffix=suffix)
+
+
+class ETH3DPipeline(EvalPipeline):
+    default_conf = {
+        "data": {
+            "name": "eth3d",
+            "batch_size": 1,
+            "train_batch_size": 1,
+            "val_batch_size": 1,
+            "test_batch_size": 1,
+            "num_workers": 16,
+        },
+        "model": {
+            "name": "gluefactory.models.two_view_pipeline",
+            "ground_truth": {
+                "name": "gluefactory.models.matchers.depth_matcher",
+                "use_lines": False,
+            },
+            "run_gt_in_forward": True,
+        },
+        "eval": {"plot_methods": [], "plot_line_methods": [], "eval_lines": False},
+    }
+
+    export_keys = [
+        "gt_matches0",
+        "matches0",
+        "matching_scores0",
+    ]
+
+    optional_export_keys = [
+        "gt_line_matches0",
+        "line_matches0",
+        "line_matching_scores0",
+    ]
+
+    def get_dataloader(self, data_conf=None):
+        data_conf = data_conf if data_conf is not None else self.default_conf["data"]
+        dataset = get_dataset("eth3d")(data_conf)
+        return dataset.get_data_loader("test")
+
+    def get_predictions(self, experiment_dir, model=None, overwrite=False):
+        pred_file = experiment_dir / "predictions.h5"
+        if not pred_file.exists() or overwrite:
+            if model is None:
+                model = load_model(self.conf.model, self.conf.checkpoint)
+            export_predictions(
+                self.get_dataloader(self.conf.data),
+                model,
+                pred_file,
+                keys=self.export_keys,
+                optional_keys=self.optional_export_keys,
+            )
+        return pred_file
+
+    def run_eval(self, loader, pred_file):
+        eval_conf = self.conf.eval
+        r = eval_dataset(loader, pred_file)
+        if self.conf.eval.eval_lines:
+            r.update(eval_dataset(loader, pred_file, conf=eval_conf, suffix="_lines"))
+        s = {}
+
+        return s, {}, r
+
+
+def plot_pr_curve(
+    models_name, results, dst_file="eth3d_pr_curve.pdf", title=None, suffix=""
+):
+    plt.figure()
+    f_scores = np.linspace(0.2, 0.9, num=8)
+    for f_score in f_scores:
+        x = np.linspace(0.01, 1)
+        y = f_score * x / (2 * x - f_score)
+        plt.plot(x[y >= 0], y[y >= 0], color=[0, 0.5, 0], alpha=0.3)
+        plt.annotate(
+            "f={0:0.1}".format(f_score),
+            xy=(0.9, y[45] + 0.02),
+            alpha=0.4,
+            fontsize=14,
+        )
+
+    plt.rcParams.update({"font.size": 12})
+    # plt.rc('legend', fontsize=10)
+    plt.grid(True)
+    plt.axis([0.0, 1.0, 0.0, 1.0])
+    plt.xticks(np.arange(0, 1.05, step=0.1), fontsize=16)
+    plt.xlabel("Recall", fontsize=18)
+    plt.ylabel("Precision", fontsize=18)
+    plt.yticks(np.arange(0, 1.05, step=0.1), fontsize=16)
+    plt.ylim([0.3, 1.0])
+    prop_cycle = plt.rcParams["axes.prop_cycle"]
+    colors = prop_cycle.by_key()["color"]
+    for m, c in zip(models_name, colors):
+        sAP_string = f'{m}: {results[m]["AP" + suffix]:.1f}'
+        plt.plot(
+            results[m]["curve_recall" + suffix],
+            results[m]["curve_precision" + suffix],
+            label=sAP_string,
+            color=c,
+        )
+
+    plt.legend(fontsize=16, loc="lower right")
+    if title:
+        plt.title(title)
+
+    plt.tight_layout(pad=0.5)
+    print(f"Saving plot to: {dst_file}")
+    plt.savefig(dst_file)
+    plt.show()
+
+
+if __name__ == "__main__":
+    dataset_name = Path(__file__).stem
+    parser = get_eval_parser()
+    args = parser.parse_intermixed_args()
+
+    default_conf = OmegaConf.create(ETH3DPipeline.default_conf)
+
+    # mingle paths
+    output_dir = Path(EVAL_PATH, dataset_name)
+    output_dir.mkdir(exist_ok=True, parents=True)
+
+    name, conf = parse_eval_args(
+        dataset_name,
+        args,
+        "configs/",
+        default_conf,
+    )
+
+    experiment_dir = output_dir / name
+    experiment_dir.mkdir(exist_ok=True)
+
+    pipeline = ETH3DPipeline(conf)
+    s, f, r = pipeline.run(
+        experiment_dir, overwrite=args.overwrite, overwrite_eval=args.overwrite_eval
+    )
+
+    # print results
+    for k, v in r.items():
+        if k.startswith("AP"):
+            print(f"{k}: {v:.2f}")
+
+    if args.plot:
+        results = {}
+        for m in conf.eval.plot_methods:
+            exp_dir = output_dir / m
+            results[m] = load_eval(exp_dir)[1]
+
+        plot_pr_curve(conf.eval.plot_methods, results, dst_file="eth3d_pr_curve.pdf")
+        if conf.eval.eval_lines:
+            for m in conf.eval.plot_line_methods:
+                exp_dir = output_dir / m
+                results[m] = load_eval(exp_dir)[1]
+            plot_pr_curve(
+                conf.eval.plot_line_methods,
+                results,
+                dst_file="eth3d_pr_curve_lines.pdf",
+                suffix="_lines",
+            )

vismatch/third_party/keypt2subpx/submodules/glue_factory/gluefactory/eval/eval_pipeline.py

@@ -0,0 +1,109 @@
+import json
+
+import h5py
+import numpy as np
+from omegaconf import OmegaConf
+
+
+def load_eval(dir):
+    summaries, results = {}, {}
+    with h5py.File(str(dir / "results.h5"), "r") as hfile:
+        for k in hfile.keys():
+            r = np.array(hfile[k])
+            if len(r.shape) < 3:
+                results[k] = r
+        for k, v in hfile.attrs.items():
+            summaries[k] = v
+    with open(dir / "summaries.json", "r") as f:
+        s = json.load(f)
+    summaries = {k: v if v is not None else np.nan for k, v in s.items()}
+    return summaries, results
+
+
+def save_eval(dir, summaries, figures, results):
+    with h5py.File(str(dir / "results.h5"), "w") as hfile:
+        for k, v in results.items():
+            arr = np.array(v)
+            if not np.issubdtype(arr.dtype, np.number):
+                arr = arr.astype("object")
+            hfile.create_dataset(k, data=arr)
+        # just to be safe, not used in practice
+        for k, v in summaries.items():
+            hfile.attrs[k] = v
+    s = {
+        k: float(v) if np.isfinite(v) else None
+        for k, v in summaries.items()
+        if not isinstance(v, list)
+    }
+    s = {**s, **{k: v for k, v in summaries.items() if isinstance(v, list)}}
+    with open(dir / "summaries.json", "w") as f:
+        json.dump(s, f, indent=4)
+
+    for fig_name, fig in figures.items():
+        fig.savefig(dir / f"{fig_name}.png")
+
+
+def exists_eval(dir):
+    return (dir / "results.h5").exists() and (dir / "summaries.json").exists()
+
+
+class EvalPipeline:
+    default_conf = {}
+
+    export_keys = []
+    optional_export_keys = []
+
+    def __init__(self, conf):
+        """Assumes"""
+        self.default_conf = OmegaConf.create(self.default_conf)
+        self.conf = OmegaConf.merge(self.default_conf, conf)
+        self._init(self.conf)
+
+    def _init(self, conf):
+        pass
+
+    @classmethod
+    def get_dataloader(self, data_conf=None):
+        """Returns a data loader with samples for each eval datapoint"""
+        raise NotImplementedError
+
+    def get_predictions(self, experiment_dir, model=None, overwrite=False):
+        """Export a prediction file for each eval datapoint"""
+        raise NotImplementedError
+
+    def run_eval(self, loader, pred_file):
+        """Run the eval on cached predictions"""
+        raise NotImplementedError
+
+    def run(self, experiment_dir, model=None, overwrite=False, overwrite_eval=False):
+        """Run export+eval loop"""
+        self.save_conf(
+            experiment_dir, overwrite=overwrite, overwrite_eval=overwrite_eval
+        )
+        pred_file = self.get_predictions(
+            experiment_dir, model=model, overwrite=overwrite
+        )
+
+        f = {}
+        if not exists_eval(experiment_dir) or overwrite_eval or overwrite:
+            s, f, r = self.run_eval(self.get_dataloader(), pred_file)
+            save_eval(experiment_dir, s, f, r)
+        s, r = load_eval(experiment_dir)
+        return s, f, r
+
+    def save_conf(self, experiment_dir, overwrite=False, overwrite_eval=False):
+        # store config
+        conf_output_path = experiment_dir / "conf.yaml"
+        if conf_output_path.exists():
+            saved_conf = OmegaConf.load(conf_output_path)
+            if (saved_conf.data != self.conf.data) or (
+                saved_conf.model != self.conf.model
+            ):
+                assert (
+                    overwrite
+                ), "configs changed, add --overwrite to rerun experiment with new conf"
+            if saved_conf.eval != self.conf.eval:
+                assert (
+                    overwrite or overwrite_eval
+                ), "eval configs changed, add --overwrite_eval to rerun evaluation"
+        OmegaConf.save(self.conf, experiment_dir / "conf.yaml")

vismatch/third_party/keypt2subpx/submodules/glue_factory/gluefactory/eval/hpatches.py

@@ -0,0 +1,203 @@
+from collections import defaultdict
+from collections.abc import Iterable
+from pathlib import Path
+from pprint import pprint
+
+import matplotlib.pyplot as plt
+import numpy as np
+import torch
+from omegaconf import OmegaConf
+from tqdm import tqdm
+
+from ..datasets import get_dataset
+from ..models.cache_loader import CacheLoader
+from ..settings import EVAL_PATH
+from ..utils.export_predictions import export_predictions
+from ..utils.tensor import map_tensor
+from ..utils.tools import AUCMetric
+from ..visualization.viz2d import plot_cumulative
+from .eval_pipeline import EvalPipeline
+from .io import get_eval_parser, load_model, parse_eval_args
+from .utils import (
+    eval_homography_dlt,
+    eval_homography_robust,
+    eval_matches_homography,
+    eval_poses,
+)
+
+
+class HPatchesPipeline(EvalPipeline):
+    default_conf = {
+        "data": {
+            "batch_size": 1,
+            "name": "hpatches",
+            "num_workers": 16,
+            "preprocessing": {
+                "resize": 480,  # we also resize during eval to have comparable metrics
+                "side": "short",
+            },
+        },
+        "model": {
+            "ground_truth": {
+                "name": None,  # remove gt matches
+            }
+        },
+        "eval": {
+            "estimator": "poselib",
+            "ransac_th": 1.0,  # -1 runs a bunch of thresholds and selects the best
+        },
+    }
+    export_keys = [
+        "keypoints0",
+        "keypoints1",
+        "keypoint_scores0",
+        "keypoint_scores1",
+        "matches0",
+        "matches1",
+        "matching_scores0",
+        "matching_scores1",
+    ]
+
+    optional_export_keys = [
+        "lines0",
+        "lines1",
+        "orig_lines0",
+        "orig_lines1",
+        "line_matches0",
+        "line_matches1",
+        "line_matching_scores0",
+        "line_matching_scores1",
+    ]
+
+    def _init(self, conf):
+        pass
+
+    @classmethod
+    def get_dataloader(self, data_conf=None):
+        data_conf = data_conf if data_conf else self.default_conf["data"]
+        dataset = get_dataset("hpatches")(data_conf)
+        return dataset.get_data_loader("test")
+
+    def get_predictions(self, experiment_dir, model=None, overwrite=False):
+        pred_file = experiment_dir / "predictions.h5"
+        if not pred_file.exists() or overwrite:
+            if model is None:
+                model = load_model(self.conf.model, self.conf.checkpoint)
+            export_predictions(
+                self.get_dataloader(self.conf.data),
+                model,
+                pred_file,
+                keys=self.export_keys,
+                optional_keys=self.optional_export_keys,
+            )
+        return pred_file
+
+    def run_eval(self, loader, pred_file):
+        assert pred_file.exists()
+        results = defaultdict(list)
+
+        conf = self.conf.eval
+
+        test_thresholds = (
+            ([conf.ransac_th] if conf.ransac_th > 0 else [0.5, 1.0, 1.5, 2.0, 2.5, 3.0])
+            if not isinstance(conf.ransac_th, Iterable)
+            else conf.ransac_th
+        )
+        pose_results = defaultdict(lambda: defaultdict(list))
+        cache_loader = CacheLoader({"path": str(pred_file), "collate": None}).eval()
+        for i, data in enumerate(tqdm(loader)):
+            pred = cache_loader(data)
+            # Remove batch dimension
+            data = map_tensor(data, lambda t: torch.squeeze(t, dim=0))
+            # add custom evaluations here
+            if "keypoints0" in pred:
+                results_i = eval_matches_homography(data, pred)
+                results_i = {**results_i, **eval_homography_dlt(data, pred)}
+            else:
+                results_i = {}
+            for th in test_thresholds:
+                pose_results_i = eval_homography_robust(
+                    data,
+                    pred,
+                    {"estimator": conf.estimator, "ransac_th": th},
+                )
+                [pose_results[th][k].append(v) for k, v in pose_results_i.items()]
+
+            # we also store the names for later reference
+            results_i["names"] = data["name"][0]
+            results_i["scenes"] = data["scene"][0]
+
+            for k, v in results_i.items():
+                results[k].append(v)
+
+        # summarize results as a dict[str, float]
+        # you can also add your custom evaluations here
+        summaries = {}
+        for k, v in results.items():
+            arr = np.array(v)
+            if not np.issubdtype(np.array(v).dtype, np.number):
+                continue
+            summaries[f"m{k}"] = round(np.median(arr), 3)
+
+        auc_ths = [1, 3, 5]
+        best_pose_results, best_th = eval_poses(
+            pose_results, auc_ths=auc_ths, key="H_error_ransac", unit="px"
+        )
+        if "H_error_dlt" in results.keys():
+            dlt_aucs = AUCMetric(auc_ths, results["H_error_dlt"]).compute()
+            for i, ath in enumerate(auc_ths):
+                summaries[f"H_error_dlt@{ath}px"] = dlt_aucs[i]
+
+        results = {**results, **pose_results[best_th]}
+        summaries = {
+            **summaries,
+            **best_pose_results,
+        }
+
+        figures = {
+            "homography_recall": plot_cumulative(
+                {
+                    "DLT": results["H_error_dlt"],
+                    self.conf.eval.estimator: results["H_error_ransac"],
+                },
+                [0, 10],
+                unit="px",
+                title="Homography ",
+            )
+        }
+
+        return summaries, figures, results
+
+
+if __name__ == "__main__":
+    dataset_name = Path(__file__).stem
+    parser = get_eval_parser()
+    args = parser.parse_intermixed_args()
+
+    default_conf = OmegaConf.create(HPatchesPipeline.default_conf)
+
+    # mingle paths
+    output_dir = Path(EVAL_PATH, dataset_name)
+    output_dir.mkdir(exist_ok=True, parents=True)
+
+    name, conf = parse_eval_args(
+        dataset_name,
+        args,
+        "configs/",
+        default_conf,
+    )
+
+    experiment_dir = output_dir / name
+    experiment_dir.mkdir(exist_ok=True)
+
+    pipeline = HPatchesPipeline(conf)
+    s, f, r = pipeline.run(
+        experiment_dir, overwrite=args.overwrite, overwrite_eval=args.overwrite_eval
+    )
+
+    # print results
+    pprint(s)
+    if args.plot:
+        for name, fig in f.items():
+            fig.canvas.manager.set_window_title(name)
+        plt.show()

vismatch/third_party/keypt2subpx/submodules/glue_factory/gluefactory/eval/inspect.py

@@ -0,0 +1,61 @@
+import argparse
+from collections import defaultdict
+from pathlib import Path
+from pprint import pprint
+
+import matplotlib
+import matplotlib.pyplot as plt
+
+from ..settings import EVAL_PATH
+from ..visualization.global_frame import GlobalFrame
+from ..visualization.two_view_frame import TwoViewFrame
+from . import get_benchmark
+from .eval_pipeline import load_eval
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("benchmark", type=str)
+    parser.add_argument("--x", type=str, default=None)
+    parser.add_argument("--y", type=str, default=None)
+    parser.add_argument("--backend", type=str, default=None)
+    parser.add_argument(
+        "--default_plot", type=str, default=TwoViewFrame.default_conf["default"]
+    )
+
+    parser.add_argument("dotlist", nargs="*")
+    args = parser.parse_intermixed_args()
+
+    output_dir = Path(EVAL_PATH, args.benchmark)
+
+    results = {}
+    summaries = defaultdict(dict)
+
+    predictions = {}
+
+    if args.backend:
+        matplotlib.use(args.backend)
+
+    bm = get_benchmark(args.benchmark)
+    loader = bm.get_dataloader()
+
+    for name in args.dotlist:
+        experiment_dir = output_dir / name
+        pred_file = experiment_dir / "predictions.h5"
+        s, results[name] = load_eval(experiment_dir)
+        predictions[name] = pred_file
+        for k, v in s.items():
+            summaries[k][name] = v
+
+    pprint(summaries)
+
+    plt.close("all")
+
+    frame = GlobalFrame(
+        {"child": {"default": args.default_plot}, **vars(args)},
+        results,
+        loader,
+        predictions,
+        child_frame=TwoViewFrame,
+    )
+    frame.draw()
+    plt.show()

vismatch/third_party/keypt2subpx/submodules/glue_factory/gluefactory/eval/io.py

@@ -0,0 +1,109 @@
+import argparse
+from pathlib import Path
+from pprint import pprint
+from typing import Optional
+
+import pkg_resources
+from omegaconf import OmegaConf
+
+from ..models import get_model
+from ..settings import TRAINING_PATH
+from ..utils.experiments import load_experiment
+
+
+def parse_config_path(name_or_path: Optional[str], defaults: str) -> Path:
+    default_configs = {}
+    for c in pkg_resources.resource_listdir("gluefactory", str(defaults)):
+        if c.endswith(".yaml"):
+            default_configs[Path(c).stem] = Path(
+                pkg_resources.resource_filename("gluefactory", defaults + c)
+            )
+    if name_or_path is None:
+        return None
+    if name_or_path in default_configs:
+        return default_configs[name_or_path]
+    path = Path(name_or_path)
+    if not path.exists():
+        raise FileNotFoundError(
+            f"Cannot find the config file: {name_or_path}. "
+            f"Not in the default configs {list(default_configs.keys())} "
+            "and not an existing path."
+        )
+    return Path(path)
+
+
+def extract_benchmark_conf(conf, benchmark):
+    mconf = OmegaConf.create(
+        {
+            "model": conf.get("model", {}),
+        }
+    )
+    if "benchmarks" in conf.keys():
+        return OmegaConf.merge(mconf, conf.benchmarks.get(benchmark, {}))
+    else:
+        return mconf
+
+
+def parse_eval_args(benchmark, args, configs_path, default=None):
+    conf = {"data": {}, "model": {}, "eval": {}}
+    if args.conf:
+        conf_path = parse_config_path(args.conf, configs_path)
+        custom_conf = OmegaConf.load(conf_path)
+        conf = extract_benchmark_conf(OmegaConf.merge(conf, custom_conf), benchmark)
+        args.tag = (
+            args.tag if args.tag is not None else conf_path.name.replace(".yaml", "")
+        )
+
+    cli_conf = OmegaConf.from_cli(args.dotlist)
+    conf = OmegaConf.merge(conf, cli_conf)
+    conf.checkpoint = args.checkpoint if args.checkpoint else conf.get("checkpoint")
+
+    if conf.checkpoint and not conf.checkpoint.endswith(".tar"):
+        checkpoint_conf = OmegaConf.load(
+            TRAINING_PATH / conf.checkpoint / "config.yaml"
+        )
+        conf = OmegaConf.merge(extract_benchmark_conf(checkpoint_conf, benchmark), conf)
+
+    if default:
+        conf = OmegaConf.merge(default, conf)
+
+    if args.tag is not None:
+        name = args.tag
+    elif args.conf and conf.checkpoint:
+        name = f"{args.conf}_{conf.checkpoint}"
+    elif args.conf:
+        name = args.conf
+    elif conf.checkpoint:
+        name = conf.checkpoint
+    if len(args.dotlist) > 0 and not args.tag:
+        name = name + "_" + ":".join(args.dotlist)
+    print("Running benchmark:", benchmark)
+    print("Experiment tag:", name)
+    print("Config:")
+    pprint(OmegaConf.to_container(conf))
+    return name, conf
+
+
+def load_model(model_conf, checkpoint):
+    if checkpoint:
+        model = load_experiment(checkpoint, conf=model_conf).eval()
+    else:
+        model = get_model("two_view_pipeline")(model_conf).eval()
+    if not model.is_initialized():
+        raise ValueError(
+            "The provided model has non-initialized parameters. "
+            + "Try to load a checkpoint instead."
+        )
+    return model
+
+
+def get_eval_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--tag", type=str, default=None)
+    parser.add_argument("--checkpoint", type=str, default=None)
+    parser.add_argument("--conf", type=str, default=None)
+    parser.add_argument("--overwrite", action="store_true")
+    parser.add_argument("--overwrite_eval", action="store_true")
+    parser.add_argument("--plot", action="store_true")
+    parser.add_argument("dotlist", nargs="*")
+    return parser