vismatch 1.1.1__py3-none-any.whl

vismatch/im_models/duster.py

@@ -0,0 +1,104 @@
+import numpy as np
+import os
+import torchvision.transforms as tfm
+import py3_wget
+import torch
+
+# Monkey patch torch.load to use weights_only=False by default for compatibility with PyTorch 2.6+
+_original_torch_load = torch.load
+
+
+def _patched_torch_load(*args, **kwargs):
+    if "weights_only" not in kwargs:
+        kwargs["weights_only"] = False
+    return _original_torch_load(*args, **kwargs)
+
+
+torch.load = _patched_torch_load
+
+from vismatch.utils import add_to_path, resize_to_divisible
+from vismatch import WEIGHTS_DIR, THIRD_PARTY_DIR, BaseMatcher
+
+add_to_path(THIRD_PARTY_DIR.joinpath("duster"))
+
+from dust3r.inference import inference
+from dust3r.model import AsymmetricCroCo3DStereo
+from dust3r.image_pairs import make_pairs
+from dust3r.cloud_opt import global_aligner, GlobalAlignerMode
+from dust3r.utils.geometry import find_reciprocal_matches, xy_grid
+
+
+class Dust3rMatcher(BaseMatcher):
+    model_path = WEIGHTS_DIR.joinpath("duster_vit_large.pth")
+    vit_patch_size = 16
+
+    def __init__(self, device="cpu", *args, **kwargs):
+        super().__init__(device, **kwargs)
+        self.normalize = tfm.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
+
+        self.verbose = False
+
+        self.download_weights()
+        self.model = AsymmetricCroCo3DStereo.from_pretrained(self.model_path).to(device)
+
+    @staticmethod
+    def download_weights():
+        url = "https://download.europe.naverlabs.com/ComputerVision/DUSt3R/DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth"
+
+        if not os.path.isfile(Dust3rMatcher.model_path):
+            print("Downloading Dust3r(ViT large)... (takes a while)")
+            py3_wget.download_file(url, Dust3rMatcher.model_path)
+
+    def preprocess(self, img):
+        _, h, w = img.shape
+        orig_shape = h, w
+
+        img = resize_to_divisible(img, self.vit_patch_size)
+
+        img = self.normalize(img).unsqueeze(0)
+
+        return img, orig_shape
+
+    def _forward(self, img0, img1):
+        img0, img0_orig_shape = self.preprocess(img0)
+        img1, img1_orig_shape = self.preprocess(img1)
+
+        images = [
+            {"img": img0, "idx": 0, "instance": 0},
+            {"img": img1, "idx": 1, "instance": 1},
+        ]
+        pairs = make_pairs(images, scene_graph="complete", prefilter=None, symmetrize=True)
+        output = inference(pairs, self.model, self.device, batch_size=1, verbose=self.verbose)
+
+        scene = global_aligner(
+            output,
+            device=self.device,
+            mode=GlobalAlignerMode.PairViewer,
+            verbose=self.verbose,
+        )
+        # retrieve useful values from scene:
+        confidence_masks = scene.get_masks()
+        pts3d = scene.get_pts3d()
+        imgs = scene.imgs
+        pts2d_list, pts3d_list = [], []
+
+        for i in range(2):
+            conf_i = confidence_masks[i].cpu().numpy()
+            pts2d_list.append(xy_grid(*imgs[i].shape[:2][::-1])[conf_i])  # imgs[i].shape[:2] = (H, W)
+            pts3d_list.append(pts3d[i].detach().cpu().numpy()[conf_i])
+
+        # return if there is no 3d points found on either one of the image
+        if pts3d_list[0].shape[0] == 0 or pts3d_list[1].shape[0] == 0:
+            return np.empty((0, 2)), np.empty((0, 2)), None, None, None, None
+        reciprocal_in_P2, nn2_in_P1, _ = find_reciprocal_matches(*pts3d_list)
+
+        mkpts1 = pts2d_list[1][reciprocal_in_P2]
+        mkpts0 = pts2d_list[0][nn2_in_P1][reciprocal_in_P2]
+
+        # duster sometimes requires reshaping an image to fit vit patch size evenly, so we need to
+        # rescale kpts to the original img
+        H0, W0, H1, W1 = *img0.shape[-2:], *img1.shape[-2:]
+        mkpts0 = self.rescale_coords(mkpts0, *img0_orig_shape, H0, W0)
+        mkpts1 = self.rescale_coords(mkpts1, *img1_orig_shape, H1, W1)
+
+        return mkpts0, mkpts1, None, None, None, None

vismatch/im_models/edm.py

@@ -0,0 +1,64 @@
+# implementation inspired by https://github.com/chicleee/EDM/blob/main/demo_single_pair.ipynb
+
+import torchvision.transforms as tfm
+from safetensors.torch import load_file
+
+from huggingface_hub import snapshot_download
+from vismatch import THIRD_PARTY_DIR, BaseMatcher
+from vismatch.utils import resize_to_divisible, add_to_path
+
+add_to_path(THIRD_PARTY_DIR.joinpath("EDM"), insert=0)
+
+from src.edm import EDM
+from src.config.default import get_cfg_defaults
+from src.utils.misc import lower_config
+
+
+class EDMMatcher(BaseMatcher):
+    divisible_size = 32
+
+    def __init__(self, device="cpu", thresh=0.2, **kwargs):
+        super().__init__(device, **kwargs)
+        self.thresh = thresh
+        self.matcher = self.build_matcher()
+
+    def build_matcher(self):
+        # Get default configurations
+        config = get_cfg_defaults()
+        config.merge_from_file(THIRD_PARTY_DIR / "EDM/configs/edm/outdoor/edm_base.py")
+        config.merge_from_file(THIRD_PARTY_DIR / "EDM/configs/data/megadepth_test_1500.py")
+
+        config.EDM.COARSE.MCONF_THR = self.thresh
+        config.EDM.COARSE.BORDER_RM = 2
+        config = lower_config(config)
+
+        matcher = EDM(config=config["edm"])
+
+        # Load model from HuggingFace
+        weights_path = f"{snapshot_download('vismatch/edm')}/edm.safetensors"
+        matcher.load_state_dict(load_file(weights_path))
+
+        return matcher.eval().to(self.device)
+
+    def preprocess(self, img):
+        _, h, w = img.shape
+        orig_shape = h, w
+        img = resize_to_divisible(img, self.divisible_size)
+        return tfm.Grayscale()(img).unsqueeze(0), orig_shape
+
+    def _forward(self, img0, img1):
+        img0, img0_orig_shape = self.preprocess(img0)
+        img1, img1_orig_shape = self.preprocess(img1)
+
+        batch = {"image0": img0, "image1": img1}
+
+        self.matcher(batch)
+
+        mkpts0 = batch["mkpts0_f"]
+        mkpts1 = batch["mkpts1_f"]
+
+        H0, W0, H1, W1 = *img0.shape[-2:], *img1.shape[-2:]
+        mkpts0 = self.rescale_coords(mkpts0, *img0_orig_shape, H0, W0)
+        mkpts1 = self.rescale_coords(mkpts1, *img1_orig_shape, H1, W1)
+
+        return mkpts0, mkpts1, None, None, None, None

vismatch/im_models/efficient_loftr.py

@@ -0,0 +1,60 @@
+import torch
+from huggingface_hub import snapshot_download
+from safetensors.torch import load_file
+from copy import deepcopy
+import torchvision.transforms as tfm
+
+
+from vismatch import THIRD_PARTY_DIR, BaseMatcher
+from vismatch.utils import resize_to_divisible, add_to_path
+
+add_to_path(THIRD_PARTY_DIR.joinpath("EfficientLoFTR"), insert=0)
+
+from src.loftr import LoFTR, full_default_cfg, opt_default_cfg, reparameter
+
+
+class EfficientLoFTRMatcher(BaseMatcher):
+    divisible_size = 32
+
+    def __init__(self, device="cpu", cfg="full", **kwargs):
+        super().__init__(device, **kwargs)
+
+        self.precision = kwargs.get("precision", self.get_precision())
+
+        model_path = f"{snapshot_download('vismatch/eloftr')}/eloftr_outdoors.safetensors"
+
+        self.matcher = LoFTR(config=deepcopy(full_default_cfg if cfg == "full" else opt_default_cfg))
+
+        state_dict = load_file(model_path)
+        self.matcher.load_state_dict(state_dict)
+
+        self.matcher = reparameter(self.matcher).to(self.device).eval()
+
+    def get_precision(self):
+        return "fp16"
+
+    def preprocess(self, img):
+        _, h, w = img.shape
+        orig_shape = h, w
+        img = resize_to_divisible(img, self.divisible_size)
+        return tfm.Grayscale()(img).unsqueeze(0), orig_shape
+
+    def _forward(self, img0, img1):
+        img0, img0_orig_shape = self.preprocess(img0)
+        img1, img1_orig_shape = self.preprocess(img1)
+
+        batch = {"image0": img0, "image1": img1}
+        if self.precision == "mp" and self.device == "cuda":
+            with torch.autocast(enabled=True, device_type="cuda"):
+                self.matcher(batch)
+        else:
+            self.matcher(batch)
+
+        mkpts0 = batch["mkpts0_f"]
+        mkpts1 = batch["mkpts1_f"]
+
+        H0, W0, H1, W1 = *img0.shape[-2:], *img1.shape[-2:]
+        mkpts0 = self.rescale_coords(mkpts0, *img0_orig_shape, H0, W0)
+        mkpts1 = self.rescale_coords(mkpts1, *img1_orig_shape, H1, W1)
+
+        return mkpts0, mkpts1, None, None, None, None

vismatch/im_models/gim.py

@@ -0,0 +1,187 @@
+import torch
+from safetensors.torch import load_file
+
+from kornia.color import rgb_to_grayscale
+
+from huggingface_hub import snapshot_download
+from vismatch import THIRD_PARTY_DIR, BaseMatcher
+from vismatch.utils import load_module, add_to_path
+
+BASE_PATH = THIRD_PARTY_DIR.joinpath("gim")
+add_to_path(BASE_PATH)
+from dkm.models.model_zoo.DKMv3 import DKMv3
+
+
+class GIM_DKM(BaseMatcher):
+    def __init__(self, device="cpu", max_num_keypoints=5000, **kwargs):
+        super().__init__(device, **kwargs)
+        self.model = DKMv3(weights=None, h=672, w=896)
+        self.max_num_keypoints = max_num_keypoints
+        self.load_weights()
+        self.model = self.model.eval().to(device)
+
+    def load_weights(self):
+        weights_path = f"{snapshot_download('vismatch/gim-dkm')}/gim_dkm_100h.safetensors"
+        state_dict = load_file(weights_path)
+        for k in list(state_dict.keys()):
+            if k.startswith("model."):
+                state_dict[k.replace("model.", "", 1)] = state_dict.pop(k)
+            if "encoder.net.fc" in k:
+                state_dict.pop(k)
+        self.model.load_state_dict(state_dict)
+
+    def preprocess(self, img):
+        # this version of DKM requires PIL images as input
+        # return Image.fromarray(np.uint8(255*tensor_to_image(img)))
+        if img.ndim < 4:
+            img = img.unsqueeze(0)
+        return img
+
+    def _forward(self, img0, img1):
+        height0, width0 = img0.shape[-2:]
+        height1, width1 = img1.shape[-2:]
+
+        img0 = self.preprocess(img0)  # now as PIL img
+        img1 = self.preprocess(img1)  # now as PIL img
+        dense_matches, dense_certainty = self.model.match(img0, img1, device=self.device)
+        torch.cuda.empty_cache()
+        # sample matching keypoints from dense warp
+        sparse_matches, mconf = self.model.sample(dense_matches, dense_certainty, self.max_num_keypoints)
+        torch.cuda.empty_cache()
+        mkpts0 = sparse_matches[:, :2]
+        mkpts1 = sparse_matches[:, 2:]
+
+        # convert to px coords
+        mkpts0 = torch.stack(
+            (width0 * (mkpts0[:, 0] + 1) / 2, height0 * (mkpts0[:, 1] + 1) / 2),
+            dim=-1,
+        )
+        mkpts1 = torch.stack(
+            (width1 * (mkpts1[:, 0] + 1) / 2, height1 * (mkpts1[:, 1] + 1) / 2),
+            dim=-1,
+        )
+
+        # b_ids = torch.where(mconf[None])[0]
+
+        return mkpts0, mkpts1, None, None, None, None
+
+
+class GIM_LightGlue(BaseMatcher):
+    def __init__(self, device="cpu", max_keypoints=2048, **kwargs):
+        super().__init__(device, **kwargs)
+        # load the altered version of gluefactory
+        load_module("gluefactory_gim", BASE_PATH.joinpath("gluefactory/__init__.py"))
+
+        from gluefactory_gim.superpoint import SuperPoint
+        from gluefactory_gim.models.matchers.lightglue import LightGlue
+
+        # Download weights from HF and get paths from cache
+        repo = snapshot_download("vismatch/gim-lightglue")
+        self.ckpt_path = f"{repo}/gim_lightglue_100h.ckpt"
+        self.superpoint_v1_path = f"{repo}/superpoint_v1.pth"
+
+        self.detector = SuperPoint(
+            {
+                "max_num_keypoints": max_keypoints,
+                "force_num_keypoints": True,
+                "detection_threshold": 0.0,
+                "nms_radius": 3,
+                "trainable": False,
+                "weights_path": self.superpoint_v1_path,
+            }
+        )
+
+        self.model = LightGlue(
+            {
+                "filter_threshold": 0.1,
+                "flash": False,
+                "checkpointed": True,
+            }
+        )
+
+        self.load_weights()
+
+    def load_weights(self):
+        state_dict = torch.load(self.ckpt_path, map_location="cpu", weights_only=True)
+        if "state_dict" in state_dict.keys():
+            state_dict = state_dict["state_dict"]
+        for k in list(state_dict.keys()):
+            if k.startswith("model."):
+                state_dict.pop(k)
+            if k.startswith("superpoint."):
+                state_dict[k.replace("superpoint.", "", 1)] = state_dict.pop(k)
+        self.detector.load_state_dict(state_dict)
+
+        state_dict = torch.load(self.ckpt_path, map_location="cpu", weights_only=True)
+        if "state_dict" in state_dict.keys():
+            state_dict = state_dict["state_dict"]
+        for k in list(state_dict.keys()):
+            if k.startswith("superpoint."):
+                state_dict.pop(k)
+            if k.startswith("model."):
+                state_dict[k.replace("model.", "", 1)] = state_dict.pop(k)
+        self.model.load_state_dict(state_dict)
+
+        self.detector = self.detector.eval().to(self.device)
+        self.model = self.model.eval().to(self.device)
+
+    def preprocess(self, img):
+        # convert to grayscale
+        return rgb_to_grayscale(img.unsqueeze(0))
+
+    def _forward(self, img0, img1):
+        img0 = self.preprocess(img0)
+        img1 = self.preprocess(img1)
+
+        data = dict(image0=img0, image1=img1)
+
+        scale0 = torch.tensor([1.0, 1.0]).to(self.device)[None]
+        scale1 = torch.tensor([1.0, 1.0]).to(self.device)[None]
+
+        size0 = torch.tensor(data["image0"].shape[-2:][::-1])[None]
+        size1 = torch.tensor(data["image1"].shape[-2:][::-1])[None]
+
+        data.update(dict(size0=size0, size1=size1))
+        data.update(dict(scale0=scale0, scale1=scale1))
+
+        pred = {}
+        pred.update(
+            {
+                k + "0": v
+                for k, v in self.detector(
+                    {
+                        "image": data["image0"],
+                        "image_size": data["size0"],
+                    }
+                ).items()
+            }
+        )
+        pred.update(
+            {
+                k + "1": v
+                for k, v in self.detector(
+                    {
+                        "image": data["image1"],
+                        "image_size": data["size1"],
+                    }
+                ).items()
+            }
+        )
+
+        pred.update(self.model({**pred, **data, **{"resize0": data["size0"], "resize1": data["size1"]}}))
+
+        kpts0 = torch.cat([kp * s for kp, s in zip(pred["keypoints0"], data["scale0"][:, None])])
+        kpts1 = torch.cat([kp * s for kp, s in zip(pred["keypoints1"], data["scale1"][:, None])])
+
+        desc0, desc1 = pred["descriptors0"], pred["descriptors1"]
+
+        m_bids = torch.nonzero(pred["keypoints0"].sum(dim=2) > -1)[:, 0]
+        matches = pred["matches"]
+        bs = data["image0"].size(0)
+
+        mkpts0 = torch.cat([kpts0[m_bids == b_id][matches[b_id][..., 0]] for b_id in range(bs)])
+        mkpts1 = torch.cat([kpts1[m_bids == b_id][matches[b_id][..., 1]] for b_id in range(bs)])
+        # b_ids = torch.cat([m_bids[m_bids == b_id][matches[b_id][..., 0]] for b_id in range(bs)])
+        # mconf = torch.cat(pred['scores'])
+
+        return mkpts0, mkpts1, kpts0, kpts1, desc0[0], desc1[0]

vismatch/im_models/handcrafted.py

@@ -0,0 +1,81 @@
+import cv2
+import numpy as np
+import torch
+
+from vismatch import BaseMatcher
+from vismatch.utils import to_numpy
+
+
+class HandcraftedBaseMatcher(BaseMatcher):
+    """
+    This class is the parent for all methods that use a handcrafted detector/descriptor,
+    It implements the forward which is the same regardless of the feature extractor of choice.
+    Therefore this class should *NOT* be instatiated, as it needs its children to define
+    the extractor/detector.
+    """
+
+    def __init__(self, device="cpu", **kwargs):
+        super().__init__(device, **kwargs)
+
+    @staticmethod
+    def preprocess(im_tensor: torch.Tensor) -> np.ndarray:
+        # convert tensors to np 255-based for openCV
+        im_arr = to_numpy(im_tensor).transpose(1, 2, 0)
+        im = cv2.cvtColor(im_arr, cv2.COLOR_RGB2GRAY)
+        im = cv2.normalize(im, None, 0, 255, cv2.NORM_MINMAX).astype("uint8")
+
+        return im
+
+    def _forward(self, img0, img1):
+        # convert tensors to numpy 255-based for OpenCV
+        img0 = self.preprocess(img0)
+        img1 = self.preprocess(img1)
+
+        # find the keypoints and descriptors with SIFT
+        kp0, des0 = self.det_descr.detectAndCompute(img0, None)
+        kp1, des1 = self.det_descr.detectAndCompute(img1, None)
+
+        # BFMatcher with default params
+
+        raw_matches = self.bf.knnMatch(des0, des1, k=self.k_neighbors)
+
+        # Apply ratio test
+        good = []
+        for m, n in raw_matches:
+            if m.distance < self.threshold * n.distance:
+                good.append(m)
+
+        mkpts0, mkpts1 = [], []
+        for good_match in good:
+            kpt_0 = np.array(kp0[good_match.queryIdx].pt)
+            kpt_1 = np.array(kp1[good_match.trainIdx].pt)
+
+            mkpts0.append(kpt_0)
+            mkpts1.append(kpt_1)
+
+        mkpts0 = np.array(mkpts0, dtype=np.float32)
+        mkpts1 = np.array(mkpts1, dtype=np.float32)
+
+        keypoints_0 = np.array([(x.pt[0], x.pt[1]) for x in kp0])
+        keypoints_1 = np.array([(x.pt[0], x.pt[1]) for x in kp1])
+
+        return mkpts0, mkpts1, keypoints_0, keypoints_1, des0, des1
+
+
+class SiftNNMatcher(HandcraftedBaseMatcher):
+    def __init__(self, device="cpu", max_num_keypoints=2048, lowe_thresh=0.75, *args, **kwargs):
+        super().__init__(device, **kwargs)
+        self.threshold = lowe_thresh
+        self.det_descr = cv2.SIFT_create(max_num_keypoints)
+
+        self.bf = cv2.BFMatcher()
+        self.k_neighbors = 2
+
+
+class OrbNNMatcher(HandcraftedBaseMatcher):
+    def __init__(self, device="cpu", max_num_keypoints=2048, lowe_thresh=0.75, *args, **kwargs):
+        super().__init__(device, **kwargs)
+        self.threshold = lowe_thresh
+        self.det_descr = cv2.ORB_create(max_num_keypoints)
+        self.bf = cv2.BFMatcher(cv2.NORM_HAMMING)
+        self.k_neighbors = 2

vismatch/im_models/keypt2subpx.py

@@ -0,0 +1,154 @@
+import torch
+import numpy as np
+
+from omegaconf import OmegaConf
+import torchvision.transforms as tfm
+
+from vismatch import get_matcher, BaseMatcher, THIRD_PARTY_DIR
+from vismatch.utils import to_numpy, to_tensor, load_module, add_to_path
+
+BASE_PATH = THIRD_PARTY_DIR.joinpath("keypt2subpx")
+add_to_path(BASE_PATH)
+
+load_module("gluefactory", BASE_PATH.joinpath("submodules/glue_factory/gluefactory/__init__.py"))
+from dataprocess.superpoint_densescore import SuperPoint
+
+add_to_path(THIRD_PARTY_DIR.joinpath("LightGlue"))
+from lightglue import LightGlue
+from lightglue.utils import rbd, batch_to_device
+
+
+class Keypt2SubpxMatcher(BaseMatcher):
+    # Maps our detector names to third-party library's expected names
+    THIRDPARTY_NAMES = {
+        "superpoint-lightglue": "splg",
+        "aliked-lightglue": "aliked",
+        "xfeat": "xfeat",
+        "xfeat-lightglue": "xfeat",
+        "dedode": "dedode",
+    }
+
+    def __init__(self, device="cpu", detector_name: str | None = None, **kwargs):
+        super().__init__(device, **kwargs)
+
+        self.detector_name = detector_name
+        if detector_name == "superpoint-lightglue":
+            self.matcher = SuperPointDense(self.device)
+        else:
+            self.matcher = get_matcher(detector_name, device=device, **kwargs)
+        detector = self.THIRDPARTY_NAMES[detector_name]
+        self.keypt2subpx = (
+            torch.hub.load("KimSinjeong/keypt2subpx", "Keypt2Subpx", pretrained=True, detector=detector, verbose=False)
+            .eval()
+            .to(self.device)
+        )
+
+    def get_match_idxs(self, mkpts: np.ndarray | torch.Tensor, kpts: np.ndarray | torch.Tensor) -> np.ndarray:
+        idxs = []
+        kpts = to_numpy(kpts)
+
+        for mkpt in to_numpy(mkpts):
+            idx = np.flatnonzero(np.all(kpts == mkpt, axis=1)).squeeze().item()
+            idxs.append(idx)
+        return np.asarray(idxs)
+
+    def get_scoremap(self, img=None, idx=None):
+        assert img is not None or idx is not None, "Must provide either image or idx"
+        if self.detector_name in ["xfeat", "xfeat-lightglue", "dedode"]:
+            return None
+        elif self.detector_name == "aliked-lightglue":
+            # https://github.com/cvg/LightGlue/blob/edb2b838efb2ecfe3f88097c5fad9887d95aedad/lightglue/aliked.py#L707
+            return self.matcher.extractor.extract_dense_map(img[None, ...])[-1].squeeze(0)
+        elif self.detector_name == "superpoint-lightglue":
+            return self.matcher.get_scoremap(idx)
+
+    def _forward(self, img0, img1):
+        mkpts0, mkpts1, keypoints0, keypoints1, descriptors0, descriptors1 = self.matcher._forward(img0, img1)
+        if len(mkpts0):  # only run subpx refinement if kpts are found
+            matching_idxs0, matching_idxs1 = (
+                self.get_match_idxs(mkpts0, keypoints0),
+                self.get_match_idxs(mkpts1, keypoints1),
+            )
+            mdesc0, mdesc1 = descriptors0[matching_idxs0], descriptors1[matching_idxs1]
+
+            scores0, scores1 = self.get_scoremap(img0, 0), self.get_scoremap(img1, 1)
+            mkpts0, mkpts1 = self.keypt2subpx(
+                to_tensor(mkpts0, self.device),
+                to_tensor(mkpts1, self.device),
+                img0,
+                img1,
+                mdesc0,
+                mdesc1,
+                scores0,
+                scores1,
+            )
+        return mkpts0, mkpts1, keypoints0, keypoints1, descriptors0, descriptors1
+
+
+class SuperPointDense(BaseMatcher):
+    # SuperPoint, with Dense Scoremap for Keypt2Subpx refinement
+    modelconf = {
+        "name": "two_view_pipeline",
+        "extractor": {
+            "name": "superpoint_densescore",
+            "max_num_keypoints": 2048,
+            "force_num_keypoints": False,
+            "detection_threshold": 0.0,
+            "nms_radius": 3,
+            "remove_borders": 3,
+            "trainable": False,
+        },
+        "matcher": {
+            "name": "matchers.lightglue_wrapper",
+            "weights": "superpoint",
+            "depth_confidence": -1,
+            "width_confidence": -1,
+            "filter_threshold": 0.1,
+            "trainable": False,
+        },
+        "ground_truth": {"name": "matchers.depth_matcher", "th_positive": 3, "th_negative": 5, "th_epi": 5},
+        "allow_no_extract": True,
+    }
+
+    def __init__(self, device="cpu", **kwargs):
+        super().__init__(device, **kwargs)
+
+        self.config = OmegaConf.create(self.modelconf)
+        self.extractor = SuperPoint(self.config).to(self.device).eval()
+        self.matcher = LightGlue(features="superpoint", depth_confidence=-1, width_confidence=-1).to(self.device)
+        self.scoremaps = {}
+
+    def preprocess(self, img):
+        return tfm.Grayscale()(img).unsqueeze(0)
+
+    def get_scoremap(self, idx):
+        return self.scoremaps[idx]
+
+    def _forward(self, img0, img1):
+        img0 = self.preprocess(img0)
+        img1 = self.preprocess(img1)
+
+        feats0 = self.extractor({"image": img0})
+        feats1 = self.extractor({"image": img1})
+
+        self.scoremaps[0] = feats0["keypoint_scores"]
+        self.scoremaps[1] = feats1["keypoint_scores"]
+
+        # requires keys ['keypoints', 'keypoint_scores', 'descriptors', 'image_size']
+        matches01 = self.matcher({"image0": feats0, "image1": feats1})
+        data = [feats0, feats1, matches01]
+        # remove batch dim and move to target device
+        feats0, feats1, matches01 = [batch_to_device(rbd(x), self.device) for x in data]
+
+        kpts0, kpts1, matches = (
+            feats0["keypoints"],
+            feats1["keypoints"],
+            matches01["matches"],
+        )
+
+        desc0 = feats0["descriptors"]
+        desc1 = feats1["descriptors"]
+
+        mkpts0, mkpts1 = kpts0[matches[..., 0]], kpts1[matches[..., 1]]
+
+        return mkpts0, mkpts1, kpts0, kpts1, desc0, desc1