vismatch 1.1.1__py3-none-any.whl

vismatch/third_party/MatchAnything/imcui/hloc/pipelines/7Scenes/create_gt_sfm.py

@@ -0,0 +1,134 @@
+from pathlib import Path
+
+import numpy as np
+import PIL.Image
+import pycolmap
+import torch
+from tqdm import tqdm
+
+from ...utils.read_write_model import read_model, write_model
+
+
+def scene_coordinates(p2D, R_w2c, t_w2c, depth, camera):
+    assert len(depth) == len(p2D)
+    p2D_norm = np.stack(pycolmap.Camera(camera._asdict()).image_to_world(p2D))
+    p2D_h = np.concatenate([p2D_norm, np.ones_like(p2D_norm[:, :1])], 1)
+    p3D_c = p2D_h * depth[:, None]
+    p3D_w = (p3D_c - t_w2c) @ R_w2c
+    return p3D_w
+
+
+def interpolate_depth(depth, kp):
+    h, w = depth.shape
+    kp = kp / np.array([[w - 1, h - 1]]) * 2 - 1
+    assert np.all(kp > -1) and np.all(kp < 1)
+    depth = torch.from_numpy(depth)[None, None]
+    kp = torch.from_numpy(kp)[None, None]
+    grid_sample = torch.nn.functional.grid_sample
+
+    # To maximize the number of points that have depth:
+    # do bilinear interpolation first and then nearest for the remaining points
+    interp_lin = grid_sample(depth, kp, align_corners=True, mode="bilinear")[0, :, 0]
+    interp_nn = torch.nn.functional.grid_sample(
+        depth, kp, align_corners=True, mode="nearest"
+    )[0, :, 0]
+    interp = torch.where(torch.isnan(interp_lin), interp_nn, interp_lin)
+    valid = ~torch.any(torch.isnan(interp), 0)
+
+    interp_depth = interp.T.numpy().flatten()
+    valid = valid.numpy()
+    return interp_depth, valid
+
+
+def image_path_to_rendered_depth_path(image_name):
+    parts = image_name.split("/")
+    name = "_".join(["".join(parts[0].split("-")), parts[1]])
+    name = name.replace("color", "pose")
+    name = name.replace("png", "depth.tiff")
+    return name
+
+
+def project_to_image(p3D, R, t, camera, eps: float = 1e-4, pad: int = 1):
+    p3D = (p3D @ R.T) + t
+    visible = p3D[:, -1] >= eps  # keep points in front of the camera
+    p2D_norm = p3D[:, :-1] / p3D[:, -1:].clip(min=eps)
+    p2D = np.stack(pycolmap.Camera(camera._asdict()).world_to_image(p2D_norm))
+    size = np.array([camera.width - pad - 1, camera.height - pad - 1])
+    valid = np.all((p2D >= pad) & (p2D <= size), -1)
+    valid &= visible
+    return p2D[valid], valid
+
+
+def correct_sfm_with_gt_depth(sfm_path, depth_folder_path, output_path):
+    cameras, images, points3D = read_model(sfm_path)
+    for imgid, img in tqdm(images.items()):
+        image_name = img.name
+        depth_name = image_path_to_rendered_depth_path(image_name)
+
+        depth = PIL.Image.open(Path(depth_folder_path) / depth_name)
+        depth = np.array(depth).astype("float64")
+        depth = depth / 1000.0  # mm to meter
+        depth[(depth == 0.0) | (depth > 1000.0)] = np.nan
+
+        R_w2c, t_w2c = img.qvec2rotmat(), img.tvec
+        camera = cameras[img.camera_id]
+        p3D_ids = img.point3D_ids
+        p3Ds = np.stack([points3D[i].xyz for i in p3D_ids[p3D_ids != -1]], 0)
+
+        p2Ds, valids_projected = project_to_image(p3Ds, R_w2c, t_w2c, camera)
+        invalid_p3D_ids = p3D_ids[p3D_ids != -1][~valids_projected]
+        interp_depth, valids_backprojected = interpolate_depth(depth, p2Ds)
+        scs = scene_coordinates(
+            p2Ds[valids_backprojected],
+            R_w2c,
+            t_w2c,
+            interp_depth[valids_backprojected],
+            camera,
+        )
+        invalid_p3D_ids = np.append(
+            invalid_p3D_ids,
+            p3D_ids[p3D_ids != -1][valids_projected][~valids_backprojected],
+        )
+        for p3did in invalid_p3D_ids:
+            if p3did == -1:
+                continue
+            else:
+                obs_imgids = points3D[p3did].image_ids
+                invalid_imgids = list(np.where(obs_imgids == img.id)[0])
+                points3D[p3did] = points3D[p3did]._replace(
+                    image_ids=np.delete(obs_imgids, invalid_imgids),
+                    point2D_idxs=np.delete(
+                        points3D[p3did].point2D_idxs, invalid_imgids
+                    ),
+                )
+
+        new_p3D_ids = p3D_ids.copy()
+        sub_p3D_ids = new_p3D_ids[new_p3D_ids != -1]
+        valids = np.ones(np.count_nonzero(new_p3D_ids != -1), dtype=bool)
+        valids[~valids_projected] = False
+        valids[valids_projected] = valids_backprojected
+        sub_p3D_ids[~valids] = -1
+        new_p3D_ids[new_p3D_ids != -1] = sub_p3D_ids
+        img = img._replace(point3D_ids=new_p3D_ids)
+
+        assert len(img.point3D_ids[img.point3D_ids != -1]) == len(
+            scs
+        ), f"{len(scs)}, {len(img.point3D_ids[img.point3D_ids != -1])}"
+        for i, p3did in enumerate(img.point3D_ids[img.point3D_ids != -1]):
+            points3D[p3did] = points3D[p3did]._replace(xyz=scs[i])
+        images[imgid] = img
+
+    output_path.mkdir(parents=True, exist_ok=True)
+    write_model(cameras, images, points3D, output_path)
+
+
+if __name__ == "__main__":
+    dataset = Path("datasets/7scenes")
+    outputs = Path("outputs/7Scenes")
+
+    SCENES = ["chess", "fire", "heads", "office", "pumpkin", "redkitchen", "stairs"]
+    for scene in SCENES:
+        sfm_path = outputs / scene / "sfm_superpoint+superglue"
+        depth_path = dataset / f"depth/7scenes_{scene}/train/depth"
+        output_path = outputs / scene / "sfm_superpoint+superglue+depth"
+        correct_sfm_with_gt_depth(sfm_path, depth_path, output_path)

vismatch/third_party/MatchAnything/imcui/hloc/pipelines/7Scenes/pipeline.py

@@ -0,0 +1,139 @@
+import argparse
+from pathlib import Path
+
+from ... import (
+    extract_features,
+    localize_sfm,
+    logger,
+    match_features,
+    pairs_from_covisibility,
+    triangulation,
+)
+from ..Cambridge.utils import create_query_list_with_intrinsics, evaluate
+from .create_gt_sfm import correct_sfm_with_gt_depth
+from .utils import create_reference_sfm
+
+SCENES = ["chess", "fire", "heads", "office", "pumpkin", "redkitchen", "stairs"]
+
+
+def run_scene(
+    images,
+    gt_dir,
+    retrieval,
+    outputs,
+    results,
+    num_covis,
+    use_dense_depth,
+    depth_dir=None,
+):
+    outputs.mkdir(exist_ok=True, parents=True)
+    ref_sfm_sift = outputs / "sfm_sift"
+    ref_sfm = outputs / "sfm_superpoint+superglue"
+    query_list = outputs / "query_list_with_intrinsics.txt"
+
+    feature_conf = {
+        "output": "feats-superpoint-n4096-r1024",
+        "model": {
+            "name": "superpoint",
+            "nms_radius": 3,
+            "max_keypoints": 4096,
+        },
+        "preprocessing": {
+            "globs": ["*.color.png"],
+            "grayscale": True,
+            "resize_max": 1024,
+        },
+    }
+    matcher_conf = match_features.confs["superglue"]
+    matcher_conf["model"]["sinkhorn_iterations"] = 5
+
+    test_list = gt_dir / "list_test.txt"
+    create_reference_sfm(gt_dir, ref_sfm_sift, test_list)
+    create_query_list_with_intrinsics(gt_dir, query_list, test_list)
+
+    features = extract_features.main(feature_conf, images, outputs, as_half=True)
+
+    sfm_pairs = outputs / f"pairs-db-covis{num_covis}.txt"
+    pairs_from_covisibility.main(ref_sfm_sift, sfm_pairs, num_matched=num_covis)
+    sfm_matches = match_features.main(
+        matcher_conf, sfm_pairs, feature_conf["output"], outputs
+    )
+    if not (use_dense_depth and ref_sfm.exists()):
+        triangulation.main(
+            ref_sfm, ref_sfm_sift, images, sfm_pairs, features, sfm_matches
+        )
+    if use_dense_depth:
+        assert depth_dir is not None
+        ref_sfm_fix = outputs / "sfm_superpoint+superglue+depth"
+        correct_sfm_with_gt_depth(ref_sfm, depth_dir, ref_sfm_fix)
+        ref_sfm = ref_sfm_fix
+
+    loc_matches = match_features.main(
+        matcher_conf, retrieval, feature_conf["output"], outputs
+    )
+
+    localize_sfm.main(
+        ref_sfm,
+        query_list,
+        retrieval,
+        features,
+        loc_matches,
+        results,
+        covisibility_clustering=False,
+        prepend_camera_name=True,
+    )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--scenes", default=SCENES, choices=SCENES, nargs="+")
+    parser.add_argument("--overwrite", action="store_true")
+    parser.add_argument(
+        "--dataset",
+        type=Path,
+        default="datasets/7scenes",
+        help="Path to the dataset, default: %(default)s",
+    )
+    parser.add_argument(
+        "--outputs",
+        type=Path,
+        default="outputs/7scenes",
+        help="Path to the output directory, default: %(default)s",
+    )
+    parser.add_argument("--use_dense_depth", action="store_true")
+    parser.add_argument(
+        "--num_covis",
+        type=int,
+        default=30,
+        help="Number of image pairs for SfM, default: %(default)s",
+    )
+    args = parser.parse_args()
+
+    gt_dirs = args.dataset / "7scenes_sfm_triangulated/{scene}/triangulated"
+    retrieval_dirs = args.dataset / "7scenes_densevlad_retrieval_top_10"
+
+    all_results = {}
+    for scene in args.scenes:
+        logger.info(f'Working on scene "{scene}".')
+        results = (
+            args.outputs
+            / scene
+            / "results_{}.txt".format("dense" if args.use_dense_depth else "sparse")
+        )
+        if args.overwrite or not results.exists():
+            run_scene(
+                args.dataset / scene,
+                Path(str(gt_dirs).format(scene=scene)),
+                retrieval_dirs / f"{scene}_top10.txt",
+                args.outputs / scene,
+                results,
+                args.num_covis,
+                args.use_dense_depth,
+                depth_dir=args.dataset / f"depth/7scenes_{scene}/train/depth",
+            )
+        all_results[scene] = results
+
+    for scene in args.scenes:
+        logger.info(f'Evaluate scene "{scene}".')
+        gt_dir = Path(str(gt_dirs).format(scene=scene))
+        evaluate(gt_dir, all_results[scene], gt_dir / "list_test.txt")

vismatch/third_party/MatchAnything/imcui/hloc/pipelines/7Scenes/utils.py

@@ -0,0 +1,34 @@
+import logging
+
+import numpy as np
+
+from hloc.utils.read_write_model import read_model, write_model
+
+logger = logging.getLogger(__name__)
+
+
+def create_reference_sfm(full_model, ref_model, blacklist=None, ext=".bin"):
+    """Create a new COLMAP model with only training images."""
+    logger.info("Creating the reference model.")
+    ref_model.mkdir(exist_ok=True)
+    cameras, images, points3D = read_model(full_model, ext)
+
+    if blacklist is not None:
+        with open(blacklist, "r") as f:
+            blacklist = f.read().rstrip().split("\n")
+
+    images_ref = dict()
+    for id_, image in images.items():
+        if blacklist and image.name in blacklist:
+            continue
+        images_ref[id_] = image
+
+    points3D_ref = dict()
+    for id_, point3D in points3D.items():
+        ref_ids = [i for i in point3D.image_ids if i in images_ref]
+        if len(ref_ids) == 0:
+            continue
+        points3D_ref[id_] = point3D._replace(image_ids=np.array(ref_ids))
+
+    write_model(cameras, images_ref, points3D_ref, ref_model, ".bin")
+    logger.info(f"Kept {len(images_ref)} images out of {len(images)}.")

vismatch/third_party/MatchAnything/imcui/hloc/pipelines/Aachen/pipeline.py

@@ -0,0 +1,109 @@
+import argparse
+from pathlib import Path
+from pprint import pformat
+
+from ... import (
+    colmap_from_nvm,
+    extract_features,
+    localize_sfm,
+    logger,
+    match_features,
+    pairs_from_covisibility,
+    pairs_from_retrieval,
+    triangulation,
+)
+
+
+def run(args):
+    # Setup the paths
+    dataset = args.dataset
+    images = dataset / "images_upright/"
+
+    outputs = args.outputs  # where everything will be saved
+    sift_sfm = outputs / "sfm_sift"  # from which we extract the reference poses
+    reference_sfm = outputs / "sfm_superpoint+superglue"  # the SfM model we will build
+    sfm_pairs = (
+        outputs / f"pairs-db-covis{args.num_covis}.txt"
+    )  # top-k most covisible in SIFT model
+    loc_pairs = (
+        outputs / f"pairs-query-netvlad{args.num_loc}.txt"
+    )  # top-k retrieved by NetVLAD
+    results = outputs / f"Aachen_hloc_superpoint+superglue_netvlad{args.num_loc}.txt"
+
+    # list the standard configurations available
+    logger.info("Configs for feature extractors:\n%s", pformat(extract_features.confs))
+    logger.info("Configs for feature matchers:\n%s", pformat(match_features.confs))
+
+    # pick one of the configurations for extraction and matching
+    retrieval_conf = extract_features.confs["netvlad"]
+    feature_conf = extract_features.confs["superpoint_aachen"]
+    matcher_conf = match_features.confs["superglue"]
+
+    features = extract_features.main(feature_conf, images, outputs)
+
+    colmap_from_nvm.main(
+        dataset / "3D-models/aachen_cvpr2018_db.nvm",
+        dataset / "3D-models/database_intrinsics.txt",
+        dataset / "aachen.db",
+        sift_sfm,
+    )
+    pairs_from_covisibility.main(sift_sfm, sfm_pairs, num_matched=args.num_covis)
+    sfm_matches = match_features.main(
+        matcher_conf, sfm_pairs, feature_conf["output"], outputs
+    )
+
+    triangulation.main(
+        reference_sfm, sift_sfm, images, sfm_pairs, features, sfm_matches
+    )
+
+    global_descriptors = extract_features.main(retrieval_conf, images, outputs)
+    pairs_from_retrieval.main(
+        global_descriptors,
+        loc_pairs,
+        args.num_loc,
+        query_prefix="query",
+        db_model=reference_sfm,
+    )
+    loc_matches = match_features.main(
+        matcher_conf, loc_pairs, feature_conf["output"], outputs
+    )
+
+    localize_sfm.main(
+        reference_sfm,
+        dataset / "queries/*_time_queries_with_intrinsics.txt",
+        loc_pairs,
+        features,
+        loc_matches,
+        results,
+        covisibility_clustering=False,
+    )  # not required with SuperPoint+SuperGlue
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--dataset",
+        type=Path,
+        default="datasets/aachen",
+        help="Path to the dataset, default: %(default)s",
+    )
+    parser.add_argument(
+        "--outputs",
+        type=Path,
+        default="outputs/aachen",
+        help="Path to the output directory, default: %(default)s",
+    )
+    parser.add_argument(
+        "--num_covis",
+        type=int,
+        default=20,
+        help="Number of image pairs for SfM, default: %(default)s",
+    )
+    parser.add_argument(
+        "--num_loc",
+        type=int,
+        default=50,
+        help="Number of image pairs for loc, default: %(default)s",
+    )
+    args = parser.parse_args()
+    run(args)

vismatch/third_party/MatchAnything/imcui/hloc/pipelines/Aachen_v1_1/pipeline.py

@@ -0,0 +1,104 @@
+import argparse
+from pathlib import Path
+from pprint import pformat
+
+from ... import (
+    extract_features,
+    localize_sfm,
+    logger,
+    match_features,
+    pairs_from_covisibility,
+    pairs_from_retrieval,
+    triangulation,
+)
+
+
+def run(args):
+    # Setup the paths
+    dataset = args.dataset
+    images = dataset / "images_upright/"
+    sift_sfm = dataset / "3D-models/aachen_v_1_1"
+
+    outputs = args.outputs  # where everything will be saved
+    reference_sfm = outputs / "sfm_superpoint+superglue"  # the SfM model we will build
+    sfm_pairs = (
+        outputs / f"pairs-db-covis{args.num_covis}.txt"
+    )  # top-k most covisible in SIFT model
+    loc_pairs = (
+        outputs / f"pairs-query-netvlad{args.num_loc}.txt"
+    )  # top-k retrieved by NetVLAD
+    results = (
+        outputs / f"Aachen-v1.1_hloc_superpoint+superglue_netvlad{args.num_loc}.txt"
+    )
+
+    # list the standard configurations available
+    logger.info("Configs for feature extractors:\n%s", pformat(extract_features.confs))
+    logger.info("Configs for feature matchers:\n%s", pformat(match_features.confs))
+
+    # pick one of the configurations for extraction and matching
+    retrieval_conf = extract_features.confs["netvlad"]
+    feature_conf = extract_features.confs["superpoint_max"]
+    matcher_conf = match_features.confs["superglue"]
+
+    features = extract_features.main(feature_conf, images, outputs)
+
+    pairs_from_covisibility.main(sift_sfm, sfm_pairs, num_matched=args.num_covis)
+    sfm_matches = match_features.main(
+        matcher_conf, sfm_pairs, feature_conf["output"], outputs
+    )
+
+    triangulation.main(
+        reference_sfm, sift_sfm, images, sfm_pairs, features, sfm_matches
+    )
+
+    global_descriptors = extract_features.main(retrieval_conf, images, outputs)
+    pairs_from_retrieval.main(
+        global_descriptors,
+        loc_pairs,
+        args.num_loc,
+        query_prefix="query",
+        db_model=reference_sfm,
+    )
+    loc_matches = match_features.main(
+        matcher_conf, loc_pairs, feature_conf["output"], outputs
+    )
+
+    localize_sfm.main(
+        reference_sfm,
+        dataset / "queries/*_time_queries_with_intrinsics.txt",
+        loc_pairs,
+        features,
+        loc_matches,
+        results,
+        covisibility_clustering=False,
+    )  # not required with SuperPoint+SuperGlue
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--dataset",
+        type=Path,
+        default="datasets/aachen_v1.1",
+        help="Path to the dataset, default: %(default)s",
+    )
+    parser.add_argument(
+        "--outputs",
+        type=Path,
+        default="outputs/aachen_v1.1",
+        help="Path to the output directory, default: %(default)s",
+    )
+    parser.add_argument(
+        "--num_covis",
+        type=int,
+        default=20,
+        help="Number of image pairs for SfM, default: %(default)s",
+    )
+    parser.add_argument(
+        "--num_loc",
+        type=int,
+        default=50,
+        help="Number of image pairs for loc, default: %(default)s",
+    )
+    args = parser.parse_args()
+    run(args)

vismatch/third_party/MatchAnything/imcui/hloc/pipelines/Aachen_v1_1/pipeline_loftr.py

@@ -0,0 +1,104 @@
+import argparse
+from pathlib import Path
+from pprint import pformat
+
+from ... import (
+    extract_features,
+    localize_sfm,
+    logger,
+    match_dense,
+    pairs_from_covisibility,
+    pairs_from_retrieval,
+    triangulation,
+)
+
+
+def run(args):
+    # Setup the paths
+    dataset = args.dataset
+    images = dataset / "images_upright/"
+    sift_sfm = dataset / "3D-models/aachen_v_1_1"
+
+    outputs = args.outputs  # where everything will be saved
+    outputs.mkdir()
+    reference_sfm = outputs / "sfm_loftr"  # the SfM model we will build
+    sfm_pairs = (
+        outputs / f"pairs-db-covis{args.num_covis}.txt"
+    )  # top-k most covisible in SIFT model
+    loc_pairs = (
+        outputs / f"pairs-query-netvlad{args.num_loc}.txt"
+    )  # top-k retrieved by NetVLAD
+    results = outputs / f"Aachen-v1.1_hloc_loftr_netvlad{args.num_loc}.txt"
+
+    # list the standard configurations available
+    logger.info("Configs for dense feature matchers:\n%s", pformat(match_dense.confs))
+
+    # pick one of the configurations for extraction and matching
+    retrieval_conf = extract_features.confs["netvlad"]
+    matcher_conf = match_dense.confs["loftr_aachen"]
+
+    pairs_from_covisibility.main(sift_sfm, sfm_pairs, num_matched=args.num_covis)
+    features, sfm_matches = match_dense.main(
+        matcher_conf, sfm_pairs, images, outputs, max_kps=8192, overwrite=False
+    )
+
+    triangulation.main(
+        reference_sfm, sift_sfm, images, sfm_pairs, features, sfm_matches
+    )
+
+    global_descriptors = extract_features.main(retrieval_conf, images, outputs)
+    pairs_from_retrieval.main(
+        global_descriptors,
+        loc_pairs,
+        args.num_loc,
+        query_prefix="query",
+        db_model=reference_sfm,
+    )
+    features, loc_matches = match_dense.main(
+        matcher_conf,
+        loc_pairs,
+        images,
+        outputs,
+        features=features,
+        max_kps=None,
+        matches=sfm_matches,
+    )
+
+    localize_sfm.main(
+        reference_sfm,
+        dataset / "queries/*_time_queries_with_intrinsics.txt",
+        loc_pairs,
+        features,
+        loc_matches,
+        results,
+        covisibility_clustering=False,
+    )  # not required with loftr
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--dataset",
+        type=Path,
+        default="datasets/aachen_v1.1",
+        help="Path to the dataset, default: %(default)s",
+    )
+    parser.add_argument(
+        "--outputs",
+        type=Path,
+        default="outputs/aachen_v1.1",
+        help="Path to the output directory, default: %(default)s",
+    )
+    parser.add_argument(
+        "--num_covis",
+        type=int,
+        default=20,
+        help="Number of image pairs for SfM, default: %(default)s",
+    )
+    parser.add_argument(
+        "--num_loc",
+        type=int,
+        default=50,
+        help="Number of image pairs for loc, default: %(default)s",
+    )
+    args = parser.parse_args()