frontveg 0.1.dev1__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- frontveg/__init__.py +17 -11
- frontveg/_tests/test_widget.py +66 -66
- frontveg/_version.py +2 -2
- frontveg/_widget.py +129 -132
- frontveg/napari.yaml +30 -14
- {frontveg-0.1.dev1.dist-info → frontveg-0.2.0.dist-info}/METADATA +23 -9
- frontveg-0.2.0.dist-info/RECORD +12 -0
- {frontveg-0.1.dev1.dist-info → frontveg-0.2.0.dist-info}/WHEEL +1 -1
- {frontveg-0.1.dev1.dist-info → frontveg-0.2.0.dist-info}/licenses/LICENSE +28 -28
- {frontveg-0.1.dev1.dist-info → frontveg-0.2.0.dist-info}/top_level.txt +0 -1
- frontveg/utils.py +0 -95
- frontveg-0.1.dev1.dist-info/RECORD +0 -44
- sam2/__init__.py +0 -11
- sam2/automatic_mask_generator.py +0 -454
- sam2/build_sam.py +0 -167
- sam2/configs/sam2/sam2_hiera_b+.yaml +0 -113
- sam2/configs/sam2/sam2_hiera_l.yaml +0 -117
- sam2/configs/sam2/sam2_hiera_s.yaml +0 -116
- sam2/configs/sam2/sam2_hiera_t.yaml +0 -118
- sam2/modeling/__init__.py +0 -5
- sam2/modeling/backbones/__init__.py +0 -5
- sam2/modeling/backbones/hieradet.py +0 -317
- sam2/modeling/backbones/image_encoder.py +0 -134
- sam2/modeling/backbones/utils.py +0 -95
- sam2/modeling/memory_attention.py +0 -169
- sam2/modeling/memory_encoder.py +0 -181
- sam2/modeling/position_encoding.py +0 -221
- sam2/modeling/sam/__init__.py +0 -5
- sam2/modeling/sam/mask_decoder.py +0 -295
- sam2/modeling/sam/prompt_encoder.py +0 -182
- sam2/modeling/sam/transformer.py +0 -360
- sam2/modeling/sam2_base.py +0 -907
- sam2/modeling/sam2_utils.py +0 -323
- sam2/sam2_hiera_b+.yaml +0 -1
- sam2/sam2_hiera_l.yaml +0 -1
- sam2/sam2_hiera_s.yaml +0 -1
- sam2/sam2_hiera_t.yaml +0 -1
- sam2/sam2_image_predictor.py +0 -466
- sam2/sam2_video_predictor.py +0 -1172
- sam2/utils/__init__.py +0 -5
- sam2/utils/amg.py +0 -348
- sam2/utils/misc.py +0 -349
- sam2/utils/transforms.py +0 -118
- {frontveg-0.1.dev1.dist-info → frontveg-0.2.0.dist-info}/entry_points.txt +0 -0
sam2/build_sam.py
DELETED
@@ -1,167 +0,0 @@
|
|
1
|
-
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
2
|
-
# All rights reserved.
|
3
|
-
|
4
|
-
# This source code is licensed under the license found in the
|
5
|
-
# LICENSE file in the root directory of this source tree.
|
6
|
-
|
7
|
-
import logging
|
8
|
-
import os
|
9
|
-
|
10
|
-
import torch
|
11
|
-
from hydra import compose
|
12
|
-
from hydra.utils import instantiate
|
13
|
-
from omegaconf import OmegaConf
|
14
|
-
|
15
|
-
import sam2
|
16
|
-
|
17
|
-
# Check if the user is running Python from the parent directory of the sam2 repo
|
18
|
-
# (i.e. the directory where this repo is cloned into) -- this is not supported since
|
19
|
-
# it could shadow the sam2 package and cause issues.
|
20
|
-
if os.path.isdir(os.path.join(sam2.__path__[0], "sam2")):
|
21
|
-
# If the user has "sam2/sam2" in their path, they are likey importing the repo itself
|
22
|
-
# as "sam2" rather than importing the "sam2" python package (i.e. "sam2/sam2" directory).
|
23
|
-
# This typically happens because the user is running Python from the parent directory
|
24
|
-
# that contains the sam2 repo they cloned.
|
25
|
-
raise RuntimeError(
|
26
|
-
"You're likely running Python from the parent directory of the sam2 repository "
|
27
|
-
"(i.e. the directory where https://github.com/facebookresearch/sam2 is cloned into). "
|
28
|
-
"This is not supported since the `sam2` Python package could be shadowed by the "
|
29
|
-
"repository name (the repository is also named `sam2` and contains the Python package "
|
30
|
-
"in `sam2/sam2`). Please run Python from another directory (e.g. from the repo dir "
|
31
|
-
"rather than its parent dir, or from your home directory) after installing SAM 2."
|
32
|
-
)
|
33
|
-
|
34
|
-
|
35
|
-
HF_MODEL_ID_TO_FILENAMES = {
|
36
|
-
"facebook/sam2-hiera-tiny": (
|
37
|
-
"configs/sam2/sam2_hiera_t.yaml",
|
38
|
-
"sam2_hiera_tiny.pt",
|
39
|
-
),
|
40
|
-
"facebook/sam2-hiera-small": (
|
41
|
-
"configs/sam2/sam2_hiera_s.yaml",
|
42
|
-
"sam2_hiera_small.pt",
|
43
|
-
),
|
44
|
-
"facebook/sam2-hiera-base-plus": (
|
45
|
-
"configs/sam2/sam2_hiera_b+.yaml",
|
46
|
-
"sam2_hiera_base_plus.pt",
|
47
|
-
),
|
48
|
-
"facebook/sam2-hiera-large": (
|
49
|
-
"configs/sam2/sam2_hiera_l.yaml",
|
50
|
-
"sam2_hiera_large.pt",
|
51
|
-
),
|
52
|
-
"facebook/sam2.1-hiera-tiny": (
|
53
|
-
"configs/sam2.1/sam2.1_hiera_t.yaml",
|
54
|
-
"sam2.1_hiera_tiny.pt",
|
55
|
-
),
|
56
|
-
"facebook/sam2.1-hiera-small": (
|
57
|
-
"configs/sam2.1/sam2.1_hiera_s.yaml",
|
58
|
-
"sam2.1_hiera_small.pt",
|
59
|
-
),
|
60
|
-
"facebook/sam2.1-hiera-base-plus": (
|
61
|
-
"configs/sam2.1/sam2.1_hiera_b+.yaml",
|
62
|
-
"sam2.1_hiera_base_plus.pt",
|
63
|
-
),
|
64
|
-
"facebook/sam2.1-hiera-large": (
|
65
|
-
"configs/sam2.1/sam2.1_hiera_l.yaml",
|
66
|
-
"sam2.1_hiera_large.pt",
|
67
|
-
),
|
68
|
-
}
|
69
|
-
|
70
|
-
|
71
|
-
def build_sam2(
|
72
|
-
config_file,
|
73
|
-
ckpt_path=None,
|
74
|
-
device="cuda",
|
75
|
-
mode="eval",
|
76
|
-
hydra_overrides_extra=[],
|
77
|
-
apply_postprocessing=True,
|
78
|
-
**kwargs,
|
79
|
-
):
|
80
|
-
|
81
|
-
if apply_postprocessing:
|
82
|
-
hydra_overrides_extra = hydra_overrides_extra.copy()
|
83
|
-
hydra_overrides_extra += [
|
84
|
-
# dynamically fall back to multi-mask if the single mask is not stable
|
85
|
-
"++model.sam_mask_decoder_extra_args.dynamic_multimask_via_stability=true",
|
86
|
-
"++model.sam_mask_decoder_extra_args.dynamic_multimask_stability_delta=0.05",
|
87
|
-
"++model.sam_mask_decoder_extra_args.dynamic_multimask_stability_thresh=0.98",
|
88
|
-
]
|
89
|
-
# Read config and init model
|
90
|
-
cfg = compose(config_name=config_file, overrides=hydra_overrides_extra)
|
91
|
-
OmegaConf.resolve(cfg)
|
92
|
-
model = instantiate(cfg.model, _recursive_=True)
|
93
|
-
_load_checkpoint(model, ckpt_path)
|
94
|
-
model = model.to(device)
|
95
|
-
if mode == "eval":
|
96
|
-
model.eval()
|
97
|
-
return model
|
98
|
-
|
99
|
-
|
100
|
-
def build_sam2_video_predictor(
|
101
|
-
config_file,
|
102
|
-
ckpt_path=None,
|
103
|
-
device="cuda",
|
104
|
-
mode="eval",
|
105
|
-
hydra_overrides_extra=[],
|
106
|
-
apply_postprocessing=True,
|
107
|
-
**kwargs,
|
108
|
-
):
|
109
|
-
hydra_overrides = [
|
110
|
-
"++model._target_=sam2.sam2_video_predictor.SAM2VideoPredictor",
|
111
|
-
]
|
112
|
-
if apply_postprocessing:
|
113
|
-
hydra_overrides_extra = hydra_overrides_extra.copy()
|
114
|
-
hydra_overrides_extra += [
|
115
|
-
# dynamically fall back to multi-mask if the single mask is not stable
|
116
|
-
"++model.sam_mask_decoder_extra_args.dynamic_multimask_via_stability=true",
|
117
|
-
"++model.sam_mask_decoder_extra_args.dynamic_multimask_stability_delta=0.05",
|
118
|
-
"++model.sam_mask_decoder_extra_args.dynamic_multimask_stability_thresh=0.98",
|
119
|
-
# the sigmoid mask logits on interacted frames with clicks in the memory encoder so that the encoded masks are exactly as what users see from clicking
|
120
|
-
"++model.binarize_mask_from_pts_for_mem_enc=true",
|
121
|
-
# fill small holes in the low-res masks up to `fill_hole_area` (before resizing them to the original video resolution)
|
122
|
-
"++model.fill_hole_area=8",
|
123
|
-
]
|
124
|
-
hydra_overrides.extend(hydra_overrides_extra)
|
125
|
-
|
126
|
-
# Read config and init model
|
127
|
-
cfg = compose(config_name=config_file, overrides=hydra_overrides)
|
128
|
-
OmegaConf.resolve(cfg)
|
129
|
-
model = instantiate(cfg.model, _recursive_=True)
|
130
|
-
_load_checkpoint(model, ckpt_path)
|
131
|
-
model = model.to(device)
|
132
|
-
if mode == "eval":
|
133
|
-
model.eval()
|
134
|
-
return model
|
135
|
-
|
136
|
-
|
137
|
-
def _hf_download(model_id):
|
138
|
-
from huggingface_hub import hf_hub_download
|
139
|
-
|
140
|
-
config_name, checkpoint_name = HF_MODEL_ID_TO_FILENAMES[model_id]
|
141
|
-
ckpt_path = hf_hub_download(repo_id=model_id, filename=checkpoint_name)
|
142
|
-
return config_name, ckpt_path
|
143
|
-
|
144
|
-
|
145
|
-
def build_sam2_hf(model_id, **kwargs):
|
146
|
-
config_name, ckpt_path = _hf_download(model_id)
|
147
|
-
return build_sam2(config_file=config_name, ckpt_path=ckpt_path, **kwargs)
|
148
|
-
|
149
|
-
|
150
|
-
def build_sam2_video_predictor_hf(model_id, **kwargs):
|
151
|
-
config_name, ckpt_path = _hf_download(model_id)
|
152
|
-
return build_sam2_video_predictor(
|
153
|
-
config_file=config_name, ckpt_path=ckpt_path, **kwargs
|
154
|
-
)
|
155
|
-
|
156
|
-
|
157
|
-
def _load_checkpoint(model, ckpt_path):
|
158
|
-
if ckpt_path is not None:
|
159
|
-
sd = torch.load(ckpt_path, map_location="cpu", weights_only=True)["model"]
|
160
|
-
missing_keys, unexpected_keys = model.load_state_dict(sd)
|
161
|
-
if missing_keys:
|
162
|
-
logging.error(missing_keys)
|
163
|
-
raise RuntimeError()
|
164
|
-
if unexpected_keys:
|
165
|
-
logging.error(unexpected_keys)
|
166
|
-
raise RuntimeError()
|
167
|
-
logging.info("Loaded checkpoint sucessfully")
|
@@ -1,113 +0,0 @@
|
|
1
|
-
# @package _global_
|
2
|
-
|
3
|
-
# Model
|
4
|
-
model:
|
5
|
-
_target_: sam2.modeling.sam2_base.SAM2Base
|
6
|
-
image_encoder:
|
7
|
-
_target_: sam2.modeling.backbones.image_encoder.ImageEncoder
|
8
|
-
scalp: 1
|
9
|
-
trunk:
|
10
|
-
_target_: sam2.modeling.backbones.hieradet.Hiera
|
11
|
-
embed_dim: 112
|
12
|
-
num_heads: 2
|
13
|
-
neck:
|
14
|
-
_target_: sam2.modeling.backbones.image_encoder.FpnNeck
|
15
|
-
position_encoding:
|
16
|
-
_target_: sam2.modeling.position_encoding.PositionEmbeddingSine
|
17
|
-
num_pos_feats: 256
|
18
|
-
normalize: true
|
19
|
-
scale: null
|
20
|
-
temperature: 10000
|
21
|
-
d_model: 256
|
22
|
-
backbone_channel_list: [896, 448, 224, 112]
|
23
|
-
fpn_top_down_levels: [2, 3] # output level 0 and 1 directly use the backbone features
|
24
|
-
fpn_interp_model: nearest
|
25
|
-
|
26
|
-
memory_attention:
|
27
|
-
_target_: sam2.modeling.memory_attention.MemoryAttention
|
28
|
-
d_model: 256
|
29
|
-
pos_enc_at_input: true
|
30
|
-
layer:
|
31
|
-
_target_: sam2.modeling.memory_attention.MemoryAttentionLayer
|
32
|
-
activation: relu
|
33
|
-
dim_feedforward: 2048
|
34
|
-
dropout: 0.1
|
35
|
-
pos_enc_at_attn: false
|
36
|
-
self_attention:
|
37
|
-
_target_: sam2.modeling.sam.transformer.RoPEAttention
|
38
|
-
rope_theta: 10000.0
|
39
|
-
feat_sizes: [32, 32]
|
40
|
-
embedding_dim: 256
|
41
|
-
num_heads: 1
|
42
|
-
downsample_rate: 1
|
43
|
-
dropout: 0.1
|
44
|
-
d_model: 256
|
45
|
-
pos_enc_at_cross_attn_keys: true
|
46
|
-
pos_enc_at_cross_attn_queries: false
|
47
|
-
cross_attention:
|
48
|
-
_target_: sam2.modeling.sam.transformer.RoPEAttention
|
49
|
-
rope_theta: 10000.0
|
50
|
-
feat_sizes: [32, 32]
|
51
|
-
rope_k_repeat: True
|
52
|
-
embedding_dim: 256
|
53
|
-
num_heads: 1
|
54
|
-
downsample_rate: 1
|
55
|
-
dropout: 0.1
|
56
|
-
kv_in_dim: 64
|
57
|
-
num_layers: 4
|
58
|
-
|
59
|
-
memory_encoder:
|
60
|
-
_target_: sam2.modeling.memory_encoder.MemoryEncoder
|
61
|
-
out_dim: 64
|
62
|
-
position_encoding:
|
63
|
-
_target_: sam2.modeling.position_encoding.PositionEmbeddingSine
|
64
|
-
num_pos_feats: 64
|
65
|
-
normalize: true
|
66
|
-
scale: null
|
67
|
-
temperature: 10000
|
68
|
-
mask_downsampler:
|
69
|
-
_target_: sam2.modeling.memory_encoder.MaskDownSampler
|
70
|
-
kernel_size: 3
|
71
|
-
stride: 2
|
72
|
-
padding: 1
|
73
|
-
fuser:
|
74
|
-
_target_: sam2.modeling.memory_encoder.Fuser
|
75
|
-
layer:
|
76
|
-
_target_: sam2.modeling.memory_encoder.CXBlock
|
77
|
-
dim: 256
|
78
|
-
kernel_size: 7
|
79
|
-
padding: 3
|
80
|
-
layer_scale_init_value: 1e-6
|
81
|
-
use_dwconv: True # depth-wise convs
|
82
|
-
num_layers: 2
|
83
|
-
|
84
|
-
num_maskmem: 7
|
85
|
-
image_size: 1024
|
86
|
-
# apply scaled sigmoid on mask logits for memory encoder, and directly feed input mask as output mask
|
87
|
-
sigmoid_scale_for_mem_enc: 20.0
|
88
|
-
sigmoid_bias_for_mem_enc: -10.0
|
89
|
-
use_mask_input_as_output_without_sam: true
|
90
|
-
# Memory
|
91
|
-
directly_add_no_mem_embed: true
|
92
|
-
# use high-resolution feature map in the SAM mask decoder
|
93
|
-
use_high_res_features_in_sam: true
|
94
|
-
# output 3 masks on the first click on initial conditioning frames
|
95
|
-
multimask_output_in_sam: true
|
96
|
-
# SAM heads
|
97
|
-
iou_prediction_use_sigmoid: True
|
98
|
-
# cross-attend to object pointers from other frames (based on SAM output tokens) in the encoder
|
99
|
-
use_obj_ptrs_in_encoder: true
|
100
|
-
add_tpos_enc_to_obj_ptrs: false
|
101
|
-
only_obj_ptrs_in_the_past_for_eval: true
|
102
|
-
# object occlusion prediction
|
103
|
-
pred_obj_scores: true
|
104
|
-
pred_obj_scores_mlp: true
|
105
|
-
fixed_no_obj_ptr: true
|
106
|
-
# multimask tracking settings
|
107
|
-
multimask_output_for_tracking: true
|
108
|
-
use_multimask_token_for_obj_ptr: true
|
109
|
-
multimask_min_pt_num: 0
|
110
|
-
multimask_max_pt_num: 1
|
111
|
-
use_mlp_for_obj_ptr_proj: true
|
112
|
-
# Compilation flag
|
113
|
-
compile_image_encoder: False
|
@@ -1,117 +0,0 @@
|
|
1
|
-
# @package _global_
|
2
|
-
|
3
|
-
# Model
|
4
|
-
model:
|
5
|
-
_target_: sam2.modeling.sam2_base.SAM2Base
|
6
|
-
image_encoder:
|
7
|
-
_target_: sam2.modeling.backbones.image_encoder.ImageEncoder
|
8
|
-
scalp: 1
|
9
|
-
trunk:
|
10
|
-
_target_: sam2.modeling.backbones.hieradet.Hiera
|
11
|
-
embed_dim: 144
|
12
|
-
num_heads: 2
|
13
|
-
stages: [2, 6, 36, 4]
|
14
|
-
global_att_blocks: [23, 33, 43]
|
15
|
-
window_pos_embed_bkg_spatial_size: [7, 7]
|
16
|
-
window_spec: [8, 4, 16, 8]
|
17
|
-
neck:
|
18
|
-
_target_: sam2.modeling.backbones.image_encoder.FpnNeck
|
19
|
-
position_encoding:
|
20
|
-
_target_: sam2.modeling.position_encoding.PositionEmbeddingSine
|
21
|
-
num_pos_feats: 256
|
22
|
-
normalize: true
|
23
|
-
scale: null
|
24
|
-
temperature: 10000
|
25
|
-
d_model: 256
|
26
|
-
backbone_channel_list: [1152, 576, 288, 144]
|
27
|
-
fpn_top_down_levels: [2, 3] # output level 0 and 1 directly use the backbone features
|
28
|
-
fpn_interp_model: nearest
|
29
|
-
|
30
|
-
memory_attention:
|
31
|
-
_target_: sam2.modeling.memory_attention.MemoryAttention
|
32
|
-
d_model: 256
|
33
|
-
pos_enc_at_input: true
|
34
|
-
layer:
|
35
|
-
_target_: sam2.modeling.memory_attention.MemoryAttentionLayer
|
36
|
-
activation: relu
|
37
|
-
dim_feedforward: 2048
|
38
|
-
dropout: 0.1
|
39
|
-
pos_enc_at_attn: false
|
40
|
-
self_attention:
|
41
|
-
_target_: sam2.modeling.sam.transformer.RoPEAttention
|
42
|
-
rope_theta: 10000.0
|
43
|
-
feat_sizes: [32, 32]
|
44
|
-
embedding_dim: 256
|
45
|
-
num_heads: 1
|
46
|
-
downsample_rate: 1
|
47
|
-
dropout: 0.1
|
48
|
-
d_model: 256
|
49
|
-
pos_enc_at_cross_attn_keys: true
|
50
|
-
pos_enc_at_cross_attn_queries: false
|
51
|
-
cross_attention:
|
52
|
-
_target_: sam2.modeling.sam.transformer.RoPEAttention
|
53
|
-
rope_theta: 10000.0
|
54
|
-
feat_sizes: [32, 32]
|
55
|
-
rope_k_repeat: True
|
56
|
-
embedding_dim: 256
|
57
|
-
num_heads: 1
|
58
|
-
downsample_rate: 1
|
59
|
-
dropout: 0.1
|
60
|
-
kv_in_dim: 64
|
61
|
-
num_layers: 4
|
62
|
-
|
63
|
-
memory_encoder:
|
64
|
-
_target_: sam2.modeling.memory_encoder.MemoryEncoder
|
65
|
-
out_dim: 64
|
66
|
-
position_encoding:
|
67
|
-
_target_: sam2.modeling.position_encoding.PositionEmbeddingSine
|
68
|
-
num_pos_feats: 64
|
69
|
-
normalize: true
|
70
|
-
scale: null
|
71
|
-
temperature: 10000
|
72
|
-
mask_downsampler:
|
73
|
-
_target_: sam2.modeling.memory_encoder.MaskDownSampler
|
74
|
-
kernel_size: 3
|
75
|
-
stride: 2
|
76
|
-
padding: 1
|
77
|
-
fuser:
|
78
|
-
_target_: sam2.modeling.memory_encoder.Fuser
|
79
|
-
layer:
|
80
|
-
_target_: sam2.modeling.memory_encoder.CXBlock
|
81
|
-
dim: 256
|
82
|
-
kernel_size: 7
|
83
|
-
padding: 3
|
84
|
-
layer_scale_init_value: 1e-6
|
85
|
-
use_dwconv: True # depth-wise convs
|
86
|
-
num_layers: 2
|
87
|
-
|
88
|
-
num_maskmem: 7
|
89
|
-
image_size: 1024
|
90
|
-
# apply scaled sigmoid on mask logits for memory encoder, and directly feed input mask as output mask
|
91
|
-
sigmoid_scale_for_mem_enc: 20.0
|
92
|
-
sigmoid_bias_for_mem_enc: -10.0
|
93
|
-
use_mask_input_as_output_without_sam: true
|
94
|
-
# Memory
|
95
|
-
directly_add_no_mem_embed: true
|
96
|
-
# use high-resolution feature map in the SAM mask decoder
|
97
|
-
use_high_res_features_in_sam: true
|
98
|
-
# output 3 masks on the first click on initial conditioning frames
|
99
|
-
multimask_output_in_sam: true
|
100
|
-
# SAM heads
|
101
|
-
iou_prediction_use_sigmoid: True
|
102
|
-
# cross-attend to object pointers from other frames (based on SAM output tokens) in the encoder
|
103
|
-
use_obj_ptrs_in_encoder: true
|
104
|
-
add_tpos_enc_to_obj_ptrs: false
|
105
|
-
only_obj_ptrs_in_the_past_for_eval: true
|
106
|
-
# object occlusion prediction
|
107
|
-
pred_obj_scores: true
|
108
|
-
pred_obj_scores_mlp: true
|
109
|
-
fixed_no_obj_ptr: true
|
110
|
-
# multimask tracking settings
|
111
|
-
multimask_output_for_tracking: true
|
112
|
-
use_multimask_token_for_obj_ptr: true
|
113
|
-
multimask_min_pt_num: 0
|
114
|
-
multimask_max_pt_num: 1
|
115
|
-
use_mlp_for_obj_ptr_proj: true
|
116
|
-
# Compilation flag
|
117
|
-
compile_image_encoder: False
|
@@ -1,116 +0,0 @@
|
|
1
|
-
# @package _global_
|
2
|
-
|
3
|
-
# Model
|
4
|
-
model:
|
5
|
-
_target_: sam2.modeling.sam2_base.SAM2Base
|
6
|
-
image_encoder:
|
7
|
-
_target_: sam2.modeling.backbones.image_encoder.ImageEncoder
|
8
|
-
scalp: 1
|
9
|
-
trunk:
|
10
|
-
_target_: sam2.modeling.backbones.hieradet.Hiera
|
11
|
-
embed_dim: 96
|
12
|
-
num_heads: 1
|
13
|
-
stages: [1, 2, 11, 2]
|
14
|
-
global_att_blocks: [7, 10, 13]
|
15
|
-
window_pos_embed_bkg_spatial_size: [7, 7]
|
16
|
-
neck:
|
17
|
-
_target_: sam2.modeling.backbones.image_encoder.FpnNeck
|
18
|
-
position_encoding:
|
19
|
-
_target_: sam2.modeling.position_encoding.PositionEmbeddingSine
|
20
|
-
num_pos_feats: 256
|
21
|
-
normalize: true
|
22
|
-
scale: null
|
23
|
-
temperature: 10000
|
24
|
-
d_model: 256
|
25
|
-
backbone_channel_list: [768, 384, 192, 96]
|
26
|
-
fpn_top_down_levels: [2, 3] # output level 0 and 1 directly use the backbone features
|
27
|
-
fpn_interp_model: nearest
|
28
|
-
|
29
|
-
memory_attention:
|
30
|
-
_target_: sam2.modeling.memory_attention.MemoryAttention
|
31
|
-
d_model: 256
|
32
|
-
pos_enc_at_input: true
|
33
|
-
layer:
|
34
|
-
_target_: sam2.modeling.memory_attention.MemoryAttentionLayer
|
35
|
-
activation: relu
|
36
|
-
dim_feedforward: 2048
|
37
|
-
dropout: 0.1
|
38
|
-
pos_enc_at_attn: false
|
39
|
-
self_attention:
|
40
|
-
_target_: sam2.modeling.sam.transformer.RoPEAttention
|
41
|
-
rope_theta: 10000.0
|
42
|
-
feat_sizes: [32, 32]
|
43
|
-
embedding_dim: 256
|
44
|
-
num_heads: 1
|
45
|
-
downsample_rate: 1
|
46
|
-
dropout: 0.1
|
47
|
-
d_model: 256
|
48
|
-
pos_enc_at_cross_attn_keys: true
|
49
|
-
pos_enc_at_cross_attn_queries: false
|
50
|
-
cross_attention:
|
51
|
-
_target_: sam2.modeling.sam.transformer.RoPEAttention
|
52
|
-
rope_theta: 10000.0
|
53
|
-
feat_sizes: [32, 32]
|
54
|
-
rope_k_repeat: True
|
55
|
-
embedding_dim: 256
|
56
|
-
num_heads: 1
|
57
|
-
downsample_rate: 1
|
58
|
-
dropout: 0.1
|
59
|
-
kv_in_dim: 64
|
60
|
-
num_layers: 4
|
61
|
-
|
62
|
-
memory_encoder:
|
63
|
-
_target_: sam2.modeling.memory_encoder.MemoryEncoder
|
64
|
-
out_dim: 64
|
65
|
-
position_encoding:
|
66
|
-
_target_: sam2.modeling.position_encoding.PositionEmbeddingSine
|
67
|
-
num_pos_feats: 64
|
68
|
-
normalize: true
|
69
|
-
scale: null
|
70
|
-
temperature: 10000
|
71
|
-
mask_downsampler:
|
72
|
-
_target_: sam2.modeling.memory_encoder.MaskDownSampler
|
73
|
-
kernel_size: 3
|
74
|
-
stride: 2
|
75
|
-
padding: 1
|
76
|
-
fuser:
|
77
|
-
_target_: sam2.modeling.memory_encoder.Fuser
|
78
|
-
layer:
|
79
|
-
_target_: sam2.modeling.memory_encoder.CXBlock
|
80
|
-
dim: 256
|
81
|
-
kernel_size: 7
|
82
|
-
padding: 3
|
83
|
-
layer_scale_init_value: 1e-6
|
84
|
-
use_dwconv: True # depth-wise convs
|
85
|
-
num_layers: 2
|
86
|
-
|
87
|
-
num_maskmem: 7
|
88
|
-
image_size: 1024
|
89
|
-
# apply scaled sigmoid on mask logits for memory encoder, and directly feed input mask as output mask
|
90
|
-
sigmoid_scale_for_mem_enc: 20.0
|
91
|
-
sigmoid_bias_for_mem_enc: -10.0
|
92
|
-
use_mask_input_as_output_without_sam: true
|
93
|
-
# Memory
|
94
|
-
directly_add_no_mem_embed: true
|
95
|
-
# use high-resolution feature map in the SAM mask decoder
|
96
|
-
use_high_res_features_in_sam: true
|
97
|
-
# output 3 masks on the first click on initial conditioning frames
|
98
|
-
multimask_output_in_sam: true
|
99
|
-
# SAM heads
|
100
|
-
iou_prediction_use_sigmoid: True
|
101
|
-
# cross-attend to object pointers from other frames (based on SAM output tokens) in the encoder
|
102
|
-
use_obj_ptrs_in_encoder: true
|
103
|
-
add_tpos_enc_to_obj_ptrs: false
|
104
|
-
only_obj_ptrs_in_the_past_for_eval: true
|
105
|
-
# object occlusion prediction
|
106
|
-
pred_obj_scores: true
|
107
|
-
pred_obj_scores_mlp: true
|
108
|
-
fixed_no_obj_ptr: true
|
109
|
-
# multimask tracking settings
|
110
|
-
multimask_output_for_tracking: true
|
111
|
-
use_multimask_token_for_obj_ptr: true
|
112
|
-
multimask_min_pt_num: 0
|
113
|
-
multimask_max_pt_num: 1
|
114
|
-
use_mlp_for_obj_ptr_proj: true
|
115
|
-
# Compilation flag
|
116
|
-
compile_image_encoder: False
|
@@ -1,118 +0,0 @@
|
|
1
|
-
# @package _global_
|
2
|
-
|
3
|
-
# Model
|
4
|
-
model:
|
5
|
-
_target_: sam2.modeling.sam2_base.SAM2Base
|
6
|
-
image_encoder:
|
7
|
-
_target_: sam2.modeling.backbones.image_encoder.ImageEncoder
|
8
|
-
scalp: 1
|
9
|
-
trunk:
|
10
|
-
_target_: sam2.modeling.backbones.hieradet.Hiera
|
11
|
-
embed_dim: 96
|
12
|
-
num_heads: 1
|
13
|
-
stages: [1, 2, 7, 2]
|
14
|
-
global_att_blocks: [5, 7, 9]
|
15
|
-
window_pos_embed_bkg_spatial_size: [7, 7]
|
16
|
-
neck:
|
17
|
-
_target_: sam2.modeling.backbones.image_encoder.FpnNeck
|
18
|
-
position_encoding:
|
19
|
-
_target_: sam2.modeling.position_encoding.PositionEmbeddingSine
|
20
|
-
num_pos_feats: 256
|
21
|
-
normalize: true
|
22
|
-
scale: null
|
23
|
-
temperature: 10000
|
24
|
-
d_model: 256
|
25
|
-
backbone_channel_list: [768, 384, 192, 96]
|
26
|
-
fpn_top_down_levels: [2, 3] # output level 0 and 1 directly use the backbone features
|
27
|
-
fpn_interp_model: nearest
|
28
|
-
|
29
|
-
memory_attention:
|
30
|
-
_target_: sam2.modeling.memory_attention.MemoryAttention
|
31
|
-
d_model: 256
|
32
|
-
pos_enc_at_input: true
|
33
|
-
layer:
|
34
|
-
_target_: sam2.modeling.memory_attention.MemoryAttentionLayer
|
35
|
-
activation: relu
|
36
|
-
dim_feedforward: 2048
|
37
|
-
dropout: 0.1
|
38
|
-
pos_enc_at_attn: false
|
39
|
-
self_attention:
|
40
|
-
_target_: sam2.modeling.sam.transformer.RoPEAttention
|
41
|
-
rope_theta: 10000.0
|
42
|
-
feat_sizes: [32, 32]
|
43
|
-
embedding_dim: 256
|
44
|
-
num_heads: 1
|
45
|
-
downsample_rate: 1
|
46
|
-
dropout: 0.1
|
47
|
-
d_model: 256
|
48
|
-
pos_enc_at_cross_attn_keys: true
|
49
|
-
pos_enc_at_cross_attn_queries: false
|
50
|
-
cross_attention:
|
51
|
-
_target_: sam2.modeling.sam.transformer.RoPEAttention
|
52
|
-
rope_theta: 10000.0
|
53
|
-
feat_sizes: [32, 32]
|
54
|
-
rope_k_repeat: True
|
55
|
-
embedding_dim: 256
|
56
|
-
num_heads: 1
|
57
|
-
downsample_rate: 1
|
58
|
-
dropout: 0.1
|
59
|
-
kv_in_dim: 64
|
60
|
-
num_layers: 4
|
61
|
-
|
62
|
-
memory_encoder:
|
63
|
-
_target_: sam2.modeling.memory_encoder.MemoryEncoder
|
64
|
-
out_dim: 64
|
65
|
-
position_encoding:
|
66
|
-
_target_: sam2.modeling.position_encoding.PositionEmbeddingSine
|
67
|
-
num_pos_feats: 64
|
68
|
-
normalize: true
|
69
|
-
scale: null
|
70
|
-
temperature: 10000
|
71
|
-
mask_downsampler:
|
72
|
-
_target_: sam2.modeling.memory_encoder.MaskDownSampler
|
73
|
-
kernel_size: 3
|
74
|
-
stride: 2
|
75
|
-
padding: 1
|
76
|
-
fuser:
|
77
|
-
_target_: sam2.modeling.memory_encoder.Fuser
|
78
|
-
layer:
|
79
|
-
_target_: sam2.modeling.memory_encoder.CXBlock
|
80
|
-
dim: 256
|
81
|
-
kernel_size: 7
|
82
|
-
padding: 3
|
83
|
-
layer_scale_init_value: 1e-6
|
84
|
-
use_dwconv: True # depth-wise convs
|
85
|
-
num_layers: 2
|
86
|
-
|
87
|
-
num_maskmem: 7
|
88
|
-
image_size: 1024
|
89
|
-
# apply scaled sigmoid on mask logits for memory encoder, and directly feed input mask as output mask
|
90
|
-
# SAM decoder
|
91
|
-
sigmoid_scale_for_mem_enc: 20.0
|
92
|
-
sigmoid_bias_for_mem_enc: -10.0
|
93
|
-
use_mask_input_as_output_without_sam: true
|
94
|
-
# Memory
|
95
|
-
directly_add_no_mem_embed: true
|
96
|
-
# use high-resolution feature map in the SAM mask decoder
|
97
|
-
use_high_res_features_in_sam: true
|
98
|
-
# output 3 masks on the first click on initial conditioning frames
|
99
|
-
multimask_output_in_sam: true
|
100
|
-
# SAM heads
|
101
|
-
iou_prediction_use_sigmoid: True
|
102
|
-
# cross-attend to object pointers from other frames (based on SAM output tokens) in the encoder
|
103
|
-
use_obj_ptrs_in_encoder: true
|
104
|
-
add_tpos_enc_to_obj_ptrs: false
|
105
|
-
only_obj_ptrs_in_the_past_for_eval: true
|
106
|
-
# object occlusion prediction
|
107
|
-
pred_obj_scores: true
|
108
|
-
pred_obj_scores_mlp: true
|
109
|
-
fixed_no_obj_ptr: true
|
110
|
-
# multimask tracking settings
|
111
|
-
multimask_output_for_tracking: true
|
112
|
-
use_multimask_token_for_obj_ptr: true
|
113
|
-
multimask_min_pt_num: 0
|
114
|
-
multimask_max_pt_num: 1
|
115
|
-
use_mlp_for_obj_ptr_proj: true
|
116
|
-
# Compilation flag
|
117
|
-
# HieraT does not currently support compilation, should always be set to False
|
118
|
-
compile_image_encoder: False
|
sam2/modeling/__init__.py
DELETED