ultralytics 8.2.71__py3-none-any.whl → 8.2.73__py3-none-any.whl

ultralytics/models/sam2/modules/decoders.py

@@ -1,305 +0,0 @@
-# Ultralytics YOLO 🚀, AGPL-3.0 license
-
-from typing import List, Optional, Tuple, Type
-
-import torch
-from torch import nn
-
-from ultralytics.nn.modules import MLP, LayerNorm2d
-
-
-class MaskDecoder(nn.Module):
-    """Transformer-based decoder predicting instance segmentation masks from image and prompt embeddings."""
-
-    def __init__(
-        self,
-        transformer_dim: int,
-        transformer: nn.Module,
-        num_multimask_outputs: int = 3,
-        activation: Type[nn.Module] = nn.GELU,
-        iou_head_depth: int = 3,
-        iou_head_hidden_dim: int = 256,
-        use_high_res_features: bool = False,
-        iou_prediction_use_sigmoid=False,
-        dynamic_multimask_via_stability=False,
-        dynamic_multimask_stability_delta=0.05,
-        dynamic_multimask_stability_thresh=0.98,
-        pred_obj_scores: bool = False,
-        pred_obj_scores_mlp: bool = False,
-        use_multimask_token_for_obj_ptr: bool = False,
-    ) -> None:
-        """
-        Initializes the MaskDecoder module for predicting instance segmentation masks.
-
-        Args:
-            transformer_dim (int): Channel dimension of the transformer.
-            transformer (nn.Module): Transformer used to predict masks.
-            num_multimask_outputs (int): Number of masks to predict when disambiguating masks.
-            activation (Type[nn.Module]): Type of activation to use when upscaling masks.
-            iou_head_depth (int): Depth of the MLP used to predict mask quality.
-            iou_head_hidden_dim (int): Hidden dimension of the MLP used to predict mask quality.
-            use_high_res_features (bool): Whether to use high-resolution features.
-            iou_prediction_use_sigmoid (bool): Whether to use sigmoid for IOU prediction.
-            dynamic_multimask_via_stability (bool): Whether to use dynamic multimask via stability.
-            dynamic_multimask_stability_delta (float): Delta value for dynamic multimask stability.
-            dynamic_multimask_stability_thresh (float): Threshold for dynamic multimask stability.
-            pred_obj_scores (bool): Whether to predict object scores.
-            pred_obj_scores_mlp (bool): Whether to use MLP for object score prediction.
-            use_multimask_token_for_obj_ptr (bool): Whether to use multimask token for object pointer.
-
-        Attributes:
-            transformer_dim (int): Channel dimension of the transformer.
-            transformer (nn.Module): Transformer used to predict masks.
-            num_multimask_outputs (int): Number of masks to predict when disambiguating masks.
-            iou_token (nn.Embedding): Embedding for IOU token.
-            num_mask_tokens (int): Total number of mask tokens.
-            mask_tokens (nn.Embedding): Embedding for mask tokens.
-            pred_obj_scores (bool): Whether to predict object scores.
-            obj_score_token (nn.Embedding): Embedding for object score token.
-            use_multimask_token_for_obj_ptr (bool): Whether to use multimask token for object pointer.
-            output_upscaling (nn.Sequential): Upscaling layers for output.
-            use_high_res_features (bool): Whether to use high-resolution features.
-            conv_s0 (nn.Conv2d): Convolutional layer for high-resolution features (s0).
-            conv_s1 (nn.Conv2d): Convolutional layer for high-resolution features (s1).
-            output_hypernetworks_mlps (nn.ModuleList): List of MLPs for output hypernetworks.
-            iou_prediction_head (MLP): MLP for IOU prediction.
-            pred_obj_score_head (nn.Linear | MLP): Linear layer or MLP for object score prediction.
-            dynamic_multimask_via_stability (bool): Whether to use dynamic multimask via stability.
-            dynamic_multimask_stability_delta (float): Delta value for dynamic multimask stability.
-        """
-        super().__init__()
-        self.transformer_dim = transformer_dim
-        self.transformer = transformer
-
-        self.num_multimask_outputs = num_multimask_outputs
-
-        self.iou_token = nn.Embedding(1, transformer_dim)
-        self.num_mask_tokens = num_multimask_outputs + 1
-        self.mask_tokens = nn.Embedding(self.num_mask_tokens, transformer_dim)
-
-        self.pred_obj_scores = pred_obj_scores
-        if self.pred_obj_scores:
-            self.obj_score_token = nn.Embedding(1, transformer_dim)
-        self.use_multimask_token_for_obj_ptr = use_multimask_token_for_obj_ptr
-
-        self.output_upscaling = nn.Sequential(
-            nn.ConvTranspose2d(transformer_dim, transformer_dim // 4, kernel_size=2, stride=2),
-            LayerNorm2d(transformer_dim // 4),
-            activation(),
-            nn.ConvTranspose2d(transformer_dim // 4, transformer_dim // 8, kernel_size=2, stride=2),
-            activation(),
-        )
-        self.use_high_res_features = use_high_res_features
-        if use_high_res_features:
-            self.conv_s0 = nn.Conv2d(transformer_dim, transformer_dim // 8, kernel_size=1, stride=1)
-            self.conv_s1 = nn.Conv2d(transformer_dim, transformer_dim // 4, kernel_size=1, stride=1)
-
-        self.output_hypernetworks_mlps = nn.ModuleList(
-            [MLP(transformer_dim, transformer_dim, transformer_dim // 8, 3) for _ in range(self.num_mask_tokens)]
-        )
-
-        self.iou_prediction_head = MLP(
-            transformer_dim,
-            iou_head_hidden_dim,
-            self.num_mask_tokens,
-            iou_head_depth,
-            sigmoid=iou_prediction_use_sigmoid,
-        )
-        if self.pred_obj_scores:
-            self.pred_obj_score_head = nn.Linear(transformer_dim, 1)
-            if pred_obj_scores_mlp:
-                self.pred_obj_score_head = MLP(transformer_dim, transformer_dim, 1, 3)
-
-        # When outputting a single mask, optionally we can dynamically fall back to the best
-        # multimask output token if the single mask output token gives low stability scores.
-        self.dynamic_multimask_via_stability = dynamic_multimask_via_stability
-        self.dynamic_multimask_stability_delta = dynamic_multimask_stability_delta
-        self.dynamic_multimask_stability_thresh = dynamic_multimask_stability_thresh
-
-    def forward(
-        self,
-        image_embeddings: torch.Tensor,
-        image_pe: torch.Tensor,
-        sparse_prompt_embeddings: torch.Tensor,
-        dense_prompt_embeddings: torch.Tensor,
-        multimask_output: bool,
-        repeat_image: bool,
-        high_res_features: Optional[List[torch.Tensor]] = None,
-    ) -> Tuple[torch.Tensor, torch.Tensor]:
-        """
-        Predicts masks given image and prompt embeddings.
-
-        Args:
-            image_embeddings (torch.Tensor): Embeddings from the image encoder.
-            image_pe (torch.Tensor): Positional encoding with the shape of image_embeddings.
-            sparse_prompt_embeddings (torch.Tensor): Embeddings of the points and boxes.
-            dense_prompt_embeddings (torch.Tensor): Embeddings of the mask inputs.
-            multimask_output (bool): Whether to return multiple masks or a single mask.
-            repeat_image (bool): Flag to repeat the image embeddings.
-            high_res_features (List[torch.Tensor] | None): Optional high-resolution features.
-
-        Returns:
-            (Tuple[torch.Tensor, torch.Tensor, torch.Tensor]): A tuple containing:
-                - masks (torch.Tensor): Batched predicted masks.
-                - iou_pred (torch.Tensor): Batched predictions of mask quality.
-                - sam_tokens_out (torch.Tensor): Batched SAM token for mask output.
-
-        Examples:
-            >>> image_embeddings = torch.rand(1, 256, 64, 64)
-            >>> image_pe = torch.rand(1, 256, 64, 64)
-            >>> sparse_prompt_embeddings = torch.rand(1, 2, 256)
-            >>> dense_prompt_embeddings = torch.rand(1, 256, 64, 64)
-            >>> decoder = MaskDecoder(256, transformer)
-            >>> masks, iou_pred, sam_tokens_out = decoder.forward(image_embeddings, image_pe,
-            ...     sparse_prompt_embeddings, dense_prompt_embeddings, True, False)
-        """
-        masks, iou_pred, mask_tokens_out, object_score_logits = self.predict_masks(
-            image_embeddings=image_embeddings,
-            image_pe=image_pe,
-            sparse_prompt_embeddings=sparse_prompt_embeddings,
-            dense_prompt_embeddings=dense_prompt_embeddings,
-            repeat_image=repeat_image,
-            high_res_features=high_res_features,
-        )
-
-        # Select the correct mask or masks for output
-        if multimask_output:
-            masks = masks[:, 1:, :, :]
-            iou_pred = iou_pred[:, 1:]
-        elif self.dynamic_multimask_via_stability and not self.training:
-            masks, iou_pred = self._dynamic_multimask_via_stability(masks, iou_pred)
-        else:
-            masks = masks[:, 0:1, :, :]
-            iou_pred = iou_pred[:, 0:1]
-
-        if multimask_output and self.use_multimask_token_for_obj_ptr:
-            sam_tokens_out = mask_tokens_out[:, 1:]  # [b, 3, c] shape
-        else:
-            # Take the mask output token. Here we *always* use the token for single mask output.
-            # At test time, even if we track after 1-click (and using multimask_output=True),
-            # we still take the single mask token here. The rationale is that we always track
-            # after multiple clicks during training, so the past tokens seen during training
-            # are always the single mask token (and we'll let it be the object-memory token).
-            sam_tokens_out = mask_tokens_out[:, 0:1]  # [b, 1, c] shape
-
-        # Prepare output
-        return masks, iou_pred, sam_tokens_out, object_score_logits
-
-    def predict_masks(
-        self,
-        image_embeddings: torch.Tensor,
-        image_pe: torch.Tensor,
-        sparse_prompt_embeddings: torch.Tensor,
-        dense_prompt_embeddings: torch.Tensor,
-        repeat_image: bool,
-        high_res_features: Optional[List[torch.Tensor]] = None,
-    ) -> Tuple[torch.Tensor, torch.Tensor]:
-        """Predicts instance segmentation masks from image and prompt embeddings using a transformer architecture."""
-        # Concatenate output tokens
-        s = 0
-        if self.pred_obj_scores:
-            output_tokens = torch.cat(
-                [
-                    self.obj_score_token.weight,
-                    self.iou_token.weight,
-                    self.mask_tokens.weight,
-                ],
-                dim=0,
-            )
-            s = 1
-        else:
-            output_tokens = torch.cat([self.iou_token.weight, self.mask_tokens.weight], dim=0)
-        output_tokens = output_tokens.unsqueeze(0).expand(sparse_prompt_embeddings.size(0), -1, -1)
-        tokens = torch.cat((output_tokens, sparse_prompt_embeddings), dim=1)
-
-        # Expand per-image data in batch direction to be per-mask
-        if repeat_image:
-            src = torch.repeat_interleave(image_embeddings, tokens.shape[0], dim=0)
-        else:
-            assert image_embeddings.shape[0] == tokens.shape[0]
-            src = image_embeddings
-        src = src + dense_prompt_embeddings
-        assert image_pe.size(0) == 1, "image_pe should have size 1 in batch dim (from `get_dense_pe()`)"
-        pos_src = torch.repeat_interleave(image_pe, tokens.shape[0], dim=0)
-        b, c, h, w = src.shape
-
-        # Run the transformer
-        hs, src = self.transformer(src, pos_src, tokens)
-        iou_token_out = hs[:, s, :]
-        mask_tokens_out = hs[:, s + 1 : (s + 1 + self.num_mask_tokens), :]
-
-        # Upscale mask embeddings and predict masks using the mask tokens
-        src = src.transpose(1, 2).view(b, c, h, w)
-        if not self.use_high_res_features:
-            upscaled_embedding = self.output_upscaling(src)
-        else:
-            dc1, ln1, act1, dc2, act2 = self.output_upscaling
-            feat_s0, feat_s1 = high_res_features
-            upscaled_embedding = act1(ln1(dc1(src) + feat_s1))
-            upscaled_embedding = act2(dc2(upscaled_embedding) + feat_s0)
-
-        hyper_in_list: List[torch.Tensor] = []
-        for i in range(self.num_mask_tokens):
-            hyper_in_list.append(self.output_hypernetworks_mlps[i](mask_tokens_out[:, i, :]))
-        hyper_in = torch.stack(hyper_in_list, dim=1)
-        b, c, h, w = upscaled_embedding.shape
-        masks = (hyper_in @ upscaled_embedding.view(b, c, h * w)).view(b, -1, h, w)
-
-        # Generate mask quality predictions
-        iou_pred = self.iou_prediction_head(iou_token_out)
-        if self.pred_obj_scores:
-            assert s == 1
-            object_score_logits = self.pred_obj_score_head(hs[:, 0, :])
-        else:
-            # Obj scores logits - default to 10.0, i.e. assuming the object is present, sigmoid(10)=1
-            object_score_logits = 10.0 * iou_pred.new_ones(iou_pred.shape[0], 1)
-
-        return masks, iou_pred, mask_tokens_out, object_score_logits
-
-    def _get_stability_scores(self, mask_logits):
-        """Computes mask stability scores based on IoU between upper and lower thresholds."""
-        mask_logits = mask_logits.flatten(-2)
-        stability_delta = self.dynamic_multimask_stability_delta
-        area_i = torch.sum(mask_logits > stability_delta, dim=-1).float()
-        area_u = torch.sum(mask_logits > -stability_delta, dim=-1).float()
-        stability_scores = torch.where(area_u > 0, area_i / area_u, 1.0)
-        return stability_scores
-
-    def _dynamic_multimask_via_stability(self, all_mask_logits, all_iou_scores):
-        """
-        Dynamically selects the most stable mask output based on stability scores and IoU predictions.
-
-        When outputting a single mask, if the stability score from the current single-mask output (based on output token
-        0) falls below a threshold, we instead select from multi-mask outputs (based on output token 1~3) the mask with
-        the highest predicted IoU score.
-
-        This is intended to ensure a valid mask for both clicking and tracking.
-        """
-        # The best mask from multimask output tokens (1~3)
-        multimask_logits = all_mask_logits[:, 1:, :, :]
-        multimask_iou_scores = all_iou_scores[:, 1:]
-        best_scores_inds = torch.argmax(multimask_iou_scores, dim=-1)
-        batch_inds = torch.arange(multimask_iou_scores.size(0), device=all_iou_scores.device)
-        best_multimask_logits = multimask_logits[batch_inds, best_scores_inds]
-        best_multimask_logits = best_multimask_logits.unsqueeze(1)
-        best_multimask_iou_scores = multimask_iou_scores[batch_inds, best_scores_inds]
-        best_multimask_iou_scores = best_multimask_iou_scores.unsqueeze(1)
-
-        # The mask from singlemask output token 0 and its stability score
-        singlemask_logits = all_mask_logits[:, 0:1, :, :]
-        singlemask_iou_scores = all_iou_scores[:, 0:1]
-        stability_scores = self._get_stability_scores(singlemask_logits)
-        is_stable = stability_scores >= self.dynamic_multimask_stability_thresh
-
-        # Dynamically fall back to best multimask output upon low stability scores.
-        mask_logits_out = torch.where(
-            is_stable[..., None, None].expand_as(singlemask_logits),
-            singlemask_logits,
-            best_multimask_logits,
-        )
-        iou_scores_out = torch.where(
-            is_stable.expand_as(singlemask_iou_scores),
-            singlemask_iou_scores,
-            best_multimask_iou_scores,
-        )
-        return mask_logits_out, iou_scores_out

ultralytics/models/sam2/modules/encoders.py

@@ -1,332 +0,0 @@
-# Ultralytics YOLO 🚀, AGPL-3.0 license
-
-from typing import List, Optional, Tuple
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-from ultralytics.models.sam.modules.encoders import PatchEmbed
-
-from .sam2_blocks import CXBlock, Fuser, MaskDownSampler, MultiScaleBlock, PositionEmbeddingSine
-
-
-class MemoryEncoder(nn.Module):
-    """Encodes pixel features and masks into a memory representation for efficient image segmentation."""
-
-    def __init__(
-        self,
-        out_dim,
-        in_dim=256,  # in_dim of pix_feats
-    ):
-        """Initializes the MemoryEncoder module for encoding pixel features and masks in SAM-like models."""
-        super().__init__()
-
-        self.mask_downsampler = MaskDownSampler(kernel_size=3, stride=2, padding=1)
-
-        self.pix_feat_proj = nn.Conv2d(in_dim, in_dim, kernel_size=1)
-        self.fuser = Fuser(CXBlock(dim=256), num_layers=2)
-        self.position_encoding = PositionEmbeddingSine(num_pos_feats=64)
-        self.out_proj = nn.Identity()
-        if out_dim != in_dim:
-            self.out_proj = nn.Conv2d(in_dim, out_dim, kernel_size=1)
-
-    def forward(
-        self,
-        pix_feat: torch.Tensor,
-        masks: torch.Tensor,
-        skip_mask_sigmoid: bool = False,
-    ) -> Tuple[torch.Tensor, torch.Tensor]:
-        """Processes pixel features and masks, fusing them to generate encoded memory representations."""
-        if not skip_mask_sigmoid:
-            masks = F.sigmoid(masks)
-        masks = self.mask_downsampler(masks)
-
-        # Fuse pix_feats and downsampled masks, in case the visual features are on CPU, cast them to CUDA
-        pix_feat = pix_feat.to(masks.device)
-
-        x = self.pix_feat_proj(pix_feat)
-        x = x + masks
-        x = self.fuser(x)
-        x = self.out_proj(x)
-
-        pos = self.position_encoding(x).to(x.dtype)
-
-        return {"vision_features": x, "vision_pos_enc": [pos]}
-
-
-class ImageEncoder(nn.Module):
-    """Encodes images using a trunk-neck architecture, producing multiscale features and positional encodings."""
-
-    def __init__(
-        self,
-        trunk: nn.Module,
-        neck: nn.Module,
-        scalp: int = 0,
-    ):
-        """Initializes an image encoder with a trunk, neck, and optional scalp for feature extraction."""
-        super().__init__()
-        self.trunk = trunk
-        self.neck = neck
-        self.scalp = scalp
-        assert (
-            self.trunk.channel_list == self.neck.backbone_channel_list
-        ), f"Channel dims of trunk {self.trunk.channel_list} and neck {self.neck.backbone_channel_list} do not match."
-
-    def forward(self, sample: torch.Tensor):
-        """Processes image input through trunk and neck, returning features, positional encodings, and FPN outputs."""
-        features, pos = self.neck(self.trunk(sample))
-        if self.scalp > 0:
-            # Discard the lowest resolution features
-            features, pos = features[: -self.scalp], pos[: -self.scalp]
-
-        src = features[-1]
-        output = {
-            "vision_features": src,
-            "vision_pos_enc": pos,
-            "backbone_fpn": features,
-        }
-        return output
-
-
-class FpnNeck(nn.Module):
-    """Feature Pyramid Network (FPN) neck variant for multiscale feature fusion in object detection models."""
-
-    def __init__(
-        self,
-        d_model: int,
-        backbone_channel_list: List[int],
-        kernel_size: int = 1,
-        stride: int = 1,
-        padding: int = 0,
-        fpn_interp_model: str = "bilinear",
-        fuse_type: str = "sum",
-        fpn_top_down_levels: Optional[List[int]] = None,
-    ):
-        """
-        Initializes a modified Feature Pyramid Network (FPN) neck.
-
-        This FPN variant removes the output convolution and uses bicubic interpolation for feature resizing,
-        similar to ViT positional embedding interpolation.
-
-        Args:
-            d_model (int): Dimension of the model.
-            backbone_channel_list (List[int]): List of channel dimensions from the backbone.
-            kernel_size (int): Kernel size for the convolutional layers.
-            stride (int): Stride for the convolutional layers.
-            padding (int): Padding for the convolutional layers.
-            fpn_interp_model (str): Interpolation mode for FPN feature resizing.
-            fuse_type (str): Type of feature fusion, either 'sum' or 'avg'.
-            fpn_top_down_levels (Optional[List[int]]): Levels to have top-down features in outputs.
-
-        Attributes:
-            position_encoding (PositionEmbeddingSine): Sinusoidal positional encoding.
-            convs (nn.ModuleList): List of convolutional layers for each backbone level.
-            backbone_channel_list (List[int]): List of channel dimensions from the backbone.
-            fpn_interp_model (str): Interpolation mode for FPN feature resizing.
-            fuse_type (str): Type of feature fusion.
-            fpn_top_down_levels (List[int]): Levels with top-down feature propagation.
-
-        Examples:
-            >>> backbone_channels = [64, 128, 256, 512]
-            >>> fpn_neck = FpnNeck(256, backbone_channels)
-            >>> print(fpn_neck)
-        """
-        super().__init__()
-        self.position_encoding = PositionEmbeddingSine(num_pos_feats=256)
-        self.convs = nn.ModuleList()
-        self.backbone_channel_list = backbone_channel_list
-        for dim in backbone_channel_list:
-            current = nn.Sequential()
-            current.add_module(
-                "conv",
-                nn.Conv2d(
-                    in_channels=dim,
-                    out_channels=d_model,
-                    kernel_size=kernel_size,
-                    stride=stride,
-                    padding=padding,
-                ),
-            )
-
-            self.convs.append(current)
-        self.fpn_interp_model = fpn_interp_model
-        assert fuse_type in ["sum", "avg"]
-        self.fuse_type = fuse_type
-
-        # levels to have top-down features in its outputs
-        # e.g. if fpn_top_down_levels is [2, 3], then only outputs of level 2 and 3
-        # have top-down propagation, while outputs of level 0 and level 1 have only
-        # lateral features from the same backbone level.
-        if fpn_top_down_levels is None:
-            # default is to have top-down features on all levels
-            fpn_top_down_levels = range(len(self.convs))
-        self.fpn_top_down_levels = list(fpn_top_down_levels)
-
-    def forward(self, xs: List[torch.Tensor]):
-        """
-        Performs forward pass through the Feature Pyramid Network (FPN) neck.
-
-        Args:
-            xs (List[torch.Tensor]): List of input tensors from the backbone, with shape (B, C, H, W) for each tensor.
-
-        Returns:
-            (Tuple[List[torch.Tensor], List[torch.Tensor]]): A tuple containing two lists:
-                - out: List of output feature maps after FPN processing, with shape (B, d_model, H, W) for each tensor.
-                - pos: List of positional encodings corresponding to each output feature map.
-
-        Examples:
-            >>> fpn_neck = FpnNeck(d_model=256, backbone_channel_list=[64, 128, 256, 512])
-            >>> inputs = [torch.rand(1, c, 32, 32) for c in [64, 128, 256, 512]]
-            >>> outputs, positions = fpn_neck(inputs)
-        """
-        out = [None] * len(self.convs)
-        pos = [None] * len(self.convs)
-        assert len(xs) == len(self.convs)
-        # fpn forward pass
-        # see https://github.com/facebookresearch/detectron2/blob/main/detectron2/modeling/backbone/fpn.py
-        prev_features = None
-        # forward in top-down order (from low to high resolution)
-        n = len(self.convs) - 1
-        for i in range(n, -1, -1):
-            x = xs[i]
-            lateral_features = self.convs[n - i](x)
-            if i in self.fpn_top_down_levels and prev_features is not None:
-                top_down_features = F.interpolate(
-                    prev_features.to(dtype=torch.float32),
-                    scale_factor=2.0,
-                    mode=self.fpn_interp_model,
-                    align_corners=(None if self.fpn_interp_model == "nearest" else False),
-                    antialias=False,
-                )
-                prev_features = lateral_features + top_down_features
-                if self.fuse_type == "avg":
-                    prev_features /= 2
-            else:
-                prev_features = lateral_features
-            x_out = prev_features
-            out[i] = x_out
-            pos[i] = self.position_encoding(x_out).to(x_out.dtype)
-
-        return out, pos
-
-
-class Hiera(nn.Module):
-    """Hierarchical vision transformer for efficient multiscale feature extraction in image processing tasks."""
-
-    def __init__(
-        self,
-        embed_dim: int = 96,  # initial embed dim
-        num_heads: int = 1,  # initial number of heads
-        drop_path_rate: float = 0.0,  # stochastic depth
-        q_pool: int = 3,  # number of q_pool stages
-        q_stride: Tuple[int, int] = (2, 2),  # downsample stride bet. stages
-        stages: Tuple[int, ...] = (2, 3, 16, 3),  # blocks per stage
-        dim_mul: float = 2.0,  # dim_mul factor at stage shift
-        head_mul: float = 2.0,  # head_mul factor at stage shift
-        window_pos_embed_bkg_spatial_size: Tuple[int, int] = (14, 14),
-        # window size per stage, when not using global att.
-        window_spec: Tuple[int, ...] = (
-            8,
-            4,
-            14,
-            7,
-        ),
-        # global attn in these blocks
-        global_att_blocks: Tuple[int, ...] = (
-            12,
-            16,
-            20,
-        ),
-        return_interm_layers=True,  # return feats from every stage
-    ):
-        """Initializes a Hiera model with configurable architecture for hierarchical vision transformers."""
-        super().__init__()
-
-        assert len(stages) == len(window_spec)
-        self.window_spec = window_spec
-
-        depth = sum(stages)
-        self.q_stride = q_stride
-        self.stage_ends = [sum(stages[:i]) - 1 for i in range(1, len(stages) + 1)]
-        assert 0 <= q_pool <= len(self.stage_ends[:-1])
-        self.q_pool_blocks = [x + 1 for x in self.stage_ends[:-1]][:q_pool]
-        self.return_interm_layers = return_interm_layers
-
-        self.patch_embed = PatchEmbed(
-            embed_dim=embed_dim,
-            kernel_size=(7, 7),
-            stride=(4, 4),
-            padding=(3, 3),
-        )
-        # Which blocks have global att?
-        self.global_att_blocks = global_att_blocks
-
-        # Windowed positional embedding (https://arxiv.org/abs/2311.05613)
-        self.window_pos_embed_bkg_spatial_size = window_pos_embed_bkg_spatial_size
-        self.pos_embed = nn.Parameter(torch.zeros(1, embed_dim, *self.window_pos_embed_bkg_spatial_size))
-        self.pos_embed_window = nn.Parameter(torch.zeros(1, embed_dim, self.window_spec[0], self.window_spec[0]))
-
-        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
-
-        cur_stage = 1
-        self.blocks = nn.ModuleList()
-
-        for i in range(depth):
-            dim_out = embed_dim
-            # lags by a block, so first block of
-            # next stage uses an initial window size
-            # of previous stage and final window size of current stage
-            window_size = self.window_spec[cur_stage - 1]
-
-            if self.global_att_blocks is not None:
-                window_size = 0 if i in self.global_att_blocks else window_size
-
-            if i - 1 in self.stage_ends:
-                dim_out = int(embed_dim * dim_mul)
-                num_heads = int(num_heads * head_mul)
-                cur_stage += 1
-
-            block = MultiScaleBlock(
-                dim=embed_dim,
-                dim_out=dim_out,
-                num_heads=num_heads,
-                drop_path=dpr[i],
-                q_stride=self.q_stride if i in self.q_pool_blocks else None,
-                window_size=window_size,
-            )
-
-            embed_dim = dim_out
-            self.blocks.append(block)
-
-        self.channel_list = (
-            [self.blocks[i].dim_out for i in self.stage_ends[::-1]]
-            if return_interm_layers
-            else [self.blocks[-1].dim_out]
-        )
-
-    def _get_pos_embed(self, hw: Tuple[int, int]) -> torch.Tensor:
-        """Generate positional embeddings by interpolating and combining window and background embeddings."""
-        h, w = hw
-        window_embed = self.pos_embed_window
-        pos_embed = F.interpolate(self.pos_embed, size=(h, w), mode="bicubic")
-        pos_embed = pos_embed + window_embed.tile([x // y for x, y in zip(pos_embed.shape, window_embed.shape)])
-        pos_embed = pos_embed.permute(0, 2, 3, 1)
-        return pos_embed
-
-    def forward(self, x: torch.Tensor) -> List[torch.Tensor]:
-        """Performs hierarchical vision transformer forward pass, returning multiscale feature maps."""
-        x = self.patch_embed(x)
-        # x: (B, H, W, C)
-
-        # Add pos embed
-        x = x + self._get_pos_embed(x.shape[1:3])
-
-        outputs = []
-        for i, blk in enumerate(self.blocks):
-            x = blk(x)
-            if (i == self.stage_ends[-1]) or (i in self.stage_ends and self.return_interm_layers):
-                feats = x.permute(0, 3, 1, 2)
-                outputs.append(feats)
-
-        return outputs