xttmp 2.3.0__py3-none-any.whl

xttmp/__init__.py

@@ -0,0 +1 @@
+__all__ = ['api', 'model', 'util']

xttmp/api/__init__.py

@@ -0,0 +1,5 @@
+from .instancing_model import instancing_model
+from .evaluate import inference_task, evaluate_task
+
+__all__ = ['instancing_model', 'inference_task', 'evaluate_task']
+

xttmp/api/evaluate.py

@@ -0,0 +1,163 @@
+# demo_vidstream
+import matplotlib.pyplot as plt
+import numpy as np
+import time
+import torch
+
+from . import instancing_model
+from ..model import *
+from ..util.evaluate_module import (get_ROC_curve_data, compute_AUC, 
+                                    get_thres_recall_data, compute_AR,
+                                    get_P_R_curve_data, compute_AP, )
+from ..util.compute_module import matrix_to_sparse_list
+
+
+def inference_task(modelName, 
+                   inputpath, 
+                   inputType = 'ImgstreamReader', 
+                   startFrame = 0, 
+                   endFrame = None, 
+                   device = 'cpu',
+                   **kwargs):
+    ''' Instantiate the model '''
+    objModel = instancing_model(modelName, device=device)
+
+    ''' Dynamically create a video stream reader or other input type '''
+    inputModule = globals().get(inputType)
+    if inputModule is None:
+        raise ValueError(f"Unknown inputType: {inputType}")
+
+    objIptStream = inputModule(inputpath, startFrame, endFrame)
+
+    objNMS = MatrixNMS(15)
+
+    ''' Initialize the model '''
+    # set the parameter list
+    objModel.set_para(**kwargs)
+    # init
+    objModel.init_config()
+
+    totalRunningTime = 0
+    results = []
+    directions = []
+    ''' Run '''
+    while objIptStream.hasFrame:
+        # Read the next frame from the video stream
+        grayImg, _ = objIptStream.get_next_frame()
+        if device != 'cpu':
+            grayImg = torch.from_numpy(grayImg).to(device=device).float().unsqueeze(0).unsqueeze(0)
+        
+        # Perform inference using the model
+        result, runTime = objModel.process(grayImg)
+        totalRunningTime += runTime
+
+        # postprocessing
+        if device != 'cpu':
+            torch.cuda.synchronize()
+            result = {k: v.squeeze(0).squeeze(0).cpu().numpy() for k, v in result.items()}
+        # response
+        response = result['response']
+        if np.max(response) == 0:
+            results.append([])
+            continue
+        response = objNMS.nms(result['response'])
+        maxOpt = np.max(response)
+        if maxOpt > 0:
+            response /= np.max(response)
+            responseListType = matrix_to_sparse_list(response.astype(np.float64))
+        else:
+            responseListType = []
+        results.append(responseListType)          
+
+        # direction
+        direction  = result['direction']
+        if (direction is not None) and len(direction) and len(responseListType):
+            directionListType = [[y, x, float(direction[x, y])] for y, x, _ in responseListType]
+        else:
+            directionListType = []
+        directions.append(directionListType)
+
+    return results, directions, totalRunningTime
+
+
+def evaluate_task(modelOpt, groundTruth, aucPara = 40, gTError = 1, startFrame = 0, endFrame = None, plotFigures=True):
+    
+
+    ''' ROC curve Part'''
+    # get ROC data
+    RPIList, FPPIList, _ = get_ROC_curve_data(modelOpt, 
+                                              groundTruth, 
+                                              rangeOfFPPI = [0, aucPara], 
+                                              gTError = gTError,
+                                              startFrame = startFrame,
+                                              endFrame = endFrame)
+    
+    # calculate AUC
+    rocOfAUC = compute_AUC(RPIList, FPPIList, rangeOfFPPI=[0, aucPara])
+
+    # plot ROC curve
+    if plotFigures:
+        rocFig, ax1 = plt.subplots()
+        ax1.plot(FPPIList, RPIList)
+        ax1.set_xlim(0, aucPara)
+        ax1.set_ylim(0, 1)
+
+        ax1.set_xlabel('False Positive Rate (FPPI)')
+        ax1.set_ylabel('Recall (RPI)')
+        ax1.set_title('ROC Curve')
+
+    ''' mR Part'''
+    # get meanRecall data
+    RPIList1, thresholdList1 = get_thres_recall_data(modelOpt, 
+                                                     groundTruth,
+                                                     gTError = gTError,
+                                                     startFrame = startFrame, 
+                                                     endFrame = endFrame)
+    
+    # calculate mean Recall
+    AR = compute_AR(RPIList1, thresholdList1, rangeOfThreshold=[0.5, 1])
+    
+    # plot mR curve
+    if plotFigures:
+        mRFig, ax2 = plt.subplots()
+        ax2.plot(thresholdList1, RPIList1)
+        ax2.set_xlim(0.5, 1)
+        ax2.set_ylim(0, 1)
+
+        ax2.set_xlabel('Threshold')
+        ax2.set_ylabel('Recall')
+        ax2.set_title('Threshold-Recall Curve')
+
+    ''' P-R curve Part'''
+    # get meanRecall data
+    rList2, pList2, _ = get_P_R_curve_data(modelOpt, 
+                                        groundTruth,
+                                        intervalOfRecall = 0.02,
+                                        gTError = gTError,
+                                        startFrame = startFrame, 
+                                        endFrame = endFrame)
+    
+    # calculate mean Recall
+    AP = compute_AP(rList2, pList2)
+
+    # plot mR curve
+    if plotFigures:
+        PRFig, ax3 = plt.subplots()
+        ax3.plot(rList2, pList2)
+        ax3.set_xlim(0, 1)
+        ax3.set_ylim(0, 1)
+
+        ax3.set_xlabel('Recall')
+        ax3.set_ylabel('Precision')
+        ax3.set_title('P-R Curve')
+    
+    if plotFigures:
+        figHandle = {'ROC': rocFig,
+                    'mR':  mRFig,
+                    'PR':  PRFig,
+                    }
+
+        return rocOfAUC, AR, AP, figHandle
+    else:
+        return rocOfAUC, AR, AP
+

xttmp/api/get_visualize_handle.py

@@ -0,0 +1,29 @@
+from ..util.iostream import Visualization
+
+def get_visualize_handle(className=None, 
+                         showThreshold=None, 
+                         width = 8, 
+                         height = 5, 
+                         dpi = 100):
+    """
+    Returns a handle to a visualization object based on the given class name.
+
+    Parameters:
+        className (str): Name of the visualization class. If None, a default visualization object is created.
+        showThreshold (bool): Whether to show threshold. Default is None.
+
+    Returns:
+        Visualization: Handle to the visualization object.
+    """
+
+    if className and showThreshold is not None:
+        objVisualization = Visualization(className, showThreshold)
+    elif className:
+        objVisualization = Visualization(className)
+    else:
+        objVisualization = Visualization()
+    
+    objVisualization.create_fig_handle(width=width, height =height, dpi=dpi)
+
+    return objVisualization
+

xttmp/api/instancing_model.py

@@ -0,0 +1,35 @@
+from ..model import * # Import all models
+
+
+def instancing_model(model_name, device = 'cpu', model_para=None):
+    """
+    Instantiate a model object based on the given model name.
+
+    Parameters:
+        model_name (str): Name of the model to instantiate. If None, a GUI for model selection will be opened.
+        model_para: Parameters for model instantiation (optional).
+
+    Returns:
+        BaseModel: The instantiated model object.
+    """
+    
+    # Instantiate the model
+    _model_name =  globals().get(model_name)
+    if _model_name:
+        model = _model_name()
+    else:
+        print(f"Class {model_name} not found.")
+
+    # Process additional parameters if provided
+    if model_para is not None:
+        # Handle model parameters
+        pass
+
+    model.setup()
+    model.to(device=device)  # Move the model to the specified device
+
+    return model
+
+
+
+

xttmp/core/apgstmd_core.py

@@ -0,0 +1,188 @@
+from collections import deque
+
+import torch
+from torch.nn import functional as F
+
+from .base_core import BaseCore
+from ..util.create_kernel import create_attention_kernel, create_prediction_kernel
+from ..core.math_operator import compute_temporal_conv_inplace
+
+
+class AttentionModule(BaseCore):
+    """
+    AttentionModule class for attention mechanism.
+    
+    This class implements the attention mechanism module in the ApgSTMD.
+    """
+    
+    def __init__(self):
+        """
+        Constructor method.
+        
+        Initializes the AttentionModule object.
+        """
+        super().__init__()
+        self.kernal_size = 17
+        self.zeta_list = [2, 2.5, 3, 3.5]
+        self.theta_list = torch.tensor([0, torch.pi/4, torch.pi/2, 3*torch.pi/4])
+        self.alpha = 1
+        self.register_buffer('attention_kernel', torch.empty(0))
+        self.setup()
+    
+    def setup(self):
+        """
+        Initialization method.
+        
+        Initializes the attention kernel.
+        """
+
+        self.r = len(self.zeta_list)
+        self.s = len(self.theta_list)
+        _attention_kernel = create_attention_kernel(
+            self.kernal_size,
+            self.zeta_list,
+            self.theta_list
+        )
+        _stacked_kernel = torch.stack([torch.stack(row) for row in _attention_kernel])
+        self.attention_kernel.data = _stacked_kernel.reshape(self.r * self.s, 1, self.kernal_size, self.kernal_size)
+    
+    def forward(self, retina_opt, prediction_map):
+        """
+        Processing method (Optimized with F.conv2d).
+        
+        Processes the retina_opt and prediction_map to generate the
+        attention-optimal output.
+        """
+        if prediction_map is None:
+            self.Opt = retina_opt
+            return self.Opt
+
+        # 1. 准备输入数据
+        map_retina_opt = retina_opt * prediction_map
+                    
+        B, C, H, W = map_retina_opt.shape
+
+        # 为了对每个 Channel 独立应用这 r*s 个卷积核，
+        # 我们把 Batch 和 Channel 维度合并，把输入变形为 (B*C, 1, H, W)
+        x = map_retina_opt.reshape(B * C, 1, H, W)
+
+        # 2. 单次并发计算所有的卷积
+        # 此时输出形状为 (B*C, r*s, H, W)
+        conv_out = F.conv2d(x, self.attention_kernel, padding='same')
+
+        # 3. 执行 Min 和 Max 聚合操作
+        # 将输出重塑为 (B*C, r, s, H, W) 以便按维度进行聚合
+        conv_out = conv_out.view(B * C, self.r, self.s, H, W)
+        min_out = torch.min(conv_out, dim=2)[0]  # shape: (B*C, r, H, W)
+        attention_response = torch.max(min_out, dim=1)[0]  # shape: (B*C, H, W)
+
+        # 4. 恢复原始形状并计算最终结果
+        attention_response = attention_response.view(B, C, H, W)
+
+        self.Opt = retina_opt + self.alpha * attention_response
+        
+        return self.Opt
+
+
+class PredictionModule(BaseCore):
+    """
+    PredictionModule class for ApgSTMD.
+    
+    This class implements the prediction module in the ApgSTMD.
+    """
+    
+    def __init__(self):
+        """
+        Constructor method.
+        
+        Initializes the PredictionModule object.
+        """
+        super().__init__()
+        self.velocity = None
+        self.intDeltaT = 25
+        self.sizeFilter = 25
+        self.numFilter = 8
+        self.zeta = 2
+        self.eta = 2.5
+        self.kappa = 0.02
+        self.mu = 0.75
+        self.beta = 1
+        self.register_buffer('time_attenuation_kernel', torch.empty(0))
+        self.register_buffer('prediction_kernel', torch.empty(0))
+
+        self.setup()
+    
+    def setup(self):
+        """
+        initiate config for prediction module.
+        """        
+        self.intDeltaT = max(int(self.intDeltaT), 1)
+        
+        if self.velocity is None:
+            self.velocity = 25 / 4 / self.intDeltaT
+        
+        _prediction_kernel = create_prediction_kernel(
+            self.velocity,
+            self.intDeltaT,
+            self.sizeFilter,
+            self.numFilter,
+            self.zeta,
+            self.eta
+        )
+        self.prediction_kernel.data = torch.stack(_prediction_kernel).unsqueeze(1)
+
+        self.time_attenuation_kernel = torch.exp(self.kappa * torch.arange(-self.intDeltaT, 1))
+
+        self.reset()  # 初始化历史帧缓存
+
+    def reset(self):
+        self.prediction_gain_buffer = deque(maxlen=self.intDeltaT)
+        self.prediction_map_buffer = deque(maxlen=self.intDeltaT)
+    
+    def forward(self, lobula_opt):
+        """
+        Processing method (Highly Optimized with Vectorization).
+        
+        Processes the input lobula_opt to predict motion and update
+        prediction map.
+        """
+        num_direction = lobula_opt.shape[1] 
+        
+        if len(self.prediction_gain_buffer) > 0:
+            # 计算滤波器输入 (广播机制同时处理所有方向通道)
+            filter_input = self.mu * lobula_opt + (1 - self.mu) * self.prediction_gain_buffer[0] 
+        else:
+            filter_input = lobula_opt
+        
+        # 分组卷积 (Depthwise Convolution)
+        # 一次 F.conv2d 计算出全部 num_direction 个通道的空间卷积，彻底消除 for 循环
+        prediction_gain = F.conv2d(filter_input, 
+                                   self.prediction_kernel, 
+                                   padding='same', 
+                                   groups=num_direction)
+        
+        # 更新最新一帧的历史
+        self.prediction_gain_buffer.append(prediction_gain)
+
+        # ==================== 2. Prediction Map =====================
+        # 在 num_direction (通道) 维度上求和 -> shape: (1, 1, H, W)
+        tobe_prediction_map = torch.sum(prediction_gain, dim=1, keepdim=True)
+
+        # ==================== 3. Facilitated STMD Output ============
+        temporal_conv_out = compute_temporal_conv_inplace(
+            self.prediction_gain_buffer,
+            self.time_attenuation_kernel
+        )
+        
+        # 一步计算出所有特征通道的 facilitated_opt -> shape: (1, num_direction, H, W)
+        self.Opt = lobula_opt + self.beta * temporal_conv_out
+
+        # ==================== 4. Memorizer update ===================
+        max_tobe_pre_map = torch.max(tobe_prediction_map)
+        
+        self.prediction_map_buffer.append( (tobe_prediction_map > max_tobe_pre_map * 2e-1).squeeze())
+
+        # prediction_map = self.cell_prediction_map[0]
+        return self.Opt, self.prediction_map_buffer[0]
+
+

xttmp/core/apgstmdv2_core.py

@@ -0,0 +1,79 @@
+import numpy as np
+from cv2 import filter2D, BORDER_CONSTANT
+
+from .base_core import BaseCore
+from ..util.create_kernel import create_prediction_kernel
+
+
+class PredictionModule(BaseCore):
+    """PredictionModule class for ApgSTMD."""
+    
+    def __init__(self):
+        """Constructor method."""
+        # Initializes the PredictionModule object
+        super().__init__()
+        
+        # Parameters
+        self.velocity = 0.1  # Velocity: v_{opt} (Default: 0.25)
+        self.intDeltaT = 0  # Delta time
+        self.sizeFilter = 20  # Size of filter
+        self.numFilter = 8  # Number of filters
+        self.zeta = 2  # Zeta parameter
+        self.eta = 2.5  # Eta parameter
+        self.beta = 1  # Beta parameter
+
+        # Hidden properties
+        self.predictionKernel = None  # Prediction kernel
+
+    def setup(self):
+        """Initialization method."""
+        # Initializes the prediction module
+
+        self.predictionKernel = create_prediction_kernel(
+            self.velocity,
+            self.intDeltaT,
+            self.sizeFilter,
+            self.numFilter,
+            self.zeta,
+            self.eta
+        )
+
+    def forward(self, lobulaOpt):
+        """Processing method."""
+        # Processes the input lobulaOpt to predict motion and update prediction map
+
+        numDict = len(lobulaOpt)
+        imgH, imgW = lobulaOpt[0].shape
+
+        predictionGain = []
+        for idxD in range(numDict):
+            predictionGain.append(
+                filter2D(
+                    lobulaOpt[idxD],
+                    -1,
+                    self.predictionKernel[idxD],
+                    borderType=BORDER_CONSTANT
+                )
+            )
+
+        # Prediction Map
+        predictionMap = np.zeros((imgH, imgW))
+        for idxD in range(numDict):
+            predictionMap += predictionGain[idxD]
+
+        # Facilitated STMD Output
+        facilitatedOpt = []
+        for idxD in range(numDict):
+            facilitatedOpt.append(
+                lobulaOpt[idxD] + self.beta * predictionGain[idxD]
+            )
+
+        # Memorizer update
+        maxPreMap = np.max(predictionMap)
+            # Logical Matrix
+        predictionMap = (predictionMap > maxPreMap * 2e-1)
+
+        # Output
+        self.Opt = facilitatedOpt
+        
+        return facilitatedOpt, predictionMap

xttmp/core/base_core.py

@@ -0,0 +1,36 @@
+from abc import ABC, abstractmethod
+
+import torch
+
+
+class BaseCore(ABC, torch.nn.Module):
+    """
+    Abstract base class for core processing components.
+    """
+
+    def __init__(self):
+        """
+        Constructor.
+        """
+        super().__init__()
+
+        self.output = None
+
+    def setup(self, *args, **kwargs):
+        """
+        Abstract method for initialization.
+        """
+        pass
+
+    def reset(self):
+        """
+        Abstract method for resetting the state.
+        """
+        pass
+
+    @abstractmethod
+    def forward(self, *args, **kwargs):
+        """
+        Abstract method for processing.
+        """
+        pass