xttmp 2.3.0__py3-none-any.whl

xttmp/model/facilitated_model.py

@@ -0,0 +1,230 @@
+import warnings
+
+from .backbone import DSTMDBackbone
+from ..core import stmdplus_core, apgstmd_core
+from ..util.compute_module import compute_response, compute_direction
+
+
+class STMDPlus(DSTMDBackbone):
+    """ STMDPlus: A facilitated model based on DSTMD with an additional contrast pathway.
+
+    Ref:
+        * Wang H, Peng J, Zheng X, et al. "A robust visual system for small target motion detection against cluttered moving backgrounds." IEEE Transactions on Neural Networks and Learning Systems, 2019, 31(3): 839-853.
+
+    Description:
+        The STMDPlus model builds upon the DSTMD architecture, enhancing target detection accuracy in cluttered moving backgrounds by introducing a contrast pathway. This pathway provides a complementary processing mechanism to improve robustness against dynamic noise and varying background contrasts. The model parameters align with those specified in the reference.
+
+    Parameters:
+        Retina:
+            - sigma1: Controls the standard deviation for Gaussian blur applied in the retina layer, reducing noise and emphasizing potential targets. (Eq. 1)
+        
+        Lamina:
+            - n1, tau1: Order and time constant of the first gamma bandpass filter in the lamina. (Eq. 3)
+            - n2, tau2: Order and time constant of the second gamma bandpass filter.
+        
+        Medulla:
+            - n3, tau3: Order and time constant of gamma delay in the Mi1 pathway. (Eq. 11)
+            - n4, tau4: Order and time constant of gamma delay in the Tm1 pathway. (Eq. 11)
+            - n5, tau5: Order and time constant for another gamma delay variant in the Tm1 pathway. (Eq. 11)
+        
+        Lobula:
+            - alpha1: Parameter modulating signal strength for the lobula, contributing to directional selectivity. (Eq. 10)
+            - A, B: Lateral inhibition coefficients. (Eq. 13)
+            - e, rho, sigma4, sigma5: Lateral inhibition coefficients. (Eq. 14)
+
+        Contrast Pathway:
+            - eta: Parameter adjusting the contrast pathway’s influence. (Eq. 15)
+            - alpha2: Signal modulation factor within the contrast pathway. (Eq. 17)
+    """
+
+
+    # Bind model parameters and their corresponding parameter pointers.
+    __paraMappingList = {
+        # retina
+        'sigma1'    : 'retina.sigma', # Eq. (1)
+        # lamina
+        'n1'        : 'lamina.order1',  # Eq. (3)
+        'tau1'      : 'lamina.tau1',
+        'n2'        : 'lamina.order2',
+        'tau2'      : 'lamina.tau2',
+        # medulla
+        'n3'        : 'medulla.mi1_para4.order',  # Eq. (11)
+        'tau3'      : 'medulla.mi1_para4.tau',
+        'n4'        : 'medulla.tm1_para5.order',
+        'tau4'      : 'medulla.tm1_para5.tau',
+        'n5'        : 'medulla.tm1_para6.order',
+        'tau5'      : 'medulla.tm1_para6.tau',
+        # lobula
+        'alpha1'    : 'lobula.alpha1',  # Eq. (10)
+        'A'         : 'lobula.hLateralInhi.A',  # Eq. (13)
+        'B'         : 'lobula.hLateralInhi.B', 
+        'e'         : 'lobula.hLateralInhi.e',  # Eq. (14)
+        'rho'       : 'lobula.hLateralInhi.rho', 
+        'sigma4'    : 'lobula.hLateralInhi.sigma1', 
+        'sigma5'    : 'lobula.hLateralInhi.sigma2', 
+        # Contrast pathway
+        'eta'       : 'contrast_pathway.eta', # Eq. (15)
+        'alpha2'    : 'contrast_pathway.alpha2', # Eq. (17)
+        } 
+    
+    def __init__(self):
+        """ Constructor method """
+        super().__init__()
+
+        # Initialize contrast pathway and mushroom body components
+        self.contrast_pathway = stmdplus_core.ContrastPathway()
+        self.mushroom_body = stmdplus_core.MushroomBody()
+
+    def forward(self, iptMatrix):
+        """ Defines the structure of the STMDPlus model. """      
+
+        # A. Ommatidia (Retina)
+        retina_output = self.retina.forward(iptMatrix)
+
+        # B. Motion Pathway (Lamina, Medulla, Lobula)
+        lamina_output = self.lamina.forward(retina_output)
+        medulla_tm3_output, medulla_mi1_p4_output, medulla_tm1_p5_output, medulla_tm1_p6_output = \
+            self.medulla.forward(lamina_output)
+        lobula_output = self.lobula.forward(medulla_tm3_output, medulla_mi1_p4_output, 
+                                            medulla_tm1_p5_output, medulla_tm1_p6_output)
+
+        # C. Contrast Pathway
+        contrast_output = self.contrast_pathway.forward(retina_output)
+
+        # D. Mushroom Body
+        mushroom_body_output = self.mushroom_body.forward(
+            lobula_output, contrast_output)
+
+        # Compute response and direction
+        self.model_output['response'] = compute_response(mushroom_body_output)
+        self.model_output['direction'] = compute_direction(mushroom_body_output)
+
+        return self.model_output
+
+
+class ApgSTMD(STMDPlus):
+    """ ApgSTMD: Attention-Prediction-guided Small Target Motion Detector
+
+    Ref: 
+        * Wang H, Zhao J, Wang H, et al. Attention and prediction-guided motion detection for low-contrast small moving targets[J]. IEEE Transactions on Cybernetics, 2022, 53(10): 6340-6352.
+
+    Description:
+        The ApgSTMD model extends the STMDPlus model by introducing attention and prediction pathways to improve target detection.
+
+    Parameters:            
+        Retina:
+            - sigma1: Standard deviation of the Gaussian blur applied in the retina layer to pre-forward input images by smoothing, reducing background noise. (Eq. 2)
+        
+        Lamina:
+            - n1, tau1: Order and time constant of the first gamma bandpass filter in the lamina. (Eq. 6)
+            - n2, tau2: Order and time constant of the second gamma bandpass filter. (Eq. 6)
+        
+        Medulla:
+            - n3, tau3: Order and time constant in the Mi1 pathway. (Eq. 14)
+            - n4, tau4: Order and time constant in the Tm1 pathway. (Eq. 14)
+            - n5, tau5: Additional delay component in Tm1. (Eq. 14)
+        
+        Lobula:
+            - gamma: Signal modulation parameter in the lobula, aiding in selective attention towards targets by enhancing certain spatial patterns. (Eq. 13)
+            - A, B: Coefficients for lateral inhibition, reducing background clutter through inhibition of surrounding non-target signals. (Eq. 15)
+            - e, rho, sigma4, sigma5: Parameters controlling non-linear inhibition effects. (Eq. 16)
+        
+        Attention Pathway:
+            - eta_list: A list of attentional weights adjusting model responsiveness to spatial regions with potential target information. (Eq. 3)
+            - theta_list: (Eq. 3)
+        
+        Prediction Pathway:
+            - zeta, eta: Parameters for prediction kernels. (Eq. 20)
+            - kappa: Parameter adjusting the weighting for predicted versus observed signals, balancing real-time data with anticipated target trajectories. (Eq. 23)
+    """
+
+    # Bind model parameters and their corresponding parameter pointers.
+    __paraMappingList = {
+        # retina
+        'sigma1'    : 'retina.sigma', # Eq. (2)
+        # lamina
+        'n1'        : 'lamina.order1',  # Eq. (6)
+        'tau1'      : 'lamina.tau1',
+        'n2'        : 'lamina.order2',
+        'tau2'      : 'lamina.tau2',
+        # medulla
+        'n3'        : 'medulla.mi1_para4.order',  # Eq. (14)
+        'tau3'      : 'medulla.mi1_para4.tau',
+        'n4'        : 'medulla.tm1_para5.order',
+        'tau4'      : 'medulla.tm1_para5.tau',
+        'n5'        : 'medulla.tm1_para6.order',
+        'tau5'      : 'medulla.tm1_para6.tau',
+        # lobula
+        'gamma'    : 'lobula.alpha1',  # Eq. (13)
+        'A'         : 'lobula.hLateralInhi.A',  # Eq. (15)
+        'B'         : 'lobula.hLateralInhi.B', 
+        'e'         : 'lobula.hLateralInhi.e',  # Eq. (16)
+        'rho'       : 'lobula.hLateralInhi.rho', 
+        'sigma4'    : 'lobula.hLateralInhi.sigma1', 
+        'sigma5'    : 'lobula.hLateralInhi.sigma2', 
+        # Attention Pathway
+        'zeta_list'  : 'attention_pathway.zeta_list', # Eq. (3)
+        'theta_list': 'attention_pathway.theta_list',
+        # Prediction Pathway
+        'zeta'      : 'prediction_pathway.zeta', # Eq. (20)
+        'eta'       : 'prediction_pathway.eta', # Eq. (20)
+        'kappa'     : 'prediction_pathway.kappa', # Eq. (23)
+        } 
+
+    def __init__(self):
+        """
+        Constructor method
+        """
+        super().__init__()
+
+        # Initialize attention pathway and prediction pathway components
+        self.attention_pathway = apgstmd_core.AttentionModule()
+        self.prediction_pathway = apgstmd_core.PredictionModule()
+
+        # Set properties of Lobula's LateralInhibition module
+        self.lobula.hLateralInhi.B = 3.5
+        self.lobula.hLateralInhi.sigma1 = 1.25
+        self.lobula.hLateralInhi.sigma2 = 2.5
+        self.lobula.hLateralInhi.e = 1.2
+
+        self.predictionMap = None
+
+    def forward(self, x):
+        """ Defines the structure of the ApgSTMD model. """
+
+        # Preprocessing Module
+        retina_output = self.retina(x)
+
+        # Attention Module
+        attention_output = self.attention_pathway(
+            retina_output, self.predictionMap)
+
+        # STMD-based Neural Network
+        lamina_output = self.lamina(attention_output)
+        medulla_tm3_output, medulla_mi1_p4_output, medulla_tm1_p5_output, medulla_tm1_p6_output = \
+            self.medulla(lamina_output)
+
+        lobula_output = self.lobula(medulla_tm3_output, medulla_mi1_p4_output, 
+                                    medulla_tm1_p5_output, medulla_tm1_p6_output)
+
+        # STMDPlus
+        contrast_output = self.contrast_pathway(retina_output)
+        mushroom_body_output = self.mushroom_body(
+            lobula_output, contrast_output)
+
+        # Prediction Module
+        #   prediction_output is the facilitated STMD output Q(x; y; t; theta) in Eq. (23)
+        prediction_output, self.predictionMap = self.prediction_pathway(mushroom_body_output)
+
+        # Compute response and direction
+        self.model_output['response'] = compute_response(prediction_output)
+        self.model_output['direction'] = compute_direction(prediction_output)
+
+        return self.model_output
+
+
+
+
+
+
+

xttmp/model/feedback_model.py

@@ -0,0 +1,271 @@
+from copy import deepcopy
+
+import torch
+
+from .backbone import ESTMDBackbone, FracSTMD
+from ..core import feedbackstmd_core, fstmd_core, stfeedbackstmd_core
+
+
+class FeedbackSTMD(ESTMDBackbone):
+    """ FeedbackSTMD: Small Target Motion Detector with feedback pathway in lobula
+
+    Ref: 
+        * Wang H, Wang H, Zhao J, et al. A time-delay feedback neural network for discriminating small, fast-moving targets in complex dynamic environments[J]. IEEE Transactions on Neural Networks and Learning Systems, 2021, 34(1): 316-330.
+
+    Description:
+        The FeedbackSTMD model is an enhancement of the ESTMD backbone with a feedback pathway integrated in the lobula. 
+
+    Parameters:
+        Retina:
+            - sigma1: Standard deviation of the Gaussian blur in the retina. (Eq. 2)
+
+        Lamina:
+            - n1, tau1: Order and time constant for the first gamma bandpass filter in the lamina. (Eq. 3)
+            - n2, tau2: Order and time constant for the second gamma filter.
+
+        Medulla:
+            - n3, tau3: Tuple containing orders and time constants for Tm1 and Mi1 pathways, both crucial in delaying signals to provide temporal integration for motion detection. (Eq. 7)
+        
+        Lobula:
+            - alpha: Feedback constant in the lobula. (Eq. 9)
+            - n4, tau4: Order and time constant in the lobula’s gamma delay, supporting time-delay feedback that strengthens temporal coherence in target tracking.
+            - eta: Parameter in the Gaussian kernel. (Eq. 10)
+            - A, B: Lateral inhibition coefficients. (Eq. 15)
+            - e, rho, sigma2, sigma3: Parameters that modulate nonlinear inhibition. (Eq. 16)
+    """
+
+    # Bind model parameters and their corresponding parameter pointers.
+    __paraMappingList = {
+        # retina
+        'sigma1'    : 'retina.sigma', # Eq. (2)
+        # lamina
+        'n1'        : 'lamina.order1', # Eq. (3)
+        'tau1'      : 'lamina.tau1',
+        'n2'        : 'lamina.order1',
+        'tau2'      : 'lamina.tau1',
+        # medulla
+        'n3'        : ('medulla.tm1.order', 'medulla.mi1.order'), # Eq. (7)
+        'tau3'      : ('medulla.tm1.tau', 'medulla.mi1.tau'), 
+        # lobula
+        'alpha'     : 'lobula.alpha', # Eq. (9)
+        'n4'        : 'lobula.gamma_delay.order',
+        'tau4'      : 'lobula.gamma_delay.tau', 
+        'eta'       : 'lobula.sigma', # Eq. (10)
+        'A'         : 'lobula.spatial_inhibition.A', # Eq. (15)
+        'B'         : 'lobula.spatial_inhibition.B',
+        'e'         : 'lobula.spatial_inhibition.e', # Eq. (16)
+        'rho'       : 'lobula.spatial_inhibition.rho',
+        'sigma2'    : 'lobula.spatial_inhibition.sigma1',
+        'sigma3'    : 'lobula.spatial_inhibition.sigma2',
+        }
+    
+    def __init__(self):
+        """
+        FeedbackSTMD Constructor method
+        Initializes an instance of the FeedbackSTMD class.
+        """
+        # Call superclass constructor
+        super().__init__()
+
+        # Customize Lobula component
+        self.lobula = feedbackstmd_core.Lobula()
+        
+        # Customize Lamina's GammaBankPassFilter properties
+        self.lamina.order1 = 4
+        self.lamina.tau1 = 8
+        self.lamina.order2 = 16
+        self.lamina.tau2 = 32
+
+        # Customize Medulla's Tm1 component properties
+        self.medulla.tm1.order = 9
+        self.medulla.tm1.tau = 45
+
+    def forward(self, x):
+        """ MODEL_STRUCTURE Method
+
+        Defines the structure of the FeedbackSTMD model.
+        """
+        # Process input matrix through model components
+        retina_output = self.retina(x)
+        lamina_output = self.lamina(retina_output)
+        medulla_ON, medulla_OFF = self.medulla(lamina_output)
+        self.model_output['response'] = self.lobula(medulla_ON, medulla_OFF)
+
+        return self.model_output
+
+
+class FSTMD(ESTMDBackbone):
+    """ FSTMD: Small Target Motion Detector with feedback loop between lobula and lamina
+
+    Ref: 
+        * Ling J, Wang H, Xu M, et al. Mathematical study of neural feedback roles in small target motion detection[J]. Frontiers in Neurorobotics, 2022, 16: 984430.
+
+    Description:
+        The FSTMD model introduces a feedback loop between the lobula and lamina layers.
+
+    Parameters:
+        Retina:
+            - sigma1: Standard deviation of the Gaussian blur in the retina, pre-processing the visual input by smoothing noise and high-frequency signals. (Eq. 1)
+
+        Lamina:
+            - n1, tau1: Order and time constant for the first gamma bandpass filter in the lamina. (Eq. 6)
+            - n2, tau2: Order and time constant for the second gamma filter.
+
+        Medulla:
+            - n3, tau3: Tuple containing the order and time constants for the gamma delay functions in Tm1 and Mi1 pathways, which provide temporal integration critical for target motion analysis. (Eq. 9)
+
+        Lobula:
+            - e, rho, sigma2, sigma3: Parameter controlling the strength of lateral inhibition in the lobula. (Eq. 13)
+
+        Feedback Pathway:
+            - n4, tau4: Order and time constant for the gamma delay in the feedback pathway, adjusting temporal coherence to align with the dynamics of target motion. (Eq. 4)
+            - a: Feedback constant, regulating the strength of the feedback signal from the lobula to the lamina, balancing sensitivity and stability in target detection by dynamically adjusting the lamina’s response to target motion. (Eq. 4)
+    """
+
+    # Bind model parameters and their corresponding parameter pointers.
+    __paraMappingList = {
+        # retina
+        'sigma1'    : 'retina.sigma', # Eq. (1)
+        # lamina
+        'n1'        : 'lamina.order1', # Eq. (6)
+        'tau1'      : 'lamina.tau1',
+        'n2'        : 'lamina.order2',
+        'tau2'      : 'lamina.tau2',
+        # medulla
+        'n3'        : ('medulla.tm1.order', 'medulla.mi1.order'), # Eq. (9)
+        'tau3'      : ('medulla.tm1.tau', 'medulla.mi1.tau'), 
+        # lobula
+        'e'         : 'lobula.spatial_inhibition.e', # Eq. (13)
+        'rho'       : 'lobula.spatial_inhibition.rho',
+        'sigma2'    : 'lobula.spatial_inhibition.sigma1',
+        'sigma3'    : 'lobula.spatial_inhibition.sigma2',
+        # feedback pathway
+        'n4'        : 'feedback_pathway.order', # Eq. (4)
+        'tau4'      : 'feedback_pathway.tau', 
+        'a'         : 'feedback_pathway.feedback_coefficient', # Eq. (4)
+        }
+    
+    def __init__(self):
+        """ FSTMD Constructor method
+
+        Initializes an instance of the FSTMD class.
+        """
+        # Call superclass constructor
+        super().__init__()
+        
+        # Initialize feedback pathway component
+        self.feedback_pathway = fstmd_core.FeedbackPathway()
+
+        self.maxIterationNum = 10
+        self.iterationThreshold = 1e-3
+
+        # Customize Medulla's Tm1 component properties
+        self.medulla.tm1.order = 5
+
+    def forward(self, x):
+        """ MODEL_STRUCTURE Method
+
+        Defines the structure of the FSTMD model.
+        """
+        last_feedback_output = torch.ones_like(x)
+        self.feedback_output = torch.zeros_like(x)
+
+        # Retina layer
+        retina_output = self.retina(x)
+
+        # Feedback loop
+        iteration_count = 1
+        self.set_loop_state(False)
+        while iteration_count < self.maxIterationNum and torch.max(
+                torch.abs(self.feedback_output - last_feedback_output)) > self.iterationThreshold:
+            last_feedback_output = self.feedback_output.clone()
+
+            # Execute feedback loop
+            lamina_output = self.lamina(retina_output + self.feedback_output)
+            medulla_ON, medulla_OFF = self.medulla(lamina_output)
+            lobula_output, correlation_output = self.lobula(medulla_ON, medulla_OFF)
+            self.feedback_output = self.feedback_pathway(correlation_output)
+
+            iteration_count += 1
+            self.set_loop_state(True)
+
+        # Set model response
+        self.model_output['response'] = lobula_output
+
+        return self.model_output
+
+    def set_loop_state(self, state):
+        """ Sets the loop state of certain components. """
+        # Disable circshift for certain components
+        self.lamina.in_loop = state
+        self.medulla.tm1.in_loop = state
+        self.feedback_pathway.in_loop = state
+
+
+class STFeedbackSTMD(ESTMDBackbone):
+    """ STFeedbackSTMD - Small Target Motion Detection With Spatio-Temporal Feedback
+
+    # !!! Notice: This model has not been reproduced yet
+    
+    Ref: 
+        [1] Wang H, Zhong Z, Lei F, et al. Bio-Inspired Small Target Motion Detection With Spatio-Temporal Feedback in Natural Scenes[J]. IEEE Transactions on Image Processing, 2023.
+
+    
+    """
+
+    # Bind model parameters and their corresponding parameter pointers.
+    __paraMappingList = {} 
+
+    def __init__(self):
+        """
+        Constructor method
+        """
+        super().__init__()       
+
+        # Customize Medulla and Lobula component
+        self.medulla = stfeedbackstmd_core.Medulla()
+        self.lobula = stfeedbackstmd_core.Lobula()
+
+
+class FracSTMD_F(FracSTMD):
+
+    # Bind model parameters and their corresponding parameter pointers.
+    __paraMappingList = deepcopy(FracSTMD._FracSTMD__paraMappingList)
+    __paraMappingList.update({
+        # lobula
+        'beta'      : 'lobula.alpha', # Eq. (9)
+        'n4'        : 'lobula.hGammaDelay.order',
+        'tau4'      : 'lobula.hGammaDelay.tau', 
+        'eta'       : 'lobula.paraGaussKernel[\'eta\']', # Eq. (10)
+    })
+
+    def __init__(self):
+        """
+        Constructor method
+        """
+        super().__init__()       
+
+        # Customize Lamina component to include fractional differentiation
+        self.lobula = feedbackstmd_core.Lobula()
+
+    def forward(self, iptMatrix):
+        """ MODEL_STRUCTURE Method
+
+        Defines the structure of the FeedbackSTMD model.
+        """
+        # Process input matrix through model components
+        self.retinaOpt = self.retina.forward(iptMatrix)
+        lamina_output = self.lamina.forward(self.retinaOpt)
+        self.medulla.forward(lamina_output)
+        self.medullaOpt = self.medulla.Opt
+        lobula_output = self.lobula.forward(self.medullaOpt[0], self.medullaOpt[1])
+
+        # Set model response
+        self.model_output['response'] = lobula_output
+
+
+
+
+
+
+

xttmp/model/haarstmd.py

@@ -0,0 +1,61 @@
+import warnings
+
+from ..core import haarstmd_core
+from .backbone import ESTMDBackbone
+from ..util.compute_module import compute_response, compute_direction
+
+class HaarSTMD(ESTMDBackbone):
+    ''' HaarSTMD: Advancing small target motion detection in dim light
+    
+    Ref: 
+        [1] Chen H, Sun X, Hu C, et al. Unveiling the power of Haar frequency domain: Advancing small target motion detection in dim light[J]. Applied Soft Computing, 2024, 167: 112281.
+    '''
+
+    # Bind model parameters and their corresponding parameter pointers.
+    __paraMappingList = {
+        'sigma1'    : 'retina.sigma',
+        'n1'        : 'lamina.order1',
+        'tau1'      : 'lamina.tau1',
+        'n2'        : 'lamina.order2',
+        'tau2'      : 'lamina.tau2',
+        'a'         : 'medulla.temporal_kernel_len',
+        'r'         : 'medulla.spatial_kernel_size',
+        'n3'        : 'medulla.delay_on.order',
+        'tau3'      : 'medulla.delay_on.tau',
+        'n4'        : 'medulla.delay_off.order',
+        'tau4'      : 'medulla.delay_off.tau',
+        'sigma2'    : 'lobula.spatial_inhibition.sigma1',
+        'sigma3'    : 'lobula.spatial_inhibition.sigma2',
+        }
+    
+    def __init__(self):
+        ''' Constructor method '''
+        super().__init__()
+
+        self.medulla = haarstmd_core.Medulla()
+        self.lobula = haarstmd_core.Lobula()
+
+        # init parameter
+        self.retina.sigma = 1
+        self.lamina.order1 = 10
+        self.lamina.tau1 = 3
+        self.lamina.order2 = 10
+        self.lamina.tau2 = 9
+        self.lobula.spatial_inhibition.sigma1 = 1.5
+        self.lobula.spatial_inhibition.sigma2 = 3
+    
+
+    def forward(self, x):
+
+        retina_output = self.retina(x)
+        lamina_output = self.lamina(retina_output)
+        correlated_spatial_output, correlated_temporal_output = self.medulla(lamina_output)
+        lobula_output = self.lobula(correlated_spatial_output, 
+                                    correlated_temporal_output)
+
+        self.model_output['response'] = compute_response(lobula_output)
+        self.model_output['direction'] = compute_direction(lobula_output)
+
+        return self.model_output
+
+