difflayers 0.1.0__py3-none-any.whl

difflayers/graph/laplacian.py

@@ -0,0 +1,76 @@
+"""
+Laplacian computation for graph-regularized Hopfield attention.
+
+Responsibility: Convert an adjacency matrix into a graph Laplacian.
+
+Provides both the unnormalized (L = D - A) and symmetric normalized
+(L_norm = D^{-1/2} L D^{-1/2}) graph Laplacians.
+
+Supports dense and sparse adjacency inputs.  When the input is a
+sparse_coo_tensor every intermediate is kept sparse so that downstream
+diffusion can exploit O(kN) sparse matrix-vector products.
+
+Complexity (dense N×N input):
+    compute_laplacian            : O(N²) time, O(N²) space
+    compute_normalized_laplacian : O(N²) time, O(N²) space
+"""
+
+import torch
+from torch import Tensor
+
+
+def _is_sparse(A: Tensor) -> bool:
+    return A.is_sparse
+
+
+def compute_laplacian(A: Tensor) -> Tensor:
+    """
+    Compute the unnormalized graph Laplacian L = D - A.
+
+    Works with both dense and sparse_coo adjacency matrices.
+    Sparse input → sparse output (preserves O(kN) downstream products).
+
+    Args:
+        A: (N, N) adjacency / similarity matrix (dense or sparse_coo).
+
+    Returns:
+        L: (N, N) graph Laplacian, same layout (dense or sparse_coo).
+           Eigenvalues in [0, max_degree].
+    """
+    if _is_sparse(A):
+        A_dense = A.to_dense()
+        D = torch.diag(A_dense.sum(dim=-1))
+        L_dense = D - A_dense
+        # Return dense — sparse Laplacian construction is complex and rarely
+        # needed; the sparse benefit is captured at the matmul call site.
+        return L_dense
+
+    D = torch.diag(A.sum(dim=-1))
+    return D - A
+
+
+def compute_normalized_laplacian(A: Tensor) -> Tensor:
+    """
+    Compute the symmetric-normalized graph Laplacian:
+        L_norm = D^{-1/2} (D - A) D^{-1/2}
+
+    Eigenvalues are in [0, 2], which gives a stable diffusion range
+    eta in (0, 0.5).  Isolated nodes (degree = 0) are handled safely
+    by setting their inverse-sqrt degree to 0.
+
+    Works with both dense and sparse_coo adjacency matrices; always
+    returns a dense Laplacian (used for eigendecomposition / precompute).
+
+    Args:
+        A: (N, N) adjacency / similarity matrix (dense or sparse_coo).
+
+    Returns:
+        L_norm: (N, N) dense normalized graph Laplacian.
+    """
+    A_dense = A.to_dense() if _is_sparse(A) else A
+    deg = A_dense.sum(dim=-1)                        # (N,)
+    d_inv_sqrt = deg.pow(-0.5)
+    d_inv_sqrt[torch.isinf(d_inv_sqrt)] = 0.0       # isolated nodes -> 0
+    D_inv_sqrt = torch.diag(d_inv_sqrt)              # (N, N)
+    L = torch.diag(deg) - A_dense
+    return D_inv_sqrt @ L @ D_inv_sqrt

difflayers/graph/laplacian_builder.py

@@ -0,0 +1,64 @@
+"""
+Laplacian builder for Graph-Regularized Hopfield attention.
+
+Responsibility: Compute the graph Laplacian L from adjacency W.
+This is the *only* class that calls ``compute_laplacian`` or
+``compute_normalized_laplacian``; it is used exclusively when an explicit
+Laplacian is required — i.e. for ``SpectralDiffusion`` (eigendecomposition)
+and energy tracking (``EnergyTracker``).
+
+For ``FactoredDiffusion`` the Laplacian is never formed; that mode uses
+(W, deg) directly, so ``LaplacianBuilder`` is not needed there.
+
+Usage::
+
+    builder = LaplacianBuilder(normalized=True)
+    L = builder.build(W)          # W: (N, N) dense or sparse
+
+Complexity:
+    normalized=False : O(N²) time, O(N²) space
+    normalized=True  : O(N²) time, O(N²) space   (D^{-1/2} L D^{-1/2})
+
+Memory: O(N²) — always returns a dense Laplacian
+"""
+
+from __future__ import annotations
+
+from torch import Tensor
+
+from .laplacian import compute_laplacian, compute_normalized_laplacian
+
+
+class LaplacianBuilder:
+    """
+    Computes the graph Laplacian L from adjacency W.
+
+    Two variants:
+      * ``normalized=False`` — unnormalized L = D - A.
+        Eigenvalues in ``[0, max_degree]``.
+      * ``normalized=True``  — symmetric-normalized L_norm = D^{-1/2}(D-A)D^{-1/2}.
+        Eigenvalues in ``[0, 2]``; stable for η ∈ (0, 0.5).
+
+    Args:
+        normalized: If True (default), produce the symmetric-normalized
+                    Laplacian.  Recommended for diffusion stability.
+    """
+
+    def __init__(self, normalized: bool = True) -> None:
+        self.normalized = normalized
+
+    def build(self, W: Tensor) -> Tensor:
+        """
+        Compute the graph Laplacian from adjacency W.
+
+        Args:
+            W: (N, N) adjacency matrix — dense or sparse_coo.
+
+        Returns:
+            L: (N, N) dense graph Laplacian (float32).
+
+        Complexity: O(N²) time, O(N²) space.
+        """
+        if self.normalized:
+            return compute_normalized_laplacian(W)
+        return compute_laplacian(W)

difflayers/transformer.py

@@ -0,0 +1,212 @@
+import torch
+import torch.nn as nn
+
+from copy import deepcopy
+from torch import Tensor
+from torch.nn.modules import Module
+from typing import Optional, Tuple, Union
+
+from . import Hopfield
+
+
+class HopfieldEncoderLayer(Module):
+    """
+    Module with underlying Hopfield association to be used as an encoder in transformer-like architectures.
+    """
+
+    def __init__(self,
+                 hopfield_association: Hopfield,
+                 dim_feedforward: int = 2048,
+                 dropout: float = 0.1,
+                 activation: str = r'relu'
+                 ):
+        """
+        Initialise a new instance of a Hopfield association-based encoder module.
+
+        :param hopfield_association: instance of Hopfield association module
+        :param dim_feedforward: depth of the linear projections applied internally
+        :param activation: activation to be applied on the result of the internal linear projections
+        :param dropout: dropout probability to be applied internally
+        """
+        super(HopfieldEncoderLayer, self).__init__()
+        self.hopfield_association = deepcopy(hopfield_association)
+
+        self.linear_residual = nn.Linear(self.hopfield_association.state_pattern_dim, dim_feedforward)
+        self.dropout_residual = nn.Dropout(dropout)
+        self.linear_output = nn.Linear(dim_feedforward, self.hopfield_association.state_pattern_dim)
+
+        self.norm_residual = nn.LayerNorm(self.hopfield_association.state_pattern_dim)
+        self.norm_output = nn.LayerNorm(self.hopfield_association.state_pattern_dim)
+        self.dropout_hopfield_association = nn.Dropout(dropout)
+        self.dropout_output = nn.Dropout(dropout)
+
+        self.activation_residual = getattr(torch, activation, None)
+        assert self.activation_residual is not None, r'invalid activation function supplied.'
+        self.reset_parameters()
+
+    def reset_parameters(self) -> None:
+        """
+        Reset parameters, including Hopfield association.
+
+        :return: None
+        """
+        for module in (self.hopfield_association, self.linear_residual,
+                       self.linear_output, self.norm_residual, self.norm_output):
+            if hasattr(module, r'reset_parameters'):
+                module.reset_parameters()
+
+    def forward(self, src: Tensor, src_mask: Optional[Tensor] = None,
+                src_key_padding_mask: Optional[Tensor] = None) -> Tensor:
+        """
+        Apply Hopfield encoding on specified data.
+
+        :param src: data to be processed by Hopfield encoder module
+        :param src_mask: mask to be applied on association matrix
+        :param src_key_padding_mask: mask to be applied on stored patterns
+        :return: Hopfield-encoded input data
+        """
+        data_associated = self.hopfield_association(
+            input=src, stored_pattern_padding_mask=src_key_padding_mask, association_mask=src_mask)
+        src = src + self.dropout_hopfield_association(input=data_associated)
+        src = self.norm_residual(input=src)
+
+        result_residual_inner = self.activation_residual(input=self.linear_residual(input=src))
+        data_associated = self.linear_output(input=self.dropout_residual(input=result_residual_inner))
+        src = src + self.dropout_output(input=data_associated)
+
+        return self.norm_output(input=src)
+
+    def get_association_matrix(self, input: Union[Tensor, Tuple[Tensor, Tensor, Tensor]]) -> Tensor:
+        """
+        Fetch Hopfield association matrix gathered by passing through the specified data.
+
+        :param input: data to be passed through the Hopfield association
+        :return: association matrix as computed by the Hopfield core module
+        """
+        return self.hopfield_association.get_association_matrix(input=input)
+
+    @property
+    def batch_first(self) -> int:
+        return self.hopfield_association.batch_first
+
+    @property
+    def input_size(self) -> int:
+        return self.hopfield_association.input_size
+
+    @property
+    def output_size(self) -> int:
+        return self.linear_output.out_features
+
+
+class HopfieldDecoderLayer(Module):
+    """
+    Module with underlying Hopfield associations to be used as a decoder in transformer-like architectures.
+    """
+
+    def __init__(self,
+                 hopfield_association_self: Hopfield,
+                 hopfield_association_cross: Hopfield,
+                 dim_feedforward: int = 2048,
+                 dropout: float = 0.1,
+                 activation: str = r'relu'
+                 ):
+        """
+        Initialise a new instance of a Hopfield association-based encoder module.
+
+        :param hopfield_association_self: instance of Hopfield self-association module
+        :param hopfield_association_cross: instance of Hopfield cross-association module
+        :param dim_feedforward: depth of the linear projections applied internally
+        :param dropout: dropout probability to be applied internally
+        :param activation: activation to be applied on the result of the internal linear projections
+        """
+        super(HopfieldDecoderLayer, self).__init__()
+        self.hopfield_association_self = deepcopy(hopfield_association_self)
+        self.hopfield_association_cross = deepcopy(hopfield_association_cross)
+
+        self.linear_residual = nn.Linear(self.hopfield_association_self.state_pattern_dim, dim_feedforward)
+        self.dropout_residual = nn.Dropout(dropout)
+        self.linear_output = nn.Linear(dim_feedforward, self.hopfield_association_self.state_pattern_dim)
+
+        self.norm_residual_self = nn.LayerNorm(self.hopfield_association_self.state_pattern_dim)
+        self.norm_residual_cross = nn.LayerNorm(self.hopfield_association_self.state_pattern_dim)
+        self.norm_output = nn.LayerNorm(self.hopfield_association_self.state_pattern_dim)
+        self.dropout_hopfield_association_self = nn.Dropout(dropout)
+        self.dropout_hopfield_association_cross = nn.Dropout(dropout)
+        self.dropout_output = nn.Dropout(dropout)
+
+        self.activation_residual = getattr(torch, activation, None)
+        assert self.activation_residual is not None, r'invalid activation function supplied.'
+        self.reset_parameters()
+
+    def reset_parameters(self) -> None:
+        """
+        Reset parameters, including Hopfield association.
+
+        :return: None
+        """
+        for module in (self.hopfield_association_self, self.hopfield_association_cross,
+                       self.linear_residual, self.linear_output, self.norm_residual_self,
+                       self.norm_residual_cross, self.norm_output):
+            if hasattr(module, r'reset_parameters'):
+                module.reset_parameters()
+
+    def forward(self, tgt: Tensor, memory: Tensor, tgt_mask: Optional[Tensor] = None,
+                memory_mask: Optional[Tensor] = None, tgt_key_padding_mask: Optional[Tensor] = None,
+                memory_key_padding_mask: Optional[Tensor] = None) -> Tensor:
+        """
+        Apply Hopfield decoding on specified data.
+
+        :param tgt: data to be processed by Hopfield decoder module (self-association)
+        :param memory: data to be processed by Hopfield encoder module (cross-association)
+        :param tgt_mask: mask to be applied on self-association matrix
+        :param memory_mask: mask to be applied on cross-association matrix
+        :param tgt_key_padding_mask: mask to be applied on stored patterns
+        :param memory_key_padding_mask: mask to be applied on state patterns as well as pattern projection
+        :return: Hopfield-decoded input
+        """
+        data_associated = self.hopfield_association_self(
+            input=tgt, stored_pattern_padding_mask=tgt_key_padding_mask,
+            association_mask=tgt_mask)
+        tgt = tgt + self.dropout_hopfield_association_self(input=data_associated)
+        tgt = self.norm_residual_self(input=tgt)
+
+        data_associated = self.hopfield_association_cross(
+            input=(memory, tgt, memory), stored_pattern_padding_mask=memory_key_padding_mask,
+            association_mask=memory_mask)
+        tgt = tgt + self.dropout_hopfield_association_cross(input=data_associated)
+        tgt = self.norm_residual_cross(input=tgt)
+
+        result_residual_inner = self.activation_residual(input=self.linear_residual(input=tgt))
+        data_associated = self.linear_output(input=self.dropout_residual(input=result_residual_inner))
+        tgt = tgt + self.dropout_output(input=data_associated)
+        return self.norm_output(input=tgt)
+
+    def get_association_matrix_self(self, input: Union[Tensor, Tuple[Tensor, Tensor, Tensor]]) -> Tensor:
+        """
+        Fetch Hopfield self-association matrix gathered by passing through the specified data.
+
+        :param input: data to be passed through the Hopfield association
+        :return: association matrix as computed by the Hopfield core module
+        """
+        return self.hopfield_association_self.get_association_matrix(input=input)
+
+    def get_association_matrix_cross(self, input: Union[Tensor, Tuple[Tensor, Tensor, Tensor]]) -> Tensor:
+        """
+        Fetch Hopfield cross-association matrix gathered by passing through the specified data.
+
+        :param input: data to be passed through the Hopfield association
+        :return: association matrix as computed by the Hopfield core module
+        """
+        return self.hopfield_association_cross.get_association_matrix(input=input)
+
+    @property
+    def batch_first(self) -> int:
+        return self.hopfield_association_self.batch_first
+
+    @property
+    def input_size(self) -> int:
+        return self.hopfield_association_self.input_size
+
+    @property
+    def output_size(self) -> int:
+        return self.linear_output_self.out_features

difflayers-0.1.0.dist-info/METADATA

@@ -0,0 +1,210 @@
+Metadata-Version: 2.4
+Name: difflayers
+Version: 0.1.0
+Summary: difflayers: Diffusion-Augmented Hopfield Networks
+Home-page: https://github.com/hopfileds/hopfield-layers
+Author: Priyam Ghosh
+Author-email: Priyam Ghosh <priyamghosh9753@gmail.com>
+License: BSD
+Project-URL: Homepage, https://github.com/hopfileds/hopfield-layers
+Project-URL: Repository, https://github.com/hopfileds/hopfield-layers
+Project-URL: Bug Tracker, https://github.com/hopfileds/hopfield-layers/issues
+Keywords: hopfield networks,deep learning,attention,diffusion,graph
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch>=1.9.0
+Requires-Dist: numpy>=1.20.0
+Requires-Dist: scipy>=1.7.0
+Dynamic: author
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
+
+# Hopfield Networks is All You Need
+
+_Hubert Ramsauer<sup>1</sup>, Bernhard Schäfl<sup>1</sup>, Johannes Lehner<sup>1</sup>, Philipp Seidl<sup>1</sup>,
+Michael Widrich<sup>1</sup>, Lukas Gruber<sup>1</sup>, Markus Holzleitner<sup>1</sup>, Milena Pavlović<sup>3, 4</sup>,
+Geir Kjetil Sandve<sup>4</sup>, Victor Greiff<sup>3</sup>, David Kreil<sup>2</sup>, Michael Kopp<sup>2</sup>, Günter
+Klambauer<sup>1</sup>, Johannes Brandstetter<sup>1</sup>, Sepp Hochreiter<sup>1, 2</sup>_
+
+<sup>1</sup> ELLIS Unit Linz and LIT AI Lab, Institute for Machine Learning, Johannes Kepler University Linz, Austria  
+<sup>2</sup> Institute of Advanced Research in Artificial Intelligence (IARAI)  
+<sup>3</sup> Department of Immunology, University of Oslo, Norway  
+<sup>4</sup> Department of Informatics, University of Oslo, Norway
+
+---
+
+##### Detailed blog post on this paper as well as the necessary background on Hopfield networks at [this link](https://ml-jku.github.io/hopfield-layers/).
+
+---
+
+The transformer and BERT models pushed the performance on NLP tasks to new levels via their attention mechanism. We show
+that this attention mechanism is the update rule of a modern Hopfield network with continuous states. This new Hopfield
+network can store exponentially (with the dimension) many patterns,converges with one update, and has exponentially
+small retrieval errors. The number of stored patterns must be traded off against convergence speed and retrieval error.
+The new Hopfield network has three types of energy minima (fixed points of the update):
+
+1. global fixed point averaging over all patterns,
+2. metastable states averaging over a subset of patterns, and
+3. fixed points which store a single pattern.
+
+Transformers learn an attention mechanism by constructing an embedding of patterns and queries into an associative
+space. Transformer and BERT models operate in their first layers preferably in the global averaging regime, while they
+operate in higher layers in metastable states. The gradient in transformers is maximal in the regime of metastable
+states, is uniformly distributed when averaging globally, and vanishes when a fixed point is near a stored pattern.
+Based on the Hopfield network interpretation, we analyzed learning of transformer and BERT architectures. Learning
+starts with attention heads that average and then most of them switch to metastable states. However, the majority of
+heads in the first layers still averages and can be replaced by averaging operations like the Gaussian weighting that we
+propose. In contrast, heads in the last layers steadily learn and seem to use metastable states to collect information
+created in lower layers. These heads seem a promising target for improving transformers. Neural networks that integrate
+Hopfield networks that are equivalent to attention heads outperform other methods on immune repertoire classification,
+where the Hopfield net stores several hundreds of thousands of patterns.
+
+With _this_ repository, we provide a PyTorch implementation of a new layer called “Hopfield” which allows to equip deep
+learning architectures with Hopfield networks as new memory concepts.
+
+The full paper is available at [https://arxiv.org/abs/2008.02217](https://arxiv.org/abs/2008.02217).
+
+## Requirements
+
+The software was developed and tested on the following 64-bit operating systems:
+
+- CentOS Linux release 8.1.1911 (Core)
+- macOS 10.15.5 (Catalina)
+
+As the development environment, [Python](https://www.python.org) 3.8.3 in combination
+with [PyTorch](https://pytorch.org) 1.6.0 was used (a version of at least 1.5.0 should be sufficient). More details on
+how to install PyTorch are available on the [official project page](https://pytorch.org).
+
+## Installation
+
+The recommended way to install the software is to use `pip/pip3`:
+
+```bash
+$ pip3 install git+https://github.com/ml-jku/hopfield-layers
+```
+
+To successfully run the [Jupyter notebooks](https://jupyter.org) contained in [examples](examples/), additional
+third-party modules are needed:
+
+```bash
+$ pip3 install -r examples/requirements.txt
+```
+
+The installation of the [Jupyter software](https://jupyter.org/install.html) itself is not covered. More details on how
+to install Jupyter are available at the [official installation page](https://jupyter.org/install.html).
+
+## Usage
+
+To get up and running with Hopfield-based networks, only <i>one</i> argument needs to be set, the size (depth) of the
+input.
+
+```python
+from hflayers import Hopfield
+
+hopfield = Hopfield(input_size=...)
+```
+
+It is also possible to replace commonly used pooling functions with a Hopfield-based one. Internally, a <i>state
+pattern</i> is trained, which in turn is used to compute pooling weights with respect to the input.
+
+```python
+from hflayers import HopfieldPooling
+
+hopfield_pooling = HopfieldPooling(input_size=...)
+```
+
+A second variant of our Hopfield-based modules is one which employs a trainable but fixed lookup mechanism. Internally,
+one or multiple <i>stored patterns</i> and <i>pattern projections</i> are trained (optionally in a non-shared manner),
+which in turn are used as a lookup mechanism independent of the input data.
+
+```python
+from hflayers import HopfieldLayer
+
+hopfield_lookup = HopfieldLayer(input_size=...)
+```
+
+The usage is as <i>simple</i> as with the main module, but equally <i>powerful</i>.
+
+## Examples
+
+Generally, the Hopfield layer is designed to be used to implement or to substitute different layers like:
+
+- <b>Pooling layers:</b> We consider the Hopfield layer as a pooling layer if only one static state (query) pattern
+  exists. Then, it is de facto a pooling over the sequence, which results from the softmax values applied on the stored
+  patterns. Therefore, our Hopfield layer can act as a pooling layer.
+
+- <b>Permutation equivariant layers:</b> Our Hopfield layer can be used as a plug-in replacement for permutation
+  equivariant layers. Since the Hopfield layer is an associative memory it assumes no dependency between the input
+  patterns.
+
+- <b>GRU & LSTM layers:</b> Our Hopfield layer can be used as a plug-in replacement for GRU & LSTM layers. Optionally,
+  for substituting GRU & LSTM layers, positional encoding might be considered.
+
+- <b>Attention layers:</b>  Our Hopfield layer can act as an attention layer, where state (query) and stored (key)
+  patterns are different, and need to be associated.
+
+The folder [examples](examples/) contains multiple demonstrations on how to use the <code>Hopfield</code>, <code>
+HopfieldPooling</code> as well as the <code>HopfieldLayer</code> modules. To successfully run the
+contained [Jupyter notebooks](https://jupyter.org), additional third-party modules
+like [pandas](https://pandas.pydata.org) and [seaborn](https://seaborn.pydata.org) are required.
+
+- [Bit Pattern Set](examples/bit_pattern/bit_pattern_demo.ipynb): The dataset of this demonstration falls into the
+  category of <i>binary classification</i> tasks in the domain of <i>Multiple Instance Learning (MIL)</i> problems. Each
+  bag comprises a collection of bit pattern instances, wheres each instance is a sequence of <b>0s</b> and <b>1s</b>.
+  The positive class has specific bit patterns injected, which are absent in the negative one. This demonstration shows,
+  that <code>Hopfield</code>, <code>HopfieldPooling</code> and <code>HopfieldLayer</code> are capable of learning and
+  filtering each bag with respect to the class-defining bit patterns.
+
+- [Latch Sequence Set](examples/latch_sequence/latch_sequence_demo.ipynb): We study an easy example of learning
+  long-term dependencies by using a simple <i>latch task</i>,
+  see [Hochreiter and Mozer](https://link.springer.com/chapter/10.1007/3-540-44668-0_92). The essence of this task is
+  that a sequence of inputs is presented, beginning with one of two symbols, <b>A</b> or <b>B</b>, and after a variable
+  number of time steps, the model has to output a corresponding symbol. Thus, the task requires memorizing the original
+  input over time. It has to be noted, that both class-defining symbols must only appear at the first position of a
+  sequence. This task was specifically designed to demonstrate the capability of recurrent neural networks to capture
+  long term dependencies. This demonstration shows, that <code>Hopfield</code>, <code>HopfieldPooling</code> and <code>
+  HopfieldLayer</code> adapt extremely fast to this specific task, concentrating only on the first entry of the
+  sequence.
+
+- [Attention-based Deep Multiple Instance Learning](examples/mnist_bags/mnist_bags_demo.ipynb): The dataset of this
+  demonstration falls into the category of <i>binary classification</i> tasks in the domain of <i>Multiple Instance
+  Learning (MIL)</i> problems, see [Ilse and Tomczak](https://arxiv.org/abs/1802.04712). Each bag comprises a collection
+  of <b>28x28</b> grayscale images/instances, whereas each instance is a sequence of pixel values in the range
+  of <b>[0; 255]</b>. The amount of instances per pag is drawn from a Gaussian with specified mean and variance. The
+  positive class is defined by the presence of the target number/digit, whereas the negative one by its absence.
+
+## Disclaimer
+
+Some implementations of this repository are based on existing ones of the
+official [PyTorch repository v1.6.0](https://github.com/pytorch/pytorch/tree/v1.6.0) and accordingly extended and
+modified. In the following, the involved parts are listed:
+
+- The implementation of [HopfieldCore](hflayers/activation.py#L16) is based on the implementation
+  of [MultiheadAttention](https://github.com/pytorch/pytorch/blob/b31f58de6fa8bbda5353b3c77d9be4914399724d/torch/nn/modules/activation.py#L771)
+  .
+- The implementation of [hopfield_core_forward](hflayers/functional.py#L8) is based on the implementation
+  of [multi_head_attention_forward](https://github.com/pytorch/pytorch/blob/b31f58de6fa8bbda5353b3c77d9be4914399724d/torch/nn/functional.py#L3854)
+  .
+- The implementation of [HopfieldEncoderLayer](hflayers/transformer.py#L12) is based on the implementation
+  of [TransformerEncoderLayer](https://github.com/pytorch/pytorch/blob/b31f58de6fa8bbda5353b3c77d9be4914399724d/torch/nn/modules/transformer.py#L241)
+  .
+- The implementation of [HopfieldDecoderLayer](hflayers/transformer.py#L101) is based on the implementation
+  of [TransformerDecoderLayer](https://github.com/pytorch/pytorch/blob/b31f58de6fa8bbda5353b3c77d9be4914399724d/torch/nn/modules/transformer.py#L303)
+  .
+
+## License
+
+This repository is BSD-style licensed (see [LICENSE](LICENSE)), except where noted otherwise.

difflayers-0.1.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+difflayers/__init__.py,sha256=vpmvaQa-IOOUieCJHc8GYtDHYIhyQKhhmG0JzCynACs,47270
+difflayers/activation.py,sha256=gl23vFjb_7cjTolftGZfSwwBOlwRZHOrpz89a7kLBHk,19207
+difflayers/attention_operator.py,sha256=sqstQXzVZZkjloB_0NWGxRO_H0fItIlWwtaZCW3Q1L4,5743
+difflayers/diffused_attention.py,sha256=-JvUHzYTMrdke8Q4TWRFR8vgvEaeuOa1LfOiF2Pv3GQ,19264
+difflayers/diffusion.py,sha256=eWsNJOrdphjGYQbJ6SD8B9px3ns6c1xN8Dq58V9LqlU,13656
+difflayers/dynamics_engine.py,sha256=hgiej34ETREmqDhf5xfW1pVD5bhcgFadZYGVkyW5peA,20287
+difflayers/functional.py,sha256=aiwPffKCuTmBO7HE9Ezom4xxym4OILz-MQvDvH10otA,26693
+difflayers/transformer.py,sha256=IPDQK6Qvu7EpLAjigk5e2TU5NT74Lud77OlmarKwtfA,9727
+difflayers/auxiliary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+difflayers/auxiliary/data.py,sha256=YcsJIjIJz3CJuXba4nQIpA5rLVGZkcW5kSzyhpSJTh4,10392
+difflayers/graph/__init__.py,sha256=EFGF-v4_NxJ1CrU3P1G-iZle9Xz9iY-UakdugPEAYp0,642
+difflayers/graph/build_graph.py,sha256=1Tubtd3SMPLMHSuW8nnf4JQUjtND-mPyjCkkQpTrBj8,2555
+difflayers/graph/builder.py,sha256=WeX2xwUALIRST3mliG2pnSkl44Fri5a6QANMLQZUzfU,4646
+difflayers/graph/laplacian.py,sha256=k5cDHur9J881PAoBa3Hmtj4WjmSCIUWJ9ALUlxMIAu4,2528
+difflayers/graph/laplacian_builder.py,sha256=7DzOfiqFWZu5vMwHfREktY5_G3743NyvjwnpwfjLQhs,2013
+difflayers-0.1.0.dist-info/licenses/LICENSE,sha256=AjYSib8nN4uc727CcxTOJNX6ehJIHD8p8Ph0nX01Ifs,3387
+difflayers-0.1.0.dist-info/METADATA,sha256=a5WWZHoSdjlDEjsLxcq8loDcH1wIWx2xM-Pi43cCKpk,11895
+difflayers-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+difflayers-0.1.0.dist-info/top_level.txt,sha256=HRiuZ0X4NWI1tQEmeMlDqbRK17xpGTyNPnYqDezMias,11
+difflayers-0.1.0.dist-info/RECORD,,

difflayers-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

difflayers-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,79 @@
+From Hopfield layers:
+
+Copyright (c) 2020, Institute for Machine Learning, Johannes Kepler University Linz (Bernhard Schäfl)
+All rights reserved.
+
+All other contributions:
+Copyright (c) 2020 the respective contributors
+All rights reserved.
+
+From PyTorch:
+
+Copyright (c) 2016-     Facebook, Inc            (Adam Paszke)
+Copyright (c) 2014-     Facebook, Inc            (Soumith Chintala)
+Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert)
+Copyright (c) 2012-2014 Deepmind Technologies    (Koray Kavukcuoglu)
+Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu)
+Copyright (c) 2011-2013 NYU                      (Clement Farabet)
+Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston)
+Copyright (c) 2006      Idiap Research Institute (Samy Bengio)
+Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz)
+
+From Caffe2:
+
+Copyright (c) 2016-present, Facebook Inc. All rights reserved.
+
+All contributions by Facebook:
+Copyright (c) 2016 Facebook Inc.
+ 
+All contributions by Google:
+Copyright (c) 2015 Google Inc.
+All rights reserved.
+ 
+All contributions by Yangqing Jia:
+Copyright (c) 2015 Yangqing Jia
+All rights reserved.
+ 
+All contributions from Caffe:
+Copyright(c) 2013, 2014, 2015, the respective contributors
+All rights reserved.
+ 
+All other contributions:
+Copyright(c) 2015, 2016 the respective contributors
+All rights reserved.
+ 
+Caffe2 uses a copyright model similar to Caffe: each contributor holds
+copyright over their contributions to Caffe2. The project versioning records
+all such contribution and copyright details. If a contributor wants to further
+mark their specific copyright on a particular contribution, they should
+indicate their copyright solely in the commit message of the change when it is
+committed.
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America
+   and IDIAP Research Institute nor the names of its contributors may be
+   used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

difflayers-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+difflayers