engression 0.0.0.dev0__py3-none-any.whl

engression/__init__.py

@@ -0,0 +1,10 @@
+from .engression import engression
+# from .engression_bagged import engression_bagged
+
+try:
+    # pylint: disable=wrong-import-position
+    import torch
+except ModuleNotFoundError:
+    raise ModuleNotFoundError(
+        "No module named 'torch', and engression depends on PyTorch (aka 'torch')."
+        "Visit https://pytorch.org/ for installation instructions.")

engression/data/loader.py

@@ -0,0 +1,51 @@
+import torch
+from torch.utils.data import TensorDataset, DataLoader
+
+def make_dataloader(x, y, batch_size=128, shuffle=True, num_workers=0):
+    """Make dataloader.
+
+    Args:
+        x (torch.Tensor): data of predictors.
+        y (torch.Tensor): data of responses.
+        batch_size (int, optional): _description_. Defaults to 128.
+        shuffle (bool, optional): _description_. Defaults to True.
+        num_workers (int, optional): _description_. Defaults to 0.
+
+    Returns:
+        _type_: _description_
+    """
+    dataset = TensorDataset(x, y)
+    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers)
+    return dataloader
+
+def partition_data(x_full, y_full, cut_quantile=0.3, split_train="smaller"):
+    """Partition data into training and test sets.
+
+    Args:
+        x_full (torch.Tensor): full data of x.
+        y_full (torch.Tensor): full data of y.
+        cut_quantile (float, optional): quantile of the cutting point of x. Defaults to 0.3.
+        split_train (str, optional): which subset is used for for training. choices=["smaller", "larger"]. Defaults to "smaller".
+
+    Returns:
+        tuple of torch.Tensors: training and test data.
+    """
+    # Split data into training and test sets.
+    x_cut = torch.quantile(x_full, cut_quantile)
+    train_idx = x_full <= x_cut if split_train == "smaller" else x_full >= x_cut
+    x_tr = x_full[train_idx]
+    y_tr = y_full[train_idx]
+    x_te = x_full[~train_idx]
+    y_te = y_full[~train_idx]
+    
+    # Standardize data based on training statistics.
+    x_tr_mean = x_tr.mean()
+    x_tr_std = x_tr.std()
+    y_tr_mean = y_tr.mean()
+    y_tr_std = y_tr.std()
+    x_tr = (x_tr - x_tr_mean)/x_tr_std
+    y_tr = (y_tr - y_tr_mean)/y_tr_std
+    x_te = (x_te - x_tr_mean)/x_tr_std
+    y_te = (y_te - y_tr_mean)/y_tr_std
+    x_full_normal = (x_full - x_tr_mean)/x_tr_std
+    return x_tr.unsqueeze(1), y_tr.unsqueeze(1), x_te.unsqueeze(1), y_te.unsqueeze(1), x_full_normal

engression/data/simulator.py

@@ -0,0 +1,55 @@
+import torch
+import torch.nn as nn
+import numpy as np
+
+
+def preanm_simulator(true_function='softplus', n=10000, x_lower=0, x_upper=2, noise_std=1, noise_dist='gaussian', train=True, device=torch.device('cpu')):
+    """Data simulator for a pre-additive noise model (pre-ANM).
+
+    Args:
+        true_function (str, optional): true function g^\star. Defaults to 'softplus'. Choices: ['softplus', 'cubic','square', 'log'].
+        n (int, optional): sample size. Defaults to 10000.
+        x_lower (int, optional): lower bound of the training support. Defaults to 0.
+        x_upper (int, optional): upper bound of the training support. Defaults to 2.
+        noise_std (int, optional): standard deviation of the noise. Defaults to 1.
+        noise_dist (str, optional): noise distribution. Defaults to 'gaussian'. Choices: ['gaussian', 'uniform'].
+        train (bool, optional): generate data for training. Defaults to True.
+        device (str or torch.device, optional): _description_. Defaults to torch.device('cpu').
+
+    Returns:
+        tuple of torch.Tensors: data simulated from a pre-ANM.
+    """
+    if isinstance(true_function, str):
+        if true_function == 'softplus':
+            true_function = lambda x: nn.Softplus()(x)
+        elif true_function == 'cubic':
+            true_function = lambda x: x.pow(3)/3
+        elif true_function == 'square':
+            true_function = lambda x: (nn.functional.relu(x)).pow(2)/2
+        elif true_function == 'log':
+            true_function = lambda x: (x/3 + np.log(3) - 2/3)*(x <= 2) + (torch.log(1 + x*(x > 2)))*(x > 2) 
+            
+    if isinstance(device, str):
+            device = torch.device(device)
+            
+    if train:
+        x = torch.rand(n, 1)*(x_upper - x_lower) + x_lower
+        if noise_dist == 'gaussian':
+            eps = torch.randn(n, 1)*noise_std
+        else:
+            assert noise_dist == 'uniform'
+            eps = (torch.rand(n, 1) - 0.5)*noise_std*np.sqrt(12)
+        xn = x + eps
+        y = true_function(xn)
+        return x.to(device), y.to(device)
+    
+    else:
+        x_eval = torch.linspace(x_lower, x_upper, n).unsqueeze(1)
+        y_eval_med = true_function(x_eval)
+        gen_sample_size = 10000
+        x_rep = torch.repeat_interleave(x_eval, (gen_sample_size * torch.ones(n)).long(), dim=0)
+        x_rep = x_rep + torch.randn(x_rep.size(0), 1)*noise_std
+        y_eval_mean = true_function(x_rep)
+        y_eval_mean = list(torch.split(y_eval_mean, gen_sample_size))
+        y_eval_mean = torch.cat([y_eval_mean[i].mean().unsqueeze(0) for i in range(n)], dim=0).unsqueeze(1)
+        return x_eval.to(device), y_eval_med.to(device), y_eval_mean.to(device)

engression/engression.py

@@ -0,0 +1,358 @@
+import torch
+import matplotlib.pyplot as plt
+
+from .loss_func import *
+from .models import StoNet
+from .data.loader import make_dataloader
+from .utils import *
+
+
+class Engressor(object):
+
+    def __init__(self, 
+                 in_dim, out_dim, num_layer=2, hidden_dim=100, noise_dim=100,
+                 lr=0.001, num_epoches=500, batch_size=None, device="cpu", standardize=True): 
+        """Engressor class
+
+        Args:
+            in_dim (int): input dimension
+            out_dim (int): output dimension
+            num_layer (int, optional): number of layers. Defaults to 2.
+            hidden_dim (int, optional): number of neurons per layer. Defaults to 100.
+            noise_dim (int, optional): noise dimension. Defaults to 100.
+            lr (float, optional): learning rate. Defaults to 0.001.
+            num_epoches (int, optional): number of epoches. Defaults to 500.
+            batch_size (int, optional): batch size. Defaults to None, referring to the full batch.
+            device (str or torch.device, optional): device. Defaults to "cpu". Choices = ["cpu", "cuda"].
+            standardize (bool, optional): whether to standardize data. Defaults to True.
+        """
+        super().__init__()
+        self.num_layer = num_layer
+        self.hidden_dim = hidden_dim
+        self.noise_dim = noise_dim
+        self.lr = lr
+        self.num_epoches = num_epoches
+        self.batch_size = batch_size
+        if isinstance(device, str):
+            if device == "gpu" or device == "cuda":
+                device = torch.device("cuda")
+            else:
+                device = torch.device(device)
+        self.device = device
+        check_for_gpu(self.device)
+        self.standardize = standardize
+        self.x_mean = None
+        self.x_std = None
+        self.y_mean = None
+        self.y_std = None
+        
+        self.model = StoNet(in_dim, out_dim, num_layer, hidden_dim, noise_dim).to(self.device)
+        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=lr)
+        
+        self.tr_loss = None
+            
+    def train_mode(self):
+        self.model.train()
+        
+    def eval_mode(self):
+        self.model.eval()
+        
+    def summary(self):
+        """Print the model architecture and hyperparameters."""
+        print("Engression model with\n" +
+              "\t number of layers: {}\n".format(self.num_layer) +
+              "\t hidden dimensions: {}\n".format(self.hidden_dim) +
+              "\t noise dimensions: {}\n".format(self.noise_dim) +
+              "\t number of epochs: {}\n".format(self.num_epoches) +
+              "\t batch size: {}\n".format(self.batch_size) +
+              "\t learning rate: {}\n".format(self.lr) +
+              "\t standardization: {}\n".format(self.standardize) +
+              "\t training mode: {}\n".format(self.model.training) +
+              "\t device: {}\n".format(self.device))
+        print("Training loss (original scale):\n" +
+              "\t energy-loss: {:.2f}, \n\tE(|Y-Yhat|): {:.2f}, \n\tE(|Yhat-Yhat'|): {:.2f}".format(
+                  self.tr_loss[0], self.tr_loss[1], self.tr_loss[2]))
+
+        
+    def _standardize_data_and_record_stats(self, x, y):
+        """Standardize the data and record the mean and standard deviation of the training data.
+
+        Args:
+            x (torch.Tensor): training data of predictors.
+            y (torch.Tensor): training data of responses.
+
+        Returns:
+            torch.Tensor: standardized data.
+        """
+        self.x_mean = torch.mean(x, dim=0)
+        self.x_std = torch.std(x, dim=0)
+        self.y_mean = torch.mean(y, dim=0)
+        self.y_std = torch.std(y, dim=0)
+        return (x - self.x_mean) / self.x_std, (y - self.y_mean) / self.y_std
+
+    def standardize_data(self, x, y=None):
+        """Standardize the data, if self.standardize is True.
+
+        Args:
+            x (torch.Tensor): training data of predictors.
+            y (torch.Tensor, optional): _description_. Defaults to None.
+
+        Returns:
+            torch.Tensor: standardized or original data.
+        """
+        if y is None:
+            if self.standardize:
+                return (x - self.x_mean) / self.x_std
+            else:
+                return x
+        else:
+            if self.standardize:
+                return (x - self.x_mean) / self.x_std, (y - self.y_mean) / self.y_std
+            else:
+                return x, y
+    
+    def unstandardize_data(self, y, x=None):
+        """Transform the predictions back to the original scale, if self.standardize is True.
+
+        Args:
+            y (torch.Tensor): data in the standardized scale
+
+        Returns:
+            torch.Tensor: data in the original scale
+        """
+        if x is None:
+            if self.standardize:
+                return y * self.y_std + self.y_mean
+            else:
+                return y
+        else:
+            if self.standardize:
+                return x * self.x_std + self.x_mean, y * self.y_std + self.y_mean
+            else:
+                return x, y
+        
+    def train(self, x, y, num_epoches=None, batch_size=512, print_every_nepoch=100, print_times_per_epoch=1, standardize=True, verbose=True):
+        """Training function.
+
+        Args:
+            x (torch.Tensor): training data of predictors.
+            y (torch.Tensor): trainging data of responses.
+            num_epoches (int, optional): number of training epochs. Defaults to None.
+            batch_size (int, optional): batch size for mini-batch SGD. Defaults to 512.
+            print_every_nepoch (int, optional): print losses every print_every_nepoch number of epochs. Defaults to 100.
+            print_times_per_epoch (int, optional): print losses for print_times_per_epoch times per epoch. Defaults to 1.
+            standardize (bool, optional): standardize the data. Defaults to True.
+        """
+        self.train_mode()
+        if num_epoches is None:
+            num_epoches = self.num_epoches
+        if batch_size is None:
+            batch_size = self.batch_size
+        if standardize:
+            self.standardize = standardize
+            
+        x = vectorize(x)
+        y = vectorize(y)
+        x = x.to(self.device)
+        y = y.to(self.device)
+        if self.standardize: 
+            print("Data is standardized during training only; the printed training losses are on the standardized scale, \n" +
+                  "while during evaluation, the predictions, evaluation metrics, and plots will be on the original scale.\n")
+            x, y = self._standardize_data_and_record_stats(x, y)
+        
+        if batch_size >= x.size(0)//2:
+            print("Batch is larger than half of the sample size. Training based on full-batch gradient descent.")
+            self.batch_size = x.size(0)
+            for epoch_idx in range(num_epoches):
+                self.model.zero_grad()
+                y_sample1 = self.model(x)
+                y_sample2 = self.model(x)
+                loss, loss1, loss2 = energy_loss_two_sample(y, y_sample1, y_sample2, verbose=verbose)
+                loss.backward()
+                self.optimizer.step()
+                if epoch_idx == 0 or  (epoch_idx + 1) % print_every_nepoch == 0:
+                    print("[Epoch {} ({:.0f}%)] energy-loss: {:.4f},  E(|Y-Yhat|): {:.4f},  E(|Yhat-Yhat'|): {:.4f}".format(
+                        epoch_idx + 1, 100 * epoch_idx / num_epoches, loss.item(), loss1.item(), loss2.item()))
+        else:
+            train_loader = make_dataloader(x, y, batch_size=batch_size, shuffle=True)
+            print("Training based on mini-batch gradient descent with a batch size of {}.".format(batch_size))
+            for epoch_idx in range(num_epoches):
+                for batch_idx, (x_batch, y_batch) in enumerate(train_loader):
+                    self.model.zero_grad()
+                    y_sample1 = self.model(x_batch)
+                    y_sample2 = self.model(x_batch)
+                    loss, loss1, loss2 = energy_loss_two_sample(y_batch, y_sample1, y_sample2, verbose=verbose)
+                    # loss = (y_batch - y_sample1).pow(2).mean()
+                    loss.backward()
+                    self.optimizer.step()
+                    if epoch_idx == 0 or (epoch_idx + 1) % print_every_nepoch == 0:
+                        if (batch_idx + 1) % (len(train_loader) // print_times_per_epoch) == 0:
+                            print("[Epoch {} ({:.0f}%), batch {}]: energy-loss: {:.4f},  E(|Y-Yhat|): {:.4f},  E(|Yhat-Yhat'|): {:.4f}".format(
+                                epoch_idx + 1, 100 * epoch_idx / num_epoches, batch_idx + 1, loss.item(), loss1.item(), loss2.item()))
+                            # print("[Epoch {}, batch {}]: loss: {:.4f}".format(
+                            #     epoch_idx + 1, batch_idx + 1, loss.item()))
+
+        # Evaluate performance on the training data (on the original scale)
+        self.model.eval()
+        x, y = self.unstandardize_data(y, x)
+        self.tr_loss = self.eval_loss(x, y, loss_type="energy", verbose=True)
+        
+        print("\nTraining loss on the original (non-standardized) scale:\n" +
+              "\tEnergy-loss: {:.4f},  E(|Y-Yhat|): {:.4f},  E(|Yhat-Yhat'|): {:.4f}".format(
+                  self.tr_loss[0], self.tr_loss[1], self.tr_loss[2]))
+        
+        print("\nPrediction-loss E(|Y-Yhat|) and variance-loss E(|Yhat-Yhat'|) should ideally be equally large" +
+              "\n-- consider training for more epochs or adjusting hyperparameters if there is a mismatch ")
+    
+    def predict(self, x, target="mean", sample_size=100):
+        """Point prediction.
+
+        Args:
+            x (torch.Tensor): data of predictors.
+            target (str or float or list, optional): single-valued functional to predict. float refers to the quantiles. Defaults to ["mean"].
+            sample_size (int, optional): sample sizes for each x. Defaults to 100.
+
+        Returns:
+            torch.Tensor or list of torch.Tensor: point predictions.
+        """
+        self.eval_mode()  
+        x = vectorize(x)
+        x = x.to(self.device)
+        x = self.standardize_data(x)
+        y_pred = self.model.predict(x, target, sample_size)
+        y_pred = self.unstandardize_data(y_pred)
+        return y_pred
+    
+    def sample(self, x, sample_size=100, expand_dim=True):
+        """Sample new response data.
+
+        Args:
+            x (torch.Tensor): test data of predictors.
+            target (str or float or list, optional): single-valued functional to predict. float refers to the quantiles. Defaults to ["mean"].
+            sample_size (int, optional): sample sizes for each x. Defaults to 100.
+
+        Returns:
+            torch.Tensor or list of torch.Tensor: samples.
+                - [:,:,i] gives the i-th sample of all x.
+                - [i,:,:] gives all samples of x_i.
+        """
+        self.eval_mode()
+        x = vectorize(x)
+        x = x.to(self.device)
+        x = self.standardize_data(x)
+        y_samples = self.model.sample(x, sample_size, expand_dim=expand_dim)
+        y_samples = self.unstandardize_data(y_samples)
+        return y_samples
+    
+    def eval_loss(self, x, y, loss_type="l2", sample_size=None, verbose=False):
+        """Compute the loss for evaluation.
+
+        Args:
+            x (torch.Tensor): data of predictors.
+            y (torch.Tensor): data of responses.
+            loss_type (str, optional): loss type. Defaults to "l2". Choices: ["l2", "l1", "energy", "cor"].
+            sample_size (int, optional): sample sizes for each x. Defaults to 100.
+        
+        Returns:
+            float: evaluation loss.
+        """
+        if sample_size is None:
+            sample_size = 2 if loss_type == "energy" else 100
+        self.eval_mode()
+        x = vectorize(x)
+        y = vectorize(y)
+        x = x.to(self.device)
+        y = y.to(self.device)
+        if loss_type == "l2":
+            y_pred = self.predict(x, target="mean", sample_size=sample_size)
+            loss = (y - y_pred).pow(2).mean()
+        elif loss_type == "cor":
+            y_pred = self.predict(x, target="mean", sample_size=sample_size)
+            loss = cor(y, y_pred)
+        elif loss_type == "l1":
+            y_pred = self.predict(x, target=0.5, sample_size=sample_size)
+            loss = (y - y_pred).abs().mean()
+        else:
+            assert loss_type == "energy"
+            y_samples = self.sample(x, sample_size=sample_size, expand_dim=False)
+            loss = energy_loss(y, y_samples, verbose=verbose)
+        if not verbose:
+            return loss.item()
+        else:
+            loss, loss1, loss2 = loss
+            return loss.item(), loss1.item(), loss2.item()
+    
+    def plot(self, x_te, y_te, x_tr=None, y_tr=None, x_idx=0, y_idx=0, 
+             target="mean", sample_size=100, save_dir=None,
+             alpha=0.8, ymin=None, ymax=None):
+        """Plot true data and predictions.
+
+        Args:
+            x_te (torch.Tensor): test data of predictors
+            y_te (torch.Tensor): test data of responses
+            x_tr (torch.Tensor): training data of predictors
+            y_tr (torch.Tensor): training data of responses
+            x_idx (int, optional): index of the predictor to plot (if there are multiple). Defaults to 0.
+            y_idx (int, optional): index of the response to plot (if there are multiple). Defaults to 0.
+            target (str or float, optional): target quantity. Defaults to "mean". Choice: ["mean", "median", "sample", float].
+            sample_size (int, optional): sample sizes for each x. Defaults to 100.
+            save_dir (str, optional): directory to save the plot. Defaults to None.
+            alpha (float, optional): transparency of the sampled data points. Defaults to 0.8.
+            ymin (float, optional): minimum value of y in the plot. Defaults to None.
+            ymax (float, optional): maximum value of y in the plot. Defaults to None.
+        """
+        if x_tr is not None and y_tr is not None:
+            # Plot training data as well.
+            x_tr = vectorize(x_tr)
+            y_tr = vectorize(y_tr)
+            plt.scatter(x_tr[:,x_idx].cpu(), y_tr[:,y_idx].cpu(), s=1, label="training data", color="silver")
+            plt.scatter(x_te[:,x_idx].cpu(), y_te[:,y_idx].cpu(), s=1, label="test data", color="gold")
+            x = torch.cat((x_tr, x_te), dim=0)
+            y = torch.cat((y_tr, y_te), dim=0)
+        else:
+            # Plot only the test data.
+            x_te = vectorize(x_te)
+            y_te = vectorize(y_te)
+            plt.scatter(x_te[:,x_idx].cpu(), y_te[:,y_idx].cpu(), s=1, label="true data", color="silver")
+            x = x_te
+            y = y_te
+        x = x.to(self.device)
+        y = y.to(self.device)
+        
+        if target != "sample":
+            y_pred = self.predict(x, target=target, sample_size=sample_size)
+            plt.scatter(x[:,x_idx].cpu(), y_pred[:,y_idx].cpu(), s=1, label="predictions", color="lightskyblue")
+        else:
+            y_sample = self.sample(x, sample_size=sample_size, expand_dim=False)
+            x_rep = x.repeat(sample_size, 1)
+            plt.scatter(x_rep[:,x_idx].cpu(), y_sample[:,y_idx].cpu(), s=1, label="samples", color="lightskyblue", alpha=alpha)
+        plt.legend(markerscale=2)
+        plt.ylim(ymin, ymax)
+        if x.shape[1] == 1:
+            plt.xlabel(r"$x$")
+        else:
+            plt.xlabel(r"$x_{}$".format(x_idx))
+        if y.shape[1] == 1:
+            plt.ylabel(r"$y$")
+        else:
+            plt.ylabel(r"$y_{}$".format(y_idx))
+        if save_dir is not None:
+            make_folder(save_dir)
+            plt.savefig(save_dir, bbox_inches="tight")
+            plt.close()
+        else:
+            plt.show()
+
+
+def engression(x, y, 
+               num_layer=2, hidden_dim=100, noise_dim=100,
+               lr=0.001, num_epoches=500, batch_size=None, 
+               print_every_nepoch=100, print_times_per_epoch=1,
+               device="cpu", standardize=True,
+               verbose=True): 
+    engressor = Engressor(in_dim=x.shape[1], out_dim=y.shape[1], num_layer=num_layer, hidden_dim=hidden_dim, noise_dim=noise_dim, 
+                          lr=lr, num_epoches=num_epoches, batch_size=batch_size, device=device, standardize=standardize)
+    engressor.train(x, y, num_epoches=num_epoches, batch_size=batch_size, 
+                    print_every_nepoch=print_every_nepoch, print_times_per_epoch=print_times_per_epoch, 
+                    standardize=standardize, verbose=verbose)
+    return engressor

engression/engression_bagged.py

@@ -0,0 +1,282 @@
+# coming soon.
+
+import torch
+import numpy as np
+import matplotlib.pyplot as plt
+
+from .engression import Engressor
+from .loss_func import *
+from .models import StoNet
+from .data.loader import make_dataloader
+from .utils import *
+
+
+class BaggedEngressor(object):
+    def __init__(self, 
+                 in_dim, out_dim, num_layer=2, hidden_dim=100, noise_dim=100,
+                 lr=0.001, num_epoches=500, batch_size=None, device="cpu", standardize=True,
+                 ensemble_size=10, val_loss_type="energy"): 
+        """Engressor with bagging.
+
+        Args:
+            in_dim (int): input dimension
+            out_dim (int): output dimension
+            num_layer (int, optional): number of layers. Defaults to 2.
+            hidden_dim (int, optional): number of neurons per layer. Defaults to 100.
+            noise_dim (int, optional): noise dimension. Defaults to 100.
+            lr (float, optional): learning rate. Defaults to 0.001.
+            num_epoches (int, optional): number of epoches. Defaults to 500.
+            batch_size (int, optional): batch size. Defaults to None, referring to the full batch.
+            device (str or torch.device, optional): device. Defaults to "cpu".
+            standardize (bool, optional): whether to standardize data. Defaults to True.
+            ensemble_size (int, optional): number of models for ensemble. Defaults to 10.
+            val_loss_type (str, optional): loss type for validation. Defaults to "energy". Choices: ["l1", "l2", "energy"].
+        """
+        super().__init__()
+        self.lr = lr
+        self.num_epoches = num_epoches
+        self.batch_size = batch_size
+        self.ensemble_size = ensemble_size
+        self.num_models_for_each_sample = ensemble_size // 2
+        if isinstance(device, str):
+            device = torch.device(device)
+        self.device = device
+        self.standardize = standardize
+        self.x_mean = None
+        self.x_std = None
+        self.y_mean = None
+        self.y_std = None
+        self.split_mask = None
+        
+        # Build ensemble models.
+        self.models = []        
+        for _ in range(ensemble_size):
+            self.models.append(Engressor(in_dim, out_dim, num_layer, hidden_dim, noise_dim, 
+                                         lr=lr, device=device, standardize=standardize))
+        self.val_loss_type = val_loss_type
+
+    def train_mode(self):
+        for i in range(self.ensemble_size):
+            self.models[i].train_mode()
+        
+    def eval_mode(self):
+        for i in range(self.ensemble_size):
+            self.models[i].eval_mode()
+        
+    def train(self, x, y, num_epoches=None, batch_size=None, standardize=True, val_loss_type="", val_sample_size=100, verbose=True):
+        """_summary_
+
+        Args:
+            x (_type_): _description_
+            y (_type_): _description_
+            num_epoches (_type_, optional): _description_. Defaults to None.
+            batch_size (_type_, optional): _description_. Defaults to None.
+            standardize (bool, optional): _description_. Defaults to True.
+                - Standardization scheme for bagging: 
+                    for each model, all data are standardized using the mean and standard deviation of the training data for this model alone.
+            val_loss_type (str, optional): _description_. Defaults to "".
+            val_sample_size (int, optional): _description_. Defaults to 100.
+            verbose (bool, optional): _description_. Defaults to True.
+        """
+        self.train_mode()
+        if num_epoches is None:
+            num_epoches = self.num_epoches
+        if batch_size is None:
+            batch_size = self.batch_size
+        if standardize:
+            self.standardize = standardize
+
+        x = vectorize(x)
+        y = vectorize(y)
+        x = x.to(self.device)
+        y = y.to(self.device)
+            
+        # Mask matrix for splitting training and validation data.
+        data_size = x.shape[0]
+        rng = np.random.default_rng(21875667591346)
+        self.split_mask = torch.from_numpy(rng.multivariate_hypergeometric([1]*self.ensemble_size, self.num_models_for_each_sample, size=data_size))
+        
+        # Training.
+        for i in range(self.ensemble_size):
+            model = self.models[i]
+            train_idx = self.split_mask[:,i]
+            x_train = x[train_idx]
+            y_train = y[train_idx]
+            x_val = x[~train_idx]
+            y_val = y[~train_idx]
+            model.train(x_train, y_train, num_epoches=num_epoches, batch_size=batch_size, standardize=self.standardize, verbose=False)
+            val_loss = model.eval_loss(x_val, y_val, loss_type="energy", sample_size=2)
+            train_loss = model.eval_loss(x_train, y_train, loss_type="energy", sample_size=2)
+            standardize_str = "(standardized)" if self.standardize else ""
+            print("[Model {}] train_loss{}: {:.4f}, val_loss{}: {:.4f}".format(i + 1, standardize_str, train_loss, standardize_str, val_loss))
+        
+        # Validation.
+        val_loss_final = self.validate_bagged(x, y, loss_type=val_loss_type, sample_size=val_sample_size)
+        print("Final validation {} loss: {:.4f}".format(val_loss_type, val_loss_final))
+    
+    def validate_bagged(self, x, y, loss_type="energy", sample_size=100):
+        """Evaluate the bagged model on the validation data.
+
+        Args:
+            x (torch.Tensor): training data for predictors.
+            y (torch.Tensor): training data for responses.
+            loss_type (str, optional): type of the loss. Defaults to "energy".
+            sample_size (int, optional): _description_. Defaults to 100.
+
+        Returns:
+            float: loss of the bagged model on the validation data.
+        """
+        self.eval_mode()
+        x = vectorize(x)
+        y = vectorize(y)
+        x = x.to(self.device)
+        y = y.to(self.device)
+        
+        # Samples from the bagged model on the validation data.
+        sample_size_per_model = sample_size // self.num_models_for_each_sample
+        y_samples = torch.zeros((y.shape[0], y.shape[1], sample_size_per_model, self.ensemble_size), device=self.device)
+        for i in range(self.ensemble_size):
+            model = self.models[i]
+            val_idx = self.split_mask[:,i] == 0
+            y_samples[val_idx, :, :, i] = model.sample(x[val_idx], sample_size=sample_size_per_model)
+        val_idx = (1 - self.split_mask).nonzero(as_tuple=True)
+        y_samples = y_samples[val_idx[0], :, :, val_idx[1]]
+        y_samples = y_samples.reshape(y.shape[0], self.num_models_for_each_sample, y.shape[1], sample_size_per_model).permute(0, 2, 3, 1)
+        
+        # Compute the loss (on the original scale).
+        if loss_type == "energy":
+            loss = energy_loss(y, y_samples, verbose=False)
+        elif loss_type == "l2":
+            y_pred = y_samples.mean(dim=len(y_samples.shape) - 1)
+            loss = (y - y_pred).pow(2).mean()
+        elif loss_type == "l1":
+            y_pred = y_samples.median(dim=len(y_samples.shape) - 1)
+            loss = (y - y_pred).abs().mean()
+        return loss.item()
+    
+    def sample(self, x, sample_size=100):
+        self.eval_mode()
+        x = vectorize(x)
+        x = x.to(self.device)
+        sample_size_per_model = sample_size // self.ensemble_size
+        y_samples = []
+        for i in range(self.ensemble_size):
+            model = self.models[i]
+            if i == self.ensemble_size:
+                sample_size_per_model += sample_size % self.ensemble_size
+            y_samples.append(model.sample(x, sample_size=sample_size_per_model))
+        y_samples = torch.cat(y_samples, dim=-1)
+        return y_samples
+    
+    def predict(self, x, target="mean", sample_size=100):
+        self.eval_mode()
+        x = vectorize(x)
+        x = x.to(self.device)
+        y_samples = self.sample(x, sample_size=sample_size)
+        if not isinstance(target, list):
+            target = [target]
+        y_pred = []
+        extremes = []
+        for t in target:
+            if t == "mean":
+                y_pred.append(y_samples.mean(dim=len(y_samples.shape) - 1))
+            else:
+                if t == "median":
+                    t = 0.5
+                assert isinstance(t, float)
+                y_pred.append(y_samples.quantile(t, dim=len(y_samples.shape) - 1))
+                if min(t, 1 - t) * sample_size < 10:
+                    extremes.append(t)
+        
+        if len(extremes) > 0:
+            print("Warning: the estimate for quantiles at {} with a sample size of {} could be inaccurate. Please increase the `sample_size`.".format(extremes, sample_size))
+
+        if len(y_pred) == 1:
+            return y_pred[0]
+        else:
+            return y_pred
+        
+    def eval_loss(self, x, y, loss_type="energy", sample_size=100):
+        """Evaluate the bagged model.
+
+        Args:
+            x (_type_): _description_
+            y (_type_): _description_
+            loss_type (str, optional): _description_. Defaults to "energy".
+            sample_size (int, optional): _description_. Defaults to 100.
+
+        Returns:
+            _type_: _description_
+        """
+        self.eval_mode()
+        x = vectorize(x)
+        y = vectorize(y)
+        x = x.to(self.device)
+        y = y.to(self.device)
+        if loss_type == "l2":
+            y_pred = self.predict(x, target="mean", sample_size=sample_size)
+            loss = (y - y_pred).pow(2).mean()
+        elif loss_type == "l1":
+            y_pred = self.predict(x, target=0.5, sample_size=sample_size)
+            loss = (y - y_pred).abs().mean()
+        else:
+            assert loss_type == "energy"
+            y_samples = self.sample(x, sample_size=sample_size, expand_dim=False)
+            loss = energy_loss(y, y_samples, verbose=False)
+        return loss.item()
+    
+    def plot(self, x, y, x_idx=0, y_idx=0, target="mean", sample_size=100, save_dir=None,
+             alpha=0.8, ymin=None, ymax=None):
+        """Plot true data and predictions.
+
+        Args:
+            x (torch.Tensor): data of predictors
+            y (torch.Tensor): data of responses
+            x_idx (int, optional): index of the predictor to plot (if there are multiple). Defaults to 0.
+            y_idx (int, optional): index of the response to plot (if there are multiple). Defaults to 0.
+            target (str or float, optional): target quantity. Defaults to "mean". Choice: ["mean", "median", "sample", float].
+            sample_size (int, optional): sample size used for estimation. Defaults to 100.
+            save_dir (str, optional): directory to save the plot. Defaults to None.
+            alpha (float, optional): transparency of the sampled data points. Defaults to 0.8.
+            ymin (float, optional): minimum value of y in the plot. Defaults to None.
+            ymax (float, optional): maximum value of y in the plot. Defaults to None.
+        """
+        x = vectorize(x)
+        y = vectorize(y)
+        x = x.to(self.device)
+        y = y.to(self.device)
+        plt.scatter(x[:,x_idx].cpu(), y[:,y_idx].cpu(), s=1, label="true data", color="silver")
+        if target != "sample":
+            y_pred = self.predict(x, target=target, sample_size=sample_size)
+            plt.scatter(x[:,x_idx].cpu(), y_pred[:,y_idx].cpu(), s=1, label="predictions", color="lightskyblue")
+        else:
+            y_samples = self.sample(x, sample_size=sample_size, expand_dim=False)
+            x_rep = x.repeat(sample_size, 1)
+            plt.scatter(x_rep[:,x_idx].cpu(), y_samples[:,y_idx].cpu(), s=1, label="samples", color="lightskyblue", alpha=alpha)
+        plt.legend(markerscale=2)
+        plt.ylim(ymin, ymax)
+        if x.shape[1] == 1:
+            plt.xlabel(r"$x$")
+        else:
+            plt.xlabel(r"$x_{}$".format(x_idx))
+        if y.shape[1] == 1:
+            plt.ylabel(r"$y$")
+        else:
+            plt.ylabel(r"$y_{}$".format(y_idx))
+        if save_dir is not None:
+            make_folder(save_dir)
+            plt.savefig(save_dir, bbox_inches="tight")
+            plt.close()
+        else:
+            plt.show()
+
+
+def engression_bagged(x, y, 
+                       num_layer=2, hidden_dim=100, noise_dim=100,
+                       lr=0.001, num_epoches=500, batch_size=None, device="cpu",
+                       nfolds=10, loss_type="energy"):
+    engressor = BaggedEngressor(in_dim=x.shape[1], out_dim=y.shape[1], num_layer=num_layer, hidden_dim=hidden_dim, noise_dim=noise_dim, 
+                            lr=lr, num_epoches=num_epoches, batch_size=batch_size, device=device, nfolds=nfolds)
+    engressor.train(x, y)
+    print("Average validation {} loss: {:.4f}".format(loss_type, np.mean(engressor.val_losses)))
+    return engressor

engression/loss_func.py

@@ -0,0 +1,52 @@
+import torch
+from .utils import vectorize
+
+def energy_loss(x_true, x_est, beta=1, verbose=False):
+    """Loss function based on the energy score.
+
+    Args:
+        x_true (torch.Tensor): iid samples from the true distribution.
+        x_est (list of torch.Tensor): a list of iid samples from the estimated distribution. ## todo: not a list!!!
+        beta (float): power parameter in the energy score.
+        verbose (bool): whether to return two terms of the loss.
+
+    Returns:
+        loss (torch.Tensor): energy loss.
+    """
+    x_true = vectorize(x_true).unsqueeze(1)
+    if not isinstance(x_est, list):
+        x_est = list(torch.split(x_est, x_true.shape[0], dim=0))
+    m = len(x_est)
+    x_est = [vectorize(x_est[i]).unsqueeze(1) for i in range(m)]
+    x_est = torch.cat(x_est, dim=1)
+        
+    s1 = torch.norm(x_est - x_true, 2, dim=2).pow(beta).mean()
+    s2 = torch.cdist(x_est, x_est, 2).pow(beta).mean() * m / (m - 1)
+    if verbose:
+        return torch.cat([(s1 - s2 / 2).reshape(1), s1.reshape(1), s2.reshape(1)], dim=0)
+    else:
+        return (s1 - s2 / 2)
+    
+
+def energy_loss_two_sample(x0, x, xp, beta=1, verbose=False):
+    """Loss function based on the energy score (estimated based on two samples).
+    
+    Args:
+        x0 (torch.Tensor): iid samples from the true distribution.
+        x (torch.Tensor): iid samples from the estimated distribution.
+        xp (torch.Tensor): iid samples from the estimated distribution.
+        beta (float): power parameter in the energy score.
+        verbose (bool):  whether to return two terms of the loss.
+    
+    Returns:
+        loss (torch.Tensor): energy loss.
+    """
+    x0 = vectorize(x0)
+    x = vectorize(x)
+    xp = vectorize(xp)
+    s1 = torch.norm(x - x0, 2, dim=1).pow(beta).mean() / 2 + torch.norm(xp - x0, 2, dim=1).pow(beta).mean() / 2
+    s2 = torch.norm(x - xp, 2, dim=1).pow(beta).mean() 
+    if verbose:
+        return torch.cat([(s1 - s2/2).reshape(1), s1.reshape(1), s2.reshape(1)], dim=0)
+    else:
+        return (s1 - s2/2)

engression/models.py

@@ -0,0 +1,179 @@
+import torch
+import torch.nn as nn
+
+
+class StoLayer(nn.Module):    
+    """A stochastic layer.
+
+    Args:
+        in_dim (int): input dimension 
+        out_dim (int): output dimension
+        noise_dim (int, optional): noise dimension. Defaults to 100.
+    """
+    def __init__(self, in_dim, out_dim, noise_dim=100):
+        super().__init__()
+        self.in_dim = in_dim
+        self.out_dim = out_dim
+        self.noise_dim = noise_dim
+        layer = [
+            nn.Linear(in_dim + noise_dim, out_dim),
+            nn.BatchNorm1d(out_dim),
+            nn.ReLU(inplace=True),
+        ]
+        self.layer = nn.Sequential(*layer)
+    
+    def forward(self, x, inject_noise=True):
+        eps = torch.randn(x.size(0), self.noise_dim, device=x.device) if inject_noise else torch.zeros(x.size(0), self.noise_dim, device=x.device)
+        x = torch.cat([x, eps], dim=1)
+        return self.layer(x)
+    
+    
+class StoNet(nn.Module):
+    """Stochastic neural network.
+
+    Args:
+        in_dim (int): input dimension 
+        out_dim (int): output dimension
+        num_layer (int, optional): number of layers. Defaults to 2.
+        hidden_dim (int, optional): number of neurons per layer. Defaults to 100.
+        noise_dim (int, optional): noise dimension. Defaults to 100.
+    """
+    def __init__(self, in_dim, out_dim, num_layer=2, hidden_dim=100, noise_dim=100):
+        super().__init__()
+        self.in_dim = in_dim
+        self.out_dim = out_dim
+        self.num_layer = num_layer
+        self.hidden_dim = hidden_dim
+        self.noise_dim = noise_dim
+        
+        self.input_layer = StoLayer(in_dim, hidden_dim, noise_dim)
+        if num_layer > 2:
+            inter_layer = [StoLayer(hidden_dim, hidden_dim, noise_dim)]
+            for i in range(num_layer - 3):
+                inter_layer.append(StoLayer(hidden_dim, hidden_dim, noise_dim))
+            self.inter_layer = nn.Sequential(*inter_layer)
+        self.out_layer = nn.Linear(hidden_dim, out_dim)
+                
+    def predict(self, x, target=["mean"], sample_size=100):
+        """Point prediction.
+
+        Args:
+            x (torch.Tensor): _description_
+            target (str or float or list, optional): single-valued functional to predict. float refers to the quantiles. Defaults to ["mean"].
+            sample_size (int, optional): sample sizes for each x. Defaults to 100.
+
+        Returns:
+            torch.Tensor or list of torch.Tensor: point predictions
+                - [:,:,i] gives the i-th sample of all x.
+                - [i,:,:] gives all samples of x_i.
+            
+        Here we do not call `sample` but directly call `forward`.
+        """
+        # eval_size = x.size(0)
+        # # Sampling
+        # with torch.no_grad():
+        #     x_rep = torch.repeat_interleave(x, (sample_size * torch.ones(eval_size, device=x.device)).long(), dim=0)
+        #     y_pred = self.forward(x=x_rep).detach()
+        #     y_pred = list(torch.split(y_pred, sample_size))
+            
+        # if not isinstance(target, list):
+        #     target = [target]
+        # results = []
+        # extremes = []
+        # for t in target:
+        #     if t == "mean":
+        #         results.append(torch.cat([y_pred[i].mean().unsqueeze(0) for i in range(eval_size)], dim=0).unsqueeze(1))
+        #     else:
+        #         if t == "median":
+        #             t = 0.5
+        #         assert isinstance(t, float)
+        #         results.append(torch.cat([y_pred[i].quantile(t).unsqueeze(0) for i in range(eval_size)], dim=0).unsqueeze(1))
+        #         if min(t, 1-t) * sample_size < 10:
+        #             extremes.append(t)
+        
+        samples = self.sample(x=x, sample_size=sample_size, expand_dim=True)
+        if not isinstance(target, list):
+            target = [target]
+        results = []
+        extremes = []
+        for t in target:
+            if t == "mean":
+                results.append(samples.mean(dim=len(samples.shape) - 1))
+            else:
+                if t == "median":
+                    t = 0.5
+                assert isinstance(t, float)
+                results.append(samples.quantile(t, dim=len(samples.shape) - 1))
+                if min(t, 1 - t) * sample_size < 10:
+                    extremes.append(t)
+        
+        if len(extremes) > 0:
+            print("Warning: the estimate for quantiles at {} with a sample size of {} could be inaccurate. Please increase the `sample_size`.".format(extremes, sample_size))
+
+        if len(results) == 1:
+            return results[0]
+        else:
+            return results
+    
+    def sample(self, x, sample_size=100, expand_dim=True):
+        """Sample new response data.
+
+        Args:
+            x (torch.Tensor): new data of predictors of shape [data_size, covariate_dim]
+            sample_size (int, optional): new sample size. Defaults to 100.
+            expand_dim (bool, optional): whether to expand the sample dimension. Defaults to True.
+
+        Returns:
+            torch.Tensor of shape (data_size, response_dim, sample_size), where response_dim could have multiple channels.
+        """
+        eval_size = x.size(0)
+        with torch.no_grad():
+            # x_rep = torch.repeat_interleave(x, (sample_size * torch.ones(eval_size, device=x.device)).long(), dim=0)
+            x_rep = x.repeat(sample_size, 1)
+            samples = self.forward(x=x_rep).detach()
+        if not expand_dim:
+            return samples
+        else:
+            expand_dim = len(samples.shape)
+            samples = samples.unsqueeze(expand_dim)
+            samples = list(torch.split(samples, eval_size))
+            samples = torch.cat(samples, dim=expand_dim)
+            return samples
+            # without expanding dimensions:
+            # samples.reshape(-1, *samples.shape[1:-1])
+        
+    def forward(self, x, inject_noise=True):
+        x = self.input_layer(x, inject_noise)
+        for i in range(self.num_layer - 2):
+            x = self.inter_layer[i](x, inject_noise)
+        x = self.out_layer(x)
+        return x
+
+
+class Net(nn.Module):
+    """Deterministic neural network.
+
+    Args:
+        in_dim (int, optional): input dimension. Defaults to 1.
+        out_dim (int, optional): output dimension. Defaults to 1.
+        num_layer (int, optional): number of layers. Defaults to 2.
+        hidden_dim (int, optional): number of neurons per layer. Defaults to 100.
+    """
+    def __init__(self, in_dim=1, out_dim=1, num_layer=2, hidden_dim=100):
+        super().__init__()
+        net = [
+            nn.Linear(in_dim, hidden_dim),
+            nn.BatchNorm1d(hidden_dim),
+            nn.ReLU(inplace=True),
+        ]
+        for _ in range(num_layer - 2):
+            net += [
+                nn.Linear(hidden_dim, hidden_dim),
+                nn.BatchNorm1d(hidden_dim),
+                nn.ReLU(inplace=True),
+            ]
+        net.append(nn.Linear(hidden_dim, out_dim))
+        self.net = nn.Sequential(*net)
+
+    def forward(self, x):
+        return self.net(x)

engression/utils.py

@@ -0,0 +1,65 @@
+import os
+import torch
+
+def vectorize(x, multichannel=False):
+    """Vectorize data in any shape.
+
+    Args:
+        x (torch.Tensor): input data
+        multichannel (bool, optional): whether to keep the multiple channels (in the second dimension). Defaults to False.
+
+    Returns:
+        torch.Tensor: data of shape (sample_size, dimension) or (sample_size, num_channel, dimension) if multichannel is True.
+    """
+    if len(x.shape) == 1:
+        return x.unsqueeze(1)
+    if len(x.shape) == 2:
+        return x
+    else:
+        if not multichannel: # one channel
+            return x.reshape(x.shape[0], -1)
+        else: # multi-channel
+            return x.reshape(x.shape[0], x.shape[1], -1)
+        
+def cor(x, y):
+    """Compute the correlation between two signals.
+
+    Args:
+        x (torch.Tensor): input data
+        y (torch.Tensor): input data
+
+    Returns:
+        torch.Tensor: correlation between x and y
+    """
+    x = vectorize(x)
+    y = vectorize(y)
+    x = x - x.mean(0)
+    y = y - y.mean(0)
+    return ((x * y).mean()) / (x.std(unbiased=False) * y.std(unbiased=False))
+
+def make_folder(name):
+    """Make a folder.
+
+    Args:
+        name (str): folder name.
+    """
+    if not os.path.exists(name):
+        print('Creating folder: {}'.format(name))
+        os.makedirs(name)
+
+def check_for_gpu(device):
+    """Check if a CUDA device is available.
+
+    Args:
+        device (torch.device): current set device.
+    """
+    if device == torch.device('cuda'):
+        if torch.cuda.is_available():
+            print("GPU is available, running on GPU.\n")
+        else:
+            print("GPU is NOT available, running instead on CPU.\n")
+    else:
+        if torch.cuda.is_available():
+            print("Warning: You have a CUDA device, so you may consider using GPU for potential acceleration\n by setting device to 'cuda'.\n")
+        else:
+            print("Running on CPU.\n")

engression-0.0.0.dev0.dist-info/METADATA

@@ -0,0 +1,47 @@
+Metadata-Version: 2.1
+Name: engression
+Version: 0.0.0.dev0
+Summary: Engression
+Home-page: https://github.com/xwshen51/engression
+Author: Xinwei Shen
+Author-email: xinwei.shen@stat.math.ethz.ch
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Description-Content-Type: text/markdown
+Requires-Dist: torch
+Requires-Dist: numpy
+Requires-Dist: matplotlib
+
+# Engression
+
+Engression is a nonlinear regression methodology proposed in the paper "[*Engression: Extrapolation for Nonlinear Regression?*]()" by Xinwei Shen and Nicolai Meinshausen. 
+This directory contains the Python implementations of engression.
+
+## Installation
+The latest release of the Python package can be installed through pip:
+```sh
+pip install engression
+```
+
+The development version can be installed from github
+
+```sh
+$ git clone https://github.com/xwshen51/engression.git  # Download the package 
+$ cd engression
+$ pip install -r requirements.txt  # Install the requirements
+$ python setup.py install develop --user
+```
+
+
+## Usage Example
+
+### Python
+```python
+import engression
+
+```
+
+
+## Contact information
+If you meet any problems with the code, please submit an issue or contact Xinwei Shen (`xinwei.shen@stat.math.ethz.ch`).

engression-0.0.0.dev0.dist-info/RECORD

@@ -0,0 +1,13 @@
+engression/__init__.py,sha256=aS5GIvTRA2rpCrZOExqu6E9SvGcBZnI3E_VCsEqpFbc,365
+engression/engression.py,sha256=PAhm9uE95IDRI-mr_42iL3Ztd5poHBfmmHgWWswktCM,16497
+engression/engression_bagged.py,sha256=-_czVMqRWD2pJhHv0sZYfC4L95M9EwYwRskMiDbuXZ0,12725
+engression/loss_func.py,sha256=aE5KpdhARtChHUnJ-7knQrpG6vBFZdiMjg2Rd_c5Qcg,2026
+engression/models.py,sha256=NFNkhDNCO5Sl2C0RkKY2K8Eosh-_L75GRMx5ZGYY6tM,6999
+engression/utils.py,sha256=2fwA-DT4wjPzGZwGOaQfB4yYVyGhLfJJTAPGUZYysxU,1910
+engression/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+engression/data/loader.py,sha256=cITbaS5VH4hCWihCKnJWpX-xVS7glhwhDtmj1atWHUk,1980
+engression/data/simulator.py,sha256=i5wONmiQUxymg_lsVuJGB3q0-ZjFp6jzqFsXc-mIl3Y,2673
+engression-0.0.0.dev0.dist-info/METADATA,sha256=XFOKrupfz-3ozN3-p0WSNq7tA61zO1Qk48v3ISzF6OY,1295
+engression-0.0.0.dev0.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+engression-0.0.0.dev0.dist-info/top_level.txt,sha256=N0yRTczPTx-rOI-BgXZrjo8FB422Z2eH7uB0U99tZMw,11
+engression-0.0.0.dev0.dist-info/RECORD,,

engression-0.0.0.dev0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.40.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

engression-0.0.0.dev0.dist-info/top_level.txt

@@ -0,0 +1 @@
+engression