kernel-elastic-autoencoder 1.0.0__py3-none-any.whl

kernel_elastic_autoencoder/losses.py

@@ -0,0 +1,127 @@
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+
+class Loss(nn.Module):
+    """Module for Kernel-Elastic Autoencoder loss.
+
+    Layer storing hyperparameters, computing loss on forward pass. Losses are computed as:
+
+    $\\mathcal{L}(\\lambda, \\delta)=\\mathcal{L}_{WCEL}(\\lambda, \\delta) + m$-$MMD(\\lambda)$
+    """
+
+    def __init__(
+        self,
+        *,
+        hp_lambda: float,
+        hp_delta: float,
+        hp_sigma: float,
+        kernel_dist_size: int,
+        padding_idx: int,
+        embedding_dim: int,
+        pooling_dim: int,
+    ):
+        """Instantiates a Loss object.
+
+        Args:
+            hp_lambda: Hyperparameter $\\lambda$, as used in WCEL and m-MMD losses. Roughly, controls how strongly the
+                shape of the latent vector distribution is penalized.
+            hp_delta: Hyperparameter $\\delta$, as used in WCEL loss. Roughly, controls the relative weights of the
+                vanilla-AE and VAE objectives in the reconstruction loss.
+            hp_sigma: Hyperparameter $\\sigma$, as used in the Kernel function applied in m-MMD loss. Roughly,
+                used as a scaling factor to control the sizes of gradients produced by the m-MMD loss.
+            kernel_dist_size: Size of the sampled distribution of vectors used to penalize the shape of the latent
+                vector distribution through the kernel.
+            padding_idx: Index of padding token. Used internally to zero vectors corresponding to padding tokens through
+                embedding layers. Should be fetched with Tokenizer.pad_token_id as specified in the Tokenizer protocol.
+            embedding_dim: Embedding dimension used by nn.Embedding layers.
+            pooling_dim: Sequence length dimension to which inputs are pooled after condition concatenation through the
+                encoder. Proportional to the dimension of latent vectors.
+        """
+        super().__init__()
+        self.hp_lambda = hp_lambda
+        """Hyperparameter $\\lambda$, as used in WCEL and m-MMD losses. Roughly, controls how strongly the
+                shape of the latent vector distribution is penalized."""
+        self.hp_delta = hp_delta
+        """Hyperparameter $\\delta$, as used in WCEL loss. Roughly, controls the relative weights of the
+                vanilla-AE and VAE objectives in the reconstruction loss."""
+        self.hp_sigma = hp_sigma
+        """Hyperparameter $\\sigma$, as used in the Kernel function applied in m-MMD loss. Roughly,
+                used as a scaling factor to control the sizes of gradients produced by the m-MMD loss."""
+        self.kernel_dist_size = kernel_dist_size
+        """Size of the sampled distribution of vectors used to penalize the shape of the latent
+                vector distribution through the kernel."""
+        self.padding_idx = padding_idx
+        """Index of padding token. Used internally to zero vectors corresponding to padding tokens through
+                embedding layers. Should be fetched with Tokenizer.pad_token_id as specified in the Tokenizer protocol."""
+        self.embedding_dim = embedding_dim
+        """Embedding dimension used by nn.Embedding layers."""
+        self.pooling_dim = pooling_dim
+        """Sequence length dimension to which inputs are pooled after condition concatenation through the
+                encoder. Proportional to the dimension of latent vectors."""
+
+        self.register_buffer("_loc", torch.zeros(self.pooling_dim * self.embedding_dim))
+        self.register_buffer("_cov", torch.eye(self.pooling_dim * self.embedding_dim))
+
+    def forward(
+        self,
+        prediction: torch.Tensor,
+        prediction_noise: torch.Tensor,
+        ground_truth: torch.Tensor,
+        latents: torch.Tensor,
+    ) -> torch.Tensor:
+        """Forward pass through the loss module. May also be called using Loss.__call__.
+
+        Args:
+            prediction: Tensor of dimension (B, M, L) containing teacher forcing prediction logits without noise
+                before the decoder.
+            prediction_noise: Tensor of dimension (B, M, L) containing teacher forcing prediction logits with added noise
+                before the decoder.
+            ground_truth: Tensor of dimension (B, M) containing the indices of ground truth tokens for each sequence.
+            latents: Tensor of dimension (B, P * E) containing latent vectors produced by the encoder with added
+                noise.
+
+        Returns:
+            torch.Tensor: Tensor of dimension (1) containing loss value.
+        """
+        return self._weighted_cross_entropy_loss(
+            prediction, prediction_noise, ground_truth
+        ) + self._modified_maximum_mean_discrepancy(latents)
+
+    def _weighted_cross_entropy_loss(
+        self,
+        prediction: torch.Tensor,
+        prediction_noise: torch.Tensor,
+        ground_truth: torch.Tensor,
+    ) -> torch.Tensor:
+        ground_truth.to(torch.long)
+        log_softmax = torch.log_softmax(prediction, dim=-1)
+        log_softmax_noise = torch.log_softmax(prediction_noise, dim=-1)
+        Y = F.one_hot(ground_truth, num_classes=prediction.size(-1))
+        Y[..., self.padding_idx] = 0
+        t1 = (Y * log_softmax_noise).sum(dim=(-1, -2))
+        t2 = (Y * log_softmax).sum(dim=(-1, -2))
+        coeff = -1 / (self.hp_lambda + self.hp_delta + 1)
+        loss = coeff * (t1 + ((self.hp_lambda + self.hp_delta) * t2))
+        return loss.mean()
+
+    def _modified_maximum_mean_discrepancy(
+        self,
+        latents: torch.Tensor,
+    ) -> torch.Tensor:
+        mvn = torch.distributions.MultivariateNormal(
+            loc=self._loc,  # type: ignore
+            covariance_matrix=self._cov,  # type: ignore
+        )
+        samples = mvn.rsample((self.kernel_dist_size,))
+        square_difference_sum = torch.cdist(latents, samples, p=2.0).pow(2)
+        kernel_pairwise_sum = torch.exp(
+            ((-1 / (self.pooling_dim * self.embedding_dim)) * square_difference_sum)
+            / (2 * (self.hp_sigma**2))
+        ).sum()
+        loss = self.hp_lambda * (
+            torch.tensor([1])
+            - ((1 / (latents.size(0) * self.kernel_dist_size)) * kernel_pairwise_sum)
+        )
+        return loss

kernel_elastic_autoencoder/model.py

@@ -0,0 +1,222 @@
+from pathlib import Path
+from typing import Any
+
+import torch
+from huggingface_hub import PyTorchModelHubMixin
+from huggingface_hub.hub_mixin import T, DataclassInstance
+from torch import nn
+
+from kernel_elastic_autoencoder.config import ModelConfig
+from kernel_elastic_autoencoder.layers import Decoder, Encoder, TrainingNoise
+
+CODERS = {
+    ModelConfig: (
+        lambda x: x.model_dump(),
+        lambda data: ModelConfig.model_validate(data),
+    )
+}
+
+
+class Model(
+    nn.Module,
+    PyTorchModelHubMixin,
+    coders=CODERS,
+    library_name="kernel_elastic_autoencoder",
+    repo_url="https://github.com/cancelradius/kae",
+    paper_url="https://arxiv.org/abs/2310.08685",
+):
+    """Base class for Kernel-Elastic Autoencoder models.
+
+    Defines a barebones API for model calls with tensor inputs. For easy use with standard data formats, calling Model
+    through the Pipeline API may be preferred.
+    """
+
+    def __init__(self, config: dict | ModelConfig):
+        """Instantiates a Model object.
+
+        Args:
+            config: Dictionary or ModelConfig schema defining model parameters. Will be validated with ModelConfig
+                regardless of input type.
+        """
+        super().__init__()
+
+        self.config = config
+        """Configuration object for Hugging Face Hub compatible serialization. Not recommended to use, as it is 
+        internal-use. Use Model.config_typed instead."""
+        config_typed = ModelConfig.model_validate(config)
+        self.config_typed: ModelConfig = config_typed
+        """Type-validated config in a Pydantic ModelConfig schema, recommended for public API use."""
+
+        self.encoder = Encoder(
+            max_len=config_typed.input.max_len,
+            vocab_size=config_typed.input.vocab_size,
+            condition_channels=config_typed.input.condition_channels,
+            embedding_dim=config_typed.common.embedding_dim,
+            pooling_dim=config_typed.common.pooling_dim,
+            padding_idx=config_typed.common.padding_idx,
+            padding_value=config_typed.common.padding_value,
+            num_layers=config_typed.encoder.num_layers,
+            num_heads=config_typed.encoder.num_heads,
+            feedforward_scale=config_typed.encoder.feedforward_scale,
+            dropout=config_typed.encoder.dropout,
+        )
+        """Model encoder module."""
+        self.noise = TrainingNoise()
+        """Noise-adding module for training purposes."""
+        self.decoder = Decoder(
+            max_len=config_typed.input.max_len,
+            vocab_size=config_typed.input.vocab_size,
+            condition_channels=config_typed.input.condition_channels,
+            embedding_dim=config_typed.common.embedding_dim,
+            pooling_dim=config_typed.common.pooling_dim,
+            padding_idx=config_typed.common.padding_idx,
+            padding_value=config_typed.common.padding_value,
+            num_layers=config_typed.decoder.num_layers,
+            num_heads=config_typed.decoder.num_heads,
+            feedforward_scale=config_typed.decoder.feedforward_scale,
+            dropout=config_typed.decoder.dropout,
+        )
+        """Model decoder module."""
+
+    @classmethod
+    def from_pretrained(
+        cls: type[T],
+        pretrained_model_name_or_path: str | Path,
+        *,
+        force_download: bool = False,
+        token: str | bool | None = None,
+        cache_dir: str | Path | None = None,
+        local_files_only: bool = False,
+        revision: str | None = None,
+        **model_kwargs,
+    ) -> T:
+        return cls.from_pretrained(
+            pretrained_model_name_or_path,
+            force_download=force_download,
+            token=token,
+            cache_dir=cache_dir,
+            local_files_only=local_files_only,
+            revision=revision,
+            **model_kwargs,
+        )
+
+    def save_pretrained(
+        self,
+        save_directory: str | Path,
+        *,
+        config: dict | DataclassInstance | None = None,
+        repo_id: str | None = None,
+        push_to_hub: bool = False,
+        model_card_kwargs: dict[str, Any] | None = None,
+        **push_to_hub_kwargs,
+    ) -> str | None:
+        return super().save_pretrained(
+            save_directory,
+            config=config,
+            repo_id=repo_id,
+            push_to_hub=push_to_hub,
+            model_card_kwargs=model_card_kwargs,
+            **push_to_hub_kwargs,
+        )
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        conditions: torch.Tensor,
+        token_mask: torch.Tensor,
+        condition_mask: torch.Tensor,
+    ):
+        """Forward pass for training loop. Produces all tensors needed for computing loss with
+        Loss.forward.
+
+        Args:
+            input_ids: Tensor of dimension (B, M) containing input ids for each sequence.
+            conditions: Tensor of dimension (B, C) containing condition values for each sequence.
+            token_mask: Tensor of dimension (B, M) containing boolean padding masks for each sequence.
+            condition_mask: Tensor of dimension (B, C) containing boolean condition padding masks for each sequence.
+
+        Returns:
+            torch.Tensor: Tensor of dimension (B, M, L) containing teacher forcing prediction logits without noise
+                before the decoder.
+            torch.Tensor: Tensor of dimension (B, M, L) containing teacher forcing prediction logits with added noise
+                before the decoder.
+            torch.Tensor: Tensor of dimension (B, P * E) containing latent vectors produced by the encoder with added
+                noise.
+        """
+        padding_mask = torch.cat((token_mask, condition_mask), 1)
+        latents, condition_embeddings = self.encoder(
+            input_ids, conditions, padding_mask
+        )
+        latents_noise = self.noise(latents)
+
+        prediction = self.decoder(
+            input_ids[:, :-1],
+            latents,
+            condition_embeddings,
+            padding_mask[: self.config_typed.input.max_len - 1],
+        )
+        prediction_noise = self.decoder(
+            input_ids[:, :-1],
+            latents_noise,
+            condition_embeddings,
+            padding_mask[: self.config_typed.input.max_len - 1],
+        )
+        return prediction, prediction_noise, latents_noise
+
+    def encode(
+        self,
+        input_ids: torch.Tensor,
+        conditions: torch.Tensor,
+        token_mask: torch.Tensor | None,
+        condition_mask: torch.Tensor,
+    ) -> torch.Tensor:
+        """Basic interface for a forward pass through the Model.model.encoder module.
+
+        Args:
+            input_ids: Tensor of dimension (B, M) containing input ids for each sequence.
+            conditions: Tensor of dimension (B, C) containing condition values for each sequence.
+            token_mask: Tensor of dimension (B, M) containing boolean padding masks for each sequence.
+            condition_mask: Tensor of dimension (B, C) containing boolean condition padding masks for each sequence.
+
+        Returns:
+            torch.Tensor: Tensor of dimension (B, P * E) containing latent vectors produced by the encoder.
+        """
+        padding_mask = (
+            torch.cat((token_mask, condition_mask), 1)
+            if (token_mask is not None)
+            else torch.cat((torch.full_like(input_ids, True), condition_mask), 1)
+        )
+        return self.encoder(input_ids, conditions, padding_mask)[0]
+
+    def decode(
+        self,
+        current_output: torch.Tensor,
+        latents: torch.Tensor,
+        condition_embeddings: torch.Tensor,
+        token_mask: torch.Tensor | None,
+        condition_mask: torch.Tensor,
+    ) -> torch.Tensor:
+        """Basic interface for a forward pass through the Model.model.decoder module.
+
+        Args:
+            current_output: Tensor of dimension (B, S) containing previous output ids for each sequence. In
+                autoregressive generation, this should initially represent a single start-of-sequence token.
+            latents: Tensor of dimension (B, P * E) containing latent vectors. May be produced using Model.encode.
+            condition_embeddings: Tensor of dimension (B, C, E) containing condition embeddings for each sequence. May
+                be produced using Model.embed_conditions.
+            token_mask: Tensor of dimension (B, S) containing boolean padding masks for each sequence. If None is
+                passed, the absence of padding is assumed.
+            condition_mask: Tensor of dimension (B, C) containing boolean condition padding masks for each sequence.
+
+        Returns:
+            torch.Tensor: Tensor of dimension (B, S, L) containing prediction logits produced by the decoder.
+        """
+        padding_mask = (
+            torch.cat((token_mask, condition_mask), 1)
+            if (token_mask is not None)
+            else torch.cat((torch.full_like(current_output, False), condition_mask), 1)
+        ).to(torch.bool)
+        return self.decoder(current_output, latents, condition_embeddings, padding_mask)
+
+    def embed_conditions(self, conditions):
+        return self.encoder.embedding.conditional_embedding(conditions)

kernel_elastic_autoencoder/pipeline.py

@@ -0,0 +1,203 @@
+from collections.abc import Callable, Iterable
+
+import torch
+from pydantic import BaseModel, ConfigDict
+
+from kernel_elastic_autoencoder.collate import Collator, DataframeCollator
+from kernel_elastic_autoencoder.config import ModelConfig
+from kernel_elastic_autoencoder.model import Model
+from kernel_elastic_autoencoder.sample import Sampler, Top1Sampler
+from kernel_elastic_autoencoder.tokenizer import Tokenizer
+
+
+class CompletionIntermediate(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    latents: torch.Tensor
+    input_ids: torch.Tensor
+    batches_completed: torch.Tensor
+    condition_embeddings: torch.Tensor
+    condition_mask: torch.Tensor
+
+
+class Completion(BaseModel):
+    """Return schema for Pipeline.completion."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    outputs: Iterable[str]
+    """Iterable of generated output sequences."""
+    condition_embeddings: torch.Tensor
+    """Tensor containing generated condition embeddings for reuse."""
+    condition_mask: torch.Tensor
+    """Tensor containing generated condition masks for reuse."""
+
+
+class Pipeline[_T]:
+    """User-facing pipeline for inference.
+
+    Defines an easy-to-use API for decoder-only inference with a pretrained model.
+
+    Examples:
+        Wrapping a pretrained model and tokenizer:
+        >>> model = Model.from_pretrained("./checkpoint")
+        >>> tokenizer = MyClassImplementingTokenizer.from_pretrained("./checkpoint/tokenizer")
+        >>> pipe = Pipeline(model, tokenizer, MyClassImplementingCollator, MyClassImplementingSampler)
+        >>> compl = pipe.completion(latents, dataset, seq_feature, cond_features)
+        >>> print(compl.outputs)
+
+        TODO: Complete examples.
+    """
+
+    def __init__(
+        self,
+        model: Model,
+        tokenizer: Tokenizer,
+        collator: Callable[[ModelConfig, Tokenizer], Collator[_T]] = DataframeCollator,
+        sampler: Callable[[Tokenizer], Sampler] = Top1Sampler,
+        device: torch.device | None = None,
+    ) -> None:
+        """Instantiates a Pipeline object.
+
+        TODO: Pipeline setup will be streamlined such that collators and samplers are by default fetched from model
+            configuration.
+
+        Args:
+            model: Pre-trained Model object for inference. Can be obtained from Model.from_pretrained.
+            tokenizer: Pre-configured Tokenizer object. The Tokenizer protocol supports tokenizers inheriting from
+                transformers.PreTrainedTokenizerBase, so such tokenizers may be loaded from HuggingFace Hub.
+            collator: Constructor for a data collator inheriting from Collator. Determines input types to Pipeline methods.
+            sampler: Constructor for a data sampler inheriting from Sampler. Determines how outputs are produced from
+                Pipeline methods.
+            device: Torch device used for inference.
+        """
+        self.model = model.to(device)
+        """Pre-trained Model object for inference. Moved to Pipeline.device, and placed in eval() mode."""
+        self.model.eval()
+        self.tokenizer = tokenizer
+        """Pre-configured Tokenizer object. The Tokenizer Protocol supports tokenizers inheriting from
+            transformers.PreTrainedTokenizerBase, so such tokenizers may be loaded from HuggingFace Hub."""
+        self.device = device
+        """Torch device used for inference."""
+
+        self.collator = collator(model.config_typed, tokenizer)
+        """Data collator inheriting from Collator, instantiated by passing Pipeline.model.config_typed and Pipeline.tokenizer."""
+        self.sampler = sampler(tokenizer)
+        """Data sampler inheriting from Sampler, instantiated by passing Pipeline.tokenizer."""
+
+    @torch.inference_mode()
+    def _completion_entry(
+        self,
+        latents: torch.Tensor,
+        dataset: _T,
+        seq_feature: str,
+        cond_features: list[str],
+        device: torch.device | None = None,
+        **kwargs,
+    ) -> CompletionIntermediate:
+        ds = self.collator(
+            dataset=dataset,
+            seq_feature=seq_feature,
+            cond_features=cond_features,
+            padding=False,
+            add_special_tokens=False,
+            device=device,
+            **kwargs,
+        )
+        conds_embed = self.model.embed_conditions(ds.conditions)
+        return CompletionIntermediate(
+            latents=latents,
+            input_ids=ds.input_ids,
+            batches_completed=torch.zeros(ds.input_ids.size(0), dtype=torch.bool),
+            condition_embeddings=conds_embed,
+            condition_mask=ds.condition_mask,
+        )
+
+    @torch.inference_mode()
+    def _completion_step(
+        self, intermediate: CompletionIntermediate, **kwargs
+    ) -> CompletionIntermediate:
+        intermediate.input_ids = torch.cat(
+            [
+                torch.full(
+                    (intermediate.input_ids.size(0), 1), self.tokenizer.bos_token_id
+                ),
+                intermediate.input_ids,
+            ],
+            dim=1,
+        )
+        logits = self.model.decode(
+            current_output=intermediate.input_ids,
+            latents=intermediate.latents,
+            condition_embeddings=intermediate.condition_embeddings,
+            token_mask=None,
+            condition_mask=intermediate.condition_mask,
+        )
+        new_toks = self.sampler.sample_ids(logits[:, -1:])
+        intermediate.batches_completed |= (
+            new_toks.squeeze(-1) == self.tokenizer.eos_token_id
+        )
+        torch.where(
+            intermediate.batches_completed.unsqueeze(-1),
+            self.tokenizer.pad_token_id,
+            new_toks,
+        )
+        intermediate.input_ids = torch.cat([intermediate.input_ids, new_toks], dim=1)
+        return intermediate
+
+    @torch.inference_mode()
+    def _completion_exit(
+        self, intermediate: CompletionIntermediate, **kwargs
+    ) -> Completion:
+        return Completion(
+            outputs=self.tokenizer.decode(
+                intermediate.input_ids, skip_special_tokens=True
+            ),
+            condition_embeddings=intermediate.condition_embeddings,
+            condition_mask=intermediate.condition_mask,
+        )
+
+    def completion(
+        self,
+        latents: torch.Tensor,
+        dataset: _T,
+        seq_feature: str,
+        cond_features: list[str],
+        device: torch.device | None = None,
+        **kwargs,
+    ) -> Completion:
+        """Completes each batch in an input dataset with latent and condition guidance through the decoder.
+
+        Using decoder-only inference, the model completes each batch in the provided dataset. Dataset input typing is
+        uniquely defined by the used Collator. The input dataset is passed through the collator, then subject to
+        autoregressive decoder-only inference through the model, where new tokens are sampled from intermediates and
+        stored in a tensor format. Finally, on reaching end-of-sequence tokens or the maximum sequence length, the
+        sequence is decoded by the tokenizer and returned along with the generated condition embeddings and masks in a
+        Completion schema.
+
+        TODO: Callbacks are planned to enable easy custom inference functions and streaming of text.
+
+        Args:
+            latents: Tensor of dimension (B, P * E) containing latent vectors for the batches.
+            dataset: Dataset of type specified in Pipeline.collator.
+            seq_feature: String to pass to collator containing the feature or column containing text sequences to
+                complete.
+            cond_features: List of strings to pass to collator containing features or columns containing numerical
+                condition values.
+            device: Torch device used for inference.
+            **kwargs: Additional keyword arguments passed to Pipeline.collator.
+
+        Returns:
+            Completion: Completion object, containing outputs, as well as condition embeddings and masks for reuse.
+        """
+        intermediate = self._completion_entry(
+            latents=latents,
+            dataset=dataset,
+            seq_feature=seq_feature,
+            cond_features=cond_features,
+            device=device,
+            **kwargs,
+        )
+        while (
+            intermediate.input_ids.size(1) < self.model.config_typed.input.max_len
+        ) and (not intermediate.batches_completed.all()):
+            intermediate = self._completion_step(intermediate=intermediate)
+        return self._completion_exit(intermediate=intermediate)

kernel_elastic_autoencoder/sample.py

@@ -0,0 +1,58 @@
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+
+import torch
+
+from kernel_elastic_autoencoder.tokenizer import Tokenizer
+
+
+class Sampler(ABC):
+    """Base class for samplers.
+
+    Provides a specification for sampling text sequences from logits.
+    """
+
+    def __init__(self, tokenizer: Tokenizer):
+        """Instantiates a Sampler object.
+
+        Args:
+            tokenizer: Object implementing the Tokenizer protocol.
+        """
+        self.tokenizer = tokenizer
+        """Object implementing the Tokenizer protocol."""
+
+    def __call__(
+        self, logits: torch.Tensor, skip_special_tokens: bool, **kwargs
+    ) -> Iterable[str]:
+        ids = self.sample_ids(logits, **kwargs)
+        sampled = self.tokenizer.decode(ids, skip_special_tokens=skip_special_tokens)
+        return sampled
+
+    @abstractmethod
+    def sample_ids(self, logits: torch.Tensor, **kwargs) -> torch.Tensor:
+        """Interface method for implementing index sampling from logits.
+
+        Args:
+            logits: Tensor of dimension (B, S, L) containing logits.
+            **kwargs: Keyword arguments.
+
+        Returns:
+            torch.Tensor: Tensor of dimension (B, S) containing vocabulary indices.
+        """
+        ...
+
+
+class Top1Sampler(Sampler):
+    def sample_ids(self, logits: torch.Tensor, **kwargs) -> torch.Tensor:
+        """Implementation of index sampling from logits choosing the highest-probability token.
+
+        Args:
+            logits: Tensor of dimension (B, S, L) containing logits.
+            **kwargs: Keyword arguments.
+
+        Returns:
+            torch.Tensor: Tensor of dimension (B, S) containing vocabulary indices.
+        """
+        return (
+            torch.topk(logits, k=1, dim=-1, **kwargs).indices.squeeze(-1).to(torch.long)
+        )