xinference 1.9.1__py3-none-any.whl → 1.10.1__py3-none-any.whl

xinference/thirdparty/indextts/utils/maskgct_utils.py

@@ -0,0 +1,259 @@
+import torch
+import librosa
+import json5
+from huggingface_hub import hf_hub_download
+from transformers import SeamlessM4TFeatureExtractor, Wav2Vec2BertModel
+import safetensors
+import numpy as np
+
+from indextts.utils.maskgct.models.codec.kmeans.repcodec_model import RepCodec
+from indextts.utils.maskgct.models.tts.maskgct.maskgct_s2a import MaskGCT_S2A
+from indextts.utils.maskgct.models.codec.amphion_codec.codec import CodecEncoder, CodecDecoder
+import time
+
+
+def _load_config(config_fn, lowercase=False):
+    """Load configurations into a dictionary
+
+    Args:
+        config_fn (str): path to configuration file
+        lowercase (bool, optional): whether changing keys to lower case. Defaults to False.
+
+    Returns:
+        dict: dictionary that stores configurations
+    """
+    with open(config_fn, "r") as f:
+        data = f.read()
+    config_ = json5.loads(data)
+    if "base_config" in config_:
+        # load configurations from new path
+        p_config_path = os.path.join(os.getenv("WORK_DIR"), config_["base_config"])
+        p_config_ = _load_config(p_config_path)
+        config_ = override_config(p_config_, config_)
+    if lowercase:
+        # change keys in config_ to lower case
+        config_ = get_lowercase_keys_config(config_)
+    return config_
+
+
+def load_config(config_fn, lowercase=False):
+    """Load configurations into a dictionary
+
+    Args:
+        config_fn (str): path to configuration file
+        lowercase (bool, optional): _description_. Defaults to False.
+
+    Returns:
+        JsonHParams: an object that stores configurations
+    """
+    config_ = _load_config(config_fn, lowercase=lowercase)
+    # create an JsonHParams object with configuration dict
+    cfg = JsonHParams(**config_)
+    return cfg
+
+
+class JsonHParams:
+    def __init__(self, **kwargs):
+        for k, v in kwargs.items():
+            if type(v) == dict:
+                v = JsonHParams(**v)
+            self[k] = v
+
+    def keys(self):
+        return self.__dict__.keys()
+
+    def items(self):
+        return self.__dict__.items()
+
+    def values(self):
+        return self.__dict__.values()
+
+    def __len__(self):
+        return len(self.__dict__)
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        return setattr(self, key, value)
+
+    def __contains__(self, key):
+        return key in self.__dict__
+
+    def __repr__(self):
+        return self.__dict__.__repr__()
+
+
+def build_semantic_model(path_='./models/tts/maskgct/ckpt/wav2vec2bert_stats.pt'):
+    semantic_model = Wav2Vec2BertModel.from_pretrained("facebook/w2v-bert-2.0")
+    semantic_model.eval()
+    stat_mean_var = torch.load(path_)
+    semantic_mean = stat_mean_var["mean"]
+    semantic_std = torch.sqrt(stat_mean_var["var"])
+    return semantic_model, semantic_mean, semantic_std
+
+
+def build_semantic_codec(cfg):
+    semantic_codec = RepCodec(cfg=cfg)
+    semantic_codec.eval()
+    return semantic_codec
+
+
+def build_s2a_model(cfg, device):
+    soundstorm_model = MaskGCT_S2A(cfg=cfg)
+    soundstorm_model.eval()
+    soundstorm_model.to(device)
+    return soundstorm_model
+
+
+def build_acoustic_codec(cfg, device):
+    codec_encoder = CodecEncoder(cfg=cfg.encoder)
+    codec_decoder = CodecDecoder(cfg=cfg.decoder)
+    codec_encoder.eval()
+    codec_decoder.eval()
+    codec_encoder.to(device)
+    codec_decoder.to(device)
+    return codec_encoder, codec_decoder
+
+
+class Inference_Pipeline():
+    def __init__(
+            self,
+            semantic_model,
+            semantic_codec,
+            semantic_mean,
+            semantic_std,
+            codec_encoder,
+            codec_decoder,
+            s2a_model_1layer,
+            s2a_model_full,
+            ):
+        self.semantic_model = semantic_model
+        self.semantic_codec = semantic_codec
+        self.semantic_mean = semantic_mean
+        self.semantic_std = semantic_std
+
+        self.codec_encoder = codec_encoder
+        self.codec_decoder = codec_decoder
+        self.s2a_model_1layer = s2a_model_1layer
+        self.s2a_model_full = s2a_model_full
+
+    @torch.no_grad()
+    def get_emb(self, input_features, attention_mask):
+        vq_emb = self.semantic_model(
+            input_features=input_features,
+            attention_mask=attention_mask,
+            output_hidden_states=True,
+        )
+        feat = vq_emb.hidden_states[17]  # (B, T, C)
+        feat = (feat - self.semantic_mean.to(feat)) / self.semantic_std.to(feat)
+        return feat
+
+    @torch.no_grad()
+    def extract_acoustic_code(self, speech):
+        vq_emb = self.codec_encoder(speech.unsqueeze(1))
+        _, vq, _, _, _ = self.codec_decoder.quantizer(vq_emb)
+        acoustic_code = vq.permute(1, 2, 0)
+        return acoustic_code
+
+    @torch.no_grad()
+    def get_scode(self, inputs):
+        semantic_code, feat = self.semantic_codec.quantize(inputs)
+        # vq = self.semantic_codec.quantizer.vq2emb(semantic_code.unsqueeze(1))
+        # vq = vq.transpose(1,2)
+        return semantic_code
+
+    @torch.no_grad()
+    def semantic2acoustic(
+        self,
+        combine_semantic_code,
+        acoustic_code,
+        n_timesteps=[25, 10, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
+        cfg=2.5,
+        rescale_cfg=0.75,
+    ):
+        semantic_code = combine_semantic_code
+
+        cond = self.s2a_model_1layer.cond_emb(semantic_code)
+        prompt = acoustic_code[:, :, :]
+        predict_1layer = self.s2a_model_1layer.reverse_diffusion(
+            cond=cond,
+            prompt=prompt,
+            temp=1.5,
+            filter_thres=0.98,
+            n_timesteps=n_timesteps[:1],
+            cfg=cfg,
+            rescale_cfg=rescale_cfg,
+        )
+
+        cond = self.s2a_model_full.cond_emb(semantic_code)
+        prompt = acoustic_code[:, :, :]
+        predict_full = self.s2a_model_full.reverse_diffusion(
+            cond=cond,
+            prompt=prompt,
+            temp=1.5,
+            filter_thres=0.98,
+            n_timesteps=n_timesteps,
+            cfg=cfg,
+            rescale_cfg=rescale_cfg,
+            gt_code=predict_1layer,
+        )
+
+        vq_emb = self.codec_decoder.vq2emb(
+            predict_full.permute(2, 0, 1), n_quantizers=12
+        )
+        recovered_audio = self.codec_decoder(vq_emb)
+        prompt_vq_emb = self.codec_decoder.vq2emb(
+            prompt.permute(2, 0, 1), n_quantizers=12
+        )
+        recovered_prompt_audio = self.codec_decoder(prompt_vq_emb)
+        recovered_prompt_audio = recovered_prompt_audio[0][0].cpu().numpy()
+        recovered_audio = recovered_audio[0][0].cpu().numpy()
+        combine_audio = np.concatenate([recovered_prompt_audio, recovered_audio])
+
+        return combine_audio, recovered_audio
+
+    def s2a_inference(
+        self,
+        prompt_speech_path,
+        combine_semantic_code,
+        cfg=2.5,
+        n_timesteps_s2a=[25, 10, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
+        cfg_s2a=2.5,
+        rescale_cfg_s2a=0.75,
+    ):
+        speech = librosa.load(prompt_speech_path, sr=24000)[0]
+        acoustic_code = self.extract_acoustic_code(
+            torch.tensor(speech).unsqueeze(0).to(combine_semantic_code.device)
+        )
+        _, recovered_audio = self.semantic2acoustic(
+            combine_semantic_code,
+            acoustic_code,
+            n_timesteps=n_timesteps_s2a,
+            cfg=cfg_s2a,
+            rescale_cfg=rescale_cfg_s2a,
+        )
+
+        return recovered_audio
+
+    @torch.no_grad()
+    def gt_inference(
+        self,
+        prompt_speech_path,
+        combine_semantic_code,
+    ):
+        speech = librosa.load(prompt_speech_path, sr=24000)[0]
+        '''
+        acoustic_code = self.extract_acoustic_code(
+            torch.tensor(speech).unsqueeze(0).to(combine_semantic_code.device)
+        )
+        prompt = acoustic_code[:, :, :]
+        prompt_vq_emb = self.codec_decoder.vq2emb(
+            prompt.permute(2, 0, 1), n_quantizers=12
+        )
+        '''
+
+        prompt_vq_emb = self.codec_encoder(torch.tensor(speech).unsqueeze(0).unsqueeze(1).to(combine_semantic_code.device))
+        recovered_prompt_audio = self.codec_decoder(prompt_vq_emb)
+        recovered_prompt_audio = recovered_prompt_audio[0][0].cpu().numpy()
+        return recovered_prompt_audio

xinference/thirdparty/indextts/utils/text_utils.py

@@ -0,0 +1,41 @@
+import re
+
+from textstat import textstat
+
+
+def contains_chinese(text):
+    # 正则表达式，用于匹配中文字符 + 数字 -> 都认为是 zh
+    if re.search(r'[\u4e00-\u9fff0-9]', text):
+        return True
+    return False
+
+
+def get_text_syllable_num(text):
+    chinese_char_pattern = re.compile(r'[\u4e00-\u9fff]')
+    number_char_pattern = re.compile(r'[0-9]')
+    syllable_num = 0
+    tokens = re.findall(r'[\u4e00-\u9fff]+|[a-zA-Z]+|[0-9]+', text)
+    # print(tokens)
+    if contains_chinese(text):
+        for token in tokens:
+            if chinese_char_pattern.search(token) or number_char_pattern.search(token):
+                syllable_num += len(token)
+            else:
+                syllable_num += textstat.syllable_count(token)
+    else:
+        syllable_num = textstat.syllable_count(text)
+
+    return syllable_num
+
+
+def get_text_tts_dur(text):
+    min_speed = 3  # 2.18 #
+    max_speed = 5.50
+
+    ratio = 0.8517 if contains_chinese(text) else 1.0
+
+    syllable_num = get_text_syllable_num(text)
+    max_dur = syllable_num * ratio / max_speed
+    min_dur = syllable_num * ratio / min_speed
+
+    return max_dur, min_dur

xinference/thirdparty/indextts/utils/typical_sampling.py

@@ -0,0 +1,30 @@
+import torch
+from transformers import TypicalLogitsWarper as BaseTypicalLogitsWarper
+
+class TypicalLogitsWarper(BaseTypicalLogitsWarper):
+    def __init__(self, mass: float = 0.9, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1):
+        super().__init__(mass=mass, filter_value=filter_value, min_tokens_to_keep=min_tokens_to_keep)
+
+    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
+        # calculate entropy
+        normalized = torch.nn.functional.log_softmax(scores, dim=-1)
+        p = torch.exp(normalized)
+        ent = -(normalized * p).nansum(-1, keepdim=True)
+
+        # shift and sort
+        shifted_scores = torch.abs((-normalized) - ent)
+        sorted_scores, sorted_indices = torch.sort(shifted_scores, descending=False)
+        sorted_logits = scores.gather(-1, sorted_indices)
+        cumulative_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1)
+
+        # Remove tokens with cumulative mass above the threshold
+        last_ind = (cumulative_probs < self.mass).sum(dim=1)
+        last_ind[last_ind < 0] = 0
+        sorted_indices_to_remove = sorted_scores > sorted_scores.gather(1, last_ind.view(-1, 1))
+        if self.min_tokens_to_keep > 1:
+            # Keep at least min_tokens_to_keep (set to min_tokens_to_keep-1 because we add the first one below)
+            sorted_indices_to_remove[..., : self.min_tokens_to_keep] = 0
+        indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
+
+        scores = scores.masked_fill(indices_to_remove, self.filter_value)
+        return scores

xinference/thirdparty/indextts/utils/utils.py

@@ -0,0 +1,93 @@
+import os
+import re
+import random
+import torch
+import torchaudio
+
+MATPLOTLIB_FLAG = False
+
+
+def load_audio(audiopath, sampling_rate):
+    audio, sr = torchaudio.load(audiopath)
+    #print(f"wave shape: {audio.shape}, sample_rate: {sr}")
+
+    if audio.size(0) > 1:  # mix to mono
+        audio = audio[0].unsqueeze(0)
+
+    if sr != sampling_rate:
+        try:
+            audio = torchaudio.functional.resample(audio, sr, sampling_rate)
+        except Exception as e:
+            print(f"Warning: {audiopath}, wave shape: {audio.shape}, sample_rate: {sr}")
+            return None
+    # clip audio invalid values
+    audio.clip_(-1, 1)
+    return audio
+
+
+def tokenize_by_CJK_char(line: str) -> str: 
+    """  
+    Tokenize a line of text with CJK char.
+
+    Note: All return charaters will be upper case.
+
+    Example:                                                                                                                                                                                                                                                                    
+      input = "你好世界是 hello world 的中文"
+      output = "你 好 世 界 是 HELLO WORLD 的 中 文"
+
+    Args:
+      line:
+        The input text.
+
+    Return:
+      A new string tokenize by CJK char.
+    """
+    # The CJK ranges is from https://github.com/alvations/nltk/blob/79eed6ddea0d0a2c212c1060b477fc268fec4d4b/nltk/tokenize/util.py
+    pattern = re.compile(
+        r"([\u1100-\u11ff\u2e80-\ua4cf\ua840-\uD7AF\uF900-\uFAFF\uFE30-\uFE4F\uFF65-\uFFDC\U00020000-\U0002FFFF])"
+    )    
+    chars = pattern.split(line.strip().upper())
+    return " ".join([w.strip() for w in chars if w.strip()])
+
+
+def make_pad_mask(lengths: torch.Tensor, max_len: int = 0) -> torch.Tensor:
+    """Make mask tensor containing indices of padded part.
+
+    See description of make_non_pad_mask.
+
+    Args:
+        lengths (torch.Tensor): Batch of lengths (B,).
+    Returns:
+        torch.Tensor: Mask tensor containing indices of padded part.
+
+    Examples:
+        >>> lengths = [5, 3, 2]
+        >>> make_pad_mask(lengths)
+        masks = [[0, 0, 0, 0 ,0],
+                 [0, 0, 0, 1, 1],
+                 [0, 0, 1, 1, 1]]
+    """
+    batch_size = lengths.size(0)
+    max_len = max_len if max_len > 0 else lengths.max().item()
+    seq_range = torch.arange(0,
+                             max_len,
+                             dtype=torch.int64,
+                             device=lengths.device)
+    seq_range_expand = seq_range.unsqueeze(0).expand(batch_size, max_len)
+    seq_length_expand = lengths.unsqueeze(-1)
+    mask = seq_range_expand >= seq_length_expand
+    return mask
+
+
+def safe_log(x: torch.Tensor, clip_val: float = 1e-7) -> torch.Tensor:
+    """
+    Computes the element-wise logarithm of the input tensor with clipping to avoid near-zero values.
+
+    Args:
+        x (Tensor): Input tensor.
+        clip_val (float, optional): Minimum value to clip the input tensor. Defaults to 1e-7.
+
+    Returns:
+        Tensor: Element-wise logarithm of the input tensor with clipping applied.
+    """
+    return torch.log(torch.clip(x, min=clip_val))

xinference/thirdparty/indextts/utils/webui_utils.py

@@ -0,0 +1,42 @@
+import gradio as gr
+
+
+def html_center(text, label='p'):
+    return f"""<div style="text-align: center; margin: 100; padding: 50;">
+                <{label} style="margin: 0; padding: 0;">{text}</{label}>
+                </div>"""
+
+
+def html_left(text, label='p'):
+    return f"""<div style="text-align: left; margin: 0; padding: 0;">
+                <{label} style="margin: 0; padding: 0;">{text}</{label}>
+                </div>"""
+
+
+def next_page(page_number,sentences):
+    new_page_number = int(page_number) + 1
+    update_page_number = gr.update(value=str(new_page_number))
+    update_prev_page = gr.update(visible=True, interactive=True)
+    if len(sentences.values) <= new_page_number * 20:
+        update_next_page = gr.update(visible=False, interactive=False)
+    else:
+        update_next_page = gr.update(visible=True, interactive=True)
+    return update_page_number, update_next_page, update_prev_page
+
+
+def prev_page(page_number):
+    new_page_number = int(page_number) - 1
+    update_page_number = gr.update(value=str(new_page_number))
+    if new_page_number == 1:
+        update_prev_page = gr.update(visible=False, interactive=False)
+    else:
+        update_prev_page = gr.update(visible=True, interactive=True)
+    update_next_page = gr.update(visible=True, interactive=True)
+    return update_page_number, update_next_page, update_prev_page
+
+
+def update_current_texts(page_number,sentences):
+    start_index = (int(page_number) - 1) * 20
+    end_index = int(page_number) * 20
+    current_texts = sentences.values[start_index:end_index if end_index < len(sentences.values) else len(sentences.values)]
+    return gr.update(values=current_texts)