import torch torch.manual_seed(0) torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True import random random.seed(0) import numpy as np np.random.seed(0) import librosa from copy import deepcopy from huggingface_hub import hf_hub_download import spaces import yaml import re import numpy as np import torch import torch.nn.functional as F import torchaudio from ipa_uk import ipa from unicodedata import normalize from ukrainian_word_stress import Stressifier, StressSymbol stressify = Stressifier() from models import * from utils import * from text_utils import TextCleaner textclenaer = TextCleaner() device = 'cuda' if torch.cuda.is_available() else 'cpu' to_mel = torchaudio.transforms.MelSpectrogram( n_mels=80, n_fft=2048, win_length=1200, hop_length=300) mean, std = -4, 4 def length_to_mask(lengths): mask = torch.arange(lengths.max()).unsqueeze(0).expand(lengths.shape[0], -1).type_as(lengths) mask = torch.gt(mask+1, lengths.unsqueeze(1)) return mask def load_state_dict(model, params): for key in model: if key in params: print('%s loaded' % key) try: model[key].load_state_dict(params[key]) except: from collections import OrderedDict state_dict = params[key] new_state_dict = OrderedDict() for k, v in state_dict.items(): name = k[7:] # remove `module.` new_state_dict[name] = v model[key].load_state_dict(new_state_dict, strict=False) config = yaml.safe_load(open('config.yml')) # load pretrained ASR model ASR_config = config.get('ASR_config', False) ASR_path = config.get('ASR_path', False) text_aligner = load_ASR_models(ASR_path, ASR_config) # load pretrained F0 model F0_path = config.get('F0_path', False) pitch_extractor = load_F0_models(F0_path) # load BERT model from Utils.PLBERT.util import load_plbert plbert = load_plbert('weights/plbert.bin', 'Utils/PLBERT/config.yml') model_single = build_model(recursive_munch(config['model_params_single']), text_aligner, pitch_extractor, plbert) model_multi = build_model(recursive_munch(config['model_params_multi']), deepcopy(text_aligner), deepcopy(pitch_extractor), deepcopy(plbert)) multi_path = hf_hub_download(repo_id='patriotyk/styletts2_ukrainian_multispeaker', filename="pytorch_model.bin") params_multi = torch.load(multi_path, map_location='cpu') single_path = hf_hub_download(repo_id='patriotyk/styletts2_ukrainian_single', filename="pytorch_model.bin") params_single = torch.load(single_path, map_location='cpu') load_state_dict(model_single, params_single) _ = [model_single[key].eval() for key in model_single] _ = [model_single[key].to(device) for key in model_single] load_state_dict(model_multi, params_multi) _ = [model_multi[key].eval() for key in model_multi] _ = [model_multi[key].to(device) for key in model_multi] models = { 'multi': model_multi, 'single': model_single } def preprocess(wave): wave_tensor = torch.from_numpy(wave).float() mel_tensor = to_mel(wave_tensor) mel_tensor = (torch.log(1e-5 + mel_tensor.unsqueeze(0)) - mean) / std return mel_tensor def compute_style(voice_audio): wave, sr = librosa.load(voice_audio, sr=24000) audio, index = librosa.effects.trim(wave, top_db=30) if sr != 24000: audio = librosa.resample(audio, sr, 24000) mel_tensor = preprocess(audio).to(device) with torch.no_grad(): ref_s = models['multi'].style_encoder(mel_tensor.unsqueeze(1)) ref_p = models['multi'].predictor_encoder(mel_tensor.unsqueeze(1)) return torch.cat([ref_s, ref_p], dim=1) def split_to_parts(text): split_symbols = '.?!:' parts = [''] index = 0 for s in text: parts[index] += s if s in split_symbols and len(parts[index]) > 150: index += 1 parts.append('') return parts def _inf(model, text, ref_s, speed, s_prev, noise, alpha, beta, diffusion_steps, embedding_scale): model = models[model] text = text.strip() text = text.replace('"', '') text = text.replace('+', 'ˈ') text = normalize('NFKC', text) text = re.sub(r'[᠆‐‑‒–—―⁻₋−⸺⸻]', '-', text) text = re.sub(r' - ', ': ', text) ps = ipa(stressify(text)) print(ps) tokens = textclenaer(ps) tokens.insert(0, 0) tokens = torch.LongTensor(tokens).to(device).unsqueeze(0) with torch.no_grad(): input_lengths = torch.LongTensor([tokens.shape[-1]]).to(tokens.device) text_mask = length_to_mask(input_lengths).to(tokens.device) t_en = model.text_encoder(tokens, input_lengths, text_mask) bert_dur = model.bert(tokens, attention_mask=(~text_mask).int()) d_en = model.bert_encoder(bert_dur).transpose(-1, -2) if ref_s is None: s_pred = model.sampler(noise, embedding=bert_dur[0].unsqueeze(0), num_steps=diffusion_steps, embedding_scale=embedding_scale).squeeze(0) else: s_pred = model.sampler(noise = noise, embedding=bert_dur, embedding_scale=embedding_scale, features=ref_s, # reference from the same speaker as the embedding num_steps=diffusion_steps).squeeze(1) if s_prev is not None: # convex combination of previous and current style s_pred = alpha * s_prev + (1 - alpha) * s_pred s = s_pred[:, 128:] ref = s_pred[:, :128] if ref_s is not None: ref = alpha * ref + (1 - alpha) * ref_s[:, :128] s = beta * s + (1 - beta) * ref_s[:, 128:] d = model.predictor.text_encoder(d_en, s, input_lengths, text_mask) x, _ = model.predictor.lstm(d) duration = model.predictor.duration_proj(x) duration = torch.sigmoid(duration).sum(axis=-1)/speed pred_dur = torch.round(duration.squeeze()).clamp(min=1) if ref_s is not None: pred_dur[0] = 30 pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data)) c_frame = 0 for i in range(pred_aln_trg.size(0)): pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1 c_frame += int(pred_dur[i].data) # encode prosody en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device)) F0_pred, N_pred = model.predictor.F0Ntrain(en, s) asr = (t_en @ pred_aln_trg.unsqueeze(0).to(device)) out = model.decoder(asr, F0_pred, N_pred, ref.squeeze().unsqueeze(0)) if ref_s is not None: out = out[:,:, 14500:] return out.squeeze().cpu().numpy(), s_pred, ps @spaces.GPU def inference(model, text, voice_audio, progress, speed=1, alpha=0.4, beta=0.4, diffusion_steps=10, embedding_scale=1.2): wavs = [] s_prev = None #sentences = text.split('|') sentences = split_to_parts(text) phonemes = '' noise = torch.randn(1,1,256).to(device) ref_s = compute_style(voice_audio) if voice_audio else None for text in progress.tqdm(sentences): if text.strip() == "": continue wav, s_prev, ps = _inf(model, text, ref_s, speed, s_prev, noise, alpha=alpha, beta=beta, diffusion_steps=diffusion_steps, embedding_scale=embedding_scale) wavs.append(wav) phonemes += ' ' + ps return np.concatenate(wavs), phonemes