File size: 3,068 Bytes
d347764 e1e6bb7 d347764 9428350 d347764 9428350 e1e6bb7 d347764 9428350 d347764 9428350 d347764 9428350 c022e43 594c7cc 9428350 d347764 a57914c d347764 f805e49 c6f1d54 f805e49 f424f82 594c7cc f424f82 82a1991 f424f82 82a1991 9a22fd3 f424f82 c737803 7b8d763 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 |
import gradio as gr
import numpy as np
import torch
from datasets import load_dataset
from transformers import VitsModel, VitsTokenizer, pipeline
device = "cuda:0" if torch.cuda.is_available() else "cpu"
# load speech translation checkpoint
asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-base", device=device)
# load text-to-speech checkpoint and speaker embeddings
# processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")
model = VitsModel.from_pretrained("facebook/mms-tts-por").to(device)
tokenizer = VitsTokenizer.from_pretrained("facebook/mms-tts-por")
# embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
# speaker_embeddings = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0)
def translate(audio):
outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"task": "transcribe", "language": "pt"})
return outputs["text"]
def synthesise(text):
inputs = tokenizer(text=text, return_tensors="pt")
with torch.no_grad():
output = model(**inputs)
output = output.waveform[0]
# inputs = processor(text=text, return_tensors="pt")
# speech = model.generate_speech(inputs["input_ids"].to(device), speaker_embeddings.to(device), vocoder=vocoder)
return output.cpu()
def speech_to_speech_translation(audio):
translated_text = translate(audio)
print(translated_text)
synthesised_speech = synthesise(translated_text)
synthesised_speech = (synthesised_speech.numpy() * 32767).astype(np.int16)
return 16000, synthesised_speech
title = "Cascaded STST"
description = """
Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in English. Demo uses OpenAI's [Whisper Base](https://huggingface.co/openai/whisper-base) model for speech translation, and Microsoft's
[SpeechT5 TTS](https://huggingface.co/microsoft/speecht5_tts) model for text-to-speech:
"""
mic_translate = gr.Interface(
fn=speech_to_speech_translation,
inputs=gr.Audio(source="microphone", type="filepath"),
outputs=gr.Audio(label="Generated Speech", type="numpy"),
title=title,
description=description,
)
file_translate = gr.Interface(
fn=speech_to_speech_translation,
inputs=gr.Audio(source="upload", type="filepath"),
outputs=gr.Audio(label="Generated Speech", type="numpy"),
examples=[["./example.wav"]],
title=title,
description=description,
)
with gr.Blocks() as demo:
# with gr.Row():
# audio_in = gr.Microphone(label="Input audio", type="filepath")
# audio_out = gr.Audio(label="Output audio", type="numpy", autoplay=True)
# audio_in.stop_recording(speech_to_speech_translation, inputs=[audio_in], outputs=[audio_out])
gr.TabbedInterface([mic_translate, file_translate], ["Microphone", "Audio File"])
demo.launch() |