|
import gradio as gr |
|
from model.CLAPSep import CLAPSep |
|
import torch |
|
import librosa |
|
import numpy as np |
|
|
|
|
|
model_config = {"lan_embed_dim": 1024, |
|
"depths": [1, 1, 1, 1], |
|
"embed_dim": 128, |
|
"encoder_embed_dim": 128, |
|
"phase": False, |
|
"spec_factor": 8, |
|
"d_attn": 640, |
|
"n_masker_layer": 3, |
|
"conv": False} |
|
DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu' |
|
CLAP_path = "model/music_audioset_epoch_15_esc_90.14.pt" |
|
|
|
|
|
model = CLAPSep(model_config, CLAP_path).to(DEVICE) |
|
ckpt = torch.load('model/best_model.ckpt', map_location=DEVICE) |
|
model.load_state_dict(ckpt, strict=False) |
|
model.eval() |
|
|
|
|
|
|
|
def inference(audio_file_path: str, text_p: str, audio_file_path_p: str, text_n: str, audio_file_path_n: str): |
|
|
|
with torch.no_grad(): |
|
embed_pos, embed_neg = torch.chunk(model.clap_model.get_text_embedding([text_p, text_n], |
|
use_tensor=True), dim=0, chunks=2) |
|
embed_pos = torch.zeros_like(embed_pos) if text_p == '' else embed_pos |
|
embed_neg = torch.zeros_like(embed_neg) if text_n == '' else embed_neg |
|
embed_pos += (model.clap_model.get_audio_embedding_from_filelist( |
|
[audio_file_path_p]) if audio_file_path_p is not None else torch.zeros_like(embed_pos)) |
|
embed_neg += (model.clap_model.get_audio_embedding_from_filelist( |
|
[audio_file_path_n]) if audio_file_path_n is not None else torch.zeros_like(embed_neg)) |
|
|
|
|
|
|
|
print(f"Separate audio from [{audio_file_path}] with textual query p: [{text_p}] and n: [{text_n}]") |
|
|
|
mixture, _ = librosa.load(audio_file_path, sr=32000) |
|
|
|
pad = (320000 - (len(mixture) % 320000))if len(mixture) % 320000 != 0 else 0 |
|
|
|
mixture =torch.tensor(np.pad(mixture,(0,pad))) |
|
|
|
max_value = torch.max(torch.abs(mixture)) |
|
if max_value > 1: |
|
mixture *= 0.9 / max_value |
|
|
|
mixture_chunks = torch.chunk(mixture, dim=0, chunks=len(mixture)//320000) |
|
sep_segments = [] |
|
for chunk in mixture_chunks: |
|
with torch.no_grad(): |
|
sep_segments.append(model.inference_from_data(chunk.unsqueeze(0), embed_pos, embed_neg)) |
|
|
|
sep_segment = torch.concat(sep_segments, dim=1) |
|
|
|
return 32000, sep_segment.squeeze().numpy() |
|
|
|
|
|
with gr.Blocks(title="CLAPSep") as demo: |
|
with gr.Row(): |
|
with gr.Column(): |
|
input_audio = gr.Audio(label="Mixture", type="filepath") |
|
text_p = gr.Textbox(label="Positive Query Text") |
|
text_n = gr.Textbox(label="Negative Query Text") |
|
query_audio_p = gr.Audio(label="Positive Query Audio (optional)", type="filepath") |
|
query_audio_n = gr.Audio(label="Negative Query Audio (optional)", type="filepath") |
|
with gr.Column(): |
|
with gr.Column(): |
|
output_audio = gr.Audio(label="Separation Result", scale=10) |
|
button = gr.Button( |
|
"Separate", |
|
variant="primary", |
|
scale=2, |
|
size="lg", |
|
interactive=True, |
|
) |
|
button.click( |
|
fn=inference, inputs=[input_audio, text_p, query_audio_p, text_n, query_audio_n], outputs=[output_audio] |
|
) |
|
|
|
|
|
demo.queue().launch(share=True) |
