import gradio as gr from model.CLAPSep import CLAPSep import torch import librosa import numpy as np model_config = {"lan_embed_dim": 1024, "depths": [1, 1, 1, 1], "embed_dim": 128, "encoder_embed_dim": 128, "phase": False, "spec_factor": 8, "d_attn": 640, "n_masker_layer": 3, "conv": False} DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu' CLAP_path = "model/music_audioset_epoch_15_esc_90.14.pt" model = CLAPSep(model_config, CLAP_path).to(DEVICE) ckpt = torch.load('model/best_model.ckpt', map_location=DEVICE) model.load_state_dict(ckpt, strict=False) model.eval() def inference(audio_file_path: str, text_p: str, audio_file_path_p: str, text_n: str, audio_file_path_n: str): # handling queries with torch.no_grad(): embed_pos, embed_neg = torch.chunk(model.clap_model.get_text_embedding([text_p, text_n], use_tensor=True), dim=0, chunks=2) embed_pos = torch.zeros_like(embed_pos) if text_p == '' else embed_pos embed_neg = torch.zeros_like(embed_neg) if text_n == '' else embed_neg embed_pos += (model.clap_model.get_audio_embedding_from_filelist( [audio_file_path_p]) if audio_file_path_p is not None else torch.zeros_like(embed_pos)) embed_neg += (model.clap_model.get_audio_embedding_from_filelist( [audio_file_path_n]) if audio_file_path_n is not None else torch.zeros_like(embed_neg)) print(f"Separate audio from [{audio_file_path}] with textual query p: [{text_p}] and n: [{text_n}]") mixture, _ = librosa.load(audio_file_path, sr=32000) pad = (320000 - (len(mixture) % 320000))if len(mixture) % 320000 != 0 else 0 mixture =torch.tensor(np.pad(mixture,(0,pad))) max_value = torch.max(torch.abs(mixture)) if max_value > 1: mixture *= 0.9 / max_value mixture_chunks = torch.chunk(mixture, dim=0, chunks=len(mixture)//320000) sep_segments = [] for chunk in mixture_chunks: with torch.no_grad(): sep_segments.append(model.inference_from_data(chunk.unsqueeze(0), embed_pos, embed_neg)) sep_segment = torch.concat(sep_segments, dim=1) return 32000, sep_segment.squeeze().numpy() with gr.Blocks(title="CLAPSep") as demo: with gr.Row(): with gr.Column(): input_audio = gr.Audio(label="Mixture", type="filepath") text_p = gr.Textbox(label="Positive Query Text") text_n = gr.Textbox(label="Negative Query Text") query_audio_p = gr.Audio(label="Positive Query Audio (optional)", type="filepath") query_audio_n = gr.Audio(label="Negative Query Audio (optional)", type="filepath") with gr.Column(): with gr.Column(): output_audio = gr.Audio(label="Separation Result", scale=10) button = gr.Button( "Separate", variant="primary", scale=2, size="lg", interactive=True, ) button.click( fn=inference, inputs=[input_audio, text_p, query_audio_p, text_n, query_audio_n], outputs=[output_audio] ) demo.queue().launch(share=True)