infer.py

import os
import json
import time
import argparse
import numpy as np
import soundfile as sf
import torch
import torch.nn.functional as F

from modules.FastSVC import SVCNN
from modules.wavlm_encoder import WavLMEncoder
from utils.pitch_ld_extraction import extract_loudness, extract_pitch_ref as extract_pitch
from utils.tools import ConfigWrapper, fast_cosine_dist

# the 6th layer features of wavlm are used to audio synthesis
SPEAKER_INFORMATION_LAYER = 6
# the mean of last 5 layers features of wavlm are used for matching in kNN
CONTENT_INFORMATION_LAYER = [20, 21, 22, 23, 24]


def VoiceConverter(test_utt: str, ref_utt: str, out_path: str, svc_mdl: SVCNN, wavlm_encoder: WavLMEncoder, f0_factor: float, speech_enroll=False, device=torch.device('cpu')):
    """
    Perform singing voice conversion and save the resulting waveform to `out_path`.

    Args:
        test_utt (str): Path to the source singing waveform (24kHz, single-channel).
        ref_utt (str): Path to the reference waveform from the target speaker (single-channel, not less than 16kHz).
        out_path (str): Path to save the converted singing audio.
        svc_mdl (SVCNN): Loaded FastSVC model with neural harmonic filters.
        wavlm_encoder (WavLMEncoder): Loaded WavLM Encoder.
        f0_factor (float): F0 shift factor.
        speech_enroll (bool, optional): Whether the reference audio is a speech clip or a singing clip. Defaults to False.
        device (torch.device, optional): Device to perform the conversion on. Defaults to cpu.

    """
    # Preprocess audio and extract features.
    print('Processing feats.')
    applied_weights = F.one_hot(torch.tensor(CONTENT_INFORMATION_LAYER), num_classes=25).float().mean(axis=0).to(device)[:, None]

    ld = extract_loudness(test_utt)
    pitch, f0_factor = extract_pitch(test_utt, ref_utt, predefined_factor=f0_factor, speech_enroll=speech_enroll)
    assert pitch.shape[0] == ld.shape[0], f'{test_utt} Length Mismatch: pitch length ({pitch.shape[0]}), ld length ({ld.shape[0]}).'

    query_feats = wavlm_encoder.get_features(test_utt, weights=applied_weights)
    matching_set = wavlm_encoder.get_features(ref_utt, weights=applied_weights)
    synth_set = wavlm_encoder.get_features(ref_utt, output_layer=SPEAKER_INFORMATION_LAYER)

    # Calculate the distance between the query feats and the matching feats
    dists = fast_cosine_dist(query_feats, matching_set, device=device)
    best = dists.topk(k=4, largest=False, dim=-1)
    # Replace query features with corresponding nearest synth feats
    prematched_wavlm = synth_set[best.indices].mean(dim=1).transpose(0, 1)  # (T, 1024)

    # Align the features: the hop_size of the wavlm feature is twice that of pitch and loudness.
    seq_len = prematched_wavlm.shape[1] * 2
    if seq_len > pitch.shape[0]:
        p = seq_len - pitch.shape[0]
        pitch = np.pad(pitch, (0, p), mode='edge')
        ld = np.pad(ld, (0, p), mode='edge')
    else:
        pitch = pitch[:seq_len]
        ld = ld[:seq_len]

    in_feats = [prematched_wavlm.unsqueeze(0), torch.from_numpy(pitch).to(dtype=torch.float).unsqueeze(0),
                torch.from_numpy(ld).to(dtype=torch.float).unsqueeze(0)]
    in_feats = tuple([x_.to(device) for x_ in in_feats])

    # Inference
    print('Inferencing.')
    with torch.no_grad():
        y_ = svc_mdl(*in_feats)

    # Save converted audio.
    print('Saving audio.')
    os.makedirs(os.path.dirname(os.path.abspath(out_path)), exist_ok=True)
    y_ = y_.unsqueeze(0)
    y_ = np.clip(y_.view(-1).cpu().numpy(), -1, 1)
    sf.write(out_path, y_, 24000)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--src_wav_path',
        required=True, type=str, help='The audio path for the source singing utterance.'
    )
    parser.add_argument(
        '--ref_wav_path',
        required=True, type=str, help='The audio path for the reference utterance.'
    )
    parser.add_argument(
        '--out_path',
        required=True, type=str, help='The audio path for the reference utterance.'
    )
    parser.add_argument(
        '-cfg', '--config_file',
        type=str, default='configs/config.json',
        help='The model configuration file.'
    )
    parser.add_argument(
        '-ckpt', '--ckpt_path',
        type=str,
        default='pretrained/model.pkl',
        help='The model checkpoint path for loading.'
    )
    parser.add_argument(
        '-f0factor', '--f0_factor', type=float, default=0.0,
        help='Adjust the pitch of the source singing to match the vocal range of the target singer. \
            The default value is 0.0, which means no pitch adjustment is applied (equivalent to f0_factor = 1.0)'
    )
    parser.add_argument(
        '--speech_enroll', action='store_true',
        help='When using speech as the reference audio, the pitch of the reference audio will be increased by 1.2 times \
            when performing pitch shift to cover the pitch gap between singing and speech. \
            Note: This option is invalid when f0_factor is specified.'
    )

    args = parser.parse_args()

    t0 = time.time()

    f0factor = args.f0_factor
    speech_enroll_flag = args.speech_enroll

    if torch.cuda.is_available():
        device = torch.device('cuda')
    else:
        device = torch.device('cpu')
    print(f'using {device} for inference.')

    # Loading model and parameters.
    # load svc model
    cfg = args.config_file
    model_path = args.ckpt_path

    print('Loading svc model configurations.')
    with open(cfg) as f:
        config = ConfigWrapper(**json.load(f))

    svc_mdl = SVCNN(config)

    state_dict = torch.load(model_path, map_location='cpu')
    svc_mdl.load_state_dict(state_dict['model']['generator'], strict=False)
    svc_mdl.to(device)
    svc_mdl.eval()
    # load wavlm model
    wavlm_encoder = WavLMEncoder(ckpt_path='pretrained/WavLM-Large.pt', device=device)
    print('wavlm loaded.')
    # End loading model and parameters.
    t1 = time.time()
    print(f'loading models cost {t1-t0:.2f}s.')

    VoiceConverter(test_utt=args.src_wav_path, ref_utt=args.ref_wav_path, out_path=args.out_path,
                   svc_mdl=svc_mdl, wavlm_encoder=wavlm_encoder,
                   f0_factor=f0factor, speech_enroll=speech_enroll_flag, device=device)

    t2 = time.time()
    print(f'converting costs {t2-t1:.2f}s.')