models.py

# from https://github.com/jaywalnut310/vits
# from https://github.com/ncsoft/avocodo
import math
import torch
from torch import nn
from torch.nn import functional as F
from torch.nn import Conv1d, ConvTranspose1d, Conv2d
from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm

import modules
import attentions
import commons
from commons import init_weights, get_padding
#for Q option
#from functions import vq, vq_st

from analysis import Pitch
from pqmf import PQMF


class StochasticDurationPredictor(nn.Module):

    def __init__(self,
                 in_channels,
                 filter_channels,
                 kernel_size,
                 p_dropout,
                 n_flows=4,
                 gin_channels=0):
        super().__init__()
        # it needs to be removed from future version.
        filter_channels = in_channels
        self.in_channels = in_channels
        self.filter_channels = filter_channels
        self.kernel_size = kernel_size
        self.p_dropout = p_dropout
        self.n_flows = n_flows
        self.gin_channels = gin_channels

        self.log_flow = modules.Log()
        self.flows = nn.ModuleList()
        self.flows.append(modules.ElementwiseAffine(2))
        for i in range(n_flows):
            self.flows.append(
                modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3))
            self.flows.append(modules.Flip())

        self.post_pre = nn.Conv1d(1, filter_channels, 1)
        self.post_proj = nn.Conv1d(filter_channels, filter_channels, 1)
        self.post_convs = modules.DDSConv(filter_channels,
                                          kernel_size,
                                          n_layers=3,
                                          p_dropout=p_dropout)
        self.post_flows = nn.ModuleList()
        self.post_flows.append(modules.ElementwiseAffine(2))
        for i in range(4):
            self.post_flows.append(
                modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3))
            self.post_flows.append(modules.Flip())

        self.pre = nn.Conv1d(in_channels, filter_channels, 1)
        self.proj = nn.Conv1d(filter_channels, filter_channels, 1)
        self.convs = modules.DDSConv(filter_channels,
                                     kernel_size,
                                     n_layers=3,
                                     p_dropout=p_dropout)
        if gin_channels != 0:
            self.cond = nn.Conv1d(gin_channels, filter_channels, 1)

    def forward(self,
                x,
                x_mask,
                w=None,
                g=None,
                reverse=False,
                noise_scale=1.0):
        x = torch.detach(x)
        x = self.pre(x)
        if g is not None:
            g = torch.detach(g)
            x = x + self.cond(g)
        x = self.convs(x, x_mask)
        x = self.proj(x) * x_mask

        if not reverse:
            flows = self.flows
            assert w is not None

            logdet_tot_q = 0
            h_w = self.post_pre(w)
            h_w = self.post_convs(h_w, x_mask)
            h_w = self.post_proj(h_w) * x_mask
            e_q = torch.randn(w.size(0), 2, w.size(2)).to(
                device=x.device, dtype=x.dtype) * x_mask
            z_q = e_q
            for flow in self.post_flows:
                z_q, logdet_q = flow(z_q, x_mask, g=(x + h_w))
                logdet_tot_q += logdet_q
            z_u, z1 = torch.split(z_q, [1, 1], 1)
            u = torch.sigmoid(z_u) * x_mask
            z0 = (w - u) * x_mask
            logdet_tot_q += torch.sum(
                (F.logsigmoid(z_u) + F.logsigmoid(-z_u)) * x_mask, [1, 2])
            logq = torch.sum(
                -0.5 * (math.log(2 * math.pi) +
                        (e_q**2)) * x_mask, [1, 2]) - logdet_tot_q

            logdet_tot = 0
            z0, logdet = self.log_flow(z0, x_mask)
            logdet_tot += logdet
            z = torch.cat([z0, z1], 1)
            for flow in flows:
                z, logdet = flow(z, x_mask, g=x, reverse=reverse)
                logdet_tot = logdet_tot + logdet
            nll = torch.sum(0.5 * (math.log(2 * math.pi) +
                                   (z**2)) * x_mask, [1, 2]) - logdet_tot
            return nll + logq  # [b]
        else:
            flows = list(reversed(self.flows))
            flows = flows[:-2] + [flows[-1]]  # remove a useless vflow
            z = torch.randn(x.size(0), 2, x.size(2)).to(
                device=x.device, dtype=x.dtype) * noise_scale
            for flow in flows:
                z = flow(z, x_mask, g=x, reverse=reverse)
            z0, z1 = torch.split(z, [1, 1], 1)
            logw = z0
            return logw


class DurationPredictor(nn.Module):

    def __init__(self,
                 in_channels,
                 filter_channels,
                 kernel_size,
                 p_dropout,
                 gin_channels=0):
        super().__init__()

        self.in_channels = in_channels
        self.filter_channels = filter_channels
        self.kernel_size = kernel_size
        self.p_dropout = p_dropout
        self.gin_channels = gin_channels

        self.drop = nn.Dropout(p_dropout)
        self.conv_1 = nn.Conv1d(in_channels,
                                filter_channels,
                                kernel_size,
                                padding=kernel_size // 2)
        self.norm_1 = modules.LayerNorm(filter_channels)
        self.conv_2 = nn.Conv1d(filter_channels,
                                filter_channels,
                                kernel_size,
                                padding=kernel_size // 2)
        self.norm_2 = modules.LayerNorm(filter_channels)
        self.proj = nn.Conv1d(filter_channels, 1, 1)

        if gin_channels != 0:
            self.cond = nn.Conv1d(gin_channels, in_channels, 1)

    def forward(self, x, x_mask, g=None):
        x = torch.detach(x)
        if g is not None:
            g = torch.detach(g)
            x = x + self.cond(g)
        x = self.conv_1(x * x_mask)
        x = torch.relu(x)
        x = self.norm_1(x)
        x = self.drop(x)
        x = self.conv_2(x * x_mask)
        x = torch.relu(x)
        x = self.norm_2(x)
        x = self.drop(x)
        x = self.proj(x * x_mask)
        return x * x_mask


class TextEncoder(nn.Module):

    def __init__(self, n_vocab, out_channels, hidden_channels, filter_channels,
                 n_heads, n_layers, kernel_size, p_dropout):
        super().__init__()
        self.n_vocab = n_vocab
        self.out_channels = out_channels
        self.hidden_channels = hidden_channels
        self.filter_channels = filter_channels
        self.n_heads = n_heads
        self.n_layers = n_layers
        self.kernel_size = kernel_size
        self.p_dropout = p_dropout

        self.emb = nn.Embedding(n_vocab, hidden_channels)
        nn.init.normal_(self.emb.weight, 0.0, hidden_channels**-0.5)
        self.emb_t = nn.Embedding(6, hidden_channels)
        nn.init.normal_(self.emb_t.weight, 0.0, hidden_channels**-0.5)

        self.encoder = attentions.Encoder(hidden_channels, filter_channels,
                                          n_heads, n_layers, kernel_size,
                                          p_dropout)
        self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)

    def forward(self, x, t, x_lengths):
        t_zero = (t == 0)
        emb_t = self.emb_t(t)
        emb_t[t_zero, :] = 0
        x = (self.emb(x) + emb_t) * math.sqrt(
            self.hidden_channels)  # [b, t, h]
        #x = torch.transpose(x, 1, -1)  # [b, h, t]
        x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(1)),
                                 1).to(x.dtype)
        #x = self.encoder(x * x_mask, x_mask)
        x = torch.einsum('btd,but->bdt', x, x_mask)
        x = self.encoder(x, x_mask)
        stats = self.proj(x) * x_mask

        m, logs = torch.split(stats, self.out_channels, dim=1)
        return x, m, logs, x_mask


class ResidualCouplingBlock(nn.Module):

    def __init__(self,
                 channels,
                 hidden_channels,
                 kernel_size,
                 dilation_rate,
                 n_layers,
                 n_flows=4,
                 gin_channels=0):
        super().__init__()
        self.channels = channels
        self.hidden_channels = hidden_channels
        self.kernel_size = kernel_size
        self.dilation_rate = dilation_rate
        self.n_layers = n_layers
        self.n_flows = n_flows
        self.gin_channels = gin_channels

        self.flows = nn.ModuleList()
        for i in range(n_flows):
            self.flows.append(
                modules.ResidualCouplingLayer(channels,
                                              hidden_channels,
                                              kernel_size,
                                              dilation_rate,
                                              n_layers,
                                              gin_channels=gin_channels,
                                              mean_only=True))
            self.flows.append(modules.Flip())

    def forward(self, x, x_mask, g=None, reverse=False):
        if not reverse:
            for flow in self.flows:
                x, _ = flow(x, x_mask, g=g, reverse=reverse)
        else:
            for flow in reversed(self.flows):
                x = flow(x, x_mask, g=g, reverse=reverse)
        return x


class PosteriorEncoder(nn.Module):

    def __init__(self,
                 in_channels,
                 out_channels,
                 hidden_channels,
                 kernel_size,
                 dilation_rate,
                 n_layers,
                 gin_channels=0):
        super().__init__()
        self.in_channels = in_channels
        self.out_channels = out_channels
        self.hidden_channels = hidden_channels
        self.kernel_size = kernel_size
        self.dilation_rate = dilation_rate
        self.n_layers = n_layers
        self.gin_channels = gin_channels

        self.pre = nn.Conv1d(in_channels, hidden_channels, 1)
        self.enc = modules.WN(hidden_channels,
                              kernel_size,
                              dilation_rate,
                              n_layers,
                              gin_channels=gin_channels)
        self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)

    def forward(self, x, x_lengths, g=None):
        x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)),
                                 1).to(x.dtype)
        x = self.pre(x) * x_mask
        x = self.enc(x, x_mask, g=g)
        stats = self.proj(x) * x_mask
        m, logs = torch.split(stats, self.out_channels, dim=1)
        z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask
        return z, m, logs, x_mask


class Generator(nn.Module):

    def __init__(self,
                 initial_channel,
                 resblock,
                 resblock_kernel_sizes,
                 resblock_dilation_sizes,
                 upsample_rates,
                 upsample_initial_channel,
                 upsample_kernel_sizes,
                 gin_channels=0):
        super(Generator, self).__init__()
        self.num_kernels = len(resblock_kernel_sizes)
        self.num_upsamples = len(upsample_rates)
        self.conv_pre = Conv1d(initial_channel,
                               upsample_initial_channel,
                               7,
                               1,
                               padding=3)
        resblock = modules.ResBlock1 if resblock == '1' else modules.ResBlock2

        self.ups = nn.ModuleList()
        for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
            self.ups.append(
                weight_norm(
                    ConvTranspose1d(upsample_initial_channel // (2**i),
                                    upsample_initial_channel // (2**(i + 1)),
                                    k,
                                    u,
                                    padding=(k - u) // 2)))

        self.resblocks = nn.ModuleList()
        self.conv_posts = nn.ModuleList()
        for i in range(len(self.ups)):
            ch = upsample_initial_channel // (2**(i + 1))
            for j, (k, d) in enumerate(
                    zip(resblock_kernel_sizes, resblock_dilation_sizes)):
                self.resblocks.append(resblock(ch, k, d))
            if i >= len(self.ups) - 3:
                self.conv_posts.append(
                    Conv1d(ch, 1, 7, 1, padding=3, bias=False))
        self.ups.apply(init_weights)

        if gin_channels != 0:
            self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)

    def forward(self, x, g=None):
        x = self.conv_pre(x)
        if g is not None:
            x = x + self.cond(g)

        for i in range(self.num_upsamples):
            x = F.leaky_relu(x, modules.LRELU_SLOPE)
            x = self.ups[i](x)
            xs = None
            for j in range(self.num_kernels):
                xs = xs + self.resblocks[i * self.num_kernels + j](x) if xs is not None \
                     else self.resblocks[i * self.num_kernels + j](x)
            x = xs / self.num_kernels
        x = F.leaky_relu(x)
        x = self.conv_posts[-1](x)
        x = torch.tanh(x)

        return x

    def hier_forward(self, x, g=None):
        outs = []
        x = self.conv_pre(x)
        if g is not None:
            x = x + self.cond(g)

        for i in range(self.num_upsamples):
            x = F.leaky_relu(x, modules.LRELU_SLOPE)
            x = self.ups[i](x)
            xs = None
            for j in range(self.num_kernels):
                xs = xs + self.resblocks[i * self.num_kernels + j](x) if xs is not None \
                     else self.resblocks[i * self.num_kernels + j](x)
            x = xs / self.num_kernels
            if i >= self.num_upsamples - 3:
                _x = F.leaky_relu(x)
                _x = self.conv_posts[i - self.num_upsamples + 3](_x)
                _x = torch.tanh(_x)
                outs.append(_x)
        return outs

    def remove_weight_norm(self):
        print('Removing weight norm...')
        for l in self.ups:
            remove_weight_norm(l)
        for l in self.resblocks:
            l.remove_weight_norm()


class DiscriminatorP(nn.Module):

    def __init__(self,
                 period,
                 kernel_size=5,
                 stride=3,
                 use_spectral_norm=False):
        super(DiscriminatorP, self).__init__()
        self.period = period
        self.use_spectral_norm = use_spectral_norm
        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
        self.convs = nn.ModuleList([
            norm_f(
                Conv2d(1,
                       32, (kernel_size, 1), (stride, 1),
                       padding=(get_padding(kernel_size, 1), 0))),
            norm_f(
                Conv2d(32,
                       128, (kernel_size, 1), (stride, 1),
                       padding=(get_padding(kernel_size, 1), 0))),
            norm_f(
                Conv2d(128,
                       512, (kernel_size, 1), (stride, 1),
                       padding=(get_padding(kernel_size, 1), 0))),
            norm_f(
                Conv2d(512,
                       1024, (kernel_size, 1), (stride, 1),
                       padding=(get_padding(kernel_size, 1), 0))),
            norm_f(
                Conv2d(1024,
                       1024, (kernel_size, 1),
                       1,
                       padding=(get_padding(kernel_size, 1), 0))),
        ])
        self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))

    def forward(self, x):
        fmap = []

        # 1d to 2d
        b, c, t = x.shape
        if t % self.period != 0:  # pad first
            n_pad = self.period - (t % self.period)
            x = F.pad(x, (0, n_pad), "reflect")
            t = t + n_pad
        x = x.view(b, c, t // self.period, self.period)

        for l in self.convs:
            x = l(x)
            x = F.leaky_relu(x, modules.LRELU_SLOPE)
            fmap.append(x)
        x = self.conv_post(x)
        fmap.append(x)
        x = torch.flatten(x, 1, -1)

        return x, fmap


class DiscriminatorS(nn.Module):

    def __init__(self, use_spectral_norm=False):
        super(DiscriminatorS, self).__init__()
        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
        self.convs = nn.ModuleList([
            norm_f(Conv1d(1, 16, 15, 1, padding=7)),
            norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)),
            norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)),
            norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)),
            norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)),
            norm_f(Conv1d(1024, 1024, 5, 1, padding=2)),
        ])
        self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))

    def forward(self, x):
        fmap = []

        for l in self.convs:
            x = l(x)
            x = F.leaky_relu(x, modules.LRELU_SLOPE)
            fmap.append(x)
        x = self.conv_post(x)
        fmap.append(x)
        x = torch.flatten(x, 1, -1)

        return x, fmap


class MultiPeriodDiscriminator(nn.Module):

    def __init__(self, use_spectral_norm=False):
        super(MultiPeriodDiscriminator, self).__init__()
        periods = [2, 3, 5, 7, 11]

        discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)]
        discs = discs + \
            [DiscriminatorP(i, use_spectral_norm=use_spectral_norm)
             for i in periods]
        self.discriminators = nn.ModuleList(discs)

    def forward(self, y, y_hat):
        y_d_rs = []
        y_d_gs = []
        fmap_rs = []
        fmap_gs = []
        for i, d in enumerate(self.discriminators):
            y_d_r, fmap_r = d(y)
            y_d_g, fmap_g = d(y_hat)
            y_d_rs.append(y_d_r)
            y_d_gs.append(y_d_g)
            fmap_rs.append(fmap_r)
            fmap_gs.append(fmap_g)

        return y_d_rs, y_d_gs, fmap_rs, fmap_gs


##### Avocodo
class CoMBDBlock(torch.nn.Module):

    def __init__(
            self,
            h_u,  # List[int],
            d_k,  # List[int],
            d_s,  # List[int],
            d_d,  # List[int],
            d_g,  # List[int],
            d_p,  # List[int],
            op_f,  # int,
            op_k,  # int,
            op_g,  # int,
            use_spectral_norm=False):
        super(CoMBDBlock, self).__init__()
        norm_f = weight_norm if use_spectral_norm is False else spectral_norm

        self.convs = nn.ModuleList()
        filters = [[1, h_u[0]]]
        for i in range(len(h_u) - 1):
            filters.append([h_u[i], h_u[i + 1]])
        for _f, _k, _s, _d, _g, _p in zip(filters, d_k, d_s, d_d, d_g, d_p):
            self.convs.append(
                norm_f(
                    Conv1d(in_channels=_f[0],
                           out_channels=_f[1],
                           kernel_size=_k,
                           stride=_s,
                           dilation=_d,
                           groups=_g,
                           padding=_p)))
        self.projection_conv = norm_f(
            Conv1d(in_channels=filters[-1][1],
                   out_channels=op_f,
                   kernel_size=op_k,
                   groups=op_g))

    def forward(self, x, b_y, b_y_hat):
        fmap_r = []
        fmap_g = []
        for block in self.convs:
            x = block(x)
            x = F.leaky_relu(x, 0.2)
            f_r, f_g = x.split([b_y, b_y_hat], dim=0)
            fmap_r.append(f_r.tile([2, 1, 1]) if b_y < b_y_hat else f_r)
            fmap_g.append(f_g)
        x = self.projection_conv(x)
        x_r, x_g = x.split([b_y, b_y_hat], dim=0)
        return x_r.tile([2, 1, 1
                         ]) if b_y < b_y_hat else x_r, x_g, fmap_r, fmap_g


class CoMBD(torch.nn.Module):

    def __init__(self, use_spectral_norm=False):
        super(CoMBD, self).__init__()
        self.pqmf_list = nn.ModuleList([
            PQMF(4, 192, 0.13, 10.0),  #lv2
            PQMF(2, 256, 0.25, 10.0)  #lv1
        ])
        combd_h_u = [[16, 64, 256, 1024, 1024, 1024] for _ in range(3)]
        combd_d_k = [[7, 11, 11, 11, 11, 5], [11, 21, 21, 21, 21, 5],
                     [15, 41, 41, 41, 41, 5]]
        combd_d_s = [[1, 1, 4, 4, 4, 1] for _ in range(3)]
        combd_d_d = [[1, 1, 1, 1, 1, 1] for _ in range(3)]
        combd_d_g = [[1, 4, 16, 64, 256, 1] for _ in range(3)]

        combd_d_p = [[3, 5, 5, 5, 5, 2], [5, 10, 10, 10, 10, 2],
                     [7, 20, 20, 20, 20, 2]]
        combd_op_f = [1, 1, 1]
        combd_op_k = [3, 3, 3]
        combd_op_g = [1, 1, 1]

        self.blocks = nn.ModuleList()
        for _h_u, _d_k, _d_s, _d_d, _d_g, _d_p, _op_f, _op_k, _op_g in zip(
                combd_h_u,
                combd_d_k,
                combd_d_s,
                combd_d_d,
                combd_d_g,
                combd_d_p,
                combd_op_f,
                combd_op_k,
                combd_op_g,
        ):
            self.blocks.append(
                CoMBDBlock(
                    _h_u,
                    _d_k,
                    _d_s,
                    _d_d,
                    _d_g,
                    _d_p,
                    _op_f,
                    _op_k,
                    _op_g,
                ))

    def _block_forward(self, ys, ys_hat, blocks):
        outs_real = []
        outs_fake = []
        f_maps_real = []
        f_maps_fake = []
        for y, y_hat, block in zip(ys, ys_hat,
                                   blocks):  #y:B, y_hat: 2B if i!=-1 else B,B
            b_y = y.shape[0]
            b_y_hat = y_hat.shape[0]
            cat_y = torch.cat([y, y_hat], dim=0)
            out_real, out_fake, f_map_r, f_map_g = block(cat_y, b_y, b_y_hat)
            outs_real.append(out_real)
            outs_fake.append(out_fake)
            f_maps_real.append(f_map_r)
            f_maps_fake.append(f_map_g)
        return outs_real, outs_fake, f_maps_real, f_maps_fake

    def _pqmf_forward(self, ys, ys_hat):
        # preprocess for multi_scale forward
        multi_scale_inputs_hat = []
        for pqmf_ in self.pqmf_list:
            multi_scale_inputs_hat.append(pqmf_.analysis(ys_hat[-1])[:, :1, :])

        # real
        # for hierarchical forward
        #outs_real_, f_maps_real_ = self._block_forward(
        #    ys, self.blocks)

        # for multi_scale forward
        #outs_real, f_maps_real = self._block_forward(
        #        ys[:-1], self.blocks[:-1], outs_real, f_maps_real)
        #outs_real.extend(outs_real[:-1])
        #f_maps_real.extend(f_maps_real[:-1])

        #outs_real = [torch.cat([o,o], dim=0) if i!=len(outs_real_)-1 else o for i,o in enumerate(outs_real_)]
        #f_maps_real = [[torch.cat([fmap,fmap], dim=0) if i!=len(f_maps_real_)-1 else fmap for fmap in fmaps ] \
        #        for i,fmaps in enumerate(f_maps_real_)]

        inputs_fake = [
            torch.cat([y, multi_scale_inputs_hat[i]], dim=0)
            if i != len(ys_hat) - 1 else y for i, y in enumerate(ys_hat)
        ]
        outs_real, outs_fake, f_maps_real, f_maps_fake = self._block_forward(
            ys, inputs_fake, self.blocks)

        # predicted
        # for hierarchical forward
        #outs_fake, f_maps_fake = self._block_forward(
        #    inputs_fake, self.blocks)

        #outs_real_, f_maps_real_ = self._block_forward(
        #    ys, self.blocks)
        # for multi_scale forward
        #outs_fake, f_maps_fake = self._block_forward(
        #    multi_scale_inputs_hat, self.blocks[:-1], outs_fake, f_maps_fake)

        return outs_real, outs_fake, f_maps_real, f_maps_fake

    def forward(self, ys, ys_hat):
        outs_real, outs_fake, f_maps_real, f_maps_fake = self._pqmf_forward(
            ys, ys_hat)
        return outs_real, outs_fake, f_maps_real, f_maps_fake


class MDC(torch.nn.Module):

    def __init__(self,
                 in_channels,
                 out_channels,
                 strides,
                 kernel_size,
                 dilations,
                 use_spectral_norm=False):
        super(MDC, self).__init__()
        norm_f = weight_norm if not use_spectral_norm else spectral_norm
        self.d_convs = nn.ModuleList()
        for _k, _d in zip(kernel_size, dilations):
            self.d_convs.append(
                norm_f(
                    Conv1d(in_channels=in_channels,
                           out_channels=out_channels,
                           kernel_size=_k,
                           dilation=_d,
                           padding=get_padding(_k, _d))))
        self.post_conv = norm_f(
            Conv1d(in_channels=out_channels,
                   out_channels=out_channels,
                   kernel_size=3,
                   stride=strides,
                   padding=get_padding(_k, _d)))
        self.softmax = torch.nn.Softmax(dim=-1)

    def forward(self, x):
        _out = None
        for _l in self.d_convs:
            _x = torch.unsqueeze(_l(x), -1)
            _x = F.leaky_relu(_x, 0.2)
            _out = torch.cat([_out, _x], axis=-1) if _out is not None \
                   else _x
        x = torch.sum(_out, dim=-1)
        x = self.post_conv(x)
        x = F.leaky_relu(x, 0.2)  # @@

        return x


class SBDBlock(torch.nn.Module):

    def __init__(self,
                 segment_dim,
                 strides,
                 filters,
                 kernel_size,
                 dilations,
                 use_spectral_norm=False):
        super(SBDBlock, self).__init__()
        norm_f = weight_norm if not use_spectral_norm else spectral_norm
        self.convs = nn.ModuleList()
        filters_in_out = [(segment_dim, filters[0])]
        for i in range(len(filters) - 1):
            filters_in_out.append([filters[i], filters[i + 1]])

        for _s, _f, _k, _d in zip(strides, filters_in_out, kernel_size,
                                  dilations):
            self.convs.append(
                MDC(in_channels=_f[0],
                    out_channels=_f[1],
                    strides=_s,
                    kernel_size=_k,
                    dilations=_d,
                    use_spectral_norm=use_spectral_norm))
        self.post_conv = norm_f(
            Conv1d(in_channels=_f[1],
                   out_channels=1,
                   kernel_size=3,
                   stride=1,
                   padding=3 // 2))  # @@

    def forward(self, x):
        fmap_r = []
        fmap_g = []
        for _l in self.convs:
            x = _l(x)
            f_r, f_g = torch.chunk(x, 2, dim=0)
            fmap_r.append(f_r)
            fmap_g.append(f_g)
        x = self.post_conv(x)  # @@
        x_r, x_g = torch.chunk(x, 2, dim=0)
        return x_r, x_g, fmap_r, fmap_g


class MDCDConfig:

    def __init__(self):
        self.pqmf_params = [16, 256, 0.03, 10.0]
        self.f_pqmf_params = [64, 256, 0.1, 9.0]
        self.filters = [[64, 128, 256, 256, 256], [64, 128, 256, 256, 256],
                        [64, 128, 256, 256, 256], [32, 64, 128, 128, 128]]
        self.kernel_sizes = [[[7, 7, 7], [7, 7, 7], [7, 7, 7], [7, 7, 7],
                              [7, 7, 7]],
                             [[5, 5, 5], [5, 5, 5], [5, 5, 5], [5, 5, 5],
                              [5, 5, 5]],
                             [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3],
                              [3, 3, 3]],
                             [[5, 5, 5], [5, 5, 5], [5, 5, 5], [5, 5, 5],
                              [5, 5, 5]]]
        self.dilations = [[[5, 7, 11], [5, 7, 11], [5, 7, 11], [5, 7, 11],
                           [5, 7, 11]],
                          [[3, 5, 7], [3, 5, 7], [3, 5, 7], [3, 5, 7],
                           [3, 5, 7]],
                          [[1, 2, 3], [1, 2, 3], [1, 2, 3], [1, 2, 3],
                           [1, 2, 3]],
                          [[1, 2, 3], [1, 2, 3], [1, 2, 3], [2, 3, 5],
                           [2, 3, 5]]]
        self.strides = [[1, 1, 3, 3, 1], [1, 1, 3, 3, 1], [1, 1, 3, 3, 1],
                        [1, 1, 3, 3, 1]]
        self.band_ranges = [[0, 6], [0, 11], [0, 16], [0, 64]]
        self.transpose = [False, False, False, True]
        self.segment_size = 8192


class SBD(torch.nn.Module):

    def __init__(self, use_spectral_norm=False):
        super(SBD, self).__init__()
        self.config = MDCDConfig()
        self.pqmf = PQMF(*self.config.pqmf_params)
        if True in self.config.transpose:
            self.f_pqmf = PQMF(*self.config.f_pqmf_params)
        else:
            self.f_pqmf = None

        self.discriminators = torch.nn.ModuleList()

        for _f, _k, _d, _s, _br, _tr in zip(self.config.filters,
                                            self.config.kernel_sizes,
                                            self.config.dilations,
                                            self.config.strides,
                                            self.config.band_ranges,
                                            self.config.transpose):
            if _tr:
                segment_dim = self.config.segment_size // _br[1] - _br[0]
            else:
                segment_dim = _br[1] - _br[0]

            self.discriminators.append(
                SBDBlock(segment_dim=segment_dim,
                         filters=_f,
                         kernel_size=_k,
                         dilations=_d,
                         strides=_s,
                         use_spectral_norm=use_spectral_norm))

    def forward(self, y, y_hat):
        y_d_rs = []
        y_d_gs = []
        fmap_rs = []
        fmap_gs = []
        y_in = self.pqmf.analysis(y)
        y_hat_in = self.pqmf.analysis(y_hat)
        y_in_f = self.f_pqmf.analysis(y)
        y_hat_in_f = self.f_pqmf.analysis(y_hat)

        for d, br, tr in zip(self.discriminators, self.config.band_ranges,
                             self.config.transpose):
            if not tr:
                _y_in = y_in[:, br[0]:br[1], :]
                _y_hat_in = y_hat_in[:, br[0]:br[1], :]
            else:
                _y_in = y_in_f[:, br[0]:br[1], :]
                _y_hat_in = y_hat_in_f[:, br[0]:br[1], :]
                _y_in = torch.transpose(_y_in, 1, 2)
                _y_hat_in = torch.transpose(_y_hat_in, 1, 2)
            #y_d_r, fmap_r = d(_y_in)
            #y_d_g, fmap_g = d(_y_hat_in)
            cat_y = torch.cat([_y_in, _y_hat_in], dim=0)
            y_d_r, y_d_g, fmap_r, fmap_g = d(cat_y)
            y_d_rs.append(y_d_r)
            fmap_rs.append(fmap_r)
            y_d_gs.append(y_d_g)
            fmap_gs.append(fmap_g)

        return y_d_rs, y_d_gs, fmap_rs, fmap_gs


class AvocodoDiscriminator(nn.Module):

    def __init__(self, use_spectral_norm=False):
        super(AvocodoDiscriminator, self).__init__()
        self.combd = CoMBD(use_spectral_norm)
        self.sbd = SBD(use_spectral_norm)

    def forward(self, y, ys_hat):
        ys = [
            self.combd.pqmf_list[0].analysis(y)[:, :1],  #lv2
            self.combd.pqmf_list[1].analysis(y)[:, :1],  #lv1
            y
        ]
        y_c_rs, y_c_gs, fmap_c_rs, fmap_c_gs = self.combd(ys, ys_hat)
        y_s_rs, y_s_gs, fmap_s_rs, fmap_s_gs = self.sbd(y, ys_hat[-1])
        y_c_rs.extend(y_s_rs)
        y_c_gs.extend(y_s_gs)
        fmap_c_rs.extend(fmap_s_rs)
        fmap_c_gs.extend(fmap_s_gs)
        return y_c_rs, y_c_gs, fmap_c_rs, fmap_c_gs


##### Avocodo


class YingDecoder(nn.Module):

    def __init__(self,
                 hidden_channels,
                 kernel_size,
                 dilation_rate,
                 n_layers,
                 yin_start,
                 yin_scope,
                 yin_shift_range,
                 gin_channels=0):
        super().__init__()
        self.in_channels = yin_scope
        self.out_channels = yin_scope
        self.hidden_channels = hidden_channels
        self.kernel_size = kernel_size
        self.dilation_rate = dilation_rate
        self.n_layers = n_layers
        self.gin_channels = gin_channels

        self.yin_start = yin_start
        self.yin_scope = yin_scope
        self.yin_shift_range = yin_shift_range

        self.pre = nn.Conv1d(self.in_channels, hidden_channels, 1)
        self.dec = modules.WN(hidden_channels,
                              kernel_size,
                              dilation_rate,
                              n_layers,
                              gin_channels=gin_channels)
        self.proj = nn.Conv1d(hidden_channels, self.out_channels, 1)

    def crop_scope(self, x, yin_start,
                   scope_shift):  # x: tensor [B,C,T] #scope_shift: tensor [B]
        return torch.stack([
            x[i, yin_start + scope_shift[i]:yin_start + self.yin_scope +
              scope_shift[i], :] for i in range(x.shape[0])
        ],
                           dim=0)

    def infer(self, z_yin, z_mask, g=None):
        B = z_yin.shape[0]
        scope_shift = torch.randint(-self.yin_shift_range,
                                    self.yin_shift_range, (B, ),
                                    dtype=torch.int)
        z_yin_crop = self.crop_scope(z_yin, self.yin_start, scope_shift)
        x = self.pre(z_yin_crop) * z_mask
        x = self.dec(x, z_mask, g=g)
        yin_hat_crop = self.proj(x) * z_mask
        return yin_hat_crop

    def forward(self, z_yin, yin_gt, z_mask, g=None):
        B = z_yin.shape[0]
        scope_shift = torch.randint(-self.yin_shift_range,
                                    self.yin_shift_range, (B, ),
                                    dtype=torch.int)
        z_yin_crop = self.crop_scope(z_yin, self.yin_start, scope_shift)
        yin_gt_shifted_crop = self.crop_scope(yin_gt, self.yin_start,
                                              scope_shift)
        yin_gt_crop = self.crop_scope(yin_gt, self.yin_start,
                                      torch.zeros_like(scope_shift))
        x = self.pre(z_yin_crop) * z_mask
        x = self.dec(x, z_mask, g=g)
        yin_hat_crop = self.proj(x) * z_mask
        return yin_gt_crop, yin_gt_shifted_crop, yin_hat_crop, z_yin_crop, scope_shift


# For Q option
#class VQEmbedding(nn.Module):
#
#    def __init__(self, codebook_size,
#                 code_channels):
#        super().__init__()
#        self.embedding = nn.Embedding(codebook_size, code_channels)
#        self.embedding.weight.data.uniform_(-1. / codebook_size,
#                                            1. / codebook_size)
#
#    def forward(self, z_e_x):
#        z_e_x_ = z_e_x.permute(0, 2, 1).contiguous()
#        latent_indices = vq(z_e_x_, self.embedding.weight)
#        z_q = self.embedding(latent_indices).permute(0, 2, 1)
#        return z_q
#
#    def straight_through(self, z_e_x):
#        z_e_x_ = z_e_x.permute(0, 2, 1).contiguous()
#        z_q_x_st_, indices = vq_st(z_e_x_, self.embedding.weight.detach())
#        z_q_x_st = z_q_x_st_.permute(0, 2, 1).contiguous()
#
#        z_q_x_flatten = torch.index_select(self.embedding.weight,
#                                           dim=0,
#                                           index=indices)
#        z_q_x_ = z_q_x_flatten.view_as(z_e_x_)
#        z_q_x = z_q_x_.permute(0, 2, 1).contiguous()
#        return z_q_x_st, z_q_x


class SynthesizerTrn(nn.Module):
    """
    Synthesizer for Training
    """

    def __init__(
            self,
            n_vocab,
            spec_channels,
            segment_size,
            midi_start,
            midi_end,
            octave_range,
            inter_channels,
            hidden_channels,
            filter_channels,
            n_heads,
            n_layers,
            kernel_size,
            p_dropout,
            resblock,
            resblock_kernel_sizes,
            resblock_dilation_sizes,
            upsample_rates,
            upsample_initial_channel,
            upsample_kernel_sizes,
            yin_channels,
            yin_start,
            yin_scope,
            yin_shift_range,
            n_speakers=0,
            gin_channels=0,
            use_sdp=True,
            #codebook_size=256, #for Q option
            **kwargs):

        super().__init__()
        self.n_vocab = n_vocab
        self.spec_channels = spec_channels
        self.inter_channels = inter_channels
        self.hidden_channels = hidden_channels
        self.filter_channels = filter_channels
        self.n_heads = n_heads
        self.n_layers = n_layers
        self.kernel_size = kernel_size
        self.p_dropout = p_dropout
        self.resblock = resblock
        self.resblock_kernel_sizes = resblock_kernel_sizes
        self.resblock_dilation_sizes = resblock_dilation_sizes
        self.upsample_rates = upsample_rates
        self.upsample_initial_channel = upsample_initial_channel
        self.upsample_kernel_sizes = upsample_kernel_sizes
        self.segment_size = segment_size
        self.n_speakers = n_speakers
        self.gin_channels = gin_channels

        self.yin_channels = yin_channels
        self.yin_start = yin_start
        self.yin_scope = yin_scope

        self.use_sdp = use_sdp
        self.enc_p = TextEncoder(n_vocab, inter_channels, hidden_channels,
                                 filter_channels, n_heads, n_layers,
                                 kernel_size, p_dropout)
        self.dec = Generator(
            inter_channels - yin_channels +
            yin_scope,
            resblock,
            resblock_kernel_sizes,
            resblock_dilation_sizes,
            upsample_rates,
            upsample_initial_channel,
            upsample_kernel_sizes,
            gin_channels=gin_channels)

        self.enc_spec = PosteriorEncoder(spec_channels,
                                         inter_channels - yin_channels,
                                         inter_channels - yin_channels,
                                         5,
                                         1,
                                         16,
                                         gin_channels=gin_channels)

        self.enc_pitch = PosteriorEncoder(yin_channels,
                                          yin_channels,
                                          yin_channels,
                                          5,
                                          1,
                                          16,
                                          gin_channels=gin_channels)

        self.flow = ResidualCouplingBlock(inter_channels,
                                          hidden_channels,
                                          5,
                                          1,
                                          4,
                                          gin_channels=gin_channels)

        if use_sdp:
            self.dp = StochasticDurationPredictor(hidden_channels,
                                                  192,
                                                  3,
                                                  0.5,
                                                  4,
                                                  gin_channels=gin_channels)
        else:
            self.dp = DurationPredictor(hidden_channels,
                                        256,
                                        3,
                                        0.5,
                                        gin_channels=gin_channels)

        self.yin_dec = YingDecoder(yin_scope,
                                   5,
                                   1,
                                   4,
                                   yin_start,
                                   yin_scope,
                                   yin_shift_range,
                                   gin_channels=gin_channels)

        #self.vq = VQEmbedding(codebook_size, inter_channels - yin_channels)#inter_channels // 2)
        self.emb_g = nn.Embedding(self.n_speakers, gin_channels)

        self.pitch = Pitch(midi_start=midi_start,
                           midi_end=midi_end,
                           octave_range=octave_range)

    def crop_scope(
            self,
            x,
            scope_shift=0):  # x: list #need to modify for non-scalar shift
        return [
            i[:, self.yin_start + scope_shift:self.yin_start + self.yin_scope +
              scope_shift, :] for i in x
        ]

    def crop_scope_tensor(
            self, x,
            scope_shift):  # x: tensor [B,C,T] #scope_shift: tensor [B]
        return torch.stack([
            x[i, self.yin_start + scope_shift[i]:self.yin_start +
              self.yin_scope + scope_shift[i], :] for i in range(x.shape[0])
        ],
                           dim=0)

    def yin_dec_infer(self, z_yin, z_mask, sid=None):
        if self.n_speakers > 0:
            g = self.emb_g(sid).unsqueeze(-1)  # [b, h, 1]
        else:
            g = None
        return self.yin_dec.infer(z_yin, z_mask, g)

    def forward(self,
                x,
                t,
                x_lengths,
                y,
                y_lengths,
                ying,
                ying_lengths,
                sid=None,
                scope_shift=0):
        x, m_p, logs_p, x_mask = self.enc_p(x, t, x_lengths)
        if self.n_speakers > 0:
            g = self.emb_g(sid).unsqueeze(-1)  # [b, h, 1]
        else:
            g = None

        z_spec, m_spec, logs_spec, spec_mask = self.enc_spec(y, y_lengths, g=g)

        #for Q option
        #z_spec_q_st, z_spec_q = self.vq.straight_through(z_spec)
        #z_spec_q_st = z_spec_q_st * spec_mask
        #z_spec_q = z_spec_q * spec_mask

        z_yin, m_yin, logs_yin, yin_mask = self.enc_pitch(ying, y_lengths, g=g)
        z_yin_crop, logs_yin_crop, m_yin_crop = self.crop_scope(
            [z_yin, logs_yin, m_yin], scope_shift)

        #yin dec loss
        yin_gt_crop, yin_gt_shifted_crop, yin_dec_crop, z_yin_crop_shifted, scope_shift = self.yin_dec(
            z_yin, ying, yin_mask, g)

        z = torch.cat([z_spec, z_yin], dim=1)
        logs_q = torch.cat([logs_spec, logs_yin], dim=1)
        m_q = torch.cat([m_spec, m_yin], dim=1)
        y_mask = spec_mask

        z_p = self.flow(z, y_mask, g=g)

        z_dec = torch.cat([z_spec, z_yin_crop], dim=1)

        z_dec_shifted = torch.cat([z_spec.detach(), z_yin_crop_shifted], dim=1)
        z_dec_ = torch.cat([z_dec, z_dec_shifted], dim=0)

        with torch.no_grad():
            # negative cross-entropy
            s_p_sq_r = torch.exp(-2 * logs_p)  # [b, d, t]
            # [b, 1, t_s]
            neg_cent1 = torch.sum(-0.5 * math.log(2 * math.pi) - logs_p, [1],
                                  keepdim=True)
            # [b, t_t, d] x [b, d, t_s] = [b, t_t, t_s], z_p: [b,d,t]
            #neg_cent2 = torch.matmul(-0.5 * (z_p**2).transpose(1, 2), s_p_sq_r)
            neg_cent2 = torch.einsum('bdt, bds -> bts', -0.5 * (z_p**2),
                                     s_p_sq_r)
            # [b, t_t, d] x [b, d, t_s] = [b, t_t, t_s]
            #neg_cent3 = torch.matmul(z_p.transpose(1, 2), (m_p * s_p_sq_r))
            neg_cent3 = torch.einsum('bdt, bds -> bts', z_p, (m_p * s_p_sq_r))
            neg_cent4 = torch.sum(-0.5 * (m_p**2) * s_p_sq_r, [1],
                                  keepdim=True)  # [b, 1, t_s]
            neg_cent = neg_cent1 + neg_cent2 + neg_cent3 + neg_cent4

            attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(
                y_mask, -1)
            from monotonic_align import maximum_path
            attn = maximum_path(neg_cent,
                                attn_mask.squeeze(1)).unsqueeze(1).detach()

        w = attn.sum(2)
        if self.use_sdp:
            l_length = self.dp(x, x_mask, w, g=g)
            l_length = l_length / torch.sum(x_mask)
        else:
            logw_ = torch.log(w + 1e-6) * x_mask
            logw = self.dp(x, x_mask, g=g)
            l_length = torch.sum(
                (logw - logw_)**2, [1, 2]) / torch.sum(x_mask)  # for averaging

        # expand prior
        m_p = torch.einsum('bctn, bdn -> bdt', attn, m_p)
        logs_p = torch.einsum('bctn, bdn -> bdt', attn, logs_p)

        #z_slice, ids_slice = commons.rand_slice_segments(z_dec, y_lengths, self.segment_size)
        #o = self.dec(z_slice, g=g)
        z_slice, ids_slice = commons.rand_slice_segments_for_cat(
            z_dec_, torch.cat([y_lengths, y_lengths], dim=0),
            self.segment_size)
        o_ = self.dec.hier_forward(z_slice, g=torch.cat([g, g], dim=0))
        o = [torch.chunk(o_hier, 2, dim=0)[0] for o_hier in o_]

        o_pad = F.pad(o_[-1], (768, 768 + (-o_[-1].shape[-1]) % 256 + 256 *
                               (o_[-1].shape[-1] % 256 == 0)),
                      mode='constant').squeeze(1)
        yin_hat = self.pitch.yingram(o_pad)
        yin_hat_crop = self.crop_scope([yin_hat])[0]
        yin_hat_shifted = self.crop_scope_tensor(
            torch.chunk(yin_hat, 2, dim=0)[0], scope_shift)
        return o, l_length, attn, ids_slice, x_mask, y_mask, o_, \
               (z, z_p, m_p, logs_p, m_q, logs_q), \
               (z_dec_), \
               (z_spec, m_spec, logs_spec, spec_mask, z_yin, m_yin, logs_yin, yin_mask), \
               (yin_gt_crop, yin_gt_shifted_crop, yin_dec_crop, yin_hat_crop, scope_shift, yin_hat_shifted)

    def infer(self,
              x,
              t,
              x_lengths,
              sid=None,
              noise_scale=1,
              length_scale=1,
              noise_scale_w=1.,
              max_len=None,
              scope_shift=0):  #need to fix #vector scope shift needed
        x, m_p, logs_p, x_mask = self.enc_p(x, t, x_lengths)
        if self.n_speakers > 0:
            g = self.emb_g(sid).unsqueeze(-1)  # [b, h, 1]
        else:
            g = None

        if self.use_sdp:
            logw = self.dp(x,
                           x_mask,
                           g=g,
                           reverse=True,
                           noise_scale=noise_scale_w)
        else:
            logw = self.dp(x, x_mask, g=g)
        w = torch.exp(logw) * x_mask * length_scale
        w_ceil = torch.ceil(w)
        y_lengths = torch.clamp_min(torch.sum(w_ceil, [1, 2]), 1).long()
        y_mask = torch.unsqueeze(commons.sequence_mask(y_lengths, None),
                                 1).to(x_mask.dtype)
        attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
        attn = commons.generate_path(w_ceil, attn_mask)

        m_p = torch.einsum('bctn, bdn -> bdt', attn, m_p)
        logs_p = torch.einsum('bctn, bdn -> bdt', attn, logs_p)

        z_p = m_p + torch.randn_like(m_p) * torch.exp(logs_p) * noise_scale
        z = self.flow(z_p, y_mask, g=g, reverse=True)
        z_spec, z_yin = torch.split(z,
                                    self.inter_channels - self.yin_channels,
                                    dim=1)
        z_yin_crop = self.crop_scope([z_yin], scope_shift)[0]
        z_crop = torch.cat([z_spec, z_yin_crop], dim=1)
        o = self.dec((z_crop * y_mask)[:, :, :max_len], g=g)
        return o, attn, y_mask, (z_crop, z, z_p, m_p, logs_p)

    def infer_pre_decoder(self,
                          x,
                          t,
                          x_lengths,
                          sid=None,
                          noise_scale=1.,
                          length_scale=1.,
                          noise_scale_w=1.,
                          max_len=None,
                          scope_shift=0):
        x, m_p, logs_p, x_mask = self.enc_p(x, t, x_lengths)
        if self.n_speakers > 0:
            g = self.emb_g(sid).unsqueeze(-1)  # [b, h, 1]
        else:
            g = None

        if self.use_sdp:
            logw = self.dp(x,
                           x_mask,
                           g=g,
                           reverse=True,
                           noise_scale=noise_scale_w)
        else:
            logw = self.dp(x, x_mask, g=g)
        w = torch.exp(logw) * x_mask * length_scale
        w_ceil = torch.ceil(w)
        y_lengths = torch.clamp_min(torch.sum(w_ceil, [1, 2]), 1).long()
        y_mask = torch.unsqueeze(commons.sequence_mask(y_lengths, None),
                                 1).to(x_mask.dtype)
        attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
        attn = commons.generate_path(w_ceil, attn_mask)

        m_p = torch.einsum('bctn, bdn -> bdt', attn, m_p)
        logs_p = torch.einsum('bctn, bdn -> bdt', attn, logs_p)

        z_p = m_p + torch.randn_like(m_p) * torch.exp(logs_p) * noise_scale
        z = self.flow(z_p, y_mask, g=g, reverse=True)
        z_spec, z_yin = torch.split(z,
                                    self.inter_channels - self.yin_channels,
                                    dim=1)
        z_yin_crop = self.crop_scope([z_yin], scope_shift)[0]
        z_crop = torch.cat([z_spec, z_yin_crop], dim=1)
        decoder_inputs = z_crop * y_mask
        return decoder_inputs, attn, y_mask, (z_crop, z, z_p, m_p, logs_p)

    def infer_pre_lr(
        self,
        x,
        t,
        x_lengths,
        sid=None,
        length_scale=1,
        noise_scale_w=1.,
    ):
        x, m_p, logs_p, x_mask = self.enc_p(x, t, x_lengths)
        if self.n_speakers > 0:
            g = self.emb_g(sid).unsqueeze(-1)  # [b, h, 1]
        else:
            g = None

        if self.use_sdp:
            logw = self.dp(x,
                           x_mask,
                           g=g,
                           reverse=True,
                           noise_scale=noise_scale_w)
        else:
            logw = self.dp(x, x_mask, g=g)
        w = torch.exp(logw) * x_mask * length_scale
        w_ceil = torch.ceil(w)
        return w_ceil, x, m_p, logs_p, x_mask, g

    def infer_lr(self, w_ceil, x, m_p, logs_p, x_mask):
        y_lengths = torch.clamp_min(torch.sum(w_ceil, [1, 2]), 1).long()
        y_mask = torch.unsqueeze(commons.sequence_mask(y_lengths, None),
                                 1).to(x_mask.dtype)
        attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
        attn = commons.generate_path(w_ceil, attn_mask)

        m_p = torch.einsum('bctn, bdn -> bdt', attn, m_p)
        logs_p = torch.einsum('bctn, bdn -> bdt', attn, logs_p)
        return m_p, logs_p, y_mask

    def infer_post_lr_pre_decoder(self,
                                  m_p,
                                  logs_p,
                                  g,
                                  y_mask,
                                  noise_scale=1,
                                  scope_shift=0):

        z_p = m_p + torch.randn_like(m_p) * torch.exp(logs_p) * noise_scale
        z = self.flow(z_p, y_mask, g=g, reverse=True)
        z_spec, z_yin = torch.split(z,
                                    self.inter_channels - self.yin_channels,
                                    dim=1)

        z_yin_crop = self.crop_scope([z_yin], scope_shift)[0]
        z_crop = torch.cat([z_spec, z_yin_crop], dim=1)
        decoder_inputs = z_crop * y_mask

        return decoder_inputs, y_mask, (z_crop, z, z_p, m_p, logs_p)

    def infer_decode_chunk(self, decoder_inputs, sid=None):
        if self.n_speakers > 0:
            g = self.emb_g(sid).unsqueeze(-1)  # [b, h, 1]
        else:
            g = None
        return self.dec(decoder_inputs, g=g)