waveglow: Cant load waveglow checkpoint into inference code even after convert

@Jackson-Kang

@CookiePPP can give better answer than me

I think you give me too much credit 😪 I’d like to provide parameter improvements but I haven’t found any major improvements that you can’t find already done in the Squeezewave repo.

Sorry, what I wanted to say is that it will be good to focus on other issues like fp32 instead of reducing parameters. If you feel that I am rude, I apologize.

In my experiences, most of speed degragation problem is mostly come from framework version.

Maybe. I am using torch 1.4 and numpy 1.16, meanwhile, in the requirements there are torch 1.0 and numpy 1.13. But I cant use numpy 1.13, because tensorflow 1.15.2 need numpy higher than 1.15. I dont know how Nvidia get it working with numpy 1.13

Update: I tried torch 1.0, the Tacotron inference time is slightly reduced, but Waveglow time is almost doubled

Sorry for late sharing… Yesterday, I change “convert_model.py” into the codes below.

Since my model has not trained yet, I couldn’t test this codes. Hope that this codes are helpful to you and plz let me know whether it works or not.

Explanation of code change) The problem is that the trained model (saved model) does not contain the number of “n_layers” Conv modules, “cond_layers”, but does have one integrated Conv module with (C_in=640, C_out = n_channels * n_layers). Thus, I separate one integrated Conv module into the number of “n_layers” convolutions.

Thank you.

Changed code)

import sys
import copy
import torch

def _check_model_old_version(model):
    if hasattr(model.WN[0], 'res_layers'):
        return True
    else:
        return False

def update_model(old_model):
    if not _check_model_old_version(old_model):
        return old_model
    new_model = copy.deepcopy(old_model)
    for idx in range(0, len(new_model.WN)):
        wavenet = new_model.WN[idx]
        wavenet.res_skip_layers = torch.nn.ModuleList()
        n_channels = wavenet.n_channels
        n_layers = wavenet.n_layers
        for i in range(0, n_layers):
            if i < n_layers - 1:
                res_skip_channels = 2*n_channels
            else:
                res_skip_channels = n_channels
            res_skip_layer = torch.nn.Conv1d(n_channels, res_skip_channels, 1)
            skip_layer = torch.nn.utils.remove_weight_norm(wavenet.skip_layers[i])
            if i < n_layers - 1:
                res_layer = torch.nn.utils.remove_weight_norm(wavenet.res_layers[i])
                res_skip_layer.weight = torch.nn.Parameter(torch.cat([res_layer.weight, skip_layer.weight]))
                res_skip_layer.bias = torch.nn.Parameter(torch.cat([res_layer.bias, skip_layer.bias]))
            else:
                res_skip_layer.weight = torch.nn.Parameter(skip_layer.weight)
                res_skip_layer.bias = torch.nn.Parameter(skip_layer.bias)
            res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight')
            wavenet.res_skip_layers.append(res_skip_layer)
        del wavenet.res_layers
        del wavenet.skip_layers
    return new_model

def _check_model_trained_with_custom_data(model):
    if hasattr(model.WN[0], 'cond_layer'):
        return True
    else:
        return False


def update_model_custom(old_model):
    if not _check_model_trained_with_custom_data(old_model):
        return old_model

    new_model = copy.deepcopy(old_model)

    for idx in range(0, len(new_model.WN)):
        wavenet = new_model.WN[idx]
        wavenet.cond_layers = torch.nn.ModuleList()
        n_channels = wavenet.n_channels
        n_layers = wavenet.n_layers

        removed_weightnorm = torch.nn.utils.remove_weight_norm(wavenet.cond_layer)
        removed_weightnorm.weight = torch.nn.Parameter(removed_weightnorm.weight.view(n_layers, n_channels*2, 640, -1))
        removed_weightnorm.bias = torch.nn.Parameter(removed_weightnorm.bias.view(n_layers, n_channels*2))

        print("{}".format(idx+1))
        for cond_param_name, cond_params in removed_weightnorm.named_parameters():

            #torch.nn.utils.remove_weight_norm()
            if cond_param_name == "bias":
                print("\tcond_layer_param [bias]: ", cond_params.data.size())

            else:
                print("\tweight norm: ", cond_param_name)
                print("\t\tcond_layer_params : ", cond_params.data.size())

            for i in range(0, n_layers):
                conv = torch.nn.Conv1d(640, n_channels*2, 1)
                conv.weight = torch.nn.Parameter(removed_weightnorm.weight[i])
                conv.bias = torch.nn.Parameter(removed_weightnorm.bias[i])
                print("\tconv {}-th size -  \tweight: {}\tbias: {}".format(i, conv.weight.size(), conv.bias.size()))

                conv = torch.nn.utils.weight_norm(conv, name='weight')
                wavenet.cond_layers.append(conv)

        del wavenet.cond_layer
        print("[WN END]\n\n")

    return new_model


if __name__ == '__main__':
    old_model_path = sys.argv[1]
    new_model_path = sys.argv[2]
    model = torch.load(old_model_path)
    model['model'] = update_model(model['model'])
    model['model'] = update_model_custom(model['model'])

    torch.save(model, new_model_path)