o j@sddlZddlmZmZddlmZddlmZddlm Z m Z m Z m Z m Z ddlZddlZddlmZddlZddlmZddlmZddlmZdd lmZdd lmZdd l m!Z!dd l"m#Z#dd l$m%Z%m&Z&ddl'm(Z(m)Z)ddl*m+Z+m,Z,m-Z-ddl.m/Z/ddl0m1Z1m2Z2ddl3m4Z4ddl5m6Z6ddl7m8Z8m9Z9m:Z:m;Z;mZ>ddl?m@Z@ddlAmBZBmCZCmDZDmEZEddlFmGZGmHZHddlFmIZJddlFmKZLddlMmNZNddlOmPZPddlQmRZRddlSmTZTddlUmVZVdrd!d"ZWdrd#d$ZXdrd%d&ZYd'ejZd(ejZfd)d*Z[d+e ejZd,e\d(ejZfd-d.Z]dsd1ej^d2e_d3e_fd4d5Z`dtd7ejZd8e\d(ejZfd9d:Zad;e e\d(ejZfde\d(e e\e\ffd?d@Zciadiaeefd1ej^fdAdBZgdCej^fdDdEZhdFeifdGdHZjdudKdLZkdvdMdNZldOdPZmdQdRZIdrdSdTZndrdUdVZodrdWdXZpdYdZZqd[d\Zrdrd]d^Zsdwd_etd`eidaeufdbdcZvGdddedee+ZwGdfdgdge,ZxeGdhdidieZyeGdjdkdkeZzeGdldmdmeZ{Gdndodoe6Z|Gdpdqdqej^Z}dS)xN) dataclassfield)chain)Path)DictListOptionalTupleUnion)Coqpit)mel)nn)autocast) functional) DataLoader)WeightedRandomSampler)DistributedSamplerDistributedSamplerWrapper) get_optimizer get_scheduler) F0Dataset TTSDataset _parse_sample) AcousticModel)ForwardSumLossVitsDiscriminatorLoss)VitsDiscriminator) BaseTTSE2E)average_over_durationscompute_attn_prior rand_segmentssegment sequence_mask)SpeakerManager) TTSTokenizer)plot_alignmentplot_avg_pitch plot_pitchplot_spectrogram)build_mel_basis compute_f0) db_to_amp) mel_to_wav)AudioProcessor) load_fsspec)MultiScaleSTFTLoss)HifiganGenerator) plot_resultsFcCs,|durt|}t|}|r|S|SN)npasarraytorch from_numpycuda)aux_idr7r9P/home/kuhnn/.local/lib/python3.10/site-packages/TTS/tts/models/delightful_tts.py id_to_torch(s   r;cCs>|durt|}t|}|d}|r|S|S)Nr)r3r4r5r6floatsqueeze unsqueezer7)d_vectorr7r9r9r:embedding_to_torch1s r@cCs*|durdStj||d}|r|S|S)Ndtype)r5 as_tensorr7)np_arrayrBr7tensorr9r9r:numpy_to_torch;s rFlengthsreturncCsR|jd}t|}tjd||jdd|d}||dd|k}|S)Nrdevice)shaper5maxitemarangerJr>expand)rG batch_sizemax_lenidsmaskr9r9r:get_mask_from_lengthsDs  rV input_elerSc Cstjttjg}|D]/}t|jdkr%t|d|| dfdd}nt|ddd|| dfdd}| |q t |}|S)NrLrconstant) r5jitannotaterTensorlenrMFpadsizeappendstack)rWrSout_listbatchone_batch_padded out_paddedr9r9r:r_Ls "  r_rY{Gz?mmeanstdcCs.|jj}|ddkr|jj||dSdS)NConvrK) __class____name__findweightdatanormal_)rhrirj classnamer9r9r: init_weightsXsrslensstridecCst||Sr2)r5ceilint)rurvr9r9r: stride_lens^sryrMcCs0t|dks Jdt|td|dS)Nrtz.Can only initialize 2-D embedding matrices ...rL)r]r5randnr3sqrtrMr9r9r:initialize_embeddingsbsr} kernel_sizecCs|d}|||ddfS)NrtrLr9)r~r_r9r9r:calc_same_paddinghsrcCs$t|dd}t|r|dSdS)Nreset_parameters)getattrcallabler)rhrr9r9r: weights_resetqs  rmdlcCs8i}|D]\}}d|vr|j}|||<q|S)Nro)named_parametersrpsumrO)r dict_sumsnamewvaluer9r9r:get_module_weights_sumysr file_pathcCs6t|\}}|dk|dkdksJ||fS)z^Load the audio file normalized in [-1, 1] Return Shapes: - x: :math:`[1, T]` rLrKr) torchaudioloadr)rxsrr9r9r: load_audios  rrLh㈵>cCsttj||d|S)Nmin)r5logclamp)rCclip_valr9r9r: _amp_to_dbsrcCst||Sr2)r5exp)rrr9r9r: _db_to_amprcC t|}|Sr2)r magnitudesoutputr9r9r: amp_to_dbrcCrr2)rrr9r9r:r+rr+c Cs|d}t|dkrtdt|t|dkr#tdt|t|jdt|j}t|d|}|tvrIt|j |j|jdt|<tj j j | dt||dt||dfd d }|d}tj||||t||d d d d d }|S)NrL min value is ? max value is _rBrJrtreflectmodeFT hop_length win_lengthwindowcenterpad_mode normalizedonesidedreturn_complex)r=r5rprintrNstrrBrJ hann_windowtor rr_r>rxstft)yn_fftrrr dtype_devicewnsize_dtype_devicespecr9r9r: _wav_to_specs8   rcCs0t|||||d}t|ddd}|S)zk Args Shapes: - y : :math:`[B, 1, T]` Return Shapes: - spec : :math:`[B,C,T]` rrtrKư>)rr5r{powrrrrrrrr9r9r: wav_to_specsrcCs<t|||||d}t|ddd}tj|dddS)NrrtrKrrLT)dimkeepdim)rr5r{rrnormrr9r9r: wav_to_energysrcCs"|d|d|jd|j}|S)Nrr)rrfmax n_fft_lenr9r9r:name_mel_basissrcCs\t|||}|tvr t|||||}t|j|j|jdt|<tt||}t |}|S)zk Args Shapes: - spec : :math:`[B,C,T]` Return Shapes: - mel : :math:`[B,C,T]` r) r mel_basislibrosa_mel_fnr5r6rrBrJmatmulr)rrnum_mels sample_ratefminr mel_basis_keyr r9r9r: spec_to_mels rc Csh|d}t|dkrtdt|t|dkr#tdt|t|||} t|dt|jdt|j} | t vrVt |||||d} t | j |j|jdt | <| t vrht |j |j|jdt | <tjjj|dt||d t||d fd d }|d}tj||||t | |d d d d d } t| d dd} tt | | } t| } | S)zr Args Shapes: - y : :math:`[B, 1, T_y]` Return Shapes: - spec : :math:`[B,C,T_spec]` rLrrrrr)rrn_melsrrrrtrrFTrrKr)r=r5rrrNrrrBrJrrr6rrr rr_r>rxrr{rrrr) rrrrrrrrrrrr rr9r9r: wav_to_melsH  $   ritems attr_name multi_dictcstfdd|DtfddD}tfddD}d|tfdd|D}|tj|}durUtfdd|D}||9}t|t|fS) z0Create balancer weight for torch WeightedSamplercg|]}|qSr9r9.0rO)rr9r: /z2get_attribute_balancer_weights..csg|]}|qSr9)indexrl)unique_attr_namesr9r:r1cs"g|] }tt|kdqSr)r]r3wherer)attr_names_samplesr9r:r2s"rcsg|]}|qSr9r9r) weight_attrr9r:r4rNcsg|] }|dqS)r)getr)rrr9r:r7s) r3arrayuniquetolistlinalgrr5r6r<)rrrattr_idx attr_countdataset_samples_weightmultiplier_samplesr9)rrrrrr:get_attribute_balancer_weights-s  rcsVeZdZdZ    ddeeeeefdeffdd Zdd d Z d d Z Z S)ForwardTTSE2eF0Datasetz5Override F0Dataset to avoid slow computing of pitchesFNrTsamples cache_pathcstj||||||ddS)N)rapverboserprecompute_num_workers normalize_f0)super__init__)selfrrrrrrrlr9r:rCs  zForwardTTSE2eF0Dataset.__init__cCsrt|\}}t|d|jj|jj|jj|jj|jjd}|j d|jjdkr/|dd}|r7t |||S)Nr)rrr pitch_fmax pitch_fminrrLrK) rr*numpyrrrrrrrMr3save)rwav_file pitch_filewavrf0r9r9r:_compute_and_save_pitchUs    z.ForwardTTSE2eF0Dataset._compute_and_save_pitchcCs@|||j}tj|s|j||d}nt|}|tj S)zH compute pitch and return a numpy array of pitch values )rr) create_pitch_file_pathrospathexistsrr3rastypefloat32)rr audio_namerpitchr9r9r:compute_or_loadfs    z&ForwardTTSE2eF0Dataset.compute_or_load)FNrTr2) rm __module__ __qualname____doc__r rrrrrr  __classcell__r9r9rr:r@s rcs@eZdZfddZddZddZeddZd d ZZ S) ForwardTTSE2eDatasetcs|dd}d|d<|d|_tj|i|||_|jjj|_|d|_|jr9t |j|j |d|dd|_ |jdurHt j |jdd dSdS) Nr*Fattn_prior_cache_pathr f0_cache_pathr)rrrrT)exist_ok)poprrrr* tokenizer characterspad_idrrr f0_datasetrmakedirs)rargskwargsr*rr9r:rss"     zForwardTTSE2eDataset.__init__c Cs|j|}t|d|dd}t|dd}|d}t|d\}}tj |d}z | ||d}Wn t ||t d} |j rP||d} t||jks_|jd|jkrl|jd7_||jSd} |jdurz||||} ||t||| ||d |d | |d d S) N audio_file root_path/rtextrrL speaker_namelanguageaudio_unique_name) raw_text token_ids token_lenrr rr# language_name attn_priorr%)rr relative_to with_suffixrreplacerrrbasename get_token_idsrOSErrorr*get_f0r] max_text_lenrM min_audio_lenrescue_item_idx __getitem__rload_or_compute_attn_prior) ridxrO rel_wav_pathr&rr wav_filenamer'rr*r9r9r:r5s>    z ForwardTTSE2eDataset.__getitem__cCs`tj|j|d}tj|rt|St|}|jd|j j }t ||}t |||S)z-Load or compute and save the attention prior.z.npyrL) rrjoinrrr3rr]rMrrrr)rr'rr8attn_prior_filer(mel_lenr*r9r9r:r6s    z/ForwardTTSE2eDataset.load_or_compute_attn_priorcCs@g}|jD]}t|^}}}tj|dd}||q|S)N)rrrrgetsizera)rrurOrr audio_lenr9r9r:rGs   zForwardTTSE2eDataset.lengthscst}fdddDtdddD}td}||}ddd D}t|}t|}||}d } |jrdd dd D} t| } t| } t|d | } | |j} t||} t|d |} | |j} | |j} t|D]E}d|}t|| |d d|f<d |}t|| |d d d | d f<|jrȈd |}t|| |dd t|f<q| ||| | ||dddddddd urdd Sd d S)a Return Shapes: - tokens: :math:`[B, T]` - token_lens :math:`[B]` - token_rel_lens :math:`[B]` - pitch :math:`[B, T]` - waveform: :math:`[B, 1, T]` - waveform_lens: :math:`[B]` - waveform_rel_lens: :math:`[B]` - speaker_names: :math:`[B]` - language_names: :math:`[B]` - audiofile_paths: :math:`[B]` - raw_texts: :math:`[B]` - attn_prior: :math:`[[T_token, T_mel]]` cs i|] fddDqS)crr9r9)rdickr9r:rrz>ForwardTTSE2eDataset.collate_fn...r9)rrdrBr: s z3ForwardTTSE2eDataset.collate_fn..rcSsg|]}t|qSr9)r])rrr9r9r:rrz3ForwardTTSE2eDataset.collate_fn..r'r(cSg|]}|jdqSrLr|rrr9r9r:rrrNcSrFrr|)rpr9r9r:rrr rLr#r)r%rr&r*) text_input text_lengths text_rel_lensr waveform waveform_lenswaveform_rel_lens speaker_nameslanguage_namesaudio_unique_names audio_filesr& attn_priors) r]rNr5 LongTensorr* FloatTensorzero_rranger`)rrdBr2 token_lenstoken_rel_lenswav_lens wav_lens_max wav_rel_lens pitch_padded pitch_lenspitch_lens_max token_padded wav_paddedir'rr r9rDr: collate_fnsZ         "  zForwardTTSE2eDataset.collate_fn) rmrrrr5r6propertyrGrerr9r9rr:rrs , rc@seZdZUdZeed<edddZee ed<edddZ eee ed<ed ddZ ee ed <d Z e ed <ed ddZ ee ed<dZeed<edddZee ed<edddZee ed<dZeeed<dS) VocoderConfig1resblock_type_decodercCgdS)N) r9r9r9r9r:zVocoderConfig.)default_factoryresblock_kernel_sizes_decodercCsgdgdgdgS)N)rLrkr9r9r9r9r:rnrresblock_dilation_sizes_decodercCrj)N)r>r>rtrtr9r9r9r9r:rnroupsample_rates_decoder upsample_initial_channel_decodercCrj)N)r=r=rwr9r9r9r9r:rnroupsample_kernel_sizes_decoderFuse_spectral_norm_discriminatorcCrj)N)rwrwrwrwr9r9r9r9r:rn roupsampling_rates_discriminatorcCrj)N)rtrkrrrlrmr9r9r9r9r:rn!roperiods_discriminatorNpretrained_model_path)rmrrrir__annotations__rrqrrxrsrtrvrxryboolrzr{r|rr9r9r9r:rgs   rgc@s:eZdZUdZeed<dZeed<dZeed<dZeed<dZ e ed <d Z e ed <d Z eed <dZ e ed<dZe ed<dZeed<dZe ed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZe ed <d!Ze ed"<d#Zeed$<dZeed%<dZeed&<dZeed'<d(Zeed)<d*Z e ed+<dS),DelightfulTtsAudioConfigi"Vrrirfft_sizerYmel_fmini@mel_fmaxdrg@rrrFresample preemphasis ref_level_db do_sound_normznp.log10log_funcTdo_trim_silence-trim_db do_rms_normNdb_levelg?power<griffin_lim_iters spec_gaindo_amp_to_db_lineardo_amp_to_db_meli min_level_dbg@max_norm)!rmrrrrxr}rrrrr<rrrrrr~rrrrrrrrrrrrrrrrr9r9r9r:r%s4                        rc@seZdZUdZeed<dZeed<dZeed<dZeed<d Z eed <d Z e ed <d Z eed<d Z eed<dZe ed<dZeed<dZeed<d Zeed<d Ze ed<dZeed<dZeed<dZeed<dZeed<gdZdZeed<gdZd d gZdZeed!<d"Ze ed#<dZeed$<d%Zeed&<dZeed'<d%Z eed(<d)Z!e ed*<d+Z"eed,<dZ#eed-<d.Z$e%ed/<d0Z&eed1<d2Z'e(ed3<d2Z)e(ed4<d5Z*eed6<d.Z+e%ed7<d0Z,eed8<d.Z-e%ed9<d.Z.e%ed:<d.Z/e%ed;<d.Z0e%ed<<d.Z1e%ed=<d.Z2e%ed><d.Z3e%ed?<d@Z4e edA<d2S)BDelightfulTtsArgsr num_chars spec_segment_sizerun_hidden_conformer_encodern_layers_conformer_encoderr>n_heads_conformer_encoderg?dropout_conformer_encoderrl&kernel_size_conv_mod_conformer_encoder'kernel_size_depthwise_conformer_encoderg333333? lrelu_slopen_hidden_conformer_decodern_layers_conformer_decodern_heads_conformer_decoderdropout_conformer_decoderrm&kernel_size_conv_mod_conformer_decoder'kernel_size_depthwise_conformer_decoderrw#bottleneck_size_p_reference_encoder#bottleneck_size_u_reference_encoder)rr@rrrkref_enc_size_reference_encoder)rLrtrLrtrLrL"ref_enc_gru_size_reference_encoderg?'ref_attention_dropout_reference_encodertoken_num_reference_encoderrr'predictor_kernel_size_reference_encodern_hidden_variance_adaptorkernel_size_variance_adaptor?dropout_variance_adaptorrn_bins_variance_adaptor emb_kernel_size_variance_adaptorFuse_speaker_embeddingr num_speakersN speakers_file d_vector_fileispeaker_embedding_channelsuse_d_vector_file d_vector_dimfreeze_vocoderfreeze_text_encoderfreeze_duration_predictorfreeze_pitch_predictorfreeze_energy_predictorfreeze_basis_vectors_predictorfreeze_decoderr length_scale)5rmrrrrxr}rrrrrr<rrrrrrrrrrr!ref_enc_filters_reference_encoderr!ref_enc_strides_reference_encoderref_enc_pad_reference_encoderrrrrrrrrrrr~rrrrrrrrrrrrrrrr9r9r9r:rBs\                                         rcsLeZdZdZ  d|dedddeffdd Zed d Zed d Z ed dZ e j ddZ eddZ e j ddZ eddZ e j ddZ eddZd}ddZdefddZddZdd Zd!d"Z     d~d#ejd$ejd%ejd&ejd'ejd(ejd)ejd*ejd+ejd,ejdefd-d.Zeddd/ddfd0d1Zeddd/fd2d3Zd4ed5ejd6efd7d8Z d4ed5ejd6efd9d:Z!dded?d@dAedBef dCdDZ#d4ed>ed?d@dAedBeddf dEdFZ$dGdHZ%dIdJZ&   ddKe'dLe'dMej(fdNdOZ)dKe'fdPdQZ*ede+eeffdRdSZ,d>ed?d@dAedBeddf dTdUZ-d4edefdVdWZ.dXdYZ/dded[e0fd\d]Z1 ddedAed^e2d_e3e4ee4e4fd`e2daedbeddcfdddeZ5dfdgZ6de4fdhdiZ7de4fdjdkZ8de4fdldmZ9dndoZ:e; pdddqd_e3e4e4e4effdrdsZdxdyZ?d}dzd{Z@ZAS) DelightfulTTSa Paper:: https://arxiv.org/pdf/2110.12612.pdf Paper Abstract:: This paper describes the Microsoft end-to-end neural text to speech (TTS) system: DelightfulTTS for Blizzard Challenge 2021. The goal of this challenge is to synthesize natural and high-quality speech from text, and we approach this goal in two perspectives: The first is to directly model and generate waveform in 48 kHz sampling rate, which brings higher perception quality than previous systems with 16 kHz or 24 kHz sampling rate; The second is to model the variation information in speech through a systematic design, which improves the prosody and naturalness. Specifically, for 48 kHz modeling, we predict 16 kHz mel-spectrogram in acoustic model, and propose a vocoder called HiFiNet to directly generate 48 kHz waveform from predicted 16 kHz mel-spectrogram, which can better trade off training efficiency, modelling stability and voice quality. We model variation information systematically from both explicit (speaker ID, language ID, pitch and duration) and implicit (utterance-level and phoneme-level prosody) perspectives: 1) For speaker and language ID, we use lookup embedding in training and inference; 2) For pitch and duration, we extract the values from paired text-speech data in training and use two predictors to predict the values in inference; 3) For utterance-level and phoneme-level prosody, we use two reference encoders to extract the values in training, and use two separate predictors to predict the values in inference. Additionally, we introduce an improved Conformer block to better model the local and global dependency in acoustic model. For task SH1, DelightfulTTS achieves 4.17 mean score in MOS test and 4.35 in SMOS test, which indicates the effectiveness of our proposed system Model training:: text --> ForwardTTS() --> spec_hat --> rand_seg_select()--> GANVocoder() --> waveform_seg spec --------^ Examples: >>> from TTS.tts.models.forward_tts_e2e import ForwardTTSE2e, ForwardTTSE2eConfig >>> config = ForwardTTSE2eConfig() >>> model = ForwardTTSE2e(config) Nconfigrr$speaker_managercstj||||d||_||||d|_|jjj|j _ |jjj|j _t |j ||d|_ t |jjjd|jjj|jjj|jjj|jjj|jjj|jjjddddd |_|jjrjt|jjj|jjjd|_dSdS)N)rrrr)rrrrLrF)inference_paddingconv_pre_weight_normconv_post_weight_normconv_post_bias)use_spectral_normperiods)rrr_set_model_argsinit_multispeakerbinary_loss_weightraudiorr out_channelsracoustic_modelr0vocoderrirsrqrxrvrtwaveform_decoderinit_discriminatorrryr{disc)rrrrrrr9r:rs8   zDelightfulTTS.__init__cCst|jSr2)next parametersrJrr9r9r:rJszDelightfulTTS.devicecC|jjSr2)r energy_scalerrr9r9r:rzDelightfulTTS.energy_scalercCrr2rrrr9r9r:rrzDelightfulTTS.length_scalecC ||j_dSr2rrrr9r9r:r cCrr2r pitch_meanrr9r9r:rrzDelightfulTTS.pitch_meancCrr2rrr9r9r:rrcCrr2r pitch_stdrr9r9r:rrzDelightfulTTS.pitch_stdcCrr2rrr9r9r:rrcCs&t|jj|jj|jj|jj|jjdS)N)rrrrr)r)rrrrrrrr9r9r:rszDelightfulTTS.mel_basisrHcCs|jjdk|_d|_dS)NrT)rsteps_to_start_discriminator train_discupdate_energy_scalerrr9r9r:init_for_trainings  zDelightfulTTS.init_for_trainingcCsZd|_|jj|_d|_|jr|jj|_|jj|j_|jjr!||jjr+|dSdS)zjInit for multi-speaker training. Args: config (Coqpit): Model configuration. rN) embedded_speaker_dimrraudio_transformrr_init_speaker_embeddingr_init_d_vectorrrr9r9r:rs    zDelightfulTTS.init_multispeakercCs0|jdkrtd|jj|_|jj|j_dSdS)Nrz. > initialization of speaker-embedding layers.)rrrrrrr9r9r:rs  z%DelightfulTTS._init_speaker_embeddingcCs,t|dr td|jj|_|jj|j_dS)Nemb_gzI[!] Speaker embedding layer already initialized before d_vector settings.)hasattr ValueErrorrrrrr9r9r:rs  zDelightfulTTS._init_d_vectorcCs|jjr|jD]}d|_q |jjr|jD]}d|_q|jjr-|j D]}d|_q'|jj r<|j D]}d|_q6|jj rK|j D]}d|_qE|jjrZ|jD]}d|_qTdSdSNF)rrr paramseters requires_gradr text_encoderrrdurarion_predictorrpitch_predictorrenergy_predictorrdecoder)rparamr9r9r:_freeze_layerss(zDelightfulTTS._freeze_layersr x_lengths spec_lengthsrrMr energyrT d_vectors speaker_idxc  Cs|j|||||||| | d } | d} t| dd||jjddd\} }| ddur1| dd }nd}|j| |d }t|||j j |jj|j j dd }i| }| d|d <||d<||d <||d<|S)aCModel's forward pass. Args: x (torch.LongTensor): Input character sequences. x_lengths (torch.LongTensor): Input sequence lengths. spec_lengths (torch.LongTensor): Spectrogram sequnce lengths. Defaults to None. spec (torch.FloatTensor): Spectrogram frames. Only used when the alignment network is on. Defaults to None. waveform (torch.FloatTensor): Waveform. Defaults to None. pitch (torch.FloatTensor): Pitch values for each spectrogram frame. Only used when the pitch predictor is on. Defaults to None. energy (torch.FloatTensor): Spectral energy values for each spectrogram frame. Only used when the energy predictor is on. Defaults to None. attn_priors (torch.FloatTentrasor): Attention priors for the aligner network. Defaults to None. aux_input (Dict): Auxiliary model inputs for multi-speaker training. Defaults to `{"d_vectors": 0, "speaker_ids": None}`. Shapes: - x: :math:`[B, T_max]` - x_lengths: :math:`[B]` - spec_lengths: :math:`[B]` - spec: :math:`[B, T_max2, C_spec]` - waveform: :math:`[B, 1, T_max2 * hop_length]` - g: :math:`[B, C]` - pitch: :math:`[B, 1, T_max2]` - energy: :math:`[B, 1, T_max2]` ) tokenssrc_lensmel_lensmelspitchesenergiesrTrr model_outputsrLrtT)rr segment_sizelet_short_samples pad_shortspk_embNrKrgracoustic_model_outputs waveform_seg slice_ids) rr transposerrr>rdetachr!rr)rrrrrrMr rrTrrencoder_outputs vocoder_inputvocoder_input_slicesrrvocoder_outputwav_segr r9r9r:forward(sF$     zDelightfulTTS.forwardr speaker_idsc Cst|jj||d|d||ddd}|ddd}|ddur'|dd}nd}|j||d }i|} || d<| S) Nrr!)rrrpitch_transformenergy_transform p_control d_controlr rLrtrrKr)r inferencerr>r) rr aux_inputr"r#rrrrr r9r9r:r&us" zDelightfulTTS.inferencecCs&|jj||d|dd}i|}|S)Nrr!)rrr)rr&)rrr'rr r9r9r:inference_spec_decodersz$DelightfulTTS.inference_spec_decoderrd criterion optimizer_idxc Cs|dkrw|d}|d}|d}|d}|d}|d} |d} |d } |d } |d } |j|||||| | | | | d }||_|jru||d |d\}}}}tdd||||d}Wd||fS1slwY||fSdS|dkr|d}tdd=t||jd|jj dd}t |jd |j j |j j |j j|j j|j j|j j|j jdd }d}d}d}Wdn1swY|jr||jd |jd\}}}}tdd||d=id|jdddd|dd|dd|jdd|jd d!|jd"d#|jd#d$|jd%d&|jd&d'|dd|jdd(|jd d)|jd)d*|jd*d+|jd+d,|jd,d-|jd-d.|jd/d0|jd0d1|jd2|d3|d4|d5|d6|d7|j }|d|d8<|d|d9<|d|d|d:<|d|d|d;<Wdn 1swY|j|fStd<)>NrrJrK mel_input mel_lengthsrMr rr!rTr) rrrrrMr rrTrrr rF)enabled)scores_disc_fakescores_disc_realNNrLrTr rrrrrrrrr mel_outputrrt mel_targetr  dur_output dr_log_pred dur_target dr_log_target pitch_output pitch_pred pitch_target energy_output energy_pred energy_targetr waveform_hat p_prosody_refp_prosody_pred u_prosody_refu_prosody_predaligner_logprob aligner_hard aligner_mas aligner_softr feats_fake feats_real scores_fake spec_slicespec_slice_hat skip_discavg_text_lengthavg_mel_lengthavg_text_batch_occupancyavg_mel_batch_occupancyz [!] Unexpected `optimizer_idx`.r9)rmodel_outputs_cacherrrrr!r<rrrrrrrrrrrrrrirNr)rrdr)r*r token_lenghtsr r rMr rr!rTroutputs scores_d_faker scores_d_real loss_dict mel_slice mel_slice_hat feats_d_fake feats_d_realr9r9r: train_steps                        'zDelightfulTTS.train_stepcCs||||Sr2)r[)rrdr)r*r9r9r: eval_step rzDelightfulTTS.eval_steptraincCsRii}}|dd}|dd}|d}|dj} |dj} |dj} t| dddt| jdddt| ddd}t|dd d j} t|dd d j} |j|d dj}t | |ddt | |ddd }| |t|ddd j}t|ddd j}|j|d dj}t ||ddt ||ddd}| ||dddj}t|jdd|d<t dt | jddd|j d|jj}|||d<|dd}|dd}t|||j|d}| ||dd}|||d<||fS)NrLr alignmentsr+rF output_fig) prediction ground_truth alignmentr:)rrr9rJ)pitch_ground_truthpitch_avg_predictedr=r<)energy_ground_truthenergy_avg_predicted alignments_dp alignment_hatrgainbaser rz/encoder_audior r)y_hatrr name_prefixz/vocoder_audior9)rpcpurr(Tr%absrdecoder&updatemel_to_wav_numpydb_to_amp_numpyrrrr1rr=r)rrdrSrofiguresaudiosr r^r+ pred_specgt_spec align_img pitch_avg pitch_avg_hatchars pitch_figures energy_avgenergy_avg_hatenergy_figuresalignments_hat encoder_audiornrvocoder_figures sample_voicer9r9r:_logsP              zDelightfulTTS._logrSloggerLoggerassetsstepscCs6|j||dd\}}|||||||jjdS)aCreate visualizations and waveform examples. For example, here you can plot spectrograms and generate sample sample waveforms from these spectrograms to be projected onto Tensorboard. Args: batch (Dict): Model inputs used at the previous training step. outputs (Dict): Model outputs generated at the previous training step. Returns: Tuple[Dict, np.ndarray]: training plots and output waveform. vocoder/rdrSroN)r train_figures train_audiosrrrrdrSrrrrwrxr9r9r: train_logKs zDelightfulTTS.train_logcCs6|j||dd\}}|||||||jjdS)Nrr)r eval_figures eval_audiosrrrr9r9r:eval_log^s zDelightfulTTS.eval_logcCst|jdr |jj}n|j}d\}}}t|tr:t|dkr#|d}nt|dkr.|\}}nt|dkr9|\}}}n|}d\}}t|dro|jr\|durR|j}n|jj |dd d }n|j ro|duri|j }n|jj |}||||d S) N model_argsNNNrLrrtrkr0rF num_samples randomize)r" speaker_id style_wavr?) rrr isinstancelistr]rrget_random_embeddingget_mean_embeddingr get_random_id name_to_id)r sentence_inforr"r#rrr?r9r9r:!get_aux_input_from_test_sentencescs0              z/DelightfulTTS.get_aux_input_from_test_sentencesc CsZi}|d}|d}tt|dddf|jj|jj|jj|jj|jj |jj |jj dd d dd}t |d|jj|jj|jjd} |j|jj|dd } | d d } |d } tt| ddddf| } ||j|j} t| dd dd|d<t||d<t| ||d<t| | |d<t| | |d<t|| |d<|S)Nr rFr1rrL)rrrrenr$zr durationsrtr_rc spectrogrampitch_from_wavpitch_avg_from_wavpitch_avg_predenergy_avg_pred)rprr5r6rrrrrrrrrr*rr ids_to_text text_to_idsr-rrrr%r(r'r&r=)rr"rrcrSrwrrrr  input_textrr|pitch_avg_pred_denormr9r9r: plot_outputssH      " zDelightfulTTS.plot_outputsr"rr?c Kst|j}tj|jj|ddtjd}d}|dur4|jj r4t |t r4|jj r4|j j |}t||d}|durE|jjrE|j j|ddd}t||d}t|tj|d}|d}|j|||d|d} | d dj} | d } | | || d } | S) NrrAr7Frrr )r'r"r r^rr^ text_inputsrS)rris_cudar3r4rrint32rrrrrrr;rrr@rFr5longr>r&rprpr) rr"rr?r"rrr _speaker_idrSrr^ return_dictr9r9r: synthesizes8     zDelightfulTTS.synthesizec Cst|j}tj|jj|ddtjd}|durt||d}|dur)t ||d}t |t j |d}| d}|j|||dd}|ddj}t|ddd }td ||jd |jj}|d } |dddf| ||d } | S)NrrArrr )rr'r rLrjrmr^rr9)rrrr3r4rrrr;r@rFr5rr>r(rprrqrvrurrr) rr"rr?rrrSSrr^rr9r9r:synthesize_with_gls2   z DelightfulTTS.synthesize_with_glc Cstdi}i}|jj}t|D]H\}}||}|j|d|j|d|dd}|j|d|d|dd} |dj|d|<| dj|d |<t |d d d |d |<q||dS)zGeneric test run for `tts` models used by `Trainer`. You can override this for a different behaviour. Returns: Tuple[Dict, Dict]: Test figures and audios to be projected to Tensorboard. z! | > Synthesizing test sentences.r"rr?)rrr?)rr?rz{}-audioz{}-audio_encoderr^Fr_z {}-alignment)rwrx) rrtest_sentences enumeraterrrrqformatr%) rr test_audios test_figuresrr7s_info aux_inputsrS outputs_glr9r9r:test_runs*   zDelightfulTTS.test_runcCs,|||d|jjj|||ddS)Nrxrw)rrrrr)rrSrrrr9r9r:test_log!szDelightfulTTS.test_logcsd}d}jdurjjrjjrfdd|dD}|dur)t|}||d<jdurJjjrJjjrJjjfdd|dD}t|}||d<||d<|S) zGCompute speaker, langugage IDs and d_vector for the batch if necessary.Ncsg|]}jj|qSr9)rr)rsnrr9r:r.z.DelightfulTTS.format_batch..rPr!csg|]}|dqS) embeddingr9rH)d_vector_mappingr9r:r7rrRr) rrPrrr5rU embeddingsrrV)rrdr!rr9)rrr: format_batch's  zDelightfulTTS.format_batchc Cs|j}t|d|j|j|j|j|j|j|jdd |d<|ddur5|dddddd|dj dfnd|d<|dj d|d |d <|dt |d  d |d<|j jr|d }tj|dj d |d |d |djd|d <t|dj d D]}t|||d |d||j d d||j d f<qd|d<t|d|j|j|jdd|d<||d|d<|S)z#Compute spectrograms on the device.rMF)rrrrrrrrr+r NrtrOr,rLrTrrKrIr)rrrr)rrrrrrrrrrMrxr"r>ruse_attn_priorsr5zerosrNrJrXr6rr)rrdacattn_priors_nprdr9r9r:format_batch_on_device>sJ <    0 z$DelightfulTTS.format_batch_on_devicerLdatasetc Csd}|j}t|ddrC|jD]2\}}td|d|d|j|d}t|t|||d\}} } ||}td| d| q|durOt|t |} nd} | durc|d kr_t |} | Sd} | S|d krkt | n| } | S) Nuse_weighted_samplerFz) > Using weighted sampler for attribute 'z' with alpha '')rrrz > Attribute weights for 'z' | > rL) rrweighted_sampler_attrsrrweighted_sampler_multipliersrrrr]rr) rrrnum_gpusweights data_itemsralphar attr_names attr_weightssamplerr9r9r: get_samplerws,  zDelightfulTTS.get_sampleris_evalrrrrankrc Cs|r |js d}|St||j|rdn|j|j|j|j|j|j|j |j |j |j |j r,|jnd||j|jd} |dkr=t| ||| |} t| |rO|jn|jdd| | j|r[|jn|jdd}| jj|_| jj|_|S)Nr)rrbatch_group_size min_text_lenr2r3 max_audio_lenphoneme_cache_pathrr*rrrrstart_by_longestrLFT)rRshuffle drop_lastrre num_workers pin_memory)run_evalrrrrRrr2r3rrrr*rrrrrdistbarrierpreprocess_samplesrreval_batch_sizerenum_eval_loader_workersnum_loader_workersrrirrjr) rrrrrrrrloaderrrr9r9r:get_data_loadersJ - zDelightfulTTS.get_data_loadercCst|jt|jgSr2)rrDelightfulTTSLossrr9r9r: get_criterionszDelightfulTTS.get_criterioncCsVt|jj|jj|jj|j}tdd|D}t|jj|jj|jj|d}||gS)zInitiate and return the GAN optimizers based on the config parameters. It returnes 2 optimizers in a list. First one is for the generator and the second one is for the discriminator. Returns: List: optimizers. css"|] \}}|ds|VqdS)zdisc.N) startswith)rrCparamsr9r9r: s z.DelightfulTTS.get_optimizer..)r) rr optimizeroptimizer_paramslr_discrrrlr_gen)roptimizer_discgen_parameters optimizer_genr9r9r:rszDelightfulTTS.get_optimizercCs|jj|jjgS)zSet the initial learning rates for each optimizer. Returns: List: learning rates for each optimizer. )rrrrr9r9r:get_lrszDelightfulTTS.get_lrcCs8t|jj|jj|d}t|jj|jj|d}||gS)zSet the schedulers for each optimizer. Args: optimizer (List[`torch.optim.Optimizer`]): List of optimizers. Returns: List: Schedulers, one for each optimizer. rrL)rrlr_scheduler_genlr_scheduler_gen_paramslr_scheduler_disclr_scheduler_disc_params)rr scheduler_D scheduler_Gr9r9r:rs zDelightfulTTS.get_schedulercCs|jdSr2)revalrtrainerr9r9r: on_epoch_endszDelightfulTTS.on_epoch_endFDelightfulTTSConfigcCs8t|\}}t|j|}tj|d}t||||dS)zInitiate model from config Args: config (ForwardTTSE2eConfig): Model config. samples (Union[List[List], List[Dict]]): Training samples to parse speaker ids for training. Defaults to None. )r)rrrr)r$init_from_configr#rr-r)rrrr new_configrrr9r9r:r s  zDelightfulTTS.init_from_configcCs>t|tdd}||d|r||jrJdSdS)u0Load model from a checkpoint created by the 👟rp) map_locationmodelN)r.r5rJload_state_dictrtraining)rrcheckpoint_pathrstater9r9r:load_checkpoints zDelightfulTTS.load_checkpointcCs>|j|j|jd}t|dr|jj|d<|jjr |S)zOCustom state dict of the model with all the necessary components for inference.)rrr rr!)rto_dictr state_dictrrrPr)r save_stater9r9r:get_state_dicts   zDelightfulTTS.get_state_dictcCs0|||}|j|d<|j|d<t||dS)zSave model to a file.rrN)rrrr5r)rrrrr9r9r:rs   zDelightfulTTS.savecCs.t|j|jjdd|_|j|jjk|_dS)zEnable the discriminator training based on `steps_to_start_discriminator` Args: trainer (Trainer): Trainer object. rN)r epochs_donerbinary_loss_warmup_epochsrtotal_steps_donerrrr9r9r:on_train_step_start#s z!DelightfulTTS.on_train_step_startr0)rHN)NNNNN)r]rrGr2rF)Brmrrrr r#rrfrJrrsetterrrrrrrrrr5rUrVrrno_gradr&r(dictr Modulerxr[r\rrrrrrrErrr rrrrrrr~r rrrrrrr staticmethodr rrrrrr9r9rr:rvs""(               M   v ; "#% 4% 9! :    rcsXeZdZfddZeddZeddZeddZ d d d ZZ S)rcstt|_t|_t|_t di|j |_ |j |_ |j |_ |j|_|j|_|j|_|j|_|j|_|j|_|j|_|j|_|j|_|j|_|j|_dS)Nr9)rrr MSELossmse_lossL1Lossmae_lossrforward_sum_lossr/multi_scale_stft_loss_paramsmulti_scale_stft_lossmel_loss_alphaaligner_loss_alphapitch_loss_alphaenergy_loss_alphau_prosody_loss_alphap_prosody_loss_alphadur_loss_alphachar_dur_loss_alphabinary_align_loss_alphabinary_alignment_loss_alphavocoder_mel_loss_alphafeat_loss_alphagen_loss_alphamulti_scale_stft_loss_alpharrr9r:r0s$    zDelightfulTTSLoss.__init__cCs.ttj||dkdd}| |S)zBinary loss that forces soft alignments to match the hard alignments as explained in `https://arxiv.org/pdf/2108.10447.pdf`. rLg-q=r)r5rrr)alignment_hardalignment_softlog_sumr9r9r:_binary_alignment_lossGs z(DelightfulTTSLoss._binary_alignment_lossc Cs`d}t||D]$\}}t||D]\}}|}|}|tt||7}qq|dS)Nrrt)zipr<rr5rirr)rHfeats_generatedlossdrdgrlglr9r9r: feature_lossOs zDelightfulTTSLoss.feature_losscCsFd}g}|D]}|}td|d}||||7}q||fS)NrrLrt)r<r5rira)rIr< gen_lossesr>rr9r9r:generator_lossYs  z DelightfulTTSLoss.generator_lossNFc. Cs i}t| |j}t||j}d|_d|_d|_||dddf}||dddf}|||} | } d|| |d||d}!|}d|||}"|||}#||dddf}||dddf}|||}$||dddf}| |dddf} ||| }%| || |}&| |j |#|j |"|j |!|j |$|j|%|j|&|j}'|jdkr|dur|||}(|'|j|(|}'|r|j|(||d<n|j|(|d<|j|&|d<|j | |d<|j |#|d <|j |"|d <|j |!|d <|j|$|d <|j|%|d <|'|d<|sM|j||d|j})|j|dd|j}*|)|d<|*|d<|d|)|*|d<tjj|||j}+|j| | d\},}-|,|j},|-|j}-|+|d<|,|d<|-|d<|d|+|-|,|d<|S)a) Shapes: - mel_output: :math:`(B, C_mel, T_mel)` - mel_target: :math:`(B, C_mel, T_mel)` - mel_lens: :math:`(B)` - dur_output: :math:`(B, T_src)` - dur_target: :math:`(B, T_src)` - pitch_output: :math:`(B, 1, T_src)` - pitch_target: :math:`(B, 1, T_src)` - energy_output: :math:`(B, 1, T_src)` - energy_target: :math:`(B, 1, T_src)` - src_lens: :math:`(B)` - waveform: :math:`(B, 1, T_wav)` - waveform_hat: :math:`(B, 1, T_wav)` - p_prosody_ref: :math:`(B, T_src, 4)` - p_prosody_pred: :math:`(B, T_src, 4)` - u_prosody_ref: :math:`(B, 1, 256) - u_prosody_pred: :math:`(B, 1, 256) - aligner_logprob: :math:`(B, 1, T_mel, T_src)` - aligner_hard: :math:`(B, T_mel, T_src)` - aligner_soft: :math:`(B, T_mel, T_src)` - spec_slice: :math:`(B, C_mel, T_mel)` - spec_slice_hat: :math:`(B, C_mel, T_mel)` FNrrKrloss_binary_alignment loss_alignerloss_mel loss_durationloss_u_prosodyloss_p_prosody loss_pitch loss_energyr<)rHr;)rIvocoder_loss_featvocoder_loss_gen)rnrvocoder_loss_melvocoder_loss_stft_mgvocoder_loss_stft_sc)r"rrJr masked_selectr$rr>r"r%r(r.r,r-r*r+r)r1r9rAr3rCr4r5r rl1_lossr2r'r5).rr2r3r r4r6r8r:r;r=rrMr>r?r@rArBrCrDrFrrGrHrIrJrKrLrVsrc_maskmel_maskmasked_mel_predictions mel_targetsmel_lossp_prosody_lossu_prosody_loss duration_loss pitch_loss energy_lossr% total_lossbinary_alignment_loss loss_featloss_genrF loss_stft_mg loss_stft_scr9r9r:res5        zDelightfulTTSLoss.forward)NNNNNNF) rmrrrr r9rArCrrr9r9rr:r/s      rr)rYrg)rt)rLrrGr2)~r dataclassesrr itertoolsrpathlibrtypingrrrr r rr3r5torch.distributed distributedrrcoqpitr librosa.filtersr rr torch.cuda.amp.autocast_modertorch.nnrr^torch.utils.datartorch.utils.data.samplerr trainer.torchrrtrainer.trainer_utilsrrTTS.tts.datasets.datasetrrr,TTS.tts.layers.delightful_tts.acoustic_modelrTTS.tts.layers.lossesrr!TTS.tts.layers.vits.discriminatorrTTS.tts.models.base_ttsrTTS.tts.utils.helpersrrr r!r"TTS.tts.utils.speakersr#TTS.tts.utils.text.tokenizerr$TTS.tts.utils.visualr%r&r'r( TTS.utils.audio.numpy_transformsr)r*r+rvr,ruTTS.utils.audio.processorr- TTS.utils.ior.TTS.vocoder.layers.lossesr/$TTS.vocoder.models.hifigan_generatorr0TTS.vocoder.utils.generic_utilsr1r;r@rFr\rVrxr_rr<rsryr}rrrrrrrrrrrrrrrrrrrrrrrgrrrrr9r9r9r:s                             %  :2& 3@