o
    
j/                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZmZ d dlmZ d dl m!Z! dej"j#_$dej"j#_%e&d ej'( Z)ej'* Z+e,de) e,de+ d!dede-de-fddZ.dd Z/dd Z0dd Z1e2dkre \Z3Z4Z5Z6Z7Z8ze1e3 W dS  e9y   ee5 z	e:d  W Y dS  e;y   e <d  Y Y dS w  e=y   ee5 e>  e:d  Y dS w dS )"    N)
DataLoader)copy_model_filessave_best_modelsave_checkpoint)NoamLR)get_optimizer)EncoderDataset)setup_encoder_model)init_training)plot_embeddings)load_tts_samples)AudioProcessor)count_parametersremove_experiment_folder)PerfectBatchSampler)check_updateTi1  z > Using CUDA: z > Number of GPUs: Fapis_valverbosec           	      C   s   |st jnt j}|st jnt j}tt | |rtntt j||||s"t j	nd t j
ddd	}| }t|j||| |d| dd}t||k ra|rTtd| dt| d	td
| dt| d|rh|t t|t j||jd}||| fS )Nuse_torch_specF)	voice_lennum_utter_per_classnum_classes_in_batchr   augmentation_configr      T)
batch_sizer   num_gpusshuffle	drop_lastz"config.eval_num_classes_in_batch (z) need to be <= z0 (Number total of Classes in the Eval dataset) !zconfig.num_classes_in_batch (z1 (Number total of Classes in the Train dataset) !)num_workersbatch_sampler
collate_fn)cr   eval_num_utter_per_classr   eval_num_classes_in_batchr   meta_data_evalmeta_data_trainr   audio_augmentationmodel_paramsgetget_class_listr   itemslenRuntimeErrorset_classestrain_classesr   num_loader_workersr!   get_map_classid_to_classname)	r   r   r   r   r   datasetclassessamplerloader r6   H/home/kuhnn/.local/lib/python3.10/site-packages/TTS/bin/train_encoder.pysetup_loader"   sP   


r8   c              	   C   s&  d}t |D ]h\}}t X |\}}t|tjtjdd|j	}t|tjtjddd|j	}t
rD|jdd}|jdd}| |}	||	tj|	j	d tj d|}
||
 7 }W d    n1 siw   Y  q|t| }t|d|i dt|	   tji}t|| |S )Nr   r   Tnon_blockingloss	UMAP Plot)	enumeratetorchno_grad	transposeviewr"   r#   r$   reshapeshapeuse_cudacudaitemr,   dashboard_logger
eval_statsr   detachcpunumpyr   eval_figures)model	criteriondata_loaderglobal_step	eval_loss_datainputslabelsoutputsr<   eval_avg_lossfiguresr6   r6   r7   
evaluationU   s<   
rZ   c                 C   s  |    td}d}t }	ttjD ]`}
d}d}t|D ]
\}}t }|\}}t|	tj
tjdd|j}t|	tj
tjddd|j}t |	 }|d7 }tjr_|  |  trq|jdd}|jdd}| |}||	tj|jd tj d|}|  t| tj\}}|  t | }||7 }|| 7 }tjdkrtjnd}|dkrd| | |d | |  n|}|jd d }|tj dkr| ||||d}t|| d	t|   tji}t || |tj! dkrt"d
#|| |||||dd |tj$ dkr%t%t| |d ||
t&|' d t }	qt"d t"d#|
|t(| |||dd tj)rt| *  t+| |||}t"d t"d t"d#|
|dd t,||t| |d ||
t&|' d
}|    q||fS )Ninfr   r   r9   Tr:   lr)r<   r\   	grad_norm	step_timeavg_loader_timer=   zq   | > Step:{}  Loss:{:.5f}  GradNorm:{:.5f}  StepTime:{:.2f}  LoaderTime:{:.2f}  AvGLoaderTime:{:.2f}  LR:{:.6f}flush)rO    zU>>> Epoch:{}  AvgLoss: {:.5f} GradNorm:{:.5f}  EpochTime:{:.2f} AvGLoaderTime:{:.2f} z

z--> EVAL PERFORMANCEz!   | > Epoch:{}  AvgLoss: {:.5f} )-trainfloattimeranger"   epochsr>   r?   rA   rB   r   r   rC   rD   lr_decaystep	zero_gradrE   rF   backwardr   	grad_cliprG   r0   param_groupssteps_plot_statsrH   train_epoch_statsr   rJ   rK   rL   train_figures
print_stepprintformat	save_stepr   OUT_PATH
state_dictr,   run_evalevalrZ   r   )rN   	optimizer	schedulerrO   rP   eval_data_loaderrQ   	best_lossr_   end_timeepochtot_loss
epoch_timerS   rT   
start_timerU   rV   loader_timerW   r<   r]   r^   r0   
current_lrtrain_statsrY   rR   r6   r6   r7   rc   ~   s   

rc   c                 C   s\  t di tj}tt}ttjtjtj|}ttj	dd\a
at|ddd\}a}tjr7t|ddd\}}}nd }tt}|t|}	tjdkrWtjdkrW|t_ttti d | jrr|jt| jdt|	d\}	| _td	| j dd
 nd| _tjrt|tj| jd d}
nd }
t|}td|dd
 tr|  }|	   | j}t!|||
|	|||\}}d S )NT)
eval_splitF)r   r   softmaxprotospeaker_encoder)
new_fields)rx   rE   rO   z > Model restored from step %dr`   r   r   )warmup_steps
last_epochz
 > Model has {} parametersr6   )"r   r"   audior	   r   ry   optimizer_paramsr\   r   datasetsr&   r%   r8   r/   rw   r,   get_criterionr<   rN   map_classid_to_classnamer   ru   restore_pathload_checkpointrE   restore_steprr   rh   r   r   r   rs   rF   rc   )argsr   rN   ry   train_data_loaderr   r{   rS   num_classesrO   rz   
num_paramsrQ   r6   r6   r7   main  s:   r   __main__r   )FF)?ossysre   	tracebackr?   torch.utils.datar   
trainer.ior   r   r   trainer.torchr   trainer.trainer_utilsr   TTS.encoder.datasetr   TTS.encoder.utils.generic_utilsr	   TTS.encoder.utils.trainingr
   TTS.encoder.utils.visualr   TTS.tts.datasetsr   TTS.utils.audior   TTS.utils.generic_utilsr   r   TTS.utils.samplersr   TTS.utils.trainingr   backendscudnnenabled	benchmarkmanual_seedrF   is_availablerE   device_countr   rr   boolr8   rZ   rc   r   __name__r   r"   ru   
AUDIO_PATHc_loggerrH   KeyboardInterruptexit
SystemExit_exit	Exception	print_excr6   r6   r6   r7   <module>   s^   






3) 3