o
    
j                     @   s0   d dl Z d dlZd dlmZ G dd deZdS )    N)	Optimizerc                       s4   e Zd Zd fdd	Z fdd	ZdddZ  ZS )RAdamMbP?g?g+?:0yE>r   Tc           	         s4  |dk rt d||dk rt d|d|d   kr"dk s,n t d|d d|d   kr8dk sBn t d|d || _t|ttfrt|dkrt|d tr|D ]%}d	|v r|d	 d |d ksu|d	 d |d krd
d tdD |d< q[t||||dd tdD d}t	 
|| d S )Ng        zInvalid learning rate: {}zInvalid epsilon value: {}r         ?z%Invalid beta parameter at index 0: {}   z%Invalid beta parameter at index 1: {}betasc                 S      g | ]}g d qS )NNN .0_r   r   B/home/kuhnn/.local/lib/python3.10/site-packages/TTS/utils/radam.py
<listcomp>       z"RAdam.__init__.<locals>.<listcomp>
   bufferc                 S   r
   r   r   r   r   r   r   r      r   )lrr	   epsweight_decayr   )
ValueErrorformatdegenerated_to_sgd
isinstancelisttuplelendictrangesuper__init__)	selfparamsr   r	   r   r   r   paramdefaults	__class__r   r   r"   
   s$   (0zRAdam.__init__c                    s   t  | d S N)r!   __setstate__)r#   stater'   r   r   r*      s   zRAdam.__setstate__Nc                 C   s  d }|d ur	| }| j D ]f}|d D ]^}|jd u rq|jj }|jr)td|j }| j| }t|dkrLd|d< t	||d< t	||d< n|d 
||d< |d 
||d< |d |d }}	|d \}
}|	|j||d| d	 ||
j|d|
 d
 |d  d7  < |d t|d d  }|d |d kr|d |d }}nc|d |d< ||d  }dd|  d }|d|d  | d|   }||d< |dkrtd| |d  |d  |d  | | |d  d|
|d    }n| jr	dd|
|d    }nd}||d< |dkrF|d dkr)|j||d  |d  d
 |	 |d }|j||| |d  d	 |j| q|dkrr|d dkr`|j||d  |d  d
 |j|| |d  d
 |j| qq|S )Nr$   z'RAdam does not support sparse gradientsr   stepexp_avg
exp_avg_sqr	   r   )value)alphar   r            r   r   r   r   )param_groupsgraddatafloat	is_sparseRuntimeErrorr+   r   torch
zeros_liketype_asmul_addcmul_add_intmathsqrtr   addcdiv_copy_)r#   closurelossgrouppr6   p_data_fp32r+   r-   r.   beta1beta2bufferedN_sma	step_sizebeta2_t	N_sma_maxdenomr   r   r   r,   !   s   


	

Bz
RAdam.step)r   r   r   r   Tr)   )__name__
__module____qualname__r"   r*   r,   __classcell__r   r   r'   r   r   	   s    r   )rB   r;   torch.optim.optimizerr   r   r   r   r   r   <module>   s   