o
    
jh                     @   s   d dl Z d dlm  mZ d dlmZ d dl mZ dd Zdd Zd#d	d
Z	dd Z
d$ddZdd ZG dd dejZG dd dejZG dd dejZG dd dejZG dd dejZG dd dejZG dd  d ejZG d!d" d"ejZdS )%    N	rearrange)nnc                 C   s   | d uS N )valr   r   V/home/kuhnn/.local/lib/python3.10/site-packages/TTS/tts/layers/tortoise/transformer.pyexists	   s   r	   c                 C   s   t | r| S |S r   )r	   )r   dr   r   r   default      r      c                 C   s*   t | tr	t| } t | tr| S | f| S r   )
isinstancelisttuple)r   depthr   r   r   
cast_tuple   s   
r   c                 C   s   t | jj S r   )torchfinfodtypemaxtr   r   r   max_neg_value   r   r      c                 C   s0   | | } | t j| |dd  } | | j|dS )NTdimkeepdimr   )r   amaxdetachsoftmax)r   r   alphar   r   r   stable_softmax   s   r$   c           
         s   dd t |D }fdd| D }|D ]2 |  tt|  D ]"\}\\}}}t fdd|\}}	i ||i ||	f||< q%q|S )Nc                 S   s   g | ]}t  t  fqS r   )dict).0_r   r   r   
<listcomp>"       zroute_args.<locals>.<listcomp>c                    s   g | ]}| v r|qS r   r   )r&   key)routerr   r   r(   #   r)   c                    s   | r iS i S r   r   )route)r*   r   r   r   <lambda>(   s    zroute_args.<locals>.<lambda>)rangekeys	enumeratezipmap)
r+   argsr   routed_argsmatched_keysf_argsg_argsroutes
new_f_args
new_g_argsr   )r*   r+   r   r   
route_args!   s   "r;   c                       s*   e Zd Zi df fdd	Zdd Z  ZS )SequentialSequence        c                    sB   t    t fdd| D sJ d | _|| _|| _d S )Nc                 3   s     | ]}t |t  kV  qd S r   )len)r&   r,   layersr   r   	<genexpr>1   s    
z.SequentialSequence.__init__.<locals>.<genexpr>zSeach argument route map must have the same depth as the number of sequential layers)super__init__allvaluesr@   
args_routelayer_dropout)selfr@   rF   rG   	__class__r?   r   rC   /   s   


zSequentialSequence.__init__c           	      K   sf   t | j|t| j}tt| j|}|D ]\\}}\}}|||fi | }|||fi | }q|S r   )r;   rF   r>   r@   r   r1   )	rH   xkwargsr3   layers_and_argsfgr6   r7   r   r   r   forward8   s   zSequentialSequence.forward__name__
__module____qualname__rC   rP   __classcell__r   r   rI   r   r<   .   s    	r<   c                       $   e Zd Z fddZdd Z  ZS )	DivideMaxc                    s   t    || _d S r   )rB   rC   r   )rH   r   rI   r   r   rC   C   s   

zDivideMax.__init__c                 C   s   |j | jdd }|| S )NTr   )r    r   r!   )rH   rK   maxesr   r   r   rP   G   s   zDivideMax.forwardrQ   r   r   rI   r   rW   B   s    rW   c                       rV   )
LayerScalec                    s\   t    |dkrd}n|dkr|dkrd}nd}tdd||}t|| _|| _d S )N   g?   gh㈵>gư>r   )	rB   rC   r   zerosfill_r   	Parameterscalefn)rH   r   r   r`   init_epsr_   rI   r   r   rC   N   s   

zLayerScale.__init__c                 K   s   | j |fi || j S r   )r`   r_   rH   rK   rL   r   r   r   rP   [   s   zLayerScale.forwardrQ   r   r   rI   r   rY   M   s    rY   c                       s&   e Zd Zd fdd	Zdd Z  ZS )PreNormFc                    s8   t    t|| _|rt|nt | _|| _d S r   )rB   rC   r   	LayerNormnormIdentitynorm_outr`   )rH   r   r`   sandwichrI   r   r   rC   c   s   

zPreNorm.__init__c                 K   s&   |  |}| j|fi |}| |S r   )re   r`   rg   rb   r   r   r   rP   i   s   

zPreNorm.forward)FrQ   r   r   rI   r   rc   b   s    rc   c                   @   s   e Zd Zdd ZdS )GEGLUc                 C   s    |j ddd\}}|t| S )N   r   r   )chunkFgelu)rH   rK   gatesr   r   r   rP   s   s   zGEGLU.forwardN)rR   rS   rT   rP   r   r   r   r   ri   r   s    ri   c                       s&   e Zd Zd fdd	Zdd Z  ZS )FeedForwardr=         @c              	      sD   t    tt||| d t t|t|| || _d S )Nrj   )rB   rC   r   
SequentialLinearri   Dropoutnet)rH   r   dropoutmultrI   r   r   rC   y   s   

zFeedForward.__init__c                 C   s
   |  |S r   )rt   )rH   rK   r   r   r   rP      s   
zFeedForward.forward)r=   rp   rQ   r   r   rI   r   ro   x   s    	ro   c                       s(   e Zd Zd
 fdd	Zddd	Z  ZS )	AttentionT   @   r=   c                    sd   t    || }|| _|| _|d | _|| _tj||d dd| _t	t||t
|| _d S )Ng         F)bias)rB   rC   headsseq_lenr_   causalr   rr   to_qkvrq   rs   to_out)rH   r   r}   r~   r|   dim_headru   	inner_dimrI   r   r   rC      s   

 zAttention.__init__Nc                    s  g |j | j|jR \}}} }tj}| |jddd}t fdd|\}	}
}|	| j }	t	d|	|
}t
|}t|rKt|d}|| | ~| jrn|j dd  \}}tj|||d	|| d
  }||| ||dd}t	d||}t|d}| |}|S )Nrz   r   r   c                    s   t | d dS )Nzb n (h d) -> b h n dhr   r   r   r   r   r-      s    z#Attention.forward.<locals>.<lambda>zb h i d, b h j d -> b h i jzb j -> b () () j)devicer   zb h i j, b h j d -> b h i dzb h n d -> b n (h d))shaper|   r   r   r"   r   rk   r2   r_   einsumr   r	   r   masked_fill_r~   onestriu_boolr   )rH   rK   maskbnr'   r   r"   qkvqkvdots
mask_valueijattnoutr   r   r   rP      s(   "

"

zAttention.forward)Trx   ry   r=   r   rQ   r   r   rI   r   rw      s    rw   c                	       s8   e Zd Zddddddddd fdd	
Zd
d Z  ZS )TransformerTrx   ry      r=   F)r~   r|   r   ff_multattn_dropout
ff_dropoutsparse_attnsandwich_normc                   s   t    tg }t|
|}tt||D ]5\}}
t||||||d}t|||	d}|	tt
||d t|||dt
||d t|||dg qt}d| }d|i}|||d| _d S )N)r~   r}   r|   r   ru   )rv   ru   r   )rh   ))TFr   )rF   )rB   rC   r   
ModuleListr   r1   r.   rw   ro   appendrY   rc   r<   r@   )rH   r   r   r}   r~   r|   r   r   r   r   r   r   r@   sparse_layerindr   ffexecute_type
route_attnattn_route_maprI   r   r   rC      s0   


		zTransformer.__init__c                 K   s   | j |fi |S r   r?   rb   r   r   r   rP      s   zTransformer.forwardrQ   r   r   rI   r   r      s    .r   )r   )r   r   )r   torch.nn.functionalr   
functionalrl   einopsr   r	   r   r   r   r$   r;   Moduler<   rW   rY   rc   ri   ro   rw   r   r   r   r   r   <module>   s$    

,