o
    
j$                     @   s  d dl mZ d dlmZ d dlZd dlm  mZ d dl	m
Z
mZ d dlmZ d dlmZ d dlmZmZ dd	 Zd
d ZeeZG dd dejZdd Zdd	 Zdd ZG dd dejZG dd dejZG dd dejZd!ddZG dd dejZG dd  d ejZdS )"    )
namedtuplewrapsN)	rearrangerepeat)	Rearrange)version)einsumnnc                 C      | d uS N )valr   r   X/home/kuhnn/.local/lib/python3.10/site-packages/TTS/tts/layers/xtts/perceiver_encoder.pyexists      r   c                    s   d t  fdd}|S )NFc                    s    rd S d | S )NTr   xcalledfnr   r   inner   s   zonce.<locals>.innerr   )r   r   r   r   r   once   s   r   c                       s:   e Zd Zd fdd	Zdd Zddd	Zdd
dZ  ZS )Attend        Fc                    s   t    || _t|| _|| _| jdd dd || _|r-t	
tjt	
dk r-J dtdg d| _| ddd| _d | _tj rG|sId S tjtd	}|jd
krj|jdkrjtd | ddd| _d S td | ddd| _d S )NmaskF
persistentz2.0.0zGin order to use flash attention, you must be using pytorch 2.0 or aboveEfficientAttentionConfig)enable_flashenable_mathenable_mem_efficientTcuda   r   zCA100 GPU detected, using flash attention if input tensor is on cudazWNon-A100 GPU detected, using math or mem efficient attention if input tensor is on cuda)super__init__dropoutr
   Dropoutattn_dropoutcausalregister_buffer	use_flashr   parsetorch__version__r   config
cpu_configcuda_configr"   is_availableget_device_propertiesdevicemajorminor
print_once)selfr&   r)   r+   device_properties	__class__r   r   r%   &   s,   
zAttend.__init__c                 C   s`   t | jr| jjd |kr| jd |d |f S tj||f|tjdd}| jd|dd |S )N)r4   dtype   r   Fr   )r   r   shaper-   onesbooltriur*   )r8   nr4   r   r   r   r   get_maskD   s
   zAttend.get_maskNc              	   C   s   g |j |j d |jR \}}}}}}	|jdkr!t|d|}|jdkr.t|d|}t|r?t|d}|d||d}|	rD| jn| j}
t	j
jjdi |
  tj||||| jr`| jnd| jd}W d    |S 1 sqw   Y  |S )	N   zb ... -> b 1 ...b j -> b 1 1 jr<   r   )	attn_mask	dropout_p	is_causalr   )r?   is_cudandimr   	expand_asr   expandr1   r0   r-   backendsr"   
sdp_kernel_asdictFscaled_dot_product_attentiontrainingr&   r)   )r8   qkvr   _headsq_lenk_lenrK   r/   outr   r   r   
flash_attnL   s"   (



zAttend.flash_attnc                 C   s   |j d |j}}|j d d }| jr| j||||dS |jdkr#dnd}td| d	||| }	t|rGt|d
}|	| t	
|	jj }	| jr\| ||}
|	|
t	
|	jj }	|	jdd}| |}td| d||}|S )z
        einstein notation
        b - batch
        h - heads
        n, i, j - sequence length (base sequence length, source, target)
        d - feature dimension
        rE   r<         ࿩r   rF   zb j dzb h j dz	b h i d, z -> b h i jrG   dimz	b h i j, z -> b h i d)r?   r4   r+   r]   rL   r	   r   r   masked_fillr-   finfor=   maxr)   rD   softmaxr(   )r8   rU   rV   rW   r   rC   r4   scalekv_einsum_eqsimcausal_maskattnr\   r   r   r   forwardl   s    	

zAttend.forward)r   FFr   )__name__
__module____qualname__r%   rD   r]   rk   __classcell__r   r   r:   r   r   %   s
    
 r   c                  G   s   t jtt|  S r   )r
   
Sequentialfilterr   )modsr   r   r   rp      s   rp   c                 C   r   r   r   r   r   r   r   r      r   c                 C   s   t | r| S t|r| S |S r   )r   callable)r   dr   r   r   default   s   ru   c                       s(   e Zd Zd fdd	ZdddZ  ZS )	RMSNormTNc                    s^   t    t|| _| jrt||d nd | _|d | _|r*tt	
|| _d S d | _d S )N   g      ?)r$   r%   r   condr
   Linearto_gamma_betarf   	Parameterr-   r@   gamma)r8   ra   rf   dim_condr:   r   r   r%      s
   


$zRMSNorm.__init__c                 C   st   t | jd}tj|dd| j | }| js|S t|sJ | |jddd\}}t	dd ||f\}}|| | S )Nr>   r<   r`   rw   c                 S   s
   t | dS )Nzb d -> b 1 dr   tr   r   r   <lambda>   s   
 z!RMSNorm.forward.<locals>.<lambda>)
ru   r|   rR   	normalizerf   rx   r   rz   chunkmap)r8   r   rx   r|   r\   betar   r   r   rk      s   zRMSNorm.forward)TNr   rl   rm   rn   r%   rk   ro   r   r   r:   r   rv      s    rv   c                       s(   e Zd Z fddZ fddZ  ZS )CausalConv1dc                    sH   t  j|i | | j\}| j\}| j\}|dksJ ||d  | _d S )Nr>   )r$   r%   kernel_sizedilationstridecausal_padding)r8   argskwargsr   r   r   r:   r   r   r%      s   zCausalConv1d.__init__c                    s"   t j|| jdfdd}t |S )Nr   r   )value)rR   padr   r$   rk   )r8   r   causal_padded_xr:   r   r   rk      s   zCausalConv1d.forwardr   r   r   r:   r   r      s    	r   c                   @   s   e Zd Zdd ZdS )GEGLUc                 C   s    |j ddd\}}t|| S )Nrw   r<   r`   )r   rR   gelu)r8   r   gater   r   r   rk      s   zGEGLU.forwardN)rl   rm   rn   rk   r   r   r   r   r      s    r      Fc                 C   s^   t | | d d }d }|rttdt||dtd}tt| |d t |t|| S )Nrw   rF   zb n d -> b d nzb d n -> b n d)intr
   rp   r   r   ry   r   )ra   multcausal_conv	dim_innerconvr   r   r   FeedForward   s   
$r   c                       s8   e Zd Zdddddddd fd	d

ZdddZ  ZS )PerceiverResamplerrw   N    @   r#   r   F)depthdim_contextnum_latentsdim_headrY   ff_multuse_flash_attnc          
         s   t    t||}||krt||nt | _tt	||| _
tjj| j
dd tg | _t|D ]}	| jtt||||ddt||dg q6t|| _d S )Ng{Gz?)stdT)ra   r   rY   r+   cross_attn_include_queries)ra   r   )r$   r%   ru   r
   ry   Identityproj_contextr{   r-   randnlatentsinitnormal_
ModuleListlayersrangeappend	Attentionr   rv   norm)
r8   ra   r   r   r   r   rY   r   r   rX   r:   r   r   r%      s*   


zPerceiverResampler.__init__c                 C   s\   |j d }| |}t| jd|d}| jD ]\}}||||d| }||| }q| |S )Nr   zn d -> b n d)br_   )r?   r   r   r   r   r   )r8   r   r   batchr   rj   ffr   r   r   rk   	  s   


zPerceiverResampler.forwardr   r   r   r   r:   r   r      s    'r   c                       s8   e Zd Zdddddddd fdd
Zdd	d
Z  ZS )r   NFr   r#   r   )r   r)   r   rY   r&   r+   r   c          
         s   t    |d | _|| _|| _|| }	t||}t|||d| _tj	||	dd| _
tj	||	d dd| _tj	|	|dd| _d S )Nr^   )r)   r&   r+   F)biasrw   )r$   r%   rf   rY   r   ru   r   attendr
   ry   to_qto_kvto_out)
r8   ra   r   r)   r   rY   r&   r+   r   r   r:   r   r   r%     s   


zAttention.__init__c           	         s   | j t| }t||}|r| jrtj||fdd}| |g| |jdddR \}}}t	 fdd|||f\}}}| j
||||d}t|d}| |S )	NrE   r`   rw   r<   c                    s   t | d dS )Nzb n (h d) -> b h n dhr~   r   r   r   r   r   :  s    z#Attention.forward.<locals>.<lambda>r_   zb h n d -> b n (h d))rY   r   ru   r   r-   catr   r   r   r   r   r   r   )	r8   r   contextr   has_contextrU   rV   rW   r\   r   r   r   rk   1  s   

(

zAttention.forward)NNr   r   r   r:   r   r     s    r   )r   F) collectionsr   	functoolsr   r-   torch.nn.functionalr
   
functionalrR   einopsr   r   einops.layers.torchr   	packagingr   r	   r   r   printr7   Moduler   rp   ru   rv   Conv1dr   r   r   r   r   r   r   r   r   <module>   s*   u
6