o j$@sdZddlZddlmZddlZddlmZddlmZddlm Z Gdddej Z Gd d d ej Z Gd d d ej ZGd ddej ZeGdddeZGdddej ZdS)zc Much of this code is adapted from Andrej Karpathy's NanoGPT (https://github.com/karpathy/nanoGPT) N) dataclass)Coqpit)nn) functionalcs(eZdZdZfddZddZZS) LayerNormzNLayerNorm but with an optional bias. PyTorch doesn't support simply bias=Falsecs@ttt||_|rtt||_dSd|_dSN) super__init__r Parametertorchonesweightzerosbias)selfndimr __class__L/home/kuhnn/.local/lib/python3.10/site-packages/TTS/tts/layers/bark/model.pyr s $zLayerNorm.__init__cCst||jj|j|jdS)Ngh㈵>)F layer_normr shaperrxrrrforwardszLayerNorm.forward)__name__ __module__ __qualname____doc__r r __classcell__rrrrrs rc&eZdZfddZdddZZS)CausalSelfAttentionc st|j|jdksJtj|jd|j|jd|_tj|j|j|jd|_t |j |_ t |j |_ |j|_|j|_|j |_ t tjjd|_|jsg|dtt|j|jdd|j|jdSdS)Nrrscaled_dot_product_attentionr)rr n_embdn_headrLinearrc_attnc_projDropoutdropout attn_dropout resid_dropouthasattrr rflashregister_buffertrilr block_sizeviewrconfigrrrr s$  zCausalSelfAttention.__init__NFcCs|\}}}||j|jdd\}}} ||||j||jdd}||||j||jdd}| |||j||jdd} |durc|d} |d} tj| |fdd}tj| | fdd} |j d} |durq|| f} nd} |j r|dur}d}nd}tj j j ||| |j|d}n=||dd d t|d }||jdddd| || d| fdktd }tj|d d}||}|| }|dd|||}|||}|| fS) Ndimr&rTF) dropout_p is_causalg?z-inf)sizer*splitr'r5r( transposer catrr1rrr%r-mathsqrt masked_fillrfloatrsoftmaxr. contiguousr/r+)rrpast_kv use_cacheBTCqkvpast_key past_valueFULL_Tpresentr=yattrrrr4s6     $6 zCausalSelfAttention.forwardNFrrrr rr rrrrr"s r"cs$eZdZfddZddZZS)MLPcs^ttj|jd|j|jd|_tjd|j|j|jd|_t|j |_ t |_ dS)Nr$) rr rr)r'rc_fcr+r,r-GELUgelur6rrrr gs z MLP.__init__cCs,||}||}||}||}|Sr)r[r]r+r-rrrrrns    z MLP.forwardrXrrrrrYfs rYcr!)BlockcsLtt|j|jd|_t||_t|j|jd|_t ||_ ||_ dS)Nr$) rr rr'rln_1r"attnln_2rYmlp layer_idx)rr7rcrrrr ws    zBlock.__init__NFcCs>|j||||d\}}||}||||}||fS)NrIrJ)r`r_rbra)rrrIrJ attn_outputprev_kvsrrrrsz Block.forwardrWrXrrrrr^vs r^c@sneZdZUdZeed<dZeed<dZeed<dZeed<dZ eed<d Z eed <d Z e ed <d Z eed<dS) GPTConfigir4i@'input_vocab_sizeoutput_vocab_size n_layerr(ir'gr-TrN)rrrr4int__annotations__rhrirkr(r'r-rFrboolrrrrrgs       rgcs0eZdZfddZd ddZd dd ZZS) GPTc stjdus JjdusJjdusJ|_ttt jj t jj t j t fddtjDtj jdd|_tjj jdd|_dS)Ncsg|]}t|qSr)r^).0idxr7rr sz GPT.__init__..r$)wtewpedrophln_fF)rr rhrir4r7r ModuleDictdict Embeddingr'r,r- ModuleListrangerkrr transformerr)lm_headr6rrrrr s   z GPT.__init__TcCsBtdd|D}|r||jjj8}||jjj8}|S)a8 Return the number of parameters in the model. For non-embedding count (default), the position embeddings get subtracted. The token embeddings would too, except due to the parameter sharing these params are actually used as weights in the final layer, so we include them. css|]}|VqdSr)numel)rpprrr sz%GPT.get_num_params..)sum parametersr~rtr rru)r non_embeddingn_paramsrrrget_num_paramss zGPT.get_num_paramsFNcCs|j}|\}}|dur|dksJ|j|} n]|r-|jddks%J|jdd}n||jjks?Jd|d|jj|rqtj|j|ddddf|j|ddddf|j|ddddfgdd} n|j|} |durd} t dgt |jj }n |ddd } |durtj | || tj |d }|d}|jd|fksJ|j|} |j| | } |rd nd} tt|jj |D]\}\}}|| ||d \} }|r| |f} q|j| } || ddd gddf}|| fS)Nr&iz"Cannot forward sequence of length z, block size is only ir9rr;)dtypedevicerrdr>)rr?r~rtrr7r4r rBtuplelenrwarangelong unsqueezerurv enumerateziprxr)rrq merge_contextrI position_idsrJr_ttok_emb past_lengthpos_embrnew_kvblock past_layer_kvkvlogitsrrrrsL  6      z GPT.forward)T)FNNF)rrrr rrr rrrrros   ro)rrC dataclassesrr coqpitrrtorch.nnrrModulerr"rYr^rgrorrrrs     L