o j@sdZddlZddlmZddlZddlmZddlmZddl m Z m Z m Z Gdd d ej ZGd d d ej ZGd d d e ZeGddde ZdS)zc Much of this code is adapted from Andrej Karpathy's NanoGPT (https://github.com/karpathy/nanoGPT) N) dataclass)nn) functional)GPTMLP GPTConfigc$eZdZfddZddZZS)NonCausalSelfAttentioncst|j|jdksJtj|jd|j|jd|_tj|j|j|jd|_t |j |_ t |j |_ |j|_|j|_|j |_ t tjjdoN|j dk|_dS)Nrbiasscaled_dot_product_attentiong)super__init__n_embdn_headrLinearr c_attnc_projDropoutdropout attn_dropout resid_dropouthasattrtorchrflashselfconfig __class__Q/home/kuhnn/.local/lib/python3.10/site-packages/TTS/tts/layers/bark/model_fine.pyrs zNonCausalSelfAttention.__init__c Cs |\}}}||j|jdd\}}}||||j||jdd}||||j||jdd}||||j||jdd}|jrWtj j j |||d|j dd}n"||dddt |d} tj| dd} || } | |}|dd|||}|||}|S) NdimrF) attn_mask dropout_p is_causalg?)sizersplitrviewr transposerrrrrrmathsqrtFsoftmaxr contiguousrr) rxBTCqkvyattr"r"r#forward s    $ zNonCausalSelfAttention.forward__name__ __module__ __qualname__rr> __classcell__r"r"r r#r s r cr ) FineBlockcs>tt|j|_t||_t|j|_t ||_ dSN) rrr LayerNormrln_1r attnln_2rmlprr r"r#r=s  zFineBlock.__init__cCs,||||}||||}|SrE)rHrGrJrI)rr5r"r"r#r>DszFineBlock.forwardr?r"r"r r#rD<s rDcs.eZdZfddZddZdddZZS) FineGPTc st|`|_j|_tttfddt jDt j j t jtfddt jDtj d|_tfddt j|jD|_t |jjD]}|j|j|jj|d_q^dS)Ncsg|] }tjjqSr")r Embeddinginput_vocab_sizer.0_rr"r# Ssz$FineGPT.__init__..csg|]}tqSr")rDrNrQr"r#rRWs)wteswpedrophln_fcs g|] }tjjjddqS)Fr )rrroutput_vocab_sizerNrQr"r#rR\sr)rrlm_headr n_codes_totalr ModuleDictdict ModuleListrangerL block_sizerrrn_layerrF transformer n_codes_givenlm_headsweightrS)rrir rQr#rKs.     zFineGPT.__init__cs&j}\}}}||jjksJd|d|jj|dks$Jd||jks0J|||ftjd|tj|dd}fddt |j j D}tj |dd } |j |} | ddddddd|d fjdd } |j | | } |j jD]} | | } qy|j | } |j||jj| } | S) Nz"Cannot forward sequence of length z, block size is only rzcannot predict 0th codebook)dtypedevicecs0g|]\}}|dddd|fdqS)Nr+) unsqueeze)rOrewteidxr"r#rRos$z#FineGPT.forward..r+r%r)rgr,rr_rZrarangelongrh enumeraterarScatrTsumrUrVrWrcrb)rpred_idxrkrgbtcodespostok_embstok_embpos_embr5blocklogitsr"rjr#r>ds(   ,   zFineGPT.forwardTcCsLtdd|D}|r$|jjD] }||j8}q||jjj8}|S)a8 Return the number of parameters in the model. For non-embedding count (default), the position embeddings get subtracted. The token embeddings would too, except due to the parameter sharing these params are actually used as weights in the final layer, so we include them. css|]}|VqdSrE)numel)rOpr"r"r# sz)FineGPT.get_num_params..)rp parametersrarSrdr{rT)r non_embeddingn_paramsrir"r"r#get_num_params|s  zFineGPT.get_num_params)T)r@rArBrr>rrCr"r"r r#rKJs rKc@s&eZdZUdZeed<dZeed<dS) FineGPTConfigrZrrbN)r@rArBrZint__annotations__rbr"r"r"r#rs r)__doc__r0 dataclassesrrrtorch.nnrr2modelrrrModuler rDrKrr"r"r"r#s   -A