o jZf@sddlZddlZddlZddlmZddlmZmZddlZddlZ ddl Z ddl Z ddl Z ddl mZddlmZddlmZddlmZddlmZdd lmZdd lmZmZeeZd d Z d dZ!ddZ"gfdee#fddZ$ddZ%gfde#dee#fddZ&d7ddZ'd8dd Z(d9d"d#Z)d$d%Z*  &   ' (  )  )d:d*d+Z+d,d-Z,  &   ' . /  )d;d0d1Z-  2 ) d}sz"generate_voice..dimhubert_tokenizer) model_pathhubert)checkpoint_path) map_location)input_sample_hzrG)r-r,r+) isinstancestr torchaudior/rconfigr]encodecchannels unsqueezetodevicetorchno_gradrcatsqueezecpunumpyrmake_sure_tokenizer_installedLOCAL_MODEL_PATHSr r load_from_checkpointforward get_tokenr.savez) r9r:r;rUencoded_framescodeshubert_manager hubert_modelrsemantic_vectorssemantic_tokensrrrr@is&     &r@ffffff?F皙?Tc & Kst|tsJt|}t|dksJtdd|Ds"| durX|dur*|d} | dur2| d} t| tjrUt| jdkrUt| dkrU| dkrU| |j j dksWJnd} t t|j||j j}t|dkrtt|dt|dd}td|d |dd}tj|ddt|f|j jd d }| dur| tj} | d d} tj| ddt| f|j jd d } n t |j jgd} tt|| t |j jggtjd}|jdd ksJtn||j}d}t j |dd}d}d}d}t!|D]9}| r|dur|dddgf}n|}|j"|d| |d\}}|ddd|j j f}| rEt||dd|j jgff}|dur|j}|#}|$%#tj&'}t(|ddd}||}t)t*|}||k}|dd+|dd<d|d<tj, |||<t|}||#|}|durt-|t ||.d\} }!t/d ||| dk<tj*||dd}"tj0|"dd}#| r|#|j j ks|dur|"d|kr|1d|nYtj2||#dfdd}|d|j j37}|dur||kr|1d|n3||dkr|1d|n#~~~"~#t dt4td||g}$|$|kr=|1|$||$}q|5|$%'6d d}%Wdn 1s^wYt|%dkrst|%|j j ksuJt7|%S)aGenerate semantic tokens from text. Args: text (str): The text to generate semantic tokens from. model (BarkModel): The BarkModel to use for generating the semantic tokens. history_prompt (tuple): A tuple of (semantic_history, coarse_history, fine_history) to use as a prompt for the generation. temp (float): The temperature to use for the generation. top_k (int): The number of top tokens to consider for the generation. top_p (float): The cumulative probability to consider for the generation. silent (bool): Whether to silence the tqdm progress bar. min_eos_p (float): The minimum probability to consider for the end of sentence token. max_gen_duration_s (float): The maximum duration in seconds to generate for. allow_early_stop (bool): Whether to allow the generation to stop early. base (tuple): A tuple of (semantic_history, coarse_history, fine_history) to use as a base for the generation. use_kv_caching (bool): Whether to use key-value caching for the generation. **kwargs: Additional keyword arguments. They are ignored. Returns: np.ndarray: The generated semantic tokens. rcs|]}|duVqdSrrrlvrrr z)generate_text_semantic..Nr8dz(warning, text too long, lopping of last %constant)constant_valuesmodeii)disabletotalroT) merge_context use_cachepast_kvFInfrp num_samples)8rxryrr$rallr.ndarrayrcminmaxr{SEMANTIC_VOCAB_SIZEarrayrrTEXT_ENCODING_OFFSETroundloggerwarningpadTEXT_PAD_TOKENastypeint64SEMANTIC_PAD_TOKENr from_numpyhstackSEMANTIC_INFER_TOKENr rrtqdmrangesemantic_modeltypedetachrfloat32rargsortcumsumrcopyinftopksizefloat multinomialupdaterSEMANTIC_RATE_HZrLcloserr )&rr:history_prompttemptop_ktop_psilent min_eos_pmax_gen_duration_sallow_early_stopbaseuse_kv_cachingkwargssemantic_history encoded_textpx n_tot_stepspbar pbar_statetot_generated_duration_skv_cachenx_inputlogitsrelevant_logits logits_device logits_dtypesorted_indices sorted_logitscumulative_probssorted_indices_to_remover_probs item_nextreq_pbar_stateoutrrrgenerate_text_semantics#       "      & $=rcCsbt|jdks J|}|dur*td|jdD]}||ddf||7<q|d}|S)NrGr8rF)r$rcrrravel)arr offset_sizerflat_arrrrr_flatten_codebooks)s rv<c / Cst|tjr#t|jdkr#t|dkr#|dkr#||jjdks%Jd|kr0dks3JJ||dks;J|jj |jj |jj } t t || } tdd|Ds^| dur(|durl|} | d}| d}| durx| d}| d}t|tjrt|jdkrt|dkr|dkr||jjdkrt|tjrt|jd kr|jd|jj kr|jd dkr|dkr||jjdkrt|jd t|dt| |jj dksJt||jj|jj}t| t|t|d t t t|| g}t t|| }|| dtj}|| dtj}|dd }ntjgtjd }tjgtjd }t tt t|| |jj |jj }|dkr[||jj dks]Jt||gtj}|tj}t|}tt|d|j}t|d|j}t t||}d}tjt|||d D]_}|t t|| }|ddtd|| gdf}|ddddf}t|dd|jd fd|jj }t|t!|jj"gd|j|dd| dfg}d}t|D]}||krq||jj dk}| r"|dur"|ddd gf}n|}|j#|| |d\}}|jjdt ||jj} |jjd t ||jj}!|dd| |!f}"|dur|"j}#|"$}$|"%&$tj'(}"t)|"ddd }%|"|%}&t*tj+j,-|&}'|'|k}(|(dd .|(dd<d|(d<tj/ |"|%|(<t|"}"|"|#$|$}"|durt0|"t||"1d \})}t2d |"|"|)d k<tj+j,j-|"|d d}*tj3|*dd}+|+| 7}+tj4||+dfdd}tj4||+dfdd}~~"~*~+|d7}q~q~Wdn 1swY|%&(5t|d},~t|,|ks.J|,6d |jj j7|jj}-td|jj D]}.|-|.ddf|.|jj8<qBt8|-S)abGenerate coarse audio codes from semantic tokens. Args: x_semantic (np.ndarray): The semantic tokens to generate coarse audio codes from. model (BarkModel): The BarkModel to use for generating the coarse audio codes. history_prompt (tuple): A tuple of (semantic_history, coarse_history, fine_history) to use as a prompt for the generation. temp (float): The temperature to use for the generation. top_k (int): The number of top tokens to consider for the generation. top_p (float): The cumulative probability to consider for the generation. silent (bool): Whether to silence the tqdm progress bar. max_coarse_history (int): The maximum number of coarse audio codes to use as history. sliding_window_len (int): The length of the sliding window to use for the generation. base (tuple): A tuple of (semantic_history, coarse_history, fine_history) to use as a base for the generation. use_kv_caching (bool): Whether to use key-value caching for the generation. Returns: np.ndarray: The generated coarse audio codes. r8rrrrcsrrrrrrrr_rz"generate_coarse..NrGrodtype)rrrr)rrFrrpr)9rxr.rr$rcrrr{rCOARSE_RATE_HZrN_COARSE_CODEBOOKSrLfloorr CODEBOOK_SIZErrrint32rrr rrrrceilrrrrCOARSE_SEMANTIC_PAD_TOKENtensorCOARSE_INFER_TOKEN coarse_modelrrrrrrrnnrrrrrrrrrrreshapeTr )/ x_semanticr:rrrrrmax_coarse_historysliding_window_lenrrsemantic_to_coarse_ratiomax_semantic_historyr1x_semantic_historyx_coarse_historyn_semantic_hist_providedn_coarse_hist_providedn_stepsx_coarsebase_semantic_idx x_semantic_in x_coarse_inn_window_stepsn_stepr semantic_idxx_inr is_major_steprrlogit_start_idx logit_end_idxrrrrrrrrrrgen_coarse_arrgen_coarse_audio_arrrrrrgenerate_coarse3s           "      B$r?c s<t|tjr8t|jdkr8d|jdkr|jjdkr8nJ|jddkr8|dkr8||jj dks:Jt dd|DsG|dur|durO|d}|durW|d}t|tjrt|jdkr|jd|jjkr|jddkr|dkr||jj dksJnd}|jd}t |t |jj||jdf|jj g tj}|dur| tj}t|ddddf tj|g}|ddddfjd} nd} d} |jddkrd|jd} t|tj |jj| ftjd |jj g}tdtt|jdd| d gd} tt|j|j}tjt| |d D]} t| d |jddg} t| | d |jdd g}|| }|| | dddfd}t||jjD]R}|||}|dur|d|dd|jj f}t|d }n%|dddd|jj f|}tj|d d tfddt|dD}||d|d|f<~~qbt||jjD]}|d|d|f|||d||f<q~q)| !j}~Wdn 1swY|dd| df}| dkr |ddd| f}|jd |jd ksJt"|S)aGenerate full audio codes from coarse audio codes. Args: x_coarse_gen (np.ndarray): The coarse audio codes to generate full audio codes from. model (BarkModel): The BarkModel to use for generating the full audio codes. history_prompt (tuple): A tuple of (semantic_history, coarse_history, fine_history) to use as a prompt for the generation. temp (float): The temperature to use for the generation. silent (bool): Whether to silence the tqdm progress bar. base (tuple): A tuple of (semantic_history, coarse_history, fine_history) to use as a base for the generation. Returns: np.ndarray: The generated full audio codes. rGr8rcsrrrrrrrrrz generate_fine..NirErrF)rrorpcsg|] }tj|ddqS)r8r)rr)rlrrrrrnBsz!generate_fine..)#rxr.rr$rcr{N_FINE_CODEBOOKSrrrrvstackzerosrrrrLrr rrrrrrr fine_modelargmaxrrrrrrr ) x_coarse_genr:rrrrx_fine_historyn_coarsein_arr n_historyn_remove_from_endn_loopsr start_idxstart_fill_idxrel_start_fill_idx in_bufferrrrcodebook_preds gen_fine_arrrr!r generate_fines $     .     r4cCsXt|d}||j}|dd}|jj|}|j|}| }|S)z:Turn quantized audio codes into audio array using encodec.Nrr8) rrrr transposer| quantizerrdecoderrrrr) fine_tokensr:rembr audio_arrrrr codec_decodeVs   r;)rErF)rRrS)r_) NrNNFrNTNT) NrNNFrrNT)Nr TN)0loggingrrrtypingrrrXrr.rrzr encodec.utilsr scipy.specialrtorch.nnrr)TTS.tts.layers.bark.hubert.hubert_managerr(TTS.tts.layers.bark.hubert.kmeans_hubertr $TTS.tts.layers.bark.hubert.tokenizerr TTS.tts.layers.bark.load_modelr r getLogger__name__rrrrryr*r5rArQr^rkr@rrrr4r;rrrrst            2   5 r